Tweaks and comments for ci-improvements, stability, and readibility (#23658)

* Tweaks and comments for ci-improvements, stability, and readibility
- Made naming more obvious and intuitive
- CI supports grafana e2e plus more granular design
  (side effect that it takes slightly longer, but negligable)
- disable strict e2e errors through environment variable
- Added tools for linting and for building plugin-ci

* feedback from stevev - remove version. Causes problems

packages/grafana-toolkit/config/circleci/config.yml

 version: 2.1
 
+parameters:
+  ssh-fingerprint:
+    type: string
+    default: "dc:60:ab:c7:2d:8c:82:50:2a:2a:97:1a:c0:66:83:14"
+
 aliases:
   # Workflow filters
   - &filter-only-master
@@ -10,23 +15,40 @@ aliases:
       only: /^v[1-9]*[0-9]+\.[1-9]*[0-9]+\.x$/
 
 workflows:
-  version: 2
   plugin_workflow:
     jobs:
-      - compile_dependencies
-      - build_plugin:
-          requires: 
-            - compile_dependencies
-      - package_and_report:
+      - yarn_install
+      - build_docs:
+          requires:
+            - yarn_install
+      - build_frontend:
+          requires:
+            - yarn_install
+      - build_backend:
+          requires:
+            - yarn_install
+      - package:
+          requires:
+            - build_frontend
+            - build_backend
+            - build_docs
+      - provisioning:
           requires:
-            - build_plugin
-      - test_integration:
+            - build_frontend
+            - build_backend
+            - build_docs
+      - e2e_canary:
           requires:
-            - package_and_report
+            - provisioning
+            - package
+      - report:
+          requires:
+            - package
+            - e2e_canary
       - approve_release:
           type: approval
           requires:
-            - test_integration
+            - report
           filters: *filter-only-release
       - publish_github_release:
           requires:
@@ -37,71 +59,111 @@ executors:
   default_exec: # declares a reusable executor
     docker:
       - image: srclosson/grafana-plugin-ci-alpine:latest
+  e2e_exec: 
+    docker:
+      - image: srclosson/grafana-plugin-ci-e2e:latest
 
 jobs:
-  compile_dependencies:
+  yarn_install:
     executor: default_exec
     steps:
       - checkout
       - restore_cache:
+          name: restore node_modules
           keys:
           - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
       - run:
           name: Install dependencies
           command: |
             mkdir ci
-            [ -L ~/project/node_modules/.bin/grafana-toolkit ] || yarn install --frozen-lockfile
+            [ -f ~/project/node_modules/.bin/grafana-toolkit ] || yarn install --frozen-lockfile
       - save_cache:
+          name: save node_modules
           paths:
             - ~/project/node_modules
           key: build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
-      
-  build_plugin:
+      - save_cache:
+          name: save cypress cache
+          paths:
+            - ~/.cache/Cypress
+          key: cypress-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+
+  build_docs:
+    executor: default_exec
+    steps:
+      - checkout
+      - restore_cache:
+          name: restore node_modules
+          keys:
+          - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+      - run:
+          name: Build docs
+          command: |
+            ./node_modules/.bin/grafana-toolkit plugin:ci-docs
+            [ -d "dist" ] || circleci-agent step halt
+      - persist_to_workspace:
+          root: .
+          paths:
+            - dist
+
+  build_frontend:
     executor: default_exec
     steps:
       - checkout
       - restore_cache:
+          name: restore node_modules
           keys:
           - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
       - run:
-          name: Build and test frontend and docs
-          command:  |
-            npx grafana-toolkit plugin:ci-docs
-            npx grafana-toolkit plugin:ci-build
+          name: Build and test frontend
+          command: ./node_modules/.bin/grafana-toolkit plugin:ci-build
+      - persist_to_workspace:
+          root: .
+          paths:
+            - dist
+
+  build_backend:
+    executor: default_exec
+    steps:
+      - checkout
+      - restore_cache:
+          name: restore node_modules
+          keys:
+          - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
       - run:
-          name: Build and test Backend
+          name: Exit if no backend
           command: |
-            if [ -f "Magefile.go" ]; then
-              mage -v buildAll
-              mage -v lint
-              mage -v coverage
-            fi
+            [ -f "Magefile.go" ] || circleci-agent step halt
       - run:
-          name: Move results to ci folder
-          command:  |
-            npx grafana-toolkit plugin:ci-build --finish
+          name: Build backend
+          command: mage -v buildAll
+      - run:
+          name: Test backend
+          command: |
+            mage -v lint
+            mage -v coverage
       - persist_to_workspace:
           root: .
           paths:
-            - ci
+            - dist
 
-  package_and_report:
+  package:
     executor: default_exec
     steps:
     - checkout
     - restore_cache:
+        name: restore node_modules
         keys:
         - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
     - attach_workspace:
         at: .
     - run:
-        name: Package distribution
-        command: |
-          npx grafana-toolkit plugin:ci-package
+        name: Move results to ci folder
+        command: ./node_modules/.bin/grafana-toolkit plugin:ci-build --finish
     - run:
-        name: Toolkit report
+        name: Package distribution
         command: |
-          npx grafana-toolkit plugin:ci-report
+          ./node_modules/.bin/grafana-toolkit plugin:ci-package
     - persist_to_workspace:
         root: .
         paths:
@@ -112,44 +174,100 @@ jobs:
     - store_artifacts:
         path: ci
 
-  test_integration:
+  provisioning:
     executor: default_exec
     steps:
     - checkout
+    - add_ssh_keys:
+        fingerprints:
+          - << pipeline.parameters.ssh-fingerprint >>
+    - run:
+        name: Checkout provisioning files
+        command: git clone git@github.com:grafana/plugin-provisioning.git
+    - run:
+        name: Prepare task output dir
+        command: |
+          mkdir ci # Avoid error if not exists
+          mkdir ci/jobs # Avoid error if not exists
+          mkdir ci/jobs/provisioning
+          mv plugin-provisioning/plugins/google-sheets/provisioning/* ci/jobs/provisioning
+    - persist_to_workspace:
+        root: .
+        paths:
+        - ci/jobs/provisioning
+
+  e2e_canary:
+    executor: e2e_exec
+    steps:
+    - checkout
     - attach_workspace:
         at: .
     - restore_cache:
+        name: Restore node_modules
         keys:
         - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+    - restore_cache:
+        name: Restore cypress cache
+        keys:
+        - cypress-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+    - run:
+        name: Exit if no e2e tests configured
+        command: |
+          [ -d cypress ] || circleci-agent step halt
     - run:
         name: Setup Grafana (local install)
         command: |
-          ginstall latest
+          ginstall canary
           /opt/grafana/bin/grafana-server -config ci/grafana-test-env/custom.ini -homepath /opt/grafana &
+          ## To make sure grafana has started up
+          while ! $(netstat -tulpn | grep 3000 >/dev/null 2>&1); do sleep 1; done 
           /opt/grafana/bin/grafana-cli --version
+
+    - run:
+        name: Copy provisioning files
+        command: cp -r ci/jobs/provisioning provisioning/
     - run:
         name: Run e2e tests
         command: |
-          [ -d cypress ] && npx grafana-e2e run || echo "skipping e2e"
+          # If the tests fail, but GRAFANACI_STRICT_E2E=0, don't worry about it
+          ./node_modules/.bin/grafana-e2e run \
+          || (
+            [ "$GRAFANACI_STRICT_E2E" -eq 0 ] && echo "Bypassing fail. ci-nostrict enabled"
+          )
     - run:
+        when: always
         name: Prepare task output dir
         command: |
           # TODO: probably move all of this to `@grafana/toolkit plugin:ci-test`
-          mkdir -m 0755 -p ci/jobs/test_integration
+          mkdir ci/jobs/e2e_canary
           # only copy if they exist
-          if [ -d cypress ]; then 
-            [ -d cypress/screenshots ] && cp cypress/screenshots/ ci/jobs/test_integration 
-            [ -d cypress/videos ] && cp cypress/videos/ ci/jobs/test_integration
-          fi
+          cp cypress/screenshots/*.* ci/jobs/e2e_canary 2>/dev/null || :
+          cp cypress/videos/*.* ci/jobs/e2e_canary 2>/dev/null || :
     - persist_to_workspace:
         root: .
         paths:
-        - ci/jobs/test_integration
+        - ci/jobs/e2e_canary
     - store_test_results:
-        path: ci/jobs/test_integration
+        path: ci/jobs/e2e_canary
     - store_artifacts:
-        path: ci/jobs/test_integration
+        path: ci/jobs/e2e_canary
 
+  report:
+    executor: default_exec
+    steps:
+    - checkout
+    - attach_workspace:
+        at: .
+    - restore_cache:
+        name: restore node_modules
+        keys:
+          - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+    - run:
+        name: Toolkit report
+        command: |
+          ./node_modules/.bin/grafana-toolkit plugin:ci-report
+    - store_artifacts:
+        path: ci
 
   publish_github_release:
     executor: default_exec
@@ -161,10 +279,11 @@ jobs:
       - attach_workspace:
           at: .
       - restore_cache:
+          name: restore node_modules
           keys:
           - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
       - run:
           name: "Publish Release on GitHub"
           command: |
-            npx grafana-toolkit plugin:github-publish
+            ./node_modules/.bin/grafana-toolkit plugin:github-publish
 

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy-common.sh

@@ -36,3 +36,17 @@ untar_file () {
 
     tar -C "$dest" -xf "$src" && /bin/rm -rf "$src"
 }
+
+##
+# WIP: Just started this and not finished.
+# The intent it to download a release from a git repo,
+# compile, and install
+get_latest_release () {
+	tarsrc=$(curl -sL "https://api.github.com/repos/$1/$2/releases/latest" | jq ".tarball_url" | tr -d '"')
+	wget -O /tmp/autoretrieved.tar.gz "$tarsrc"
+	origdir=$PWD
+	reponame=$(tar zxvf autoretrieved.tar.gz | tail -1 | awk -F / '{print $1}')
+	cd "/tmp/$reponame"
+	#perform compile
+	cd $origdir
+}

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy.sh

@@ -47,6 +47,19 @@ mkdir -pv /tmp/mage $HOME/go/bin
 git clone https://github.com/magefile/mage.git /tmp/mage
 cd /tmp/mage && go run bootstrap.go
 mv $HOME/go/bin/mage /usr/local/bin
+
+wget -O - -q https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s -- -b /usr/local/bin v2.2.0
+
+source "/etc/profile"
+sh -l -c "go get -u github.com/mgechev/revive"
+for file in $(ls $HOME/go/bin); do
+	mv -v $HOME/go/bin/$file /usr/local/bin/$file
+done
+
+ls -l /usr/local/bin
+revive --help
+file /usr/local/bin/revive
+
 # Cleanup after yourself
 /bin/rm -rf /tmp/mage 
 /bin/rm -rf $HOME/go