CI: Add Google Cloud SDK to plugin CI Docker images (#27444)

* grafana-plugin-ci-alpine: Upgrading Go,golangci-lint, adding gcloud

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

* grafana-plugin-ci: Upgrading Go,golangci-lint, adding gcloud

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

* grafana-plugin-ci-alpine: Minor cleanup

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

* Don't specify depth when cloning Mage

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

* grafana-plugin-ci-e2e: Upgrading Go, golangci-lint

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

* ci-deploy: Upgrading gcloud

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

* Update packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/README.md

Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>

* grafana-plugin-ci-alpine: Install Python

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>

.drone.yml

@@ -369,7 +369,7 @@ steps:
   - package
 
 - name: publish-storybook
-  image: grafana/grafana-ci-deploy:1.2.5
+  image: grafana/grafana-ci-deploy:1.2.6
   commands:
   - printenv GCP_KEY | base64 -d > /tmp/gcpkey.json
   - gcloud auth activate-service-account --key-file=/tmp/gcpkey.json
@@ -459,7 +459,7 @@ steps:
   - end-to-end-tests
 
 - name: publish-packages
-  image: grafana/grafana-ci-deploy:1.2.5
+  image: grafana/grafana-ci-deploy:1.2.6
   commands:
   - echo ./bin/grabpl publish-packages --edition oss
   depends_on:

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/Dockerfile

 FROM alpine:3.12
+
 USER root
-ADD scripts scripts
-ADD install /usr/local
+
+COPY scripts scripts
+COPY install /usr/local
+
 WORKDIR scripts
+
 RUN ./deploy.sh

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/README.md

@@ -14,7 +14,7 @@ The home directory will be `/home/circleci`
 - npm is installed globally
 
 ## Go
-- Go 1.14 is installed in `/usr/local/bin/go`
+- Go is installed in `/usr/local/bin/go`
 - golangci-lint is installed in `/usr/local/bin/golangci-lint`
 - mage is installed in `/home/circleci/go/bin/mage`
 
@@ -45,7 +45,7 @@ To test, your CircleCI config will need a run section with something similar to
 
 
 # Building
-To build, cd to `<srcroot>/packages/grafana-toolkit/docker/grafana-plugin-ci`
+To build, cd to `<srcroot>/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine`
 ```
 ./build.sh
 ```

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/build.sh

@@ -4,7 +4,7 @@ set -eo pipefail
 source ./common.sh
 
 #
-# No longer required, but useful to keep just in case we want to deploy 
+# No longer required, but useful to keep just in case we want to deploy
 # changes in toolkit directly to the docker image
 #
 if [ -n "$INCLUDE_TOOLKIT" ]; then
@@ -16,12 +16,7 @@ if [ -n "$INCLUDE_TOOLKIT" ]; then
 	cp -v ../../tsconfig.json install/grafana-toolkit
 fi
 
-
-output=$(docker build . | tee /dev/tty)
-hash=$(echo "$output" | tail -1 | sed -ne "s/^Successfully built \(.*\)/\1/p")
-if [ ${#hash} -gt 0 ]; then
-	docker tag "$hash" $DOCKER_IMAGE_NAME:latest
-	docker push $DOCKER_IMAGE_NAME:latest
-fi
+docker build -t ${DOCKER_IMAGE_NAME} .
+docker push $DOCKER_IMAGE_NAME
 
 [ -n "$INCLUDE_TOOLKIT" ] && /bin/rm -rfv install/grafana-toolkit

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/common.sh

@@ -4,4 +4,4 @@
 ## Common variable declarations
 ##
 
-DOCKER_IMAGE_NAME="grafana/grafana-plugin-ci:latest-alpine"
+DOCKER_IMAGE_NAME="grafana/grafana-plugin-ci:1.0.1-alpine"

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy-common.sh

@@ -23,8 +23,8 @@ get_file () {
     [ -n "$2" ] && dest=$2 || do_exit "destination required" 2
     sha=$3
     file=$(basename $dest)
-    
-    wget "$url" -O "$dest"
+
+    curl -fL "${url}" -o "$dest"
     if [ -n "$sha" ]; then
         echo "$sha  $dest" | sha256sum || do_exit "Checksum validation failed for $file. Exiting" 1
     fi
@@ -43,7 +43,7 @@ untar_file () {
 # compile, and install
 get_latest_release () {
 	tarsrc=$(curl -sL "https://api.github.com/repos/$1/$2/releases/latest" | jq ".tarball_url" | tr -d '"')
-	wget -O /tmp/autoretrieved.tar.gz "$tarsrc"
+	curl -fL -o /tmp/autoretrieved.tar.gz "$tarsrc"
 	origdir=$PWD
 	reponame=$(tar zxvf autoretrieved.tar.gz | tail -1 | awk -F / '{print $1}')
 	cd "/tmp/$reponame"

packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy.sh

 #!/bin/sh
 set -eo pipefail
+
 source "./deploy-common.sh"
 
 # Make libgcc compatible
@@ -9,8 +10,7 @@ mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2
 rm /bin/cp
 mv /usr/local/bin/cp /bin/cp
 
-sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories
-apk add --no-cache nodejs npm yarn build-base openssh git-lfs perl-utils
+apk add --no-cache curl nodejs npm yarn build-base openssh git-lfs perl-utils coreutils python3
 
 #
 # Only relevant for testing, but cypress does not work with musl/alpine.
@@ -18,17 +18,17 @@ apk add --no-cache nodejs npm yarn build-base openssh git-lfs perl-utils
 # apk add --no-cache xvfb glib nss nspr gdk-pixbuf "gtk+3.0" pango atk cairo dbus-libs libxcomposite libxrender libxi libxtst libxrandr libxscrnsaver alsa-lib at-spi2-atk at-spi2-core cups-libs gcompat libc6-compat
 
 # Install Go
-filename="go1.14.linux-amd64.tar.gz"
-get_file "https://dl.google.com/go/$filename" "/tmp/$filename" "08df79b46b0adf498ea9f320a0f23d6ec59e9003660b4c9c1ce8e5e2c6f823ca"
+filename="go1.15.1.linux-amd64.tar.gz"
+get_file "https://dl.google.com/go/$filename" "/tmp/$filename" "70ac0dbf60a8ee9236f337ed0daa7a4c3b98f6186d4497826f68e97c0c0413f6"
 untar_file "/tmp/$filename"
 
 # Install golangci-lint
-GOLANGCILINT_VERSION=1.28.0
+GOLANGCILINT_VERSION=1.31.0
 filename="golangci-lint-${GOLANGCILINT_VERSION}-linux-amd64"
 get_file "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCILINT_VERSION}/$filename.tar.gz" \
-    "/tmp/$filename.tar.gz" \
-    "179d34edf4baf6454a7081fbaaf74dc99397a3be8e1a535dee04d835a977bf76"
-untar_file "/tmp/$filename.tar.gz"
+    "/tmp/${filename}.tar.gz" \
+    "9a5d47b51442d68b718af4c7350f4406cdc087e2236a5b9ae52f37aebede6cb3"
+untar_file "/tmp/${filename}.tar.gz"
 ln -s /usr/local/${filename}/golangci-lint /usr/local/bin/golangci-lint
 ln -s /usr/local/go/bin/go /usr/local/bin/go
 ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
@@ -43,13 +43,12 @@ get_file "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-l
     "b4138199aa755ebfe171b57cc46910b13258ace5fbc4eaa099c42607cd0bff32"
 chmod +x /usr/local/bin/cc-test-reporter
 
-wget -O /usr/local/bin/grabpl "https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.7/grabpl"
-chmod +x /usr/local/bin/grabpl
+curl -fL -o /usr/local/bin/grabpl "https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.7/grabpl"
 
 apk add --no-cache git
 # Install Mage
 mkdir -pv /tmp/mage $HOME/go/bin
-git clone --depth 1 https://github.com/magefile/mage.git /tmp/mage
+git clone https://github.com/magefile/mage.git /tmp/mage
 cd /tmp/mage && go run bootstrap.go
 mv $HOME/go/bin/mage /usr/local/bin
 
@@ -66,6 +65,16 @@ current_dir=$PWD
 cd /usr/local/grafana-toolkit && yarn install && cd $current_dir
 ln -s /usr/local/grafana-toolkit/bin/grafana-toolkit.js /usr/local/bin/grafana-toolkit
 
+GOOGLE_SDK_VERSION=308.0.0
+GOOGLE_SDK_CHECKSUM=9e8e31d9503340fc912374311ac1fffbfc5b59748d20b681f9aca3de2b68deb5
+
+curl -fLO https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz
+echo "${GOOGLE_SDK_CHECKSUM} google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz" | sha256sum --check --status
+tar xvzf google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz -C /opt
+rm google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz
+ln -s /opt/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
+ln -s /opt/google-cloud-sdk/bin/gcloud /usr/bin/gcloud
+
 # Cleanup after yourself
 /bin/rm -rf /tmp/mage
 /bin/rm -rf $HOME/go

packages/grafana-toolkit/docker/grafana-plugin-ci-e2e/Dockerfile

-FROM debian:buster-slim 
-USER root 
-ADD scripts scripts
-ADD install /usr/local
+FROM debian:buster-slim
+
+USER root
+
+COPY scripts scripts
+COPY install /usr/local
+
 RUN cd scripts && ./deploy.sh

packages/grafana-toolkit/docker/grafana-plugin-ci-e2e/build.sh

@@ -4,7 +4,7 @@ set -eo pipefail
 source ./common.sh
 
 #
-# No longer required, but useful to keep just in case we want to deploy 
+# No longer required, but useful to keep just in case we want to deploy
 # changes in toolkit directly to the docker image
 #
 if [ -n "$INCLUDE_TOOLKIT" ]; then
@@ -16,12 +16,7 @@ if [ -n "$INCLUDE_TOOLKIT" ]; then
 	cp -v ../../tsconfig.json install/grafana-toolkit
 fi
 
-
-output=$(docker build . | tee /dev/tty)
-hash=$(echo "$output" | tail -1 | sed -ne "s/^Successfully built \(.*\)/\1/p")
-if [ ${#hash} -gt 0 ]; then
-	docker tag "$hash" $DOCKER_IMAGE_NAME:latest
-	docker push $DOCKER_IMAGE_NAME:latest
-fi
+docker build -t ${DOCKER_IMAGE_NAME} .
+docker push $DOCKER_IMAGE_NAME
 
 [ -n "$INCLUDE_TOOLKIT" ] && /bin/rm -rfv install/grafana-toolkit

packages/grafana-toolkit/docker/grafana-plugin-ci-e2e/common.sh

@@ -4,4 +4,4 @@
 ## Common variable declarations
 ##
 
-DOCKER_IMAGE_NAME="srclosson/grafana-plugin-ci-e2e"
+DOCKER_IMAGE_NAME="grafana/grafana-plugin-ci-e2e:1.0.0"

packages/grafana-toolkit/docker/grafana-plugin-ci-e2e/scripts/deploy-common.sh

@@ -11,7 +11,6 @@ do_exit() {
     exit $exit_code
 }
 
-
 ##
 # Get file, get's a file, validates the SHA
 # @param filename
@@ -23,7 +22,7 @@ get_file () {
     [ -n "$2" ] && dest=$2 || do_exit "destination required" -2
     sha=$3
     file=$(basename $dest)
-    
+
     wget "$url" -O "$dest"
     if [ -n "$sha" ]; then
         echo "$sha $dest" | sha256sum --check --status || do_exit "Checksum validation failed for $file. Exiting" -1
@@ -35,4 +34,4 @@ untar_file () {
     [ -n "$2" ] && dest=$2 || dest="/usr/local"
 
     tar -C "$dest" -xf "$src" && /bin/rm -rf "$src"
-}
\ No newline at end of file
+}

packages/grafana-toolkit/docker/grafana-plugin-ci-e2e/scripts/deploy.sh

@@ -5,9 +5,9 @@ source "/etc/profile"
 source "./deploy-slim.sh"
 source "./deploy-common.sh"
 
-NODEVER="v12.16.2-linux-x64"
+NODEVER="v12.18.3-linux-x64"
 # Install Node
-wget -O - "https://nodejs.org/dist/v12.16.2/node-${NODEVER}.tar.xz" | tar Jvxf - -C "/tmp"
+wget -O - "https://nodejs.org/dist/v12.18.3/node-${NODEVER}.tar.xz" | tar Jvxf - -C "/tmp"
 
 # Move node to /usr/local so it's in the path
 pushd /tmp/node-${NODEVER}
@@ -22,15 +22,16 @@ source "/etc/profile"
 npm i -g yarn
 
 # Install Go
-filename="go1.14.linux-amd64.tar.gz"
-get_file "https://dl.google.com/go/$filename" "/tmp/$filename" "08df79b46b0adf498ea9f320a0f23d6ec59e9003660b4c9c1ce8e5e2c6f823ca"
+filename="go1.15.1.linux-amd64.tar.gz"
+get_file "https://dl.google.com/go/$filename" "/tmp/$filename" "70ac0dbf60a8ee9236f337ed0daa7a4c3b98f6186d4497826f68e97c0c0413f6"
 untar_file "/tmp/$filename"
 
 # Install golangci-lint
-filename="golangci-lint-1.26.0-linux-amd64"
-get_file "https://github.com/golangci/golangci-lint/releases/download/v1.26.0/$filename.tar.gz" \
+GOLANGCILINT_VERSION=1.31.0
+filename="golangci-lint-${GOLANGCILINT_VERSION}-linux-amd64"
+get_file "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCILINT_VERSION}/$filename.tar.gz" \
     "/tmp/$filename.tar.gz" \
-    "59b0e49a4578fea574648a2fd5174ed61644c667ea1a1b54b8082fde15ef94fd"
+    "9a5d47b51442d68b718af4c7350f4406cdc087e2236a5b9ae52f37aebede6cb3"
 untar_file "/tmp/$filename.tar.gz"
 ln -s /usr/local/${filename}/golangci-lint /usr/local/bin/golangci-lint
 ln -s /usr/local/go/bin/go /usr/local/bin/go
@@ -48,7 +49,7 @@ chmod +x /usr/local/bin/grabpl
 
 # Install Mage
 mkdir -pv /tmp/mage $HOME/go/bin
-git clone --depth 1 https://github.com/magefile/mage.git /tmp/mage
+git clone https://github.com/magefile/mage.git /tmp/mage
 pushd /tmp/mage && go run bootstrap.go && popd
 mv $HOME/go/bin/mage /usr/local/bin
 # Cleanup after yourself

packages/grafana-toolkit/docker/grafana-plugin-ci/Dockerfile

 FROM circleci/node:12-browsers
-USER root 
-ADD scripts scripts
+USER root
+COPY scripts scripts
 WORKDIR scripts
 RUN ./deploy.sh
-ADD install/gget /usr/local/bin/gget
+COPY install/gget /usr/local/bin/gget

packages/grafana-toolkit/docker/grafana-plugin-ci/build.sh

 #!/bin/bash
+set -eo pipefail
+
 source ./common.sh
 
-output=$(docker build . | tee /dev/tty)
-hash=$(echo "$output" | tail -1 | sed -ne "s/^Successfully built \(.*\)/\1/p")
-docker tag "$hash" $DOCKER_IMAGE_NAME:latest
-docker push $DOCKER_IMAGE_NAME:latest
+docker build -t ${DOCKER_IMAGE_NAME} .
+docker push $DOCKER_IMAGE_NAME
 

packages/grafana-toolkit/docker/grafana-plugin-ci/common.sh

@@ -4,4 +4,4 @@
 ## Common variable declarations
 ##
 
-DOCKER_IMAGE_NAME="srclosson/grafana-plugin-ci"
\ No newline at end of file
+DOCKER_IMAGE_NAME="grafana/grafana-plugin-ci:1.0.0"

packages/grafana-toolkit/docker/grafana-plugin-ci/scripts/deploy.sh

@@ -3,20 +3,20 @@ source "./deploy-common.sh"
 
 # Install Go
 filename="go1.15.1.linux-amd64.tar.gz"
-get_file "https://dl.google.com/go/$filename" "/tmp/$filename" "08df79b46b0adf498ea9f320a0f23d6ec59e9003660b4c9c1ce8e5e2c6f823ca"
+get_file "https://dl.google.com/go/$filename" "/tmp/$filename" "70ac0dbf60a8ee9236f337ed0daa7a4c3b98f6186d4497826f68e97c0c0413f6"
 untar_file "/tmp/$filename"
 
-
 # Install golangci-lint
-filename="golangci-lint-1.23.7-linux-amd64.tar.gz"
-get_file "https://github.com/golangci/golangci-lint/releases/download/v1.23.7/$filename" \
-    "/tmp/$filename" \
-    "34df1794a2ea8e168b3c98eed3cc0f3e13ed4cba735e4e40ef141df5c41bc086"
-untar_file "/tmp/$filename"
-chmod 755 /usr/local/bin/golangci-lint
-ln -s /usr/local/golangci-lint-1.23.7-linux-amd64/golangci-lint /usr/local/bin/golangci-lint
+GOLANGCILINT_VERSION=1.31.0
+filename="golangci-lint-${GOLANGCILINT_VERSION}-linux-amd64"
+get_file "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCILINT_VERSION}/$filename.tar.gz" \
+    "/tmp/${filename}.tar.gz" \
+    "9a5d47b51442d68b718af4c7350f4406cdc087e2236a5b9ae52f37aebede6cb3"
+untar_file "/tmp/${filename}.tar.gz"
+ln -s /usr/local/${filename}/golangci-lint /usr/local/bin/golangci-lint
 ln -s /usr/local/go/bin/go /usr/local/bin/go
 ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
+chmod 755 /usr/local/bin/golangci-lint
 
 # Install dependencies
 apt-get update -y && apt-get install -y adduser libfontconfig1 locate && /bin/rm -rf /var/lib/apt/lists/*
@@ -32,9 +32,20 @@ chmod +x /usr/local/bin/grabpl
 
 # Install Mage
 mkdir -pv /tmp/mage $HOME/go/bin
-git clone --depth 1 https://github.com/magefile/mage.git /tmp/mage
+git clone https://github.com/magefile/mage.git /tmp/mage
 pushd /tmp/mage && go run bootstrap.go && popd
 mv $HOME/go/bin/mage /usr/local/bin
+
+GOOGLE_SDK_VERSION=308.0.0
+GOOGLE_SDK_CHECKSUM=9e8e31d9503340fc912374311ac1fffbfc5b59748d20b681f9aca3de2b68deb5
+
+curl -fLO https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz
+echo "${GOOGLE_SDK_CHECKSUM} google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz" | sha256sum --check --status
+tar xvzf google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz -C /opt
+rm google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz
+ln -s /opt/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
+ln -s /opt/google-cloud-sdk/bin/gcloud /usr/bin/gcloud
+
 # Cleanup after yourself
 /bin/rm -rf /tmp/mage
 /bin/rm -rf $HOME/go

scripts/build/ci-deploy/Dockerfile

@@ -10,9 +10,14 @@ FROM circleci/python:2.7-stretch-node
 
 USER root
 
+ARG GOOGLE_SDK_VERSION=308.0.0
+ARG GOOGLE_SDK_CHECKSUM=9e8e31d9503340fc912374311ac1fffbfc5b59748d20b681f9aca3de2b68deb5
+
 RUN pip install -U awscli crcmod && \
-    curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-222.0.0-linux-x86_64.tar.gz | \
-      tar xvzf - -C /opt && \
+    curl -fLO https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz && \
+    echo "${GOOGLE_SDK_CHECKSUM} google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz" | sha256sum --check --status && \
+    tar xzf google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz -C /opt && \
+    rm google-cloud-sdk-${GOOGLE_SDK_VERSION}-linux-x86_64.tar.gz && \
     apt update && \
     apt install -y createrepo expect && \
     apt-get autoremove -y && \

scripts/build/ci-deploy/build-deploy.sh

 #!/bin/bash
+set -eo pipefail
 
-_version="1.2.5"
+_version="1.2.6"
 _tag="grafana/grafana-ci-deploy:${_version}"
 
 docker build -t $_tag .

scripts/lib.star

 build_image = 'grafana/build-container:1.2.26'
-publish_image = 'grafana/grafana-ci-deploy:1.2.5'
+publish_image = 'grafana/grafana-ci-deploy:1.2.6'
 grafana_docker_image = 'grafana/drone-grafana-docker:0.2.0'
 alpine_image = 'alpine:3.12'
 windows_image = 'mcr.microsoft.com/windows:1809'