Build: Split up task in the CI pipeline to ease running outside circleci (#18861)

* build: make sign rpm packages not depend on checking out private key

* build: move commands from circleci config into verify signed packages script

* build: split update and publish of deb and rpm into two scripts

* use files argument for sign and verify packages

* validate files argument for sign and verify packages

* update test publish of deb/rpm readme

.circleci/config.yml

@@ -215,14 +215,14 @@ jobs:
           name: build and package grafana
           name: build and package grafana
           command: './scripts/build/build-all.sh'
           command: './scripts/build/build-all.sh'
       - run:
       - run:
+          name: Prepare GPG private key
+          command: './scripts/build/prepare_signing_key.sh'
+      - run:
           name: sign packages
           name: sign packages
-          command: './scripts/build/sign_packages.sh'
+          command: './scripts/build/sign_packages.sh dist/*.rpm'
       - run:
       - run:
           name: verify signed packages
           name: verify signed packages
-          command: |
-            mkdir -p ~/.rpmdb/pubkeys
-            curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key
-            ./scripts/build/verify_signed_packages.sh dist/*.rpm
+          command: './scripts/build/verify_signed_packages.sh dist/*.rpm'
       - run:
       - run:
           name: sha-sum packages
           name: sha-sum packages
           command: 'go run build.go sha-dist'
           command: 'go run build.go sha-dist'
@@ -250,8 +250,11 @@ jobs:
           name: build and package grafana
           name: build and package grafana
           command: './scripts/build/build.sh'
           command: './scripts/build/build.sh'
       - run:
       - run:
+          name: Prepare GPG private key
+          command: './scripts/build/prepare_signing_key.sh'
+      - run:
           name: sign packages
           name: sign packages
-          command: './scripts/build/sign_packages.sh'
+          command: './scripts/build/sign_packages.sh dist/*.rpm'
       - run:
       - run:
           name: sha-sum packages
           name: sha-sum packages
           command: 'go run build.go sha-dist'
           command: 'go run build.go sha-dist'
@@ -361,8 +364,11 @@ jobs:
           name: package grafana
           name: package grafana
           command: './scripts/build/build.sh --fast --package-only'
           command: './scripts/build/build.sh --fast --package-only'
       - run:
       - run:
+          name: Prepare GPG private key
+          command: './scripts/build/prepare_signing_key.sh'
+      - run:
           name: sign packages
           name: sign packages
-          command: './scripts/build/sign_packages.sh'
+          command: './scripts/build/sign_packages.sh dist/*.rpm'
       - run:
       - run:
           name: sha-sum packages
           name: sha-sum packages
           command: 'go run build.go sha-dist'
           command: 'go run build.go sha-dist'
@@ -436,8 +442,11 @@ jobs:
           name: build and package enterprise
           name: build and package enterprise
           command: './scripts/build/build.sh -enterprise'
           command: './scripts/build/build.sh -enterprise'
       - run:
       - run:
+          name: Prepare GPG private key
+          command: './scripts/build/prepare_signing_key.sh'
+      - run:
           name: sign packages
           name: sign packages
-          command: './scripts/build/sign_packages.sh'
+          command: './scripts/build/sign_packages.sh dist/*.rpm'
       - run:
       - run:
           name: sha-sum packages
           name: sha-sum packages
           command: 'go run build.go sha-dist'
           command: 'go run build.go sha-dist'
@@ -477,14 +486,14 @@ jobs:
         name: build and package grafana
         name: build and package grafana
         command: './scripts/build/build-all.sh -enterprise'
         command: './scripts/build/build-all.sh -enterprise'
     - run:
     - run:
+        name: Prepare GPG private key
+        command: './scripts/build/prepare_signing_key.sh'
+    - run:
         name: sign packages
         name: sign packages
-        command: './scripts/build/sign_packages.sh'
+        command: './scripts/build/sign_packages.sh dist/*.rpm'
     - run:
     - run:
         name: verify signed packages
         name: verify signed packages
-        command: |
-          mkdir -p ~/.rpmdb/pubkeys
-          curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key
-          ./scripts/build/verify_signed_packages.sh dist/*.rpm
+        command: './scripts/build/verify_signed_packages.sh dist/*.rpm'
     - run:
     - run:
         name: sha-sum packages
         name: sha-sum packages
         command: 'go run build.go sha-dist'
         command: 'go run build.go sha-dist'
@@ -538,14 +547,23 @@ jobs:
           name: Deploy to Grafana.com
           name: Deploy to Grafana.com
           command: './scripts/build/publish.sh --enterprise'
           command: './scripts/build/publish.sh --enterprise'
       - run:
       - run:
+          name: Prepare GPG private key
+          command: './scripts/build/prepare_signing_key.sh'
+      - run:
           name: Load GPG private key
           name: Load GPG private key
-          command: './scripts/build/load-signing-key.sh'
+          command: './scripts/build/update_repo/load-signing-key.sh'
       - run:
       - run:
           name: Update Debian repository
           name: Update Debian repository
           command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
           command: './scripts/build/update_repo/update-deb.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
       - run:
       - run:
+          name: Publish Debian repository
+          command: './scripts/build/update_repo/publish-deb.sh "enterprise"'
+      - run:
           name: Update RPM repository
           name: Update RPM repository
           command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
           command: './scripts/build/update_repo/update-rpm.sh "enterprise" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "enterprise-dist"'
+      - run:
+          name: Publish RPM repository
+          command: './scripts/build/update_repo/publish-rpm.sh "enterprise" "$CIRCLE_TAG"'
 
 
 
 
   deploy-master:
   deploy-master:
@@ -592,14 +610,23 @@ jobs:
           name: Deploy to Grafana.com
           name: Deploy to Grafana.com
           command: './scripts/build/publish.sh'
           command: './scripts/build/publish.sh'
       - run:
       - run:
+          name: Prepare GPG private key
+          command: './scripts/build/prepare_signing_key.sh'
+      - run:
           name: Load GPG private key
           name: Load GPG private key
-          command: './scripts/build/load-signing-key.sh'
+          command: './scripts/build/update_repo/load-signing-key.sh'
       - run:
       - run:
           name: Update Debian repository
           name: Update Debian repository
           command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
           command: './scripts/build/update_repo/update-deb.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
       - run:
       - run:
+          name: Publish Debian repository
+          command: './scripts/build/update_repo/publish-deb.sh "oss"'
+      - run:
           name: Update RPM repository
           name: Update RPM repository
           command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
           command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
+      - run:
+          name: Publish RPM repository
+          command: './scripts/build/update_repo/publish-rpm.sh "oss" "$CIRCLE_TAG"'
 
 
   build-oss-msi:
   build-oss-msi:
     docker:
     docker:

scripts/build/prepare_signing_key.sh

+#!/bin/bash
+
+set -e
+
+git clone git@github.com:torkelo/private.git ~/private-repo
+cp ~/private-repo/signing/private.key /private.key

scripts/build/sign_packages.sh

 #!/bin/bash
 #!/bin/bash
 
 
-git clone git@github.com:torkelo/private.git ~/private-repo
+set -e
 
 
-gpg --allow-secret-key-import --import ~/private-repo/signing/private.key
+_files=$*
+
+if [ -z "$_files" ]; then
+    echo "_files (arg 1) has to be set"
+    exit 1
+fi
+
+if [ -z "$GPG_KEY_PASSWORD" ]; then
+    echo "GPG_KEY_PASSWORD has to be set"
+    exit 1
+fi
+
+gpg --allow-secret-key-import --import /private.key
 
 
 cp ./scripts/build/rpmmacros ~/.rpmmacros
 cp ./scripts/build/rpmmacros ~/.rpmmacros
 
 
-for package in dist/*.rpm; do
+for package in $_files; do
     [ -e "$package" ] || continue
     [ -e "$package" ] || continue
     ./scripts/build/sign_expect "$GPG_KEY_PASSWORD" "$package"
     ./scripts/build/sign_expect "$GPG_KEY_PASSWORD" "$package"
 done
 done

scripts/build/update_repo/README.md

@@ -7,22 +7,23 @@
 It's possible to test the repo updates for rpm and deb by running the test scripts within a docker container like this. Tests are being executed by using two buckets on gcp setup for testing.
 It's possible to test the repo updates for rpm and deb by running the test scripts within a docker container like this. Tests are being executed by using two buckets on gcp setup for testing.
 
 
 ```bash
 ```bash
-docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.0 bash # 1.2.0 is the newest image at the time of writing
+docker run -ti --rm -u 0:0 grafana/grafana-ci-deploy:1.2.2 bash # 1.2.2 is the newest image at the time of writing
 # in the container:
 # in the container:
-mkdir -p /go/src/github.com/grafana/dist
-cd /go/src/github.com/grafana
+mkdir -p /dist
 
 
 #outside of container:
 #outside of container:
 cd <grafana project dir>/..
 cd <grafana project dir>/..
-docker cp grafana <container_name>:/go/src/github.com/grafana/.
+docker cp grafana <container_name>:/
 docker cp <gpg.key used for signing> <container_name>:/private.key
 docker cp <gpg.key used for signing> <container_name>:/private.key
 
 
 #in container:
 #in container:
-gpg --batch --allow-secret-key-import --import /private.key
+./scripts/build/update_repo/load-signing-key.sh
 cd dist && wget https://dl.grafana.com/oss/release/grafana_5.4.3_amd64.deb && wget https://dl.grafana.com/oss/release/grafana-5.4.3-1.x86_64.rpm && cd ..
 cd dist && wget https://dl.grafana.com/oss/release/grafana_5.4.3_amd64.deb && wget https://dl.grafana.com/oss/release/grafana-5.4.3-1.x86_64.rpm && cd ..
 
 
-#run these scripts:
-./script/build/update_repo/test-update-deb-repo.sh <gpg key password>
-./script/build/update_repo/test-update-rpm-repo.sh <gpg key password>
+#run these scripts to update local deb and rpm repos and publish them:
+./scripts/build/update_repo/test-update-deb-repo.sh <gpg key password>
+./scripts/build/update_repo/test-publish-deb-repo.sh
+./scripts/build/update_repo/test-update-rpm-repo.sh <gpg key password>
+./scripts/build/update_repo/test-publish-rpm-repo.sh
 
 
 ```
 ```

scripts/build/load-signing-key.sh → scripts/build/update_repo/load-signing-key.sh

@@ -2,6 +2,5 @@
 
 
 set -e
 set -e
 
 
-git clone git@github.com:torkelo/private.git ~/private-repo
-gpg --batch --allow-secret-key-import --import ~/private-repo/signing/private.key
+gpg --batch --allow-secret-key-import --import /private.key
 pkill gpg-agent
 pkill gpg-agent
\ No newline at end of file

scripts/build/update_repo/publish-deb.sh

+#!/usr/bin/env bash
+
+RELEASE_TYPE="${1:-}"
+GCP_DB_BUCKET="${2:-grafana-aptly-db}"
+GCP_REPO_BUCKET="${3:-grafana-repo}"
+
+if [ -z "$RELEASE_TYPE" ]; then
+    echo "RELEASE_TYPE (arg 1) has to be set"
+    exit 1
+fi
+
+if [[ "$RELEASE_TYPE" != "oss" && "$RELEASE_TYPE" != "enterprise" ]]; then
+    echo "RELEASE_TYPE (arg 1) must be either oss or enterprise."
+    exit 1
+fi
+
+set -e
+
+# Update the repo and db on gcp
+
+gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE"
+
+# Uploads the binaries before the metadata (to prevent 404's for debs)
+gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool"
+
+gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb"
+
+# usage:
+#
+# deb https://packages.grafana.com/oss/deb stable main

scripts/build/update_repo/publish-rpm.sh

+#!/usr/bin/env bash
+
+RELEASE_TYPE="${1:-}"
+RELEASE_TAG="${2:-}"
+GCP_REPO_BUCKET="${3:-grafana-repo}"
+
+REPO="rpm"
+
+if [ -z "$RELEASE_TYPE" ]; then
+    echo "RELEASE_TYPE (arg 1) has to be set"
+    exit 1
+fi
+
+if [[ "$RELEASE_TYPE" != "oss" && "$RELEASE_TYPE" != "enterprise" ]]; then
+    echo "RELEASE_TYPE (arg 1) must be either oss or enterprise."
+    exit 1
+fi
+
+if echo "$RELEASE_TAG" | grep -q "beta"; then
+    REPO="rpm-beta"
+fi
+
+set -e
+
+# Setup environment
+BUCKET="gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/$REPO"
+
+# Update the repo and db on gcp
+gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses
+gsutil -m rsync -r -d /rpm-repo "$BUCKET"
+
+# usage:
+# [grafana]
+# name=grafana
+# baseurl=https://packages.grafana.com/oss/rpm
+# repo_gpgcheck=1
+# enabled=1
+# gpgcheck=1
+# gpgkey=https://packages.grafana.com/gpg.key
+# sslverify=1
+# sslcacert=/etc/pki/tls/certs/ca-bundle.crt

scripts/build/update_repo/test-publish-deb-repo.sh

+#!/usr/bin/env bash
+
+./scripts/build/update_repo/publish-deb.sh "oss" "grafana-testing-aptly-db" "grafana-testing-repo"

scripts/build/update_repo/test-publish-rpm-repo.sh

+#!/usr/bin/env bash
+
+./scripts/build/update_repo/publish-rpm.sh "oss" "v5.4.3" "grafana-testing-repo"

scripts/build/update_repo/test-update-deb-repo.sh

@@ -2,4 +2,4 @@
 
 
 GPG_PASS=${1:-}
 GPG_PASS=${1:-}
 
 
-./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db" "grafana-testing-repo"
+./scripts/build/update_repo/update-deb.sh "oss" "$GPG_PASS" "v5.4.3" "dist" "grafana-testing-aptly-db"

scripts/build/update_repo/update-deb.sh

@@ -5,7 +5,6 @@ GPG_PASS="${2:-}"
 RELEASE_TAG="${3:-}"
 RELEASE_TAG="${3:-}"
 DIST_PATH="${4:-}"
 DIST_PATH="${4:-}"
 GCP_DB_BUCKET="${5:-grafana-aptly-db}"
 GCP_DB_BUCKET="${5:-grafana-aptly-db}"
-GCP_REPO_BUCKET="${6:-grafana-repo}"
 
 
 REPO="grafana"
 REPO="grafana"
 
 
@@ -54,15 +53,6 @@ rm /tmp/sign-this /tmp/sign-this.asc
 aptly publish update stable filesystem:repo:grafana
 aptly publish update stable filesystem:repo:grafana
 aptly publish update beta filesystem:repo:grafana
 aptly publish update beta filesystem:repo:grafana
 
 
-# Update the repo and db on gcp
-
-gsutil -m rsync -r -d /deb-repo/db "gs://$GCP_DB_BUCKET/$RELEASE_TYPE"
-
-# Uploads the binaries before the metadata (to prevent 404's for debs)
-gsutil -m rsync -r /deb-repo/repo/grafana/pool "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb/pool"
-
-gsutil -m rsync -r -d /deb-repo/repo/grafana "gs://$GCP_REPO_BUCKET/$RELEASE_TYPE/deb"
-
 # usage:
 # usage:
 #
 #
 # deb https://packages.grafana.com/oss/deb stable main
 # deb https://packages.grafana.com/oss/deb stable main

scripts/build/update_repo/update-rpm.sh

@@ -46,10 +46,6 @@ rm /rpm-repo/repodata/repomd.xml.asc || true
 pkill gpg-agent || true
 pkill gpg-agent || true
 ./scripts/build/update_repo/sign-rpm-repo.sh "$GPG_PASS"
 ./scripts/build/update_repo/sign-rpm-repo.sh "$GPG_PASS"
 
 
-# Update the repo and db on gcp
-gsutil -m cp /rpm-repo/*.rpm "$BUCKET" # sync binaries first to avoid cache misses
-gsutil -m rsync -r -d /rpm-repo "$BUCKET"
-
 # usage:
 # usage:
 # [grafana]
 # [grafana]
 # name=grafana
 # name=grafana

scripts/build/verify_signed_packages.sh

@@ -2,6 +2,14 @@
 
 
 _files=$*
 _files=$*
 
 
+if [ -z "$_files" ]; then
+    echo "_files (arg 1) has to be set"
+    exit 1
+fi
+
+mkdir -p ~/.rpmdb/pubkeys
+curl -s https://packages.grafana.com/gpg.key > ~/.rpmdb/pubkeys/grafana.key
+
 ALL_SIGNED=0
 ALL_SIGNED=0
 
 
 for file in $_files; do
 for file in $_files; do