Skip to content

N
nexpie-grafana-theme
Commit d2b957b7 by Arve Knudsen Committed by GitHub
Options

Drone: Add release pipelines (#28083) 

* Drone: Implement release pipeline

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

* Drone: Implement test-release pipeline

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
parent 61749dee
Showing with 424 additions and 111 deletions
.circleci/config.yml
......@@ -2,11 +2,6 @@ version: 2.1
aliases:
# Workflow filters
- &filter-only-release
branches:
only: chore/test-release-pipeline
tags:
only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
- &filter-only-master
branches:
only: master
......@@ -110,13 +105,6 @@ jobs:
resource_class: large
executor: grafana-build
steps:
- run:
name: Exit if enterprise and forked PR
command: |
if [[ "<< parameters.edition >>" == "enterprise" && -n "$CIRCLE_PR_NUMBER" ]]; then
echo "Nothing to do for forked PRs, so marking this step successful"
circleci step halt
fi
- checkout
- restore_cache:
keys:
......@@ -188,13 +176,8 @@ jobs:
- run:
name: Build internal Grafana plug-ins
command: |
if [[ -n "$CIRCLE_PR_NUMBER" ]]; then
# This is a forked PR, so don't sign as it requires an API secret
/tmp/grabpl build-plugins --jobs 2 --edition << parameters.edition >>
else
export GRAFANA_API_KEY=$GRAFANA_COM_API_KEY
/tmp/grabpl build-plugins --jobs 2 --edition << parameters.edition >> --sign --signing-admin
fi
export GRAFANA_API_KEY=$GRAFANA_COM_API_KEY
/tmp/grabpl build-plugins --jobs 2 --edition << parameters.edition >> --sign --signing-admin
- run:
name: Move artifacts
command: |
......@@ -341,38 +324,6 @@ jobs:
command: "./scripts/ci-job-succeeded.sh"
when: on_success
release-next-packages:
executor: grafana-build
steps:
- run:
name: Exit if forked PR
command: |
if [[ -n "$CIRCLE_PR_NUMBER" ]]; then
echo "Nothing to do for forked PRs, so marking this step successful"
circleci step halt
fi
- checkout
- run:
name: CI job started
command: ./scripts/ci-job-started.sh
- run:
name: Bootstrap lerna
command: npx lerna bootstrap
- run:
name: npm - Prepare auth token
command: "echo //registry.npmjs.org/:_authToken=$NPM_TOKEN >> ~/.npmrc"
- run:
name: Release next packages
command: ./scripts/circle-release-next-packages.sh
- run:
name: CI job failed
command: ./scripts/ci-job-failed.sh
when: on_fail
- run:
name: CI job succeeded
command: ./scripts/ci-job-succeeded.sh
when: on_success
package-oss:
executor: grafana-build
steps:
......@@ -389,11 +340,6 @@ jobs:
- run:
name: Package Grafana
command: |
if [[ -n "$CIRCLE_PR_NUMBER" ]]; then
echo Using test GPG key pair since building a forked PR
source scripts/build/gpg-test-vars.sh
fi
# Necessary for signing bundled plugins
export GRAFANA_API_KEY=$GRAFANA_COM_API_KEY
if [[ -n $CIRCLE_TAG ]]; then
......@@ -528,11 +474,6 @@ jobs:
yarn install --frozen-lockfile --no-progress
yarn storybook:build
if [[ -n "$CIRCLE_PR_NUMBER" ]]; then
echo "Nothing to do for forked PRs, so marking this step successful"
circleci step halt
fi
if [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then
# We're testing the release pipeline
echo Testing release
......@@ -830,9 +771,6 @@ jobs:
elif [[ $CIRCLE_BRANCH == "chore/test-release-pipeline" ]]; then
# We're testing the release pipeline
./scripts/circle-test-frontend.sh --edition << parameters.edition >> v7.0.0-test
else
# A master build
./scripts/circle-test-frontend.sh --edition << parameters.edition >> --build-id $CIRCLE_WORKFLOW_ID
fi
- store_test_results:
path: reports/junit
......@@ -867,10 +805,6 @@ jobs:
# We're testing the release pipeline
/tmp/grabpl test-backend --github-token "${GITHUB_GRAFANABOT_TOKEN}" --edition << parameters.edition >> \
v7.0.0-test
else
# A master build
/tmp/grabpl test-backend --github-token "${GITHUB_GRAFANABOT_TOKEN}" --edition << parameters.edition >> \
--build-id $CIRCLE_WORKFLOW_ID
fi
- run:
name: CI job failed
......@@ -881,45 +815,9 @@ jobs:
command: "./scripts/ci-job-succeeded.sh"
when: on_success
build-docs-website:
executor: base
steps:
- checkout
- setup_remote_docker
- run:
name: CI job started
command: "./scripts/ci-job-started.sh"
- run:
name: Build Grafana docs website
command: |
# Use latest revision here, since we want to catch if it breaks
IMAGE=grafana/docs-base:latest
# In order to copy sources into the remote container, we need to employ a trick of creating a container
# with a volume, that we copy the sources into. Then, we launch the build container, with the volume
# from the other container
docker create -v /hugo/content/docs/grafana --name docs alpine:3.11 /bin/true
docker cp ${PWD}/docs/sources docs:/hugo/content/docs/grafana/latest
docker run --volumes-from docs $IMAGE /bin/bash -c 'make prod'
- run:
name: CI job failed
command: "./scripts/ci-job-failed.sh"
when: on_fail
- run:
name: CI job succeeded
command: "./scripts/ci-job-succeeded.sh"
when: on_success
release-packages:
executor: node
steps:
- run:
name: Exit if forked PR
command: |
if [[ -n "$CIRCLE_PR_NUMBER" ]]; then
echo "Nothing to do for forked PRs, so marking this step successful"
circleci step halt
fi
- checkout
- run:
name: CI job started
......@@ -1223,7 +1121,7 @@ workflows:
requires:
- package-enterprise
- release-packages:
filters: *filter-only-release
filters: *filter-release
requires:
- end-to-end-tests
- mysql-integration-test
......
.drone.star
load('scripts/lib.star', 'pr_pipelines', 'master_pipelines')
load('scripts/pr.star', 'pr_pipelines')
load('scripts/master.star', 'master_pipelines')
load('scripts/release.star', 'release_pipelines', 'test_release_pipelines')
def main(ctx):
edition = 'oss'
return pr_pipelines(edition=edition) + master_pipelines(edition=edition)
return pr_pipelines(edition=edition) + master_pipelines(edition=edition) + release_pipelines() + \
test_release_pipelines()
scripts/circle-release-next-packages.sh
......@@ -7,7 +7,7 @@ PACKAGES=(ui toolkit data runtime e2e e2e-selectors)
source "$(dirname "$0")/helpers/exit-if-fail.sh"
function prepare_version_commit () {
echo $'\nCommitting version changes. This commit will not be checked-in!'
echo $'\nCommitting version changes. This commit will not be pushed!'
git config --global user.email "drone@grafana.com"
git config --global user.name "Drone"
git commit -am "Version commit"
......@@ -48,9 +48,9 @@ else
# repository. We will not push this commit to the origin repository.
prepare_version_commit
# Frontend packages have already been versioned and built by the
# build-frontend step in drone. We will only unpublish the previous
# canary version and publish the current built version as the new
# Frontend packages have already been versioned and built by the
# build-frontend step in drone. We will only unpublish the previous
# canary version and publish the current built version as the new
# latest canary build.
for PACKAGE in "${PACKAGES[@]}"
do
......
scripts/master.star 0 → 100644
load(
'scripts/lib.star',
'pipeline',
'lint_backend_step',
'codespell_step',
'shellcheck_step',
'test_backend_step',
'test_frontend_step',
'build_backend_step',
'build_frontend_step',
'build_plugins_step',
'package_step',
'e2e_tests_server_step',
'e2e_tests_step',
'build_storybook_step',
'build_docs_website_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'get_windows_steps',
'benchmark_ldap_step',
'ldap_service',
'enterprise_downstream_step',
'frontend_metrics_step',
'publish_storybook_step',
'release_next_npm_packages_step',
'upload_packages_step',
'deploy_to_kubernetes_step',
'publish_packages_step',
'notify_pipeline',
'integration_test_services',
)
ver_mode = 'master'
def get_steps(edition, is_downstream=False):
publish = edition != 'enterprise' or is_downstream
steps = [
enterprise_downstream_step(edition),
lint_backend_step(edition),
codespell_step(),
shellcheck_step(),
test_backend_step(),
test_frontend_step(),
frontend_metrics_step(edition=edition),
build_backend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_plugins_step(edition=edition, sign=True),
package_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
e2e_tests_server_step(),
e2e_tests_step(),
build_storybook_step(edition=edition, ver_mode=ver_mode),
publish_storybook_step(edition=edition, ver_mode=ver_mode),
build_docs_website_step(),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=publish),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=publish),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
release_next_npm_packages_step(edition),
upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
deploy_to_kubernetes_step(edition=edition, is_downstream=is_downstream),
]
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream)
publish_steps = [
publish_packages_step(edition=edition, is_downstream=is_downstream),
]
return steps, windows_steps, publish_steps
def master_pipelines(edition):
services = integration_test_services()
trigger = {
'event': ['push',],
'branch': 'master',
}
steps, windows_steps, publish_steps = get_steps(edition=edition)
if edition == 'enterprise':
steps.append(benchmark_ldap_step())
services.append(ldap_service())
pipelines = [
pipeline(
name='build-master', edition=edition, trigger=trigger, services=services, steps=steps,
ver_mode=ver_mode,
),
pipeline(
name='windows-master', edition=edition, trigger=trigger, steps=windows_steps, platform='windows',
depends_on=['build-master'], ver_mode=ver_mode,
),
]
if edition != 'enterprise':
pipelines.append(pipeline(
name='publish-master', edition=edition, trigger=trigger, steps=publish_steps,
depends_on=['build-master', 'windows-master',], install_deps=False, ver_mode=ver_mode,
))
pipelines.append(notify_pipeline(
name='notify-master', slack_channel='grafana-ci-notifications', trigger=trigger,
depends_on=['build-master', 'windows-master', 'publish-master'],
))
else:
# Add downstream enterprise pipelines triggerable from OSS builds
trigger = {
'event': ['custom',],
}
steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True)
pipelines.append(pipeline(
name='build-master-downstream', edition=edition, trigger=trigger, services=services, steps=steps,
is_downstream=True, ver_mode=ver_mode,
))
pipelines.append(pipeline(
name='windows-master-downstream', edition=edition, trigger=trigger, steps=windows_steps,
platform='windows', depends_on=['build-master-downstream'], is_downstream=True, ver_mode=ver_mode,
))
pipelines.append(pipeline(
name='publish-master-downstream', edition=edition, trigger=trigger, steps=publish_steps,
depends_on=['build-master-downstream', 'windows-master-downstream'], is_downstream=True, install_deps=False,
ver_mode=ver_mode,
))
pipelines.append(notify_pipeline(
name='notify-master-downstream', slack_channel='grafana-enterprise-ci-notifications', trigger=trigger,
depends_on=['build-master-downstream', 'windows-master-downstream', 'publish-master-downstream'],
))
return pipelines
scripts/pr.star 0 → 100644
load(
'scripts/lib.star',
'pipeline',
'lint_backend_step',
'codespell_step',
'shellcheck_step',
'test_backend_step',
'test_frontend_step',
'build_backend_step',
'build_frontend_step',
'build_plugins_step',
'package_step',
'e2e_tests_server_step',
'e2e_tests_step',
'build_storybook_step',
'build_docs_website_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'get_windows_steps',
'benchmark_ldap_step',
'ldap_service',
'integration_test_services',
)
ver_mode = 'pr'
def pr_pipelines(edition):
services = integration_test_services()
variants = ['linux-x64', 'linux-x64-musl', 'osx64', 'win64',]
steps = [
lint_backend_step(edition),
codespell_step(),
shellcheck_step(),
test_backend_step(),
test_frontend_step(),
build_backend_step(edition=edition, ver_mode=ver_mode, variants=variants),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition),
package_step(edition=edition, ver_mode=ver_mode, variants=variants),
e2e_tests_server_step(),
e2e_tests_step(),
build_storybook_step(edition=edition, ver_mode=ver_mode),
build_docs_website_step(),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64',]),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
]
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
if edition == 'enterprise':
steps.append(benchmark_ldap_step())
services.append(ldap_service())
trigger = {
'event': ['pull_request',],
}
return [
pipeline(
name='test-pr', edition=edition, trigger=trigger, services=services, steps=steps,
ver_mode=ver_mode,
),
pipeline(
name='windows-pr', edition=edition, trigger=trigger, steps=windows_steps, platform='windows',
ver_mode=ver_mode,
),
]
scripts/release.star 0 → 100644
load(
'scripts/lib.star',
'test_release_ver',
'build_image',
'publish_image',
'pipeline',
'lint_backend_step',
'codespell_step',
'shellcheck_step',
'test_backend_step',
'test_frontend_step',
'build_backend_step',
'build_frontend_step',
'build_plugins_step',
'package_step',
'e2e_tests_server_step',
'e2e_tests_step',
'build_storybook_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'get_windows_steps',
'benchmark_ldap_step',
'ldap_service',
'frontend_metrics_step',
'publish_storybook_step',
'upload_packages_step',
'publish_packages_step',
'notify_pipeline',
'integration_test_services',
)
def release_npm_packages_step(edition, ver_mode):
if edition == 'enterprise':
return None
commands = [
'./node_modules/.bin/lerna bootstrap',
'echo "//registry.npmjs.org/:_authToken=$${NPM_TOKEN}" >> ~/.npmrc',
]
if ver_mode == 'release':
commands.append('./scripts/build/release-packages.sh ${DRONE_TAG}')
return {
'name': 'release-npm-packages',
'image': build_image,
'depends_on': [
'end-to-end-tests',
],
'environment': {
'NPM_TOKEN': {
'from_secret': 'npm_token',
},
},
'commands': commands,
}
def get_steps(edition, ver_mode):
steps = [
lint_backend_step(edition),
codespell_step(),
shellcheck_step(),
test_backend_step(),
test_frontend_step(),
build_backend_step(edition=edition, ver_mode=ver_mode),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition, sign=True),
package_step(edition=edition, ver_mode=ver_mode),
e2e_tests_server_step(),
e2e_tests_step(),
build_storybook_step(edition=edition, ver_mode=ver_mode),
publish_storybook_step(edition=edition, ver_mode=ver_mode),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=True),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=True),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
release_npm_packages_step(edition=edition, ver_mode=ver_mode),
upload_packages_step(edition=edition, ver_mode=ver_mode),
]
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
return steps, windows_steps
def get_oss_pipelines(trigger, ver_mode):
services = integration_test_services()
steps, windows_steps = get_steps(edition='oss', ver_mode=ver_mode)
return [
pipeline(
name='oss-build-{}'.format(ver_mode), edition='oss', trigger=trigger, services=services, steps=steps,
ver_mode=ver_mode,
),
pipeline(
name='oss-windows-{}'.format(ver_mode), edition='oss', trigger=trigger, steps=windows_steps,
platform='windows', depends_on=['oss-build-{}'.format(ver_mode)], ver_mode=ver_mode,
),
]
def get_enterprise_pipelines(trigger, ver_mode):
services = integration_test_services()
steps, windows_steps = get_steps(edition='enterprise', ver_mode=ver_mode)
return [
pipeline(
name='enterprise-build-{}'.format(ver_mode), edition='enterprise', trigger=trigger, services=services, steps=steps,
ver_mode=ver_mode,
),
pipeline(
name='enterprise-windows-{}'.format(ver_mode), edition='enterprise', trigger=trigger, steps=windows_steps, platform='windows',
depends_on=['enterprise-build-{}'.format(ver_mode)], ver_mode=ver_mode,
),
]
def release_pipelines():
ver_mode = 'release'
services = integration_test_services()
trigger = {
'ref': ['refs/tags/v*',],
}
# The release pipelines include also enterprise ones, so both editions are built for a release.
# We could also solve this by triggering a downstream build for the enterprise repo, but by including enterprise
# in OSS release builds, we simplify the UX for the release engineer.
oss_pipelines = get_oss_pipelines(ver_mode=ver_mode, trigger=trigger)
enterprise_pipelines = get_enterprise_pipelines(ver_mode=ver_mode, trigger=trigger)
publish_pipeline = pipeline(
name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss', steps=[
{
'name': 'publish-packages',
'image': publish_image,
'depends_on': [
'initialize',
],
'environment': {
'GRAFANA_COM_API_KEY': {
'from_secret': 'grafana_api_key',
},
},
'commands': [
'./bin/grabpl publish-packages --edition oss ${DRONE_TAG}',
'./bin/grabpl publish-packages --edition enterprise ${DRONE_TAG}',
],
},
], depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines], install_deps=False,
ver_mode=ver_mode,
)
pipelines = oss_pipelines + enterprise_pipelines + [publish_pipeline,]
pipelines.append(notify_pipeline(
name='notify-{}'.format(ver_mode), slack_channel='grafana-ci-notifications', trigger=trigger,
depends_on=[p['name'] for p in pipelines],
))
return pipelines
def test_release_pipelines():
ver_mode = 'test-release'
services = integration_test_services()
trigger = {
'event': ['custom',],
}
oss_pipelines = get_oss_pipelines(ver_mode=ver_mode, trigger=trigger)
enterprise_pipelines = get_enterprise_pipelines(ver_mode=ver_mode, trigger=trigger)
publish_cmd = './bin/grabpl publish-packages --edition {{}} --dry-run {}'.format(test_release_ver)
publish_pipeline = pipeline(
name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss', steps=[
{
'name': 'publish-packages-oss',
'image': publish_image,
'depends_on': [
'initialize',
],
'environment': {
'GRAFANA_COM_API_KEY': {
'from_secret': 'grafana_api_key',
},
},
'commands': [
publish_cmd.format('oss'),
],
},
{
'name': 'publish-packages-enterprise',
'image': publish_image,
'depends_on': [
'initialize',
],
'environment': {
'GRAFANA_COM_API_KEY': {
'from_secret': 'grafana_api_key',
},
},
'commands': [
publish_cmd.format('enterprise'),
],
},
], depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines], install_deps=False,
ver_mode=ver_mode,
)
pipelines = oss_pipelines + enterprise_pipelines + [publish_pipeline,]
pipelines.append(notify_pipeline(
name='notify-{}'.format(ver_mode), slack_channel='grafana-ci-notifications', trigger=trigger,
depends_on=[p['name'] for p in pipelines],
))
return pipelines
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment