Toolkit: include a github release utility (#22520)

* Adding git publish to toolkit

* grafana-toolkit new feature plugin:github-release

* Feedback from code review
1. Used async await and exec for executing commands
2. Using dist folder to get plugin information

* Reverting change to plugin.json back to original value.

* reverted changes to yarn.lock

* reverted changes to yarn.lock

* feedback from code review

* feedback from code review 2

* moved constants to recommended functions

* styling changes and reverting yarn.lock

* removing changes to package.json

* replced execLine with execa

* better error detection around the publish token

* made simpler with commitHash from build

* Testing showed a number of required changes:
- Make the sha configurable
  or through environment variable
  or through git config.
- Allow a release to be recreated
- Set name and repo from git config as this is what
ghr is expecting anyway.
- Appropriate errors if the user
  tries to run a release without
  doing a ci-build and ci-package first.

* Using spinner.
Took out extra dependencies out of project.json
wrote tests manually.

* Updated tests. Now passing

* Adding git publish to toolkit

* grafana-toolkit new feature plugin:github-release

* Feedback from code review
1. Used async await and exec for executing commands
2. Using dist folder to get plugin information

* Reverting change to plugin.json back to original value.

* reverted changes to yarn.lock

* reverted changes to yarn.lock

* feedback from code review

* feedback from code review 2

* moved constants to recommended functions

* styling changes and reverting yarn.lock

* removing changes to package.json

* replced execLine with execa

* better error detection around the publish token

* made simpler with commitHash from build

* Testing showed a number of required changes:
- Make the sha configurable
  or through environment variable
  or through git config.
- Allow a release to be recreated
- Set name and repo from git config as this is what
ghr is expecting anyway.
- Appropriate errors if the user
  tries to run a release without
  doing a ci-build and ci-package first.

* Using spinner.
Took out extra dependencies out of project.json
wrote tests manually.

* Updated tests. Now passing

* updated test for reducers, from master

* package.json and yarn.lock from master

packages/grafana-toolkit/src/cli/index.ts

@@ -14,6 +14,7 @@ import { pluginTestTask } from './tasks/plugin.tests';
 import { searchTestDataSetupTask } from './tasks/searchTestDataSetup';
 import { searchTestDataSetupTask } from './tasks/searchTestDataSetup';
 import { closeMilestoneTask } from './tasks/closeMilestone';
 import { closeMilestoneTask } from './tasks/closeMilestone';
 import { pluginDevTask } from './tasks/plugin.dev';
 import { pluginDevTask } from './tasks/plugin.dev';
+import { githubPublishTask } from './tasks/plugin.utils';
 import {
 import {
   ciBuildPluginTask,
   ciBuildPluginTask,
   ciBuildPluginDocsTask,
   ciBuildPluginDocsTask,
@@ -208,6 +209,22 @@ export const run = (includeInternalScripts = false) => {
       });
       });
     });
     });
 
 
+  program
+    .command('plugin:github-publish')
+    .option('--dryrun', 'Do a dry run only', false)
+    .option('--verbose', 'Print verbose', false)
+    .option('--commitHash <hashKey>', 'Specify the commit hash')
+    .option('--recreate', 'Recreate the release if already present')
+    .description('Publish to github ... etc etc etc')
+    .action(async cmd => {
+      await execTask(githubPublishTask)({
+        dryrun: cmd.dryrun,
+        verbose: cmd.verbose,
+        commitHash: cmd.commitHash,
+        recreate: cmd.recreate,
+      });
+    });
+
   // Test the manifest creation
   // Test the manifest creation
   program
   program
     .command('manifest')
     .command('manifest')

packages/grafana-toolkit/src/cli/tasks/manifest.test.ts

@@ -21,6 +21,7 @@ describe('Manifest', () => {
         "plugin.create.ts",
         "plugin.create.ts",
         "plugin.dev.ts",
         "plugin.dev.ts",
         "plugin.tests.ts",
         "plugin.tests.ts",
+        "plugin.utils.ts",
         "precommit.ts",
         "precommit.ts",
         "searchTestDataSetup.ts",
         "searchTestDataSetup.ts",
         "task.ts",
         "task.ts",

packages/grafana-toolkit/src/cli/tasks/plugin.ci.ts

@@ -118,6 +118,14 @@ const packagePluginRunner: TaskRunner<PluginCIOptions> = async () => {
   const packagesDir = path.resolve(ciDir, 'packages');
   const packagesDir = path.resolve(ciDir, 'packages');
   const distDir = path.resolve(ciDir, 'dist');
   const distDir = path.resolve(ciDir, 'dist');
   const docsDir = path.resolve(ciDir, 'docs');
   const docsDir = path.resolve(ciDir, 'docs');
+  const jobsDir = path.resolve(ciDir, 'jobs');
+
+  fs.exists(jobsDir, jobsDirExists => {
+    if (!jobsDirExists) {
+      throw 'You must run plugin:ci-build prior to running plugin:ci-package';
+    }
+  });
+
   const grafanaEnvDir = path.resolve(ciDir, 'grafana-test-env');
   const grafanaEnvDir = path.resolve(ciDir, 'grafana-test-env');
   await execa('rimraf', [packagesDir, distDir, grafanaEnvDir]);
   await execa('rimraf', [packagesDir, distDir, grafanaEnvDir]);
   fs.mkdirSync(packagesDir);
   fs.mkdirSync(packagesDir);

packages/grafana-toolkit/src/cli/tasks/plugin.utils.ts

+import { Task, TaskRunner } from './task';
+import { getPluginJson } from '../../config/utils/pluginValidation';
+import { GitHubRelease } from '../utils/githubRelease';
+import { getPluginId } from '../../config/utils/getPluginId';
+import { getCiFolder } from '../../plugins/env';
+import { useSpinner } from '../utils/useSpinner';
+
+import path = require('path');
+
+// @ts-ignore
+import execa = require('execa');
+
+interface Command extends Array<any> {}
+
+const releaseNotes = async (): Promise<string> => {
+  const { stdout } = await execa.shell(`awk \'BEGIN {FS="##"; RS=""} FNR==3 {print; exit}\' CHANGELOG.md`);
+  return stdout;
+};
+
+const checkoutBranch = async (branchName: string): Promise<Command> => {
+  const currentBranch = await execa.shell(`git rev-parse --abbrev-ref HEAD`);
+  const branchesAvailable = await execa.shell(
+    `(git branch -a | grep ${branchName} | grep -v remote) || echo 'No release found'`
+  );
+
+  if (currentBranch.stdout !== branchName) {
+    if (branchesAvailable.stdout.trim() === branchName) {
+      return ['git', ['checkout', branchName]];
+    } else {
+      return ['git', ['checkout', '-b', branchName]];
+    }
+  }
+  return [];
+};
+
+const gitUrlParse = (url: string): { owner: string; name: string } => {
+  let matchResult: RegExpMatchArray | null = [];
+
+  if (url.match(/^git@github.com/)) {
+    // We have an ssh style url.
+    matchResult = url.match(/^git@github.com:(.*?)\/(.*?)\.git/);
+  }
+
+  if (url.match(/^https:\/\/github.com\//)) {
+    // We have an https style url
+    matchResult = url.match(/^https:\/\/github.com\/(.*?)\/(.*?)\/.git/);
+  }
+
+  if (matchResult && matchResult.length > 2) {
+    return {
+      owner: matchResult[1],
+      name: matchResult[2],
+    };
+  }
+
+  throw `Coult not find a suitable git repository. Received [${url}]`;
+};
+
+const prepareRelease = useSpinner<any>('Preparing release', async ({ dryrun, verbose }) => {
+  const ciDir = getCiFolder();
+  const distDir = path.resolve(ciDir, 'dist');
+  const distContentDir = path.resolve(distDir, getPluginId());
+  const pluginJsonFile = path.resolve(distContentDir, 'plugin.json');
+  const pluginVersion = getPluginJson(pluginJsonFile).info.version;
+  const GIT_EMAIL = 'eng@grafana.com';
+  const GIT_USERNAME = 'CircleCI Automation';
+
+  const githubPublishScript: Command = [
+    ['git', ['config', 'user.email', GIT_EMAIL]],
+    ['git', ['config', 'user.name', GIT_USERNAME]],
+    await checkoutBranch(`release-${pluginVersion}`),
+    ['git', ['add', '--force', distDir], { dryrun }],
+    [
+      'git',
+      ['commit', '-m', `automated release ${pluginVersion} [skip ci]`],
+      {
+        dryrun,
+        okOnError: [/nothing to commit/g, /nothing added to commit/g, /no changes added to commit/g],
+      },
+    ],
+    ['git', ['tag', '-f', pluginVersion]],
+    ['git', ['push', '-f', 'origin', `release-${pluginVersion}`], { dryrun }],
+  ];
+
+  for (let line of githubPublishScript) {
+    const opts = line.length === 3 ? line[2] : {};
+    const command = line[0];
+    const args = line[1];
+
+    try {
+      if (verbose) {
+        console.log('executing >>', line);
+      }
+
+      if (line.length > 0 && line[0].length > 0) {
+        if (opts['dryrun']) {
+          line[1].push('--dry-run');
+        }
+        const { stdout } = await execa(command, args);
+        if (verbose) {
+          console.log(stdout);
+        }
+      } else {
+        if (verbose) {
+          console.log('skipping empty line');
+        }
+      }
+    } catch (ex) {
+      const err: string = ex.message;
+      if (opts['okOnError'] && Array.isArray(opts['okOnError'])) {
+        let trueError = true;
+        for (let regex of opts['okOnError']) {
+          if (err.match(regex)) {
+            trueError = false;
+            break;
+          }
+        }
+
+        if (!trueError) {
+          // This is not an error
+          continue;
+        }
+      }
+      console.error(err);
+      process.exit(-1);
+    }
+  }
+});
+
+interface GithubPluglishReleaseOptions {
+  commitHash?: string;
+  recreate?: boolean;
+  githubToken: string;
+  gitRepoOwner: string;
+  gitRepoName: string;
+}
+
+const createRelease = useSpinner<GithubPluglishReleaseOptions>(
+  'Creating release',
+  async ({ commitHash, recreate, githubToken, gitRepoName, gitRepoOwner }) => {
+    const gitRelease = new GitHubRelease(githubToken, gitRepoOwner, gitRepoName, await releaseNotes(), commitHash);
+    return gitRelease.release(recreate || false);
+  }
+);
+
+export interface GithubPublishOptions {
+  dryrun?: boolean;
+  verbose?: boolean;
+  commitHash?: string;
+  recreate?: boolean;
+}
+
+const githubPublishRunner: TaskRunner<GithubPublishOptions> = async ({ dryrun, verbose, commitHash, recreate }) => {
+  if (!process.env['CIRCLE_REPOSITORY_URL']) {
+    throw `The release plugin requires you specify the repository url as environment variable CIRCLE_REPOSITORY_URL`;
+  }
+
+  if (!process.env['GITHUB_TOKEN']) {
+    throw `Github publish requires that you set the environment variable GITHUB_TOKEN to a valid github api token.
+    See: https://github.com/settings/tokens for more details.`;
+  }
+
+  const parsedUrl = gitUrlParse(process.env['CIRCLE_REPOSITORY_URL']);
+  const githubToken = process.env['GITHUB_TOKEN'];
+
+  await prepareRelease({
+    dryrun,
+    verbose,
+  });
+
+  await createRelease({
+    commitHash,
+    recreate,
+    githubToken,
+    gitRepoOwner: parsedUrl.owner,
+    gitRepoName: parsedUrl.name,
+  });
+};
+
+export const githubPublishTask = new Task<GithubPublishOptions>('Github Publish', githubPublishRunner);

packages/grafana-toolkit/src/cli/utils/githubRelease.ts

+import { getPluginId } from '../../config/utils/getPluginId';
+import { getPluginJson } from '../../config/utils/pluginValidation';
+import { getCiFolder } from '../../plugins/env';
+import path = require('path');
+// @ts-ignore
+import execa = require('execa');
+
+const ghrPlatform = (): string => {
+  switch (process.platform) {
+    case 'win32':
+      return 'windows';
+    case 'darwin':
+      return 'darwin';
+    case 'linux':
+      return 'linux';
+    default:
+      return process.platform;
+  }
+};
+
+class GitHubRelease {
+  token: string;
+  username: string;
+  repository: string;
+  releaseNotes: string;
+  commitHash?: string;
+
+  constructor(token: string, username: string, repository: string, releaseNotes: string, commitHash?: string) {
+    this.token = token;
+    this.username = username;
+    this.repository = repository;
+    this.releaseNotes = releaseNotes;
+    this.commitHash = commitHash;
+  }
+
+  /**
+   * Get the ghr binary to perform the release
+   */
+  private async getGhr(): Promise<string> {
+    const GHR_VERSION = '0.13.0';
+    const GHR_ARCH = process.arch === 'x64' ? 'amd64' : '386';
+    const GHR_PLATFORM = ghrPlatform();
+    const GHR_EXTENSION = process.platform === 'linux' ? 'tar.gz' : 'zip';
+    const outName = `./ghr.${GHR_EXTENSION}`;
+    const archiveName = `ghr_v${GHR_VERSION}_${GHR_PLATFORM}_${GHR_ARCH}`;
+    const exeName = process.platform === 'linux' ? 'ghr' : 'ghr.exe';
+    const exeNameFullPath = path.resolve(process.cwd(), archiveName, exeName);
+    const ghrUrl = `https://github.com/tcnksm/ghr/releases/download/v${GHR_VERSION}/${archiveName}.${GHR_EXTENSION}`;
+    await execa('wget', [ghrUrl, `--output-document=${outName}`]);
+    if (GHR_EXTENSION === 'tar.gz') {
+      await execa('tar', ['zxvf', outName]);
+    } else {
+      await execa('unzip', ['-p', outName]);
+    }
+
+    if (process.platform === 'linux') {
+      await execa('chmod', ['755', exeNameFullPath]);
+    }
+
+    return exeNameFullPath;
+  }
+
+  async release(recreate: boolean) {
+    const ciDir = getCiFolder();
+    const distDir = path.resolve(ciDir, 'dist');
+    const distContentDir = path.resolve(distDir, getPluginId());
+    const pluginJsonFile = path.resolve(distContentDir, 'plugin.json');
+    const pluginInfo = getPluginJson(pluginJsonFile).info;
+    const PUBLISH_DIR = path.resolve(getCiFolder(), 'packages');
+    const commitHash = this.commitHash || pluginInfo.build?.hash;
+
+    // Get the ghr binary according to platform
+    const ghrExe = await this.getGhr();
+
+    if (!commitHash) {
+      throw 'The release plugin was not able to locate a commithash for release. Either build using the ci, or specify the commit hash with --commitHash <value>';
+    }
+
+    const args = [
+      '-t',
+      this.token,
+      '-u',
+      this.username,
+      '-r',
+      this.repository, // should override --- may not be the same
+      '-c',
+      commitHash,
+      '-n',
+      `${this.repository}_v${pluginInfo.version}`,
+      '-b',
+      this.releaseNotes,
+      `v${pluginInfo.version}`,
+      PUBLISH_DIR,
+    ];
+
+    if (recreate) {
+      args.splice(12, 0, '-recreate');
+    }
+
+    const { stdout } = await execa(ghrExe, args);
+
+    console.log(stdout);
+  }
+}
+
+export { GitHubRelease };