Add Windows MSI generation to build process (#16502)

- add MSI generation to build process

.circleci/config.yml

@@ -552,6 +552,21 @@ jobs:
           name: Update RPM repository
           name: Update RPM repository
           command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
           command: './scripts/build/update_repo/update-rpm.sh "oss" "$GPG_KEY_PASSWORD" "$CIRCLE_TAG" "dist"'
 
 
+  build-oss-msi:
+    docker:
+      - image: grafana/wix-toolset-ci:v3
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: Build OSS MSI
+          command: './scripts/build/ci-msi-build/ci-msi-build-oss.sh'
+      - persist_to_workspace:
+          root: .
+          paths:
+            - dist/grafana-*.msi
+
   store-build-artifacts:
   store-build-artifacts:
     docker:
     docker:
       - image: circleci/node:8
       - image: circleci/node:8
@@ -590,6 +605,7 @@ workflows:
             - backend-lint
             - backend-lint
             - mysql-integration-test
             - mysql-integration-test
             - postgres-integration-test
             - postgres-integration-test
+            - build-oss-msi
           filters: *filter-only-master
           filters: *filter-only-master
       - grafana-docker-master:
       - grafana-docker-master:
           requires:
           requires:
@@ -613,6 +629,16 @@ workflows:
             - postgres-integration-test
             - postgres-integration-test
             - build-all-enterprise
             - build-all-enterprise
           filters: *filter-only-master
           filters: *filter-only-master
+      - build-oss-msi:
+          requires:
+            - build-all
+            - test-backend
+            - test-frontend
+            - codespell
+            - backend-lint
+            - mysql-integration-test
+            - postgres-integration-test
+          filters: *filter-only-master
 
 
   release:
   release:
     jobs:
     jobs:
@@ -641,6 +667,7 @@ workflows:
             - backend-lint
             - backend-lint
             - mysql-integration-test
             - mysql-integration-test
             - postgres-integration-test
             - postgres-integration-test
+            - build-oss-msi
           filters: *filter-only-release
           filters: *filter-only-release
       - deploy-enterprise-release:
       - deploy-enterprise-release:
           requires:
           requires:
@@ -664,6 +691,16 @@ workflows:
             - mysql-integration-test
             - mysql-integration-test
             - postgres-integration-test
             - postgres-integration-test
           filters: *filter-only-release
           filters: *filter-only-release
+      - build-oss-msi:
+          requires:
+            - build-all
+            - test-backend
+            - test-frontend
+            - codespell
+            - backend-lint
+            - mysql-integration-test
+            - postgres-integration-test
+          filters: *filter-only-master
 
 
   build-branches-and-prs:
   build-branches-and-prs:
       jobs:
       jobs:

scripts/build/ci-msi-build/README.md

+# Grafana MSI Generator
+
+Creates a docker image that can be included within CircleCI or run locally to generate an MSI for Grafana.
+
+## Docker Image
+
+The docker image is created and published via CircleCI, and can also be built locally.
+
+The image is self contained with all of the code in `/master`.
+The detection process expects a zip file in `/master/dist`.
+
+There are two patterns that will be matched for a build in the dist directory:
+```
+grafana-6.0.0-ca0bc2c5pre3.windows-amd64.zip
+grafana-5.4.3.windows-amd64.zip
+```
+
+### Building an MSI
+
+The process is automated to expect a dist directory, and will build an msi for first matching grafana-*.windows-amd64.zip file found.
+
+```
+grafana-5.4.3.windows-amd64.zip
+```
+
+## CircleCI
+
+
+
+## Manual
+
+A wrapper script takes a single argument for the path to a zip file, or searches for a file in dist.
+
+A manual build can be initiated using docker-compose
+```
+cd oss
+docker-compose up --build
+```
+## Automated
+
+## Testing
+
+## Change Log
+v1.0.0 - initial commit

scripts/build/ci-msi-build/ci-msi-build-oss.sh

+#!/bin/bash
+set -e
+WORKING_DIRECTORY=`pwd`
+# copy zip file to /tmp/dist
+mkdir -p /tmp/dist
+cp ./dist/*.zip /tmp/dist
+echo "Contents of /tmp/dist"
+ls -al /tmp/dist
+
+# nssm download has been unreliable, use a cached copy of it
+echo "Caching NSSM"
+mkdir -p /tmp/cache
+cp ./scripts/build/ci-msi-build/oss/cache/nssm-2.24.zip /tmp/cache
+# a build can be specified, which will be pulled down
+#python3 generator/build.py --build 5.4.3
+#echo "LIGHT config"
+#ls -al /home/xclient/wix/light.exe.config
+#cat /home/xclient/wix/light.exe.config
+#cp ./scripts/build/ci-msi-build/oss/light.exe.config /home/xclient/wix/light.exe.config
+#cat /home/xclient/wix/light.exe.config
+cd ./scripts/build/ci-msi-build/oss
+echo "Building MSI"
+python3 generator/build.py "$@"
+chmod a+x /tmp/scratch/*.msi
+echo "MSI: Copy to $WORKING_DIRECTORY/dist"
+cp /tmp/scratch/*.msi $WORKING_DIRECTORY/dist
+echo "MSI: contents of $WORKING_DIRECTORY/dist"
+ls -al $WORKING_DIRECTORY/dist

scripts/build/ci-msi-build/oss/Makefile

+all: build
+
+build:
+	pip3 install -r requirements.txt
+	python3 generator/build.py

scripts/build/ci-msi-build/oss/ci-wrapper.sh

+#!/bin/bash
+
+# Build will be found in ./dist and ./dist-enterprise
+# integrated circleci will have all of the code in /master
+# and the builds will be found in $HOME
+mkdir -p /tmp/dist
+if [ -d '/home/xclient/repo/dist/' ]; then
+  ls -al /home/xclient/repo/dist/
+  cp /home/xclient/repo/dist/*.zip /tmp/dist/
+  echo "Contents of /tmp/dist"
+  ls -al /tmp/dist
+fi
+# nssm download has been unreliable, use a cached copy of it
+echo "Caching NSSM"
+mkdir -p /tmp/cache
+cp /master/cache/nssm-2.24.zip /tmp/cache
+# a build can be specified, which will be pulled down
+#python3 generator/build.py --build 5.4.3
+echo "LIGHT config"
+ls -al /home/xclient/wix/light.exe.config
+cat /home/xclient/wix/light.exe.config
+cp /master/light.exe.config /home/xclient/wix/light.exe.config
+cat /home/xclient/wix/light.exe.config
+cd /master
+echo "Building MSI"
+python3 generator/build.py "$@"
+#
+#

scripts/build/ci-msi-build/oss/docker-compose.yml

+version: '3'
+services:
+  wix:
+    build: './docker'
+    command: /oss/wrapper.sh
+    # important: wine is setup for the user xclient
+    user: xclient
+    volumes:
+      - ../oss:/oss
+      - ../master/templates:/oss/templates
+      - ../master/resources:/oss/resources
+    environment:
+      - TERM=linux

scripts/build/ci-msi-build/oss/generator/utils.py

+import zipfile
+import os
+import glob
+import re
+import shutil
+import wget
+from jinja2 import Template, Environment, FileSystemLoader
+
+
+
+def extract_zip(filename, target_dir):
+    with zipfile.ZipFile(filename, 'r') as zip_ref:
+        zip_ref.extractall(target_dir)
+
+
+def get_nssm(tmpPath, version):
+    if not os.path.isdir(tmpPath):
+        os.mkdir(tmpPath)
+    target_filename = '{}/nssm-{}.zip'.format(tmpPath, version)
+    exists = os.path.isfile(target_filename)
+    if exists:
+        return target_filename
+    url = 'https://nssm.cc/release/nssm-{}.zip'.format(version)
+    print('NSSM url is {}'.format(url))
+    filename = wget.download(url, out=target_filename, bar=wget.bar_thermometer)
+    return filename
+
+
+def get_zip(version, target_filename):
+    exists = os.path.isfile(target_filename)
+    if exists:
+        return target_filename
+    url = 'https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana-{}.windows-amd64.zip'.format(version)
+    #url = 'https://dl.grafana.com/enterprise/release/grafana-enterprise-{}.windows-amd64.zip'.format(version)
+    filename = wget.download(url, out=target_filename, bar=wget.bar_thermometer)
+    return filename
+
+
+#
+#
+#
+def detect_version(dist_path):
+    detectedVersion = ''
+    detectedHash = ''
+    isEnterprise = False
+    print("Detecting Version...")
+    # grafana-6.0.0-ca0bc2c5pre3.windows-amd64.zip
+    # get files in directory matching pattern
+    fileList = glob.glob(dist_path + '/grafana*.windows-amd64.zip')
+    print(fileList)
+    if len(fileList) == 0:
+        print('Skipping detection, no matches')
+        return
+    firstFile = fileList[0]
+    p1 = re.search(r'grafana-(\d\.\d\.\d)\.windows-amd64.zip$', firstFile)
+    p2 = re.search(r'grafana-(\d\.\d\.\d)-(.*)\.windows-amd64.zip$', firstFile)
+    if p1:
+        detectedVersion = p1.group(1)
+    if p2:
+        detectedVersion = p2.group(1)
+        detectedHash = p2.group(2)
+    return detectedVersion, detectedHash, isEnterprise
+
+    #if os.path.isdir(dist_path + 'enterprise-dist'):
+    #    # grafana-enterprise-6.0.0-29b28127pre3.windows-amd64.zip
+    #    # get files in directory matching pattern
+    #    fileList = glob.glob(dist_path + '/enterprise-dist/grafana*.windows-amd64.zip')
+    #    firstFile = fileList[0]
+    #    p1 = re.search(r'grafana-enterprise-(\d\.\d\.\d)\.windows-amd64.zip$', firstFile)
+    #    p2 = re.search(r'grafana-enterprise-(\d\.\d\.\d)-(.*)\.windows-amd64.zip$', firstFile)
+    #    if p1:
+    #        detectedVersion = p1.group(1)
+    #        isEnterprise = True
+    #    if p2:
+    #        detectedVersion = p2.group(1)
+    #        detectedHash = p2.group(2)
+    #        isEnterprise = True
+    #    return detectedVersion, detectedHash, isEnterprise
+
+
+def generate_product_wxs(env, config, features, scratch_file, target_dir):
+    template = env.get_template('common/product.wxs.j2')
+    output = template.render(config=config, features=features)
+    fh = open(scratch_file, 'w')
+    fh.write(output)
+    fh.close()
+    shutil.copy2(scratch_file, target_dir)
+
+def generate_service_wxs(env, grafana_version, scratch_file, target_dir, nssm_version='2.24'):
+    template = env.get_template('common/grafana-service.wxs.j2')
+    output = template.render(grafana_version=grafana_version, nssm_version=nssm_version)
+    fh = open(scratch_file, 'w')
+    fh.write(output)
+    fh.close()
+    shutil.copy2(scratch_file, target_dir)
+
+def generate_firewall_wxs(env, grafana_version, scratch_file, target_dir):
+    os.system("ls -al templates")
+    template = env.get_template('common/grafana-firewall.wxs.j2')
+    output = template.render(grafana_version=grafana_version)
+    fh = open(scratch_file, 'w')
+    fh.write(output)
+    fh.close()
+    shutil.copy2(scratch_file, target_dir)
+
+
+def generate_oracle_environment_wxs(env, instant_client_version, scratch_file, target_dir):
+    template = env.get_template('oracle/oracle-environment.wxs.j2')
+    output = template.render(instant_client_version=instant_client_version)
+    fh = open(scratch_file, 'w')
+    fh.write(output)
+    fh.close()
+    shutil.copy2(scratch_file, target_dir)
+
+def copy_static_files(target_dir):
+    for item in os.listdir('resources/images'):
+        s = os.path.join('resources/images', item)
+        d = os.path.join(target_dir, item)
+        shutil.copy2(s, d)
+    for item in os.listdir('resources/license'):
+        s = os.path.join('resources/license', item)
+        d = os.path.join(target_dir, item)
+        shutil.copy2(s, d)

scripts/build/ci-msi-build/oss/light.exe.config

+<?xml version="1.0" encoding="utf-8" ?>
+<!-- Copyright (c) .NET Foundation and contributors. All rights reserved. Licensed under the Microsoft Reciprocal License. See LICENSE.TXT file in the project root for full license information. -->
+
+<configuration>
+    <startup useLegacyV2RuntimeActivationPolicy="true">
+        <supportedRuntime version="v4.0" />
+        <supportedRuntime version="v2.0.50727" />
+    </startup>
+    <runtime>
+        <loadFromRemoteSources enabled="true"/>
+		 <AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false;Switch.System.IO.BlockLongPaths=false" />
+    </runtime>
+</configuration>
\ No newline at end of file

scripts/build/ci-msi-build/oss/requirements.txt

+Jinja2>=2.10
+MarkupSafe>=1.1.0
+wget>=3.2

scripts/build/ci-msi-build/oss/templates/common/grafana-firewall.wxs.j2

+<?xml version="1.0" encoding="utf-8"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi"
+     xmlns:fire="http://schemas.microsoft.com/wix/FirewallExtension">
+  <Fragment>
+      <ComponentGroup Id="GrafanaFirewallExceptionsGroup">
+        <Component Id="FirewallGrafanaServer" Guid="7278f07d-de6f-497f-9267-d5feb5216a5c" Directory="INSTALLDIR">
+          <File KeyPath="yes" Source="SourceDir\grafana\bin\grafana-server.exe">
+             <fire:FirewallException
+              Id="FWX1"
+              Name="Grafana Server TCP 3000"
+              Port="3000"
+              Profile="all"
+              Protocol="tcp"
+              Scope="any"/>
+          </File>
+        </Component>
+      </ComponentGroup>
+  </Fragment>
+</Wix>

scripts/build/ci-msi-build/oss/templates/common/grafana-service.wxs.j2

+<?xml version="1.0" encoding="utf-8"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi"
+     xmlns:util="http://schemas.microsoft.com/wix/UtilExtension">
+
+  <Fragment>
+    <ComponentGroup Id="GrafanaServiceX64">
+      <Component Id="nssm_component" Guid="*" Directory="INSTALLDIR">
+        <File Id="nssm" KeyPath="yes" Source="SourceDir\nssm-{{ nssm_version }}\win64\nssm.exe" />
+
+        <ServiceInstall Id="ServiceInstall"
+          Account="LocalSystem"
+          ErrorControl="normal"
+          Name="Grafana"
+          Start="auto"
+          Type="ownProcess"
+          Vital="yes"
+          Description="Grafana by Grafana Labs"
+          DisplayName="Grafana">
+          <ServiceConfig OnInstall="yes" OnReinstall="yes" DelayedAutoStart="no" />
+        </ServiceInstall>
+
+        <ServiceControl Id="ControlService"
+          Name="Grafana"
+          Wait="yes"
+          Start="install"
+          Stop="both"
+          Remove="uninstall"
+        />
+
+        <RegistryKey Root="HKLM" Key="SYSTEM\CurrentControlSet\Services\Grafana">
+          <RegistryKey Key="Parameters">
+            <RegistryValue Name="AppDirectory" Value="[INSTALLDIR]grafana" Type="expandable" />
+            <RegistryValue Name="Application" Value="[INSTALLDIR]grafana\bin\grafana-server.exe" Type="expandable" />
+            <RegistryValue Name="AppParameters" Value='' Type="expandable" />
+
+            <RegistryValue Name="AppEnvironmentExtra" Type="multiString">
+              <MultiStringValue>LOG_LEVEL=DEBUG</MultiStringValue>
+            </RegistryValue>
+         
+            <RegistryValue Name="AppStdout" Value="[LOGDIR]grafana-service.log" Type="expandable" />
+            <RegistryValue Name="AppStderr" Value="[LOGDIR]grafana-service.log" Type="expandable" />
+            <RegistryValue Name="AppRotateFiles" Value="1" Type="integer" />
+            <RegistryValue Name="AppRotateOnline" Value="1" Type="integer" />
+
+            <!-- Rotate after 100 MB -->
+            <RegistryValue Name="AppRotateBytes" Value="104857600" Type="integer" />
+            <RegistryValue Name="AppStdoutCopyAndTruncate" Value="1" Type="integer" />
+            <RegistryValue Name="AppStderrCopyAndTruncate" Value="1" Type="integer" />
+            <RegistryValue Name="AppRotateDelay" Value="1000" Type="integer" />
+
+            <RegistryKey Key="AppExit">
+              <RegistryValue Type="string" Value="Restart" />
+            </RegistryKey>
+          </RegistryKey>
+        </RegistryKey>
+      </Component>
+    </ComponentGroup>
+  </Fragment>
+</Wix>

scripts/build/ci-msi-build/oss/templates/common/product.wxs.j2

+<?xml version="1.0"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+  <Product Id="*"
+    UpgradeCode="{{ config.upgrade_code }}"
+    Name="{{ config.product_name }}"
+    Version="{{ config.grafana_version }}"
+    Manufacturer="{{ config.manufacturer }}"
+    Language="1033">
+
+    <Package
+      Platform="x64"
+      InstallerVersion="200"
+      Compressed="yes"
+      Comments="Windows Installer Package"/>
+
+    <Media Id="1" Cabinet="product.cab" EmbedCab="yes"/>
+
+    <Icon Id="icon.ico" SourceFile="grafana_icon.ico"/>
+
+    <WixVariable Id="WixUILicenseRtf" Value="LICENSE.rtf" />
+    <WixVariable Id="WixUIBannerBmp" Value="grafana_top_banner_white.bmp" />
+    <WixVariable Id="WixUIDialogBmp" Value="grafana_dialog_background.bmp" />
+
+    <Property Id="ARPPRODUCTICON" Value="icon.ico" />
+    <Property Id="ARPHELPLINK" Value="https://www.grafana.com" />
+    <Property Id="ARPURLINFOABOUT" Value="https://www.grafana.com" />
+
+    <Directory Id="TARGETDIR" Name="SourceDir">
+      <Directory Id="ProgramFiles64Folder">
+        <Directory Id="INSTALLDIR" Name="GrafanaLabs">
+          {% for feature in features %}
+            {% for component_group in feature.component_groups %}
+              <Directory Id="{{component_group.directory}}"/>
+            {% endfor %}
+          {% endfor %}
+        </Directory>
+      </Directory>
+    </Directory>
+
+    <Feature Id="DefaultFeature" Title="Grafana" Display="expand" ConfigurableDirectory="INSTALLDIR">
+      {% for feature in features %}
+        <Feature Id="{{ feature.name }}Feature" Title="{{ feature.title }}" Level="1">
+          {% for component_group in feature.component_groups %}
+            <ComponentGroupRef Id="{{ component_group.ref_id }}"/>
+          {% endfor %}
+        </Feature>
+      {% endfor %}
+    </Feature>
+
+    <Property Id="WIXUI_INSTALLDIR" Value="INSTALLDIR" />
+		<UIRef Id="WixUI_FeatureTree"/>
+
+   </Product>
+</Wix>

scripts/build/ci-msi-build/oss/templates/oracle/oracle-environment.wxs.j2

+<?xml version="1.0" encoding="utf-8"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi"
+     xmlns:fire="http://schemas.microsoft.com/wix/FirewallExtension">
+  <Fragment>
+      <ComponentGroup Id="OracleEnvironment">
+        <Component Id="ORACLE_ENV" Guid="13ed28fd-a2d8-45e3-9c0f-8ec8d9e3ad16" Directory="INSTALLDIR">
+          <Environment Id="OCI_LIB64" Name="OCI_LIB64" Value="[INSTALLDIR]instantclient_{{ instant_client_version }}" Permanent="yes" Part="last" Action="set" System="yes" />
+          <Environment Id="PATH" Name="PATH" Value="[INSTALLDIR]instantclient_{{instant_client_version}}" Permanent="yes" Part="last" Action="set" System="yes" />
+        </Component>
+      </ComponentGroup>
+  </Fragment>
+</Wix>

scripts/build/ci-msi-build/oss/wrapper.sh

+#!/bin/bash
+cd /oss
+make