Commit 0bfbb1c1 by Leonard Gram Committed by GitHub

Merge pull request #11920 from grafana/crosscompile

Crosscompile and packages Grafana on arm, windows, linux and darwin
parents 64053e19 bea4741a
aliases:
# Workflow filters
- &filter-only-release
branches:
ignore: /.*/
tags:
only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
- &filter-not-release
tags:
ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
version: 2
jobs:
......@@ -46,7 +57,6 @@ jobs:
command: 'sudo npm install -g yarn --quiet'
- restore_cache:
key: dependency-cache-{{ checksum "yarn.lock" }}
# Could we skip this step if the cache has been restored? `[ -d node_modules ] || yarn install ...` should be able to apply to build step as well
- run:
name: yarn install
command: 'yarn install --pure-lockfile --no-progress'
......@@ -68,15 +78,27 @@ jobs:
name: build backend and run go tests
command: './scripts/circle-test-backend.sh'
build:
build-all:
docker:
- image: grafana/build-container:v0.1
- image: grafana/build-container:1.0.0
working_directory: /go/src/github.com/grafana/grafana
steps:
- checkout
- run:
name: prepare build tools
command: '/tmp/bootstrap.sh'
- restore_cache:
key: phantomjs-binaries-{{ checksum "scripts/build/download-phantomjs.sh" }}
- run:
name: download phantomjs binaries
command: './scripts/build/download-phantomjs.sh'
- save_cache:
key: phantomjs-binaries-{{ checksum "scripts/build/download-phantomjs.sh" }}
paths:
- /tmp/phantomjs
- run:
name: build and package grafana
command: './scripts/build/build.sh'
command: './scripts/build/build-all.sh'
- run:
name: sign packages
command: './scripts/build/sign_packages.sh'
......@@ -92,6 +114,8 @@ jobs:
- dist/grafana*
- scripts/*.sh
- scripts/publish
- store_artifacts:
path: dist
build-enterprise:
docker:
......@@ -154,45 +178,43 @@ workflows:
version: 2
test-and-build:
jobs:
- build-all:
filters: *filter-not-release
- codespell:
filters:
tags:
only: /.*/
filters: *filter-not-release
- gometalinter:
filters:
tags:
only: /.*/
- build:
filters:
tags:
only: /.*/
filters: *filter-not-release
- test-frontend:
filters:
tags:
only: /.*/
filters: *filter-not-release
- test-backend:
filters:
tags:
only: /.*/
filters: *filter-not-release
- deploy-master:
requires:
- build-all
- test-backend
- test-frontend
- build
- codespell
- gometalinter
filters:
branches:
only: master
release:
jobs:
- build-all:
filters: *filter-only-release
- codespell:
filters: *filter-only-release
- gometalinter:
filters: *filter-only-release
- test-frontend:
filters: *filter-only-release
- test-backend:
filters: *filter-only-release
- deploy-release:
requires:
- build-all
- test-backend
- test-frontend
- build
filters:
branches:
ignore: /.*/
tags:
only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
# - build-enterprise:
# filters:
# tags:
# only: /.*/
- codespell
- gometalinter
filters: *filter-only-release
......@@ -12,6 +12,10 @@ module.exports = function (grunt) {
platform: process.platform.replace('win32', 'windows'),
};
if (grunt.option('platform')) {
config.platform = grunt.option('platform');
}
if (grunt.option('arch')) {
config.arch = grunt.option('arch');
} else {
......
......@@ -38,16 +38,3 @@ artifacts:
- path: grafana-*windows-*.*
name: binzip
type: zip
deploy:
- provider: Environment
name: GrafanaReleaseMaster
on:
buildType: master
- provider: Environment
name: GrafanaReleaseRelease
on:
buildType: release
......@@ -27,8 +27,7 @@ var (
goarch string
goos string
gocc string
gocxx string
cgo string
cgo bool
pkgArch string
version string = "v1"
// deb & rpm does not support semver so have to handle their version a little differently
......@@ -53,8 +52,7 @@ func main() {
flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
flag.StringVar(&gocc, "cc", "", "CC")
flag.StringVar(&gocxx, "cxx", "", "CXX")
flag.StringVar(&cgo, "cgo-enabled", "", "CGO_ENABLED")
flag.BoolVar(&cgo, "cgo-enabled", cgo, "Enable cgo")
flag.StringVar(&pkgArch, "pkg-arch", "", "PKG ARCH")
flag.StringVar(&phjsToRelease, "phjs", "", "PhantomJS binary")
flag.BoolVar(&race, "race", race, "Use race detector")
......@@ -93,20 +91,24 @@ func main() {
build("grafana-server", "./pkg/cmd/grafana-server", []string{})
case "build":
clean()
//clean()
for _, binary := range binaries {
build(binary, "./pkg/cmd/"+binary, []string{})
}
case "build-frontend":
grunt(gruntBuildArg("build")...)
case "test":
test("./pkg/...")
grunt("test")
case "package":
grunt(gruntBuildArg("release")...)
if runtime.GOOS != "windows" {
createLinuxPackages()
}
grunt(gruntBuildArg("build")...)
packageGrafana()
case "package-only":
packageGrafana()
case "pkg-rpm":
grunt(gruntBuildArg("release")...)
......@@ -131,6 +133,22 @@ func main() {
}
}
func packageGrafana() {
platformArg := fmt.Sprintf("--platform=%v", goos)
previousPkgArch := pkgArch
if pkgArch == "" {
pkgArch = goarch
}
postProcessArgs := gruntBuildArg("package")
postProcessArgs = append(postProcessArgs, platformArg)
grunt(postProcessArgs...)
pkgArch = previousPkgArch
if goos == "linux" && goarch == "amd64"{
createLinuxPackages()
}
}
func makeLatestDistCopies() {
files, err := ioutil.ReadDir("dist")
if err != nil {
......@@ -138,9 +156,9 @@ func makeLatestDistCopies() {
}
latestMapping := map[string]string{
".deb": "dist/grafana_latest_amd64.deb",
".rpm": "dist/grafana-latest-1.x86_64.rpm",
".tar.gz": "dist/grafana-latest.linux-x64.tar.gz",
"_amd64.deb": "dist/grafana_latest_amd64.deb",
".x86_64.rpm": "dist/grafana-latest-1.x86_64.rpm",
".linux-amd64.tar.gz": "dist/grafana-latest.linux-x64.tar.gz",
}
for _, file := range files {
......@@ -386,7 +404,8 @@ func test(pkg string) {
}
func build(binaryName, pkg string, tags []string) {
binary := "./bin/" + binaryName
binary := fmt.Sprintf("./bin/%s-%s/%s", goos, goarch, binaryName)
if goos == "windows" {
binary += ".exe"
}
......@@ -408,6 +427,7 @@ func build(binaryName, pkg string, tags []string) {
if !isDev {
setBuildEnv()
runPrint("go", "version")
fmt.Printf("Targeting %s/%s\n", goos, goarch)
}
runPrint("go", args...)
......@@ -451,6 +471,14 @@ func clean() {
func setBuildEnv() {
os.Setenv("GOOS", goos)
if goos == "windows" {
// require windows >=7
os.Setenv("CGO_CFLAGS", "-D_WIN32_WINNT=0x0601")
}
if goarch != "amd64" || goos != "linux" {
// needed for all other archs
cgo = true
}
if strings.HasPrefix(goarch, "armv") {
os.Setenv("GOARCH", "arm")
os.Setenv("GOARM", goarch[4:])
......@@ -460,15 +488,12 @@ func setBuildEnv() {
if goarch == "386" {
os.Setenv("GO386", "387")
}
if cgo != "" {
os.Setenv("CGO_ENABLED", cgo)
if cgo {
os.Setenv("CGO_ENABLED", "1")
}
if gocc != "" {
os.Setenv("CC", gocc)
}
if gocxx != "" {
os.Setenv("CXX", gocxx)
}
}
func getGitSha() string {
......
#!/bin/bash
#
# This script is executed from within the container.
#
CCARMV7=arm-linux-gnueabihf-gcc
CCARM64=aarch64-linux-gnu-gcc
CCOSX64=/tmp/osxcross/target/bin/o64-clang
CCWIN64=x86_64-w64-mingw32-gcc
CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc
GOPATH=/go
REPO_PATH=$GOPATH/src/github.com/grafana/grafana
cd /go/src/github.com/grafana/grafana
echo "current dir: $(pwd)"
if [ "$CIRCLE_TAG" != "" ]; then
echo "Building releases from tag $CIRCLE_TAG"
go run build.go -goarch armv7 -cc ${CCARMV7} -includeBuildNumber=false build
go run build.go -goarch arm64 -cc ${CCARM64} -includeBuildNumber=false build
go run build.go -goos darwin -cc ${CCOSX64} -includeBuildNumber=false build
go run build.go -goos windows -cc ${CCWIN64} -includeBuildNumber=false build
CC=${CCX64} go run build.go -includeBuildNumber=false build
else
echo "Building incremental build for $CIRCLE_BRANCH"
go run build.go -goarch armv7 -cc ${CCARMV7} -buildNumber=${CIRCLE_BUILD_NUM} build
go run build.go -goarch arm64 -cc ${CCARM64} -buildNumber=${CIRCLE_BUILD_NUM} build
go run build.go -goos darwin -cc ${CCOSX64} -buildNumber=${CIRCLE_BUILD_NUM} build
go run build.go -goos windows -cc ${CCWIN64} -buildNumber=${CIRCLE_BUILD_NUM} build
CC=${CCX64} go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build
fi
yarn install --pure-lockfile --no-progress
echo "current dir: $(pwd)"
if [ -d "dist" ]; then
rm -rf dist
fi
if [ "$CIRCLE_TAG" != "" ]; then
echo "Building frontend from tag $CIRCLE_TAG"
go run build.go -includeBuildNumber=false build-frontend
echo "Packaging a release from tag $CIRCLE_TAG"
go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest
go run build.go -goos linux -pkg-arch armv7 -includeBuildNumber=false package-only
go run build.go -goos linux -pkg-arch arm64 -includeBuildNumber=false package-only
if [ -d '/tmp/phantomjs/darwin' ]; then
cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs
else
echo 'PhantomJS binaries for darwin missing!'
fi
go run build.go -goos darwin -pkg-arch amd64 -includeBuildNumber=false package-only
if [ -d '/tmp/phantomjs/windows' ]; then
cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe
rm tools/phantomjs/phantomjs
else
echo 'PhantomJS binaries for darwin missing!'
fi
go run build.go -goos windows -pkg-arch amd64 -includeBuildNumber=false package-only
else
echo "Building frontend for $CIRCLE_BRANCH"
go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build-frontend
echo "Packaging incremental build for $CIRCLE_BRANCH"
go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest
go run build.go -goos linux -pkg-arch armv7 -buildNumber=${CIRCLE_BUILD_NUM} package-only
go run build.go -goos linux -pkg-arch arm64 -buildNumber=${CIRCLE_BUILD_NUM} package-only
if [ -d '/tmp/phantomjs/darwin' ]; then
cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs
else
echo 'PhantomJS binaries for darwin missing!'
fi
go run build.go -goos darwin -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only
if [ -d '/tmp/phantomjs/windows' ]; then
cp /tmp/phantomjs/windows/phantomjs.exe tools/phantomjs/phantomjs.exe
rm tools/phantomjs/phantomjs
else
echo 'PhantomJS binaries for windows missing!'
fi
go run build.go -goos windows -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only
fi
......@@ -4,6 +4,8 @@
# This script is executed from within the container.
#
CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc
GOPATH=/go
REPO_PATH=$GOPATH/src/github.com/grafana/grafana
......@@ -11,23 +13,29 @@ cd /go/src/github.com/grafana/grafana
echo "current dir: $(pwd)"
if [ "$CIRCLE_TAG" != "" ]; then
echo "Building a release from tag $CIRCLE_TAG"
go run build.go -buildNumber=${CIRCLE_BUILD_NUM} -includeBuildNumber=false build
echo "Building releases from tag $CIRCLE_TAG"
CC=${CCX64} go run build.go -includeBuildNumber=false build
else
echo "Building incremental build for $CIRCLE_BRANCH"
go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build
CC=${CCX64} go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build
fi
yarn install --pure-lockfile --no-progress
source /etc/profile.d/rvm.sh
echo "current dir: $(pwd)"
if [ -d "dist" ]; then
rm -rf dist
fi
if [ "$CIRCLE_TAG" != "" ]; then
echo "Building frontend from tag $CIRCLE_TAG"
go run build.go -includeBuildNumber=false build-frontend
echo "Packaging a release from tag $CIRCLE_TAG"
go run build.go -buildNumber=${CIRCLE_BUILD_NUM} -includeBuildNumber=false package latest
go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest
else
echo "Building frontend for $CIRCLE_BRANCH"
go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build-frontend
echo "Packaging incremental build for $CIRCLE_BRANCH"
go run build.go -buildNumber=${CIRCLE_BUILD_NUM} package latest
go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest
fi
#!/bin/bash -e
if [ ! -d '/tmp/phantomjs' ]; then
_version="2.1.1"
curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-$_version-windows.zip > /tmp/phantomjs-win.zip
curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-$_version-macosx.zip > /tmp/phantomjs-mac.zip
cd /tmp
unzip /tmp/phantomjs-win.zip
unzip /tmp/phantomjs-mac.zip
mkdir -p /tmp/phantomjs/windows /tmp/phantomjs/darwin
cp /tmp/phantomjs-$_version-windows/bin/phantomjs.exe /tmp/phantomjs/windows/phantomjs.exe
cp /tmp/phantomjs-$_version-macosx/bin/phantomjs /tmp/phantomjs/darwin/phantomjs
fi
......@@ -18,8 +18,15 @@ import (
var apiUrl = flag.String("apiUrl", "https://grafana.com/api", "api url")
var apiKey = flag.String("apiKey", "", "api key")
var version = ""
var versionRe = regexp.MustCompile(`grafana-(.*)\.(linux|windows)`)
var versionRe = regexp.MustCompile(`grafana-(.*)(\.|_)(arm64|armv7|darwin|linux|windows|x86_64)`)
var debVersionRe = regexp.MustCompile(`grafana_(.*)_(arm64|armv7|amd64)\.deb`)
var builds = []build{}
var architectureMapping = map[string]string{
"amd64":"amd64",
"armv7":"armv7",
"arm64":"arm64",
"x86_64":"amd64",
}
func main() {
flag.Parse()
......@@ -60,45 +67,68 @@ func main() {
}
}
func packageWalker(path string, f os.FileInfo, err error) error {
if f.Name() == "dist" || strings.Contains(f.Name(), "sha256") || strings.Contains(f.Name(), "latest") {
return nil
}
log.Printf("Finding package file %s", f.Name())
result := versionRe.FindSubmatch([]byte(f.Name()))
func mapPackage(path string, name string, shaBytes []byte) (build, error) {
log.Printf("Finding package file %s", name)
result := versionRe.FindSubmatch([]byte(name))
debResult := debVersionRe.FindSubmatch([]byte(name))
if len(result) > 0 {
version = string(result[1])
log.Printf("Version detected: %v", version)
}
shaBytes, err := ioutil.ReadFile(path + ".sha256")
if err != nil {
log.Fatalf("Failed to read sha256 file %v", err)
} else if (len(debResult) > 0) {
version = string(debResult[1])
}
os := ""
if strings.Contains(f.Name(), "linux-x64.tar.gz") {
if strings.Contains(name, "linux") {
os = "linux"
}
if strings.HasSuffix(f.Name(), "windows-x64.zip") {
if strings.HasSuffix(name, "windows-amd64.zip") {
os = "win"
}
if strings.HasSuffix(f.Name(), ".rpm") {
if strings.HasSuffix(name, "darwin-amd64.tar.gz") {
os = "darwin"
}
if strings.HasSuffix(name, ".rpm") {
os = "rhel"
}
if strings.HasSuffix(f.Name(), ".deb") {
if strings.HasSuffix(name, ".deb") {
os = "deb"
}
builds = append(builds, build{
arch := ""
for archListed, archReal := range architectureMapping {
if strings.Contains(name, archListed) {
arch = archReal
break
}
}
return build{
Os: os,
Arch: "amd64",
Url: "https://s3-us-west-2.amazonaws.com/grafana-releases/master/" + f.Name(),
Arch: arch,
Url: "https://s3-us-west-2.amazonaws.com/grafana-releases/master/" + name,
Sha256: string(shaBytes),
})
}, nil
}
func packageWalker(path string, f os.FileInfo, err error) error {
if f.Name() == "dist" || strings.Contains(f.Name(), "sha256") || strings.Contains(f.Name(), "latest") {
return nil
}
shaBytes, err := ioutil.ReadFile(path + ".sha256")
if err != nil {
log.Fatalf("Failed to read sha256 file %v", err)
}
build, err := mapPackage(path, f.Name(), shaBytes)
if err != nil {
return err
}
builds = append(builds, build)
return nil
}
......
package main
import (
"testing"
)
type testPackage struct {
path string
version string
os string
arch string
}
var testData = []testPackage{
{
path: "grafana-5.2.0-474pre1.arm64.rpm",
version: "5.2.0-474pre1",
os: "rhel",
arch: "arm64",
},
{
path: "grafana-5.2.0-474pre1.armv7.rpm",
version: "5.2.0-474pre1",
os: "rhel",
arch: "armv7",
},
{
path: "grafana-5.2.0-474pre1.darwin-amd64.tar.gz",
version: "5.2.0-474pre1",
os: "darwin",
arch: "amd64",
},
{
path: "grafana-5.2.0-474pre1.linux-amd64.tar.gz",
version: "5.2.0-474pre1",
os: "linux",
arch: "amd64",
},
{
path: "grafana-5.2.0-474pre1.linux-arm64.tar.gz",
version: "5.2.0-474pre1",
os: "linux",
arch: "arm64",
},
{
path: "grafana-5.2.0-474pre1.linux-armv7.tar.gz",
version: "5.2.0-474pre1",
os: "linux",
arch: "armv7",
},
{
path: "grafana-5.2.0-474pre1.windows-amd64.zip",
version: "5.2.0-474pre1",
os: "win",
arch: "amd64",
},
{
path: "grafana-5.2.0-474pre1.x86_64.rpm",
version: "5.2.0-474pre1",
os: "rhel",
arch: "amd64",
},
{
path: "grafana_5.2.0-474pre1_amd64.deb",
version: "5.2.0-474pre1",
os: "deb",
arch: "amd64",
},
{
path: "grafana_5.2.0-474pre1_arm64.deb",
version: "5.2.0-474pre1",
os: "deb",
arch: "arm64",
},
{
path: "grafana_5.2.0-474pre1_armv7.deb",
version: "5.2.0-474pre1",
os: "deb",
arch: "armv7",
},
}
func TestFileWalker(t *testing.T) {
for _, packageInfo := range testData {
version = ""
actualPackageInfo, err := mapPackage(packageInfo.path, packageInfo.path, []byte{})
if err != nil {
t.Error(err)
continue
}
if version != packageInfo.version {
t.Errorf("Testing (%v), expected %v to be %v.", packageInfo.path, version, packageInfo.version)
}
if actualPackageInfo.Os != packageInfo.os {
t.Errorf("Testing (%v), expected %v to be %v.", packageInfo.path, actualPackageInfo.Os, packageInfo.os)
}
if actualPackageInfo.Arch != packageInfo.arch {
t.Errorf("Testing (%v), expected %v to be %v.", packageInfo.path, actualPackageInfo.Arch, packageInfo.arch)
}
}
}
......@@ -3,13 +3,20 @@ var path = require('path');
module.exports = function(grunt) {
"use strict";
// build, then zip and upload to s3
// build then zip
grunt.registerTask('release', [
'build',
'build-post-process',
'compress:release'
]);
// package into archives
grunt.registerTask('package', [
'clean:temp',
'build-post-process',
'compress:release'
]);
grunt.registerTask('build-post-process', function() {
grunt.config('copy.public_to_temp', {
expand: true,
......@@ -18,7 +25,7 @@ module.exports = function(grunt) {
dest: '<%= tempDir %>/public/',
});
grunt.config('copy.backend_bin', {
cwd: 'bin',
cwd: 'bin/<%= platform %>-<%= arch %>',
expand: true,
src: ['*'],
options: { mode: true},
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment