Skip to content

N
nexpie-grafana-theme
Commit 82b84c59 by Leonard Gram Committed by GitHub
Options

Merge pull request #13968 from xlson/configurable-release-publisher 

Configurable release publisher
parents 30e92461 1de35c43
Showing with 316 additions and 90 deletions
.circleci/config.yml
...@@ -335,6 +335,9 @@ jobs: ...@@ -335,6 +335,9 @@ jobs:
- run: - run:
name: deploy to gcp name: deploy to gcp
command: '/opt/google-cloud-sdk/bin/gsutil cp ./enterprise-dist/* gs://$GCP_BUCKET_NAME/enterprise/master' command: '/opt/google-cloud-sdk/bin/gsutil cp ./enterprise-dist/* gs://$GCP_BUCKET_NAME/enterprise/master'
- run:
name: Deploy to grafana.com
command: 'cd enterprise-dist && scripts/build/release_publisher/release_publisher -apikey ${GRAFANA_COM_API_KEY} -enterprise -from-local'
deploy-enterprise-release: deploy-enterprise-release:
......
scripts/build/publish.sh
#/bin/sh #!/bin/sh
# no relation to publish.go # no relation to publish.go
......
scripts/build/release_publisher/externalrelease.go 0 → 100644
package main
import (
"fmt"
"io/ioutil"
"net/http"
"strings"
"time"
)
type releaseFromExternalContent struct {
getter urlGetter
rawVersion string
artifactConfigurations []buildArtifact
}
func (re releaseFromExternalContent) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) {
version := re.rawVersion[1:]
isBeta := strings.Contains(version, "beta")
builds := []build{}
for _, ba := range re.artifactConfigurations {
sha256, err := re.getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(baseArchiveUrl, version, isBeta)))
if err != nil {
return nil, err
}
builds = append(builds, newBuild(baseArchiveUrl, ba, version, isBeta, sha256))
}
r := release{
Version: version,
ReleaseDate: time.Now(),
Stable: !isBeta,
Beta: isBeta,
Nightly: false,
WhatsNewUrl: whatsNewUrl,
ReleaseNotesUrl: releaseNotesUrl,
Builds: builds,
}
return &r, nil
}
type urlGetter interface {
getContents(url string) (string, error)
}
type getHttpContents struct{}
func (getHttpContents) getContents(url string) (string, error) {
response, err := http.Get(url)
if err != nil {
return "", err
}
defer response.Body.Close()
all, err := ioutil.ReadAll(response.Body)
if err != nil {
return "", err
}
return string(all), nil
}
scripts/build/release_publisher/localrelease.go 0 → 100644
package main
import (
"fmt"
"github.com/pkg/errors"
"io/ioutil"
"log"
"os"
"path/filepath"
"regexp"
"strings"
"time"
)
type releaseLocalSources struct {
path string
artifactConfigurations []buildArtifact
}
func (r releaseLocalSources) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error) {
buildData := r.findBuilds(baseArchiveUrl)
rel := release{
Version: buildData.version,
ReleaseDate: time.Now(),
Stable: false,
Beta: false,
Nightly: true,
WhatsNewUrl: whatsNewUrl,
ReleaseNotesUrl: releaseNotesUrl,
Builds: buildData.builds,
}
return &rel, nil
}
type buildData struct {
version string
builds []build
}
func (r releaseLocalSources) findBuilds(baseArchiveUrl string) buildData {
data := buildData{}
filepath.Walk(r.path, createBuildWalker(r.path, &data, r.artifactConfigurations, baseArchiveUrl))
return data
}
func createBuildWalker(path string, data *buildData, archiveTypes []buildArtifact, baseArchiveUrl string) func(path string, f os.FileInfo, err error) error {
return func(path string, f os.FileInfo, err error) error {
if err != nil {
log.Printf("error: %v", err)
}
if f.Name() == path || strings.HasSuffix(f.Name(), ".sha256") {
return nil
}
for _, archive := range archiveTypes {
if strings.HasSuffix(f.Name(), archive.urlPostfix) {
shaBytes, err := ioutil.ReadFile(path + ".sha256")
if err != nil {
log.Fatalf("Failed to read sha256 file %v", err)
}
version, err := grabVersion(f.Name(), archive.urlPostfix)
if err != nil {
log.Println(err)
continue
}
data.version = version
data.builds = append(data.builds, build{
Os: archive.os,
Url: archive.getUrl(baseArchiveUrl, version, false),
Sha256: string(shaBytes),
Arch: archive.arch,
})
return nil
}
}
return nil
}
}
func grabVersion(name string, suffix string) (string, error) {
match := regexp.MustCompile(fmt.Sprintf(`grafana(-enterprise)?[-_](.*)%s`, suffix)).FindSubmatch([]byte(name))
if len(match) > 0 {
return string(match[2]), nil
}
return "", errors.New("No version found.")
}
scripts/build/release_publisher/main.go
...@@ -7,13 +7,13 @@ import ( ...@@ -7,13 +7,13 @@ import (
"os" "os"
) )
var baseUri string = "https://grafana.com/api"
func main() { func main() {
var version string var version string
var whatsNewUrl string var whatsNewUrl string
var releaseNotesUrl string var releaseNotesUrl string
var dryRun bool var dryRun bool
var enterprise bool
var fromLocal bool
var apiKey string var apiKey string
flag.StringVar(&version, "version", "", "Grafana version (ex: --version v5.2.0-beta1)") flag.StringVar(&version, "version", "", "Grafana version (ex: --version v5.2.0-beta1)")
...@@ -21,20 +21,55 @@ func main() { ...@@ -21,20 +21,55 @@ func main() {
flag.StringVar(&releaseNotesUrl, "rn", "", "Grafana version (ex: --rn https://community.grafana.com/t/release-notes-v5-2-x/7894)") flag.StringVar(&releaseNotesUrl, "rn", "", "Grafana version (ex: --rn https://community.grafana.com/t/release-notes-v5-2-x/7894)")
flag.StringVar(&apiKey, "apikey", "", "Grafana.com API key (ex: --apikey ABCDEF)") flag.StringVar(&apiKey, "apikey", "", "Grafana.com API key (ex: --apikey ABCDEF)")
flag.BoolVar(&dryRun, "dry-run", false, "--dry-run") flag.BoolVar(&dryRun, "dry-run", false, "--dry-run")
flag.BoolVar(&enterprise, "enterprise", false, "--enterprise")
flag.BoolVar(&fromLocal, "from-local", false, "--from-local")
flag.Parse() flag.Parse()
if len(os.Args) == 1 { if len(os.Args) == 1 {
fmt.Println("Usage: go run publisher.go main.go --version <v> --wn <what's new url> --rn <release notes url> --apikey <api key> --dry-run false") fmt.Println("Usage: go run publisher.go main.go --version <v> --wn <what's new url> --rn <release notes url> --apikey <api key> --dry-run false --enterprise false")
fmt.Println("example: go run publisher.go main.go --version v5.2.0-beta2 --wn http://docs.grafana.org/guides/whats-new-in-v5-2/ --rn https://community.grafana.com/t/release-notes-v5-2-x/7894 --apikey ASDF123 --dry-run true") fmt.Println("example: go run publisher.go main.go --version v5.2.0-beta2 --wn http://docs.grafana.org/guides/whats-new-in-v5-2/ --rn https://community.grafana.com/t/release-notes-v5-2-x/7894 --apikey ASDF123 --dry-run --enterprise")
os.Exit(1) os.Exit(1)
} }
if dryRun { if dryRun {
log.Println("Dry-run has been enabled.") log.Println("Dry-run has been enabled.")
} }
var baseUrl string
var builder releaseBuilder
var product string
if fromLocal {
path, _ := os.Getwd()
builder = releaseLocalSources{
path: path,
artifactConfigurations: buildArtifactConfigurations,
}
} else {
builder = releaseFromExternalContent{
getter: getHttpContents{},
rawVersion: version,
artifactConfigurations: buildArtifactConfigurations,
}
}
p := publisher{apiKey: apiKey} if enterprise {
if err := p.doRelease(version, whatsNewUrl, releaseNotesUrl, dryRun); err != nil { baseUrl = "https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/release/grafana-enterprise"
product = "grafana-enterprise"
} else {
baseUrl = "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana"
product = "grafana"
}
p := publisher{
apiKey: apiKey,
apiUri: "https://grafana.com/api",
product: product,
dryRun: dryRun,
enterprise: enterprise,
baseArchiveUrl: baseUrl,
builder: builder,
}
if err := p.doRelease(whatsNewUrl, releaseNotesUrl); err != nil {
log.Fatalf("error: %v", err) log.Fatalf("error: %v", err)
} }
} }
scripts/build/release_publisher/publisher.go
...@@ -12,53 +12,47 @@ import ( ...@@ -12,53 +12,47 @@ import (
) )
type publisher struct { type publisher struct {
apiKey string apiKey string
apiUri string
product string
dryRun bool
enterprise bool
baseArchiveUrl string
builder releaseBuilder
} }
func (p *publisher) doRelease(version string, whatsNewUrl string, releaseNotesUrl string, dryRun bool) error { type releaseBuilder interface {
currentRelease, err := newRelease(version, whatsNewUrl, releaseNotesUrl, buildArtifactConfigurations, getHttpContents{}) prepareRelease(baseArchiveUrl, whatsNewUrl string, releaseNotesUrl string) (*release, error)
}
func (p *publisher) doRelease(whatsNewUrl string, releaseNotesUrl string) error {
currentRelease, err := p.builder.prepareRelease(p.baseArchiveUrl, whatsNewUrl, releaseNotesUrl)
if err != nil { if err != nil {
return err return err
} }
if dryRun { if err := p.postRelease(currentRelease); err != nil {
relJson, err := json.Marshal(currentRelease) return err
if err != nil {
return err
}
log.Println(string(relJson))
for _, b := range currentRelease.Builds {
artifactJson, err := json.Marshal(b)
if err != nil {
return err
}
log.Println(string(artifactJson))
}
} else {
if err := p.postRelease(currentRelease); err != nil {
return err
}
} }
return nil return nil
} }
func (p *publisher) postRelease(r *release) error { func (p *publisher) postRelease(r *release) error {
err := p.postRequest("/grafana/versions", r, fmt.Sprintf("Create Release %s", r.Version)) err := p.postRequest("/versions", r, fmt.Sprintf("Create Release %s", r.Version))
if err != nil { if err != nil {
return err return err
} }
err = p.postRequest("/grafana/versions/"+r.Version, r, fmt.Sprintf("Update Release %s", r.Version)) err = p.postRequest("/versions/"+r.Version, r, fmt.Sprintf("Update Release %s", r.Version))
if err != nil { if err != nil {
return err return err
} }
for _, b := range r.Builds { for _, b := range r.Builds {
err = p.postRequest(fmt.Sprintf("/grafana/versions/%s/packages", r.Version), b, fmt.Sprintf("Create Build %s %s", b.Os, b.Arch)) err = p.postRequest(fmt.Sprintf("/versions/%s/packages", r.Version), b, fmt.Sprintf("Create Build %s %s", b.Os, b.Arch))
if err != nil { if err != nil {
return err return err
} }
err = p.postRequest(fmt.Sprintf("/grafana/versions/%s/packages/%s/%s", r.Version, b.Arch, b.Os), b, fmt.Sprintf("Update Build %s %s", b.Os, b.Arch)) err = p.postRequest(fmt.Sprintf("/versions/%s/packages/%s/%s", r.Version, b.Arch, b.Os), b, fmt.Sprintf("Update Build %s %s", b.Os, b.Arch))
if err != nil { if err != nil {
return err return err
} }
...@@ -67,15 +61,13 @@ func (p *publisher) postRelease(r *release) error { ...@@ -67,15 +61,13 @@ func (p *publisher) postRelease(r *release) error {
return nil return nil
} }
const baseArhiveUrl = "https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana"
type buildArtifact struct { type buildArtifact struct {
os string os string
arch string arch string
urlPostfix string urlPostfix string
} }
func (t buildArtifact) getUrl(version string, isBeta bool) string { func (t buildArtifact) getUrl(baseArchiveUrl, version string, isBeta bool) string {
prefix := "-" prefix := "-"
rhelReleaseExtra := "" rhelReleaseExtra := ""
...@@ -87,7 +79,7 @@ func (t buildArtifact) getUrl(version string, isBeta bool) string { ...@@ -87,7 +79,7 @@ func (t buildArtifact) getUrl(version string, isBeta bool) string {
rhelReleaseExtra = "-1" rhelReleaseExtra = "-1"
} }
url := strings.Join([]string{baseArhiveUrl, prefix, version, rhelReleaseExtra, t.urlPostfix}, "") url := strings.Join([]string{baseArchiveUrl, prefix, version, rhelReleaseExtra, t.urlPostfix}, "")
return url return url
} }
...@@ -149,48 +141,32 @@ var buildArtifactConfigurations = []buildArtifact{ ...@@ -149,48 +141,32 @@ var buildArtifactConfigurations = []buildArtifact{
}, },
} }
func newRelease(rawVersion string, whatsNewUrl string, releaseNotesUrl string, artifactConfigurations []buildArtifact, getter urlGetter) (*release, error) { func newBuild(baseArchiveUrl string, ba buildArtifact, version string, isBeta bool, sha256 string) build {
version := rawVersion[1:]
now := time.Now()
isBeta := strings.Contains(version, "beta")
builds := []build{}
for _, ba := range artifactConfigurations {
sha256, err := getter.getContents(fmt.Sprintf("%s.sha256", ba.getUrl(version, isBeta)))
if err != nil {
return nil, err
}
builds = append(builds, newBuild(ba, version, isBeta, sha256))
}
r := release{
Version: version,
ReleaseDate: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local),
Stable: !isBeta,
Beta: isBeta,
Nightly: false,
WhatsNewUrl: whatsNewUrl,
ReleaseNotesUrl: releaseNotesUrl,
Builds: builds,
}
return &r, nil
}
func newBuild(ba buildArtifact, version string, isBeta bool, sha256 string) build {
return build{ return build{
Os: ba.os, Os: ba.os,
Url: ba.getUrl(version, isBeta), Url: ba.getUrl(baseArchiveUrl, version, isBeta),
Sha256: sha256, Sha256: sha256,
Arch: ba.arch, Arch: ba.arch,
} }
} }
func (p *publisher) apiUrl(url string) string {
return fmt.Sprintf("%s/%s%s", p.apiUri, p.product, url)
}
func (p *publisher) postRequest(url string, obj interface{}, desc string) error { func (p *publisher) postRequest(url string, obj interface{}, desc string) error {
jsonBytes, err := json.Marshal(obj) jsonBytes, err := json.Marshal(obj)
if err != nil { if err != nil {
return err return err
} }
req, err := http.NewRequest(http.MethodPost, baseUri+url, bytes.NewReader(jsonBytes))
if p.dryRun {
log.Println(fmt.Sprintf("POST to %s:", p.apiUrl(url)))
log.Println(string(jsonBytes))
return nil
}
req, err := http.NewRequest(http.MethodPost, p.apiUrl(url), bytes.NewReader(jsonBytes))
if err != nil { if err != nil {
return err return err
} }
...@@ -243,24 +219,3 @@ type build struct { ...@@ -243,24 +219,3 @@ type build struct {
Sha256 string `json:"sha256"` Sha256 string `json:"sha256"`
Arch string `json:"arch"` Arch string `json:"arch"`
} }
type urlGetter interface {
getContents(url string) (string, error)
}
type getHttpContents struct{}
func (getHttpContents) getContents(url string) (string, error) {
response, err := http.Get(url)
if err != nil {
return "", err
}
defer response.Body.Close()
all, err := ioutil.ReadAll(response.Body)
if err != nil {
return "", err
}
return string(all), nil
}
scripts/build/release_publisher/publisher_test.go
...@@ -2,16 +2,24 @@ package main ...@@ -2,16 +2,24 @@ package main
import "testing" import "testing"
func TestNewRelease(t *testing.T) { func TestPreparingReleaseFromRemote(t *testing.T) {
versionIn := "v5.2.0-beta1" versionIn := "v5.2.0-beta1"
expectedVersion := "5.2.0-beta1" expectedVersion := "5.2.0-beta1"
whatsNewUrl := "https://whatsnews.foo/" whatsNewUrl := "https://whatsnews.foo/"
relNotesUrl := "https://relnotes.foo/" relNotesUrl := "https://relnotes.foo/"
expectedArch := "amd64" expectedArch := "amd64"
expectedOs := "linux" expectedOs := "linux"
buildArtifacts := []buildArtifact{{expectedOs, expectedArch, ".linux-amd64.tar.gz"}} buildArtifacts := []buildArtifact{{expectedOs,expectedArch, ".linux-amd64.tar.gz"}}
rel, _ := newRelease(versionIn, whatsNewUrl, relNotesUrl, buildArtifacts, mockHttpGetter{}) var builder releaseBuilder
builder = releaseFromExternalContent{
getter: mockHttpGetter{},
rawVersion: versionIn,
artifactConfigurations: buildArtifactConfigurations,
}
rel, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-releases/release/grafana", whatsNewUrl, relNotesUrl)
if !rel.Beta || rel.Stable { if !rel.Beta || rel.Stable {
t.Errorf("%s should have been tagged as beta (not stable), but wasn't .", versionIn) t.Errorf("%s should have been tagged as beta (not stable), but wasn't .", versionIn)
...@@ -41,3 +49,71 @@ type mockHttpGetter struct{} ...@@ -41,3 +49,71 @@ type mockHttpGetter struct{}
func (mockHttpGetter) getContents(url string) (string, error) { func (mockHttpGetter) getContents(url string) (string, error) {
return url, nil return url, nil
} }
func TestPreparingReleaseFromLocal(t *testing.T) {
whatsNewUrl := "https://whatsnews.foo/"
relNotesUrl := "https://relnotes.foo/"
expectedVersion := "5.4.0-123pre1"
expectedBuilds := 4
var builder releaseBuilder
testDataPath := "local_test_data"
builder = releaseLocalSources{
path: testDataPath,
artifactConfigurations: buildArtifactConfigurations,
}
relAll, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl)
if relAll.Stable || !relAll.Nightly {
t.Error("Expected a nightly release but wasn't.")
}
if relAll.ReleaseNotesUrl != relNotesUrl {
t.Errorf("expected releaseNotesUrl to be %s, but it was %s", relNotesUrl, relAll.ReleaseNotesUrl)
}
if relAll.WhatsNewUrl != whatsNewUrl {
t.Errorf("expected whatsNewUrl to be %s, but it was %s", whatsNewUrl, relAll.WhatsNewUrl)
}
if relAll.Beta {
t.Errorf("Expected release to be nightly, not beta.")
}
if relAll.Version != expectedVersion {
t.Errorf("Expected version=%s, but got=%s", expectedVersion, relAll.Version)
}
if len(relAll.Builds) != expectedBuilds {
t.Errorf("Expected %v builds, but was %v", expectedBuilds, len(relAll.Builds))
}
expectedArch := "amd64"
expectedOs := "win"
builder = releaseLocalSources{
path: testDataPath,
artifactConfigurations: []buildArtifact{{
os: expectedOs,
arch: expectedArch,
urlPostfix: ".windows-amd64.zip",
}},
}
relOne, _ := builder.prepareRelease("https://s3-us-west-2.amazonaws.com/grafana-enterprise-releases/master/grafana-enterprise", whatsNewUrl, relNotesUrl)
if len(relOne.Builds) != 1 {
t.Errorf("Expected 1 artifact, but was %v", len(relOne.Builds))
}
build := relOne.Builds[0]
if build.Arch != expectedArch {
t.Fatalf("Expected arch to be %s, but was %s", expectedArch, build.Arch)
}
if build.Os != expectedOs {
t.Fatalf("Expected os to be %s, but was %s", expectedOs, build.Os)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment