Merge pull request #12798 from grafana/grafana-docker

Merges the Grafana docker build into the main repository.

.circleci/config.yml

@@ -5,9 +5,11 @@ aliases:
       ignore: /.*/
     tags:
       only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
-  - &filter-not-release
+  - &filter-not-release-or-master
     tags:
       ignore: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
+    branches:
+      ignore: master
   - &filter-only-master
     branches:
       only: master
@@ -89,7 +91,7 @@ jobs:
           name: run linters
           command: 'gometalinter.v2 --enable-gc --vendor --deadline 10m --disable-all --enable=deadcode --enable=ineffassign --enable=structcheck --enable=unconvert --enable=varcheck ./...'
       - run:
-          name: run go vet 
+          name: run go vet
           command: 'go vet ./pkg/...'
 
   test-frontend:
@@ -156,8 +158,53 @@ jobs:
             - dist/grafana*
             - scripts/*.sh
             - scripts/publish
-      - store_artifacts:
-          path: dist
+
+  build:
+    docker:
+     - image: grafana/build-container:1.0.0
+    working_directory: /go/src/github.com/grafana/grafana
+    steps:
+      - checkout
+      - run:
+          name: prepare build tools
+          command: '/tmp/bootstrap.sh'
+      - run:
+          name: build and package grafana
+          command: './scripts/build/build.sh'
+      - run:
+          name: sign packages
+          command: './scripts/build/sign_packages.sh'
+      - run:
+          name: sha-sum packages
+          command: 'go run build.go sha-dist'
+      - persist_to_workspace:
+          root: .
+          paths:
+            - dist/grafana*
+
+  grafana-docker-master:
+    docker:
+      - image: docker:stable-git
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - setup_remote_docker
+      - run: docker info
+      - run: cp dist/grafana-latest.linux-x64.tar.gz packaging/docker
+      - run: cd packaging/docker && ./build-deploy.sh "master-${CIRCLE_SHA1}"
+
+  grafana-docker-release:
+      docker:
+        - image: docker:stable-git
+      steps:
+        - checkout
+        - attach_workspace:
+            at: .
+        - setup_remote_docker
+        - run: docker info
+        - run: cp dist/grafana-latest.linux-x64.tar.gz packaging/docker
+        - run: cd packaging/docker && ./build-deploy.sh "${CIRCLE_TAG}"
 
   build-enterprise:
     docker:
@@ -214,9 +261,6 @@ jobs:
           name: Trigger Windows build
           command: './scripts/trigger_windows_build.sh ${APPVEYOR_TOKEN} ${CIRCLE_SHA1} master'
       - run:
-          name: Trigger Docker build
-          command: './scripts/trigger_docker_build.sh ${TRIGGER_GRAFANA_PACKER_CIRCLECI_TOKEN} master-$(echo "${CIRCLE_SHA1}" | cut -b1-7)'
-      - run:
           name: Publish to Grafana.com
           command: |
             rm dist/grafana-master-$(echo "${CIRCLE_SHA1}" | cut -b1-7).linux-x64.tar.gz
@@ -237,30 +281,27 @@ jobs:
       - run:
           name: Trigger Windows build
           command: './scripts/trigger_windows_build.sh ${APPVEYOR_TOKEN} ${CIRCLE_SHA1} release'
-      - run:
-          name: Trigger Docker build
-          command: './scripts/trigger_docker_build.sh ${TRIGGER_GRAFANA_PACKER_CIRCLECI_TOKEN} ${CIRCLE_TAG}'
 
 workflows:
   version: 2
-  test-and-build:
+  build-master:
     jobs:
       - build-all:
           filters: *filter-only-master
       - build-enterprise:
           filters: *filter-only-master
       - codespell:
-          filters: *filter-not-release
+          filters: *filter-only-master
       - gometalinter:
-          filters: *filter-not-release
+          filters: *filter-only-master
       - test-frontend:
-          filters: *filter-not-release
+          filters: *filter-only-master
       - test-backend:
-          filters: *filter-not-release
+          filters: *filter-only-master
       - mysql-integration-test:
-          filters: *filter-not-release
+          filters: *filter-only-master
       - postgres-integration-test:
-          filters: *filter-not-release
+          filters: *filter-only-master
       - deploy-master:
           requires:
             - build-all
@@ -270,7 +311,17 @@ workflows:
             - gometalinter
             - mysql-integration-test
             - postgres-integration-test
-          filters: *filter-only-master           
+          filters: *filter-only-master
+      - grafana-docker-master:
+          requires:
+            - build-all
+            - test-backend
+            - test-frontend
+            - codespell
+            - gometalinter
+            - mysql-integration-test
+            - postgres-integration-test
+          filters: *filter-only-master
       - deploy-enterprise-master:
           requires:
             - build-all
@@ -309,3 +360,30 @@ workflows:
             - mysql-integration-test
             - postgres-integration-test
           filters: *filter-only-release
+      - grafana-docker-release:
+          requires:
+            - build-all
+            - test-backend
+            - test-frontend
+            - codespell
+            - gometalinter
+            - mysql-integration-test
+            - postgres-integration-test
+          filters: *filter-only-release
+
+  build-branches-and-prs:
+      jobs:
+        - build:
+            filters: *filter-not-release-or-master
+        - codespell:
+            filters: *filter-not-release-or-master
+        - gometalinter:
+            filters: *filter-not-release-or-master
+        - test-frontend:
+            filters: *filter-not-release-or-master
+        - test-backend:
+            filters: *filter-not-release-or-master
+        - mysql-integration-test:
+            filters: *filter-not-release-or-master
+        - postgres-integration-test:
+            filters: *filter-not-release-or-master

.gitignore

@@ -58,6 +58,7 @@ debug.test
 /examples/*/dist
 /packaging/**/*.rpm
 /packaging/**/*.deb
+/packaging/**/*.tar.gz
 
 # Ignore OSX indexing
 .DS_Store

Makefile

@@ -24,6 +24,12 @@ build-js:
 
 build: build-go build-js
 
+build-docker-dev:
+	@echo "\033[92mInfo:\033[0m the frontend code is expected to be built already."
+	go run build.go -goos linux -pkg-arch amd64 ${OPT} build package-only latest
+	cp dist/grafana-latest.linux-x64.tar.gz packaging/docker
+	cd packaging/docker && docker build --tag grafana/grafana:dev .
+
 test-go:
 	go test -v ./pkg/...
 
@@ -36,4 +42,4 @@ run:
 	./bin/grafana-server
 
 protoc:
-	protoc -I pkg/tsdb/models pkg/tsdb/models/*.proto --go_out=plugins=grpc:pkg/tsdb/models/.
\ No newline at end of file
+	protoc -I pkg/tsdb/models pkg/tsdb/models/*.proto --go_out=plugins=grpc:pkg/tsdb/models/.

README.md

@@ -74,6 +74,15 @@ bra run
 
 Open grafana in your browser (default: `http://localhost:3000`) and login with admin user (default: `user/pass = admin/admin`).
 
+### Building a docker image (on linux/amd64)
+
+This builds a docker image from your local sources:
+
+1. Build the frontend `go run build.go build-frontend`
+2. Build the docker image `make build-docker-dev`
+
+The resulting image will be tagged as `grafana/grafana:dev`
+
 ### Dev config
 
 Create a custom.ini in the conf directory to override default configuration options.

packaging/docker/Dockerfile

+FROM debian:stretch-slim
+
+ARG GRAFANA_TGZ="grafana-latest.linux-x64.tar.gz"
+ARG GF_UID="472"
+ARG GF_GID="472"
+
+ENV PATH=/usr/share/grafana/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+    GF_PATHS_CONFIG="/etc/grafana/grafana.ini" \
+    GF_PATHS_DATA="/var/lib/grafana" \
+    GF_PATHS_HOME="/usr/share/grafana" \
+    GF_PATHS_LOGS="/var/log/grafana" \
+    GF_PATHS_PLUGINS="/var/lib/grafana/plugins" \
+    GF_PATHS_PROVISIONING="/etc/grafana/provisioning"
+
+RUN apt-get update && apt-get install -qq -y tar libfontconfig ca-certificates && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY ${GRAFANA_TGZ} /tmp/grafana.tar.gz
+
+RUN mkdir -p "$GF_PATHS_HOME/.aws" && \
+    tar xfvz /tmp/grafana.tar.gz --strip-components=1 -C "$GF_PATHS_HOME" && \
+    rm /tmp/grafana.tar.gz && \
+    groupadd -r -g $GF_GID grafana && \
+    useradd -r -u $GF_UID -g grafana grafana && \
+    mkdir -p "$GF_PATHS_PROVISIONING/datasources" \
+             "$GF_PATHS_PROVISIONING/dashboards" \
+             "$GF_PATHS_LOGS" \
+             "$GF_PATHS_PLUGINS" \
+             "$GF_PATHS_DATA" && \
+    cp "$GF_PATHS_HOME/conf/sample.ini" "$GF_PATHS_CONFIG" && \
+    cp "$GF_PATHS_HOME/conf/ldap.toml" /etc/grafana/ldap.toml && \
+    chown -R grafana:grafana "$GF_PATHS_DATA" "$GF_PATHS_HOME/.aws" "$GF_PATHS_LOGS" "$GF_PATHS_PLUGINS" && \
+    chmod 777 "$GF_PATHS_DATA" "$GF_PATHS_HOME/.aws" "$GF_PATHS_LOGS" "$GF_PATHS_PLUGINS"
+
+EXPOSE 3000
+
+COPY ./run.sh /run.sh
+
+USER grafana
+WORKDIR /
+ENTRYPOINT [ "/run.sh" ]

packaging/docker/README.md

+# Grafana Docker image
+
+[![CircleCI](https://circleci.com/gh/grafana/grafana-docker.svg?style=svg)](https://circleci.com/gh/grafana/grafana-docker)
+
+## Running your Grafana container
+
+Start your container binding the external port `3000`.
+
+```bash
+docker run -d --name=grafana -p 3000:3000 grafana/grafana
+```
+
+Try it out, default admin user is admin/admin.
+
+## How to use the container
+
+Further documentation can be found at http://docs.grafana.org/installation/docker/
+
+## Changelog
+
+### v5.1.5, v5.2.0-beta2
+* Fix: config keys ending with _FILE are not respected [#170](https://github.com/grafana/grafana-docker/issues/170)
+
+### v5.2.0-beta1
+* Support for Docker Secrets
+
+### v5.1.0
+* Major restructuring of the container
+* Usage of `chown` removed
+* File permissions incompatibility with previous versions
+  * user id changed from 104 to 472
+  * group id changed from 107 to 472
+* Runs as the grafana user by default (instead of root)
+* All default volumes removed
+
+### v4.2.0
+* Plugins are now installed into ${GF_PATHS_PLUGINS}
+* Building the container now requires a full url to the deb package instead of just version
+* Fixes bug caused by installing multiple plugins
+
+### v4.0.0-beta2
+* Plugins dir (`/var/lib/grafana/plugins`) is no longer a separate volume
+
+### v3.1.1
+* Make it possible to install specific plugin version https://github.com/grafana/grafana-docker/issues/59#issuecomment-260584026
\ No newline at end of file

packaging/docker/build-deploy.sh

+#!/bin/sh
+
+_grafana_version=$1
+./build.sh "$_grafana_version"
+docker login -u "$DOCKER_USER" -p "$DOCKER_PASS"
+
+./push_to_docker_hub.sh "$_grafana_version"
+
+if echo "$_grafana_version" | grep -q "^master-"; then
+  apk add --no-cache curl
+  ./deploy_to_k8s.sh "grafana/grafana-dev:$_grafana_version"
+fi

packaging/docker/build.sh

+#!/bin/sh
+
+_grafana_tag=$1
+
+# If the tag starts with v, treat this as a official release
+if echo "$_grafana_tag" | grep -q "^v"; then
+	_grafana_version=$(echo "${_grafana_tag}" | cut -d "v" -f 2)
+	_docker_repo=${2:-grafana/grafana}
+else
+	_grafana_version=$_grafana_tag
+	_docker_repo=${2:-grafana/grafana-dev}
+fi
+
+echo "Building ${_docker_repo}:${_grafana_version}"
+
+docker build \
+	--tag "${_docker_repo}:${_grafana_version}" \
+	--no-cache=true .
+
+# Tag as 'latest' for official release; otherwise tag as grafana/grafana:master
+if echo "$_grafana_tag" | grep -q "^v"; then
+	docker tag "${_docker_repo}:${_grafana_version}" "${_docker_repo}:latest"
+else
+	docker tag "${_docker_repo}:${_grafana_version}" "grafana/grafana:master"
+fi

packaging/docker/custom/Dockerfile

+ARG GRAFANA_VERSION="latest"
+
+FROM grafana/grafana:${GRAFANA_VERSION}
+
+USER grafana
+
+ARG GF_INSTALL_PLUGINS=""
+
+RUN if [ ! -z "${GF_INSTALL_PLUGINS}" ]; then \
+    OLDIFS=$IFS; \
+        IFS=','; \
+    for plugin in ${GF_INSTALL_PLUGINS}; do \
+        IFS=$OLDIFS; \
+        grafana-cli --pluginsDir "$GF_PATHS_PLUGINS" plugins install ${plugin}; \
+    done; \
+fi

packaging/docker/deploy_to_k8s.sh

+#!/bin/sh
+
+curl -s --header "Content-Type: application/json" \
+     --data "{\"build_parameters\": {\"CIRCLE_JOB\": \"deploy\", \"IMAGE_NAMES\": \"$1\"}}" \
+     --request POST \
+     https://circleci.com/api/v1.1/project/github/raintank/deployment_tools/tree/master?circle-token=$CIRCLE_TOKEN

packaging/docker/push_to_docker_hub.sh

+#!/bin/sh
+
+_grafana_tag=$1
+
+# If the tag starts with v, treat this as a official release
+if echo "$_grafana_tag" | grep -q "^v"; then
+	_grafana_version=$(echo "${_grafana_tag}" | cut -d "v" -f 2)
+	_docker_repo=${2:-grafana/grafana}
+else
+	_grafana_version=$_grafana_tag
+	_docker_repo=${2:-grafana/grafana-dev}
+fi
+
+echo "pushing ${_docker_repo}:${_grafana_version}"
+docker push "${_docker_repo}:${_grafana_version}"
+
+if echo "$_grafana_tag" | grep -q "^v"; then
+	echo "pushing ${_docker_repo}:latest"
+	docker push "${_docker_repo}:latest"
+else
+	echo "pushing grafana/grafana:master"
+	docker push grafana/grafana:master
+fi

packaging/docker/run.sh

+#!/bin/bash -e
+
+PERMISSIONS_OK=0
+
+if [ ! -r "$GF_PATHS_CONFIG" ]; then
+    echo "GF_PATHS_CONFIG='$GF_PATHS_CONFIG' is not readable."
+    PERMISSIONS_OK=1
+fi
+
+if [ ! -w "$GF_PATHS_DATA" ]; then
+    echo "GF_PATHS_DATA='$GF_PATHS_DATA' is not writable."
+    PERMISSIONS_OK=1
+fi
+
+if [ ! -r "$GF_PATHS_HOME" ]; then
+    echo "GF_PATHS_HOME='$GF_PATHS_HOME' is not readable."
+    PERMISSIONS_OK=1
+fi
+
+if [ $PERMISSIONS_OK -eq 1 ]; then
+    echo "You may have issues with file permissions, more information here: http://docs.grafana.org/installation/docker/#migration-from-a-previous-version-of-the-docker-container-to-5-1-or-later"
+fi
+
+if [ ! -d "$GF_PATHS_PLUGINS" ]; then
+    mkdir "$GF_PATHS_PLUGINS"
+fi
+
+if [ ! -z ${GF_AWS_PROFILES+x} ]; then
+    > "$GF_PATHS_HOME/.aws/credentials"
+
+    for profile in ${GF_AWS_PROFILES}; do
+        access_key_varname="GF_AWS_${profile}_ACCESS_KEY_ID"
+        secret_key_varname="GF_AWS_${profile}_SECRET_ACCESS_KEY"
+        region_varname="GF_AWS_${profile}_REGION"
+
+        if [ ! -z "${!access_key_varname}" -a ! -z "${!secret_key_varname}" ]; then
+            echo "[${profile}]" >> "$GF_PATHS_HOME/.aws/credentials"
+            echo "aws_access_key_id = ${!access_key_varname}" >> "$GF_PATHS_HOME/.aws/credentials"
+            echo "aws_secret_access_key = ${!secret_key_varname}" >> "$GF_PATHS_HOME/.aws/credentials"
+            if [ ! -z "${!region_varname}" ]; then
+                echo "region = ${!region_varname}" >> "$GF_PATHS_HOME/.aws/credentials"
+            fi
+        fi
+    done
+
+    chmod 600 "$GF_PATHS_HOME/.aws/credentials"
+fi
+
+# Convert all environment variables with names ending in __FILE into the content of
+# the file that they point at and use the name without the trailing __FILE.
+# This can be used to carry in Docker secrets.
+for VAR_NAME in $(env | grep '^GF_[^=]\+__FILE=.\+' | sed -r "s/([^=]*)__FILE=.*/\1/g"); do
+    VAR_NAME_FILE="$VAR_NAME"__FILE
+    if [ "${!VAR_NAME}" ]; then
+        echo >&2 "ERROR: Both $VAR_NAME and $VAR_NAME_FILE are set (but are exclusive)"
+        exit 1
+    fi
+    echo "Getting secret $VAR_NAME from ${!VAR_NAME_FILE}"
+    export "$VAR_NAME"="$(< "${!VAR_NAME_FILE}")"
+    unset "$VAR_NAME_FILE"
+done
+
+export HOME="$GF_PATHS_HOME"
+
+if [ ! -z "${GF_INSTALL_PLUGINS}" ]; then
+  OLDIFS=$IFS
+  IFS=','
+  for plugin in ${GF_INSTALL_PLUGINS}; do
+    IFS=$OLDIFS
+    grafana-cli --pluginsDir "${GF_PATHS_PLUGINS}" plugins install ${plugin}
+  done
+fi
+
+exec grafana-server                                         \
+  --homepath="$GF_PATHS_HOME"                               \
+  --config="$GF_PATHS_CONFIG"                               \
+  "$@"                                                      \
+  cfg:default.log.mode="console"                            \
+  cfg:default.paths.data="$GF_PATHS_DATA"                   \
+  cfg:default.paths.logs="$GF_PATHS_LOGS"                   \
+  cfg:default.paths.plugins="$GF_PATHS_PLUGINS"             \
+  cfg:default.paths.provisioning="$GF_PATHS_PROVISIONING"

scripts/build/build.sh

@@ -14,12 +14,14 @@ echo "current dir: $(pwd)"
 
 if [ "$CIRCLE_TAG" != "" ]; then
   echo "Building releases from tag $CIRCLE_TAG"
-  CC=${CCX64} go run build.go -includeBuildNumber=false build
+  OPT="-includeBuildNumber=false"
 else
   echo "Building incremental build for $CIRCLE_BRANCH"
-  CC=${CCX64} go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build
+  OPT="-buildNumber=${CIRCLE_BUILD_NUM}"
 fi
 
+CC=${CCX64} go run build.go ${OPT} build
+
 yarn install --pure-lockfile --no-progress
 
 echo "current dir: $(pwd)"
@@ -28,14 +30,8 @@ if [ -d "dist" ]; then
   rm -rf dist
 fi
 
-if [ "$CIRCLE_TAG" != "" ]; then
-  echo "Building frontend from tag $CIRCLE_TAG"
-  go run build.go -includeBuildNumber=false build-frontend
-  echo "Packaging a release from tag $CIRCLE_TAG"
-  go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest
-else
-  echo "Building frontend for $CIRCLE_BRANCH"
-  go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build-frontend
-  echo "Packaging incremental build for $CIRCLE_BRANCH"
-  go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest
-fi
+echo "Building frontend"
+go run build.go ${OPT} build-frontend
+
+echo "Packaging"
+go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only latest