Add monitoring mixing for Grafana (#28285)

Co-authored-by: Tom Wilkie <tom.wilkie@gmail.com>

grafana-mixin/.gitignore

+alerts.yaml
+rules.yaml
+dashboards_out
\ No newline at end of file

grafana-mixin/Makefile

+JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 2 --string-style s --comment-style s
+
+all: fmt lint build clean
+
+fmt:
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		xargs -n 1 -- $(JSONNET_FMT) -i
+
+lint:
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		while read f; do \
+			$(JSONNET_FMT) "$$f" | diff -u "$$f" -; \
+		done
+
+	mixtool lint mixin.libsonnet
+
+build:
+	mixtool generate all mixin.libsonnet
+
+clean:
+	rm -rf dashboards_out alerts.yaml rules.yaml
\ No newline at end of file

grafana-mixin/README.md

+# Grafana Mixin
+
+_This is a work in progress. We aim for it to become a good role model for alerts
+and dashboards eventually, but it is not quite there yet._
+
+The Grafana Mixin is a set of configurable, reusable, and extensible alerts and
+dashboards based on the metrics exported by Grafana. The mixin creates
+recording and alerting rules for Prometheus and suitable dashboard descriptions
+for Grafana.
+
+To use them, you need to have `mixtool` and `jsonnetfmt` installed. If you
+have a working Go development environment, it's easiest to run the following:
+
+```bash
+$ go get github.com/monitoring-mixins/mixtool/cmd/mixtool
+$ go get github.com/google/go-jsonnet/cmd/jsonnetfmt
+```
+
+You can then build the Prometheus rules files `alerts.yaml` and
+`rules.yaml` and a directory `dashboard_out` with the JSON dashboard files
+for Grafana:
+
+```bash
+$ make build
+```
+
+For more advanced uses of mixins, see
+https://github.com/monitoring-mixins/docs.

grafana-mixin/alerts/alerts.yaml

+groups:
+- name: GrafanaAlerts
+  rules:
+  - alert: GrafanaRequestsFailing
+    for: 5m
+    expr: |
+      100 * namespace_job_handler_statuscode:http_request_total:rate5m{handler!~"/datasources/proxy/:id.*|/ds/query|/tsdb/query", statuscode=~"5.."}
+      /
+      namespace_job_handler_statuscode:http_request_total:rate5m{handler!~"/datasources/proxy/:id.*|/ds/query|/tsdb/query"}
+      > 0.5
+    labels:
+      severity: 'critical'
+    annotations:
+      message: "'{{ $labels.namespace }}' / '{{ $labels.job }}' / '{{ $labels.handler }}' is experiencing {{ $value | humanize }}% errors"

grafana-mixin/mixin.libsonnet

+{
+  grafanaDashboards: {
+    'grafana-overview.json': (import 'dashboards/grafana-overview.json'),
+  },
+
+  // Helper function to ensure that we don't override other rules, by forcing
+  // the patching of the groups list, and not the overall rules object.
+  local importRules(rules) = {
+    groups+: std.native('parseYaml')(rules)[0].groups,
+  },
+
+  prometheusRules+: importRules(importstr 'rules/rules.yaml'),
+
+  prometheusAlerts+: importRules(importstr 'alerts/alerts.yaml'),
+}

grafana-mixin/rules/rules.yaml

+groups:
+  - name: grafana_rules
+    rules:
+    # Record error rate of http requests excluding dataproxy, /ds/query and /tsdb/query requests
+    - record: namespace_job_handler_statuscode:http_request_total:rate5m
+      expr: |
+          sum by (namespace, job, handler, statuscode) (rate(http_request_total[5m]))

scripts/lib.star

@@ -224,6 +224,7 @@ def lint_backend_step(edition):
             'revive -formatter stylish -config scripts/go/configs/revive.toml ./pkg/...',
             './scripts/revive-strict',
             './scripts/tidy-check.sh',
+            './scripts/mixin-check.sh,
         ],
     }
 

scripts/mixin-check.sh

+#!/bin/bash
+set -eo pipefail
+
+cd grafana-mixin
+go install github.com/monitoring-mixins/mixtool/cmd/mixtool
+go install github.com/google/go-jsonnet/cmd/jsonnetfmt
+make lint build