Alerting: Add support for Sensu Go notification channel (#28012)

* Add support for Sensu Go notification channel

Similar to current support for the older "Core" version of
Sensu, this commit add support for the newer version.

Closes #19908

Signed-off-by: Todd Campbell <todd@sensu.io>

* fix linter errors

Signed-off-by: Todd Campbell <todd@sensu.io>

* Apply suggestions from code review

PR review suggestions

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* Fix no new variables error

* Replace convey testing with testify

Signed-off-by: Todd Campbell <todd@sensu.io>

* Wording suggestions

Signed-off-by: Todd Campbell <todd@sensu.io>

* Add docker compose environment for testing/maintenance

Signed-off-by: Todd Campbell <todd@sensu.io>

* Renamed and fixed docker-compose.yaml to work in devenv

Signed-off-by: Todd Campbell <todd@sensu.io>

* Change sensugo web UI port to 3080 so as not to conflict with grafana

Signed-off-by: Todd Campbell <todd@sensu.io>

* Apply suggestions from code review

Set the API key as a secure value.

Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>

* Add Sensu Go information to notifications doc

Signed-off-by: Todd Campbell <todd@sensu.io>

* Update pkg/services/alerting/notifiers/sensugo.go

Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>

* change assert to require for Error/NoError tests

Signed-off-by: Todd Campbell <todd@sensu.io>

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>

devenv/docker/blocks/sensugo/backend.yml

+---
+# Sensu backend configuration
+
+##
+# backend configuration
+##
+state-dir: "/var/lib/sensu/sensu-backend"
+#cache-dir: "/var/cache/sensu/sensu-backend"
+#config-file: "/etc/sensu/backend.yml"
+#debug: false
+#deregistration-handler: "example_handler"
+#log-level: "warn" # available log levels: panic, fatal, error, warn, info, debug
+
+##
+# agent configuration
+##
+#agent-host: "[::]" # listen on all IPv4 and IPv6 addresses
+#agent-port: 8081
+
+##
+# api configuration
+##
+#api-listen-address: "[::]:8080" # listen on all IPv4 and IPv6 addresses
+#api-url: "http://localhost:8080"
+
+##
+# dashboard configuration
+##
+#dashboard-cert-file: "/path/to/ssl/cert.pem"
+#dashboard-key-file: "/path/to/ssl/key.pem"
+#dashboard-host: "[::]" # listen on all IPv4 and IPv6 addresses
+#dashboard-port: 3000
+
+##
+# ssl configuration
+##
+#cert-file: "/path/to/ssl/cert.pem"
+#key-file: "/path/to/ssl/key.pem"
+#trusted-ca-file: "/path/to/trusted-certificate-authorities.pem"
+#insecure-skip-tls-verify: false
+
+##
+# store configuration
+##
+#etcd-advertise-client-urls: "http://localhost:2379"
+#etcd-cert-file: "/path/to/ssl/cert.pem"
+#etcd-client-cert-auth: false
+#etcd-initial-advertise-peer-urls: "http://127.0.0.1:2380"
+#etcd-initial-cluster: "default=http://127.0.0.1:2380"
+#etcd-initial-cluster-state: "new" # new or existing
+#etcd-initial-cluster-token: "sensu"
+#etcd-key-file: "/path/to/ssl/key.pem"
+#etcd-listen-client-urls: "http://127.0.0.1:2379"
+#etcd-listen-peer-urls: "http://127.0.0.1:2380"
+#etcd-name: "default"
+#etcd-peer-cert-file: "/path/to/ssl/cert.pem"
+#etcd-peer-client-cert-auth: false
+#etcd-peer-key-file: "/path/to/ssl/key.pem"
+#etcd-peer-trusted-ca-file: "/path/to/ssl/key.pem"
+#etcd-trusted-ca-file: "/path/to/ssl/key.pem"
+#no-embed-etcd: false
+#etcd-cipher-suits
+#  - TLS_EXAMPLE

devenv/docker/blocks/sensugo/docker-compose.yaml

+  sensu-backend:
+    image: sensu/sensu:latest
+    container_name: sensu-backend
+    ports:
+      - "3080:3000"
+      - "8080:8080"
+      - "8081:8081"
+    volumes:
+      - ./docker/blocks/sensugo/backend.yml:/etc/sensu/backend.yml
+      - sensu-backend-data:/var/lib/sensu/etcd
+    environment:
+      SENSU_BACKEND_CLUSTER_ADMIN_USERNAME: admin
+      SENSU_BACKEND_CLUSTER_ADMIN_PASSWORD: Password123
+    command: "sensu-backend start --log-level info"
+
+volumes:
+  sensu-backend-data: {}
+

devenv/docker/blocks/sensugo/notes.md

+# Notes on Sensu Go Docker Block
+
+The API Key needed to connect to Sensu Go has to be created manually.
+
+## Create the API Key
+
+`docker exec -it sensu-backend /bin/ash`
+
+Configure the `sensuctl` command using the pre-set username and password:
+
+```bash
+sensuctl configure -n --url http://127.0.0.1:8080 --username admin --password 'Password123' --namespace default
+```
+
+Generate the API Key:
+
+```bash
+sensuctl api-key grant admin
+```
+
+The output should look similar to this:
+
+```
+Created: /api/core/v2/apikeys/0a1b2c3d-4e5f-6a7b-8c9d-0e1f2a3b4c5d
+```
+
+
+## Configuring the notification channel
+
+### Backend URL
+
+The Backend URL is the API port (8080) forwarded to the container, it should be
+`http://localhost:8080`
+
+### API Key
+
+The `0a1b2c3d-4e5f-6a7b-8c9d-0e1f2a3b4c5d` in the output above is the API Key
+to use in configuring the Sensu Go notification channel.
+

docs/sources/administration/provisioning.md

@@ -479,6 +479,17 @@ The following sections detail the supported settings and secure settings for eac
 | username |                |
 | password | yes            |
 
+#### Alert notification `sensugo`
+
+| Name      | Secure setting |
+| --------  | -------------- |
+| url       |                |
+| apikey    | yes            |
+| entity    |                |
+| check     |                |
+| handler   |                |
+| namespace |                |
+
 #### Alert notification `prometheus-alertmanager`
 
 | Name              | Secure setting |

docs/sources/alerting/notifications.md

@@ -64,6 +64,7 @@ OpsGenie | `opsgenie` | yes, external only | yes
 Prometheus Alertmanager | `prometheus-alertmanager` | yes, external only | yes
 Pushover | `pushover` | yes | no
 Sensu | `sensu` | yes, external only | no
+[Sensu Go](#sensu-go) | `sensugo` | yes, external only | no
 [Slack](#slack) | `slack` | yes | no
 Telegram | `telegram` | yes | no
 Threema | `threema` | yes, external only | no
@@ -212,6 +213,10 @@ Alertmanager handles alerts sent by client applications such as Prometheus serve
 
 [Zenduty](https://www.zenduty.com) is an incident alerting and response orchestration platform that not alerts the right teams via SMS, Phone(Voice), Email, Slack, Microsoft Teams and Push notifications(Android/iOS) whenever a Grafana alert is triggered, but also helps you rapidly triage and remediate critical, user impacting incidents. Grafana alert are sent to Zenduty through Grafana's native webhook dispatcher. Refer the Zenduty-Grafana [integration documentation](https://docs.zenduty.com/docs/grafana) for configuring the integration.
 
+### Sensu Go
+
+[Sensu](https://sensu.io) is a complete solution for monitoring and observability at scale. Sensu Go is designed to give you visibility into everything you care about: traditional server closets, containers, applications, the cloud, and more. Grafana notifications can be sent to Sensu Go as events via the API. This operation requires an API Key. Refer to the [Sensu Go documentation](https://docs.sensu.io/sensu-go/latest/operations/control-access/use-apikeys/#api-key-authentication) for information on creating this key.
+
 ## Enable images in notifications {#external-image-store}
 
 Grafana can render the panel associated with the alert rule as a PNG image and include that in the notification. Read more about the requirements and how to configure

pkg/services/alerting/notifiers/sensugo.go

+package notifiers
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
+)
+
+func init() {
+	alerting.RegisterNotifier(&alerting.NotifierPlugin{
+		Type:        "sensugo",
+		Name:        "Sensu Go",
+		Description: "Sends HTTP POST request to a Sensu Go API",
+		Heading:     "Sensu Go Settings",
+		Factory:     NewSensuGoNotifier,
+		Options: []alerting.NotifierOption{
+			{
+				Label:        "Backend URL",
+				Element:      alerting.ElementTypeInput,
+				InputType:    alerting.InputTypeText,
+				Placeholder:  "http://sensu-api.local:8080",
+				PropertyName: "url",
+				Required:     true,
+			},
+			{
+				Label:        "API Key",
+				Element:      alerting.ElementTypeInput,
+				InputType:    alerting.InputTypePassword,
+				Description:  "API Key to auth to Sensu Go backend",
+				PropertyName: "apikey",
+				Required:     true,
+				Secure:       true,
+			},
+			{
+				Label:        "Proxy entity name",
+				Element:      alerting.ElementTypeInput,
+				InputType:    alerting.InputTypeText,
+				Description:  "If empty, rule name will be used",
+				PropertyName: "entity",
+			},
+			{
+				Label:        "Check name",
+				Element:      alerting.ElementTypeInput,
+				InputType:    alerting.InputTypeText,
+				Description:  "If empty, rule id will be used",
+				PropertyName: "check",
+			},
+			{
+				Label:        "Handler",
+				Element:      alerting.ElementTypeInput,
+				InputType:    alerting.InputTypeText,
+				PropertyName: "handler",
+			},
+			{
+				Label:        "Namespace",
+				Element:      alerting.ElementTypeInput,
+				InputType:    alerting.InputTypeText,
+				Placeholder:  "default",
+				PropertyName: "namespace",
+			},
+		},
+	})
+}
+
+// NewSensuGoNotifier is the constructor for the Sensu Go Notifier.
+func NewSensuGoNotifier(model *models.AlertNotification) (alerting.Notifier, error) {
+	url := model.Settings.Get("url").MustString()
+	apikey := model.DecryptedValue("apikey", model.Settings.Get("apikey").MustString())
+
+	if url == "" {
+		return nil, alerting.ValidationError{Reason: "Could not find URL property in settings"}
+	}
+	if apikey == "" {
+		return nil, alerting.ValidationError{Reason: "Could not find the API Key property in settings"}
+	}
+
+	return &SensuGoNotifier{
+		NotifierBase: NewNotifierBase(model),
+		URL:          url,
+		Entity:       model.Settings.Get("entity").MustString(),
+		Check:        model.Settings.Get("check").MustString(),
+		Namespace:    model.Settings.Get("namespace").MustString(),
+		Handler:      model.Settings.Get("handler").MustString(),
+		APIKey:       apikey,
+		log:          log.New("alerting.notifier.sensugo"),
+	}, nil
+}
+
+// SensuGoNotifier is responsible for sending
+// alert notifications to Sensu Go.
+type SensuGoNotifier struct {
+	NotifierBase
+	URL       string
+	Entity    string
+	Check     string
+	Namespace string
+	Handler   string
+	APIKey    string
+	log       log.Logger
+}
+
+// Notify send alert notification to Sensu Go
+func (sn *SensuGoNotifier) Notify(evalContext *alerting.EvalContext) error {
+	sn.log.Info("Sending Sensu Go result")
+
+	var namespace string
+
+	bodyJSON := simplejson.New()
+	// Sensu Go alerts require an entity and a check. We set it to the user-specified
+	// value (optional), else we fallback and use the grafana rule anme  and ruleID.
+	if sn.Entity != "" {
+		bodyJSON.SetPath([]string{"entity", "metadata", "name"}, sn.Entity)
+	} else {
+		// Sensu Go alerts cannot have spaces in them
+		bodyJSON.SetPath([]string{"entity", "metadata", "name"}, strings.ReplaceAll(evalContext.Rule.Name, " ", "_"))
+	}
+	if sn.Check != "" {
+		bodyJSON.SetPath([]string{"check", "metadata", "name"}, sn.Check)
+	} else {
+		bodyJSON.SetPath([]string{"check", "metadata", "name"}, "grafana_rule_"+strconv.FormatInt(evalContext.Rule.ID, 10))
+	}
+	// Sensu Go requires the entity in an event specify its namespace.  We set it to
+	// the user-specified value (optional), else we fallback and use default
+	if sn.Namespace != "" {
+		bodyJSON.SetPath([]string{"entity", "metadata", "namespace"}, sn.Namespace)
+		namespace = sn.Namespace
+	} else {
+		bodyJSON.SetPath([]string{"entity", "metadata", "namespace"}, "default")
+		namespace = "default"
+	}
+	// Sensu Go needs check output
+	if evalContext.Rule.Message != "" {
+		bodyJSON.SetPath([]string{"check", "output"}, evalContext.Rule.Message)
+	} else {
+		bodyJSON.SetPath([]string{"check", "output"}, "Grafana Metric Condition Met")
+	}
+	// Sensu GO requires that the check portion of the event have an interval
+	bodyJSON.SetPath([]string{"check", "interval"}, 86400)
+
+	switch evalContext.Rule.State {
+	case "alerting":
+		bodyJSON.SetPath([]string{"check", "status"}, 2)
+	case "no_data":
+		bodyJSON.SetPath([]string{"check", "status"}, 1)
+	default:
+		bodyJSON.SetPath([]string{"check", "status"}, 0)
+	}
+
+	if sn.Handler != "" {
+		bodyJSON.SetPath([]string{"check", "handlers"}, []string{sn.Handler})
+	}
+
+	ruleURL, err := evalContext.GetRuleURL()
+	if err == nil {
+		bodyJSON.Set("ruleUrl", ruleURL)
+	}
+
+	labels := map[string]string{
+		"ruleName": evalContext.Rule.Name,
+		"ruleId":   strconv.FormatInt(evalContext.Rule.ID, 10),
+		"ruleURL":  ruleURL,
+	}
+
+	if sn.NeedsImage() && evalContext.ImagePublicURL != "" {
+		labels["imageUrl"] = evalContext.ImagePublicURL
+	}
+
+	bodyJSON.SetPath([]string{"check", "metadata", "labels"}, labels)
+
+	body, err := bodyJSON.MarshalJSON()
+	if err != nil {
+		return err
+	}
+
+	cmd := &models.SendWebhookSync{
+		Url:        fmt.Sprintf("%s/api/core/v2/namespaces/%s/events", strings.TrimSuffix(sn.URL, "/"), namespace),
+		Body:       string(body),
+		HttpMethod: "POST",
+		HttpHeader: map[string]string{
+			"Content-Type":  "application/json",
+			"Authorization": fmt.Sprintf("Key %s", sn.APIKey),
+		},
+	}
+	if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
+		sn.log.Error("Failed to send Sensu Go event", "error", err, "sensugo", sn.Name)
+		return err
+	}
+
+	return nil
+}

pkg/services/alerting/notifiers/sensugo_test.go

+package notifiers
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+func TestSensuGoNotifier(t *testing.T) {
+	json := `{ }`
+
+	settingsJSON, err := simplejson.NewJson([]byte(json))
+	require.NoError(t, err)
+	model := &models.AlertNotification{
+		Name:     "Sensu Go",
+		Type:     "sensugo",
+		Settings: settingsJSON,
+	}
+
+	_, err = NewSensuGoNotifier(model)
+	require.Error(t, err)
+
+	json = `
+	{
+		"url": "http://sensu-api.example.com:8080",
+		"entity": "grafana_instance_01",
+		"check": "grafana_rule_0",
+		"namespace": "default",
+		"handler": "myhandler",
+		"apikey": "abcdef0123456789abcdef"
+	}`
+
+	settingsJSON, err = simplejson.NewJson([]byte(json))
+	require.NoError(t, err)
+	model = &models.AlertNotification{
+		Name:     "Sensu Go",
+		Type:     "sensugo",
+		Settings: settingsJSON,
+	}
+
+	not, err := NewSensuGoNotifier(model)
+	require.NoError(t, err)
+	sensuGoNotifier := not.(*SensuGoNotifier)
+
+	assert.Equal(t, "Sensu Go", sensuGoNotifier.Name)
+	assert.Equal(t, "sensugo", sensuGoNotifier.Type)
+	assert.Equal(t, "http://sensu-api.example.com:8080", sensuGoNotifier.URL)
+	assert.Equal(t, "grafana_instance_01", sensuGoNotifier.Entity)
+	assert.Equal(t, "grafana_rule_0", sensuGoNotifier.Check)
+	assert.Equal(t, "default", sensuGoNotifier.Namespace)
+	assert.Equal(t, "myhandler", sensuGoNotifier.Handler)
+	assert.Equal(t, "abcdef0123456789abcdef", sensuGoNotifier.APIKey)
+}

public/app/types/alerting.ts

@@ -40,6 +40,7 @@ export type NotifierType =
   | 'hipchat'
   | 'email'
   | 'sensu'
+  | 'sensugo'
   | 'googlechat'
   | 'threema'
   | 'teams'