feat(alerting): add slack/email support for execution errors

pkg/services/alerting/eval_context.go

@@ -28,36 +28,46 @@ type EvalContext struct {
 	ImageOnDiskPath string
 }
 
-func (a *EvalContext) GetDurationMs() float64 {
-	return float64(a.EndTime.Nanosecond()-a.StartTime.Nanosecond()) / float64(1000000)
+type StateDescription struct {
+	Color string
+	Text  string
+	Data  string
 }
 
-func (c *EvalContext) GetColor() string {
-	if !c.Firing {
-		return "#36a64f"
+func (c *EvalContext) GetStateModel() *StateDescription {
+	if c.Error != nil {
+		return &StateDescription{
+			Color: "#D63232",
+			Text:  "EXECUTION ERROR",
 		}
-
-	if c.Rule.Severity == m.AlertSeverityWarning {
-		return "#fd821b"
-	} else {
-		return "#D63232"
 	}
-}
 
-func (c *EvalContext) GetStateText() string {
 	if !c.Firing {
-		return "OK"
+		return &StateDescription{
+			Color: "#36a64f",
+			Text:  "OK",
+		}
 	}
 
 	if c.Rule.Severity == m.AlertSeverityWarning {
-		return "WARNING"
+		return &StateDescription{
+			Color: "#fd821b",
+			Text:  "WARNING",
+		}
 	} else {
-		return "CRITICAL"
+		return &StateDescription{
+			Color: "#D63232",
+			Text:  "CRITICAL",
 		}
+	}
+}
+
+func (a *EvalContext) GetDurationMs() float64 {
+	return float64(a.EndTime.Nanosecond()-a.StartTime.Nanosecond()) / float64(1000000)
 }
 
 func (c *EvalContext) GetNotificationTitle() string {
-	return "[" + c.GetStateText() + "] " + c.Rule.Name
+	return "[" + c.GetStateModel().Text + "] " + c.Rule.Name
 }
 
 func (c *EvalContext) getDashboardSlug() (string, error) {

pkg/services/alerting/notifier.go

@@ -48,7 +48,6 @@ func (n *RootNotifier) Notify(context *EvalContext) {
 
 	for _, notifier := range notifiers {
 		n.log.Info("Sending notification", "firing", context.Firing, "type", notifier.GetType())
-
 		go notifier.Notify(context)
 	}
 }

pkg/services/alerting/notifiers/email.go

@@ -54,7 +54,7 @@ func (this *EmailNotifier) Notify(context *alerting.EvalContext) {
 			"State":         context.Rule.State,
 			"Name":          context.Rule.Name,
 			"Severity":      context.Rule.Severity,
-			"SeverityColor": context.GetColor(),
+			"SeverityColor": context.GetStateModel().Color,
 			"Message":       context.Rule.Message,
 			"RuleUrl":       ruleUrl,
 			"ImageLink":     context.ImagePublicUrl,

pkg/services/alerting/notifiers/slack.go

@@ -61,13 +61,26 @@ func (this *SlackNotifier) Notify(context *alerting.EvalContext) {
 		}
 	}
 
+	if context.Error != nil {
+		fields = append(fields, map[string]interface{}{
+			"title": "Error message",
+			"value": context.Error.Error(),
+			"short": false,
+		})
+	}
+
+	message := ""
+	if context.Rule.State != m.AlertStateOK { //dont add message when going back to alert state ok.
+		message = context.Rule.Message
+	}
+
 	body := map[string]interface{}{
 		"attachments": []map[string]interface{}{
 			{
-				"color":       context.GetColor(),
+				"color":       context.GetStateModel().Color,
 				"title":       context.GetNotificationTitle(),
 				"title_link":  ruleUrl,
-				"text":        context.Rule.Message,
+				"text":        message,
 				"fields":      fields,
 				"image_url":   context.ImagePublicUrl,
 				"footer":      "Grafana v" + setting.BuildVersion,

pkg/services/alerting/result_handler.go

@@ -65,7 +65,7 @@ func (handler *DefaultResultHandler) Handle(ctx *EvalContext) {
 			Type:      annotations.AlertType,
 			AlertId:   ctx.Rule.Id,
 			Title:     ctx.Rule.Name,
-			Text:      ctx.GetStateText(),
+			Text:      ctx.GetStateModel().Text,
 			NewState:  string(ctx.Rule.State),
 			PrevState: string(oldState),
 			Timestamp: time.Now(),