feat(alerting): lots of progress on notifications, refactored them out to their…

feat(alerting): lots of progress on notifications, refactored them out to their own package, restored webhook notitication and added slack notification

pkg/cmd/grafana-server/main.go

@@ -16,7 +16,7 @@ import (
 	"github.com/grafana/grafana/pkg/login"
 	"github.com/grafana/grafana/pkg/metrics"
 	"github.com/grafana/grafana/pkg/plugins"
-	"github.com/grafana/grafana/pkg/services/alerting"
+	alertingInit "github.com/grafana/grafana/pkg/services/alerting/init"
 	"github.com/grafana/grafana/pkg/services/eventpublisher"
 	"github.com/grafana/grafana/pkg/services/notifications"
 	"github.com/grafana/grafana/pkg/services/search"
@@ -68,7 +68,7 @@ func main() {
 	social.NewOAuthService()
 	eventpublisher.Init()
 	plugins.Init()
-	alerting.Init()
+	alertingInit.Init()
 
 	if err := notifications.Init(); err != nil {
 		log.Fatal(3, "Notification service failed to initialize", err)

pkg/services/alerting/alerting.go → pkg/services/alerting/init/init.go

-package alerting
+package init
 
 import (
+	"github.com/grafana/grafana/pkg/services/alerting"
+	_ "github.com/grafana/grafana/pkg/services/alerting/notifiers"
 	"github.com/grafana/grafana/pkg/setting"
 	_ "github.com/grafana/grafana/pkg/tsdb/graphite"
 )
 
-var (
-	maxAlertExecutionRetries = 3
-)
-
-var engine *Engine
+var engine *alerting.Engine
 
 func Init() {
 	if !setting.AlertingEnabled {
 		return
 	}
 
-	engine = NewEngine()
+	engine = alerting.NewEngine()
 	engine.Start()
 }

pkg/services/alerting/models.go

@@ -11,22 +11,9 @@ type AlertJob struct {
 	Offset  int64
 	Delay   bool
 	Running bool
-	RetryCount int
 	Rule    *AlertRule
 }
 
-func (aj *AlertJob) Retryable() bool {
-	return aj.RetryCount < maxAlertExecutionRetries
-}
-
-func (aj *AlertJob) ResetRetry() {
-	aj.RetryCount = 0
-}
-
-func (aj *AlertJob) IncRetry() {
-	aj.RetryCount++
-}
-
 type AlertResultContext struct {
 	Firing      bool
 	IsTestRun   bool

pkg/services/alerting/notifier.go

@@ -2,17 +2,13 @@ package alerting
 
 import (
 	"errors"
-	"fmt"
-	"strings"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/log"
 	m "github.com/grafana/grafana/pkg/models"
-	"github.com/grafana/grafana/pkg/setting"
 )
 
 type RootNotifier struct {
-	NotifierBase
 	log log.Logger
 }
 
@@ -22,6 +18,10 @@ func NewRootNotifier() *RootNotifier {
 	}
 }
 
+func (n *RootNotifier) GetType() string {
+	return "root"
+}
+
 func (n *RootNotifier) Notify(context *AlertResultContext) {
 	n.log.Info("Sending notifications for", "ruleId", context.Rule.Id)
 
@@ -46,7 +46,7 @@ func (n *RootNotifier) getNotifiers(orgId int64, notificationIds []int64) ([]Not
 
 	var result []Notifier
 	for _, notification := range query.Result {
-		if not, err := NewNotificationFromDBModel(notification); err != nil {
+		if not, err := n.getNotifierFor(notification); err != nil {
 			return nil, err
 		} else {
 			result = append(result, not)
@@ -56,47 +56,40 @@ func (n *RootNotifier) getNotifiers(orgId int64, notificationIds []int64) ([]Not
 	return result, nil
 }
 
-type NotifierBase struct {
-	Name string
-	Type string
-}
-
-func (n *NotifierBase) GetType() string {
-	return n.Type
-}
-
-type EmailNotifier struct {
-	NotifierBase
-	Addresses []string
-	log       log.Logger
-}
-
-func (this *EmailNotifier) Notify(context *AlertResultContext) {
-	this.log.Info("Sending alert notification to", "addresses", this.Addresses)
-
-	slugQuery := &m.GetDashboardSlugByIdQuery{Id: context.Rule.DashboardId}
-	if err := bus.Dispatch(slugQuery); err != nil {
-		this.log.Error("Failed to load dashboard", "error", err)
-		return
+func (n *RootNotifier) getNotifierFor(model *m.AlertNotification) (Notifier, error) {
+	factory, found := notifierFactories[model.Type]
+	if !found {
+		return nil, errors.New("Unsupported notification type")
 	}
 
-	ruleLink := fmt.Sprintf("%sdashboard/db/%s?fullscreen&edit&tab=alert&panelId=%d", setting.AppUrl, slugQuery.Result, context.Rule.PanelId)
-
-	cmd := &m.SendEmailCommand{
-		Data: map[string]interface{}{
-			"RuleState": context.Rule.State,
-			"RuleName":  context.Rule.Name,
-			"Severity":  context.Rule.Severity,
-			"RuleLink":  ruleLink,
-		},
-		To:       this.Addresses,
-		Template: "alert_notification.html",
-	}
+	return factory(model)
+	// if model.Type == "email" {
+	// 	addressesString := model.Settings.Get("addresses").MustString()
+	//
+	// 	if addressesString == "" {
+	// 		return nil, fmt.Errorf("Could not find addresses in settings")
+	// 	}
+	//
+	// 		NotifierBase: NotifierBase{
+	// 			Name: model.Name,
+	// 			Type: model.Type,
+	// 		},
+	// 		Addresses: strings.Split(addressesString, "\n"),
+	// 		log:       log.New("alerting.notification.email"),
+	// 	}, nil
+	// }
 
-	err := bus.Dispatch(cmd)
-	if err != nil {
-		this.log.Error("Failed to send alert notification email", "error", err)
-	}
+	// url := settings.Get("url").MustString()
+	// if url == "" {
+	// 	return nil, fmt.Errorf("Could not find url propertie in settings")
+	// }
+	//
+	// return &WebhookNotifier{
+	// 	Url:      url,
+	// 	User:     settings.Get("user").MustString(),
+	// 	Password: settings.Get("password").MustString(),
+	// 	log:      log.New("alerting.notification.webhook"),
+	// }, nil
 }
 
 // type WebhookNotifier struct {
@@ -126,35 +119,10 @@ func (this *EmailNotifier) Notify(context *AlertResultContext) {
 // 	bus.Dispatch(cmd)
 // }
 
-func NewNotificationFromDBModel(model *m.AlertNotification) (Notifier, error) {
-	if model.Type == "email" {
-		addressesString := model.Settings.Get("addresses").MustString()
-
-		if addressesString == "" {
-			return nil, fmt.Errorf("Could not find addresses in settings")
-		}
-
-		return &EmailNotifier{
-			NotifierBase: NotifierBase{
-				Name: model.Name,
-				Type: model.Type,
-			},
-			Addresses: strings.Split(addressesString, "\n"),
-			log:       log.New("alerting.notification.email"),
-		}, nil
-	}
+type NotifierFactory func(notification *m.AlertNotification) (Notifier, error)
 
-	return nil, errors.New("Unsupported notification type")
+var notifierFactories map[string]NotifierFactory = make(map[string]NotifierFactory)
 
-	// url := settings.Get("url").MustString()
-	// if url == "" {
-	// 	return nil, fmt.Errorf("Could not find url propertie in settings")
-	// }
-	//
-	// return &WebhookNotifier{
-	// 	Url:      url,
-	// 	User:     settings.Get("user").MustString(),
-	// 	Password: settings.Get("password").MustString(),
-	// 	log:      log.New("alerting.notification.webhook"),
-	// }, nil
+func RegisterNotifier(typeName string, factory NotifierFactory) {
+	notifierFactories[typeName] = factory
 }

pkg/services/alerting/notifiers/base.go

+package notifiers
+
+type NotifierBase struct {
+	Name string
+	Type string
+}
+
+func (n *NotifierBase) GetType() string {
+	return n.Type
+}

pkg/services/alerting/notifiers/common.go

+package notifiers
+
+import (
+	"fmt"
+
+	"github.com/grafana/grafana/pkg/bus"
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+func getRuleLink(rule *alerting.AlertRule) (string, error) {
+	slugQuery := &m.GetDashboardSlugByIdQuery{Id: rule.DashboardId}
+	if err := bus.Dispatch(slugQuery); err != nil {
+		return "", err
+	}
+
+	ruleLink := fmt.Sprintf("%sdashboard/db/%s?fullscreen&edit&tab=alert&panelId=%d", setting.AppUrl, slugQuery.Result, rule.PanelId)
+	return ruleLink, nil
+}

pkg/services/alerting/notifiers/email.go

+package notifiers
+
+import (
+	"strings"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/log"
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
+)
+
+func init() {
+	alerting.RegisterNotifier("email", NewEmailNotifier)
+}
+
+type EmailNotifier struct {
+	NotifierBase
+	Addresses []string
+	log       log.Logger
+}
+
+func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
+	addressesString := model.Settings.Get("addresses").MustString()
+
+	if addressesString == "" {
+		return nil, alerting.AlertValidationError{Reason: "Could not find addresses in settings"}
+	}
+
+	return &EmailNotifier{
+		NotifierBase: NotifierBase{
+			Name: model.Name,
+			Type: model.Type,
+		},
+		Addresses: strings.Split(addressesString, "\n"),
+		log:       log.New("alerting.notifier.email"),
+	}, nil
+}
+
+func (this *EmailNotifier) Notify(context *alerting.AlertResultContext) {
+	this.log.Info("Sending alert notification to", "addresses", this.Addresses)
+
+	ruleLink, err := getRuleLink(context.Rule)
+	if err != nil {
+		this.log.Error("Failed get rule link", "error", err)
+		return
+	}
+
+	cmd := &m.SendEmailCommand{
+		Data: map[string]interface{}{
+			"RuleState": context.Rule.State,
+			"RuleName":  context.Rule.Name,
+			"Severity":  context.Rule.Severity,
+			"RuleLink":  ruleLink,
+		},
+		To:       this.Addresses,
+		Template: "alert_notification.html",
+	}
+
+	if err := bus.Dispatch(cmd); err != nil {
+		this.log.Error("Failed to send alert notification email", "error", err)
+	}
+}

pkg/services/alerting/notifiers/email_test.go

+package notifiers
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	m "github.com/grafana/grafana/pkg/models"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestEmailNotifier(t *testing.T) {
+	Convey("Email notifier tests", t, func() {
+
+		Convey("Parsing alert notification from settings", func() {
+			Convey("empty settings should return error", func() {
+				json := `{ }`
+
+				settingsJSON, _ := simplejson.NewJson([]byte(json))
+				model := &m.AlertNotification{
+					Name:     "ops",
+					Type:     "email",
+					Settings: settingsJSON,
+				}
+
+				_, err := NewEmailNotifier(model)
+				So(err, ShouldNotBeNil)
+			})
+
+			Convey("from settings", func() {
+				json := `
+				{
+					"addresses": "ops@grafana.org"
+				}`
+
+				settingsJSON, _ := simplejson.NewJson([]byte(json))
+				model := &m.AlertNotification{
+					Name:     "ops",
+					Type:     "email",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewEmailNotifier(model)
+				emailNotifier := not.(*EmailNotifier)
+
+				So(err, ShouldBeNil)
+				So(emailNotifier.Name, ShouldEqual, "ops")
+				So(emailNotifier.Type, ShouldEqual, "email")
+				So(emailNotifier.Addresses[0], ShouldEqual, "ops@grafana.org")
+			})
+		})
+	})
+}

pkg/services/alerting/notifiers/slack.go

+package notifiers
+
+import (
+	"fmt"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
+)
+
+func init() {
+	alerting.RegisterNotifier("slack", NewSlackNotifier)
+}
+
+func NewSlackNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
+	url := model.Settings.Get("url").MustString()
+	if url == "" {
+		return nil, alerting.AlertValidationError{Reason: "Could not find url property in settings"}
+	}
+
+	return &SlackNotifier{
+		NotifierBase: NotifierBase{
+			Name: model.Name,
+			Type: model.Type,
+		},
+		Url: url,
+		log: log.New("alerting.notifier.slack"),
+	}, nil
+}
+
+type SlackNotifier struct {
+	NotifierBase
+	Url string
+	log log.Logger
+}
+
+func (this *SlackNotifier) Notify(context *alerting.AlertResultContext) {
+	this.log.Info("Executing slack notification", "ruleId", context.Rule.Id, "notification", this.Name)
+
+	rule := context.Rule
+
+	ruleLink, err := getRuleLink(rule)
+	if err != nil {
+		this.log.Error("Failed get rule link", "error", err)
+		return
+	}
+
+	stateText := string(rule.Severity)
+	if !context.Firing {
+		stateText = "ok"
+	}
+
+	text := fmt.Sprintf("[%s]: <%s|%s>", stateText, ruleLink, rule.Name)
+
+	body := simplejson.New()
+	body.Set("text", text)
+
+	data, _ := body.MarshalJSON()
+	cmd := &m.SendWebhook{Url: this.Url, Body: string(data)}
+
+	if err := bus.Dispatch(cmd); err != nil {
+		this.log.Error("Failed to send slack notification", "error", err, "webhook", this.Name)
+	}
+}

pkg/services/alerting/notifiers/webhook.go

+package notifiers
+
+import (
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
+)
+
+func init() {
+	alerting.RegisterNotifier("webhook", NewWebHookNotifier)
+}
+
+func NewWebHookNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
+	url := model.Settings.Get("url").MustString()
+	if url == "" {
+		return nil, alerting.AlertValidationError{Reason: "Could not find url property in settings"}
+	}
+
+	return &WebhookNotifier{
+		NotifierBase: NotifierBase{
+			Name: model.Name,
+			Type: model.Type,
+		},
+		Url:      url,
+		User:     model.Settings.Get("user").MustString(),
+		Password: model.Settings.Get("password").MustString(),
+		log:      log.New("alerting.notifier.webhook"),
+	}, nil
+}
+
+type WebhookNotifier struct {
+	NotifierBase
+	Url      string
+	User     string
+	Password string
+	log      log.Logger
+}
+
+func (this *WebhookNotifier) Notify(context *alerting.AlertResultContext) {
+	this.log.Info("Sending webhook")
+
+	bodyJSON := simplejson.New()
+	bodyJSON.Set("name", context.Rule.Name)
+	bodyJSON.Set("firing", context.Firing)
+	bodyJSON.Set("severity", context.Rule.Severity)
+
+	body, _ := bodyJSON.MarshalJSON()
+
+	cmd := &m.SendWebhook{
+		Url:      this.Url,
+		User:     this.User,
+		Password: this.Password,
+		Body:     string(body),
+	}
+
+	if err := bus.Dispatch(cmd); err != nil {
+		this.log.Error("Failed to send webhook", "error", err, "webhook", this.Name)
+	}
+}

pkg/services/alerting/notifiers/webhook_test.go

+package notifiers
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	m "github.com/grafana/grafana/pkg/models"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestWebhookNotifier(t *testing.T) {
+	Convey("Webhook notifier tests", t, func() {
+
+		Convey("Parsing alert notification from settings", func() {
+			Convey("empty settings should return error", func() {
+				json := `{ }`
+
+				settingsJSON, _ := simplejson.NewJson([]byte(json))
+				model := &m.AlertNotification{
+					Name:     "ops",
+					Type:     "email",
+					Settings: settingsJSON,
+				}
+
+				_, err := NewWebHookNotifier(model)
+				So(err, ShouldNotBeNil)
+			})
+
+			Convey("from settings", func() {
+				json := `
+				{
+          "url": "http://google.com"
+				}`
+
+				settingsJSON, _ := simplejson.NewJson([]byte(json))
+				model := &m.AlertNotification{
+					Name:     "ops",
+					Type:     "email",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewWebHookNotifier(model)
+				emailNotifier := not.(*WebhookNotifier)
+
+				So(err, ShouldBeNil)
+				So(emailNotifier.Name, ShouldEqual, "ops")
+				So(emailNotifier.Type, ShouldEqual, "email")
+				So(emailNotifier.Url, ShouldEqual, "http://google.com")
+			})
+		})
+	})
+}

pkg/services/alerting/scheduler.go

@@ -30,7 +30,6 @@ func (s *SchedulerImpl) Update(alerts []*AlertRule) {
 		} else {
 			job = &AlertJob{
 				Running: false,
-				RetryCount: 0,
 			}
 		}
 

pkg/services/notifications/webhook.go

@@ -2,6 +2,8 @@ package notifications
 
 import (
 	"bytes"
+	"fmt"
+	"io/ioutil"
 	"net/http"
 	"time"
 
@@ -39,6 +41,8 @@ func processWebhookQueue() {
 }
 
 func sendWebRequest(webhook *Webhook) error {
+	webhookLog.Debug("Sending webhook", "url", webhook.Url)
+
 	client := http.Client{
 		Timeout: time.Duration(3 * time.Second),
 	}
@@ -56,8 +60,17 @@ func sendWebRequest(webhook *Webhook) error {
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
 
+	_, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("Webhook response code %s", resp.StatusCode)
+	}
+
+	defer resp.Body.Close()
 	return nil
 }
 

public/app/features/alerting/notification_edit_ctrl.ts

@@ -41,6 +41,10 @@ export class AlertNotificationEditCtrl {
       });
     }
   }
+
+  typeChanged() {
+    this.model.settings = {};
+  }
 }
 
 coreModule.controller('AlertNotificationEditCtrl', AlertNotificationEditCtrl);

public/app/features/alerting/partials/notification_edit.html

@@ -20,7 +20,7 @@
 			<div class="gf-form-select-wrapper width-15">
 				<select class="gf-form-input"
 					ng-model="ctrl.model.type"
-					ng-options="t for t in ['webhook', 'email']"
+					ng-options="t for t in ['webhook', 'email', 'slack']"
 					ng-change="ctrl.typeChanged(notification, $index)">
 				</select>
 			</div>
@@ -45,6 +45,14 @@
 		</div>
 	</div>
 
+  <div class="gf-form-group" ng-show="ctrl.model.type === 'slack'">
+    <h3 class="page-heading">Slack settings</h3>
+		<div class="gf-form">
+			<span class="gf-form-label width-6">Url</span>
+			<input type="text" class="gf-form-input max-width-26" ng-model="ctrl.model.settings.url" placeholder="Slack incoming webhook url"></input>
+		</div>
+	</div>
+
 	<div class="gf-form-group section" ng-show="ctrl.model.type === 'email'">
     <h3 class="page-heading">Email addresses</h3>
 		<div class="gf-form">