Skip to content

N
nexpie-grafana-theme
Commit 55b560a4 by bergquist
Options

feat(alerting): filter notifications based on severity 

closes #5851
parent b1ed641d
Showing with 202 additions and 148 deletions
pkg/services/alerting/interfaces.go
package alerting
import "time"
import (
"time"
"github.com/grafana/grafana/pkg/models"
)
type EvalHandler interface {
Eval(context *EvalContext)
......@@ -15,6 +19,7 @@ type Notifier interface {
Notify(alertResult *EvalContext)
GetType() string
NeedsImage() bool
MatchSeverity(result models.AlertSeverityType) bool
}
type Condition interface {
......
pkg/services/alerting/notifier.go
......@@ -28,10 +28,14 @@ func (n *RootNotifier) NeedsImage() bool {
return false
}
func (n *RootNotifier) MatchSeverity(result m.AlertSeverityType) bool {
return false
}
func (n *RootNotifier) Notify(context *EvalContext) {
n.log.Info("Sending notifications for", "ruleId", context.Rule.Id)
notifiers, err := n.getNotifiers(context.Rule.OrgId, context.Rule.Notifications)
notifiers, err := n.getNotifiers(context.Rule.OrgId, context.Rule.Notifications, context)
if err != nil {
n.log.Error("Failed to read notifications", "error", err)
return
......@@ -87,7 +91,7 @@ func (n *RootNotifier) uploadImage(context *EvalContext) error {
return nil
}
func (n *RootNotifier) getNotifiers(orgId int64, notificationIds []int64) ([]Notifier, error) {
func (n *RootNotifier) getNotifiers(orgId int64, notificationIds []int64, context *EvalContext) ([]Notifier, error) {
query := &m.GetAlertNotificationsToSendQuery{OrgId: orgId, Ids: notificationIds}
if err := bus.Dispatch(query); err != nil {
......@@ -96,17 +100,19 @@ func (n *RootNotifier) getNotifiers(orgId int64, notificationIds []int64) ([]Not
var result []Notifier
for _, notification := range query.Result {
if not, err := n.getNotifierFor(notification); err != nil {
if not, err := n.createNotifierFor(notification); err != nil {
return nil, err
} else {
result = append(result, not)
if shouldUseNotification(not, context) {
result = append(result, not)
}
}
}
return result, nil
}
func (n *RootNotifier) getNotifierFor(model *m.AlertNotification) (Notifier, error) {
func (n *RootNotifier) createNotifierFor(model *m.AlertNotification) (Notifier, error) {
factory, found := notifierFactories[model.Type]
if !found {
return nil, errors.New("Unsupported notification type")
......@@ -115,6 +121,18 @@ func (n *RootNotifier) getNotifierFor(model *m.AlertNotification) (Notifier, err
return factory(model)
}
func shouldUseNotification(notifier Notifier, context *EvalContext) bool {
if !context.Firing {
return true
}
if context.Error != nil {
return true
}
return notifier.MatchSeverity(context.Rule.Severity)
}
type NotifierFactory func(notification *m.AlertNotification) (Notifier, error)
var notifierFactories map[string]NotifierFactory = make(map[string]NotifierFactory)
......
pkg/services/alerting/notifier_test.go
package alerting
// func TestAlertNotificationExtraction(t *testing.T) {
// Convey("Notifier tests", t, func() {
// Convey("rules for sending notifications", func() {
// dummieNotifier := NotifierImpl{}
//
// result := &AlertResult{
// State: alertstates.Critical,
// }
//
// notifier := &Notification{
// Name: "Test Notifier",
// Type: "TestType",
// SendCritical: true,
// SendWarning: true,
// }
//
// Convey("Should send notification", func() {
// So(dummieNotifier.ShouldDispath(result, notifier), ShouldBeTrue)
// })
//
// Convey("warn:false and state:warn should not send", func() {
// result.State = alertstates.Warn
// notifier.SendWarning = false
// So(dummieNotifier.ShouldDispath(result, notifier), ShouldBeFalse)
// })
// })
//
// Convey("Parsing alert notification from settings", func() {
// Convey("Parsing email", func() {
// Convey("empty settings should return error", func() {
// json := `{ }`
//
// settingsJSON, _ := simplejson.NewJson([]byte(json))
// model := &m.AlertNotification{
// Name: "ops",
// Type: "email",
// Settings: settingsJSON,
// }
//
// _, err := NewNotificationFromDBModel(model)
// So(err, ShouldNotBeNil)
// })
//
// Convey("from settings", func() {
// json := `
// {
// "to": "ops@grafana.org"
// }`
//
// settingsJSON, _ := simplejson.NewJson([]byte(json))
// model := &m.AlertNotification{
// Name: "ops",
// Type: "email",
// Settings: settingsJSON,
// }
//
// not, err := NewNotificationFromDBModel(model)
//
// So(err, ShouldBeNil)
// So(not.Name, ShouldEqual, "ops")
// So(not.Type, ShouldEqual, "email")
// So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.EmailNotifier")
//
// email := not.Notifierr.(*EmailNotifier)
// So(email.To, ShouldEqual, "ops@grafana.org")
// })
// })
//
// Convey("Parsing webhook", func() {
// Convey("empty settings should return error", func() {
// json := `{ }`
//
// settingsJSON, _ := simplejson.NewJson([]byte(json))
// model := &m.AlertNotification{
// Name: "ops",
// Type: "webhook",
// Settings: settingsJSON,
// }
//
// _, err := NewNotificationFromDBModel(model)
// So(err, ShouldNotBeNil)
// })
//
// Convey("from settings", func() {
// json := `
// {
// "url": "http://localhost:3000",
// "username": "username",
// "password": "password"
// }`
//
// settingsJSON, _ := simplejson.NewJson([]byte(json))
// model := &m.AlertNotification{
// Name: "slack",
// Type: "webhook",
// Settings: settingsJSON,
// }
//
// not, err := NewNotificationFromDBModel(model)
//
// So(err, ShouldBeNil)
// So(not.Name, ShouldEqual, "slack")
// So(not.Type, ShouldEqual, "webhook")
// So(reflect.TypeOf(not.Notifierr).Elem().String(), ShouldEqual, "alerting.WebhookNotifier")
//
// webhook := not.Notifierr.(*WebhookNotifier)
// So(webhook.Url, ShouldEqual, "http://localhost:3000")
// })
// })
// })
// })
// }
import (
"testing"
"fmt"
"github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
type FakeNotifier struct {
FakeMatchResult bool
}
func (fn *FakeNotifier) GetType() string {
return "FakeNotifier"
}
func (fn *FakeNotifier) NeedsImage() bool {
return true
}
func (fn *FakeNotifier) Notify(alertResult *EvalContext) {}
func (fn *FakeNotifier) MatchSeverity(result models.AlertSeverityType) bool {
return fn.FakeMatchResult
}
func TestAlertNotificationExtraction(t *testing.T) {
Convey("Notifier tests", t, func() {
Convey("none firing alerts", func() {
ctx := &EvalContext{
Firing: false,
Rule: &Rule{
Severity: models.AlertSeverityCritical,
},
}
notifier := &FakeNotifier{FakeMatchResult: false}
So(shouldUseNotification(notifier, ctx), ShouldBeTrue)
})
Convey("exeuction error cannot be ignored", func() {
ctx := &EvalContext{
Firing: true,
Error: fmt.Errorf("I used to be a programmer just like you"),
Rule: &Rule{
Severity: models.AlertSeverityCritical,
},
}
notifier := &FakeNotifier{FakeMatchResult: false}
So(shouldUseNotification(notifier, ctx), ShouldBeTrue)
})
Convey("firing alert that match", func() {
ctx := &EvalContext{
Firing: true,
Rule: &Rule{
Severity: models.AlertSeverityCritical,
},
}
notifier := &FakeNotifier{FakeMatchResult: true}
So(shouldUseNotification(notifier, ctx), ShouldBeTrue)
})
Convey("firing alert that dont match", func() {
ctx := &EvalContext{
Firing: true,
Rule: &Rule{
Severity: models.AlertSeverityCritical,
},
}
notifier := &FakeNotifier{FakeMatchResult: false}
So(shouldUseNotification(notifier, ctx), ShouldBeFalse)
})
})
}
pkg/services/alerting/notifiers/base.go
package notifiers
import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
)
type NotifierBase struct {
Name string
Type string
Name string
Type string
SeverityFilter models.AlertSeverityType
}
func NewNotifierBase(name, notifierType string, model *simplejson.Json) NotifierBase {
base := NotifierBase{Name: name, Type: notifierType}
severityFilter := models.AlertSeverityType(model.Get("severityFilter").MustString(""))
if severityFilter == models.AlertSeverityCritical || severityFilter == models.AlertSeverityWarning {
base.SeverityFilter = severityFilter
}
return base
}
func (n *NotifierBase) MatchSeverity(result models.AlertSeverityType) bool {
if !n.SeverityFilter.IsValid() {
return true
}
return n.SeverityFilter == result
}
func (n *NotifierBase) GetType() string {
......
pkg/services/alerting/notifiers/base_test.go 0 → 100644
package notifiers
import (
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestBaseNotifier(t *testing.T) {
Convey("Parsing base notification severity", t, func() {
Convey("matches", func() {
json := `
{
"severityFilter": "critical"
}`
settingsJSON, _ := simplejson.NewJson([]byte(json))
not := NewNotifierBase("ops", "email", settingsJSON)
So(not.MatchSeverity(m.AlertSeverityCritical), ShouldBeTrue)
})
Convey("does not match", func() {
json := `
{
"severityFilter": "critical"
}`
settingsJSON, _ := simplejson.NewJson([]byte(json))
not := NewNotifierBase("ops", "email", settingsJSON)
So(not.MatchSeverity(m.AlertSeverityWarning), ShouldBeFalse)
})
})
}
pkg/services/alerting/notifiers/email.go
......@@ -29,12 +29,9 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
}
return &EmailNotifier{
NotifierBase: NotifierBase{
Name: model.Name,
Type: model.Type,
},
Addresses: strings.Split(addressesString, "\n"),
log: log.New("alerting.notifier.email"),
NotifierBase: NewNotifierBase(model.Name, model.Type, model.Settings),
Addresses: strings.Split(addressesString, "\n"),
log: log.New("alerting.notifier.email"),
}, nil
}
......
pkg/services/alerting/notifiers/slack.go
......@@ -23,12 +23,9 @@ func NewSlackNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
}
return &SlackNotifier{
NotifierBase: NotifierBase{
Name: model.Name,
Type: model.Type,
},
Url: url,
log: log.New("alerting.notifier.slack"),
NotifierBase: NewNotifierBase(model.Name, model.Type, model.Settings),
Url: url,
log: log.New("alerting.notifier.slack"),
}, nil
}
......
pkg/services/alerting/notifiers/webhook.go
......@@ -20,14 +20,11 @@ func NewWebHookNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
}
return &WebhookNotifier{
NotifierBase: NotifierBase{
Name: model.Name,
Type: model.Type,
},
Url: url,
User: model.Settings.Get("user").MustString(),
Password: model.Settings.Get("password").MustString(),
log: log.New("alerting.notifier.webhook"),
NotifierBase: NewNotifierBase(model.Name, model.Type, model.Settings),
Url: url,
User: model.Settings.Get("user").MustString(),
Password: model.Settings.Get("password").MustString(),
log: log.New("alerting.notifier.webhook"),
}, nil
}
......
pkg/services/alerting/test_notification.go
......@@ -28,7 +28,7 @@ func handleNotificationTestCommand(cmd *NotificationTestCommand) error {
Settings: cmd.Settings,
}
notifiers, err := notifier.getNotifierFor(model)
notifiers, err := notifier.createNotifierFor(model)
if err != nil {
log.Error2("Failed to create notifier", "error", err.Error())
......
public/app/features/alerting/notification_edit_ctrl.ts
......@@ -17,7 +17,9 @@ export class AlertNotificationEditCtrl {
} else {
this.model = {
type: 'email',
settings: {},
settings: {
severityFilter: 'none'
},
isDefault: false
};
}
......
public/app/features/alerting/partials/notification_edit.html
......@@ -12,11 +12,11 @@
<div class="gf-form-group">
<div class="gf-form">
<span class="gf-form-label width-8">Name</span>
<span class="gf-form-label width-12">Name</span>
<input type="text" class="gf-form-input max-width-15" ng-model="ctrl.model.name" required></input>
</div>
<div class="gf-form">
<span class="gf-form-label width-8">Type</span>
<span class="gf-form-label width-12">Type</span>
<div class="gf-form-select-wrapper width-15">
<select class="gf-form-input"
ng-model="ctrl.model.type"
......@@ -26,10 +26,19 @@
</div>
</div>
<div class="gf-form">
<span class="gf-form-label width-12">Severity filter</span>
<div class="gf-form-select-wrapper width-15">
<select class="gf-form-input"
ng-model="ctrl.model.settings.severityFilter"
ng-options="t for t in ['none', 'critical', 'warning']">
</select>
</div>
</div>
<div class="gf-form">
<gf-form-switch
class="gf-form"
label="All alerts"
label-class="width-8"
label="Send on all alerts"
label-class="width-12"
checked="ctrl.model.isDefault"
tooltip="Use this notification for all alerts">
</gf-form-switch>
......@@ -91,5 +100,5 @@
<button ng-click="ctrl.testNotification()" class="btn btn-secondary">Send</button>
</div>
</div>
</div>
</div>
</div>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment