Skip to content

N
nexpie-grafana-theme
Commit f001815d by Carl Bergquist Committed by GitHub
Options

alerting: no notification when going from nodata -> pending (#16905) 

ref #16496
parent b8674f4e
Showing with 76 additions and 62 deletions
pkg/services/alerting/eval_context_test.go
...@@ -6,6 +6,8 @@ import ( ...@@ -6,6 +6,8 @@ import (
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
) )
...@@ -195,12 +197,10 @@ func TestGetStateFromEvalContext(t *testing.T) { ...@@ -195,12 +197,10 @@ func TestGetStateFromEvalContext(t *testing.T) {
} }
for _, tc := range tcs { for _, tc := range tcs {
ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}}) evalContext := NewEvalContext(context.Background(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
tc.applyFn(ctx) tc.applyFn(evalContext)
have := ctx.GetNewState() newState := evalContext.GetNewState()
if have != tc.expected { assert.Equal(t, tc.expected, newState, "failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, string(newState))
t.Errorf("failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, string(have))
}
} }
} }
pkg/services/alerting/notifiers/base.go
...@@ -48,12 +48,15 @@ func NewNotifierBase(model *models.AlertNotification) NotifierBase { ...@@ -48,12 +48,15 @@ func NewNotifierBase(model *models.AlertNotification) NotifierBase {
// ShouldNotify checks this evaluation should send an alert notification // ShouldNotify checks this evaluation should send an alert notification
func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalContext, notiferState *models.AlertNotificationState) bool { func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalContext, notiferState *models.AlertNotificationState) bool {
prevState := context.PrevAlertState
newState := context.Rule.State
// Only notify on state change. // Only notify on state change.
if context.PrevAlertState == context.Rule.State && !n.SendReminder { if prevState == newState && !n.SendReminder {
return false return false
} }
if context.PrevAlertState == context.Rule.State && n.SendReminder { if prevState == newState && n.SendReminder {
// Do not notify if interval has not elapsed // Do not notify if interval has not elapsed
lastNotify := time.Unix(notiferState.UpdatedAt, 0) lastNotify := time.Unix(notiferState.UpdatedAt, 0)
if notiferState.UpdatedAt != 0 && lastNotify.Add(n.Frequency).After(time.Now()) { if notiferState.UpdatedAt != 0 && lastNotify.Add(n.Frequency).After(time.Now()) {
...@@ -61,32 +64,35 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC ...@@ -61,32 +64,35 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC
} }
// Do not notify if alert state is OK or pending even on repeated notify // Do not notify if alert state is OK or pending even on repeated notify
if context.Rule.State == models.AlertStateOK || context.Rule.State == models.AlertStatePending { if newState == models.AlertStateOK || newState == models.AlertStatePending {
return false return false
} }
} }
// Do not notify when we become OK for the first time. unknownOrNoData := prevState == models.AlertStateUnknown || prevState == models.AlertStateNoData
if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStateOK { okOrPending := newState == models.AlertStatePending || newState == models.AlertStateOK
// Do not notify when new state is ok/pending when previous is unknown or no_data
if unknownOrNoData && okOrPending {
return false return false
} }
// Do not notify when we become OK for the first time. // Do not notify when we become Pending for the first
if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStatePending { if prevState == models.AlertStateNoData && newState == models.AlertStatePending {
return false return false
} }
// Do not notify when we become OK from pending // Do not notify when we become OK from pending
if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK { if prevState == models.AlertStatePending && newState == models.AlertStateOK {
return false return false
} }
// Do not notify when we OK -> Pending // Do not notify when we OK -> Pending
if context.PrevAlertState == models.AlertStateOK && context.Rule.State == models.AlertStatePending { if prevState == models.AlertStateOK && newState == models.AlertStatePending {
return false return false
} }
// Do not notifu if state pending and it have been updated last minute // Do not notify if state pending and it have been updated last minute
if notiferState.State == models.AlertNotificationStatePending { if notiferState.State == models.AlertNotificationStatePending {
lastUpdated := time.Unix(notiferState.UpdatedAt, 0) lastUpdated := time.Unix(notiferState.UpdatedAt, 0)
if lastUpdated.Add(1 * time.Minute).After(time.Now()) { if lastUpdated.Add(1 * time.Minute).After(time.Now()) {
...@@ -95,7 +101,7 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC ...@@ -95,7 +101,7 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC
} }
// Do not notify when state is OK if DisableResolveMessage is set to true // Do not notify when state is OK if DisableResolveMessage is set to true
if context.Rule.State == models.AlertStateOK && n.DisableResolveMessage { if newState == models.AlertStateOK && n.DisableResolveMessage {
return false return false
} }
......
pkg/services/alerting/notifiers/base_test.go
...@@ -5,8 +5,10 @@ import ( ...@@ -5,8 +5,10 @@ import (
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/alerting"
. "github.com/smartystreets/goconvey/convey" . "github.com/smartystreets/goconvey/convey"
) )
...@@ -16,76 +18,76 @@ func TestShouldSendAlertNotification(t *testing.T) { ...@@ -16,76 +18,76 @@ func TestShouldSendAlertNotification(t *testing.T) {
tcs := []struct { tcs := []struct {
name string name string
prevState m.AlertStateType prevState models.AlertStateType
newState m.AlertStateType newState models.AlertStateType
sendReminder bool sendReminder bool
frequency time.Duration frequency time.Duration
state *m.AlertNotificationState state *models.AlertNotificationState
expect bool expect bool
}{ }{
{ {
name: "pending -> ok should not trigger an notification", name: "pending -> ok should not trigger an notification",
newState: m.AlertStateOK, newState: models.AlertStateOK,
prevState: m.AlertStatePending, prevState: models.AlertStatePending,
sendReminder: false, sendReminder: false,
expect: false, expect: false,
}, },
{ {
name: "ok -> alerting should trigger an notification", name: "ok -> alerting should trigger an notification",
newState: m.AlertStateAlerting, newState: models.AlertStateAlerting,
prevState: m.AlertStateOK, prevState: models.AlertStateOK,
sendReminder: false, sendReminder: false,
expect: true, expect: true,
}, },
{ {
name: "ok -> pending should not trigger an notification", name: "ok -> pending should not trigger an notification",
newState: m.AlertStatePending, newState: models.AlertStatePending,
prevState: m.AlertStateOK, prevState: models.AlertStateOK,
sendReminder: false, sendReminder: false,
expect: false, expect: false,
}, },
{ {
name: "ok -> ok should not trigger an notification", name: "ok -> ok should not trigger an notification",
newState: m.AlertStateOK, newState: models.AlertStateOK,
prevState: m.AlertStateOK, prevState: models.AlertStateOK,
sendReminder: false, sendReminder: false,
expect: false, expect: false,
}, },
{ {
name: "ok -> ok with reminder should not trigger an notification", name: "ok -> ok with reminder should not trigger an notification",
newState: m.AlertStateOK, newState: models.AlertStateOK,
prevState: m.AlertStateOK, prevState: models.AlertStateOK,
sendReminder: true, sendReminder: true,
expect: false, expect: false,
}, },
{ {
name: "alerting -> ok should trigger an notification", name: "alerting -> ok should trigger an notification",
newState: m.AlertStateOK, newState: models.AlertStateOK,
prevState: m.AlertStateAlerting, prevState: models.AlertStateAlerting,
sendReminder: false, sendReminder: false,
expect: true, expect: true,
}, },
{ {
name: "alerting -> ok should trigger an notification when reminders enabled", name: "alerting -> ok should trigger an notification when reminders enabled",
newState: m.AlertStateOK, newState: models.AlertStateOK,
prevState: m.AlertStateAlerting, prevState: models.AlertStateAlerting,
frequency: time.Minute * 10, frequency: time.Minute * 10,
sendReminder: true, sendReminder: true,
state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-time.Minute).Unix()}, state: &models.AlertNotificationState{UpdatedAt: tnow.Add(-time.Minute).Unix()},
expect: true, expect: true,
}, },
{ {
name: "alerting -> alerting with reminder and no state should trigger", name: "alerting -> alerting with reminder and no state should trigger",
newState: m.AlertStateAlerting, newState: models.AlertStateAlerting,
prevState: m.AlertStateAlerting, prevState: models.AlertStateAlerting,
frequency: time.Minute * 10, frequency: time.Minute * 10,
sendReminder: true, sendReminder: true,
...@@ -93,78 +95,84 @@ func TestShouldSendAlertNotification(t *testing.T) { ...@@ -93,78 +95,84 @@ func TestShouldSendAlertNotification(t *testing.T) {
}, },
{ {
name: "alerting -> alerting with reminder and last notification sent 1 minute ago should not trigger", name: "alerting -> alerting with reminder and last notification sent 1 minute ago should not trigger",
newState: m.AlertStateAlerting, newState: models.AlertStateAlerting,
prevState: m.AlertStateAlerting, prevState: models.AlertStateAlerting,
frequency: time.Minute * 10, frequency: time.Minute * 10,
sendReminder: true, sendReminder: true,
state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-time.Minute).Unix()}, state: &models.AlertNotificationState{UpdatedAt: tnow.Add(-time.Minute).Unix()},
expect: false, expect: false,
}, },
{ {
name: "alerting -> alerting with reminder and last notifciation sent 11 minutes ago should trigger", name: "alerting -> alerting with reminder and last notifciation sent 11 minutes ago should trigger",
newState: m.AlertStateAlerting, newState: models.AlertStateAlerting,
prevState: m.AlertStateAlerting, prevState: models.AlertStateAlerting,
frequency: time.Minute * 10, frequency: time.Minute * 10,
sendReminder: true, sendReminder: true,
state: &m.AlertNotificationState{UpdatedAt: tnow.Add(-11 * time.Minute).Unix()}, state: &models.AlertNotificationState{UpdatedAt: tnow.Add(-11 * time.Minute).Unix()},
expect: true, expect: true,
}, },
{ {
name: "OK -> alerting with notifciation state pending and updated 30 seconds ago should not trigger", name: "OK -> alerting with notifciation state pending and updated 30 seconds ago should not trigger",
newState: m.AlertStateAlerting, newState: models.AlertStateAlerting,
prevState: m.AlertStateOK, prevState: models.AlertStateOK,
state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-30 * time.Second).Unix()}, state: &models.AlertNotificationState{State: models.AlertNotificationStatePending, UpdatedAt: tnow.Add(-30 * time.Second).Unix()},
expect: false, expect: false,
}, },
{ {
name: "OK -> alerting with notifciation state pending and updated 2 minutes ago should trigger", name: "OK -> alerting with notifciation state pending and updated 2 minutes ago should trigger",
newState: m.AlertStateAlerting, newState: models.AlertStateAlerting,
prevState: m.AlertStateOK, prevState: models.AlertStateOK,
state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()}, state: &models.AlertNotificationState{State: models.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()},
expect: true, expect: true,
}, },
{ {
name: "unknown -> ok", name: "unknown -> ok",
prevState: m.AlertStateUnknown, prevState: models.AlertStateUnknown,
newState: m.AlertStateOK, newState: models.AlertStateOK,
expect: false, expect: false,
}, },
{ {
name: "unknown -> pending", name: "unknown -> pending",
prevState: m.AlertStateUnknown, prevState: models.AlertStateUnknown,
newState: m.AlertStatePending, newState: models.AlertStatePending,
expect: false, expect: false,
}, },
{ {
name: "unknown -> alerting", name: "unknown -> alerting",
prevState: m.AlertStateUnknown, prevState: models.AlertStateUnknown,
newState: m.AlertStateAlerting, newState: models.AlertStateAlerting,
expect: true, expect: true,
}, },
{
name: "no_data -> pending",
prevState: models.AlertStateNoData,
newState: models.AlertStatePending,
expect: false,
},
} }
for _, tc := range tcs { for _, tc := range tcs {
evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
State: tc.prevState, State: tc.prevState,
}) })
if tc.state == nil { if tc.state == nil {
tc.state = &m.AlertNotificationState{} tc.state = &models.AlertNotificationState{}
} }
evalContext.Rule.State = tc.newState evalContext.Rule.State = tc.newState
nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency} nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency}
if nb.ShouldNotify(evalContext.Ctx, evalContext, tc.state) != tc.expect { r := nb.ShouldNotify(evalContext.Ctx, evalContext, tc.state)
t.Errorf("failed test %s.\n expected \n%+v \nto return: %v", tc.name, tc, tc.expect) assert.Equal(t, r, tc.expect, "failed test %s. expected %+v to return: %v", tc.name, tc, tc.expect)
}
} }
} }
...@@ -172,7 +180,7 @@ func TestBaseNotifier(t *testing.T) { ...@@ -172,7 +180,7 @@ func TestBaseNotifier(t *testing.T) {
Convey("default constructor for notifiers", t, func() { Convey("default constructor for notifiers", t, func() {
bJson := simplejson.New() bJson := simplejson.New()
model := &m.AlertNotification{ model := &models.AlertNotification{
Uid: "1", Uid: "1",
Name: "name", Name: "name",
Type: "email", Type: "email",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment