Skip to content

N
nexpie-grafana-theme
Commit ccfd9c89 by bergquist
Options

introduces hard coded deboucing for alerting

parent 5469a1a5
Showing with 145 additions and 70 deletions
pkg/services/alerting/eval_context.go
...@@ -69,7 +69,7 @@ func (c *EvalContext) GetStateModel() *StateDescription { ...@@ -69,7 +69,7 @@ func (c *EvalContext) GetStateModel() *StateDescription {
Text: "Alerting", Text: "Alerting",
} }
default: default:
panic("Unknown rule state " + c.Rule.State) panic("Unknown rule state for alert notifications " + c.Rule.State)
} }
} }
...@@ -125,11 +125,26 @@ func (c *EvalContext) GetNewState() m.AlertStateType { ...@@ -125,11 +125,26 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
return c.PrevAlertState return c.PrevAlertState
} }
return c.Rule.ExecutionErrorState.ToAlertState() return c.Rule.ExecutionErrorState.ToAlertState()
}
if c.Firing && c.Rule.DebounceDuration != 0 {
since := time.Now().Sub(c.Rule.LastStateChange)
if since > c.Rule.DebounceDuration {
return m.AlertStateAlerting
}
if c.PrevAlertState == m.AlertStateAlerting {
return m.AlertStateAlerting
}
return m.AlertStatePending
}
} else if c.Firing { if c.Firing {
return m.AlertStateAlerting return m.AlertStateAlerting
}
} else if c.NoDataFound { if c.NoDataFound {
c.log.Info("Alert Rule returned no data", c.log.Info("Alert Rule returned no data",
"ruleId", c.Rule.Id, "ruleId", c.Rule.Id,
"name", c.Rule.Name, "name", c.Rule.Name,
......
pkg/services/alerting/eval_context_test.go
...@@ -2,11 +2,11 @@ package alerting ...@@ -2,11 +2,11 @@ package alerting
import ( import (
"context" "context"
"fmt" "errors"
"testing" "testing"
"time"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
) )
func TestStateIsUpdatedWhenNeeded(t *testing.T) { func TestStateIsUpdatedWhenNeeded(t *testing.T) {
...@@ -31,71 +31,123 @@ func TestStateIsUpdatedWhenNeeded(t *testing.T) { ...@@ -31,71 +31,123 @@ func TestStateIsUpdatedWhenNeeded(t *testing.T) {
}) })
} }
func TestAlertingEvalContext(t *testing.T) { func TestGetStateFromEvalContext(t *testing.T) {
Convey("Should compute and replace properly new rule state", t, func() { tcs := []struct {
name string
expected models.AlertStateType
applyFn func(ec *EvalContext)
focus bool
}{
{
name: "ok -> alerting",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.Firing = true
ec.PrevAlertState = models.AlertStateOK
},
},
{
name: "ok -> error(alerting)",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Error = errors.New("test error")
ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
},
},
{
name: "ok -> pending. since its been firing for less than FOR",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
ec.Rule.DebounceDuration = time.Minute * 5
},
},
{
name: "ok -> alerting. since its been firing for more than FOR",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
ec.Rule.DebounceDuration = time.Minute * 2
},
},
{
name: "alerting -> alerting. should not update regardless of FOR",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateAlerting
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
ec.Rule.DebounceDuration = time.Minute * 2
},
},
{
name: "ok -> ok. should not update regardless of FOR",
expected: models.AlertStateOK,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
ec.Rule.DebounceDuration = time.Minute * 2
},
},
{
name: "ok -> error(keep_last)",
expected: models.AlertStateOK,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Error = errors.New("test error")
ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
},
},
{
name: "pending -> error(keep_last)",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Error = errors.New("test error")
ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
},
},
{
name: "ok -> no_data(alerting)",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Rule.NoDataState = models.NoDataSetAlerting
ec.NoDataFound = true
},
},
{
name: "ok -> no_data(keep_last)",
expected: models.AlertStateOK,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Rule.NoDataState = models.NoDataKeepState
ec.NoDataFound = true
},
},
{
name: "pending -> no_data(keep_last)",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.NoDataState = models.NoDataKeepState
ec.NoDataFound = true
},
},
}
for _, tc := range tcs {
ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}}) ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
dummieError := fmt.Errorf("dummie error")
Convey("ok -> alerting", func() { tc.applyFn(ctx)
ctx.PrevAlertState = models.AlertStateOK have := ctx.GetNewState()
ctx.Firing = true if have != tc.expected {
t.Errorf("failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, string(have))
ctx.Rule.State = ctx.GetNewState() }
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting) }
})
Convey("ok -> error(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
})
Convey("ok -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
})
Convey("pending -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
})
Convey("ok -> no_data(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetAlerting
ctx.NoDataFound = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
})
Convey("ok -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
})
Convey("pending -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
})
})
} }
pkg/services/alerting/result_handler.go
...@@ -73,6 +73,9 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { ...@@ -73,6 +73,9 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
// when two servers are raising. This makes sure that the server // when two servers are raising. This makes sure that the server
// with the last state change always sends a notification. // with the last state change always sends a notification.
evalContext.Rule.StateChanges = cmd.Result.StateChanges evalContext.Rule.StateChanges = cmd.Result.StateChanges
// Update the last state change of the alert rule in memory
evalContext.Rule.LastStateChange = time.Now()
} }
// save annotation // save annotation
......
pkg/services/alerting/rule.go
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"regexp" "regexp"
"strconv" "strconv"
"time"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
...@@ -18,6 +19,8 @@ type Rule struct { ...@@ -18,6 +19,8 @@ type Rule struct {
Frequency int64 Frequency int64
Name string Name string
Message string Message string
LastStateChange time.Time
DebounceDuration time.Duration
NoDataState m.NoDataOption NoDataState m.NoDataOption
ExecutionErrorState m.ExecutionErrorOption ExecutionErrorState m.ExecutionErrorOption
State m.AlertStateType State m.AlertStateType
...@@ -100,6 +103,8 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { ...@@ -100,6 +103,8 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
model.Message = ruleDef.Message model.Message = ruleDef.Message
model.Frequency = ruleDef.Frequency model.Frequency = ruleDef.Frequency
model.State = ruleDef.State model.State = ruleDef.State
model.LastStateChange = ruleDef.NewStateDate
model.DebounceDuration = time.Minute * 2 // hard coded for now
model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data")) model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting")) model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
model.StateChanges = ruleDef.StateChanges model.StateChanges = ruleDef.StateChanges
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment