Commit 489f087f by bergquist

feat(alerting): reduce states. Make exeuction result configurable.

ref #6444
parent f0b591b8
...@@ -40,7 +40,6 @@ var ( ...@@ -40,7 +40,6 @@ var (
M_Alerting_Result_State_Ok Counter M_Alerting_Result_State_Ok Counter
M_Alerting_Result_State_Paused Counter M_Alerting_Result_State_Paused Counter
M_Alerting_Result_State_NoData Counter M_Alerting_Result_State_NoData Counter
M_Alerting_Result_State_ExecError Counter
M_Alerting_Result_State_Pending Counter M_Alerting_Result_State_Pending Counter
M_Alerting_Active_Alerts Counter M_Alerting_Active_Alerts Counter
M_Alerting_Notification_Sent_Slack Counter M_Alerting_Notification_Sent_Slack Counter
...@@ -102,7 +101,6 @@ func initMetricVars(settings *MetricSettings) { ...@@ -102,7 +101,6 @@ func initMetricVars(settings *MetricSettings) {
M_Alerting_Result_State_Ok = RegCounter("alerting.result", "state", "ok") M_Alerting_Result_State_Ok = RegCounter("alerting.result", "state", "ok")
M_Alerting_Result_State_Paused = RegCounter("alerting.result", "state", "paused") M_Alerting_Result_State_Paused = RegCounter("alerting.result", "state", "paused")
M_Alerting_Result_State_NoData = RegCounter("alerting.result", "state", "no_data") M_Alerting_Result_State_NoData = RegCounter("alerting.result", "state", "no_data")
M_Alerting_Result_State_ExecError = RegCounter("alerting.result", "state", "exec_error")
M_Alerting_Result_State_Pending = RegCounter("alerting.result", "state", "pending") M_Alerting_Result_State_Pending = RegCounter("alerting.result", "state", "pending")
M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts") M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts")
......
...@@ -9,10 +9,10 @@ import ( ...@@ -9,10 +9,10 @@ import (
type AlertStateType string type AlertStateType string
type AlertSeverityType string type AlertSeverityType string
type NoDataOption string type NoDataOption string
type ExecutionErrorOption string
const ( const (
AlertStateNoData AlertStateType = "no_data" AlertStateNoData AlertStateType = "no_data"
AlertStateExecError AlertStateType = "execution_error"
AlertStatePaused AlertStateType = "paused" AlertStatePaused AlertStateType = "paused"
AlertStateAlerting AlertStateType = "alerting" AlertStateAlerting AlertStateType = "alerting"
AlertStateOK AlertStateType = "ok" AlertStateOK AlertStateType = "ok"
...@@ -22,22 +22,34 @@ const ( ...@@ -22,22 +22,34 @@ const (
const ( const (
NoDataSetNoData NoDataOption = "no_data" NoDataSetNoData NoDataOption = "no_data"
NoDataSetAlerting NoDataOption = "alerting" NoDataSetAlerting NoDataOption = "alerting"
NoDataSetOK NoDataOption = "ok"
NoDataKeepState NoDataOption = "keep_state" NoDataKeepState NoDataOption = "keep_state"
) )
const (
ExecutionErrorSetAlerting ExecutionErrorOption = "alerting"
ExecutionErrorKeepState ExecutionErrorOption = "keep_state"
)
func (s AlertStateType) IsValid() bool { func (s AlertStateType) IsValid() bool {
return s == AlertStateOK || s == AlertStateNoData || s == AlertStateExecError || s == AlertStatePaused || s == AlertStatePending return s == AlertStateOK || s == AlertStateNoData || s == AlertStatePaused || s == AlertStatePending
} }
func (s NoDataOption) IsValid() bool { func (s NoDataOption) IsValid() bool {
return s == NoDataSetNoData || s == NoDataSetAlerting || s == NoDataSetOK || s == NoDataKeepState return s == NoDataSetNoData || s == NoDataSetAlerting || s == NoDataKeepState
} }
func (s NoDataOption) ToAlertState() AlertStateType { func (s NoDataOption) ToAlertState() AlertStateType {
return AlertStateType(s) return AlertStateType(s)
} }
func (s ExecutionErrorOption) IsValid() bool {
return s == ExecutionErrorSetAlerting || s == ExecutionErrorKeepState
}
func (s ExecutionErrorOption) ToAlertState() AlertStateType {
return AlertStateType(s)
}
type Alert struct { type Alert struct {
Id int64 Id int64
Version int64 Version int64
......
...@@ -31,6 +31,17 @@ type EvalContext struct { ...@@ -31,6 +31,17 @@ type EvalContext struct {
Ctx context.Context Ctx context.Context
} }
func NewEvalContext(alertCtx context.Context, rule *Rule) *EvalContext {
return &EvalContext{
Ctx: alertCtx,
StartTime: time.Now(),
Rule: rule,
Logs: make([]*ResultLogEntry, 0),
EvalMatches: make([]*EvalMatch, 0),
log: log.New("alerting.evalContext"),
}
}
type StateDescription struct { type StateDescription struct {
Color string Color string
Text string Text string
...@@ -49,11 +60,6 @@ func (c *EvalContext) GetStateModel() *StateDescription { ...@@ -49,11 +60,6 @@ func (c *EvalContext) GetStateModel() *StateDescription {
Color: "#888888", Color: "#888888",
Text: "No Data", Text: "No Data",
} }
case m.AlertStateExecError:
return &StateDescription{
Color: "#000",
Text: "Execution Error",
}
case m.AlertStateAlerting: case m.AlertStateAlerting:
return &StateDescription{ return &StateDescription{
Color: "#D63232", Color: "#D63232",
...@@ -73,25 +79,7 @@ func (c *EvalContext) ShouldSendNotification() bool { ...@@ -73,25 +79,7 @@ func (c *EvalContext) ShouldSendNotification() bool {
return false return false
} }
alertState := c.Rule.State return true
if c.NoDataFound {
if c.Rule.NoDataState == m.NoDataKeepState {
return false
}
alertState = c.Rule.NoDataState.ToAlertState()
}
if c.Error != nil {
if c.Rule.ExecutionErrorState == m.NoDataKeepState {
return false
}
alertState = c.Rule.ExecutionErrorState.ToAlertState()
}
return alertState != c.PrevAlertState
} }
func (a *EvalContext) GetDurationMs() float64 { func (a *EvalContext) GetDurationMs() float64 {
...@@ -128,14 +116,3 @@ func (c *EvalContext) GetRuleUrl() (string, error) { ...@@ -128,14 +116,3 @@ func (c *EvalContext) GetRuleUrl() (string, error) {
return ruleUrl, nil return ruleUrl, nil
} }
} }
func NewEvalContext(alertCtx context.Context, rule *Rule) *EvalContext {
return &EvalContext{
Ctx: alertCtx,
StartTime: time.Now(),
Rule: rule,
Logs: make([]*ResultLogEntry, 0),
EvalMatches: make([]*EvalMatch, 0),
log: log.New("alerting.evalContext"),
}
}
...@@ -2,7 +2,6 @@ package alerting ...@@ -2,7 +2,6 @@ package alerting
import ( import (
"context" "context"
"fmt"
"testing" "testing"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
...@@ -12,7 +11,6 @@ import ( ...@@ -12,7 +11,6 @@ import (
func TestAlertingEvalContext(t *testing.T) { func TestAlertingEvalContext(t *testing.T) {
Convey("Eval context", t, func() { Convey("Eval context", t, func() {
ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}}) ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
err := fmt.Errorf("Dummie error!")
Convey("Should update alert state", func() { Convey("Should update alert state", func() {
...@@ -45,66 +43,6 @@ func TestAlertingEvalContext(t *testing.T) { ...@@ -45,66 +43,6 @@ func TestAlertingEvalContext(t *testing.T) {
So(ctx.ShouldSendNotification(), ShouldBeTrue) So(ctx.ShouldSendNotification(), ShouldBeTrue)
}) })
Convey("alerting -> ok", func() {
ctx.PrevAlertState = models.AlertStateAlerting
ctx.Rule.State = models.AlertStateOK
So(ctx.ShouldSendNotification(), ShouldBeTrue)
})
Convey("ok -> no_data(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetAlerting
ctx.Rule.State = models.AlertStateAlerting
So(ctx.ShouldSendNotification(), ShouldBeTrue)
})
Convey("ok -> no_data(ok)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetOK
ctx.NoDataFound = true
ctx.Rule.State = models.AlertStateNoData
So(ctx.ShouldSendNotification(), ShouldBeFalse)
})
Convey("ok -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.Rule.State = models.AlertStateNoData
ctx.NoDataFound = true
So(ctx.ShouldSendNotification(), ShouldBeFalse)
})
Convey("ok -> execution_error(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateExecError
ctx.Rule.ExecutionErrorState = models.NoDataSetAlerting
ctx.Error = err
So(ctx.ShouldSendNotification(), ShouldBeTrue)
})
Convey("ok -> execution_error(ok)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateExecError
ctx.Rule.ExecutionErrorState = models.NoDataSetOK
ctx.Error = err
So(ctx.ShouldSendNotification(), ShouldBeFalse)
})
Convey("ok -> execution_error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateExecError
ctx.Rule.ExecutionErrorState = models.NoDataKeepState
ctx.Error = err
So(ctx.ShouldSendNotification(), ShouldBeFalse)
})
}) })
}) })
} }
...@@ -27,27 +27,52 @@ func NewResultHandler() *DefaultResultHandler { ...@@ -27,27 +27,52 @@ func NewResultHandler() *DefaultResultHandler {
} }
} }
func (handler *DefaultResultHandler) GetStateFromEvaluation(evalContext *EvalContext) m.AlertStateType {
if evalContext.Error != nil {
handler.log.Error("Alert Rule Result Error",
"ruleId", evalContext.Rule.Id,
"name", evalContext.Rule.Name,
"error", evalContext.Error,
"changing state to", evalContext.Rule.ExecutionErrorState.ToAlertState())
if evalContext.Rule.ExecutionErrorState == m.ExecutionErrorKeepState {
return evalContext.PrevAlertState
} else {
return evalContext.Rule.ExecutionErrorState.ToAlertState()
}
} else if evalContext.Firing {
return m.AlertStateAlerting
} else if evalContext.NoDataFound {
handler.log.Info("Alert Rule returned no data",
"ruleId", evalContext.Rule.Id,
"name", evalContext.Rule.Name,
"changing state to", evalContext.Rule.NoDataState.ToAlertState())
if evalContext.Rule.NoDataState == m.NoDataKeepState {
return evalContext.PrevAlertState
} else {
return evalContext.Rule.NoDataState.ToAlertState()
}
}
return m.AlertStateOK
}
func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
evalContext.PrevAlertState = evalContext.Rule.State evalContext.PrevAlertState = evalContext.Rule.State
executionError := "" executionError := ""
annotationData := simplejson.New() annotationData := simplejson.New()
evalContext.Rule.State = handler.GetStateFromEvaluation(evalContext)
if evalContext.Error != nil { if evalContext.Error != nil {
handler.log.Error("Alert Rule Result Error", "ruleId", evalContext.Rule.Id, "error", evalContext.Error)
evalContext.Rule.State = m.AlertStateExecError
executionError = evalContext.Error.Error() executionError = evalContext.Error.Error()
annotationData.Set("errorMessage", executionError) annotationData.Set("errorMessage", executionError)
} else if evalContext.Firing {
evalContext.Rule.State = m.AlertStateAlerting
annotationData = simplejson.NewFromAny(evalContext.EvalMatches)
} else {
if evalContext.NoDataFound {
if evalContext.Rule.NoDataState != m.NoDataKeepState {
evalContext.Rule.State = evalContext.Rule.NoDataState.ToAlertState()
}
} else {
evalContext.Rule.State = m.AlertStateOK
} }
if evalContext.Firing {
annotationData = simplejson.NewFromAny(evalContext.EvalMatches)
} }
countStateResult(evalContext.Rule.State) countStateResult(evalContext.Rule.State)
...@@ -88,8 +113,6 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { ...@@ -88,8 +113,6 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
if evalContext.ShouldSendNotification() { if evalContext.ShouldSendNotification() {
handler.notifier.Notify(evalContext) handler.notifier.Notify(evalContext)
} else {
handler.log.Info("Notfication not sent", "prev state", evalContext.PrevAlertState, "new state", evalContext.Rule.State)
} }
} }
...@@ -108,7 +131,5 @@ func countStateResult(state m.AlertStateType) { ...@@ -108,7 +131,5 @@ func countStateResult(state m.AlertStateType) {
metrics.M_Alerting_Result_State_Paused.Inc(1) metrics.M_Alerting_Result_State_Paused.Inc(1)
case m.AlertStateNoData: case m.AlertStateNoData:
metrics.M_Alerting_Result_State_NoData.Inc(1) metrics.M_Alerting_Result_State_NoData.Inc(1)
case m.AlertStateExecError:
metrics.M_Alerting_Result_State_ExecError.Inc(1)
} }
} }
package alerting package alerting
// import ( import (
// "context" "context"
// "testing" "testing"
//
// "github.com/grafana/grafana/pkg/models" "fmt"
// . "github.com/smartystreets/goconvey/convey"
// ) "github.com/grafana/grafana/pkg/models"
// . "github.com/smartystreets/goconvey/convey"
// func TestAlertResultHandler(t *testing.T) { )
// Convey("Test result Handler", t, func() {
// func TestAlertingResultHandler(t *testing.T) {
// handler := NewResultHandler() Convey("Result handler", t, func() {
// evalContext := NewEvalContext(context.TODO(), &Rule{}) ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
// dummieError := fmt.Errorf("dummie")
// Convey("Should update", func() { handler := NewResultHandler()
//
// Convey("when no earlier alert state", func() { Convey("Should update alert state", func() {
// oldState := models.AlertStateOK
// Convey("ok -> alerting", func() {
// evalContext.Rule.State = models.AlertStateAlerting ctx.PrevAlertState = models.AlertStateOK
// evalContext.Rule.NoDataState = models.NoDataKeepState ctx.Firing = true
// evalContext.NoDataFound = true
// So(handler.GetStateFromEvaluation(ctx), ShouldEqual, models.AlertStateAlerting)
// So(handler.shouldUpdateAlertState(evalContext, oldState), ShouldBeFalse) So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
// }) })
// })
// }) Convey("ok -> error(alerting)", func() {
// } ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
})
Convey("ok -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
Convey("pending -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
Convey("ok -> no_data(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetAlerting
ctx.NoDataFound = true
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
})
Convey("ok -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
Convey("pending -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
})
})
}
...@@ -19,7 +19,7 @@ type Rule struct { ...@@ -19,7 +19,7 @@ type Rule struct {
Name string Name string
Message string Message string
NoDataState m.NoDataOption NoDataState m.NoDataOption
ExecutionErrorState m.NoDataOption ExecutionErrorState m.ExecutionErrorOption
State m.AlertStateType State m.AlertStateType
Conditions []Condition Conditions []Condition
Notifications []int64 Notifications []int64
...@@ -78,6 +78,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { ...@@ -78,6 +78,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
model.Frequency = ruleDef.Frequency model.Frequency = ruleDef.Frequency
model.State = ruleDef.State model.State = ruleDef.State
model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data")) model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
for _, v := range ruleDef.Settings.Get("notifications").MustArray() { for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
jsonModel := simplejson.NewFromAny(v) jsonModel := simplejson.NewFromAny(v)
......
...@@ -37,10 +37,14 @@ var reducerTypes = [ ...@@ -37,10 +37,14 @@ var reducerTypes = [
]; ];
var noDataModes = [ var noDataModes = [
{text: 'OK', value: 'ok'},
{text: 'Alerting', value: 'alerting'}, {text: 'Alerting', value: 'alerting'},
{text: 'No Data', value: 'no_data'}, {text: 'No Data', value: 'no_data'},
{text: 'Keep Last', value: 'keep_last'}, {text: 'Keep Last State', value: 'keep_state'},
];
var executionErrorModes = [
{text: 'Alerting', value: 'alerting'},
{text: 'Keep Last State', value: 'keep_state'},
]; ];
function createReducerPart(model) { function createReducerPart(model) {
...@@ -48,7 +52,6 @@ function createReducerPart(model) { ...@@ -48,7 +52,6 @@ function createReducerPart(model) {
return new QueryPart(model, def); return new QueryPart(model, def);
} }
function getStateDisplayModel(state) { function getStateDisplayModel(state) {
switch (state) { switch (state) {
case 'ok': { case 'ok': {
...@@ -113,6 +116,7 @@ export default { ...@@ -113,6 +116,7 @@ export default {
conditionTypes: conditionTypes, conditionTypes: conditionTypes,
evalFunctions: evalFunctions, evalFunctions: evalFunctions,
noDataModes: noDataModes, noDataModes: noDataModes,
executionErrorModes: executionErrorModes,
reducerTypes: reducerTypes, reducerTypes: reducerTypes,
createReducerPart: createReducerPart, createReducerPart: createReducerPart,
joinEvalMatches: joinEvalMatches, joinEvalMatches: joinEvalMatches,
......
...@@ -19,6 +19,7 @@ export class AlertTabCtrl { ...@@ -19,6 +19,7 @@ export class AlertTabCtrl {
conditionModels: any; conditionModels: any;
evalFunctions: any; evalFunctions: any;
noDataModes: any; noDataModes: any;
executionErrorModes: any;
addNotificationSegment; addNotificationSegment;
notifications; notifications;
alertNotifications; alertNotifications;
...@@ -42,6 +43,7 @@ export class AlertTabCtrl { ...@@ -42,6 +43,7 @@ export class AlertTabCtrl {
this.evalFunctions = alertDef.evalFunctions; this.evalFunctions = alertDef.evalFunctions;
this.conditionTypes = alertDef.conditionTypes; this.conditionTypes = alertDef.conditionTypes;
this.noDataModes = alertDef.noDataModes; this.noDataModes = alertDef.noDataModes;
this.executionErrorModes = alertDef.executionErrorModes;
this.appSubUrl = config.appSubUrl; this.appSubUrl = config.appSubUrl;
} }
...@@ -140,6 +142,7 @@ export class AlertTabCtrl { ...@@ -140,6 +142,7 @@ export class AlertTabCtrl {
} }
alert.noDataState = alert.noDataState || 'no_data'; alert.noDataState = alert.noDataState || 'no_data';
alert.executionErrorState = alert.executionErrorState || 'alerting';
alert.frequency = alert.frequency || '60s'; alert.frequency = alert.frequency || '60s';
alert.handler = alert.handler || 1; alert.handler = alert.handler || 1;
alert.notifications = alert.notifications || []; alert.notifications = alert.notifications || [];
......
...@@ -82,7 +82,7 @@ ...@@ -82,7 +82,7 @@
<div class="gf-form-group"> <div class="gf-form-group">
<div class="gf-form"> <div class="gf-form">
<span class="gf-form-label">If no data points or all values are null</span> <span class="gf-form-label width-18">If no data points or all values are null</span>
<span class="gf-form-label query-keyword">SET STATE TO</span> <span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper"> <div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes"> <select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
...@@ -90,6 +90,15 @@ ...@@ -90,6 +90,15 @@
</div> </div>
</div> </div>
<div class="gf-form">
<span class="gf-form-label width-18">If query exeuction throws an error</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
</select>
</div>
</div>
<div class="gf-form-button-row"> <div class="gf-form-button-row">
<button class="btn btn-inverse" ng-click="ctrl.test()"> <button class="btn btn-inverse" ng-click="ctrl.test()">
Test Rule Test Rule
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment