Commit 489f087f by bergquist

feat(alerting): reduce states. Make exeuction result configurable.

ref #6444
parent f0b591b8
......@@ -40,7 +40,6 @@ var (
M_Alerting_Result_State_Ok Counter
M_Alerting_Result_State_Paused Counter
M_Alerting_Result_State_NoData Counter
M_Alerting_Result_State_ExecError Counter
M_Alerting_Result_State_Pending Counter
M_Alerting_Active_Alerts Counter
M_Alerting_Notification_Sent_Slack Counter
......@@ -102,7 +101,6 @@ func initMetricVars(settings *MetricSettings) {
M_Alerting_Result_State_Ok = RegCounter("alerting.result", "state", "ok")
M_Alerting_Result_State_Paused = RegCounter("alerting.result", "state", "paused")
M_Alerting_Result_State_NoData = RegCounter("alerting.result", "state", "no_data")
M_Alerting_Result_State_ExecError = RegCounter("alerting.result", "state", "exec_error")
M_Alerting_Result_State_Pending = RegCounter("alerting.result", "state", "pending")
M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts")
......
......@@ -9,10 +9,10 @@ import (
type AlertStateType string
type AlertSeverityType string
type NoDataOption string
type ExecutionErrorOption string
const (
AlertStateNoData AlertStateType = "no_data"
AlertStateExecError AlertStateType = "execution_error"
AlertStatePaused AlertStateType = "paused"
AlertStateAlerting AlertStateType = "alerting"
AlertStateOK AlertStateType = "ok"
......@@ -22,22 +22,34 @@ const (
const (
NoDataSetNoData NoDataOption = "no_data"
NoDataSetAlerting NoDataOption = "alerting"
NoDataSetOK NoDataOption = "ok"
NoDataKeepState NoDataOption = "keep_state"
)
const (
ExecutionErrorSetAlerting ExecutionErrorOption = "alerting"
ExecutionErrorKeepState ExecutionErrorOption = "keep_state"
)
func (s AlertStateType) IsValid() bool {
return s == AlertStateOK || s == AlertStateNoData || s == AlertStateExecError || s == AlertStatePaused || s == AlertStatePending
return s == AlertStateOK || s == AlertStateNoData || s == AlertStatePaused || s == AlertStatePending
}
func (s NoDataOption) IsValid() bool {
return s == NoDataSetNoData || s == NoDataSetAlerting || s == NoDataSetOK || s == NoDataKeepState
return s == NoDataSetNoData || s == NoDataSetAlerting || s == NoDataKeepState
}
func (s NoDataOption) ToAlertState() AlertStateType {
return AlertStateType(s)
}
func (s ExecutionErrorOption) IsValid() bool {
return s == ExecutionErrorSetAlerting || s == ExecutionErrorKeepState
}
func (s ExecutionErrorOption) ToAlertState() AlertStateType {
return AlertStateType(s)
}
type Alert struct {
Id int64
Version int64
......
......@@ -31,6 +31,17 @@ type EvalContext struct {
Ctx context.Context
}
func NewEvalContext(alertCtx context.Context, rule *Rule) *EvalContext {
return &EvalContext{
Ctx: alertCtx,
StartTime: time.Now(),
Rule: rule,
Logs: make([]*ResultLogEntry, 0),
EvalMatches: make([]*EvalMatch, 0),
log: log.New("alerting.evalContext"),
}
}
type StateDescription struct {
Color string
Text string
......@@ -49,11 +60,6 @@ func (c *EvalContext) GetStateModel() *StateDescription {
Color: "#888888",
Text: "No Data",
}
case m.AlertStateExecError:
return &StateDescription{
Color: "#000",
Text: "Execution Error",
}
case m.AlertStateAlerting:
return &StateDescription{
Color: "#D63232",
......@@ -73,25 +79,7 @@ func (c *EvalContext) ShouldSendNotification() bool {
return false
}
alertState := c.Rule.State
if c.NoDataFound {
if c.Rule.NoDataState == m.NoDataKeepState {
return false
}
alertState = c.Rule.NoDataState.ToAlertState()
}
if c.Error != nil {
if c.Rule.ExecutionErrorState == m.NoDataKeepState {
return false
}
alertState = c.Rule.ExecutionErrorState.ToAlertState()
}
return alertState != c.PrevAlertState
return true
}
func (a *EvalContext) GetDurationMs() float64 {
......@@ -128,14 +116,3 @@ func (c *EvalContext) GetRuleUrl() (string, error) {
return ruleUrl, nil
}
}
func NewEvalContext(alertCtx context.Context, rule *Rule) *EvalContext {
return &EvalContext{
Ctx: alertCtx,
StartTime: time.Now(),
Rule: rule,
Logs: make([]*ResultLogEntry, 0),
EvalMatches: make([]*EvalMatch, 0),
log: log.New("alerting.evalContext"),
}
}
......@@ -2,7 +2,6 @@ package alerting
import (
"context"
"fmt"
"testing"
"github.com/grafana/grafana/pkg/models"
......@@ -12,7 +11,6 @@ import (
func TestAlertingEvalContext(t *testing.T) {
Convey("Eval context", t, func() {
ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
err := fmt.Errorf("Dummie error!")
Convey("Should update alert state", func() {
......@@ -45,66 +43,6 @@ func TestAlertingEvalContext(t *testing.T) {
So(ctx.ShouldSendNotification(), ShouldBeTrue)
})
Convey("alerting -> ok", func() {
ctx.PrevAlertState = models.AlertStateAlerting
ctx.Rule.State = models.AlertStateOK
So(ctx.ShouldSendNotification(), ShouldBeTrue)
})
Convey("ok -> no_data(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetAlerting
ctx.Rule.State = models.AlertStateAlerting
So(ctx.ShouldSendNotification(), ShouldBeTrue)
})
Convey("ok -> no_data(ok)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetOK
ctx.NoDataFound = true
ctx.Rule.State = models.AlertStateNoData
So(ctx.ShouldSendNotification(), ShouldBeFalse)
})
Convey("ok -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.Rule.State = models.AlertStateNoData
ctx.NoDataFound = true
So(ctx.ShouldSendNotification(), ShouldBeFalse)
})
Convey("ok -> execution_error(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateExecError
ctx.Rule.ExecutionErrorState = models.NoDataSetAlerting
ctx.Error = err
So(ctx.ShouldSendNotification(), ShouldBeTrue)
})
Convey("ok -> execution_error(ok)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateExecError
ctx.Rule.ExecutionErrorState = models.NoDataSetOK
ctx.Error = err
So(ctx.ShouldSendNotification(), ShouldBeFalse)
})
Convey("ok -> execution_error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateExecError
ctx.Rule.ExecutionErrorState = models.NoDataKeepState
ctx.Error = err
So(ctx.ShouldSendNotification(), ShouldBeFalse)
})
})
})
}
......@@ -27,27 +27,52 @@ func NewResultHandler() *DefaultResultHandler {
}
}
func (handler *DefaultResultHandler) GetStateFromEvaluation(evalContext *EvalContext) m.AlertStateType {
if evalContext.Error != nil {
handler.log.Error("Alert Rule Result Error",
"ruleId", evalContext.Rule.Id,
"name", evalContext.Rule.Name,
"error", evalContext.Error,
"changing state to", evalContext.Rule.ExecutionErrorState.ToAlertState())
if evalContext.Rule.ExecutionErrorState == m.ExecutionErrorKeepState {
return evalContext.PrevAlertState
} else {
return evalContext.Rule.ExecutionErrorState.ToAlertState()
}
} else if evalContext.Firing {
return m.AlertStateAlerting
} else if evalContext.NoDataFound {
handler.log.Info("Alert Rule returned no data",
"ruleId", evalContext.Rule.Id,
"name", evalContext.Rule.Name,
"changing state to", evalContext.Rule.NoDataState.ToAlertState())
if evalContext.Rule.NoDataState == m.NoDataKeepState {
return evalContext.PrevAlertState
} else {
return evalContext.Rule.NoDataState.ToAlertState()
}
}
return m.AlertStateOK
}
func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
evalContext.PrevAlertState = evalContext.Rule.State
executionError := ""
annotationData := simplejson.New()
evalContext.Rule.State = handler.GetStateFromEvaluation(evalContext)
if evalContext.Error != nil {
handler.log.Error("Alert Rule Result Error", "ruleId", evalContext.Rule.Id, "error", evalContext.Error)
evalContext.Rule.State = m.AlertStateExecError
executionError = evalContext.Error.Error()
annotationData.Set("errorMessage", executionError)
} else if evalContext.Firing {
evalContext.Rule.State = m.AlertStateAlerting
annotationData = simplejson.NewFromAny(evalContext.EvalMatches)
} else {
if evalContext.NoDataFound {
if evalContext.Rule.NoDataState != m.NoDataKeepState {
evalContext.Rule.State = evalContext.Rule.NoDataState.ToAlertState()
}
} else {
evalContext.Rule.State = m.AlertStateOK
}
if evalContext.Firing {
annotationData = simplejson.NewFromAny(evalContext.EvalMatches)
}
countStateResult(evalContext.Rule.State)
......@@ -88,8 +113,6 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
if evalContext.ShouldSendNotification() {
handler.notifier.Notify(evalContext)
} else {
handler.log.Info("Notfication not sent", "prev state", evalContext.PrevAlertState, "new state", evalContext.Rule.State)
}
}
......@@ -108,7 +131,5 @@ func countStateResult(state m.AlertStateType) {
metrics.M_Alerting_Result_State_Paused.Inc(1)
case m.AlertStateNoData:
metrics.M_Alerting_Result_State_NoData.Inc(1)
case m.AlertStateExecError:
metrics.M_Alerting_Result_State_ExecError.Inc(1)
}
}
package alerting
// import (
// "context"
// "testing"
//
// "github.com/grafana/grafana/pkg/models"
// . "github.com/smartystreets/goconvey/convey"
// )
//
// func TestAlertResultHandler(t *testing.T) {
// Convey("Test result Handler", t, func() {
//
// handler := NewResultHandler()
// evalContext := NewEvalContext(context.TODO(), &Rule{})
//
// Convey("Should update", func() {
//
// Convey("when no earlier alert state", func() {
// oldState := models.AlertStateOK
//
// evalContext.Rule.State = models.AlertStateAlerting
// evalContext.Rule.NoDataState = models.NoDataKeepState
// evalContext.NoDataFound = true
//
// So(handler.shouldUpdateAlertState(evalContext, oldState), ShouldBeFalse)
// })
// })
// })
// }
import (
"context"
"testing"
"fmt"
"github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestAlertingResultHandler(t *testing.T) {
Convey("Result handler", t, func() {
ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
dummieError := fmt.Errorf("dummie")
handler := NewResultHandler()
Convey("Should update alert state", func() {
Convey("ok -> alerting", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Firing = true
So(handler.GetStateFromEvaluation(ctx), ShouldEqual, models.AlertStateAlerting)
So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
})
Convey("ok -> error(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
})
Convey("ok -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
Convey("pending -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
Convey("ok -> no_data(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetAlerting
ctx.NoDataFound = true
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
})
Convey("ok -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
Convey("pending -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = handler.GetStateFromEvaluation(ctx)
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
})
})
})
}
......@@ -19,7 +19,7 @@ type Rule struct {
Name string
Message string
NoDataState m.NoDataOption
ExecutionErrorState m.NoDataOption
ExecutionErrorState m.ExecutionErrorOption
State m.AlertStateType
Conditions []Condition
Notifications []int64
......@@ -78,6 +78,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
model.Frequency = ruleDef.Frequency
model.State = ruleDef.State
model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
jsonModel := simplejson.NewFromAny(v)
......
......@@ -37,10 +37,14 @@ var reducerTypes = [
];
var noDataModes = [
{text: 'OK', value: 'ok'},
{text: 'Alerting', value: 'alerting'},
{text: 'No Data', value: 'no_data'},
{text: 'Keep Last', value: 'keep_last'},
{text: 'Keep Last State', value: 'keep_state'},
];
var executionErrorModes = [
{text: 'Alerting', value: 'alerting'},
{text: 'Keep Last State', value: 'keep_state'},
];
function createReducerPart(model) {
......@@ -48,7 +52,6 @@ function createReducerPart(model) {
return new QueryPart(model, def);
}
function getStateDisplayModel(state) {
switch (state) {
case 'ok': {
......@@ -113,6 +116,7 @@ export default {
conditionTypes: conditionTypes,
evalFunctions: evalFunctions,
noDataModes: noDataModes,
executionErrorModes: executionErrorModes,
reducerTypes: reducerTypes,
createReducerPart: createReducerPart,
joinEvalMatches: joinEvalMatches,
......
......@@ -19,6 +19,7 @@ export class AlertTabCtrl {
conditionModels: any;
evalFunctions: any;
noDataModes: any;
executionErrorModes: any;
addNotificationSegment;
notifications;
alertNotifications;
......@@ -42,6 +43,7 @@ export class AlertTabCtrl {
this.evalFunctions = alertDef.evalFunctions;
this.conditionTypes = alertDef.conditionTypes;
this.noDataModes = alertDef.noDataModes;
this.executionErrorModes = alertDef.executionErrorModes;
this.appSubUrl = config.appSubUrl;
}
......@@ -140,6 +142,7 @@ export class AlertTabCtrl {
}
alert.noDataState = alert.noDataState || 'no_data';
alert.executionErrorState = alert.executionErrorState || 'alerting';
alert.frequency = alert.frequency || '60s';
alert.handler = alert.handler || 1;
alert.notifications = alert.notifications || [];
......
......@@ -82,7 +82,7 @@
<div class="gf-form-group">
<div class="gf-form">
<span class="gf-form-label">If no data points or all values are null</span>
<span class="gf-form-label width-18">If no data points or all values are null</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
......@@ -90,6 +90,15 @@
</div>
</div>
<div class="gf-form">
<span class="gf-form-label width-18">If query exeuction throws an error</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
</select>
</div>
</div>
<div class="gf-form-button-row">
<button class="btn btn-inverse" ng-click="ctrl.test()">
Test Rule
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment