feat(alerting): progress on handling no data in alert query, #5860

pkg/metrics/metrics.go

@@ -33,7 +33,7 @@ var (
 	M_Alerting_Result_State_Warning        Counter
 	M_Alerting_Result_State_Ok             Counter
 	M_Alerting_Result_State_Paused         Counter
-	M_Alerting_Result_State_Pending        Counter
+	M_Alerting_Result_State_Unknown        Counter
 	M_Alerting_Result_State_ExecutionError Counter
 	M_Alerting_Active_Alerts               Counter
 	M_Alerting_Notification_Sent_Slack     Counter
@@ -81,7 +81,7 @@ func initMetricVars(settings *MetricSettings) {
 	M_Alerting_Result_State_Warning = RegCounter("alerting.result", "state", "warning")
 	M_Alerting_Result_State_Ok = RegCounter("alerting.result", "state", "ok")
 	M_Alerting_Result_State_Paused = RegCounter("alerting.result", "state", "paused")
-	M_Alerting_Result_State_Pending = RegCounter("alerting.result", "state", "pending")
+	M_Alerting_Result_State_Unknown = RegCounter("alerting.result", "state", "unknown")
 	M_Alerting_Result_State_ExecutionError = RegCounter("alerting.result", "state", "execution_error")
 
 	M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts")

pkg/models/alert.go

@@ -10,7 +10,7 @@ type AlertStateType string
 type AlertSeverityType string
 
 const (
-	AlertStatePending        AlertStateType = "pending"
+	AlertStateUnknown        AlertStateType = "unknown"
 	AlertStateExeuctionError AlertStateType = "execution_error"
 	AlertStatePaused         AlertStateType = "paused"
 	AlertStateCritical       AlertStateType = "critical"
@@ -19,7 +19,7 @@ const (
 )
 
 func (s AlertStateType) IsValid() bool {
-	return s == AlertStateOK || s == AlertStatePending || s == AlertStateExeuctionError || s == AlertStatePaused || s == AlertStateCritical || s == AlertStateWarning
+	return s == AlertStateOK || s == AlertStateUnknown || s == AlertStateExeuctionError || s == AlertStatePaused || s == AlertStateCritical || s == AlertStateWarning
 }
 
 const (

pkg/services/alerting/conditions/evaluator.go

@@ -5,25 +5,21 @@ import (
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/services/alerting"
-	"github.com/grafana/grafana/pkg/tsdb"
 )
 
 var (
 	defaultTypes []string = []string{"gt", "lt"}
 	rangedTypes  []string = []string{"within_range", "outside_range"}
-	paramlessTypes []string = []string{"no_value"}
 )
 
 type AlertEvaluator interface {
-	Eval(timeSeries *tsdb.TimeSeries, reducedValue float64) bool
+	Eval(reducedValue *float64) bool
 }
 
-type ParameterlessEvaluator struct {
-	Type string
-}
+type NoDataEvaluator struct{}
 
-func (e *ParameterlessEvaluator) Eval(series *tsdb.TimeSeries, reducedValue float64) bool {
-	return len(series.Points) == 0
+func (e *NoDataEvaluator) Eval(reducedValue *float64) bool {
+	return reducedValue == nil
 }
 
 type ThresholdEvaluator struct {
@@ -47,14 +43,12 @@ func newThresholdEvaludator(typ string, model *simplejson.Json) (*ThresholdEvalu
 	return defaultEval, nil
 }
 
-func (e *ThresholdEvaluator) Eval(series *tsdb.TimeSeries, reducedValue float64) bool {
+func (e *ThresholdEvaluator) Eval(reducedValue *float64) bool {
 	switch e.Type {
 	case "gt":
-		return reducedValue > e.Threshold
+		return *reducedValue > e.Threshold
 	case "lt":
-		return reducedValue < e.Threshold
-	case "no_value":
-		return len(series.Points) == 0
+		return *reducedValue < e.Threshold
 	}
 
 	return false
@@ -88,12 +82,12 @@ func newRangedEvaluator(typ string, model *simplejson.Json) (*RangedEvaluator, e
 	return rangedEval, nil
 }
 
-func (e *RangedEvaluator) Eval(series *tsdb.TimeSeries, reducedValue float64) bool {
+func (e *RangedEvaluator) Eval(reducedValue *float64) bool {
 	switch e.Type {
 	case "within_range":
-		return (e.Lower < reducedValue && e.Upper > reducedValue) || (e.Upper < reducedValue && e.Lower > reducedValue)
+		return (e.Lower < *reducedValue && e.Upper > *reducedValue) || (e.Upper < *reducedValue && e.Lower > *reducedValue)
 	case "outside_range":
-		return (e.Upper < reducedValue && e.Lower < reducedValue) || (e.Upper > reducedValue && e.Lower > reducedValue)
+		return (e.Upper < *reducedValue && e.Lower < *reducedValue) || (e.Upper > *reducedValue && e.Lower > *reducedValue)
 	}
 
 	return false
@@ -113,8 +107,8 @@ func NewAlertEvaluator(model *simplejson.Json) (AlertEvaluator, error) {
 		return newRangedEvaluator(typ, model)
 	}
 
-	if inSlice(typ, paramlessTypes) {
-		return &ParameterlessEvaluator{Type: typ}, nil
+	if typ == "no_data" {
+		return &NoDataEvaluator{}, nil
 	}
 
 	return nil, alerting.ValidationError{Reason: "Evaludator invalid evaluator type"}

pkg/services/alerting/conditions/evaluator_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/tsdb"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
@@ -15,19 +14,7 @@ func evalutorScenario(json string, reducedValue float64, datapoints ...float64)
 	evaluator, err := NewAlertEvaluator(jsonModel)
 	So(err, ShouldBeNil)
 
-	var timeserie [][2]float64
-	dummieTimestamp := float64(521452145)
-
-	for _, v := range datapoints {
-		timeserie = append(timeserie, [2]float64{v, dummieTimestamp})
-	}
-
-	tsdb := &tsdb.TimeSeries{
-		Name:   "test time serie",
-		Points: timeserie,
-	}
-
-	return evaluator.Eval(tsdb, reducedValue)
+	return evaluator.Eval(reducedValue)
 }
 
 func TestEvalutors(t *testing.T) {

pkg/services/alerting/conditions/query.go

@@ -40,22 +40,27 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) {
 
 	for _, series := range seriesList {
 		reducedValue := c.Reducer.Reduce(series)
-		evalMatch := c.Evaluator.Eval(series, reducedValue)
+		evalMatch := c.Evaluator.Eval(reducedValue)
 
 		if context.IsTestRun {
 			context.Logs = append(context.Logs, &alerting.ResultLogEntry{
-				Message: fmt.Sprintf("Condition[%d]: Eval: %v, Metric: %s, Value: %1.3f", c.Index, evalMatch, series.Name, reducedValue),
+				Message: fmt.Sprintf("Condition[%d]: Eval: %v, Metric: %s, Value: %1.3f", c.Index, evalMatch, series.Name, *reducedValue),
 			})
 		}
 
 		if evalMatch {
 			context.EvalMatches = append(context.EvalMatches, &alerting.EvalMatch{
 				Metric: series.Name,
-				Value:  reducedValue,
+				Value:  *reducedValue,
 			})
 		}
 
 		context.Firing = evalMatch
+
+		// handle no data scenario
+		if reducedValue == nil {
+			context.NoDataFound = true
+		}
 	}
 }
 

pkg/services/alerting/conditions/reducer.go

@@ -3,14 +3,18 @@ package conditions
 import "github.com/grafana/grafana/pkg/tsdb"
 
 type QueryReducer interface {
-	Reduce(timeSeries *tsdb.TimeSeries) float64
+	Reduce(timeSeries *tsdb.TimeSeries) *float64
 }
 
 type SimpleReducer struct {
 	Type string
 }
 
-func (s *SimpleReducer) Reduce(series *tsdb.TimeSeries) float64 {
+func (s *SimpleReducer) Reduce(series *tsdb.TimeSeries) *float64 {
+	if len(series.Points) == 0 {
+		return nil
+	}
+
 	var value float64 = 0
 
 	switch s.Type {
@@ -46,7 +50,7 @@ func (s *SimpleReducer) Reduce(series *tsdb.TimeSeries) float64 {
 		value = float64(len(series.Points))
 	}
 
-	return value
+	return &value
 }
 
 func NewSimpleReducer(typ string) *SimpleReducer {

pkg/services/alerting/eval_context.go

@@ -26,6 +26,7 @@ type EvalContext struct {
 	dashboardSlug   string
 	ImagePublicUrl  string
 	ImageOnDiskPath string
+	NoDataFound     bool
 }
 
 type StateDescription struct {

pkg/services/alerting/result_handler.go

@@ -41,8 +41,13 @@ func (handler *DefaultResultHandler) Handle(ctx *EvalContext) {
 		ctx.Rule.State = m.AlertStateType(ctx.Rule.Severity)
 		annotationData = simplejson.NewFromAny(ctx.EvalMatches)
 	} else {
+		// handle no data case
+		if ctx.NoDataFound {
+			ctx.Rule.State = ctx.Rule.NoDataState
+		} else {
 			ctx.Rule.State = m.AlertStateOK
 		}
+	}
 
 	countStateResult(ctx.Rule.State)
 	if ctx.Rule.State != oldState {
@@ -91,8 +96,8 @@ func countStateResult(state m.AlertStateType) {
 		metrics.M_Alerting_Result_State_Ok.Inc(1)
 	case m.AlertStatePaused:
 		metrics.M_Alerting_Result_State_Paused.Inc(1)
-	case m.AlertStatePending:
-		metrics.M_Alerting_Result_State_Pending.Inc(1)
+	case m.AlertStateUnknown:
+		metrics.M_Alerting_Result_State_Unknown.Inc(1)
 	case m.AlertStateExeuctionError:
 		metrics.M_Alerting_Result_State_ExecutionError.Inc(1)
 	}

pkg/services/alerting/rule.go

@@ -18,6 +18,7 @@ type Rule struct {
 	Frequency     int64
 	Name          string
 	Message       string
+	NoDataState   m.AlertStateType
 	State         m.AlertStateType
 	Severity      m.AlertSeverityType
 	Conditions    []Condition
@@ -67,6 +68,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Frequency = ruleDef.Frequency
 	model.Severity = ruleDef.Severity
 	model.State = ruleDef.State
+	model.NoDataState = m.AlertStateType(ruleDef.Settings.Get("noDataState").MustString("unknown"))
 
 	for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
 		jsonModel := simplejson.NewFromAny(v)

pkg/services/alerting/rule_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/models"
+	m "github.com/grafana/grafana/pkg/models"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
@@ -45,6 +45,7 @@ func TestAlertRuleModel(t *testing.T) {
 				"name": "name2",
 				"description": "desc2",
 				"handler": 0,
+				"noDataMode": "critical",
 				"enabled": true,
 				"frequency": "60s",
         "conditions": [
@@ -63,7 +64,7 @@ func TestAlertRuleModel(t *testing.T) {
 			alertJSON, jsonErr := simplejson.NewJson([]byte(json))
 			So(jsonErr, ShouldBeNil)
 
-			alert := &models.Alert{
+			alert := &m.Alert{
 				Id:          1,
 				OrgId:       1,
 				DashboardId: 1,
@@ -80,6 +81,10 @@ func TestAlertRuleModel(t *testing.T) {
 			Convey("Can read notifications", func() {
 				So(len(alertRule.Notifications), ShouldEqual, 2)
 			})
+
+			Convey("Can read noDataMode", func() {
+				So(len(alertRule.NoDataMode), ShouldEqual, m.AlertStateCritical)
+			})
 		})
 	})
 }

pkg/services/sqlstore/alert.go

@@ -159,7 +159,7 @@ func upsertAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *xor
 		} else {
 			alert.Updated = time.Now()
 			alert.Created = time.Now()
-			alert.State = m.AlertStatePending
+			alert.State = m.AlertStateUnknown
 			alert.NewStateDate = time.Now()
 
 			_, err := sess.Insert(alert)

pkg/tsdb/graphite/graphite.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb"
 )
 
@@ -47,6 +48,10 @@ func (e *GraphiteExecutor) Execute(queries tsdb.QuerySlice, context *tsdb.QueryC
 		formData["target"] = []string{query.Query}
 	}
 
+	if setting.Env == setting.DEV {
+		glog.Debug("Graphite request", "params", formData)
+	}
+
 	req, err := e.createRequest(formData)
 	if err != nil {
 		result.Error = err
@@ -71,6 +76,10 @@ func (e *GraphiteExecutor) Execute(queries tsdb.QuerySlice, context *tsdb.QueryC
 			Name:   series.Target,
 			Points: series.DataPoints,
 		})
+
+		if setting.Env == setting.DEV {
+			glog.Debug("Graphite response", "target", series.Target, "datapoints", len(series.DataPoints))
+		}
 	}
 
 	result.QueryResults["A"] = queryRes

public/app/features/alerting/alert_def.ts

@@ -36,6 +36,13 @@ var reducerTypes = [
   {text: 'count()', value: 'count'},
 ];
 
+var noDataModes = [
+  {text: 'OK', value: 'ok'},
+  {text: 'Critical', value: 'critical'},
+  {text: 'Warning', value: 'warning'},
+  {text: 'Unknown', value: 'unknown'},
+];
+
 function createReducerPart(model) {
   var def = new QueryPartDef({type: model.type, defaultParams: []});
   return new QueryPart(model, def);
@@ -69,9 +76,9 @@ function getStateDisplayModel(state) {
         stateClass: 'alert-state-warning'
       };
     }
-    case 'pending': {
+    case 'unknown': {
       return {
-        text: 'PENDING',
+        text: 'UNKNOWN',
         iconClass: "fa fa-question",
         stateClass: 'alert-state-warning'
       };
@@ -100,6 +107,7 @@ export default {
   conditionTypes: conditionTypes,
   evalFunctions: evalFunctions,
   severityLevels: severityLevels,
+  noDataModes: noDataModes,
   reducerTypes: reducerTypes,
   createReducerPart: createReducerPart,
 };

public/app/features/alerting/alert_list_ctrl.ts

@@ -13,7 +13,7 @@ export class AlertListCtrl {
   stateFilters = [
     {text: 'All', value: null},
     {text: 'OK', value: 'ok'},
-    {text: 'Pending', value: 'pending'},
+    {text: 'Unknown', value: 'unknown'},
     {text: 'Warning', value: 'warning'},
     {text: 'Critical', value: 'critical'},
     {text: 'Execution Error', value: 'execution_error'},

public/app/features/alerting/alert_tab_ctrl.ts

@@ -18,6 +18,7 @@ export class AlertTabCtrl {
   conditionModels: any;
   evalFunctions: any;
   severityLevels: any;
+  noDataModes: any;
   addNotificationSegment;
   notifications;
   alertNotifications;
@@ -41,6 +42,7 @@ export class AlertTabCtrl {
     this.evalFunctions = alertDef.evalFunctions;
     this.conditionTypes = alertDef.conditionTypes;
     this.severityLevels = alertDef.severityLevels;
+    this.noDataModes = alertDef.noDataModes;
     this.appSubUrl = config.appSubUrl;
   }
 
@@ -138,6 +140,7 @@ export class AlertTabCtrl {
       alert.conditions.push(this.buildDefaultCondition());
     }
 
+    alert.noDataState = alert.noDataState || 'unknown';
     alert.severity = alert.severity || 'critical';
     alert.frequency = alert.frequency || '60s';
     alert.handler = alert.handler || 1;

public/app/features/alerting/partials/alert_tab.html

@@ -53,18 +53,19 @@
 						<span class="gf-form-label query-keyword width-5" ng-if="$index===0">WHEN</span>
 					</div>
           <div class="gf-form">
-						<query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
+						<query-part-editor class="gf-form-label query-part" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
 						</query-part-editor>
+            <span class="gf-form-label query-keyword">OF</span>
 					</div>
 					<div class="gf-form">
-						<span class="gf-form-label">Reducer</span>
-						<query-part-editor class="gf-form-label query-part" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
+						<query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
 						</query-part-editor>
 					</div>
 					<div class="gf-form">
 						<metric-segment-model property="conditionModel.evaluator.type" options="ctrl.evalFunctions" custom="false" css-class="query-keyword" on-change="ctrl.evaluatorTypeChanged(conditionModel.evaluator)"></metric-segment-model>
 						<input class="gf-form-input max-width-7" type="number" ng-hide="conditionModel.evaluator.params.length === 0" ng-model="conditionModel.evaluator.params[0]" ng-change="ctrl.evaluatorParamsChanged()"></input>
             <label class="gf-form-label query-keyword" ng-show="conditionModel.evaluator.params.length === 2">TO</label>
+            <input class="gf-form-input max-width-7" type="number" ng-if="conditionModel.evaluator.params.length === 2" ng-model="conditionModel.evaluator.params[1]" ng-change="ctrl.evaluatorParamsChanged()"></input>
 					</div>
 					<div class="gf-form">
 						<label class="gf-form-label">
@@ -88,6 +89,18 @@
 					</label>
 				</div>
 
+			</div>
+
+			<div class="gf-form-group">
+				<div class="gf-form">
+          <span class="gf-form-label">If no data points or all values are null</span>
+          <span class="gf-form-label query-keyword">SET STATE TO</span>
+					<div class="gf-form-select-wrapper">
+						<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
+						</select>
+					</div>
+				</div>
+
 				<div class="gf-form-button-row">
 					<button class="btn btn-inverse" ng-click="ctrl.test()">
 						Test Rule

public/sass/_variables.dark.scss

@@ -39,7 +39,6 @@ $brand-primary:         $orange;
 $brand-success:         $green;
 $brand-warning:         $brand-primary;
 $brand-danger:          $red;
-$brand-text-highlight:  #f7941d;
 
 // Status colors
 // -------------------------

public/sass/_variables.light.scss

@@ -44,7 +44,6 @@ $brand-primary:         $orange;
 $brand-success:         $green;
 $brand-warning:         $orange;
 $brand-danger:          $red;
-$brand-text-highlight:  #f7941d;
 
 // Status colors
 // -------------------------