feat(alerting): more work on handling result and saving state

pkg/services/alerting/engine.go

@@ -71,23 +71,25 @@ func (e *Engine) alertingTicker() {
 }
 
 func (e *Engine) execDispatch() {
-	defer func() {
-		if err := recover(); err != nil {
-			e.log.Error("Scheduler Panic: stopping executor", "error", err, "stack", log.Stack(1))
-		}
-	}()
-
 	for job := range e.execQueue {
-		log.Trace("Alerting: engine:execDispatch() starting job %s", job.Rule.Name)
-		e.executeJob(job)
+		e.log.Debug("Starting executing alert rule %s", job.Rule.Name)
+		go e.executeJob(job)
 	}
 }
 
 func (e *Engine) executeJob(job *AlertJob) {
+	defer func() {
+		if err := recover(); err != nil {
+			e.log.Error("Execute Alert Panic", "error", err, "stack", log.Stack(1))
+		}
+	}()
+
 	job.Running = true
 	context := NewAlertResultContext(job.Rule)
 	e.handler.Execute(context)
 	job.Running = false
+
+	e.resultQueue <- context
 }
 
 func (e *Engine) resultHandler() {

pkg/services/alerting/handler.go

@@ -55,6 +55,7 @@ func (e *HandlerImpl) eval(context *AlertResultContext) {
 	}
 
 	context.EndTime = time.Now()
+	context.DoneChan <- true
 }
 
 // func (e *HandlerImpl) executeQuery(job *AlertJob) (tsdb.TimeSeriesSlice, error) {

pkg/services/alerting/models.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
 )
 
 type AlertJob struct {
@@ -38,6 +39,7 @@ type AlertResultContext struct {
 	Rule        *AlertRule
 	DoneChan    chan bool
 	CancelChan  chan bool
+	log         log.Logger
 }
 
 func (a *AlertResultContext) GetDurationSeconds() float64 {
@@ -51,6 +53,7 @@ func NewAlertResultContext(rule *AlertRule) *AlertResultContext {
 		Logs:       make([]*AlertResultLogEntry, 0),
 		DoneChan:   make(chan bool, 1),
 		CancelChan: make(chan bool, 1),
+		log:        log.New("alerting.engine"),
 	}
 }
 

pkg/services/alerting/result_handler.go

 package alerting
 
-import "github.com/grafana/grafana/pkg/log"
+import (
+	"time"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting/alertstates"
+)
 
 type ResultHandler interface {
 	Handle(result *AlertResultContext)
@@ -19,44 +27,51 @@ func NewResultHandler() *ResultHandlerImpl {
 }
 
 func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
-	// if handler.shouldUpdateState(result) {
-	// 	cmd := &m.UpdateAlertStateCommand{
-	// 		AlertId:         result.Rule.Id,
-	// 		State:           result.Rule.Severity,
-	// 		Info:            result.Description,
-	// 		OrgId:           result.Rule.OrgId,
-	// 		TriggeredAlerts: simplejson.NewFromAny(result.Details),
-	// 	}
-	//
-	// 	if err := bus.Dispatch(cmd); err != nil {
-	// 		handler.log.Error("Failed to save state", "error", err)
-	// 	}
-	//
-	// 	handler.log.Debug("will notify about new state", "new state", result.State)
-	// 	handler.notifier.Notify(result)
-	// }
+	newState := alertstates.Ok
+	if result.Triggered {
+		newState = result.Rule.Severity
+	}
+
+	handler.log.Info("Handle result", "newState", newState)
+	handler.log.Info("Handle result", "triggered", result.Triggered)
+
+	if handler.shouldUpdateState(result, newState) {
+		cmd := &m.UpdateAlertStateCommand{
+			AlertId:         result.Rule.Id,
+			Info:            result.Description,
+			OrgId:           result.Rule.OrgId,
+			State:           newState,
+			TriggeredAlerts: simplejson.NewFromAny(result.Details),
+		}
+
+		if err := bus.Dispatch(cmd); err != nil {
+			handler.log.Error("Failed to save state", "error", err)
+		}
+
+		//handler.log.Debug("will notify about new state", "new state", result.State)
+		//handler.notifier.Notify(result)
+	}
 }
 
-func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResultContext) bool {
-	// query := &m.GetLastAlertStateQuery{
-	// 	AlertId: result.AlertJob.Rule.Id,
-	// 	OrgId:   result.AlertJob.Rule.OrgId,
-	// }
-	//
-	// if err := bus.Dispatch(query); err != nil {
-	// 	log.Error2("Failed to read last alert state", "error", err)
-	// 	return false
-	// }
-	//
-	// if query.Result == nil {
-	// 	return true
-	// }
-	//
-	// lastExecution := query.Result.Created
-	// asdf := result.StartTime.Add(time.Minute * -15)
-	// olderThen15Min := lastExecution.Before(asdf)
-	// changedState := query.Result.State != result.State
-	//
-	// return changedState || olderThen15Min
-	return false
+func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResultContext, newState string) bool {
+	query := &m.GetLastAlertStateQuery{
+		AlertId: result.Rule.Id,
+		OrgId:   result.Rule.OrgId,
+	}
+
+	if err := bus.Dispatch(query); err != nil {
+		log.Error2("Failed to read last alert state", "error", err)
+		return false
+	}
+
+	if query.Result == nil {
+		return true
+	}
+
+	lastExecution := query.Result.Created
+	asdf := result.StartTime.Add(time.Minute * -15)
+	olderThen15Min := lastExecution.Before(asdf)
+	changedState := query.Result.State != newState
+
+	return changedState || olderThen15Min
 }

pkg/services/alerting/result_handler_test.go

 package alerting
 
+// import (
+// 	"testing"
+// 	"time"
+//
+// 	"github.com/grafana/grafana/pkg/bus"
+// 	m "github.com/grafana/grafana/pkg/models"
+// 	"github.com/grafana/grafana/pkg/services/alerting/alertstates"
+//
+// 	. "github.com/smartystreets/goconvey/convey"
+// )
+//
 // func TestAlertResultHandler(t *testing.T) {
 // 	Convey("Test result Handler", t, func() {
 // 		resultHandler := ResultHandlerImpl{}
-// 		mockResult := &AlertResult{
-// 			State: alertstates.Ok,
-// 			AlertJob: &AlertJob{
-// 				Rule: &AlertRule{
-// 					Id:    1,
-// 					OrgId: 1,
-// 				},
+// 		mockResult := &AlertResultContext{
+// 			Triggered: false,
+// 			Rule: &AlertRule{
+// 				Id:    1,
+// 				OrgId 1,
 // 			},
 // 		}
 // 		mockAlertState := &m.AlertState{}
@@ -30,7 +39,7 @@ package alerting
 // 				mockAlertState = &m.AlertState{
 // 					State: alertstates.Critical,
 // 				}
-// 				mockResult.State = alertstates.Ok
+// 				mockResult.Triggered = false
 // 				So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
 // 			})
 //
@@ -40,7 +49,7 @@ package alerting
 // 					State:   alertstates.Critical,
 // 					Created: now.Add(time.Minute * -30),
 // 				}
-// 				mockResult.State = alertstates.Critical
+// 				mockResult.Triggered = true
 // 				mockResult.StartTime = time.Now()
 // 				So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
 // 			})

public/app/plugins/panel/graph/alert_tab_ctrl.ts

@@ -46,8 +46,8 @@ export class AlertTabCtrl {
     {text: '<', value: '<'},
   ];
   severityLevels = [
-    {text: 'Critical', value: 'critical'},
-    {text: 'Warning', value: 'warning'},
+    {text: 'Critical', value: 'CRITICAL'},
+    {text: 'Warning', value: 'WARN'},
   ];
 
   /** @ngInject */