Skip to content

N
nexpie-grafana-theme
Commit 9022e871 by bergquist
Options

avoid sending full notification state to pending/complete

parent 341d8af6
Showing with 48 additions and 32 deletions
pkg/models/alert_notifications.go
...@@ -98,12 +98,16 @@ type AlertNotificationState struct { ...@@ -98,12 +98,16 @@ type AlertNotificationState struct {
} }
type SetAlertNotificationStateToPendingCommand struct { type SetAlertNotificationStateToPendingCommand struct {
Id int64
AlertRuleStateUpdatedVersion int64 AlertRuleStateUpdatedVersion int64
State *AlertNotificationState Version int64
ResultVersion int64
} }
type SetAlertNotificationStateToCompleteCommand struct { type SetAlertNotificationStateToCompleteCommand struct {
State *AlertNotificationState Id int64
Version int64
} }
type GetOrCreateNotificationStateQuery struct { type GetOrCreateNotificationStateQuery struct {
......
pkg/services/alerting/notifier.go
...@@ -76,7 +76,8 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no ...@@ -76,7 +76,8 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no
} }
cmd := &m.SetAlertNotificationStateToCompleteCommand{ cmd := &m.SetAlertNotificationStateToCompleteCommand{
State: notifierState.state, Id: not.GetNotifierId(),
Version: notifierState.state.Version,
} }
if err = bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { if err = bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
...@@ -94,7 +95,8 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no ...@@ -94,7 +95,8 @@ func (n *notificationService) sendAndMarkAsComplete(evalContext *EvalContext, no
func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *NotifierState) error { func (n *notificationService) sendNotification(evalContext *EvalContext, notifierState *NotifierState) error {
if !evalContext.IsTestRun { if !evalContext.IsTestRun {
setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{ setPendingCmd := &m.SetAlertNotificationStateToPendingCommand{
State: notifierState.state, Id: notifierState.state.NotifierId,
Version: notifierState.state.Version,
AlertRuleStateUpdatedVersion: evalContext.Rule.StateChanges, AlertRuleStateUpdatedVersion: evalContext.Rule.StateChanges,
} }
...@@ -106,6 +108,10 @@ func (n *notificationService) sendNotification(evalContext *EvalContext, notifie ...@@ -106,6 +108,10 @@ func (n *notificationService) sendNotification(evalContext *EvalContext, notifie
if err != nil { if err != nil {
return err return err
} }
// We need to update state version to be able to log
// unexpected version conflicts when marking notifications as ok
notifierState.state.Version = setPendingCmd.ResultVersion
} }
return n.sendAndMarkAsComplete(evalContext, notifierState) return n.sendAndMarkAsComplete(evalContext, notifierState)
......
pkg/services/sqlstore/alert_notification.go
...@@ -239,12 +239,11 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { ...@@ -239,12 +239,11 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error { func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToCompleteCommand) error {
return withDbSession(ctx, func(sess *DBSession) error { return withDbSession(ctx, func(sess *DBSession) error {
version := cmd.State.Version version := cmd.Version
var current m.AlertNotificationState var current m.AlertNotificationState
sess.ID(cmd.State.Id).Get(&current) sess.ID(cmd.Id).Get(&current)
cmd.State.State = m.AlertNotificationStateCompleted newVersion := cmd.Version + 1
cmd.State.Version++
sql := `UPDATE alert_notification_state SET sql := `UPDATE alert_notification_state SET
state = ?, state = ?,
...@@ -253,7 +252,7 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA ...@@ -253,7 +252,7 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA
WHERE WHERE
id = ?` id = ?`
_, err := sess.Exec(sql, cmd.State.State, cmd.State.Version, timeNow().Unix(), cmd.State.Id) _, err := sess.Exec(sql, m.AlertNotificationStateCompleted, newVersion, timeNow().Unix(), cmd.Id)
if err != nil { if err != nil {
return err return err
...@@ -269,10 +268,7 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA ...@@ -269,10 +268,7 @@ func SetAlertNotificationStateToCompleteCommand(ctx context.Context, cmd *m.SetA
func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToPendingCommand) error { func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAlertNotificationStateToPendingCommand) error {
return withDbSession(ctx, func(sess *DBSession) error { return withDbSession(ctx, func(sess *DBSession) error {
currentVersion := cmd.State.Version newVersion := cmd.Version + 1
cmd.State.State = m.AlertNotificationStatePending
cmd.State.Version++
sql := `UPDATE alert_notification_state SET sql := `UPDATE alert_notification_state SET
state = ?, state = ?,
version = ?, version = ?,
...@@ -283,12 +279,12 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl ...@@ -283,12 +279,12 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl
(version = ? OR alert_rule_state_updated_version < ?)` (version = ? OR alert_rule_state_updated_version < ?)`
res, err := sess.Exec(sql, res, err := sess.Exec(sql,
cmd.State.State, m.AlertNotificationStatePending,
cmd.State.Version, newVersion,
timeNow().Unix(), timeNow().Unix(),
cmd.AlertRuleStateUpdatedVersion, cmd.AlertRuleStateUpdatedVersion,
cmd.State.Id, cmd.Id,
currentVersion, cmd.Version,
cmd.AlertRuleStateUpdatedVersion) cmd.AlertRuleStateUpdatedVersion)
if err != nil { if err != nil {
...@@ -300,6 +296,8 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl ...@@ -300,6 +296,8 @@ func SetAlertNotificationStateToPendingCommand(ctx context.Context, cmd *m.SetAl
return m.ErrAlertNotificationStateVersionConflict return m.ErrAlertNotificationStateVersionConflict
} }
cmd.ResultVersion = newVersion
return nil return nil
}) })
} }
......
pkg/services/sqlstore/alert_notification_test.go
...@@ -42,13 +42,16 @@ func TestAlertNotificationSQLAccess(t *testing.T) { ...@@ -42,13 +42,16 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
Convey("Update existing state to pending with correct version should update database", func() { Convey("Update existing state to pending with correct version should update database", func() {
s := *query.Result s := *query.Result
cmd := models.SetAlertNotificationStateToPendingCommand{ cmd := models.SetAlertNotificationStateToPendingCommand{
State: &s, Id: s.Id,
Version: s.Version,
AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion,
} }
err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(cmd.State.Version, ShouldEqual, 1) So(cmd.ResultVersion, ShouldEqual, 1)
So(cmd.State.State, ShouldEqual, models.AlertNotificationStatePending)
query2 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} query2 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID}
err = GetOrCreateAlertNotificationState(context.Background(), query2) err = GetOrCreateAlertNotificationState(context.Background(), query2)
...@@ -58,11 +61,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) { ...@@ -58,11 +61,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
So(query2.Result.UpdatedAt, ShouldEqual, now.Unix()) So(query2.Result.UpdatedAt, ShouldEqual, now.Unix())
Convey("Update existing state to completed should update database", func() { Convey("Update existing state to completed should update database", func() {
s := *cmd.State s := *query.Result
cmd := models.SetAlertNotificationStateToCompleteCommand{ setStateCmd := models.SetAlertNotificationStateToCompleteCommand{
State: &s, Id: s.Id,
Version: cmd.ResultVersion,
} }
err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd) err := SetAlertNotificationStateToCompleteCommand(context.Background(), &setStateCmd)
So(err, ShouldBeNil) So(err, ShouldBeNil)
query3 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID} query3 := &models.GetOrCreateNotificationStateQuery{AlertId: alertID, OrgId: orgID, NotifierId: notifierID}
...@@ -74,10 +78,10 @@ func TestAlertNotificationSQLAccess(t *testing.T) { ...@@ -74,10 +78,10 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
}) })
Convey("Update existing state to completed should update database, but return version mismatch", func() { Convey("Update existing state to completed should update database, but return version mismatch", func() {
cmd.State.Version = 1000 s := *query.Result
s := *cmd.State
cmd := models.SetAlertNotificationStateToCompleteCommand{ cmd := models.SetAlertNotificationStateToCompleteCommand{
State: &s, Id: s.Id,
Version: 1000,
} }
err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd) err := SetAlertNotificationStateToCompleteCommand(context.Background(), &cmd)
So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict)
...@@ -95,7 +99,9 @@ func TestAlertNotificationSQLAccess(t *testing.T) { ...@@ -95,7 +99,9 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
s := *query.Result s := *query.Result
s.Version = 1000 s.Version = 1000
cmd := models.SetAlertNotificationStateToPendingCommand{ cmd := models.SetAlertNotificationStateToPendingCommand{
State: &s, Id: s.NotifierId,
Version: s.Version,
AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion,
} }
err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd)
So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict) So(err, ShouldEqual, models.ErrAlertNotificationStateVersionConflict)
...@@ -104,21 +110,23 @@ func TestAlertNotificationSQLAccess(t *testing.T) { ...@@ -104,21 +110,23 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
Convey("Updating existing state to pending with incorrect version since alert rule state update version is higher", func() { Convey("Updating existing state to pending with incorrect version since alert rule state update version is higher", func() {
s := *query.Result s := *query.Result
cmd := models.SetAlertNotificationStateToPendingCommand{ cmd := models.SetAlertNotificationStateToPendingCommand{
State: &s, Id: s.Id,
Version: s.Version,
AlertRuleStateUpdatedVersion: 1000, AlertRuleStateUpdatedVersion: 1000,
} }
err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd)
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(cmd.State.Version, ShouldEqual, 1) So(cmd.ResultVersion, ShouldEqual, 1)
So(cmd.State.State, ShouldEqual, models.AlertNotificationStatePending)
}) })
Convey("different version and same alert state change version should return error", func() { Convey("different version and same alert state change version should return error", func() {
s := *query.Result s := *query.Result
s.Version = 1000 s.Version = 1000
cmd := models.SetAlertNotificationStateToPendingCommand{ cmd := models.SetAlertNotificationStateToPendingCommand{
State: &s, Id: s.Id,
Version: s.Version,
AlertRuleStateUpdatedVersion: s.AlertRuleStateUpdatedVersion,
} }
err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd) err := SetAlertNotificationStateToPendingCommand(context.Background(), &cmd)
So(err, ShouldNotBeNil) So(err, ShouldNotBeNil)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment