AlertingNG: pause/unpause definitions via the API (#30627)

* AlertingNG: pause/unpause definitions via the API

* Apply suggestions from code review

Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>

* Enable pausing/unpausing multiple definitions

Co-authored-by: Will Browne <wbrowne@users.noreply.github.com>

pkg/services/ngalert/api.go

 package ngalert
 
 import (
+	"fmt"
+
 	"github.com/go-macaron/binding"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana/pkg/api/response"
@@ -21,6 +23,8 @@ func (ng *AlertNG) registerAPIEndpoints() {
 		alertDefinitions.Delete("/:alertDefinitionUID", ng.validateOrgAlertDefinition, routing.Wrap(ng.deleteAlertDefinitionEndpoint))
 		alertDefinitions.Post("/", middleware.ReqSignedIn, binding.Bind(saveAlertDefinitionCommand{}), routing.Wrap(ng.createAlertDefinitionEndpoint))
 		alertDefinitions.Put("/:alertDefinitionUID", ng.validateOrgAlertDefinition, binding.Bind(updateAlertDefinitionCommand{}), routing.Wrap(ng.updateAlertDefinitionEndpoint))
+		alertDefinitions.Post("/pause", ng.validateOrgAlertDefinition, binding.Bind(updateAlertDefinitionPausedCommand{}), routing.Wrap(ng.alertDefinitionPauseEndpoint))
+		alertDefinitions.Post("/unpause", ng.validateOrgAlertDefinition, binding.Bind(updateAlertDefinitionPausedCommand{}), routing.Wrap(ng.alertDefinitionUnpauseEndpoint))
 	})
 
 	ng.RouteRegister.Group("/api/ngalert/", func(schedulerRouter routing.RouteRegister) {
@@ -180,3 +184,27 @@ func (ng *AlertNG) unpauseScheduler() response.Response {
 	}
 	return response.JSON(200, util.DynMap{"message": "alert definition scheduler unpaused"})
 }
+
+// alertDefinitionPauseEndpoint handles POST /api/alert-definitions/pause.
+func (ng *AlertNG) alertDefinitionPauseEndpoint(c *models.ReqContext, cmd updateAlertDefinitionPausedCommand) response.Response {
+	cmd.OrgID = c.SignedInUser.OrgId
+	cmd.Paused = true
+
+	err := ng.updateAlertDefinitionPaused(&cmd)
+	if err != nil {
+		return response.Error(500, "Failed to pause alert definition", err)
+	}
+	return response.JSON(200, util.DynMap{"message": fmt.Sprintf("%d alert definitions paused", cmd.ResultCount)})
+}
+
+// alertDefinitionUnpauseEndpoint handles POST /api/alert-definitions/unpause.
+func (ng *AlertNG) alertDefinitionUnpauseEndpoint(c *models.ReqContext, cmd updateAlertDefinitionPausedCommand) response.Response {
+	cmd.OrgID = c.SignedInUser.OrgId
+	cmd.Paused = false
+
+	err := ng.updateAlertDefinitionPaused(&cmd)
+	if err != nil {
+		return response.Error(500, "Failed to unpause alert definition", err)
+	}
+	return response.JSON(200, util.DynMap{"message": fmt.Sprintf("%d alert definitions unpaused", cmd.ResultCount)})
+}

pkg/services/ngalert/database.go

@@ -212,7 +212,7 @@ func (ng *AlertNG) getOrgAlertDefinitions(query *listAlertDefinitionsQuery) erro
 func (ng *AlertNG) getAlertDefinitions(query *listAlertDefinitionsQuery) error {
 	return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
 		alerts := make([]*AlertDefinition, 0)
-		q := "SELECT uid, org_id, interval_seconds, version FROM alert_definition"
+		q := "SELECT uid, org_id, interval_seconds, version, paused FROM alert_definition"
 		if err := sess.SQL(q).Find(&alerts); err != nil {
 			return err
 		}
@@ -221,6 +221,39 @@ func (ng *AlertNG) getAlertDefinitions(query *listAlertDefinitionsQuery) error {
 		return nil
 	})
 }
+
+func (ng *AlertNG) updateAlertDefinitionPaused(cmd *updateAlertDefinitionPausedCommand) error {
+	return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
+		placeHolders := strings.Builder{}
+		const separator = ", "
+		separatorVar := separator
+		params := []interface{}{cmd.Paused, cmd.OrgID}
+		for i, UID := range cmd.UIDs {
+			if i == len(cmd.UIDs)-1 {
+				separatorVar = ""
+			}
+			placeHolders.WriteString(fmt.Sprintf("?%s", separatorVar))
+			params = append(params, UID)
+		}
+		sql := fmt.Sprintf("UPDATE alert_definition SET paused = ? WHERE org_id = ? AND uid IN (%s)", placeHolders.String())
+
+		// prepend sql statement to params
+		var i interface{}
+		params = append(params, i)
+		copy(params[1:], params[0:])
+		params[0] = sql
+
+		res, err := sess.Exec(params...)
+		if err != nil {
+			return err
+		}
+		if cmd.ResultCount, err = res.RowsAffected(); err != nil {
+			ng.log.Debug("failed to get rows affected: %w", err)
+		}
+		return nil
+	})
+}
+
 func generateNewAlertDefinitionUID(sess *sqlstore.DBSession, orgID int64) (string, error) {
 	for i := 0; i < 3; i++ {
 		uid := util.GenerateShortUID()

pkg/services/ngalert/database_mig.go

@@ -46,6 +46,10 @@ func addAlertDefinitionMigrations(mg *migrator.Migrator) {
 	}
 	mg.AddMigration("add unique index in alert_definition on org_id and title columns", migrator.NewAddIndexMigration(alertDefinition, uniqueIndices[0]))
 	mg.AddMigration("add unique index in alert_definition on org_id and uid columns", migrator.NewAddIndexMigration(alertDefinition, uniqueIndices[1]))
+
+	mg.AddMigration("Add column paused in alert_definition", migrator.NewAddColumnMigration(alertDefinition, &migrator.Column{
+		Name: "paused", Type: migrator.DB_Bool, Nullable: false, Default: "0",
+	}))
 }
 
 func addAlertDefinitionVersionMigrations(mg *migrator.Migrator) {

pkg/services/ngalert/models.go

@@ -21,6 +21,7 @@ type AlertDefinition struct {
 	IntervalSeconds int64             `json:"intervalSeconds"`
 	Version         int64             `json:"version"`
 	UID             string            `xorm:"uid" json:"uid"`
+	Paused          bool              `json:"paused"`
 }
 
 type alertDefinitionKey struct {
@@ -101,3 +102,11 @@ type listAlertDefinitionsQuery struct {
 
 	Result []*AlertDefinition
 }
+
+type updateAlertDefinitionPausedCommand struct {
+	OrgID  int64    `json:"-"`
+	UIDs   []string `json:"uids"`
+	Paused bool     `json:"-"`
+
+	ResultCount int64
+}

pkg/services/ngalert/schedule.go

@@ -185,6 +185,10 @@ func (ng *AlertNG) alertingTicker(grafanaCtx context.Context) error {
 			}
 			readyToRun := make([]readyToRunItem, 0)
 			for _, item := range alertDefinitions {
+				if item.Paused {
+					continue
+				}
+
 				key := item.getKey()
 				itemVersion := item.Version
 				newRoutine := !ng.schedule.registry.exists(key)

pkg/services/ngalert/schedule_test.go

@@ -123,6 +123,33 @@ func TestAlertingTicker(t *testing.T) {
 		tick := advanceClock(t, mockedClock)
 		assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
 	})
+
+	// pause alert definition
+	err = ng.updateAlertDefinitionPaused(&updateAlertDefinitionPausedCommand{UIDs: []string{alerts[2].UID}, OrgID: alerts[2].OrgID, Paused: true})
+	require.NoError(t, err)
+	t.Logf("alert definition: %v paused", alerts[2].getKey())
+
+	expectedAlertDefinitionsEvaluated = []alertDefinitionKey{}
+	t.Run(fmt.Sprintf("on 8th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
+		tick := advanceClock(t, mockedClock)
+		assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
+	})
+
+	expectedAlertDefinitionsStopped = []alertDefinitionKey{alerts[2].getKey()}
+	t.Run(fmt.Sprintf("on 8th tick alert definitions: %s should be stopped", concatenate(expectedAlertDefinitionsStopped)), func(t *testing.T) {
+		assertStopRun(t, stopAppliedCh, expectedAlertDefinitionsStopped...)
+	})
+
+	// unpause alert definition
+	err = ng.updateAlertDefinitionPaused(&updateAlertDefinitionPausedCommand{UIDs: []string{alerts[2].UID}, OrgID: alerts[2].OrgID, Paused: false})
+	require.NoError(t, err)
+	t.Logf("alert definition: %v unpaused", alerts[2].getKey())
+
+	expectedAlertDefinitionsEvaluated = []alertDefinitionKey{alerts[0].getKey(), alerts[2].getKey()}
+	t.Run(fmt.Sprintf("on 9th tick alert definitions: %s should be evaluated", concatenate(expectedAlertDefinitionsEvaluated)), func(t *testing.T) {
+		tick := advanceClock(t, mockedClock)
+		assertEvalRun(t, evalAppliedCh, tick, expectedAlertDefinitionsEvaluated...)
+	})
 }
 
 func assertEvalRun(t *testing.T, ch <-chan evalAppliedInfo, tick time.Time, keys ...alertDefinitionKey) {