Annotation: Add clean up job for old annotations (#26156)

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>
Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

conf/defaults.ini

@@ -599,6 +599,36 @@ max_attempts = 3
 # Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend
 min_interval_seconds = 1
 
+# Configures for how long alert annotations are stored. Default is 0, which keeps them forever.
+# This setting should be expressed as an duration. Ex 6h (hours), 10d (days), 2w (weeks), 1M (month).
+max_annotation_age =
+
+# Configures max number of alert annotations that Grafana stores. Default value is 0, which keeps all alert annotations.
+max_annotations_to_keep =
+
+#################################### Annotations #########################
+
+[annotations.dashboard]
+# Dashboard annotations means that annotations are associated with the dashboard they are created on.
+
+# Configures how long dashboard annotations are stored. Default is 0, which keeps them forever.
+# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+max_age =
+
+# Configures max number of dashboard annotations that Grafana stores. Default value is 0, which keeps all dashboard annotations.
+max_annotations_to_keep =
+
+[annotations.api]
+# API annotations means that the annotations have been created using the API without any
+# association with a dashboard.
+
+# Configures how long Grafana stores API annotations. Default is 0, which keeps them forever.
+# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+max_age =
+
+# Configures max number of API annotations that Grafana keeps. Default value is 0, which keeps all API annotations.
+max_annotations_to_keep =
+
 #################################### Explore #############################
 [explore]
 # Enable the Explore section

conf/sample.ini

@@ -591,6 +591,36 @@
 # Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend
 ;min_interval_seconds = 1
 
+# Configures for how long alert annotations are stored. Default is 0, which keeps them forever.
+# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+;max_annotation_age =
+
+# Configures max number of alert annotations that Grafana stores. Default value is 0, which keeps all alert annotations.
+;max_annotations_to_keep =
+
+#################################### Annotations #########################
+
+[annotations.dashboard]
+# Dashboard annotations means that annotations are associated with the dashboard they are created on.
+
+# Configures how long dashboard annotations are stored. Default is 0, which keeps them forever.
+# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+;max_age =
+
+# Configures max number of dashboard annotations that Grafana stores. Default value is 0, which keeps all dashboard annotations.
+;max_annotations_to_keep =
+
+[annotations.api]
+# API annotations means that the annotations have been created using the API without any
+# association with a dashboard.
+
+# Configures how long Grafana stores API annotations. Default is 0, which keeps them forever.
+# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+;max_age =
+
+# Configures max number of API annotations that Grafana keeps. Default value is 0, which keeps all API annotations.
+;max_annotations_to_keep =
+
 #################################### Explore #############################
 [explore]
 # Enable the Explore section

docs/sources/administration/configuration.md

@@ -976,6 +976,43 @@ Sets the minimum interval between rule evaluations. Default value is `1`.
 
 > **Note.** This setting has precedence over each individual rule frequency. If a rule frequency is lower than this value, then this value is enforced.
 
+### max_annotation_age =
+
+Configures for how long alert annotations are stored. Default is 0, which keeps them forever.
+This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+
+### max_annotations_to_keep =
+
+Configures max number of alert annotations that Grafana stores. Default value is 0, which keeps all alert annotations.
+
+<hr>
+
+## [annotations.dashboard]
+
+Dashboard annotations means that annotations are associated with the dashboard they are created on.
+
+### max_age
+
+Configures how long dashboard annotations are stored. Default is 0, which keeps them forever.
+This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+
+### max_annotations_to_keep
+
+Configures max number of dashboard annotations that Grafana stores. Default value is 0, which keeps all dashboard annotations.
+
+## [annotations.api]
+
+API annotations means that the annotations have been created using the API without any association with a dashboard.
+
+### max_age
+
+Configures how long Grafana stores API annotations. Default is 0, which keeps them forever.
+This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+
+### max_annotations_to_keep
+
+Configures max number of API annotations that Grafana keeps. Default value is 0, which keeps all API annotations.
+
 <hr>
 
 ## [explore]

pkg/services/annotations/annotations.go

 package annotations
 
-import "github.com/grafana/grafana/pkg/components/simplejson"
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/setting"
+)
 
 type Repository interface {
 	Save(item *Item) error
@@ -9,6 +14,11 @@ type Repository interface {
 	Delete(params *DeleteParams) error
 }
 
+// AnnotationCleaner is responsible for cleaning up old annotations
+type AnnotationCleaner interface {
+	CleanAnnotations(ctx context.Context, cfg *setting.Cfg) error
+}
+
 type ItemQuery struct {
 	OrgId        int64    `json:"orgId"`
 	From         int64    `json:"from"`
@@ -43,6 +53,15 @@ type DeleteParams struct {
 }
 
 var repositoryInstance Repository
+var cleanerInstance AnnotationCleaner
+
+func GetAnnotationCleaner() AnnotationCleaner {
+	return cleanerInstance
+}
+
+func SetAnnotationCleaner(rep AnnotationCleaner) {
+	cleanerInstance = rep
+}
 
 func GetRepository() Repository {
 	return repositoryInstance
@@ -74,6 +93,10 @@ type Item struct {
 	Title string
 }
 
+func (i Item) TableName() string {
+	return "annotation"
+}
+
 type ItemDTO struct {
 	Id          int64            `json:"id"`
 	AlertId     int64            `json:"alertId"`

pkg/services/cleanup/cleanup.go

@@ -12,6 +12,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/serverlock"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/registry"
+	"github.com/grafana/grafana/pkg/services/annotations"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
@@ -37,9 +38,14 @@ func (srv *CleanUpService) Run(ctx context.Context) error {
 	for {
 		select {
 		case <-ticker.C:
+			ctxWithTimeout, cancelFn := context.WithTimeout(ctx, time.Minute*9)
+			defer cancelFn()
+
 			srv.cleanUpTmpFiles()
 			srv.deleteExpiredSnapshots()
 			srv.deleteExpiredDashboardVersions()
+			srv.cleanUpOldAnnotations(ctxWithTimeout)
+
 			err := srv.ServerLockService.LockAndExecute(ctx, "delete old login attempts",
 				time.Minute*10, func() {
 					srv.deleteOldLoginAttempts()
@@ -53,6 +59,14 @@ func (srv *CleanUpService) Run(ctx context.Context) error {
 	}
 }
 
+func (srv *CleanUpService) cleanUpOldAnnotations(ctx context.Context) {
+	cleaner := annotations.GetAnnotationCleaner()
+	err := cleaner.CleanAnnotations(ctx, srv.Cfg)
+	if err != nil {
+		srv.log.Error("failed to clean up old annotations", "error", err)
+	}
+}
+
 func (srv *CleanUpService) cleanUpTmpFiles() {
 	if _, err := os.Stat(srv.Cfg.ImagesDir); os.IsNotExist(err) {
 		return

pkg/services/sqlstore/annotation_cleanup.go

+package sqlstore
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+// AnnotationCleanupService is responseible for cleaning old annotations.
+type AnnotationCleanupService struct {
+	batchSize int64
+	log       log.Logger
+}
+
+const (
+	alertAnnotationType     = "alert_id <> 0"
+	dashboardAnnotationType = "dashboard_id <> 0 AND alert_id = 0"
+	apiAnnotationType       = "alert_id = 0 AND dashboard_id = 0"
+)
+
+// CleanAnnotations deletes old annotations created by
+// alert rules, API requests and human made in the UI.
+func (acs *AnnotationCleanupService) CleanAnnotations(ctx context.Context, cfg *setting.Cfg) error {
+	err := acs.cleanAnnotations(ctx, cfg.AlertingAnnotationCleanupSetting, alertAnnotationType)
+	if err != nil {
+		return err
+	}
+
+	err = acs.cleanAnnotations(ctx, cfg.APIAnnotationCleanupSettings, apiAnnotationType)
+	if err != nil {
+		return err
+	}
+
+	return acs.cleanAnnotations(ctx, cfg.DashboardAnnotationCleanupSettings, dashboardAnnotationType)
+}
+
+func (acs *AnnotationCleanupService) cleanAnnotations(ctx context.Context, cfg setting.AnnotationCleanupSettings, annotationType string) error {
+	if cfg.MaxAge > 0 {
+		cutoffDate := time.Now().Add(-cfg.MaxAge).UnixNano() / int64(time.Millisecond)
+		deleteQuery := `DELETE FROM annotation WHERE id IN (SELECT id FROM (SELECT id FROM annotation WHERE %s AND created < %v ORDER BY id DESC %s) a)`
+		sql := fmt.Sprintf(deleteQuery, annotationType, cutoffDate, dialect.Limit(acs.batchSize))
+
+		err := acs.executeUntilDoneOrCancelled(ctx, sql)
+		if err != nil {
+			return err
+		}
+	}
+
+	if cfg.MaxCount > 0 {
+		deleteQuery := `DELETE FROM annotation WHERE id IN (SELECT id FROM (SELECT id FROM annotation WHERE %s ORDER BY id DESC %s) a)`
+		sql := fmt.Sprintf(deleteQuery, annotationType, dialect.LimitOffset(acs.batchSize, cfg.MaxCount))
+		return acs.executeUntilDoneOrCancelled(ctx, sql)
+	}
+
+	return nil
+}
+
+func (acs *AnnotationCleanupService) executeUntilDoneOrCancelled(ctx context.Context, sql string) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			var affected int64
+			err := withDbSession(ctx, func(session *DBSession) error {
+				res, err := session.Exec(sql)
+				if err != nil {
+					return err
+				}
+
+				affected, err = res.RowsAffected()
+
+				return err
+			})
+			if err != nil {
+				return err
+			}
+
+			if affected == 0 {
+				return nil
+			}
+		}
+	}
+}

pkg/services/sqlstore/annotation_cleanup_test.go

+package sqlstore
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/annotations"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAnnotationCleanUp(t *testing.T) {
+	fakeSQL := InitTestDB(t)
+
+	t.Cleanup(func() {
+		_ = fakeSQL.WithDbSession(context.Background(), func(session *DBSession) error {
+			_, err := session.Exec("DELETE FROM annotation")
+			require.Nil(t, err, "cleaning up all annotations should not cause problems")
+			return err
+		})
+	})
+
+	createTestAnnotations(t, fakeSQL, 21, 6)
+	assertAnnotationCount(t, fakeSQL, "", 21)
+
+	tests := []struct {
+		name                     string
+		cfg                      *setting.Cfg
+		alertAnnotationCount     int64
+		dashboardAnnotationCount int64
+		APIAnnotationCount       int64
+	}{
+		{
+			name: "default settings should not delete any annotations",
+			cfg: &setting.Cfg{
+				AlertingAnnotationCleanupSetting:   settingsFn(0, 0),
+				DashboardAnnotationCleanupSettings: settingsFn(0, 0),
+				APIAnnotationCleanupSettings:       settingsFn(0, 0),
+			},
+			alertAnnotationCount:     7,
+			dashboardAnnotationCount: 7,
+			APIAnnotationCount:       7,
+		},
+		{
+			name: "should remove annotations created before cut off point",
+			cfg: &setting.Cfg{
+				AlertingAnnotationCleanupSetting:   settingsFn(time.Hour*48, 0),
+				DashboardAnnotationCleanupSettings: settingsFn(time.Hour*48, 0),
+				APIAnnotationCleanupSettings:       settingsFn(time.Hour*48, 0),
+			},
+			alertAnnotationCount:     5,
+			dashboardAnnotationCount: 5,
+			APIAnnotationCount:       5,
+		},
+		{
+			name: "should only keep three annotations",
+			cfg: &setting.Cfg{
+				AlertingAnnotationCleanupSetting:   settingsFn(0, 3),
+				DashboardAnnotationCleanupSettings: settingsFn(0, 3),
+				APIAnnotationCleanupSettings:       settingsFn(0, 3),
+			},
+			alertAnnotationCount:     3,
+			dashboardAnnotationCount: 3,
+			APIAnnotationCount:       3,
+		},
+		{
+			name: "running the max count delete again should not remove any annotations",
+			cfg: &setting.Cfg{
+				AlertingAnnotationCleanupSetting:   settingsFn(0, 3),
+				DashboardAnnotationCleanupSettings: settingsFn(0, 3),
+				APIAnnotationCleanupSettings:       settingsFn(0, 3),
+			},
+			alertAnnotationCount:     3,
+			dashboardAnnotationCount: 3,
+			APIAnnotationCount:       3,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cleaner := &AnnotationCleanupService{batchSize: 1, log: log.New("test-logger")}
+			err := cleaner.CleanAnnotations(context.Background(), test.cfg)
+			require.NoError(t, err)
+
+			assertAnnotationCount(t, fakeSQL, alertAnnotationType, test.alertAnnotationCount)
+			assertAnnotationCount(t, fakeSQL, dashboardAnnotationType, test.dashboardAnnotationCount)
+			assertAnnotationCount(t, fakeSQL, apiAnnotationType, test.APIAnnotationCount)
+		})
+	}
+}
+
+func TestOldAnnotationsAreDeletedFirst(t *testing.T) {
+	fakeSQL := InitTestDB(t)
+
+	t.Cleanup(func() {
+		_ = fakeSQL.WithDbSession(context.Background(), func(session *DBSession) error {
+			_, err := session.Exec("DELETE FROM annotation")
+			require.Nil(t, err, "cleaning up all annotations should not cause problems")
+			return err
+		})
+	})
+
+	//create some test annotations
+	a := annotations.Item{
+		DashboardId: 1,
+		OrgId:       1,
+		UserId:      1,
+		PanelId:     1,
+		AlertId:     10,
+		Text:        "",
+		Created:     time.Now().AddDate(-10, 0, -10).UnixNano() / int64(time.Millisecond),
+	}
+
+	session := fakeSQL.NewSession()
+	defer session.Close()
+
+	_, err := session.Insert(a)
+	require.NoError(t, err, "cannot insert annotation")
+	_, err = session.Insert(a)
+	require.NoError(t, err, "cannot insert annotation")
+
+	a.AlertId = 20
+	_, err = session.Insert(a)
+	require.NoError(t, err, "cannot insert annotation")
+
+	// run the clean up task to keep one annotation.
+	cleaner := &AnnotationCleanupService{batchSize: 1, log: log.New("test-logger")}
+	err = cleaner.cleanAnnotations(context.Background(), setting.AnnotationCleanupSettings{MaxCount: 1}, alertAnnotationType)
+	require.NoError(t, err)
+
+	// assert that the last annotations were kept
+	countNew, err := session.Where("alert_id = 20").Count(&annotations.Item{})
+	require.NoError(t, err)
+	require.Equal(t, int64(1), countNew, "the last annotations should be kept")
+
+	countOld, err := session.Where("alert_id = 10").Count(&annotations.Item{})
+	require.NoError(t, err)
+	require.Equal(t, int64(0), countOld, "the two first annotations should have been deleted.")
+}
+
+func assertAnnotationCount(t *testing.T, fakeSQL *SqlStore, sql string, expectedCount int64) {
+	t.Helper()
+
+	session := fakeSQL.NewSession()
+	defer session.Close()
+	count, err := session.Where(sql).Count(&annotations.Item{})
+	require.NoError(t, err)
+	require.Equal(t, expectedCount, count)
+}
+
+func createTestAnnotations(t *testing.T, sqlstore *SqlStore, expectedCount int, oldAnnotations int) {
+	t.Helper()
+
+	cutoffDate := time.Now()
+
+	for i := 0; i < expectedCount; i++ {
+		a := &annotations.Item{
+			DashboardId: 1,
+			OrgId:       1,
+			UserId:      1,
+			PanelId:     1,
+			Text:        "",
+		}
+
+		// mark every third as an API annotation
+		// that doesnt belong to a dashboard
+		if i%3 == 1 {
+			a.DashboardId = 0
+		}
+
+		// mark every third annotation as an alert annotation
+		if i%3 == 0 {
+			a.AlertId = 10
+			a.DashboardId = 2
+		}
+
+		// create epoch as int annotations.go line 40
+		a.Created = cutoffDate.UnixNano() / int64(time.Millisecond)
+
+		// set a really old date for the first six annotations
+		if i < oldAnnotations {
+			a.Created = cutoffDate.AddDate(-10, 0, -10).UnixNano() / int64(time.Millisecond)
+		}
+
+		_, err := sqlstore.NewSession().Insert(a)
+		require.NoError(t, err, "should be able to save annotation", err)
+	}
+}
+
+func settingsFn(maxAge time.Duration, maxCount int64) setting.AnnotationCleanupSettings {
+	return setting.AnnotationCleanupSettings{MaxAge: maxAge, MaxCount: maxCount}
+}

pkg/services/sqlstore/sqlstore.go

@@ -96,6 +96,7 @@ func (ss *SqlStore) Init() error {
 
 	// Init repo instances
 	annotations.SetRepository(&SqlAnnotationRepo{})
+	annotations.SetAnnotationCleaner(&AnnotationCleanupService{batchSize: 100, log: log.New("annotationcleaner")})
 	ss.Bus.SetTransactionManager(ss)
 
 	// Register handlers

pkg/services/sqlstore/sqlutil/sqlutil.go

@@ -11,6 +11,7 @@ type TestDB struct {
 }
 
 func Sqlite3TestDB() TestDB {
+	// To run all tests in a local test database, set ConnStr to "grafana_test.db"
 	return TestDB{
 		DriverName: "sqlite3",
 		ConnStr:    ":memory:",

pkg/setting/setting.go

@@ -19,6 +19,7 @@ import (
 	"github.com/go-macaron/session"
 	ini "gopkg.in/ini.v1"
 
+	"github.com/grafana/grafana/pkg/components/gtime"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/util"
 )
@@ -302,6 +303,11 @@ type Cfg struct {
 	FeatureToggles map[string]bool
 
 	AnonymousHideVersion bool
+
+	// Annotations
+	AlertingAnnotationCleanupSetting   AnnotationCleanupSettings
+	DashboardAnnotationCleanupSettings AnnotationCleanupSettings
+	APIAnnotationCleanupSettings       AnnotationCleanupSettings
 }
 
 // IsExpressionsEnabled returns whether the expressions feature is enabled.
@@ -396,6 +402,33 @@ func applyEnvVariableOverrides(file *ini.File) error {
 	return nil
 }
 
+func (cfg *Cfg) readAnnotationSettings() {
+	dashboardAnnotation := cfg.Raw.Section("annotations.dashboard")
+	apiIAnnotation := cfg.Raw.Section("annotations.api")
+	alertingSection := cfg.Raw.Section("alerting")
+
+	var newAnnotationCleanupSettings = func(section *ini.Section, maxAgeField string) AnnotationCleanupSettings {
+		maxAge, err := gtime.ParseInterval(section.Key(maxAgeField).MustString(""))
+		if err != nil {
+			maxAge = 0
+		}
+
+		return AnnotationCleanupSettings{
+			MaxAge:   maxAge,
+			MaxCount: section.Key("max_annotations_to_keep").MustInt64(0),
+		}
+	}
+
+	cfg.AlertingAnnotationCleanupSetting = newAnnotationCleanupSettings(alertingSection, "max_annotation_age")
+	cfg.DashboardAnnotationCleanupSettings = newAnnotationCleanupSettings(dashboardAnnotation, "max_age")
+	cfg.APIAnnotationCleanupSettings = newAnnotationCleanupSettings(apiIAnnotation, "max_age")
+}
+
+type AnnotationCleanupSettings struct {
+	MaxAge   time.Duration
+	MaxCount int64
+}
+
 func envKey(sectionName string, keyName string) string {
 	sN := strings.ToUpper(strings.Replace(sectionName, ".", "_", -1))
 	sN = strings.Replace(sN, "-", "_", -1)
@@ -758,6 +791,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 	cfg.readSessionConfig()
 	cfg.readSmtpSettings()
 	cfg.readQuotaSettings()
+	cfg.readAnnotationSettings()
 
 	if VerifyEmailEnabled && !cfg.Smtp.Enabled {
 		log.Warnf("require_email_validation is enabled but smtp is disabled")