Alerting: new min_interval_seconds options to enforce a minimum eval frequency (#21188)

* add min_interval_seconds setting to alerting config

It will let operator enforce a minimum time for the scheduler to enqueue evaluations

* Introduce UI modifications

* Update docs

Co-authored-by: Martin <uepoch@users.noreply.github.com>

conf/defaults.ini

@@ -593,6 +593,8 @@ notification_timeout_seconds = 30
 # Default setting for max attempts to sending alert notifications. Default value is 3
 # Default setting for max attempts to sending alert notifications. Default value is 3
 max_attempts = 3
 max_attempts = 3
 
 
+# Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend
+min_interval_seconds = 1
 
 
 #################################### Explore #############################
 #################################### Explore #############################
 [explore]
 [explore]

conf/sample.ini

@@ -584,6 +584,9 @@
 # Default setting for max attempts to sending alert notifications. Default value is 3
 # Default setting for max attempts to sending alert notifications. Default value is 3
 ;max_attempts = 3
 ;max_attempts = 3
 
 
+# Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend
+;min_interval_seconds = 1
+
 #################################### Explore #############################
 #################################### Explore #############################
 [explore]
 [explore]
 # Enable the Explore section
 # Enable the Explore section

docs/sources/alerting/rules.md

@@ -45,6 +45,7 @@ Currently only the graph panel supports alert rules.
 ### Name and Evaluation interval
 ### Name and Evaluation interval
 
 
 Here you can specify the name of the alert rule and how often the scheduler should evaluate the alert rule.
 Here you can specify the name of the alert rule and how often the scheduler should evaluate the alert rule.
+**Note:** You can set a minimum interval in the `alerting.min_interval_seconds` config field, to set a minimum time between evaluations. Check out the [[configuration]]({{< relref "../installation/configuration.md" >}}#min-interval-seconds) page for more information.
 
 
 ### For
 ### For
 
 

docs/sources/installation/configuration.md

@@ -722,6 +722,12 @@ Default setting for alert notification timeout. Default value is `30`
 
 
 Default setting for max attempts to sending alert notifications. Default value is `3`
 Default setting for max attempts to sending alert notifications. Default value is `3`
 
 
+### min_interval_seconds
+
+Default setting for minimum interval between rule evaluations. Default value is `1`
+
+> **Note.** This setting has precedence over each individual rule frequency. Therefore, if a rule frequency is lower than this value, this value will be enforced.
+
 ## [rendering]
 ## [rendering]
 
 
 Options to configure a remote HTTP image rendering service, e.g. using https://github.com/grafana/grafana-image-renderer.
 Options to configure a remote HTTP image rendering service, e.g. using https://github.com/grafana/grafana-image-renderer.

packages/grafana-runtime/src/config.ts

@@ -34,6 +34,7 @@ export class GrafanaBootConfig {
   alertingEnabled = false;
   alertingEnabled = false;
   alertingErrorOrTimeout = '';
   alertingErrorOrTimeout = '';
   alertingNoDataOrNullValues = '';
   alertingNoDataOrNullValues = '';
+  alertingMinInterval = 1;
   authProxyEnabled = false;
   authProxyEnabled = false;
   exploreEnabled = false;
   exploreEnabled = false;
   ldapEnabled = false;
   ldapEnabled = false;

pkg/api/frontendsettings.go

@@ -176,6 +176,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *m.ReqContext) (map[string]interf
 		"alertingEnabled":            setting.AlertingEnabled,
 		"alertingEnabled":            setting.AlertingEnabled,
 		"alertingErrorOrTimeout":     setting.AlertingErrorOrTimeout,
 		"alertingErrorOrTimeout":     setting.AlertingErrorOrTimeout,
 		"alertingNoDataOrNullValues": setting.AlertingNoDataOrNullValues,
 		"alertingNoDataOrNullValues": setting.AlertingNoDataOrNullValues,
+		"alertingMinInterval":        setting.AlertingMinInterval,
 		"exploreEnabled":             setting.ExploreEnabled,
 		"exploreEnabled":             setting.ExploreEnabled,
 		"googleAnalyticsId":          setting.GoogleAnalyticsId,
 		"googleAnalyticsId":          setting.GoogleAnalyticsId,
 		"disableLoginForm":           setting.DisableLoginForm,
 		"disableLoginForm":           setting.DisableLoginForm,

pkg/services/alerting/scheduler.go

@@ -6,6 +6,7 @@ import (
 
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
 )
 )
 
 
 type schedulerImpl struct {
 type schedulerImpl struct {
@@ -61,7 +62,13 @@ func (s *schedulerImpl) Tick(tickTime time.Time, execQueue chan *Job) {
 			continue
 			continue
 		}
 		}
 
 
-		if now%job.Rule.Frequency == 0 {
+		// Check the job frequency against the minimum interval required
+		interval := job.Rule.Frequency
+		if interval < setting.AlertingMinInterval {
+			interval = setting.AlertingMinInterval
+		}
+
+		if now%interval == 0 {
 			if job.Offset > 0 {
 			if job.Offset > 0 {
 				job.OffsetWait = true
 				job.OffsetWait = true
 			} else {
 			} else {

pkg/setting/setting.go

@@ -201,6 +201,7 @@ var (
 	AlertingEvaluationTimeout   time.Duration
 	AlertingEvaluationTimeout   time.Duration
 	AlertingNotificationTimeout time.Duration
 	AlertingNotificationTimeout time.Duration
 	AlertingMaxAttempts         int
 	AlertingMaxAttempts         int
+	AlertingMinInterval         int64
 
 
 	// Explore UI
 	// Explore UI
 	ExploreEnabled bool
 	ExploreEnabled bool
@@ -961,6 +962,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 	notificationTimeoutSeconds := alerting.Key("notification_timeout_seconds").MustInt64(30)
 	notificationTimeoutSeconds := alerting.Key("notification_timeout_seconds").MustInt64(30)
 	AlertingNotificationTimeout = time.Second * time.Duration(notificationTimeoutSeconds)
 	AlertingNotificationTimeout = time.Second * time.Duration(notificationTimeoutSeconds)
 	AlertingMaxAttempts = alerting.Key("max_attempts").MustInt(3)
 	AlertingMaxAttempts = alerting.Key("max_attempts").MustInt(3)
+	AlertingMinInterval = alerting.Key("min_interval_seconds").MustInt64(1)
 
 
 	explore := iniFile.Section("explore")
 	explore := iniFile.Section("explore")
 	ExploreEnabled = explore.Key("enabled").MustBool(true)
 	ExploreEnabled = explore.Key("enabled").MustBool(true)

public/app/features/alerting/AlertTabCtrl.ts

@@ -12,6 +12,7 @@ import { DataQuery, DataSourceApi } from '@grafana/data';
 import { PanelModel } from 'app/features/dashboard/state';
 import { PanelModel } from 'app/features/dashboard/state';
 import { getDefaultCondition } from './getAlertingValidationMessage';
 import { getDefaultCondition } from './getAlertingValidationMessage';
 import { CoreEvents } from 'app/types';
 import { CoreEvents } from 'app/types';
+import kbn from 'app/core/utils/kbn';
 
 
 export class AlertTabCtrl {
 export class AlertTabCtrl {
   panel: PanelModel;
   panel: PanelModel;
@@ -31,6 +32,9 @@ export class AlertTabCtrl {
   appSubUrl: string;
   appSubUrl: string;
   alertHistory: any;
   alertHistory: any;
   newAlertRuleTag: any;
   newAlertRuleTag: any;
+  alertingMinIntervalSecs: number;
+  alertingMinInterval: string;
+  frequencyWarning: any;
 
 
   /** @ngInject */
   /** @ngInject */
   constructor(
   constructor(
@@ -51,6 +55,8 @@ export class AlertTabCtrl {
     this.executionErrorModes = alertDef.executionErrorModes;
     this.executionErrorModes = alertDef.executionErrorModes;
     this.appSubUrl = config.appSubUrl;
     this.appSubUrl = config.appSubUrl;
     this.panelCtrl._enableAlert = this.enable;
     this.panelCtrl._enableAlert = this.enable;
+    this.alertingMinIntervalSecs = config.alertingMinInterval;
+    this.alertingMinInterval = kbn.secondsToHms(config.alertingMinInterval);
   }
   }
 
 
   $onInit() {
   $onInit() {
@@ -178,6 +184,8 @@ export class AlertTabCtrl {
       return;
       return;
     }
     }
 
 
+    this.checkFrequency();
+
     alert.conditions = alert.conditions || [];
     alert.conditions = alert.conditions || [];
     if (alert.conditions.length === 0) {
     if (alert.conditions.length === 0) {
       alert.conditions.push(getDefaultCondition());
       alert.conditions.push(getDefaultCondition());
@@ -232,6 +240,23 @@ export class AlertTabCtrl {
     this.panelCtrl.render();
     this.panelCtrl.render();
   }
   }
 
 
+  checkFrequency() {
+    this.frequencyWarning = '';
+
+    try {
+      const frequencySecs = kbn.interval_to_seconds(this.alert.frequency);
+      if (frequencySecs < this.alertingMinIntervalSecs) {
+        this.frequencyWarning =
+          'A minimum evaluation interval of ' +
+          this.alertingMinInterval +
+          ' have been configured in Grafana and will be used for this alert rule. ' +
+          'Please contact the administrator to configure a lower interval.';
+      }
+    } catch (err) {
+      this.frequencyWarning = err;
+    }
+  }
+
   graphThresholdChanged(evt: any) {
   graphThresholdChanged(evt: any) {
     for (const condition of this.alert.conditions) {
     for (const condition of this.alert.conditions) {
       if (condition.type === 'query') {
       if (condition.type === 'query') {

public/app/features/alerting/partials/alert_tab.html

@@ -13,7 +13,7 @@
           </div>
           </div>
           <div class="gf-form">
           <div class="gf-form">
             <span class="gf-form-label width-9">Evaluate every</span>
             <span class="gf-form-label width-9">Evaluate every</span>
-            <input class="gf-form-input max-width-6" type="text" ng-model="ctrl.alert.frequency">
+            <input class="gf-form-input max-width-6" type="text" ng-model="ctrl.alert.frequency" ng-blur="ctrl.checkFrequency()">
           </div>
           </div>
           <div class="gf-form max-width-11">
           <div class="gf-form max-width-11">
             <label class="gf-form-label width-5">For</label>
             <label class="gf-form-label width-5">For</label>
@@ -31,6 +31,12 @@
             </info-popover>
             </info-popover>
           </div>
           </div>
         </div>
         </div>
+        <div class="gf-form" ng-if="ctrl.frequencyWarning">
+          <label class="gf-form-label text-warning">
+            <i
+            class="fa fa-warning"></i> {{ctrl.frequencyWarning}}
+          </label>
+        </div>
       </div>
       </div>
 
 
       <div class="gf-form-group">
       <div class="gf-form-group">