Skip to content

N
nexpie-grafana-theme
Commit 93772649 by bergquist
Options

feat(alerting): add global usage metrics for alerting 

closes #5786
parent ae15de11
Showing with 46 additions and 4 deletions
pkg/metrics/metrics.go
...@@ -29,9 +29,18 @@ var ( ...@@ -29,9 +29,18 @@ var (
M_Api_Dashboard_Snapshot_External Counter M_Api_Dashboard_Snapshot_External Counter
M_Api_Dashboard_Snapshot_Get Counter M_Api_Dashboard_Snapshot_Get Counter
M_Models_Dashboard_Insert Counter M_Models_Dashboard_Insert Counter
M_Alerting_Result_Critical Counter
M_Alerting_Result_Warning Counter
M_Alerting_Result_Info Counter
M_Alerting_Result_Ok Counter
M_Alerting_Active_Alerts Counter
M_Alerting_Notification_Sent_Slack Counter
M_Alerting_Notification_Sent_Email Counter
M_Alerting_Notification_Sent_Webhook Counter
// Timers // Timers
M_DataSource_ProxyReq_Timer Timer M_DataSource_ProxyReq_Timer Timer
M_Alerting_Exeuction_Time Timer
) )
func initMetricVars(settings *MetricSettings) { func initMetricVars(settings *MetricSettings) {
...@@ -66,6 +75,16 @@ func initMetricVars(settings *MetricSettings) { ...@@ -66,6 +75,16 @@ func initMetricVars(settings *MetricSettings) {
M_Models_Dashboard_Insert = RegCounter("models.dashboard.insert") M_Models_Dashboard_Insert = RegCounter("models.dashboard.insert")
M_Alerting_Result_Critical = RegCounter("alerting.result", "severity", "critical")
M_Alerting_Result_Warning = RegCounter("alerting.result", "severity", "warning")
M_Alerting_Result_Info = RegCounter("alerting.result", "severity", "info")
M_Alerting_Result_Ok = RegCounter("alerting.result", "severity", "ok")
M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts")
M_Alerting_Notification_Sent_Slack = RegCounter("alerting.notifcations_sent", "type", "slack")
M_Alerting_Notification_Sent_Email = RegCounter("alerting.notifcations_sent", "type", "email")
M_Alerting_Notification_Sent_Webhook = RegCounter("alerting.notifcations_sent", "type", "webhook")
// Timers // Timers
M_DataSource_ProxyReq_Timer = RegTimer("api.dataproxy.request.all") M_DataSource_ProxyReq_Timer = RegTimer("api.dataproxy.request.all")
M_Alerting_Exeuction_Time = RegTimer("alerting.execution_time")
} }
pkg/services/alerting/eval_handler.go
...@@ -5,10 +5,7 @@ import ( ...@@ -5,10 +5,7 @@ import (
"time" "time"
"github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/log"
) "github.com/grafana/grafana/pkg/metrics"
var (
descriptionFmt = "Actual value: %1.2f for %s. "
) )
type DefaultEvalHandler struct { type DefaultEvalHandler struct {
...@@ -55,5 +52,7 @@ func (e *DefaultEvalHandler) eval(context *EvalContext) { ...@@ -55,5 +52,7 @@ func (e *DefaultEvalHandler) eval(context *EvalContext) {
} }
context.EndTime = time.Now() context.EndTime = time.Now()
elapsedTime := context.EndTime.Sub(context.StartTime)
metrics.M_Alerting_Exeuction_Time.Update(elapsedTime)
context.DoneChan <- true context.DoneChan <- true
} }
pkg/services/alerting/notifier.go
...@@ -48,6 +48,7 @@ func (n *RootNotifier) Notify(context *EvalContext) { ...@@ -48,6 +48,7 @@ func (n *RootNotifier) Notify(context *EvalContext) {
for _, notifier := range notifiers { for _, notifier := range notifiers {
n.log.Info("Sending notification", "firing", context.Firing, "type", notifier.GetType()) n.log.Info("Sending notification", "firing", context.Firing, "type", notifier.GetType())
go notifier.Notify(context) go notifier.Notify(context)
} }
} }
......
pkg/services/alerting/notifiers/email.go
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics"
m "github.com/grafana/grafana/pkg/models" m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/alerting"
) )
...@@ -38,6 +39,7 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) { ...@@ -38,6 +39,7 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
func (this *EmailNotifier) Notify(context *alerting.EvalContext) { func (this *EmailNotifier) Notify(context *alerting.EvalContext) {
this.log.Info("Sending alert notification to", "addresses", this.Addresses) this.log.Info("Sending alert notification to", "addresses", this.Addresses)
metrics.M_Alerting_Notification_Sent_Email.Inc(1)
ruleUrl, err := context.GetRuleUrl() ruleUrl, err := context.GetRuleUrl()
if err != nil { if err != nil {
......
pkg/services/alerting/notifiers/slack.go
...@@ -6,6 +6,7 @@ import ( ...@@ -6,6 +6,7 @@ import (
"github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics"
m "github.com/grafana/grafana/pkg/models" m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/alerting"
) )
...@@ -38,6 +39,7 @@ type SlackNotifier struct { ...@@ -38,6 +39,7 @@ type SlackNotifier struct {
func (this *SlackNotifier) Notify(context *alerting.EvalContext) { func (this *SlackNotifier) Notify(context *alerting.EvalContext) {
this.log.Info("Executing slack notification", "ruleId", context.Rule.Id, "notification", this.Name) this.log.Info("Executing slack notification", "ruleId", context.Rule.Id, "notification", this.Name)
metrics.M_Alerting_Notification_Sent_Slack.Inc(1)
ruleUrl, err := context.GetRuleUrl() ruleUrl, err := context.GetRuleUrl()
if err != nil { if err != nil {
......
pkg/services/alerting/notifiers/webhook.go
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
"github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics"
m "github.com/grafana/grafana/pkg/models" m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting" "github.com/grafana/grafana/pkg/services/alerting"
) )
...@@ -40,6 +41,7 @@ type WebhookNotifier struct { ...@@ -40,6 +41,7 @@ type WebhookNotifier struct {
func (this *WebhookNotifier) Notify(context *alerting.EvalContext) { func (this *WebhookNotifier) Notify(context *alerting.EvalContext) {
this.log.Info("Sending webhook") this.log.Info("Sending webhook")
metrics.M_Alerting_Notification_Sent_Webhook.Inc(1)
bodyJSON := simplejson.New() bodyJSON := simplejson.New()
bodyJSON.Set("title", context.GetNotificationTitle()) bodyJSON.Set("title", context.GetNotificationTitle())
......
pkg/services/alerting/reader.go
...@@ -6,6 +6,7 @@ import ( ...@@ -6,6 +6,7 @@ import (
"github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics"
m "github.com/grafana/grafana/pkg/models" m "github.com/grafana/grafana/pkg/models"
) )
...@@ -58,6 +59,7 @@ func (arr *DefaultRuleReader) Fetch() []*Rule { ...@@ -58,6 +59,7 @@ func (arr *DefaultRuleReader) Fetch() []*Rule {
} }
} }
metrics.M_Alerting_Active_Alerts.Inc(int64(len(res)))
return res return res
} }
......
pkg/services/alerting/result_handler.go
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics"
m "github.com/grafana/grafana/pkg/models" m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/annotations" "github.com/grafana/grafana/pkg/services/annotations"
) )
...@@ -37,6 +38,7 @@ func (handler *DefaultResultHandler) Handle(ctx *EvalContext) { ...@@ -37,6 +38,7 @@ func (handler *DefaultResultHandler) Handle(ctx *EvalContext) {
ctx.Rule.State = m.AlertStateOK ctx.Rule.State = m.AlertStateOK
} }
countSeverity(ctx.Rule.Severity)
if ctx.Rule.State != oldState { if ctx.Rule.State != oldState {
handler.log.Info("New state change", "alertId", ctx.Rule.Id, "newState", ctx.Rule.State, "oldState", oldState) handler.log.Info("New state change", "alertId", ctx.Rule.Id, "newState", ctx.Rule.State, "oldState", oldState)
...@@ -69,3 +71,16 @@ func (handler *DefaultResultHandler) Handle(ctx *EvalContext) { ...@@ -69,3 +71,16 @@ func (handler *DefaultResultHandler) Handle(ctx *EvalContext) {
handler.notifier.Notify(ctx) handler.notifier.Notify(ctx)
} }
} }
func countSeverity(state m.AlertSeverityType) {
switch state {
case m.AlertSeverityOK:
metrics.M_Alerting_Result_Ok.Inc(1)
case m.AlertSeverityInfo:
metrics.M_Alerting_Result_Info.Inc(1)
case m.AlertSeverityWarning:
metrics.M_Alerting_Result_Warning.Inc(1)
case m.AlertSeverityCritical:
metrics.M_Alerting_Result_Critical.Inc(1)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment