Skip to content

N
nexpie-grafana-theme
Commit 93772649 by bergquist
Options

feat(alerting): add global usage metrics for alerting 

closes #5786
parent ae15de11
Showing with 66 additions and 24 deletions
pkg/metrics/metrics.go
......@@ -9,29 +9,38 @@ func init() {
}
var (
M_Instance_Start Counter
M_Page_Status_200 Counter
M_Page_Status_500 Counter
M_Page_Status_404 Counter
M_Api_Status_500 Counter
M_Api_Status_404 Counter
M_Api_User_SignUpStarted Counter
M_Api_User_SignUpCompleted Counter
M_Api_User_SignUpInvite Counter
M_Api_Dashboard_Save Timer
M_Api_Dashboard_Get Timer
M_Api_Dashboard_Search Timer
M_Api_Admin_User_Create Counter
M_Api_Login_Post Counter
M_Api_Login_OAuth Counter
M_Api_Org_Create Counter
M_Api_Dashboard_Snapshot_Create Counter
M_Api_Dashboard_Snapshot_External Counter
M_Api_Dashboard_Snapshot_Get Counter
M_Models_Dashboard_Insert Counter
M_Instance_Start Counter
M_Page_Status_200 Counter
M_Page_Status_500 Counter
M_Page_Status_404 Counter
M_Api_Status_500 Counter
M_Api_Status_404 Counter
M_Api_User_SignUpStarted Counter
M_Api_User_SignUpCompleted Counter
M_Api_User_SignUpInvite Counter
M_Api_Dashboard_Save Timer
M_Api_Dashboard_Get Timer
M_Api_Dashboard_Search Timer
M_Api_Admin_User_Create Counter
M_Api_Login_Post Counter
M_Api_Login_OAuth Counter
M_Api_Org_Create Counter
M_Api_Dashboard_Snapshot_Create Counter
M_Api_Dashboard_Snapshot_External Counter
M_Api_Dashboard_Snapshot_Get Counter
M_Models_Dashboard_Insert Counter
M_Alerting_Result_Critical Counter
M_Alerting_Result_Warning Counter
M_Alerting_Result_Info Counter
M_Alerting_Result_Ok Counter
M_Alerting_Active_Alerts Counter
M_Alerting_Notification_Sent_Slack Counter
M_Alerting_Notification_Sent_Email Counter
M_Alerting_Notification_Sent_Webhook Counter
// Timers
M_DataSource_ProxyReq_Timer Timer
M_Alerting_Exeuction_Time Timer
)
func initMetricVars(settings *MetricSettings) {
......@@ -66,6 +75,16 @@ func initMetricVars(settings *MetricSettings) {
M_Models_Dashboard_Insert = RegCounter("models.dashboard.insert")
M_Alerting_Result_Critical = RegCounter("alerting.result", "severity", "critical")
M_Alerting_Result_Warning = RegCounter("alerting.result", "severity", "warning")
M_Alerting_Result_Info = RegCounter("alerting.result", "severity", "info")
M_Alerting_Result_Ok = RegCounter("alerting.result", "severity", "ok")
M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts")
M_Alerting_Notification_Sent_Slack = RegCounter("alerting.notifcations_sent", "type", "slack")
M_Alerting_Notification_Sent_Email = RegCounter("alerting.notifcations_sent", "type", "email")
M_Alerting_Notification_Sent_Webhook = RegCounter("alerting.notifcations_sent", "type", "webhook")
// Timers
M_DataSource_ProxyReq_Timer = RegTimer("api.dataproxy.request.all")
M_Alerting_Exeuction_Time = RegTimer("alerting.execution_time")
}
pkg/services/alerting/eval_handler.go
......@@ -5,10 +5,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/log"
)
var (
descriptionFmt = "Actual value: %1.2f for %s. "
"github.com/grafana/grafana/pkg/metrics"
)
type DefaultEvalHandler struct {
......@@ -55,5 +52,7 @@ func (e *DefaultEvalHandler) eval(context *EvalContext) {
}
context.EndTime = time.Now()
elapsedTime := context.EndTime.Sub(context.StartTime)
metrics.M_Alerting_Exeuction_Time.Update(elapsedTime)
context.DoneChan <- true
}
pkg/services/alerting/notifier.go
......@@ -48,6 +48,7 @@ func (n *RootNotifier) Notify(context *EvalContext) {
for _, notifier := range notifiers {
n.log.Info("Sending notification", "firing", context.Firing, "type", notifier.GetType())
go notifier.Notify(context)
}
}
......
pkg/services/alerting/notifiers/email.go
......@@ -5,6 +5,7 @@ import (
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
)
......@@ -38,6 +39,7 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
func (this *EmailNotifier) Notify(context *alerting.EvalContext) {
this.log.Info("Sending alert notification to", "addresses", this.Addresses)
metrics.M_Alerting_Notification_Sent_Email.Inc(1)
ruleUrl, err := context.GetRuleUrl()
if err != nil {
......
pkg/services/alerting/notifiers/slack.go
......@@ -6,6 +6,7 @@ import (
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
)
......@@ -38,6 +39,7 @@ type SlackNotifier struct {
func (this *SlackNotifier) Notify(context *alerting.EvalContext) {
this.log.Info("Executing slack notification", "ruleId", context.Rule.Id, "notification", this.Name)
metrics.M_Alerting_Notification_Sent_Slack.Inc(1)
ruleUrl, err := context.GetRuleUrl()
if err != nil {
......
pkg/services/alerting/notifiers/webhook.go
......@@ -4,6 +4,7 @@ import (
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
)
......@@ -40,6 +41,7 @@ type WebhookNotifier struct {
func (this *WebhookNotifier) Notify(context *alerting.EvalContext) {
this.log.Info("Sending webhook")
metrics.M_Alerting_Notification_Sent_Webhook.Inc(1)
bodyJSON := simplejson.New()
bodyJSON.Set("title", context.GetNotificationTitle())
......
pkg/services/alerting/reader.go
......@@ -6,6 +6,7 @@ import (
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics"
m "github.com/grafana/grafana/pkg/models"
)
......@@ -58,6 +59,7 @@ func (arr *DefaultRuleReader) Fetch() []*Rule {
}
}
metrics.M_Alerting_Active_Alerts.Inc(int64(len(res)))
return res
}
......
pkg/services/alerting/result_handler.go
......@@ -5,6 +5,7 @@ import (
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/annotations"
)
......@@ -37,6 +38,7 @@ func (handler *DefaultResultHandler) Handle(ctx *EvalContext) {
ctx.Rule.State = m.AlertStateOK
}
countSeverity(ctx.Rule.Severity)
if ctx.Rule.State != oldState {
handler.log.Info("New state change", "alertId", ctx.Rule.Id, "newState", ctx.Rule.State, "oldState", oldState)
......@@ -69,3 +71,16 @@ func (handler *DefaultResultHandler) Handle(ctx *EvalContext) {
handler.notifier.Notify(ctx)
}
}
func countSeverity(state m.AlertSeverityType) {
switch state {
case m.AlertSeverityOK:
metrics.M_Alerting_Result_Ok.Inc(1)
case m.AlertSeverityInfo:
metrics.M_Alerting_Result_Info.Inc(1)
case m.AlertSeverityWarning:
metrics.M_Alerting_Result_Warning.Inc(1)
case m.AlertSeverityCritical:
metrics.M_Alerting_Result_Critical.Inc(1)
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment