Skip to content

N
nexpie-grafana-theme
Commit 8253b9dd by Andrei Lavrov Committed by GitHub
Options

Alerting: override PagerDuty 'dedup_key' via tags (#27356) 

* Alerting: override PagerDuty 'dedup_key' via tags

Added ability to override PagerDuty 'dedup_key' via Alert tags

Updated unit tests

* Updated documentation

* add comma

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

* add space

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>
parent e71eb133
Showing with 14 additions and 5 deletions
docs/sources/alerting/notifications.md
...@@ -125,13 +125,14 @@ Severity | Level for dynamic notifications, default is `critical` (1) ...@@ -125,13 +125,14 @@ Severity | Level for dynamic notifications, default is `critical` (1)
Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok
Message in details | Removes the Alert message from the PD summary field and puts it into custom details instead (2) Message in details | Removes the Alert message from the PD summary field and puts it into custom details instead (2)
>**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level. >**Note:** The tags `Severity`, `Class`, `Group`, `dedup_key`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level.
>Using Message In Details will change the structure of the `custom_details` field in the PagerDuty Event. >Using Message In Details will change the structure of the `custom_details` field in the PagerDuty Event.
This might break custom event rules in your PagerDuty rules if you rely on the fields in `payload.custom_details`. This might break custom event rules in your PagerDuty rules if you rely on the fields in `payload.custom_details`.
Move any existing rules using `custom_details.myMetric` to `custom_details.queries.myMetric`. Move any existing rules using `custom_details.myMetric` to `custom_details.queries.myMetric`.
This behavior will become the default in a future version of Grafana. This behavior will become the default in a future version of Grafana.
> Using `dedup_key` tag will override grafana generated `dedup_key` with a custom key.
### Webhook ### Webhook
The webhook notification is a simple way to send information about a state change over HTTP to a custom endpoint. The webhook notification is a simple way to send information about a state change over HTTP to a custom endpoint.
......
pkg/services/alerting/notifiers/pagerduty.go
...@@ -171,10 +171,9 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext ...@@ -171,10 +171,9 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
// set default, override in following case switch if defined // set default, override in following case switch if defined
payloadJSON.Set("component", "Grafana") payloadJSON.Set("component", "Grafana")
payloadJSON.Set("severity", pn.Severity) payloadJSON.Set("severity", pn.Severity)
dedupKey := "alertId-" + strconv.FormatInt(evalContext.Rule.ID, 10)
for _, tag := range evalContext.Rule.AlertRuleTags { for _, tag := range evalContext.Rule.AlertRuleTags {
customData.Set(tag.Key, tag.Value)
// Override tags appropriately if they are in the PagerDuty v2 API // Override tags appropriately if they are in the PagerDuty v2 API
switch strings.ToLower(tag.Key) { switch strings.ToLower(tag.Key) {
case "group": case "group":
...@@ -183,6 +182,11 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext ...@@ -183,6 +182,11 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
payloadJSON.Set("class", tag.Value) payloadJSON.Set("class", tag.Value)
case "component": case "component":
payloadJSON.Set("component", tag.Value) payloadJSON.Set("component", tag.Value)
case "dedup_key":
if len(tag.Value) > 254 {
tag.Value = tag.Value[0:254]
}
dedupKey = tag.Value
case "severity": case "severity":
// Only set severity if it's one of the PD supported enum values // Only set severity if it's one of the PD supported enum values
// Info, Warning, Error, or Critical (case insensitive) // Info, Warning, Error, or Critical (case insensitive)
...@@ -199,6 +203,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext ...@@ -199,6 +203,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
pn.log.Warn("Ignoring invalid severity tag", "severity", sev) pn.log.Warn("Ignoring invalid severity tag", "severity", sev)
} }
} }
customData.Set(tag.Key, tag.Value)
} }
var summary string var summary string
...@@ -220,7 +225,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext ...@@ -220,7 +225,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
bodyJSON := simplejson.New() bodyJSON := simplejson.New()
bodyJSON.Set("routing_key", pn.Key) bodyJSON.Set("routing_key", pn.Key)
bodyJSON.Set("event_action", eventType) bodyJSON.Set("event_action", eventType)
bodyJSON.Set("dedup_key", "alertId-"+strconv.FormatInt(evalContext.Rule.ID, 10)) bodyJSON.Set("dedup_key", dedupKey)
bodyJSON.Set("payload", payloadJSON) bodyJSON.Set("payload", payloadJSON)
ruleURL, err := evalContext.GetRuleURL() ruleURL, err := evalContext.GetRuleURL()
......
pkg/services/alerting/notifiers/pagerduty_test.go
...@@ -2,6 +2,7 @@ package notifiers ...@@ -2,6 +2,7 @@ package notifiers
import ( import (
"context" "context"
"strings"
"testing" "testing"
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
...@@ -269,6 +270,7 @@ func TestPagerdutyNotifier(t *testing.T) { ...@@ -269,6 +270,7 @@ func TestPagerdutyNotifier(t *testing.T) {
{Key: "class", Value: "aClass"}, {Key: "class", Value: "aClass"},
{Key: "component", Value: "aComponent"}, {Key: "component", Value: "aComponent"},
{Key: "severity", Value: "warning"}, {Key: "severity", Value: "warning"},
{Key: "dedup_key", Value: "key-" + strings.Repeat("x", 260)},
}, },
}) })
evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png" evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
...@@ -282,7 +284,7 @@ func TestPagerdutyNotifier(t *testing.T) { ...@@ -282,7 +284,7 @@ func TestPagerdutyNotifier(t *testing.T) {
diff := cmp.Diff(map[string]interface{}{ diff := cmp.Diff(map[string]interface{}{
"client": "Grafana", "client": "Grafana",
"client_url": "", "client_url": "",
"dedup_key": "alertId-0", "dedup_key": "key-" + strings.Repeat("x", 250),
"event_action": "trigger", "event_action": "trigger",
"links": []interface{}{ "links": []interface{}{
map[string]interface{}{ map[string]interface{}{
...@@ -297,6 +299,7 @@ func TestPagerdutyNotifier(t *testing.T) { ...@@ -297,6 +299,7 @@ func TestPagerdutyNotifier(t *testing.T) {
"class": "aClass", "class": "aClass",
"component": "aComponent", "component": "aComponent",
"severity": "warning", "severity": "warning",
"dedup_key": "key-" + strings.Repeat("x", 250),
"keyOnly": "", "keyOnly": "",
}, },
"severity": "warning", "severity": "warning",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment