Alerting: override PagerDuty 'dedup_key' via tags (#27356)

* Alerting: override PagerDuty 'dedup_key' via tags

Added ability to override PagerDuty 'dedup_key' via Alert tags

Updated unit tests

* Updated documentation

* add comma

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

* add space

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

docs/sources/alerting/notifications.md

@@ -125,13 +125,14 @@ Severity | Level for dynamic notifications, default is `critical` (1)
 Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok
 Message in details | Removes the Alert message from the PD summary field and puts it into custom details instead (2)
 
->**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level.
+>**Note:** The tags `Severity`, `Class`, `Group`, `dedup_key`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level.
 
 >Using Message In Details will change the structure of the `custom_details` field in the PagerDuty Event.
 This might break custom event rules in your PagerDuty rules if you rely on the fields in `payload.custom_details`.
 Move any existing rules using `custom_details.myMetric` to `custom_details.queries.myMetric`.
 This behavior will become the default in a future version of Grafana.
 
+> Using `dedup_key` tag will override grafana generated `dedup_key` with a custom key.
 ### Webhook
 
 The webhook notification is a simple way to send information about a state change over HTTP to a custom endpoint.

pkg/services/alerting/notifiers/pagerduty.go

@@ -171,10 +171,9 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
 	// set default, override in following case switch if defined
 	payloadJSON.Set("component", "Grafana")
 	payloadJSON.Set("severity", pn.Severity)
+	dedupKey := "alertId-" + strconv.FormatInt(evalContext.Rule.ID, 10)
 
 	for _, tag := range evalContext.Rule.AlertRuleTags {
-		customData.Set(tag.Key, tag.Value)
-
 		// Override tags appropriately if they are in the PagerDuty v2 API
 		switch strings.ToLower(tag.Key) {
 		case "group":
@@ -183,6 +182,11 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
 			payloadJSON.Set("class", tag.Value)
 		case "component":
 			payloadJSON.Set("component", tag.Value)
+		case "dedup_key":
+			if len(tag.Value) > 254 {
+				tag.Value = tag.Value[0:254]
+			}
+			dedupKey = tag.Value
 		case "severity":
 			// Only set severity if it's one of the PD supported enum values
 			// Info, Warning, Error, or Critical (case insensitive)
@@ -199,6 +203,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
 				pn.log.Warn("Ignoring invalid severity tag", "severity", sev)
 			}
 		}
+		customData.Set(tag.Key, tag.Value)
 	}
 
 	var summary string
@@ -220,7 +225,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
 	bodyJSON := simplejson.New()
 	bodyJSON.Set("routing_key", pn.Key)
 	bodyJSON.Set("event_action", eventType)
-	bodyJSON.Set("dedup_key", "alertId-"+strconv.FormatInt(evalContext.Rule.ID, 10))
+	bodyJSON.Set("dedup_key", dedupKey)
 	bodyJSON.Set("payload", payloadJSON)
 
 	ruleURL, err := evalContext.GetRuleURL()

pkg/services/alerting/notifiers/pagerduty_test.go

@@ -2,6 +2,7 @@ package notifiers
 
 import (
 	"context"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -269,6 +270,7 @@ func TestPagerdutyNotifier(t *testing.T) {
 						{Key: "class", Value: "aClass"},
 						{Key: "component", Value: "aComponent"},
 						{Key: "severity", Value: "warning"},
+						{Key: "dedup_key", Value: "key-" + strings.Repeat("x", 260)},
 					},
 				})
 				evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
@@ -282,7 +284,7 @@ func TestPagerdutyNotifier(t *testing.T) {
 				diff := cmp.Diff(map[string]interface{}{
 					"client":       "Grafana",
 					"client_url":   "",
-					"dedup_key":    "alertId-0",
+					"dedup_key":    "key-" + strings.Repeat("x", 250),
 					"event_action": "trigger",
 					"links": []interface{}{
 						map[string]interface{}{
@@ -297,6 +299,7 @@ func TestPagerdutyNotifier(t *testing.T) {
 							"class":     "aClass",
 							"component": "aComponent",
 							"severity":  "warning",
+							"dedup_key": "key-" + strings.Repeat("x", 250),
 							"keyOnly":   "",
 						},
 						"severity":  "warning",