Alerting: Enable Alert rule tags to override PagerDuty Severity setting (#22736)

docs/sources/alerting/notifications.md

@@ -110,7 +110,7 @@ Integration Key | Integration key for PagerDuty.
 Severity | Level for dynamic notifications, default is `critical`
 Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok
 
-**Note:** The tags `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef).  If an alert panel defines these tag keys they will be transposed to the root of the event sent to Pagerduty.  This means they will be available within the Pagerduty UI and Filtering tools.
+**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level.
 
 ### Webhook
 

pkg/services/alerting/notifiers/pagerduty.go

@@ -98,6 +98,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
 
 	// set default, override in following case switch if defined
 	payloadJSON.Set("component", "Grafana")
+	payloadJSON.Set("severity", pn.Severity)
 
 	for _, tag := range evalContext.Rule.AlertRuleTags {
 		customData.Set(tag.Key, tag.Value)
@@ -110,6 +111,21 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
 			payloadJSON.Set("class", tag.Value)
 		case "component":
 			payloadJSON.Set("component", tag.Value)
+		case "severity":
+			// Only set severity if it's one of the PD supported enum values
+			// Info, Warning, Error, or Critical (case insensitive)
+			switch sev := strings.ToLower(tag.Value); sev {
+			case "info":
+				fallthrough
+			case "warning":
+				fallthrough
+			case "error":
+				fallthrough
+			case "critical":
+				payloadJSON.Set("severity", sev)
+			default:
+				pn.log.Warn("Ignoring invalid severity tag", "severity", sev)
+			}
 		}
 	}
 
@@ -122,7 +138,6 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
 	if hostname, err := os.Hostname(); err == nil {
 		payloadJSON.Set("source", hostname)
 	}
-	payloadJSON.Set("severity", pn.Severity)
 	payloadJSON.Set("timestamp", time.Now())
 	payloadJSON.Set("custom_details", customData)
 	bodyJSON := simplejson.New()

pkg/services/alerting/notifiers/pagerduty_test.go

@@ -197,6 +197,7 @@ func TestPagerdutyNotifier(t *testing.T) {
 						{Key: "group", Value: "aGroup"},
 						{Key: "class", Value: "aClass"},
 						{Key: "component", Value: "aComponent"},
+						{Key: "severity", Value: "warning"},
 					},
 				})
 				evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
@@ -224,6 +225,162 @@ func TestPagerdutyNotifier(t *testing.T) {
 							"group":     "aGroup",
 							"class":     "aClass",
 							"component": "aComponent",
+							"severity":  "warning",
+							"keyOnly":   "",
+						},
+						"severity":  "warning",
+						"summary":   "someRule - someMessage",
+						"timestamp": "<<PRESENCE>>",
+						"class":     "aClass",
+						"group":     "aGroup",
+					},
+					"images": []interface{}{
+						map[string]interface{}{
+							"src": "http://somewhere.com/omg_dont_panic.png",
+						},
+					},
+					"routing_key": "abcdefgh0123456789",
+				}, payload.Interface(), cmp.Comparer(presenceComparer))
+				So(diff, ShouldBeEmpty)
+			})
+
+			Convey("should support multiple levels of severity", func() {
+				json := `{
+					"integrationKey": "abcdefgh0123456789",
+					"autoResolve": false
+				}`
+
+				settingsJSON, err := simplejson.NewJson([]byte(json))
+				So(err, ShouldBeNil)
+
+				model := &models.AlertNotification{
+					Name:     "pagerduty_testing",
+					Type:     "pagerduty",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewPagerdutyNotifier(model)
+				So(err, ShouldBeNil)
+
+				pagerdutyNotifier := not.(*PagerdutyNotifier)
+
+				evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
+					ID:      0,
+					Name:    "someRule",
+					Message: "someMessage",
+					State:   models.AlertStateAlerting,
+					AlertRuleTags: []*models.Tag{
+						{Key: "keyOnly"},
+						{Key: "group", Value: "aGroup"},
+						{Key: "class", Value: "aClass"},
+						{Key: "component", Value: "aComponent"},
+						{Key: "severity", Value: "info"},
+					},
+				})
+				evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
+				evalContext.IsTestRun = true
+
+				payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext)
+				So(err, ShouldBeNil)
+				payload, err := simplejson.NewJson(payloadJSON)
+				So(err, ShouldBeNil)
+
+				diff := cmp.Diff(map[string]interface{}{
+					"client":       "Grafana",
+					"client_url":   "",
+					"dedup_key":    "alertId-0",
+					"event_action": "trigger",
+					"links": []interface{}{
+						map[string]interface{}{
+							"href": "",
+						},
+					},
+					"payload": map[string]interface{}{
+						"source":    "<<PRESENCE>>",
+						"component": "aComponent",
+						"custom_details": map[string]interface{}{
+							"group":     "aGroup",
+							"class":     "aClass",
+							"component": "aComponent",
+							"severity":  "info",
+							"keyOnly":   "",
+						},
+						"severity":  "info",
+						"summary":   "someRule - someMessage",
+						"timestamp": "<<PRESENCE>>",
+						"class":     "aClass",
+						"group":     "aGroup",
+					},
+					"images": []interface{}{
+						map[string]interface{}{
+							"src": "http://somewhere.com/omg_dont_panic.png",
+						},
+					},
+					"routing_key": "abcdefgh0123456789",
+				}, payload.Interface(), cmp.Comparer(presenceComparer))
+				So(diff, ShouldBeEmpty)
+			})
+
+			Convey("should ignore invalid severity for PD but keep the tag", func() {
+				json := `{
+					"integrationKey": "abcdefgh0123456789",
+					"autoResolve": false,
+					"severity": "critical"
+				}`
+
+				settingsJSON, err := simplejson.NewJson([]byte(json))
+				So(err, ShouldBeNil)
+
+				model := &models.AlertNotification{
+					Name:     "pagerduty_testing",
+					Type:     "pagerduty",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewPagerdutyNotifier(model)
+				So(err, ShouldBeNil)
+
+				pagerdutyNotifier := not.(*PagerdutyNotifier)
+
+				evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
+					ID:      0,
+					Name:    "someRule",
+					Message: "someMessage",
+					State:   models.AlertStateAlerting,
+					AlertRuleTags: []*models.Tag{
+						{Key: "keyOnly"},
+						{Key: "group", Value: "aGroup"},
+						{Key: "class", Value: "aClass"},
+						{Key: "component", Value: "aComponent"},
+						{Key: "severity", Value: "llama"},
+					},
+				})
+				evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
+				evalContext.IsTestRun = true
+
+				payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext)
+				So(err, ShouldBeNil)
+				payload, err := simplejson.NewJson(payloadJSON)
+				So(err, ShouldBeNil)
+
+				diff := cmp.Diff(map[string]interface{}{
+					"client":       "Grafana",
+					"client_url":   "",
+					"dedup_key":    "alertId-0",
+					"event_action": "trigger",
+					"links": []interface{}{
+						map[string]interface{}{
+							"href": "",
+						},
+					},
+					"payload": map[string]interface{}{
+						"source":    "<<PRESENCE>>",
+						"component": "aComponent",
+						"custom_details": map[string]interface{}{
+							"group":     "aGroup",
+							"class":     "aClass",
+							"component": "aComponent",
+							"severity":  "llama",
 							"keyOnly":   "",
 						},
 						"severity":  "critical",