AlertingNG: Modify queries and transform endpoint to get datasource UIDs (#30297)

* Pass skipCache from context

* Use macaron Params instead of ParamsEscape for UIDs

* Modify queries and transform to get datasource UIDs

* Update github.com/grafana/grafana-plugin-sdk-go to v0.83.0

go.mod

@@ -43,7 +43,7 @@ require (
 	github.com/google/uuid v1.1.2
 	github.com/gosimple/slug v1.9.0
 	github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4
-	github.com/grafana/grafana-plugin-sdk-go v0.81.0
+	github.com/grafana/grafana-plugin-sdk-go v0.83.0
 	github.com/grafana/loki v1.6.2-0.20201026154740-6978ee5d7387
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.1
 	github.com/hashicorp/go-hclog v0.14.1

go.sum

@@ -669,8 +669,8 @@ github.com/gosimple/slug v1.9.0 h1:r5vDcYrFz9BmfIAMC829un9hq7hKM4cHUrsv36LbEqs=
 github.com/gosimple/slug v1.9.0/go.mod h1:AMZ+sOVe65uByN3kgEyf9WEBKBCSS+dJjMX9x4vDJbg=
 github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4 h1:SPdxCL9BChFTlyi0Khv64vdCW4TMna8+sxL7+Chx+Ag=
 github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4/go.mod h1:nc0XxBzjeGcrMltCDw269LoWF9S8ibhgxolCdA1R8To=
-github.com/grafana/grafana-plugin-sdk-go v0.81.0 h1:/4OwkOh9UDC0aWY4DHNrKiY0itUHCZFq34OjEB2u8v8=
-github.com/grafana/grafana-plugin-sdk-go v0.81.0/go.mod h1:exQQHhClzHs2gOwjPSO4FOKwjjZ8VrnzbbABHX8LB6U=
+github.com/grafana/grafana-plugin-sdk-go v0.83.0 h1:X84eJMLSx0KOTRW1EziXkqLw9pSmK0RggWC98ImX/9g=
+github.com/grafana/grafana-plugin-sdk-go v0.83.0/go.mod h1:exQQHhClzHs2gOwjPSO4FOKwjjZ8VrnzbbABHX8LB6U=
 github.com/grafana/loki v1.6.2-0.20201026154740-6978ee5d7387 h1:iwcM8lkYJ3EhytGLJ2BvRSwutb0QWoI7EWbYv3yJRsY=
 github.com/grafana/loki v1.6.2-0.20201026154740-6978ee5d7387/go.mod h1:jHA1OHnPsuj3LLgMXmFopsKDt4ARHHUhrmT3JrGf71g=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=

pkg/expr/nodes.go

@@ -123,13 +123,15 @@ const (
 // DSNode is a DPNode that holds a datasource request.
 type DSNode struct {
 	baseNode
-	query        json.RawMessage
-	datasourceID int64
-	orgID        int64
-	queryType    string
-	timeRange    backend.TimeRange
-	intervalMS   int64
-	maxDP        int64
+	query         json.RawMessage
+	datasourceID  int64
+	datasourceUID string
+
+	orgID      int64
+	queryType  string
+	timeRange  backend.TimeRange
+	intervalMS int64
+	maxDP      int64
 }
 
 // NodeType returns the data pipeline node type.
@@ -157,14 +159,24 @@ func buildDSNode(dp *simple.DirectedGraph, rn *rawNode, orgID int64) (*DSNode, e
 	}
 
 	rawDsID, ok := rn.Query["datasourceId"]
-	if !ok {
-		return nil, fmt.Errorf("no datasourceId in expression data source request for refId %v", rn.RefID)
-	}
-	floatDsID, ok := rawDsID.(float64)
-	if !ok {
-		return nil, fmt.Errorf("expected datasourceId to be a float64, got type %T for refId %v", rawDsID, rn.RefID)
+	switch ok {
+	case true:
+		floatDsID, ok := rawDsID.(float64)
+		if !ok {
+			return nil, fmt.Errorf("expected datasourceId to be a float64, got type %T for refId %v", rawDsID, rn.RefID)
+		}
+		dsNode.datasourceID = int64(floatDsID)
+	default:
+		rawDsUID, ok := rn.Query["datasourceUid"]
+		if !ok {
+			return nil, fmt.Errorf("neither datasourceId or datasourceUid in expression data source request for refId %v", rn.RefID)
+		}
+		strDsUID, ok := rawDsUID.(string)
+		if !ok {
+			return nil, fmt.Errorf("expected datasourceUid to be a string, got type %T for refId %v", rawDsUID, rn.RefID)
+		}
+		dsNode.datasourceUID = strDsUID
 	}
-	dsNode.datasourceID = int64(floatDsID)
 
 	var floatIntervalMS float64
 	if rawIntervalMS := rn.Query["intervalMs"]; ok {
@@ -192,7 +204,8 @@ func (dn *DSNode) Execute(ctx context.Context, vars mathexp.Vars) (mathexp.Resul
 	pc := backend.PluginContext{
 		OrgID: dn.orgID,
 		DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
-			ID: dn.datasourceID,
+			ID:  dn.datasourceID,
+			UID: dn.datasourceUID,
 		},
 	}
 

pkg/expr/service.go

@@ -14,6 +14,10 @@ const DatasourceName = "__expr__"
 // expression command.
 const DatasourceID = -100
 
+// DatasourceUID is the fake datasource uid used in requests to identify it as an
+// expression command.
+const DatasourceUID = "-100"
+
 // Service is service representation for expression handling.
 type Service struct {
 }

pkg/expr/transform.go

@@ -131,14 +131,17 @@ func QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.Que
 	}
 
 	datasourceID := int64(0)
+	var datasourceUID string
 
 	if req.PluginContext.DataSourceInstanceSettings != nil {
 		datasourceID = req.PluginContext.DataSourceInstanceSettings.ID
+		datasourceUID = req.PluginContext.DataSourceInstanceSettings.UID
 	}
 
 	getDsInfo := &models.GetDataSourceQuery{
 		OrgId: req.PluginContext.OrgID,
 		Id:    datasourceID,
+		Uid:   datasourceUID,
 	}
 
 	if err := bus.Dispatch(getDsInfo); err != nil {

pkg/services/ngalert/api.go

@@ -31,7 +31,7 @@ func (ng *AlertNG) registerAPIEndpoints() {
 
 // conditionEvalEndpoint handles POST /api/alert-definitions/eval.
 func (ng *AlertNG) conditionEvalEndpoint(c *models.ReqContext, dto evalAlertConditionCommand) response.Response {
-	if err := ng.validateCondition(dto.Condition, c.SignedInUser); err != nil {
+	if err := ng.validateCondition(dto.Condition, c.SignedInUser, c.SkipCache); err != nil {
 		return response.Error(400, "invalid condition", err)
 	}
 
@@ -54,14 +54,14 @@ func (ng *AlertNG) conditionEvalEndpoint(c *models.ReqContext, dto evalAlertCond
 
 // alertDefinitionEvalEndpoint handles GET /api/alert-definitions/eval/:alertDefinitionUID.
 func (ng *AlertNG) alertDefinitionEvalEndpoint(c *models.ReqContext) response.Response {
-	alertDefinitionUID := c.ParamsEscape(":alertDefinitionUID")
+	alertDefinitionUID := c.Params(":alertDefinitionUID")
 
 	condition, err := ng.LoadAlertCondition(alertDefinitionUID, c.SignedInUser.OrgId)
 	if err != nil {
 		return response.Error(400, "Failed to load alert definition conditions", err)
 	}
 
-	if err := ng.validateCondition(*condition, c.SignedInUser); err != nil {
+	if err := ng.validateCondition(*condition, c.SignedInUser, c.SkipCache); err != nil {
 		return response.Error(400, "invalid condition", err)
 	}
 
@@ -87,7 +87,7 @@ func (ng *AlertNG) alertDefinitionEvalEndpoint(c *models.ReqContext) response.Re
 
 // getAlertDefinitionEndpoint handles GET /api/alert-definitions/:alertDefinitionUID.
 func (ng *AlertNG) getAlertDefinitionEndpoint(c *models.ReqContext) response.Response {
-	alertDefinitionUID := c.ParamsEscape(":alertDefinitionUID")
+	alertDefinitionUID := c.Params(":alertDefinitionUID")
 
 	query := getAlertDefinitionByUIDQuery{
 		UID:   alertDefinitionUID,
@@ -103,7 +103,7 @@ func (ng *AlertNG) getAlertDefinitionEndpoint(c *models.ReqContext) response.Res
 
 // deleteAlertDefinitionEndpoint handles DELETE /api/alert-definitions/:alertDefinitionUID.
 func (ng *AlertNG) deleteAlertDefinitionEndpoint(c *models.ReqContext) response.Response {
-	alertDefinitionUID := c.ParamsEscape(":alertDefinitionUID")
+	alertDefinitionUID := c.Params(":alertDefinitionUID")
 
 	cmd := deleteAlertDefinitionByUIDCommand{
 		UID:   alertDefinitionUID,
@@ -119,10 +119,10 @@ func (ng *AlertNG) deleteAlertDefinitionEndpoint(c *models.ReqContext) response.
 
 // updateAlertDefinitionEndpoint handles PUT /api/alert-definitions/:alertDefinitionUID.
 func (ng *AlertNG) updateAlertDefinitionEndpoint(c *models.ReqContext, cmd updateAlertDefinitionCommand) response.Response {
-	cmd.UID = c.ParamsEscape(":alertDefinitionUID")
+	cmd.UID = c.Params(":alertDefinitionUID")
 	cmd.OrgID = c.SignedInUser.OrgId
 
-	if err := ng.validateCondition(cmd.Condition, c.SignedInUser); err != nil {
+	if err := ng.validateCondition(cmd.Condition, c.SignedInUser, c.SkipCache); err != nil {
 		return response.Error(400, "invalid condition", err)
 	}
 
@@ -137,7 +137,7 @@ func (ng *AlertNG) updateAlertDefinitionEndpoint(c *models.ReqContext, cmd updat
 func (ng *AlertNG) createAlertDefinitionEndpoint(c *models.ReqContext, cmd saveAlertDefinitionCommand) response.Response {
 	cmd.OrgID = c.SignedInUser.OrgId
 
-	if err := ng.validateCondition(cmd.Condition, c.SignedInUser); err != nil {
+	if err := ng.validateCondition(cmd.Condition, c.SignedInUser, c.SkipCache); err != nil {
 		return response.Error(400, "invalid condition", err)
 	}
 

pkg/services/ngalert/eval/alert_query.go

@@ -68,12 +68,12 @@ type AlertQuery struct {
 	// RelativeTimeRange is the relative Start and End of the query as sent by the frontend.
 	RelativeTimeRange RelativeTimeRange `json:"relativeTimeRange"`
 
-	DatasourceID int64 `json:"-"`
+	DatasourceUID string `json:"-"`
 
 	// JSON is the raw JSON query and includes the above properties as well as custom properties.
 	Model json.RawMessage `json:"model"`
 
-	modelProps map[string]interface{} `json:"-"`
+	modelProps map[string]interface{}
 }
 
 func (aq *AlertQuery) setModelProps() error {
@@ -102,20 +102,20 @@ func (aq *AlertQuery) setDatasource() error {
 	}
 
 	if dsName == expr.DatasourceName {
-		aq.DatasourceID = expr.DatasourceID
-		aq.modelProps["datasourceId"] = expr.DatasourceID
+		aq.DatasourceUID = expr.DatasourceUID
+		aq.modelProps["datasourceUid"] = expr.DatasourceUID
 		return nil
 	}
 
-	i, ok := aq.modelProps["datasourceId"]
+	i, ok := aq.modelProps["datasourceUid"]
 	if !ok {
-		return fmt.Errorf("failed to get datasourceId from query model")
+		return fmt.Errorf("failed to get datasourceUid from query model")
 	}
-	dsID, ok := i.(float64)
+	dsUID, ok := i.(string)
 	if !ok {
-		return fmt.Errorf("failed to cast datasourceId to float64: %v", i)
+		return fmt.Errorf("failed to cast datasourceUid to string: %v", i)
 	}
-	aq.DatasourceID = int64(dsID)
+	aq.DatasourceUID = dsUID
 	return nil
 }
 
@@ -125,7 +125,7 @@ func (aq *AlertQuery) IsExpression() (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	return aq.DatasourceID == expr.DatasourceID, nil
+	return aq.DatasourceUID == expr.DatasourceUID, nil
 }
 
 // setMaxDatapoints sets the model maxDataPoints if it's missing or invalid
@@ -206,12 +206,12 @@ func (aq *AlertQuery) getIntervalDuration() (time.Duration, error) {
 }
 
 // GetDatasource returns the query datasource identifier.
-func (aq *AlertQuery) GetDatasource() (int64, error) {
+func (aq *AlertQuery) GetDatasource() (string, error) {
 	err := aq.setDatasource()
 	if err != nil {
-		return 0, err
+		return "", err
 	}
-	return aq.DatasourceID, nil
+	return aq.DatasourceUID, nil
 }
 
 func (aq *AlertQuery) getModel() ([]byte, error) {

pkg/services/ngalert/eval/alert_query_test.go

@@ -13,14 +13,14 @@ import (
 
 func TestAlertQuery(t *testing.T) {
 	testCases := []struct {
-		desc                 string
-		alertQuery           AlertQuery
-		expectedIsExpression bool
-		expectedDatasource   string
-		expectedDatasourceID int64
-		expectedMaxPoints    int64
-		expectedIntervalMS   int64
-		err                  error
+		desc                  string
+		alertQuery            AlertQuery
+		expectedIsExpression  bool
+		expectedDatasource    string
+		expectedDatasourceUID string
+		expectedMaxPoints     int64
+		expectedIntervalMS    int64
+		err                   error
 	}{
 		{
 			desc: "given an expression query",
@@ -32,11 +32,11 @@ func TestAlertQuery(t *testing.T) {
 					"extraParam": "some text"
 				}`),
 			},
-			expectedIsExpression: true,
-			expectedDatasource:   expr.DatasourceName,
-			expectedDatasourceID: int64(expr.DatasourceID),
-			expectedMaxPoints:    int64(defaultMaxDataPoints),
-			expectedIntervalMS:   int64(defaultIntervalMS),
+			expectedIsExpression:  true,
+			expectedDatasource:    expr.DatasourceName,
+			expectedDatasourceUID: expr.DatasourceUID,
+			expectedMaxPoints:     int64(defaultMaxDataPoints),
+			expectedIntervalMS:    int64(defaultIntervalMS),
 		},
 		{
 			desc: "given a query",
@@ -44,16 +44,16 @@ func TestAlertQuery(t *testing.T) {
 				RefID: "A",
 				Model: json.RawMessage(`{
 					"datasource": "my datasource",
-					"datasourceId": 1,
+					"datasourceUid": "000000001",
 					"queryType": "metricQuery",
 					"extraParam": "some text"
 				}`),
 			},
-			expectedIsExpression: false,
-			expectedDatasource:   "my datasource",
-			expectedDatasourceID: 1,
-			expectedMaxPoints:    int64(defaultMaxDataPoints),
-			expectedIntervalMS:   int64(defaultIntervalMS),
+			expectedIsExpression:  false,
+			expectedDatasource:    "my datasource",
+			expectedDatasourceUID: "000000001",
+			expectedMaxPoints:     int64(defaultMaxDataPoints),
+			expectedIntervalMS:    int64(defaultIntervalMS),
 		},
 		{
 			desc: "given a query with valid maxDataPoints",
@@ -61,17 +61,17 @@ func TestAlertQuery(t *testing.T) {
 				RefID: "A",
 				Model: json.RawMessage(`{
 					"datasource": "my datasource",
-					"datasourceId": 1,
+					"datasourceUid": "000000001",
 					"queryType": "metricQuery",
 					"maxDataPoints": 200,
 					"extraParam": "some text"
 				}`),
 			},
-			expectedIsExpression: false,
-			expectedDatasource:   "my datasource",
-			expectedDatasourceID: 1,
-			expectedMaxPoints:    200,
-			expectedIntervalMS:   int64(defaultIntervalMS),
+			expectedIsExpression:  false,
+			expectedDatasource:    "my datasource",
+			expectedDatasourceUID: "000000001",
+			expectedMaxPoints:     200,
+			expectedIntervalMS:    int64(defaultIntervalMS),
 		},
 		{
 			desc: "given a query with invalid maxDataPoints",
@@ -79,17 +79,17 @@ func TestAlertQuery(t *testing.T) {
 				RefID: "A",
 				Model: json.RawMessage(`{
 					"datasource": "my datasource",
-					"datasourceId": 1,
+					"datasourceUid": "000000001",
 					"queryType": "metricQuery",
 					"maxDataPoints": "invalid",
 					"extraParam": "some text"
 				}`),
 			},
-			expectedIsExpression: false,
-			expectedDatasource:   "my datasource",
-			expectedDatasourceID: 1,
-			expectedMaxPoints:    int64(defaultMaxDataPoints),
-			expectedIntervalMS:   int64(defaultIntervalMS),
+			expectedIsExpression:  false,
+			expectedDatasource:    "my datasource",
+			expectedDatasourceUID: "000000001",
+			expectedMaxPoints:     int64(defaultMaxDataPoints),
+			expectedIntervalMS:    int64(defaultIntervalMS),
 		},
 		{
 			desc: "given a query with zero maxDataPoints",
@@ -97,17 +97,17 @@ func TestAlertQuery(t *testing.T) {
 				RefID: "A",
 				Model: json.RawMessage(`{
 					"datasource": "my datasource",
-					"datasourceId": 1,
+					"datasourceUid": "000000001",
 					"queryType": "metricQuery",
 					"maxDataPoints": 0,
 					"extraParam": "some text"
 				}`),
 			},
-			expectedIsExpression: false,
-			expectedDatasource:   "my datasource",
-			expectedDatasourceID: 1,
-			expectedMaxPoints:    int64(defaultMaxDataPoints),
-			expectedIntervalMS:   int64(defaultIntervalMS),
+			expectedIsExpression:  false,
+			expectedDatasource:    "my datasource",
+			expectedDatasourceUID: "000000001",
+			expectedMaxPoints:     int64(defaultMaxDataPoints),
+			expectedIntervalMS:    int64(defaultIntervalMS),
 		},
 		{
 			desc: "given a query with valid intervalMs",
@@ -115,17 +115,17 @@ func TestAlertQuery(t *testing.T) {
 				RefID: "A",
 				Model: json.RawMessage(`{
 					"datasource": "my datasource",
-					"datasourceId": 1,
+					"datasourceUid": "000000001",
 					"queryType": "metricQuery",
 					"intervalMs": 2000,
 					"extraParam": "some text"
 				}`),
 			},
-			expectedIsExpression: false,
-			expectedDatasource:   "my datasource",
-			expectedDatasourceID: 1,
-			expectedMaxPoints:    int64(defaultMaxDataPoints),
-			expectedIntervalMS:   2000,
+			expectedIsExpression:  false,
+			expectedDatasource:    "my datasource",
+			expectedDatasourceUID: "000000001",
+			expectedMaxPoints:     int64(defaultMaxDataPoints),
+			expectedIntervalMS:    2000,
 		},
 		{
 			desc: "given a query with invalid intervalMs",
@@ -133,17 +133,17 @@ func TestAlertQuery(t *testing.T) {
 				RefID: "A",
 				Model: json.RawMessage(`{
 					"datasource": "my datasource",
-					"datasourceId": 1,
+					"datasourceUid": "000000001",
 					"queryType": "metricQuery",
 					"intervalMs": "invalid",
 					"extraParam": "some text"	
 				}`),
 			},
-			expectedIsExpression: false,
-			expectedDatasource:   "my datasource",
-			expectedDatasourceID: 1,
-			expectedMaxPoints:    int64(defaultMaxDataPoints),
-			expectedIntervalMS:   int64(defaultIntervalMS),
+			expectedIsExpression:  false,
+			expectedDatasource:    "my datasource",
+			expectedDatasourceUID: "000000001",
+			expectedMaxPoints:     int64(defaultMaxDataPoints),
+			expectedIntervalMS:    int64(defaultIntervalMS),
 		},
 		{
 			desc: "given a query with invalid intervalMs",
@@ -151,17 +151,17 @@ func TestAlertQuery(t *testing.T) {
 				RefID: "A",
 				Model: json.RawMessage(`{
 					"datasource": "my datasource",
-					"datasourceId": 1,
+					"datasourceUid": "000000001",
 					"queryType": "metricQuery",
 					"intervalMs": 0,
 					"extraParam": "some text"
 				}`),
 			},
-			expectedIsExpression: false,
-			expectedDatasource:   "my datasource",
-			expectedDatasourceID: 1,
-			expectedMaxPoints:    int64(defaultMaxDataPoints),
-			expectedIntervalMS:   int64(defaultIntervalMS),
+			expectedIsExpression:  false,
+			expectedDatasource:    "my datasource",
+			expectedDatasourceUID: "000000001",
+			expectedMaxPoints:     int64(defaultMaxDataPoints),
+			expectedIntervalMS:    int64(defaultIntervalMS),
 		},
 	}
 
@@ -176,7 +176,7 @@ func TestAlertQuery(t *testing.T) {
 			t.Run("can set datasource for expression", func(t *testing.T) {
 				err := tc.alertQuery.setDatasource()
 				require.NoError(t, err)
-				require.Equal(t, tc.expectedDatasourceID, tc.alertQuery.DatasourceID)
+				require.Equal(t, tc.expectedDatasourceUID, tc.alertQuery.DatasourceUID)
 			})
 
 			t.Run("can set queryType for expression", func(t *testing.T) {
@@ -204,17 +204,18 @@ func TestAlertQuery(t *testing.T) {
 				err = json.Unmarshal(blob, &model)
 				require.NoError(t, err)
 
+				fmt.Printf(">>>>>>> %+v %+v\n", tc.alertQuery, model)
 				i, ok := model["datasource"]
 				require.True(t, ok)
 				datasource, ok := i.(string)
 				require.True(t, ok)
 				require.Equal(t, tc.expectedDatasource, datasource)
 
-				i, ok = model["datasourceId"]
+				i, ok = model["datasourceUid"]
 				require.True(t, ok)
-				datasourceID, ok := i.(float64)
+				datasourceUID, ok := i.(string)
 				require.True(t, ok)
-				require.Equal(t, tc.expectedDatasourceID, int64(datasourceID))
+				require.Equal(t, tc.expectedDatasourceUID, datasourceUID)
 
 				i, ok = model["maxDataPoints"]
 				require.True(t, ok)

pkg/services/ngalert/validator.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/grafana/grafana/pkg/expr"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 )
@@ -35,7 +34,7 @@ func (ng *AlertNG) validateAlertDefinition(alertDefinition *AlertDefinition, req
 }
 
 // validateCondition validates that condition queries refer to existing datasources
-func (ng *AlertNG) validateCondition(c eval.Condition, user *models.SignedInUser) error {
+func (ng *AlertNG) validateCondition(c eval.Condition, user *models.SignedInUser, skipCache bool) error {
 	var refID string
 
 	if len(c.QueriesAndExpressions) == 0 {
@@ -47,18 +46,22 @@ func (ng *AlertNG) validateCondition(c eval.Condition, user *models.SignedInUser
 			refID = c.RefID
 		}
 
-		datasourceID, err := query.GetDatasource()
+		datasourceUID, err := query.GetDatasource()
 		if err != nil {
 			return err
 		}
 
-		if datasourceID == expr.DatasourceID {
+		isExpression, err := query.IsExpression()
+		if err != nil {
+			return err
+		}
+		if isExpression {
 			continue
 		}
 
-		_, err = ng.DatasourceCache.GetDatasource(datasourceID, user, false)
+		_, err = ng.DatasourceCache.GetDatasourceByUID(datasourceUID, user, skipCache)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to get datasource: %s: %w", datasourceUID, err)
 		}
 	}