azuremonitor: handle multi-dimensions on backend

pkg/tsdb/azuremonitor/azuremonitor.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strings"
 	"time"
 
 	"github.com/grafana/grafana/pkg/api/pluginproxy"
@@ -24,7 +25,7 @@ import (
 )
 
 var (
-	slog log.Logger
+	azlog log.Logger
 )
 
 // AzureMonitorExecutor executes queries for the Azure Monitor datasource - all four services
@@ -47,7 +48,7 @@ func NewAzureMonitorExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint,
 }
 
 func init() {
-	slog = log.New("tsdb.azuremonitor")
+	azlog = log.New("tsdb.azuremonitor")
 	tsdb.RegisterTsdbQueryEndpoint("grafana-azure-monitor-datasource", NewAzureMonitorExecutor)
 }
 
@@ -61,7 +62,6 @@ func (e *AzureMonitorExecutor) Query(ctx context.Context, dsInfo *models.DataSou
 	queryType := tsdbQuery.Queries[0].Model.Get("queryType").MustString("")
 
 	switch queryType {
-	case "azureMonitorTimeSeriesQuery":
 	case "Azure Monitor":
 		fallthrough
 	default:
@@ -112,26 +112,39 @@ func (e *AzureMonitorExecutor) buildQueries(tsdbQuery *tsdb.TsdbQuery) ([]*Azure
 		var target string
 
 		azureMonitorTarget := query.Model.Get("azureMonitor").MustMap()
+		azlog.Debug("AzureMonitor", "target", azureMonitorTarget)
 
 		urlComponents := make(map[string]string)
-		urlComponents["resourceGroup"] = azureMonitorTarget["resourceGroup"].(string)
-		urlComponents["metricDefinition"] = azureMonitorTarget["metricDefinition"].(string)
-		urlComponents["resourceName"] = azureMonitorTarget["resourceName"].(string)
-
-		azureURL := fmt.Sprintf("resourceGroups/%s/providers/%s/%s/providers/microsoft.insights/metrics", urlComponents["resourceGroup"], urlComponents["metricDefinition"], urlComponents["resourceName"])
+		urlComponents["resourceGroup"] = fmt.Sprintf("%v", azureMonitorTarget["resourceGroup"])
+		urlComponents["metricDefinition"] = fmt.Sprintf("%v", azureMonitorTarget["metricDefinition"])
+		urlComponents["resourceName"] = fmt.Sprintf("%v", azureMonitorTarget["resourceName"])
+
+		ub := URLBuilder{
+			ResourceGroup:    urlComponents["resourceGroup"],
+			MetricDefinition: urlComponents["metricDefinition"],
+			ResourceName:     urlComponents["resourceName"],
+		}
+		azureURL := ub.Build()
 
-		alias := azureMonitorTarget["alias"].(string)
+		alias := fmt.Sprintf("%v", azureMonitorTarget["alias"])
 
 		params := url.Values{}
 		params.Add("api-version", "2018-01-01")
 		params.Add("timespan", fmt.Sprintf("%v/%v", startTime.UTC().Format(time.RFC3339), endTime.UTC().Format(time.RFC3339)))
-		params.Add("interval", azureMonitorTarget["timeGrain"].(string))
-		params.Add("aggregation", azureMonitorTarget["aggregation"].(string))
-		params.Add("metricnames", azureMonitorTarget["metricName"].(string))
+		params.Add("interval", fmt.Sprintf("%v", azureMonitorTarget["timeGrain"]))
+		params.Add("aggregation", fmt.Sprintf("%v", azureMonitorTarget["aggregation"]))
+		params.Add("metricnames", fmt.Sprintf("%v", azureMonitorTarget["metricName"]))
+
+		dimension := fmt.Sprintf("%v", azureMonitorTarget["dimension"])
+		dimensionFilter := strings.TrimSpace(fmt.Sprintf("%v", azureMonitorTarget["dimensionFilter"]))
+		if azureMonitorTarget["dimension"] != nil && azureMonitorTarget["dimensionFilter"] != nil && dimensionFilter != "" {
+			params.Add("$filter", fmt.Sprintf("%s eq '%s'", dimension, dimensionFilter))
+		}
+
 		target = params.Encode()
 
 		if setting.Env == setting.DEV {
-			slog.Debug("Azuremonitor request", "params", params)
+			azlog.Debug("Azuremonitor request", "params", params)
 		}
 
 		azureMonitorQueries = append(azureMonitorQueries, &AzureMonitorQuery{
@@ -174,6 +187,7 @@ func (e *AzureMonitorExecutor) executeQuery(ctx context.Context, query *AzureMon
 		opentracing.HTTPHeaders,
 		opentracing.HTTPHeadersCarrier(req.Header))
 
+	azlog.Debug("AzureMonitor", "Request URL", req.URL.String())
 	res, err := ctxhttp.Do(ctx, e.httpClient, req)
 	if err != nil {
 		queryResult.Error = err
@@ -213,7 +227,7 @@ func (e *AzureMonitorExecutor) createRequest(ctx context.Context, dsInfo *models
 
 	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
 	if err != nil {
-		slog.Error("Failed to create request", "error", err)
+		azlog.Error("Failed to create request", "error", err)
 		return nil, fmt.Errorf("Failed to create request. error: %v", err)
 	}
 
@@ -233,14 +247,14 @@ func (e *AzureMonitorExecutor) unmarshalResponse(res *http.Response) (AzureMonit
 	}
 
 	if res.StatusCode/100 != 2 {
-		slog.Error("Request failed", "status", res.Status, "body", string(body))
+		azlog.Error("Request failed", "status", res.Status, "body", string(body))
 		return AzureMonitorResponse{}, fmt.Errorf(string(body))
 	}
 
 	var data AzureMonitorResponse
 	err = json.Unmarshal(body, &data)
 	if err != nil {
-		slog.Error("Failed to unmarshal AzureMonitor response", "error", err, "status", res.Status, "body", string(body))
+		azlog.Error("Failed to unmarshal AzureMonitor response", "error", err, "status", res.Status, "body", string(body))
 		return AzureMonitorResponse{}, err
 	}
 
@@ -248,14 +262,24 @@ func (e *AzureMonitorExecutor) unmarshalResponse(res *http.Response) (AzureMonit
 }
 
 func (e *AzureMonitorExecutor) parseResponse(queryRes *tsdb.QueryResult, data AzureMonitorResponse, query *AzureMonitorQuery) error {
-	slog.Info("AzureMonitor", "Response", data)
+	azlog.Debug("AzureMonitor", "Response", data)
 
-	for _, series := range data.Value {
+	if len(data.Value) == 0 {
+		return nil
+	}
+
+	for _, series := range data.Value[0].Timeseries {
 		points := make([]tsdb.TimePoint, 0)
 
-		defaultMetricName := fmt.Sprintf("%s.%s", query.UrlComponents["resourceName"], series.Name.LocalizedValue)
+		metadataName := ""
+		metadataValue := ""
+		if len(series.Metadatavalues) > 0 {
+			metadataName = series.Metadatavalues[0].Name.LocalizedValue
+			metadataValue = series.Metadatavalues[0].Value
+		}
+		defaultMetricName := formatLegendKey(query.UrlComponents["resourceName"], data.Value[0].Name.LocalizedValue, metadataName, metadataValue)
 
-		for _, point := range series.Timeseries[0].Data {
+		for _, point := range series.Data {
 			var value float64
 			switch query.Params.Get("aggregation") {
 			case "Average":

pkg/tsdb/azuremonitor/azuremonitor_test.go

@@ -178,6 +178,33 @@ func TestAzureMonitor(t *testing.T) {
 				So(res.Series[0].Points[0][0].Float64, ShouldEqual, 4)
 				So(res.Series[0].Points[0][1].Float64, ShouldEqual, 1549723440000)
 			})
+
+			Convey("when data from query aggregated as total and has dimension filter", func() {
+				data, err := loadTestFile("./test-data/6-azure-monitor-response-multi-dimension.json")
+				So(err, ShouldBeNil)
+
+				res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "A"}
+				query := &AzureMonitorQuery{
+					UrlComponents: map[string]string{
+						"resourceName": "grafana",
+					},
+					Params: url.Values{
+						"aggregation": {"Average"},
+					},
+				}
+				err = executor.parseResponse(res, data, query)
+				So(err, ShouldBeNil)
+				So(len(res.Series), ShouldEqual, 3)
+
+				So(res.Series[0].Name, ShouldEqual, "grafana{blobtype=PageBlob}.Blob Count")
+				So(res.Series[0].Points[0][0].Float64, ShouldEqual, 3)
+
+				So(res.Series[1].Name, ShouldEqual, "grafana{blobtype=BlockBlob}.Blob Count")
+				So(res.Series[1].Points[0][0].Float64, ShouldEqual, 1)
+
+				So(res.Series[2].Name, ShouldEqual, "grafana{blobtype=Azure Data Lake Storage}.Blob Count")
+				So(res.Series[2].Points[0][0].Float64, ShouldEqual, 0)
+			})
 		})
 	})
 }

pkg/tsdb/azuremonitor/legend-key.go

+package azuremonitor
+
+import "fmt"
+
+// formatLegendKey builds the legend key or timeseries name
+func formatLegendKey(resourceName string, metricName string, metadataName string, metadataValue string) string {
+	if len(metadataName) > 0 {
+		return fmt.Sprintf("%s{%s=%s}.%s", resourceName, metadataName, metadataValue, metricName)
+	}
+	return fmt.Sprintf("%s.%s", resourceName, metricName)
+}

pkg/tsdb/azuremonitor/test-data/1-azure-monitor-response-avg.json

@@ -4,7 +4,7 @@
   "interval": "PT1M",
   "value": [
     {
-      "id": "\/subscriptions\/44693801-6ee6-49de-9b2d-9106972f9572\/resourceGroups\/grafanastaging\/providers\/Microsoft.Compute\/virtualMachines\/grafana\/providers\/Microsoft.Insights\/metrics\/Percentage CPU",
+      "id": "\/subscriptions\/xxx\/resourceGroups\/grafanastaging\/providers\/Microsoft.Compute\/virtualMachines\/grafana\/providers\/Microsoft.Insights\/metrics\/Percentage CPU",
       "type": "Microsoft.Insights\/metrics",
       "name": {
         "value": "Percentage CPU",

pkg/tsdb/azuremonitor/test-data/2-azure-monitor-response-total.json

@@ -4,7 +4,7 @@
   "interval": "PT1M",
   "value": [
     {
-      "id": "\/subscriptions\/44693801-6ee6-49de-9b2d-9106972f9572\/resourceGroups\/grafanastaging\/providers\/Microsoft.Compute\/virtualMachines\/grafana\/providers\/Microsoft.Insights\/metrics\/Percentage CPU",
+      "id": "\/subscriptions\/xxx\/resourceGroups\/grafanastaging\/providers\/Microsoft.Compute\/virtualMachines\/grafana\/providers\/Microsoft.Insights\/metrics\/Percentage CPU",
       "type": "Microsoft.Insights\/metrics",
       "name": {
         "value": "Percentage CPU",

pkg/tsdb/azuremonitor/test-data/3-azure-monitor-response-maximum.json

@@ -4,7 +4,7 @@
   "interval": "PT1M",
   "value": [
     {
-      "id": "\/subscriptions\/44693801-6ee6-49de-9b2d-9106972f9572\/resourceGroups\/grafanastaging\/providers\/Microsoft.Compute\/virtualMachines\/grafana\/providers\/Microsoft.Insights\/metrics\/Percentage CPU",
+      "id": "\/subscriptions\/xxx\/resourceGroups\/grafanastaging\/providers\/Microsoft.Compute\/virtualMachines\/grafana\/providers\/Microsoft.Insights\/metrics\/Percentage CPU",
       "type": "Microsoft.Insights\/metrics",
       "name": {
         "value": "Percentage CPU",

pkg/tsdb/azuremonitor/test-data/4-azure-monitor-response-minimum.json

@@ -4,7 +4,7 @@
   "interval": "PT1M",
   "value": [
     {
-      "id": "\/subscriptions\/44693801-6ee6-49de-9b2d-9106972f9572\/resourceGroups\/grafanastaging\/providers\/Microsoft.Compute\/virtualMachines\/grafana\/providers\/Microsoft.Insights\/metrics\/Percentage CPU",
+      "id": "\/subscriptions\/xxx\/resourceGroups\/grafanastaging\/providers\/Microsoft.Compute\/virtualMachines\/grafana\/providers\/Microsoft.Insights\/metrics\/Percentage CPU",
       "type": "Microsoft.Insights\/metrics",
       "name": {
         "value": "Percentage CPU",

pkg/tsdb/azuremonitor/test-data/5-azure-monitor-response-count.json

@@ -4,7 +4,7 @@
   "interval": "PT1M",
   "value": [
     {
-      "id": "\/subscriptions\/44693801-6ee6-49de-9b2d-9106972f9572\/resourceGroups\/grafanastaging\/providers\/Microsoft.Compute\/virtualMachines\/grafana\/providers\/Microsoft.Insights\/metrics\/Percentage CPU",
+      "id": "\/subscriptions\/xxx\/resourceGroups\/grafanastaging\/providers\/Microsoft.Compute\/virtualMachines\/grafana\/providers\/Microsoft.Insights\/metrics\/Percentage CPU",
       "type": "Microsoft.Insights\/metrics",
       "name": {
         "value": "Percentage CPU",

pkg/tsdb/azuremonitor/test-data/6-azure-monitor-response-multi-dimension.json

+{
+  "cost": 0,
+  "timespan": "2019-02-09T15:21:39Z\/2019-02-09T21:21:39Z",
+  "interval": "PT1H",
+  "value": [
+    {
+      "id": "\/subscriptions\/xxx\/resourceGroups\/grafanastaging\/providers\/Microsoft.Storage\/storageAccounts\/grafanastaging\/blobServices\/default\/providers\/Microsoft.Insights\/metrics\/BlobCount",
+      "type": "Microsoft.Insights\/metrics",
+      "name": {
+        "value": "BlobCount",
+        "localizedValue": "Blob Count"
+      },
+      "unit": "Count",
+      "timeseries": [
+        {
+          "metadatavalues": [
+            {
+              "name": {
+                "value": "blobtype",
+                "localizedValue": "blobtype"
+              },
+              "value": "PageBlob"
+            }
+          ],
+          "data": [
+            {
+              "timeStamp": "2019-02-09T15:21:00Z",
+              "average": 3
+            },
+            {
+              "timeStamp": "2019-02-09T16:21:00Z",
+              "average": 3
+            },
+            {
+              "timeStamp": "2019-02-09T17:21:00Z",
+              "average": 3
+            },
+            {
+              "timeStamp": "2019-02-09T18:21:00Z",
+              "average": 3
+            },
+            {
+              "timeStamp": "2019-02-09T19:21:00Z",
+              "average": 3
+            },
+            {
+              "timeStamp": "2019-02-09T20:21:00Z"
+            }
+          ]
+        },
+        {
+          "metadatavalues": [
+            {
+              "name": {
+                "value": "blobtype",
+                "localizedValue": "blobtype"
+              },
+              "value": "BlockBlob"
+            }
+          ],
+          "data": [
+            {
+              "timeStamp": "2019-02-09T15:21:00Z",
+              "average": 1
+            },
+            {
+              "timeStamp": "2019-02-09T16:21:00Z",
+              "average": 1
+            },
+            {
+              "timeStamp": "2019-02-09T17:21:00Z",
+              "average": 1
+            },
+            {
+              "timeStamp": "2019-02-09T18:21:00Z",
+              "average": 1
+            },
+            {
+              "timeStamp": "2019-02-09T19:21:00Z",
+              "average": 1
+            },
+            {
+              "timeStamp": "2019-02-09T20:21:00Z"
+            }
+          ]
+        },
+        {
+          "metadatavalues": [
+            {
+              "name": {
+                "value": "blobtype",
+                "localizedValue": "blobtype"
+              },
+              "value": "Azure Data Lake Storage"
+            }
+          ],
+          "data": [
+            {
+              "timeStamp": "2019-02-09T15:21:00Z",
+              "average": 0
+            },
+            {
+              "timeStamp": "2019-02-09T16:21:00Z",
+              "average": 0
+            },
+            {
+              "timeStamp": "2019-02-09T17:21:00Z",
+              "average": 0
+            },
+            {
+              "timeStamp": "2019-02-09T18:21:00Z",
+              "average": 0
+            },
+            {
+              "timeStamp": "2019-02-09T19:21:00Z",
+              "average": 0
+            },
+            {
+              "timeStamp": "2019-02-09T20:21:00Z"
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "namespace": "Microsoft.Storage\/storageAccounts\/blobServices",
+  "resourceregion": "westeurope"
+}

pkg/tsdb/azuremonitor/url-builder.go

+package azuremonitor
+
+import (
+	"fmt"
+	"strings"
+)
+
+// URLBuilder builds the URL for calling the Azure Monitor API
+type URLBuilder struct {
+	ResourceGroup    string
+	MetricDefinition string
+	ResourceName     string
+}
+
+// Build checks the metric definition property to see which form of the url
+// should be returned
+func (ub *URLBuilder) Build() string {
+
+	if strings.Count(ub.MetricDefinition, "/") > 1 {
+		rn := strings.Split(ub.ResourceName, "/")
+		lastIndex := strings.LastIndex(ub.MetricDefinition, "/")
+		service := ub.MetricDefinition[lastIndex+1:]
+		md := ub.MetricDefinition[0:lastIndex]
+		return fmt.Sprintf("resourceGroups/%s/providers/%s/%s/%s/%s/providers/microsoft.insights/metrics", ub.ResourceGroup, md, rn[0], service, rn[1])
+	}
+
+	return fmt.Sprintf("resourceGroups/%s/providers/%s/%s/providers/microsoft.insights/metrics", ub.ResourceGroup, ub.MetricDefinition, ub.ResourceName)
+}

pkg/tsdb/azuremonitor/url-builder_test.go

+package azuremonitor
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestURLBuilder(t *testing.T) {
+	Convey("AzureMonitor URL Builder", t, func() {
+
+		Convey("when metric definition is in the short form", func() {
+			ub := &URLBuilder{
+				ResourceGroup:    "rg",
+				MetricDefinition: "Microsoft.Compute/virtualMachines",
+				ResourceName:     "rn",
+			}
+
+			url := ub.Build()
+			So(url, ShouldEqual, "resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/rn/providers/microsoft.insights/metrics")
+		})
+
+		Convey("when metric definition is Microsoft.Storage/storageAccounts/blobServices", func() {
+			ub := &URLBuilder{
+				ResourceGroup:    "rg",
+				MetricDefinition: "Microsoft.Storage/storageAccounts/blobServices",
+				ResourceName:     "rn1/default",
+			}
+
+			url := ub.Build()
+			So(url, ShouldEqual, "resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/rn1/blobServices/default/providers/microsoft.insights/metrics")
+		})
+
+		Convey("when metric definition is Microsoft.Storage/storageAccounts/fileServices", func() {
+			ub := &URLBuilder{
+				ResourceGroup:    "rg",
+				MetricDefinition: "Microsoft.Storage/storageAccounts/fileServices",
+				ResourceName:     "rn1/default",
+			}
+
+			url := ub.Build()
+			So(url, ShouldEqual, "resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/rn1/fileServices/default/providers/microsoft.insights/metrics")
+		})
+	})
+}