TestData: Query variable support (nested + glob queries) (#18413)

* TestData: added support for nested data source variable queries, and test dashboard

* Added drilldown dashboards

* Fixed typescript issue

devenv/dev-dashboards/feature-templating/testdata-nested-variables-drilldown.json

+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "links": [],
+  "panels": [
+    {
+      "content": "## Data center = $datacenter\n\n### server = $server\n\n#### pod = $pod",
+      "gridPos": {
+        "h": 6,
+        "w": 15,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "mode": "markdown",
+      "options": {},
+      "targets": [
+        {
+          "refId": "A",
+          "scenarioId": "random_walk"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Panel Title",
+      "type": "text"
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"],
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 9,
+        "x": 15,
+        "y": 0
+      },
+      "id": 6,
+      "interval": null,
+      "links": [
+        {
+          "targetBlank": true,
+          "title": "Overview dashboard",
+          "url": "d/-Y-tnEDWk/dashboard-tests-nested-template-variables?orgId=1&${__all_variables}&${__url_time_range}"
+        }
+      ],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "options": {},
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": true,
+        "ymax": null,
+        "ymin": null
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "refId": "A",
+          "scenarioId": "random_walk"
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Panel drilldown link test",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "avg"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "gdev-testdata",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 13,
+        "w": 24,
+        "x": 0,
+        "y": 6
+      },
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "alias": "$datacenter.$server.$pod",
+          "refId": "A",
+          "scenarioId": "random_walk"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Panel Title",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 19,
+  "style": "dark",
+  "tags": ["gdev", "templating"],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+          "text": "A",
+          "value": ["A"]
+        },
+        "datasource": "TestData DB-1",
+        "definition": "*",
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "datacenter",
+        "options": [],
+        "query": "*",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "text": "AA",
+          "value": ["AA"]
+        },
+        "datasource": "TestData DB-1",
+        "definition": "$datacenter.*",
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "server",
+        "options": [],
+        "query": "$datacenter.*",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "text": "All",
+          "value": ["$__all"]
+        },
+        "datasource": "gdev-testdata",
+        "definition": "$datacenter.$server.*",
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "pod",
+        "options": [],
+        "query": "$datacenter.$server.*",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
+  },
+  "timezone": "",
+  "title": "Templating - Nested Variables Drilldown",
+  "uid": "O6GmNPvWk",
+  "version": 8
+}

devenv/dev-dashboards/feature-templating/testdata-nested-variables.json

+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "iteration": 1565097360786,
+  "links": [],
+  "panels": [
+    {
+      "content": "## Data center = $datacenter\n\n### server = $server\n\n#### pod = $pod",
+      "gridPos": {
+        "h": 6,
+        "w": 14,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "mode": "markdown",
+      "options": {},
+      "targets": [
+        {
+          "refId": "A",
+          "scenarioId": "random_walk"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Panel Title",
+      "type": "text"
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"],
+      "datasource": "gdev-testdata",
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 10,
+        "x": 14,
+        "y": 0
+      },
+      "id": 6,
+      "interval": null,
+      "links": [
+        {
+          "targetBlank": true,
+          "title": "Drilldown detail dashboard",
+          "url": "d/O6GmNPvWk/dashboard-tests-nested-template-variables-drilldown?orgId=1&${__all_variables}&${__url_time_range}"
+        }
+      ],
+      "mappingType": 1,
+      "mappingTypes": [],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "options": {},
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": true,
+        "ymax": null,
+        "ymin": null
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "refId": "A",
+          "scenarioId": "random_walk"
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Panel drilldown link test",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "avg"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "gdev-testdata",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 13,
+        "w": 24,
+        "x": 0,
+        "y": 6
+      },
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "alias": "$datacenter.$server.$pod",
+          "refId": "A",
+          "scenarioId": "random_walk"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Panel Title",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 19,
+  "style": "dark",
+  "tags": ["gdev", "templating"],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+          "text": "A",
+          "value": ["A"]
+        },
+        "datasource": "gdev-testdata",
+        "definition": "*",
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "datacenter",
+        "options": [],
+        "query": "*",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "text": "AA",
+          "value": ["AA"]
+        },
+        "datasource": "gdev-testdata",
+        "definition": "$datacenter.*",
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "server",
+        "options": [],
+        "query": "$datacenter.*",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": null,
+        "current": {
+          "text": "All",
+          "value": ["$__all"]
+        },
+        "datasource": "gdev-testdata",
+        "definition": "$datacenter.$server.*",
+        "hide": 0,
+        "includeAll": true,
+        "label": null,
+        "multi": true,
+        "name": "pod",
+        "options": [],
+        "query": "$datacenter.$server.*",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
+  },
+  "timezone": "",
+  "title": "Templating - Nested Template Variables",
+  "uid": "-Y-tnEDWk",
+  "version": 11
+}

public/app/plugins/datasource/testdata/QueryEditor.tsx

@@ -11,7 +11,7 @@ import { SelectableValue } from '@grafana/data';
 
 // Types
 import { QueryEditorProps } from '@grafana/ui';
-import { TestDataDatasource } from './datasource';
+import { TestDataDataSource } from './datasource';
 import { TestDataQuery, Scenario } from './types';
 
 interface State {
@@ -19,7 +19,7 @@ interface State {
   current: Scenario | null;
 }
 
-type Props = QueryEditorProps<TestDataDatasource, TestDataQuery>;
+type Props = QueryEditorProps<TestDataDataSource, TestDataQuery>;
 
 export class QueryEditor extends PureComponent<Props> {
   backendSrv = getBackendSrv();

public/app/plugins/datasource/testdata/TestInfoTab.tsx

@@ -3,9 +3,9 @@ import React, { PureComponent } from 'react';
 
 // Types
 import { PluginConfigPageProps, DataSourcePlugin } from '@grafana/ui';
-import { TestDataDatasource } from './datasource';
+import { TestDataDataSource } from './datasource';
 
-interface Props extends PluginConfigPageProps<DataSourcePlugin<TestDataDatasource>> {}
+interface Props extends PluginConfigPageProps<DataSourcePlugin<TestDataDataSource>> {}
 
 export class TestInfoTab extends PureComponent<Props> {
   constructor(props: Props) {

public/app/plugins/datasource/testdata/datasource.ts

 import _ from 'lodash';
-import { DataSourceApi, DataQueryRequest, DataSourceInstanceSettings, DataStreamObserver } from '@grafana/ui';
-
+import {
+  DataSourceApi,
+  DataQueryRequest,
+  DataSourceInstanceSettings,
+  DataStreamObserver,
+  MetricFindValue,
+} from '@grafana/ui';
 import { TableData, TimeSeries } from '@grafana/data';
 import { TestDataQuery, Scenario } from './types';
 import { getBackendSrv } from 'app/core/services/backend_srv';
 import { StreamHandler } from './StreamHandler';
+import { queryMetricTree } from './metricTree';
+import templateSrv from 'app/features/templating/template_srv';
 
 type TestData = TimeSeries | TableData;
 
@@ -12,7 +19,7 @@ export interface TestDataRegistry {
   [key: string]: TestData[];
 }
 
-export class TestDataDatasource extends DataSourceApi<TestDataQuery> {
+export class TestDataDataSource extends DataSourceApi<TestDataQuery> {
   streams = new StreamHandler();
 
   /** @ngInject */
@@ -23,15 +30,11 @@ export class TestDataDatasource extends DataSourceApi<TestDataQuery> {
   query(options: DataQueryRequest<TestDataQuery>, observer: DataStreamObserver) {
     const queries = options.targets.map(item => {
       return {
-        refId: item.refId,
-        scenarioId: item.scenarioId,
+        ...item,
         intervalMs: options.intervalMs,
         maxDataPoints: options.maxDataPoints,
         datasourceId: this.id,
-        stringInput: item.stringInput,
-        points: item.points,
-        alias: item.alias,
-        ...item,
+        alias: templateSrv.replace(item.alias || ''),
       };
     });
 
@@ -122,4 +125,14 @@ export class TestDataDatasource extends DataSourceApi<TestDataQuery> {
   getScenarios(): Promise<Scenario[]> {
     return getBackendSrv().get('/api/tsdb/testdata/scenarios');
   }
+
+  metricFindQuery(query: string) {
+    return new Promise<MetricFindValue[]>((resolve, reject) => {
+      setTimeout(() => {
+        const children = queryMetricTree(templateSrv.replace(query));
+        const items = children.map(item => ({ value: item.name, text: item.name }));
+        resolve(items);
+      }, 100);
+    });
+  }
 }

public/app/plugins/datasource/testdata/metricTree.test.ts

+import { queryMetricTree } from './metricTree';
+
+describe('MetricTree', () => {
+  it('queryMetric tree return right tree nodes', () => {
+    const nodes = queryMetricTree('*');
+    expect(nodes[0].children[0].name).toBe('AA');
+    expect(nodes[0].children[1].name).toBe('AB');
+  });
+
+  it('queryMetric tree return right tree nodes', () => {
+    const nodes = queryMetricTree('A.AB.ABC.*');
+    expect(nodes[0].name).toBe('ABCA');
+  });
+
+  it('queryMetric tree supports glob paths', () => {
+    const nodes = queryMetricTree('A.{AB,AC}.*').map(i => i.name);
+    expect(nodes).toEqual(['ABA', 'ABB', 'ABC', 'ACA', 'ACB', 'ACC']);
+  });
+});

public/app/plugins/datasource/testdata/metricTree.ts

+export interface TreeNode {
+  name: string;
+  children: TreeNode[];
+}
+
+/*
+ *  Builds a nested tree like
+ *  [
+ *    {
+ *      name: 'A',
+ *      children: [
+ *        { name: 'AA', children: [] },
+ *        { name: 'AB', children: [] },
+ *      ]
+ *    }
+ *  ]
+ */
+function buildMetricTree(parent: string, depth: number): TreeNode[] {
+  const chars = ['A', 'B', 'C'];
+  const children: TreeNode[] = [];
+
+  if (depth > 3) {
+    return [];
+  }
+
+  for (const letter of chars) {
+    const nodeName = `${parent}${letter}`;
+    children.push({
+      name: nodeName,
+      children: buildMetricTree(nodeName, depth + 1),
+    });
+  }
+
+  return children;
+}
+
+function queryTree(children: TreeNode[], query: string[], queryIndex: number): TreeNode[] {
+  if (query[queryIndex] === '*') {
+    return children;
+  }
+
+  const nodeQuery = query[queryIndex];
+  let result: TreeNode[] = [];
+  let namesToMatch = [nodeQuery];
+
+  // handle glob queries
+  if (nodeQuery.startsWith('{')) {
+    namesToMatch = nodeQuery.replace(/\{|\}/g, '').split(',');
+  }
+
+  for (const node of children) {
+    for (const nameToMatch of namesToMatch) {
+      if (node.name === nameToMatch) {
+        result = result.concat(queryTree(node.children, query, queryIndex + 1));
+      }
+    }
+  }
+
+  return result;
+}
+
+export function queryMetricTree(query: string): TreeNode[] {
+  const children = buildMetricTree('', 0);
+  return queryTree(children, query.split('.'), 0);
+}

public/app/plugins/datasource/testdata/module.tsx

 import { DataSourcePlugin } from '@grafana/ui';
-import { TestDataDatasource } from './datasource';
+import { TestDataDataSource } from './datasource';
 import { TestDataQueryCtrl } from './query_ctrl';
 import { TestInfoTab } from './TestInfoTab';
 import { ConfigEditor } from './ConfigEditor';
@@ -10,7 +10,7 @@ class TestDataAnnotationsQueryCtrl {
   static template = '<h2>Annotation scenario</h2>';
 }
 
-export const plugin = new DataSourcePlugin(TestDataDatasource)
+export const plugin = new DataSourcePlugin(TestDataDataSource)
   .setConfigEditor(ConfigEditor)
   .setQueryCtrl(TestDataQueryCtrl)
   .setAnnotationQueryCtrl(TestDataAnnotationsQueryCtrl)