Merge pull request #14052 from grafana/davkal/explore-query-importer

Explore: POC for datasource query importers

public/app/features/explore/Explore.tsx

@@ -3,8 +3,9 @@ import { hot } from 'react-hot-loader';
 import Select from 'react-select';
 import _ from 'lodash';
 
+import { DataSource } from 'app/types/datasources';
 import { ExploreState, ExploreUrlState, HistoryItem, Query, QueryTransaction, ResultType } from 'app/types/explore';
-import { RawTimeRange } from 'app/types/series';
+import { RawTimeRange, DataQuery } from 'app/types/series';
 import kbn from 'app/core/utils/kbn';
 import colors from 'app/core/utils/colors';
 import store from 'app/core/store';
@@ -16,6 +17,7 @@ import PickerOption from 'app/core/components/Picker/PickerOption';
 import IndicatorsContainer from 'app/core/components/Picker/IndicatorsContainer';
 import NoOptionsMessage from 'app/core/components/Picker/NoOptionsMessage';
 import TableModel, { mergeTablesIntoModel } from 'app/core/table_model';
+import { DatasourceSrv } from 'app/features/plugins/datasource_srv';
 
 import QueryRows from './QueryRows';
 import Graph from './Graph';
@@ -24,7 +26,6 @@ import Table from './Table';
 import ErrorBoundary from './ErrorBoundary';
 import TimePicker from './TimePicker';
 import { ensureQueries, generateQueryKey, hasQuery } from './utils/query';
-import { DataSource } from 'app/types/datasources';
 
 const MAX_HISTORY_ITEMS = 100;
 
@@ -77,7 +78,7 @@ function updateHistory(history: HistoryItem[], datasourceId: string, queries: st
 }
 
 interface ExploreProps {
-  datasourceSrv: any;
+  datasourceSrv: DatasourceSrv;
   onChangeSplit: (split: boolean, state?: ExploreState) => void;
   onSaveState: (key: string, state: ExploreState) => void;
   position: string;
@@ -92,6 +93,7 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
   /**
    * Current query expressions of the rows including their modifications, used for running queries.
    * Not kept in component state to prevent edit-render roundtrips.
+   * TODO: make this generic (other datasources might not have string representations of current query state)
    */
   queryExpressions: string[];
   /**
@@ -164,7 +166,7 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
     }
   }
 
-  async setDatasource(datasource: DataSource) {
+  async setDatasource(datasource: any, origin?: DataSource) {
     const supportsGraph = datasource.meta.metrics;
     const supportsLogs = datasource.meta.logs;
     const supportsTable = datasource.meta.metrics;
@@ -193,12 +195,33 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
       datasource.init();
     }
 
-    // Keep queries but reset edit state
+    // Check if queries can be imported from previously selected datasource
+    let queryExpressions = this.queryExpressions;
+    if (origin) {
+      if (origin.meta.id === datasource.meta.id) {
+        // Keep same queries if same type of datasource
+        queryExpressions = [...this.queryExpressions];
+      } else if (datasource.importQueries) {
+        // Datasource-specific importers, wrapping to satisfy interface
+        const wrappedQueries: DataQuery[] = this.queryExpressions.map((query, index) => ({
+          refId: String(index),
+          expr: query,
+        }));
+        const modifiedQueries: DataQuery[] = await datasource.importQueries(wrappedQueries, origin.meta);
+        queryExpressions = modifiedQueries.map(({ expr }) => expr);
+      } else {
+        // Default is blank queries
+        queryExpressions = this.queryExpressions.map(() => '');
+      }
+    }
+
+    // Reset edit state with new queries
     const nextQueries = this.state.queries.map((q, i) => ({
       ...q,
       key: generateQueryKey(i),
-      query: this.queryExpressions[i],
+      query: queryExpressions[i],
     }));
+    this.queryExpressions = queryExpressions;
 
     // Custom components
     const StartPage = datasource.pluginExports.ExploreStartPage;
@@ -258,6 +281,7 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
   };
 
   onChangeDatasource = async option => {
+    const origin = this.state.datasource;
     this.setState({
       datasource: null,
       datasourceError: null,
@@ -266,7 +290,7 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
     });
     const datasourceName = option.value;
     const datasource = await this.props.datasourceSrv.get(datasourceName);
-    this.setDatasource(datasource);
+    this.setDatasource(datasource as any, origin);
   };
 
   onChangeQuery = (value: string, index: number, override?: boolean) => {

public/app/features/plugins/datasource_srv.ts

@@ -22,7 +22,7 @@ export class DatasourceSrv {
     this.datasources = {};
   }
 
-  get(name?): Promise<DataSourceApi> {
+  get(name?: string): Promise<DataSourceApi> {
     if (!name) {
       return this.get(config.defaultDatasource);
     }
@@ -40,7 +40,7 @@ export class DatasourceSrv {
     return this.loadDatasource(name);
   }
 
-  loadDatasource(name) {
+  loadDatasource(name: string): Promise<DataSourceApi> {
     const dsConfig = config.datasources[name];
     if (!dsConfig) {
       return this.$q.reject({ message: 'Datasource named ' + name + ' was not found' });

public/app/plugins/datasource/logging/datasource.ts

 import _ from 'lodash';
 
 import * as dateMath from 'app/core/utils/datemath';
+import { LogsStream, LogsModel, makeSeriesForLogs } from 'app/core/logs_model';
+import { PluginMeta, DataQuery } from 'app/types';
 
 import LanguageProvider from './language_provider';
 import { mergeStreamsToLogs } from './result_transformer';
-import { LogsStream, LogsModel, makeSeriesForLogs } from 'app/core/logs_model';
 
 export const DEFAULT_LIMIT = 1000;
 
@@ -111,6 +112,10 @@ export default class LoggingDatasource {
     });
   }
 
+  async importQueries(queries: DataQuery[], originMeta: PluginMeta): Promise<DataQuery[]> {
+    return this.languageProvider.importQueries(queries, originMeta.id);
+  }
+
   metadataRequest(url) {
     // HACK to get label values for {job=|}, will be replaced when implementing LoggingQueryField
     const apiUrl = url.replace('v1', 'prom');

public/app/plugins/datasource/logging/language_provider.test.ts

+import Plain from 'slate-plain-serializer';
+
+import LanguageProvider from './language_provider';
+
+describe('Language completion provider', () => {
+  const datasource = {
+    metadataRequest: () => ({ data: { data: [] } }),
+  };
+
+  it('returns default suggestions on emtpty context', () => {
+    const instance = new LanguageProvider(datasource);
+    const result = instance.provideCompletionItems({ text: '', prefix: '', wrapperClasses: [] });
+    expect(result.context).toBeUndefined();
+    expect(result.refresher).toBeUndefined();
+    expect(result.suggestions.length).toEqual(0);
+  });
+
+  describe('label suggestions', () => {
+    it('returns default label suggestions on label context', () => {
+      const instance = new LanguageProvider(datasource);
+      const value = Plain.deserialize('{}');
+      const range = value.selection.merge({
+        anchorOffset: 1,
+      });
+      const valueWithSelection = value.change().select(range).value;
+      const result = instance.provideCompletionItems({
+        text: '',
+        prefix: '',
+        wrapperClasses: ['context-labels'],
+        value: valueWithSelection,
+      });
+      expect(result.context).toBe('context-labels');
+      expect(result.suggestions).toEqual([{ items: [{ label: 'job' }, { label: 'namespace' }], label: 'Labels' }]);
+    });
+  });
+});
+
+describe('Query imports', () => {
+  const datasource = {
+    metadataRequest: () => ({ data: { data: [] } }),
+  };
+
+  it('returns empty queries for unknown origin datasource', async () => {
+    const instance = new LanguageProvider(datasource);
+    const result = await instance.importQueries([{ refId: 'bar', expr: 'foo' }], 'unknown');
+    expect(result).toEqual([{ refId: 'bar', expr: '' }]);
+  });
+
+  describe('prometheus query imports', () => {
+    it('returns empty query from metric-only query', async () => {
+      const instance = new LanguageProvider(datasource);
+      const result = await instance.importPrometheusQuery('foo');
+      expect(result).toEqual('');
+    });
+
+    it('returns empty query from selector query if label is not available', async () => {
+      const datasourceWithLabels = {
+        metadataRequest: url => (url === '/api/prom/label' ? { data: { data: ['other'] } } : { data: { data: [] } }),
+      };
+      const instance = new LanguageProvider(datasourceWithLabels);
+      const result = await instance.importPrometheusQuery('{foo="bar"}');
+      expect(result).toEqual('{}');
+    });
+
+    it('returns selector query from selector query with common labels', async () => {
+      const datasourceWithLabels = {
+        metadataRequest: url => (url === '/api/prom/label' ? { data: { data: ['foo'] } } : { data: { data: [] } }),
+      };
+      const instance = new LanguageProvider(datasourceWithLabels);
+      const result = await instance.importPrometheusQuery('metric{foo="bar",baz="42"}');
+      expect(result).toEqual('{foo="bar"}');
+    });
+  });
+});

public/app/plugins/datasource/logging/language_provider.ts

@@ -8,9 +8,9 @@ import {
   TypeaheadInput,
   TypeaheadOutput,
 } from 'app/types/explore';
-
-import { parseSelector } from 'app/plugins/datasource/prometheus/language_utils';
+import { parseSelector, labelRegexp, selectorRegexp } from 'app/plugins/datasource/prometheus/language_utils';
 import PromqlSyntax from 'app/plugins/datasource/prometheus/promql';
+import { DataQuery } from 'app/types';
 
 const DEFAULT_KEYS = ['job', 'namespace'];
 const EMPTY_SELECTOR = '{}';
@@ -158,6 +158,56 @@ export default class LoggingLanguageProvider extends LanguageProvider {
     return { context, refresher, suggestions };
   }
 
+  async importQueries(queries: DataQuery[], datasourceType: string): Promise<DataQuery[]> {
+    if (datasourceType === 'prometheus') {
+      return Promise.all(
+        queries.map(async query => {
+          const expr = await this.importPrometheusQuery(query.expr);
+          return {
+            ...query,
+            expr,
+          };
+        })
+      );
+    }
+    return queries.map(query => ({
+      ...query,
+      expr: '',
+    }));
+  }
+
+  async importPrometheusQuery(query: string): Promise<string> {
+    // Consider only first selector in query
+    const selectorMatch = query.match(selectorRegexp);
+    if (selectorMatch) {
+      const selector = selectorMatch[0];
+      const labels = {};
+      selector.replace(labelRegexp, (_, key, operator, value) => {
+        labels[key] = { value, operator };
+        return '';
+      });
+
+      // Keep only labels that exist on origin and target datasource
+      await this.start(); // fetches all existing label keys
+      const commonLabels = {};
+      for (const key in labels) {
+        const existingKeys = this.labelKeys[EMPTY_SELECTOR];
+        if (existingKeys.indexOf(key) > -1) {
+          // Should we check for label value equality here?
+          commonLabels[key] = labels[key];
+        }
+      }
+      const labelKeys = Object.keys(commonLabels).sort();
+      const cleanSelector = labelKeys
+        .map(key => `${key}${commonLabels[key].operator}${commonLabels[key].value}`)
+        .join(',');
+
+      return ['{', cleanSelector, '}'].join('');
+    }
+
+    return '';
+  }
+
   async fetchLogLabels() {
     const url = '/api/prom/label';
     try {

public/app/plugins/datasource/prometheus/language_utils.ts

@@ -24,8 +24,8 @@ export function processLabels(labels, withName = false) {
 }
 
 // const cleanSelectorRegexp = /\{(\w+="[^"\n]*?")(,\w+="[^"\n]*?")*\}/;
-const selectorRegexp = /\{[^}]*?\}/;
-const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g;
+export const selectorRegexp = /\{[^}]*?\}/;
+export const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g;
 export function parseSelector(query: string, cursorOffset = 1): { labelKeys: any[]; selector: string } {
   if (!query.match(selectorRegexp)) {
     // Special matcher for metrics

public/app/types/datasources.ts

@@ -18,8 +18,6 @@ export interface DataSource {
   readOnly: boolean;
   meta?: PluginMeta;
   pluginExports?: PluginExports;
-  init?: () => void;
-  testDatasource?: () => Promise<any>;
 }
 
 export interface DataSourcesState {

public/app/types/series.ts

 import { Moment } from 'moment';
+import { PluginMeta } from './plugins';
 
 export enum LoadingState {
   NotStarted = 'NotStarted',
@@ -70,6 +71,7 @@ export interface DataQueryResponse {
 
 export interface DataQuery {
   refId: string;
+  [key: string]: any;
 }
 
 export interface DataQueryOptions {
@@ -87,5 +89,14 @@ export interface DataQueryOptions {
 }
 
 export interface DataSourceApi {
+  /**
+   * Imports queries from a different datasource
+   */
+  importQueries?(queries: DataQuery[], originMeta: PluginMeta): Promise<DataQuery[]>;
+  /**
+   * Initializes a datasource after instantiation
+   */
+  init?: () => void;
   query(options: DataQueryOptions): Promise<DataQueryResponse>;
+  testDatasource?: () => Promise<any>;
 }