Skip to content

N
nexpie-grafana-theme
Commit 82ae7c6e by David Committed by GitHub
Options

Merge pull request #12167 from grafana/davkal/ifql-helpers 

Query helpers, Annotations, and Template Variables for IFQL datasource
parents 17a2ce13 cdba2bd1
Showing with 294 additions and 78 deletions
public/app/plugins/datasource/influxdb-ifql/README.md
...@@ -14,13 +14,16 @@ Read more about InfluxDB here: ...@@ -14,13 +14,16 @@ Read more about InfluxDB here:
[http://docs.grafana.org/datasources/influxdb/](http://docs.grafana.org/datasources/influxdb/) [http://docs.grafana.org/datasources/influxdb/](http://docs.grafana.org/datasources/influxdb/)
## Supported Template Variable Macros:
* List all measurements for a given database: `measurements(database)`
* List all tags for a given database and measurement: `tags(database, measurement)`
* List all tag values for a given database, measurement, and tag: `tag_valuess(database, measurement, tag)`
* List all field keys for a given database and measurement: `field_keys(database, measurement)`
## Roadmap ## Roadmap
- Sync Grafana time ranges with `range()`
- Template variable expansion
- Syntax highlighting - Syntax highlighting
- Tab completion (functions, values) - Tab completion (functions, values)
- Result helpers (result counts, table previews)
- Annotations support
- Alerting integration - Alerting integration
- Explore UI integration - Explore UI integration
public/app/plugins/datasource/influxdb-ifql/datasource.ts
...@@ -2,7 +2,14 @@ import _ from 'lodash'; ...@@ -2,7 +2,14 @@ import _ from 'lodash';
import * as dateMath from 'app/core/utils/datemath'; import * as dateMath from 'app/core/utils/datemath';
import { getTableModelFromResult, getTimeSeriesFromResult, parseResults } from './response_parser'; import {
getAnnotationsFromResult,
getTableModelFromResult,
getTimeSeriesFromResult,
getValuesFromResult,
parseResults,
} from './response_parser';
import expandMacros from './metric_find_query';
function serializeParams(params) { function serializeParams(params) {
if (!params) { if (!params) {
...@@ -54,25 +61,21 @@ export default class InfluxDatasource { ...@@ -54,25 +61,21 @@ export default class InfluxDatasource {
this.supportMetrics = true; this.supportMetrics = true;
} }
prepareQueries(options) { prepareQueryTarget(target, options) {
const targets = _.cloneDeep(options.targets); // Replace grafana variables
const timeFilter = this.getTimeFilter(options); const timeFilter = this.getTimeFilter(options);
options.scopedVars.range = { value: timeFilter }; options.scopedVars.range = { value: timeFilter };
const interpolated = this.templateSrv.replace(target.query, options.scopedVars);
// Filter empty queries and replace grafana variables return {
const queryTargets = targets.filter(t => t.query).map(t => { ...target,
const interpolated = this.templateSrv.replace(t.query, options.scopedVars); query: interpolated,
return { };
...t,
query: interpolated,
};
});
return queryTargets;
} }
query(options) { query(options) {
const queryTargets = this.prepareQueries(options); const queryTargets = options.targets
.filter(target => target.query)
.map(target => this.prepareQueryTarget(target, options));
if (queryTargets.length === 0) { if (queryTargets.length === 0) {
return Promise.resolve({ data: [] }); return Promise.resolve({ data: [] });
} }
...@@ -81,13 +84,9 @@ export default class InfluxDatasource { ...@@ -81,13 +84,9 @@ export default class InfluxDatasource {
const { query, resultFormat } = target; const { query, resultFormat } = target;
if (resultFormat === 'table') { if (resultFormat === 'table') {
return ( return this._seriesQuery(query, options)
this._seriesQuery(query, options) .then(response => parseResults(response.data))
.then(response => parseResults(response.data)) .then(results => results.map(getTableModelFromResult));
// Keep only first result from each request
.then(results => results[0])
.then(getTableModelFromResult)
);
} else { } else {
return this._seriesQuery(query, options) return this._seriesQuery(query, options)
.then(response => parseResults(response.data)) .then(response => parseResults(response.data))
...@@ -108,18 +107,42 @@ export default class InfluxDatasource { ...@@ -108,18 +107,42 @@ export default class InfluxDatasource {
}); });
} }
var timeFilter = this.getTimeFilter({ rangeRaw: options.rangeRaw }); const { query } = options.annotation;
var query = options.annotation.query.replace('$timeFilter', timeFilter); const queryOptions = {
query = this.templateSrv.replace(query, null, 'regex'); scopedVars: {},
...options,
silent: true,
};
const target = this.prepareQueryTarget({ query }, queryOptions);
return {}; return this._seriesQuery(target.query, queryOptions).then(response => {
const results = parseResults(response.data);
if (results.length === 0) {
throw { message: 'No results in response from InfluxDB' };
}
const annotations = _.flatten(results.map(result => getAnnotationsFromResult(result, options.annotation)));
return annotations;
});
} }
metricFindQuery(query: string, options?: any) { metricFindQuery(query: string, options?: any) {
// TODO not implemented const interpreted = expandMacros(query);
var interpolated = this.templateSrv.replace(query, null, 'regex');
// Use normal querier in silent mode
return this._seriesQuery(interpolated, options).then(_.curry(parseResults)(query)); const queryOptions = {
rangeRaw: { to: 'now', from: 'now - 1h' },
scopedVars: {},
...options,
silent: true,
};
const target = this.prepareQueryTarget({ query: interpreted }, queryOptions);
return this._seriesQuery(target.query, queryOptions).then(response => {
const results = parseResults(response.data);
const values = _.uniq(_.flatten(results.map(getValuesFromResult)));
return values
.filter(value => value && value[0] !== '_') // Ignore internal fields
.map(value => ({ text: value }));
});
} }
_seriesQuery(query: string, options?: any) { _seriesQuery(query: string, options?: any) {
......
public/app/plugins/datasource/influxdb-ifql/metric_find_query.ts 0 → 100644
// MACROS
// List all measurements for a given database: `measurements(database)`
const MEASUREMENTS_REGEXP = /^\s*measurements\((.+)\)\s*$/;
// List all tags for a given database and measurement: `tags(database, measurement)`
const TAGS_REGEXP = /^\s*tags\((.+)\s*,\s*(.+)\)\s*$/;
// List all tag values for a given database, measurement, and tag: `tag_valuess(database, measurement, tag)`
const TAG_VALUES_REGEXP = /^\s*tag_values\((.+)\s*,\s*(.+)\s*,\s*(.+)\)\s*$/;
// List all field keys for a given database and measurement: `field_keys(database, measurement)`
const FIELD_KEYS_REGEXP = /^\s*field_keys\((.+)\s*,\s*(.+)\)\s*$/;
export default function expandMacros(query) {
const measurementsQuery = query.match(MEASUREMENTS_REGEXP);
if (measurementsQuery) {
const database = measurementsQuery[1];
return `from(db:"${database}")
|> range($range)
|> group(by:["_measurement"])
|> distinct(column:"_measurement")
|> group(none:true)`;
}
const tagsQuery = query.match(TAGS_REGEXP);
if (tagsQuery) {
const database = tagsQuery[1];
const measurement = tagsQuery[2];
return `from(db:"${database}")
|> range($range)
|> filter(fn:(r) => r._measurement == "${measurement}")
|> keys()`;
}
const tagValuesQuery = query.match(TAG_VALUES_REGEXP);
if (tagValuesQuery) {
const database = tagValuesQuery[1];
const measurement = tagValuesQuery[2];
const tag = tagValuesQuery[3];
return `from(db:"${database}")
|> range($range)
|> filter(fn:(r) => r._measurement == "${measurement}")
|> group(by:["${tag}"])
|> distinct(column:"${tag}")
|> group(none:true)`;
}
const fieldKeysQuery = query.match(FIELD_KEYS_REGEXP);
if (fieldKeysQuery) {
const database = fieldKeysQuery[1];
const measurement = fieldKeysQuery[2];
return `from(db:"${database}")
|> range($range)
|> filter(fn:(r) => r._measurement == "${measurement}")
|> group(by:["_field"])
|> distinct(column:"_field")
|> group(none:true)`;
}
// By default return pure query
return query;
}
public/app/plugins/datasource/influxdb-ifql/partials/annotations.editor.html
<div class="gf-form-group"> <div class="gf-form-group">
<div class="gf-form"> <div class="gf-form">
<input type="text" class="gf-form-input" ng-model='ctrl.annotation.query' placeholder="select text from events where $timeFilter limit 1000"></input> <input type="text" class="gf-form-input" ng-model='ctrl.annotation.query' placeholder='from(db:"telegraf") |> range($range)'></input>
</div> </div>
</div> </div>
<h5 class="section-heading">Field mappings <tip>If your influxdb query returns more than one field you need to specify the column names below. An annotation event is composed of a title, tags, and an additional text field.</tip></h5> <h5 class="section-heading">Field mappings
<tip>If your influxdb query returns more than one field you need to specify the column names below. An annotation event is composed
of a title, tags, and an additional text field.</tip>
</h5>
<div class="gf-form-group"> <div class="gf-form-group">
<div class="gf-form-inline"> <div class="gf-form-inline">
<div class="gf-form"> <div class="gf-form">
...@@ -16,9 +18,5 @@ ...@@ -16,9 +18,5 @@
<span class="gf-form-label width-4">Tags</span> <span class="gf-form-label width-4">Tags</span>
<input type="text" class="gf-form-input max-width-10" ng-model='ctrl.annotation.tagsColumn' placeholder=""></input> <input type="text" class="gf-form-input max-width-10" ng-model='ctrl.annotation.tagsColumn' placeholder=""></input>
</div> </div>
<div class="gf-form" ng-show="ctrl.annotation.titleColumn">
<span class="gf-form-label width-4">Title <em class="muted">(deprecated)</em></span>
<input type="text" class="gf-form-input max-width-10" ng-model='ctrl.annotation.titleColumn' placeholder=""></input>
</div>
</div> </div>
</div> </div>
\ No newline at end of file
public/app/plugins/datasource/influxdb-ifql/partials/query.editor.html
<query-editor-row query-ctrl="ctrl" can-collapse="true" has-text-edit-mode="true"> <query-editor-row query-ctrl="ctrl" can-collapse="true" has-text-edit-mode="true">
<div class="gf-form"> <div class="gf-form">
<textarea rows="3" class="gf-form-input" ng-model="ctrl.target.query" spellcheck="false" placeholder="IFQL Query" ng-model-onblur <textarea rows="10" class="gf-form-input" ng-model="ctrl.target.query" spellcheck="false" placeholder="IFQL Query" ng-model-onblur
ng-change="ctrl.refresh()"></textarea> ng-change="ctrl.refresh()"></textarea>
<!-- Result preview -->
<textarea rows="10" class="gf-form-input" ng-model="ctrl.dataPreview" readonly></textarea>
</div> </div>
<div class="gf-form-inline"> <div class="gf-form-inline">
<div class="gf-form"> <div class="gf-form">
...@@ -12,9 +14,15 @@ ...@@ -12,9 +14,15 @@
ng-change="ctrl.refresh()"></select> ng-change="ctrl.refresh()"></select>
</div> </div>
</div> </div>
<div class="gf-form max-width-25" ng-hide="ctrl.target.resultFormat === 'table'"> <div class="gf-form" ng-if="ctrl.panelCtrl.loading">
<label class="gf-form-label query-keyword">ALIAS BY</label> <label class="gf-form-label">
<input type="text" class="gf-form-input" ng-model="ctrl.target.alias" spellcheck='false' placeholder="Naming pattern" ng-blur="ctrl.refresh()"> <i class="fa fa-spinner fa-spin"></i> Loading</label>
</div>
<div class="gf-form" ng-if="!ctrl.panelCtrl.loading">
<label class="gf-form-label">Result tables</label>
<input type="text" class="gf-form-input" ng-model="ctrl.resultTableCount" disabled="disabled">
<label class="gf-form-label">Result records</label>
<input type="text" class="gf-form-input" ng-model="ctrl.resultRecordCount" disabled="disabled">
</div> </div>
<div class="gf-form gf-form--grow"> <div class="gf-form gf-form--grow">
<div class="gf-form-label gf-form-label--grow"></div> <div class="gf-form-label gf-form-label--grow"></div>
......
public/app/plugins/datasource/influxdb-ifql/plugin.json
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
"id": "influxdb-ifql", "id": "influxdb-ifql",
"defaultMatchFormat": "regex values", "defaultMatchFormat": "regex values",
"metrics": true, "metrics": true,
"annotations": false, "annotations": true,
"alerting": false, "alerting": false,
"queryOptions": { "queryOptions": {
"minInterval": true "minInterval": true
......
public/app/plugins/datasource/influxdb-ifql/query_ctrl.ts
import appEvents from 'app/core/app_events';
import { QueryCtrl } from 'app/plugins/sdk'; import { QueryCtrl } from 'app/plugins/sdk';
function makeDefaultQuery(database) { function makeDefaultQuery(database) {
...@@ -9,18 +10,46 @@ function makeDefaultQuery(database) { ...@@ -9,18 +10,46 @@ function makeDefaultQuery(database) {
export class InfluxIfqlQueryCtrl extends QueryCtrl { export class InfluxIfqlQueryCtrl extends QueryCtrl {
static templateUrl = 'partials/query.editor.html'; static templateUrl = 'partials/query.editor.html';
dataPreview: string;
resultRecordCount: string;
resultTableCount: string;
resultFormats: any[]; resultFormats: any[];
/** @ngInject **/ /** @ngInject **/
constructor($scope, $injector) { constructor($scope, $injector) {
super($scope, $injector); super($scope, $injector);
this.resultRecordCount = '';
this.resultTableCount = '';
if (this.target.query === undefined) { if (this.target.query === undefined) {
this.target.query = makeDefaultQuery(this.datasource.database); this.target.query = makeDefaultQuery(this.datasource.database);
} }
this.resultFormats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }]; this.resultFormats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }];
appEvents.on('ds-request-response', this.onResponseReceived, $scope);
this.panelCtrl.events.on('refresh', this.onRefresh, $scope);
this.panelCtrl.events.on('data-received', this.onDataReceived, $scope);
} }
onDataReceived = dataList => {
this.resultRecordCount = dataList.reduce((count, model) => {
const records = model.type === 'table' ? model.rows.length : model.datapoints.length;
return count + records;
}, 0);
this.resultTableCount = dataList.length;
};
onResponseReceived = response => {
this.dataPreview = response.data;
};
onRefresh = () => {
this.dataPreview = '';
this.resultRecordCount = '';
this.resultTableCount = '';
};
getCollapsedText() { getCollapsedText() {
return this.target.query; return this.target.query;
} }
......
public/app/plugins/datasource/influxdb-ifql/response_parser.ts
import Papa from 'papaparse'; import Papa from 'papaparse';
import flatten from 'lodash/flatten';
import groupBy from 'lodash/groupBy'; import groupBy from 'lodash/groupBy';
import TableModel from 'app/core/table_model'; import TableModel from 'app/core/table_model';
...@@ -6,17 +7,25 @@ import TableModel from 'app/core/table_model'; ...@@ -6,17 +7,25 @@ import TableModel from 'app/core/table_model';
const filterColumnKeys = key => key && key[0] !== '_' && key !== 'result' && key !== 'table'; const filterColumnKeys = key => key && key[0] !== '_' && key !== 'result' && key !== 'table';
const IGNORE_FIELDS_FOR_NAME = ['result', '', 'table']; const IGNORE_FIELDS_FOR_NAME = ['result', '', 'table'];
export const getTagsFromRecord = record =>
Object.keys(record)
.filter(key => key[0] !== '_')
.filter(key => IGNORE_FIELDS_FOR_NAME.indexOf(key) === -1)
.reduce((tags, key) => {
tags[key] = record[key];
return tags;
}, {});
export const getNameFromRecord = record => { export const getNameFromRecord = record => {
// Measurement and field // Measurement and field
const metric = [record._measurement, record._field]; const metric = [record._measurement, record._field];
// Add tags // Add tags
const tags = Object.keys(record) const tags = getTagsFromRecord(record);
.filter(key => key[0] !== '_') const tagValues = Object.keys(tags).map(key => `${key}=${tags[key]}`);
.filter(key => IGNORE_FIELDS_FOR_NAME.indexOf(key) === -1)
.map(key => `${key}=${record[key]}`);
return [...metric, ...tags].join(' '); return [...metric, ...tagValues].join(' ');
}; };
const parseCSV = (input: string) => const parseCSV = (input: string) =>
...@@ -36,6 +45,33 @@ export function parseResults(response: string): any[] { ...@@ -36,6 +45,33 @@ export function parseResults(response: string): any[] {
return response.trim().split(/\n\s*\s/); return response.trim().split(/\n\s*\s/);
} }
export function getAnnotationsFromResult(result: string, options: any) {
const data = parseCSV(result);
if (data.length === 0) {
return [];
}
const annotations = [];
const textSelector = options.textCol || '_value';
const tagsSelector = options.tagsCol || '';
const tagSelection = tagsSelector.split(',').map(t => t.trim());
data.forEach(record => {
// Remove empty values, then split in different tags for comma separated values
const tags = getTagsFromRecord(record);
const tagValues = flatten(tagSelection.filter(tag => tags[tag]).map(tag => tags[tag].split(',')));
annotations.push({
annotation: options,
time: parseTime(record._time),
tags: tagValues,
text: record[textSelector],
});
});
return annotations;
}
export function getTableModelFromResult(result: string) { export function getTableModelFromResult(result: string) {
const data = parseCSV(result); const data = parseCSV(result);
...@@ -86,3 +122,8 @@ export function getTimeSeriesFromResult(result: string) { ...@@ -86,3 +122,8 @@ export function getTimeSeriesFromResult(result: string) {
return seriesList; return seriesList;
} }
export function getValuesFromResult(result: string) {
const data = parseCSV(result);
return data.map(record => record['_value']);
}
public/app/plugins/datasource/influxdb-ifql/specs/datasource.jest.ts
...@@ -13,41 +13,27 @@ describe('InfluxDB (IFQL)', () => { ...@@ -13,41 +13,27 @@ describe('InfluxDB (IFQL)', () => {
targets: [], targets: [],
}; };
let queries: any[]; describe('prepareQueryTarget()', () => {
let target: any;
describe('prepareQueries()', () => {
it('filters empty queries', () => {
queries = ds.prepareQueries(DEFAULT_OPTIONS);
expect(queries.length).toBe(0);
queries = ds.prepareQueries({
...DEFAULT_OPTIONS,
targets: [{ query: '' }],
});
expect(queries.length).toBe(0);
});
it('replaces $range variable', () => { it('replaces $range variable', () => {
queries = ds.prepareQueries({ target = ds.prepareQueryTarget({ query: 'from(db: "test") |> range($range)' }, DEFAULT_OPTIONS);
...DEFAULT_OPTIONS, expect(target.query).toBe('from(db: "test") |> range(start: -3h)');
targets: [{ query: 'from(db: "test") |> range($range)' }],
});
expect(queries.length).toBe(1);
expect(queries[0].query).toBe('from(db: "test") |> range(start: -3h)');
}); });
it('replaces $range variable with custom dates', () => { it('replaces $range variable with custom dates', () => {
const to = moment(); const to = moment();
const from = moment().subtract(1, 'hours'); const from = moment().subtract(1, 'hours');
queries = ds.prepareQueries({ target = ds.prepareQueryTarget(
...DEFAULT_OPTIONS, { query: 'from(db: "test") |> range($range)' },
rangeRaw: { to, from }, {
targets: [{ query: 'from(db: "test") |> range($range)' }], ...DEFAULT_OPTIONS,
}); rangeRaw: { to, from },
expect(queries.length).toBe(1); }
);
const start = from.toISOString(); const start = from.toISOString();
const stop = to.toISOString(); const stop = to.toISOString();
expect(queries[0].query).toBe(`from(db: "test") |> range(start: ${start}, stop: ${stop})`); expect(target.query).toBe(`from(db: "test") |> range(start: ${start}, stop: ${stop})`);
}); });
}); });
}); });
public/app/plugins/datasource/influxdb-ifql/specs/metric_find_query.jest.ts 0 → 100644
import expandMacros from '../metric_find_query';
describe('metric find query', () => {
describe('expandMacros()', () => {
it('returns a non-macro query unadulterated', () => {
const query = 'from(db:"telegraf") |> last()';
const result = expandMacros(query);
expect(result).toBe(query);
});
it('returns a measurement query for measurements()', () => {
const query = ' measurements(mydb) ';
const result = expandMacros(query).replace(/\s/g, '');
expect(result).toBe(
'from(db:"mydb")|>range($range)|>group(by:["_measurement"])|>distinct(column:"_measurement")|>group(none:true)'
);
});
it('returns a tags query for tags()', () => {
const query = ' tags(mydb , mymetric) ';
const result = expandMacros(query).replace(/\s/g, '');
expect(result).toBe('from(db:"mydb")|>range($range)|>filter(fn:(r)=>r._measurement=="mymetric")|>keys()');
});
it('returns a tag values query for tag_values()', () => {
const query = ' tag_values(mydb , mymetric, mytag) ';
const result = expandMacros(query).replace(/\s/g, '');
expect(result).toBe(
'from(db:"mydb")|>range($range)|>filter(fn:(r)=>r._measurement=="mymetric")' +
'|>group(by:["mytag"])|>distinct(column:"mytag")|>group(none:true)'
);
});
it('returns a field keys query for field_keys()', () => {
const query = ' field_keys(mydb , mymetric) ';
const result = expandMacros(query).replace(/\s/g, '');
expect(result).toBe(
'from(db:"mydb")|>range($range)|>filter(fn:(r)=>r._measurement=="mymetric")' +
'|>group(by:["_field"])|>distinct(column:"_field")|>group(none:true)'
);
});
});
});
public/app/plugins/datasource/influxdb-ifql/specs/response_parser.jest.ts
import { import {
getAnnotationsFromResult,
getNameFromRecord, getNameFromRecord,
getTableModelFromResult, getTableModelFromResult,
getTimeSeriesFromResult, getTimeSeriesFromResult,
getValuesFromResult,
parseResults, parseResults,
parseValue, parseValue,
} from '../response_parser'; } from '../response_parser';
...@@ -15,6 +17,17 @@ describe('influxdb ifql response parser', () => { ...@@ -15,6 +17,17 @@ describe('influxdb ifql response parser', () => {
}); });
}); });
describe('getAnnotationsFromResult()', () => {
it('expects a list of annotations', () => {
const results = parseResults(response);
const annotations = getAnnotationsFromResult(results[0], { tagsCol: 'cpu' });
expect(annotations.length).toBe(300);
expect(annotations[0].tags.length).toBe(1);
expect(annotations[0].tags[0]).toBe('cpu-total');
expect(annotations[0].text).toBe('0');
});
});
describe('getTableModelFromResult()', () => { describe('getTableModelFromResult()', () => {
it('expects a table model', () => { it('expects a table model', () => {
const results = parseResults(response); const results = parseResults(response);
...@@ -33,6 +46,14 @@ describe('influxdb ifql response parser', () => { ...@@ -33,6 +46,14 @@ describe('influxdb ifql response parser', () => {
}); });
}); });
describe('getValuesFromResult()', () => {
it('returns all values from the _value field in the response', () => {
const results = parseResults(response);
const values = getValuesFromResult(results[0]);
expect(values.length).toBe(300);
});
});
describe('getNameFromRecord()', () => { describe('getNameFromRecord()', () => {
it('expects name based on measurements and tags', () => { it('expects name based on measurements and tags', () => {
const record = { const record = {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment