Commit 45a15657 by David Committed by GitHub

Merge pull request #12143 from grafana/davkal/ifql-datasource

InfluxDB IFQL datasource (beta)
parents 9e7651a1 64f58747
......@@ -157,6 +157,7 @@
"moment": "^2.18.1",
"mousetrap": "^1.6.0",
"mousetrap-global-bind": "^1.1.0",
"papaparse": "^4.4.0",
"prismjs": "^1.6.0",
"prop-types": "^15.6.0",
"react": "^16.2.0",
......
......@@ -85,6 +85,13 @@ func getFrontendSettingsMap(c *m.ReqContext) (map[string]interface{}, error) {
dsMap["database"] = ds.Database
dsMap["url"] = url
}
if ds.Type == m.DS_INFLUXDB_IFQL {
dsMap["username"] = ds.User
dsMap["password"] = ds.Password
dsMap["database"] = ds.Database
dsMap["url"] = url
}
}
if ds.Type == m.DS_ES {
......@@ -95,6 +102,10 @@ func getFrontendSettingsMap(c *m.ReqContext) (map[string]interface{}, error) {
dsMap["database"] = ds.Database
}
if ds.Type == m.DS_INFLUXDB_IFQL {
dsMap["database"] = ds.Database
}
if ds.Type == m.DS_PROMETHEUS {
// add unproxied server URL for link to Prometheus web UI
dsMap["directUrl"] = ds.Url
......
......@@ -12,6 +12,7 @@ const (
DS_GRAPHITE = "graphite"
DS_INFLUXDB = "influxdb"
DS_INFLUXDB_08 = "influxdb_08"
DS_INFLUXDB_IFQL = "influxdb-ifql"
DS_ES = "elasticsearch"
DS_OPENTSDB = "opentsdb"
DS_CLOUDWATCH = "cloudwatch"
......
......@@ -44,4 +44,8 @@ export default class TableModel {
this.columnMap[col.text] = col;
}
}
addRow(row) {
this.rows.push(row);
}
}
......@@ -4,6 +4,7 @@ import * as elasticsearchPlugin from 'app/plugins/datasource/elasticsearch/modul
import * as opentsdbPlugin from 'app/plugins/datasource/opentsdb/module';
import * as grafanaPlugin from 'app/plugins/datasource/grafana/module';
import * as influxdbPlugin from 'app/plugins/datasource/influxdb/module';
import * as influxdbIfqlPlugin from 'app/plugins/datasource/influxdb-ifql/module';
import * as mixedPlugin from 'app/plugins/datasource/mixed/module';
import * as mysqlPlugin from 'app/plugins/datasource/mysql/module';
import * as postgresPlugin from 'app/plugins/datasource/postgres/module';
......@@ -30,6 +31,7 @@ const builtInPlugins = {
'app/plugins/datasource/opentsdb/module': opentsdbPlugin,
'app/plugins/datasource/grafana/module': grafanaPlugin,
'app/plugins/datasource/influxdb/module': influxdbPlugin,
'app/plugins/datasource/influxdb-ifql/module': influxdbIfqlPlugin,
'app/plugins/datasource/mixed/module': mixedPlugin,
'app/plugins/datasource/mysql/module': mysqlPlugin,
'app/plugins/datasource/postgres/module': postgresPlugin,
......
# InfluxDB (IFQL) Datasource [BETA] - Native Plugin
Grafana ships with **built in** support for InfluxDB (>= 1.4.1).
Use this datasource if you want to use IFQL to query your InfluxDB.
Feel free to run this datasource side-by-side with the non-IFQL datasource.
If you point both datasources to the same InfluxDB instance, you can switch query mode by switching the datasources.
Read more about IFQL here:
[https://github.com/influxdata/ifql](https://github.com/influxdata/ifql)
Read more about InfluxDB here:
[http://docs.grafana.org/datasources/influxdb/](http://docs.grafana.org/datasources/influxdb/)
## Roadmap
- Sync Grafana time ranges with `range()`
- Template variable expansion
- Syntax highlighting
- Tab completion (functions, values)
- Result helpers (result counts, table previews)
- Annotations support
- Alerting integration
- Explore UI integration
import _ from 'lodash';
import * as dateMath from 'app/core/utils/datemath';
import { getTableModelFromResult, getTimeSeriesFromResult, parseResults } from './response_parser';
function serializeParams(params) {
if (!params) {
return '';
}
return _.reduce(
params,
(memo, value, key) => {
if (value === null || value === undefined) {
return memo;
}
memo.push(encodeURIComponent(key) + '=' + encodeURIComponent(value));
return memo;
},
[]
).join('&');
}
const MAX_SERIES = 20;
export default class InfluxDatasource {
type: string;
url: string;
username: string;
password: string;
name: string;
orgName: string;
database: any;
basicAuth: any;
withCredentials: any;
interval: any;
supportAnnotations: boolean;
supportMetrics: boolean;
/** @ngInject */
constructor(instanceSettings, private backendSrv, private templateSrv) {
this.type = 'influxdb-ifql';
this.url = instanceSettings.url.trim();
this.username = instanceSettings.username;
this.password = instanceSettings.password;
this.name = instanceSettings.name;
this.orgName = instanceSettings.orgName || 'defaultorgname';
this.database = instanceSettings.database;
this.basicAuth = instanceSettings.basicAuth;
this.withCredentials = instanceSettings.withCredentials;
this.interval = (instanceSettings.jsonData || {}).timeInterval;
this.supportAnnotations = true;
this.supportMetrics = true;
}
prepareQueries(options) {
const targets = _.cloneDeep(options.targets);
const timeFilter = this.getTimeFilter(options);
options.scopedVars.range = { value: timeFilter };
// Filter empty queries and replace grafana variables
const queryTargets = targets.filter(t => t.query).map(t => {
const interpolated = this.templateSrv.replace(t.query, options.scopedVars);
return {
...t,
query: interpolated,
};
});
return queryTargets;
}
query(options) {
const queryTargets = this.prepareQueries(options);
if (queryTargets.length === 0) {
return Promise.resolve({ data: [] });
}
const queries = queryTargets.map(target => {
const { query, resultFormat } = target;
if (resultFormat === 'table') {
return (
this._seriesQuery(query, options)
.then(response => parseResults(response.data))
// Keep only first result from each request
.then(results => results[0])
.then(getTableModelFromResult)
);
} else {
return this._seriesQuery(query, options)
.then(response => parseResults(response.data))
.then(results => results.map(getTimeSeriesFromResult));
}
});
return Promise.all(queries).then((series: any) => {
let seriesList = _.flattenDeep(series).slice(0, MAX_SERIES);
return { data: seriesList };
});
}
annotationQuery(options) {
if (!options.annotation.query) {
return Promise.reject({
message: 'Query missing in annotation definition',
});
}
var timeFilter = this.getTimeFilter({ rangeRaw: options.rangeRaw });
var query = options.annotation.query.replace('$timeFilter', timeFilter);
query = this.templateSrv.replace(query, null, 'regex');
return {};
}
metricFindQuery(query: string, options?: any) {
// TODO not implemented
var interpolated = this.templateSrv.replace(query, null, 'regex');
return this._seriesQuery(interpolated, options).then(_.curry(parseResults)(query));
}
_seriesQuery(query: string, options?: any) {
if (!query) {
return Promise.resolve({ data: '' });
}
return this._influxRequest('POST', '/v1/query', { q: query }, options);
}
testDatasource() {
const query = `from(db:"${this.database}") |> last()`;
return this._influxRequest('POST', '/v1/query', { q: query })
.then(res => {
if (res && res.trim()) {
return { status: 'success', message: 'Data source connected and database found.' };
}
return {
status: 'error',
message:
'Data source connected, but has no data. Verify the "Database" field and make sure the database has data.',
};
})
.catch(err => {
return { status: 'error', message: err.message };
});
}
_influxRequest(method: string, url: string, data: any, options?: any) {
let params: any = {
orgName: this.orgName,
};
if (this.username) {
params.u = this.username;
params.p = this.password;
}
// data sent as GET param
_.extend(params, data);
data = null;
let req: any = {
method: method,
url: this.url + url,
params: params,
data: data,
precision: 'ms',
inspect: { type: this.type },
paramSerializer: serializeParams,
};
req.headers = req.headers || {};
if (this.basicAuth || this.withCredentials) {
req.withCredentials = true;
}
if (this.basicAuth) {
req.headers.Authorization = this.basicAuth;
}
return this.backendSrv.datasourceRequest(req).then(
result => {
return result;
},
function(err) {
if (err.status !== 0 || err.status >= 300) {
if (err.data && err.data.error) {
throw {
message: 'InfluxDB Error: ' + err.data.error,
data: err.data,
config: err.config,
};
} else {
throw {
message: 'Network Error: ' + err.statusText + '(' + err.status + ')',
data: err.data,
config: err.config,
};
}
}
}
);
}
getTimeFilter(options) {
const from = this.getInfluxTime(options.rangeRaw.from, false);
const to = this.getInfluxTime(options.rangeRaw.to, true);
if (to === 'now') {
return `start: ${from}`;
}
return `start: ${from}, stop: ${to}`;
}
getInfluxTime(date, roundUp) {
if (_.isString(date)) {
if (date === 'now') {
return date;
}
const parts = /^now\s*-\s*(\d+)([d|h|m|s])$/.exec(date);
if (parts) {
const amount = parseInt(parts[1]);
const unit = parts[2];
return '-' + amount + unit;
}
date = dateMath.parse(date, roundUp);
}
return date.toISOString();
}
}
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 19.2.1, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 78.8 79.9" style="enable-background:new 0 0 78.8 79.9;" xml:space="preserve">
<style type="text/css">
.st0{fill:url(#symbol_1_);}
</style>
<g id="influxdb_logo">
<linearGradient id="symbol_1_" gradientUnits="userSpaceOnUse" x1="41.5273" y1="41.85" x2="319.2919" y2="41.85" gradientTransform="matrix(1 0 0 -1 0 81.8)">
<stop offset="0" style="stop-color:#4591ED"/>
<stop offset="1" style="stop-color:#00C9FF"/>
</linearGradient>
<path id="symbol_2_" class="st0" d="M78.7,48.2L71.1,15c-0.4-1.8-2.1-3.6-3.9-4.1L32.3,0.2c-0.5-0.1-1-0.2-1.5-0.2
c-1.5,0-3.1,0.6-4,1.5l-25,23.2c-1.3,1.2-2.1,3.6-1.7,5.4l8.1,35.5c0.4,1.8,2.1,3.6,3.9,4.1l32.6,10c0.5,0.1,1,0.2,1.5,0.2
c1.5,0,3.1-0.6,4-1.5l26.7-24.8C78.4,52.4,79.1,50,78.7,48.2z M35.9,8l23.9,7.3c0.9,0.3,0.9,0.7,0,0.9l-12.6,2.9
c-1,0.2-2.3-0.2-2.9-0.9l-8.8-9.5C34.8,8.1,35,7.8,35.9,8z M50.8,50.9c0.2,1-0.4,1.5-1.3,1.2l-25.8-7.9c-0.9-0.3-1.1-1.1-0.4-1.7
l19.8-18.4c0.7-0.7,1.5-0.4,1.7,0.5L50.8,50.9z M8.3,27.5L29.3,8c0.7-0.7,1.8-0.6,2.5,0.1l10.5,11.3c0.7,0.7,0.6,1.8-0.1,2.5
l-21,19.5c-0.7,0.7-1.8,0.6-2.5-0.1L8.2,30C7.6,29.3,7.6,28.2,8.3,27.5z M13.4,58.5L7.8,34.2c-0.2-1,0.1-1.1,0.8-0.4l8.8,9.5
c0.7,0.7,1,2.1,0.7,3l-3.8,12.3C14.1,59.4,13.6,59.4,13.4,58.5z M44.1,72.6l-27.3-8.4c-0.9-0.3-1.5-1.3-1.2-2.2l4.5-14.8
c0.3-0.9,1.3-1.5,2.2-1.2l27.3,8.4c0.9,0.3,1.5,1.3,1.2,2.2l-4.5,14.8C46,72.4,45,72.9,44.1,72.6z M68.4,52.7l-18.3,17
c-0.7,0.7-1.1,0.4-0.8-0.5l3.8-12.3c0.3-0.9,1.3-1.9,2.3-2.1L68,51.9C68.9,51.7,69.1,52.1,68.4,52.7z M70.4,49.1l-15.1,3.4
c-1,0.2-1.9-0.4-2.1-1.3l-6.4-27.9c-0.2-1,0.4-1.9,1.3-2.1l15.1-3.4c1-0.2,1.9,0.4,2.1,1.3L71.7,47C71.9,47.9,71.3,48.9,70.4,49.1z
"/>
</g>
</svg>
import InfluxDatasource from './datasource';
import { InfluxIfqlQueryCtrl } from './query_ctrl';
class InfluxConfigCtrl {
static templateUrl = 'partials/config.html';
}
class InfluxAnnotationsQueryCtrl {
static templateUrl = 'partials/annotations.editor.html';
}
export {
InfluxDatasource as Datasource,
InfluxIfqlQueryCtrl as QueryCtrl,
InfluxConfigCtrl as ConfigCtrl,
InfluxAnnotationsQueryCtrl as AnnotationsQueryCtrl,
};
<div class="gf-form-group">
<div class="gf-form">
<input type="text" class="gf-form-input" ng-model='ctrl.annotation.query' placeholder="select text from events where $timeFilter limit 1000"></input>
</div>
</div>
<h5 class="section-heading">Field mappings <tip>If your influxdb query returns more than one field you need to specify the column names below. An annotation event is composed of a title, tags, and an additional text field.</tip></h5>
<div class="gf-form-group">
<div class="gf-form-inline">
<div class="gf-form">
<span class="gf-form-label width-4">Text</span>
<input type="text" class="gf-form-input max-width-10" ng-model='ctrl.annotation.textColumn' placeholder=""></input>
</div>
<div class="gf-form">
<span class="gf-form-label width-4">Tags</span>
<input type="text" class="gf-form-input max-width-10" ng-model='ctrl.annotation.tagsColumn' placeholder=""></input>
</div>
<div class="gf-form" ng-show="ctrl.annotation.titleColumn">
<span class="gf-form-label width-4">Title <em class="muted">(deprecated)</em></span>
<input type="text" class="gf-form-input max-width-10" ng-model='ctrl.annotation.titleColumn' placeholder=""></input>
</div>
</div>
</div>
<datasource-http-settings current="ctrl.current" no-direct-access="true" suggest-url="http://localhost:8093">
</datasource-http-settings>
<h3 class="page-heading">InfluxDB Details</h3>
<div class="gf-form-group">
<div class="gf-form-inline">
<div class="gf-form max-width-30">
<span class="gf-form-label width-7">Default Database</span>
<input type="text" class="gf-form-input" ng-model='ctrl.current.database' placeholder="" required></input>
</div>
</div>
<div class="gf-form-inline">
<div class="gf-form max-width-15">
<span class="gf-form-label width-7">User</span>
<input type="text" class="gf-form-input" ng-model='ctrl.current.user' placeholder=""></input>
</div>
<div class="gf-form max-width-15">
<span class="gf-form-label width-7">Password</span>
<input type="password" class="gf-form-input" ng-model='ctrl.current.password' placeholder=""></input>
</div>
</div>
</div>
\ No newline at end of file
<query-editor-row query-ctrl="ctrl" can-collapse="true" has-text-edit-mode="true">
<div class="gf-form">
<textarea rows="3" class="gf-form-input" ng-model="ctrl.target.query" spellcheck="false" placeholder="IFQL Query" ng-model-onblur
ng-change="ctrl.refresh()"></textarea>
</div>
<div class="gf-form-inline">
<div class="gf-form">
<label class="gf-form-label query-keyword">FORMAT AS</label>
<div class="gf-form-select-wrapper">
<select class="gf-form-input gf-size-auto" ng-model="ctrl.target.resultFormat" ng-options="f.value as f.text for f in ctrl.resultFormats"
ng-change="ctrl.refresh()"></select>
</div>
</div>
<div class="gf-form max-width-25" ng-hide="ctrl.target.resultFormat === 'table'">
<label class="gf-form-label query-keyword">ALIAS BY</label>
<input type="text" class="gf-form-input" ng-model="ctrl.target.alias" spellcheck='false' placeholder="Naming pattern" ng-blur="ctrl.refresh()">
</div>
<div class="gf-form gf-form--grow">
<div class="gf-form-label gf-form-label--grow"></div>
</div>
</div>
</query-editor-row>
\ No newline at end of file
{
"type": "datasource",
"name": "InfluxDB (IFQL) [BETA]",
"id": "influxdb-ifql",
"defaultMatchFormat": "regex values",
"metrics": true,
"annotations": false,
"alerting": false,
"queryOptions": {
"minInterval": true
},
"info": {
"description": "InfluxDB Data Source for IFQL Queries for Grafana",
"author": {
"name": "Grafana Project",
"url": "https://grafana.com"
},
"logos": {
"small": "img/influxdb_logo.svg",
"large": "img/influxdb_logo.svg"
},
"version": "5.1.0"
}
}
\ No newline at end of file
import { QueryCtrl } from 'app/plugins/sdk';
function makeDefaultQuery(database) {
return `from(db: "${database}")
|> range($range)
|> limit(n:1000)
`;
}
export class InfluxIfqlQueryCtrl extends QueryCtrl {
static templateUrl = 'partials/query.editor.html';
resultFormats: any[];
/** @ngInject **/
constructor($scope, $injector) {
super($scope, $injector);
if (this.target.query === undefined) {
this.target.query = makeDefaultQuery(this.datasource.database);
}
this.resultFormats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }];
}
getCollapsedText() {
return this.target.query;
}
}
import Papa from 'papaparse';
import groupBy from 'lodash/groupBy';
import TableModel from 'app/core/table_model';
const filterColumnKeys = key => key && key[0] !== '_' && key !== 'result' && key !== 'table';
const IGNORE_FIELDS_FOR_NAME = ['result', '', 'table'];
export const getNameFromRecord = record => {
// Measurement and field
const metric = [record._measurement, record._field];
// Add tags
const tags = Object.keys(record)
.filter(key => key[0] !== '_')
.filter(key => IGNORE_FIELDS_FOR_NAME.indexOf(key) === -1)
.map(key => `${key}=${record[key]}`);
return [...metric, ...tags].join(' ');
};
const parseCSV = (input: string) =>
Papa.parse(input, {
header: true,
comments: '#',
}).data;
export const parseValue = (input: string) => {
const value = parseFloat(input);
return isNaN(value) ? null : value;
};
export const parseTime = (input: string) => Date.parse(input);
export function parseResults(response: string): any[] {
return response.trim().split(/\n\s*\s/);
}
export function getTableModelFromResult(result: string) {
const data = parseCSV(result);
const table = new TableModel();
if (data.length > 0) {
// First columns are fixed
const firstColumns = [
{ text: 'Time', id: '_time' },
{ text: 'Measurement', id: '_measurement' },
{ text: 'Field', id: '_field' },
];
// Dynamically add columns for tags
const firstRecord = data[0];
const tags = Object.keys(firstRecord)
.filter(filterColumnKeys)
.map(key => ({ id: key, text: key }));
const valueColumn = { id: '_value', text: 'Value' };
const columns = [...firstColumns, ...tags, valueColumn];
columns.forEach(c => table.addColumn(c));
// Add rows
data.forEach(record => {
const row = columns.map(c => record[c.id]);
table.addRow(row);
});
}
return table;
}
export function getTimeSeriesFromResult(result: string) {
const data = parseCSV(result);
if (data.length === 0) {
return [];
}
// Group results by table ID (assume one table per timeseries for now)
const tables = groupBy(data, 'table');
const seriesList = Object.keys(tables)
.map(id => tables[id])
.map(series => {
const datapoints = series.map(record => [parseValue(record._value), parseTime(record._time)]);
const alias = getNameFromRecord(series[0]);
return { datapoints, target: alias };
});
return seriesList;
}
import moment from 'moment';
import { TemplateSrv } from 'app/features/templating/template_srv';
import Datasource from '../datasource';
describe('InfluxDB (IFQL)', () => {
const templateSrv = new TemplateSrv();
const ds = new Datasource({ url: '' }, {}, templateSrv);
const DEFAULT_OPTIONS = {
rangeRaw: { to: 'now', from: 'now - 3h' },
scopedVars: {},
targets: [],
};
let queries: any[];
describe('prepareQueries()', () => {
it('filters empty queries', () => {
queries = ds.prepareQueries(DEFAULT_OPTIONS);
expect(queries.length).toBe(0);
queries = ds.prepareQueries({
...DEFAULT_OPTIONS,
targets: [{ query: '' }],
});
expect(queries.length).toBe(0);
});
it('replaces $range variable', () => {
queries = ds.prepareQueries({
...DEFAULT_OPTIONS,
targets: [{ query: 'from(db: "test") |> range($range)' }],
});
expect(queries.length).toBe(1);
expect(queries[0].query).toBe('from(db: "test") |> range(start: -3h)');
});
it('replaces $range variable with custom dates', () => {
const to = moment();
const from = moment().subtract(1, 'hours');
queries = ds.prepareQueries({
...DEFAULT_OPTIONS,
rangeRaw: { to, from },
targets: [{ query: 'from(db: "test") |> range($range)' }],
});
expect(queries.length).toBe(1);
const start = from.toISOString();
const stop = to.toISOString();
expect(queries[0].query).toBe(`from(db: "test") |> range(start: ${start}, stop: ${stop})`);
});
});
});
import {
getNameFromRecord,
getTableModelFromResult,
getTimeSeriesFromResult,
parseResults,
parseValue,
} from '../response_parser';
import response from './sample_response_csv';
describe('influxdb ifql response parser', () => {
describe('parseResults()', () => {
it('expects three results', () => {
const results = parseResults(response);
expect(results.length).toBe(2);
});
});
describe('getTableModelFromResult()', () => {
it('expects a table model', () => {
const results = parseResults(response);
const table = getTableModelFromResult(results[0]);
expect(table.columns.length).toBe(6);
expect(table.rows.length).toBe(300);
});
});
describe('getTimeSeriesFromResult()', () => {
it('expects time series', () => {
const results = parseResults(response);
const series = getTimeSeriesFromResult(results[0]);
expect(series.length).toBe(50);
expect(series[0].datapoints.length).toBe(6);
});
});
describe('getNameFromRecord()', () => {
it('expects name based on measurements and tags', () => {
const record = {
'': '',
result: '',
table: '0',
_start: '2018-06-02T06:35:25.651942602Z',
_stop: '2018-06-02T07:35:25.651942602Z',
_time: '2018-06-02T06:35:31Z',
_value: '0',
_field: 'usage_guest',
_measurement: 'cpu',
cpu: 'cpu-total',
host: 'kenobi-3.local',
};
expect(getNameFromRecord(record)).toBe('cpu usage_guest cpu=cpu-total host=kenobi-3.local');
});
});
describe('parseValue()', () => {
it('parses a number', () => {
expect(parseValue('42.3')).toBe(42.3);
});
it('parses a non-number to null', () => {
expect(parseValue('foo')).toBe(null);
});
});
});
......@@ -7928,6 +7928,10 @@ pako@~1.0.5:
version "1.0.6"
resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.6.tgz#0101211baa70c4bca4a0f63f2206e97b7dfaf258"
papaparse@^4.4.0:
version "4.4.0"
resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-4.4.0.tgz#6bcdbda80873e00cfb0bdcd7a4571c72a9a40168"
parallel-transform@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/parallel-transform/-/parallel-transform-1.1.0.tgz#d410f065b05da23081fcd10f28854c29bda33b06"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment