Skip to content

N
nexpie-grafana-theme
Commit a230aa10 by Zoltán Bedi Committed by GitHub
Options

Prometheus: refactor to DataFrame (#27737) 

* Add typing to prometheus response

* Refactor result_transformer to return DataFrame

* Refactor + test fixes

* Fix Prometheus data source test

* Modify heatmap function + add back tests

* Update performInstantQuery return type

* Remove duplicate code from result_transformer

* Address review comments

* Update metric labels retrival logic to be safer
parent f97f12f6
Showing with 570 additions and 580 deletions
packages/grafana-runtime/src/services/backendSrv.ts
......@@ -108,7 +108,7 @@ export interface FetchErrorDataProps {
export interface FetchError<T extends FetchErrorDataProps = any> {
status: number;
statusText?: string;
data: T | string;
data: T;
cancelled?: boolean;
isHandled?: boolean;
config: BackendSrvRequest;
......
public/app/plugins/datasource/prometheus/datasource.test.ts
......@@ -681,32 +681,32 @@ describe('PrometheusDatasource', () => {
it('should be same length', () => {
expect(results.data.length).toBe(2);
expect(results.data[0].datapoints.length).toBe((end - start) / step + 1);
expect(results.data[1].datapoints.length).toBe((end - start) / step + 1);
expect(results.data[0].length).toBe((end - start) / step + 1);
expect(results.data[1].length).toBe((end - start) / step + 1);
});
it('should fill null until first datapoint in response', () => {
expect(results.data[0].datapoints[0][1]).toBe(start * 1000);
expect(results.data[0].datapoints[0][0]).toBe(null);
expect(results.data[0].datapoints[1][1]).toBe((start + step * 1) * 1000);
expect(results.data[0].datapoints[1][0]).toBe(3846);
expect(results.data[0].fields[0].values.get(0)).toBe(start * 1000);
expect(results.data[0].fields[1].values.get(0)).toBe(null);
expect(results.data[0].fields[0].values.get(1)).toBe((start + step * 1) * 1000);
expect(results.data[0].fields[1].values.get(1)).toBe(3846);
});
it('should fill null after last datapoint in response', () => {
const length = (end - start) / step + 1;
expect(results.data[0].datapoints[length - 2][1]).toBe((end - step * 1) * 1000);
expect(results.data[0].datapoints[length - 2][0]).toBe(3848);
expect(results.data[0].datapoints[length - 1][1]).toBe(end * 1000);
expect(results.data[0].datapoints[length - 1][0]).toBe(null);
expect(results.data[0].fields[0].values.get(length - 2)).toBe((end - step * 1) * 1000);
expect(results.data[0].fields[1].values.get(length - 2)).toBe(3848);
expect(results.data[0].fields[0].values.get(length - 1)).toBe(end * 1000);
expect(results.data[0].fields[1].values.get(length - 1)).toBe(null);
});
it('should fill null at gap between series', () => {
expect(results.data[0].datapoints[2][1]).toBe((start + step * 2) * 1000);
expect(results.data[0].datapoints[2][0]).toBe(null);
expect(results.data[1].datapoints[1][1]).toBe((start + step * 1) * 1000);
expect(results.data[1].datapoints[1][0]).toBe(null);
expect(results.data[1].datapoints[3][1]).toBe((start + step * 3) * 1000);
expect(results.data[1].datapoints[3][0]).toBe(null);
expect(results.data[0].fields[0].values.get(2)).toBe((start + step * 2) * 1000);
expect(results.data[0].fields[1].values.get(2)).toBe(null);
expect(results.data[1].fields[0].values.get(1)).toBe((start + step * 1) * 1000);
expect(results.data[1].fields[1].values.get(1)).toBe(null);
expect(results.data[1].fields[0].values.get(3)).toBe((start + step * 3) * 1000);
expect(results.data[1].fields[1].values.get(3)).toBe(null);
});
});
......
public/app/plugins/datasource/prometheus/datasource.ts
// Libraries
import cloneDeep from 'lodash/cloneDeep';
import LRU from 'lru-cache';
// Services & Utils
import {
AnnotationEvent,
CoreApp,
DataQueryError,
DataQueryRequest,
DataQueryResponse,
DataQueryResponseData,
DataSourceApi,
DataSourceInstanceSettings,
dateMath,
DateTime,
LoadingState,
rangeUtil,
ScopedVars,
TimeRange,
TimeSeries,
rangeUtil,
} from '@grafana/data';
import { forkJoin, merge, Observable, of, throwError } from 'rxjs';
import { BackendSrvRequest, FetchError, getBackendSrv } from '@grafana/runtime';
import { safeStringifyValue } from 'app/core/utils/explore';
import { getTimeSrv } from 'app/features/dashboard/services/TimeSrv';
import templateSrv from 'app/features/templating/template_srv';
import cloneDeep from 'lodash/cloneDeep';
import defaults from 'lodash/defaults';
import LRU from 'lru-cache';
import { forkJoin, merge, Observable, of, pipe, throwError } from 'rxjs';
import { catchError, filter, map, tap } from 'rxjs/operators';
import PrometheusMetricFindQuery from './metric_find_query';
import { ResultTransformer } from './result_transformer';
import PrometheusLanguageProvider from './language_provider';
import { BackendSrvRequest, getBackendSrv } from '@grafana/runtime';
import addLabelToQuery from './add_label_to_query';
import { getQueryHints } from './query_hints';
import PrometheusLanguageProvider from './language_provider';
import { expandRecordingRules } from './language_utils';
// Types
import { PromOptions, PromQuery, PromQueryRequest } from './types';
import { safeStringifyValue } from 'app/core/utils/explore';
import templateSrv from 'app/features/templating/template_srv';
import { getTimeSrv } from 'app/features/dashboard/services/TimeSrv';
import TableModel from 'app/core/table_model';
import { defaults } from 'lodash';
import PrometheusMetricFindQuery from './metric_find_query';
import { getQueryHints } from './query_hints';
import { getOriginalMetricName, renderTemplate, transform } from './result_transformer';
import {
isFetchErrorResponse,
PromDataErrorResponse,
PromDataSuccessResponse,
PromMatrixData,
PromOptions,
PromQuery,
PromQueryRequest,
PromScalarData,
PromVectorData,
} from './types';
export const ANNOTATION_QUERY_STEP_DEFAULT = '60s';
export interface PromDataQueryResponse {
data: {
status: string;
data: {
resultType: string;
results?: DataQueryResponseData[];
result?: DataQueryResponseData[];
};
};
cancelled?: boolean;
}
export interface PromLabelQueryResponse {
data: {
status: string;
data: string[];
};
cancelled?: boolean;
}
export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions> {
type: string;
editorSrc: string;
......@@ -73,7 +56,6 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
httpMethod: string;
languageProvider: PrometheusLanguageProvider;
lookupsDisabled: boolean;
resultTransformer: ResultTransformer;
customQueryParameters: any;
constructor(instanceSettings: DataSourceInstanceSettings<PromOptions>) {
......@@ -88,7 +70,6 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
this.queryTimeout = instanceSettings.jsonData.queryTimeout;
this.httpMethod = instanceSettings.jsonData.httpMethod || 'GET';
this.directUrl = instanceSettings.jsonData.directUrl;
this.resultTransformer = new ResultTransformer(templateSrv);
this.ruleMappings = {};
this.languageProvider = new PrometheusLanguageProvider(this);
this.lookupsDisabled = instanceSettings.jsonData.disableMetricsLookup ?? false;
......@@ -172,38 +153,6 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
return templateSrv.variableExists(target.expr);
}
processResult = (
response: any,
query: PromQueryRequest,
target: PromQuery,
responseListLength: number,
scopedVars?: ScopedVars,
mixedQueries?: boolean
) => {
// Keeping original start/end for transformers
const transformerOptions = {
format: target.format,
step: query.step,
legendFormat: target.legendFormat,
start: query.start,
end: query.end,
query: query.expr,
responseListLength,
scopedVars,
refId: target.refId,
valueWithRefId: target.valueWithRefId,
meta: {
/** Fix for showing of Prometheus results in Explore table.
* We want to show result of instant query always in table and result of range query based on target.runAll;
*/
preferredVisualisationType: target.instant ? 'table' : mixedQueries ? 'graph' : undefined,
},
};
const series = this.resultTransformer.transform(response, transformerOptions);
return series;
};
prepareTargets = (options: DataQueryRequest<PromQuery>, start: number, end: number) => {
const queries: PromQueryRequest[] = [];
const activeTargets: PromQuery[] = [];
......@@ -283,17 +232,13 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
const subQueries = queries.map((query, index) => {
const target = activeTargets[index];
let observable = query.instant
? this.performInstantQuery(query, end)
: this.performTimeSeriesQuery(query, query.start, query.end);
return observable.pipe(
const filterAndMapResponse = pipe(
// Decrease the counter here. We assume that each request returns only single value and then completes
// (should hold until there is some streaming requests involved).
tap(() => runningQueriesCount--),
filter((response: any) => (response.cancelled ? false : true)),
map((response: any) => {
const data = this.processResult(response, query, target, queries.length, undefined, mixedQueries);
const data = transform(response, { query, target, responseListLength: queries.length, mixedQueries });
return {
data,
key: query.requestId,
......@@ -301,6 +246,12 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
} as DataQueryResponse;
})
);
if (query.instant) {
return this.performInstantQuery(query, end).pipe(filterAndMapResponse);
}
return this.performTimeSeriesQuery(query, query.start, query.end).pipe(filterAndMapResponse);
});
return merge(...subQueries);
......@@ -313,24 +264,26 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
requestId: string,
scopedVars: ScopedVars
) {
const observables: Array<Observable<Array<TableModel | TimeSeries>>> = queries.map((query, index) => {
const observables = queries.map((query, index) => {
const target = activeTargets[index];
let observable = query.instant
? this.performInstantQuery(query, end)
: this.performTimeSeriesQuery(query, query.start, query.end);
return observable.pipe(
const filterAndMapResponse = pipe(
filter((response: any) => (response.cancelled ? false : true)),
map((response: any) => {
const data = this.processResult(response, query, target, queries.length, scopedVars);
const data = transform(response, { query, target, responseListLength: queries.length, scopedVars });
return data;
})
);
if (query.instant) {
return this.performInstantQuery(query, end).pipe(filterAndMapResponse);
}
return this.performTimeSeriesQuery(query, query.start, query.end).pipe(filterAndMapResponse);
});
return forkJoin(observables).pipe(
map((results: Array<Array<TableModel | TimeSeries>>) => {
map(results => {
const data = results.reduce((result, current) => {
return [...result, ...current];
}, []);
......@@ -465,8 +418,11 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
}
}
return this._request(url, data, { requestId: query.requestId, headers: query.headers }).pipe(
catchError(err => {
return this._request<PromDataSuccessResponse<PromMatrixData>>(url, data, {
requestId: query.requestId,
headers: query.headers,
}).pipe(
catchError((err: FetchError<PromDataErrorResponse<PromMatrixData>>) => {
if (err.cancelled) {
return of(err);
}
......@@ -493,8 +449,11 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
}
}
return this._request(url, data, { requestId: query.requestId, headers: query.headers }).pipe(
catchError(err => {
return this._request<PromDataSuccessResponse<PromVectorData | PromScalarData>>(url, data, {
requestId: query.requestId,
headers: query.headers,
}).pipe(
catchError((err: FetchError<PromDataErrorResponse<PromVectorData | PromScalarData>>) => {
if (err.cancelled) {
return of(err);
}
......@@ -587,17 +546,11 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
};
const query = this.createQuery(queryModel, queryOptions, start, end);
const self = this;
const response: PromDataQueryResponse = await this.performTimeSeriesQuery(
query,
query.start,
query.end
).toPromise();
const response = await this.performTimeSeriesQuery(query, query.start, query.end).toPromise();
const eventList: AnnotationEvent[] = [];
const splitKeys = tagKeys.split(',');
if (response.cancelled) {
if (isFetchErrorResponse(response) && response.cancelled) {
return [];
}
......@@ -620,8 +573,8 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
value[0] = timestampValue;
});
const activeValues = series.values.filter((value: Record<number, string>) => parseFloat(value[1]) >= 1);
const activeValuesTimestamps: number[] = activeValues.map((value: number[]) => value[0]);
const activeValues = series.values.filter(value => parseFloat(value[1]) >= 1);
const activeValuesTimestamps = activeValues.map(value => value[0]);
// Instead of creating singular annotation for each active event we group events into region if they are less
// then `step` apart.
......@@ -644,9 +597,9 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
time: timestamp,
timeEnd: timestamp,
annotation,
title: self.resultTransformer.renderTemplate(titleFormat, series.metric),
title: renderTemplate(titleFormat, series.metric),
tags,
text: self.resultTransformer.renderTemplate(textFormat, series.metric),
text: renderTemplate(textFormat, series.metric),
};
}
......@@ -676,7 +629,7 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
const response = await this.performInstantQuery(query, now / 1000).toPromise();
return response.data.status === 'success'
? { status: 'success', message: 'Data source is working' }
: { status: 'error', message: response.error };
: { status: 'error', message: response.data.error };
}
interpolateVariablesInQueries(queries: PromQuery[], scopedVars: ScopedVars): PromQuery[] {
......@@ -764,7 +717,7 @@ export class PrometheusDatasource extends DataSourceApi<PromQuery, PromOptions>
}
getOriginalMetricName(labelData: { [key: string]: string }) {
return this.resultTransformer.getOriginalMetricName(labelData);
return getOriginalMetricName(labelData);
}
}
......
public/app/plugins/datasource/prometheus/metric_find_query.ts
import _ from 'lodash';
import { map } from 'rxjs/operators';
import { MetricFindValue, TimeRange } from '@grafana/data';
import { PromDataQueryResponse, PrometheusDatasource } from './datasource';
import { PrometheusDatasource } from './datasource';
import { PromQueryRequest } from './types';
import { getTimeSrv } from 'app/features/dashboard/services/TimeSrv';
......@@ -137,7 +137,7 @@ export default class PrometheusMetricFindQuery {
const end = this.datasource.getPrometheusTime(this.range.to, true);
const instantQuery: PromQueryRequest = { expr: query } as PromQueryRequest;
return this.datasource.performInstantQuery(instantQuery, end).pipe(
map((result: PromDataQueryResponse) => {
map(result => {
return _.map(result.data.data.result, metricData => {
let text = metricData.metric.__name__ || '';
delete metricData.metric.__name__;
......
public/app/plugins/datasource/prometheus/result_transformer.test.ts
import { ResultTransformer } from './result_transformer';
import { DataQueryResponseData } from '@grafana/data';
import { DataFrame } from '@grafana/data';
import { transform } from './result_transformer';
describe('Prometheus Result Transformer', () => {
const ctx: any = {};
beforeEach(() => {
ctx.templateSrv = {
replace: (str: string) => str,
};
ctx.resultTransformer = new ResultTransformer(ctx.templateSrv);
});
const options: any = { target: {}, query: {} };
describe('When nothing is returned', () => {
test('should return empty series', () => {
it('should return empty array', () => {
const response = {
status: 'success',
data: {
resultType: '',
result: (null as unknown) as DataQueryResponseData[],
result: null,
},
};
const series = ctx.resultTransformer.transform({ data: response }, {});
const series = transform({ data: response } as any, options);
expect(series).toEqual([]);
});
test('should return empty table', () => {
it('should return empty array', () => {
const response = {
status: 'success',
data: {
resultType: '',
result: (null as unknown) as DataQueryResponseData[],
result: null,
},
};
const table = ctx.resultTransformer.transform({ data: response }, { format: 'table' });
expect(table).toMatchObject([{ type: 'table', rows: [] }]);
const result = transform({ data: response } as any, { ...options, target: { format: 'table' } });
expect(result).toHaveLength(0);
});
});
......@@ -44,48 +36,65 @@ describe('Prometheus Result Transformer', () => {
result: [
{
metric: { __name__: 'test', job: 'testjob' },
values: [[1443454528, '3846']],
values: [
[1443454528, '3846'],
[1443454530, '3848'],
],
},
{
metric: {
__name__: 'test',
__name__: 'test2',
instance: 'localhost:8080',
job: 'otherjob',
},
values: [[1443454529, '3847']],
values: [
[1443454529, '3847'],
[1443454531, '3849'],
],
},
],
},
};
it('should return table model', () => {
const table = ctx.resultTransformer.transformMetricDataToTable(response.data.result, 0, 'A');
expect(table.type).toBe('table');
expect(table.rows).toEqual([
[1443454528000, 'test', '', 'testjob', 3846],
[1443454529000, 'test', 'localhost:8080', 'otherjob', 3847],
]);
expect(table.columns).toMatchObject([
{ text: 'Time', type: 'time' },
{ text: '__name__', filterable: true },
{ text: 'instance', filterable: true },
{ text: 'job' },
{ text: 'Value' },
it('should return data frame', () => {
const result = transform({ data: response } as any, {
...options,
target: {
responseListLength: 0,
refId: 'A',
format: 'table',
},
});
expect(result[0].fields[0].values.toArray()).toEqual([
1443454528000,
1443454530000,
1443454529000,
1443454531000,
]);
expect(table.columns[4].filterable).toBeUndefined();
expect(table.refId).toBe('A');
expect(result[0].fields[0].name).toBe('Time');
expect(result[0].fields[1].values.toArray()).toEqual(['test', 'test', 'test2', 'test2']);
expect(result[0].fields[1].name).toBe('__name__');
expect(result[0].fields[1].config.filterable).toBe(true);
expect(result[0].fields[2].values.toArray()).toEqual(['', '', 'localhost:8080', 'localhost:8080']);
expect(result[0].fields[2].name).toBe('instance');
expect(result[0].fields[3].values.toArray()).toEqual(['testjob', 'testjob', 'otherjob', 'otherjob']);
expect(result[0].fields[3].name).toBe('job');
expect(result[0].fields[4].values.toArray()).toEqual([3846, 3848, 3847, 3849]);
expect(result[0].fields[4].name).toEqual('Value');
expect(result[0].refId).toBe('A');
});
it('should column title include refId if response count is more than 2', () => {
const table = ctx.resultTransformer.transformMetricDataToTable(response.data.result, 2, 'B');
expect(table.type).toBe('table');
expect(table.columns).toMatchObject([
{ text: 'Time', type: 'time' },
{ text: '__name__' },
{ text: 'instance' },
{ text: 'job' },
{ text: 'Value #B' },
]);
it('should include refId if response count is more than 2', () => {
const result = transform({ data: response } as any, {
...options,
target: {
refId: 'B',
format: 'table',
},
responseListLength: 2,
});
expect(result[0].fields[4].name).toEqual('Value #B');
});
});
......@@ -103,31 +112,37 @@ describe('Prometheus Result Transformer', () => {
},
};
it('should return table model', () => {
const table = ctx.resultTransformer.transformMetricDataToTable(response.data.result);
expect(table.type).toBe('table');
expect(table.rows).toEqual([[1443454528000, 'test', 'testjob', 3846]]);
expect(table.columns).toMatchObject([
{ text: 'Time', type: 'time' },
{ text: '__name__' },
{ text: 'job' },
{ text: 'Value' },
]);
it('should return data frame', () => {
const result = transform({ data: response } as any, { ...options, target: { format: 'table' } });
expect(result[0].fields[0].values.toArray()).toEqual([1443454528000]);
expect(result[0].fields[0].name).toBe('Time');
expect(result[0].fields[1].values.toArray()).toEqual(['test']);
expect(result[0].fields[1].name).toBe('__name__');
expect(result[0].fields[2].values.toArray()).toEqual(['testjob']);
expect(result[0].fields[2].name).toBe('job');
expect(result[0].fields[3].values.toArray()).toEqual([3846]);
expect(result[0].fields[3].name).toEqual('Value');
});
it('should return table model with le label values parsed as numbers', () => {
const table = ctx.resultTransformer.transformMetricDataToTable([
{
metric: { le: '102' },
value: [1594908838, '0'],
it('should return le label values parsed as numbers', () => {
const response = {
status: 'success',
data: {
resultType: 'vector',
result: [
{
metric: { le: '102' },
value: [1594908838, '0'],
},
],
},
]);
expect(table.type).toBe('table');
expect(table.rows).toEqual([[1594908838000, 102, 0]]);
};
const result = transform({ data: response } as any, { ...options, target: { format: 'table' } });
expect(result[0].fields[1].values.toArray()).toEqual([102]);
});
});
describe('When resultFormat is time series and instant = true', () => {
describe('When instant = true', () => {
const response = {
status: 'success',
data: {
......@@ -141,158 +156,99 @@ describe('Prometheus Result Transformer', () => {
},
};
it('should return time series', () => {
const timeSeries = ctx.resultTransformer.transform({ data: response }, {});
expect(timeSeries[0].target).toBe('test{job="testjob"}');
expect(timeSeries[0].title).toBe('test{job="testjob"}');
it('should return data frame', () => {
const result: DataFrame[] = transform({ data: response } as any, { ...options, query: { instant: true } });
expect(result[0].name).toBe('test{job="testjob"}');
});
});
describe('When resultFormat is heatmap', () => {
const response = {
const getResponse = (result: any) => ({
status: 'success',
data: {
resultType: 'matrix',
result: [
{
metric: { __name__: 'test', job: 'testjob', le: '1' },
values: [
[1445000010, '10'],
[1445000020, '10'],
[1445000030, '0'],
],
},
{
metric: { __name__: 'test', job: 'testjob', le: '2' },
values: [
[1445000010, '20'],
[1445000020, '10'],
[1445000030, '30'],
],
},
{
metric: { __name__: 'test', job: 'testjob', le: '3' },
values: [
[1445000010, '30'],
[1445000020, '10'],
[1445000030, '40'],
],
},
],
result,
},
});
const options = {
format: 'heatmap',
start: 1445000010,
end: 1445000030,
legendFormat: '{{le}}',
};
it('should convert cumulative histogram to regular', () => {
const options = {
format: 'heatmap',
start: 1445000010,
end: 1445000030,
legendFormat: '{{le}}',
};
const result = ctx.resultTransformer.transform({ data: response }, options);
expect(result).toEqual([
const response = getResponse([
{
target: '1',
title: '1',
query: undefined,
datapoints: [
[10, 1445000010000],
[10, 1445000020000],
[0, 1445000030000],
metric: { __name__: 'test', job: 'testjob', le: '1' },
values: [
[1445000010, '10'],
[1445000020, '10'],
[1445000030, '0'],
],
tags: { __name__: 'test', job: 'testjob', le: '1' },
},
{
target: '2',
title: '2',
query: undefined,
datapoints: [
[10, 1445000010000],
[0, 1445000020000],
[30, 1445000030000],
metric: { __name__: 'test', job: 'testjob', le: '2' },
values: [
[1445000010, '20'],
[1445000020, '10'],
[1445000030, '30'],
],
tags: { __name__: 'test', job: 'testjob', le: '2' },
},
{
target: '3',
title: '3',
query: undefined,
datapoints: [
[10, 1445000010000],
[0, 1445000020000],
[10, 1445000030000],
metric: { __name__: 'test', job: 'testjob', le: '3' },
values: [
[1445000010, '30'],
[1445000020, '10'],
[1445000030, '40'],
],
tags: { __name__: 'test', job: 'testjob', le: '3' },
},
]);
const result = transform({ data: response } as any, { query: options, target: options } as any);
expect(result[0].fields[0].values.toArray()).toEqual([1445000010000, 1445000020000, 1445000030000]);
expect(result[0].fields[1].values.toArray()).toEqual([10, 10, 0]);
expect(result[1].fields[0].values.toArray()).toEqual([1445000010000, 1445000020000, 1445000030000]);
expect(result[1].fields[1].values.toArray()).toEqual([10, 0, 30]);
expect(result[2].fields[0].values.toArray()).toEqual([1445000010000, 1445000020000, 1445000030000]);
expect(result[2].fields[1].values.toArray()).toEqual([10, 0, 10]);
});
it('should handle missing datapoints', () => {
const seriesList = [
const response = getResponse([
{
datapoints: [
[1, 1000],
[2, 2000],
metric: { __name__: 'test', job: 'testjob', le: '1' },
values: [
[1445000010, '1'],
[1445000020, '2'],
],
},
{
datapoints: [
[2, 1000],
[5, 2000],
[1, 3000],
metric: { __name__: 'test', job: 'testjob', le: '2' },
values: [
[1445000010, '2'],
[1445000020, '5'],
[1445000030, '1'],
],
},
{
datapoints: [
[3, 1000],
[7, 2000],
metric: { __name__: 'test', job: 'testjob', le: '3' },
values: [
[1445000010, '3'],
[1445000020, '7'],
],
},
];
const expected = [
{
datapoints: [
[1, 1000],
[2, 2000],
],
},
{
datapoints: [
[1, 1000],
[3, 2000],
[1, 3000],
],
},
{
datapoints: [
[1, 1000],
[2, 2000],
],
},
];
const result = ctx.resultTransformer.transformToHistogramOverTime(seriesList);
expect(result).toEqual(expected);
});
it('should throw error when data in wrong format', () => {
const seriesList = [{ rows: [] as any[] }, { datapoints: [] as any[] }];
expect(() => {
ctx.resultTransformer.transformToHistogramOverTime(seriesList);
}).toThrow();
});
it('should throw error when prometheus returned non-timeseries', () => {
// should be { metric: {}, values: [] } for timeseries
const metricData = { metric: {}, value: [] as any[] };
expect(() => {
ctx.resultTransformer.transformMetricData(metricData, { step: 1 }, 1000, 2000);
}).toThrow();
]);
const result = transform({ data: response } as any, { query: options, target: options } as any);
expect(result[0].fields[1].values.toArray()).toEqual([1, 2]);
expect(result[1].fields[1].values.toArray()).toEqual([1, 3, 1]);
expect(result[2].fields[1].values.toArray()).toEqual([1, 2]);
});
});
describe('When resultFormat is time series', () => {
it('should transform matrix into timeseries', () => {
describe('When the response is a matrix', () => {
it('should transform into a data frame', () => {
const response = {
status: 'success',
data: {
......@@ -309,31 +265,20 @@ describe('Prometheus Result Transformer', () => {
],
},
};
const options = {
format: 'timeseries',
start: 0,
end: 2,
refId: 'B',
};
const result = ctx.resultTransformer.transform({ data: response }, options);
expect(result).toEqual([
{
target: 'test{job="testjob"}',
title: 'test{job="testjob"}',
query: undefined,
datapoints: [
[10, 0],
[10, 1000],
[0, 2000],
],
tags: { job: 'testjob' },
refId: 'B',
const result: DataFrame[] = transform({ data: response } as any, {
...options,
query: {
start: 0,
end: 2,
},
]);
});
expect(result[0].fields[0].values.toArray()).toEqual([0, 1000, 2000]);
expect(result[0].fields[1].values.toArray()).toEqual([10, 10, 0]);
expect(result[0].name).toBe('test{job="testjob"}');
});
it('should fill timeseries with null values', () => {
it('should fill null values', () => {
const response = {
status: 'success',
data: {
......@@ -349,27 +294,11 @@ describe('Prometheus Result Transformer', () => {
],
},
};
const options = {
format: 'timeseries',
step: 1,
start: 0,
end: 2,
};
const result = ctx.resultTransformer.transform({ data: response }, options);
expect(result).toEqual([
{
target: 'test{job="testjob"}',
title: 'test{job="testjob"}',
query: undefined,
datapoints: [
[null, 0],
[10, 1000],
[0, 2000],
],
tags: { job: 'testjob' },
},
]);
const result = transform({ data: response } as any, { ...options, query: { step: 1, start: 0, end: 2 } });
expect(result[0].fields[0].values.toArray()).toEqual([0, 1000, 2000]);
expect(result[0].fields[1].values.toArray()).toEqual([null, 10, 0]);
});
it('should use __name__ label as series name', () => {
......@@ -389,15 +318,15 @@ describe('Prometheus Result Transformer', () => {
},
};
const options = {
format: 'timeseries',
step: 1,
start: 0,
end: 2,
};
const result = ctx.resultTransformer.transform({ data: response }, options);
expect(result[0].target).toEqual('test{job="testjob"}');
const result = transform({ data: response } as any, {
...options,
query: {
step: 1,
start: 0,
end: 2,
},
});
expect(result[0].name).toEqual('test{job="testjob"}');
});
it('should set frame name to undefined if no __name__ label but there are other labels', () => {
......@@ -417,17 +346,15 @@ describe('Prometheus Result Transformer', () => {
},
};
const options = {
format: 'timeseries',
step: 1,
query: 'Some query',
start: 0,
end: 2,
};
const result = ctx.resultTransformer.transform({ data: response }, options);
expect(result[0].target).toBe('{job="testjob"}');
expect(result[0].tags.job).toEqual('testjob');
const result = transform({ data: response } as any, {
...options,
query: {
step: 1,
start: 0,
end: 2,
},
});
expect(result[0].name).toBe('{job="testjob"}');
});
it('should align null values with step', () => {
......@@ -446,35 +373,10 @@ describe('Prometheus Result Transformer', () => {
],
},
};
const options = {
format: 'timeseries',
step: 2,
start: 0,
end: 8,
refId: 'A',
meta: { custom: { hello: '1' } },
};
const result = ctx.resultTransformer.transform({ data: response }, options);
expect(result).toEqual([
{
target: 'test{job="testjob"}',
title: 'test{job="testjob"}',
meta: {
custom: { hello: '1' },
},
query: undefined,
refId: 'A',
datapoints: [
[null, 0],
[null, 2000],
[10, 4000],
[null, 6000],
[10, 8000],
],
tags: { job: 'testjob' },
},
]);
const result = transform({ data: response } as any, { ...options, query: { step: 2, start: 0, end: 8 } });
expect(result[0].fields[0].values.toArray()).toEqual([0, 2000, 4000, 6000, 8000]);
expect(result[0].fields[1].values.toArray()).toEqual([null, null, 10, null, 10]);
});
});
});
public/app/plugins/datasource/prometheus/result_transformer.ts
import _ from 'lodash';
import TableModel from 'app/core/table_model';
import { TimeSeries, FieldType, Labels, formatLabels, QueryResultMeta } from '@grafana/data';
import { TemplateSrv } from 'app/features/templating/template_srv';
export class ResultTransformer {
constructor(private templateSrv: TemplateSrv) {}
transform(response: any, options: any): Array<TableModel | TimeSeries> {
const prometheusResult = response.data.data.result;
if (options.format === 'table') {
return [
this.transformMetricDataToTable(
prometheusResult,
options.responseListLength,
options.refId,
options.meta,
options.valueWithRefId
),
];
} else if (prometheusResult && options.format === 'heatmap') {
let seriesList: TimeSeries[] = [];
for (const metricData of prometheusResult) {
seriesList.push(this.transformMetricData(metricData, options, options.start, options.end));
}
seriesList.sort(sortSeriesByLabel);
seriesList = this.transformToHistogramOverTime(seriesList);
return seriesList;
} else if (prometheusResult) {
const seriesList: TimeSeries[] = [];
for (const metricData of prometheusResult) {
if (response.data.data.resultType === 'matrix') {
seriesList.push(this.transformMetricData(metricData, options, options.start, options.end));
} else if (response.data.data.resultType === 'vector') {
seriesList.push(this.transformInstantMetricData(metricData, options));
}
}
return seriesList;
}
import {
ArrayVector,
DataFrame,
Field,
FieldType,
formatLabels,
MutableField,
ScopedVars,
TIME_SERIES_TIME_FIELD_NAME,
TIME_SERIES_VALUE_FIELD_NAME,
} from '@grafana/data';
import { FetchResponse } from '@grafana/runtime';
import templateSrv from 'app/features/templating/template_srv';
import {
isMatrixData,
MatrixOrVectorResult,
PromDataSuccessResponse,
PromMetric,
PromQuery,
PromQueryRequest,
PromValue,
TransformOptions,
} from './types';
export function transform(
response: FetchResponse<PromDataSuccessResponse>,
transformOptions: {
query: PromQueryRequest;
target: PromQuery;
responseListLength: number;
scopedVars?: ScopedVars;
mixedQueries?: boolean;
}
) {
// Create options object from transformOptions
const options: TransformOptions = {
format: transformOptions.target.format,
step: transformOptions.query.step,
legendFormat: transformOptions.target.legendFormat,
start: transformOptions.query.start,
end: transformOptions.query.end,
query: transformOptions.query.expr,
responseListLength: transformOptions.responseListLength,
scopedVars: transformOptions.scopedVars,
refId: transformOptions.target.refId,
valueWithRefId: transformOptions.target.valueWithRefId,
meta: {
/**
* Fix for showing of Prometheus results in Explore table.
* We want to show result of instant query always in table and result of range query based on target.runAll;
*/
preferredVisualisationType: getPreferredVisualisationType(
transformOptions.query.instant,
transformOptions.mixedQueries
),
},
};
const prometheusResult = response.data.data;
if (!prometheusResult.result) {
return [];
}
transformMetricData(metricData: any, options: any, start: number, end: number): TimeSeries {
const dps = [];
const { name, labels, title } = this.createLabelInfo(metricData.metric, options);
// Return early if result type is scalar
if (prometheusResult.resultType === 'scalar') {
return [
{
meta: options.meta,
refId: options.refId,
length: 1,
fields: [getTimeField([prometheusResult.result]), getValueField([prometheusResult.result])],
},
];
}
// Return early again if the format is table, this needs special transformation.
if (options.format === 'table') {
const tableData = transformMetricDataToTable(prometheusResult.result, options);
return [tableData];
}
const stepMs = parseFloat(options.step) * 1000;
let baseTimestamp = start * 1000;
// Process matrix and vector results to DataFrame
const dataFrame: DataFrame[] = [];
prometheusResult.result.forEach((data: MatrixOrVectorResult) => dataFrame.push(transformToDataFrame(data, options)));
if (metricData.values === undefined) {
throw new Error('Prometheus heatmap error: data should be a time series');
}
// When format is heatmap use the already created data frames and transform it more
if (options.format === 'heatmap') {
dataFrame.sort(sortSeriesByLabel);
const seriesList = transformToHistogramOverTime(dataFrame);
return seriesList;
}
// Return matrix or vector result as DataFrame[]
return dataFrame;
}
function getPreferredVisualisationType(isInstantQuery?: boolean, mixedQueries?: boolean) {
if (isInstantQuery) {
return 'table';
}
return mixedQueries ? 'graph' : undefined;
}
/**
* Transforms matrix and vector result from Prometheus result to DataFrame
*/
function transformToDataFrame(data: MatrixOrVectorResult, options: TransformOptions): DataFrame {
const { name } = createLabelInfo(data.metric, options);
const fields: Field[] = [];
if (isMatrixData(data)) {
const stepMs = options.step ? options.step * 1000 : NaN;
let baseTimestamp = options.start * 1000;
const dps: PromValue[] = [];
for (const value of metricData.values) {
for (const value of data.values) {
let dpValue: number | null = parseFloat(value[1]);
if (_.isNaN(dpValue)) {
if (isNaN(dpValue)) {
dpValue = null;
}
const timestamp = parseFloat(value[0]) * 1000;
const timestamp = value[0] * 1000;
for (let t = baseTimestamp; t < timestamp; t += stepMs) {
dps.push([null, t]);
dps.push([t, null]);
}
baseTimestamp = timestamp + stepMs;
dps.push([dpValue, timestamp]);
dps.push([timestamp, dpValue]);
}
const endTimestamp = end * 1000;
const endTimestamp = options.end * 1000;
for (let t = baseTimestamp; t <= endTimestamp; t += stepMs) {
dps.push([null, t]);
dps.push([t, null]);
}
fields.push(getTimeField(dps, true));
fields.push(getValueField(dps, undefined, false));
} else {
fields.push(getTimeField([data.value]));
fields.push(getValueField([data.value]));
}
return {
meta: options.meta,
refId: options.refId,
length: fields[0].values.length,
fields,
name,
};
}
function transformMetricDataToTable(md: MatrixOrVectorResult[], options: TransformOptions): DataFrame {
if (!md || md.length === 0) {
return {
datapoints: dps,
refId: options.refId,
target: name ?? '',
tags: labels,
title,
meta: options.meta,
refId: options.refId,
length: 0,
fields: [],
};
}
transformMetricDataToTable(
md: any,
resultCount: number,
refId: string,
meta: QueryResultMeta,
valueWithRefId?: boolean
): TableModel {
const table = new TableModel();
table.refId = refId;
table.meta = meta;
let i: number, j: number;
const metricLabels: { [key: string]: number } = {};
if (!md || md.length === 0) {
return table;
}
// Collect all labels across all metrics
_.each(md, series => {
for (const label in series.metric) {
if (!metricLabels.hasOwnProperty(label)) {
metricLabels[label] = 1;
}
}
const valueText = options.responseListLength > 1 || options.valueWithRefId ? `Value #${options.refId}` : 'Value';
const timeField = getTimeField([]);
const metricFields = Object.keys(md.reduce((acc, series) => ({ ...acc, ...series.metric }), {}))
.sort()
.map(label => {
return {
name: label,
config: { filterable: true },
type: FieldType.other,
values: new ArrayVector(),
};
});
const valueField = getValueField([], valueText);
md.forEach(d => {
if (isMatrixData(d)) {
d.values.forEach(val => {
timeField.values.add(val[0] * 1000);
metricFields.forEach(metricField => metricField.values.add(getLabelValue(d.metric, metricField.name)));
valueField.values.add(parseFloat(val[1]));
});
} else {
timeField.values.add(d.value[0] * 1000);
metricFields.forEach(metricField => metricField.values.add(getLabelValue(d.metric, metricField.name)));
valueField.values.add(parseFloat(d.value[1]));
}
});
return {
meta: options.meta,
refId: options.refId,
length: timeField.values.length,
fields: [timeField, ...metricFields, valueField],
};
}
// Sort metric labels, create columns for them and record their index
const sortedLabels = _.keys(metricLabels).sort();
table.columns.push({ text: 'Time', type: FieldType.time });
_.each(sortedLabels, (label, labelIndex) => {
metricLabels[label] = labelIndex + 1;
table.columns.push({ text: label, filterable: true });
});
const valueText = resultCount > 1 || valueWithRefId ? `Value #${refId}` : 'Value';
table.columns.push({ text: valueText });
function getLabelValue(metric: PromMetric, label: string): string | number {
if (metric.hasOwnProperty(label)) {
if (label === 'le') {
return parseHistogramLabel(metric[label]);
}
return metric[label];
}
return '';
}
// Populate rows, set value to empty string when label not present.
_.each(md, series => {
if (series.value) {
series.values = [series.value];
}
if (series.values) {
for (i = 0; i < series.values.length; i++) {
const values = series.values[i];
const reordered: any = [values[0] * 1000];
if (series.metric) {
for (j = 0; j < sortedLabels.length; j++) {
const label = sortedLabels[j];
if (series.metric.hasOwnProperty(label)) {
if (label === 'le') {
reordered.push(parseHistogramLabel(series.metric[label]));
} else {
reordered.push(series.metric[label]);
}
} else {
reordered.push('');
}
}
}
reordered.push(parseFloat(values[1]));
table.rows.push(reordered);
}
}
});
function getTimeField(data: PromValue[], isMs = false): MutableField {
return {
name: TIME_SERIES_TIME_FIELD_NAME,
type: FieldType.time,
config: {},
values: new ArrayVector<number>(data.map(val => (isMs ? val[0] : val[0] * 1000))),
};
}
return table;
}
function getValueField(
data: PromValue[],
valueName: string = TIME_SERIES_VALUE_FIELD_NAME,
parseValue = true
): MutableField {
return {
name: valueName,
type: FieldType.number,
config: {},
values: new ArrayVector<number | null>(data.map(val => (parseValue ? parseFloat(val[1]) : val[1]))),
};
}
transformInstantMetricData(md: any, options: any): TimeSeries {
const dps = [];
const { name, labels } = this.createLabelInfo(md.metric, options);
dps.push([parseFloat(md.value[1]), md.value[0] * 1000]);
return { target: name ?? '', title: name, datapoints: dps, tags: labels, refId: options.refId, meta: options.meta };
function createLabelInfo(labels: { [key: string]: string }, options: TransformOptions) {
if (options?.legendFormat) {
const title = renderTemplate(templateSrv.replace(options.legendFormat, options?.scopedVars), labels);
return { name: title, labels };
}
createLabelInfo(labels: { [key: string]: string }, options: any): { name?: string; labels: Labels; title?: string } {
if (options?.legendFormat) {
const title = this.renderTemplate(this.templateSrv.replace(options.legendFormat, options?.scopedVars), labels);
return { name: title, title, labels };
}
let { __name__, ...labelsWithoutName } = labels;
const { __name__, ...labelsWithoutName } = labels;
const labelPart = formatLabels(labelsWithoutName);
const title = `${__name__ ?? ''}${labelPart}`;
let title = __name__ || '';
return { name: title, labels: labelsWithoutName };
}
const labelPart = formatLabels(labelsWithoutName);
export function getOriginalMetricName(labelData: { [key: string]: string }) {
const metricName = labelData.__name__ || '';
delete labelData.__name__;
const labelPart = Object.entries(labelData)
.map(label => `${label[0]}="${label[1]}"`)
.join(',');
return `${metricName}{${labelPart}}`;
}
if (!title && !labelPart) {
title = options.query;
export function renderTemplate(aliasPattern: string, aliasData: { [key: string]: string }) {
const aliasRegex = /\{\{\s*(.+?)\s*\}\}/g;
return aliasPattern.replace(aliasRegex, (_match, g1) => {
if (aliasData[g1]) {
return aliasData[g1];
}
return '';
});
}
title = `${__name__ ?? ''}${labelPart}`;
return { name: title, title, labels: labelsWithoutName };
}
getOriginalMetricName(labelData: { [key: string]: string }) {
const metricName = labelData.__name__ || '';
delete labelData.__name__;
const labelPart = Object.entries(labelData)
.map(label => `${label[0]}="${label[1]}"`)
.join(',');
return `${metricName}{${labelPart}}`;
}
renderTemplate(aliasPattern: string, aliasData: { [key: string]: string }) {
const aliasRegex = /\{\{\s*(.+?)\s*\}\}/g;
return aliasPattern.replace(aliasRegex, (match, g1) => {
if (aliasData[g1]) {
return aliasData[g1];
}
return '';
});
}
transformToHistogramOverTime(seriesList: TimeSeries[]) {
/* t1 = timestamp1, t2 = timestamp2 etc.
function transformToHistogramOverTime(seriesList: DataFrame[]) {
/* t1 = timestamp1, t2 = timestamp2 etc.
t1 t2 t3 t1 t2 t3
le10 10 10 0 => 10 10 0
le20 20 10 30 => 10 0 30
le30 30 10 35 => 10 0 5
*/
for (let i = seriesList.length - 1; i > 0; i--) {
const topSeries = seriesList[i].datapoints;
const bottomSeries = seriesList[i - 1].datapoints;
if (!topSeries || !bottomSeries) {
throw new Error('Prometheus heatmap transform error: data should be a time series');
}
for (let j = 0; j < topSeries.length; j++) {
const bottomPoint = bottomSeries[j] || [0];
topSeries[j][0]! -= bottomPoint[0]!;
}
for (let i = seriesList.length - 1; i > 0; i--) {
const topSeries = seriesList[i].fields.find(s => s.name === TIME_SERIES_VALUE_FIELD_NAME);
const bottomSeries = seriesList[i - 1].fields.find(s => s.name === TIME_SERIES_VALUE_FIELD_NAME);
if (!topSeries || !bottomSeries) {
throw new Error('Prometheus heatmap transform error: data should be a time series');
}
return seriesList;
for (let j = 0; j < topSeries.values.length; j++) {
const bottomPoint = bottomSeries.values.get(j) || [0];
topSeries.values.toArray()[j] -= bottomPoint;
}
}
return seriesList;
}
function sortSeriesByLabel(s1: TimeSeries, s2: TimeSeries): number {
function sortSeriesByLabel(s1: DataFrame, s2: DataFrame): number {
let le1, le2;
try {
// fail if not integer. might happen with bad queries
le1 = parseHistogramLabel(s1.target);
le2 = parseHistogramLabel(s2.target);
le1 = parseHistogramLabel(s1.name ?? '');
le2 = parseHistogramLabel(s2.name ?? '');
} catch (err) {
console.error(err);
return 0;
......
public/app/plugins/datasource/prometheus/types.ts
import { DataQuery, DataSourceJsonData } from '@grafana/data';
import { DataQuery, DataSourceJsonData, QueryResultMeta, ScopedVars } from '@grafana/data';
import { FetchError } from '@grafana/runtime';
export interface PromQuery extends DataQuery {
expr: string;
......@@ -41,3 +42,77 @@ export interface PromMetricsMetadataItem {
export interface PromMetricsMetadata {
[metric: string]: PromMetricsMetadataItem[];
}
export interface PromDataSuccessResponse<T = PromData> {
status: 'success';
data: T;
}
export interface PromDataErrorResponse<T = PromData> {
status: 'error';
errorType: string;
error: string;
data: T;
}
export type PromData = PromMatrixData | PromVectorData | PromScalarData;
export interface PromVectorData {
resultType: 'vector';
result: Array<{
metric: PromMetric;
value: PromValue;
}>;
}
export interface PromMatrixData {
resultType: 'matrix';
result: Array<{
metric: PromMetric;
values: PromValue[];
}>;
}
export interface PromScalarData {
resultType: 'scalar';
result: PromValue;
}
export type PromValue = [number, any];
export interface PromMetric {
__name__?: string;
[index: string]: any;
}
export function isFetchErrorResponse(response: any): response is FetchError {
return 'cancelled' in response;
}
export function isMatrixData(result: MatrixOrVectorResult): result is PromMatrixData['result'][0] {
return 'values' in result;
}
export type MatrixOrVectorResult = PromMatrixData['result'][0] | PromVectorData['result'][0];
export interface TransformOptions {
format?: string;
step?: number;
legendFormat?: string;
start: number;
end: number;
query: string;
responseListLength: number;
scopedVars?: ScopedVars;
refId: string;
valueWithRefId?: boolean;
meta: QueryResultMeta;
}
export interface PromLabelQueryResponse {
data: {
status: string;
data: string[];
};
cancelled?: boolean;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment