Skip to content

N
nexpie-grafana-theme
Commit ae575158 by Alexander Zobnin Committed by GitHub
Options

Elasticsearch: Remove timeSrv dependency (#29770) 

* Elasticsearch: use time range from queries or default one

* Fix tests

* Fix language provider tests

* Elasticsearch: remove timeSrv dependency for logs context (#29841)

* Elasticsearch: remove timeSrv dependency for logs context

* Add tests & slighlty improve getIndexList

Co-authored-by: Elfo404 <gio.ricci@grafana.com>
parent 1ee97457
Showing with 113 additions and 75 deletions
public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx
import React from 'react';
import { EventsWithValidation, regexValidation, LegacyForms } from '@grafana/ui';
const { Select, Input, FormField } = LegacyForms;
import { ElasticsearchOptions } from '../types';
import { ElasticsearchOptions, Interval } from '../types';
import { DataSourceSettings, SelectableValue } from '@grafana/data';
const indexPatternTypes = [
......@@ -170,7 +170,9 @@ const jsonDataChangeHandler = (key: keyof ElasticsearchOptions, value: Props['va
});
};
const intervalHandler = (value: Props['value'], onChange: Props['onChange']) => (option: SelectableValue<string>) => {
const intervalHandler = (value: Props['value'], onChange: Props['onChange']) => (
option: SelectableValue<Interval | 'none'>
) => {
const { database } = value;
// If option value is undefined it will send its label instead so we have to convert made up value to undefined here.
const newInterval = option.value === 'none' ? undefined : option.value;
......
public/app/plugins/datasource/elasticsearch/datasource.test.ts
......@@ -15,7 +15,6 @@ import {
import _ from 'lodash';
import { ElasticDatasource, enhanceDataFrame } from './datasource';
import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__
import { TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { ElasticsearchOptions, ElasticsearchQuery } from './types';
import { Filters } from './components/QueryEditor/BucketAggregationsEditor/aggregations';
......@@ -61,8 +60,6 @@ describe('ElasticDatasource', function(this: any) {
getAdhocFilters: jest.fn(() => []),
};
const timeSrv: any = createTimeSrv('now-1h');
interface TestContext {
ds: ElasticDatasource;
}
......@@ -88,15 +85,8 @@ describe('ElasticDatasource', function(this: any) {
}
function createDatasource(instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>) {
createDatasourceWithTime(instanceSettings, timeSrv as TimeSrv);
}
function createDatasourceWithTime(
instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>,
timeSrv: TimeSrv
) {
instanceSettings.jsonData = instanceSettings.jsonData || ({} as ElasticsearchOptions);
ctx.ds = new ElasticDatasource(instanceSettings, templateSrv as TemplateSrv, timeSrv);
ctx.ds = new ElasticDatasource(instanceSettings, templateSrv as TemplateSrv);
}
describe('When testing datasource with index pattern', () => {
......@@ -506,14 +496,11 @@ describe('ElasticDatasource', function(this: any) {
};
beforeEach(() => {
createDatasourceWithTime(
{
url: ELASTICSEARCH_MOCK_URL,
database: '[asd-]YYYY.MM.DD',
jsonData: { interval: 'Daily', esVersion: 50 } as ElasticsearchOptions,
} as DataSourceInstanceSettings<ElasticsearchOptions>,
twoWeekTimeSrv
);
createDatasource({
url: ELASTICSEARCH_MOCK_URL,
database: '[asd-]YYYY.MM.DD',
jsonData: { interval: 'Daily', esVersion: 50 } as ElasticsearchOptions,
} as DataSourceInstanceSettings<ElasticsearchOptions>);
});
it('should return fields of the newest available index', async () => {
......@@ -534,7 +521,8 @@ describe('ElasticDatasource', function(this: any) {
return Promise.reject({ status: 404 });
});
const fieldObjects = await ctx.ds.getFields();
const range = twoWeekTimeSrv.timeRange();
const fieldObjects = await ctx.ds.getFields(undefined, range);
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp', 'beat.hostname']);
......@@ -545,6 +533,7 @@ describe('ElasticDatasource', function(this: any) {
.subtract(2, 'day')
.format('YYYY.MM.DD');
const range = twoWeekTimeSrv.timeRange();
datasourceRequestMock.mockImplementation(options => {
if (options.url === `${ELASTICSEARCH_MOCK_URL}/asd-${twoDaysBefore}/_mapping`) {
return Promise.resolve(basicResponse);
......@@ -554,7 +543,7 @@ describe('ElasticDatasource', function(this: any) {
expect.assertions(2);
try {
await ctx.ds.getFields();
await ctx.ds.getFields(undefined, range);
} catch (e) {
expect(e).toStrictEqual({ status: 500 });
expect(datasourceRequestMock).toBeCalledTimes(1);
......@@ -562,13 +551,14 @@ describe('ElasticDatasource', function(this: any) {
});
it('should not retry more than 7 indices', async () => {
const range = twoWeekTimeSrv.timeRange();
datasourceRequestMock.mockImplementation(() => {
return Promise.reject({ status: 404 });
});
expect.assertions(2);
try {
await ctx.ds.getFields();
await ctx.ds.getFields(undefined, range);
} catch (e) {
expect(e).toStrictEqual({ status: 404 });
expect(datasourceRequestMock).toBeCalledTimes(7);
......@@ -840,8 +830,7 @@ describe('ElasticDatasource', function(this: any) {
timeField: '@time',
},
} as DataSourceInstanceSettings<ElasticsearchOptions>,
templateSrv as TemplateSrv,
timeSrv as TimeSrv
templateSrv as TemplateSrv
);
(dataSource as any).post = jest.fn(() => Promise.resolve({ responses: [] }));
dataSource.query(createElasticQuery());
......
public/app/plugins/datasource/elasticsearch/datasource.ts
......@@ -12,6 +12,10 @@ import {
LogRowModel,
Field,
MetricFindValue,
TimeRange,
DefaultTimeRange,
DateTime,
dateTime,
} from '@grafana/data';
import LanguageProvider from './language_provider';
import { ElasticResponse } from './elastic_response';
......@@ -21,7 +25,6 @@ import { toUtc } from '@grafana/data';
import { defaultBucketAgg, hasMetricOfType } from './query_def';
import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { DataLinkConfig, ElasticsearchOptions, ElasticsearchQuery } from './types';
import { RowContextOptions } from '@grafana/ui/src/components/Logs/LogRowContextProvider';
import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils';
......@@ -65,8 +68,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
constructor(
instanceSettings: DataSourceInstanceSettings<ElasticsearchOptions>,
private readonly templateSrv: TemplateSrv = getTemplateSrv(),
private readonly timeSrv: TimeSrv = getTimeSrv()
private readonly templateSrv: TemplateSrv = getTemplateSrv()
) {
super(instanceSettings);
this.basicAuth = instanceSettings.basicAuth;
......@@ -140,9 +142,8 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
*
* @param url the url to query the index on, for example `/_mapping`.
*/
private get(url: string) {
const range = this.timeSrv.timeRange();
const indexList = this.indexPattern.getIndexList(range.from.valueOf(), range.to.valueOf());
private get(url: string, range = DefaultTimeRange) {
const indexList = this.indexPattern.getIndexList(range.from, range.to);
if (_.isArray(indexList) && indexList.length) {
return this.requestAllIndices(indexList, url).then((results: any) => {
results.data.$$config = results.config;
......@@ -370,7 +371,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
);
}
getQueryHeader(searchType: any, timeFrom: any, timeTo: any) {
getQueryHeader(searchType: any, timeFrom?: DateTime, timeTo?: DateTime): string {
const queryHeader: any = {
search_type: searchType,
ignore_unavailable: true,
......@@ -447,9 +448,13 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
getLogRowContext = async (row: LogRowModel, options?: RowContextOptions): Promise<{ data: DataFrame[] }> => {
const sortField = row.dataFrame.fields.find(f => f.name === 'sort');
const searchAfter = sortField?.values.get(row.rowIndex) || [row.timeEpochMs];
const range = this.timeSrv.timeRange();
const direction = options?.direction === 'FORWARD' ? 'asc' : 'desc';
const header = this.getQueryHeader('query_then_fetch', range.from, range.to);
const sort = options?.direction === 'FORWARD' ? 'asc' : 'desc';
const header =
options?.direction === 'FORWARD'
? this.getQueryHeader('query_then_fetch', dateTime(row.timeEpochMs))
: this.getQueryHeader('query_then_fetch', undefined, dateTime(row.timeEpochMs));
const limit = options?.limit ?? 10;
const esQuery = JSON.stringify({
size: limit,
......@@ -459,8 +464,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
{
range: {
[this.timeField]: {
gte: range.from.valueOf(),
lte: range.to.valueOf(),
[options?.direction === 'FORWARD' ? 'gte' : 'lte']: row.timeEpochMs,
format: 'epoch_millis',
},
},
......@@ -468,14 +472,14 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
],
},
},
sort: [{ [this.timeField]: direction }, { _doc: direction }],
sort: [{ [this.timeField]: sort }, { _doc: sort }],
search_after: searchAfter,
});
const payload = [header, esQuery].join('\n') + '\n';
const url = this.getMultiSearchUrl();
const response = await this.post(url, payload);
const targets: ElasticsearchQuery[] = [{ refId: `${row.dataFrame.refId}`, metrics: [], isLogsQuery: true }];
const elasticResponse = new ElasticResponse(targets, transformHitsBasedOnDirection(response, direction));
const elasticResponse = new ElasticResponse(targets, transformHitsBasedOnDirection(response, sort));
const logResponse = elasticResponse.getLogs(this.logMessageField, this.logLevelField);
const dataFrame = _.first(logResponse.data);
if (!dataFrame) {
......@@ -576,9 +580,9 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
}
// TODO: instead of being a string, this could be a custom type representing all the elastic types
async getFields(type?: string): Promise<MetricFindValue[]> {
async getFields(type?: string, range?: TimeRange): Promise<MetricFindValue[]> {
const configuredEsVersion = this.esVersion;
return this.get('/_mapping').then((result: any) => {
return this.get('/_mapping', range).then((result: any) => {
const typeMap: any = {
float: 'number',
double: 'number',
......@@ -663,8 +667,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
});
}
getTerms(queryDef: any) {
const range = this.timeSrv.timeRange();
getTerms(queryDef: any, range = DefaultTimeRange) {
const searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count';
const header = this.getQueryHeader(searchType, range.from, range.to);
let esQuery = JSON.stringify(this.queryBuilder.getTermsQuery(queryDef));
......@@ -698,18 +701,19 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
return '_msearch';
}
metricFindQuery(query: string): Promise<MetricFindValue[]> {
metricFindQuery(query: string, options?: any): Promise<MetricFindValue[]> {
const range = options?.range;
const parsedQuery = JSON.parse(query);
if (query) {
if (parsedQuery.find === 'fields') {
parsedQuery.field = this.templateSrv.replace(parsedQuery.field, {}, 'lucene');
return this.getFields(query);
return this.getFields(query, range);
}
if (parsedQuery.find === 'terms') {
parsedQuery.field = this.templateSrv.replace(parsedQuery.field, {}, 'lucene');
parsedQuery.query = this.templateSrv.replace(parsedQuery.query || '*', {}, 'lucene');
return this.getTerms(query);
return this.getTerms(query, range);
}
}
......
public/app/plugins/datasource/elasticsearch/index_pattern.ts
import { toUtc, dateTime } from '@grafana/data';
import { toUtc, dateTime, DateTime, DurationUnit } from '@grafana/data';
import { Interval } from './types';
type IntervalMap = Record<
Interval,
{
startOf: DurationUnit;
amount: DurationUnit;
}
>;
const intervalMap: any = {
const intervalMap: IntervalMap = {
Hourly: { startOf: 'hour', amount: 'hours' },
Daily: { startOf: 'day', amount: 'days' },
Weekly: { startOf: 'isoWeek', amount: 'weeks' },
Weekly: { startOf: 'week', amount: 'weeks' },
Monthly: { startOf: 'month', amount: 'months' },
Yearly: { startOf: 'year', amount: 'years' },
};
......@@ -11,7 +20,7 @@ const intervalMap: any = {
export class IndexPattern {
private dateLocale = 'en';
constructor(private pattern: any, private interval?: string) {}
constructor(private pattern: string, private interval?: keyof typeof intervalMap) {}
getIndexForToday() {
if (this.interval) {
......@@ -23,16 +32,21 @@ export class IndexPattern {
}
}
getIndexList(from: any, to: any) {
getIndexList(from?: DateTime, to?: DateTime) {
// When no `from` or `to` is provided, we request data from 7 subsequent/previous indices
// for the provided index pattern.
// This is useful when requesting log context where the only time data we have is the log
// timestamp.
const indexOffset = 7;
if (!this.interval) {
return this.pattern;
}
const intervalInfo = intervalMap[this.interval];
const start = dateTime(from)
const start = dateTime(from || dateTime(to).add(-indexOffset, intervalInfo.amount))
.utc()
.startOf(intervalInfo.startOf);
const endEpoch = dateTime(to)
const endEpoch = dateTime(to || dateTime(from).add(indexOffset, intervalInfo.amount))
.utc()
.startOf(intervalInfo.startOf)
.valueOf();
......
public/app/plugins/datasource/elasticsearch/language_provider.test.ts
import LanguageProvider from './language_provider';
import { PromQuery } from '../prometheus/types';
import { ElasticDatasource } from './datasource';
import { DataSourceInstanceSettings, dateTime } from '@grafana/data';
import { DataSourceInstanceSettings } from '@grafana/data';
import { ElasticsearchOptions } from './types';
import { TemplateSrv } from '../../../features/templating/template_srv';
import { TimeSrv } from '../../../features/dashboard/services/TimeSrv';
const templateSrvStub = {
getAdhocFilters: jest.fn(() => [] as any[]),
replace: jest.fn((a: string) => a),
} as any;
const timeSrvStub = {
timeRange(): any {
return {
from: dateTime(1531468681),
to: dateTime(1531489712),
};
},
} as any;
const dataSource = new ElasticDatasource(
{
url: 'http://es.com',
......@@ -30,8 +20,7 @@ const dataSource = new ElasticDatasource(
timeField: '@time',
},
} as DataSourceInstanceSettings<ElasticsearchOptions>,
templateSrvStub as TemplateSrv,
timeSrvStub as TimeSrv
templateSrvStub as TemplateSrv
);
describe('transform prometheus query to elasticsearch query', () => {
it('Prometheus query with exact equals labels ( 2 labels ) and metric __name__', () => {
......
public/app/plugins/datasource/elasticsearch/specs/index_pattern.test.ts
///<amd-dependency path="test/specs/helpers" name="helpers" />
import { IndexPattern } from '../index_pattern';
import { toUtc, getLocale, setLocale } from '@grafana/data';
import { toUtc, getLocale, setLocale, dateTime } from '@grafana/data';
describe('IndexPattern', () => {
const originalLocale = getLocale();
......@@ -31,8 +31,8 @@ describe('IndexPattern', () => {
describe('no interval', () => {
test('should return correct index', () => {
const pattern = new IndexPattern('my-metrics');
const from = new Date(2015, 4, 30, 1, 2, 3);
const to = new Date(2015, 5, 1, 12, 5, 6);
const from = dateTime(new Date(2015, 4, 30, 1, 2, 3));
const to = dateTime(new Date(2015, 5, 1, 12, 5, 6));
expect(pattern.getIndexList(from, to)).toEqual('my-metrics');
});
});
......@@ -40,8 +40,8 @@ describe('IndexPattern', () => {
describe('daily', () => {
test('should return correct index list', () => {
const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
const from = new Date(1432940523000);
const to = new Date(1433153106000);
const from = dateTime(new Date(1432940523000));
const to = dateTime(new Date(1433153106000));
const expected = ['asd-2015.05.29', 'asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01'];
......@@ -51,8 +51,8 @@ describe('IndexPattern', () => {
test('should format date using western arabic numerals regardless of locale', () => {
setLocale('ar_SA'); // saudi-arabic, formatting for YYYY.MM.DD looks like "٢٠٢٠.٠٩.٠٣"
const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
const from = new Date(1432940523000);
const to = new Date(1433153106000);
const from = dateTime(new Date(1432940523000));
const to = dateTime(new Date(1433153106000));
const expected = ['asd-2015.05.29', 'asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01'];
......@@ -60,4 +60,42 @@ describe('IndexPattern', () => {
});
});
});
describe('when getting index list from single date', () => {
it('Should return index matching the starting time and subsequent ones', () => {
const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
const from = dateTime(new Date(1432940523000));
const expected = [
'asd-2015.05.29',
'asd-2015.05.30',
'asd-2015.05.31',
'asd-2015.06.01',
'asd-2015.06.02',
'asd-2015.06.03',
'asd-2015.06.04',
'asd-2015.06.05',
];
expect(pattern.getIndexList(from)).toEqual(expected);
});
it('Should return index matching the starting time and previous ones', () => {
const pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
const to = dateTime(new Date(1432940523000));
const expected = [
'asd-2015.05.22',
'asd-2015.05.23',
'asd-2015.05.24',
'asd-2015.05.25',
'asd-2015.05.26',
'asd-2015.05.27',
'asd-2015.05.28',
'asd-2015.05.29',
];
expect(pattern.getIndexList(undefined, to)).toEqual(expected);
});
});
});
public/app/plugins/datasource/elasticsearch/types.ts
......@@ -8,10 +8,12 @@ import {
MetricAggregationType,
} from './components/QueryEditor/MetricAggregationsEditor/aggregations';
export type Interval = 'Hourly' | 'Daily' | 'Weekly' | 'Monthly' | 'Yearly';
export interface ElasticsearchOptions extends DataSourceJsonData {
timeField: string;
esVersion: number;
interval?: string;
interval?: Interval;
timeInterval: string;
maxConcurrentShardRequests?: number;
logMessageField?: string;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment