Skip to content

N
nexpie-grafana-theme
Commit 4c9cb415 by Andrej Ocenas Committed by David
Options

Elastic: Fix parsing for millisecond number timestamps (#20290) 

* Stop parsing 'field'

* Remove time field from query for docs queries

* Test refactor to test all versions

* Fix tests
parent c43aa348
Showing with 462 additions and 560 deletions
packages/grafana-data/src/dataframe/MutableDataFrame.test.ts
......@@ -58,7 +58,7 @@ describe('Apending DataFrame', () => {
]);
// Add a time value that has an array type
frame.add({ time: [300] });
frame.add({ time: 300 });
expect(frame.toArray()).toEqual([
{ time: 100, name: 'a', value: 1, value2: null }, // 1
{ time: 200, name: 'BB', value: 20, value2: null }, // 2
......
packages/grafana-data/src/dataframe/MutableDataFrame.ts
......@@ -231,11 +231,6 @@ export class MutableDataFrame<T = any> implements DataFrame, MutableVector<T> {
field.parse = makeFieldParser(val, field);
}
val = field.parse(val);
} else if (field.type === FieldType.time && isArray(val)) {
if (!field.parse) {
field.parse = (val: any[]) => val[0] || undefined;
}
val = field.parse(val);
}
if (val === undefined) {
......
public/app/plugins/datasource/elasticsearch/elastic_response.ts
......@@ -424,40 +424,26 @@ export class ElasticResponse {
throw this.getErrorFromElasticResponse(this.response, response.error);
}
const hits = response.hits;
// We keep a list of all props so that we can create all the fields in the dataFrame, this can lead
// to wide sparse dataframes in case the scheme is different per document.
let propNames: string[] = [];
let propName, hit, doc: any, i;
for (i = 0; i < hits.hits.length; i++) {
hit = hits.hits[i];
for (const hit of response.hits.hits) {
const flattened = hit._source ? flatten(hit._source, null) : {};
doc = {};
doc[this.targets[0].timeField] = null;
doc = {
...doc,
const doc = {
_id: hit._id,
_type: hit._type,
_index: hit._index,
_source: { ...flattened },
...flattened,
};
// Note: the order of for...in is arbitrary amd implementation dependant
// and should probably not be relied upon.
for (propName in hit.fields) {
for (const propName of Object.keys(doc)) {
if (propNames.indexOf(propName) === -1) {
propNames.push(propName);
}
doc[propName] = hit.fields[propName];
}
for (propName in doc) {
if (propNames.indexOf(propName) === -1) {
propNames.push(propName);
}
}
doc._source = { ...flattened };
docs.push(doc);
}
......@@ -468,9 +454,7 @@ export class ElasticResponse {
series.addField({
name: this.targets[0].timeField,
type: FieldType.time,
}).parse = (v: any) => {
return v[0] || '';
};
});
if (logMessageField) {
series.addField({
......
public/app/plugins/datasource/elasticsearch/query_builder.ts
......@@ -5,7 +5,7 @@ export class ElasticQueryBuilder {
timeField: string;
esVersion: number;
constructor(options: any) {
constructor(options: { timeField: string; esVersion: number }) {
this.timeField = options.timeField;
this.esVersion = options.esVersion;
}
......@@ -129,11 +129,6 @@ export class ElasticQueryBuilder {
}
query.script_fields = {};
if (this.esVersion < 5) {
query.fielddata_fields = [this.timeField];
} else {
query.docvalue_fields = [this.timeField];
}
return query;
}
......
public/app/plugins/datasource/elasticsearch/specs/elastic_response.test.ts
......@@ -871,9 +871,6 @@ describe('ElasticResponse', () => {
host: 'djisaodjsoad',
message: 'hello, i am a message',
},
fields: {
'@timestamp': ['2019-06-24T09:51:19.765Z'],
},
},
{
_id: 'kdospaidopa',
......@@ -884,9 +881,6 @@ describe('ElasticResponse', () => {
host: 'dsalkdakdop',
message: 'hello, i am also message',
},
fields: {
'@timestamp': ['2019-06-24T09:52:19.765Z'],
},
},
],
},
......
public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts
import { ElasticQueryBuilder } from '../query_builder';
describe('ElasticQueryBuilder', () => {
let builder: any;
beforeEach(() => {
builder = new ElasticQueryBuilder({ timeField: '@timestamp' });
});
it('with defaults', () => {
const query = builder.build({
metrics: [{ type: 'Count', id: '0' }],
timeField: '@timestamp',
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '1' }],
});
expect(query.query.bool.filter[0].range['@timestamp'].gte).toBe('$timeFrom');
expect(query.aggs['1'].date_histogram.extended_bounds.min).toBe('$timeFrom');
});
it('with defaults on es5.x', () => {
const builder5x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 5,
});
const query = builder5x.build({
metrics: [{ type: 'Count', id: '0' }],
timeField: '@timestamp',
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '1' }],
});
expect(query.query.bool.filter[0].range['@timestamp'].gte).toBe('$timeFrom');
expect(query.aggs['1'].date_histogram.extended_bounds.min).toBe('$timeFrom');
});
it('with multiple bucket aggs', () => {
const query = builder.build({
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{ type: 'terms', field: '@host', id: '2' },
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
});
expect(query.aggs['2'].terms.field).toBe('@host');
expect(query.aggs['2'].aggs['3'].date_histogram.field).toBe('@timestamp');
});
const builder = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 2 });
const builder5x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 5 });
const builder56 = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 56 });
const builder6x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 60 });
const builder7x = new ElasticQueryBuilder({ timeField: '@timestamp', esVersion: 70 });
const allBuilders = [builder, builder5x, builder56, builder6x, builder7x];
allBuilders.forEach(builder => {
describe(`version ${builder.esVersion}`, () => {
it('should return query with defaults', () => {
const query = builder.build({
metrics: [{ type: 'Count', id: '0' }],
timeField: '@timestamp',
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '1' }],
});
expect(query.query.bool.filter[0].range['@timestamp'].gte).toBe('$timeFrom');
expect(query.aggs['1'].date_histogram.extended_bounds.min).toBe('$timeFrom');
});
it('with multiple bucket aggs', () => {
const query = builder.build({
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{ type: 'terms', field: '@host', id: '2' },
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
});
it('with select field', () => {
const query = builder.build(
{
metrics: [{ type: 'avg', field: '@value', id: '1' }],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
},
100,
1000
);
const aggs = query.aggs['2'].aggs;
expect(aggs['1'].avg.field).toBe('@value');
});
expect(query.aggs['2'].terms.field).toBe('@host');
expect(query.aggs['2'].aggs['3'].date_histogram.field).toBe('@timestamp');
});
it('with term agg and order by term', () => {
const query = builder.build(
{
metrics: [
{ type: 'count', id: '1' },
{ type: 'avg', field: '@value', id: '5' },
],
bucketAggs: [
it('with select field', () => {
const query = builder.build(
{
type: 'terms',
field: '@host',
settings: { size: 5, order: 'asc', orderBy: '_term' },
id: '2',
metrics: [{ type: 'avg', field: '@value', id: '1' }],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
},
100,
1000
);
const firstLevel = query.aggs['2'];
expect(firstLevel.terms.order._term).toBe('asc');
});
it('with term agg and order by term on es6.x', () => {
const builder6x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 60,
});
const query = builder6x.build(
{
metrics: [
{ type: 'count', id: '1' },
{ type: 'avg', field: '@value', id: '5' },
],
bucketAggs: [
100,
'1000'
);
const aggs = query.aggs['2'].aggs;
expect(aggs['1'].avg.field).toBe('@value');
});
it('term agg and order by term', () => {
const target = {
metrics: [
{ type: 'count', id: '1' },
{ type: 'avg', field: '@value', id: '5' },
],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: 5, order: 'asc', orderBy: '_term' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
};
const query = builder.build(target, 100, '1000');
const firstLevel = query.aggs['2'];
if (builder.esVersion >= 60) {
expect(firstLevel.terms.order._key).toBe('asc');
} else {
expect(firstLevel.terms.order._term).toBe('asc');
}
});
it('with term agg and order by metric agg', () => {
const query = builder.build(
{
type: 'terms',
field: '@host',
settings: { size: 5, order: 'asc', orderBy: '_term' },
id: '2',
metrics: [
{ type: 'count', id: '1' },
{ type: 'avg', field: '@value', id: '5' },
],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: 5, order: 'asc', orderBy: '5' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
},
100,
// @ts-ignore
1000
);
const firstLevel = query.aggs['2'];
expect(firstLevel.terms.order._key).toBe('asc');
});
100,
'1000'
);
it('with term agg and order by metric agg', () => {
const query = builder.build(
{
metrics: [
{ type: 'count', id: '1' },
{ type: 'avg', field: '@value', id: '5' },
],
bucketAggs: [
{
type: 'terms',
field: '@host',
settings: { size: 5, order: 'asc', orderBy: '5' },
id: '2',
},
{ type: 'date_histogram', field: '@timestamp', id: '3' },
],
},
100,
1000
);
const firstLevel = query.aggs['2'];
const secondLevel = firstLevel.aggs['3'];
expect(firstLevel.aggs['5'].avg.field).toBe('@value');
expect(secondLevel.aggs['5'].avg.field).toBe('@value');
});
const firstLevel = query.aggs['2'];
const secondLevel = firstLevel.aggs['3'];
expect(firstLevel.aggs['5'].avg.field).toBe('@value');
expect(secondLevel.aggs['5'].avg.field).toBe('@value');
});
it('with metric percentiles', () => {
const query = builder.build(
{
metrics: [
it('with metric percentiles', () => {
const query = builder.build(
{
id: '1',
type: 'percentiles',
field: '@load_time',
settings: {
percents: [1, 2, 3, 4],
},
metrics: [
{
id: '1',
type: 'percentiles',
field: '@load_time',
settings: {
percents: [1, 2, 3, 4],
},
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
},
100,
1000
);
100,
'1000'
);
const firstLevel = query.aggs['3'];
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['1'].percentiles.field).toBe('@load_time');
expect(firstLevel.aggs['1'].percentiles.percents).toEqual([1, 2, 3, 4]);
});
expect(firstLevel.aggs['1'].percentiles.field).toBe('@load_time');
expect(firstLevel.aggs['1'].percentiles.percents).toEqual([1, 2, 3, 4]);
});
it('with filters aggs', () => {
const query = builder.build({
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
id: '2',
type: 'filters',
settings: {
filters: [{ query: '@metric:cpu' }, { query: '@metric:logins.count' }],
it('with filters aggs', () => {
const query = builder.build({
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
id: '2',
type: 'filters',
settings: {
filters: [{ query: '@metric:cpu' }, { query: '@metric:logins.count' }],
},
},
{ type: 'date_histogram', field: '@timestamp', id: '4' },
],
});
expect(query.aggs['2'].filters.filters['@metric:cpu'].query_string.query).toBe('@metric:cpu');
expect(query.aggs['2'].filters.filters['@metric:logins.count'].query_string.query).toBe('@metric:logins.count');
expect(query.aggs['2'].aggs['4'].date_histogram.field).toBe('@timestamp');
});
it('should return correct query for raw_document metric', () => {
const target = {
metrics: [{ type: 'raw_document', id: '1', settings: {} }],
timeField: '@timestamp',
bucketAggs: [] as any[],
};
const query = builder.build(target);
expect(query).toMatchObject({
size: 500,
query: {
bool: {
filter: [
{
range: {
'@timestamp': {
format: 'epoch_millis',
gte: '$timeFrom',
lte: '$timeTo',
},
},
},
{
query_string: {
analyze_wildcard: true,
query: undefined,
},
},
],
},
},
},
{ type: 'date_histogram', field: '@timestamp', id: '4' },
],
});
expect(query.aggs['2'].filters.filters['@metric:cpu'].query_string.query).toBe('@metric:cpu');
expect(query.aggs['2'].filters.filters['@metric:logins.count'].query_string.query).toBe('@metric:logins.count');
expect(query.aggs['2'].aggs['4'].date_histogram.field).toBe('@timestamp');
});
it('with filters aggs on es5.x', () => {
const builder5x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 5,
});
const query = builder5x.build({
metrics: [{ type: 'count', id: '1' }],
timeField: '@timestamp',
bucketAggs: [
{
id: '2',
type: 'filters',
settings: {
filters: [{ query: '@metric:cpu' }, { query: '@metric:logins.count' }],
sort: {
'@timestamp': {
order: 'desc',
unmapped_type: 'boolean',
},
},
},
{ type: 'date_histogram', field: '@timestamp', id: '4' },
],
});
expect(query.aggs['2'].filters.filters['@metric:cpu'].query_string.query).toBe('@metric:cpu');
expect(query.aggs['2'].filters.filters['@metric:logins.count'].query_string.query).toBe('@metric:logins.count');
expect(query.aggs['2'].aggs['4'].date_histogram.field).toBe('@timestamp');
});
it('with raw_document metric', () => {
const query = builder.build({
metrics: [{ type: 'raw_document', id: '1', settings: {} }],
timeField: '@timestamp',
bucketAggs: [],
});
expect(query.size).toBe(500);
});
it('with raw_document metric size set', () => {
const query = builder.build({
metrics: [{ type: 'raw_document', id: '1', settings: { size: 1337 } }],
timeField: '@timestamp',
bucketAggs: [],
});
expect(query.size).toBe(1337);
});
it('with moving average', () => {
const query = builder.build({
metrics: [
{
id: '3',
type: 'sum',
field: '@value',
},
{
id: '2',
type: 'moving_avg',
field: '3',
pipelineAgg: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('3');
});
it('with moving average doc count', () => {
const query = builder.build({
metrics: [
{
id: '3',
type: 'count',
field: 'select field',
},
{
id: '2',
type: 'moving_avg',
field: '3',
pipelineAgg: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '4' }],
});
const firstLevel = query.aggs['4'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('_count');
});
it('with broken moving average', () => {
const query = builder.build({
metrics: [
{
id: '3',
type: 'sum',
field: '@value',
},
{
id: '2',
type: 'moving_avg',
pipelineAgg: '3',
},
{
id: '4',
type: 'moving_avg',
pipelineAgg: 'Metric to apply moving average',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('3');
expect(firstLevel.aggs['4']).toBe(undefined);
});
it('with derivative', () => {
const query = builder.build({
metrics: [
{
id: '3',
type: 'sum',
field: '@value',
},
{
id: '2',
type: 'derivative',
pipelineAgg: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
script_fields: {},
});
});
it('should set query size from settings when raw_documents', () => {
const query = builder.build({
metrics: [{ type: 'raw_document', id: '1', settings: { size: 1337 } }],
timeField: '@timestamp',
bucketAggs: [],
});
expect(query.size).toBe(1337);
});
it('with moving average', () => {
const query = builder.build({
metrics: [
{
id: '3',
type: 'sum',
field: '@value',
},
{
id: '2',
type: 'moving_avg',
field: '3',
pipelineAgg: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['3'];
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('3');
});
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('3');
});
it('with derivative doc count', () => {
const query = builder.build({
metrics: [
{
id: '3',
type: 'count',
field: 'select field',
},
{
id: '2',
type: 'derivative',
pipelineAgg: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '4' }],
});
it('with moving average doc count', () => {
const query = builder.build({
metrics: [
{
id: '3',
type: 'count',
field: 'select field',
},
{
id: '2',
type: 'moving_avg',
field: '3',
pipelineAgg: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '4' }],
});
const firstLevel = query.aggs['4'];
const firstLevel = query.aggs['4'];
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('_count');
});
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('_count');
});
it('with bucket_script', () => {
const query = builder.build({
metrics: [
{
id: '1',
type: 'sum',
field: '@value',
},
{
id: '3',
type: 'max',
field: '@value',
},
{
field: 'select field',
id: '4',
meta: {},
pipelineVariables: [
it('with broken moving average', () => {
const query = builder.build({
metrics: [
{
name: 'var1',
pipelineAgg: '1',
id: '3',
type: 'sum',
field: '@value',
},
{
name: 'var2',
id: '2',
type: 'moving_avg',
pipelineAgg: '3',
},
{
id: '4',
type: 'moving_avg',
pipelineAgg: 'Metric to apply moving average',
},
],
settings: {
script: 'params.var1 * params.var2',
},
type: 'bucket_script',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
});
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['2'];
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['4']).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '1', var2: '3' });
});
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg).not.toBe(undefined);
expect(firstLevel.aggs['2'].moving_avg.buckets_path).toBe('3');
expect(firstLevel.aggs['4']).toBe(undefined);
});
it('with bucket_script doc count', () => {
const query = builder.build({
metrics: [
{
id: '3',
type: 'count',
field: 'select field',
},
{
field: 'select field',
id: '4',
meta: {},
pipelineVariables: [
it('with derivative', () => {
const query = builder.build({
metrics: [
{
name: 'var1',
id: '3',
type: 'sum',
field: '@value',
},
{
id: '2',
type: 'derivative',
pipelineAgg: '3',
},
],
settings: {
script: 'params.var1 * 1000',
},
type: 'bucket_script',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
});
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
});
const firstLevel = query.aggs['2'];
const firstLevel = query.aggs['3'];
expect(firstLevel.aggs['4']).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '_count' });
});
it('with histogram', () => {
const query = builder.build({
metrics: [{ id: '1', type: 'count' }],
bucketAggs: [
{
type: 'histogram',
field: 'bytes',
id: '3',
settings: { interval: 10, min_doc_count: 2, missing: 5 },
},
],
});
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('3');
});
const firstLevel = query.aggs['3'];
expect(firstLevel.histogram.field).toBe('bytes');
expect(firstLevel.histogram.interval).toBe(10);
expect(firstLevel.histogram.min_doc_count).toBe(2);
expect(firstLevel.histogram.missing).toBe(5);
});
it('with derivative doc count', () => {
const query = builder.build({
metrics: [
{
id: '3',
type: 'count',
field: 'select field',
},
{
id: '2',
type: 'derivative',
pipelineAgg: '3',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '4' }],
});
it('with adhoc filters', () => {
const query = builder.build(
{
metrics: [{ type: 'Count', id: '0' }],
timeField: '@timestamp',
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
},
[
{ key: 'key1', operator: '=', value: 'value1' },
{ key: 'key2', operator: '=', value: 'value2' },
{ key: 'key2', operator: '!=', value: 'value2' },
{ key: 'key3', operator: '<', value: 'value3' },
{ key: 'key4', operator: '>', value: 'value4' },
{ key: 'key5', operator: '=~', value: 'value5' },
{ key: 'key6', operator: '!~', value: 'value6' },
]
);
expect(query.query.bool.must[0].match_phrase['key1'].query).toBe('value1');
expect(query.query.bool.must[1].match_phrase['key2'].query).toBe('value2');
expect(query.query.bool.must_not[0].match_phrase['key2'].query).toBe('value2');
expect(query.query.bool.filter[2].range['key3'].lt).toBe('value3');
expect(query.query.bool.filter[3].range['key4'].gt).toBe('value4');
expect(query.query.bool.filter[4].regexp['key5']).toBe('value5');
expect(query.query.bool.filter[5].bool.must_not.regexp['key6']).toBe('value6');
});
const firstLevel = query.aggs['4'];
// terms query ES<6.0 - check ordering for _term and doc_type
expect(firstLevel.aggs['2']).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative).not.toBe(undefined);
expect(firstLevel.aggs['2'].derivative.buckets_path).toBe('_count');
});
it('getTermsQuery(default case) es<6.0 should set asc sorting on _term', () => {
const query = builder.getTermsQuery({});
expect(query.aggs['1'].terms.order._term).toBe('asc');
expect(query.aggs['1'].terms.order._key).toBeUndefined();
expect(query.aggs['1'].terms.order._count).toBeUndefined();
});
it('with bucket_script', () => {
const query = builder.build({
metrics: [
{
id: '1',
type: 'sum',
field: '@value',
},
{
id: '3',
type: 'max',
field: '@value',
},
{
field: 'select field',
id: '4',
meta: {},
pipelineVariables: [
{
name: 'var1',
pipelineAgg: '1',
},
{
name: 'var2',
pipelineAgg: '3',
},
],
settings: {
script: 'params.var1 * params.var2',
},
type: 'bucket_script',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
});
it('getTermsQuery(order:desc) es<6.0 should set desc sorting on _term', () => {
const query = builder.getTermsQuery({ order: 'desc' });
expect(query.aggs['1'].terms.order._term).toBe('desc');
expect(query.aggs['1'].terms.order._key).toBeUndefined();
expect(query.aggs['1'].terms.order._count).toBeUndefined();
});
const firstLevel = query.aggs['2'];
it('getTermsQuery(orderBy:doc_count) es<6.0 should set desc sorting on _count', () => {
const query = builder.getTermsQuery({ orderBy: 'doc_count' });
expect(query.aggs['1'].terms.order._term).toBeUndefined();
expect(query.aggs['1'].terms.order._key).toBeUndefined();
expect(query.aggs['1'].terms.order._count).toBe('desc');
});
expect(firstLevel.aggs['4']).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '1', var2: '3' });
});
it('getTermsQuery(orderBy:doc_count, order:asc) es<6.0 should set asc sorting on _count', () => {
const query = builder.getTermsQuery({ orderBy: 'doc_count', order: 'asc' });
expect(query.aggs['1'].terms.order._term).toBeUndefined();
expect(query.aggs['1'].terms.order._key).toBeUndefined();
expect(query.aggs['1'].terms.order._count).toBe('asc');
});
it('with bucket_script doc count', () => {
const query = builder.build({
metrics: [
{
id: '3',
type: 'count',
field: 'select field',
},
{
field: 'select field',
id: '4',
meta: {},
pipelineVariables: [
{
name: 'var1',
pipelineAgg: '3',
},
],
settings: {
script: 'params.var1 * 1000',
},
type: 'bucket_script',
},
],
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '2' }],
});
// terms query ES>=6.0 - check ordering for _key and doc_type
const firstLevel = query.aggs['2'];
it('getTermsQuery(default case) es6.x should set asc sorting on _key', () => {
const builder6x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 60,
});
const query = builder6x.getTermsQuery({});
expect(query.aggs['1'].terms.order._term).toBeUndefined();
expect(query.aggs['1'].terms.order._key).toBe('asc');
expect(query.aggs['1'].terms.order._count).toBeUndefined();
});
expect(firstLevel.aggs['4']).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script).not.toBe(undefined);
expect(firstLevel.aggs['4'].bucket_script.buckets_path).toMatchObject({ var1: '_count' });
});
it('getTermsQuery(order:desc) es6.x should set desc sorting on _key', () => {
const builder6x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 60,
});
const query = builder6x.getTermsQuery({ order: 'desc' });
expect(query.aggs['1'].terms.order._term).toBeUndefined();
expect(query.aggs['1'].terms.order._key).toBe('desc');
expect(query.aggs['1'].terms.order._count).toBeUndefined();
});
it('with histogram', () => {
const query = builder.build({
metrics: [{ id: '1', type: 'count' }],
bucketAggs: [
{
type: 'histogram',
field: 'bytes',
id: '3',
settings: { interval: 10, min_doc_count: 2, missing: 5 },
},
],
});
it('getTermsQuery(orderBy:doc_count) es6.x should set desc sorting on _count', () => {
const builder6x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 60,
});
const query = builder6x.getTermsQuery({ orderBy: 'doc_count' });
expect(query.aggs['1'].terms.order._term).toBeUndefined();
expect(query.aggs['1'].terms.order._key).toBeUndefined();
expect(query.aggs['1'].terms.order._count).toBe('desc');
});
const firstLevel = query.aggs['3'];
expect(firstLevel.histogram.field).toBe('bytes');
expect(firstLevel.histogram.interval).toBe(10);
expect(firstLevel.histogram.min_doc_count).toBe(2);
expect(firstLevel.histogram.missing).toBe(5);
});
it('getTermsQuery(orderBy:doc_count, order:asc) es6.x should set asc sorting on _count', () => {
const builder6x = new ElasticQueryBuilder({
timeField: '@timestamp',
esVersion: 60,
it('with adhoc filters', () => {
const query = builder.build(
{
metrics: [{ type: 'Count', id: '0' }],
timeField: '@timestamp',
bucketAggs: [{ type: 'date_histogram', field: '@timestamp', id: '3' }],
},
[
{ key: 'key1', operator: '=', value: 'value1' },
{ key: 'key2', operator: '=', value: 'value2' },
{ key: 'key2', operator: '!=', value: 'value2' },
{ key: 'key3', operator: '<', value: 'value3' },
{ key: 'key4', operator: '>', value: 'value4' },
{ key: 'key5', operator: '=~', value: 'value5' },
{ key: 'key6', operator: '!~', value: 'value6' },
]
);
expect(query.query.bool.must[0].match_phrase['key1'].query).toBe('value1');
expect(query.query.bool.must[1].match_phrase['key2'].query).toBe('value2');
expect(query.query.bool.must_not[0].match_phrase['key2'].query).toBe('value2');
expect(query.query.bool.filter[2].range['key3'].lt).toBe('value3');
expect(query.query.bool.filter[3].range['key4'].gt).toBe('value4');
expect(query.query.bool.filter[4].regexp['key5']).toBe('value5');
expect(query.query.bool.filter[5].bool.must_not.regexp['key6']).toBe('value6');
});
describe('getTermsQuery', () => {
function testGetTermsQuery(queryDef: any) {
const query = builder.getTermsQuery(queryDef);
return query.aggs['1'].terms.order;
}
function checkSort(order: any, expected: string) {
if (builder.esVersion < 60) {
expect(order._term).toBe(expected);
expect(order._key).toBeUndefined();
} else {
expect(order._term).toBeUndefined();
expect(order._key).toBe(expected);
}
}
it('should set correct default sorting', () => {
const order = testGetTermsQuery({});
checkSort(order, 'asc');
expect(order._count).toBeUndefined();
});
it('should set correct explicit sorting', () => {
const order = testGetTermsQuery({ order: 'desc' });
console.log({ order });
checkSort(order, 'desc');
expect(order._count).toBeUndefined();
});
it('getTermsQuery(orderBy:doc_count) should set desc sorting on _count', () => {
const query = builder.getTermsQuery({ orderBy: 'doc_count' });
expect(query.aggs['1'].terms.order._term).toBeUndefined();
expect(query.aggs['1'].terms.order._key).toBeUndefined();
expect(query.aggs['1'].terms.order._count).toBe('desc');
});
it('getTermsQuery(orderBy:doc_count, order:asc) should set asc sorting on _count', () => {
const query = builder.getTermsQuery({ orderBy: 'doc_count', order: 'asc' });
expect(query.aggs['1'].terms.order._term).toBeUndefined();
expect(query.aggs['1'].terms.order._key).toBeUndefined();
expect(query.aggs['1'].terms.order._count).toBe('asc');
});
});
it('getTermsQuery should request documents and date histogram', () => {
const query = builder.getLogsQuery({}, '');
console.log({ query });
expect(query).toHaveProperty('query.bool.filter');
expect(query.aggs['2']).toHaveProperty('date_histogram');
});
});
const query = builder6x.getTermsQuery({ orderBy: 'doc_count', order: 'asc' });
expect(query.aggs['1'].terms.order._term).toBeUndefined();
expect(query.aggs['1'].terms.order._key).toBeUndefined();
expect(query.aggs['1'].terms.order._count).toBe('asc');
});
// Logs query
it('getTermsQuery should request documents and date histogram', () => {
const query = builder.getLogsQuery({});
expect(query).toHaveProperty('query.bool.filter');
expect(query.aggs['2']).toHaveProperty('date_histogram');
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment