Commit 491a4dc9 by Chris Cowan Committed by GitHub

Elasticsearch: View in context feature for logs (#28764)

* Elasticsearch: View in context feature for logs

* Fixing unused type

* Limit show context to esVersion > 5

* Fixing scope for showContextToggle; removing console.log

* Fix typing; adding check for lineField

* Update test to reflect new sorting keys for logs

* Removing sort from metadata fields

* Adding comment for clarity

* Fixing scenerio where the data is missing

* remove export & use optional chaining

Co-authored-by: Elfo404 <gio.ricci@grafana.com>
parent 8b212901
...@@ -91,6 +91,16 @@ export class LogsContainer extends PureComponent<LogsContainerProps> { ...@@ -91,6 +91,16 @@ export class LogsContainer extends PureComponent<LogsContainerProps> {
return []; return [];
}; };
showContextToggle = (): boolean => {
const { datasourceInstance } = this.props;
if (datasourceInstance?.showContextToggle) {
return datasourceInstance.showContextToggle();
}
return false;
};
getFieldLinks = (field: Field, rowIndex: number) => { getFieldLinks = (field: Field, rowIndex: number) => {
return getFieldLinksForExplore(field, rowIndex, this.props.splitOpen, this.props.range); return getFieldLinksForExplore(field, rowIndex, this.props.splitOpen, this.props.range);
}; };
...@@ -157,7 +167,7 @@ export class LogsContainer extends PureComponent<LogsContainerProps> { ...@@ -157,7 +167,7 @@ export class LogsContainer extends PureComponent<LogsContainerProps> {
timeZone={timeZone} timeZone={timeZone}
scanning={scanning} scanning={scanning}
scanRange={range.raw} scanRange={range.raw}
showContextToggle={this.props.datasourceInstance?.showContextToggle} showContextToggle={this.showContextToggle}
width={width} width={width}
getRowContext={this.getLogRowContext} getRowContext={this.getLogRowContext}
getFieldLinks={this.getFieldLinks} getFieldLinks={this.getFieldLinks}
......
...@@ -9,6 +9,8 @@ import { ...@@ -9,6 +9,8 @@ import {
DataLink, DataLink,
PluginMeta, PluginMeta,
DataQuery, DataQuery,
LogRowModel,
Field,
MetricFindValue, MetricFindValue,
} from '@grafana/data'; } from '@grafana/data';
import LanguageProvider from './language_provider'; import LanguageProvider from './language_provider';
...@@ -21,6 +23,7 @@ import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime'; ...@@ -21,6 +23,7 @@ import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv'; import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv'; import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { DataLinkConfig, ElasticsearchOptions, ElasticsearchQuery } from './types'; import { DataLinkConfig, ElasticsearchOptions, ElasticsearchQuery } from './types';
import { RowContextOptions } from '@grafana/ui/src/components/Logs/LogRowContextProvider';
import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils'; import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils';
import { import {
isMetricAggregationWithField, isMetricAggregationWithField,
...@@ -432,6 +435,74 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic ...@@ -432,6 +435,74 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
return text; return text;
} }
/**
* This method checks to ensure the user is running a 5.0+ cluster. This is
* necessary bacause the query being used for the getLogRowContext relies on the
* search_after feature.
*/
showContextToggle(): boolean {
return this.esVersion > 5;
}
getLogRowContext = async (row: LogRowModel, options?: RowContextOptions): Promise<{ data: DataFrame[] }> => {
const sortField = row.dataFrame.fields.find(f => f.name === 'sort');
const searchAfter = sortField?.values.get(row.rowIndex) || [row.timeEpochMs];
const range = this.timeSrv.timeRange();
const direction = options?.direction === 'FORWARD' ? 'asc' : 'desc';
const header = this.getQueryHeader('query_then_fetch', range.from, range.to);
const limit = options?.limit ?? 10;
const esQuery = JSON.stringify({
size: limit,
query: {
bool: {
filter: [
{
range: {
[this.timeField]: {
gte: range.from.valueOf(),
lte: range.to.valueOf(),
format: 'epoch_millis',
},
},
},
],
},
},
sort: [{ [this.timeField]: direction }, { _doc: direction }],
search_after: searchAfter,
});
const payload = [header, esQuery].join('\n') + '\n';
const url = this.getMultiSearchUrl();
const response = await this.post(url, payload);
const targets: ElasticsearchQuery[] = [{ refId: `${row.dataFrame.refId}`, metrics: [], isLogsQuery: true }];
const elasticResponse = new ElasticResponse(targets, transformHitsBasedOnDirection(response, direction));
const logResponse = elasticResponse.getLogs(this.logMessageField, this.logLevelField);
const dataFrame = _.first(logResponse.data);
if (!dataFrame) {
return { data: [] };
}
/**
* The LogRowContextProvider requires there is a field in the dataFrame.fields
* named `ts` for timestamp and `line` for the actual log line to display.
* Unfortunatly these fields are hardcoded and are required for the lines to
* be properly displayed. This code just copies the fields based on this.timeField
* and this.logMessageField and recreates the dataFrame so it works.
*/
const timestampField = dataFrame.fields.find((f: Field) => f.name === this.timeField);
const lineField = dataFrame.fields.find((f: Field) => f.name === this.logMessageField);
if (timestampField && lineField) {
return {
data: [
{
...dataFrame,
fields: [...dataFrame.fields, { ...timestampField, name: 'ts' }, { ...lineField, name: 'line' }],
},
],
};
}
return logResponse;
};
query(options: DataQueryRequest<ElasticsearchQuery>): Promise<DataQueryResponse> { query(options: DataQueryRequest<ElasticsearchQuery>): Promise<DataQueryResponse> {
let payload = ''; let payload = '';
const targets = this.interpolateVariablesInQueries(_.cloneDeep(options.targets), options.scopedVars); const targets = this.interpolateVariablesInQueries(_.cloneDeep(options.targets), options.scopedVars);
...@@ -758,3 +829,22 @@ export function enhanceDataFrame(dataFrame: DataFrame, dataLinks: DataLinkConfig ...@@ -758,3 +829,22 @@ export function enhanceDataFrame(dataFrame: DataFrame, dataLinks: DataLinkConfig
field.config.links = [...(field.config.links || []), link]; field.config.links = [...(field.config.links || []), link];
} }
} }
function transformHitsBasedOnDirection(response: any, direction: 'asc' | 'desc') {
if (direction === 'desc') {
return response;
}
const actualResponse = response.responses[0];
return {
...response,
responses: [
{
...actualResponse,
hits: {
...actualResponse.hits,
hits: actualResponse.hits.hits.reverse(),
},
},
],
};
}
...@@ -372,6 +372,7 @@ export class ElasticResponse { ...@@ -372,6 +372,7 @@ export class ElasticResponse {
_id: hit._id, _id: hit._id,
_type: hit._type, _type: hit._type,
_index: hit._index, _index: hit._index,
sort: hit.sort,
}; };
if (hit._source) { if (hit._source) {
...@@ -552,6 +553,7 @@ type Doc = { ...@@ -552,6 +553,7 @@ type Doc = {
_type: string; _type: string;
_index: string; _index: string;
_source?: any; _source?: any;
sort?: Array<string | number>;
}; };
/** /**
...@@ -572,6 +574,7 @@ const flattenHits = (hits: Doc[]): { docs: Array<Record<string, any>>; propNames ...@@ -572,6 +574,7 @@ const flattenHits = (hits: Doc[]): { docs: Array<Record<string, any>>; propNames
_id: hit._id, _id: hit._id,
_type: hit._type, _type: hit._type,
_index: hit._index, _index: hit._index,
sort: hit.sort,
_source: { ...flattened }, _source: { ...flattened },
...flattened, ...flattened,
}; };
......
...@@ -131,8 +131,14 @@ export class ElasticQueryBuilder { ...@@ -131,8 +131,14 @@ export class ElasticQueryBuilder {
documentQuery(query: any, size: number) { documentQuery(query: any, size: number) {
query.size = size; query.size = size;
query.sort = {}; query.sort = [
query.sort[this.timeField] = { order: 'desc', unmapped_type: 'boolean' }; {
[this.timeField]: { order: 'desc', unmapped_type: 'boolean' },
},
{
_doc: { order: 'desc' },
},
];
// fields field not supported on ES 5.x // fields field not supported on ES 5.x
if (this.esVersion < 5) { if (this.esVersion < 5) {
......
...@@ -243,12 +243,7 @@ describe('ElasticQueryBuilder', () => { ...@@ -243,12 +243,7 @@ describe('ElasticQueryBuilder', () => {
], ],
}, },
}, },
sort: { sort: [{ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } }, { _doc: { order: 'desc' } }],
'@timestamp': {
order: 'desc',
unmapped_type: 'boolean',
},
},
script_fields: {}, script_fields: {},
}); });
}); });
...@@ -573,7 +568,10 @@ describe('ElasticQueryBuilder', () => { ...@@ -573,7 +568,10 @@ describe('ElasticQueryBuilder', () => {
}; };
expect(query.query).toEqual(expectedQuery); expect(query.query).toEqual(expectedQuery);
expect(query.sort).toEqual({ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } }); expect(query.sort).toEqual([
{ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } },
{ _doc: { order: 'desc' } },
]);
const expectedAggs = { const expectedAggs = {
// FIXME: It's pretty weak to include this '1' in the test as it's not part of what we are testing here and // FIXME: It's pretty weak to include this '1' in the test as it's not part of what we are testing here and
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment