Elasticsearch: View in context feature for logs (#28764)

* Elasticsearch: View in context feature for logs

* Fixing unused type

* Limit show context to esVersion > 5

* Fixing scope for showContextToggle; removing console.log

* Fix typing; adding check for lineField

* Update test to reflect new sorting keys for logs

* Removing sort from metadata fields

* Adding comment for clarity

* Fixing scenerio where the data is missing

* remove export & use optional chaining

Co-authored-by: Elfo404 <gio.ricci@grafana.com>

public/app/features/explore/LogsContainer.tsx

@@ -91,6 +91,16 @@ export class LogsContainer extends PureComponent<LogsContainerProps> {
     return [];
   };
 
+  showContextToggle = (): boolean => {
+    const { datasourceInstance } = this.props;
+
+    if (datasourceInstance?.showContextToggle) {
+      return datasourceInstance.showContextToggle();
+    }
+
+    return false;
+  };
+
   getFieldLinks = (field: Field, rowIndex: number) => {
     return getFieldLinksForExplore(field, rowIndex, this.props.splitOpen, this.props.range);
   };
@@ -157,7 +167,7 @@ export class LogsContainer extends PureComponent<LogsContainerProps> {
               timeZone={timeZone}
               scanning={scanning}
               scanRange={range.raw}
-              showContextToggle={this.props.datasourceInstance?.showContextToggle}
+              showContextToggle={this.showContextToggle}
               width={width}
               getRowContext={this.getLogRowContext}
               getFieldLinks={this.getFieldLinks}

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -9,6 +9,8 @@ import {
   DataLink,
   PluginMeta,
   DataQuery,
+  LogRowModel,
+  Field,
   MetricFindValue,
 } from '@grafana/data';
 import LanguageProvider from './language_provider';
@@ -21,6 +23,7 @@ import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
 import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
 import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
 import { DataLinkConfig, ElasticsearchOptions, ElasticsearchQuery } from './types';
+import { RowContextOptions } from '@grafana/ui/src/components/Logs/LogRowContextProvider';
 import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils';
 import {
   isMetricAggregationWithField,
@@ -432,6 +435,74 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
     return text;
   }
 
+  /**
+   * This method checks to ensure the user is running a 5.0+ cluster. This is
+   * necessary bacause the query being used for the getLogRowContext relies on the
+   * search_after feature.
+   */
+  showContextToggle(): boolean {
+    return this.esVersion > 5;
+  }
+
+  getLogRowContext = async (row: LogRowModel, options?: RowContextOptions): Promise<{ data: DataFrame[] }> => {
+    const sortField = row.dataFrame.fields.find(f => f.name === 'sort');
+    const searchAfter = sortField?.values.get(row.rowIndex) || [row.timeEpochMs];
+    const range = this.timeSrv.timeRange();
+    const direction = options?.direction === 'FORWARD' ? 'asc' : 'desc';
+    const header = this.getQueryHeader('query_then_fetch', range.from, range.to);
+    const limit = options?.limit ?? 10;
+    const esQuery = JSON.stringify({
+      size: limit,
+      query: {
+        bool: {
+          filter: [
+            {
+              range: {
+                [this.timeField]: {
+                  gte: range.from.valueOf(),
+                  lte: range.to.valueOf(),
+                  format: 'epoch_millis',
+                },
+              },
+            },
+          ],
+        },
+      },
+      sort: [{ [this.timeField]: direction }, { _doc: direction }],
+      search_after: searchAfter,
+    });
+    const payload = [header, esQuery].join('\n') + '\n';
+    const url = this.getMultiSearchUrl();
+    const response = await this.post(url, payload);
+    const targets: ElasticsearchQuery[] = [{ refId: `${row.dataFrame.refId}`, metrics: [], isLogsQuery: true }];
+    const elasticResponse = new ElasticResponse(targets, transformHitsBasedOnDirection(response, direction));
+    const logResponse = elasticResponse.getLogs(this.logMessageField, this.logLevelField);
+    const dataFrame = _.first(logResponse.data);
+    if (!dataFrame) {
+      return { data: [] };
+    }
+    /**
+     * The LogRowContextProvider requires there is a field in the dataFrame.fields
+     * named `ts` for timestamp and `line` for the actual log line to display.
+     * Unfortunatly these fields are hardcoded and are required for the lines to
+     * be properly displayed. This code just copies the fields based on this.timeField
+     * and this.logMessageField and recreates the dataFrame so it works.
+     */
+    const timestampField = dataFrame.fields.find((f: Field) => f.name === this.timeField);
+    const lineField = dataFrame.fields.find((f: Field) => f.name === this.logMessageField);
+    if (timestampField && lineField) {
+      return {
+        data: [
+          {
+            ...dataFrame,
+            fields: [...dataFrame.fields, { ...timestampField, name: 'ts' }, { ...lineField, name: 'line' }],
+          },
+        ],
+      };
+    }
+    return logResponse;
+  };
+
   query(options: DataQueryRequest<ElasticsearchQuery>): Promise<DataQueryResponse> {
     let payload = '';
     const targets = this.interpolateVariablesInQueries(_.cloneDeep(options.targets), options.scopedVars);
@@ -758,3 +829,22 @@ export function enhanceDataFrame(dataFrame: DataFrame, dataLinks: DataLinkConfig
     field.config.links = [...(field.config.links || []), link];
   }
 }
+
+function transformHitsBasedOnDirection(response: any, direction: 'asc' | 'desc') {
+  if (direction === 'desc') {
+    return response;
+  }
+  const actualResponse = response.responses[0];
+  return {
+    ...response,
+    responses: [
+      {
+        ...actualResponse,
+        hits: {
+          ...actualResponse.hits,
+          hits: actualResponse.hits.hits.reverse(),
+        },
+      },
+    ],
+  };
+}

public/app/plugins/datasource/elasticsearch/elastic_response.ts

@@ -372,6 +372,7 @@ export class ElasticResponse {
         _id: hit._id,
         _type: hit._type,
         _index: hit._index,
+        sort: hit.sort,
       };
 
       if (hit._source) {
@@ -552,6 +553,7 @@ type Doc = {
   _type: string;
   _index: string;
   _source?: any;
+  sort?: Array<string | number>;
 };
 
 /**
@@ -572,6 +574,7 @@ const flattenHits = (hits: Doc[]): { docs: Array<Record<string, any>>; propNames
       _id: hit._id,
       _type: hit._type,
       _index: hit._index,
+      sort: hit.sort,
       _source: { ...flattened },
       ...flattened,
     };

public/app/plugins/datasource/elasticsearch/query_builder.ts

@@ -131,8 +131,14 @@ export class ElasticQueryBuilder {
 
   documentQuery(query: any, size: number) {
     query.size = size;
-    query.sort = {};
-    query.sort[this.timeField] = { order: 'desc', unmapped_type: 'boolean' };
+    query.sort = [
+      {
+        [this.timeField]: { order: 'desc', unmapped_type: 'boolean' },
+      },
+      {
+        _doc: { order: 'desc' },
+      },
+    ];
 
     // fields field not supported on ES 5.x
     if (this.esVersion < 5) {

public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts

@@ -243,12 +243,7 @@ describe('ElasticQueryBuilder', () => {
               ],
             },
           },
-          sort: {
-            '@timestamp': {
-              order: 'desc',
-              unmapped_type: 'boolean',
-            },
-          },
+          sort: [{ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } }, { _doc: { order: 'desc' } }],
           script_fields: {},
         });
       });
@@ -573,7 +568,10 @@ describe('ElasticQueryBuilder', () => {
           };
           expect(query.query).toEqual(expectedQuery);
 
-          expect(query.sort).toEqual({ '@timestamp': { order: 'desc', unmapped_type: 'boolean' } });
+          expect(query.sort).toEqual([
+            { '@timestamp': { order: 'desc', unmapped_type: 'boolean' } },
+            { _doc: { order: 'desc' } },
+          ]);
 
           const expectedAggs = {
             // FIXME: It's pretty weak to include this '1' in the test as it's not part of what we are testing here and