Elasticsearch: Create Raw Doc metric to render raw JSON docs in columns in the…

Elasticsearch: Create Raw Doc metric to render raw JSON docs in columns in the new table panel (#26233)

* test

* WIP: Create v2 version

* Update tests, remove conosole logs, refactor

* Remove incorrect types

* Update type

* Rename legacy and new metrics

* Update

* Run request when Raw Data tto Raw Document switch

* Fix size updating

* Remove _source field from table results as we are showing each source field as column

* Remove _source just for metrics, not logs

* Revert "Remove _source just for metrics, not logs"

This reverts commit 611b6922f762afa0e76fd8679a9b8160bca74e6a.

* Revert "Remove _source field from table results as we are showing each source field as column"

This reverts commit 31a9d5f81b79b91a12a7e0b74f172ff8afc4bdd3.

* Add vis preference for logs

* Update visualisation to logs

* Revert "Revert "Remove _source just for metrics""

This reverts commit a102ab2894e7a9ca6eee307aa9b60dfea169c718.

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

public/app/plugins/datasource/elasticsearch/elastic_response.ts

@@ -391,21 +391,60 @@ export class ElasticResponse {
   }
 
   getTimeSeries() {
-    const seriesList = [];
+    if (this.targets.some((target: any) => target.metrics.some((metric: any) => metric.type === 'raw_data'))) {
+      return this.processResponseToDataFrames(false);
+    }
+    return this.processResponseToSeries();
+  }
 
-    for (let i = 0; i < this.response.responses.length; i++) {
-      const response = this.response.responses[i];
+  getLogs(logMessageField?: string, logLevelField?: string): DataQueryResponse {
+    return this.processResponseToDataFrames(true, logMessageField, logLevelField);
+  }
+
+  processResponseToDataFrames(
+    isLogsRequest: boolean,
+    logMessageField?: string,
+    logLevelField?: string
+  ): DataQueryResponse {
+    const dataFrame: DataFrame[] = [];
+
+    for (let n = 0; n < this.response.responses.length; n++) {
+      const response = this.response.responses[n];
       if (response.error) {
         throw this.getErrorFromElasticResponse(this.response, response.error);
       }
 
       if (response.hits && response.hits.hits.length > 0) {
-        this.processHits(response.hits, seriesList);
+        const { propNames, docs } = flattenHits(response.hits.hits);
+        if (docs.length > 0) {
+          let series = createEmptyDataFrame(
+            propNames,
+            this.targets[0].timeField,
+            isLogsRequest,
+            logMessageField,
+            logLevelField
+          );
+
+          // Add a row for each document
+          for (const doc of docs) {
+            if (logLevelField) {
+              // Remap level field based on the datasource config. This field is then used in explore to figure out the
+              // log level. We may rewrite some actual data in the level field if they are different.
+              doc['level'] = doc[logLevelField];
+            }
+
+            series.add(doc);
+          }
+          if (isLogsRequest) {
+            series = addPreferredVisualisationType(series, 'logs');
+          }
+          dataFrame.push(series);
+        }
       }
 
       if (response.aggregations) {
         const aggregations = response.aggregations;
-        const target = this.targets[i];
+        const target = this.targets[n];
         const tmpSeriesList: any[] = [];
         const table = new TableModel();
 
@@ -413,50 +452,42 @@ export class ElasticResponse {
         this.trimDatapoints(tmpSeriesList, target);
         this.nameSeries(tmpSeriesList, target);
 
-        for (let y = 0; y < tmpSeriesList.length; y++) {
-          seriesList.push(tmpSeriesList[y]);
+        if (table.rows.length > 0) {
+          dataFrame.push(toDataFrame(table));
         }
 
-        if (table.rows.length > 0) {
-          seriesList.push(table);
+        for (let y = 0; y < tmpSeriesList.length; y++) {
+          let series = toDataFrame(tmpSeriesList[y]);
+
+          // When log results, show aggregations only in graph. Log fields are then going to be shown in table.
+          if (isLogsRequest) {
+            series = addPreferredVisualisationType(series, 'graph');
+          }
+
+          dataFrame.push(series);
         }
       }
     }
 
-    return { data: seriesList };
+    return { data: dataFrame };
   }
 
-  getLogs(logMessageField?: string, logLevelField?: string): DataQueryResponse {
-    const dataFrame: DataFrame[] = [];
+  processResponseToSeries = () => {
+    const seriesList = [];
 
-    for (let n = 0; n < this.response.responses.length; n++) {
-      const response = this.response.responses[n];
+    for (let i = 0; i < this.response.responses.length; i++) {
+      const response = this.response.responses[i];
       if (response.error) {
         throw this.getErrorFromElasticResponse(this.response, response.error);
       }
 
-      const { propNames, docs } = flattenHits(response.hits.hits);
-      if (docs.length > 0) {
-        let series = createEmptyDataFrame(propNames, this.targets[0].timeField, logMessageField, logLevelField);
-
-        // Add a row for each document
-        for (const doc of docs) {
-          if (logLevelField) {
-            // Remap level field based on the datasource config. This field is then used in explore to figure out the
-            // log level. We may rewrite some actual data in the level field if they are different.
-            doc['level'] = doc[logLevelField];
-          }
-
-          series.add(doc);
-        }
-
-        series = addPreferredVisualisationType(series, 'logs');
-        dataFrame.push(series);
+      if (response.hits && response.hits.hits.length > 0) {
+        this.processHits(response.hits, seriesList);
       }
 
       if (response.aggregations) {
         const aggregations = response.aggregations;
-        const target = this.targets[n];
+        const target = this.targets[i];
         const tmpSeriesList: any[] = [];
         const table = new TableModel();
 
@@ -465,18 +496,17 @@ export class ElasticResponse {
         this.nameSeries(tmpSeriesList, target);
 
         for (let y = 0; y < tmpSeriesList.length; y++) {
-          let series = toDataFrame(tmpSeriesList[y]);
-
-          // When log results, show aggregations only in graph. Log fields are then going to be shown in table.
-          series = addPreferredVisualisationType(series, 'graph');
+          seriesList.push(tmpSeriesList[y]);
+        }
 
-          dataFrame.push(series);
+        if (table.rows.length > 0) {
+          seriesList.push(table);
         }
       }
     }
 
-    return { data: dataFrame };
-  }
+    return { data: seriesList };
+  };
 }
 
 type Doc = {
@@ -532,6 +562,7 @@ const flattenHits = (hits: Doc[]): { docs: Array<Record<string, any>>; propNames
 const createEmptyDataFrame = (
   propNames: string[],
   timeField: string,
+  isLogsRequest: boolean,
   logMessageField?: string,
   logLevelField?: string
 ): MutableDataFrame => {
@@ -549,13 +580,6 @@ const createEmptyDataFrame = (
     }).parse = (v: any) => {
       return v || '';
     };
-  } else {
-    series.addField({
-      name: '_source',
-      type: FieldType.string,
-    }).parse = (v: any) => {
-      return JSON.stringify(v, null, 2);
-    };
   }
 
   if (logLevelField) {
@@ -574,6 +598,10 @@ const createEmptyDataFrame = (
     if (fieldNames.includes(propName)) {
       continue;
     }
+    // Do not add _source field (besides logs) as we are showing each _source field in table instead.
+    if (!isLogsRequest && propName === '_source') {
+      continue;
+    }
 
     series.addField({
       name: propName,

public/app/plugins/datasource/elasticsearch/metric_agg.ts

@@ -101,7 +101,8 @@ export class ElasticMetricAggCtrl {
           $scope.updateMovingAvgModelSettings();
           break;
         }
-        case 'raw_document': {
+        case 'raw_document':
+        case 'raw_data': {
           $scope.agg.settings.size = $scope.agg.settings.size || 500;
           $scope.settingsLinkText = 'Size: ' + $scope.agg.settings.size;
           $scope.target.metrics.splice(0, $scope.target.metrics.length, $scope.agg);
@@ -164,7 +165,10 @@ export class ElasticMetricAggCtrl {
       $scope.showOptions = false;
 
       // reset back to metric/group by query
-      if ($scope.target.bucketAggs.length === 0 && $scope.agg.type !== 'raw_document') {
+      if (
+        $scope.target.bucketAggs.length === 0 &&
+        ($scope.agg.type !== 'raw_document' || $scope.agg.type !== 'raw_data')
+      ) {
         $scope.target.bucketAggs = [queryDef.defaultBucketAgg()];
       }
 

public/app/plugins/datasource/elasticsearch/partials/metric_agg.html

@@ -101,7 +101,7 @@
     <label class="gf-form-label width-10">Percentiles</label>
     <input type="text" class="gf-form-input max-width-12" ng-model="agg.settings.percents" array-join ng-blur="onChange()"></input>
   </div>
-  <div class="gf-form offset-width-7" ng-if="agg.type === 'raw_document'">
+  <div class="gf-form offset-width-7" ng-if="agg.type === 'raw_document' ||  agg.type === 'raw_data'">
     <label class="gf-form-label width-10">Size</label>
     <input type="number" class="gf-form-input max-width-12" ng-model="agg.settings.size" ng-blur="onChange()"></input>
   </div>

public/app/plugins/datasource/elasticsearch/query_builder.ts

@@ -212,7 +212,8 @@ export class ElasticQueryBuilder {
     // If target doesn't have bucketAggs and type is not raw_document, it is invalid query.
     if (target.bucketAggs.length === 0) {
       metric = target.metrics[0];
-      if (!metric || metric.type !== 'raw_document') {
+
+      if (!metric || !(metric.type === 'raw_document' || metric.type === 'raw_data')) {
         throw { message: 'Invalid query' };
       }
     }
@@ -221,7 +222,7 @@ export class ElasticQueryBuilder {
      * Check if metric type is raw_document. If metric doesn't have size (or size is 0), update size to 500.
      * Otherwise it will not be a valid query and error will be thrown.
      */
-    if (target.metrics?.[0]?.type === 'raw_document') {
+    if (target.metrics?.[0]?.type === 'raw_document' || target.metrics?.[0]?.type === 'raw_data') {
       metric = target.metrics[0];
       const size = (metric.settings && metric.settings.size !== 0 && metric.settings.size) || 500;
       return this.documentQuery(query, size);

public/app/plugins/datasource/elasticsearch/query_ctrl.ts

@@ -51,9 +51,15 @@ export class ElasticQueryCtrl extends QueryCtrl {
   }
 
   queryUpdated() {
+    // As Raw Data and Raw Document have the same request, we need to run refresh if they are updated
+    const isPossiblyRawDataSwitch = this.target.metrics.some(
+      (metric: any) => metric.type === 'raw_data' || metric.type === 'raw_document'
+    );
     const newJson = angular.toJson(this.datasource.queryBuilder.build(this.target), true);
     if (this.rawQueryOld && newJson !== this.rawQueryOld) {
       this.refresh();
+    } else if (isPossiblyRawDataSwitch) {
+      this.refresh();
     }
 
     this.rawQueryOld = newJson;

public/app/plugins/datasource/elasticsearch/query_def.ts

@@ -79,7 +79,8 @@ export const metricAggTypes = [
     supportsMultipleBucketPaths: true,
     minVersion: 2,
   },
-  { text: 'Raw Document', value: 'raw_document', requiresField: false },
+  { text: 'Raw Document (legacy)', value: 'raw_document', requiresField: false },
+  { text: 'Raw Data', value: 'raw_data', requiresField: false },
   { text: 'Logs', value: 'logs', requiresField: false },
 ];
 

public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts

@@ -92,25 +92,25 @@ describe('ElasticQueryDef', () => {
   describe('pipeline aggs depending on esverison', () => {
     describe('using esversion undefined', () => {
       test('should not get pipeline aggs', () => {
-        expect(queryDef.getMetricAggTypes(undefined).length).toBe(10);
+        expect(queryDef.getMetricAggTypes(undefined).length).toBe(11);
       });
     });
 
     describe('using esversion 1', () => {
       test('should not get pipeline aggs', () => {
-        expect(queryDef.getMetricAggTypes(1).length).toBe(10);
+        expect(queryDef.getMetricAggTypes(1).length).toBe(11);
       });
     });
 
     describe('using esversion 2', () => {
       test('should get pipeline aggs', () => {
-        expect(queryDef.getMetricAggTypes(2).length).toBe(14);
+        expect(queryDef.getMetricAggTypes(2).length).toBe(15);
       });
     });
 
     describe('using esversion 5', () => {
       test('should get pipeline aggs', () => {
-        expect(queryDef.getMetricAggTypes(5).length).toBe(14);
+        expect(queryDef.getMetricAggTypes(5).length).toBe(15);
       });
     });
   });