Elasticsearch: Allow fields starting with underscore (#27520)

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -378,8 +378,12 @@ describe('ElasticDatasource', function(this: any) {
               mappings: {
                 metricsets: {
                   _all: {},
+                  _meta: {
+                    test: 'something',
+                  },
                   properties: {
                     '@timestamp': { type: 'date' },
+                    __timestamp: { type: 'date' },
                     beat: {
                       properties: {
                         name: {
@@ -426,6 +430,7 @@ describe('ElasticDatasource', function(this: any) {
       const fields = _.map(fieldObjects, 'text');
       expect(fields).toEqual([
         '@timestamp',
+        '__timestamp',
         'beat.name.raw',
         'beat.name',
         'beat.hostname',
@@ -455,7 +460,7 @@ describe('ElasticDatasource', function(this: any) {
       });
 
       const fields = _.map(fieldObjects, 'text');
-      expect(fields).toEqual(['@timestamp']);
+      expect(fields).toEqual(['@timestamp', '__timestamp']);
     });
   });
 

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -22,6 +22,20 @@ import { TemplateSrv } from 'app/features/templating/template_srv';
 import { TimeSrv } from 'app/features/dashboard/services/TimeSrv';
 import { DataLinkConfig, ElasticsearchOptions, ElasticsearchQuery } from './types';
 
+// Those are metadata fields as defined in https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-fields.html#_identity_metadata_fields.
+// custom fields can start with underscores, therefore is not safe to exclude anything that starts with one.
+const ELASTIC_META_FIELDS = [
+  '_index',
+  '_type',
+  '_id',
+  '_source',
+  '_size',
+  '_field_names',
+  '_ignored',
+  '_routing',
+  '_meta',
+];
+
 export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, ElasticsearchOptions> {
   basicAuth?: string;
   withCredentials?: boolean;
@@ -426,6 +440,10 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
     });
   }
 
+  isMetadataField(fieldName: string) {
+    return ELASTIC_META_FIELDS.includes(fieldName);
+  }
+
   getFields(query: any) {
     const configuredEsVersion = this.esVersion;
     return this.get('/_mapping').then((result: any) => {
@@ -441,8 +459,8 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
         nested: 'nested',
       };
 
-      function shouldAddField(obj: any, key: any, query: any) {
-        if (key[0] === '_') {
+      const shouldAddField = (obj: any, key: string, query: any) => {
+        if (this.isMetadataField(key)) {
           return false;
         }
 
@@ -452,7 +470,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
 
         // equal query type filter, or via typemap translation
         return query.type === obj.type || query.type === typeMap[obj.type];
-      }
+      };
 
       // Store subfield names: [system, process, cpu, total] -> system.process.cpu.total
       const fieldNameParts: any = [];