Elasticsearch: Fix default max concurrent shard requests (#17770)

Elasticsearch v7.0 changed the behavior of max concurrent shard
requests and the default to 5. v5.6 and before 7.0 the default
is 256. This adds some additional behavior given certain
version is selected when configure the datasource to set
default max concurrent shard requests.
Changing from a version pre-v7.0+ to v7.0+ sets max
concurrent shard requests to 5.
Changing from a version v7.0+ to a pre-v7.0 sets max
concurrent shard requests to 256.

Fixes #17454

docs/sources/installation/upgrading.md

@@ -175,3 +175,10 @@ this new setting.
 In 6.2 we completely removed the backend session storage since we replaced the previous login session implementation with an auth token.
 If you are using Auth proxy with LDAP an shared cached is used in Grafana so you might want configure [remote_cache] instead. If not
 Grafana will fallback to using the database as an shared cache.
+
+### Upgrading Elasticsearch to v7.0+
+
+The semantics of `max concurrent shard requests` changed in Elasticsearch v7.0, see [release notes](https://www.elastic.co/guide/en/elasticsearch/reference/7.0/breaking-changes-7.0.html#semantics-changed-max-concurrent-shared-requests) for reference.
+
+If you upgrade Elasticsearch to v7.0+ you should make sure to update the datasource configuration in Grafana so that version
+is `7.0+` and `max concurrent shard requests` properly configured. 256 was the default in pre v7.0 versions. In v7.0 and above 5 is the default.

public/app/plugins/datasource/elasticsearch/config_ctrl.ts

 import _ from 'lodash';
+import { ElasticsearchOptions } from './types';
+import { DataSourceInstanceSettings } from '@grafana/ui';
+import { getMaxConcurrenShardRequestOrDefault } from './datasource';
 
 export class ElasticConfigCtrl {
   static templateUrl = 'public/app/plugins/datasource/elasticsearch/partials/config.html';
-  current: any;
+  current: DataSourceInstanceSettings<ElasticsearchOptions>;
 
   /** @ngInject */
   constructor($scope) {
@@ -44,4 +47,8 @@ export class ElasticConfigCtrl {
       this.current.database = def.example || 'es-index-name';
     }
   }
+
+  versionChanged() {
+    this.current.jsonData.maxConcurrentShardRequests = getMaxConcurrenShardRequestOrDefault(this.current.jsonData);
+  }
 }

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -537,3 +537,16 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
     return false;
   }
 }
+
+export function getMaxConcurrenShardRequestOrDefault(options: ElasticsearchOptions): number {
+  if (options.maxConcurrentShardRequests === 5 && options.esVersion < 70) {
+    return 256;
+  }
+
+  if (options.maxConcurrentShardRequests === 256 && options.esVersion >= 70) {
+    return 5;
+  }
+
+  const defaultMaxConcurrentShardRequests = options.esVersion >= 70 ? 5 : 256;
+  return options.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests;
+}

public/app/plugins/datasource/elasticsearch/partials/config.html

@@ -26,7 +26,7 @@
 	<div class="gf-form">
     <span class="gf-form-label width-9">Version</span>
     <span class="gf-form-select-wrapper">
-        <select class="gf-form-input gf-size-auto" ng-model="ctrl.current.jsonData.esVersion" ng-options="f.value as f.name for f in ctrl.esVersions"></select>
+        <select class="gf-form-input gf-size-auto" ng-model="ctrl.current.jsonData.esVersion" ng-options="f.value as f.name for f in ctrl.esVersions" ng-change="ctrl.versionChanged()"></select>
     </span>
 	</div>
     <div class="gf-form max-width-30" ng-if="ctrl.current.jsonData.esVersion>=56">

public/app/plugins/datasource/elasticsearch/specs/datasource.test.ts

 import angular, { IQService } from 'angular';
 import * as dateMath from '@grafana/ui/src/utils/datemath';
 import _ from 'lodash';
-import { ElasticDatasource } from '../datasource';
+import { ElasticDatasource, getMaxConcurrenShardRequestOrDefault } from '../datasource';
 import { toUtc, dateTime } from '@grafana/ui/src/utils/moment_wrapper';
 import { BackendSrv } from 'app/core/services/backend_srv';
 import { TimeSrv } from 'app/features/dashboard/services/TimeSrv';
@@ -646,3 +646,27 @@ describe('ElasticDatasource', function(this: any) {
     });
   });
 });
+
+describe('getMaxConcurrenShardRequestOrDefault', () => {
+  const testCases = [
+    { version: 50, expectedMaxConcurrentShardRequests: 256 },
+    { version: 50, maxConcurrentShardRequests: 50, expectedMaxConcurrentShardRequests: 50 },
+    { version: 56, expectedMaxConcurrentShardRequests: 256 },
+    { version: 56, maxConcurrentShardRequests: 256, expectedMaxConcurrentShardRequests: 256 },
+    { version: 56, maxConcurrentShardRequests: 5, expectedMaxConcurrentShardRequests: 256 },
+    { version: 56, maxConcurrentShardRequests: 200, expectedMaxConcurrentShardRequests: 200 },
+    { version: 70, expectedMaxConcurrentShardRequests: 5 },
+    { version: 70, maxConcurrentShardRequests: 256, expectedMaxConcurrentShardRequests: 5 },
+    { version: 70, maxConcurrentShardRequests: 5, expectedMaxConcurrentShardRequests: 5 },
+    { version: 70, maxConcurrentShardRequests: 6, expectedMaxConcurrentShardRequests: 6 },
+  ];
+
+  testCases.forEach(tc => {
+    it(`version = ${tc.version}, maxConcurrentShardRequests = ${tc.maxConcurrentShardRequests}`, () => {
+      const options = { esVersion: tc.version, maxConcurrentShardRequests: tc.maxConcurrentShardRequests };
+      expect(getMaxConcurrenShardRequestOrDefault(options as ElasticsearchOptions)).toBe(
+        tc.expectedMaxConcurrentShardRequests
+      );
+    });
+  });
+});