Elasticsearch: add frozen indices search support (#27472)

* Elasticsearch: Add frozen indices support for alerting

pkg/tsdb/elasticsearch/client/client.go

@@ -308,12 +308,20 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
 }
 
 func (c *baseClientImpl) getMultiSearchQueryParameters() string {
+	var qs []string
+
 	if c.version >= 70 {
 		maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5)
-		return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests)
+		qs = append(qs, fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests))
+
+		includeFrozen := c.getSettings().Get("includeFrozen").MustBool(false)
+
+		if includeFrozen {
+			qs = append(qs, "ignore_throttled=false")
+		}
 	}
 
-	return ""
+	return strings.Join(qs, "&")
 }
 
 func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder {

pkg/tsdb/elasticsearch/client/client_test.go

@@ -250,6 +250,7 @@ func TestClient(t *testing.T) {
 				"maxConcurrentShardRequests": 100,
 				"timeField":                  "@timestamp",
 				"interval":                   "Daily",
+				"includeFrozen":              true,
 			}),
 		}, func(sc *scenarioContext) {
 			sc.responseBody = `{
@@ -271,6 +272,7 @@ func TestClient(t *testing.T) {
 					So(sc.request, ShouldNotBeNil)
 					So(sc.request.Method, ShouldEqual, http.MethodPost)
 					So(sc.request.URL.Path, ShouldEqual, "/_msearch")
+					So(sc.request.URL.RawQuery, ShouldNotContainSubstring, "ignore_throttled=")
 
 					So(sc.requestBody, ShouldNotBeNil)
 
@@ -312,6 +314,7 @@ func TestClient(t *testing.T) {
 				"maxConcurrentShardRequests": 6,
 				"timeField":                  "@timestamp",
 				"interval":                   "Daily",
+				"includeFrozen":              true,
 			}),
 		}, func(sc *scenarioContext) {
 			sc.responseBody = `{
@@ -333,7 +336,8 @@ func TestClient(t *testing.T) {
 					So(sc.request, ShouldNotBeNil)
 					So(sc.request.Method, ShouldEqual, http.MethodPost)
 					So(sc.request.URL.Path, ShouldEqual, "/_msearch")
-					So(sc.request.URL.RawQuery, ShouldEqual, "max_concurrent_shard_requests=6")
+					So(sc.request.URL.RawQuery, ShouldContainSubstring, "max_concurrent_shard_requests=6")
+					So(sc.request.URL.RawQuery, ShouldContainSubstring, "ignore_throttled=false")
 
 					So(sc.requestBody, ShouldNotBeNil)
 
@@ -350,6 +354,7 @@ func TestClient(t *testing.T) {
 					So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15")
 					So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true)
 					So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch")
+					So(jHeader.Get("ignore_throttled").MustBool(), ShouldBeFalse)
 
 					Convey("and replace $__interval variable", func() {
 						So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname")

public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx

@@ -18,6 +18,7 @@ export const ConfigEditor = (props: Props) => {
       jsonData: {
         ...options.jsonData,
         timeField: options.jsonData.timeField || '@timestamp',
+        includeFrozen: options.jsonData.includeFrozen ?? false,
         esVersion,
         maxConcurrentShardRequests:
           options.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests(esVersion),

public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx

 import React from 'react';
 import { EventsWithValidation, regexValidation, LegacyForms } from '@grafana/ui';
-const { Select, Input, FormField } = LegacyForms;
+const { Select, Input, FormField, Switch } = LegacyForms;
 import { ElasticsearchOptions } from '../types';
-import { DataSourceSettings, SelectableValue } from '@grafana/data';
+import { DataSourceSettings, onUpdateDatasourceJsonDataOptionChecked, SelectableValue } from '@grafana/data';
 
 const indexPatternTypes = [
   { label: 'No pattern', value: 'none' },
@@ -144,11 +144,29 @@ export const ElasticDetails = (props: Props) => {
             />
           </div>
         </div>
+        {value.jsonData.esVersion >= 70 && (
+          <div className="gf-form-group">
+            <div className="gf-form-inline">
+              <Switch
+                label="Include Frozen Indices"
+                checked={value.jsonData.includeFrozen ?? false}
+                onChange={onUpdateDatasourceJsonDataOptionChecked(
+                  {
+                    options: value,
+                    onOptionsChange: onChange,
+                  },
+                  'includeFrozen'
+                )}
+              />
+            </div>
+          </div>
+        )}
       </div>
     </>
   );
 };
 
+// TODO: Use change handlers from @grafana/data
 const changeHandler = (
   key: keyof DataSourceSettings<ElasticsearchOptions>,
   value: Props['value'],
@@ -160,6 +178,7 @@ const changeHandler = (
   });
 };
 
+// TODO: Use change handlers from @grafana/data
 const jsonDataChangeHandler = (key: keyof ElasticsearchOptions, value: Props['value'], onChange: Props['onChange']) => (
   event: React.SyntheticEvent<HTMLInputElement | HTMLSelectElement>
 ) => {

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -864,6 +864,69 @@ describe('ElasticDatasource', function(this: any) {
       expect(typeof JSON.parse(query.split('\n')[1]).query.bool.filter[0].range['@time'].gte).toBe('number');
     });
   });
+
+  describe('getMultiSearchUrl', () => {
+    describe('When esVersion >= 70', () => {
+      it('Should add correct params to URL if "includeFrozen" is enabled', () => {
+        const datasSurce = new ElasticDatasource(
+          {
+            jsonData: {
+              esVersion: 70,
+              includeFrozen: true,
+            },
+          } as DataSourceInstanceSettings<ElasticsearchOptions>,
+          templateSrv,
+          timeSrv
+        );
+
+        expect(datasSurce.getMultiSearchUrl()).toMatch(/ignore_throttled=false/);
+      });
+
+      it('Should NOT add ignore_throttled if "includeFrozen" is disabled', () => {
+        const datasSurce = new ElasticDatasource(
+          {
+            jsonData: {
+              esVersion: 70,
+              includeFrozen: false,
+            },
+          } as DataSourceInstanceSettings<ElasticsearchOptions>,
+          templateSrv,
+          timeSrv
+        );
+
+        expect(datasSurce.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/);
+      });
+    });
+
+    describe('When esVersion <= 70', () => {
+      it('Should NOT add ignore_throttled params regardless of includeFrozen', () => {
+        const datasSurceWithIncludeFrozen = new ElasticDatasource(
+          {
+            jsonData: {
+              esVersion: 60,
+              includeFrozen: true,
+            },
+          } as DataSourceInstanceSettings<ElasticsearchOptions>,
+          templateSrv,
+          timeSrv
+        );
+
+        const datasSurceWithoutIncludeFrozen = new ElasticDatasource(
+          {
+            jsonData: {
+              esVersion: 60,
+              includeFrozen: false,
+            },
+          } as DataSourceInstanceSettings<ElasticsearchOptions>,
+          templateSrv,
+          timeSrv
+        );
+
+        expect(datasSurceWithIncludeFrozen.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/);
+        expect(datasSurceWithoutIncludeFrozen.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/);
+      });
+    });
+  });
 });
 
 describe('enhanceDataFrame', () => {

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -52,6 +52,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
   logLevelField?: string;
   dataLinks: DataLinkConfig[];
   languageProvider: LanguageProvider;
+  includeFrozen: boolean;
 
   /** @ngInject */
   constructor(
@@ -79,6 +80,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
     this.logMessageField = settingsData.logMessageField || '';
     this.logLevelField = settingsData.logLevelField || '';
     this.dataLinks = settingsData.dataLinks || [];
+    this.includeFrozen = settingsData.includeFrozen ?? false;
 
     if (this.logMessageField === '') {
       this.logMessageField = undefined;
@@ -559,11 +561,17 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
   }
 
   getMultiSearchUrl() {
+    const searchParams = new URLSearchParams();
+
     if (this.esVersion >= 70 && this.maxConcurrentShardRequests) {
-      return `_msearch?max_concurrent_shard_requests=${this.maxConcurrentShardRequests}`;
+      searchParams.append('max_concurrent_shard_requests', '' + this.maxConcurrentShardRequests);
+    }
+
+    if (this.esVersion >= 70 && this.includeFrozen) {
+      searchParams.append('ignore_throttled', 'false');
     }
 
-    return '_msearch';
+    return (`_msearch?` + searchParams.toString()).replace(/\?+$/, '');
   }
 
   metricFindQuery(query: any) {

public/app/plugins/datasource/elasticsearch/types.ts

@@ -9,6 +9,7 @@ export interface ElasticsearchOptions extends DataSourceJsonData {
   logMessageField?: string;
   logLevelField?: string;
   dataLinks?: DataLinkConfig[];
+  includeFrozen?: boolean;
 }
 
 export interface ElasticsearchAggregation {