Skip to content

N
nexpie-grafana-theme
Commit 61a3160c by Giordano Ricci Committed by GitHub
Options

Elasticsearch: add frozen indices search support (#27472) 

* Elasticsearch: Add frozen indices support for alerting
parent 590f1407
Showing with 112 additions and 7 deletions
pkg/tsdb/elasticsearch/client/client.go
...@@ -308,12 +308,20 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque ...@@ -308,12 +308,20 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
} }
func (c *baseClientImpl) getMultiSearchQueryParameters() string { func (c *baseClientImpl) getMultiSearchQueryParameters() string {
var qs []string
if c.version >= 70 { if c.version >= 70 {
maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5) maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5)
return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests) qs = append(qs, fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests))
includeFrozen := c.getSettings().Get("includeFrozen").MustBool(false)
if includeFrozen {
qs = append(qs, "ignore_throttled=false")
}
} }
return "" return strings.Join(qs, "&")
} }
func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder { func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder {
......
pkg/tsdb/elasticsearch/client/client_test.go
...@@ -250,6 +250,7 @@ func TestClient(t *testing.T) { ...@@ -250,6 +250,7 @@ func TestClient(t *testing.T) {
"maxConcurrentShardRequests": 100, "maxConcurrentShardRequests": 100,
"timeField": "@timestamp", "timeField": "@timestamp",
"interval": "Daily", "interval": "Daily",
"includeFrozen": true,
}), }),
}, func(sc *scenarioContext) { }, func(sc *scenarioContext) {
sc.responseBody = `{ sc.responseBody = `{
...@@ -271,6 +272,7 @@ func TestClient(t *testing.T) { ...@@ -271,6 +272,7 @@ func TestClient(t *testing.T) {
So(sc.request, ShouldNotBeNil) So(sc.request, ShouldNotBeNil)
So(sc.request.Method, ShouldEqual, http.MethodPost) So(sc.request.Method, ShouldEqual, http.MethodPost)
So(sc.request.URL.Path, ShouldEqual, "/_msearch") So(sc.request.URL.Path, ShouldEqual, "/_msearch")
So(sc.request.URL.RawQuery, ShouldNotContainSubstring, "ignore_throttled=")
So(sc.requestBody, ShouldNotBeNil) So(sc.requestBody, ShouldNotBeNil)
...@@ -312,6 +314,7 @@ func TestClient(t *testing.T) { ...@@ -312,6 +314,7 @@ func TestClient(t *testing.T) {
"maxConcurrentShardRequests": 6, "maxConcurrentShardRequests": 6,
"timeField": "@timestamp", "timeField": "@timestamp",
"interval": "Daily", "interval": "Daily",
"includeFrozen": true,
}), }),
}, func(sc *scenarioContext) { }, func(sc *scenarioContext) {
sc.responseBody = `{ sc.responseBody = `{
...@@ -333,7 +336,8 @@ func TestClient(t *testing.T) { ...@@ -333,7 +336,8 @@ func TestClient(t *testing.T) {
So(sc.request, ShouldNotBeNil) So(sc.request, ShouldNotBeNil)
So(sc.request.Method, ShouldEqual, http.MethodPost) So(sc.request.Method, ShouldEqual, http.MethodPost)
So(sc.request.URL.Path, ShouldEqual, "/_msearch") So(sc.request.URL.Path, ShouldEqual, "/_msearch")
So(sc.request.URL.RawQuery, ShouldEqual, "max_concurrent_shard_requests=6") So(sc.request.URL.RawQuery, ShouldContainSubstring, "max_concurrent_shard_requests=6")
So(sc.request.URL.RawQuery, ShouldContainSubstring, "ignore_throttled=false")
So(sc.requestBody, ShouldNotBeNil) So(sc.requestBody, ShouldNotBeNil)
...@@ -350,6 +354,7 @@ func TestClient(t *testing.T) { ...@@ -350,6 +354,7 @@ func TestClient(t *testing.T) {
So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15") So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15")
So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true) So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true)
So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch") So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch")
So(jHeader.Get("ignore_throttled").MustBool(), ShouldBeFalse)
Convey("and replace $__interval variable", func() { Convey("and replace $__interval variable", func() {
So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname") So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname")
......
public/app/plugins/datasource/elasticsearch/configuration/ConfigEditor.tsx
...@@ -18,6 +18,7 @@ export const ConfigEditor = (props: Props) => { ...@@ -18,6 +18,7 @@ export const ConfigEditor = (props: Props) => {
jsonData: { jsonData: {
...options.jsonData, ...options.jsonData,
timeField: options.jsonData.timeField || '@timestamp', timeField: options.jsonData.timeField || '@timestamp',
includeFrozen: options.jsonData.includeFrozen ?? false,
esVersion, esVersion,
maxConcurrentShardRequests: maxConcurrentShardRequests:
options.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests(esVersion), options.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests(esVersion),
......
public/app/plugins/datasource/elasticsearch/configuration/ElasticDetails.tsx
import React from 'react'; import React from 'react';
import { EventsWithValidation, regexValidation, LegacyForms } from '@grafana/ui'; import { EventsWithValidation, regexValidation, LegacyForms } from '@grafana/ui';
const { Select, Input, FormField } = LegacyForms; const { Select, Input, FormField, Switch } = LegacyForms;
import { ElasticsearchOptions } from '../types'; import { ElasticsearchOptions } from '../types';
import { DataSourceSettings, SelectableValue } from '@grafana/data'; import { DataSourceSettings, onUpdateDatasourceJsonDataOptionChecked, SelectableValue } from '@grafana/data';
const indexPatternTypes = [ const indexPatternTypes = [
{ label: 'No pattern', value: 'none' }, { label: 'No pattern', value: 'none' },
...@@ -144,11 +144,29 @@ export const ElasticDetails = (props: Props) => { ...@@ -144,11 +144,29 @@ export const ElasticDetails = (props: Props) => {
/> />
</div> </div>
</div> </div>
{value.jsonData.esVersion >= 70 && (
<div className="gf-form-group">
<div className="gf-form-inline">
<Switch
label="Include Frozen Indices"
checked={value.jsonData.includeFrozen ?? false}
onChange={onUpdateDatasourceJsonDataOptionChecked(
{
options: value,
onOptionsChange: onChange,
},
'includeFrozen'
)}
/>
</div>
</div>
)}
</div> </div>
</> </>
); );
}; };
// TODO: Use change handlers from @grafana/data
const changeHandler = ( const changeHandler = (
key: keyof DataSourceSettings<ElasticsearchOptions>, key: keyof DataSourceSettings<ElasticsearchOptions>,
value: Props['value'], value: Props['value'],
...@@ -160,6 +178,7 @@ const changeHandler = ( ...@@ -160,6 +178,7 @@ const changeHandler = (
}); });
}; };
// TODO: Use change handlers from @grafana/data
const jsonDataChangeHandler = (key: keyof ElasticsearchOptions, value: Props['value'], onChange: Props['onChange']) => ( const jsonDataChangeHandler = (key: keyof ElasticsearchOptions, value: Props['value'], onChange: Props['onChange']) => (
event: React.SyntheticEvent<HTMLInputElement | HTMLSelectElement> event: React.SyntheticEvent<HTMLInputElement | HTMLSelectElement>
) => { ) => {
......
public/app/plugins/datasource/elasticsearch/datasource.test.ts
...@@ -864,6 +864,69 @@ describe('ElasticDatasource', function(this: any) { ...@@ -864,6 +864,69 @@ describe('ElasticDatasource', function(this: any) {
expect(typeof JSON.parse(query.split('\n')[1]).query.bool.filter[0].range['@time'].gte).toBe('number'); expect(typeof JSON.parse(query.split('\n')[1]).query.bool.filter[0].range['@time'].gte).toBe('number');
}); });
}); });
describe('getMultiSearchUrl', () => {
describe('When esVersion >= 70', () => {
it('Should add correct params to URL if "includeFrozen" is enabled', () => {
const datasSurce = new ElasticDatasource(
{
jsonData: {
esVersion: 70,
includeFrozen: true,
},
} as DataSourceInstanceSettings<ElasticsearchOptions>,
templateSrv,
timeSrv
);
expect(datasSurce.getMultiSearchUrl()).toMatch(/ignore_throttled=false/);
});
it('Should NOT add ignore_throttled if "includeFrozen" is disabled', () => {
const datasSurce = new ElasticDatasource(
{
jsonData: {
esVersion: 70,
includeFrozen: false,
},
} as DataSourceInstanceSettings<ElasticsearchOptions>,
templateSrv,
timeSrv
);
expect(datasSurce.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/);
});
});
describe('When esVersion <= 70', () => {
it('Should NOT add ignore_throttled params regardless of includeFrozen', () => {
const datasSurceWithIncludeFrozen = new ElasticDatasource(
{
jsonData: {
esVersion: 60,
includeFrozen: true,
},
} as DataSourceInstanceSettings<ElasticsearchOptions>,
templateSrv,
timeSrv
);
const datasSurceWithoutIncludeFrozen = new ElasticDatasource(
{
jsonData: {
esVersion: 60,
includeFrozen: false,
},
} as DataSourceInstanceSettings<ElasticsearchOptions>,
templateSrv,
timeSrv
);
expect(datasSurceWithIncludeFrozen.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/);
expect(datasSurceWithoutIncludeFrozen.getMultiSearchUrl()).not.toMatch(/ignore_throttled=false/);
});
});
});
}); });
describe('enhanceDataFrame', () => { describe('enhanceDataFrame', () => {
......
public/app/plugins/datasource/elasticsearch/datasource.ts
...@@ -52,6 +52,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic ...@@ -52,6 +52,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
logLevelField?: string; logLevelField?: string;
dataLinks: DataLinkConfig[]; dataLinks: DataLinkConfig[];
languageProvider: LanguageProvider; languageProvider: LanguageProvider;
includeFrozen: boolean;
/** @ngInject */ /** @ngInject */
constructor( constructor(
...@@ -79,6 +80,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic ...@@ -79,6 +80,7 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
this.logMessageField = settingsData.logMessageField || ''; this.logMessageField = settingsData.logMessageField || '';
this.logLevelField = settingsData.logLevelField || ''; this.logLevelField = settingsData.logLevelField || '';
this.dataLinks = settingsData.dataLinks || []; this.dataLinks = settingsData.dataLinks || [];
this.includeFrozen = settingsData.includeFrozen ?? false;
if (this.logMessageField === '') { if (this.logMessageField === '') {
this.logMessageField = undefined; this.logMessageField = undefined;
...@@ -559,11 +561,17 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic ...@@ -559,11 +561,17 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
} }
getMultiSearchUrl() { getMultiSearchUrl() {
const searchParams = new URLSearchParams();
if (this.esVersion >= 70 && this.maxConcurrentShardRequests) { if (this.esVersion >= 70 && this.maxConcurrentShardRequests) {
return `_msearch?max_concurrent_shard_requests=${this.maxConcurrentShardRequests}`; searchParams.append('max_concurrent_shard_requests', '' + this.maxConcurrentShardRequests);
}
if (this.esVersion >= 70 && this.includeFrozen) {
searchParams.append('ignore_throttled', 'false');
} }
return '_msearch'; return (`_msearch?` + searchParams.toString()).replace(/\?+$/, '');
} }
metricFindQuery(query: any) { metricFindQuery(query: any) {
......
public/app/plugins/datasource/elasticsearch/types.ts
...@@ -9,6 +9,7 @@ export interface ElasticsearchOptions extends DataSourceJsonData { ...@@ -9,6 +9,7 @@ export interface ElasticsearchOptions extends DataSourceJsonData {
logMessageField?: string; logMessageField?: string;
logLevelField?: string; logLevelField?: string;
dataLinks?: DataLinkConfig[]; dataLinks?: DataLinkConfig[];
includeFrozen?: boolean;
} }
export interface ElasticsearchAggregation { export interface ElasticsearchAggregation {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment