Commit 1a2dad9d by David Committed by GitHub

Prometheus: disable dynamic label lookup on big datasources (#20936)

* Prometheus: disable dynamic label lookup on big datasources

- when a prometheus datasource has more than 10000 metrics, label lookup
for the query field is disabled
- installations of that size have slow typehead lookup times and make
the editor sluggish

* Raise dynamic lookup threshold to 10000 metrics

* Run start tasks again
parent 7665dcc8
......@@ -23,7 +23,6 @@ import { PrometheusDatasource } from '../datasource';
import PromQlLanguageProvider from '../language_provider';
const HISTOGRAM_GROUP = '__histograms__';
const METRIC_MARK = 'metric';
const PRISM_SYNTAX = 'promql';
export const RECORDING_RULES_GROUP = '__recording_rules__';
......@@ -138,6 +137,7 @@ class PromQueryField extends React.PureComponent<PromQueryFieldProps, PromQueryF
componentDidMount() {
if (this.languageProvider) {
Prism.languages[PRISM_SYNTAX] = this.languageProvider.syntax;
this.refreshMetrics(makePromiseCancelable(this.languageProvider.start()));
}
this.refreshHint();
......@@ -228,17 +228,11 @@ class PromQueryField extends React.PureComponent<PromQueryFieldProps, PromQueryF
};
onUpdateLanguage = () => {
const { histogramMetrics, metrics } = this.languageProvider;
const { histogramMetrics, metrics, lookupsDisabled, lookupMetricsThreshold } = this.languageProvider;
if (!metrics) {
return;
}
Prism.languages[PRISM_SYNTAX] = this.languageProvider.syntax;
Prism.languages[PRISM_SYNTAX][METRIC_MARK] = {
alias: 'variable',
pattern: new RegExp(`(?:^|\\s)(${metrics.join('|')})(?:$|\\s)`),
};
// Build metrics tree
const metricsByPrefix = groupMetricsByPrefix(metrics);
const histogramOptions = histogramMetrics.map((hm: any) => ({ label: hm, value: hm }));
......@@ -250,7 +244,16 @@ class PromQueryField extends React.PureComponent<PromQueryFieldProps, PromQueryF
]
: metricsByPrefix;
this.setState({ metricsOptions, syntaxLoaded: true });
// Hint for big disabled lookups
let hint: QueryHint;
if (lookupsDisabled) {
hint = {
label: `Dynamic label lookup is disabled for datasources with more than ${lookupMetricsThreshold} metrics.`,
type: 'INFO',
};
}
this.setState({ hint, metricsOptions, syntaxLoaded: true });
};
onTypeahead = async (typeahead: TypeaheadInput): Promise<TypeaheadOutput> => {
......
......@@ -154,6 +154,7 @@ describe('Language completion provider', () => {
describe('label suggestions', () => {
it('returns default label suggestions on label context and no metric', async () => {
const instance = new LanguageProvider(datasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('{}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(1).value;
......@@ -173,6 +174,7 @@ describe('Language completion provider', () => {
getTimeRange: () => ({ start: 0, end: 1 }),
} as any) as PrometheusDatasource;
const instance = new LanguageProvider(datasources);
instance.lookupsDisabled = false;
const value = Plain.deserialize('metric{}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(7).value;
......@@ -204,6 +206,7 @@ describe('Language completion provider', () => {
getTimeRange: () => ({ start: 0, end: 1 }),
} as any) as PrometheusDatasource;
const instance = new LanguageProvider(datasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('{job1="foo",job2!="foo",job3=~"foo",__name__="metric",}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(54).value;
......@@ -224,6 +227,7 @@ describe('Language completion provider', () => {
return { data: { data: ['value1', 'value2'] } };
},
} as any) as PrometheusDatasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('{job!=}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(8).value;
......@@ -245,6 +249,7 @@ describe('Language completion provider', () => {
it('returns a refresher on label context and unavailable metric', async () => {
const instance = new LanguageProvider(datasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('metric{}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(7).value;
......@@ -263,6 +268,7 @@ describe('Language completion provider', () => {
...datasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as any) as PrometheusDatasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('metric{bar=ba}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(13).value;
......@@ -282,6 +288,7 @@ describe('Language completion provider', () => {
...datasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as any) as PrometheusDatasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('sum(metric{foo="xx"}) by ()');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(26).value;
......@@ -300,6 +307,7 @@ describe('Language completion provider', () => {
...datasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as any) as PrometheusDatasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('sum(metric) by ()');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(16).value;
......@@ -318,6 +326,7 @@ describe('Language completion provider', () => {
...datasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as any) as PrometheusDatasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('sum(\nmetric\n)\nby ()');
const aggregationTextBlock = value.document.getBlocks().get(3);
const ed = new SlateEditor({ value });
......@@ -343,6 +352,7 @@ describe('Language completion provider', () => {
...datasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as any) as PrometheusDatasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('sum(rate(metric[1h])) by ()');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(26).value;
......@@ -366,6 +376,7 @@ describe('Language completion provider', () => {
...datasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as any) as PrometheusDatasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('sum(rate(metric{label1="value"}[1h])) by ()');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(42).value;
......@@ -386,6 +397,7 @@ describe('Language completion provider', () => {
it('returns no suggestions inside an unclear aggregation context using alternate syntax', async () => {
const instance = new LanguageProvider(datasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('sum by ()');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(8).value;
......@@ -404,6 +416,7 @@ describe('Language completion provider', () => {
...datasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as any) as PrometheusDatasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('sum by () (metric)');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(8).value;
......@@ -429,6 +442,7 @@ describe('Language completion provider', () => {
} as any) as PrometheusDatasource;
const instance = new LanguageProvider(datasource);
instance.lookupsDisabled = false;
const value = Plain.deserialize('{}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(1).value;
......@@ -447,6 +461,57 @@ describe('Language completion provider', () => {
expect((datasource.metadataRequest as Mock).mock.calls.length).toBe(2);
});
});
describe('dynamic lookup protection for big installations', () => {
it('dynamic lookup is enabled if number of metrics is reasonably low', async () => {
const datasource: PrometheusDatasource = ({
metadataRequest: () => ({ data: { data: ['foo'] as string[] } }),
getTimeRange: () => ({ start: 0, end: 1 }),
} as any) as PrometheusDatasource;
const instance = new LanguageProvider(datasource, { lookupMetricsThreshold: 1 });
expect(instance.lookupsDisabled).toBeTruthy();
await instance.start();
expect(instance.lookupsDisabled).toBeFalsy();
});
it('dynamic lookup is disabled if number of metrics is higher than threshold', async () => {
const datasource: PrometheusDatasource = ({
metadataRequest: () => ({ data: { data: ['foo', 'bar'] as string[] } }),
getTimeRange: () => ({ start: 0, end: 1 }),
} as any) as PrometheusDatasource;
const instance = new LanguageProvider(datasource, { lookupMetricsThreshold: 1 });
expect(instance.lookupsDisabled).toBeTruthy();
await instance.start();
expect(instance.lookupsDisabled).toBeTruthy();
});
it('does not issue label-based metadata requests when lookup is disabled', async () => {
const datasource: PrometheusDatasource = ({
metadataRequest: jest.fn(() => ({ data: { data: ['foo', 'bar'] as string[] } })),
getTimeRange: jest.fn(() => ({ start: 0, end: 1 })),
} as any) as PrometheusDatasource;
const instance = new LanguageProvider(datasource, { lookupMetricsThreshold: 1 });
const value = Plain.deserialize('{}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(1).value;
const args = {
text: '',
prefix: '',
wrapperClasses: ['context-labels'],
value: valueWithSelection,
};
expect(instance.lookupsDisabled).toBeTruthy();
expect((datasource.metadataRequest as Mock).mock.calls.length).toBe(0);
await instance.start();
expect(instance.lookupsDisabled).toBeTruthy();
expect((datasource.metadataRequest as Mock).mock.calls.length).toBe(1);
await instance.provideCompletionItems(args);
expect((datasource.metadataRequest as Mock).mock.calls.length).toBe(1);
});
});
});
const simpleMetricLabelsResponse = {
......
......@@ -14,6 +14,7 @@ const DEFAULT_KEYS = ['job', 'instance'];
const EMPTY_SELECTOR = '{}';
const HISTORY_ITEM_COUNT = 5;
const HISTORY_COUNT_CUTOFF = 1000 * 60 * 60 * 24; // 24h
export const DEFAULT_LOOKUP_METRICS_THRESHOLD = 10000; // number of metrics defining an installation that's too big
const wrapLabel = (label: string): CompletionItem => ({ label });
......@@ -46,6 +47,8 @@ export default class PromQlLanguageProvider extends LanguageProvider {
metrics?: string[];
startTask: Promise<any>;
datasource: PrometheusDatasource;
lookupMetricsThreshold: number;
lookupsDisabled: boolean; // Dynamically set to true for big/slow instances
/**
* Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does
......@@ -54,13 +57,18 @@ export default class PromQlLanguageProvider extends LanguageProvider {
*/
private labelsCache = new LRU<string, Record<string, string[]>>(10);
constructor(datasource: PrometheusDatasource) {
constructor(datasource: PrometheusDatasource, initialValues?: Partial<PromQlLanguageProvider>) {
super();
this.datasource = datasource;
this.histogramMetrics = [];
this.timeRange = { start: 0, end: 0 };
this.metrics = [];
// Disable lookups until we know the instance is small enough
this.lookupMetricsThreshold = DEFAULT_LOOKUP_METRICS_THRESHOLD;
this.lookupsDisabled = true;
Object.assign(this, initialValues);
}
// Strip syntax chars
......@@ -83,22 +91,11 @@ export default class PromQlLanguageProvider extends LanguageProvider {
return [];
};
start = () => {
if (!this.startTask) {
this.startTask = this.fetchMetrics();
}
return this.startTask;
};
fetchMetrics = async () => {
this.metrics = await this.fetchMetricNames();
start = async (): Promise<any[]> => {
this.metrics = await this.request('/api/v1/label/__name__/values');
this.lookupsDisabled = this.metrics.length > this.lookupMetricsThreshold;
this.processHistogramMetrics(this.metrics);
return Promise.resolve([]);
};
fetchMetricNames = async (): Promise<string[]> => {
return this.request('/api/v1/label/__name__/values');
return [];
};
processHistogramMetrics = (data: string[]) => {
......@@ -333,6 +330,9 @@ export default class PromQlLanguageProvider extends LanguageProvider {
};
async getLabelValues(selector: string, withName?: boolean) {
if (this.lookupsDisabled) {
return undefined;
}
try {
if (selector === EMPTY_SELECTOR) {
return await this.fetchDefaultLabels();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment