Skip to content

N
nexpie-grafana-theme
Commit 54a3f5fd by David Committed by GitHub
Options

Loki: use series API for stream facetting (#21332)

parent 2cf538a4
Showing with 248 additions and 99 deletions
public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap
......@@ -56,11 +56,43 @@ exports[`LokiExploreQueryEditor should render component 1`] = `
"languageProvider": LokiLanguageProvider {
"cleanText": [Function],
"datasource": [Circular],
"fetchSeriesLabels": [Function],
"getBeginningCompletionItems": [Function],
"getTermCompletionItems": [Function],
"labelKeys": Object {},
"labelValues": Object {},
"labelKeys": Array [],
"labelsCache": LRUCache {
Symbol(max): 10,
Symbol(lengthCalculator): [Function],
Symbol(allowStale): false,
Symbol(maxAge): 0,
Symbol(dispose): undefined,
Symbol(noDisposeOnSet): false,
Symbol(updateAgeOnGet): false,
Symbol(cache): Map {},
Symbol(lruList): Yallist {
"head": null,
"length": 0,
"tail": null,
},
Symbol(length): 0,
},
"request": [Function],
"seriesCache": LRUCache {
Symbol(max): 10,
Symbol(lengthCalculator): [Function],
Symbol(allowStale): false,
Symbol(maxAge): 0,
Symbol(dispose): undefined,
Symbol(noDisposeOnSet): false,
Symbol(updateAgeOnGet): false,
Symbol(cache): Map {},
Symbol(lruList): Yallist {
"head": null,
"length": 0,
"tail": null,
},
Symbol(length): 0,
},
"start": [Function],
},
"metadataRequest": [Function],
......
public/app/plugins/datasource/loki/components/useLokiSyntaxAndLabels.test.ts
......@@ -59,9 +59,9 @@ describe('useLokiSyntax hook', () => {
await waitForNextUpdate();
expect(result.current.logLabelOptions).toEqual(logLabelOptionsMock2);
languageProvider.fetchLabelValues = (key: string) => {
languageProvider.fetchLabelValues = (key: string, absoluteRange: AbsoluteTimeRange) => {
languageProvider.logLabelOptions = logLabelOptionsMock3;
return Promise.resolve();
return Promise.resolve([]);
};
act(() => result.current.setActiveOption([activeOptionMock]));
......
public/app/plugins/datasource/loki/datasource.ts
......@@ -390,9 +390,7 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
async metadataRequest(url: string, params?: Record<string, string>) {
const res = await this._request(url, params, { silent: true }).toPromise();
return {
data: { data: res.data.data || res.data.values || [] },
};
return res.data.data || res.data.values || [];
}
async metricFindQuery(query: string) {
......@@ -423,7 +421,7 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
async labelNamesQuery() {
const url = (await this.getVersion()) === 'v0' ? `${LEGACY_LOKI_ENDPOINT}/label` : `${LOKI_ENDPOINT}/label`;
const result = await this.metadataRequest(url);
return result.data.data.map((value: string) => ({ text: value }));
return result.map((value: string) => ({ text: value }));
}
async labelValuesQuery(label: string) {
......@@ -432,7 +430,7 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
? `${LEGACY_LOKI_ENDPOINT}/label/${label}/values`
: `${LOKI_ENDPOINT}/label/${label}/values`;
const result = await this.metadataRequest(url);
return result.data.data.map((value: string) => ({ text: value }));
return result.map((value: string) => ({ text: value }));
}
interpolateQueryExpr(value: any, variable: any) {
......
public/app/plugins/datasource/loki/language_provider.test.ts
import Plain from 'slate-plain-serializer';
import { Editor as SlateEditor } from 'slate';
import LanguageProvider, { LABEL_REFRESH_INTERVAL, LokiHistoryItem, rangeToParams } from './language_provider';
import { AbsoluteTimeRange } from '@grafana/data';
......@@ -85,34 +84,53 @@ describe('Language completion provider', () => {
});
});
describe('label suggestions', () => {
it('returns default label suggestions on label context', async () => {
const instance = new LanguageProvider(datasource);
const value = Plain.deserialize('{}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(1).value;
const result = await instance.provideCompletionItems(
{
text: '',
prefix: '',
wrapperClasses: ['context-labels'],
value: valueWithSelection,
},
{ absoluteRange: rangeMock }
);
describe('label key suggestions', () => {
it('returns all label suggestions on empty selector', async () => {
const datasource = makeMockLokiDatasource({ label1: [], label2: [] });
const provider = await getLanguageProvider(datasource);
const input = createTypeaheadInput('{}', '', '', 1);
const result = await provider.provideCompletionItems(input, { absoluteRange: rangeMock });
expect(result.context).toBe('context-labels');
expect(result.suggestions).toEqual([{ items: [{ label: 'job' }, { label: 'namespace' }], label: 'Labels' }]);
expect(result.suggestions).toEqual([{ items: [{ label: 'label1' }, { label: 'label2' }], label: 'Labels' }]);
});
it('returns label suggestions from Loki', async () => {
it('returns all label suggestions on selector when starting to type', async () => {
const datasource = makeMockLokiDatasource({ label1: [], label2: [] });
const provider = await getLanguageProvider(datasource);
const input = createTypeaheadInput('{}', '');
const input = createTypeaheadInput('{l}', '', '', 2);
const result = await provider.provideCompletionItems(input, { absoluteRange: rangeMock });
expect(result.context).toBe('context-labels');
expect(result.suggestions).toEqual([{ items: [{ label: 'label1' }, { label: 'label2' }], label: 'Labels' }]);
});
});
describe('label suggestions facetted', () => {
it('returns facetted label suggestions based on selector', async () => {
const datasource = makeMockLokiDatasource(
{ label1: [], label2: [] },
{ '{foo="bar"}': [{ label1: 'label_val1' }] }
);
const provider = await getLanguageProvider(datasource);
const input = createTypeaheadInput('{foo="bar",}', '', '', 11);
const result = await provider.provideCompletionItems(input, { absoluteRange: rangeMock });
expect(result.context).toBe('context-labels');
expect(result.suggestions).toEqual([{ items: [{ label: 'label1' }], label: 'Labels' }]);
});
it('returns facetted label suggestions for multipule selectors', async () => {
const datasource = makeMockLokiDatasource(
{ label1: [], label2: [] },
{ '{baz="42",foo="bar"}': [{ label2: 'label_val2' }] }
);
const provider = await getLanguageProvider(datasource);
const input = createTypeaheadInput('{baz="42",foo="bar",}', '', '', 20);
const result = await provider.provideCompletionItems(input, { absoluteRange: rangeMock });
expect(result.context).toBe('context-labels');
expect(result.suggestions).toEqual([{ items: [{ label: 'label2' }], label: 'Labels' }]);
});
});
describe('label suggestions', () => {
it('returns label values suggestions from Loki', async () => {
const datasource = makeMockLokiDatasource({ label1: ['label1_val1', 'label1_val2'], label2: [] });
const provider = await getLanguageProvider(datasource);
......
public/app/plugins/datasource/loki/language_provider.ts
// Libraries
import _ from 'lodash';
import LRU from 'lru-cache';
// Services & Utils
import { parseSelector, labelRegexp, selectorRegexp } from 'app/plugins/datasource/prometheus/language_utils';
import {
parseSelector,
labelRegexp,
selectorRegexp,
processLabels,
} from 'app/plugins/datasource/prometheus/language_utils';
import syntax, { FUNCTIONS } from './syntax';
// Types
......@@ -12,7 +18,7 @@ import { PromQuery } from '../prometheus/types';
import { RATE_RANGES } from '../prometheus/promql';
import LokiDatasource from './datasource';
import { CompletionItem, TypeaheadInput, TypeaheadOutput } from '@grafana/ui';
import { CompletionItem, TypeaheadInput, TypeaheadOutput, CompletionItemGroup } from '@grafana/ui';
import { Grammar } from 'prismjs';
const DEFAULT_KEYS = ['job', 'namespace'];
......@@ -50,20 +56,27 @@ export function addHistoryMetadata(item: CompletionItem, history: LokiHistoryIte
}
export default class LokiLanguageProvider extends LanguageProvider {
labelKeys?: { [index: string]: string[] }; // metric -> [labelKey,...]
labelValues?: { [index: string]: { [index: string]: string[] } }; // metric -> labelKey -> [labelValue,...]
labelKeys?: string[];
logLabelOptions: any[];
logLabelFetchTs?: number;
started: boolean;
initialRange: AbsoluteTimeRange;
datasource: LokiDatasource;
lookupsDisabled: boolean; // Dynamically set to true for big/slow instances
/**
* Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does
* not account for different size of a response. If that is needed a `length` function can be added in the options.
* 10 as a max size is totally arbitrary right now.
*/
private seriesCache = new LRU<string, Record<string, string[]>>(10);
private labelsCache = new LRU<string, string[]>(10);
constructor(datasource: LokiDatasource, initialValues?: any) {
super();
this.datasource = datasource;
this.labelKeys = {};
this.labelValues = {};
this.labelKeys = [];
Object.assign(this, initialValues);
}
......@@ -75,8 +88,14 @@ export default class LokiLanguageProvider extends LanguageProvider {
return syntax;
}
request = (url: string, params?: any): Promise<{ data: { data: string[] } }> => {
return this.datasource.metadataRequest(url, params);
request = async (url: string, params?: any): Promise<any> => {
try {
return await this.datasource.metadataRequest(url, params);
} catch (error) {
console.error(error);
}
return undefined;
};
/**
......@@ -95,12 +114,7 @@ export default class LokiLanguageProvider extends LanguageProvider {
};
getLabelKeys(): string[] {
return this.labelKeys[EMPTY_SELECTOR];
}
async getLabelValues(key: string): Promise<string[]> {
await this.fetchLabelValues(key, this.initialRange);
return this.labelValues[EMPTY_SELECTOR][key];
return this.labelKeys;
}
/**
......@@ -219,42 +233,66 @@ export default class LokiLanguageProvider extends LanguageProvider {
{ text, wrapperClasses, labelKey, value }: TypeaheadInput,
{ absoluteRange }: any
): Promise<TypeaheadOutput> {
let context: string;
const suggestions = [];
let context = 'context-labels';
const suggestions: CompletionItemGroup[] = [];
const line = value.anchorBlock.getText();
const cursorOffset: number = value.selection.anchor.offset;
const cursorOffset = value.selection.anchor.offset;
const isValueStart = text.match(/^(=|=~|!=|!~)/);
// Use EMPTY_SELECTOR until series API is implemented for facetting
const selector = EMPTY_SELECTOR;
// Get normalized selector
let selector;
let parsedSelector;
try {
parsedSelector = parseSelector(line, cursorOffset);
} catch {}
selector = parsedSelector.selector;
} catch {
selector = EMPTY_SELECTOR;
}
if (!isValueStart && selector === EMPTY_SELECTOR) {
// start task gets all labels
await this.start();
const allLabels = this.getLabelKeys();
return { context, suggestions: [{ label: `Labels`, items: allLabels.map(wrapLabel) }] };
}
const existingKeys = parsedSelector ? parsedSelector.labelKeys : [];
if ((text && text.match(/^!?=~?/)) || wrapperClasses.includes('attr-value')) {
// Label values
if (labelKey && this.labelValues[selector]) {
let labelValues = this.labelValues[selector][labelKey];
let labelValues;
// Query labels for selector
if (selector) {
if (selector === EMPTY_SELECTOR && labelKey) {
const labelValuesForKey = await this.getLabelValues(labelKey);
labelValues = { [labelKey]: labelValuesForKey };
} else {
labelValues = await this.getSeriesLabels(selector, absoluteRange);
}
}
if (!labelValues) {
await this.fetchLabelValues(labelKey, absoluteRange);
labelValues = this.labelValues[selector][labelKey];
console.warn(`Server did not return any values for selector = ${selector}`);
return { context, suggestions };
}
if ((text && isValueStart) || wrapperClasses.includes('attr-value')) {
// Label values
if (labelKey && labelValues[labelKey]) {
context = 'context-label-values';
suggestions.push({
label: `Label values for "${labelKey}"`,
items: labelValues.map(wrapLabel),
items: labelValues[labelKey].map(wrapLabel),
});
}
} else {
// Label keys
const labelKeys = this.labelKeys[selector] || DEFAULT_KEYS;
const labelKeys = labelValues ? Object.keys(labelValues) : DEFAULT_KEYS;
if (labelKeys) {
const possibleKeys = _.difference(labelKeys, existingKeys);
if (possibleKeys.length) {
context = 'context-labels';
suggestions.push({ label: `Labels`, items: possibleKeys.map(wrapLabel) });
const newItems = possibleKeys.map(key => ({ label: key }));
const newSuggestion: CompletionItemGroup = { label: `Labels`, items: newItems };
suggestions.push(newSuggestion);
}
}
}
......@@ -302,7 +340,7 @@ export default class LokiLanguageProvider extends LanguageProvider {
// Keep only labels that exist on origin and target datasource
await this.start(); // fetches all existing label keys
const existingKeys = this.labelKeys[EMPTY_SELECTOR];
const existingKeys = this.labelKeys;
let labelsToKeep: { [key: string]: { value: any; operator: any } } = {};
if (existingKeys && existingKeys.length) {
// Check for common labels
......@@ -325,22 +363,31 @@ export default class LokiLanguageProvider extends LanguageProvider {
return ['{', cleanSelector, '}'].join('');
}
async getSeriesLabels(selector: string, absoluteRange: AbsoluteTimeRange) {
if (this.lookupsDisabled) {
return undefined;
}
try {
return await this.fetchSeriesLabels(selector, absoluteRange);
} catch (error) {
// TODO: better error handling
console.error(error);
return undefined;
}
}
/**
* Fetches all label keys
* @param absoluteRange Fetches
*/
async fetchLogLabels(absoluteRange: AbsoluteTimeRange): Promise<any> {
const url = '/api/prom/label';
try {
this.logLabelFetchTs = Date.now();
const rangeParams = absoluteRange ? rangeToParams(absoluteRange) : {};
const res = await this.request(url, rangeParams);
const labelKeys = res.data.data.slice().sort();
this.labelKeys = {
...this.labelKeys,
[EMPTY_SELECTOR]: labelKeys,
};
this.labelValues = {
[EMPTY_SELECTOR]: {},
};
this.logLabelOptions = labelKeys.map((key: string) => ({ label: key, value: key, isLeaf: false }));
this.labelKeys = res.slice().sort();
this.logLabelOptions = this.labelKeys.map((key: string) => ({ label: key, value: key, isLeaf: false }));
} catch (e) {
console.error(e);
}
......@@ -353,12 +400,64 @@ export default class LokiLanguageProvider extends LanguageProvider {
}
}
async fetchLabelValues(key: string, absoluteRange: AbsoluteTimeRange) {
/**
* Fetch labels for a selector. This is cached by it's args but also by the global timeRange currently selected as
* they can change over requested time.
* @param name
*/
fetchSeriesLabels = async (match: string, absoluteRange: AbsoluteTimeRange): Promise<Record<string, string[]>> => {
const rangeParams: { start?: number; end?: number } = absoluteRange ? rangeToParams(absoluteRange) : {};
const url = '/loki/api/v1/series';
const { start, end } = rangeParams;
const cacheKey = this.generateCacheKey(url, start, end, match);
const params = { match, start, end };
let value = this.seriesCache.get(cacheKey);
if (!value) {
// Clear value when requesting new one. Empty object being truthy also makes sure we don't request twice.
this.seriesCache.set(cacheKey, {});
const data = await this.request(url, params);
const { values } = processLabels(data);
value = values;
this.seriesCache.set(cacheKey, value);
}
return value;
};
// Cache key is a bit different here. We round up to a minute the intervals.
// The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every
// millisecond while still actually getting all the keys for the correct interval. This still can create problems
// when user does not the newest values for a minute if already cached.
generateCacheKey(url: string, start: number, end: number, param: string): string {
return [url, this.roundTime(start), this.roundTime(end), param].join();
}
// Round nanos epoch to nearest 5 minute interval
roundTime(nanos: number): number {
return nanos ? Math.floor(nanos / NS_IN_MS / 1000 / 60 / 5) : 0;
}
async getLabelValues(key: string): Promise<string[]> {
return await this.fetchLabelValues(key, this.initialRange);
}
async fetchLabelValues(key: string, absoluteRange: AbsoluteTimeRange): Promise<string[]> {
const url = `/api/prom/label/${key}/values`;
let values: string[] = [];
const rangeParams: { start?: number; end?: number } = absoluteRange ? rangeToParams(absoluteRange) : {};
const { start, end } = rangeParams;
const cacheKey = this.generateCacheKey(url, start, end, key);
const params = { start, end };
let value = this.labelsCache.get(cacheKey);
if (!value) {
try {
const rangeParams = absoluteRange ? rangeToParams(absoluteRange) : {};
const res = await this.request(url, rangeParams);
const values = res.data.data.slice().sort();
// Clear value when requesting new one. Empty object being truthy also makes sure we don't request twice.
this.labelsCache.set(cacheKey, []);
const res = await this.request(url, params);
values = res.slice().sort();
value = values;
this.labelsCache.set(cacheKey, value);
// Add to label options
this.logLabelOptions = this.logLabelOptions.map(keyOption => {
......@@ -370,19 +469,10 @@ export default class LokiLanguageProvider extends LanguageProvider {
}
return keyOption;
});
// Add to key map
const exisingValues = this.labelValues[EMPTY_SELECTOR];
const nextValues = {
...exisingValues,
[key]: values,
};
this.labelValues = {
...this.labelValues,
[EMPTY_SELECTOR]: nextValues,
};
} catch (e) {
console.error(e);
}
}
return value;
}
}
public/app/plugins/datasource/loki/mocks.ts
......@@ -3,34 +3,45 @@ import { DataSourceSettings } from '@grafana/data';
import { LokiOptions } from './types';
import { createDatasourceSettings } from '../../../features/datasources/mocks';
export function makeMockLokiDatasource(labelsAndValues: { [label: string]: string[] }): LokiDatasource {
interface Labels {
[label: string]: string[];
}
interface Series {
[label: string]: string;
}
interface SeriesForSelector {
[selector: string]: Series[];
}
export function makeMockLokiDatasource(labelsAndValues: Labels, series?: SeriesForSelector): LokiDatasource {
const legacyLokiLabelsAndValuesEndpointRegex = /^\/api\/prom\/label\/(\w*)\/values/;
const lokiLabelsAndValuesEndpointRegex = /^\/loki\/api\/v1\/label\/(\w*)\/values/;
const lokiSeriesEndpointRegex = /^\/loki\/api\/v1\/series/;
const legacyLokiLabelsEndpoint = `${LEGACY_LOKI_ENDPOINT}/label`;
const lokiLabelsEndpoint = `${LOKI_ENDPOINT}/label`;
const labels = Object.keys(labelsAndValues);
return {
metadataRequest: (url: string) => {
let responseData;
metadataRequest: (url: string, params?: { [key: string]: string }) => {
if (url === legacyLokiLabelsEndpoint || url === lokiLabelsEndpoint) {
responseData = labels;
return labels;
} else {
const match = url.match(legacyLokiLabelsAndValuesEndpointRegex) || url.match(lokiLabelsAndValuesEndpointRegex);
if (match) {
responseData = labelsAndValues[match[1]];
}
}
if (responseData) {
return {
data: {
data: responseData,
},
};
const legacyLabelsMatch = url.match(legacyLokiLabelsAndValuesEndpointRegex);
const labelsMatch = url.match(lokiLabelsAndValuesEndpointRegex);
const seriesMatch = url.match(lokiSeriesEndpointRegex);
if (legacyLabelsMatch) {
return labelsAndValues[legacyLabelsMatch[1]] || [];
} else if (labelsMatch) {
return labelsAndValues[labelsMatch[1]] || [];
} else if (seriesMatch) {
return series[params.match] || [];
} else {
throw new Error(`Unexpected url error, ${url}`);
}
}
},
} as any;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment