Commit bfba47c6 by Ryan McKinley Committed by Torkel Ödegaard

Refactor: Move LogLevel and Labels utils to @grafana/ui (#16285)

* rename Tags to Labels in SeriesData

* move some logs stuff to grafana/ui

* add roundtrip tests
parent d0d5b385
......@@ -7,4 +7,5 @@ export * from './theme';
export * from './graph';
export * from './threshold';
export * from './input';
export * from './logs';
export * from './displayValue';
/**
* Mapping of log level abbreviation to canonical log level.
* Supported levels are reduce to limit color variation.
*/
export enum LogLevel {
emerg = 'critical',
alert = 'critical',
crit = 'critical',
critical = 'critical',
warn = 'warning',
warning = 'warning',
err = 'error',
eror = 'error',
error = 'error',
info = 'info',
notice = 'info',
dbug = 'debug',
debug = 'debug',
trace = 'trace',
unknown = 'unknown',
}
......@@ -8,5 +8,7 @@ export * from './csv';
export * from './statsCalculator';
export * from './displayValue';
export * from './deprecationWarning';
export * from './logs';
export * from './labels';
export { getMappedValue } from './valueMappings';
export * from './validate';
import { parseLabels, formatLabels, findCommonLabels, findUniqueLabels } from './labels';
describe('parseLabels()', () => {
it('returns no labels on empty labels string', () => {
expect(parseLabels('')).toEqual({});
expect(parseLabels('{}')).toEqual({});
});
it('returns labels on labels string', () => {
expect(parseLabels('{foo="bar", baz="42"}')).toEqual({ foo: 'bar', baz: '42' });
});
});
describe('formatLabels()', () => {
it('returns no labels on empty label set', () => {
expect(formatLabels({})).toEqual('');
expect(formatLabels({}, 'foo')).toEqual('foo');
});
it('returns label string on label set', () => {
expect(formatLabels({ foo: 'bar', baz: '42' })).toEqual('{baz="42", foo="bar"}');
});
});
describe('findCommonLabels()', () => {
it('returns no common labels on empty sets', () => {
expect(findCommonLabels([{}])).toEqual({});
expect(findCommonLabels([{}, {}])).toEqual({});
});
it('returns no common labels on differing sets', () => {
expect(findCommonLabels([{ foo: 'bar' }, {}])).toEqual({});
expect(findCommonLabels([{}, { foo: 'bar' }])).toEqual({});
expect(findCommonLabels([{ baz: '42' }, { foo: 'bar' }])).toEqual({});
expect(findCommonLabels([{ foo: '42', baz: 'bar' }, { foo: 'bar' }])).toEqual({});
});
it('returns the single labels set as common labels', () => {
expect(findCommonLabels([{ foo: 'bar' }])).toEqual({ foo: 'bar' });
});
});
describe('findUniqueLabels()', () => {
it('returns no uncommon labels on empty sets', () => {
expect(findUniqueLabels({}, {})).toEqual({});
});
it('returns all labels given no common labels', () => {
expect(findUniqueLabels({ foo: '"bar"' }, {})).toEqual({ foo: '"bar"' });
});
it('returns all labels except the common labels', () => {
expect(findUniqueLabels({ foo: '"bar"', baz: '"42"' }, { foo: '"bar"' })).toEqual({ baz: '"42"' });
});
});
import { Labels } from '../types/data';
/**
* Regexp to extract Prometheus-style labels
*/
const labelRegexp = /\b(\w+)(!?=~?)"([^"\n]*?)"/g;
/**
* Returns a map of label keys to value from an input selector string.
*
* Example: `parseLabels('{job="foo", instance="bar"}) // {job: "foo", instance: "bar"}`
*/
export function parseLabels(labels: string): Labels {
const labelsByKey: Labels = {};
labels.replace(labelRegexp, (_, key, operator, value) => {
labelsByKey[key] = value;
return '';
});
return labelsByKey;
}
/**
* Returns a map labels that are common to the given label sets.
*/
export function findCommonLabels(labelsSets: Labels[]): Labels {
return labelsSets.reduce(
(acc, labels) => {
if (!labels) {
throw new Error('Need parsed labels to find common labels.');
}
if (!acc) {
// Initial set
acc = { ...labels };
} else {
// Remove incoming labels that are missing or not matching in value
Object.keys(labels).forEach(key => {
if (acc[key] === undefined || acc[key] !== labels[key]) {
delete acc[key];
}
});
// Remove common labels that are missing from incoming label set
Object.keys(acc).forEach(key => {
if (labels[key] === undefined) {
delete acc[key];
}
});
}
return acc;
},
(undefined as unknown) as Labels
);
}
/**
* Returns a map of labels that are in `labels`, but not in `commonLabels`.
*/
export function findUniqueLabels(labels: Labels, commonLabels: Labels): Labels {
const uncommonLabels: Labels = { ...labels };
Object.keys(commonLabels).forEach(key => {
delete uncommonLabels[key];
});
return uncommonLabels;
}
/**
* Serializes the given labels to a string.
*/
export function formatLabels(labels: Labels, defaultValue = ''): string {
if (!labels || Object.keys(labels).length === 0) {
return defaultValue;
}
const labelKeys = Object.keys(labels).sort();
const cleanSelector = labelKeys.map(key => `${key}="${labels[key]}"`).join(', ');
return ['{', cleanSelector, '}'].join('');
}
import { LogLevel } from '../types/logs';
import { getLogLevel } from './logs';
describe('getLoglevel()', () => {
it('returns no log level on empty line', () => {
expect(getLogLevel('')).toBe(LogLevel.unknown);
});
it('returns no log level on when level is part of a word', () => {
expect(getLogLevel('this is information')).toBe(LogLevel.unknown);
});
it('returns same log level for long and short version', () => {
expect(getLogLevel('[Warn]')).toBe(LogLevel.warning);
expect(getLogLevel('[Warning]')).toBe(LogLevel.warning);
expect(getLogLevel('[Warn]')).toBe('warning');
});
it('returns log level on line contains a log level', () => {
expect(getLogLevel('warn: it is looking bad')).toBe(LogLevel.warn);
expect(getLogLevel('2007-12-12 12:12:12 [WARN]: it is looking bad')).toBe(LogLevel.warn);
});
it('returns first log level found', () => {
expect(getLogLevel('WARN this could be a debug message')).toBe(LogLevel.warn);
});
});
import { LogLevel } from '../types/logs';
import { SeriesData, FieldType } from '../types/data';
/**
* Returns the log level of a log line.
* Parse the line for level words. If no level is found, it returns `LogLevel.unknown`.
*
* Example: `getLogLevel('WARN 1999-12-31 this is great') // LogLevel.warn`
*/
export function getLogLevel(line: string): LogLevel {
if (!line) {
return LogLevel.unknown;
}
for (const key of Object.keys(LogLevel)) {
const regexp = new RegExp(`\\b${key}\\b`, 'i');
if (regexp.test(line)) {
const level = (LogLevel as any)[key];
if (level) {
return level;
}
}
}
return LogLevel.unknown;
}
export function addLogLevelToSeries(series: SeriesData, lineIndex: number): SeriesData {
return {
...series, // Keeps Tags, RefID etc
fields: [...series.fields, { name: 'LogLevel', type: FieldType.string }],
rows: series.rows.map(row => {
const line = row[lineIndex];
return [...row, getLogLevel(line)];
}),
};
}
......@@ -89,7 +89,7 @@ export function guessFieldTypeFromValue(v: any): FieldType {
/**
* Looks at the data to guess the column type. This ignores any existing setting
*/
function guessFieldTypeFromTable(series: SeriesData, index: number): FieldType | undefined {
export function guessFieldTypeFromSeries(series: SeriesData, index: number): FieldType | undefined {
const column = series.fields[index];
// 1. Use the column name to guess
......@@ -129,7 +129,7 @@ export const guessFieldTypes = (series: SeriesData): SeriesData => {
// Replace it with a calculated version
return {
...field,
type: guessFieldTypeFromTable(series, index),
type: guessFieldTypeFromSeries(series, index),
};
}),
};
......@@ -162,7 +162,7 @@ export const toLegacyResponseData = (series: SeriesData): TimeSeries | TableData
const { fields, rows } = series;
if (fields.length === 2) {
const type = guessFieldTypeFromTable(series, 1);
const type = guessFieldTypeFromSeries(series, 1);
if (type === FieldType.time) {
return {
target: fields[0].name || series.name,
......
import _ from 'lodash';
import { colors, TimeSeries } from '@grafana/ui';
import { colors, TimeSeries, Labels, LogLevel } from '@grafana/ui';
import { getThemeColor } from 'app/core/utils/colors';
/**
* Mapping of log level abbreviation to canonical log level.
* Supported levels are reduce to limit color variation.
*/
export enum LogLevel {
emerg = 'critical',
alert = 'critical',
crit = 'critical',
critical = 'critical',
warn = 'warning',
warning = 'warning',
err = 'error',
eror = 'error',
error = 'error',
info = 'info',
notice = 'info',
dbug = 'debug',
debug = 'debug',
trace = 'trace',
unknown = 'unknown',
}
export const LogLevelColor = {
[LogLevel.critical]: colors[7],
[LogLevel.warning]: colors[1],
......@@ -46,7 +24,7 @@ export interface LogRowModel {
entry: string;
hasAnsi: boolean;
key: string; // timestamp + labels
labels: LogsStreamLabels;
labels: Labels;
logLevel: LogLevel;
raw: string;
searchWords?: string[];
......@@ -54,7 +32,7 @@ export interface LogRowModel {
timeFromNow: string;
timeEpochMs: number;
timeLocal: string;
uniqueLabels?: LogsStreamLabels;
uniqueLabels?: Labels;
}
export interface LogLabelStatsModel {
......@@ -72,7 +50,7 @@ export enum LogsMetaKind {
export interface LogsMetaItem {
label: string;
value: string | number | LogsStreamLabels;
value: string | number | Labels;
kind: LogsMetaKind;
}
......@@ -88,8 +66,8 @@ export interface LogsStream {
labels: string;
entries: LogsStreamEntry[];
search?: string;
parsedLabels?: LogsStreamLabels;
uniqueLabels?: LogsStreamLabels;
parsedLabels?: Labels;
uniqueLabels?: Labels;
}
export interface LogsStreamEntry {
......@@ -99,10 +77,6 @@ export interface LogsStreamEntry {
timestamp?: string;
}
export interface LogsStreamLabels {
[key: string]: string;
}
export enum LogsDedupDescription {
none = 'No de-duplication',
exact = 'De-duplication of successive lines that are identical, ignoring ISO datetimes.',
......
import React, { PureComponent } from 'react';
import { LogsStreamLabels, LogRowModel } from 'app/core/logs_model';
import { LogRowModel } from 'app/core/logs_model';
import { LogLabel } from './LogLabel';
import { Labels } from '@grafana/ui';
interface Props {
getRows?: () => LogRowModel[];
labels: LogsStreamLabels;
labels: Labels;
plain?: boolean;
onClickLabel?: (label: string, value: string) => void;
}
......
......@@ -2,10 +2,10 @@ import _ from 'lodash';
import React, { PureComponent } from 'react';
import * as rangeUtil from 'app/core/utils/rangeutil';
import { RawTimeRange, Switch } from '@grafana/ui';
import { RawTimeRange, Switch, LogLevel } from '@grafana/ui';
import TimeSeries from 'app/core/time_series2';
import { LogsDedupDescription, LogsDedupStrategy, LogsModel, LogLevel, LogsMetaKind } from 'app/core/logs_model';
import { LogsDedupDescription, LogsDedupStrategy, LogsModel, LogsMetaKind } from 'app/core/logs_model';
import ToggleButtonGroup, { ToggleButton } from 'app/core/components/ToggleButtonGroup/ToggleButtonGroup';
......
import React, { PureComponent } from 'react';
import { hot } from 'react-hot-loader';
import { connect } from 'react-redux';
import { RawTimeRange, TimeRange } from '@grafana/ui';
import { RawTimeRange, TimeRange, LogLevel } from '@grafana/ui';
import { ExploreId, ExploreItemState } from 'app/types/explore';
import { LogsModel, LogsDedupStrategy, LogLevel } from 'app/core/logs_model';
import { LogsModel, LogsDedupStrategy } from 'app/core/logs_model';
import { StoreState } from 'app/types';
import { toggleLogs, changeDedupStrategy } from './state/actions';
......
......@@ -7,6 +7,7 @@ import {
DataSourceSelectItem,
DataSourceApi,
QueryFixAction,
LogLevel,
} from '@grafana/ui/src/types';
import {
ExploreId,
......@@ -18,7 +19,6 @@ import {
ExploreUIState,
} from 'app/types/explore';
import { actionCreatorFactory, noPayloadActionCreatorFactory, ActionOf } from 'app/core/redux/actionCreatorFactory';
import { LogLevel } from 'app/core/logs_model';
/** Higher order actions
*
......
import { LogLevel, LogsStream } from 'app/core/logs_model';
import { LogsStream } from 'app/core/logs_model';
import {
findCommonLabels,
findUniqueLabels,
formatLabels,
getLogLevel,
mergeStreamsToLogs,
parseLabels,
} from './result_transformer';
describe('getLoglevel()', () => {
it('returns no log level on empty line', () => {
expect(getLogLevel('')).toBe(LogLevel.unknown);
});
it('returns no log level on when level is part of a word', () => {
expect(getLogLevel('this is information')).toBe(LogLevel.unknown);
});
it('returns same log level for long and short version', () => {
expect(getLogLevel('[Warn]')).toBe(LogLevel.warning);
expect(getLogLevel('[Warning]')).toBe(LogLevel.warning);
expect(getLogLevel('[Warn]')).toBe('warning');
});
it('returns log level on line contains a log level', () => {
expect(getLogLevel('warn: it is looking bad')).toBe(LogLevel.warn);
expect(getLogLevel('2007-12-12 12:12:12 [WARN]: it is looking bad')).toBe(LogLevel.warn);
});
it('returns first log level found', () => {
expect(getLogLevel('WARN this could be a debug message')).toBe(LogLevel.warn);
});
});
describe('parseLabels()', () => {
it('returns no labels on empty labels string', () => {
expect(parseLabels('')).toEqual({});
expect(parseLabels('{}')).toEqual({});
});
it('returns labels on labels string', () => {
expect(parseLabels('{foo="bar", baz="42"}')).toEqual({ foo: 'bar', baz: '42' });
});
});
describe('formatLabels()', () => {
it('returns no labels on empty label set', () => {
expect(formatLabels({})).toEqual('');
expect(formatLabels({}, 'foo')).toEqual('foo');
});
it('returns label string on label set', () => {
expect(formatLabels({ foo: 'bar', baz: '42' })).toEqual('{baz="42", foo="bar"}');
});
});
describe('findCommonLabels()', () => {
it('returns no common labels on empty sets', () => {
expect(findCommonLabels([{}])).toEqual({});
expect(findCommonLabels([{}, {}])).toEqual({});
});
it('returns no common labels on differing sets', () => {
expect(findCommonLabels([{ foo: 'bar' }, {}])).toEqual({});
expect(findCommonLabels([{}, { foo: 'bar' }])).toEqual({});
expect(findCommonLabels([{ baz: '42' }, { foo: 'bar' }])).toEqual({});
expect(findCommonLabels([{ foo: '42', baz: 'bar' }, { foo: 'bar' }])).toEqual({});
});
it('returns the single labels set as common labels', () => {
expect(findCommonLabels([{ foo: 'bar' }])).toEqual({ foo: 'bar' });
});
});
describe('findUniqueLabels()', () => {
it('returns no uncommon labels on empty sets', () => {
expect(findUniqueLabels({}, {})).toEqual({});
});
it('returns all labels given no common labels', () => {
expect(findUniqueLabels({ foo: '"bar"' }, {})).toEqual({ foo: '"bar"' });
});
it('returns all labels except the common labels', () => {
expect(findUniqueLabels({ foo: '"bar"', baz: '"42"' }, { foo: '"bar"' })).toEqual({ baz: '"42"' });
});
});
import { mergeStreamsToLogs, logStreamToSeriesData, seriesDataToLogStream } from './result_transformer';
describe('mergeStreamsToLogs()', () => {
it('returns empty logs given no streams', () => {
......@@ -201,3 +115,37 @@ describe('mergeStreamsToLogs()', () => {
]);
});
});
describe('convert SeriesData to/from LogStream', () => {
const streams = [
{
labels: '{foo="bar"}',
entries: [
{
line: "foo: 'bar'",
ts: '1970-01-01T00:00:00Z',
},
],
},
{
labels: '{bar="foo"}',
entries: [
{
line: "bar: 'foo'",
ts: '1970-01-01T00:00:00Z',
},
],
},
];
it('converts streams to series', () => {
const data = streams.map(stream => logStreamToSeriesData(stream));
expect(data.length).toBe(2);
expect(data[0].labels['foo']).toEqual('bar');
expect(data[0].rows[0][0]).toEqual(streams[0].entries[0].ts);
const roundtrip = data.map(series => seriesDataToLogStream(series));
expect(roundtrip.length).toBe(2);
expect(roundtrip[0].labels).toEqual(streams[0].labels);
});
});
......@@ -2,120 +2,27 @@ import ansicolor from 'vendor/ansicolor/ansicolor';
import _ from 'lodash';
import moment from 'moment';
import {
LogLevel,
LogsMetaItem,
LogsModel,
LogRowModel,
LogsStream,
LogsStreamEntry,
LogsStreamLabels,
LogsMetaKind,
} from 'app/core/logs_model';
import { LogsMetaItem, LogsModel, LogRowModel, LogsStream, LogsStreamEntry, LogsMetaKind } from 'app/core/logs_model';
import { hasAnsiCodes } from 'app/core/utils/text';
import { DEFAULT_MAX_LINES } from './datasource';
/**
* Returns the log level of a log line.
* Parse the line for level words. If no level is found, it returns `LogLevel.unknown`.
*
* Example: `getLogLevel('WARN 1999-12-31 this is great') // LogLevel.warn`
*/
export function getLogLevel(line: string): LogLevel {
if (!line) {
return LogLevel.unknown;
}
let level: LogLevel;
Object.keys(LogLevel).forEach(key => {
if (!level) {
const regexp = new RegExp(`\\b${key}\\b`, 'i');
if (regexp.test(line)) {
level = LogLevel[key];
}
}
});
if (!level) {
level = LogLevel.unknown;
}
return level;
}
/**
* Regexp to extract Prometheus-style labels
*/
const labelRegexp = /\b(\w+)(!?=~?)"([^"\n]*?)"/g;
/**
* Returns a map of label keys to value from an input selector string.
*
* Example: `parseLabels('{job="foo", instance="bar"}) // {job: "foo", instance: "bar"}`
*/
export function parseLabels(labels: string): LogsStreamLabels {
const labelsByKey: LogsStreamLabels = {};
labels.replace(labelRegexp, (_, key, operator, value) => {
labelsByKey[key] = value;
return '';
});
return labelsByKey;
}
/**
* Returns a map labels that are common to the given label sets.
*/
export function findCommonLabels(labelsSets: LogsStreamLabels[]): LogsStreamLabels {
return labelsSets.reduce((acc, labels) => {
if (!labels) {
throw new Error('Need parsed labels to find common labels.');
}
if (!acc) {
// Initial set
acc = { ...labels };
} else {
// Remove incoming labels that are missing or not matching in value
Object.keys(labels).forEach(key => {
if (acc[key] === undefined || acc[key] !== labels[key]) {
delete acc[key];
}
});
// Remove common labels that are missing from incoming label set
Object.keys(acc).forEach(key => {
if (labels[key] === undefined) {
delete acc[key];
}
});
}
return acc;
}, undefined);
}
/**
* Returns a map of labels that are in `labels`, but not in `commonLabels`.
*/
export function findUniqueLabels(labels: LogsStreamLabels, commonLabels: LogsStreamLabels): LogsStreamLabels {
const uncommonLabels: LogsStreamLabels = { ...labels };
Object.keys(commonLabels).forEach(key => {
delete uncommonLabels[key];
});
return uncommonLabels;
}
/**
* Serializes the given labels to a string.
*/
export function formatLabels(labels: LogsStreamLabels, defaultValue = ''): string {
if (!labels || Object.keys(labels).length === 0) {
return defaultValue;
}
const labelKeys = Object.keys(labels).sort();
const cleanSelector = labelKeys.map(key => `${key}="${labels[key]}"`).join(', ');
return ['{', cleanSelector, '}'].join('');
}
import {
parseLabels,
SeriesData,
findUniqueLabels,
Labels,
findCommonLabels,
getLogLevel,
FieldType,
formatLabels,
guessFieldTypeFromSeries,
} from '@grafana/ui';
export function processEntry(
entry: LogsStreamEntry,
labels: string,
parsedLabels: LogsStreamLabels,
uniqueLabels: LogsStreamLabels,
parsedLabels: Labels,
uniqueLabels: Labels,
search: string
): LogRowModel {
const { line } = entry;
......@@ -201,3 +108,48 @@ export function mergeStreamsToLogs(streams: LogsStream[], limit = DEFAULT_MAX_LI
rows: sortedRows,
};
}
export function logStreamToSeriesData(stream: LogsStream): SeriesData {
let labels: Labels = stream.parsedLabels;
if (!labels && stream.labels) {
labels = parseLabels(stream.labels);
}
return {
labels,
fields: [{ name: 'ts', type: FieldType.time }, { name: 'line', type: FieldType.string }],
rows: stream.entries.map(entry => {
return [entry.ts || entry.timestamp, entry.line];
}),
};
}
export function seriesDataToLogStream(series: SeriesData): LogsStream {
let timeIndex = -1;
let lineIndex = -1;
for (let i = 0; i < series.fields.length; i++) {
const field = series.fields[i];
const type = field.type || guessFieldTypeFromSeries(series, i);
if (timeIndex < 0 && type === FieldType.time) {
timeIndex = i;
}
if (lineIndex < 0 && type === FieldType.string) {
lineIndex = i;
}
}
if (timeIndex < 0) {
throw new Error('Series does not have a time field');
}
if (lineIndex < 0) {
throw new Error('Series does not have a line field');
}
return {
labels: formatLabels(series.labels),
parsedLabels: series.labels,
entries: series.rows.map(row => {
return {
line: row[lineIndex],
ts: row[timeIndex],
};
}),
};
}
......@@ -9,10 +9,11 @@ import {
DataSourceApi,
QueryHint,
ExploreStartPageProps,
LogLevel,
} from '@grafana/ui';
import { Emitter, TimeSeries } from 'app/core/core';
import { LogsModel, LogsDedupStrategy, LogLevel } from 'app/core/logs_model';
import { LogsModel, LogsDedupStrategy } from 'app/core/logs_model';
import TableModel from 'app/core/table_model';
export interface CompletionItem {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment