Commit 619b4b48 by Torkel Ödegaard Committed by GitHub

Merge pull request #14336 from grafana/davkal/explore-line-parsing

Explore: Logging line parsing and field stats
parents a89e21c5 7cb456ea
......@@ -95,6 +95,57 @@ export enum LogsDedupStrategy {
signature = 'signature',
}
export interface LogsParser {
/**
* Value-agnostic matcher for a field label.
* Used to filter rows, and first capture group contains the value.
*/
buildMatcher: (label: string) => RegExp;
/**
* Regex to find a field in the log line.
* First capture group contains the label value, second capture group the value.
*/
fieldRegex: RegExp;
/**
* Function to verify if this is a valid parser for the given line.
* The parser accepts the line unless it returns undefined.
*/
test: (line: string) => any;
}
export const LogsParsers: { [name: string]: LogsParser } = {
JSON: {
buildMatcher: label => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"([^"]*)"`),
fieldRegex: /"(\w+)"\s*:\s*"([^"]*)"/,
test: line => {
try {
return JSON.parse(line);
} catch (error) {}
},
},
logfmt: {
buildMatcher: label => new RegExp(`(?:^|\\s)${label}=("[^"]*"|\\S+)`),
fieldRegex: /(?:^|\s)(\w+)=("[^"]*"|\S+)/,
test: line => LogsParsers.logfmt.fieldRegex.test(line),
},
};
export function calculateFieldStats(rows: LogRow[], extractor: RegExp): LogsLabelStat[] {
// Consider only rows that satisfy the matcher
const rowsWithField = rows.filter(row => extractor.test(row.entry));
const rowCount = rowsWithField.length;
// Get field value counts for eligible rows
const countsByValue = _.countBy(rowsWithField, row => (row as LogRow).entry.match(extractor)[1]);
const sortedCounts = _.chain(countsByValue)
.map((count, value) => ({ count, value, proportion: count / rowCount }))
.sortBy('count')
.reverse()
.value();
return sortedCounts;
}
export function calculateLogsLabelStats(rows: LogRow[], label: string): LogsLabelStat[] {
// Consider only rows that have the given label
const rowsWithLabel = rows.filter(row => row.labels[label] !== undefined);
......@@ -151,6 +202,19 @@ export function dedupLogRows(logs: LogsModel, strategy: LogsDedupStrategy): Logs
};
}
export function getParser(line: string): LogsParser {
let parser;
try {
if (LogsParsers.JSON.test(line)) {
parser = LogsParsers.JSON;
}
} catch (error) {}
if (!parser && LogsParsers.logfmt.test(line)) {
parser = LogsParsers.logfmt;
}
return parser;
}
export function filterLogLevels(logs: LogsModel, hiddenLogLevels: Set<LogLevel>): LogsModel {
if (hiddenLogLevels.size === 0) {
return logs;
......
import { calculateLogsLabelStats, dedupLogRows, LogsDedupStrategy, LogsModel } from '../logs_model';
import {
calculateFieldStats,
calculateLogsLabelStats,
dedupLogRows,
getParser,
LogsDedupStrategy,
LogsModel,
LogsParsers,
} from '../logs_model';
describe('dedupLogRows()', () => {
test('should return rows as is when dedup is set to none', () => {
......@@ -107,6 +115,50 @@ describe('dedupLogRows()', () => {
});
});
describe('calculateFieldStats()', () => {
test('should return no stats for empty rows', () => {
expect(calculateFieldStats([], /foo=(.*)/)).toEqual([]);
});
test('should return no stats if extractor does not match', () => {
const rows = [
{
entry: 'foo=bar',
},
];
expect(calculateFieldStats(rows as any, /baz=(.*)/)).toEqual([]);
});
test('should return stats for found field', () => {
const rows = [
{
entry: 'foo="42 + 1"',
},
{
entry: 'foo=503 baz=foo',
},
{
entry: 'foo="42 + 1"',
},
{
entry: 't=2018-12-05T07:44:59+0000 foo=503',
},
];
expect(calculateFieldStats(rows as any, /foo=("[^"]*"|\S+)/)).toMatchObject([
{
value: '"42 + 1"',
count: 2,
},
{
value: '503',
count: 2,
},
]);
});
});
describe('calculateLogsLabelStats()', () => {
test('should return no stats for empty rows', () => {
expect(calculateLogsLabelStats([], '')).toEqual([]);
......@@ -159,3 +211,70 @@ describe('calculateLogsLabelStats()', () => {
]);
});
});
describe('getParser()', () => {
test('should return no parser on empty line', () => {
expect(getParser('')).toBeUndefined();
});
test('should return no parser on unknown line pattern', () => {
expect(getParser('To Be or not to be')).toBeUndefined();
});
test('should return logfmt parser on key value patterns', () => {
expect(getParser('foo=bar baz="41 + 1')).toEqual(LogsParsers.logfmt);
});
test('should return JSON parser on JSON log lines', () => {
// TODO implement other JSON value types than string
expect(getParser('{"foo": "bar", "baz": "41 + 1"}')).toEqual(LogsParsers.JSON);
});
});
describe('LogsParsers', () => {
describe('logfmt', () => {
const parser = LogsParsers.logfmt;
test('should detect format', () => {
expect(parser.test('foo')).toBeFalsy();
expect(parser.test('foo=bar')).toBeTruthy();
});
test('should have a valid fieldRegex', () => {
const match = 'foo=bar'.match(parser.fieldRegex);
expect(match).toBeDefined();
expect(match[1]).toBe('foo');
expect(match[2]).toBe('bar');
});
test('should build a valid value matcher', () => {
const matcher = parser.buildMatcher('foo');
const match = 'foo=bar'.match(matcher);
expect(match).toBeDefined();
expect(match[1]).toBe('bar');
});
});
describe('JSON', () => {
const parser = LogsParsers.JSON;
test('should detect format', () => {
expect(parser.test('foo')).toBeFalsy();
expect(parser.test('{"foo":"bar"}')).toBeTruthy();
});
test('should have a valid fieldRegex', () => {
const match = '{"foo":"bar"}'.match(parser.fieldRegex);
expect(match).toBeDefined();
expect(match[1]).toBe('foo');
expect(match[2]).toBe('bar');
});
test('should build a valid value matcher', () => {
const matcher = parser.buildMatcher('foo');
const match = '{"foo":"bar"}'.match(matcher);
expect(match).toBeDefined();
expect(match[1]).toBe('bar');
});
});
});
......@@ -24,7 +24,7 @@ function StatsRow({ active, count, proportion, value }: LogsLabelStat) {
}
const STATS_ROW_LIMIT = 5;
class Stats extends PureComponent<{
export class Stats extends PureComponent<{
stats: LogsLabelStat[];
label: string;
value: string;
......@@ -48,15 +48,21 @@ class Stats extends PureComponent<{
const otherProportion = otherCount / total;
return (
<>
<div className="logs-stats__info">
{label}: {total} of {rowCount} rows have that label
<span className="logs-stats__icon fa fa-window-close" onClick={onClickClose} />
<div className="logs-stats">
<div className="logs-stats__header">
<span className="logs-stats__title">
{label}: {total} of {rowCount} rows have that label
</span>
<span className="logs-stats__close fa fa-remove" onClick={onClickClose} />
</div>
<div className="logs-stats__body">
{topRows.map(stat => <StatsRow key={stat.value} {...stat} active={stat.value === value} />)}
{insertActiveRow && activeRow && <StatsRow key={activeRow.value} {...activeRow} active />}
{otherCount > 0 && (
<StatsRow key="__OTHERS__" count={otherCount} value="Other" proportion={otherProportion} />
)}
</div>
{topRows.map(stat => <StatsRow key={stat.value} {...stat} active={stat.value === value} />)}
{insertActiveRow && <StatsRow key={activeRow.value} {...activeRow} active />}
{otherCount > 0 && <StatsRow key="__OTHERS__" count={otherCount} value="Other" proportion={otherProportion} />}
</>
</div>
);
}
}
......
......@@ -10,16 +10,20 @@ import {
LogsModel,
dedupLogRows,
filterLogLevels,
getParser,
LogLevel,
LogsMetaKind,
LogsLabelStat,
LogsParser,
LogRow,
calculateFieldStats,
} from 'app/core/logs_model';
import { findHighlightChunksInText } from 'app/core/utils/text';
import { Switch } from 'app/core/components/Switch/Switch';
import ToggleButtonGroup, { ToggleButton } from 'app/core/components/ToggleButtonGroup/ToggleButtonGroup';
import Graph from './Graph';
import LogLabels from './LogLabels';
import LogLabels, { Stats } from './LogLabels';
const PREVIEW_LIMIT = 100;
......@@ -38,6 +42,19 @@ const graphOptions = {
},
};
/**
* Renders a highlighted field.
* When hovering, a stats icon is shown.
*/
const FieldHighlight = onClick => props => {
return (
<span className={props.className} style={props.style}>
{props.children}
<span className="logs-row__field-highlight--icon fa fa-signal" onClick={() => onClick(props.children)} />
</span>
);
};
interface RowProps {
allRows: LogRow[];
highlighterExpressions?: string[];
......@@ -49,57 +66,171 @@ interface RowProps {
onClickLabel?: (label: string, value: string) => void;
}
function Row({
allRows,
highlighterExpressions,
onClickLabel,
row,
showDuplicates,
showLabels,
showLocalTime,
showUtc,
}: RowProps) {
const previewHighlights = highlighterExpressions && !_.isEqual(highlighterExpressions, row.searchWords);
const highlights = previewHighlights ? highlighterExpressions : row.searchWords;
const needsHighlighter = highlights && highlights.length > 0;
const highlightClassName = classnames('logs-row__match-highlight', {
'logs-row__match-highlight--preview': previewHighlights,
});
return (
<div className="logs-row">
{showDuplicates && (
<div className="logs-row__duplicates">{row.duplicates > 0 ? `${row.duplicates + 1}x` : null}</div>
)}
<div className={row.logLevel ? `logs-row__level logs-row__level--${row.logLevel}` : ''} />
{showUtc && (
<div className="logs-row__time" title={`Local: ${row.timeLocal} (${row.timeFromNow})`}>
{row.timestamp}
</div>
)}
{showLocalTime && (
<div className="logs-row__time" title={`${row.timestamp} (${row.timeFromNow})`}>
{row.timeLocal}
</div>
)}
{showLabels && (
<div className="logs-row__labels">
<LogLabels allRows={allRows} labels={row.uniqueLabels} onClickLabel={onClickLabel} />
</div>
)}
<div className="logs-row__message">
{needsHighlighter ? (
<Highlighter
textToHighlight={row.entry}
searchWords={highlights}
findChunks={findHighlightChunksInText}
highlightClassName={highlightClassName}
/>
) : (
row.entry
interface RowState {
fieldCount: number;
fieldLabel: string;
fieldStats: LogsLabelStat[];
fieldValue: string;
parsed: boolean;
parser: LogsParser;
parsedFieldHighlights: string[];
showFieldStats: boolean;
}
/**
* Renders a log line.
*
* When user hovers over it for a certain time, it lazily parses the log line.
* Once a parser is found, it will determine fields, that will be highlighted.
* When the user requests stats for a field, they will be calculated and rendered below the row.
*/
class Row extends PureComponent<RowProps, RowState> {
mouseMessageTimer: NodeJS.Timer;
state = {
fieldCount: 0,
fieldLabel: null,
fieldStats: null,
fieldValue: null,
parsed: false,
parser: null,
parsedFieldHighlights: [],
showFieldStats: false,
};
componentWillUnmount() {
clearTimeout(this.mouseMessageTimer);
}
onClickClose = () => {
this.setState({ showFieldStats: false });
};
onClickHighlight = (fieldText: string) => {
const { allRows } = this.props;
const { parser } = this.state;
const fieldMatch = fieldText.match(parser.fieldRegex);
if (fieldMatch) {
// Build value-agnostic row matcher based on the field label
const fieldLabel = fieldMatch[1];
const fieldValue = fieldMatch[2];
const matcher = parser.buildMatcher(fieldLabel);
const fieldStats = calculateFieldStats(allRows, matcher);
const fieldCount = fieldStats.reduce((sum, stat) => sum + stat.count, 0);
this.setState({ fieldCount, fieldLabel, fieldStats, fieldValue, showFieldStats: true });
}
};
onMouseOverMessage = () => {
// Don't parse right away, user might move along
this.mouseMessageTimer = setTimeout(this.parseMessage, 500);
};
onMouseOutMessage = () => {
clearTimeout(this.mouseMessageTimer);
this.setState({ parsed: false });
};
parseMessage = () => {
if (!this.state.parsed) {
const { row } = this.props;
const parser = getParser(row.entry);
if (parser) {
// Use parser to highlight detected fields
const parsedFieldHighlights = [];
this.props.row.entry.replace(new RegExp(parser.fieldRegex, 'g'), substring => {
parsedFieldHighlights.push(substring.trim());
return '';
});
this.setState({ parsedFieldHighlights, parsed: true, parser });
}
}
};
render() {
const {
allRows,
highlighterExpressions,
onClickLabel,
row,
showDuplicates,
showLabels,
showLocalTime,
showUtc,
} = this.props;
const {
fieldCount,
fieldLabel,
fieldStats,
fieldValue,
parsed,
parsedFieldHighlights,
showFieldStats,
} = this.state;
const previewHighlights = highlighterExpressions && !_.isEqual(highlighterExpressions, row.searchWords);
const highlights = previewHighlights ? highlighterExpressions : row.searchWords;
const needsHighlighter = highlights && highlights.length > 0;
const highlightClassName = classnames('logs-row__match-highlight', {
'logs-row__match-highlight--preview': previewHighlights,
});
return (
<div className="logs-row">
{showDuplicates && (
<div className="logs-row__duplicates">{row.duplicates > 0 ? `${row.duplicates + 1}x` : null}</div>
)}
<div className={row.logLevel ? `logs-row__level logs-row__level--${row.logLevel}` : ''} />
{showUtc && (
<div className="logs-row__time" title={`Local: ${row.timeLocal} (${row.timeFromNow})`}>
{row.timestamp}
</div>
)}
{showLocalTime && (
<div className="logs-row__time" title={`${row.timestamp} (${row.timeFromNow})`}>
{row.timeLocal}
</div>
)}
{showLabels && (
<div className="logs-row__labels">
<LogLabels allRows={allRows} labels={row.uniqueLabels} onClickLabel={onClickLabel} />
</div>
)}
<div className="logs-row__message" onMouseEnter={this.onMouseOverMessage} onMouseLeave={this.onMouseOutMessage}>
{parsed && (
<Highlighter
autoEscape
highlightTag={FieldHighlight(this.onClickHighlight)}
textToHighlight={row.entry}
searchWords={parsedFieldHighlights}
highlightClassName="logs-row__field-highlight"
/>
)}
{!parsed &&
needsHighlighter && (
<Highlighter
textToHighlight={row.entry}
searchWords={highlights}
findChunks={findHighlightChunksInText}
highlightClassName={highlightClassName}
/>
)}
{!parsed && !needsHighlighter && row.entry}
{showFieldStats && (
<div className="logs-row__stats">
<Stats
stats={fieldStats}
label={fieldLabel}
value={fieldValue}
onClickClose={this.onClickClose}
rowCount={fieldCount}
/>
</div>
)}
</div>
</div>
</div>
);
);
}
}
function renderMetaItem(value: any, kind: LogsMetaKind) {
......
......@@ -158,6 +158,32 @@ $column-horizontal-spacing: 10px;
text-align: right;
}
.logs-row__field-highlight {
// Undoing mark styling
background: inherit;
padding: inherit;
border-bottom: 1px dotted $typeahead-selected-color;
.logs-row__field-highlight--icon {
margin-left: 0.5em;
cursor: pointer;
display: none;
}
}
.logs-row__stats {
margin: 5px 0;
}
.logs-row__field-highlight:hover {
color: $typeahead-selected-color;
border-bottom-style: solid;
.logs-row__field-highlight--icon {
display: inline;
}
}
.logs-label {
display: inline-block;
padding: 0 2px;
......@@ -181,21 +207,42 @@ $column-horizontal-spacing: 10px;
top: 1.25em;
left: -10px;
z-index: 100;
justify-content: space-between;
box-shadow: $popover-shadow;
}
/*
* Stats popover & message stats box
*/
.logs-stats {
background-color: $popover-bg;
color: $popover-color;
border: 1px solid $popover-border-color;
padding: 10px;
border-radius: $border-radius;
justify-content: space-between;
box-shadow: $popover-shadow;
max-width: 500px;
}
.logs-stats__header {
background-color: $popover-border-color;
padding: 6px 10px;
display: flex;
}
.logs-stats__title {
font-weight: $font-weight-semi-bold;
padding-right: $spacer;
overflow: hidden;
display: inline-block;
white-space: nowrap;
text-overflow: ellipsis;
flex-grow: 1;
}
.logs-stats__info {
margin-bottom: $spacer / 2;
.logs-stats__body {
padding: 20px 10px 10px 10px;
}
.logs-stats__icon {
margin-left: 0.5em;
.logs-stats__close {
cursor: pointer;
}
......@@ -242,6 +289,6 @@ $column-horizontal-spacing: 10px;
}
&__innerbar {
background-color: $blue;
background: $blue;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment