Commit acd52e6a by David Kaltschmidt

Explore: Logging line parsing and field stats

Lazily parse lines and display stats for field when clicked on.

- log line parsers for JSON (basic), and logfmt
- delayed parsing in case user moves to other line
- reuse label stats for field stats
parent 9cc4e052
...@@ -95,6 +95,57 @@ export enum LogsDedupStrategy { ...@@ -95,6 +95,57 @@ export enum LogsDedupStrategy {
signature = 'signature', signature = 'signature',
} }
export interface LogsParser {
/**
* Value-agnostic matcher for a field label.
* Used to filter rows, and first capture group contains the value.
*/
buildMatcher: (label: string) => RegExp;
/**
* Regex to find a field in the log line.
* First capture group contains the label value, second capture group the value.
*/
fieldRegex: RegExp;
/**
* Function to verify if this is a valid parser for the given line.
* The parser accepts the line unless it returns undefined.
*/
test: (line: string) => any;
}
export const LogsParsers: { [name: string]: LogsParser } = {
JSON: {
buildMatcher: label => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"([^"]*)"`),
fieldRegex: /"(\w+)"\s*:\s*"([^"]*)"/,
test: line => {
try {
return JSON.parse(line);
} catch (error) {}
},
},
logfmt: {
buildMatcher: label => new RegExp(`(?:^|\\s)${label}=("[^"]*"|\\S+)`),
fieldRegex: /(?:^|\s)(\w+)=("[^"]*"|\S+)/,
test: line => LogsParsers.logfmt.fieldRegex.test(line),
},
};
export function calculateFieldStats(rows: LogRow[], extractor: RegExp): LogsLabelStat[] {
// Consider only rows that satisfy the matcher
const rowsWithField = rows.filter(row => extractor.test(row.entry));
const rowCount = rowsWithField.length;
// Get field value counts for eligible rows
const countsByValue = _.countBy(rowsWithField, row => (row as LogRow).entry.match(extractor)[1]);
const sortedCounts = _.chain(countsByValue)
.map((count, value) => ({ count, value, proportion: count / rowCount }))
.sortBy('count')
.reverse()
.value();
return sortedCounts;
}
export function calculateLogsLabelStats(rows: LogRow[], label: string): LogsLabelStat[] { export function calculateLogsLabelStats(rows: LogRow[], label: string): LogsLabelStat[] {
// Consider only rows that have the given label // Consider only rows that have the given label
const rowsWithLabel = rows.filter(row => row.labels[label] !== undefined); const rowsWithLabel = rows.filter(row => row.labels[label] !== undefined);
...@@ -151,6 +202,19 @@ export function dedupLogRows(logs: LogsModel, strategy: LogsDedupStrategy): Logs ...@@ -151,6 +202,19 @@ export function dedupLogRows(logs: LogsModel, strategy: LogsDedupStrategy): Logs
}; };
} }
export function getParser(line: string): LogsParser {
let parser;
try {
if (LogsParsers.JSON.test(line)) {
parser = LogsParsers.JSON;
}
} catch (error) {}
if (!parser && LogsParsers.logfmt.test(line)) {
parser = LogsParsers.logfmt;
}
return parser;
}
export function filterLogLevels(logs: LogsModel, hiddenLogLevels: Set<LogLevel>): LogsModel { export function filterLogLevels(logs: LogsModel, hiddenLogLevels: Set<LogLevel>): LogsModel {
if (hiddenLogLevels.size === 0) { if (hiddenLogLevels.size === 0) {
return logs; return logs;
......
import { calculateLogsLabelStats, dedupLogRows, LogsDedupStrategy, LogsModel } from '../logs_model'; import {
calculateFieldStats,
calculateLogsLabelStats,
dedupLogRows,
getParser,
LogsDedupStrategy,
LogsModel,
LogsParsers,
} from '../logs_model';
describe('dedupLogRows()', () => { describe('dedupLogRows()', () => {
test('should return rows as is when dedup is set to none', () => { test('should return rows as is when dedup is set to none', () => {
...@@ -107,6 +115,50 @@ describe('dedupLogRows()', () => { ...@@ -107,6 +115,50 @@ describe('dedupLogRows()', () => {
}); });
}); });
describe('calculateFieldStats()', () => {
test('should return no stats for empty rows', () => {
expect(calculateFieldStats([], /foo=(.*)/)).toEqual([]);
});
test('should return no stats if extractor does not match', () => {
const rows = [
{
entry: 'foo=bar',
},
];
expect(calculateFieldStats(rows as any, /baz=(.*)/)).toEqual([]);
});
test('should return stats for found field', () => {
const rows = [
{
entry: 'foo="42 + 1"',
},
{
entry: 'foo=503 baz=foo',
},
{
entry: 'foo="42 + 1"',
},
{
entry: 't=2018-12-05T07:44:59+0000 foo=503',
},
];
expect(calculateFieldStats(rows as any, /foo=("[^"]*"|\S+)/)).toMatchObject([
{
value: '"42 + 1"',
count: 2,
},
{
value: '503',
count: 2,
},
]);
});
});
describe('calculateLogsLabelStats()', () => { describe('calculateLogsLabelStats()', () => {
test('should return no stats for empty rows', () => { test('should return no stats for empty rows', () => {
expect(calculateLogsLabelStats([], '')).toEqual([]); expect(calculateLogsLabelStats([], '')).toEqual([]);
...@@ -159,3 +211,70 @@ describe('calculateLogsLabelStats()', () => { ...@@ -159,3 +211,70 @@ describe('calculateLogsLabelStats()', () => {
]); ]);
}); });
}); });
describe('getParser()', () => {
test('should return no parser on empty line', () => {
expect(getParser('')).toBeUndefined();
});
test('should return no parser on unknown line pattern', () => {
expect(getParser('To Be or not to be')).toBeUndefined();
});
test('should return logfmt parser on key value patterns', () => {
expect(getParser('foo=bar baz="41 + 1')).toEqual(LogsParsers.logfmt);
});
test('should return JSON parser on JSON log lines', () => {
// TODO implement other JSON value types than string
expect(getParser('{"foo": "bar", "baz": "41 + 1"}')).toEqual(LogsParsers.JSON);
});
});
describe('LogsParsers', () => {
describe('logfmt', () => {
const parser = LogsParsers.logfmt;
test('should detect format', () => {
expect(parser.test('foo')).toBeFalsy();
expect(parser.test('foo=bar')).toBeTruthy();
});
test('should have a valid fieldRegex', () => {
const match = 'foo=bar'.match(parser.fieldRegex);
expect(match).toBeDefined();
expect(match[1]).toBe('foo');
expect(match[2]).toBe('bar');
});
test('should build a valid value matcher', () => {
const matcher = parser.buildMatcher('foo');
const match = 'foo=bar'.match(matcher);
expect(match).toBeDefined();
expect(match[1]).toBe('bar');
});
});
describe('JSON', () => {
const parser = LogsParsers.JSON;
test('should detect format', () => {
expect(parser.test('foo')).toBeFalsy();
expect(parser.test('{"foo":"bar"}')).toBeTruthy();
});
test('should have a valid fieldRegex', () => {
const match = '{"foo":"bar"}'.match(parser.fieldRegex);
expect(match).toBeDefined();
expect(match[1]).toBe('foo');
expect(match[2]).toBe('bar');
});
test('should build a valid value matcher', () => {
const matcher = parser.buildMatcher('foo');
const match = '{"foo":"bar"}'.match(matcher);
expect(match).toBeDefined();
expect(match[1]).toBe('bar');
});
});
});
...@@ -24,7 +24,7 @@ function StatsRow({ active, count, proportion, value }: LogsLabelStat) { ...@@ -24,7 +24,7 @@ function StatsRow({ active, count, proportion, value }: LogsLabelStat) {
} }
const STATS_ROW_LIMIT = 5; const STATS_ROW_LIMIT = 5;
class Stats extends PureComponent<{ export class Stats extends PureComponent<{
stats: LogsLabelStat[]; stats: LogsLabelStat[];
label: string; label: string;
value: string; value: string;
...@@ -54,7 +54,7 @@ class Stats extends PureComponent<{ ...@@ -54,7 +54,7 @@ class Stats extends PureComponent<{
<span className="logs-stats__icon fa fa-window-close" onClick={onClickClose} /> <span className="logs-stats__icon fa fa-window-close" onClick={onClickClose} />
</div> </div>
{topRows.map(stat => <StatsRow key={stat.value} {...stat} active={stat.value === value} />)} {topRows.map(stat => <StatsRow key={stat.value} {...stat} active={stat.value === value} />)}
{insertActiveRow && <StatsRow key={activeRow.value} {...activeRow} active />} {insertActiveRow && activeRow && <StatsRow key={activeRow.value} {...activeRow} active />}
{otherCount > 0 && <StatsRow key="__OTHERS__" count={otherCount} value="Other" proportion={otherProportion} />} {otherCount > 0 && <StatsRow key="__OTHERS__" count={otherCount} value="Other" proportion={otherProportion} />}
</> </>
); );
......
...@@ -10,16 +10,20 @@ import { ...@@ -10,16 +10,20 @@ import {
LogsModel, LogsModel,
dedupLogRows, dedupLogRows,
filterLogLevels, filterLogLevels,
getParser,
LogLevel, LogLevel,
LogsMetaKind, LogsMetaKind,
LogsLabelStat,
LogsParser,
LogRow, LogRow,
calculateFieldStats,
} from 'app/core/logs_model'; } from 'app/core/logs_model';
import { findHighlightChunksInText } from 'app/core/utils/text'; import { findHighlightChunksInText } from 'app/core/utils/text';
import { Switch } from 'app/core/components/Switch/Switch'; import { Switch } from 'app/core/components/Switch/Switch';
import ToggleButtonGroup, { ToggleButton } from 'app/core/components/ToggleButtonGroup/ToggleButtonGroup'; import ToggleButtonGroup, { ToggleButton } from 'app/core/components/ToggleButtonGroup/ToggleButtonGroup';
import Graph from './Graph'; import Graph from './Graph';
import LogLabels from './LogLabels'; import LogLabels, { Stats } from './LogLabels';
const PREVIEW_LIMIT = 100; const PREVIEW_LIMIT = 100;
...@@ -38,6 +42,19 @@ const graphOptions = { ...@@ -38,6 +42,19 @@ const graphOptions = {
}, },
}; };
/**
* Renders a highlighted field.
* When hovering, a stats icon is shown.
*/
const FieldHighlight = onClick => props => {
return (
<span className={props.className} style={props.style}>
{props.children}
<span className="logs-row__field-highlight--icon fa fa-signal" onClick={() => onClick(props.children)} />
</span>
);
};
interface RowProps { interface RowProps {
allRows: LogRow[]; allRows: LogRow[];
highlighterExpressions?: string[]; highlighterExpressions?: string[];
...@@ -49,57 +66,169 @@ interface RowProps { ...@@ -49,57 +66,169 @@ interface RowProps {
onClickLabel?: (label: string, value: string) => void; onClickLabel?: (label: string, value: string) => void;
} }
function Row({ interface RowState {
allRows, fieldCount: number;
highlighterExpressions, fieldLabel: string;
onClickLabel, fieldStats: LogsLabelStat[];
row, fieldValue: string;
showDuplicates, parsed: boolean;
showLabels, parser: LogsParser;
showLocalTime, parsedFieldHighlights: string[];
showUtc, showFieldStats: boolean;
}: RowProps) { }
const previewHighlights = highlighterExpressions && !_.isEqual(highlighterExpressions, row.searchWords);
const highlights = previewHighlights ? highlighterExpressions : row.searchWords; /**
const needsHighlighter = highlights && highlights.length > 0; * Renders a log line.
const highlightClassName = classnames('logs-row__match-highlight', { *
'logs-row__match-highlight--preview': previewHighlights, * When user hovers over it for a certain time, it lazily parses the log line.
}); * Once a parser is found, it will determine fields, that will be highlighted.
return ( * When the user requests stats for a field, they will be calculated and rendered below the row.
<div className="logs-row"> */
{showDuplicates && ( class Row extends PureComponent<RowProps, RowState> {
<div className="logs-row__duplicates">{row.duplicates > 0 ? `${row.duplicates + 1}x` : null}</div> mouseMessageTimer: NodeJS.Timer;
)}
<div className={row.logLevel ? `logs-row__level logs-row__level--${row.logLevel}` : ''} /> state = {
{showUtc && ( fieldCount: 0,
<div className="logs-row__time" title={`Local: ${row.timeLocal} (${row.timeFromNow})`}> fieldLabel: null,
{row.timestamp} fieldStats: null,
</div> fieldValue: null,
)} parsed: false,
{showLocalTime && ( parser: null,
<div className="logs-row__time" title={`${row.timestamp} (${row.timeFromNow})`}> parsedFieldHighlights: [],
{row.timeLocal} showFieldStats: false,
</div> };
)}
{showLabels && ( componentWillUnmount() {
<div className="logs-row__labels"> clearTimeout(this.mouseMessageTimer);
<LogLabels allRows={allRows} labels={row.uniqueLabels} onClickLabel={onClickLabel} /> }
</div>
)} onClickClose = () => {
<div className="logs-row__message"> this.setState({ showFieldStats: false });
{needsHighlighter ? ( };
<Highlighter
textToHighlight={row.entry} onClickHighlight = (fieldText: string) => {
searchWords={highlights} const { allRows } = this.props;
findChunks={findHighlightChunksInText} const { parser } = this.state;
highlightClassName={highlightClassName}
/> const fieldMatch = fieldText.match(parser.fieldRegex);
) : ( if (fieldMatch) {
row.entry // Build value-agnostic row matcher based on the field label
const fieldLabel = fieldMatch[1];
const fieldValue = fieldMatch[2];
const matcher = parser.buildMatcher(fieldLabel);
const fieldStats = calculateFieldStats(allRows, matcher);
const fieldCount = fieldStats.reduce((sum, stat) => sum + stat.count, 0);
this.setState({ fieldCount, fieldLabel, fieldStats, fieldValue, showFieldStats: true });
}
};
onMouseOverMessage = () => {
// Don't parse right away, user might move along
this.mouseMessageTimer = setTimeout(this.parseMessage, 500);
};
onMouseOutMessage = () => {
clearTimeout(this.mouseMessageTimer);
this.setState({ parsed: false });
};
parseMessage = () => {
if (!this.state.parsed) {
const { row } = this.props;
const parser = getParser(row.entry);
if (parser) {
// Use parser to highlight detected fields
const parsedFieldHighlights = [];
this.props.row.entry.replace(new RegExp(parser.fieldRegex, 'g'), substring => {
parsedFieldHighlights.push(substring.trim());
return '';
});
this.setState({ parsedFieldHighlights, parsed: true, parser });
}
}
};
render() {
const {
allRows,
highlighterExpressions,
onClickLabel,
row,
showDuplicates,
showLabels,
showLocalTime,
showUtc,
} = this.props;
const {
fieldCount,
fieldLabel,
fieldStats,
fieldValue,
parsed,
parsedFieldHighlights,
showFieldStats,
} = this.state;
const previewHighlights = highlighterExpressions && !_.isEqual(highlighterExpressions, row.searchWords);
const highlights = previewHighlights ? highlighterExpressions : row.searchWords;
const needsHighlighter = highlights && highlights.length > 0;
const highlightClassName = classnames('logs-row__match-highlight', {
'logs-row__match-highlight--preview': previewHighlights,
});
return (
<div className="logs-row">
{showDuplicates && (
<div className="logs-row__duplicates">{row.duplicates > 0 ? `${row.duplicates + 1}x` : null}</div>
)}
<div className={row.logLevel ? `logs-row__level logs__row-level-${row.logLevel}` : ''} />
{showUtc && (
<div className="logs-row__time" title={`Local: ${row.timeLocal} (${row.timeFromNow})`}>
{row.timestamp}
</div>
)}
{showLocalTime && (
<div className="logs-row__time" title={`${row.timestamp} (${row.timeFromNow})`}>
{row.timeLocal}
</div>
)} )}
{showLabels && (
<div className="logs-row__labels">
<LogLabels allRows={allRows} labels={row.uniqueLabels} onClickLabel={onClickLabel} />
</div>
)}
<div className="logs-row__message" onMouseEnter={this.onMouseOverMessage} onMouseLeave={this.onMouseOutMessage}>
{parsed && (
<Highlighter
autoEscape
highlightTag={FieldHighlight(this.onClickHighlight)}
textToHighlight={row.entry}
searchWords={parsedFieldHighlights}
highlightClassName="logs-row__field-highlight"
/>
)}
{!parsed &&
needsHighlighter && (
<Highlighter
textToHighlight={row.entry}
searchWords={highlights}
findChunks={findHighlightChunksInText}
highlightClassName={highlightClassName}
/>
)}
{!parsed && !needsHighlighter && row.entry}
{showFieldStats && (
<Stats
stats={fieldStats}
label={fieldLabel}
value={fieldValue}
onClickClose={this.onClickClose}
rowCount={fieldCount}
/>
)}
</div>
</div> </div>
</div> );
); }
} }
function renderMetaItem(value: any, kind: LogsMetaKind) { function renderMetaItem(value: any, kind: LogsMetaKind) {
......
...@@ -158,6 +158,28 @@ $column-horizontal-spacing: 10px; ...@@ -158,6 +158,28 @@ $column-horizontal-spacing: 10px;
text-align: right; text-align: right;
} }
.logs-row__field-highlight {
// Undoing mark styling
background: inherit;
padding: inherit;
border-bottom: 1px dotted $typeahead-selected-color;
.logs-row__field-highlight--icon {
margin-left: 0.5em;
cursor: pointer;
display: none;
}
}
.logs-row__field-highlight:hover {
color: $typeahead-selected-color;
border-bottom-style: solid;
.logs-row__field-highlight--icon {
display: inline;
}
}
.logs-label { .logs-label {
display: inline-block; display: inline-block;
padding: 0 2px; padding: 0 2px;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment