Skip to content

N
nexpie-grafana-theme
Commit dbd77e0a by Lukas Siatka Committed by GitHub
Options

Explore: fixes log entries sorting - changes milliseconds to nanoseconds (#24303) 

* Chore: adds timeEpochNs to LogRowModel in @grafana/data

* Chore: updates explore utils ResultProcessor getLogsResult and explore utils tests

* Chore: updates core/logs_model to include nanoseconds

* Chore: updates LogRowModel sorting key from milliseconds to nanoseconds and adds timeEpochNs to tests

* Chore: adds timeEpochNs to LogRowModel mock in Explore LiveLogs test

* Chore: fixes logs model timeEpochNs padding

* Chore: updates timeEpochNs padding in tests

* Chore: updates LogRowModel mocks

* Chore: changes isLoki to datasourceId

* Chore: adds hasFieldWithNameAndType method to FieldCache in grafana-data dataframe

* Chore: changes timeEpochNs from number to string as it can overflow Number.MAX_SAFE_INTEGER

* Chore: updates LogRowModel sorting to use milliseconds and nanoseconds

* Chore: removes datasourceId from logSeriesToLogsModel method

* Chore: updates ResultProcessor tests to include nanosecond-level precision log rows sorting
parent 2c9eed36
Showing with 71 additions and 25 deletions
packages/grafana-data/src/dataframe/FieldCache.ts
...@@ -67,6 +67,10 @@ export class FieldCache { ...@@ -67,6 +67,10 @@ export class FieldCache {
return !!this.fieldByName[name]; return !!this.fieldByName[name];
} }
hasFieldWithNameAndType(name: string, type: FieldType): boolean {
return !!this.fieldByName[name] && this.fieldByType[type].filter(field => field.name === name).length > 0;
}
/** /**
* Returns the first field with the given name. * Returns the first field with the given name.
*/ */
......
packages/grafana-data/src/types/logs.ts
...@@ -60,6 +60,9 @@ export interface LogRowModel { ...@@ -60,6 +60,9 @@ export interface LogRowModel {
searchWords?: string[]; searchWords?: string[];
timeFromNow: string; timeFromNow: string;
timeEpochMs: number; timeEpochMs: number;
// timeEpochNs stores time with nanosecond-level precision,
// as millisecond-level precision is usually not enough for proper sorting of logs
timeEpochNs: string;
timeLocal: string; timeLocal: string;
timeUtc: string; timeUtc: string;
uid: string; uid: string;
......
packages/grafana-ui/src/components/Logs/LogDetails.test.tsx
...@@ -15,6 +15,7 @@ const setup = (propOverrides?: Partial<Props>, rowOverrides?: Partial<LogRowMode ...@@ -15,6 +15,7 @@ const setup = (propOverrides?: Partial<Props>, rowOverrides?: Partial<LogRowMode
logLevel: 'error' as LogLevel, logLevel: 'error' as LogLevel,
timeFromNow: '', timeFromNow: '',
timeEpochMs: 1546297200000, timeEpochMs: 1546297200000,
timeEpochNs: '1546297200000000000',
timeLocal: '', timeLocal: '',
timeUtc: '', timeUtc: '',
hasAnsi: false, hasAnsi: false,
......
packages/grafana-ui/src/components/Logs/LogRowContextProvider.test.ts
...@@ -94,6 +94,7 @@ const row: LogRowModel = { ...@@ -94,6 +94,7 @@ const row: LogRowModel = {
raw: '4', raw: '4',
logLevel: LogLevel.info, logLevel: LogLevel.info,
timeEpochMs: 4, timeEpochMs: 4,
timeEpochNs: '4000000',
timeFromNow: '', timeFromNow: '',
timeLocal: '', timeLocal: '',
timeUtc: '', timeUtc: '',
......
packages/grafana-ui/src/components/Logs/LogRows.test.tsx
...@@ -111,6 +111,7 @@ const makeLog = (overrides: Partial<LogRowModel>): LogRowModel => { ...@@ -111,6 +111,7 @@ const makeLog = (overrides: Partial<LogRowModel>): LogRowModel => {
raw: entry, raw: entry,
timeFromNow: '', timeFromNow: '',
timeEpochMs: 1, timeEpochMs: 1,
timeEpochNs: '1000000',
timeLocal: '', timeLocal: '',
timeUtc: '', timeUtc: '',
searchWords: [], searchWords: [],
......
public/app/core/logs_model.ts
...@@ -257,6 +257,7 @@ interface LogFields { ...@@ -257,6 +257,7 @@ interface LogFields {
timeField: FieldWithIndex; timeField: FieldWithIndex;
stringField: FieldWithIndex; stringField: FieldWithIndex;
timeNanosecondField?: FieldWithIndex;
logLevelField?: FieldWithIndex; logLevelField?: FieldWithIndex;
idField?: FieldWithIndex; idField?: FieldWithIndex;
} }
...@@ -284,6 +285,9 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi ...@@ -284,6 +285,9 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
return { return {
series, series,
timeField: fieldCache.getFirstFieldOfType(FieldType.time), timeField: fieldCache.getFirstFieldOfType(FieldType.time),
timeNanosecondField: fieldCache.hasFieldWithNameAndType('tsNs', FieldType.time)
? fieldCache.getFieldByName('tsNs')
: undefined,
stringField, stringField,
logLevelField: fieldCache.getFieldByName('level'), logLevelField: fieldCache.getFieldByName('level'),
idField: getIdField(fieldCache), idField: getIdField(fieldCache),
...@@ -296,7 +300,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi ...@@ -296,7 +300,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
let hasUniqueLabels = false; let hasUniqueLabels = false;
for (const info of allSeries) { for (const info of allSeries) {
const { timeField, stringField, logLevelField, idField, series } = info; const { timeField, timeNanosecondField, stringField, logLevelField, idField, series } = info;
const labels = stringField.labels; const labels = stringField.labels;
const uniqueLabels = findUniqueLabels(labels, commonLabels); const uniqueLabels = findUniqueLabels(labels, commonLabels);
if (Object.keys(uniqueLabels).length > 0) { if (Object.keys(uniqueLabels).length > 0) {
...@@ -311,6 +315,8 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi ...@@ -311,6 +315,8 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
for (let j = 0; j < series.length; j++) { for (let j = 0; j < series.length; j++) {
const ts = timeField.values.get(j); const ts = timeField.values.get(j);
const time = dateTime(ts); const time = dateTime(ts);
const tsNs = timeNanosecondField ? timeNanosecondField.values.get(j) : undefined;
const timeEpochNs = tsNs ? tsNs : time.valueOf() + '000000';
const messageValue: unknown = stringField.values.get(j); const messageValue: unknown = stringField.values.get(j);
// This should be string but sometimes isn't (eg elastic) because the dataFrame is not strongly typed. // This should be string but sometimes isn't (eg elastic) because the dataFrame is not strongly typed.
...@@ -327,7 +333,6 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi ...@@ -327,7 +333,6 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
} else { } else {
logLevel = getLogLevel(message); logLevel = getLogLevel(message);
} }
rows.push({ rows.push({
entryFieldIndex: stringField.index, entryFieldIndex: stringField.index,
rowIndex: j, rowIndex: j,
...@@ -335,6 +340,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi ...@@ -335,6 +340,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
logLevel, logLevel,
timeFromNow: dateTimeFormatTimeAgo(ts), timeFromNow: dateTimeFormatTimeAgo(ts),
timeEpochMs: time.valueOf(), timeEpochMs: time.valueOf(),
timeEpochNs,
timeLocal: dateTimeFormat(ts, { timeZone: 'browser' }), timeLocal: dateTimeFormat(ts, { timeZone: 'browser' }),
timeUtc: dateTimeFormat(ts, { timeZone: 'utc' }), timeUtc: dateTimeFormat(ts, { timeZone: 'utc' }),
uniqueLabels, uniqueLabels,
......
public/app/core/utils/explore.test.ts
...@@ -391,6 +391,7 @@ describe('sortLogsResult', () => { ...@@ -391,6 +391,7 @@ describe('sortLogsResult', () => {
logLevel: LogLevel.info, logLevel: LogLevel.info,
raw: '', raw: '',
timeEpochMs: 0, timeEpochMs: 0,
timeEpochNs: '0',
timeFromNow: '', timeFromNow: '',
timeLocal: '', timeLocal: '',
timeUtc: '', timeUtc: '',
...@@ -407,6 +408,7 @@ describe('sortLogsResult', () => { ...@@ -407,6 +408,7 @@ describe('sortLogsResult', () => {
logLevel: LogLevel.info, logLevel: LogLevel.info,
raw: '', raw: '',
timeEpochMs: 10, timeEpochMs: 10,
timeEpochNs: '10000000',
timeFromNow: '', timeFromNow: '',
timeLocal: '', timeLocal: '',
timeUtc: '', timeUtc: '',
......
public/app/core/utils/explore.ts
...@@ -482,6 +482,7 @@ export const getRefIds = (value: any): string[] => { ...@@ -482,6 +482,7 @@ export const getRefIds = (value: any): string[] => {
}; };
export const sortInAscendingOrder = (a: LogRowModel, b: LogRowModel) => { export const sortInAscendingOrder = (a: LogRowModel, b: LogRowModel) => {
// compare milliseconds
if (a.timeEpochMs < b.timeEpochMs) { if (a.timeEpochMs < b.timeEpochMs) {
return -1; return -1;
} }
...@@ -490,10 +491,20 @@ export const sortInAscendingOrder = (a: LogRowModel, b: LogRowModel) => { ...@@ -490,10 +491,20 @@ export const sortInAscendingOrder = (a: LogRowModel, b: LogRowModel) => {
return 1; return 1;
} }
// if milliseonds are equal, compare nanoseconds
if (a.timeEpochNs < b.timeEpochNs) {
return -1;
}
if (a.timeEpochNs > b.timeEpochNs) {
return 1;
}
return 0; return 0;
}; };
const sortInDescendingOrder = (a: LogRowModel, b: LogRowModel) => { const sortInDescendingOrder = (a: LogRowModel, b: LogRowModel) => {
// compare milliseconds
if (a.timeEpochMs > b.timeEpochMs) { if (a.timeEpochMs > b.timeEpochMs) {
return -1; return -1;
} }
...@@ -502,6 +513,15 @@ const sortInDescendingOrder = (a: LogRowModel, b: LogRowModel) => { ...@@ -502,6 +513,15 @@ const sortInDescendingOrder = (a: LogRowModel, b: LogRowModel) => {
return 1; return 1;
} }
// if milliseonds are equal, compare nanoseconds
if (a.timeEpochNs > b.timeEpochNs) {
return -1;
}
if (a.timeEpochNs < b.timeEpochNs) {
return 1;
}
return 0; return 0;
}; };
......
public/app/features/explore/LiveLogs.test.tsx
...@@ -72,6 +72,7 @@ const makeLog = (overides: Partial<LogRowModel>): LogRowModel => { ...@@ -72,6 +72,7 @@ const makeLog = (overides: Partial<LogRowModel>): LogRowModel => {
raw: entry, raw: entry,
timeFromNow: '', timeFromNow: '',
timeEpochMs: 1, timeEpochMs: 1,
timeEpochNs: '1000000',
timeLocal: '', timeLocal: '',
timeUtc: '', timeUtc: '',
...overides, ...overides,
......
public/app/features/explore/utils/ResultProcessor.test.ts
...@@ -26,7 +26,8 @@ const testContext = (options: any = {}) => { ...@@ -26,7 +26,8 @@ const testContext = (options: any = {}) => {
refId: 'A', refId: 'A',
fields: [ fields: [
{ name: 'value', type: FieldType.number, values: [4, 5, 6] }, { name: 'value', type: FieldType.number, values: [4, 5, 6] },
{ name: 'time', type: FieldType.time, values: [100, 200, 300] }, { name: 'time', type: FieldType.time, values: [100, 100, 100] },
{ name: 'tsNs', type: FieldType.time, values: ['100000002', undefined, '100000001'] },
{ name: 'message', type: FieldType.string, values: ['this is a message', 'second message', 'third'] }, { name: 'message', type: FieldType.string, values: ['this is a message', 'second message', 'third'] },
], ],
}); });
...@@ -125,7 +126,8 @@ describe('ResultProcessor', () => { ...@@ -125,7 +126,8 @@ describe('ResultProcessor', () => {
expect(theResult?.fields[0].name).toEqual('value'); expect(theResult?.fields[0].name).toEqual('value');
expect(theResult?.fields[1].name).toEqual('time'); expect(theResult?.fields[1].name).toEqual('time');
expect(theResult?.fields[2].name).toEqual('message'); expect(theResult?.fields[2].name).toEqual('tsNs');
expect(theResult?.fields[3].name).toEqual('message');
expect(theResult?.fields[1].display).not.toBeNull(); expect(theResult?.fields[1].display).not.toBeNull();
expect(theResult?.length).toBe(3); expect(theResult?.length).toBe(3);
...@@ -135,19 +137,21 @@ describe('ResultProcessor', () => { ...@@ -135,19 +137,21 @@ describe('ResultProcessor', () => {
columns: [ columns: [
{ text: 'value', type: 'number' }, { text: 'value', type: 'number' },
{ text: 'time', type: 'time' }, { text: 'time', type: 'time' },
{ text: 'tsNs', type: 'time' },
{ text: 'message', type: 'string' }, { text: 'message', type: 'string' },
], ],
rows: [ rows: [
[4, 100, 'this is a message'], [4, 100, '100000000', 'this is a message'],
[5, 200, 'second message'], [5, 200, '100000000', 'second message'],
[6, 300, 'third'], [6, 300, '100000000', 'third'],
], ],
type: 'table', type: 'table',
}) })
); );
expect(theResult.fields[0].name).toEqual('value'); expect(theResult.fields[0].name).toEqual('value');
expect(theResult.fields[1].name).toEqual('time'); expect(theResult.fields[1].name).toEqual('time');
expect(theResult.fields[2].name).toEqual('message'); expect(theResult.fields[2].name).toEqual('tsNs');
expect(theResult.fields[3].name).toEqual('message');
expect(theResult.fields[1].display).not.toBeNull(); expect(theResult.fields[1].display).not.toBeNull();
expect(theResult.length).toBe(3); expect(theResult.length).toBe(3);
}); });
...@@ -166,54 +170,57 @@ describe('ResultProcessor', () => { ...@@ -166,54 +170,57 @@ describe('ResultProcessor', () => {
meta: [], meta: [],
rows: [ rows: [
{ {
rowIndex: 2, rowIndex: 0,
dataFrame: logsDataFrame, dataFrame: logsDataFrame,
entry: 'third', entry: 'this is a message',
entryFieldIndex: 2, entryFieldIndex: 3,
hasAnsi: false, hasAnsi: false,
labels: {}, labels: {},
logLevel: 'unknown', logLevel: 'unknown',
raw: 'third', raw: 'this is a message',
searchWords: [] as string[], searchWords: [] as string[],
timeEpochMs: 300, timeEpochMs: 100,
timeEpochNs: '100000002',
timeFromNow: 'fromNow() jest mocked', timeFromNow: 'fromNow() jest mocked',
timeLocal: 'format() jest mocked', timeLocal: 'format() jest mocked',
timeUtc: 'format() jest mocked', timeUtc: 'format() jest mocked',
uid: '2', uid: '0',
uniqueLabels: {}, uniqueLabels: {},
}, },
{ {
rowIndex: 1, rowIndex: 2,
dataFrame: logsDataFrame, dataFrame: logsDataFrame,
entry: 'second message', entry: 'third',
entryFieldIndex: 2, entryFieldIndex: 3,
hasAnsi: false, hasAnsi: false,
labels: {}, labels: {},
logLevel: 'unknown', logLevel: 'unknown',
raw: 'second message', raw: 'third',
searchWords: [] as string[], searchWords: [] as string[],
timeEpochMs: 200, timeEpochMs: 100,
timeEpochNs: '100000001',
timeFromNow: 'fromNow() jest mocked', timeFromNow: 'fromNow() jest mocked',
timeLocal: 'format() jest mocked', timeLocal: 'format() jest mocked',
timeUtc: 'format() jest mocked', timeUtc: 'format() jest mocked',
uid: '1', uid: '2',
uniqueLabels: {}, uniqueLabels: {},
}, },
{ {
rowIndex: 0, rowIndex: 1,
dataFrame: logsDataFrame, dataFrame: logsDataFrame,
entry: 'this is a message', entry: 'second message',
entryFieldIndex: 2, entryFieldIndex: 3,
hasAnsi: false, hasAnsi: false,
labels: {}, labels: {},
logLevel: 'unknown', logLevel: 'unknown',
raw: 'this is a message', raw: 'second message',
searchWords: [] as string[], searchWords: [] as string[],
timeEpochMs: 100, timeEpochMs: 100,
timeEpochNs: '100000000',
timeFromNow: 'fromNow() jest mocked', timeFromNow: 'fromNow() jest mocked',
timeLocal: 'format() jest mocked', timeLocal: 'format() jest mocked',
timeUtc: 'format() jest mocked', timeUtc: 'format() jest mocked',
uid: '0', uid: '1',
uniqueLabels: {}, uniqueLabels: {},
}, },
], ],
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment