Skip to content

N
nexpie-grafana-theme
Commit 8f4be08b by Andrej Ocenas Committed by GitHub
Options

Explore: Support wide data frames (#28393) 

* Change how isTimeSeries work

* Simplify the decorators and update tests
parent 0bb33839
Showing with 123 additions and 249 deletions
public/app/core/reducers/appNotification.ts
......@@ -5,6 +5,12 @@ export const initialState: AppNotificationsState = {
appNotifications: [] as AppNotification[],
};
/**
* Reducer and action to show toast notifications of various types (success, warnings, errors etc). Use to show
* transient info to user, like errors that cannot be otherwise handled or success after an action.
*
* Use factory functions in core/copy/appNotifications to create the payload.
*/
const appNotificationsSlice = createSlice({
name: 'appNotifications',
initialState,
......
public/app/features/explore/state/actions.ts
// Libraries
import { map, throttleTime } from 'rxjs/operators';
import { map, mergeMap, throttleTime } from 'rxjs/operators';
import { identity } from 'rxjs';
import { PayloadAction } from '@reduxjs/toolkit';
import { DataSourceSrv } from '@grafana/runtime';
......@@ -84,7 +84,7 @@ import {
} from './actionTypes';
import { getTimeZone } from 'app/features/profile/state/selectors';
import { getShiftedTimeRange } from 'app/core/utils/timePicker';
import { updateLocation } from '../../../core/actions';
import { notifyApp, updateLocation } from '../../../core/actions';
import { getTimeSrv, TimeSrv } from '../../dashboard/services/TimeSrv';
import { preProcessPanelData, runRequest } from '../../dashboard/state/runRequest';
import { DashboardModel, PanelModel } from 'app/features/dashboard/state';
......@@ -96,6 +96,7 @@ import {
decorateWithLogsResult,
decorateWithTableResult,
} from '../utils/decorators';
import { createErrorNotification } from '../../../core/copy/appNotification';
/**
* Adds a query row after the row with the given index.
......@@ -427,6 +428,8 @@ export const runQueries = (exploreId: ExploreId): ThunkResult<void> => {
queryResponse,
querySubscription,
history,
refreshInterval,
absoluteRange,
} = exploreItemState;
if (!hasNonEmptyQuery(queries)) {
......@@ -473,12 +476,13 @@ export const runQueries = (exploreId: ExploreId): ThunkResult<void> => {
// actually can see what is happening.
live ? throttleTime(500) : identity,
map((data: PanelData) => preProcessPanelData(data, queryResponse)),
decorateWithGraphLogsTraceAndTable(getState().explore[exploreId].datasourceInstance),
decorateWithGraphResult(),
decorateWithTableResult(),
decorateWithLogsResult(getState().explore[exploreId])
map(decorateWithGraphLogsTraceAndTable),
map(decorateWithGraphResult),
map(decorateWithLogsResult({ absoluteRange, refreshInterval })),
mergeMap(decorateWithTableResult)
)
.subscribe(data => {
.subscribe(
data => {
if (!data.error && firstResponse) {
// Side-effect: Saving history in localstorage
const nextHistory = updateHistory(history, datasourceId, queries);
......@@ -513,7 +517,13 @@ export const runQueries = (exploreId: ExploreId): ThunkResult<void> => {
dispatch(scanStopAction({ exploreId }));
}
}
});
},
error => {
dispatch(notifyApp(createErrorNotification('Query processing error', error)));
dispatch(changeLoadingStateAction({ exploreId, loadingState: LoadingState.Error }));
console.error(error);
}
);
dispatch(queryStoreSubscriptionAction({ exploreId, querySubscription: newQuerySub }));
};
......
public/app/features/explore/utils/decorators.test.ts
......@@ -3,15 +3,12 @@ jest.mock('@grafana/data/src/datetime/formatter', () => ({
dateTimeFormatTimeAgo: (ts: any) => 'fromNow() jest mocked',
}));
import { of } from 'rxjs';
import {
ArrayVector,
DataFrame,
DataQueryRequest,
DataSourceApi,
FieldType,
LoadingState,
observableTester,
PanelData,
TimeRange,
toDataFrame,
......@@ -24,7 +21,7 @@ import {
decorateWithTableResult,
} from './decorators';
import { describe } from '../../../../test/lib/common';
import { ExploreItemState, ExplorePanelData } from 'app/types';
import { ExplorePanelData } from 'app/types';
import TableModel from 'app/core/table_model';
const getTestContext = () => {
......@@ -37,6 +34,7 @@ const getTestContext = () => {
fields: [
{ name: 'time', type: FieldType.time, values: [100, 200, 300] },
{ name: 'A-series', type: FieldType.number, values: [4, 5, 6] },
{ name: 'B-series', type: FieldType.number, values: [7, 8, 9] },
],
});
......@@ -86,10 +84,8 @@ const createExplorePanelData = (args: Partial<ExplorePanelData>): ExplorePanelDa
};
describe('decorateWithGraphLogsTraceAndTable', () => {
describe('when used without error', () => {
it('then the result should be correct', done => {
it('should correctly classify the dataFrames', () => {
const { table, logs, timeSeries, emptyTable } = getTestContext();
const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi;
const series = [table, logs, timeSeries, emptyTable];
const panelData: PanelData = {
series,
......@@ -97,10 +93,7 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
timeRange: ({} as unknown) as TimeRange,
};
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)),
expect: value => {
expect(value).toEqual({
expect(decorateWithGraphLogsTraceAndTable(panelData)).toEqual({
series,
state: LoadingState.Done,
timeRange: {},
......@@ -112,15 +105,9 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
tableResult: null,
logsResult: null,
});
},
done,
});
});
});
describe('when used without frames', () => {
it('then the result should be correct', done => {
const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi;
it('should handle empty array', () => {
const series: DataFrame[] = [];
const panelData: PanelData = {
series,
......@@ -128,10 +115,7 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
timeRange: ({} as unknown) as TimeRange,
};
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)),
expect: value => {
expect(value).toEqual({
expect(decorateWithGraphLogsTraceAndTable(panelData)).toEqual({
series: [],
state: LoadingState.Done,
timeRange: {},
......@@ -143,16 +127,10 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
tableResult: null,
logsResult: null,
});
},
done,
});
});
});
describe('when used with an error', () => {
it('then the result should be correct', done => {
it('should handle query error', () => {
const { timeSeries, logs, table } = getTestContext();
const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi;
const series: DataFrame[] = [timeSeries, logs, table];
const panelData: PanelData = {
series,
......@@ -161,10 +139,7 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
timeRange: ({} as unknown) as TimeRange,
};
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)),
expect: value => {
expect(value).toEqual({
expect(decorateWithGraphLogsTraceAndTable(panelData)).toEqual({
series: [timeSeries, logs, table],
error: {},
state: LoadingState.Error,
......@@ -177,88 +152,65 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
tableResult: null,
logsResult: null,
});
},
done,
});
});
});
});
describe('decorateWithGraphResult', () => {
describe('when used without error', () => {
it('then the graphResult should be correct', done => {
it('should process the graph dataFrames', () => {
const { timeSeries } = getTestContext();
const timeField = timeSeries.fields[0];
const valueField = timeSeries.fields[1];
const panelData = createExplorePanelData({ graphFrames: [timeSeries] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphResult()),
expect: panelData => {
expect(panelData.graphResult![0]).toEqual({
console.log(decorateWithGraphResult(panelData).graphResult);
expect(decorateWithGraphResult(panelData).graphResult).toMatchObject([
{
label: 'A-series',
color: '#7EB26D',
data: [
[100, 4],
[200, 5],
[300, 6],
],
info: [],
isVisible: true,
yAxis: {
index: 1,
},
seriesIndex: 0,
timeField,
valueField,
timeStep: 100,
});
},
done,
});
});
{
label: 'B-series',
data: [
[100, 7],
[200, 8],
[300, 9],
],
isVisible: true,
yAxis: {
index: 1,
},
seriesIndex: 1,
timeStep: 100,
},
]);
});
describe('when used without error but graph frames are empty', () => {
it('then the graphResult should be null', done => {
it('returns null if it gets empty array', () => {
const panelData = createExplorePanelData({ graphFrames: [] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphResult()),
expect: panelData => {
expect(panelData.graphResult).toBeNull();
},
done,
});
});
expect(decorateWithGraphResult(panelData).graphResult).toBeNull();
});
describe('when used with error', () => {
it('then the graphResult should be null', done => {
it('returns null if panelData has error', () => {
const { timeSeries } = getTestContext();
const panelData = createExplorePanelData({ error: {}, graphFrames: [timeSeries] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphResult()),
expect: panelData => {
expect(panelData.graphResult).toBeNull();
},
done,
});
});
expect(decorateWithGraphResult(panelData).graphResult).toBeNull();
});
});
describe('decorateWithTableResult', () => {
describe('when used without error', () => {
it('then the tableResult should be correct', done => {
it('should process table type dataFrame', async () => {
const { table, emptyTable } = getTestContext();
const panelData = createExplorePanelData({ tableFrames: [table, emptyTable] });
const panelResult = await decorateWithTableResult(panelData).toPromise();
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithTableResult()),
expect: panelData => {
let theResult = panelData.tableResult;
let theResult = panelResult.tableResult;
expect(theResult?.fields[0].name).toEqual('value');
expect(theResult?.fields[1].name).toEqual('time');
......@@ -291,12 +243,9 @@ describe('decorateWithTableResult', () => {
expect(theResult.fields[3].name).toEqual('message');
expect(theResult.fields[1].display).not.toBeNull();
expect(theResult.length).toBe(3);
},
done,
});
});
it('should do join transform if all series are timeseries', done => {
it('should do join transform if all series are timeseries', async () => {
const tableFrames = [
toDataFrame({
name: 'A-series',
......@@ -316,11 +265,8 @@ describe('decorateWithTableResult', () => {
}),
];
const panelData = createExplorePanelData({ tableFrames });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithTableResult()),
expect: panelData => {
const result = panelData.tableResult;
const panelResult = await decorateWithTableResult(panelData).toPromise();
const result = panelResult.tableResult;
expect(result?.fields[0].name).toBe('Time');
expect(result?.fields[1].name).toBe('A-series');
......@@ -328,12 +274,9 @@ describe('decorateWithTableResult', () => {
expect(result?.fields[0].values.toArray()).toEqual([100, 200, 300]);
expect(result?.fields[1].values.toArray()).toEqual([4, 5, 6]);
expect(result?.fields[2].values.toArray()).toEqual([4, 5, 6]);
},
done,
});
});
it('should not override fields display property when filled', done => {
it('should not override fields display property when filled', async () => {
const tableFrames = [
toDataFrame({
name: 'A-series',
......@@ -345,64 +288,30 @@ describe('decorateWithTableResult', () => {
tableFrames[0].fields[0].display = displayFunctionMock;
const panelData = createExplorePanelData({ tableFrames });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithTableResult()),
expect: panelData => {
const data = panelData.tableResult;
expect(data?.fields[0].display).toBe(displayFunctionMock);
},
done,
});
});
const panelResult = await decorateWithTableResult(panelData).toPromise();
expect(panelResult.tableResult?.fields[0].display).toBe(displayFunctionMock);
});
describe('when used without error but table frames are empty', () => {
it('then the tableResult should be null', done => {
it('should return null when passed empty array', async () => {
const panelData = createExplorePanelData({ tableFrames: [] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithTableResult()),
expect: panelData => {
expect(panelData.tableResult).toBeNull();
},
done,
});
});
const panelResult = await decorateWithTableResult(panelData).toPromise();
expect(panelResult.tableResult).toBeNull();
});
describe('when used with error', () => {
it('then the tableResult should be null', done => {
it('returns null if panelData has error', async () => {
const { table, emptyTable } = getTestContext();
const panelData = createExplorePanelData({ error: {}, tableFrames: [table, emptyTable] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithTableResult()),
expect: panelData => {
expect(panelData.tableResult).toBeNull();
},
done,
});
});
const panelResult = await decorateWithTableResult(panelData).toPromise();
expect(panelResult.tableResult).toBeNull();
});
});
describe('decorateWithLogsResult', () => {
describe('when used without error', () => {
it('then the logsResult should be correct', done => {
it('should correctly transform logs dataFrames', () => {
const { logs } = getTestContext();
const state = ({
queryIntervals: { intervalMs: 10 },
} as unknown) as ExploreItemState;
const request = ({ timezone: 'utc', intervalMs: 60000 } as unknown) as DataQueryRequest;
const panelData = createExplorePanelData({ logsFrames: [logs], request });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithLogsResult(state)),
expect: panelData => {
const theResult = panelData.logsResult;
expect(theResult).toEqual({
expect(decorateWithLogsResult()(panelData).logsResult).toEqual({
hasUniqueLabels: false,
meta: [],
rows: [
......@@ -496,40 +405,16 @@ describe('decorateWithLogsResult', () => {
],
visibleRange: undefined,
});
},
done,
});
});
});
describe('when used without error but logs frames are empty', () => {
it('then the graphResult should be null', done => {
it('returns null if passed empty array', () => {
const panelData = createExplorePanelData({ logsFrames: [] });
const state = ({} as unknown) as ExploreItemState;
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithLogsResult(state)),
expect: panelData => {
expect(panelData.logsResult).toBeNull();
},
done,
});
});
expect(decorateWithLogsResult()(panelData).logsResult).toBeNull();
});
describe('when used with error', () => {
it('then the graphResult should be null', done => {
it('returns null if panelData has error', () => {
const { logs } = getTestContext();
const panelData = createExplorePanelData({ error: {}, logsFrames: [logs] });
const state = ({} as unknown) as ExploreItemState;
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithLogsResult(state)),
expect: panelData => {
expect(panelData.logsResult).toBeNull();
},
done,
});
});
expect(decorateWithLogsResult()(panelData).logsResult).toBeNull();
});
});
public/app/features/explore/utils/decorators.ts
import { MonoTypeOperatorFunction, of, OperatorFunction } from 'rxjs';
import { map, mergeMap } from 'rxjs/operators';
import { Observable, of } from 'rxjs';
import { map } from 'rxjs/operators';
import {
AbsoluteTimeRange,
DataFrame,
DataSourceApi,
FieldType,
getDisplayProcessor,
PanelData,
PreferredVisualisationType,
sortLogsResult,
standardTransformers,
} from '@grafana/data';
import { config } from '@grafana/runtime';
import { groupBy } from 'lodash';
import { ExploreItemState, ExplorePanelData } from '../../../types';
import { ExplorePanelData } from '../../../types';
import { getGraphSeriesModel } from '../../../plugins/panel/graph2/getGraphSeriesModel';
import { dataFrameToLogsModel } from '../../../core/logs_model';
import { refreshIntervalToSortOrder } from '../../../core/utils/explore';
export const decorateWithGraphLogsTraceAndTable = (
datasourceInstance?: DataSourceApi | null
): OperatorFunction<PanelData, ExplorePanelData> => inputStream =>
inputStream.pipe(
map(data => {
/**
* When processing response first we try to determine what kind of dataframes we got as one query can return multiple
* dataFrames with different type of data. This is later used for type specific processing. As we use this in
* Observable pipeline, it decorates the existing panelData to pass the results to later processing stages.
*/
export const decorateWithGraphLogsTraceAndTable = (data: PanelData): ExplorePanelData => {
if (data.error) {
return {
...data,
......@@ -41,26 +42,29 @@ export const decorateWithGraphLogsTraceAndTable = (
const traceFrames: DataFrame[] = [];
for (const frame of data.series) {
if (shouldShowInVisualisationTypeStrict(frame, 'logs')) {
switch (frame.meta?.preferredVisualisationType) {
case 'logs':
logsFrames.push(frame);
} else if (shouldShowInVisualisationTypeStrict(frame, 'graph')) {
break;
case 'graph':
graphFrames.push(frame);
} else if (shouldShowInVisualisationTypeStrict(frame, 'trace')) {
break;
case 'trace':
traceFrames.push(frame);
} else if (shouldShowInVisualisationTypeStrict(frame, 'table')) {
break;
case 'table':
tableFrames.push(frame);
} else if (isTimeSeries(frame, datasourceInstance?.meta.id)) {
if (shouldShowInVisualisationType(frame, 'graph')) {
break;
default:
if (isTimeSeries(frame)) {
graphFrames.push(frame);
}
if (shouldShowInVisualisationType(frame, 'table')) {
tableFrames.push(frame);
}
} else {
// We fallback to table if we do not have any better meta info about the dataframe.
tableFrames.push(frame);
}
}
}
return {
...data,
......@@ -72,12 +76,9 @@ export const decorateWithGraphLogsTraceAndTable = (
tableResult: null,
logsResult: null,
};
})
);
};
export const decorateWithGraphResult = (): MonoTypeOperatorFunction<ExplorePanelData> => inputStream =>
inputStream.pipe(
map(data => {
export const decorateWithGraphResult = (data: ExplorePanelData): ExplorePanelData => {
if (data.error) {
return { ...data, graphResult: null };
}
......@@ -94,12 +95,14 @@ export const decorateWithGraphResult = (): MonoTypeOperatorFunction<ExplorePanel
);
return { ...data, graphResult };
})
);
export const decorateWithTableResult = (): MonoTypeOperatorFunction<ExplorePanelData> => inputStream =>
inputStream.pipe(
mergeMap(data => {
};
/**
* This processing returns Observable because it uses Transformer internally which result type is also Observable.
* In this case the transformer should return single result but it is possible that in the future it could return
* multiple results and so this should be used with mergeMap or similar to unbox the internal observable.
*/
export const decorateWithTableResult = (data: ExplorePanelData): Observable<ExplorePanelData> => {
if (data.error) {
return of({ ...data, tableResult: null });
}
......@@ -148,67 +151,37 @@ export const decorateWithTableResult = (): MonoTypeOperatorFunction<ExplorePanel
return { ...data, tableResult: frame };
})
);
})
);
};
export const decorateWithLogsResult = (
state: ExploreItemState
): MonoTypeOperatorFunction<ExplorePanelData> => inputStream =>
inputStream.pipe(
map(data => {
options: { absoluteRange?: AbsoluteTimeRange; refreshInterval?: string } = {}
) => (data: ExplorePanelData): ExplorePanelData => {
if (data.error) {
return { ...data, logsResult: null };
}
const { absoluteRange, refreshInterval } = state;
if (data.logsFrames.length === 0) {
return { ...data, logsResult: null };
}
const timeZone = data.request?.timezone ?? 'browser';
const intervalMs = data.request?.intervalMs;
const newResults = dataFrameToLogsModel(data.logsFrames, intervalMs, timeZone, absoluteRange);
const sortOrder = refreshIntervalToSortOrder(refreshInterval);
const newResults = dataFrameToLogsModel(data.logsFrames, intervalMs, timeZone, options.absoluteRange);
const sortOrder = refreshIntervalToSortOrder(options.refreshInterval);
const sortedNewResults = sortLogsResult(newResults, sortOrder);
const rows = sortedNewResults.rows;
const series = sortedNewResults.series;
const logsResult = { ...sortedNewResults, rows, series };
return { ...data, logsResult };
})
);
function isTimeSeries(frame: DataFrame, datasource?: string): boolean {
// TEMP: Temporary hack. Remove when logs/metrics unification is done
if (datasource && datasource === 'cloudwatch') {
return isTimeSeriesCloudWatch(frame);
}
if (frame.fields.length === 2) {
if (frame.fields[0].type === FieldType.time) {
return true;
}
}
return false;
}
function shouldShowInVisualisationType(frame: DataFrame, visualisation: PreferredVisualisationType) {
if (frame.meta?.preferredVisualisationType && frame.meta?.preferredVisualisationType !== visualisation) {
return false;
}
return true;
}
function shouldShowInVisualisationTypeStrict(frame: DataFrame, visualisation: PreferredVisualisationType) {
return frame.meta?.preferredVisualisationType === visualisation;
}
// TEMP: Temporary hack. Remove when logs/metrics unification is done
function isTimeSeriesCloudWatch(frame: DataFrame): boolean {
return (
frame.fields.some(field => field.type === FieldType.time) &&
frame.fields.some(field => field.type === FieldType.number)
};
/**
* Check if frame contains time series, which for our purpose means 1 time column and 1 or more numeric columns.
*/
function isTimeSeries(frame: DataFrame): boolean {
const grouped = groupBy(frame.fields, field => field.type);
return Boolean(
Object.keys(grouped).length === 2 && grouped[FieldType.time]?.length === 1 && grouped[FieldType.number]
);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment