Skip to content

N
nexpie-grafana-theme
Commit e5505720 by Ryan McKinley Committed by GitHub
Options

DataFrame: round trip metadata to arrow Table (#21277)

parent cb3d91b5
Showing with 152 additions and 72 deletions
packages/grafana-data/rollup.config.ts
...@@ -20,7 +20,7 @@ const buildCjsPackage = ({ env }) => { ...@@ -20,7 +20,7 @@ const buildCjsPackage = ({ env }) => {
globals: {}, globals: {},
}, },
], ],
external: ['lodash'], // Use Lodash from grafana external: ['lodash', 'apache-arrow'], // Use Lodash & arrow from grafana
plugins: [ plugins: [
commonjs({ commonjs({
include: /node_modules/, include: /node_modules/,
......
packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.test.ts → packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts
import { resultsToDataFrames } from './ArrowDataFrame'; import { resultsToDataFrames, grafanaDataFrameToArrowTable, arrowTableToDataFrame } from './ArrowDataFrame';
import { toDataFrameDTO } from '../processDataFrame'; import { toDataFrameDTO, toDataFrame } from './processDataFrame';
import { FieldType } from '../types';
/* tslint:disable */ /* tslint:disable */
const resp = { const resp = {
...@@ -33,3 +34,29 @@ describe('GEL Utils', () => { ...@@ -33,3 +34,29 @@ describe('GEL Utils', () => {
expect(norm).toMatchSnapshot(); expect(norm).toMatchSnapshot();
}); });
}); });
describe('Read/Write arrow Table to DataFrame', () => {
test('should parse output with dataframe', () => {
const frame = toDataFrame({
name: 'Hello',
refId: 'XYZ',
meta: {
aaa: 'xyz',
anything: 'xxx',
},
fields: [
{ name: 'time', config: {}, type: FieldType.time, values: [1, 2, 3] },
{ name: 'value', config: { min: 0, max: 50, unit: 'somthing' }, type: FieldType.number, values: [1, 2, 3] },
{ name: 'str', config: {}, type: FieldType.string, values: ['a', 'b', 'c'] },
],
});
const table = grafanaDataFrameToArrowTable(frame);
expect(table.length).toEqual(frame.length);
// Now back to DataFrame
const before = JSON.stringify(toDataFrameDTO(frame), null, 2);
const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2);
expect(after).toEqual(before);
});
});
packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.ts → packages/grafana-data/src/dataframe/ArrowDataFrame.ts
import { DataFrame, FieldType, Field, Vector } from '../../types'; import { DataFrame, FieldType, Field, Vector, FieldConfig, Labels } from '../types';
import { Table, ArrowType } from 'apache-arrow'; import {
Table,
ArrowType,
Builder,
Vector as ArrowVector,
Float64,
DataType,
Utf8,
TimestampMillisecond,
Bool,
Column,
} from 'apache-arrow';
export interface ArrowDataFrame extends DataFrame { export interface ArrowDataFrame extends DataFrame {
table: Table; table: Table;
...@@ -41,29 +52,92 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame { ...@@ -41,29 +52,92 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
type = FieldType.time; type = FieldType.time;
break; break;
} }
case ArrowType.Utf8: {
type = FieldType.string;
break;
}
default: default:
console.log('UNKNOWN Type:', schema); console.log('UNKNOWN Type:', schema);
} }
// console.log(' field>', schema.metadata); const labelsJson = col.metadata.get('labels');
const configJson = col.metadata.get('config');
let config: FieldConfig = {};
let labels: Labels | undefined = undefined;
if (labelsJson) {
labels = JSON.parse(labelsJson);
}
if (configJson) {
config = JSON.parse(configJson);
}
fields.push({ fields.push({
name: col.name, name: col.name,
type, type,
config: {}, // TODO, pull from metadata config,
values, values,
labels,
}); });
} }
} }
const meta = table.schema.metadata; const meta = table.schema.metadata;
const metaJson = valueOrUndefined(meta.get('meta'));
return { return {
fields, fields,
length: table.length, length: table.length,
refId: valueOrUndefined(meta.get('refId')), refId: valueOrUndefined(meta.get('refId')),
name: valueOrUndefined(meta.get('name')), name: valueOrUndefined(meta.get('name')),
meta: metaJson ? JSON.parse(metaJson) : undefined,
table, table,
}; };
} }
function toArrowVector(field: Field): ArrowVector {
// OR: Float64Vector.from([1, 2, 3]));
let type: DataType;
if (field.type === FieldType.number) {
type = new Float64();
} else if (field.type === FieldType.time) {
type = new TimestampMillisecond();
} else if (field.type === FieldType.boolean) {
type = new Bool();
} else if (field.type === FieldType.string) {
type = new Utf8();
} else {
type = new Utf8();
}
const builder = Builder.new({ type, nullValues: [null] });
field.values.toArray().forEach(builder.append.bind(builder));
return builder.finish().toVector();
}
export function grafanaDataFrameToArrowTable(data: DataFrame): Table {
const table = Table.new(
data.fields.map(field => {
const column = Column.new(field.name, toArrowVector(field));
if (field.labels) {
column.metadata.set('labels', JSON.stringify(field.labels));
}
if (field.config) {
column.metadata.set('config', JSON.stringify(field.config));
}
return column;
})
);
const metadata = table.schema.metadata;
if (data.name) {
metadata.set('name', data.name);
}
if (data.refId) {
metadata.set('refId', data.refId);
}
if (data.meta) {
metadata.set('meta', JSON.stringify(data.meta));
}
return table;
}
export function resultsToDataFrames(rsp: any): DataFrame[] { export function resultsToDataFrames(rsp: any): DataFrame[] {
const frames: DataFrame[] = []; const frames: DataFrame[] = [];
for (const res of Object.values(rsp.results)) { for (const res of Object.values(rsp.results)) {
......
packages/grafana-data/src/dataframe/arrow/__snapshots__/ArrowDataFrame.test.ts.snap → packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap
...@@ -9,33 +9,20 @@ Array [ ...@@ -9,33 +9,20 @@ Array [
"labels": undefined, "labels": undefined,
"name": "Time", "name": "Time",
"type": "time", "type": "time",
"values": Int32Array [ "values": Array [
882710016, 1569334575000,
365389179, 1569334580000,
1587742720, 1569334585000,
365389180, 1569334590000,
-2002191872, 1569334595000,
365389181, 1569334600000,
-1297159168, 1569334605000,
365389182, 1569334610000,
-592126464, 1569334615000,
365389183, 1569334620000,
112906240, 1569334625000,
365389185, 1569334630000,
817938944, 1569334635000,
365389186,
1522971648,
365389187,
-2066962944,
365389188,
-1361930240,
365389189,
-656897536,
365389190,
48135168,
365389192,
753167872,
365389193,
], ],
}, },
Object { Object {
...@@ -43,7 +30,7 @@ Array [ ...@@ -43,7 +30,7 @@ Array [
"labels": undefined, "labels": undefined,
"name": "", "name": "",
"type": "number", "type": "number",
"values": Float64Array [ "values": Array [
3, 3,
3, 3,
3, 3,
...@@ -71,33 +58,20 @@ Array [ ...@@ -71,33 +58,20 @@ Array [
"labels": undefined, "labels": undefined,
"name": "Time", "name": "Time",
"type": "time", "type": "time",
"values": Int32Array [ "values": Array [
882710016, 1569334575000,
365389179, 1569334580000,
1587742720, 1569334585000,
365389180, 1569334590000,
-2002191872, 1569334595000,
365389181, 1569334600000,
-1297159168, 1569334605000,
365389182, 1569334610000,
-592126464, 1569334615000,
365389183, 1569334620000,
112906240, 1569334625000,
365389185, 1569334630000,
817938944, 1569334635000,
365389186,
1522971648,
365389187,
-2066962944,
365389188,
-1361930240,
365389189,
-656897536,
365389190,
48135168,
365389192,
753167872,
365389193,
], ],
}, },
Object { Object {
...@@ -105,7 +79,7 @@ Array [ ...@@ -105,7 +79,7 @@ Array [
"labels": undefined, "labels": undefined,
"name": "GB-series", "name": "GB-series",
"type": "number", "type": "number",
"values": Float64Array [ "values": Array [
0, 0,
0, 0,
0, 0,
......
packages/grafana-data/src/dataframe/index.ts
...@@ -4,10 +4,4 @@ export * from './CircularDataFrame'; ...@@ -4,10 +4,4 @@ export * from './CircularDataFrame';
export * from './MutableDataFrame'; export * from './MutableDataFrame';
export * from './processDataFrame'; export * from './processDataFrame';
export * from './dimensions'; export * from './dimensions';
export * from './ArrowDataFrame';
// NOTE: We can not export arrow in the global scope because it will crash phantomjs
// In core, this is loaded async. In plugins you can import using:
//
// import { resultsToDataFrames } from '@grafana/data/dataframe/arrow/ArrowDataFrame'
//
// export * from './arrow/ArrowDataFrame';
packages/grafana-data/src/dataframe/processDataFrame.ts
...@@ -442,11 +442,22 @@ export function getDataFrameRow(data: DataFrame, row: number): any[] { ...@@ -442,11 +442,22 @@ export function getDataFrameRow(data: DataFrame, row: number): any[] {
*/ */
export function toDataFrameDTO(data: DataFrame): DataFrameDTO { export function toDataFrameDTO(data: DataFrame): DataFrameDTO {
const fields: FieldDTO[] = data.fields.map(f => { const fields: FieldDTO[] = data.fields.map(f => {
let values = f.values.toArray();
if (!Array.isArray(values)) {
// Apache arrow will pack objects into typed arrays
// Float64Array, etc
// TODO: Float64Array could be used directly
values = [];
for (let i = 0; i < f.values.length; i++) {
values.push(f.values.get(i));
}
}
return { return {
name: f.name, name: f.name,
type: f.type, type: f.type,
config: f.config, config: f.config,
values: f.values.toArray(), values,
labels: f.labels, labels: f.labels,
}; };
}); });
......
public/app/features/expressions/ExpressionDatasource.ts
...@@ -69,7 +69,7 @@ export class ExpressionDatasourceApi extends DataSourceApi<ExpressionQuery> { ...@@ -69,7 +69,7 @@ export class ExpressionDatasourceApi extends DataSourceApi<ExpressionQuery> {
*/ */
async toDataQueryResponse(rsp: any): Promise<DataQueryResponse> { async toDataQueryResponse(rsp: any): Promise<DataQueryResponse> {
const { resultsToDataFrames } = await import( const { resultsToDataFrames } = await import(
/* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/arrow/ArrowDataFrame' /* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/ArrowDataFrame'
); );
return { data: resultsToDataFrames(rsp) }; return { data: resultsToDataFrames(rsp) };
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment