Commit 29e9b1f7 by Ryan McKinley Committed by GitHub

DataFrames: add arrow test and capture metadata parsing errors (#21524)

parent 76ba2db4
import fs from 'fs';
import path from 'path';
import { resultsToDataFrames, grafanaDataFrameToArrowTable, arrowTableToDataFrame } from './ArrowDataFrame';
import { toDataFrameDTO, toDataFrame } from './processDataFrame';
import { FieldType } from '../types';
import { Table } from 'apache-arrow';
/* tslint:disable */
const resp = {
......@@ -59,4 +63,12 @@ describe('Read/Write arrow Table to DataFrame', () => {
const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2);
expect(after).toEqual(before);
});
test('should parse output with dataframe', () => {
const fullpath = path.resolve(__dirname, './__snapshots__/all_types.golden.arrow');
const arrow = fs.readFileSync(fullpath);
const table = Table.from([arrow]);
const frame = arrowTableToDataFrame(table);
expect(toDataFrameDTO(frame)).toMatchSnapshot();
});
});
import { DataFrame, FieldType, Field, Vector, FieldConfig, Labels } from '../types';
import { DataFrame, FieldType, Field, Vector } from '../types';
import {
Table,
ArrowType,
......@@ -28,6 +28,17 @@ function valueOrUndefined(val?: string) {
return val ? val : undefined;
}
function parseOptionalMeta(str?: string): any {
if (str && str.length && str !== '{}') {
try {
return JSON.parse(str);
} catch (err) {
console.warn('Error reading JSON from arrow metadata: ', str);
}
}
return undefined;
}
export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
const fields: Field[] = [];
......@@ -59,35 +70,23 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
default:
console.log('UNKNOWN Type:', schema);
}
const labelsJson = col.metadata.get('labels');
const configJson = col.metadata.get('config');
let config: FieldConfig = {};
let labels: Labels | undefined = undefined;
if (labelsJson) {
labels = JSON.parse(labelsJson);
}
if (configJson) {
config = JSON.parse(configJson);
}
fields.push({
name: col.name,
type,
config,
values,
labels,
config: parseOptionalMeta(col.metadata.get('config')) || {},
labels: parseOptionalMeta(col.metadata.get('labels')),
});
}
}
const meta = table.schema.metadata;
const metaJson = valueOrUndefined(meta.get('meta'));
return {
fields,
length: table.length,
refId: valueOrUndefined(meta.get('refId')),
name: valueOrUndefined(meta.get('name')),
meta: metaJson ? JSON.parse(metaJson) : undefined,
meta: parseOptionalMeta(meta.get('meta')),
table,
};
}
......
......@@ -102,3 +102,325 @@ Array [
},
]
`;
exports[`Read/Write arrow Table to DataFrame should parse output with dataframe 1`] = `
Object {
"fields": Array [
Object {
"config": Object {},
"labels": undefined,
"name": "string_values",
"type": "string",
"values": Array [
"Grafana",
"❤️",
"Transforms",
],
},
Object {
"config": Object {
"decimals": 2,
"filterable": false,
"links": Array [
Object {
"targetBlank": true,
"title": "Donate - The Sloth Conservation Foundation",
"url": "https://slothconservation.com/how-to-help/donate/",
},
],
"max": null,
"min": null,
"noValue": "😤",
"nullValueMode": "null",
"title": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)",
},
"labels": undefined,
"name": "nullable_string_values",
"type": "string",
"values": Array [
"🦥",
null,
"update your unicode/font if no sloth, is 2019.",
],
},
Object {
"config": Object {
"max": 1,
"min": 0,
},
"labels": undefined,
"name": "int8_values",
"type": "number",
"values": Array [
-128,
1,
127,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_int8_values",
"type": "number",
"values": Array [
-128,
null,
127,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "int16_values",
"type": "number",
"values": Array [
-32768,
1,
32767,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_int16_values",
"type": "number",
"values": Array [
-32768,
null,
32767,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "int32_values",
"type": "number",
"values": Array [
-2147483648,
1,
2147483647,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_int32_values",
"type": "number",
"values": Array [
-2147483648,
null,
2147483647,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "int64_values",
"type": "number",
"values": Array [
"\\"-9223372036854775808\\"",
"\\"1\\"",
"\\"9223372036854775807\\"",
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_int64_values",
"type": "number",
"values": Array [
"\\"-9223372036854775808\\"",
null,
"\\"9223372036854775807\\"",
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "uint8_values",
"type": "number",
"values": Array [
0,
1,
255,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_uint8_values",
"type": "number",
"values": Array [
0,
null,
255,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "uint16_values",
"type": "number",
"values": Array [
0,
1,
65535,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_uint16_values",
"type": "number",
"values": Array [
0,
null,
65535,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "uint32_values",
"type": "number",
"values": Array [
0,
1,
4294967295,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_uint32_values",
"type": "number",
"values": Array [
0,
null,
4294967295,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "uint64_values",
"type": "number",
"values": Array [
"\\"0\\"",
"\\"1\\"",
"\\"18446744073709551615\\"",
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_uint64_values",
"type": "number",
"values": Array [
"\\"0\\"",
null,
"\\"18446744073709551615\\"",
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "float32_values",
"type": "number",
"values": Array [
1.401298464324817e-45,
1,
3.4028234663852886e+38,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_float32_values",
"type": "number",
"values": Array [
1.401298464324817e-45,
null,
3.4028234663852886e+38,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "float64_values",
"type": "number",
"values": Array [
5e-324,
1,
1.7976931348623157e+308,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_float64_values",
"type": "number",
"values": Array [
5e-324,
null,
1.7976931348623157e+308,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "bool_values",
"type": "boolean",
"values": Array [
true,
true,
false,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_bool_values",
"type": "boolean",
"values": Array [
true,
null,
false,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "timestamps",
"type": "time",
"values": Array [
1568039445000,
1568039450000,
1568039455000,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_timestamps",
"type": "time",
"values": Array [
1568039445000,
null,
1568039455000,
],
},
],
"meta": Object {
"limit": 4242,
"searchWords": Array [
"Grafana",
"❤️",
" 🦥 ",
"test",
],
},
"name": "many_types",
"refId": "A",
}
`;
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment