Skip to content

N
nexpie-grafana-theme
Commit f59d7ace by Ryan McKinley Committed by GitHub
Options

Arrow DataFrame: cast BigInt/UInt to numbers (#25811)

parent 629ce689
Showing with 78 additions and 49 deletions
packages/grafana-data/src/dataframe/ArrayDataFrame.test.ts
...@@ -7,13 +7,15 @@ describe('Array DataFrame', () => { ...@@ -7,13 +7,15 @@ describe('Array DataFrame', () => {
{ name: 'first', value: 1, time: 123 }, { name: 'first', value: 1, time: 123 },
{ name: 'second', value: 2, time: 456, extra: 'here' }, { name: 'second', value: 2, time: 456, extra: 'here' },
{ name: 'third', value: 3, time: 789 }, { name: 'third', value: 3, time: 789 },
{ name: '4th (NaN)', value: NaN, time: 1000 },
{ name: '5th (Null)', value: null, time: 1100 },
]; ];
const frame = new ArrayDataFrame(input); const frame = new ArrayDataFrame(input);
frame.name = 'Hello'; frame.name = 'Hello';
frame.refId = 'Z'; frame.refId = 'Z';
frame.setFieldType('phantom', FieldType.string, v => '🦥'); frame.setFieldType('phantom', FieldType.string, v => '🦥');
const field = frame.fields.find(f => f.name == 'value'); const field = frame.fields.find(f => f.name === 'value');
field!.config.unit = 'kwh'; field!.config.unit = 'kwh';
test('Should support functional methods', () => { test('Should support functional methods', () => {
...@@ -48,6 +50,8 @@ describe('Array DataFrame', () => { ...@@ -48,6 +50,8 @@ describe('Array DataFrame', () => {
"first", "first",
"second", "second",
"third", "third",
"4th (NaN)",
"5th (Null)",
], ],
}, },
Object { Object {
...@@ -61,6 +65,8 @@ describe('Array DataFrame', () => { ...@@ -61,6 +65,8 @@ describe('Array DataFrame', () => {
1, 1,
2, 2,
3, 3,
NaN,
null,
], ],
}, },
Object { Object {
...@@ -72,6 +78,8 @@ describe('Array DataFrame', () => { ...@@ -72,6 +78,8 @@ describe('Array DataFrame', () => {
123, 123,
456, 456,
789, 789,
1000,
1100,
], ],
}, },
Object { Object {
...@@ -83,6 +91,8 @@ describe('Array DataFrame', () => { ...@@ -83,6 +91,8 @@ describe('Array DataFrame', () => {
"🦥", "🦥",
"🦥", "🦥",
"🦥", "🦥",
"🦥",
"🦥",
], ],
}, },
], ],
......
packages/grafana-data/src/dataframe/ArrowDataFrame.ts
import { DataFrame, FieldType, Field, Vector } from '../types'; import { DataFrame, FieldType, Field, Vector } from '../types';
import { FunctionalVector } from '../vector/FunctionalVector';
import { import {
Table, Table,
...@@ -48,14 +49,18 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame { ...@@ -48,14 +49,18 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
if (col) { if (col) {
const schema = table.schema.fields[i]; const schema = table.schema.fields[i];
let type = FieldType.other; let type = FieldType.other;
const values: Vector<any> = col; let values: Vector<any> = col;
switch ((schema.typeId as unknown) as ArrowType) { switch ((schema.typeId as unknown) as ArrowType) {
case ArrowType.Decimal: case ArrowType.Decimal:
case ArrowType.Int:
case ArrowType.FloatingPoint: { case ArrowType.FloatingPoint: {
type = FieldType.number; type = FieldType.number;
break; break;
} }
case ArrowType.Int: {
type = FieldType.number;
values = new NumberColumn(col); // Cast to number
break;
}
case ArrowType.Bool: { case ArrowType.Bool: {
type = FieldType.boolean; type = FieldType.boolean;
break; break;
...@@ -73,7 +78,7 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame { ...@@ -73,7 +78,7 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
} }
fields.push({ fields.push({
name: stripFieldNamePrefix(col.name), name: col.name,
type, type,
values, values,
config: parseOptionalMeta(col.metadata.get('config')) || {}, config: parseOptionalMeta(col.metadata.get('config')) || {},
...@@ -92,17 +97,6 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame { ...@@ -92,17 +97,6 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
}; };
} }
// fieldNamePrefixSep is the delimiter used with fieldNamePrefix.
const fieldNamePrefixSep = '🦥: ';
function stripFieldNamePrefix(name: string): string {
const idx = name.indexOf(fieldNamePrefixSep);
if (idx > 0) {
return name.substring(idx + fieldNamePrefixSep.length);
}
return name;
}
function toArrowVector(field: Field): ArrowVector { function toArrowVector(field: Field): ArrowVector {
// OR: Float64Vector.from([1, 2, 3])); // OR: Float64Vector.from([1, 2, 3]));
...@@ -129,17 +123,10 @@ export function grafanaDataFrameToArrowTable(data: DataFrame): Table { ...@@ -129,17 +123,10 @@ export function grafanaDataFrameToArrowTable(data: DataFrame): Table {
if (table instanceof Table) { if (table instanceof Table) {
return table as Table; return table as Table;
} }
// Make sure the names are unique
const names = new Set<string>();
table = Table.new( table = Table.new(
data.fields.map((field, index) => { data.fields.map((field, index) => {
let name = field.name; const column = Column.new(field.name, toArrowVector(field));
if (names.has(field.name)) {
name = `${index}${fieldNamePrefixSep}${field.name}`;
}
names.add(name);
const column = Column.new(name, toArrowVector(field));
if (field.labels) { if (field.labels) {
column.metadata.set('labels', JSON.stringify(field.labels)); column.metadata.set('labels', JSON.stringify(field.labels));
} }
...@@ -161,3 +148,26 @@ export function grafanaDataFrameToArrowTable(data: DataFrame): Table { ...@@ -161,3 +148,26 @@ export function grafanaDataFrameToArrowTable(data: DataFrame): Table {
} }
return table; return table;
} }
class NumberColumn extends FunctionalVector<number> {
constructor(private col: Column) {
super();
}
get length() {
return this.col.length;
}
get(index: number): number {
const v = this.col.get(index);
if (v === null || isNaN(v)) {
return v;
}
// The conversion operations are always silent, never give errors,
// but if the bigint is too huge and won’t fit the number type,
// then extra bits will be cut off, so we should be careful doing such conversion.
// See https://javascript.info/bigint
return Number(v);
}
}
packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap
...@@ -21,6 +21,7 @@ Object { ...@@ -21,6 +21,7 @@ Object {
Object { Object {
"config": Object { "config": Object {
"decimals": 2, "decimals": 2,
"displayName": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)",
"filterable": false, "filterable": false,
"links": Array [ "links": Array [
Object { Object {
...@@ -33,7 +34,6 @@ Object { ...@@ -33,7 +34,6 @@ Object {
"min": null, "min": null,
"noValue": "😤", "noValue": "😤",
"nullValueMode": "null", "nullValueMode": "null",
"title": "Grafana ❤️ (Previous should be heart emoji) 🦥 (Previous should be sloth emoji)",
}, },
"labels": Object { "labels": Object {
"aLabelKey": "aLabelValue", "aLabelKey": "aLabelValue",
...@@ -136,11 +136,11 @@ Object { ...@@ -136,11 +136,11 @@ Object {
"name": "int64_values", "name": "int64_values",
"type": "number", "type": "number",
"values": Array [ "values": Array [
"\\"-9223372036854775808\\"", -9223372036854776000,
"\\"-9007199254740991\\"", -9007199254740991,
"\\"1\\"", 1,
"\\"9007199254740991\\"", 9007199254740991,
"\\"9223372036854775807\\"", 9223372036854776000,
], ],
}, },
Object { Object {
...@@ -149,11 +149,11 @@ Object { ...@@ -149,11 +149,11 @@ Object {
"name": "nullable_int64_values", "name": "nullable_int64_values",
"type": "number", "type": "number",
"values": Array [ "values": Array [
"\\"-9223372036854775808\\"", -9223372036854776000,
"\\"-9007199254740991\\"", -9007199254740991,
null, null,
"\\"9007199254740991\\"", 9007199254740991,
"\\"9223372036854775807\\"", 9223372036854776000,
], ],
}, },
Object { Object {
...@@ -240,11 +240,11 @@ Object { ...@@ -240,11 +240,11 @@ Object {
"name": "uint64_values", "name": "uint64_values",
"type": "number", "type": "number",
"values": Array [ "values": Array [
"\\"0\\"", 0,
"\\"0\\"", 0,
"\\"1\\"", 1,
"\\"9007199254740991\\"", 9007199254740991,
"\\"18446744073709551615\\"", 18446744073709552000,
], ],
}, },
Object { Object {
...@@ -253,11 +253,11 @@ Object { ...@@ -253,11 +253,11 @@ Object {
"name": "nullable_uint64_values", "name": "nullable_uint64_values",
"type": "number", "type": "number",
"values": Array [ "values": Array [
"\\"0\\"", 0,
"\\"0\\"", 0,
null, null,
"\\"9007199254740991\\"", 9007199254740991,
"\\"18446744073709551615\\"", 18446744073709552000,
], ],
}, },
Object { Object {
...@@ -354,6 +354,19 @@ Object { ...@@ -354,6 +354,19 @@ Object {
Object { Object {
"config": Object {}, "config": Object {},
"labels": undefined, "labels": undefined,
"name": "timestamps",
"type": "time",
"values": Array [
0,
1568039445000,
1568039450000,
9007199254.740992,
9223372036854.775,
],
},
Object {
"config": Object {},
"labels": undefined,
"name": "nullable_timestamps", "name": "nullable_timestamps",
"type": "time", "type": "time",
"values": Array [ "values": Array [
...@@ -366,13 +379,9 @@ Object { ...@@ -366,13 +379,9 @@ Object {
}, },
], ],
"meta": Object { "meta": Object {
"limit": 4242, "custom": Object {
"searchWords": Array [ "Hi": "there",
"Grafana", },
"❤️",
" 🦥 ",
"test",
],
}, },
"name": "many_types", "name": "many_types",
"refId": "A", "refId": "A",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment