DataFrame: round trip metadata to arrow Table (#21277)

packages/grafana-data/rollup.config.ts

@@ -20,7 +20,7 @@ const buildCjsPackage = ({ env }) => {
         globals: {},
       },
     ],
-    external: ['lodash'], // Use Lodash from grafana
+    external: ['lodash', 'apache-arrow'], // Use Lodash & arrow from grafana
     plugins: [
       commonjs({
         include: /node_modules/,

packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.test.ts → packages/grafana-data/src/dataframe/ArrowDataFrame.test.ts

-import { resultsToDataFrames } from './ArrowDataFrame';
-import { toDataFrameDTO } from '../processDataFrame';
+import { resultsToDataFrames, grafanaDataFrameToArrowTable, arrowTableToDataFrame } from './ArrowDataFrame';
+import { toDataFrameDTO, toDataFrame } from './processDataFrame';
+import { FieldType } from '../types';
 
 /* tslint:disable */
 const resp = {
@@ -33,3 +34,29 @@ describe('GEL Utils', () => {
     expect(norm).toMatchSnapshot();
   });
 });
+
+describe('Read/Write arrow Table to DataFrame', () => {
+  test('should parse output with dataframe', () => {
+    const frame = toDataFrame({
+      name: 'Hello',
+      refId: 'XYZ',
+      meta: {
+        aaa: 'xyz',
+        anything: 'xxx',
+      },
+      fields: [
+        { name: 'time', config: {}, type: FieldType.time, values: [1, 2, 3] },
+        { name: 'value', config: { min: 0, max: 50, unit: 'somthing' }, type: FieldType.number, values: [1, 2, 3] },
+        { name: 'str', config: {}, type: FieldType.string, values: ['a', 'b', 'c'] },
+      ],
+    });
+
+    const table = grafanaDataFrameToArrowTable(frame);
+    expect(table.length).toEqual(frame.length);
+
+    // Now back to DataFrame
+    const before = JSON.stringify(toDataFrameDTO(frame), null, 2);
+    const after = JSON.stringify(toDataFrameDTO(arrowTableToDataFrame(table)), null, 2);
+    expect(after).toEqual(before);
+  });
+});

packages/grafana-data/src/dataframe/arrow/ArrowDataFrame.ts → packages/grafana-data/src/dataframe/ArrowDataFrame.ts

-import { DataFrame, FieldType, Field, Vector } from '../../types';
-import { Table, ArrowType } from 'apache-arrow';
+import { DataFrame, FieldType, Field, Vector, FieldConfig, Labels } from '../types';
+import {
+  Table,
+  ArrowType,
+  Builder,
+  Vector as ArrowVector,
+  Float64,
+  DataType,
+  Utf8,
+  TimestampMillisecond,
+  Bool,
+  Column,
+} from 'apache-arrow';
 
 export interface ArrowDataFrame extends DataFrame {
   table: Table;
@@ -41,29 +52,92 @@ export function arrowTableToDataFrame(table: Table): ArrowDataFrame {
           type = FieldType.time;
           break;
         }
+        case ArrowType.Utf8: {
+          type = FieldType.string;
+          break;
+        }
         default:
           console.log('UNKNOWN Type:', schema);
       }
-      // console.log(' field>', schema.metadata);
+      const labelsJson = col.metadata.get('labels');
+      const configJson = col.metadata.get('config');
+
+      let config: FieldConfig = {};
+      let labels: Labels | undefined = undefined;
+      if (labelsJson) {
+        labels = JSON.parse(labelsJson);
+      }
+      if (configJson) {
+        config = JSON.parse(configJson);
+      }
 
       fields.push({
         name: col.name,
         type,
-        config: {}, // TODO, pull from metadata
+        config,
         values,
+        labels,
       });
     }
   }
   const meta = table.schema.metadata;
+  const metaJson = valueOrUndefined(meta.get('meta'));
   return {
     fields,
     length: table.length,
     refId: valueOrUndefined(meta.get('refId')),
     name: valueOrUndefined(meta.get('name')),
+    meta: metaJson ? JSON.parse(metaJson) : undefined,
     table,
   };
 }
 
+function toArrowVector(field: Field): ArrowVector {
+  // OR: Float64Vector.from([1, 2, 3]));
+
+  let type: DataType;
+  if (field.type === FieldType.number) {
+    type = new Float64();
+  } else if (field.type === FieldType.time) {
+    type = new TimestampMillisecond();
+  } else if (field.type === FieldType.boolean) {
+    type = new Bool();
+  } else if (field.type === FieldType.string) {
+    type = new Utf8();
+  } else {
+    type = new Utf8();
+  }
+  const builder = Builder.new({ type, nullValues: [null] });
+  field.values.toArray().forEach(builder.append.bind(builder));
+  return builder.finish().toVector();
+}
+
+export function grafanaDataFrameToArrowTable(data: DataFrame): Table {
+  const table = Table.new(
+    data.fields.map(field => {
+      const column = Column.new(field.name, toArrowVector(field));
+      if (field.labels) {
+        column.metadata.set('labels', JSON.stringify(field.labels));
+      }
+      if (field.config) {
+        column.metadata.set('config', JSON.stringify(field.config));
+      }
+      return column;
+    })
+  );
+  const metadata = table.schema.metadata;
+  if (data.name) {
+    metadata.set('name', data.name);
+  }
+  if (data.refId) {
+    metadata.set('refId', data.refId);
+  }
+  if (data.meta) {
+    metadata.set('meta', JSON.stringify(data.meta));
+  }
+  return table;
+}
+
 export function resultsToDataFrames(rsp: any): DataFrame[] {
   const frames: DataFrame[] = [];
   for (const res of Object.values(rsp.results)) {

packages/grafana-data/src/dataframe/arrow/__snapshots__/ArrowDataFrame.test.ts.snap → packages/grafana-data/src/dataframe/__snapshots__/ArrowDataFrame.test.ts.snap

@@ -9,33 +9,20 @@ Array [
         "labels": undefined,
         "name": "Time",
         "type": "time",
-        "values": Int32Array [
-          882710016,
-          365389179,
-          1587742720,
-          365389180,
-          -2002191872,
-          365389181,
-          -1297159168,
-          365389182,
-          -592126464,
-          365389183,
-          112906240,
-          365389185,
-          817938944,
-          365389186,
-          1522971648,
-          365389187,
-          -2066962944,
-          365389188,
-          -1361930240,
-          365389189,
-          -656897536,
-          365389190,
-          48135168,
-          365389192,
-          753167872,
-          365389193,
+        "values": Array [
+          1569334575000,
+          1569334580000,
+          1569334585000,
+          1569334590000,
+          1569334595000,
+          1569334600000,
+          1569334605000,
+          1569334610000,
+          1569334615000,
+          1569334620000,
+          1569334625000,
+          1569334630000,
+          1569334635000,
         ],
       },
       Object {
@@ -43,7 +30,7 @@ Array [
         "labels": undefined,
         "name": "",
         "type": "number",
-        "values": Float64Array [
+        "values": Array [
           3,
           3,
           3,
@@ -71,33 +58,20 @@ Array [
         "labels": undefined,
         "name": "Time",
         "type": "time",
-        "values": Int32Array [
-          882710016,
-          365389179,
-          1587742720,
-          365389180,
-          -2002191872,
-          365389181,
-          -1297159168,
-          365389182,
-          -592126464,
-          365389183,
-          112906240,
-          365389185,
-          817938944,
-          365389186,
-          1522971648,
-          365389187,
-          -2066962944,
-          365389188,
-          -1361930240,
-          365389189,
-          -656897536,
-          365389190,
-          48135168,
-          365389192,
-          753167872,
-          365389193,
+        "values": Array [
+          1569334575000,
+          1569334580000,
+          1569334585000,
+          1569334590000,
+          1569334595000,
+          1569334600000,
+          1569334605000,
+          1569334610000,
+          1569334615000,
+          1569334620000,
+          1569334625000,
+          1569334630000,
+          1569334635000,
         ],
       },
       Object {
@@ -105,7 +79,7 @@ Array [
         "labels": undefined,
         "name": "GB-series",
         "type": "number",
-        "values": Float64Array [
+        "values": Array [
           0,
           0,
           0,

packages/grafana-data/src/dataframe/index.ts

@@ -4,10 +4,4 @@ export * from './CircularDataFrame';
 export * from './MutableDataFrame';
 export * from './processDataFrame';
 export * from './dimensions';
-
-// NOTE: We can not export arrow in the global scope because it will crash phantomjs
-// In core, this is loaded async.  In plugins you can import using:
-//
-// import { resultsToDataFrames } from '@grafana/data/dataframe/arrow/ArrowDataFrame'
-//
-// export * from './arrow/ArrowDataFrame';
+export * from './ArrowDataFrame';

packages/grafana-data/src/dataframe/processDataFrame.ts

@@ -442,11 +442,22 @@ export function getDataFrameRow(data: DataFrame, row: number): any[] {
  */
 export function toDataFrameDTO(data: DataFrame): DataFrameDTO {
   const fields: FieldDTO[] = data.fields.map(f => {
+    let values = f.values.toArray();
+    if (!Array.isArray(values)) {
+      // Apache arrow will pack objects into typed arrays
+      // Float64Array, etc
+      // TODO: Float64Array could be used directly
+      values = [];
+      for (let i = 0; i < f.values.length; i++) {
+        values.push(f.values.get(i));
+      }
+    }
+
     return {
       name: f.name,
       type: f.type,
       config: f.config,
-      values: f.values.toArray(),
+      values,
       labels: f.labels,
     };
   });

public/app/features/expressions/ExpressionDatasource.ts

@@ -69,7 +69,7 @@ export class ExpressionDatasourceApi extends DataSourceApi<ExpressionQuery> {
    */
   async toDataQueryResponse(rsp: any): Promise<DataQueryResponse> {
     const { resultsToDataFrames } = await import(
-      /* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/arrow/ArrowDataFrame'
+      /* webpackChunkName: "apache-arrow-util" */ '@grafana/data/src/dataframe/ArrowDataFrame'
     );
     return { data: resultsToDataFrames(rsp) };
   }