Feat: More robust csv support (#16170)

* stream csv

* merged master

* merged master

* fix test failures

* add csv files

* update boolean parsing

* add toCSV

* add toCSV

* add toCSV

* add streaming datasource

* set time range

* streaming to a graph

* streaming datasource

* streaming table

* add server to the streaming

* remove react streaming

packages/grafana-ui/src/components/Table/TableInputCSV.story.tsx

@@ -15,9 +15,9 @@ TableInputStories.add('default', () => {
     <div style={{ width: '90%', height: '90vh' }}>
     <div style={{ width: '90%', height: '90vh' }}>
       <TableInputCSV
       <TableInputCSV
         text={'a,b,c\n1,2,3'}
         text={'a,b,c\n1,2,3'}
-        onTableParsed={(table: SeriesData, text: string) => {
-          console.log('Table', table, text);
-          action('Table')(table, text);
+        onSeriesParsed={(data: SeriesData[], text: string) => {
+          console.log('Data', data, text);
+          action('Data')(data, text);
         }}
         }}
       />
       />
     </div>
     </div>

packages/grafana-ui/src/components/Table/TableInputCSV.test.tsx

@@ -10,7 +10,7 @@ describe('TableInputCSV', () => {
       .create(
       .create(
         <TableInputCSV
         <TableInputCSV
           text={'a,b,c\n1,2,3'}
           text={'a,b,c\n1,2,3'}
-          onTableParsed={(table: SeriesData, text: string) => {
+          onSeriesParsed={(data: SeriesData[], text: string) => {
             // console.log('Table:', table, 'from:', text);
             // console.log('Table:', table, 'from:', text);
           }}
           }}
         />
         />

packages/grafana-ui/src/components/Table/TableInputCSV.tsx

 import React from 'react';
 import React from 'react';
 import debounce from 'lodash/debounce';
 import debounce from 'lodash/debounce';
-import { parseCSV, TableParseOptions, TableParseDetails } from '../../utils/processSeriesData';
 import { SeriesData } from '../../types/data';
 import { SeriesData } from '../../types/data';
 import { AutoSizer } from 'react-virtualized';
 import { AutoSizer } from 'react-virtualized';
+import { CSVConfig, readCSV } from '../../utils/csv';
 
 
 interface Props {
 interface Props {
-  options?: TableParseOptions;
+  config?: CSVConfig;
   text: string;
   text: string;
-  onTableParsed: (table: SeriesData, text: string) => void;
+  onSeriesParsed: (data: SeriesData[], text: string) => void;
 }
 }
 
 
 interface State {
 interface State {
   text: string;
   text: string;
-  table: SeriesData;
-  details: TableParseDetails;
+  data: SeriesData[];
 }
 }
 
 
 /**
 /**
@@ -23,68 +22,61 @@ class TableInputCSV extends React.PureComponent<Props, State> {
   constructor(props: Props) {
   constructor(props: Props) {
     super(props);
     super(props);
 
 
-    // Shoud this happen in onComponentMounted?
-    const { text, options, onTableParsed } = props;
-    const details = {};
-    const table = parseCSV(text, options, details);
+    const { text, config } = props;
     this.state = {
     this.state = {
       text,
       text,
-      table,
-      details,
+      data: readCSV(text, { config }),
     };
     };
-    onTableParsed(table, text);
   }
   }
 
 
   readCSV = debounce(() => {
   readCSV = debounce(() => {
-    const details = {};
-    const table = parseCSV(this.state.text, this.props.options, details);
-    this.setState({ table, details });
+    const { config } = this.props;
+    const { text } = this.state;
+
+    this.setState({ data: readCSV(text, { config }) });
   }, 150);
   }, 150);
 
 
   componentDidUpdate(prevProps: Props, prevState: State) {
   componentDidUpdate(prevProps: Props, prevState: State) {
     const { text } = this.state;
     const { text } = this.state;
-    if (text !== prevState.text || this.props.options !== prevProps.options) {
+
+    if (text !== prevState.text || this.props.config !== prevProps.config) {
       this.readCSV();
       this.readCSV();
     }
     }
+
     // If the props text has changed, replace our local version
     // If the props text has changed, replace our local version
     if (this.props.text !== prevProps.text && this.props.text !== text) {
     if (this.props.text !== prevProps.text && this.props.text !== text) {
       this.setState({ text: this.props.text });
       this.setState({ text: this.props.text });
     }
     }
 
 
-    if (this.state.table !== prevState.table) {
-      this.props.onTableParsed(this.state.table, this.state.text);
+    if (this.state.data !== prevState.data) {
+      this.props.onSeriesParsed(this.state.data, this.state.text);
     }
     }
   }
   }
 
 
-  onFooterClicked = (event: any) => {
-    console.log('Errors', this.state);
-    const message = this.state.details
-      .errors!.map(err => {
-        return err.message;
-      })
-      .join('\n');
-    alert('CSV Parsing Errors:\n' + message);
-  };
-
   onTextChange = (event: any) => {
   onTextChange = (event: any) => {
     this.setState({ text: event.target.value });
     this.setState({ text: event.target.value });
   };
   };
 
 
   render() {
   render() {
-    const { table, details } = this.state;
-
-    const hasErrors = details.errors && details.errors.length > 0;
-    const footerClassNames = hasErrors ? 'gf-table-input-csv-err' : '';
+    const { data } = this.state;
 
 
     return (
     return (
       <AutoSizer>
       <AutoSizer>
         {({ height, width }) => (
         {({ height, width }) => (
           <div className="gf-table-input-csv" style={{ width, height }}>
           <div className="gf-table-input-csv" style={{ width, height }}>
             <textarea placeholder="Enter CSV here..." value={this.state.text} onChange={this.onTextChange} />
             <textarea placeholder="Enter CSV here..." value={this.state.text} onChange={this.onTextChange} />
-            <footer onClick={this.onFooterClicked} className={footerClassNames}>
-              Rows:{table.rows.length}, Columns:{table.fields.length} &nbsp;
-              {hasErrors ? <i className="fa fa-exclamation-triangle" /> : <i className="fa fa-check-circle" />}
-            </footer>
+            {data && (
+              <footer>
+                {data.map((series, index) => {
+                  return (
+                    <span key={index}>
+                      Rows:{series.rows.length}, Columns:{series.fields.length} &nbsp;
+                      <i className="fa fa-check-circle" />
+                    </span>
+                  );
+                })}
+              </footer>
+            )}
           </div>
           </div>
         )}
         )}
       </AutoSizer>
       </AutoSizer>

packages/grafana-ui/src/components/Table/_TableInputCSV.scss

@@ -18,7 +18,3 @@
   font-size: 80%;
   font-size: 80%;
   cursor: pointer;
   cursor: pointer;
 }
 }
-
-.gf-table-input-csv footer.gf-table-input-csv-err {
-  background: yellow;
-}

packages/grafana-ui/src/utils/__snapshots__/processSeriesData.test.ts.snap → packages/grafana-ui/src/utils/__snapshots__/csv.test.ts.snap

 // Jest Snapshot v1, https://goo.gl/fbAQLP
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 
-exports[`processSeriesData basic processing should generate a header and fix widths 1`] = `
+exports[`read csv should get X and y 1`] = `
 Object {
 Object {
   "fields": Array [
   "fields": Array [
     Object {
     Object {
-      "name": "Field 1",
+      "name": "Column 1",
+      "type": "number",
     },
     },
     Object {
     Object {
-      "name": "Field 2",
+      "name": "Column 2",
+      "type": "number",
     },
     },
     Object {
     Object {
-      "name": "Field 3",
+      "name": "Column 3",
+      "type": "number",
+    },
+    Object {
+      "name": "Field 4",
+      "type": "number",
     },
     },
   ],
   ],
   "rows": Array [
   "rows": Array [
     Array [
     Array [
-      1,
-      null,
-      null,
-    ],
-    Array [
       2,
       2,
       3,
       3,
       4,
       4,
+      null,
     ],
     ],
     Array [
     Array [
       5,
       5,
       6,
       6,
       null,
       null,
+      null,
+    ],
+    Array [
+      null,
+      null,
+      null,
+      7,
     ],
     ],
   ],
   ],
 }
 }
 `;
 `;
 
 
-exports[`processSeriesData basic processing should read header and two rows 1`] = `
+exports[`read csv should read csv from local file system 1`] = `
 Object {
 Object {
   "fields": Array [
   "fields": Array [
     Object {
     Object {
       "name": "a",
       "name": "a",
+      "type": "number",
     },
     },
     Object {
     Object {
       "name": "b",
       "name": "b",
+      "type": "number",
     },
     },
     Object {
     Object {
       "name": "c",
       "name": "c",
+      "type": "number",
     },
     },
   ],
   ],
   "rows": Array [
   "rows": Array [
     Array [
     Array [
-      1,
-      2,
-      3,
+      10,
+      20,
+      30,
     ],
     ],
     Array [
     Array [
-      4,
-      5,
-      6,
+      40,
+      50,
+      60,
+    ],
+  ],
+}
+`;
+
+exports[`read csv should read csv with headers 1`] = `
+Object {
+  "fields": Array [
+    Object {
+      "name": "a",
+      "type": "number",
+      "unit": "ms",
+    },
+    Object {
+      "name": "b",
+      "type": "string",
+      "unit": "lengthm",
+    },
+    Object {
+      "name": "c",
+      "type": "boolean",
+      "unit": "s",
+    },
+  ],
+  "rows": Array [
+    Array [
+      10,
+      "20",
+      true,
+    ],
+    Array [
+      40,
+      "50",
+      false,
+    ],
+    Array [
+      40,
+      "500",
+      false,
+    ],
+    Array [
+      40,
+      "50",
+      true,
     ],
     ],
   ],
   ],
 }
 }

packages/grafana-ui/src/utils/csv.test.ts

+import { readCSV, toCSV, CSVHeaderStyle } from './csv';
+
+// Test with local CSV files
+const fs = require('fs');
+
+describe('read csv', () => {
+  it('should get X and y', () => {
+    const text = ',1\n2,3,4\n5,6\n,,,7';
+    const data = readCSV(text);
+    expect(data.length).toBe(1);
+
+    const series = data[0];
+    expect(series.fields.length).toBe(4);
+    expect(series.rows.length).toBe(3);
+
+    // Make sure everythign it padded properly
+    for (const row of series.rows) {
+      expect(row.length).toBe(series.fields.length);
+    }
+
+    expect(series).toMatchSnapshot();
+  });
+
+  it('should read csv from local file system', () => {
+    const path = __dirname + '/testdata/simple.csv';
+    expect(fs.existsSync(path)).toBeTruthy();
+
+    const csv = fs.readFileSync(path, 'utf8');
+    const data = readCSV(csv);
+    expect(data.length).toBe(1);
+    expect(data[0]).toMatchSnapshot();
+  });
+
+  it('should read csv with headers', () => {
+    const path = __dirname + '/testdata/withHeaders.csv';
+    expect(fs.existsSync(path)).toBeTruthy();
+
+    const csv = fs.readFileSync(path, 'utf8');
+    const data = readCSV(csv);
+    expect(data.length).toBe(1);
+    expect(data[0]).toMatchSnapshot();
+  });
+});
+
+function norm(csv: string): string {
+  return csv.trim().replace(/[\r]/g, '');
+}
+
+describe('write csv', () => {
+  it('should write the same CSV that we read', () => {
+    const path = __dirname + '/testdata/roundtrip.csv';
+    const csv = fs.readFileSync(path, 'utf8');
+    const data = readCSV(csv);
+    const out = toCSV(data, { headerStyle: CSVHeaderStyle.full });
+    expect(data.length).toBe(1);
+    expect(data[0].fields.length).toBe(3);
+    expect(norm(out)).toBe(norm(csv));
+
+    // Keep the name even without special formatting
+    const again = readCSV(out);
+    const shorter = toCSV(again, { headerStyle: CSVHeaderStyle.name });
+
+    const f = readCSV(shorter);
+    const fields = f[0].fields;
+    expect(fields.length).toBe(3);
+    expect(fields.map(f => f.name).join(',')).toEqual('a,b,c'); // the names
+  });
+});

packages/grafana-ui/src/utils/index.ts

@@ -4,6 +4,7 @@ export * from './colors';
 export * from './namedColorsPalette';
 export * from './namedColorsPalette';
 export * from './thresholds';
 export * from './thresholds';
 export * from './string';
 export * from './string';
+export * from './csv';
 export * from './statsCalculator';
 export * from './statsCalculator';
 export * from './displayValue';
 export * from './displayValue';
 export * from './deprecationWarning';
 export * from './deprecationWarning';

packages/grafana-ui/src/utils/processSeriesData.test.ts

-import { parseCSV, toSeriesData, guessFieldTypes, guessFieldTypeFromValue } from './processSeriesData';
+import { toSeriesData, guessFieldTypes, guessFieldTypeFromValue } from './processSeriesData';
 import { FieldType } from '../types/data';
 import { FieldType } from '../types/data';
 import moment from 'moment';
 import moment from 'moment';
 
 
-describe('processSeriesData', () => {
-  describe('basic processing', () => {
-    it('should read header and two rows', () => {
-      const text = 'a,b,c\n1,2,3\n4,5,6';
-      expect(parseCSV(text)).toMatchSnapshot();
-    });
-
-    it('should generate a header and fix widths', () => {
-      const text = '1\n2,3,4\n5,6';
-      const series = parseCSV(text, {
-        headerIsFirstLine: false,
-      });
-      expect(series.rows.length).toBe(3);
-
-      expect(series).toMatchSnapshot();
-    });
-  });
-});
-
 describe('toSeriesData', () => {
 describe('toSeriesData', () => {
   it('converts timeseries to series', () => {
   it('converts timeseries to series', () => {
     const input1 = {
     const input1 = {

packages/grafana-ui/src/utils/processSeriesData.ts

@@ -4,135 +4,9 @@ import isString from 'lodash/isString';
 import isBoolean from 'lodash/isBoolean';
 import isBoolean from 'lodash/isBoolean';
 import moment from 'moment';
 import moment from 'moment';
 
 
-import Papa, { ParseError, ParseMeta } from 'papaparse';
-
 // Types
 // Types
 import { SeriesData, Field, TimeSeries, FieldType, TableData } from '../types/index';
 import { SeriesData, Field, TimeSeries, FieldType, TableData } from '../types/index';
 
 
-// Subset of all parse options
-export interface TableParseOptions {
-  headerIsFirstLine?: boolean; // Not a papa-parse option
-  delimiter?: string; // default: ","
-  newline?: string; // default: "\r\n"
-  quoteChar?: string; // default: '"'
-  encoding?: string; // default: ""
-  comments?: boolean | string; // default: false
-}
-
-export interface TableParseDetails {
-  meta?: ParseMeta;
-  errors?: ParseError[];
-}
-
-/**
- * This makes sure the header and all rows have equal length.
- *
- * @param series (immutable)
- * @returns a series that has equal length rows, or the same
- * series if no changes were needed
- */
-export function matchRowSizes(series: SeriesData): SeriesData {
-  const { rows } = series;
-  let { fields } = series;
-
-  let sameSize = true;
-  let size = fields.length;
-  rows.forEach(row => {
-    if (size !== row.length) {
-      sameSize = false;
-      size = Math.max(size, row.length);
-    }
-  });
-  if (sameSize) {
-    return series;
-  }
-
-  // Pad Fields
-  if (size !== fields.length) {
-    const diff = size - fields.length;
-    fields = [...fields];
-    for (let i = 0; i < diff; i++) {
-      fields.push({
-        name: 'Field ' + (fields.length + 1),
-      });
-    }
-  }
-
-  // Pad Rows
-  const fixedRows: any[] = [];
-  rows.forEach(row => {
-    const diff = size - row.length;
-    if (diff > 0) {
-      row = [...row];
-      for (let i = 0; i < diff; i++) {
-        row.push(null);
-      }
-    }
-    fixedRows.push(row);
-  });
-
-  return {
-    fields,
-    rows: fixedRows,
-  };
-}
-
-function makeFields(values: any[]): Field[] {
-  return values.map((value, index) => {
-    if (!value) {
-      value = 'Field ' + (index + 1);
-    }
-    return {
-      name: value.toString().trim(),
-    };
-  });
-}
-
-/**
- * Convert CSV text into a valid SeriesData object
- *
- * @param text
- * @param options
- * @param details, if exists the result will be filled with debugging details
- */
-export function parseCSV(text: string, options?: TableParseOptions, details?: TableParseDetails): SeriesData {
-  const results = Papa.parse(text, { ...options, dynamicTyping: true, skipEmptyLines: true });
-  const { data, meta, errors } = results;
-
-  // Fill the parse details for debugging
-  if (details) {
-    details.errors = errors;
-    details.meta = meta;
-  }
-
-  if (!data || data.length < 1) {
-    // Show a more reasonable warning on empty input text
-    if (details && !text) {
-      errors.length = 0;
-      errors.push({
-        code: 'empty',
-        message: 'Empty input text',
-        type: 'warning',
-        row: 0,
-      });
-      details.errors = errors;
-    }
-    return {
-      fields: [],
-      rows: [],
-    };
-  }
-
-  // Assume the first line is the header unless the config says its not
-  const headerIsNotFirstLine = options && options.headerIsFirstLine === false;
-  const header = headerIsNotFirstLine ? [] : results.data.shift();
-
-  return matchRowSizes({
-    fields: makeFields(header),
-    rows: results.data,
-  });
-}
-
 function convertTableToSeriesData(table: TableData): SeriesData {
 function convertTableToSeriesData(table: TableData): SeriesData {
   return {
   return {
     // rename the 'text' to 'name' field
     // rename the 'text' to 'name' field

packages/grafana-ui/src/utils/statsCalculator.test.ts

-import { parseCSV } from './processSeriesData';
 import { getStatsCalculators, StatID, calculateStats } from './statsCalculator';
 import { getStatsCalculators, StatID, calculateStats } from './statsCalculator';
 
 
 import _ from 'lodash';
 import _ from 'lodash';
 
 
 describe('Stats Calculators', () => {
 describe('Stats Calculators', () => {
-  const basicTable = parseCSV('a,b,c\n10,20,30\n20,30,40');
+  const basicTable = {
+    fields: [{ name: 'a' }, { name: 'b' }, { name: 'c' }],
+    rows: [[10, 20, 30], [20, 30, 40]],
+  };
 
 
   it('should load all standard stats', () => {
   it('should load all standard stats', () => {
     const names = [
     const names = [

packages/grafana-ui/src/utils/testdata/roundtrip.csv

+#name#a,b,c
+#type#number,string,boolean
+#unit#ms,,s
+10,AA,true
+20,XX,false
+30,YY,false
+40,ZZ,true
+50,"X,Y",true
+60,"X
+Y",true
+70,BB,false
+

packages/grafana-ui/src/utils/testdata/simple.csv

+a,b,c
+10,20,30
+40,50,60

packages/grafana-ui/src/utils/testdata/withHeaders.csv

+#name#a,b,c
+#unit#ms,lengthm,s
+#type#number,string,boolean
+10,20,True
+40,50,FALSE
+"40","500",0
+40,50,1