Merge pull request #16112 from ryantxu/show-all-columns

show all numeric columns in singlestats/graph2

packages/grafana-ui/src/utils/processTableData.test.ts

-import { parseCSV, toTableData } from './processTableData';
+import { parseCSV, toTableData, guessColumnTypes, guessColumnTypeFromValue } from './processTableData';
+import { ColumnType } from '../types/data';
+import moment from 'moment';
 
 describe('processTableData', () => {
   describe('basic processing', () => {
@@ -20,23 +22,23 @@ describe('processTableData', () => {
 });
 
 describe('toTableData', () => {
-  it('converts timeseries to table skipping nulls', () => {
+  it('converts timeseries to table ', () => {
     const input1 = {
       target: 'Field Name',
       datapoints: [[100, 1], [200, 2]],
     };
+    let table = toTableData(input1);
+    expect(table.columns[0].text).toBe(input1.target);
+    expect(table.rows).toBe(input1.datapoints);
+
+    // Should fill a default name if target is empty
     const input2 = {
       // without target
       target: '',
       datapoints: [[100, 1], [200, 2]],
     };
-    const data = toTableData([null, input1, input2, null, null]);
-    expect(data.length).toBe(2);
-    expect(data[0].columns[0].text).toBe(input1.target);
-    expect(data[0].rows).toBe(input1.datapoints);
-
-    // Default name
-    expect(data[1].columns[0].text).toEqual('Value');
+    table = toTableData(input2);
+    expect(table.columns[0].text).toEqual('Value');
   });
 
   it('keeps tableData unchanged', () => {
@@ -44,15 +46,39 @@ describe('toTableData', () => {
       columns: [{ text: 'A' }, { text: 'B' }, { text: 'C' }],
       rows: [[100, 'A', 1], [200, 'B', 2], [300, 'C', 3]],
     };
-    const data = toTableData([null, input, null, null]);
-    expect(data.length).toBe(1);
-    expect(data[0]).toBe(input);
+    const table = toTableData(input);
+    expect(table).toBe(input);
+  });
+
+  it('Guess Colum Types from value', () => {
+    expect(guessColumnTypeFromValue(1)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue(1.234)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue(3.125e7)).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue(true)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue(false)).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue(new Date())).toBe(ColumnType.time);
+    expect(guessColumnTypeFromValue(moment())).toBe(ColumnType.time);
   });
 
-  it('supports null values OK', () => {
-    expect(toTableData([null, null, null, null])).toEqual([]);
-    expect(toTableData(undefined)).toEqual([]);
-    expect(toTableData((null as unknown) as any[])).toEqual([]);
-    expect(toTableData([])).toEqual([]);
+  it('Guess Colum Types from strings', () => {
+    expect(guessColumnTypeFromValue('1')).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue('1.234')).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue('3.125e7')).toBe(ColumnType.number);
+    expect(guessColumnTypeFromValue('True')).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue('FALSE')).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue('true')).toBe(ColumnType.boolean);
+    expect(guessColumnTypeFromValue('xxxx')).toBe(ColumnType.string);
+  });
+
+  it('Guess Colum Types from table', () => {
+    const table = {
+      columns: [{ text: 'A (number)' }, { text: 'B (strings)' }, { text: 'C (nulls)' }, { text: 'Time' }],
+      rows: [[123, null, null, '2000'], [null, 'Hello', null, 'XXX']],
+    };
+    const norm = guessColumnTypes(table);
+    expect(norm.columns[0].type).toBe(ColumnType.number);
+    expect(norm.columns[1].type).toBe(ColumnType.string);
+    expect(norm.columns[2].type).toBeUndefined();
+    expect(norm.columns[3].type).toBe(ColumnType.time); // based on name
   });
 });

packages/grafana-ui/src/utils/processTableData.ts

 // Libraries
 import isNumber from 'lodash/isNumber';
+import isString from 'lodash/isString';
+import isBoolean from 'lodash/isBoolean';
+import moment from 'moment';
+
 import Papa, { ParseError, ParseMeta } from 'papaparse';
 
 // Types
@@ -147,26 +151,118 @@ function convertTimeSeriesToTableData(timeSeries: TimeSeries): TableData {
   };
 }
 
-export const isTableData = (data: any): data is TableData => data && data.hasOwnProperty('columns');
+export const getFirstTimeColumn = (table: TableData): number => {
+  const { columns } = table;
+  for (let i = 0; i < columns.length; i++) {
+    if (columns[i].type === ColumnType.time) {
+      return i;
+    }
+  }
+  return -1;
+};
+
+// PapaParse Dynamic Typing regex:
+// https://github.com/mholt/PapaParse/blob/master/papaparse.js#L998
+const NUMBER = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
 
-export const toTableData = (results?: any[]): TableData[] => {
-  if (!results) {
-    return [];
+/**
+ * Given a value this will guess the best column type
+ *
+ * TODO: better Date/Time support!  Look for standard date strings?
+ */
+export function guessColumnTypeFromValue(v: any): ColumnType {
+  if (isNumber(v)) {
+    return ColumnType.number;
   }
 
-  return results
-    .filter(d => !!d)
-    .map(data => {
-      if (data.hasOwnProperty('columns')) {
-        return data as TableData;
-      }
-      if (data.hasOwnProperty('datapoints')) {
-        return convertTimeSeriesToTableData(data);
-      }
-      // TODO, try to convert JSON to table?
-      console.warn('Can not convert', data);
-      throw new Error('Unsupported data format');
-    });
+  if (isString(v)) {
+    if (NUMBER.test(v)) {
+      return ColumnType.number;
+    }
+
+    if (v === 'true' || v === 'TRUE' || v === 'True' || v === 'false' || v === 'FALSE' || v === 'False') {
+      return ColumnType.boolean;
+    }
+
+    return ColumnType.string;
+  }
+
+  if (isBoolean(v)) {
+    return ColumnType.boolean;
+  }
+
+  if (v instanceof Date || v instanceof moment) {
+    return ColumnType.time;
+  }
+
+  return ColumnType.other;
+}
+
+/**
+ * Looks at the data to guess the column type.  This ignores any existing setting
+ */
+function guessColumnTypeFromTable(table: TableData, index: number): ColumnType | undefined {
+  const column = table.columns[index];
+
+  // 1. Use the column name to guess
+  if (column.text) {
+    const name = column.text.toLowerCase();
+    if (name === 'date' || name === 'time') {
+      return ColumnType.time;
+    }
+  }
+
+  // 2. Check the first non-null value
+  for (let i = 0; i < table.rows.length; i++) {
+    const v = table.rows[i][index];
+    if (v !== null) {
+      return guessColumnTypeFromValue(v);
+    }
+  }
+
+  // Could not find anything
+  return undefined;
+}
+
+/**
+ * @returns a table Returns a copy of the table with the best guess for each column type
+ * If the table already has column types defined, they will be used
+ */
+export const guessColumnTypes = (table: TableData): TableData => {
+  for (let i = 0; i < table.columns.length; i++) {
+    if (!table.columns[i].type) {
+      // Somethign is missing a type return a modified copy
+      return {
+        ...table,
+        columns: table.columns.map((column, index) => {
+          if (column.type) {
+            return column;
+          }
+          // Replace it with a calculated version
+          return {
+            ...column,
+            type: guessColumnTypeFromTable(table, index),
+          };
+        }),
+      };
+    }
+  }
+  // No changes necessary
+  return table;
+};
+
+export const isTableData = (data: any): data is TableData => data && data.hasOwnProperty('columns');
+
+export const toTableData = (data: any): TableData => {
+  if (data.hasOwnProperty('columns')) {
+    return data as TableData;
+  }
+  if (data.hasOwnProperty('datapoints')) {
+    return convertTimeSeriesToTableData(data);
+  }
+  // TODO, try to convert JSON/Array to table?
+  console.warn('Can not convert', data);
+  throw new Error('Unsupported data format');
 };
 
 export function sortTableData(data: TableData, sortIndex?: number, reverse = false): TableData {

public/app/features/dashboard/dashgrid/DataPanel.test.tsx

+// Library
+import React from 'react';
+
+import { DataPanel, getProcessedTableData } from './DataPanel';
+
+describe('DataPanel', () => {
+  let dataPanel: DataPanel;
+
+  beforeEach(() => {
+    dataPanel = new DataPanel({
+      queries: [],
+      panelId: 1,
+      widthPixels: 100,
+      refreshCounter: 1,
+      datasource: 'xxx',
+      children: r => {
+        return <div>hello</div>;
+      },
+      onError: (message, error) => {},
+    });
+  });
+
+  it('starts with unloaded state', () => {
+    expect(dataPanel.state.isFirstLoad).toBe(true);
+  });
+
+  it('converts timeseries to table skipping nulls', () => {
+    const input1 = {
+      target: 'Field Name',
+      datapoints: [[100, 1], [200, 2]],
+    };
+    const input2 = {
+      // without target
+      target: '',
+      datapoints: [[100, 1], [200, 2]],
+    };
+    const data = getProcessedTableData([null, input1, input2, null, null]);
+    expect(data.length).toBe(2);
+    expect(data[0].columns[0].text).toBe(input1.target);
+    expect(data[0].rows).toBe(input1.datapoints);
+
+    // Default name
+    expect(data[1].columns[0].text).toEqual('Value');
+
+    // Every colun should have a name and a type
+    for (const table of data) {
+      for (const column of table.columns) {
+        expect(column.text).toBeDefined();
+        expect(column.type).toBeDefined();
+      }
+    }
+  });
+
+  it('supports null values from query OK', () => {
+    expect(getProcessedTableData([null, null, null, null])).toEqual([]);
+    expect(getProcessedTableData(undefined)).toEqual([]);
+    expect(getProcessedTableData((null as unknown) as any[])).toEqual([]);
+    expect(getProcessedTableData([])).toEqual([]);
+  });
+});

public/app/features/dashboard/dashgrid/DataPanel.tsx

@@ -15,6 +15,7 @@ import {
   TimeRange,
   ScopedVars,
   toTableData,
+  guessColumnTypes,
 } from '@grafana/ui';
 
 interface RenderProps {
@@ -46,6 +47,25 @@ export interface State {
   data?: TableData[];
 }
 
+/**
+ * All panels will be passed tables that have our best guess at colum type set
+ *
+ * This is also used by PanelChrome for snapshot support
+ */
+export function getProcessedTableData(results?: any[]): TableData[] {
+  if (!results) {
+    return [];
+  }
+
+  const tables: TableData[] = [];
+  for (const r of results) {
+    if (r) {
+      tables.push(guessColumnTypes(toTableData(r)));
+    }
+  }
+  return tables;
+}
+
 export class DataPanel extends Component<Props, State> {
   static defaultProps = {
     isVisible: true,
@@ -147,7 +167,7 @@ export class DataPanel extends Component<Props, State> {
       this.setState({
         loading: LoadingState.Done,
         response: resp,
-        data: toTableData(resp.data),
+        data: getProcessedTableData(resp.data),
         isFirstLoad: false,
       });
     } catch (err) {

public/app/features/dashboard/dashgrid/PanelChrome.tsx

@@ -19,11 +19,13 @@ import config from 'app/core/config';
 // Types
 import { DashboardModel, PanelModel } from '../state';
 import { PanelPlugin } from 'app/types';
-import { DataQueryResponse, TimeRange, LoadingState, TableData, DataQueryError, toTableData } from '@grafana/ui';
+import { DataQueryResponse, TimeRange, LoadingState, TableData, DataQueryError } from '@grafana/ui';
 import { ScopedVars } from '@grafana/ui';
 
 import templateSrv from 'app/features/templating/template_srv';
 
+import { getProcessedTableData } from './DataPanel';
+
 const DEFAULT_PLUGIN_ERROR = 'Error in plugin';
 
 export interface Props {
@@ -139,7 +141,7 @@ export class PanelChrome extends PureComponent<Props, State> {
   }
 
   get getDataForPanel() {
-    return this.hasPanelSnapshot ? toTableData(this.props.panel.snapshotData) : null;
+    return this.hasPanelSnapshot ? getProcessedTableData(this.props.panel.snapshotData) : null;
   }
 
   renderPanelPlugin(loading: LoadingState, data: TableData[], width: number, height: number): JSX.Element {

public/app/plugins/panel/graph2/GraphPanel.tsx

@@ -2,14 +2,16 @@
 import _ from 'lodash';
 import React, { PureComponent } from 'react';
 
-// Utils
-import { processTimeSeries } from '@grafana/ui/src/utils';
-
-// Components
-import { Graph } from '@grafana/ui';
-
-// Types
-import { PanelProps, NullValueMode, TimeSeriesVMs } from '@grafana/ui/src/types';
+import {
+  Graph,
+  PanelProps,
+  NullValueMode,
+  colors,
+  TimeSeriesVMs,
+  ColumnType,
+  getFirstTimeColumn,
+  processTimeSeries,
+} from '@grafana/ui';
 import { Options } from './types';
 
 interface Props extends PanelProps<Options> {}
@@ -19,12 +21,30 @@ export class GraphPanel extends PureComponent<Props> {
     const { data, timeRange, width, height } = this.props;
     const { showLines, showBars, showPoints } = this.props.options;
 
-    let vmSeries: TimeSeriesVMs;
-    if (data) {
-      vmSeries = processTimeSeries({
-        data,
-        nullValueMode: NullValueMode.Ignore,
-      });
+    const vmSeries: TimeSeriesVMs = [];
+    for (const table of data) {
+      const timeColumn = getFirstTimeColumn(table);
+      if (timeColumn < 0) {
+        continue;
+      }
+
+      for (let i = 0; i < table.columns.length; i++) {
+        const column = table.columns[i];
+
+        // Show all numeric columns
+        if (column.type === ColumnType.number) {
+          const tsvm = processTimeSeries({
+            data: [table],
+            xColumn: timeColumn,
+            yColumn: i,
+            nullValueMode: NullValueMode.Null,
+          })[0];
+
+          const colorIndex = vmSeries.length % colors.length;
+          tsvm.color = colors[colorIndex];
+          vmSeries.push(tsvm);
+        }
+      }
     }
 
     return (

public/app/plugins/panel/singlestat2/SingleStatPanel.tsx

@@ -4,7 +4,7 @@ import React, { PureComponent, CSSProperties } from 'react';
 // Types
 import { SingleStatOptions, SingleStatBaseOptions } from './types';
 
-import { DisplayValue, PanelProps, processTimeSeries, NullValueMode } from '@grafana/ui';
+import { DisplayValue, PanelProps, processTimeSeries, NullValueMode, ColumnType } from '@grafana/ui';
 import { config } from 'app/core/config';
 import { getDisplayProcessor } from '@grafana/ui';
 import { ProcessedValuesRepeater } from './ProcessedValuesRepeater';
@@ -24,13 +24,30 @@ export const getSingleStatValues = (props: PanelProps<SingleStatBaseOptions>): D
     theme: config.theme,
   });
 
-  return processTimeSeries({
-    data,
-    nullValueMode: NullValueMode.Null,
-  }).map((series, index) => {
-    const value = stat !== 'name' ? series.stats[stat] : series.label;
-    return processor(value);
-  });
+  const values: DisplayValue[] = [];
+  for (const table of data) {
+    for (let i = 0; i < table.columns.length; i++) {
+      const column = table.columns[i];
+
+      // Show all columns that are not 'time'
+      if (column.type === ColumnType.number) {
+        const series = processTimeSeries({
+          data: [table],
+          xColumn: i,
+          yColumn: i,
+          nullValueMode: NullValueMode.Null,
+        })[0];
+
+        const value = stat !== 'name' ? series.stats[stat] : series.label;
+        values.push(processor(value));
+      }
+    }
+  }
+
+  if (values.length === 0) {
+    throw { message: 'Could not find numeric data' };
+  }
+  return values;
 };
 
 export class SingleStatPanel extends PureComponent<PanelProps<SingleStatOptions>> {