PanelInspector: Adds a Raw display mode but defaults to Formatted display mode (#27306)

* PanelInspector: Fields with overrides are formatted correct in CSV

* Refactor: adds raw format

* Refactor: changes switch to Formatted values

* Tests: adds tests for applyRawFieldOverrides and getRawDisplayProcessor

* Test: change to utc timeZone

* Refactor: changes after PR comments

packages/grafana-data/src/field/displayProcessor.test.ts

-import { getDisplayProcessor } from './displayProcessor';
+import { getDisplayProcessor, getRawDisplayProcessor } from './displayProcessor';
 import { DisplayProcessor, DisplayValue } from '../types/displayValue';
 import { MappingType, ValueMapping } from '../types/valueMapping';
 import { Field, FieldConfig, FieldType, GrafanaTheme, Threshold, ThresholdsMode } from '../types';
@@ -326,3 +326,27 @@ describe('Date display options', () => {
     });
   });
 });
+
+describe('getRawDisplayProcessor', () => {
+  const processor = getRawDisplayProcessor();
+  const date = new Date('2020-01-01T00:00:00.000Z');
+  const timestamp = date.valueOf();
+
+  it.each`
+    value                             | expected
+    ${0}                              | ${'0'}
+    ${13.37}                          | ${'13.37'}
+    ${true}                           | ${'true'}
+    ${false}                          | ${'false'}
+    ${date}                           | ${`${date}`}
+    ${timestamp}                      | ${'1577836800000'}
+    ${'a string'}                     | ${'a string'}
+    ${null}                           | ${'null'}
+    ${undefined}                      | ${'undefined'}
+    ${{ value: 0, label: 'a label' }} | ${'[object Object]'}
+  `('when called with value:{$value}', ({ value, expected }) => {
+    const result = processor(value);
+
+    expect(result).toEqual({ text: expected, numeric: null });
+  });
+});

packages/grafana-data/src/field/displayProcessor.ts

@@ -4,7 +4,7 @@ import _ from 'lodash';
 // Types
 import { Field, FieldType } from '../types/dataFrame';
 import { GrafanaTheme } from '../types/theme';
-import { DisplayProcessor, DisplayValue, DecimalCount, DecimalInfo } from '../types/displayValue';
+import { DecimalCount, DecimalInfo, DisplayProcessor, DisplayValue } from '../types/displayValue';
 import { getValueFormat } from '../valueFormats/valueFormats';
 import { getMappedValue } from '../utils/valueMappings';
 import { dateTime } from '../datetime';
@@ -166,3 +166,10 @@ export function getDecimalsForValue(value: number, decimalOverride?: DecimalCoun
 
   return { decimals, scaledDecimals };
 }
+
+export function getRawDisplayProcessor(): DisplayProcessor {
+  return (value: any) => ({
+    text: `${value}`,
+    numeric: (null as unknown) as number,
+  });
+}

packages/grafana-data/src/field/fieldOverrides.test.ts

 import {
+  applyFieldOverrides,
+  applyRawFieldOverrides,
   FieldOverrideEnv,
   findNumericFieldMinMax,
-  setFieldConfigDefaults,
-  setDynamicConfigValue,
-  applyFieldOverrides,
   getLinksSupplier,
+  setDynamicConfigValue,
+  setFieldConfigDefaults,
 } from './fieldOverrides';
 import { MutableDataFrame, toDataFrame } from '../dataframe';
 import {
+  DataFrame,
+  Field,
+  FieldColorMode,
   FieldConfig,
   FieldConfigPropertyItem,
-  GrafanaTheme,
-  FieldType,
-  DataFrame,
   FieldConfigSource,
+  FieldType,
+  GrafanaTheme,
   InterpolateFunction,
+  ThresholdsMode,
 } from '../types';
-import { Registry } from '../utils';
+import { locationUtil, Registry } from '../utils';
 import { mockStandardProperties } from '../utils/tests/mockStandardProperties';
 import { FieldMatcherID } from '../transformations';
 import { FieldConfigOptionsRegistry } from './FieldConfigOptionsRegistry';
 import { getFieldDisplayName } from './fieldState';
-import { locationUtil } from '../utils';
+import { ArrayVector } from '../vector';
+import { getDisplayProcessor } from './displayProcessor';
 
 const property1: any = {
   id: 'custom.property1', // Match field properties
@@ -543,3 +548,191 @@ describe('getLinksSupplier', () => {
     );
   });
 });
+
+describe('applyRawFieldOverrides', () => {
+  const getNumberFieldConfig = () => ({
+    custom: {},
+    thresholds: {
+      mode: ThresholdsMode.Absolute,
+      steps: [
+        {
+          color: 'green',
+          value: (null as unknown) as number,
+        },
+        {
+          color: 'red',
+          value: 80,
+        },
+      ],
+    },
+    mappings: [],
+    color: {
+      mode: FieldColorMode.Thresholds,
+    },
+    min: 0,
+    max: 1599124316808,
+  });
+
+  const getEmptyConfig = () => ({
+    custom: {},
+    mappings: [],
+  });
+
+  const getDisplayValue = (frames: DataFrame[], frameIndex: number, fieldIndex: number) => {
+    const field = frames[frameIndex].fields[fieldIndex];
+    const value = field.values.get(0);
+    return field.display!(value);
+  };
+
+  const expectRawDataDisplayValue = (frames: DataFrame[], frameIndex: number) => {
+    expect(getDisplayValue(frames, frameIndex, 0)).toEqual({ text: '1599045551050', numeric: null });
+    expect(getDisplayValue(frames, frameIndex, 1)).toEqual({ text: '3.14159265359', numeric: null });
+    expect(getDisplayValue(frames, frameIndex, 2)).toEqual({ text: '0', numeric: null });
+    expect(getDisplayValue(frames, frameIndex, 3)).toEqual({ text: '0', numeric: null });
+    expect(getDisplayValue(frames, frameIndex, 4)).toEqual({ text: 'A - string', numeric: null });
+    expect(getDisplayValue(frames, frameIndex, 5)).toEqual({ text: '1599045551050', numeric: null });
+  };
+
+  const expectFormattedDataDisplayValue = (frames: DataFrame[], frameIndex: number) => {
+    expect(getDisplayValue(frames, frameIndex, 0)).toEqual({
+      color: '#F2495C',
+      numeric: 1599045551050,
+      prefix: undefined,
+      suffix: undefined,
+      text: '1599045551050',
+      threshold: {
+        color: 'red',
+        value: 80,
+      },
+    });
+
+    expect(getDisplayValue(frames, frameIndex, 1)).toEqual({
+      color: '#73BF69',
+      numeric: 3.14159265359,
+      prefix: undefined,
+      suffix: undefined,
+      text: '3.142',
+      threshold: {
+        color: 'green',
+        value: null,
+      },
+    });
+
+    expect(getDisplayValue(frames, frameIndex, 2)).toEqual({
+      color: '#73BF69',
+      numeric: 0,
+      prefix: undefined,
+      suffix: undefined,
+      text: '0',
+      threshold: {
+        color: 'green',
+        value: null,
+      },
+    });
+
+    expect(getDisplayValue(frames, frameIndex, 3)).toEqual({
+      numeric: 0,
+      prefix: undefined,
+      suffix: undefined,
+      text: '0',
+    });
+
+    expect(getDisplayValue(frames, frameIndex, 4)).toEqual({
+      numeric: NaN,
+      prefix: undefined,
+      suffix: undefined,
+      text: 'A - string',
+    });
+
+    expect(getDisplayValue(frames, frameIndex, 5)).toEqual({
+      numeric: 1599045551050,
+      prefix: undefined,
+      suffix: undefined,
+      text: '2020-09-02 11:19:11',
+    });
+  };
+
+  describe('when called', () => {
+    it('then all fields should have their display processor replaced with the raw display processor', () => {
+      const numberAsEpoc: Field = {
+        name: 'numberAsEpoc',
+        type: FieldType.number,
+        values: new ArrayVector([1599045551050]),
+        config: getNumberFieldConfig(),
+      };
+
+      const numberWithDecimals: Field = {
+        name: 'numberWithDecimals',
+        type: FieldType.number,
+        values: new ArrayVector([3.14159265359]),
+        config: {
+          ...getNumberFieldConfig(),
+          decimals: 3,
+        },
+      };
+
+      const numberAsBoolean: Field = {
+        name: 'numberAsBoolean',
+        type: FieldType.number,
+        values: new ArrayVector([0]),
+        config: getNumberFieldConfig(),
+      };
+
+      const boolean: Field = {
+        name: 'boolean',
+        type: FieldType.boolean,
+        values: new ArrayVector([0]),
+        config: getEmptyConfig(),
+      };
+
+      const string: Field = {
+        name: 'string',
+        type: FieldType.boolean,
+        values: new ArrayVector(['A - string']),
+        config: getEmptyConfig(),
+      };
+
+      const datetime: Field = {
+        name: 'datetime',
+        type: FieldType.time,
+        values: new ArrayVector([1599045551050]),
+        config: {
+          unit: 'dateTimeAsIso',
+        },
+      };
+
+      const dataFrameA: DataFrame = toDataFrame({
+        fields: [numberAsEpoc, numberWithDecimals, numberAsBoolean, boolean, string, datetime],
+      });
+
+      dataFrameA.fields[0].display = getDisplayProcessor({ field: dataFrameA.fields[0] });
+      dataFrameA.fields[1].display = getDisplayProcessor({ field: dataFrameA.fields[1] });
+      dataFrameA.fields[2].display = getDisplayProcessor({ field: dataFrameA.fields[2] });
+      dataFrameA.fields[3].display = getDisplayProcessor({ field: dataFrameA.fields[3] });
+      dataFrameA.fields[4].display = getDisplayProcessor({ field: dataFrameA.fields[4] });
+      dataFrameA.fields[5].display = getDisplayProcessor({ field: dataFrameA.fields[5], timeZone: 'utc' });
+
+      const dataFrameB: DataFrame = toDataFrame({
+        fields: [numberAsEpoc, numberWithDecimals, numberAsBoolean, boolean, string, datetime],
+      });
+
+      dataFrameB.fields[0].display = getDisplayProcessor({ field: dataFrameB.fields[0] });
+      dataFrameB.fields[1].display = getDisplayProcessor({ field: dataFrameB.fields[1] });
+      dataFrameB.fields[2].display = getDisplayProcessor({ field: dataFrameB.fields[2] });
+      dataFrameB.fields[3].display = getDisplayProcessor({ field: dataFrameB.fields[3] });
+      dataFrameB.fields[4].display = getDisplayProcessor({ field: dataFrameB.fields[4] });
+      dataFrameB.fields[5].display = getDisplayProcessor({ field: dataFrameB.fields[5], timeZone: 'utc' });
+
+      const data = [dataFrameA, dataFrameB];
+      const rawData = applyRawFieldOverrides(data);
+
+      // expect raw data is correct
+      expectRawDataDisplayValue(rawData, 0);
+      expectRawDataDisplayValue(rawData, 1);
+
+      // expect the original data is still the same
+      expectFormattedDataDisplayValue(data, 0);
+      expectFormattedDataDisplayValue(data, 1);
+    });
+  });
+});

packages/grafana-data/src/field/fieldOverrides.ts

 import {
-  DynamicConfigValue,
-  FieldConfig,
+  ApplyFieldOverrideOptions,
+  ColorScheme,
   DataFrame,
+  DataLink,
+  DataSourceInstanceSettings,
+  DynamicConfigValue,
   Field,
-  FieldType,
   FieldColorMode,
-  ColorScheme,
-  FieldOverrideContext,
-  ScopedVars,
-  ApplyFieldOverrideOptions,
+  FieldConfig,
   FieldConfigPropertyItem,
-  LinkModel,
-  InterpolateFunction,
-  ValueLinkConfig,
+  FieldOverrideContext,
+  FieldType,
   GrafanaTheme,
+  InterpolateFunction,
+  LinkModel,
+  ScopedVars,
   TimeZone,
-  DataLink,
-  DataSourceInstanceSettings,
+  ValueLinkConfig,
 } from '../types';
-import { fieldMatchers, ReducerID, reduceField } from '../transformations';
+import { fieldMatchers, reduceField, ReducerID } from '../transformations';
 import { FieldMatcher } from '../types/transformations';
 import isNumber from 'lodash/isNumber';
 import set from 'lodash/set';
 import unset from 'lodash/unset';
 import get from 'lodash/get';
-import { getDisplayProcessor } from './displayProcessor';
+import { getDisplayProcessor, getRawDisplayProcessor } from './displayProcessor';
 import { guessFieldTypeForField } from '../dataframe';
 import { standardFieldConfigEditorRegistry } from './standardFieldConfigEditorRegistry';
 import { FieldConfigOptionsRegistry } from './FieldConfigOptionsRegistry';
 import { DataLinkBuiltInVars, locationUtil } from '../utils';
 import { formattedValueToString } from '../valueFormats';
 import { getFieldDisplayValuesProxy } from './getFieldDisplayValuesProxy';
-import { getFrameDisplayName, getFieldDisplayName } from './fieldState';
+import { getFieldDisplayName, getFrameDisplayName } from './fieldState';
 import { getTimeField } from '../dataframe/processDataFrame';
 import { mapInternalLinkToExplore } from '../utils/dataLinks';
 import { getTemplateProxyForField } from './templateProxies';
@@ -427,3 +427,34 @@ export const getLinksSupplier = (
     }
   });
 };
+
+/**
+ * Return a copy of the DataFrame with raw data
+ */
+export function applyRawFieldOverrides(data: DataFrame[]): DataFrame[] {
+  if (!data || data.length === 0) {
+    return [];
+  }
+
+  const newData = [...data];
+  const processor = getRawDisplayProcessor();
+
+  for (let frameIndex = 0; frameIndex < newData.length; frameIndex++) {
+    const newFrame = { ...newData[frameIndex] };
+    const newFields = [...newFrame.fields];
+
+    for (let fieldIndex = 0; fieldIndex < newFields.length; fieldIndex++) {
+      newFields[fieldIndex] = {
+        ...newFields[fieldIndex],
+        display: processor,
+      };
+    }
+
+    newData[frameIndex] = {
+      ...newFrame,
+      fields: newFields,
+    };
+  }
+
+  return newData;
+}

packages/grafana-data/src/field/index.ts

@@ -5,6 +5,6 @@ export * from './standardFieldConfigEditorRegistry';
 export * from './overrides/processors';
 export { FieldConfigOptionsRegistry } from './FieldConfigOptionsRegistry';
 
-export { applyFieldOverrides, validateFieldConfig } from './fieldOverrides';
+export { applyFieldOverrides, validateFieldConfig, applyRawFieldOverrides } from './fieldOverrides';
 export { getFieldDisplayValuesProxy } from './getFieldDisplayValuesProxy';
 export { getFieldDisplayName, getFrameDisplayName } from './fieldState';

packages/grafana-data/src/utils/csv.test.ts

-import { readCSV, toCSV, CSVHeaderStyle } from './csv';
-import { getDataFrameRow } from '../dataframe/processDataFrame';
+import { CSVHeaderStyle, readCSV, toCSV } from './csv';
+import { getDataFrameRow, toDataFrameDTO } from '../dataframe/processDataFrame';
 
 // Test with local CSV files
 import fs from 'fs';
-import { toDataFrameDTO } from '../dataframe/processDataFrame';
 import { MutableDataFrame } from '../dataframe';
+import { getDisplayProcessor } from '../field';
 
 describe('read csv', () => {
   it('should get X and y', () => {
@@ -115,4 +115,29 @@ describe('DataFrame to CSV', () => {
       "
     `);
   });
+
+  it('should use field display processor if exists', () => {
+    const dataFrame = new MutableDataFrame({
+      fields: [
+        { name: 'Time', values: [1589455688623] },
+        {
+          name: 'Value',
+          values: [1589455688623],
+          config: {
+            unit: 'dateTimeAsIso',
+          },
+        },
+      ],
+    });
+
+    dataFrame.fields[1].display = getDisplayProcessor({ field: dataFrame.fields[1], timeZone: 'utc' });
+
+    const csv = toCSV([dataFrame]);
+    expect(csv).toMatchInlineSnapshot(`
+      "\\"Time\\",\\"Value\\"
+      1589455688623,2020-05-14 11:28:08
+
+      "
+    `);
+  });
 });

packages/grafana-data/src/utils/csv.ts

 // Libraries
-import Papa, { ParseResult, ParseConfig, Parser } from 'papaparse';
+import Papa, { ParseConfig, Parser, ParseResult } from 'papaparse';
 import defaults from 'lodash/defaults';
-import isNumber from 'lodash/isNumber';
 
 // Types
-import { DataFrame, Field, FieldType, FieldConfig } from '../types';
+import { DataFrame, Field, FieldConfig, FieldType } from '../types';
 import { guessFieldTypeFromValue } from '../dataframe/processDataFrame';
 import { MutableDataFrame } from '../dataframe/MutableDataFrame';
 import { getFieldDisplayName } from '../field';
+import { formattedValueToString } from '../valueFormats';
 
 export enum CSVHeaderStyle {
   full,
@@ -205,21 +205,11 @@ function writeValue(value: any, config: CSVConfig): string {
 }
 
 function makeFieldWriter(field: Field, config: CSVConfig): FieldWriter {
-  if (field.type) {
-    if (field.type === FieldType.boolean) {
-      return (value: any) => {
-        return value ? 'true' : 'false';
-      };
-    }
-
-    if (field.type === FieldType.number) {
-      return (value: any) => {
-        if (isNumber(value)) {
-          return value.toString();
-        }
-        return writeValue(value, config);
-      };
-    }
+  if (field.display) {
+    return (value: any) => {
+      const displayValue = field.display!(value);
+      return writeValue(formattedValueToString(displayValue), config);
+    };
   }
 
   return (value: any) => writeValue(value, config);

packages/grafana-ui/src/components/Table/DefaultCell.tsx

 import React, { FC } from 'react';
-import { TableCellProps } from './types';
 import { formattedValueToString, LinkModel } from '@grafana/data';
 
+import { TableCellProps } from './types';
+
 export const DefaultCell: FC<TableCellProps> = props => {
   const { field, cell, tableStyles, row } = props;
   let link: LinkModel<any> | undefined;
@@ -13,33 +14,33 @@ export const DefaultCell: FC<TableCellProps> = props => {
       valueRowIndex: row.index,
     })[0];
   }
-  const value = field.display ? formattedValueToString(displayValue) : displayValue;
+  const value = field.display ? formattedValueToString(displayValue) : `${displayValue}`;
+
+  if (!link) {
+    return <div className={tableStyles.tableCell}>{value}</div>;
+  }
 
   return (
     <div className={tableStyles.tableCell}>
-      {link ? (
-        <a
-          href={link.href}
-          onClick={
-            link.onClick
-              ? event => {
-                  // Allow opening in new tab
-                  if (!(event.ctrlKey || event.metaKey || event.shiftKey) && link!.onClick) {
-                    event.preventDefault();
-                    link!.onClick(event);
-                  }
+      <a
+        href={link.href}
+        onClick={
+          link.onClick
+            ? event => {
+                // Allow opening in new tab
+                if (!(event.ctrlKey || event.metaKey || event.shiftKey) && link!.onClick) {
+                  event.preventDefault();
+                  link!.onClick(event);
                 }
-              : undefined
-          }
-          target={link.target}
-          title={link.title}
-          className={tableStyles.tableCellLink}
-        >
-          {value}
-        </a>
-      ) : (
-        value
-      )}
+              }
+            : undefined
+        }
+        target={link.target}
+        title={link.title}
+        className={tableStyles.tableCellLink}
+      >
+        {value}
+      </a>
     </div>
   );
 };

public/app/features/dashboard/components/Inspector/InspectDataTab.tsx

 import React, { PureComponent } from 'react';
 import {
   applyFieldOverrides,
+  applyRawFieldOverrides,
   DataFrame,
   DataTransformerID,
   dateTimeFormat,
@@ -8,13 +9,8 @@ import {
   SelectableValue,
   toCSV,
   transformDataFrame,
-  getTimeField,
-  FieldType,
-  FormattedVector,
-  DisplayProcessor,
-  getDisplayProcessor,
 } from '@grafana/data';
-import { Button, Field, Icon, Switch, Select, Table, VerticalGroup, Container, HorizontalGroup } from '@grafana/ui';
+import { Button, Container, Field, HorizontalGroup, Icon, Select, Switch, Table, VerticalGroup } from '@grafana/ui';
 import { selectors } from '@grafana/e2e-selectors';
 import AutoSizer from 'react-virtualized-auto-sizer';
 
@@ -60,33 +56,6 @@ export class InspectDataTab extends PureComponent<Props, State> {
     const { panel } = this.props;
     const { transformId } = this.state;
 
-    // Replace the time field with a formatted time
-    const { timeIndex, timeField } = getTimeField(dataFrame);
-
-    if (timeField) {
-      // Use the configured date or standard time display
-      let processor: DisplayProcessor | undefined = timeField.display;
-      if (!processor) {
-        processor = getDisplayProcessor({
-          field: timeField,
-        });
-      }
-
-      const formattedDateField = {
-        ...timeField,
-        type: FieldType.string,
-        values: new FormattedVector(timeField.values, processor),
-      };
-
-      const fields = [...dataFrame.fields];
-      fields[timeIndex!] = formattedDateField;
-
-      dataFrame = {
-        ...dataFrame,
-        fields,
-      };
-    }
-
     const dataFrameCsv = toCSV([dataFrame]);
 
     const blob = new Blob([String.fromCharCode(0xfeff), dataFrameCsv], {
@@ -146,12 +115,16 @@ export class InspectDataTab extends PureComponent<Props, State> {
       });
     }
 
+    if (!options.withFieldConfig) {
+      return applyRawFieldOverrides(data);
+    }
+
     // We need to apply field config even though it was already applied in the PanelQueryRunner.
     // That's because transformers create new fields and data frames, so i.e. display processor is no longer there
     return applyFieldOverrides({
       data,
       theme: config.theme,
-      fieldConfig: options.withFieldConfig ? this.props.panel.fieldConfig : { defaults: {}, overrides: [] },
+      fieldConfig: this.props.panel.fieldConfig,
       replaceVariables: (value: string) => {
         return value;
       },
@@ -185,7 +158,7 @@ export class InspectDataTab extends PureComponent<Props, State> {
       }
 
       if (options.withFieldConfig) {
-        activeString += 'field configuration';
+        activeString += 'formatted data';
       }
     }
 
@@ -261,8 +234,8 @@ export class InspectDataTab extends PureComponent<Props, State> {
               )}
               {showFieldConfigsOption && (
                 <Field
-                  label="Apply field configuration"
-                  description="Table data is displayed with options defined in the Field and Override tabs."
+                  label="Formatted data"
+                  description="Table data is formatted with options defined in the Field and Override tabs."
                 >
                   <Switch
                     value={!!options.withFieldConfig}

public/app/features/dashboard/components/Inspector/PanelInspector.tsx

@@ -32,7 +32,7 @@ const PanelInspectorUnconnected: React.FC<Props> = ({ panel, dashboard, defaultT
   const dispatch = useDispatch();
   const [dataOptions, setDataOptions] = useState<GetDataOptions>({
     withTransforms: false,
-    withFieldConfig: false,
+    withFieldConfig: true,
   });
   const { data, isLoading, error } = usePanelLatestData(panel, dataOptions);
   const metaDs = useDatasourceMetadata(data);