Transformers: calculate a new field based on the row values (#23675)

packages/grafana-data/src/transformations/fieldReducer.ts

@@ -227,7 +227,7 @@ export const fieldReducers = new Registry<FieldReducerInfo>(() => [
   },
 ]);
 
-function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
+export function doStandardCalcs(field: Field, ignoreNulls: boolean, nullAsZero: boolean): FieldCalcs {
   const calcs = {
     sum: 0,
     max: -Number.MAX_VALUE,

packages/grafana-data/src/transformations/index.ts

@@ -7,6 +7,7 @@ export { FilterFieldsByNameTransformerOptions } from './transformers/filterByNam
 export { FilterFramesByRefIdTransformerOptions } from './transformers/filterByRefId';
 export { SeriesToColumnsOptions } from './transformers/seriesToColumns';
 export { ReduceTransformerOptions } from './transformers/reduce';
+export { CalculateFieldTransformerOptions } from './transformers/calculateField';
 export { OrganizeFieldsTransformerOptions } from './transformers/organize';
 export { createOrderFieldsComparer } from './transformers/order';
 export { transformDataFrame } from './transformDataFrame';

packages/grafana-data/src/transformations/transformers.ts

 import { appendTransformer } from './transformers/append';
 import { reduceTransformer } from './transformers/reduce';
+import { calculateFieldTransformer } from './transformers/calculateField';
 import { filterFieldsTransformer, filterFramesTransformer } from './transformers/filter';
 import { filterFieldsByNameTransformer } from './transformers/filterByName';
 import { noopTransformer } from './transformers/noop';
@@ -19,6 +20,7 @@ export const standardTransformers = {
   organizeFieldsTransformer,
   appendTransformer,
   reduceTransformer,
+  calculateFieldTransformer,
   seriesToColumnsTransformer,
   renameFieldsTransformer,
 };

packages/grafana-data/src/transformations/transformers/calculateField.test.ts

+import { DataTransformerID } from './ids';
+import { toDataFrame } from '../../dataframe/processDataFrame';
+import { FieldType } from '../../types/dataFrame';
+import { ReducerID } from '../fieldReducer';
+import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
+import { transformDataFrame } from '../transformDataFrame';
+import { calculateFieldTransformer } from './calculateField';
+import { DataFrameView } from '../../dataframe';
+
+const seriesToTestWith = toDataFrame({
+  fields: [
+    { name: 'A', type: FieldType.time, values: [1000, 2000] },
+    { name: 'B', type: FieldType.number, values: [1, 100] },
+    { name: 'C', type: FieldType.number, values: [2, 200] },
+    { name: 'D', type: FieldType.string, values: ['first', 'second'] },
+  ],
+});
+
+describe('calculateField transformer', () => {
+  beforeAll(() => {
+    mockTransformationsRegistry([calculateFieldTransformer]);
+  });
+
+  it('will filter and alias', () => {
+    const cfg = {
+      id: DataTransformerID.calculateField,
+      options: {
+        // defautls to sum
+        alias: 'The Total',
+      },
+    };
+
+    const filtered = transformDataFrame([cfg], [seriesToTestWith])[0];
+    const rows = new DataFrameView(filtered).toArray();
+    expect(rows).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "A": 1000,
+          "B": 1,
+          "C": 2,
+          "D": "first",
+          "The Total": 3,
+        },
+        Object {
+          "A": 2000,
+          "B": 100,
+          "C": 200,
+          "D": "second",
+          "The Total": 300,
+        },
+      ]
+    `);
+  });
+
+  it('will replace other fields', () => {
+    const cfg = {
+      id: DataTransformerID.calculateField,
+      options: {
+        reducer: ReducerID.mean,
+        replaceFields: true,
+      },
+    };
+
+    const filtered = transformDataFrame([cfg], [seriesToTestWith])[0];
+    const rows = new DataFrameView(filtered).toArray();
+    expect(rows).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "Mean": 1.5,
+        },
+        Object {
+          "Mean": 150,
+        },
+      ]
+    `);
+  });
+
+  it('will filter by name', () => {
+    const cfg = {
+      id: DataTransformerID.calculateField,
+      options: {
+        reducer: ReducerID.mean,
+        replaceFields: true,
+        include: 'B',
+      },
+    };
+
+    const filtered = transformDataFrame([cfg], [seriesToTestWith])[0];
+    const rows = new DataFrameView(filtered).toArray();
+    expect(rows).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "Mean": 1,
+        },
+        Object {
+          "Mean": 100,
+        },
+      ]
+    `);
+  });
+});

packages/grafana-data/src/transformations/transformers/calculateField.ts

+import { DataFrame, DataTransformerInfo, Vector, FieldType, Field, NullValueMode } from '../../types';
+import { DataTransformerID } from './ids';
+import { ReducerID, fieldReducers } from '../fieldReducer';
+import { getFieldMatcher } from '../matchers';
+import { FieldMatcherID } from '../matchers/ids';
+import { RowVector } from '../../vector/RowVector';
+import { ArrayVector } from '../../vector';
+import { doStandardCalcs } from '../fieldReducer';
+
+export interface CalculateFieldTransformerOptions {
+  reducer: ReducerID;
+  include?: string; // Assume all fields
+  alias?: string; // The output field name
+  replaceFields?: boolean;
+  nullValueMode?: NullValueMode;
+}
+
+export const calculateFieldTransformer: DataTransformerInfo<CalculateFieldTransformerOptions> = {
+  id: DataTransformerID.calculateField,
+  name: 'Add field from calculation',
+  description: 'Use the row values to calculate a new field',
+  defaultOptions: {
+    reducer: ReducerID.sum,
+  },
+  transformer: options => (data: DataFrame[]) => {
+    let matcher = getFieldMatcher({
+      id: FieldMatcherID.numeric,
+    });
+    if (options.include && options.include.length) {
+      matcher = getFieldMatcher({
+        id: FieldMatcherID.byName,
+        options: options.include,
+      });
+    }
+
+    const info = fieldReducers.get(options.reducer);
+    if (!info) {
+      throw new Error(`Unknown reducer: ${options.reducer}`);
+    }
+    const reducer = info.reduce ?? doStandardCalcs;
+    const ignoreNulls = options.nullValueMode === NullValueMode.Ignore;
+    const nullAsZero = options.nullValueMode === NullValueMode.AsZero;
+
+    return data.map(frame => {
+      // Find the columns that should be examined
+      const columns: Vector[] = [];
+      frame.fields.forEach(field => {
+        if (matcher(field)) {
+          columns.push(field.values);
+        }
+      });
+
+      // Prepare a "fake" field for the row
+      const iter = new RowVector(columns);
+      const row: Field = {
+        name: 'temp',
+        values: iter,
+        type: FieldType.number,
+        config: {},
+      };
+      const vals: number[] = [];
+      for (let i = 0; i < frame.length; i++) {
+        iter.rowIndex = i;
+        row.calcs = undefined; // bust the cache (just in case)
+        const val = reducer(row, ignoreNulls, nullAsZero)[options.reducer];
+        vals.push(val);
+      }
+
+      const field = {
+        name: options.alias || info.name,
+        type: FieldType.number,
+        config: {},
+        values: new ArrayVector(vals),
+      };
+
+      return {
+        ...frame,
+        fields: options.replaceFields ? [field] : [...frame.fields, field],
+      };
+    });
+  },
+};

packages/grafana-data/src/transformations/transformers/ids.ts

@@ -6,6 +6,7 @@ export enum DataTransformerID {
   order = 'order', // order fields based on user configuration
   organize = 'organize', // order, rename and filter based on user configuration
   rename = 'rename', // rename field based on user configuration
+  calculateField = 'calculateField', // Run a reducer on the row
 
   seriesToColumns = 'seriesToColumns', // former table transform timeseries_to_columns
   filterFields = 'filterFields', // Pick some fields (keep all frames)

packages/grafana-data/src/vector/RowVector.ts

+import { Vector } from '../types';
+import { vectorToArray } from './vectorToArray';
+
+/**
+ * RowVector makes the row values look like a vector
+ * @internal
+ */
+export class RowVector implements Vector {
+  constructor(private columns: Vector[]) {}
+
+  rowIndex = 0;
+
+  get length(): number {
+    return this.columns.length;
+  }
+
+  get(index: number): number {
+    return this.columns[index].get(this.rowIndex);
+  }
+
+  toArray(): number[] {
+    return vectorToArray(this);
+  }
+
+  toJSON(): number[] {
+    return vectorToArray(this);
+  }
+}

packages/grafana-ui/src/components/TransformersUI/CalculateFieldTransformerEditor.tsx

+import React, { useContext, ChangeEvent } from 'react';
+import {
+  DataTransformerID,
+  CalculateFieldTransformerOptions,
+  KeyValue,
+  standardTransformers,
+  TransformerRegistyItem,
+  TransformerUIProps,
+  FieldType,
+  ReducerID,
+  fieldReducers,
+} from '@grafana/data';
+import { ThemeContext } from '../../themes/ThemeContext';
+import { css } from 'emotion';
+import { InlineList } from '../List/InlineList';
+import { Icon } from '../Icon/Icon';
+import { Label } from '../Forms/Label';
+import { StatsPicker } from '../StatsPicker/StatsPicker';
+import { Switch } from '../Switch/Switch';
+import { Input } from '../Input/Input';
+
+interface CalculateFieldTransformerEditorProps extends TransformerUIProps<CalculateFieldTransformerOptions> {}
+
+interface CalculateFieldTransformerEditorState {
+  include: string;
+  names: string[];
+  selected: string[];
+}
+
+export class CalculateFieldTransformerEditor extends React.PureComponent<
+  CalculateFieldTransformerEditorProps,
+  CalculateFieldTransformerEditorState
+> {
+  constructor(props: CalculateFieldTransformerEditorProps) {
+    super(props);
+    this.state = {
+      include: props.options.include || '',
+      names: [],
+      selected: [],
+    };
+  }
+
+  componentDidMount() {
+    this.initOptions();
+  }
+
+  private initOptions() {
+    const { input, options } = this.props;
+    const configuredOptions = options.include ? options.include.split('|') : [];
+
+    const allNames: string[] = [];
+    const byName: KeyValue<boolean> = {};
+    for (const frame of input) {
+      for (const field of frame.fields) {
+        if (field.type !== FieldType.number) {
+          continue;
+        }
+        if (!byName[field.name]) {
+          byName[field.name] = true;
+          allNames.push(field.name);
+        }
+      }
+    }
+
+    if (configuredOptions.length) {
+      const options: string[] = [];
+      const selected: string[] = [];
+      for (const v of allNames) {
+        if (configuredOptions.includes(v)) {
+          selected.push(v);
+        }
+        options.push(v);
+      }
+
+      this.setState({
+        names: options,
+        selected: selected,
+      });
+    } else {
+      this.setState({ names: allNames, selected: [] });
+    }
+  }
+
+  onFieldToggle = (fieldName: string) => {
+    const { selected } = this.state;
+    if (selected.indexOf(fieldName) > -1) {
+      this.onChange(selected.filter(s => s !== fieldName));
+    } else {
+      this.onChange([...selected, fieldName]);
+    }
+  };
+
+  onChange = (selected: string[]) => {
+    this.setState({ selected });
+    this.props.onChange({
+      ...this.props.options,
+      include: selected.join('|'),
+    });
+  };
+
+  onToggleReplaceFields = (evt: ChangeEvent<HTMLInputElement>) => {
+    const { options } = this.props;
+    this.props.onChange({
+      ...options,
+      replaceFields: !options.replaceFields,
+    });
+  };
+
+  onAliasChanged = (evt: ChangeEvent<HTMLInputElement>) => {
+    const { options } = this.props;
+    this.props.onChange({
+      ...options,
+      alias: evt.target.value,
+    });
+  };
+
+  onStatsChange = (stats: string[]) => {
+    this.props.onChange({
+      ...this.props.options,
+      reducer: stats.length ? (stats[0] as ReducerID) : ReducerID.sum,
+    });
+  };
+
+  render() {
+    const { options } = this.props;
+    const { names, selected } = this.state;
+    const reducer = fieldReducers.get(options.reducer);
+    return (
+      <div>
+        <Label>Numeric Fields</Label>
+        <InlineList
+          items={names}
+          renderItem={(o, i) => {
+            return (
+              <span
+                className={css`
+                  margin-right: ${i === names.length - 1 ? '0' : '10px'};
+                `}
+              >
+                <FilterPill
+                  onClick={() => {
+                    this.onFieldToggle(o);
+                  }}
+                  label={o}
+                  selected={selected.indexOf(o) > -1}
+                />
+              </span>
+            );
+          }}
+        />
+        <Label>Calculation</Label>
+        <StatsPicker stats={[options.reducer]} onChange={this.onStatsChange} defaultStat={ReducerID.sum} />
+        <Label>Alias</Label>
+        <Input value={options.alias} placeholder={reducer.name} onChange={this.onAliasChanged} />
+
+        <Label>Replace all fields</Label>
+        <Switch checked={options.replaceFields} onChange={this.onToggleReplaceFields} />
+
+        {/*  nullValueMode?: NullValueMode; */}
+      </div>
+    );
+  }
+}
+
+interface FilterPillProps {
+  selected: boolean;
+  label: string;
+  onClick: React.MouseEventHandler<HTMLElement>;
+}
+const FilterPill: React.FC<FilterPillProps> = ({ label, selected, onClick }) => {
+  const theme = useContext(ThemeContext);
+  return (
+    <div
+      className={css`
+        padding: ${theme.spacing.xxs} ${theme.spacing.sm};
+        color: white;
+        background: ${selected ? theme.palette.blue95 : theme.palette.blue77};
+        border-radius: 16px;
+        display: inline-block;
+        cursor: pointer;
+      `}
+      onClick={onClick}
+    >
+      {selected && (
+        <Icon
+          className={css`
+            margin-right: 4px;
+          `}
+          name="check"
+        />
+      )}
+      {label}
+    </div>
+  );
+};
+
+export const calculateFieldTransformRegistryItem: TransformerRegistyItem<CalculateFieldTransformerOptions> = {
+  id: DataTransformerID.calculateField,
+  editor: CalculateFieldTransformerEditor,
+  transformation: standardTransformers.calculateFieldTransformer,
+  name: 'Add field from calculation',
+  description: 'Use the row values to calculate a new field',
+};

packages/grafana-ui/src/utils/standardTransformers.ts

@@ -4,6 +4,7 @@ import { filterFieldsByNameTransformRegistryItem } from '../components/Transform
 import { filterFramesByRefIdTransformRegistryItem } from '../components/TransformersUI/FilterByRefIdTransformerEditor';
 import { organizeFieldsTransformRegistryItem } from '../components/TransformersUI/OrganizeFieldsTransformerEditor';
 import { seriesToFieldsTransformerRegistryItem } from '../components/TransformersUI/SeriesToFieldsTransformerEditor';
+import { calculateFieldTransformRegistryItem } from '../components/TransformersUI/CalculateFieldTransformerEditor';
 
 export const getStandardTransformers = (): Array<TransformerRegistyItem<any>> => {
   return [
@@ -12,5 +13,6 @@ export const getStandardTransformers = (): Array<TransformerRegistyItem<any>> =>
     filterFramesByRefIdTransformRegistryItem,
     organizeFieldsTransformRegistryItem,
     seriesToFieldsTransformerRegistryItem,
+    calculateFieldTransformRegistryItem,
   ];
 };