@grafana/data: Matchers and Transforms (#16756)

* add extension framework

* add filter transformer

* more logging

* adding more tests

* make stats an extension

* make stats an extension

* test registry init

* first get a function, then call it

* move files to data package

* not used

* update to columnar

* Add more tests for nameMatcher

* Fix invert predicate

* add fluent API

* remove calc snapshot

* split Field matchers and Frame matchers

* split filter transformers

* Fix typo

packages/grafana-data/src/utils/index.ts

@@ -22,3 +22,8 @@ export { getMappedValue } from './valueMappings';
 import * as dateMath from './datemath';
 import * as rangeUtil from './rangeutil';
 export { dateMath, rangeUtil };
+
+export * from './matchers/ids';
+export * from './matchers/matchers';
+export * from './transformers/ids';
+export * from './transformers/transformers';

packages/grafana-data/src/utils/matchers/fieldTypeMatcher.test.ts

+import { FieldType } from '../../types/dataFrame';
+import { fieldMatchers } from './matchers';
+import { FieldMatcherID } from './ids';
+import { toDataFrame } from '../processDataFrame';
+
+export const simpleSeriesWithTypes = toDataFrame({
+  fields: [
+    { name: 'A', type: FieldType.time },
+    { name: 'B', type: FieldType.boolean },
+    { name: 'C', type: FieldType.string },
+  ],
+});
+
+describe('Field Type Matcher', () => {
+  const matcher = fieldMatchers.get(FieldMatcherID.byType);
+  it('finds numbers', () => {
+    for (const field of simpleSeriesWithTypes.fields) {
+      const matches = matcher.get(FieldType.number);
+      expect(matches(field)).toBe(field.type === FieldType.number);
+    }
+  });
+});

packages/grafana-data/src/utils/matchers/fieldTypeMatcher.ts

+import { Field, FieldType } from '../../types/dataFrame';
+import { FieldMatcherInfo } from './matchers';
+import { FieldMatcherID } from './ids';
+
+// General Field matcher
+const fieldTypeMacher: FieldMatcherInfo<FieldType> = {
+  id: FieldMatcherID.byType,
+  name: 'Field Type',
+  description: 'match based on the field type',
+  defaultOptions: FieldType.number,
+
+  get: (type: FieldType) => {
+    return (field: Field) => {
+      return type === field.type;
+    };
+  },
+
+  getOptionsDisplayText: (type: FieldType) => {
+    return `Field type: ${type}`;
+  },
+};
+
+// Numeric Field matcher
+// This gets its own entry so it shows up in the dropdown
+const numericMacher: FieldMatcherInfo = {
+  id: FieldMatcherID.numeric,
+  name: 'Numeric Fields',
+  description: 'Fields with type number',
+
+  get: () => {
+    return fieldTypeMacher.get(FieldType.number);
+  },
+
+  getOptionsDisplayText: () => {
+    return 'Numeric Fields';
+  },
+};
+
+// Time Field matcher
+const timeMacher: FieldMatcherInfo = {
+  id: FieldMatcherID.time,
+  name: 'Time Fields',
+  description: 'Fields with type time',
+
+  get: () => {
+    return fieldTypeMacher.get(FieldType.time);
+  },
+
+  getOptionsDisplayText: () => {
+    return 'Time Fields';
+  },
+};
+
+/**
+ * Registry Initalization
+ */
+export function getFieldTypeMatchers(): FieldMatcherInfo[] {
+  return [fieldTypeMacher, numericMacher, timeMacher];
+}

packages/grafana-data/src/utils/matchers/ids.ts

+// This needs to be in its own file to avoid circular references
+
+// Builtin Predicates
+// not using 'any' and 'never' since they are reservered keywords
+export enum MatcherID {
+  anyMatch = 'anyMatch', // checks children
+  allMatch = 'allMatch', // checks children
+  invertMatch = 'invertMatch', // checks child
+  alwaysMatch = 'alwaysMatch',
+  neverMatch = 'neverMatch',
+}
+
+export enum FieldMatcherID {
+  // Specific Types
+  numeric = 'numeric',
+  time = 'time',
+
+  // With arguments
+  byType = 'byType',
+  byName = 'byName',
+  // byIndex = 'byIndex',
+  // byLabel = 'byLabel',
+}
+
+/**
+ * Field name matchers
+ */
+export enum FrameMatcherID {
+  byName = 'byName',
+  byRefId = 'byRefId',
+  byIndex = 'byIndex',
+  byLabel = 'byLabel',
+}

packages/grafana-data/src/utils/matchers/matchers.test.ts

+import { fieldMatchers } from './matchers';
+import { FieldMatcherID } from './ids';
+
+describe('Matchers', () => {
+  it('should load all matchers', () => {
+    for (const name of Object.keys(FieldMatcherID)) {
+      const matcher = fieldMatchers.get(name);
+      expect(matcher.id).toBe(name);
+    }
+  });
+});

packages/grafana-data/src/utils/matchers/matchers.ts

+import { Field, DataFrame } from '../../types/dataFrame';
+import { Registry, RegistryItemWithOptions } from '../registry';
+
+export type FieldMatcher = (field: Field) => boolean;
+export type FrameMatcher = (frame: DataFrame) => boolean;
+
+export interface FieldMatcherInfo<TOptions = any> extends RegistryItemWithOptions<TOptions> {
+  get: (options: TOptions) => FieldMatcher;
+}
+
+export interface FrameMatcherInfo<TOptions = any> extends RegistryItemWithOptions<TOptions> {
+  get: (options: TOptions) => FrameMatcher;
+}
+
+export interface MatcherConfig<TOptions = any> {
+  id: string;
+  options?: TOptions;
+}
+
+// Load the Buildtin matchers
+import { getFieldPredicateMatchers, getFramePredicateMatchers } from './predicates';
+import { getFieldNameMatchers, getFrameNameMatchers } from './nameMatcher';
+import { getFieldTypeMatchers } from './fieldTypeMatcher';
+import { getRefIdMatchers } from './refIdMatcher';
+
+export const fieldMatchers = new Registry<FieldMatcherInfo>(() => {
+  return [
+    ...getFieldPredicateMatchers(), // Predicates
+    ...getFieldTypeMatchers(), // by type
+    ...getFieldNameMatchers(), // by name
+  ];
+});
+
+export const frameMatchers = new Registry<FrameMatcherInfo>(() => {
+  return [
+    ...getFramePredicateMatchers(), // Predicates
+    ...getFrameNameMatchers(), // by name
+    ...getRefIdMatchers(), // by query refId
+  ];
+});
+
+export function getFieldMatcher(config: MatcherConfig): FieldMatcher {
+  const info = fieldMatchers.get(config.id);
+  if (!info) {
+    throw new Error('Unknown Matcher: ' + config.id);
+  }
+  return info.get(config.options);
+}
+
+export function getFrameMatchers(config: MatcherConfig): FrameMatcher {
+  const info = frameMatchers.get(config.id);
+  if (!info) {
+    throw new Error('Unknown Matcher: ' + config.id);
+  }
+  return info.get(config.options);
+}

packages/grafana-data/src/utils/matchers/nameMatcher.test.ts

+import { getFieldMatcher } from './matchers';
+import { FieldMatcherID } from './ids';
+import { toDataFrame } from '../processDataFrame';
+
+describe('Field Name Matcher', () => {
+  it('Match all with wildcard regex', () => {
+    const seriesWithNames = toDataFrame({
+      fields: [{ name: 'A hello world' }, { name: 'AAA' }, { name: 'C' }],
+    });
+    const config = {
+      id: FieldMatcherID.byName,
+      options: '/.*/',
+    };
+
+    const matcher = getFieldMatcher(config);
+
+    for (const field of seriesWithNames.fields) {
+      expect(matcher(field)).toBe(true);
+    }
+  });
+
+  it('Match all with decimals regex', () => {
+    const seriesWithNames = toDataFrame({
+      fields: [{ name: '12' }, { name: '112' }, { name: '13' }],
+    });
+    const config = {
+      id: FieldMatcherID.byName,
+      options: '/^\\d+$/',
+    };
+
+    const matcher = getFieldMatcher(config);
+
+    for (const field of seriesWithNames.fields) {
+      expect(matcher(field)).toBe(true);
+    }
+  });
+
+  it('Match complex regex', () => {
+    const seriesWithNames = toDataFrame({
+      fields: [{ name: 'some.instance.path' }, { name: '112' }, { name: '13' }],
+    });
+    const config = {
+      id: FieldMatcherID.byName,
+      options: '/\\b(?:\\S+?\\.)+\\S+\\b$/',
+    };
+
+    const matcher = getFieldMatcher(config);
+    let resultCount = 0;
+    for (const field of seriesWithNames.fields) {
+      if (matcher(field)) {
+        resultCount++;
+      }
+      expect(resultCount).toBe(1);
+    }
+  });
+});

packages/grafana-data/src/utils/matchers/nameMatcher.ts

+import { Field, DataFrame } from '../../types/dataFrame';
+import { FieldMatcherInfo, FrameMatcherInfo } from './matchers';
+import { FieldMatcherID, FrameMatcherID } from './ids';
+import { stringToJsRegex } from '../string';
+
+// General Field matcher
+const fieldNameMacher: FieldMatcherInfo<string> = {
+  id: FieldMatcherID.byName,
+  name: 'Field Name',
+  description: 'match the field name',
+  defaultOptions: '/.*/',
+
+  get: (pattern: string) => {
+    const regex = stringToJsRegex(pattern);
+    return (field: Field) => {
+      return regex.test(field.name);
+    };
+  },
+
+  getOptionsDisplayText: (pattern: string) => {
+    return `Field name: ${pattern}`;
+  },
+};
+
+// General Field matcher
+const frameNameMacher: FrameMatcherInfo<string> = {
+  id: FrameMatcherID.byName,
+  name: 'Frame Name',
+  description: 'match the frame name',
+  defaultOptions: '/.*/',
+
+  get: (pattern: string) => {
+    const regex = stringToJsRegex(pattern);
+    return (frame: DataFrame) => {
+      return regex.test(frame.name || '');
+    };
+  },
+
+  getOptionsDisplayText: (pattern: string) => {
+    return `Frame name: ${pattern}`;
+  },
+};
+
+/**
+ * Registry Initalization
+ */
+export function getFieldNameMatchers(): FieldMatcherInfo[] {
+  return [fieldNameMacher];
+}
+
+export function getFrameNameMatchers(): FrameMatcherInfo[] {
+  return [frameNameMacher];
+}

packages/grafana-data/src/utils/matchers/predicates.test.ts

+import { FieldType } from '../../types/dataFrame';
+import { MatcherConfig, fieldMatchers } from './matchers';
+import { simpleSeriesWithTypes } from './fieldTypeMatcher.test';
+import { FieldMatcherID, MatcherID } from './ids';
+
+const matchesNumberConfig: MatcherConfig = {
+  id: FieldMatcherID.byType,
+  options: FieldType.number,
+};
+const matchesTimeConfig: MatcherConfig = {
+  id: FieldMatcherID.byType,
+  options: FieldType.time,
+};
+const both = [matchesNumberConfig, matchesTimeConfig];
+
+describe('Check Predicates', () => {
+  it('can not match both', () => {
+    const matches = fieldMatchers.get(MatcherID.allMatch).get(both);
+    for (const field of simpleSeriesWithTypes.fields) {
+      expect(matches(field)).toBe(false);
+    }
+  });
+
+  it('match either time or number', () => {
+    const matches = fieldMatchers.get(MatcherID.anyMatch).get(both);
+    for (const field of simpleSeriesWithTypes.fields) {
+      expect(matches(field)).toBe(field.type === FieldType.number || field.type === FieldType.time);
+    }
+  });
+
+  it('match not time', () => {
+    const matches = fieldMatchers.get(MatcherID.invertMatch).get(matchesTimeConfig);
+    for (const field of simpleSeriesWithTypes.fields) {
+      expect(matches(field)).toBe(field.type !== FieldType.time);
+    }
+  });
+});

packages/grafana-data/src/utils/matchers/predicates.ts

+import { Field, DataFrame } from '../../types/dataFrame';
+import { MatcherID } from './ids';
+import {
+  FrameMatcherInfo,
+  FieldMatcherInfo,
+  MatcherConfig,
+  getFieldMatcher,
+  fieldMatchers,
+  getFrameMatchers,
+  frameMatchers,
+} from './matchers';
+
+const anyFieldMatcher: FieldMatcherInfo<MatcherConfig[]> = {
+  id: MatcherID.anyMatch,
+  name: 'Any',
+  description: 'Any child matches (OR)',
+  excludeFromPicker: true,
+  defaultOptions: [], // empty array
+
+  get: (options: MatcherConfig[]) => {
+    const children = options.map(option => {
+      return getFieldMatcher(option);
+    });
+    return (field: Field) => {
+      for (const child of children) {
+        if (child(field)) {
+          return true;
+        }
+      }
+      return false;
+    };
+  },
+
+  getOptionsDisplayText: (options: MatcherConfig[]) => {
+    let text = '';
+    for (const sub of options) {
+      if (text.length > 0) {
+        text += ' OR ';
+      }
+      const matcher = fieldMatchers.get(sub.id);
+      text += matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(sub) : matcher.name;
+    }
+    return text;
+  },
+};
+
+const anyFrameMatcher: FrameMatcherInfo<MatcherConfig[]> = {
+  id: MatcherID.anyMatch,
+  name: 'Any',
+  description: 'Any child matches (OR)',
+  excludeFromPicker: true,
+  defaultOptions: [], // empty array
+
+  get: (options: MatcherConfig[]) => {
+    const children = options.map(option => {
+      return getFrameMatchers(option);
+    });
+    return (frame: DataFrame) => {
+      for (const child of children) {
+        if (child(frame)) {
+          return true;
+        }
+      }
+      return false;
+    };
+  },
+
+  getOptionsDisplayText: (options: MatcherConfig[]) => {
+    let text = '';
+    for (const sub of options) {
+      if (text.length > 0) {
+        text += ' OR ';
+      }
+      const matcher = frameMatchers.get(sub.id);
+      text += matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(sub) : matcher.name;
+    }
+    return text;
+  },
+};
+
+const allFieldsMatcher: FieldMatcherInfo<MatcherConfig[]> = {
+  id: MatcherID.allMatch,
+  name: 'All',
+  description: 'Everything matches (AND)',
+  excludeFromPicker: true,
+  defaultOptions: [], // empty array
+
+  get: (options: MatcherConfig[]) => {
+    const children = options.map(option => {
+      return getFieldMatcher(option);
+    });
+    return (field: Field) => {
+      for (const child of children) {
+        if (!child(field)) {
+          return false;
+        }
+      }
+      return true;
+    };
+  },
+
+  getOptionsDisplayText: (options: MatcherConfig[]) => {
+    let text = '';
+    for (const sub of options) {
+      if (text.length > 0) {
+        text += ' AND ';
+      }
+      const matcher = fieldMatchers.get(sub.id); // Ugho what about frame
+      text += matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(sub) : matcher.name;
+    }
+    return text;
+  },
+};
+
+const allFramesMatcher: FrameMatcherInfo<MatcherConfig[]> = {
+  id: MatcherID.allMatch,
+  name: 'All',
+  description: 'Everything matches (AND)',
+  excludeFromPicker: true,
+  defaultOptions: [], // empty array
+
+  get: (options: MatcherConfig[]) => {
+    const children = options.map(option => {
+      return getFrameMatchers(option);
+    });
+    return (frame: DataFrame) => {
+      for (const child of children) {
+        if (!child(frame)) {
+          return false;
+        }
+      }
+      return true;
+    };
+  },
+
+  getOptionsDisplayText: (options: MatcherConfig[]) => {
+    let text = '';
+    for (const sub of options) {
+      if (text.length > 0) {
+        text += ' AND ';
+      }
+      const matcher = frameMatchers.get(sub.id);
+      text += matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(sub) : matcher.name;
+    }
+    return text;
+  },
+};
+
+const notFieldMatcher: FieldMatcherInfo<MatcherConfig> = {
+  id: MatcherID.invertMatch,
+  name: 'NOT',
+  description: 'Inverts other matchers',
+  excludeFromPicker: true,
+
+  get: (option: MatcherConfig) => {
+    const check = getFieldMatcher(option);
+    return (field: Field) => {
+      return !check(field);
+    };
+  },
+
+  getOptionsDisplayText: (options: MatcherConfig) => {
+    const matcher = fieldMatchers.get(options.id);
+    const text = matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(options.options) : matcher.name;
+    return 'NOT ' + text;
+  },
+};
+
+const notFrameMatcher: FrameMatcherInfo<MatcherConfig> = {
+  id: MatcherID.invertMatch,
+  name: 'NOT',
+  description: 'Inverts other matchers',
+  excludeFromPicker: true,
+
+  get: (option: MatcherConfig) => {
+    const check = getFrameMatchers(option);
+    return (frame: DataFrame) => {
+      return !check(frame);
+    };
+  },
+
+  getOptionsDisplayText: (options: MatcherConfig) => {
+    const matcher = frameMatchers.get(options.id);
+    const text = matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(options.options) : matcher.name;
+    return 'NOT ' + text;
+  },
+};
+
+export const alwaysFieldMatcher = (field: Field) => {
+  return true;
+};
+
+export const alwaysFrameMatcher = (frame: DataFrame) => {
+  return true;
+};
+
+export const neverFieldMatcher = (field: Field) => {
+  return false;
+};
+
+export const neverFrameMatcher = (frame: DataFrame) => {
+  return false;
+};
+
+const alwaysFieldMatcherInfo: FieldMatcherInfo = {
+  id: MatcherID.alwaysMatch,
+  name: 'All Fields',
+  description: 'Always Match',
+
+  get: (option: any) => {
+    return alwaysFieldMatcher;
+  },
+
+  getOptionsDisplayText: (options: any) => {
+    return 'Always';
+  },
+};
+
+const alwaysFrameMatcherInfo: FrameMatcherInfo = {
+  id: MatcherID.alwaysMatch,
+  name: 'All Frames',
+  description: 'Always Match',
+
+  get: (option: any) => {
+    return alwaysFrameMatcher;
+  },
+
+  getOptionsDisplayText: (options: any) => {
+    return 'Always';
+  },
+};
+
+const neverFieldMatcherInfo: FieldMatcherInfo = {
+  id: MatcherID.neverMatch,
+  name: 'No Fields',
+  description: 'Never Match',
+  excludeFromPicker: true,
+
+  get: (option: any) => {
+    return neverFieldMatcher;
+  },
+
+  getOptionsDisplayText: (options: any) => {
+    return 'Never';
+  },
+};
+
+const neverFrameMatcherInfo: FrameMatcherInfo = {
+  id: MatcherID.neverMatch,
+  name: 'No Frames',
+  description: 'Never Match',
+
+  get: (option: any) => {
+    return neverFrameMatcher;
+  },
+
+  getOptionsDisplayText: (options: any) => {
+    return 'Never';
+  },
+};
+
+export function getFieldPredicateMatchers(): FieldMatcherInfo[] {
+  return [anyFieldMatcher, allFieldsMatcher, notFieldMatcher, alwaysFieldMatcherInfo, neverFieldMatcherInfo];
+}
+
+export function getFramePredicateMatchers(): FrameMatcherInfo[] {
+  return [anyFrameMatcher, allFramesMatcher, notFrameMatcher, alwaysFrameMatcherInfo, neverFrameMatcherInfo];
+}

packages/grafana-data/src/utils/matchers/refIdMatcher.ts

+import { DataFrame } from '../../types/dataFrame';
+import { FrameMatcherInfo } from './matchers';
+import { FrameMatcherID } from './ids';
+
+// General Field matcher
+const refIdMacher: FrameMatcherInfo<string> = {
+  id: FrameMatcherID.byRefId,
+  name: 'Query refId',
+  description: 'match the refId',
+  defaultOptions: 'A',
+
+  get: (pattern: string) => {
+    return (frame: DataFrame) => {
+      return pattern === frame.refId;
+    };
+  },
+
+  getOptionsDisplayText: (pattern: string) => {
+    return `RefID: ${pattern}`;
+  },
+};
+
+export function getRefIdMatchers(): FrameMatcherInfo[] {
+  return [refIdMacher];
+}

packages/grafana-data/src/utils/registry.ts

@@ -13,6 +13,18 @@ export interface RegistryItem {
   excludeFromPicker?: boolean;
 }
 
+export interface RegistryItemWithOptions<TOptions = any> extends RegistryItem {
+  /**
+   * Convert the options to a string
+   */
+  getOptionsDisplayText?: (options: TOptions) => string;
+
+  /**
+   * Default options used if nothing else is specified
+   */
+  defaultOptions?: TOptions;
+}
+
 interface RegistrySelectInfo {
   options: Array<SelectableValue<string>>;
   current: Array<SelectableValue<string>>;

packages/grafana-data/src/utils/transformers/__snapshots__/reduce.test.ts.snap

+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Reducer Transformer filters by include 1`] = `
+Object {
+  "fields": Array [
+    Object {
+      "config": Object {},
+      "name": "Field",
+      "type": "string",
+      "values": Array [
+        "A",
+        "B",
+      ],
+    },
+    Object {
+      "config": Object {
+        "title": "First",
+      },
+      "name": "first",
+      "type": "number",
+      "values": Array [
+        1,
+        "a",
+      ],
+    },
+    Object {
+      "config": Object {
+        "title": "Min",
+      },
+      "name": "min",
+      "type": "number",
+      "values": Array [
+        1,
+        null,
+      ],
+    },
+    Object {
+      "config": Object {
+        "title": "Max",
+      },
+      "name": "max",
+      "type": "number",
+      "values": Array [
+        4,
+        null,
+      ],
+    },
+    Object {
+      "config": Object {
+        "title": "Delta",
+      },
+      "name": "delta",
+      "type": "number",
+      "values": Array [
+        3,
+        0,
+      ],
+    },
+  ],
+  "labels": undefined,
+  "meta": Object {
+    "transformations": Array [
+      "reduce",
+    ],
+  },
+  "name": undefined,
+  "refId": undefined,
+}
+`;

packages/grafana-data/src/utils/transformers/append.test.ts

+import { transformDataFrame, dataTransformers } from './transformers';
+import { DataTransformerID } from './ids';
+import { toDataFrame } from '../processDataFrame';
+
+const seriesAB = toDataFrame({
+  columns: [{ text: 'A' }, { text: 'B' }],
+  rows: [
+    [1, 100], // A,B
+    [2, 200], // A,B
+  ],
+});
+
+const seriesBC = toDataFrame({
+  columns: [{ text: 'A' }, { text: 'C' }],
+  rows: [
+    [3, 3000], // A,C
+    [4, 4000], // A,C
+  ],
+});
+
+describe('Append Transformer', () => {
+  it('filters by include', () => {
+    const cfg = {
+      id: DataTransformerID.append,
+      options: {},
+    };
+    const x = dataTransformers.get(DataTransformerID.append);
+    expect(x.id).toBe(cfg.id);
+
+    const processed = transformDataFrame([cfg], [seriesAB, seriesBC])[0];
+    expect(processed.fields.length).toBe(3);
+
+    const fieldA = processed.fields[0];
+    const fieldB = processed.fields[1];
+    const fieldC = processed.fields[2];
+
+    expect(fieldA.values.toArray()).toEqual([1, 2, 3, 4]);
+    expect(fieldB.values.toArray()).toEqual([100, 200, undefined, undefined]);
+    expect(fieldC.values.toArray()).toEqual([undefined, undefined, 3000, 4000]);
+  });
+});

packages/grafana-data/src/utils/transformers/append.ts

+import { DataTransformerInfo } from './transformers';
+import { DataFrame } from '../../types/dataFrame';
+import { DataTransformerID } from './ids';
+import { DataFrameHelper } from '../dataFrameHelper';
+import { KeyValue } from '../../types/data';
+import { AppendedVectors } from '../vector';
+
+export interface AppendOptions {}
+
+export const appendTransformer: DataTransformerInfo<AppendOptions> = {
+  id: DataTransformerID.append,
+  name: 'Append',
+  description: 'Append values into a single DataFrame.  This uses the name as the key',
+  defaultOptions: {},
+
+  /**
+   * Return a modified copy of the series.  If the transform is not or should not
+   * be applied, just return the input series
+   */
+  transformer: (options: AppendOptions) => {
+    return (data: DataFrame[]) => {
+      if (data.length < 2) {
+        return data;
+      }
+
+      let length = 0;
+      const processed = new DataFrameHelper();
+      for (let i = 0; i < data.length; i++) {
+        const frame = data[i];
+        const used: KeyValue<boolean> = {};
+        for (let j = 0; j < frame.fields.length; j++) {
+          const src = frame.fields[j];
+          if (used[src.name]) {
+            continue;
+          }
+          used[src.name] = true;
+
+          let f = processed.getFieldByName(src.name);
+          if (!f) {
+            f = processed.addField({
+              ...src,
+              values: new AppendedVectors(length),
+            });
+          }
+          (f.values as AppendedVectors).append(src.values);
+        }
+
+        // Make sure all fields have their length updated
+        length += frame.length;
+        processed.length = length;
+        for (const f of processed.fields) {
+          (f.values as AppendedVectors).setLength(processed.length);
+        }
+      }
+      return [processed];
+    };
+  },
+};

packages/grafana-data/src/utils/transformers/filter.test.ts

+import { FieldType } from '../../types/dataFrame';
+import { FieldMatcherID } from '../matchers/ids';
+import { transformDataFrame } from './transformers';
+import { DataTransformerID } from './ids';
+import { toDataFrame } from '../processDataFrame';
+
+export const simpleSeriesWithTypes = toDataFrame({
+  fields: [
+    { name: 'A', type: FieldType.time, values: [1000, 2000] },
+    { name: 'B', type: FieldType.boolean, values: [true, false] },
+    { name: 'C', type: FieldType.string, values: ['a', 'b'] },
+    { name: 'D', type: FieldType.number, values: [1, 2] },
+  ],
+});
+
+describe('Filter Transformer', () => {
+  it('filters by include', () => {
+    const cfg = {
+      id: DataTransformerID.filterFields,
+      options: {
+        include: { id: FieldMatcherID.numeric },
+      },
+    };
+
+    const filtered = transformDataFrame([cfg], [simpleSeriesWithTypes])[0];
+    expect(filtered.fields.length).toBe(1);
+    expect(filtered.fields[0].name).toBe('D');
+  });
+});

packages/grafana-data/src/utils/transformers/filter.ts

+import { DataTransformerInfo, NoopDataTransformer } from './transformers';
+import { DataFrame, Field } from '../../types/dataFrame';
+import { FieldMatcherID } from '../matchers/ids';
+import { DataTransformerID } from './ids';
+import { MatcherConfig, getFieldMatcher, getFrameMatchers } from '../matchers/matchers';
+
+export interface FilterOptions {
+  include?: MatcherConfig;
+  exclude?: MatcherConfig;
+}
+
+export const filterFieldsTransformer: DataTransformerInfo<FilterOptions> = {
+  id: DataTransformerID.filterFields,
+  name: 'Filter Fields',
+  description: 'select a subset of fields',
+  defaultOptions: {
+    include: { id: FieldMatcherID.numeric },
+  },
+
+  /**
+   * Return a modified copy of the series.  If the transform is not or should not
+   * be applied, just return the input series
+   */
+  transformer: (options: FilterOptions) => {
+    if (!options.include && !options.exclude) {
+      return NoopDataTransformer;
+    }
+
+    const include = options.include ? getFieldMatcher(options.include) : null;
+    const exclude = options.exclude ? getFieldMatcher(options.exclude) : null;
+
+    return (data: DataFrame[]) => {
+      const processed: DataFrame[] = [];
+      for (const series of data) {
+        // Find the matching field indexes
+        const fields: Field[] = [];
+        for (let i = 0; i < series.fields.length; i++) {
+          const field = series.fields[i];
+          if (exclude) {
+            if (exclude(field)) {
+              continue;
+            }
+            if (!include) {
+              fields.push(field);
+            }
+          }
+          if (include && include(field)) {
+            fields.push(field);
+          }
+        }
+
+        if (!fields.length) {
+          continue;
+        }
+        const copy = {
+          ...series, // all the other properties
+          fields, // but a different set of fields
+        };
+        processed.push(copy);
+      }
+      return processed;
+    };
+  },
+};
+
+export const filterFramesTransformer: DataTransformerInfo<FilterOptions> = {
+  id: DataTransformerID.filterFrames,
+  name: 'Filter Frames',
+  description: 'select a subset of frames',
+  defaultOptions: {},
+
+  /**
+   * Return a modified copy of the series.  If the transform is not or should not
+   * be applied, just return the input series
+   */
+  transformer: (options: FilterOptions) => {
+    if (!options.include && !options.exclude) {
+      return NoopDataTransformer;
+    }
+
+    const include = options.include ? getFrameMatchers(options.include) : null;
+    const exclude = options.exclude ? getFrameMatchers(options.exclude) : null;
+
+    return (data: DataFrame[]) => {
+      const processed: DataFrame[] = [];
+      for (const series of data) {
+        if (exclude) {
+          if (exclude(series)) {
+            continue;
+          }
+          if (!include) {
+            processed.push(series);
+          }
+        }
+        if (include && include(series)) {
+          processed.push(series);
+        }
+      }
+      return processed;
+    };
+  },
+};

packages/grafana-data/src/utils/transformers/ids.ts

+export enum DataTransformerID {
+  //  join = 'join', // Pick a field and merge all series based on that field
+  append = 'append', // Merge all series together
+  //  rotate = 'rotate', // Columns to rows
+  reduce = 'reduce', // Run calculations on fields
+
+  filterFields = 'filterFields', // Pick some fields (keep all frames)
+  filterFrames = 'filterFrames', // Pick some frames (keep all fields)
+}

packages/grafana-data/src/utils/transformers/reduce.test.ts

+import { transformDataFrame } from './transformers';
+import { ReducerID } from '../fieldReducer';
+import { DataTransformerID } from './ids';
+import { toDataFrame, toDataFrameDTO } from '../processDataFrame';
+
+const seriesWithValues = toDataFrame({
+  fields: [
+    { name: 'A', values: [1, 2, 3, 4] }, // Numbers
+    { name: 'B', values: ['a', 'b', 'c', 'd'] }, // Strings
+  ],
+});
+
+describe('Reducer Transformer', () => {
+  it('filters by include', () => {
+    const cfg = {
+      id: DataTransformerID.reduce,
+      options: {
+        reducers: [ReducerID.first, ReducerID.min, ReducerID.max, ReducerID.delta],
+      },
+    };
+    const processed = transformDataFrame([cfg], [seriesWithValues])[0];
+    expect(processed.fields.length).toBe(5);
+    expect(toDataFrameDTO(processed)).toMatchSnapshot();
+  });
+});

packages/grafana-data/src/utils/transformers/reduce.ts

+import { DataTransformerInfo } from './transformers';
+import { DataFrame, FieldType, Field } from '../../types/dataFrame';
+import { MatcherConfig, getFieldMatcher } from '../matchers/matchers';
+import { alwaysFieldMatcher } from '../matchers/predicates';
+import { DataTransformerID } from './ids';
+import { ReducerID, fieldReducers, reduceField } from '../fieldReducer';
+import { KeyValue } from '../../types/data';
+import { ArrayVector } from '../vector';
+import { guessFieldTypeForField } from '../processDataFrame';
+
+export interface ReduceOptions {
+  reducers: string[];
+  fields?: MatcherConfig; // Assume all fields
+}
+
+export const reduceTransformer: DataTransformerInfo<ReduceOptions> = {
+  id: DataTransformerID.reduce,
+  name: 'Reducer',
+  description: 'Return a DataFrame with the reduction results',
+  defaultOptions: {
+    calcs: [ReducerID.min, ReducerID.max, ReducerID.mean, ReducerID.last],
+  },
+
+  /**
+   * Return a modified copy of the series.  If the transform is not or should not
+   * be applied, just return the input series
+   */
+  transformer: (options: ReduceOptions) => {
+    const matcher = options.fields ? getFieldMatcher(options.fields) : alwaysFieldMatcher;
+    const calculators = fieldReducers.list(options.reducers);
+    const reducers = calculators.map(c => c.id);
+
+    return (data: DataFrame[]) => {
+      const processed: DataFrame[] = [];
+      for (const series of data) {
+        const values: ArrayVector[] = [];
+        const fields: Field[] = [];
+        const byId: KeyValue<ArrayVector> = {};
+        values.push(new ArrayVector()); // The name
+        fields.push({
+          name: 'Field',
+          type: FieldType.string,
+          values: values[0],
+          config: {},
+        });
+        for (const info of calculators) {
+          const vals = new ArrayVector();
+          byId[info.id] = vals;
+          values.push(vals);
+          fields.push({
+            name: info.id,
+            type: FieldType.other, // UNKNOWN until after we call the functions
+            values: values[values.length - 1],
+            config: {
+              title: info.name,
+              // UNIT from original field?
+            },
+          });
+        }
+        for (let i = 0; i < series.fields.length; i++) {
+          const field = series.fields[i];
+          if (matcher(field)) {
+            const results = reduceField({
+              field,
+              reducers,
+            });
+            // Update the name list
+            values[0].buffer.push(field.name);
+            for (const info of calculators) {
+              const v = results[info.id];
+              byId[info.id].buffer.push(v);
+            }
+          }
+        }
+        for (const f of fields) {
+          const t = guessFieldTypeForField(f);
+          if (t) {
+            f.type = t;
+          }
+        }
+        processed.push({
+          ...series, // Same properties, different fields
+          fields,
+          length: values[0].length,
+        });
+      }
+      return processed;
+    };
+  },
+};

packages/grafana-data/src/utils/transformers/transformers.test.ts

+import { DataTransformerID } from './ids';
+import { dataTransformers } from './transformers';
+import { toDataFrame } from '../processDataFrame';
+import { ReducerID } from '../fieldReducer';
+import { DataFrameView } from '../dataFrameView';
+
+describe('Transformers', () => {
+  it('should load all transformeres', () => {
+    for (const name of Object.keys(DataTransformerID)) {
+      const calc = dataTransformers.get(name);
+      expect(calc.id).toBe(name);
+    }
+  });
+
+  const seriesWithValues = toDataFrame({
+    fields: [
+      { name: 'A', values: [1, 2, 3, 4] }, // Numbers
+      { name: 'B', values: ['a', 'b', 'c', 'd'] }, // Strings
+    ],
+  });
+
+  it('should use fluent API', () => {
+    const results = dataTransformers.reduce([seriesWithValues], {
+      reducers: [ReducerID.first],
+    });
+    expect(results.length).toBe(1);
+
+    const view = new DataFrameView(results[0]).toJSON();
+    expect(view).toEqual([
+      { Field: 'A', first: 1 }, // Row 0
+      { Field: 'B', first: 'a' }, // Row 1
+    ]);
+  });
+});

packages/grafana-data/src/utils/transformers/transformers.ts

+import { DataFrame } from '../../types/dataFrame';
+import { Registry, RegistryItemWithOptions } from '../registry';
+
+/**
+ * Immutable data transformation
+ */
+export type DataTransformer = (data: DataFrame[]) => DataFrame[];
+
+export interface DataTransformerInfo<TOptions = any> extends RegistryItemWithOptions {
+  transformer: (options: TOptions) => DataTransformer;
+}
+
+export interface DataTransformerConfig<TOptions = any> {
+  id: string;
+  options: TOptions;
+}
+
+// Transformer that does nothing
+export const NoopDataTransformer = (data: DataFrame[]) => data;
+
+/**
+ * Apply configured transformations to the input data
+ */
+export function transformDataFrame(options: DataTransformerConfig[], data: DataFrame[]): DataFrame[] {
+  let processed = data;
+  for (const config of options) {
+    const info = dataTransformers.get(config.id);
+    const transformer = info.transformer(config.options);
+    const after = transformer(processed);
+
+    // Add a key to the metadata if the data changed
+    if (after && after !== processed) {
+      for (const series of after) {
+        if (!series.meta) {
+          series.meta = {};
+        }
+        if (!series.meta.transformations) {
+          series.meta.transformations = [info.id];
+        } else {
+          series.meta.transformations = [...series.meta.transformations, info.id];
+        }
+      }
+      processed = after;
+    }
+  }
+  return processed;
+}
+
+// Initalize the Registry
+
+import { appendTransformer, AppendOptions } from './append';
+import { reduceTransformer, ReduceOptions } from './reduce';
+import { filterFieldsTransformer, filterFramesTransformer } from './filter';
+
+/**
+ * Registry of transformation options that can be driven by
+ * stored configuration files.
+ */
+class TransformerRegistry extends Registry<DataTransformerInfo> {
+  // ------------------------------------------------------------
+  // Nacent options for more functional programming
+  // The API to these functions should change to match the actual
+  // needs of people trying to use it.
+  //  filterFields|Frames is left off since it is likely easier to
+  //  support with `frames.filter( f => {...} )`
+  // ------------------------------------------------------------
+
+  append(data: DataFrame[], options?: AppendOptions): DataFrame | undefined {
+    return appendTransformer.transformer(options || appendTransformer.defaultOptions)(data)[0];
+  }
+
+  reduce(data: DataFrame[], options: ReduceOptions): DataFrame[] {
+    return reduceTransformer.transformer(options)(data);
+  }
+}
+
+export const dataTransformers = new TransformerRegistry(() => [
+  filterFieldsTransformer,
+  filterFramesTransformer,
+  appendTransformer,
+  reduceTransformer,
+]);

packages/grafana-data/src/utils/vector.test.ts

-import { ConstantVector, ScaledVector, ArrayVector, CircularVector } from './vector';
+import { ConstantVector, ScaledVector, ArrayVector, CircularVector, AppendedVectors } from './vector';
 
 describe('Check Proxy Vector', () => {
   it('should support constant values', () => {
@@ -156,3 +156,24 @@ describe('Check Circular Vector', () => {
     expect(v.toArray()).toEqual([3, 4, 5]);
   });
 });
+
+describe('Check Appending Vector', () => {
+  it('should transparently join them', () => {
+    const appended = new AppendedVectors();
+    appended.append(new ArrayVector([1, 2, 3]));
+    appended.append(new ArrayVector([4, 5, 6]));
+    appended.append(new ArrayVector([7, 8, 9]));
+    expect(appended.length).toEqual(9);
+
+    appended.setLength(5);
+    expect(appended.length).toEqual(5);
+    appended.append(new ArrayVector(['a', 'b', 'c']));
+    expect(appended.length).toEqual(8);
+    expect(appended.toArray()).toEqual([1, 2, 3, 4, 5, 'a', 'b', 'c']);
+
+    appended.setLength(2);
+    appended.setLength(6);
+    appended.append(new ArrayVector(['x', 'y', 'z']));
+    expect(appended.toArray()).toEqual([1, 2, undefined, undefined, undefined, undefined, 'x', 'y', 'z']);
+  });
+});

packages/grafana-data/src/utils/vector.ts

@@ -44,11 +44,8 @@ export class ConstantVector<T = any> implements Vector<T> {
   }
 
   toArray(): T[] {
-    const arr: T[] = [];
-    for (let i = 0; i < this.length; i++) {
-      arr[i] = this.value;
-    }
-    return arr;
+    const arr = new Array<T>(this.length);
+    return arr.fill(this.value);
   }
 
   toJSON(): T[] {
@@ -226,3 +223,74 @@ export class CircularVector<T = any> implements Vector<T> {
     return vectorToArray(this);
   }
 }
+
+interface AppendedVectorInfo<T> {
+  start: number;
+  end: number;
+  values: Vector<T>;
+}
+
+/**
+ * This may be more trouble than it is worth.  This trades some computation time for
+ * RAM -- rather than allocate a new array the size of all previous arrays, this just
+ * points the correct index to their original array values
+ */
+export class AppendedVectors<T = any> implements Vector<T> {
+  length = 0;
+  source: Array<AppendedVectorInfo<T>> = new Array<AppendedVectorInfo<T>>();
+
+  constructor(startAt = 0) {
+    this.length = startAt;
+  }
+
+  /**
+   * Make the vector look like it is this long
+   */
+  setLength(length: number) {
+    if (length > this.length) {
+      // make the vector longer (filling with undefined)
+      this.length = length;
+    } else if (length < this.length) {
+      // make the array shorter
+      const sources: Array<AppendedVectorInfo<T>> = new Array<AppendedVectorInfo<T>>();
+      for (const src of this.source) {
+        sources.push(src);
+        if (src.end > length) {
+          src.end = length;
+          break;
+        }
+      }
+      this.source = sources;
+      this.length = length;
+    }
+  }
+
+  append(v: Vector<T>): AppendedVectorInfo<T> {
+    const info = {
+      start: this.length,
+      end: this.length + v.length,
+      values: v,
+    };
+    this.length = info.end;
+    this.source.push(info);
+    return info;
+  }
+
+  get(index: number): T {
+    for (let i = 0; i < this.source.length; i++) {
+      const src = this.source[i];
+      if (index >= src.start && index < src.end) {
+        return src.values.get(index - src.start);
+      }
+    }
+    return (undefined as unknown) as T;
+  }
+
+  toArray(): T[] {
+    return vectorToArray(this);
+  }
+
+  toJSON(): T[] {
+    return vectorToArray(this);
+  }
+}