Transform: added merge transform that will merge multiple series/tables into one table (#25490)

* wip: added draft of series to rows.

* wip: building dataFrame structure first and then adding data.

* wip: added some refactorings of the seriesToRows transformer.

* did some refactorings to make the code easier to follow.

* added an editor for the transform.

* renamed some of the test data.

* added docs.

* fixed according to feedback.

* renamved files.

* fixed docs according to feedback.

* fixed so we don't keep labels or config values from.

* removed unused field.

* fixed spelling errors.

* fixed docs according to feedback.

docs/sources/panels/transformations.md

@@ -67,6 +67,7 @@ Grafana comes with the following transformations:
   - [Apply a transformation](#apply-a-transformation)
   - [Apply a transformation](#apply-a-transformation)
   - [Transformation types and options](#transformation-types-and-options)
   - [Transformation types and options](#transformation-types-and-options)
     - [Reduce](#reduce)
     - [Reduce](#reduce)
+    - [Merge](#merge)
     - [Filter by name](#filter-by-name)
     - [Filter by name](#filter-by-name)
     - [Filter data by query](#filter-data-by-query)
     - [Filter data by query](#filter-data-by-query)
     - [Organize fields](#organize-fields)
     - [Organize fields](#organize-fields)
@@ -93,6 +94,28 @@ After I apply the transformation, there is no time value and each column has bee
 
 
 {{< docs-imagebox img="/img/docs/transformations/reduce-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}}
 {{< docs-imagebox img="/img/docs/transformations/reduce-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}}
 
 
+### Merge
+
+Use this transformation to combine the result from multiple queries into one single result based on the time field. This is helpful when using the table panel visualization. 
+
+In the example below, we are visualizing multiple queries returning table data before applying the transformation.
+
+{{< docs-imagebox img="/img/docs/transformations/table-data-before-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}}
+
+Here is the same example after applying the merge transformation.
+
+{{< docs-imagebox img="/img/docs/transformations/table-data-after-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}}
+
+If any of the queries return time series data, then a `Metric` column containing the name of the query is added. You can be customized this value by defining `Label` on the source query.
+
+In the example below, we are visualizing multiple queries returning time series data before applying the transformation.
+
+{{< docs-imagebox img="/img/docs/transformations/time-series-before-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}}
+
+Here is the same example after applying the merge transformation.
+
+{{< docs-imagebox img="/img/docs/transformations/time-series-after-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}}
+
 ### Filter by name
 ### Filter by name
 
 
 Use this transformation to remove portions of the query results.
 Use this transformation to remove portions of the query results.

packages/grafana-data/src/dataframe/processDataFrame.ts

@@ -23,6 +23,7 @@ import { MutableDataFrame } from './MutableDataFrame';
 import { SortedVector } from '../vector/SortedVector';
 import { SortedVector } from '../vector/SortedVector';
 import { ArrayDataFrame } from './ArrayDataFrame';
 import { ArrayDataFrame } from './ArrayDataFrame';
 import { getFieldDisplayName } from '../field/fieldState';
 import { getFieldDisplayName } from '../field/fieldState';
+import { fieldIndexComparer } from '../field/fieldComparers';
 
 
 function convertTableToDataFrame(table: TableData): DataFrame {
 function convertTableToDataFrame(table: TableData): DataFrame {
   const fields = table.columns.map(c => {
   const fields = table.columns.map(c => {
@@ -391,31 +392,10 @@ export function sortDataFrame(data: DataFrame, sortIndex?: number, reverse = fal
   for (let i = 0; i < data.length; i++) {
   for (let i = 0; i < data.length; i++) {
     index.push(i);
     index.push(i);
   }
   }
-  const values = field.values;
 
 
-  // Numeric Comparison
-  let compare = (a: number, b: number) => {
-    const vA = values.get(a);
-    const vB = values.get(b);
-    return vA - vB; // works for numbers!
-  };
-
-  // String Comparison
-  if (field.type === FieldType.string) {
-    compare = (a: number, b: number) => {
-      const vA: string = values.get(a);
-      const vB: string = values.get(b);
-      return vA.localeCompare(vB);
-    };
-  }
-
-  // Run the sort function
-  index.sort(compare);
-  if (reverse) {
-    index.reverse();
-  }
+  const fieldComparer = fieldIndexComparer(field, reverse);
+  index.sort(fieldComparer);
 
 
-  // Return a copy that maps sorted values
   return {
   return {
     ...data,
     ...data,
     fields: data.fields.map(f => {
     fields: data.fields.map(f => {

packages/grafana-data/src/datetime/moment_wrapper.ts

@@ -58,6 +58,7 @@ export interface DateTime extends Object {
   fromNow: (withoutSuffix?: boolean) => string;
   fromNow: (withoutSuffix?: boolean) => string;
   from: (formaInput: DateTimeInput) => string;
   from: (formaInput: DateTimeInput) => string;
   isSame: (input?: DateTimeInput, granularity?: DurationUnit) => boolean;
   isSame: (input?: DateTimeInput, granularity?: DurationUnit) => boolean;
+  isBefore: (input?: DateTimeInput) => boolean;
   isValid: () => boolean;
   isValid: () => boolean;
   local: () => DateTime;
   local: () => DateTime;
   locale: (locale: string) => DateTime;
   locale: (locale: string) => DateTime;

packages/grafana-data/src/field/fieldComparers.ts

+import { Field, FieldType } from '../types/dataFrame';
+import { Vector } from '../types/vector';
+import { dateTime } from '../datetime';
+import isNumber from 'lodash/isNumber';
+
+type IndexComparer = (a: number, b: number) => number;
+
+export const fieldIndexComparer = (field: Field, reverse = false): IndexComparer => {
+  const values = field.values;
+
+  switch (field.type) {
+    case FieldType.number:
+      return numericIndexComparer(values, reverse);
+    case FieldType.string:
+      return stringIndexComparer(values, reverse);
+    case FieldType.boolean:
+      return booleanIndexComparer(values, reverse);
+    case FieldType.time:
+      return timeIndexComparer(values, reverse);
+    default:
+      return naturalIndexComparer(reverse);
+  }
+};
+
+export const timeComparer = (a: any, b: any): number => {
+  if (!a || !b) {
+    return falsyComparer(a, b);
+  }
+
+  if (isNumber(a) && isNumber(b)) {
+    return numericComparer(a, b);
+  }
+
+  if (dateTime(a).isBefore(b)) {
+    return -1;
+  }
+
+  if (dateTime(b).isBefore(a)) {
+    return 1;
+  }
+
+  return 0;
+};
+
+export const numericComparer = (a: number, b: number): number => {
+  return a - b;
+};
+
+export const stringComparer = (a: string, b: string): number => {
+  if (!a || !b) {
+    return falsyComparer(a, b);
+  }
+  return a.localeCompare(b);
+};
+
+export const booleanComparer = (a: boolean, b: boolean): number => {
+  return falsyComparer(a, b);
+};
+
+const falsyComparer = (a: any, b: any): number => {
+  if (!a && b) {
+    return 1;
+  }
+
+  if (a && !b) {
+    return -1;
+  }
+
+  return 0;
+};
+
+const timeIndexComparer = (values: Vector<any>, reverse: boolean): IndexComparer => {
+  return (a: number, b: number): number => {
+    const vA = values.get(a);
+    const vB = values.get(b);
+    return reverse ? timeComparer(vB, vA) : timeComparer(vA, vB);
+  };
+};
+
+const booleanIndexComparer = (values: Vector<any>, reverse: boolean): IndexComparer => {
+  return (a: number, b: number): number => {
+    const vA: boolean = values.get(a);
+    const vB: boolean = values.get(b);
+    return reverse ? booleanComparer(vB, vA) : booleanComparer(vA, vB);
+  };
+};
+
+const numericIndexComparer = (values: Vector<any>, reverse: boolean): IndexComparer => {
+  return (a: number, b: number): number => {
+    const vA: number = values.get(a);
+    const vB: number = values.get(b);
+    return reverse ? numericComparer(vB, vA) : numericComparer(vA, vB);
+  };
+};
+
+const stringIndexComparer = (values: Vector<any>, reverse: boolean): IndexComparer => {
+  return (a: number, b: number): number => {
+    const vA: string = values.get(a);
+    const vB: string = values.get(b);
+    return reverse ? stringComparer(vB, vA) : stringComparer(vA, vB);
+  };
+};
+
+const naturalIndexComparer = (reverse: boolean): IndexComparer => {
+  return (a: number, b: number): number => {
+    return reverse ? numericComparer(b, a) : numericComparer(a, b);
+  };
+};

packages/grafana-data/src/transformations/transformers.ts

@@ -11,6 +11,7 @@ import { seriesToColumnsTransformer } from './transformers/seriesToColumns';
 import { renameFieldsTransformer } from './transformers/rename';
 import { renameFieldsTransformer } from './transformers/rename';
 import { labelsToFieldsTransformer } from './transformers/labelsToFields';
 import { labelsToFieldsTransformer } from './transformers/labelsToFields';
 import { ensureColumnsTransformer } from './transformers/ensureColumns';
 import { ensureColumnsTransformer } from './transformers/ensureColumns';
+import { mergeTransformer } from './transformers/merge/merge';
 
 
 export const standardTransformers = {
 export const standardTransformers = {
   noopTransformer,
   noopTransformer,
@@ -27,4 +28,5 @@ export const standardTransformers = {
   renameFieldsTransformer,
   renameFieldsTransformer,
   labelsToFieldsTransformer,
   labelsToFieldsTransformer,
   ensureColumnsTransformer,
   ensureColumnsTransformer,
+  mergeTransformer,
 };
 };

packages/grafana-data/src/transformations/transformers/ids.ts

@@ -8,6 +8,7 @@ export enum DataTransformerID {
   rename = 'rename',
   rename = 'rename',
   calculateField = 'calculateField',
   calculateField = 'calculateField',
   seriesToColumns = 'seriesToColumns',
   seriesToColumns = 'seriesToColumns',
+  merge = 'merge',
   labelsToFields = 'labelsToFields',
   labelsToFields = 'labelsToFields',
   filterFields = 'filterFields',
   filterFields = 'filterFields',
   filterFieldsByName = 'filterFieldsByName',
   filterFieldsByName = 'filterFieldsByName',

packages/grafana-data/src/transformations/transformers/merge/DataFrameBuilder.ts

+import { MutableDataFrame } from '../../../dataframe';
+import {
+  DataFrame,
+  FieldType,
+  Field,
+  TIME_SERIES_TIME_FIELD_NAME,
+  TIME_SERIES_VALUE_FIELD_NAME,
+} from '../../../types/dataFrame';
+import { ArrayVector } from '../../../vector';
+import { omit } from 'lodash';
+import { getFrameDisplayName } from '../../../field';
+
+interface DataFrameBuilderResult {
+  dataFrame: MutableDataFrame;
+  valueMapper: ValueMapper;
+}
+
+type ValueMapper = (frame: DataFrame, valueIndex: number, timeIndex: number) => Record<string, any>;
+
+const TIME_SERIES_METRIC_FIELD_NAME = 'Metric';
+
+export class DataFrameBuilder {
+  private isOnlyTimeSeries: boolean;
+  private displayMetricField: boolean;
+  private valueFields: Record<string, Field>;
+  private timeField: Field | null;
+
+  constructor() {
+    this.isOnlyTimeSeries = true;
+    this.displayMetricField = false;
+    this.valueFields = {};
+    this.timeField = null;
+  }
+
+  addFields(frame: DataFrame, timeIndex: number): void {
+    if (frame.fields.length > 2) {
+      this.isOnlyTimeSeries = false;
+    }
+
+    if (frame.fields.length === 2) {
+      this.displayMetricField = true;
+    }
+
+    for (let index = 0; index < frame.fields.length; index++) {
+      const field = frame.fields[index];
+
+      if (index === timeIndex) {
+        if (!this.timeField) {
+          this.timeField = this.copyStructure(field, TIME_SERIES_TIME_FIELD_NAME);
+        }
+        continue;
+      }
+
+      if (!this.valueFields[field.name]) {
+        this.valueFields[field.name] = this.copyStructure(field, field.name);
+      }
+    }
+  }
+
+  build(): DataFrameBuilderResult {
+    return {
+      dataFrame: this.createDataFrame(),
+      valueMapper: this.createValueMapper(),
+    };
+  }
+
+  private createValueMapper(): ValueMapper {
+    return (frame: DataFrame, valueIndex: number, timeIndex: number) => {
+      return frame.fields.reduce((values: Record<string, any>, field, index) => {
+        const value = field.values.get(valueIndex);
+
+        if (index === timeIndex) {
+          values[TIME_SERIES_TIME_FIELD_NAME] = value;
+
+          if (this.displayMetricField) {
+            values[TIME_SERIES_METRIC_FIELD_NAME] = getFrameDisplayName(frame);
+          }
+          return values;
+        }
+
+        if (this.isOnlyTimeSeries) {
+          values[TIME_SERIES_VALUE_FIELD_NAME] = value;
+          return values;
+        }
+
+        values[field.name] = value;
+        return values;
+      }, {});
+    };
+  }
+
+  private createDataFrame(): MutableDataFrame {
+    const dataFrame = new MutableDataFrame();
+
+    if (this.timeField) {
+      dataFrame.addField(this.timeField);
+
+      if (this.displayMetricField) {
+        dataFrame.addField({
+          name: TIME_SERIES_METRIC_FIELD_NAME,
+          type: FieldType.string,
+        });
+      }
+    }
+
+    const valueFields = Object.values(this.valueFields);
+
+    if (this.isOnlyTimeSeries) {
+      if (valueFields.length > 0) {
+        dataFrame.addField({
+          ...valueFields[0],
+          name: TIME_SERIES_VALUE_FIELD_NAME,
+        });
+      }
+      return dataFrame;
+    }
+
+    for (const field of valueFields) {
+      dataFrame.addField(field);
+    }
+
+    return dataFrame;
+  }
+
+  private copyStructure(field: Field, name: string): Field {
+    return {
+      ...omit(field, ['values', 'name', 'state', 'labels', 'config']),
+      name,
+      values: new ArrayVector(),
+      config: {
+        ...omit(field.config, 'displayName'),
+      },
+    };
+  }
+}

packages/grafana-data/src/transformations/transformers/merge/DataFramesStackedByTime.ts

+import { DataFrame } from '../../../types/dataFrame';
+import { timeComparer } from '../../../field/fieldComparers';
+import { sortDataFrame } from '../../../dataframe';
+import { TimeFieldsByFrame } from './TimeFieldsByFrame';
+
+interface DataFrameStackValue {
+  valueIndex: number;
+  timeIndex: number;
+  frame: DataFrame;
+}
+export class DataFramesStackedByTime {
+  private valuesPointerByFrame: Record<number, number>;
+  private dataFrames: DataFrame[];
+  private isSorted: boolean;
+
+  constructor(private timeFields: TimeFieldsByFrame) {
+    this.valuesPointerByFrame = {};
+    this.dataFrames = [];
+    this.isSorted = false;
+  }
+
+  push(frame: DataFrame): number {
+    const index = this.dataFrames.length;
+    this.valuesPointerByFrame[index] = 0;
+    this.dataFrames.push(frame);
+    return index;
+  }
+
+  pop(): DataFrameStackValue {
+    if (!this.isSorted) {
+      this.sortByTime();
+      this.isSorted = true;
+    }
+
+    const frameIndex = this.dataFrames.reduce((champion, frame, index) => {
+      const championTime = this.peekTimeValueForFrame(champion);
+      const contenderTime = this.peekTimeValueForFrame(index);
+      return timeComparer(contenderTime, championTime) >= 0 ? champion : index;
+    }, 0);
+
+    const previousPointer = this.movePointerForward(frameIndex);
+
+    return {
+      frame: this.dataFrames[frameIndex],
+      valueIndex: previousPointer,
+      timeIndex: this.timeFields.getFieldIndex(frameIndex),
+    };
+  }
+
+  getLength(): number {
+    const frames = Object.values(this.dataFrames);
+    return frames.reduce((length: number, frame) => (length += frame.length), 0);
+  }
+
+  private peekTimeValueForFrame(frameIndex: number): any {
+    const timeField = this.timeFields.getField(frameIndex);
+    const valuePointer = this.valuesPointerByFrame[frameIndex];
+    return timeField.values.get(valuePointer);
+  }
+
+  private movePointerForward(frameIndex: number): number {
+    const currentPointer = this.valuesPointerByFrame[frameIndex];
+    this.valuesPointerByFrame[frameIndex] = currentPointer + 1;
+
+    return currentPointer;
+  }
+
+  private sortByTime() {
+    this.dataFrames = this.dataFrames.map((frame, index) => {
+      const timeFieldIndex = this.timeFields.getFieldIndex(index);
+      return sortDataFrame(frame, timeFieldIndex);
+    });
+  }
+}

packages/grafana-data/src/transformations/transformers/merge/TimeFieldsByFrame.ts

+import { isNumber } from 'lodash';
+import { Field, DataFrame } from '../../../types/dataFrame';
+import { getTimeField } from '../../../dataframe';
+
+export class TimeFieldsByFrame {
+  private timeIndexByFrameIndex: Record<number, number>;
+  private timeFieldByFrameIndex: Record<number, Field>;
+
+  constructor() {
+    this.timeIndexByFrameIndex = {};
+    this.timeFieldByFrameIndex = {};
+  }
+
+  add(frameIndex: number, frame: DataFrame): void {
+    const fieldDescription = getTimeField(frame);
+    const timeIndex = fieldDescription?.timeIndex;
+    const timeField = fieldDescription?.timeField;
+
+    if (isNumber(timeIndex)) {
+      this.timeIndexByFrameIndex[frameIndex] = timeIndex;
+    }
+
+    if (timeField) {
+      this.timeFieldByFrameIndex[frameIndex] = timeField;
+    }
+  }
+
+  getField(frameIndex: number): Field {
+    return this.timeFieldByFrameIndex[frameIndex];
+  }
+
+  getFieldIndex(frameIndex: number): number {
+    return this.timeIndexByFrameIndex[frameIndex];
+  }
+
+  getLength() {
+    return Object.keys(this.timeIndexByFrameIndex).length;
+  }
+}

packages/grafana-data/src/transformations/transformers/merge/merge.ts

+import { DataTransformerID } from '../ids';
+import { DataTransformerInfo } from '../../../types/transformations';
+import { DataFrame } from '../../../types/dataFrame';
+import { DataFrameBuilder } from './DataFrameBuilder';
+import { TimeFieldsByFrame } from './TimeFieldsByFrame';
+import { DataFramesStackedByTime } from './DataFramesStackedByTime';
+
+export interface MergeTransformerOptions {}
+
+export const mergeTransformer: DataTransformerInfo<MergeTransformerOptions> = {
+  id: DataTransformerID.merge,
+  name: 'Merge series/tables',
+  description: 'Merges multiple series/tables by time into a single serie/table',
+  defaultOptions: {},
+  transformer: (options: MergeTransformerOptions) => {
+    return (data: DataFrame[]) => {
+      if (!Array.isArray(data) || data.length <= 1) {
+        return data;
+      }
+
+      const timeFields = new TimeFieldsByFrame();
+      const framesStack = new DataFramesStackedByTime(timeFields);
+      const dataFrameBuilder = new DataFrameBuilder();
+
+      for (const frame of data) {
+        const frameIndex = framesStack.push(frame);
+        timeFields.add(frameIndex, frame);
+
+        const timeIndex = timeFields.getFieldIndex(frameIndex);
+        dataFrameBuilder.addFields(frame, timeIndex);
+      }
+
+      if (data.length !== timeFields.getLength()) {
+        return data;
+      }
+
+      const { dataFrame, valueMapper } = dataFrameBuilder.build();
+
+      for (let index = 0; index < framesStack.getLength(); index++) {
+        const { frame, valueIndex, timeIndex } = framesStack.pop();
+        dataFrame.add(valueMapper(frame, valueIndex, timeIndex));
+      }
+
+      return [dataFrame];
+    };
+  },
+};

public/app/core/components/TransformersUI/MergeTransformerEditor.tsx

+import React from 'react';
+import { DataTransformerID, standardTransformers, TransformerRegistyItem, TransformerUIProps } from '@grafana/data';
+import { MergeTransformerOptions } from '@grafana/data/src/transformations/transformers/merge/merge';
+
+export const MergeTransformerEditor: React.FC<TransformerUIProps<MergeTransformerOptions>> = ({
+  input,
+  options,
+  onChange,
+}) => {
+  return null;
+};
+
+export const mergeTransformerRegistryItem: TransformerRegistyItem<MergeTransformerOptions> = {
+  id: DataTransformerID.merge,
+  editor: MergeTransformerEditor,
+  transformation: standardTransformers.mergeTransformer,
+  name: 'Merge on time',
+  description: `Merge series/tables by time and return a single table with values as rows. 
+                Useful for showing multiple time series, tables or a combination of both visualized in a table.`,
+};

public/app/core/utils/standardTransformers.ts

@@ -6,6 +6,7 @@ import { organizeFieldsTransformRegistryItem } from '../components/TransformersU
 import { seriesToFieldsTransformerRegistryItem } from '../components/TransformersUI/SeriesToFieldsTransformerEditor';
 import { seriesToFieldsTransformerRegistryItem } from '../components/TransformersUI/SeriesToFieldsTransformerEditor';
 import { calculateFieldTransformRegistryItem } from '../components/TransformersUI/CalculateFieldTransformerEditor';
 import { calculateFieldTransformRegistryItem } from '../components/TransformersUI/CalculateFieldTransformerEditor';
 import { labelsToFieldsTransformerRegistryItem } from '../components/TransformersUI/LabelsToFieldsTransformerEditor';
 import { labelsToFieldsTransformerRegistryItem } from '../components/TransformersUI/LabelsToFieldsTransformerEditor';
+import { mergeTransformerRegistryItem } from '../components/TransformersUI/MergeTransformerEditor';
 
 
 export const getStandardTransformers = (): Array<TransformerRegistyItem<any>> => {
 export const getStandardTransformers = (): Array<TransformerRegistyItem<any>> => {
   return [
   return [
@@ -16,5 +17,6 @@ export const getStandardTransformers = (): Array<TransformerRegistyItem<any>> =>
     seriesToFieldsTransformerRegistryItem,
     seriesToFieldsTransformerRegistryItem,
     calculateFieldTransformRegistryItem,
     calculateFieldTransformRegistryItem,
     labelsToFieldsTransformerRegistryItem,
     labelsToFieldsTransformerRegistryItem,
+    mergeTransformerRegistryItem,
   ];
   ];
 };
 };