transform: add expressions to query editor (w/ feature flag) (#20072)

for use with gel which is not released yet.

package.json

@@ -203,6 +203,7 @@
     "angular-native-dragdrop": "1.2.2",
     "angular-route": "1.6.6",
     "angular-sanitize": "1.6.6",
+    "apache-arrow": "0.15.0",
     "baron": "3.0.3",
     "brace": "0.10.0",
     "calculate-size": "1.1.1",

packages/grafana-runtime/src/config.ts

@@ -12,6 +12,7 @@ export interface BuildInfo {
 
 interface FeatureToggles {
   transformations: boolean;
+  expressions: boolean;
 }
 export class GrafanaBootConfig {
   datasources: { [str: string]: DataSourceInstanceSettings } = {};
@@ -46,6 +47,7 @@ export class GrafanaBootConfig {
   pluginsToPreload: string[] = [];
   featureToggles: FeatureToggles = {
     transformations: false,
+    expressions: false,
   };
 
   constructor(options: GrafanaBootConfig) {

pkg/api/api.go

@@ -328,7 +328,6 @@ func (hs *HTTPServer) registerRoutes() {
 		// metrics
 		apiRoute.Post("/tsdb/query", bind(dtos.MetricRequest{}), Wrap(hs.QueryMetrics))
 		apiRoute.Post("/tsdb/query/v2", bind(dtos.MetricRequest{}), Wrap(hs.QueryMetricsV2))
-		apiRoute.Post("/tsdb/transform", bind(dtos.MetricRequest{}), Wrap(hs.Transform))
 		apiRoute.Get("/tsdb/testdata/scenarios", Wrap(GetTestDataScenarios))
 		apiRoute.Get("/tsdb/testdata/gensql", reqGrafanaAdmin, Wrap(GenerateSQLTestData))
 		apiRoute.Get("/tsdb/testdata/random-walk", Wrap(GetTestDataRandomWalk))

pkg/api/metrics.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"sort"
 
+	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/setting"
 
 	"github.com/grafana/grafana/pkg/api/dtos"
@@ -21,37 +22,40 @@ func (hs *HTTPServer) QueryMetricsV2(c *m.ReqContext, reqDto dtos.MetricRequest)
 		return Error(404, "Expressions feature toggle is not enabled", nil)
 	}
 
-	timeRange := tsdb.NewTimeRange(reqDto.From, reqDto.To)
-
 	if len(reqDto.Queries) == 0 {
-		return Error(400, "No queries found in query", nil)
+		return Error(500, "No queries found in query", nil)
 	}
 
-	var datasourceID int64
-	for _, query := range reqDto.Queries {
+	request := &tsdb.TsdbQuery{
+		TimeRange: tsdb.NewTimeRange(reqDto.From, reqDto.To),
+		Debug:     reqDto.Debug,
+	}
+
+	expr := false
+	var ds *m.DataSource
+	for i, query := range reqDto.Queries {
 		name, err := query.Get("datasource").String()
 		if err != nil {
 			return Error(500, "datasource missing name", err)
 		}
-		datasourceID, err = query.Get("datasourceId").Int64()
-		if err != nil {
-			return Error(400, "GEL datasource missing ID", nil)
-		}
-		if name == "-- GEL --" {
-			break
+		if name == "__expr__" {
+			expr = true
 		}
-	}
-	ds, err := hs.DatasourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache)
-	if err != nil {
-		if err == m.ErrDataSourceAccessDenied {
-			return Error(403, "Access denied to datasource", err)
-		}
-		return Error(500, "Unable to load datasource meta data", err)
-	}
 
-	request := &tsdb.TsdbQuery{TimeRange: timeRange, Debug: reqDto.Debug}
+		datasourceID, err := query.Get("datasourceId").Int64()
+		if err != nil {
+			return Error(500, "datasource missing ID", nil)
+		}
 
-	for _, query := range reqDto.Queries {
+		if i == 0 && !expr {
+			ds, err = hs.DatasourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache)
+			if err != nil {
+				if err == m.ErrDataSourceAccessDenied {
+					return Error(403, "Access denied to datasource", err)
+				}
+				return Error(500, "Unable to load datasource meta data", err)
+			}
+		}
 		request.Queries = append(request.Queries, &tsdb.Query{
 			RefId:         query.Get("refId").MustString("A"),
 			MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
@@ -59,11 +63,21 @@ func (hs *HTTPServer) QueryMetricsV2(c *m.ReqContext, reqDto dtos.MetricRequest)
 			Model:         query,
 			DataSource:    ds,
 		})
+
 	}
 
-	resp, err := tsdb.HandleRequest(c.Req.Context(), ds, request)
-	if err != nil {
-		return Error(500, "Metric request error", err)
+	var resp *tsdb.Response
+	var err error
+	if !expr {
+		resp, err = tsdb.HandleRequest(c.Req.Context(), ds, request)
+		if err != nil {
+			return Error(500, "Metric request error", err)
+		}
+	} else {
+		resp, err = plugins.Transform.Transform(c.Req.Context(), request)
+		if err != nil {
+			return Error(500, "Transform request error", err)
+		}
 	}
 
 	statusCode := 200

pkg/api/transform.go

-package api
-
-import (
-	"net/http"
-
-	"github.com/grafana/grafana/pkg/api/dtos"
-	m "github.com/grafana/grafana/pkg/models"
-	"github.com/grafana/grafana/pkg/plugins"
-	"github.com/grafana/grafana/pkg/setting"
-	"github.com/grafana/grafana/pkg/tsdb"
-)
-
-// POST /api/tsdb/transform
-// This enpoint is tempory, will be part of v2 query endpoint.
-func (hs *HTTPServer) Transform(c *m.ReqContext, reqDto dtos.MetricRequest) Response {
-	if !setting.IsExpressionsEnabled() {
-		return Error(404, "Expressions feature toggle is not enabled", nil)
-	}
-	if plugins.Transform == nil {
-		return Error(http.StatusServiceUnavailable, "transform plugin is not loaded", nil)
-	}
-
-	timeRange := tsdb.NewTimeRange(reqDto.From, reqDto.To)
-
-	if len(reqDto.Queries) == 0 {
-		return Error(400, "No queries found in query", nil)
-	}
-
-	var datasourceID int64
-	for _, query := range reqDto.Queries {
-		name, err := query.Get("datasource").String()
-		if err != nil {
-			return Error(500, "datasource missing name", err)
-		}
-		datasourceID, err = query.Get("datasourceId").Int64()
-		if err != nil {
-			return Error(400, "GEL datasource missing ID", nil)
-		}
-		if name == "-- GEL --" {
-			break
-		}
-	}
-
-	ds, err := hs.DatasourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache)
-	if err != nil {
-		if err == m.ErrDataSourceAccessDenied {
-			return Error(403, "Access denied to datasource", err)
-		}
-		return Error(500, "Unable to load datasource meta data", err)
-	}
-
-	request := &tsdb.TsdbQuery{TimeRange: timeRange, Debug: reqDto.Debug}
-
-	for _, query := range reqDto.Queries {
-		request.Queries = append(request.Queries, &tsdb.Query{
-			RefId:         query.Get("refId").MustString("A"),
-			MaxDataPoints: query.Get("maxDataPoints").MustInt64(100),
-			IntervalMs:    query.Get("intervalMs").MustInt64(1000),
-			Model:         query,
-			DataSource:    ds,
-		})
-	}
-
-	resp, err := plugins.Transform.Transform(c.Req.Context(), ds, request)
-	if err != nil {
-		return Error(500, "Transform request error", err)
-	}
-
-	statusCode := 200
-	for _, res := range resp.Results {
-		if res.Error != nil {
-			res.ErrorString = res.Error.Error()
-			resp.Message = res.ErrorString
-			statusCode = 400
-		}
-	}
-
-	return JSON(statusCode, &resp)
-}

pkg/plugins/transform_plugin.go

@@ -27,7 +27,6 @@ type TransformPlugin struct {
 
 	Executable string `json:"executable,omitempty"`
 
-	//transform.TransformPlugin
 	*TransformWrapper
 
 	client *plugin.Client
@@ -136,22 +135,10 @@ type TransformWrapper struct {
 	api    *grafanaAPI
 }
 
-func (tw *TransformWrapper) Transform(ctx context.Context, ds *models.DataSource, query *tsdb.TsdbQuery) (*tsdb.Response, error) {
-	jsonData, err := ds.JsonData.MarshalJSON()
-	if err != nil {
-		return nil, err
-	}
-
+func (tw *TransformWrapper) Transform(ctx context.Context, query *tsdb.TsdbQuery) (*tsdb.Response, error) {
 	pbQuery := &pluginv2.TransformRequest{
-		Datasource: &pluginv2.DatasourceInfo{
-			Name:                    ds.Name,
-			Type:                    ds.Type,
-			Url:                     ds.Url,
-			Id:                      ds.Id,
-			OrgId:                   ds.OrgId,
-			JsonData:                string(jsonData),
-			DecryptedSecureJsonData: ds.SecureJsonData.Decrypt(),
-		},
+		// TODO Not sure Datasource property needs be on this?
+		Datasource: &pluginv2.DatasourceInfo{},
 		TimeRange: &pluginv2.TimeRange{
 			FromRaw:     query.TimeRange.From,
 			ToRaw:       query.TimeRange.To,
@@ -217,6 +204,7 @@ func (s *grafanaAPI) QueryDatasource(ctx context.Context, req *pluginv2.QueryDat
 	if len(req.Queries) == 0 {
 		return nil, fmt.Errorf("zero queries found in datasource request")
 	}
+
 	getDsInfo := &models.GetDataSourceByIdQuery{
 		Id:    req.DatasourceId,
 		OrgId: req.OrgId,

public/app/features/dashboard/panel_editor/QueriesTab.tsx

@@ -29,6 +29,7 @@ import { PluginHelp } from 'app/core/components/PluginHelp/PluginHelp';
 import { addQuery } from 'app/core/utils/query';
 import { Unsubscribable } from 'rxjs';
 import { isSharedDashboardQuery, DashboardQueryEditor } from 'app/plugins/datasource/dashboard';
+import { expressionDatasource, ExpressionDatasourceID } from 'app/features/expressions/ExpressionDatasource';
 
 interface Props {
   panel: PanelModel;
@@ -97,9 +98,11 @@ export class QueriesTab extends PureComponent<Props, State> {
     if (datasource.meta.mixed) {
       // Set the datasource on all targets
       panel.targets.forEach(target => {
-        target.datasource = panel.datasource;
-        if (!target.datasource) {
-          target.datasource = config.defaultDatasource;
+        if (target.datasource !== ExpressionDatasourceID) {
+          target.datasource = panel.datasource;
+          if (!target.datasource) {
+            target.datasource = config.defaultDatasource;
+          }
         }
       });
     } else if (currentDS) {
@@ -107,7 +110,9 @@ export class QueriesTab extends PureComponent<Props, State> {
       if (currentDS.meta.mixed) {
         // Remove the explicit datasource
         for (const target of panel.targets) {
-          delete target.datasource;
+          if (target.datasource !== ExpressionDatasourceID) {
+            delete target.datasource;
+          }
         }
       } else if (currentDS.meta.id !== datasource.meta.id) {
         // we are changing data source type, clear queries
@@ -150,6 +155,11 @@ export class QueriesTab extends PureComponent<Props, State> {
     this.onScrollBottom();
   };
 
+  onAddExpressionClick = () => {
+    this.onUpdateQueries(addQuery(this.props.panel.targets, expressionDatasource.newQuery()));
+    this.onScrollBottom();
+  };
+
   onScrollBottom = () => {
     this.setState({ scrollTop: this.state.scrollTop + 10000 });
   };
@@ -168,6 +178,11 @@ export class QueriesTab extends PureComponent<Props, State> {
           </button>
         )}
         {isAddingMixed && this.renderMixedPicker()}
+        {config.featureToggles.expressions && (
+          <button className="btn navbar-button" onClick={this.onAddExpressionClick}>
+            Add Expression
+          </button>
+        )}
       </>
     );
   };

public/app/features/dashboard/state/runRequest.ts

@@ -14,6 +14,7 @@ import {
   DataQueryError,
 } from '@grafana/ui';
 import { LoadingState, dateMath, toDataFrame, DataFrame, guessFieldTypes } from '@grafana/data';
+import { ExpressionDatasourceID, expressionDatasource } from 'app/features/expressions/ExpressionDatasource';
 
 type MapOfResponsePackets = { [str: string]: DataQueryResponse };
 
@@ -132,6 +133,16 @@ function cancelNetworkRequestsOnUnsubscribe(req: DataQueryRequest) {
 }
 
 export function callQueryMethod(datasource: DataSourceApi, request: DataQueryRequest) {
+  console.log('CALL', request.targets);
+
+  // If any query has an expression, use the expression endpoint
+  for (const target of request.targets) {
+    if (target.datasource === ExpressionDatasourceID) {
+      return expressionDatasource.query(request);
+    }
+  }
+
+  // Otherwise it is a standard datasource request
   const returnVal = datasource.query(request);
   return from(returnVal);
 }

public/app/features/expressions/ExpressionDatasource.ts

+import {
+  DataSourceApi,
+  DataQueryRequest,
+  DataQueryResponse,
+  DataSourceInstanceSettings,
+  DataSourcePluginMeta,
+} from '@grafana/ui';
+import { ExpressionQuery, GELQueryType } from './types';
+import { ExpressionQueryEditor } from './ExpressionQueryEditor';
+import { Observable, from } from 'rxjs';
+import { config } from '@grafana/runtime';
+import { getBackendSrv } from 'app/core/services/backend_srv';
+import { gelResponseToDataFrames } from './util';
+
+/**
+ * This is a singleton that is not actually instanciated
+ */
+export class ExpressionDatasourceApi extends DataSourceApi<ExpressionQuery> {
+  constructor(instanceSettings: DataSourceInstanceSettings) {
+    super(instanceSettings);
+  }
+
+  getCollapsedText(query: ExpressionQuery) {
+    return `Expression: ${query.type}`;
+  }
+
+  query(request: DataQueryRequest): Observable<DataQueryResponse> {
+    const { targets, intervalMs, maxDataPoints, range } = request;
+
+    const orgId = (window as any).grafanaBootData.user.orgId;
+    const queries = targets.map(q => {
+      if (q.datasource === ExpressionDatasourceID) {
+        return {
+          ...q,
+          datasourceId: this.id,
+          orgId,
+        };
+      }
+      const ds = config.datasources[q.datasource || config.defaultDatasource];
+      return {
+        ...q,
+        datasourceId: ds.id,
+        intervalMs,
+        maxDataPoints,
+        orgId,
+        // ?? alias: templateSrv.replace(q.alias || ''),
+      };
+    });
+    const req: Promise<DataQueryResponse> = getBackendSrv()
+      .post('/api/tsdb/query/v2', {
+        from: range.from.valueOf().toString(),
+        to: range.to.valueOf().toString(),
+        queries: queries,
+      })
+      .then((rsp: any) => {
+        return { data: gelResponseToDataFrames(rsp) } as DataQueryResponse;
+      });
+    return from(req);
+  }
+
+  testDatasource() {
+    return Promise.resolve({});
+  }
+
+  newQuery(): ExpressionQuery {
+    return {
+      refId: '--', // Replaced with query
+      type: GELQueryType.math,
+      datasource: ExpressionDatasourceID,
+    };
+  }
+}
+
+export const ExpressionDatasourceID = '__expr__';
+export const expressionDatasource = new ExpressionDatasourceApi({
+  id: -100,
+  name: ExpressionDatasourceID,
+} as DataSourceInstanceSettings);
+expressionDatasource.meta = {
+  id: ExpressionDatasourceID,
+} as DataSourcePluginMeta;
+expressionDatasource.components = {
+  QueryEditor: ExpressionQueryEditor,
+};

public/app/features/expressions/ExpressionQueryEditor.tsx

+// Libraries
+import React, { PureComponent, ChangeEvent } from 'react';
+
+import { FormLabel, QueryEditorProps, Select, FormField } from '@grafana/ui';
+import { SelectableValue, ReducerID } from '@grafana/data';
+
+// Types
+import { ExpressionQuery, GELQueryType } from './types';
+import { ExpressionDatasourceApi } from './ExpressionDatasource';
+
+type Props = QueryEditorProps<ExpressionDatasourceApi, ExpressionQuery>;
+
+interface State {}
+
+const gelTypes: Array<SelectableValue<GELQueryType>> = [
+  { value: GELQueryType.math, label: 'Math' },
+  { value: GELQueryType.reduce, label: 'Reduce' },
+  { value: GELQueryType.resample, label: 'Resample' },
+];
+
+const reducerTypes: Array<SelectableValue<string>> = [
+  { value: ReducerID.min, label: 'Min', description: 'Get the minimum value' },
+  { value: ReducerID.max, label: 'Max', description: 'Get the maximum value' },
+  { value: ReducerID.mean, label: 'Mean', description: 'Get the average value' },
+  { value: ReducerID.sum, label: 'Sum', description: 'Get the sum of all values' },
+  { value: ReducerID.count, label: 'Count', description: 'Get the number of values' },
+];
+
+const downsamplingTypes: Array<SelectableValue<string>> = [
+  { value: ReducerID.min, label: 'Min', description: 'Fill with the minimum value' },
+  { value: ReducerID.max, label: 'Max', description: 'Fill with the maximum value' },
+  { value: ReducerID.mean, label: 'Mean', description: 'Fill with the average value' },
+  { value: ReducerID.sum, label: 'Sum', description: 'Fill with the sum of all values' },
+];
+
+const upsamplingTypes: Array<SelectableValue<string>> = [
+  { value: 'pad', label: 'pad', description: 'fill with the last known value' },
+  { value: 'backfilling', label: 'backfilling', description: 'fill with the next known value' },
+  { value: 'fillna', label: 'fillna', description: 'Fill with NaNs' },
+];
+
+export class ExpressionQueryEditor extends PureComponent<Props, State> {
+  state = {};
+
+  onSelectGELType = (item: SelectableValue<GELQueryType>) => {
+    const { query, onChange } = this.props;
+    const q = {
+      ...query,
+      type: item.value!,
+    };
+
+    if (q.type === GELQueryType.reduce) {
+      if (!q.reducer) {
+        q.reducer = ReducerID.mean;
+      }
+      q.expression = undefined;
+    } else if (q.type === GELQueryType.resample) {
+      if (!q.downsampler) {
+        q.downsampler = ReducerID.mean;
+      }
+      if (!q.upsampler) {
+        q.upsampler = 'fillna';
+      }
+      q.reducer = undefined;
+    } else {
+      q.reducer = undefined;
+    }
+
+    onChange(q);
+  };
+
+  onSelectReducer = (item: SelectableValue<string>) => {
+    const { query, onChange } = this.props;
+    onChange({
+      ...query,
+      reducer: item.value!,
+    });
+  };
+
+  onSelectUpsampler = (item: SelectableValue<string>) => {
+    const { query, onChange } = this.props;
+    onChange({
+      ...query,
+      upsampler: item.value!,
+    });
+  };
+
+  onSelectDownsampler = (item: SelectableValue<string>) => {
+    const { query, onChange } = this.props;
+    onChange({
+      ...query,
+      downsampler: item.value!,
+    });
+  };
+
+  onRuleReducer = (item: SelectableValue<string>) => {
+    const { query, onChange } = this.props;
+    onChange({
+      ...query,
+      rule: item.value!,
+    });
+  };
+
+  onExpressionChange = (evt: ChangeEvent<any>) => {
+    const { query, onChange } = this.props;
+    onChange({
+      ...query,
+      expression: evt.target.value,
+    });
+  };
+
+  onRuleChange = (evt: ChangeEvent<any>) => {
+    const { query, onChange } = this.props;
+    onChange({
+      ...query,
+      rule: evt.target.value,
+    });
+  };
+
+  render() {
+    const { query } = this.props;
+    const selected = gelTypes.find(o => o.value === query.type);
+    const reducer = reducerTypes.find(o => o.value === query.reducer);
+    const downsampler = downsamplingTypes.find(o => o.value === query.downsampler);
+    const upsampler = upsamplingTypes.find(o => o.value === query.upsampler);
+
+    return (
+      <div>
+        <div className="form-field">
+          <Select options={gelTypes} value={selected} onChange={this.onSelectGELType} />
+          {query.type === GELQueryType.reduce && (
+            <>
+              <FormLabel width={5}>Function:</FormLabel>
+              <Select options={reducerTypes} value={reducer} onChange={this.onSelectReducer} />
+              <FormField label="Fields:" labelWidth={5} onChange={this.onExpressionChange} value={query.expression} />
+            </>
+          )}
+        </div>
+        {query.type === GELQueryType.math && (
+          <textarea value={query.expression} onChange={this.onExpressionChange} className="gf-form-input" rows={2} />
+        )}
+        {query.type === GELQueryType.resample && (
+          <>
+            <div>
+              <FormField label="Series:" labelWidth={5} onChange={this.onExpressionChange} value={query.expression} />
+              <FormField label="Rule:" labelWidth={5} onChange={this.onRuleChange} value={query.rule} />
+            </div>
+            <div>
+              <FormLabel width={12}>Downsample Function:</FormLabel>
+              <Select options={downsamplingTypes} value={downsampler} onChange={this.onSelectDownsampler} />
+              <FormLabel width={12}>Upsample Function:</FormLabel>
+              <Select options={upsamplingTypes} value={upsampler} onChange={this.onSelectUpsampler} />
+            </div>
+          </>
+        )}
+      </div>
+    );
+  }
+}

public/app/features/expressions/__snapshots__/util.test.ts.snap

+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`GEL Utils should parse output with dataframe 1`] = `
+Array [
+  Object {
+    "fields": Array [
+      Object {
+        "config": Object {},
+        "name": "Time",
+        "type": "time",
+        "values": Int32Array [
+          882710016,
+          365389179,
+          1587742720,
+          365389180,
+          -2002191872,
+          365389181,
+          -1297159168,
+          365389182,
+          -592126464,
+          365389183,
+          112906240,
+          365389185,
+          817938944,
+          365389186,
+          1522971648,
+          365389187,
+          -2066962944,
+          365389188,
+          -1361930240,
+          365389189,
+          -656897536,
+          365389190,
+          48135168,
+          365389192,
+          753167872,
+          365389193,
+        ],
+      },
+      Object {
+        "config": Object {},
+        "name": "",
+        "type": "number",
+        "values": Float64Array [
+          3,
+          3,
+          3,
+          5,
+          5,
+          5,
+          3,
+          3,
+          3,
+          5,
+          5,
+          5,
+          3,
+        ],
+      },
+    ],
+    "labels": undefined,
+    "meta": undefined,
+    "name": undefined,
+    "refId": undefined,
+  },
+  Object {
+    "fields": Array [
+      Object {
+        "config": Object {},
+        "name": "Time",
+        "type": "time",
+        "values": Int32Array [
+          882710016,
+          365389179,
+          1587742720,
+          365389180,
+          -2002191872,
+          365389181,
+          -1297159168,
+          365389182,
+          -592126464,
+          365389183,
+          112906240,
+          365389185,
+          817938944,
+          365389186,
+          1522971648,
+          365389187,
+          -2066962944,
+          365389188,
+          -1361930240,
+          365389189,
+          -656897536,
+          365389190,
+          48135168,
+          365389192,
+          753167872,
+          365389193,
+        ],
+      },
+      Object {
+        "config": Object {},
+        "name": "GB-series",
+        "type": "number",
+        "values": Float64Array [
+          0,
+          0,
+          0,
+          2,
+          2,
+          2,
+          0,
+          0,
+          0,
+          2,
+          2,
+          2,
+          0,
+        ],
+      },
+    ],
+    "labels": undefined,
+    "meta": undefined,
+    "name": undefined,
+    "refId": undefined,
+  },
+]
+`;

public/app/features/expressions/types.ts

+import { DataQuery } from '@grafana/ui';
+
+export enum GELQueryType {
+  math = 'math',
+  reduce = 'reduce',
+  resample = 'resample',
+}
+
+/**
+ * For now this is a single object to cover all the types.... would likely
+ * want to split this up by type as the complexity increases
+ */
+export interface ExpressionQuery extends DataQuery {
+  type: GELQueryType;
+  reducer?: string;
+  expression?: string;
+  rule?: string;
+  downsampler?: string;
+  upsampler?: string;
+}

public/app/features/expressions/util.test.ts

+import { gelResponseToDataFrames } from './util';
+import { toDataFrameDTO } from '@grafana/data';
+
+/* tslint:disable */
+const resp = {
+  results: {
+    '': {
+      refId: '',
+      dataframes: [
+        'QVJST1cxAACsAQAAEAAAAAAACgAOAAwACwAEAAoAAAAUAAAAAAAAAQMACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAOD+//8IAAAADAAAAAIAAABHQwAABQAAAHJlZklkAAAAAP///wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAAlAAAAAQAAACG////FAAAAGAAAABgAAAAAAADAWAAAAACAAAALAAAAAQAAABQ////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAAB0////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAAAAABm////AAACAAAAAAAAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAAbAAAAHQAAAAAAAoBdAAAAAIAAAA0AAAABAAAANz///8IAAAAEAAAAAQAAAB0aW1lAAAAAAQAAAB0eXBlAAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAAC8AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA0AAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAWAAAAA0AAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABoAAAAAAAAAGgAAAAAAAAAAAAAAAAAAABoAAAAAAAAAGgAAAAAAAAAAAAAAAIAAAANAAAAAAAAAAAAAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAAAAFp00e2XHFQAIo158ZccVAPqoiH1lxxUA7K6yfmXHFQDetNx/ZccVANC6BoFlxxUAwsAwgmXHFQC0xlqDZccVAKbMhIRlxxUAmNKuhWXHFQCK2NiGZccVAHzeAohlxxUAbuQsiWXHFQAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAUQAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAUQAAAAAAAAAhAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAMAAQAAALgBAAAAAAAAwAAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABQAAAAAgAAACgAAAAEAAAA4P7//wgAAAAMAAAAAgAAAEdDAAAFAAAAcmVmSWQAAAAA////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAIAAACUAAAABAAAAIb///8UAAAAYAAAAGAAAAAAAAMBYAAAAAIAAAAsAAAABAAAAFD///8IAAAAEAAAAAYAAABudW1iZXIAAAQAAAB0eXBlAAAAAHT///8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAAAAAGb///8AAAIAAAAAAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABsAAAAdAAAAAAACgF0AAAAAgAAADQAAAAEAAAA3P///wgAAAAQAAAABAAAAHRpbWUAAAAABAAAAHR5cGUAAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAVGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAABUaW1lAAAAANgBAABBUlJPVzE=',
+        'QVJST1cxAAC8AQAAEAAAAAAACgAOAAwACwAEAAoAAAAUAAAAAAAAAQMACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAND+//8IAAAADAAAAAIAAABHQgAABQAAAHJlZklkAAAA8P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAApAAAAAQAAAB2////FAAAAGgAAABoAAAAAAADAWgAAAACAAAALAAAAAQAAABA////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAABk////CAAAABQAAAAJAAAAR0Itc2VyaWVzAAAABAAAAG5hbWUAAAAAAAAAAF7///8AAAIACQAAAEdCLXNlcmllcwASABgAFAATABIADAAAAAgABAASAAAAFAAAAGwAAAB0AAAAAAAKAXQAAAACAAAANAAAAAQAAADc////CAAAABAAAAAEAAAAdGltZQAAAAAEAAAAdHlwZQAAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAvAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAANAAAAAAAAAAFAAAAAAAAAMDAAoAGAAMAAgABAAKAAAAFAAAAFgAAAANAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaAAAAAAAAABoAAAAAAAAAAAAAAAAAAAAaAAAAAAAAABoAAAAAAAAAAAAAAACAAAADQAAAAAAAAAAAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAAAAABadNHtlxxUACKNefGXHFQD6qIh9ZccVAOyusn5lxxUA3rTcf2XHFQDQugaBZccVAMLAMIJlxxUAtMZag2XHFQCmzISEZccVAJjSroVlxxUAitjYhmXHFQB83gKIZccVAG7kLIllxxUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAABAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA4AAAAAAADAAEAAADIAQAAAAAAAMAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAND+//8IAAAADAAAAAIAAABHQgAABQAAAHJlZklkAAAA8P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAApAAAAAQAAAB2////FAAAAGgAAABoAAAAAAADAWgAAAACAAAALAAAAAQAAABA////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAABk////CAAAABQAAAAJAAAAR0Itc2VyaWVzAAAABAAAAG5hbWUAAAAAAAAAAF7///8AAAIACQAAAEdCLXNlcmllcwASABgAFAATABIADAAAAAgABAASAAAAFAAAAGwAAAB0AAAAAAAKAXQAAAACAAAANAAAAAQAAADc////CAAAABAAAAAEAAAAdGltZQAAAAAEAAAAdHlwZQAAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAA6AEAAEFSUk9XMQ==',
+      ],
+      series: [] as any[],
+      tables: null as any,
+      frames: null as any,
+    },
+  },
+};
+/* tslint:enable */
+
+describe('GEL Utils', () => {
+  // test('should parse sample GEL output', () => {
+  //   const frames = gelResponseToDataFrames(resp);
+  //   const frame = frames[0];
+  //   expect(frame.name).toEqual('BBB');
+  //   expect(frame.fields.length).toEqual(2);
+  //   expect(frame.length).toEqual(resp.Frames[0].fields[0].values.length);
+
+  //   const timeField = frame.fields[0];
+  //   expect(timeField.name).toEqual('Time');
+
+  //   // The whole response
+  //   expect(frames).toMatchSnapshot();
+  // });
+
+  test('should parse output with dataframe', () => {
+    const frames = gelResponseToDataFrames(resp);
+    for (const frame of frames) {
+      console.log('Frame', frame.refId + ' // ' + frame.labels);
+      for (const field of frame.fields) {
+        console.log(' > ', field.name, field.values.toArray());
+      }
+    }
+
+    const norm = frames.map(f => toDataFrameDTO(f));
+    expect(norm).toMatchSnapshot();
+  });
+});

public/app/features/expressions/util.ts

+import { DataFrame, FieldType, Field, Vector } from '@grafana/data';
+import { Table, ArrowType } from 'apache-arrow';
+
+export function base64StringToArrowTable(text: string) {
+  const b64 = atob(text);
+  const arr = Uint8Array.from(b64, c => {
+    return c.charCodeAt(0);
+  });
+  return Table.from(arr);
+}
+
+export function arrowTableToDataFrame(table: Table): DataFrame {
+  const fields: Field[] = [];
+  for (let i = 0; i < table.numCols; i++) {
+    const col = table.getColumnAt(i);
+    if (col) {
+      const schema = table.schema.fields[i];
+      let type = FieldType.other;
+      const values: Vector<any> = col;
+      switch ((schema.typeId as unknown) as ArrowType) {
+        case ArrowType.Decimal:
+        case ArrowType.Int:
+        case ArrowType.FloatingPoint: {
+          type = FieldType.number;
+          break;
+        }
+        case ArrowType.Bool: {
+          type = FieldType.boolean;
+          break;
+        }
+        case ArrowType.Timestamp: {
+          type = FieldType.time;
+          break;
+        }
+        default:
+          console.log('UNKNOWN Type:', schema);
+      }
+
+      fields.push({
+        name: col.name,
+        type,
+        config: {}, // TODO, pull from metadata
+        values,
+      });
+    }
+  }
+  return {
+    fields,
+    length: table.length,
+  };
+}
+
+export function gelResponseToDataFrames(rsp: any): DataFrame[] {
+  const frames: DataFrame[] = [];
+  for (const res of Object.values(rsp.results)) {
+    for (const b of (res as any).dataframes) {
+      const t = base64StringToArrowTable(b as string);
+      frames.push(arrowTableToDataFrame(t));
+    }
+  }
+  return frames;
+}

public/app/features/plugins/datasource_srv.ts

@@ -14,6 +14,9 @@ import { auto } from 'angular';
 import { TemplateSrv } from '../templating/template_srv';
 import { GrafanaRootScope } from 'app/routes/GrafanaCtrl';
 
+// Pretend Datasource
+import { expressionDatasource } from 'app/features/expressions/ExpressionDatasource';
+
 export class DatasourceSrv implements DataSourceService {
   datasources: { [name: string]: DataSourceApi };
 
@@ -56,6 +59,12 @@ export class DatasourceSrv implements DataSourceService {
   }
 
   loadDatasource(name: string): Promise<DataSourceApi> {
+    // Expression Datasource (not a real datasource)
+    if (name === expressionDatasource.name) {
+      this.datasources[name] = expressionDatasource;
+      return this.$q.when(expressionDatasource);
+    }
+
     const dsConfig = config.datasources[name];
     if (!dsConfig) {
       return this.$q.reject({ message: 'Datasource named ' + name + ' was not found' });