BackendPlugin: Change QueryData response format (#23234)

* BackendPlugin: (wip) change response format
goes with https://github.com/grafana/grafana-plugin-sdk-go/pull/109

* fix error mapping in wrapper

* latest of my sdk branch

* latest of my sdk branch

* TransformWrapper fixes

* latest of sdk branch (removes extra meta)

* add metadata in wrappers

* also set error string

* sdk: v0.35.0

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/gorilla/websocket v1.4.1
 	github.com/gosimple/slug v1.4.2
 	github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4
-	github.com/grafana/grafana-plugin-sdk-go v0.33.0
+	github.com/grafana/grafana-plugin-sdk-go v0.35.0
 	github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd
 	github.com/hashicorp/go-plugin v1.0.1
 	github.com/hashicorp/go-version v1.1.0

go.sum

@@ -128,8 +128,8 @@ github.com/gosimple/slug v1.4.2 h1:jDmprx3q/9Lfk4FkGZtvzDQ9Cj9eAmsjzeQGp24PeiQ=
 github.com/gosimple/slug v1.4.2/go.mod h1:ER78kgg1Mv0NQGlXiDe57DpCyfbNywXXZ9mIorhxAf0=
 github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4 h1:SPdxCL9BChFTlyi0Khv64vdCW4TMna8+sxL7+Chx+Ag=
 github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4/go.mod h1:nc0XxBzjeGcrMltCDw269LoWF9S8ibhgxolCdA1R8To=
-github.com/grafana/grafana-plugin-sdk-go v0.33.0 h1:+eFcOV/KioHTTRNimENZeajlkw31B+m92RNRShooPEQ=
-github.com/grafana/grafana-plugin-sdk-go v0.33.0/go.mod h1:4rVPIvfv7SzFC0AA/8T5tmDRxIsrvDJOF9p4SrQGS1M=
+github.com/grafana/grafana-plugin-sdk-go v0.35.0 h1:IxNaNq8hN3ShQ804FURFOd1ehbKOmFROztY+8vohhW8=
+github.com/grafana/grafana-plugin-sdk-go v0.35.0/go.mod h1:zX/Zz/HYDAkL1NxffOZeixqPqIVVoCTWI2AuFy4J+V4=
 github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd h1:rNuUHR+CvK1IS89MMtcF0EpcVMZtjKfPRp4MEmt/aTs=
 github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI=
 github.com/hashicorp/go-plugin v1.0.1 h1:4OtAfUGbnKC6yS48p0CtMX2oFYtzFZVv6rok3cRWgnE=
@@ -180,6 +180,7 @@ github.com/linkedin/goavro/v2 v2.9.7 h1:Vd++Rb/RKcmNJjM0HP/JJFMEWa21eUBVKPYlKehO
 github.com/linkedin/goavro/v2 v2.9.7/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
 github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de/go.mod h1:3q8WtuPQsoRbatJuy3nvq/hRSvuBJrHHr+ybPPiNvHQ=
 github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af/go.mod h1:Cqz6pqow14VObJ7peltM+2n3PWOz7yTrfUuGbVFkzN0=
+github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/mattetti/filebuffer v1.0.0 h1:ixTvQ0JjBTwWbdpDZ98lLrydo7KRi8xNRIi5RFszsbY=
 github.com/mattetti/filebuffer v1.0.0/go.mod h1:X6nyAIge2JGVmuJt2MFCqmHrb/5IHiphfHtot0s5cnI=
 github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=

pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_v2.go

@@ -2,12 +2,13 @@ package wrapper
 
 import (
 	"context"
+	"fmt"
 	"time"
 
+	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin"
 
 	"github.com/grafana/grafana-plugin-sdk-go/genproto/pluginv2"
-	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/tsdb"
@@ -81,12 +82,24 @@ func (tw *DatasourcePluginWrapperV2) Query(ctx context.Context, ds *models.DataS
 		return nil, err
 	}
 
-	return &tsdb.Response{
-		Results: map[string]*tsdb.QueryResult{
-			"": {
-				Dataframes: pbRes.Frames,
-				Meta:       simplejson.NewFromAny(pbRes.Metadata),
-			},
-		},
-	}, nil
+	tR := &tsdb.Response{
+		Results: make(map[string]*tsdb.QueryResult, len(pbRes.Responses)),
+	}
+
+	for refID, pRes := range pbRes.Responses {
+		qr := &tsdb.QueryResult{
+			RefId:      refID,
+			Dataframes: pRes.Frames,
+		}
+		if len(pRes.JsonMeta) != 0 {
+			qr.Meta = simplejson.NewFromAny(pRes.JsonMeta)
+		}
+		if pRes.Error != "" {
+			qr.Error = fmt.Errorf(pRes.Error)
+			qr.ErrorString = pRes.Error
+		}
+		tR.Results[refID] = qr
+	}
+
+	return tR, nil
 }

pkg/plugins/transform_plugin.go

@@ -103,14 +103,25 @@ func (tw *TransformWrapper) Transform(ctx context.Context, query *tsdb.TsdbQuery
 		return nil, err
 	}
 
-	return &tsdb.Response{
-		Results: map[string]*tsdb.QueryResult{
-			"": {
-				Dataframes: pbRes.Frames,
-				Meta:       simplejson.NewFromAny(pbRes.Metadata),
-			},
-		},
-	}, nil
+	tR := &tsdb.Response{
+		Results: make(map[string]*tsdb.QueryResult, len(pbRes.Responses)),
+	}
+	for refID, res := range pbRes.Responses {
+		tRes := &tsdb.QueryResult{
+			RefId:      refID,
+			Dataframes: res.Frames,
+		}
+		if len(res.JsonMeta) != 0 {
+			tRes.Meta = simplejson.NewFromAny(res.JsonMeta)
+		}
+		if res.Error != "" {
+			tRes.Error = fmt.Errorf(res.Error)
+			tRes.ErrorString = res.Error
+		}
+		tR.Results[refID] = tRes
+	}
+
+	return tR, nil
 }
 
 type transformCallback struct {
@@ -168,18 +179,16 @@ func (s *transformCallback) QueryData(ctx context.Context, req *pluginv2.QueryDa
 	}
 	// Convert tsdb results (map) to plugin-model/datasource (slice) results.
 	// Only error, tsdb.Series, and encoded Dataframes responses are mapped.
-
-	encodedFrames := [][]byte{}
+	responses := make(map[string]*pluginv2.DataResponse, len(tsdbRes.Results))
 	for refID, res := range tsdbRes.Results {
-
+		pRes := &pluginv2.DataResponse{}
 		if res.Error != nil {
-			// TODO add Errors property to Frame
-			encodedFrames = append(encodedFrames, nil)
-			continue
+			pRes.Error = res.Error.Error()
 		}
 
 		if res.Dataframes != nil {
-			encodedFrames = append(encodedFrames, res.Dataframes...)
+			pRes.Frames = res.Dataframes
+			responses[refID] = pRes
 			continue
 		}
 
@@ -193,8 +202,18 @@ func (s *transformCallback) QueryData(ctx context.Context, req *pluginv2.QueryDa
 			if err != nil {
 				return nil, err
 			}
-			encodedFrames = append(encodedFrames, encFrame)
+			pRes.Frames = append(pRes.Frames, encFrame)
 		}
+		if res.Meta != nil {
+			b, err := res.Meta.MarshalJSON()
+			if err != nil {
+				s.logger.Error("failed to marhsal json metadata", err)
+			}
+			pRes.JsonMeta = b
+		}
+		responses[refID] = pRes
 	}
-	return &pluginv2.QueryDataResponse{Frames: encodedFrames}, nil
+	return &pluginv2.QueryDataResponse{
+		Responses: responses,
+	}, nil
 }

vendor/github.com/grafana/grafana-plugin-sdk-go/data/arrow.go

@@ -657,7 +657,7 @@ func UnmarshalArrow(b []byte) (*Frame, error) {
 
 	if metaAsString, ok := getMDKey("meta", metaData); ok {
 		var err error
-		frame.Meta, err = QueryResultMetaFromJSON(metaAsString)
+		frame.Meta, err = FrameMetaFromJSON(metaAsString)
 		if err != nil {
 			return nil, err
 		}
@@ -692,3 +692,29 @@ func toJSONString(val interface{}) (string, error) {
 	}
 	return string(b), nil
 }
+
+// BytesSliceToFrames decodes a slice of encoded Arrow frames to a slice of *Frame.
+func BytesSliceToFrames(bFrames [][]byte) ([]*Frame, error) {
+	frames := make([]*Frame, len(bFrames))
+	var err error
+	for i, encodedFrame := range bFrames {
+		frames[i], err = UnmarshalArrow(encodedFrame)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return frames, nil
+}
+
+// FramesToBytesSlice encodes a slice of Frames into a slice of []byte.
+func FramesToBytesSlice(frames []*Frame) ([][]byte, error) {
+	bs := make([][]byte, len(frames))
+	var err error
+	for i, frame := range frames {
+		bs[i], err = MarshalArrow(frame)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return bs, nil
+}

vendor/github.com/grafana/grafana-plugin-sdk-go/data/frame.go

@@ -28,7 +28,7 @@ type Frame struct {
 	Fields []*Field
 
 	RefID    string
-	Meta     *QueryResultMeta
+	Meta     *FrameMeta
 	Warnings []Warning
 }
 
@@ -263,29 +263,13 @@ func FrameTestCompareOptions() []cmp.Option {
 		if x == nil && y == nil {
 			return true
 		}
-		if y == nil {
-			if math.IsNaN(float64(*x)) {
-				return true
-			}
-			if math.IsInf(float64(*x), 1) {
-				return true
-			}
-			if math.IsInf(float64(*x), -1) {
-				return true
-			}
+		if y == nil && x != nil || y != nil && x == nil {
+			return false
 		}
-		if x == nil {
-			if math.IsNaN(float64(*y)) {
-				return true
-			}
-			if math.IsInf(float64(*y), 1) {
-				return true
-			}
-			if math.IsInf(float64(*y), -1) {
-				return true
-			}
-		}
-		return *x == *y
+		return (math.IsNaN(float64(*x)) && math.IsNaN(float64(*y))) ||
+			(math.IsInf(float64(*x), 1) && math.IsInf(float64(*y), 1)) ||
+			(math.IsInf(float64(*x), -1) && math.IsInf(float64(*y), -1)) ||
+			*x == *y
 	})
 	f64s := cmp.Comparer(func(x, y float64) bool {
 		return (math.IsNaN(x) && math.IsNaN(y)) ||
@@ -297,29 +281,13 @@ func FrameTestCompareOptions() []cmp.Option {
 		if x == nil && y == nil {
 			return true
 		}
-		if y == nil {
-			if math.IsNaN(float64(*x)) {
-				return true
-			}
-			if math.IsInf(float64(*x), 1) {
-				return true
-			}
-			if math.IsInf(float64(*x), -1) {
-				return true
-			}
-		}
-		if x == nil {
-			if math.IsNaN(float64(*y)) {
-				return true
-			}
-			if math.IsInf(float64(*y), 1) {
-				return true
-			}
-			if math.IsInf(float64(*y), -1) {
-				return true
-			}
+		if y == nil && x != nil || y != nil && x == nil {
+			return false
 		}
-		return *x == *y
+		return (math.IsNaN(float64(*x)) && math.IsNaN(float64(*y))) ||
+			(math.IsInf(float64(*x), 1) && math.IsInf(float64(*y), 1)) ||
+			(math.IsInf(float64(*x), -1) && math.IsInf(float64(*y), -1)) ||
+			*x == *y
 	})
 	f32s := cmp.Comparer(func(x, y float32) bool {
 		return (math.IsNaN(float64(x)) && math.IsNaN(float64(y))) ||
@@ -369,6 +337,8 @@ func (f *Frame) StringTable(maxFields, maxRows int) (string, error) {
 
 	sb := &strings.Builder{}
 	sb.WriteString(fmt.Sprintf("Name: %v\n", f.Name))
+	sb.WriteString(fmt.Sprintf("Dimensions: %v Fields by %v Rows\n", len(f.Fields), rowLen))
+
 	table := tablewriter.NewWriter(sb)
 
 	// table formatting options
@@ -377,9 +347,6 @@ func (f *Frame) StringTable(maxFields, maxRows int) (string, error) {
 	table.SetAutoWrapText(false)
 	table.SetAlignment(tablewriter.ALIGN_LEFT)
 
-	// (caption is below the table)
-	table.SetCaption(true, fmt.Sprintf("Field Count: %v\nRow Count: %v", len(f.Fields), rowLen))
-
 	// set table headers
 	headers := make([]string, width)
 	for colIdx, field := range f.Fields {

vendor/github.com/grafana/grafana-plugin-sdk-go/data/query_result_meta.go → vendor/github.com/grafana/grafana-plugin-sdk-go/data/frame_meta.go

@@ -2,24 +2,24 @@ package data
 
 import "encoding/json"
 
-// QueryResultMeta matches:
+// FrameMeta matches:
 // https://github.com/grafana/grafana/blob/master/packages/grafana-data/src/types/data.ts#L11
 // NOTE -- in javascript this can accept any `[key: string]: any;` however
 // this interface only exposes the values we want to be exposed
-type QueryResultMeta struct {
-	// Used in Explore for highlighting
-	SearchWords []string `json:"searchWords,omitempty"`
-
-	// Used in Explore to show limit applied to search result
-	Limit int64 `json:"limit,omitempty"`
-
+type FrameMeta struct {
 	// Datasource specific values
 	Custom map[string]interface{} `json:"custom,omitempty"`
+
+	// Stats is TODO
+	Stats interface{} `json:"stats,omitempty"`
+
+	// Notices is TODO
+	Notices interface{} `json:"notices,omitempty"`
 }
 
-// QueryResultMetaFromJSON creates a QueryResultMeta from a json string
-func QueryResultMetaFromJSON(jsonStr string) (*QueryResultMeta, error) {
-	var m QueryResultMeta
+// FrameMetaFromJSON creates a QueryResultMeta from a json string
+func FrameMetaFromJSON(jsonStr string) (*FrameMeta, error) {
+	var m FrameMeta
 	err := json.Unmarshal([]byte(jsonStr), &m)
 	if err != nil {
 		return nil, err

vendor/modules.txt

@@ -142,7 +142,7 @@ github.com/gosimple/slug
 # github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4
 github.com/grafana/grafana-plugin-model/go/datasource
 github.com/grafana/grafana-plugin-model/go/renderer
-# github.com/grafana/grafana-plugin-sdk-go v0.33.0
+# github.com/grafana/grafana-plugin-sdk-go v0.35.0
 github.com/grafana/grafana-plugin-sdk-go/backend/grpcplugin
 github.com/grafana/grafana-plugin-sdk-go/data
 github.com/grafana/grafana-plugin-sdk-go/genproto/pluginv2