Commit 9d7c74ef by Marcus Efraimsson Committed by GitHub

Backend Plugins: Provide proper plugin config to plugins (#21985)

Properly provides plugin configs to backend plugins.
Uses v0.16.0 of grafana-plugin-sdk-go-

Ref #21512
Ref #19667
parent f82a6aa0
......@@ -32,7 +32,7 @@ require (
github.com/gorilla/websocket v1.4.1
github.com/gosimple/slug v1.4.2
github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4
github.com/grafana/grafana-plugin-sdk-go v0.14.0
github.com/grafana/grafana-plugin-sdk-go v0.16.0
github.com/hashicorp/go-hclog v0.8.0
github.com/hashicorp/go-plugin v1.0.1
github.com/hashicorp/go-version v1.1.0
......
......@@ -95,6 +95,7 @@ github.com/go-xorm/xorm v0.7.1 h1:Kj7mfuqctPdX60zuxP6EoEut0f3E6K66H6hcoxiHUMc=
github.com/go-xorm/xorm v0.7.1/go.mod h1:EHS1htMQFptzMaIHKyzqpHGw6C9Rtug75nsq6DA9unI=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
......@@ -125,8 +126,8 @@ github.com/gosimple/slug v1.4.2 h1:jDmprx3q/9Lfk4FkGZtvzDQ9Cj9eAmsjzeQGp24PeiQ=
github.com/gosimple/slug v1.4.2/go.mod h1:ER78kgg1Mv0NQGlXiDe57DpCyfbNywXXZ9mIorhxAf0=
github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4 h1:SPdxCL9BChFTlyi0Khv64vdCW4TMna8+sxL7+Chx+Ag=
github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4/go.mod h1:nc0XxBzjeGcrMltCDw269LoWF9S8ibhgxolCdA1R8To=
github.com/grafana/grafana-plugin-sdk-go v0.14.0 h1:7Y34uhBkmkZ1ywYNfTXYXZihRgxgCf9WQbad4iqJ+t8=
github.com/grafana/grafana-plugin-sdk-go v0.14.0/go.mod h1:Dgc2ygLBJXTTIcfIgCGG6mxRcC/aZ36rKZ1tq5B5XmY=
github.com/grafana/grafana-plugin-sdk-go v0.16.0 h1:fuoLzsQLs0RKcvXDP/cAQxaZGP1rbnoBwUaY/1yvh6k=
github.com/grafana/grafana-plugin-sdk-go v0.16.0/go.mod h1:D1MkO+4EPCWc1Wrr260hq7wbo7Ox0grnNWBygulq7aM=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI=
......@@ -417,8 +418,6 @@ gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw=
gopkg.in/redis.v2 v2.3.2/go.mod h1:4wl9PJ/CqzeHk3LVq1hNLHH8krm3+AXEgut4jVc++LU=
gopkg.in/redis.v5 v5.2.9 h1:MNZYOLPomQzZMfpN3ZtD1uyJ2IDonTTlxYiV/pEApiw=
gopkg.in/redis.v5 v5.2.9/go.mod h1:6gtv0/+A4iM08kdRfocWYB3bLX2tebpNtfKlFT6H4mY=
gopkg.in/square/go-jose.v2 v2.3.0 h1:nLzhkFyl5bkblqYBoiWJUt5JkWOzmiaBtCxdJAqJd3U=
gopkg.in/square/go-jose.v2 v2.3.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/square/go-jose.v2 v2.4.1 h1:H0TmLt7/KmzlrDOpa1F+zr0Tk90PbJYBfsVUmRLrf9Y=
gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/stretchr/testify.v1 v1.2.2/go.mod h1:QI5V/q6UbPmuhtm10CaFZxED9NreB8PnFYN9JcR6TxU=
......
......@@ -275,15 +275,28 @@ func (hs *HTTPServer) CallDatasourceResource(c *m.ReqContext) Response {
if err != nil {
return Error(500, "Failed to read request body", err)
}
jsonDataBytes, err := ds.JsonData.MarshalJSON()
if err != nil {
return Error(500, "Failed to marshal JSON data to bytes", err)
}
req := backendplugin.CallResourceRequest{
Config: backendplugin.PluginConfig{
OrgID: c.OrgId,
PluginID: plugin.Id,
Instance: &backendplugin.PluginInstance{
PluginType: plugin.Type,
JSONData: jsonDataBytes,
DecryptedSecureJSONData: ds.DecryptedValues(),
Updated: ds.Updated,
DataSourceConfig: &backendplugin.DataSourceConfig{
ID: ds.Id,
Name: ds.Name,
Type: ds.Type,
URL: ds.Url,
Database: ds.Database,
User: ds.User,
BasicAuthEnabled: ds.BasicAuth,
BasicAuthUser: ds.BasicAuthUser,
},
},
Path: c.Params("*"),
......
package api
import (
"encoding/json"
"sort"
"time"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
"github.com/grafana/grafana/pkg/api/dtos"
"github.com/grafana/grafana/pkg/bus"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
)
func (hs *HTTPServer) GetPluginList(c *m.ReqContext) Response {
func (hs *HTTPServer) GetPluginList(c *models.ReqContext) Response {
typeFilter := c.Query("type")
enabledFilter := c.Query("enabled")
embeddedFilter := c.Query("embedded")
......@@ -85,7 +87,7 @@ func (hs *HTTPServer) GetPluginList(c *m.ReqContext) Response {
return JSON(200, result)
}
func GetPluginSettingByID(c *m.ReqContext) Response {
func GetPluginSettingByID(c *models.ReqContext) Response {
pluginID := c.Params(":pluginId")
def, exists := plugins.Plugins[pluginID]
......@@ -108,9 +110,9 @@ func GetPluginSettingByID(c *m.ReqContext) Response {
State: def.State,
}
query := m.GetPluginSettingByIdQuery{PluginId: pluginID, OrgId: c.OrgId}
query := models.GetPluginSettingByIdQuery{PluginId: pluginID, OrgId: c.OrgId}
if err := bus.Dispatch(&query); err != nil {
if err != m.ErrPluginSettingNotFound {
if err != models.ErrPluginSettingNotFound {
return Error(500, "Failed to get login settings", nil)
}
} else {
......@@ -122,7 +124,7 @@ func GetPluginSettingByID(c *m.ReqContext) Response {
return JSON(200, dto)
}
func UpdatePluginSetting(c *m.ReqContext, cmd m.UpdatePluginSettingCmd) Response {
func UpdatePluginSetting(c *models.ReqContext, cmd models.UpdatePluginSettingCmd) Response {
pluginID := c.Params(":pluginId")
cmd.OrgId = c.OrgId
......@@ -139,7 +141,7 @@ func UpdatePluginSetting(c *m.ReqContext, cmd m.UpdatePluginSettingCmd) Response
return Success("Plugin settings updated")
}
func GetPluginDashboards(c *m.ReqContext) Response {
func GetPluginDashboards(c *models.ReqContext) Response {
pluginID := c.Params(":pluginId")
list, err := plugins.GetPluginDashboards(c.OrgId, pluginID)
......@@ -154,7 +156,7 @@ func GetPluginDashboards(c *m.ReqContext) Response {
return JSON(200, list)
}
func GetPluginMarkdown(c *m.ReqContext) Response {
func GetPluginMarkdown(c *models.ReqContext) Response {
pluginID := c.Params(":pluginId")
name := c.Params(":name")
......@@ -180,7 +182,7 @@ func GetPluginMarkdown(c *m.ReqContext) Response {
return resp
}
func ImportDashboard(c *m.ReqContext, apiCmd dtos.ImportDashboardCommand) Response {
func ImportDashboard(c *models.ReqContext, apiCmd dtos.ImportDashboardCommand) Response {
if apiCmd.PluginId == "" && apiCmd.Dashboard == nil {
return Error(422, "Dashboard must be set", nil)
}
......@@ -204,7 +206,7 @@ func ImportDashboard(c *m.ReqContext, apiCmd dtos.ImportDashboardCommand) Respon
}
// /api/plugins/:pluginId/health
func (hs *HTTPServer) CheckHealth(c *m.ReqContext) Response {
func (hs *HTTPServer) CheckHealth(c *models.ReqContext) Response {
pluginID := c.Params("pluginId")
resp, err := hs.BackendPluginManager.CheckHealth(c.Req.Context(), pluginID)
if err != nil {
......@@ -236,21 +238,45 @@ func (hs *HTTPServer) CheckHealth(c *m.ReqContext) Response {
}
// /api/plugins/:pluginId/resources/*
func (hs *HTTPServer) CallResource(c *m.ReqContext) Response {
func (hs *HTTPServer) CallResource(c *models.ReqContext) Response {
pluginID := c.Params("pluginId")
_, exists := plugins.Plugins[pluginID]
plugin, exists := plugins.Plugins[pluginID]
if !exists {
return Error(404, "Plugin not found, no installed plugin with that id", nil)
}
var jsonDataBytes []byte
var decryptedSecureJSONData map[string]string
var updated time.Time
ps, err := hs.getCachedPluginSettings(pluginID, c.SignedInUser)
if err != nil {
if err != models.ErrPluginSettingNotFound {
return Error(500, "Failed to get plugin settings", err)
}
} else {
jsonDataBytes, err = json.Marshal(&ps.JsonData)
if err != nil {
return Error(500, "Failed to marshal JSON data to bytes", err)
}
decryptedSecureJSONData = ps.DecryptedValues()
updated = ps.Updated
}
body, err := c.Req.Body().Bytes()
if err != nil {
return Error(500, "Failed to read request body", err)
}
req := backendplugin.CallResourceRequest{
Config: backendplugin.PluginConfig{
OrgID: c.OrgId,
PluginID: pluginID,
PluginID: plugin.Id,
PluginType: plugin.Type,
JSONData: jsonDataBytes,
DecryptedSecureJSONData: decryptedSecureJSONData,
Updated: updated,
},
Path: c.Params("*"),
Method: c.Req.Method,
......@@ -273,3 +299,22 @@ func (hs *HTTPServer) CallResource(c *m.ReqContext) Response {
header: resp.Headers,
}
}
func (hs *HTTPServer) getCachedPluginSettings(pluginID string, user *models.SignedInUser) (*models.PluginSetting, error) {
cacheKey := "plugin-setting-" + pluginID
if cached, found := hs.CacheService.Get(cacheKey); found {
ps := cached.(*models.PluginSetting)
if ps.OrgId == user.OrgId {
return ps, nil
}
}
query := models.GetPluginSettingByIdQuery{PluginId: pluginID, OrgId: user.OrgId}
if err := hs.Bus.Dispatch(&query); err != nil {
return nil, err
}
hs.CacheService.Set(cacheKey, query.Result, time.Second*5)
return query.Result, nil
}
package models
var pluginSettingDecryptionCache = secureJSONDecryptionCache{
cache: make(map[int64]cachedDecryptedJSON),
}
// DecryptedValues returns cached decrypted values from secureJsonData.
func (ps *PluginSetting) DecryptedValues() map[string]string {
pluginSettingDecryptionCache.Lock()
defer pluginSettingDecryptionCache.Unlock()
if item, present := pluginSettingDecryptionCache.cache[ps.Id]; present && ps.Updated.Equal(item.updated) {
return item.json
}
json := ps.SecureJsonData.Decrypt()
pluginSettingDecryptionCache.cache[ps.Id] = cachedDecryptedJSON{
updated: ps.Updated,
json: json,
}
return json
}
// DecryptedValue returns cached decrypted value from cached secureJsonData.
func (ps *PluginSetting) DecryptedValue(key string) (string, bool) {
value, exists := ps.DecryptedValues()[key]
return value, exists
}
// ClearPluginSettingDecryptionCache clears the datasource decryption cache.
func ClearPluginSettingDecryptionCache() {
pluginSettingDecryptionCache.Lock()
defer pluginSettingDecryptionCache.Unlock()
pluginSettingDecryptionCache.cache = make(map[int64]cachedDecryptedJSON)
}
package models
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/components/securejsondata"
)
func TestPluginSettingDecryptionCache(t *testing.T) {
t.Run("When plugin settings hasn't been updated, encrypted JSON should be fetched from cache", func(t *testing.T) {
ClearPluginSettingDecryptionCache()
ps := PluginSetting{
Id: 1,
JsonData: map[string]interface{}{},
SecureJsonData: securejsondata.GetEncryptedJsonData(map[string]string{
"password": "password",
}),
}
// Populate cache
password, ok := ps.DecryptedValue("password")
require.Equal(t, "password", password)
require.True(t, ok)
ps.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{
"password": "",
})
require.Equal(t, "password", password)
require.True(t, ok)
})
t.Run("When plugin settings is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) {
ClearPluginSettingDecryptionCache()
ps := PluginSetting{
Id: 1,
JsonData: map[string]interface{}{},
SecureJsonData: securejsondata.GetEncryptedJsonData(map[string]string{
"password": "password",
}),
}
// Populate cache
password, ok := ps.DecryptedValue("password")
require.Equal(t, "password", password)
require.True(t, ok)
ps.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{
"password": "",
})
ps.Updated = time.Now()
password, ok = ps.DecryptedValue("password")
require.Empty(t, password)
require.True(t, ok)
})
}
......@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"net/http"
"time"
"github.com/grafana/grafana-plugin-sdk-go/genproto/pluginv2"
"github.com/prometheus/client_golang/prometheus"
......@@ -205,13 +206,33 @@ func (p *BackendPlugin) callResource(ctx context.Context, req CallResourceReques
}
protoReq := &pluginv2.CallResource_Request{
Config: &pluginv2.PluginConfig{},
Config: &pluginv2.PluginConfig{
OrgId: req.Config.OrgID,
PluginId: req.Config.PluginID,
PluginType: req.Config.PluginType,
JsonData: req.Config.JSONData,
DecryptedSecureJsonData: req.Config.DecryptedSecureJSONData,
UpdatedMS: req.Config.Updated.UnixNano() / int64(time.Millisecond),
},
Path: req.Path,
Method: req.Method,
Url: req.URL,
Headers: reqHeaders,
Body: req.Body,
}
if req.Config.DataSourceConfig != nil {
protoReq.Config.DatasourceConfig = &pluginv2.DataSourceConfig{
Id: req.Config.DataSourceConfig.ID,
Name: req.Config.DataSourceConfig.Name,
Url: req.Config.DataSourceConfig.URL,
Database: req.Config.DataSourceConfig.Database,
User: req.Config.DataSourceConfig.User,
BasicAuthEnabled: req.Config.DataSourceConfig.BasicAuthEnabled,
BasicAuthUser: req.Config.DataSourceConfig.BasicAuthUser,
}
}
protoResp, err := p.core.CallResource(ctx, protoReq)
if err != nil {
if st, ok := status.FromError(err); ok {
......
......@@ -3,6 +3,7 @@ package backendplugin
import (
"encoding/json"
"strconv"
"time"
"github.com/grafana/grafana-plugin-sdk-go/genproto/pluginv2"
)
......@@ -54,18 +55,24 @@ func checkHealthResultFromProto(protoResp *pluginv2.CheckHealth_Response) *Check
}
}
type PluginInstance struct {
type DataSourceConfig struct {
ID int64
Name string
Type string
URL string
JSONData json.RawMessage
User string
Database string
BasicAuthEnabled bool
BasicAuthUser string
}
type PluginConfig struct {
PluginID string
OrgID int64
Instance *PluginInstance
PluginID string
PluginType string
JSONData json.RawMessage
DecryptedSecureJSONData map[string]string
Updated time.Time
DataSourceConfig *DataSourceConfig
}
type CallResourceRequest struct {
......
......@@ -2,6 +2,7 @@ package wrapper
import (
"context"
"time"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
......@@ -12,30 +13,40 @@ import (
"github.com/grafana/grafana/pkg/tsdb"
)
func NewDatasourcePluginWrapperV2(log log.Logger, plugin backendplugin.DatasourcePlugin) *DatasourcePluginWrapperV2 {
return &DatasourcePluginWrapperV2{DatasourcePlugin: plugin, logger: log}
func NewDatasourcePluginWrapperV2(log log.Logger, pluginId, pluginType string, plugin backendplugin.DatasourcePlugin) *DatasourcePluginWrapperV2 {
return &DatasourcePluginWrapperV2{DatasourcePlugin: plugin, logger: log, pluginId: pluginId, pluginType: pluginType}
}
type DatasourcePluginWrapperV2 struct {
backendplugin.DatasourcePlugin
logger log.Logger
pluginId string
pluginType string
}
func (tw *DatasourcePluginWrapperV2) Query(ctx context.Context, ds *models.DataSource, query *tsdb.TsdbQuery) (*tsdb.Response, error) {
jsonData, err := ds.JsonData.MarshalJSON()
jsonDataBytes, err := ds.JsonData.MarshalJSON()
if err != nil {
return nil, err
}
pbQuery := &pluginv2.DataQueryRequest{
Config: &pluginv2.PluginConfig{
OrgId: ds.OrgId,
PluginId: tw.pluginId,
PluginType: tw.pluginType,
UpdatedMS: ds.Updated.UnixNano() / int64(time.Millisecond),
JsonData: jsonDataBytes,
DecryptedSecureJsonData: ds.DecryptedValues(),
DatasourceConfig: &pluginv2.DataSourceConfig{
Id: ds.Id,
Name: ds.Name,
Type: ds.Type,
Url: ds.Url,
Id: ds.Id,
OrgId: ds.OrgId,
JsonData: string(jsonData),
DecryptedSecureJsonData: ds.SecureJsonData.Decrypt(),
Database: ds.Database,
User: ds.User,
BasicAuthEnabled: ds.BasicAuth,
BasicAuthUser: ds.BasicAuthUser,
},
},
Queries: []*pluginv2.DataQuery{},
}
......
......@@ -70,7 +70,7 @@ func (p *DataSourcePlugin) onLegacyPluginStart(pluginID string, client *backendp
func (p *DataSourcePlugin) onPluginStart(pluginID string, client *backendplugin.Client, logger log.Logger) error {
if client.DatasourcePlugin != nil {
tsdb.RegisterTsdbQueryEndpoint(pluginID, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
return wrapper.NewDatasourcePluginWrapperV2(logger, client.DatasourcePlugin), nil
return wrapper.NewDatasourcePluginWrapperV2(logger, p.Id, p.Type, client.DatasourcePlugin), nil
})
}
......
......@@ -55,7 +55,7 @@ func (p *TransformPlugin) onPluginStart(pluginID string, client *backendplugin.C
if client.DatasourcePlugin != nil {
tsdb.RegisterTsdbQueryEndpoint(pluginID, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
return wrapper.NewDatasourcePluginWrapperV2(logger, client.DatasourcePlugin), nil
return wrapper.NewDatasourcePluginWrapperV2(logger, p.Id, p.Type, client.DatasourcePlugin), nil
})
}
......@@ -122,9 +122,15 @@ func (s *transformCallback) DataQuery(ctx context.Context, req *pluginv2.DataQue
return nil, fmt.Errorf("zero queries found in datasource request")
}
datasourceID := int64(0)
if req.Config.DatasourceConfig != nil {
datasourceID = req.Config.DatasourceConfig.Id
}
getDsInfo := &models.GetDataSourceByIdQuery{
Id: req.Config.Id,
OrgId: req.Config.OrgId,
Id: datasourceID,
}
if err := bus.Dispatch(getDsInfo); err != nil {
......
......@@ -27,6 +27,46 @@ type Field struct {
// Fields is a slice of Field pointers.
type Fields []*Field
// AppendRow adds a new row to the Frame by appending to each element of vals to
// the corresponding Field in the dataframe.
// The dataframe's Fields and the Fields' Vectors must be initalized or AppendRow will panic.
// The number of arguments must match the number of Fields in the Frame and each type must coorespond
// to the Field type or AppendRow will panic.
func (f *Frame) AppendRow(vals ...interface{}) {
for i, v := range vals {
f.Fields[i].Vector.Append(v)
}
}
// AppendRowSafe adds a new row to the Frame by appending to each each element of vals to
// the corresponding Field in the dataframe. It has the some constraints as AppendRow but will
// return an error under those conditions instead of panicing.
func (f *Frame) AppendRowSafe(vals ...interface{}) error {
if len(vals) != len(f.Fields) {
return fmt.Errorf("failed to append vals to Frame. Frame has %v fields but was given %v to append", len(f.Fields), len(vals))
}
// check validity before any modification
for i, v := range vals {
if f.Fields[i] == nil {
return fmt.Errorf("can not append to uninitalized Field at field index %v", i)
}
if f.Fields[i].Vector == nil {
return fmt.Errorf("can not append to uninitalized Field Vector at field index %v", i)
}
dfPType := f.Fields[i].Vector.PrimitiveType()
if v == nil {
if !dfPType.Nullable() {
return fmt.Errorf("can not append nil to non-nullable vector with underlying type %s at field index %v", dfPType, i)
}
}
if v != nil && pTypeFromVal(v) != dfPType {
return fmt.Errorf("invalid type appending row at index %v, got %T want %v", i, v, dfPType.ItemTypeString())
}
f.Fields[i].Vector.Append(v)
}
return nil
}
// NewField returns a new instance of Field.
func NewField(name string, labels Labels, values interface{}) *Field {
var vec Vector
......
......@@ -14,6 +14,10 @@ func (v *nullablegenVector) Set(idx int, i interface{}) {
}
func (v *nullablegenVector) Append(i interface{}) {
if i == nil {
(*v) = append((*v), nil)
return
}
(*v) = append((*v), i.(*gen))
}
......@@ -21,6 +25,10 @@ func (v *nullablegenVector) At(i int) interface{} {
return (*v)[i]
}
func (v *nullablegenVector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *nullablegenVector) Len() int {
return len((*v))
}
......@@ -28,3 +36,7 @@ func (v *nullablegenVector) Len() int {
func (v *nullablegenVector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *nullablegenVector) Extend(i int) {
(*v) = append((*v), make([]*gen, i)...)
}
......@@ -27,6 +27,10 @@ func (v *genVector) At(i int) interface{} {
return (*v)[i]
}
func (v *genVector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *genVector) Len() int {
return len((*v))
}
......@@ -34,3 +38,7 @@ func (v *genVector) Len() int {
func (v *genVector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *genVector) Extend(i int) {
(*v) = append((*v), make([]gen, i)...)
}
package dataframe
import (
"database/sql"
"fmt"
"reflect"
)
// NewFromSQLRows returns a new dataframe populated with the data from rows. The Field Vector types
// will be Vectors of pointer types, []*T, if the SQL column is nullable or if the nullable property is unknown.
// Otherwise, they will be []T types.
//
// Fields will be named to match name of the SQL columns and the SQL column names must be unique (https://github.com/grafana/grafana-plugin-sdk-go/issues/59).
//
// All the types must be supported by the dataframe or a SQLStringConverter will be created and
// the resulting Field Vector type will be of type []*string.
//
// The SQLStringConverter's ConversionFunc will be applied to matching rows if it is not nil.
// Additionally, if the SQLStringConverter's Replacer is not nil, the replacement will be performed.
// A map of Field/Column index to the corresponding SQLStringConverter is returned so what conversions were
// done can be inspected.
func NewFromSQLRows(rows *sql.Rows, converters ...SQLStringConverter) (*Frame, map[int]SQLStringConverter, error) {
frame, mappers, err := newForSQLRows(rows, converters...)
if err != nil {
return nil, nil, err
}
for rows.Next() {
sRow := frame.newScannableRow()
err := rows.Scan(sRow...)
if err != nil {
return nil, nil, err
}
}
for fieldIdx, mapper := range mappers {
if mapper.ConversionFunc == nil {
continue
}
vec := frame.Fields[fieldIdx]
for i := 0; i < vec.Len(); i++ {
v, err := mapper.ConversionFunc(vec.Vector.At(i).(*string))
if err != nil {
return nil, nil, err
}
vec.Vector.Set(i, v)
}
if mapper.Replacer == nil {
continue
}
if err := Replace(frame, fieldIdx, mapper.Replacer); err != nil {
return nil, nil, err
}
}
return frame, mappers, nil
}
// newForSQLRows creates a new Frame appropriate for scanning SQL rows with
// the the new Frame's ScannableRow() method.
func newForSQLRows(rows *sql.Rows, converters ...SQLStringConverter) (*Frame, map[int]SQLStringConverter, error) {
mapping := make(map[int]SQLStringConverter)
colTypes, err := rows.ColumnTypes()
if err != nil {
return nil, nil, err
}
colNames, err := rows.Columns()
if err != nil {
return nil, nil, err
}
// In the future we can probably remove this restriction. But right now we map names to Arrow Field Names.
// Arrow Field names must be unique: https://github.com/grafana/grafana-plugin-sdk-go/issues/59
seen := map[string]int{}
for i, name := range colNames {
if j, ok := seen[name]; ok {
return nil, nil, fmt.Errorf(`duplicate column names are not allowed, found identical name "%v" at column indices %v and %v`, name, j, i)
}
seen[name] = i
}
frame := &Frame{}
for i, colType := range colTypes {
colName := colNames[i]
nullable, ok := colType.Nullable()
if !ok {
nullable = true // If we don't know if it is nullable, assume it is
}
scanType := colType.ScanType()
for _, converter := range converters {
if converter.InputScanKind == scanType.Kind() && converter.InputTypeName == colType.DatabaseTypeName() {
nullable = true // String converters are always nullable
scanType = reflect.TypeOf("")
mapping[i] = converter
}
}
var vec interface{}
if !nullable {
vec = reflect.MakeSlice(reflect.SliceOf(scanType), 0, 0).Interface()
} else {
ptrType := reflect.TypeOf(reflect.New(scanType).Interface())
// Nullabe types get passed to scan as a pointer to a pointer
vec = reflect.MakeSlice(reflect.SliceOf(ptrType), 0, 0).Interface()
}
if !ValidVectorType(vec) {
// Automatically create string mapper if we end up with an unsupported type
mapping[i] = SQLStringConverter{
Name: fmt.Sprintf("Autogenerated for column %v", i),
InputTypeName: colType.DatabaseTypeName(),
InputScanKind: colType.ScanType().Kind(),
}
ptrType := reflect.TypeOf(reflect.New(reflect.TypeOf("")).Interface())
vec = reflect.MakeSlice(reflect.SliceOf(ptrType), 0, 0).Interface()
}
frame.Fields = append(frame.Fields, NewField(colName, nil, vec))
}
return frame, mapping, nil
}
// newScannableRow adds a row to the dataframe by extending each Field's Vector. It returns
// a slice of references that can be passed to the database/sql rows.Scan() to scan directly into
// the extended Vectors of the dataframe.
func (f *Frame) newScannableRow() []interface{} {
row := make([]interface{}, len(f.Fields))
for i, field := range f.Fields {
vec := field.Vector
vec.Extend(1)
// non-nullable fields will be *T, and nullable fields will be **T
vecItemPointer := vec.PointerAt(vec.Len() - 1)
row[i] = vecItemPointer
}
return row
}
// SQLStringConverter can be used to store types not supported by
// a dataframe into a *string. When scanning, if a SQL's row's InputScanType's Kind
// and InputScanKind match that returned by the sql response, then the
// conversion func will be run on the row.
type SQLStringConverter struct {
// Name is an optional property that can be used to identify a converter
Name string
InputScanKind reflect.Kind // reflect.Type might better or worse option?
InputTypeName string
// Conversion func may be nil to do no additional operations on the string conversion.
ConversionFunc func(in *string) (*string, error)
// If the Replacer is not nil, the replacement will be performed.
Replacer *StringFieldReplacer
}
// Note: SQLStringConverter is perhaps better understood as []byte. However, currently
// the Vector type ([][]byte) is not supported. https://github.com/grafana/grafana-plugin-sdk-go/issues/57
// StringFieldReplacer is used to replace a *string Field in a dataframe. The type
// returned by the ReplaceFunc must match the type of elements of VectorType.
// Both properties must be non-nil.
type StringFieldReplacer struct {
VectorType interface{}
ReplaceFunc func(in *string) (interface{}, error)
}
// Replace will replace a *string Vector of the specified Field's index
// using the StringFieldReplacer.
func Replace(frame *Frame, fieldIdx int, replacer *StringFieldReplacer) error {
if fieldIdx > len(frame.Fields) {
return fmt.Errorf("fieldIdx is out of bounds, field len: %v", len(frame.Fields))
}
field := frame.Fields[fieldIdx]
if field.Vector.PrimitiveType() != VectorPTypeNullableString {
return fmt.Errorf("can only replace []*string vectors, vector is of type %s", field.Vector.PrimitiveType())
}
if !ValidVectorType(replacer.VectorType) {
return fmt.Errorf("can not replace column with unsupported type %T", replacer.VectorType)
}
newVector := newVector(replacer.VectorType, field.Vector.Len())
for i := 0; i < newVector.Len(); i++ {
oldVal := field.Vector.At(i).(*string) // Vector type is checked earlier above
newVal, err := replacer.ReplaceFunc(oldVal)
if err != nil {
return err
}
newVector.Set(i, newVal)
}
field.Vector = newVector
return nil
}
......@@ -27,6 +27,10 @@ func (v *uint8Vector) At(i int) interface{} {
return (*v)[i]
}
func (v *uint8Vector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *uint8Vector) Len() int {
return len((*v))
}
......@@ -35,6 +39,10 @@ func (v *uint8Vector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *uint8Vector) Extend(i int) {
(*v) = append((*v), make([]uint8, i)...)
}
//go:Uint16erate uint16ny -in=$GOFILE -out=vector.Uint16.go uint16 "Uint16=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type uint16Vector []uint16
......@@ -56,6 +64,10 @@ func (v *uint16Vector) At(i int) interface{} {
return (*v)[i]
}
func (v *uint16Vector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *uint16Vector) Len() int {
return len((*v))
}
......@@ -64,6 +76,10 @@ func (v *uint16Vector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *uint16Vector) Extend(i int) {
(*v) = append((*v), make([]uint16, i)...)
}
//go:Uint32erate uint32ny -in=$GOFILE -out=vector.Uint32.go uint32 "Uint32=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type uint32Vector []uint32
......@@ -85,6 +101,10 @@ func (v *uint32Vector) At(i int) interface{} {
return (*v)[i]
}
func (v *uint32Vector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *uint32Vector) Len() int {
return len((*v))
}
......@@ -93,6 +113,10 @@ func (v *uint32Vector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *uint32Vector) Extend(i int) {
(*v) = append((*v), make([]uint32, i)...)
}
//go:Uint64erate uint64ny -in=$GOFILE -out=vector.Uint64.go uint64 "Uint64=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type uint64Vector []uint64
......@@ -114,6 +138,10 @@ func (v *uint64Vector) At(i int) interface{} {
return (*v)[i]
}
func (v *uint64Vector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *uint64Vector) Len() int {
return len((*v))
}
......@@ -122,6 +150,10 @@ func (v *uint64Vector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *uint64Vector) Extend(i int) {
(*v) = append((*v), make([]uint64, i)...)
}
//go:Int8erate int8ny -in=$GOFILE -out=vector.Int8.go int8 "Int8=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type int8Vector []int8
......@@ -143,6 +175,10 @@ func (v *int8Vector) At(i int) interface{} {
return (*v)[i]
}
func (v *int8Vector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *int8Vector) Len() int {
return len((*v))
}
......@@ -151,6 +187,10 @@ func (v *int8Vector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *int8Vector) Extend(i int) {
(*v) = append((*v), make([]int8, i)...)
}
//go:Int16erate int16ny -in=$GOFILE -out=vector.Int16.go int16 "Int16=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type int16Vector []int16
......@@ -172,6 +212,10 @@ func (v *int16Vector) At(i int) interface{} {
return (*v)[i]
}
func (v *int16Vector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *int16Vector) Len() int {
return len((*v))
}
......@@ -180,6 +224,10 @@ func (v *int16Vector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *int16Vector) Extend(i int) {
(*v) = append((*v), make([]int16, i)...)
}
//go:Int32erate int32ny -in=$GOFILE -out=vector.Int32.go int32 "Int32=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type int32Vector []int32
......@@ -201,6 +249,10 @@ func (v *int32Vector) At(i int) interface{} {
return (*v)[i]
}
func (v *int32Vector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *int32Vector) Len() int {
return len((*v))
}
......@@ -209,6 +261,10 @@ func (v *int32Vector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *int32Vector) Extend(i int) {
(*v) = append((*v), make([]int32, i)...)
}
//go:Int64erate int64ny -in=$GOFILE -out=vector.Int64.go int64 "Int64=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type int64Vector []int64
......@@ -230,6 +286,10 @@ func (v *int64Vector) At(i int) interface{} {
return (*v)[i]
}
func (v *int64Vector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *int64Vector) Len() int {
return len((*v))
}
......@@ -238,6 +298,10 @@ func (v *int64Vector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *int64Vector) Extend(i int) {
(*v) = append((*v), make([]int64, i)...)
}
//go:Float32erate float32ny -in=$GOFILE -out=vector.Float32.go float32 "Float32=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type float32Vector []float32
......@@ -259,6 +323,10 @@ func (v *float32Vector) At(i int) interface{} {
return (*v)[i]
}
func (v *float32Vector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *float32Vector) Len() int {
return len((*v))
}
......@@ -267,6 +335,10 @@ func (v *float32Vector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *float32Vector) Extend(i int) {
(*v) = append((*v), make([]float32, i)...)
}
//go:Float64erate float64ny -in=$GOFILE -out=vector.Float64.go float64 "Float64=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type float64Vector []float64
......@@ -288,6 +360,10 @@ func (v *float64Vector) At(i int) interface{} {
return (*v)[i]
}
func (v *float64Vector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *float64Vector) Len() int {
return len((*v))
}
......@@ -296,6 +372,10 @@ func (v *float64Vector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *float64Vector) Extend(i int) {
(*v) = append((*v), make([]float64, i)...)
}
//go:Stringerate stringny -in=$GOFILE -out=vector.String.go string "String=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type stringVector []string
......@@ -317,6 +397,10 @@ func (v *stringVector) At(i int) interface{} {
return (*v)[i]
}
func (v *stringVector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *stringVector) Len() int {
return len((*v))
}
......@@ -325,6 +409,10 @@ func (v *stringVector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *stringVector) Extend(i int) {
(*v) = append((*v), make([]string, i)...)
}
//go:Boolerate boolny -in=$GOFILE -out=vector.Bool.go bool "Bool=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type boolVector []bool
......@@ -346,6 +434,10 @@ func (v *boolVector) At(i int) interface{} {
return (*v)[i]
}
func (v *boolVector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *boolVector) Len() int {
return len((*v))
}
......@@ -354,6 +446,10 @@ func (v *boolVector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *boolVector) Extend(i int) {
(*v) = append((*v), make([]bool, i)...)
}
//go:TimeTimeerate timeTimeny -in=$GOFILE -out=vector.TimeTime.go time.Time "TimeTime=uint8,uint16,uint32,uint64,int8,int16,int32,int64,float32,float64,string,bool,time.Time"
type timeTimeVector []time.Time
......@@ -375,6 +471,10 @@ func (v *timeTimeVector) At(i int) interface{} {
return (*v)[i]
}
func (v *timeTimeVector) PointerAt(i int) interface{} {
return &(*v)[i]
}
func (v *timeTimeVector) Len() int {
return len((*v))
}
......@@ -382,3 +482,7 @@ func (v *timeTimeVector) Len() int {
func (v *timeTimeVector) PrimitiveType() VectorPType {
return vectorPType(v)
}
func (v *timeTimeVector) Extend(i int) {
(*v) = append((*v), make([]time.Time, i)...)
}
......@@ -9,9 +9,11 @@ import (
type Vector interface {
Set(idx int, i interface{})
Append(i interface{})
Extend(i int)
At(i int) interface{}
Len() int
PrimitiveType() VectorPType
PointerAt(i int) interface{}
//buildArrowColumn(pool memory.Allocator, field arrow.Field) *array.Column
}
......@@ -81,6 +83,73 @@ func newVector(t interface{}, n int) (v Vector) {
return
}
// ValidVectorType returns if a primitive slice is a valid / supported Vector type
func ValidVectorType(t interface{}) bool {
switch t.(type) {
// ints
case []int8:
return true
case []*int8:
return true
case []int16:
return true
case []*int16:
return true
case []int32:
return true
case []*int32:
return true
case []int64:
return true
case []*int64:
return true
// uints
case []uint8:
return true
case []*uint8:
return true
case []uint16:
return true
case []*uint16:
return true
case []uint32:
return true
case []*uint32:
return true
case []uint64:
return true
case []*uint64:
return true
// floats
case []float32:
return true
case []*float32:
return true
case []float64:
return true
case []*float64:
return true
case []string:
return true
case []*string:
return true
case []bool:
return true
case []*bool:
return true
case []time.Time:
return true
case []*time.Time:
return true
default:
return false
}
}
// VectorPType indicates the go type underlying the Vector.
type VectorPType int
......@@ -221,3 +290,176 @@ func vectorPType(v Vector) VectorPType {
return VectorPType(-1)
}
func pTypeFromVal(v interface{}) VectorPType {
switch v.(type) {
case int8:
return VectorPTypeInt8
case *int8:
return VectorPTypeNullableInt8
case int16:
return VectorPTypeInt16
case *int16:
return VectorPTypeNullableInt16
case int32:
return VectorPTypeInt32
case *int32:
return VectorPTypeNullableInt32
case int64:
return VectorPTypeInt64
case *int64:
return VectorPTypeNullableInt64
case uint8:
return VectorPTypeUint8
case *uint8:
return VectorPTypeNullableUint8
case uint16:
return VectorPTypeUint16
case *uint16:
return VectorPTypeNullableUint16
case uint32:
return VectorPTypeUint32
case *uint32:
return VectorPTypeNullableUint32
case uint64:
return VectorPTypeUint64
case *uint64:
return VectorPTypeNullableUint64
case float32:
return VectorPTypeFloat32
case *float32:
return VectorPTypeNullableFloat32
case float64:
return VectorPTypeFloat64
case *float64:
return VectorPTypeNullableFloat64
case string:
return VectorPTypeString
case *string:
return VectorPTypeNullableString
case bool:
return VectorPTypeBool
case *bool:
return VectorPTypeNullableBool
case time.Time:
return VectorPTypeTime
case *time.Time:
return VectorPTypeNullableTime
}
return VectorPType(-1)
}
func (p VectorPType) String() string {
if p < 0 {
return "invalid/unsupported"
}
return fmt.Sprintf("[]%v", p.ItemTypeString())
}
// ItemTypeString returns the string representation of the type of element within in the vector
func (p VectorPType) ItemTypeString() string {
switch p {
case VectorPTypeInt8:
return "int8"
case VectorPTypeNullableInt8:
return "*int8"
case VectorPTypeInt16:
return "int16"
case VectorPTypeNullableInt16:
return "*int16"
case VectorPTypeInt32:
return "int32"
case VectorPTypeNullableInt32:
return "*int32"
case VectorPTypeInt64:
return "int64"
case VectorPTypeNullableInt64:
return "*int64"
case VectorPTypeUint8:
return "unit8"
case VectorPTypeNullableUint8:
return "*uint8"
case VectorPTypeUint16:
return "uint16"
case VectorPTypeNullableUint16:
return "*uint16"
case VectorPTypeUint32:
return "uint32"
case VectorPTypeNullableUint32:
return "*uint32"
case VectorPTypeUint64:
return "uint64"
case VectorPTypeNullableUint64:
return "*uint64"
case VectorPTypeFloat32:
return "float32"
case VectorPTypeNullableFloat32:
return "*float32"
case VectorPTypeFloat64:
return "float64"
case VectorPTypeNullableFloat64:
return "*float64"
case VectorPTypeString:
return "string"
case VectorPTypeNullableString:
return "*string"
case VectorPTypeBool:
return "bool"
case VectorPTypeNullableBool:
return "*bool"
case VectorPTypeTime:
return "time.Time"
case VectorPTypeNullableTime:
return "*time.Time"
}
return "invalid/unsupported type"
}
// Nullable returns if type is a nullable type
func (p VectorPType) Nullable() bool {
switch p {
case VectorPTypeNullableInt8, VectorPTypeNullableInt16, VectorPTypeNullableInt32, VectorPTypeNullableInt64:
return true
case VectorPTypeNullableUint8, VectorPTypeNullableUint16, VectorPTypeNullableUint32, VectorPTypeNullableUint64:
return true
case VectorPTypeNullableFloat32, VectorPTypeNullableFloat64:
return true
case VectorPTypeNullableString:
return true
case VectorPTypeNullableBool:
return true
case VectorPTypeNullableTime:
return true
}
return false
}
......@@ -148,7 +148,7 @@ github.com/gosimple/slug
# github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4
github.com/grafana/grafana-plugin-model/go/datasource
github.com/grafana/grafana-plugin-model/go/renderer
# github.com/grafana/grafana-plugin-sdk-go v0.14.0
# github.com/grafana/grafana-plugin-sdk-go v0.16.0
github.com/grafana/grafana-plugin-sdk-go/backend/plugin
github.com/grafana/grafana-plugin-sdk-go/dataframe
github.com/grafana/grafana-plugin-sdk-go/genproto/pluginv2
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment