mysql: minor progress on response processing

pkg/api/metrics.go

@@ -50,6 +50,12 @@ func QueryMetrics(c *middleware.Context, reqDto dtos.MetricRequest) Response {
 		return ApiError(500, "Metric request error", err)
 		return ApiError(500, "Metric request error", err)
 	}
 	}
 
 
+	for _, res := range resp.Results {
+		if res.Error != nil {
+			res.ErrorString = res.Error.Error()
+		}
+	}
+
 	return Json(200, &resp)
 	return Json(200, &resp)
 }
 }
 
 

pkg/models/test_data.go

@@ -6,14 +6,13 @@ type InsertSqlTestDataCommand struct {
 }
 }
 
 
 type SqlTestData struct {
 type SqlTestData struct {
-	Id            int64
-	Metric1       string
-	Metric2       string
-	ValueBigInt   int64
-	ValueDouble   float64
-	ValueFloat    float32
-	ValueInt      int
-	TimeEpoch     int64
-	TimeDateTime  time.Time
-	TimeTimeStamp time.Time
+	Id           int64
+	Metric1      string
+	Metric2      string
+	ValueBigInt  int64
+	ValueDouble  float64
+	ValueFloat   float32
+	ValueInt     int
+	TimeEpoch    int64
+	TimeDateTime time.Time
 }
 }

pkg/services/sqlstore/migrations/stats_mig.go

@@ -46,8 +46,8 @@ func addTestDataMigrations(mg *Migrator) {
 			{Name: "value_float", Type: DB_Float, Nullable: true},
 			{Name: "value_float", Type: DB_Float, Nullable: true},
 			{Name: "value_int", Type: DB_Int, Nullable: true},
 			{Name: "value_int", Type: DB_Int, Nullable: true},
 			{Name: "time_epoch", Type: DB_BigInt, Nullable: false},
 			{Name: "time_epoch", Type: DB_BigInt, Nullable: false},
-			{Name: "time_datetime", Type: DB_DateTime, Nullable: false},
-			{Name: "time_timestamp", Type: DB_TimeStamp, Nullable: false},
+			{Name: "time_date_time", Type: DB_DateTime, Nullable: false},
+			{Name: "time_time_stamp", Type: DB_TimeStamp, Nullable: false},
 		},
 		},
 	}
 	}
 
 

pkg/services/sqlstore/sql_test_data.go

 package sqlstore
 package sqlstore
 
 
 import (
 import (
+	"math/rand"
 	"time"
 	"time"
 
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/bus"
@@ -11,23 +12,53 @@ func init() {
 	bus.AddHandler("sql", InsertSqlTestData)
 	bus.AddHandler("sql", InsertSqlTestData)
 }
 }
 
 
-func InsertSqlTestData(cmd *m.InsertSqlTestDataCommand) error {
-	return inTransaction2(func(sess *session) error {
+func sqlRandomWalk(m1 string, m2 string, intWalker int64, floatWalker float64, sess *session) error {
 
 
-		row := &m.SqlTestData{
-			Metric1:      "server1",
-			Metric2:      "frontend",
-			ValueBigInt:  123123,
-			ValueDouble:  3.14159265359,
-			ValueFloat:   3.14159265359,
-			TimeEpoch:    time.Now().Unix(),
-			TimeDateTime: time.Now(),
-		}
+	timeWalker := time.Now().Add(time.Hour * -1)
+	now := time.Now()
+	step := time.Minute
+
+	row := &m.SqlTestData{
+		Metric1:      m1,
+		Metric2:      m2,
+		TimeEpoch:    timeWalker.Unix(),
+		TimeDateTime: timeWalker,
+	}
 
 
+	for timeWalker.Unix() < now.Unix() {
+		timeWalker = timeWalker.Add(step)
+
+		row.Id = 0
+		row.ValueBigInt += rand.Int63n(100) - 100
+		row.ValueDouble += rand.Float64() - 0.5
+		row.ValueFloat += rand.Float32() - 0.5
+		row.TimeEpoch = timeWalker.Unix()
+		row.TimeDateTime = timeWalker
+
+		sqlog.Info("Writing SQL test data row")
 		if _, err := sess.Table("test_data").Insert(row); err != nil {
 		if _, err := sess.Table("test_data").Insert(row); err != nil {
 			return err
 			return err
 		}
 		}
+	}
+
+	return nil
+}
+
+func InsertSqlTestData(cmd *m.InsertSqlTestDataCommand) error {
+	return inTransaction2(func(sess *session) error {
+		var err error
+
+		sqlog.Info("SQL TestData: Clearing previous test data")
+		res, err := sess.Exec("TRUNCATE test_data")
+		if err != nil {
+			return err
+		}
+
+		rows, _ := res.RowsAffected()
+		sqlog.Info("SQL TestData: Truncate done", "rows", rows)
+
+		sqlRandomWalk("server1", "frontend", 100, 1.123, sess)
 
 
-		return nil
+		return err
 	})
 	})
 }
 }

pkg/tsdb/models.go

@@ -45,9 +45,10 @@ func (br *BatchResult) WithError(err error) *BatchResult {
 }
 }
 
 
 type QueryResult struct {
 type QueryResult struct {
-	Error  error           `json:"error"`
-	RefId  string          `json:"refId"`
-	Series TimeSeriesSlice `json:"series"`
+	Error       error           `json:"-"`
+	ErrorString string          `json:"error"`
+	RefId       string          `json:"refId"`
+	Series      TimeSeriesSlice `json:"series"`
 }
 }
 
 
 type TimeSeries struct {
 type TimeSeries struct {

pkg/tsdb/mysql/mysql.go

@@ -4,9 +4,12 @@ import (
 	"context"
 	"context"
 	"database/sql"
 	"database/sql"
 	"fmt"
 	"fmt"
+	"strconv"
 	"sync"
 	"sync"
 
 
+	"github.com/go-xorm/core"
 	"github.com/go-xorm/xorm"
 	"github.com/go-xorm/xorm"
+	"github.com/grafana/grafana/pkg/components/null"
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/tsdb"
 	"github.com/grafana/grafana/pkg/tsdb"
@@ -74,19 +77,14 @@ func (e *MysqlExecutor) initEngine() error {
 }
 }
 
 
 func (e *MysqlExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, context *tsdb.QueryContext) *tsdb.BatchResult {
 func (e *MysqlExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, context *tsdb.QueryContext) *tsdb.BatchResult {
-	result := &tsdb.BatchResult{}
+	result := &tsdb.BatchResult{
+		QueryResults: make(map[string]*tsdb.QueryResult),
+	}
 
 
 	session := e.engine.NewSession()
 	session := e.engine.NewSession()
 	defer session.Close()
 	defer session.Close()
-
 	db := session.DB()
 	db := session.DB()
 
 
-	// queries := strings.Split(req.Query, ";")
-	//
-	// data := dataStruct{}
-	// data.Results = make([]resultsStruct, 1)
-	// data.Results[0].Series = make([]seriesStruct, 0)
-
 	for _, query := range queries {
 	for _, query := range queries {
 		rawSql := query.Model.Get("rawSql").MustString()
 		rawSql := query.Model.Get("rawSql").MustString()
 		if rawSql == "" {
 		if rawSql == "" {
@@ -100,118 +98,119 @@ func (e *MysqlExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, co
 		}
 		}
 		defer rows.Close()
 		defer rows.Close()
 
 
-		columnNames, err := rows.Columns()
+		result.QueryResults[query.RefId] = e.TransformToTimeSeries(query, rows)
+	}
+
+	for _, value := range result.QueryResults {
+		if value.Error != nil {
+			e.log.Error("error", "error", value.Error)
+		}
+	}
+
+	return result
+}
+
+func (e MysqlExecutor) TransformToTimeSeries(query *tsdb.Query, rows *core.Rows) *tsdb.QueryResult {
+	result := &tsdb.QueryResult{RefId: query.RefId}
+	pointsBySeries := make(map[string]*tsdb.TimeSeries)
+	columnNames, err := rows.Columns()
+
+	if err != nil {
+		result.Error = err
+		return result
+	}
+
+	rowData := NewStringStringScan(columnNames)
+	for rows.Next() {
+		err := rowData.Update(rows.Rows)
 		if err != nil {
 		if err != nil {
+			e.log.Error("Mysql response parsing", "error", err)
 			result.Error = err
 			result.Error = err
 			return result
 			return result
 		}
 		}
 
 
-		rc := NewStringStringScan(columnNames)
-		for rows.Next() {
-			err := rc.Update(rows.Rows)
-			if err != nil {
-				e.log.Error("Mysql response parsing", "error", err)
-				result.Error = err
-				return result
-			}
+		if rowData.metric == "" {
+			rowData.metric = "Unknown"
+		}
+
+		e.log.Info("Rows", "metric", rowData.metric, "time", rowData.time, "value", rowData.value)
+
+		if !rowData.time.Valid {
+			result.Error = fmt.Errorf("Found row with no time value")
+			return result
+		}
 
 
-			rowValues := rc.Get()
-			e.log.Info("Rows", "row", rowValues)
+		if series, exist := pointsBySeries[rowData.metric]; exist {
+			series.Points = append(series.Points, tsdb.TimePoint{rowData.value, rowData.time})
+		} else {
+			series := &tsdb.TimeSeries{Name: rowData.metric}
+			series.Points = append(series.Points, tsdb.TimePoint{rowData.value, rowData.time})
+			pointsBySeries[rowData.metric] = series
 		}
 		}
+	}
 
 
-		// for rows.Next() {
-		// 	columnValues := make([]interface{}, len(columnNames))
-		//
-		// 	err = rows.ScanSlice(&columnValues)
-		// 	if err != nil {
-		// 		result.Error = err
-		// 		return result
-		// 	}
-		//
-		// 	// bytes -> string
-		// 	for i := range columnValues {
-		// 		rowType := reflect.TypeOf(columnValues[i])
-		// 		e.log.Info("row", "type", rowType)
-		//
-		// 		rawValue := reflect.Indirect(reflect.ValueOf(columnValues[i]))
-		//
-		// 		// if rawValue is null then ignore
-		// 		if rawValue.Interface() == nil {
-		// 			continue
-		// 		}
-		//
-		// 		rawValueType := reflect.TypeOf(rawValue.Interface())
-		// 		vv := reflect.ValueOf(rawValue.Interface())
-		// 		e.log.Info("column type", "name", columnNames[i], "type", rawValueType, "vv", vv)
-		// 	}
-		// }
+	for _, value := range pointsBySeries {
+		result.Series = append(result.Series, value)
 	}
 	}
 
 
 	return result
 	return result
 }
 }
 
 
 type stringStringScan struct {
 type stringStringScan struct {
-	// cp are the column pointers
-	cp []interface{}
-	// row contains the final result
-	row      []string
-	colCount int
-	colNames []string
+	rowPtrs     []interface{}
+	rowValues   []string
+	columnNames []string
+	columnCount int
+
+	time   null.Float
+	value  null.Float
+	metric string
 }
 }
 
 
 func NewStringStringScan(columnNames []string) *stringStringScan {
 func NewStringStringScan(columnNames []string) *stringStringScan {
-	lenCN := len(columnNames)
 	s := &stringStringScan{
 	s := &stringStringScan{
-		cp:       make([]interface{}, lenCN),
-		row:      make([]string, lenCN*2),
-		colCount: lenCN,
-		colNames: columnNames,
+		columnCount: len(columnNames),
+		columnNames: columnNames,
+		rowPtrs:     make([]interface{}, len(columnNames)),
+		rowValues:   make([]string, len(columnNames)),
 	}
 	}
-	j := 0
-	for i := 0; i < lenCN; i++ {
-		s.cp[i] = new(sql.RawBytes)
-		s.row[j] = s.colNames[i]
-		j = j + 2
+
+	for i := 0; i < s.columnCount; i++ {
+		s.rowPtrs[i] = new(sql.RawBytes)
 	}
 	}
+
 	return s
 	return s
 }
 }
 
 
 func (s *stringStringScan) Update(rows *sql.Rows) error {
 func (s *stringStringScan) Update(rows *sql.Rows) error {
-	if err := rows.Scan(s.cp...); err != nil {
+	if err := rows.Scan(s.rowPtrs...); err != nil {
 		return err
 		return err
 	}
 	}
-	j := 0
-	for i := 0; i < s.colCount; i++ {
-		if rb, ok := s.cp[i].(*sql.RawBytes); ok {
-			s.row[j+1] = string(*rb)
+
+	for i := 0; i < s.columnCount; i++ {
+		if rb, ok := s.rowPtrs[i].(*sql.RawBytes); ok {
+			s.rowValues[i] = string(*rb)
+			fmt.Printf("column %s = %s", s.columnNames[i], s.rowValues[i])
+
+			switch s.columnNames[i] {
+			case "time_sec":
+				if sec, err := strconv.ParseInt(s.rowValues[i], 10, 64); err == nil {
+					s.time = null.FloatFrom(float64(sec * 1000))
+				}
+			case "value":
+				if value, err := strconv.ParseFloat(s.rowValues[i], 64); err == nil {
+					s.value = null.FloatFrom(value)
+				}
+			case "metric":
+				if value, err := strconv.ParseFloat(s.rowValues[i], 64); err == nil {
+					s.value = null.FloatFrom(value)
+				}
+			}
+
 			*rb = nil // reset pointer to discard current value to avoid a bug
 			*rb = nil // reset pointer to discard current value to avoid a bug
 		} else {
 		} else {
-			return fmt.Errorf("Cannot convert index %d column %s to type *sql.RawBytes", i, s.colNames[i])
+			return fmt.Errorf("Cannot convert index %d column %s to type *sql.RawBytes", i, s.columnNames[i])
 		}
 		}
-		j = j + 2
 	}
 	}
 	return nil
 	return nil
 }
 }
-
-func (s *stringStringScan) Get() []string {
-	return s.row
-}
-
-// type sqlDataRequest struct {
-// 	Query string `json:"query"`
-// 	Body  []byte `json:"-"`
-// }
-//
-// type seriesStruct struct {
-// 	Columns []string        `json:"columns"`
-// 	Name    string          `json:"name"`
-// 	Values  [][]interface{} `json:"values"`
-// }
-//
-// type resultsStruct struct {
-// 	Series []seriesStruct `json:"series"`
-// }
-//
-// type dataStruct struct {
-// 	Results []resultsStruct `json:"results"`
-// }

pkg/tsdb/request.go

 package tsdb
 package tsdb
 
 
-import "context"
+import (
+	"context"
+)
 
 
 type HandleRequestFunc func(ctx context.Context, req *Request) (*Response, error)
 type HandleRequestFunc func(ctx context.Context, req *Request) (*Response, error)
 
 

public/app/plugins/datasource/mysql/datasource.ts

@@ -39,6 +39,11 @@ export class MysqlDatasource {
       var data = [];
       var data = [];
       if (res.results) {
       if (res.results) {
         _.forEach(res.results, queryRes => {
         _.forEach(res.results, queryRes => {
+
+          if (queryRes.error) {
+            throw {error: queryRes.error, message: queryRes.error};
+          }
+
           for (let series of queryRes.series) {
           for (let series of queryRes.series) {
             data.push({
             data.push({
               target: series.name,
               target: series.name,

public/app/plugins/datasource/mysql/module.ts

@@ -6,6 +6,21 @@ import {QueryCtrl} from 'app/plugins/sdk';
 
 
 class MysqlQueryCtrl extends QueryCtrl {
 class MysqlQueryCtrl extends QueryCtrl {
   static templateUrl = 'partials/query.editor.html';
   static templateUrl = 'partials/query.editor.html';
+
+  resultFormats: any;
+  target: any;
+
+  constructor($scope, $injector) {
+    super($scope, $injector);
+
+    this.target.resultFormat = 'time_series';
+    this.target.alias = "{{table}}{{col_3}}";
+    this.resultFormats = [
+      {text: 'Time series', value: 'time_series'},
+      {text: 'Table', value: 'table'},
+    ];
+
+  }
 }
 }
 
 
 class MysqlConfigCtrl {
 class MysqlConfigCtrl {

public/app/plugins/datasource/mysql/partials/query.editor.html

 <query-editor-row query-ctrl="ctrl" can-collapse="false">
 <query-editor-row query-ctrl="ctrl" can-collapse="false">
   <div class="gf-form-inline">
   <div class="gf-form-inline">
 		<div class="gf-form gf-form--grow">
 		<div class="gf-form gf-form--grow">
-			<textarea rows="3" class="gf-form-input" ng-model="ctrl.target.rawSql" spellcheck="false" placeholder="query expression" data-min-length=0 data-items=100 ng-model-onblur ng-change="ctrl.refreshMetricData()"></textarea>
+			<textarea rows="6" class="gf-form-input" ng-model="ctrl.target.rawSql" spellcheck="false" placeholder="query expression" data-min-length=0 data-items=100 ng-model-onblur ng-change="ctrl.refreshMetricData()"></textarea>
 		</div>
 		</div>
 	</div>
 	</div>
+
+  <div class="gf-form-inline">
+    <div class="gf-form">
+			<label class="gf-form-label query-keyword">Format as</label>
+			<div class="gf-form-select-wrapper">
+				<select class="gf-form-input gf-size-auto" ng-model="ctrl.target.resultFormat" ng-options="f.value as f.text for f in ctrl.resultFormats" ng-change="ctrl.refresh()"></select>
+			</div>
+		</div>
+    <div class="gf-form max-width-30">
+			<label class="gf-form-label query-keyword">Name by</label>
+			<input type="text" class="gf-form-input" ng-model="ctrl.target.alias" spellcheck='false' placeholder="pattern" ng-blur="ctrl.refresh()">
+		</div>
+
+		<div class="gf-form gf-form--grow">
+			<div class="gf-form-label gf-form-label--grow"></div>
+		</div>
+	</div>
+
 </query-editor-row>
 </query-editor-row>