Skip to content

N
nexpie-grafana-theme
Commit 362162d6 by bergquist
Options

feat(mqe): add token cache

parent 69d63165
Showing with 403 additions and 50 deletions
pkg/tsdb/mqe/model_parser.go
......@@ -6,14 +6,14 @@ import (
"github.com/grafana/grafana/pkg/tsdb"
)
func NewQueryParser() *MQEQueryParser {
return &MQEQueryParser{}
func NewQueryParser() *QueryParser {
return &QueryParser{}
}
type MQEQueryParser struct{}
type QueryParser struct{}
func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource, queryContext *tsdb.QueryContext) (*MQEQuery, error) {
query := &MQEQuery{TimeRange: queryContext.TimeRange}
func (qp *QueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSource, queryContext *tsdb.QueryContext) (*Query, error) {
query := &Query{TimeRange: queryContext.TimeRange}
query.AddAppToAlias = model.Get("addAppToAlias").MustBool(false)
query.AddHostToAlias = model.Get("addHostToAlias").MustBool(false)
query.UseRawQuery = model.Get("rawQuery").MustBool(false)
......@@ -22,11 +22,11 @@ func (qp *MQEQueryParser) Parse(model *simplejson.Json, dsInfo *models.DataSourc
query.Apps = model.Get("apps").MustStringArray([]string{})
query.Hosts = model.Get("hosts").MustStringArray([]string{})
var metrics []MQEMetric
var metrics []Metric
var err error
for _, metricsObj := range model.Get("metrics").MustArray() {
metricJson := simplejson.NewFromAny(metricsObj)
var m MQEMetric
var m Metric
m.Alias = metricJson.Get("alias").MustString("")
m.Metric, err = metricJson.Get("metric").String()
......
pkg/tsdb/mqe/model_parser_test.go
......@@ -11,7 +11,7 @@ import (
func TestMQEQueryParser(t *testing.T) {
Convey("MQE query parser", t, func() {
parser := &MQEQueryParser{}
parser := &QueryParser{}
dsInfo := &models.DataSource{JsonData: simplejson.New()}
queryContext := &tsdb.QueryContext{}
......
pkg/tsdb/mqe/mqe.go
......@@ -12,19 +12,20 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
)
/*
TODO:
* response serie names with wildcards
* real caching
* performance. outgoing requests in pararell.
* frontend plugin. targetContainsTemplates
*/
type MQEExecutor struct {
*models.DataSource
queryParser *MQEQueryParser
responseParser *MQEResponseParser
queryParser *QueryParser
responseParser *ResponseParser
httpClient *http.Client
log log.Logger
tokenClient *TokenClient
......@@ -52,7 +53,7 @@ func init() {
type QueryToSend struct {
RawQuery string
QueryRef *MQEQuery
QueryRef *Query
}
func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, queryContext *tsdb.QueryContext) *tsdb.BatchResult {
......@@ -63,7 +64,7 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer
return result.WithError(err)
}
var mqeQueries []*MQEQuery
var mqeQueries []*Query
for _, v := range queries {
q, err := e.queryParser.Parse(v.Model, e.DataSource, queryContext)
if err != nil {
......@@ -82,9 +83,13 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer
rawQueries = append(rawQueries, queries...)
}
e.log.Debug("Sending request", "url", e.DataSource.Url)
queryResult := &tsdb.QueryResult{}
for _, v := range rawQueries {
e.log.Info("Mqe executor", "query", v)
if setting.Env == setting.DEV {
e.log.Debug("Executing", "query", v)
}
req, err := e.createRequest(v.RawQuery)
......@@ -108,7 +113,11 @@ func (e *MQEExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, quer
}
func (e *MQEExecutor) createRequest(query string) (*http.Request, error) {
u, _ := url.Parse(e.Url)
u, err := url.Parse(e.Url)
if err != nil {
return nil, err
}
u.Path = path.Join(u.Path, "query")
payload := simplejson.New()
......@@ -131,6 +140,5 @@ func (e *MQEExecutor) createRequest(query string) (*http.Request, error) {
req.SetBasicAuth(e.BasicAuthUser, e.BasicAuthPassword)
}
e.log.Debug("Mqe request", "url", req.URL.String())
return req, nil
}
pkg/tsdb/mqe/response_parser.go
......@@ -13,8 +13,8 @@ import (
"github.com/grafana/grafana/pkg/tsdb"
)
func NewResponseParser() *MQEResponseParser {
return &MQEResponseParser{
func NewResponseParser() *ResponseParser {
return &ResponseParser{
log: log.New("tsdb.mqe"),
}
}
......@@ -44,11 +44,11 @@ type MQESerie struct {
Tagset map[string]string `json:"tagset"`
}
type MQEResponseParser struct {
type ResponseParser struct {
log log.Logger
}
func (parser *MQEResponseParser) Parse(res *http.Response, queryRef *MQEQuery) (*tsdb.QueryResult, error) {
func (parser *ResponseParser) Parse(res *http.Response, queryRef *Query) (*tsdb.QueryResult, error) {
body, err := ioutil.ReadAll(res.Body)
defer res.Body.Close()
if err != nil {
......@@ -63,12 +63,12 @@ func (parser *MQEResponseParser) Parse(res *http.Response, queryRef *MQEQuery) (
var data *MQEResponse = &MQEResponse{}
err = json.Unmarshal(body, data)
if err != nil {
parser.log.Info("Failed to unmarshal mqe response", "error", err, "status", res.Status, "body", string(body))
parser.log.Info("Failed to unmarshal response", "error", err, "status", res.Status, "body", string(body))
return nil, err
}
if !data.Success {
return nil, fmt.Errorf("MQE request failed.")
return nil, fmt.Errorf("Request failed.")
}
var series tsdb.TimeSeriesSlice
......
pkg/tsdb/mqe/response_parser_test.go
......@@ -20,7 +20,7 @@ func TestMQEResponseParser(t *testing.T) {
parser := NewResponseParser()
Convey("Can parse response", func() {
queryRef := &MQEQuery{
queryRef := &Query{
AddAppToAlias: true,
AddHostToAlias: true,
}
......
pkg/tsdb/mqe/token_client.go
......@@ -8,15 +8,25 @@ import (
"net/http"
"net/url"
"path"
"time"
"golang.org/x/net/context/ctxhttp"
"strconv"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
"github.com/patrickmn/go-cache"
)
var tokenCache *cache.Cache
func init() {
tokenCache = cache.New(5*time.Minute, 30*time.Second)
}
type TokenClient struct {
tlog log.Logger
log log.Logger
Datasource *models.DataSource
HttpClient *http.Client
}
......@@ -25,27 +35,30 @@ func NewTokenClient(datasource *models.DataSource) *TokenClient {
httpClient, _ := datasource.GetHttpClient()
return &TokenClient{
tlog: log.New("tsdb.mqe.tokenclient"),
log: log.New("tsdb.mqe.tokenclient"),
Datasource: datasource,
HttpClient: httpClient,
}
}
var cache map[int64]*TokenBody = map[int64]*TokenBody{}
//Replace this stupid cache with internal cache from grafana master before merging
func (client *TokenClient) GetTokenData(ctx context.Context) (*TokenBody, error) {
_, excist := cache[client.Datasource.Id]
if !excist {
key := strconv.FormatInt(client.Datasource.Id, 10)
item, found := tokenCache.Get(key)
if found {
if result, ok := item.(*TokenBody); ok {
return result, nil
}
}
b, err := client.RequestTokenData(ctx)
if err != nil {
return nil, err
}
cache[client.Datasource.Id] = b
}
tokenCache.Set(key, b, cache.DefaultExpiration)
return cache[client.Datasource.Id], nil
return b, nil
}
func (client *TokenClient) RequestTokenData(ctx context.Context) (*TokenBody, error) {
......@@ -54,7 +67,7 @@ func (client *TokenClient) RequestTokenData(ctx context.Context) (*TokenBody, er
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
client.tlog.Info("Failed to create request", "error", err)
client.log.Info("Failed to create request", "error", err)
}
res, err := ctxhttp.Do(ctx, client.HttpClient, req)
......@@ -69,14 +82,14 @@ func (client *TokenClient) RequestTokenData(ctx context.Context) (*TokenBody, er
}
if res.StatusCode/100 != 2 {
client.tlog.Info("Request failed", "status", res.Status, "body", string(body))
client.log.Info("Request failed", "status", res.Status, "body", string(body))
return nil, fmt.Errorf("Request failed status: %v", res.Status)
}
var result *TokenResponse
err = json.Unmarshal(body, &result)
if err != nil {
client.tlog.Info("Failed to unmarshal graphite response", "error", err, "status", res.Status, "body", string(body))
client.log.Info("Failed to unmarshal response", "error", err, "status", res.Status, "body", string(body))
return nil, err
}
......
pkg/tsdb/mqe/types.go
......@@ -11,13 +11,13 @@ import (
"github.com/grafana/grafana/pkg/tsdb"
)
type MQEMetric struct {
type Metric struct {
Metric string
Alias string
}
type MQEQuery struct {
Metrics []MQEMetric
type Query struct {
Metrics []Metric
Hosts []string
Apps []string
AddAppToAlias bool
......@@ -32,7 +32,7 @@ var (
containsWildcardPattern *regexp.Regexp = regexp.MustCompile(`\*`)
)
func (q *MQEQuery) Build(availableSeries []string) ([]QueryToSend, error) {
func (q *Query) Build(availableSeries []string) ([]QueryToSend, error) {
var queriesToSend []QueryToSend
where := q.buildWhereClause()
......@@ -90,7 +90,7 @@ func (q *MQEQuery) Build(availableSeries []string) ([]QueryToSend, error) {
return queriesToSend, nil
}
func (q *MQEQuery) buildWhereClause() string {
func (q *Query) buildWhereClause() string {
hasApps := len(q.Apps) > 0
hasHosts := len(q.Hosts) > 0
......
pkg/tsdb/mqe/types_test.go
......@@ -25,11 +25,11 @@ func TestWildcardExpansion(t *testing.T) {
Convey("Can expanding query", t, func() {
Convey("Without wildcard series", func() {
query := &MQEQuery{
Metrics: []MQEMetric{
MQEMetric{Metric: "os.cpu.3.idle", Alias: ""},
MQEMetric{Metric: "os.cpu.2.idle", Alias: ""},
MQEMetric{Metric: "os.cpu.1.idle", Alias: "cpu"},
query := &Query{
Metrics: []Metric{
Metric{Metric: "os.cpu.3.idle", Alias: ""},
Metric{Metric: "os.cpu.2.idle", Alias: ""},
Metric{Metric: "os.cpu.1.idle", Alias: "cpu"},
},
Hosts: []string{"staples-lab-1", "staples-lab-2"},
Apps: []string{"demoapp-1", "demoapp-2"},
......@@ -47,9 +47,9 @@ func TestWildcardExpansion(t *testing.T) {
})
Convey("Containg wildcard series", func() {
query := &MQEQuery{
Metrics: []MQEMetric{
MQEMetric{Metric: "os.cpu*", Alias: ""},
query := &Query{
Metrics: []Metric{
Metric{Metric: "os.cpu*", Alias: ""},
},
Hosts: []string{"staples-lab-1"},
AddAppToAlias: false,
......
vendor/github.com/patrickmn/go-cache/CONTRIBUTORS 0 → 100644
This is a list of people who have contributed code to go-cache. They, or their
employers, are the copyright holders of the contributed code. Contributed code
is subject to the license restrictions listed in LICENSE (as they were when the
code was contributed.)
Dustin Sallings <dustin@spy.net>
Jason Mooberry <jasonmoo@me.com>
Sergey Shepelev <temotor@gmail.com>
vendor/github.com/patrickmn/go-cache/LICENSE 0 → 100644
Copyright (c) 2012-2016 Patrick Mylund Nielsen and the go-cache contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
vendor/github.com/patrickmn/go-cache/README.md 0 → 100644
# go-cache
go-cache is an in-memory key:value store/cache similar to memcached that is
suitable for applications running on a single machine. Its major advantage is
that, being essentially a thread-safe `map[string]interface{}` with expiration
times, it doesn't need to serialize or transmit its contents over the network.
Any object can be stored, for a given duration or forever, and the cache can be
safely used by multiple goroutines.
Although go-cache isn't meant to be used as a persistent datastore, the entire
cache can be saved to and loaded from a file (using `c.Items()` to retrieve the
items map to serialize, and `NewFrom()` to create a cache from a deserialized
one) to recover from downtime quickly. (See the docs for `NewFrom()` for caveats.)
### Installation
`go get github.com/patrickmn/go-cache`
### Usage
```go
import (
"fmt"
"github.com/patrickmn/go-cache"
"time"
)
func main() {
// Create a cache with a default expiration time of 5 minutes, and which
// purges expired items every 30 seconds
c := cache.New(5*time.Minute, 30*time.Second)
// Set the value of the key "foo" to "bar", with the default expiration time
c.Set("foo", "bar", cache.DefaultExpiration)
// Set the value of the key "baz" to 42, with no expiration time
// (the item won't be removed until it is re-set, or removed using
// c.Delete("baz")
c.Set("baz", 42, cache.NoExpiration)
// Get the string associated with the key "foo" from the cache
foo, found := c.Get("foo")
if found {
fmt.Println(foo)
}
// Since Go is statically typed, and cache values can be anything, type
// assertion is needed when values are being passed to functions that don't
// take arbitrary types, (i.e. interface{}). The simplest way to do this for
// values which will only be used once--e.g. for passing to another
// function--is:
foo, found := c.Get("foo")
if found {
MyFunction(foo.(string))
}
// This gets tedious if the value is used several times in the same function.
// You might do either of the following instead:
if x, found := c.Get("foo"); found {
foo := x.(string)
// ...
}
// or
var foo string
if x, found := c.Get("foo"); found {
foo = x.(string)
}
// ...
// foo can then be passed around freely as a string
// Want performance? Store pointers!
c.Set("foo", &MyStruct, cache.DefaultExpiration)
if x, found := c.Get("foo"); found {
foo := x.(*MyStruct)
// ...
}
// If you store a reference type like a pointer, slice, map or channel, you
// do not need to run Set if you modify the underlying data. The cached
// reference points to the same memory, so if you modify a struct whose
// pointer you've stored in the cache, retrieving that pointer with Get will
// point you to the same data:
foo := &MyStruct{Num: 1}
c.Set("foo", foo, cache.DefaultExpiration)
// ...
x, _ := c.Get("foo")
foo := x.(*MyStruct)
fmt.Println(foo.Num)
// ...
foo.Num++
// ...
x, _ := c.Get("foo")
foo := x.(*MyStruct)
foo.Println(foo.Num)
// will print:
// 1
// 2
}
```
### Reference
`godoc` or [http://godoc.org/github.com/patrickmn/go-cache](http://godoc.org/github.com/patrickmn/go-cache)
vendor/github.com/patrickmn/go-cache/sharded.go 0 → 100644
package cache
import (
"crypto/rand"
"math"
"math/big"
insecurerand "math/rand"
"os"
"runtime"
"time"
)
// This is an experimental and unexported (for now) attempt at making a cache
// with better algorithmic complexity than the standard one, namely by
// preventing write locks of the entire cache when an item is added. As of the
// time of writing, the overhead of selecting buckets results in cache
// operations being about twice as slow as for the standard cache with small
// total cache sizes, and faster for larger ones.
//
// See cache_test.go for a few benchmarks.
type unexportedShardedCache struct {
*shardedCache
}
type shardedCache struct {
seed uint32
m uint32
cs []*cache
janitor *shardedJanitor
}
// djb2 with better shuffling. 5x faster than FNV with the hash.Hash overhead.
func djb33(seed uint32, k string) uint32 {
var (
l = uint32(len(k))
d = 5381 + seed + l
i = uint32(0)
)
// Why is all this 5x faster than a for loop?
if l >= 4 {
for i < l-4 {
d = (d * 33) ^ uint32(k[i])
d = (d * 33) ^ uint32(k[i+1])
d = (d * 33) ^ uint32(k[i+2])
d = (d * 33) ^ uint32(k[i+3])
i += 4
}
}
switch l - i {
case 1:
case 2:
d = (d * 33) ^ uint32(k[i])
case 3:
d = (d * 33) ^ uint32(k[i])
d = (d * 33) ^ uint32(k[i+1])
case 4:
d = (d * 33) ^ uint32(k[i])
d = (d * 33) ^ uint32(k[i+1])
d = (d * 33) ^ uint32(k[i+2])
}
return d ^ (d >> 16)
}
func (sc *shardedCache) bucket(k string) *cache {
return sc.cs[djb33(sc.seed, k)%sc.m]
}
func (sc *shardedCache) Set(k string, x interface{}, d time.Duration) {
sc.bucket(k).Set(k, x, d)
}
func (sc *shardedCache) Add(k string, x interface{}, d time.Duration) error {
return sc.bucket(k).Add(k, x, d)
}
func (sc *shardedCache) Replace(k string, x interface{}, d time.Duration) error {
return sc.bucket(k).Replace(k, x, d)
}
func (sc *shardedCache) Get(k string) (interface{}, bool) {
return sc.bucket(k).Get(k)
}
func (sc *shardedCache) Increment(k string, n int64) error {
return sc.bucket(k).Increment(k, n)
}
func (sc *shardedCache) IncrementFloat(k string, n float64) error {
return sc.bucket(k).IncrementFloat(k, n)
}
func (sc *shardedCache) Decrement(k string, n int64) error {
return sc.bucket(k).Decrement(k, n)
}
func (sc *shardedCache) Delete(k string) {
sc.bucket(k).Delete(k)
}
func (sc *shardedCache) DeleteExpired() {
for _, v := range sc.cs {
v.DeleteExpired()
}
}
// Returns the items in the cache. This may include items that have expired,
// but have not yet been cleaned up. If this is significant, the Expiration
// fields of the items should be checked. Note that explicit synchronization
// is needed to use a cache and its corresponding Items() return values at
// the same time, as the maps are shared.
func (sc *shardedCache) Items() []map[string]Item {
res := make([]map[string]Item, len(sc.cs))
for i, v := range sc.cs {
res[i] = v.Items()
}
return res
}
func (sc *shardedCache) Flush() {
for _, v := range sc.cs {
v.Flush()
}
}
type shardedJanitor struct {
Interval time.Duration
stop chan bool
}
func (j *shardedJanitor) Run(sc *shardedCache) {
j.stop = make(chan bool)
tick := time.Tick(j.Interval)
for {
select {
case <-tick:
sc.DeleteExpired()
case <-j.stop:
return
}
}
}
func stopShardedJanitor(sc *unexportedShardedCache) {
sc.janitor.stop <- true
}
func runShardedJanitor(sc *shardedCache, ci time.Duration) {
j := &shardedJanitor{
Interval: ci,
}
sc.janitor = j
go j.Run(sc)
}
func newShardedCache(n int, de time.Duration) *shardedCache {
max := big.NewInt(0).SetUint64(uint64(math.MaxUint32))
rnd, err := rand.Int(rand.Reader, max)
var seed uint32
if err != nil {
os.Stderr.Write([]byte("WARNING: go-cache's newShardedCache failed to read from the system CSPRNG (/dev/urandom or equivalent.) Your system's security may be compromised. Continuing with an insecure seed.\n"))
seed = insecurerand.Uint32()
} else {
seed = uint32(rnd.Uint64())
}
sc := &shardedCache{
seed: seed,
m: uint32(n),
cs: make([]*cache, n),
}
for i := 0; i < n; i++ {
c := &cache{
defaultExpiration: de,
items: map[string]Item{},
}
sc.cs[i] = c
}
return sc
}
func unexportedNewSharded(defaultExpiration, cleanupInterval time.Duration, shards int) *unexportedShardedCache {
if defaultExpiration == 0 {
defaultExpiration = -1
}
sc := newShardedCache(shards, defaultExpiration)
SC := &unexportedShardedCache{sc}
if cleanupInterval > 0 {
runShardedJanitor(sc, cleanupInterval)
runtime.SetFinalizer(SC, stopShardedJanitor)
}
return SC
}
vendor/vendor.json
......@@ -315,6 +315,12 @@
"versionExact": "v1.21.1"
},
{
"checksumSHA1": "8z32QKTSDusa4QQyunKE4kyYXZ8=",
"path": "github.com/patrickmn/go-cache",
"revision": "e7a9def80f35fe1b170b7b8b68871d59dea117e1",
"revisionTime": "2016-11-25T23:48:19Z"
},
{
"checksumSHA1": "SMUvX2B8eoFd9wnPofwBKlN6btE=",
"path": "github.com/prometheus/client_golang/api/prometheus",
"revision": "5636dc67ae776adf5590da7349e70fbb9559972d",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment