dataproxy: added caching of datasources when doing data proxy requests, #9078

pkg/api/api.go

@@ -217,8 +217,8 @@ func (hs *HttpServer) registerRoutes() {
 		}, reqOrgAdmin)
 
 		r.Get("/frontend/settings/", GetFrontendSettings)
-		r.Any("/datasources/proxy/:id/*", reqSignedIn, ProxyDataSourceRequest)
-		r.Any("/datasources/proxy/:id", reqSignedIn, ProxyDataSourceRequest)
+		r.Any("/datasources/proxy/:id/*", reqSignedIn, hs.ProxyDataSourceRequest)
+		r.Any("/datasources/proxy/:id", reqSignedIn, hs.ProxyDataSourceRequest)
 
 		// Dashboard
 		r.Group("/dashboards", func() {

pkg/api/dataproxy.go

 package api
 
 import (
+	"fmt"
+	"time"
+
 	"github.com/grafana/grafana/pkg/api/pluginproxy"
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/metrics"
@@ -9,19 +12,35 @@ import (
 	"github.com/grafana/grafana/pkg/plugins"
 )
 
-func getDatasource(id int64, orgId int64) (*m.DataSource, error) {
+const HeaderNameNoBackendCache = "X-Grafana-NoCache"
+
+func (hs *HttpServer) getDatasourceById(id int64, orgId int64, nocache bool) (*m.DataSource, error) {
+	cacheKey := fmt.Sprintf("ds-%d", id)
+
+	if !nocache {
+		if cached, found := hs.cache.Get(cacheKey); found {
+			ds := cached.(*m.DataSource)
+			if ds.OrgId == orgId {
+				return ds, nil
+			}
+		}
+	}
+
 	query := m.GetDataSourceByIdQuery{Id: id, OrgId: orgId}
 	if err := bus.Dispatch(&query); err != nil {
 		return nil, err
 	}
 
+	hs.cache.Set(cacheKey, query.Result, time.Second*5)
 	return query.Result, nil
 }
 
-func ProxyDataSourceRequest(c *middleware.Context) {
+func (hs *HttpServer) ProxyDataSourceRequest(c *middleware.Context) {
 	c.TimeRequest(metrics.M_DataSource_ProxyReq_Timer)
 
-	ds, err := getDatasource(c.ParamsInt64(":id"), c.OrgId)
+	nocache := c.Req.Header.Get(HeaderNameNoBackendCache) == "true"
+
+	ds, err := hs.getDatasourceById(c.ParamsInt64(":id"), c.OrgId, nocache)
 
 	if err != nil {
 		c.JsonApiErr(500, "Unable to load datasource meta data", err)

pkg/api/http_server.go

@@ -9,7 +9,9 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"time"
 
+	gocache "github.com/patrickmn/go-cache"
 	macaron "gopkg.in/macaron.v1"
 
 	"github.com/grafana/grafana/pkg/api/live"
@@ -29,13 +31,15 @@ type HttpServer struct {
 	macaron       *macaron.Macaron
 	context       context.Context
 	streamManager *live.StreamManager
+	cache         *gocache.Cache
 
 	httpSrv *http.Server
 }
 
 func NewHttpServer() *HttpServer {
 	return &HttpServer{
-		log: log.New("http.server"),
+		log:   log.New("http.server"),
+		cache: gocache.New(5*time.Minute, 10*time.Minute),
 	}
 }
 

pkg/metrics/metrics.go

@@ -55,6 +55,7 @@ var (
 	M_Alerting_Notification_Sent_Pushover  Counter
 	M_Aws_CloudWatch_GetMetricStatistics   Counter
 	M_Aws_CloudWatch_ListMetrics           Counter
+	M_DB_DataSource_QueryById              Counter
 
 	// Timers
 	M_DataSource_ProxyReq_Timer Timer
@@ -130,6 +131,8 @@ func initMetricVars(settings *MetricSettings) {
 	M_Aws_CloudWatch_GetMetricStatistics = RegCounter("aws.cloudwatch.get_metric_statistics")
 	M_Aws_CloudWatch_ListMetrics = RegCounter("aws.cloudwatch.list_metrics")
 
+	M_DB_DataSource_QueryById = RegCounter("db.datasource.query_by_id")
+
 	// Timers
 	M_DataSource_ProxyReq_Timer = RegTimer("api.dataproxy.request.all")
 	M_Alerting_Execution_Time = RegTimer("alerting.execution_time")

pkg/services/sqlstore/datasource.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/securejsondata"
+	"github.com/grafana/grafana/pkg/metrics"
 	m "github.com/grafana/grafana/pkg/models"
 )
 
@@ -19,6 +20,8 @@ func init() {
 }
 
 func GetDataSourceById(query *m.GetDataSourceByIdQuery) error {
+	metrics.M_DB_DataSource_QueryById.Inc(1)
+
 	datasource := m.DataSource{OrgId: query.OrgId, Id: query.Id}
 	has, err := x.Get(&datasource)
 

public/app/core/services/backend_srv.ts

@@ -7,8 +7,9 @@ import coreModule from 'app/core/core_module';
 import appEvents from 'app/core/app_events';
 
 export class BackendSrv {
-  inFlightRequests = {};
-  HTTP_REQUEST_CANCELLED = -1;
+  private inFlightRequests = {};
+  private HTTP_REQUEST_CANCELLED = -1;
+  private noBackendCache: boolean;
 
   /** @ngInject */
   constructor(private $http, private alertSrv, private $rootScope, private $q, private $timeout, private contextSrv) {
@@ -34,6 +35,13 @@ export class BackendSrv {
     return this.request({ method: 'PUT', url: url, data: data });
   }
 
+  withNoBackendCache(callback) {
+    this.noBackendCache = true;
+    return callback().finally(() => {
+      this.noBackendCache = false;
+    });
+  }
+
   requestErrorHandler(err) {
     if (err.isHandled) {
       return;
@@ -149,6 +157,10 @@ export class BackendSrv {
         options.headers['X-DS-Authorization'] = options.headers.Authorization;
         delete options.headers.Authorization;
       }
+
+      if (this.noBackendCache) {
+        options.headers['X-Grafana-NoCache'] = 'true';
+      }
     }
 
     return this.$http(options).then(response => {

public/app/features/plugins/ds_edit_ctrl.ts

@@ -111,31 +111,31 @@ export class DataSourceEditCtrl {
   }
 
   testDatasource() {
-    this.testing = { done: false };
-
     this.datasourceSrv.get(this.current.name).then(datasource => {
       if (!datasource.testDatasource) {
-        delete this.testing;
         return;
       }
 
-      return datasource.testDatasource().then(result => {
-        this.testing.message = result.message;
-        this.testing.status = result.status;
-        this.testing.title = result.title;
-      }).catch(err => {
-        if (err.statusText) {
-          this.testing.message = err.statusText;
-          this.testing.title = "HTTP Error";
-        } else {
-          this.testing.message = err.message;
-          this.testing.title = "Unknown error";
-        }
-      });
-    }).finally(() => {
-      if (this.testing) {
+      this.testing = {done: false};
+
+      // make test call in no backend cache context
+      this.backendSrv.withNoBackendCache(() => {
+        return datasource.testDatasource().then(result => {
+          this.testing.message = result.message;
+          this.testing.status = result.status;
+          this.testing.title = result.title;
+        }).catch(err => {
+          if (err.statusText) {
+            this.testing.message = err.statusText;
+            this.testing.title = "HTTP Error";
+          } else {
+            this.testing.message = err.message;
+            this.testing.title = "Unknown error";
+          }
+        });
+      }).finally(() => {
         this.testing.done = true;
-      }
+      });
     });
   }
 

vendor/github.com/patrickmn/go-cache/CONTRIBUTORS

@@ -6,3 +6,4 @@ code was contributed.)
 Dustin Sallings <dustin@spy.net>
 Jason Mooberry <jasonmoo@me.com>
 Sergey Shepelev <temotor@gmail.com>
+Alex Edwards <ajmedwards@gmail.com>

vendor/github.com/patrickmn/go-cache/LICENSE

-Copyright (c) 2012-2016 Patrick Mylund Nielsen and the go-cache contributors
+Copyright (c) 2012-2017 Patrick Mylund Nielsen and the go-cache contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

vendor/github.com/patrickmn/go-cache/README.md

@@ -20,86 +20,62 @@ one) to recover from downtime quickly. (See the docs for `NewFrom()` for caveats
 ### Usage
 
 ```go
-	import (
-		"fmt"
-		"github.com/patrickmn/go-cache"
-		"time"
-	)
-
-	func main() {
-
-		// Create a cache with a default expiration time of 5 minutes, and which
-		// purges expired items every 30 seconds
-		c := cache.New(5*time.Minute, 30*time.Second)
-
-		// Set the value of the key "foo" to "bar", with the default expiration time
-		c.Set("foo", "bar", cache.DefaultExpiration)
-
-		// Set the value of the key "baz" to 42, with no expiration time
-		// (the item won't be removed until it is re-set, or removed using
-		// c.Delete("baz")
-		c.Set("baz", 42, cache.NoExpiration)
-
-		// Get the string associated with the key "foo" from the cache
-		foo, found := c.Get("foo")
-		if found {
-			fmt.Println(foo)
-		}
-
-		// Since Go is statically typed, and cache values can be anything, type
-		// assertion is needed when values are being passed to functions that don't
-		// take arbitrary types, (i.e. interface{}). The simplest way to do this for
-		// values which will only be used once--e.g. for passing to another
-		// function--is:
-		foo, found := c.Get("foo")
-		if found {
-			MyFunction(foo.(string))
-		}
-
-		// This gets tedious if the value is used several times in the same function.
-		// You might do either of the following instead:
-		if x, found := c.Get("foo"); found {
-			foo := x.(string)
-			// ...
-		}
-		// or
-		var foo string
-		if x, found := c.Get("foo"); found {
-			foo = x.(string)
-		}
-		// ...
-		// foo can then be passed around freely as a string
+import (
+	"fmt"
+	"github.com/patrickmn/go-cache"
+	"time"
+)
+
+func main() {
+	// Create a cache with a default expiration time of 5 minutes, and which
+	// purges expired items every 10 minutes
+	c := cache.New(5*time.Minute, 10*time.Minute)
+
+	// Set the value of the key "foo" to "bar", with the default expiration time
+	c.Set("foo", "bar", cache.DefaultExpiration)
+
+	// Set the value of the key "baz" to 42, with no expiration time
+	// (the item won't be removed until it is re-set, or removed using
+	// c.Delete("baz")
+	c.Set("baz", 42, cache.NoExpiration)
+
+	// Get the string associated with the key "foo" from the cache
+	foo, found := c.Get("foo")
+	if found {
+		fmt.Println(foo)
+	}
 
-		// Want performance? Store pointers!
-		c.Set("foo", &MyStruct, cache.DefaultExpiration)
-		if x, found := c.Get("foo"); found {
-			foo := x.(*MyStruct)
-			// ...
-		}
-
-		// If you store a reference type like a pointer, slice, map or channel, you
-		// do not need to run Set if you modify the underlying data. The cached
-		// reference points to the same memory, so if you modify a struct whose
-		// pointer you've stored in the cache, retrieving that pointer with Get will
-		// point you to the same data:
-		foo := &MyStruct{Num: 1}
-		c.Set("foo", foo, cache.DefaultExpiration)
-		// ...
-		x, _ := c.Get("foo")
-		foo := x.(*MyStruct)
-		fmt.Println(foo.Num)
-		// ...
-		foo.Num++
-		// ...
-		x, _ := c.Get("foo")
-		foo := x.(*MyStruct)
-		foo.Println(foo.Num)
+	// Since Go is statically typed, and cache values can be anything, type
+	// assertion is needed when values are being passed to functions that don't
+	// take arbitrary types, (i.e. interface{}). The simplest way to do this for
+	// values which will only be used once--e.g. for passing to another
+	// function--is:
+	foo, found := c.Get("foo")
+	if found {
+		MyFunction(foo.(string))
+	}
 
-		// will print:
-		// 1
-		// 2
+	// This gets tedious if the value is used several times in the same function.
+	// You might do either of the following instead:
+	if x, found := c.Get("foo"); found {
+		foo := x.(string)
+		// ...
+	}
+	// or
+	var foo string
+	if x, found := c.Get("foo"); found {
+		foo = x.(string)
+	}
+	// ...
+	// foo can then be passed around freely as a string
 
+	// Want performance? Store pointers!
+	c.Set("foo", &MyStruct, cache.DefaultExpiration)
+	if x, found := c.Get("foo"); found {
+		foo := x.(*MyStruct)
+			// ...
 	}
+}
 ```
 
 ### Reference

vendor/github.com/patrickmn/go-cache/cache.go

@@ -135,6 +135,36 @@ func (c *cache) Get(k string) (interface{}, bool) {
 	return item.Object, true
 }
 
+// GetWithExpiration returns an item and its expiration time from the cache.
+// It returns the item or nil, the expiration time if one is set (if the item
+// never expires a zero value for time.Time is returned), and a bool indicating
+// whether the key was found.
+func (c *cache) GetWithExpiration(k string) (interface{}, time.Time, bool) {
+	c.mu.RLock()
+	// "Inlining" of get and Expired
+	item, found := c.items[k]
+	if !found {
+		c.mu.RUnlock()
+		return nil, time.Time{}, false
+	}
+
+	if item.Expiration > 0 {
+		if time.Now().UnixNano() > item.Expiration {
+			c.mu.RUnlock()
+			return nil, time.Time{}, false
+		}
+
+		// Return the item and the expiration time
+		c.mu.RUnlock()
+		return item.Object, time.Unix(0, item.Expiration), true
+	}
+
+	// If expiration <= 0 (i.e. no expiration time set) then return the item
+	// and a zeroed time.Time
+	c.mu.RUnlock()
+	return item.Object, time.Time{}, true
+}
+
 func (c *cache) get(k string) (interface{}, bool) {
 	item, found := c.items[k]
 	if !found {
@@ -1044,7 +1074,6 @@ type janitor struct {
 }
 
 func (j *janitor) Run(c *cache) {
-	j.stop = make(chan bool)
 	ticker := time.NewTicker(j.Interval)
 	for {
 		select {
@@ -1064,6 +1093,7 @@ func stopJanitor(c *Cache) {
 func runJanitor(c *cache, ci time.Duration) {
 	j := &janitor{
 		Interval: ci,
+		stop:     make(chan bool),
 	}
 	c.janitor = j
 	go j.Run(c)

vendor/vendor.json

@@ -455,10 +455,10 @@
 			"revisionTime": "2017-07-07T05:36:02Z"
 		},
 		{
-			"checksumSHA1": "8z32QKTSDusa4QQyunKE4kyYXZ8=",
+			"checksumSHA1": "JVGDxPn66bpe6xEiexs1r+y6jF0=",
 			"path": "github.com/patrickmn/go-cache",
-			"revision": "e7a9def80f35fe1b170b7b8b68871d59dea117e1",
-			"revisionTime": "2016-11-25T23:48:19Z"
+			"revision": "a3647f8e31d79543b2d0f0ae2fe5c379d72cedc0",
+			"revisionTime": "2017-07-22T04:01:10Z"
 		},
 		{
 			"checksumSHA1": "SMUvX2B8eoFd9wnPofwBKlN6btE=",