Skip to content

N
nexpie-grafana-theme
Commit 1bce8f66 by Torkel Ödegaard
Options

Merge pull request #3736 from mtanda/cloudwatch_custom_metrics_alpha 

(cloudwatch) custom metrics support
parents 714129c8 ccb063df
Showing with 232 additions and 12 deletions
pkg/api/cloudwatch/metrics.go
...@@ -3,7 +3,14 @@ package cloudwatch ...@@ -3,7 +3,14 @@ package cloudwatch
import ( import (
"encoding/json" "encoding/json"
"sort" "sort"
"strings"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awsutil"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/middleware"
"github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/util"
) )
...@@ -11,6 +18,14 @@ import ( ...@@ -11,6 +18,14 @@ import (
var metricsMap map[string][]string var metricsMap map[string][]string
var dimensionsMap map[string][]string var dimensionsMap map[string][]string
type CustomMetricsCache struct {
Expire time.Time
Cache []string
}
var customMetricsMetricsMap map[string]map[string]map[string]*CustomMetricsCache
var customMetricsDimensionsMap map[string]map[string]map[string]*CustomMetricsCache
func init() { func init() {
metricsMap = map[string][]string{ metricsMap = map[string][]string{
"AWS/AutoScaling": {"GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity", "GroupInServiceInstances", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"}, "AWS/AutoScaling": {"GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity", "GroupInServiceInstances", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"},
...@@ -85,6 +100,9 @@ func init() { ...@@ -85,6 +100,9 @@ func init() {
"AWS/WAF": {"Rule", "WebACL"}, "AWS/WAF": {"Rule", "WebACL"},
"AWS/WorkSpaces": {"DirectoryId", "WorkspaceId"}, "AWS/WorkSpaces": {"DirectoryId", "WorkspaceId"},
} }
customMetricsMetricsMap = make(map[string]map[string]map[string]*CustomMetricsCache)
customMetricsDimensionsMap = make(map[string]map[string]map[string]*CustomMetricsCache)
} }
// Whenever this list is updated, frontend list should also be updated. // Whenever this list is updated, frontend list should also be updated.
...@@ -127,11 +145,20 @@ func handleGetMetrics(req *cwRequest, c *middleware.Context) { ...@@ -127,11 +145,20 @@ func handleGetMetrics(req *cwRequest, c *middleware.Context) {
json.Unmarshal(req.Body, reqParam) json.Unmarshal(req.Body, reqParam)
namespaceMetrics, exists := metricsMap[reqParam.Parameters.Namespace] var namespaceMetrics []string
if !exists { if !isCustomMetrics(reqParam.Parameters.Namespace) {
var exists bool
if namespaceMetrics, exists = metricsMap[reqParam.Parameters.Namespace]; !exists {
c.JsonApiErr(404, "Unable to find namespace "+reqParam.Parameters.Namespace, nil) c.JsonApiErr(404, "Unable to find namespace "+reqParam.Parameters.Namespace, nil)
return return
} }
} else {
var err error
if namespaceMetrics, err = getMetricsForCustomMetrics(req.Region, reqParam.Parameters.Namespace, req.DataSource.Database, getAllMetrics); err != nil {
c.JsonApiErr(500, "Unable to call AWS API", err)
return
}
}
sort.Sort(sort.StringSlice(namespaceMetrics)) sort.Sort(sort.StringSlice(namespaceMetrics))
result := []interface{}{} result := []interface{}{}
...@@ -151,11 +178,20 @@ func handleGetDimensions(req *cwRequest, c *middleware.Context) { ...@@ -151,11 +178,20 @@ func handleGetDimensions(req *cwRequest, c *middleware.Context) {
json.Unmarshal(req.Body, reqParam) json.Unmarshal(req.Body, reqParam)
dimensionValues, exists := dimensionsMap[reqParam.Parameters.Namespace] var dimensionValues []string
if !exists { if !isCustomMetrics(reqParam.Parameters.Namespace) {
var exists bool
if dimensionValues, exists = dimensionsMap[reqParam.Parameters.Namespace]; !exists {
c.JsonApiErr(404, "Unable to find dimension "+reqParam.Parameters.Namespace, nil) c.JsonApiErr(404, "Unable to find dimension "+reqParam.Parameters.Namespace, nil)
return return
} }
} else {
var err error
if dimensionValues, err = getDimensionsForCustomMetrics(req.Region, reqParam.Parameters.Namespace, req.DataSource.Database, getAllMetrics); err != nil {
c.JsonApiErr(500, "Unable to call AWS API", err)
return
}
}
sort.Sort(sort.StringSlice(dimensionValues)) sort.Sort(sort.StringSlice(dimensionValues))
result := []interface{}{} result := []interface{}{}
...@@ -165,3 +201,122 @@ func handleGetDimensions(req *cwRequest, c *middleware.Context) { ...@@ -165,3 +201,122 @@ func handleGetDimensions(req *cwRequest, c *middleware.Context) {
c.JSON(200, result) c.JSON(200, result)
} }
func getAllMetrics(region string, namespace string, database string) (cloudwatch.ListMetricsOutput, error) {
cfg := &aws.Config{
Region: aws.String(region),
Credentials: getCredentials(database),
}
svc := cloudwatch.New(session.New(cfg), cfg)
params := &cloudwatch.ListMetricsInput{
Namespace: aws.String(namespace),
}
var resp cloudwatch.ListMetricsOutput
err := svc.ListMetricsPages(params,
func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool {
metrics, _ := awsutil.ValuesAtPath(page, "Metrics")
for _, metric := range metrics {
resp.Metrics = append(resp.Metrics, metric.(*cloudwatch.Metric))
}
return !lastPage
})
if err != nil {
return resp, err
}
return resp, nil
}
var metricsCacheLock sync.Mutex
func getMetricsForCustomMetrics(region string, namespace string, database string, getAllMetrics func(string, string, string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) {
result, err := getAllMetrics(region, namespace, database)
if err != nil {
return []string{}, err
}
metricsCacheLock.Lock()
defer metricsCacheLock.Unlock()
if _, ok := customMetricsMetricsMap[database]; !ok {
customMetricsMetricsMap[database] = make(map[string]map[string]*CustomMetricsCache)
}
if _, ok := customMetricsMetricsMap[database][region]; !ok {
customMetricsMetricsMap[database][region] = make(map[string]*CustomMetricsCache)
}
if _, ok := customMetricsMetricsMap[database][region][namespace]; !ok {
customMetricsMetricsMap[database][region][namespace] = &CustomMetricsCache{}
customMetricsMetricsMap[database][region][namespace].Cache = make([]string, 0)
}
if customMetricsMetricsMap[database][region][namespace].Expire.After(time.Now()) {
return customMetricsMetricsMap[database][region][namespace].Cache, nil
}
customMetricsMetricsMap[database][region][namespace].Cache = make([]string, 0)
customMetricsMetricsMap[database][region][namespace].Expire = time.Now().Add(5 * time.Minute)
for _, metric := range result.Metrics {
if isDuplicate(customMetricsMetricsMap[database][region][namespace].Cache, *metric.MetricName) {
continue
}
customMetricsMetricsMap[database][region][namespace].Cache = append(customMetricsMetricsMap[database][region][namespace].Cache, *metric.MetricName)
}
return customMetricsMetricsMap[database][region][namespace].Cache, nil
}
var dimensionsCacheLock sync.Mutex
func getDimensionsForCustomMetrics(region string, namespace string, database string, getAllMetrics func(string, string, string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) {
result, err := getAllMetrics(region, namespace, database)
if err != nil {
return []string{}, err
}
dimensionsCacheLock.Lock()
defer dimensionsCacheLock.Unlock()
if _, ok := customMetricsDimensionsMap[database]; !ok {
customMetricsDimensionsMap[database] = make(map[string]map[string]*CustomMetricsCache)
}
if _, ok := customMetricsDimensionsMap[database][region]; !ok {
customMetricsDimensionsMap[database][region] = make(map[string]*CustomMetricsCache)
}
if _, ok := customMetricsDimensionsMap[database][region][namespace]; !ok {
customMetricsDimensionsMap[database][region][namespace] = &CustomMetricsCache{}
customMetricsDimensionsMap[database][region][namespace].Cache = make([]string, 0)
}
if customMetricsDimensionsMap[database][region][namespace].Expire.After(time.Now()) {
return customMetricsDimensionsMap[database][region][namespace].Cache, nil
}
customMetricsDimensionsMap[database][region][namespace].Cache = make([]string, 0)
customMetricsDimensionsMap[database][region][namespace].Expire = time.Now().Add(5 * time.Minute)
for _, metric := range result.Metrics {
for _, dimension := range metric.Dimensions {
if isDuplicate(customMetricsDimensionsMap[database][region][namespace].Cache, *dimension.Name) {
continue
}
customMetricsDimensionsMap[database][region][namespace].Cache = append(customMetricsDimensionsMap[database][region][namespace].Cache, *dimension.Name)
}
}
return customMetricsDimensionsMap[database][region][namespace].Cache, nil
}
func isDuplicate(nameList []string, target string) bool {
for _, name := range nameList {
if name == target {
return true
}
}
return false
}
func isCustomMetrics(namespace string) bool {
return strings.Index(namespace, "AWS/") != 0
}
pkg/api/cloudwatch/metrics_test.go 0 → 100644
package cloudwatch
import (
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/cloudwatch"
. "github.com/smartystreets/goconvey/convey"
)
func TestCloudWatchMetrics(t *testing.T) {
Convey("When calling getMetricsForCustomMetrics", t, func() {
region := "us-east-1"
namespace := "Foo"
database := "default"
f := func(region string, namespace string, database string) (cloudwatch.ListMetricsOutput, error) {
return cloudwatch.ListMetricsOutput{
Metrics: []*cloudwatch.Metric{
{
MetricName: aws.String("Test_MetricName"),
Dimensions: []*cloudwatch.Dimension{
{
Name: aws.String("Test_DimensionName"),
},
},
},
},
}, nil
}
metrics, _ := getMetricsForCustomMetrics(region, namespace, database, f)
Convey("Should contain Test_MetricName", func() {
So(metrics, ShouldContain, "Test_MetricName")
})
})
Convey("When calling getDimensionsForCustomMetrics", t, func() {
region := "us-east-1"
namespace := "Foo"
database := "default"
f := func(region string, namespace string, database string) (cloudwatch.ListMetricsOutput, error) {
return cloudwatch.ListMetricsOutput{
Metrics: []*cloudwatch.Metric{
{
MetricName: aws.String("Test_MetricName"),
Dimensions: []*cloudwatch.Dimension{
{
Name: aws.String("Test_DimensionName"),
},
},
},
},
}, nil
}
dimensionKeys, _ := getDimensionsForCustomMetrics(region, namespace, database, f)
Convey("Should contain Test_DimensionName", func() {
So(dimensionKeys, ShouldContain, "Test_DimensionName")
})
})
}
public/app/plugins/datasource/cloudwatch/datasource.js
...@@ -90,18 +90,20 @@ function (angular, _, moment, dateMath) { ...@@ -90,18 +90,20 @@ function (angular, _, moment, dateMath) {
return this.awsRequest({action: '__GetNamespaces'}); return this.awsRequest({action: '__GetNamespaces'});
}; };
this.getMetrics = function(namespace) { this.getMetrics = function(namespace, region) {
return this.awsRequest({ return this.awsRequest({
action: '__GetMetrics', action: '__GetMetrics',
region: region,
parameters: { parameters: {
namespace: templateSrv.replace(namespace) namespace: templateSrv.replace(namespace)
} }
}); });
}; };
this.getDimensionKeys = function(namespace) { this.getDimensionKeys = function(namespace, region) {
return this.awsRequest({ return this.awsRequest({
action: '__GetDimensions', action: '__GetDimensions',
region: region,
parameters: { parameters: {
namespace: templateSrv.replace(namespace) namespace: templateSrv.replace(namespace)
} }
...@@ -164,14 +166,14 @@ function (angular, _, moment, dateMath) { ...@@ -164,14 +166,14 @@ function (angular, _, moment, dateMath) {
return this.getNamespaces(); return this.getNamespaces();
} }
var metricNameQuery = query.match(/^metrics\(([^\)]+?)\)/); var metricNameQuery = query.match(/^metrics\(([^\)]+?)(,\s?([^,]+?))?\)/);
if (metricNameQuery) { if (metricNameQuery) {
return this.getMetrics(metricNameQuery[1]); return this.getMetrics(metricNameQuery[1], metricNameQuery[3]);
} }
var dimensionKeysQuery = query.match(/^dimension_keys\(([^\)]+?)\)/); var dimensionKeysQuery = query.match(/^dimension_keys\(([^\)]+?)(,\s?([^,]+?))?\)/);
if (dimensionKeysQuery) { if (dimensionKeysQuery) {
return this.getDimensionKeys(dimensionKeysQuery[1]); return this.getDimensionKeys(dimensionKeysQuery[1], dimensionKeysQuery[3]);
} }
var dimensionValuesQuery = query.match(/^dimension_values\(([^,]+?),\s?([^,]+?),\s?([^,]+?),\s?([^,]+?)\)/); var dimensionValuesQuery = query.match(/^dimension_values\(([^,]+?),\s?([^,]+?),\s?([^,]+?),\s?([^,]+?)\)/);
......
public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.js
...@@ -102,7 +102,7 @@ function (angular, _) { ...@@ -102,7 +102,7 @@ function (angular, _) {
var query = $q.when([]); var query = $q.when([]);
if (segment.type === 'key' || segment.type === 'plus-button') { if (segment.type === 'key' || segment.type === 'plus-button') {
query = $scope.datasource.getDimensionKeys($scope.target.namespace); query = $scope.datasource.getDimensionKeys($scope.target.namespace, $scope.target.region);
} else if (segment.type === 'value') { } else if (segment.type === 'value') {
var dimensionKey = $scope.dimSegments[$index-2].value; var dimensionKey = $scope.dimSegments[$index-2].value;
query = $scope.datasource.getDimensionValues(target.region, target.namespace, target.metricName, dimensionKey, {}); query = $scope.datasource.getDimensionValues(target.region, target.namespace, target.metricName, dimensionKey, {});
...@@ -160,7 +160,7 @@ function (angular, _) { ...@@ -160,7 +160,7 @@ function (angular, _) {
}; };
$scope.getMetrics = function() { $scope.getMetrics = function() {
return $scope.datasource.metricFindQuery('metrics(' + $scope.target.namespace + ')') return $scope.datasource.metricFindQuery('metrics(' + $scope.target.namespace + ',' + $scope.target.region + ')')
.then($scope.transformToSegments(true)); .then($scope.transformToSegments(true));
}; };
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment