Skip to content

N
nexpie-grafana-theme
Commit 40ed235b by Mitsuhiro Tanda
Options

support GetMetricData

parent 077cf9a3
Showing with 243 additions and 46 deletions
pkg/metrics/metrics.go
...@@ -44,6 +44,7 @@ var ( ...@@ -44,6 +44,7 @@ var (
M_Alerting_Notification_Sent *prometheus.CounterVec M_Alerting_Notification_Sent *prometheus.CounterVec
M_Aws_CloudWatch_GetMetricStatistics prometheus.Counter M_Aws_CloudWatch_GetMetricStatistics prometheus.Counter
M_Aws_CloudWatch_ListMetrics prometheus.Counter M_Aws_CloudWatch_ListMetrics prometheus.Counter
M_Aws_CloudWatch_GetMetricData prometheus.Counter
M_DB_DataSource_QueryById prometheus.Counter M_DB_DataSource_QueryById prometheus.Counter
// Timers // Timers
...@@ -218,6 +219,12 @@ func init() { ...@@ -218,6 +219,12 @@ func init() {
Namespace: exporterName, Namespace: exporterName,
}) })
M_Aws_CloudWatch_GetMetricData = prometheus.NewCounter(prometheus.CounterOpts{
Name: "aws_cloudwatch_get_metric_data_total",
Help: "counter for getting metric data time series from aws",
Namespace: exporterName,
})
M_DB_DataSource_QueryById = prometheus.NewCounter(prometheus.CounterOpts{ M_DB_DataSource_QueryById = prometheus.NewCounter(prometheus.CounterOpts{
Name: "db_datasource_query_by_id_total", Name: "db_datasource_query_by_id_total",
Help: "counter for getting datasource by id", Help: "counter for getting datasource by id",
...@@ -307,6 +314,7 @@ func initMetricVars() { ...@@ -307,6 +314,7 @@ func initMetricVars() {
M_Alerting_Notification_Sent, M_Alerting_Notification_Sent,
M_Aws_CloudWatch_GetMetricStatistics, M_Aws_CloudWatch_GetMetricStatistics,
M_Aws_CloudWatch_ListMetrics, M_Aws_CloudWatch_ListMetrics,
M_Aws_CloudWatch_GetMetricData,
M_DB_DataSource_QueryById, M_DB_DataSource_QueryById,
M_Alerting_Active_Alerts, M_Alerting_Active_Alerts,
M_StatTotal_Dashboards, M_StatTotal_Dashboards,
......
pkg/tsdb/cloudwatch/cloudwatch.go
...@@ -14,6 +14,7 @@ import ( ...@@ -14,6 +14,7 @@ import (
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb" "github.com/grafana/grafana/pkg/tsdb"
"golang.org/x/sync/errgroup"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/request"
...@@ -88,48 +89,63 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo ...@@ -88,48 +89,63 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo
Results: make(map[string]*tsdb.QueryResult), Results: make(map[string]*tsdb.QueryResult),
} }
errCh := make(chan error, 1) eg, ectx := errgroup.WithContext(ctx)
resCh := make(chan *tsdb.QueryResult, 1)
currentlyExecuting := 0 getMetricDataQueries := make(map[string]map[string]*CloudWatchQuery)
for i, model := range queryContext.Queries { for i, model := range queryContext.Queries {
queryType := model.Model.Get("type").MustString() queryType := model.Model.Get("type").MustString()
if queryType != "timeSeriesQuery" && queryType != "" { if queryType != "timeSeriesQuery" && queryType != "" {
continue continue
} }
currentlyExecuting++
go func(refId string, index int) { query, err := parseQuery(queryContext.Queries[i].Model)
queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext) if err != nil {
currentlyExecuting-- return nil, err
}
query.RefId = queryContext.Queries[i].RefId
if query.Id != "" {
if _, ok := getMetricDataQueries[query.Region]; !ok {
getMetricDataQueries[query.Region] = make(map[string]*CloudWatchQuery)
}
getMetricDataQueries[query.Region][query.Id] = query
continue
}
eg.Go(func() error {
queryRes, err := e.executeQuery(ectx, query, queryContext)
if err != nil { if err != nil {
errCh <- err return err
} else {
queryRes.RefId = refId
resCh <- queryRes
} }
}(model.RefId, i) result.Results[queryRes.RefId] = queryRes
return nil
})
} }
for currentlyExecuting != 0 { if len(getMetricDataQueries) > 0 {
select { for region, getMetricDataQuery := range getMetricDataQueries {
case res := <-resCh: q := getMetricDataQuery
result.Results[res.RefId] = res eg.Go(func() error {
case err := <-errCh: queryResponses, err := e.executeGetMetricDataQuery(ectx, region, q, queryContext)
return result, err if err != nil {
case <-ctx.Done(): return err
return result, ctx.Err() }
for _, queryRes := range queryResponses {
result.Results[queryRes.RefId] = queryRes
}
return nil
})
} }
} }
return result, nil if err := eg.Wait(); err != nil {
}
func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
query, err := parseQuery(parameters)
if err != nil {
return nil, err return nil, err
} }
return result, nil
}
func (e *CloudWatchExecutor) executeQuery(ctx context.Context, query *CloudWatchQuery, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
client, err := e.getClient(query.Region) client, err := e.getClient(query.Region)
if err != nil { if err != nil {
return nil, err return nil, err
...@@ -201,6 +217,139 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simpl ...@@ -201,6 +217,139 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simpl
return queryRes, nil return queryRes, nil
} }
func (e *CloudWatchExecutor) executeGetMetricDataQuery(ctx context.Context, region string, queries map[string]*CloudWatchQuery, queryContext *tsdb.TsdbQuery) ([]*tsdb.QueryResult, error) {
queryResponses := make([]*tsdb.QueryResult, 0)
// validate query
for _, query := range queries {
if !(len(query.Statistics) == 1 && len(query.ExtendedStatistics) == 0) &&
!(len(query.Statistics) == 0 && len(query.ExtendedStatistics) == 1) {
return queryResponses, errors.New("Statistics count should be 1")
}
}
client, err := e.getClient(region)
if err != nil {
return queryResponses, err
}
startTime, err := queryContext.TimeRange.ParseFrom()
if err != nil {
return queryResponses, err
}
endTime, err := queryContext.TimeRange.ParseTo()
if err != nil {
return queryResponses, err
}
params := &cloudwatch.GetMetricDataInput{
StartTime: aws.Time(startTime),
EndTime: aws.Time(endTime),
ScanBy: aws.String("TimestampAscending"),
}
for _, query := range queries {
// 1 minutes resolutin metrics is stored for 15 days, 15 * 24 * 60 = 21600
if query.HighResolution && (((endTime.Unix() - startTime.Unix()) / int64(query.Period)) > 21600) {
return nil, errors.New("too long query period")
}
mdq := &cloudwatch.MetricDataQuery{
Id: aws.String(query.Id),
ReturnData: aws.Bool(query.ReturnData),
}
if query.Expression != "" {
mdq.Expression = aws.String(query.Expression)
} else {
mdq.MetricStat = &cloudwatch.MetricStat{
Metric: &cloudwatch.Metric{
Namespace: aws.String(query.Namespace),
MetricName: aws.String(query.MetricName),
},
Period: aws.Int64(int64(query.Period)),
}
for _, d := range query.Dimensions {
mdq.MetricStat.Metric.Dimensions = append(mdq.MetricStat.Metric.Dimensions,
&cloudwatch.Dimension{
Name: d.Name,
Value: d.Value,
})
}
if len(query.Statistics) == 1 {
mdq.MetricStat.Stat = query.Statistics[0]
} else {
mdq.MetricStat.Stat = query.ExtendedStatistics[0]
}
}
params.MetricDataQueries = append(params.MetricDataQueries, mdq)
}
nextToken := ""
mdr := make(map[string]*cloudwatch.MetricDataResult)
for {
if nextToken != "" {
params.NextToken = aws.String(nextToken)
}
resp, err := client.GetMetricDataWithContext(ctx, params)
if err != nil {
return queryResponses, err
}
metrics.M_Aws_CloudWatch_GetMetricData.Add(float64(len(params.MetricDataQueries)))
for _, r := range resp.MetricDataResults {
if _, ok := mdr[*r.Id]; !ok {
mdr[*r.Id] = r
} else {
mdr[*r.Id].Timestamps = append(mdr[*r.Id].Timestamps, r.Timestamps...)
mdr[*r.Id].Values = append(mdr[*r.Id].Values, r.Values...)
}
}
if resp.NextToken == nil || *resp.NextToken == "" {
break
}
nextToken = *resp.NextToken
}
for i, r := range mdr {
if *r.StatusCode != "Complete" {
return queryResponses, fmt.Errorf("Part of query is failed: %s", *r.StatusCode)
}
queryRes := tsdb.NewQueryResult()
queryRes.RefId = queries[i].RefId
query := queries[*r.Id]
series := tsdb.TimeSeries{
Tags: map[string]string{},
Points: make([]tsdb.TimePoint, 0),
}
for _, d := range query.Dimensions {
series.Tags[*d.Name] = *d.Value
}
s := ""
if len(query.Statistics) == 1 {
s = *query.Statistics[0]
} else {
s = *query.ExtendedStatistics[0]
}
series.Name = formatAlias(query, s, series.Tags)
for j, t := range r.Timestamps {
expectedTimestamp := r.Timestamps[j].Add(time.Duration(query.Period) * time.Second)
if j > 0 && expectedTimestamp.Before(*t) {
series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(expectedTimestamp.Unix()*1000)))
}
series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(*r.Values[j]), float64((*t).Unix())*1000))
}
queryRes.Series = append(queryRes.Series, &series)
queryResponses = append(queryResponses, queryRes)
}
return queryResponses, nil
}
func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) { func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) {
var result []*cloudwatch.Dimension var result []*cloudwatch.Dimension
...@@ -257,6 +406,9 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { ...@@ -257,6 +406,9 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
return nil, err return nil, err
} }
id := model.Get("id").MustString("")
expression := model.Get("expression").MustString("")
dimensions, err := parseDimensions(model) dimensions, err := parseDimensions(model)
if err != nil { if err != nil {
return nil, err return nil, err
...@@ -295,6 +447,7 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { ...@@ -295,6 +447,7 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
alias = "{{metric}}_{{stat}}" alias = "{{metric}}_{{stat}}"
} }
returnData := model.Get("returnData").MustBool(false)
highResolution := model.Get("highResolution").MustBool(false) highResolution := model.Get("highResolution").MustBool(false)
return &CloudWatchQuery{ return &CloudWatchQuery{
...@@ -306,11 +459,18 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { ...@@ -306,11 +459,18 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
ExtendedStatistics: aws.StringSlice(extendedStatistics), ExtendedStatistics: aws.StringSlice(extendedStatistics),
Period: period, Period: period,
Alias: alias, Alias: alias,
Id: id,
Expression: expression,
ReturnData: returnData,
HighResolution: highResolution, HighResolution: highResolution,
}, nil }, nil
} }
func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string { func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string {
if len(query.Id) > 0 && len(query.Expression) > 0 {
return query.Id
}
data := map[string]string{} data := map[string]string{}
data["region"] = query.Region data["region"] = query.Region
data["namespace"] = query.Namespace data["namespace"] = query.Namespace
...@@ -338,6 +498,7 @@ func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]stri ...@@ -338,6 +498,7 @@ func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]stri
func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) { func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) {
queryRes := tsdb.NewQueryResult() queryRes := tsdb.NewQueryResult()
queryRes.RefId = query.RefId
var value float64 var value float64
for _, s := range append(query.Statistics, query.ExtendedStatistics...) { for _, s := range append(query.Statistics, query.ExtendedStatistics...) {
series := tsdb.TimeSeries{ series := tsdb.TimeSeries{
......
pkg/tsdb/cloudwatch/types.go
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
) )
type CloudWatchQuery struct { type CloudWatchQuery struct {
RefId string
Region string Region string
Namespace string Namespace string
MetricName string MetricName string
...@@ -13,5 +14,8 @@ type CloudWatchQuery struct { ...@@ -13,5 +14,8 @@ type CloudWatchQuery struct {
ExtendedStatistics []*string ExtendedStatistics []*string
Period int Period int
Alias string Alias string
Id string
Expression string
ReturnData bool
HighResolution bool HighResolution bool
} }
public/app/plugins/datasource/cloudwatch/datasource.ts
...@@ -30,7 +30,9 @@ export default class CloudWatchDatasource { ...@@ -30,7 +30,9 @@ export default class CloudWatchDatasource {
var queries = _.filter(options.targets, item => { var queries = _.filter(options.targets, item => {
return ( return (
item.hide !== true && !!item.region && !!item.namespace && !!item.metricName && !_.isEmpty(item.statistics) (item.id !== '' || item.hide !== true) &&
((!!item.region && !!item.namespace && !!item.metricName && !_.isEmpty(item.statistics)) ||
item.expression.length > 0)
); );
}).map(item => { }).map(item => {
item.region = this.templateSrv.replace(this.getActualRegion(item.region), options.scopedVars); item.region = this.templateSrv.replace(this.getActualRegion(item.region), options.scopedVars);
...@@ -38,6 +40,9 @@ export default class CloudWatchDatasource { ...@@ -38,6 +40,9 @@ export default class CloudWatchDatasource {
item.metricName = this.templateSrv.replace(item.metricName, options.scopedVars); item.metricName = this.templateSrv.replace(item.metricName, options.scopedVars);
item.dimensions = this.convertDimensionFormat(item.dimensions, options.scopedVars); item.dimensions = this.convertDimensionFormat(item.dimensions, options.scopedVars);
item.period = String(this.getPeriod(item, options)); // use string format for period in graph query, and alerting item.period = String(this.getPeriod(item, options)); // use string format for period in graph query, and alerting
item.id = this.templateSrv.replace(item.id, options.scopedVars);
item.expression = this.templateSrv.replace(item.expression, options.scopedVars);
item.returnData = typeof item.hide === 'undefined' ? true : !item.hide;
return _.extend( return _.extend(
{ {
...@@ -384,11 +389,11 @@ export default class CloudWatchDatasource { ...@@ -384,11 +389,11 @@ export default class CloudWatchDatasource {
var currentVariables = !_.isArray(variable.current.value) var currentVariables = !_.isArray(variable.current.value)
? [variable.current] ? [variable.current]
: variable.current.value.map(v => { : variable.current.value.map(v => {
return { return {
text: v, text: v,
value: v, value: v,
}; };
}); });
let useSelectedVariables = let useSelectedVariables =
selectedVariables.some(s => { selectedVariables.some(s => {
return s.value === currentVariables[0].value; return s.value === currentVariables[0].value;
...@@ -399,6 +404,9 @@ export default class CloudWatchDatasource { ...@@ -399,6 +404,9 @@ export default class CloudWatchDatasource {
scopedVar[variable.name] = v; scopedVar[variable.name] = v;
t.refId = target.refId + '_' + v.value; t.refId = target.refId + '_' + v.value;
t.dimensions[dimensionKey] = templateSrv.replace(t.dimensions[dimensionKey], scopedVar); t.dimensions[dimensionKey] = templateSrv.replace(t.dimensions[dimensionKey], scopedVar);
if (target.id) {
t.id = target.id + window.btoa(v.value).replace(/=/g, '0'); // generate unique id
}
return t; return t;
}); });
} }
......
public/app/plugins/datasource/cloudwatch/partials/query.parameter.html
<div class="gf-form-inline"> <div class="gf-form-inline" ng-if="target.expression.length === 0">
<div class="gf-form"> <div class="gf-form">
<label class="gf-form-label query-keyword width-8">Metric</label> <label class="gf-form-label query-keyword width-8">Metric</label>
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
</div> </div>
</div> </div>
<div class="gf-form-inline"> <div class="gf-form-inline" ng-if="target.expression.length === 0">
<div class="gf-form"> <div class="gf-form">
<label class="gf-form-label query-keyword width-8">Dimensions</label> <label class="gf-form-label query-keyword width-8">Dimensions</label>
<metric-segment ng-repeat="segment in dimSegments" segment="segment" get-options="getDimSegments(segment, $index)" on-change="dimSegmentChanged(segment, $index)"></metric-segment> <metric-segment ng-repeat="segment in dimSegments" segment="segment" get-options="getDimSegments(segment, $index)" on-change="dimSegmentChanged(segment, $index)"></metric-segment>
...@@ -31,18 +31,31 @@ ...@@ -31,18 +31,31 @@
</div> </div>
</div> </div>
<div class="gf-form-inline"> <div class="gf-form-inline" ng-if="target.statistics.length === 1">
<div class="gf-form"> <div class="gf-form">
<label class="gf-form-label query-keyword width-8"> <label class=" gf-form-label query-keyword width-8 ">Id</label>
<input type="text " class="gf-form-input " ng-model="target.id " spellcheck='false' ng-model-onblur ng-change="onChange() ">
</div>
<div class="gf-form max-width-30 ">
<label class="gf-form-label query-keyword width-7 ">Expression</label>
<input type="text " class="gf-form-input " ng-model="target.expression
" spellcheck='false' ng-model-onblur ng-change="onChange() ">
</div>
</div>
<div class="gf-form-inline ">
<div class="gf-form ">
<label class="gf-form-label query-keyword width-8 ">
Min period Min period
<info-popover mode="right-normal">Minimum interval between points in seconds</info-popover> <info-popover mode="right-normal ">Minimum interval between points in seconds</info-popover>
</label> </label>
<input type="text" class="gf-form-input" ng-model="target.period" spellcheck='false' placeholder="auto" ng-model-onblur ng-change="onChange()" /> <input type="text " class="gf-form-input " ng-model="target.period " spellcheck='false' placeholder="auto
" ng-model-onblur ng-change="onChange() " />
</div> </div>
<div class="gf-form max-width-30"> <div class="gf-form max-width-30 ">
<label class="gf-form-label query-keyword width-7">Alias</label> <label class="gf-form-label query-keyword width-7 ">Alias</label>
<input type="text" class="gf-form-input" ng-model="target.alias" spellcheck='false' ng-model-onblur ng-change="onChange()"> <input type="text " class="gf-form-input " ng-model="target.alias " spellcheck='false' ng-model-onblur ng-change="onChange() ">
<info-popover mode="right-absolute"> <info-popover mode="right-absolute ">
Alias replacement variables: Alias replacement variables:
<ul ng-non-bindable> <ul ng-non-bindable>
<li>{{metric}}</li> <li>{{metric}}</li>
...@@ -54,12 +67,12 @@ ...@@ -54,12 +67,12 @@
</ul> </ul>
</info-popover> </info-popover>
</div> </div>
<div class="gf-form"> <div class="gf-form ">
<gf-form-switch class="gf-form" label="HighRes" label-class="width-5" checked="target.highResolution" on-change="onChange()"> <gf-form-switch class="gf-form " label="HighRes " label-class="width-5 " checked="target.highResolution " on-change="onChange() ">
</gf-form-switch> </gf-form-switch>
</div> </div>
<div class="gf-form gf-form--grow"> <div class="gf-form gf-form--grow ">
<div class="gf-form-label gf-form-label--grow"></div> <div class="gf-form-label gf-form-label--grow "></div>
</div> </div>
</div> </div>
public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.ts
...@@ -27,6 +27,9 @@ export class CloudWatchQueryParameterCtrl { ...@@ -27,6 +27,9 @@ export class CloudWatchQueryParameterCtrl {
target.dimensions = target.dimensions || {}; target.dimensions = target.dimensions || {};
target.period = target.period || ''; target.period = target.period || '';
target.region = target.region || 'default'; target.region = target.region || 'default';
target.id = target.id || '';
target.expression = target.expression || '';
target.returnData = target.returnData || false;
target.highResolution = target.highResolution || false; target.highResolution = target.highResolution || false;
$scope.regionSegment = uiSegmentSrv.getSegmentForValue($scope.target.region, 'select region'); $scope.regionSegment = uiSegmentSrv.getSegmentForValue($scope.target.region, 'select region');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment