Skip to content

N
nexpie-grafana-theme
Commit 40ed235b by Mitsuhiro Tanda
Options

support GetMetricData

parent 077cf9a3
Showing with 237 additions and 40 deletions
pkg/metrics/metrics.go
......@@ -44,6 +44,7 @@ var (
M_Alerting_Notification_Sent *prometheus.CounterVec
M_Aws_CloudWatch_GetMetricStatistics prometheus.Counter
M_Aws_CloudWatch_ListMetrics prometheus.Counter
M_Aws_CloudWatch_GetMetricData prometheus.Counter
M_DB_DataSource_QueryById prometheus.Counter
// Timers
......@@ -218,6 +219,12 @@ func init() {
Namespace: exporterName,
})
M_Aws_CloudWatch_GetMetricData = prometheus.NewCounter(prometheus.CounterOpts{
Name: "aws_cloudwatch_get_metric_data_total",
Help: "counter for getting metric data time series from aws",
Namespace: exporterName,
})
M_DB_DataSource_QueryById = prometheus.NewCounter(prometheus.CounterOpts{
Name: "db_datasource_query_by_id_total",
Help: "counter for getting datasource by id",
......@@ -307,6 +314,7 @@ func initMetricVars() {
M_Alerting_Notification_Sent,
M_Aws_CloudWatch_GetMetricStatistics,
M_Aws_CloudWatch_ListMetrics,
M_Aws_CloudWatch_GetMetricData,
M_DB_DataSource_QueryById,
M_Alerting_Active_Alerts,
M_StatTotal_Dashboards,
......
pkg/tsdb/cloudwatch/cloudwatch.go
......@@ -14,6 +14,7 @@ import (
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
"golang.org/x/sync/errgroup"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
......@@ -88,48 +89,63 @@ func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryCo
Results: make(map[string]*tsdb.QueryResult),
}
errCh := make(chan error, 1)
resCh := make(chan *tsdb.QueryResult, 1)
eg, ectx := errgroup.WithContext(ctx)
currentlyExecuting := 0
getMetricDataQueries := make(map[string]map[string]*CloudWatchQuery)
for i, model := range queryContext.Queries {
queryType := model.Model.Get("type").MustString()
if queryType != "timeSeriesQuery" && queryType != "" {
continue
}
currentlyExecuting++
go func(refId string, index int) {
queryRes, err := e.executeQuery(ctx, queryContext.Queries[index].Model, queryContext)
currentlyExecuting--
query, err := parseQuery(queryContext.Queries[i].Model)
if err != nil {
errCh <- err
} else {
queryRes.RefId = refId
resCh <- queryRes
}
}(model.RefId, i)
return nil, err
}
query.RefId = queryContext.Queries[i].RefId
for currentlyExecuting != 0 {
select {
case res := <-resCh:
result.Results[res.RefId] = res
case err := <-errCh:
return result, err
case <-ctx.Done():
return result, ctx.Err()
if query.Id != "" {
if _, ok := getMetricDataQueries[query.Region]; !ok {
getMetricDataQueries[query.Region] = make(map[string]*CloudWatchQuery)
}
getMetricDataQueries[query.Region][query.Id] = query
continue
}
return result, nil
}
eg.Go(func() error {
queryRes, err := e.executeQuery(ectx, query, queryContext)
if err != nil {
return err
}
result.Results[queryRes.RefId] = queryRes
return nil
})
}
func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
query, err := parseQuery(parameters)
if len(getMetricDataQueries) > 0 {
for region, getMetricDataQuery := range getMetricDataQueries {
q := getMetricDataQuery
eg.Go(func() error {
queryResponses, err := e.executeGetMetricDataQuery(ectx, region, q, queryContext)
if err != nil {
return err
}
for _, queryRes := range queryResponses {
result.Results[queryRes.RefId] = queryRes
}
return nil
})
}
}
if err := eg.Wait(); err != nil {
return nil, err
}
return result, nil
}
func (e *CloudWatchExecutor) executeQuery(ctx context.Context, query *CloudWatchQuery, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) {
client, err := e.getClient(query.Region)
if err != nil {
return nil, err
......@@ -201,6 +217,139 @@ func (e *CloudWatchExecutor) executeQuery(ctx context.Context, parameters *simpl
return queryRes, nil
}
func (e *CloudWatchExecutor) executeGetMetricDataQuery(ctx context.Context, region string, queries map[string]*CloudWatchQuery, queryContext *tsdb.TsdbQuery) ([]*tsdb.QueryResult, error) {
queryResponses := make([]*tsdb.QueryResult, 0)
// validate query
for _, query := range queries {
if !(len(query.Statistics) == 1 && len(query.ExtendedStatistics) == 0) &&
!(len(query.Statistics) == 0 && len(query.ExtendedStatistics) == 1) {
return queryResponses, errors.New("Statistics count should be 1")
}
}
client, err := e.getClient(region)
if err != nil {
return queryResponses, err
}
startTime, err := queryContext.TimeRange.ParseFrom()
if err != nil {
return queryResponses, err
}
endTime, err := queryContext.TimeRange.ParseTo()
if err != nil {
return queryResponses, err
}
params := &cloudwatch.GetMetricDataInput{
StartTime: aws.Time(startTime),
EndTime: aws.Time(endTime),
ScanBy: aws.String("TimestampAscending"),
}
for _, query := range queries {
// 1 minutes resolutin metrics is stored for 15 days, 15 * 24 * 60 = 21600
if query.HighResolution && (((endTime.Unix() - startTime.Unix()) / int64(query.Period)) > 21600) {
return nil, errors.New("too long query period")
}
mdq := &cloudwatch.MetricDataQuery{
Id: aws.String(query.Id),
ReturnData: aws.Bool(query.ReturnData),
}
if query.Expression != "" {
mdq.Expression = aws.String(query.Expression)
} else {
mdq.MetricStat = &cloudwatch.MetricStat{
Metric: &cloudwatch.Metric{
Namespace: aws.String(query.Namespace),
MetricName: aws.String(query.MetricName),
},
Period: aws.Int64(int64(query.Period)),
}
for _, d := range query.Dimensions {
mdq.MetricStat.Metric.Dimensions = append(mdq.MetricStat.Metric.Dimensions,
&cloudwatch.Dimension{
Name: d.Name,
Value: d.Value,
})
}
if len(query.Statistics) == 1 {
mdq.MetricStat.Stat = query.Statistics[0]
} else {
mdq.MetricStat.Stat = query.ExtendedStatistics[0]
}
}
params.MetricDataQueries = append(params.MetricDataQueries, mdq)
}
nextToken := ""
mdr := make(map[string]*cloudwatch.MetricDataResult)
for {
if nextToken != "" {
params.NextToken = aws.String(nextToken)
}
resp, err := client.GetMetricDataWithContext(ctx, params)
if err != nil {
return queryResponses, err
}
metrics.M_Aws_CloudWatch_GetMetricData.Add(float64(len(params.MetricDataQueries)))
for _, r := range resp.MetricDataResults {
if _, ok := mdr[*r.Id]; !ok {
mdr[*r.Id] = r
} else {
mdr[*r.Id].Timestamps = append(mdr[*r.Id].Timestamps, r.Timestamps...)
mdr[*r.Id].Values = append(mdr[*r.Id].Values, r.Values...)
}
}
if resp.NextToken == nil || *resp.NextToken == "" {
break
}
nextToken = *resp.NextToken
}
for i, r := range mdr {
if *r.StatusCode != "Complete" {
return queryResponses, fmt.Errorf("Part of query is failed: %s", *r.StatusCode)
}
queryRes := tsdb.NewQueryResult()
queryRes.RefId = queries[i].RefId
query := queries[*r.Id]
series := tsdb.TimeSeries{
Tags: map[string]string{},
Points: make([]tsdb.TimePoint, 0),
}
for _, d := range query.Dimensions {
series.Tags[*d.Name] = *d.Value
}
s := ""
if len(query.Statistics) == 1 {
s = *query.Statistics[0]
} else {
s = *query.ExtendedStatistics[0]
}
series.Name = formatAlias(query, s, series.Tags)
for j, t := range r.Timestamps {
expectedTimestamp := r.Timestamps[j].Add(time.Duration(query.Period) * time.Second)
if j > 0 && expectedTimestamp.Before(*t) {
series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(expectedTimestamp.Unix()*1000)))
}
series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(*r.Values[j]), float64((*t).Unix())*1000))
}
queryRes.Series = append(queryRes.Series, &series)
queryResponses = append(queryResponses, queryRes)
}
return queryResponses, nil
}
func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) {
var result []*cloudwatch.Dimension
......@@ -257,6 +406,9 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
return nil, err
}
id := model.Get("id").MustString("")
expression := model.Get("expression").MustString("")
dimensions, err := parseDimensions(model)
if err != nil {
return nil, err
......@@ -295,6 +447,7 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
alias = "{{metric}}_{{stat}}"
}
returnData := model.Get("returnData").MustBool(false)
highResolution := model.Get("highResolution").MustBool(false)
return &CloudWatchQuery{
......@@ -306,11 +459,18 @@ func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) {
ExtendedStatistics: aws.StringSlice(extendedStatistics),
Period: period,
Alias: alias,
Id: id,
Expression: expression,
ReturnData: returnData,
HighResolution: highResolution,
}, nil
}
func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string) string {
if len(query.Id) > 0 && len(query.Expression) > 0 {
return query.Id
}
data := map[string]string{}
data["region"] = query.Region
data["namespace"] = query.Namespace
......@@ -338,6 +498,7 @@ func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]stri
func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) {
queryRes := tsdb.NewQueryResult()
queryRes.RefId = query.RefId
var value float64
for _, s := range append(query.Statistics, query.ExtendedStatistics...) {
series := tsdb.TimeSeries{
......
pkg/tsdb/cloudwatch/types.go
......@@ -5,6 +5,7 @@ import (
)
type CloudWatchQuery struct {
RefId string
Region string
Namespace string
MetricName string
......@@ -13,5 +14,8 @@ type CloudWatchQuery struct {
ExtendedStatistics []*string
Period int
Alias string
Id string
Expression string
ReturnData bool
HighResolution bool
}
public/app/plugins/datasource/cloudwatch/datasource.ts
......@@ -30,7 +30,9 @@ export default class CloudWatchDatasource {
var queries = _.filter(options.targets, item => {
return (
item.hide !== true && !!item.region && !!item.namespace && !!item.metricName && !_.isEmpty(item.statistics)
(item.id !== '' || item.hide !== true) &&
((!!item.region && !!item.namespace && !!item.metricName && !_.isEmpty(item.statistics)) ||
item.expression.length > 0)
);
}).map(item => {
item.region = this.templateSrv.replace(this.getActualRegion(item.region), options.scopedVars);
......@@ -38,6 +40,9 @@ export default class CloudWatchDatasource {
item.metricName = this.templateSrv.replace(item.metricName, options.scopedVars);
item.dimensions = this.convertDimensionFormat(item.dimensions, options.scopedVars);
item.period = String(this.getPeriod(item, options)); // use string format for period in graph query, and alerting
item.id = this.templateSrv.replace(item.id, options.scopedVars);
item.expression = this.templateSrv.replace(item.expression, options.scopedVars);
item.returnData = typeof item.hide === 'undefined' ? true : !item.hide;
return _.extend(
{
......@@ -399,6 +404,9 @@ export default class CloudWatchDatasource {
scopedVar[variable.name] = v;
t.refId = target.refId + '_' + v.value;
t.dimensions[dimensionKey] = templateSrv.replace(t.dimensions[dimensionKey], scopedVar);
if (target.id) {
t.id = target.id + window.btoa(v.value).replace(/=/g, '0'); // generate unique id
}
return t;
});
}
......
public/app/plugins/datasource/cloudwatch/partials/query.parameter.html
<div class="gf-form-inline">
<div class="gf-form-inline" ng-if="target.expression.length === 0">
<div class="gf-form">
<label class="gf-form-label query-keyword width-8">Metric</label>
......@@ -20,7 +20,7 @@
</div>
</div>
<div class="gf-form-inline">
<div class="gf-form-inline" ng-if="target.expression.length === 0">
<div class="gf-form">
<label class="gf-form-label query-keyword width-8">Dimensions</label>
<metric-segment ng-repeat="segment in dimSegments" segment="segment" get-options="getDimSegments(segment, $index)" on-change="dimSegmentChanged(segment, $index)"></metric-segment>
......@@ -31,18 +31,31 @@
</div>
</div>
<div class="gf-form-inline">
<div class="gf-form-inline" ng-if="target.statistics.length === 1">
<div class="gf-form">
<label class="gf-form-label query-keyword width-8">
<label class=" gf-form-label query-keyword width-8 ">Id</label>
<input type="text " class="gf-form-input " ng-model="target.id " spellcheck='false' ng-model-onblur ng-change="onChange() ">
</div>
<div class="gf-form max-width-30 ">
<label class="gf-form-label query-keyword width-7 ">Expression</label>
<input type="text " class="gf-form-input " ng-model="target.expression
" spellcheck='false' ng-model-onblur ng-change="onChange() ">
</div>
</div>
<div class="gf-form-inline ">
<div class="gf-form ">
<label class="gf-form-label query-keyword width-8 ">
Min period
<info-popover mode="right-normal">Minimum interval between points in seconds</info-popover>
<info-popover mode="right-normal ">Minimum interval between points in seconds</info-popover>
</label>
<input type="text" class="gf-form-input" ng-model="target.period" spellcheck='false' placeholder="auto" ng-model-onblur ng-change="onChange()" />
<input type="text " class="gf-form-input " ng-model="target.period " spellcheck='false' placeholder="auto
" ng-model-onblur ng-change="onChange() " />
</div>
<div class="gf-form max-width-30">
<label class="gf-form-label query-keyword width-7">Alias</label>
<input type="text" class="gf-form-input" ng-model="target.alias" spellcheck='false' ng-model-onblur ng-change="onChange()">
<info-popover mode="right-absolute">
<div class="gf-form max-width-30 ">
<label class="gf-form-label query-keyword width-7 ">Alias</label>
<input type="text " class="gf-form-input " ng-model="target.alias " spellcheck='false' ng-model-onblur ng-change="onChange() ">
<info-popover mode="right-absolute ">
Alias replacement variables:
<ul ng-non-bindable>
<li>{{metric}}</li>
......@@ -54,12 +67,12 @@
</ul>
</info-popover>
</div>
<div class="gf-form">
<gf-form-switch class="gf-form" label="HighRes" label-class="width-5" checked="target.highResolution" on-change="onChange()">
<div class="gf-form ">
<gf-form-switch class="gf-form " label="HighRes " label-class="width-5 " checked="target.highResolution " on-change="onChange() ">
</gf-form-switch>
</div>
<div class="gf-form gf-form--grow">
<div class="gf-form-label gf-form-label--grow"></div>
<div class="gf-form gf-form--grow ">
<div class="gf-form-label gf-form-label--grow "></div>
</div>
</div>
public/app/plugins/datasource/cloudwatch/query_parameter_ctrl.ts
......@@ -27,6 +27,9 @@ export class CloudWatchQueryParameterCtrl {
target.dimensions = target.dimensions || {};
target.period = target.period || '';
target.region = target.region || 'default';
target.id = target.id || '';
target.expression = target.expression || '';
target.returnData = target.returnData || false;
target.highResolution = target.highResolution || false;
$scope.regionSegment = uiSegmentSrv.getSegmentForValue($scope.target.region, 'select region');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment