Merge branch 'moving_avg_es_support'

CHANGELOG.md

 # 2.6.0 (2015-12-04)
 
+### New Features
+* **Elasticsearch**: Support for pipeline aggregations Moving average and derivative, closes [#3451](https://github.com/grafana/grafana/issues/3451)
+
 ### Bug Fixes
 * **metric editors**: Fix for clicking typeahead auto dropdown option, fixes [#3428](https://github.com/grafana/grafana/issues/3428)
 * **influxdb**: Fixed issue showing Group By label only on first query, fixes [#3453](https://github.com/grafana/grafana/issues/3453)

public/app/plugins/datasource/elasticsearch/elastic_response.js

@@ -15,6 +15,9 @@ function (_, queryDef) {
 
     for (y = 0; y < target.metrics.length; y++) {
       metric = target.metrics[y];
+      if (metric.hide) {
+        continue;
+      }
 
       switch(metric.type) {
         case 'count': {
@@ -76,8 +79,12 @@ function (_, queryDef) {
           newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props};
           for (i = 0; i < esAgg.buckets.length; i++) {
             bucket = esAgg.buckets[i];
-            value = bucket[metric.id].value;
-            newSeries.datapoints.push([value, bucket.key]);
+
+            value = bucket[metric.id];
+            if (value !== undefined) {
+              newSeries.datapoints.push([value.value, bucket.key]);
+            }
+
           }
           seriesList.push(newSeries);
           break;
@@ -193,7 +200,14 @@ function (_, queryDef) {
       });
     }
 
-    if (series.field) {
+    if (series.field && queryDef.isPipelineAgg(series.metric)) {
+      var appliedAgg = _.findWhere(target.metrics, { id: series.field });
+      if (appliedAgg) {
+        metricName += ' ' + queryDef.describeMetric(appliedAgg);
+      } else {
+        metricName = 'Unset';
+      }
+    } else if (series.field) {
       metricName += ' ' + series.field;
     }
 

public/app/plugins/datasource/elasticsearch/metric_agg.js

@@ -13,14 +13,21 @@ function (angular, _, queryDef) {
 
     $scope.metricAggTypes = queryDef.metricAggTypes;
     $scope.extendedStats = queryDef.extendedStats;
+    $scope.pipelineAggOptions = [];
 
     $scope.init = function() {
       $scope.agg = metricAggs[$scope.index];
       $scope.validateModel();
+      $scope.updatePipelineAggOptions();
+    };
+
+    $scope.updatePipelineAggOptions = function() {
+      $scope.pipelineAggOptions = queryDef.getPipelineAggOptions($scope.target);
     };
 
     $rootScope.onAppEvent('elastic-query-updated', function() {
       $scope.index = _.indexOf(metricAggs, $scope.agg);
+      $scope.updatePipelineAggOptions();
       $scope.validateModel();
     }, $scope);
 
@@ -30,7 +37,18 @@ function (angular, _, queryDef) {
       $scope.settingsLinkText = '';
       $scope.aggDef = _.findWhere($scope.metricAggTypes, {value: $scope.agg.type});
 
-      if (!$scope.agg.field) {
+      if (queryDef.isPipelineAgg($scope.agg.type)) {
+        $scope.agg.pipelineAgg = $scope.agg.pipelineAgg || 'select metric';
+        $scope.agg.field = $scope.agg.pipelineAgg;
+
+        var pipelineOptions = queryDef.getPipelineOptions($scope.agg);
+        if (pipelineOptions.length > 0) {
+          _.each(pipelineOptions, function(opt) {
+            $scope.agg.settings[opt.text] = $scope.agg.settings[opt.text] || opt.default;
+          });
+          $scope.settingsLinkText = 'Options';
+        }
+      } else if (!$scope.agg.field) {
         $scope.agg.field = 'select field';
       }
 
@@ -65,12 +83,18 @@ function (angular, _, queryDef) {
 
     $scope.toggleOptions = function() {
       $scope.showOptions = !$scope.showOptions;
+      $scope.updatePipelineAggOptions();
+    };
+
+    $scope.onChangeInternal = function() {
+      $scope.onChange();
     };
 
     $scope.onTypeChange = function() {
       $scope.agg.settings = {};
       $scope.agg.meta = {};
       $scope.showOptions = false;
+      $scope.updatePipelineAggOptions();
       $scope.onChange();
     };
 
@@ -94,6 +118,14 @@ function (angular, _, queryDef) {
       $scope.onChange();
     };
 
+    $scope.toggleShowMetric = function() {
+      $scope.agg.hide = !$scope.agg.hide;
+      if (!$scope.agg.hide) {
+        delete $scope.agg.hide;
+      }
+      $scope.onChange();
+    };
+
     $scope.init();
 
   });

public/app/plugins/datasource/elasticsearch/partials/metricAgg.html

-<div class="tight-form">
+<div class="tight-form" ng-class="{'tight-form-disabled': agg.hide}">
 	<ul class="tight-form-list">
 		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
 			Metric
+			&nbsp; <a ng-click="toggleShowMetric()" bs-tooltip="Click to toggle show/hide metric">
+				<i class="fa fa-eye" ng-hide="agg.hide"></i>
+				<i class="fa fa-eye-slash" ng-show="agg.hide"></i>
+			</a>
 		</li>
 		<li>
 			<metric-segment-model property="agg.type" options="metricAggTypes" on-change="onTypeChange()" custom="false" css-class="tight-form-item-large"></metric-segment-model>
 		</li>
 		<li ng-if="aggDef.requiresField">
-			<metric-segment-model property="agg.field" get-options="getFieldsInternal()" on-change="onChange()" css-class="tight-form-item-xxlarge"></metric-segment>
+			<metric-segment-model property="agg.field" get-options="getFieldsInternal()" on-change="onChange()" css-class="tight-form-item-xxlarge"></metric-segment-model>
+		</li>
+		<li ng-if="aggDef.isPipelineAgg">
+			<metric-segment-model property="agg.pipelineAgg" options="pipelineAggOptions" on-change="onChangeInternal()" custom="false" css-class="tight-form-item-xxlarge"></metric-segment-model>
 		</li>
 		<li class="tight-form-item last" ng-if="settingsLinkText">
-			<a ng-click="toggleOptions()">{{settingsLinkText}}</a>
+			<a ng-click="toggleOptions()">
+				<i class="fa fa-caret-down" ng-show="showOptions"></i>
+				<i class="fa fa-caret-right" ng-hide="showOptions"></i>
+				{{settingsLinkText}}
+			</a>
 		</li>
 	</ul>
 
@@ -26,7 +37,29 @@
 </div>
 
 <div class="tight-form" ng-if="showOptions">
-	<div class="tight-form-inner-box">
+	<div class="tight-form-inner-box tight-form-container">
+		<div class="tight-form" ng-if="agg.type === 'moving_avg'">
+			<ul class="tight-form-list">
+				<li class="tight-form-item" style="width: 75px;">
+					Window
+				</li>
+				<li>
+					<input type="number" class="input-medium tight-form-input last" ng-model="agg.settings.window" ng-blur="onChangeInternal()" spellcheck='false'>
+				</li>
+			</ul>
+			<div class="clearfix"></div>
+		</div>
+		<div class="tight-form" ng-if="agg.type === 'moving_avg'">
+			<ul class="tight-form-list">
+				<li class="tight-form-item" style="width: 75px;">
+					Model
+				</li>
+				<li>
+					<input type="text" class="input-medium tight-form-input last" ng-change="onChangeInternal()" ng-model="agg.settings.model" blur="onChange()" spellcheck='false'>
+				</li>
+			</ul>
+			<div class="clearfix"></div>
+		</div>
 		<div class="tight-form last" ng-if="agg.type === 'percentiles'">
 			<ul class="tight-form-list">
 				<li class="tight-form-item">

public/app/plugins/datasource/elasticsearch/query_builder.js

 define([
+  './query_def'
 ],
-function () {
+function (queryDef) {
   'use strict';
 
   function ElasticQueryBuilder(options) {
@@ -167,14 +168,25 @@ function () {
         continue;
       }
 
-      var metricAgg = {field: metric.field};
+      var aggField = {};
+      var metricAgg = null;
+
+      if (queryDef.isPipelineAgg(metric.type)) {
+        if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) {
+          metricAgg = { buckets_path: metric.pipelineAgg };
+        } else {
+          continue;
+        }
+      } else {
+        metricAgg = {field: metric.field};
+      }
+
       for (var prop in metric.settings) {
         if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) {
           metricAgg[prop] = metric.settings[prop];
         }
       }
 
-      var aggField = {};
       aggField[metric.type] = metricAgg;
       nestedAggs.aggs[metric.id] = aggField;
     }
@@ -217,5 +229,4 @@ function () {
   };
 
   return ElasticQueryBuilder;
-
 });

public/app/plugins/datasource/elasticsearch/query_def.js

@@ -13,6 +13,8 @@ function (_) {
       {text: "Min",  value: 'min', requiresField: true},
       {text: "Extended Stats",  value: 'extended_stats', requiresField: true},
       {text: "Percentiles",  value: 'percentiles', requiresField: true},
+      {text: "Moving Average",  value: 'moving_avg', requiresField: false, isPipelineAgg: true },
+      {text: "Derivative",  value: 'derivative', requiresField: false, isPipelineAgg: true },
       {text: "Unique Count", value: "cardinality", requiresField: true},
       {text: "Raw Document", value: "raw_document", requiresField: false}
     ],
@@ -66,6 +68,43 @@ function (_) {
       {text: '1d', value: '1d'},
     ],
 
+    pipelineOptions: {
+      'moving_avg' : [
+        {text: 'window', default: 5},
+        {text: 'model', default: 'simple'}
+      ],
+      'derivative': []
+    },
+
+    getPipelineOptions: function(metric) {
+      if (!this.isPipelineAgg(metric.type)) {
+        return [];
+      }
+
+      return this.pipelineOptions[metric.type];
+    },
+
+    isPipelineAgg: function(metricType) {
+      if (metricType) {
+        var po = this.pipelineOptions[metricType];
+        return po !== null && po !== undefined;
+      }
+
+      return false;
+    },
+
+    getPipelineAggOptions: function(targets) {
+      var self = this;
+      var result = [];
+      _.each(targets.metrics, function(metric) {
+        if (!self.isPipelineAgg(metric.type)) {
+          result.push({text: self.describeMetric(metric), value: metric.id });
+        }
+      });
+
+      return result;
+    },
+
     getOrderByOptions: function(target) {
       var self = this;
       var metricRefs = [];

public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts

@@ -155,4 +155,89 @@ describe('ElasticQueryBuilder', function() {
     expect(query.size).to.be(500);
   });
 
+  it('with moving average', function() {
+    var query = builder.build({
+      metrics: [
+        {
+          id: '3',
+          type: 'sum',
+          field: '@value'
+        },
+        {
+          id: '2',
+          type: 'moving_avg',
+          field: '3',
+          pipelineAgg: '3'
+        }
+      ],
+      bucketAggs: [
+        {type: 'date_histogram', field: '@timestamp', id: '3'}
+      ],
+    });
+
+    var firstLevel = query.aggs["3"];
+
+    expect(firstLevel.aggs["2"]).not.to.be(undefined);
+    expect(firstLevel.aggs["2"].moving_avg).not.to.be(undefined);
+    expect(firstLevel.aggs["2"].moving_avg.buckets_path).to.be("3");
+  });
+
+  it('with broken moving average', function() {
+      var query = builder.build({
+          metrics: [
+              {
+                  id: '3',
+                  type: 'sum',
+                  field: '@value'
+              },
+              {
+                  id: '2',
+                  type: 'moving_avg',
+                  pipelineAgg: '3'
+              },
+              {
+                  id: '4',
+                  type: 'moving_avg',
+                  pipelineAgg: 'Metric to apply moving average'
+              }
+          ],
+          bucketAggs: [
+              { type: 'date_histogram', field: '@timestamp', id: '3' }
+          ],
+      });
+
+      var firstLevel = query.aggs["3"];
+
+      expect(firstLevel.aggs["2"]).not.to.be(undefined);
+      expect(firstLevel.aggs["2"].moving_avg).not.to.be(undefined);
+      expect(firstLevel.aggs["2"].moving_avg.buckets_path).to.be("3");
+      expect(firstLevel.aggs["4"]).to.be(undefined);
+  });
+
+  it('with derivative', function() {
+    var query = builder.build({
+      metrics: [
+        {
+          id: '3',
+          type: 'sum',
+          field: '@value'
+        },
+        {
+          id: '2',
+          type: 'derivative',
+          pipelineAgg: '3'
+        }
+      ],
+      bucketAggs: [
+        {type: 'date_histogram', field: '@timestamp', id: '3'}
+      ],
+    });
+
+    var firstLevel = query.aggs["3"];
+
+    expect(firstLevel.aggs["2"]).not.to.be(undefined);
+    expect(firstLevel.aggs["2"].derivative).not.to.be(undefined);
+    expect(firstLevel.aggs["2"].derivative.buckets_path).to.be("3");
+  });
+
 });

public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts

+///<amd-dependency path="../query_def" name="QueryDef" />
+///<amd-dependency path="test/specs/helpers" name="helpers" />
+
+import {describe, beforeEach, it, sinon, expect, angularMocks} from 'test/lib/common';
+
+declare var helpers: any;
+declare var QueryDef: any;
+
+describe('ElasticQueryDef', function() {
+
+  describe('getPipelineAggOptions', function() {
+    describe('with zero targets', function() {
+      var response = QueryDef.getPipelineAggOptions([]);
+
+      it('should return zero', function() {
+        expect(response.length).to.be(0);
+      });
+    });
+
+    describe('with count and sum targets', function() {
+      var targets = {
+        metrics: [
+          { type: 'count', field: '@value' },
+          { type: 'sum', field: '@value' }
+        ]
+      };
+
+      var response = QueryDef.getPipelineAggOptions(targets);
+
+      it('should return zero', function() {
+        expect(response.length).to.be(2);
+      });
+    });
+
+    describe('with count and moving average targets', function() {
+      var targets = {
+        metrics: [
+          { type: 'count', field: '@value' },
+          { type: 'moving_avg', field: '@value' }
+        ]
+      };
+
+      var response = QueryDef.getPipelineAggOptions(targets);
+
+      it('should return one', function() {
+        expect(response.length).to.be(1);
+      });
+    });
+
+    describe('with derivatives targets', function() {
+      var targets = {
+        metrics: [
+          { type: 'derivative', field: '@value' }
+        ]
+      };
+
+      var response = QueryDef.getPipelineAggOptions(targets);
+
+      it('should return zero', function() {
+        expect(response.length).to.be(0);
+      });
+    });
+  });
+
+  describe('isPipelineMetric', function() {
+    describe('moving_avg', function() {
+      var result = QueryDef.isPipelineAgg('moving_avg');
+
+      it('is pipe line metric', function() {
+        expect(result).to.be(true);
+      });
+    });
+
+    describe('count', function() {
+      var result = QueryDef.isPipelineAgg('count');
+
+      it('is not pipe line metric', function() {
+        expect(result).to.be(false);
+      });
+    });
+  });
+});