Elasticsearch: Add Moving Function Pipeline Aggregation (#28131)

* Elasticsearch: Add Moving Function Pipeline Aggregation

* Removing support for moving average in 7.x

* Fixing getMetricAggTypes to handle undefined esVersion

* Adding moving_fn to go code

* Update public/app/plugins/datasource/elasticsearch/metric_agg.ts

Co-authored-by: Giordano Ricci <grdnricci@gmail.com>

* Adding test for esversion 70

* Removing default value for script, adding placeholder instead

* Fixing formatting

* Adding code for handling missing or obsolete aggregations

Co-authored-by: Giordano Ricci <grdnricci@gmail.com>

pkg/tsdb/elasticsearch/models.go

@@ -45,6 +45,7 @@ var metricAggType = map[string]string{
 	"percentiles":    "Percentiles",
 	"cardinality":    "Unique Count",
 	"moving_avg":     "Moving Average",
+	"moving_fn":      "Moving Function",
 	"cumulative_sum": "Cumulative Sum",
 	"derivative":     "Derivative",
 	"bucket_script":  "Bucket Script",
@@ -64,6 +65,7 @@ var extendedStats = map[string]string{
 
 var pipelineAggType = map[string]string{
 	"moving_avg":     "moving_avg",
+	"moving_fn":      "moving_fn",
 	"cumulative_sum": "cumulative_sum",
 	"derivative":     "derivative",
 	"bucket_script":  "bucket_script",

public/app/plugins/datasource/elasticsearch/metric_agg.ts

@@ -44,6 +44,7 @@ export class ElasticMetricAggCtrl {
       $scope.settingsLinkText = '';
       $scope.variablesLinkText = '';
       $scope.aggDef = _.find($scope.metricAggTypes, { value: $scope.agg.type });
+      $scope.isValidAgg = $scope.aggDef != null;
 
       if (queryDef.isPipelineAgg($scope.agg.type)) {
         if (queryDef.isPipelineAggWithMultipleBucketPaths($scope.agg.type)) {
@@ -105,6 +106,13 @@ export class ElasticMetricAggCtrl {
           $scope.updateMovingAvgModelSettings();
           break;
         }
+        case 'moving_fn': {
+          const movingFunctionOptions = queryDef.getPipelineOptions($scope.agg);
+          _.each(movingFunctionOptions, opt => {
+            $scope.agg.settings[opt.text] = $scope.agg.settings[opt.text] || opt.default;
+          });
+          break;
+        }
         case 'raw_document':
         case 'raw_data': {
           $scope.agg.settings.size = $scope.agg.settings.size || 500;
@@ -115,7 +123,7 @@ export class ElasticMetricAggCtrl {
           break;
         }
       }
-      if ($scope.aggDef.supportsInlineScript) {
+      if ($scope.aggDef?.supportsInlineScript) {
         // I know this stores the inline script twice
         // but having it like this simplifes the query_builder
         const inlineScript = $scope.agg.inlineScript;

public/app/plugins/datasource/elasticsearch/partials/metric_agg.html

@@ -10,12 +10,18 @@
 		</label>
 	</div>
 
-	<div class="gf-form">
+	<div class="gf-form" ng-if="isValidAgg">
 		<metric-segment-model property="agg.type" options="metricAggTypes" on-change="onTypeChange()" custom="false" css-class="width-10"></metric-segment-model>
 		<metric-segment-model ng-if="aggDef.requiresField" property="agg.field" get-options="getFieldsInternal()" on-change="onChange()" css-class="width-12"></metric-segment-model>
 		<metric-segment-model ng-if="aggDef.isPipelineAgg && !aggDef.supportsMultipleBucketPaths" property="agg.pipelineAgg" options="pipelineAggOptions" on-change="onChangeInternal()" custom="false" css-class="width-12"></metric-segment-model>
 	</div>
 
+  <div class="gf-form gf-form--grow" ng-if="!isValidAgg">
+    <label class="gf-form-label gf-form-label--grow">
+      <em>This aggregation is no longer supported by your version of Elasticsearch</em>
+    </label>
+  </div>
+
   <div class="gf-form gf-form--grow" ng-if="aggDef.isPipelineAgg && aggDef.supportsMultipleBucketPaths">
 		<label class="gf-form-label gf-form-label--grow">
 			<a ng-click="toggleVariables()">
@@ -26,7 +32,7 @@
 		</label>
 	</div>
 
-	<div class="gf-form gf-form--grow">
+	<div class="gf-form gf-form--grow" ng-if="isValidAgg">
 		<label class="gf-form-label gf-form-label--grow">
 			<a ng-click="toggleOptions()" ng-if="settingsLinkText">
 				<icon name="'angle-down'" ng-show="showOptions"></icon>
@@ -71,6 +77,25 @@
 		<input type="text" class="gf-form-input max-width-12" ng-model="agg.settings.format" ng-blur="onChangeInternal()" spellcheck='false'>
 	</div>
 
+  <div ng-if="agg.type === 'moving_fn'">
+
+    <div class="gf-form offset-width-7">
+      <label class="gf-form-label width-10">Window</label>
+      <input type="number" class="gf-form-input max-width-12" ng-model="agg.settings.window" ng-blur="onChangeInternal()" spellcheck='false'>
+    </div>
+
+    <div class="gf-form offset-width-7">
+      <label class="gf-form-label width-10">Script</label>
+      <input type="text" class="gf-form-input max-width-12" ng-model="agg.settings.script" ng-blur="onChangeInternal()" spellcheck='false' placeholder="eg. MovingFunctions.unweightedAvg(values)">
+    </div>
+
+    <div class="gf-form offset-width-7">
+      <label class="gf-form-label width-10">Shift</label>
+      <input type="number" class="gf-form-input max-width-12" ng-model="agg.settings.shift" ng-blur="onChangeInternal()" spellcheck='false'>
+    </div>
+
+  </div>
+
   <div ng-if="agg.type === 'moving_avg'">
     <div class="gf-form offset-width-7">
       <label class="gf-form-label width-10">Model</label>

public/app/plugins/datasource/elasticsearch/query_def.ts

@@ -57,6 +57,14 @@ export const metricAggTypes = [
     requiresField: false,
     isPipelineAgg: true,
     minVersion: 2,
+    maxVersion: 60,
+  },
+  {
+    text: 'Moving Function',
+    value: 'moving_fn',
+    requiresField: false,
+    isPipelineAgg: true,
+    minVersion: 70,
   },
   {
     text: 'Derivative',
@@ -151,6 +159,7 @@ export const pipelineOptions: any = {
     { text: 'predict', default: undefined },
     { text: 'minimize', default: false },
   ],
+  moving_fn: [{ text: 'window', default: 5 }, { text: 'script' }],
   derivative: [{ text: 'unit', default: undefined }],
   cumulative_sum: [{ text: 'format', default: undefined }],
   bucket_script: [],
@@ -175,8 +184,10 @@ export const movingAvgModelSettings: any = {
 
 export function getMetricAggTypes(esVersion: any) {
   return _.filter(metricAggTypes, f => {
-    if (f.minVersion) {
-      return f.minVersion <= esVersion;
+    if (f.minVersion || f.maxVersion) {
+      const minVersion = f.minVersion || 0;
+      const maxVersion = f.maxVersion || esVersion;
+      return esVersion >= minVersion && esVersion <= maxVersion;
     } else {
       return true;
     }

public/app/plugins/datasource/elasticsearch/specs/query_def.test.ts

@@ -162,8 +162,21 @@ describe('ElasticQueryDef', () => {
     });
 
     describe('using esversion 5', () => {
+      const metricAggTypes = queryDef.getMetricAggTypes(5);
       test('should get pipeline aggs', () => {
-        expect(queryDef.getMetricAggTypes(5).length).toBe(15);
+        expect(metricAggTypes.length).toBe(15);
+      });
+    });
+    describe('using esversion 70', () => {
+      const metricAggTypes = queryDef.getMetricAggTypes(70);
+      test('should get pipeline aggs', () => {
+        expect(metricAggTypes.length).toBe(15);
+      });
+      test('should get pipeline aggs with moving function', () => {
+        expect(metricAggTypes.some(m => m.value === 'moving_fn')).toBeTruthy();
+      });
+      test('should get pipeline aggs without moving average', () => {
+        expect(metricAggTypes.some(m => m.value === 'moving_avg')).toBeFalsy();
       });
     });
   });