Elasticsearch: Support extended stats and percentiles in terms order by (#28910)

Adds support to the terms aggregation for ordering by percentiles and extended stats. 

Closes #5148

Co-authored-by: Giordano Ricci <grdnricci@gmail.com>
Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

pkg/tsdb/elasticsearch/time_series_query.go

@@ -2,6 +2,7 @@ package elasticsearch
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -240,15 +241,27 @@ func addTermsAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, metrics []*Metr
 		}
 
 		if orderBy, err := bucketAgg.Settings.Get("orderBy").String(); err == nil {
-			a.Order[orderBy] = bucketAgg.Settings.Get("order").MustString("desc")
-
-			if _, err := strconv.Atoi(orderBy); err == nil {
+			/*
+			   The format for extended stats and percentiles is {metricId}[bucket_path]
+			   for everything else it's just {metricId}, _count, _term, or _key
+			*/
+			metricIdRegex := regexp.MustCompile(`^(\d+)`)
+			metricId := metricIdRegex.FindString(orderBy)
+
+			if len(metricId) > 0 {
 				for _, m := range metrics {
-					if m.ID == orderBy {
+					if m.ID == metricId {
+						if m.Type == "count" {
+							a.Order["_count"] = bucketAgg.Settings.Get("order").MustString("desc")
+						} else {
+							a.Order[orderBy] = bucketAgg.Settings.Get("order").MustString("desc")
 							b.Metric(m.ID, m.Type, m.Field, nil)
+						}
 						break
 					}
 				}
+			} else {
+				a.Order[orderBy] = bucketAgg.Settings.Get("order").MustString("desc")
 			}
 		}
 

pkg/tsdb/elasticsearch/time_series_query_test.go

@@ -127,6 +127,80 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
 			So(avgAgg.Aggregation.Type, ShouldEqual, "avg")
 		})
 
+		Convey("With term agg and order by count metric agg", func() {
+			c := newFakeClient(5)
+			_, err := executeTsdbQuery(c, `{
+				"timeField": "@timestamp",
+				"bucketAggs": [
+					{
+						"type": "terms",
+						"field": "@host",
+						"id": "2",
+						"settings": { "size": "5", "order": "asc", "orderBy": "1"	}
+					},
+					{ "type": "date_histogram", "field": "@timestamp", "id": "3" }
+				],
+				"metrics": [
+					{"type": "count", "id": "1" }
+				]
+			}`, from, to, 15*time.Second)
+			So(err, ShouldBeNil)
+			sr := c.multisearchRequests[0].Requests[0]
+
+			termsAgg := sr.Aggs[0].Aggregation.Aggregation.(*es.TermsAggregation)
+			So(termsAgg.Order["_count"], ShouldEqual, "asc")
+		})
+
+		Convey("With term agg and order by percentiles agg", func() {
+			c := newFakeClient(5)
+			_, err := executeTsdbQuery(c, `{
+				"timeField": "@timestamp",
+				"bucketAggs": [
+					{
+						"type": "terms",
+						"field": "@host",
+						"id": "2",
+						"settings": { "size": "5", "order": "asc", "orderBy": "1[95.0]"	}
+					},
+					{ "type": "date_histogram", "field": "@timestamp", "id": "3" }
+				],
+				"metrics": [
+        {"type": "percentiles", "field": "@value", "id": "1", "settings": { "percents": ["95","99"] } }
+				]
+			}`, from, to, 15*time.Second)
+			So(err, ShouldBeNil)
+			sr := c.multisearchRequests[0].Requests[0]
+
+			orderByAgg := sr.Aggs[0].Aggregation.Aggs[0]
+			So(orderByAgg.Key, ShouldEqual, "1")
+			So(orderByAgg.Aggregation.Type, ShouldEqual, "percentiles")
+		})
+
+		Convey("With term agg and order by extended stats agg", func() {
+			c := newFakeClient(5)
+			_, err := executeTsdbQuery(c, `{
+				"timeField": "@timestamp",
+				"bucketAggs": [
+					{
+						"type": "terms",
+						"field": "@host",
+						"id": "2",
+						"settings": { "size": "5", "order": "asc", "orderBy": "1[std_deviation]"	}
+					},
+					{ "type": "date_histogram", "field": "@timestamp", "id": "3" }
+				],
+				"metrics": [
+        {"type": "extended_stats", "field": "@value", "id": "1", "meta": { "std_deviation": true } }
+				]
+			}`, from, to, 15*time.Second)
+			So(err, ShouldBeNil)
+			sr := c.multisearchRequests[0].Requests[0]
+
+			orderByAgg := sr.Aggs[0].Aggregation.Aggs[0]
+			So(orderByAgg.Key, ShouldEqual, "1")
+			So(orderByAgg.Aggregation.Type, ShouldEqual, "extended_stats")
+		})
+
 		Convey("With term agg and order by term", func() {
 			c := newFakeClient(5)
 			_, err := executeTsdbQuery(c, `{

public/app/plugins/datasource/elasticsearch/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/index.tsx

@@ -4,11 +4,16 @@ import { useDispatch } from '../../../../hooks/useStatelessReducer';
 import { SettingsEditorContainer } from '../../SettingsEditorContainer';
 import { changeBucketAggregationSetting } from '../state/actions';
 import { BucketAggregation } from '../aggregations';
-import { bucketAggregationConfig, intervalOptions, orderByOptions, orderOptions, sizeOptions } from '../utils';
+import {
+  bucketAggregationConfig,
+  createOrderByOptionsFromMetrics,
+  intervalOptions,
+  orderOptions,
+  sizeOptions,
+} from '../utils';
 import { FiltersSettingsEditor } from './FiltersSettingsEditor';
 import { useDescription } from './useDescription';
 import { useQuery } from '../../ElasticsearchQueryContext';
-import { describeMetric } from '../../../../utils';
 
 const inlineFieldProps: Partial<ComponentProps<typeof InlineField>> = {
   labelWidth: 16,
@@ -22,8 +27,7 @@ export const SettingsEditor: FunctionComponent<Props> = ({ bucketAgg }) => {
   const dispatch = useDispatch();
   const { metrics } = useQuery();
   const settingsDescription = useDescription(bucketAgg);
-
-  const orderBy = [...orderByOptions, ...(metrics || []).map(m => ({ label: describeMetric(m), value: m.id }))];
+  const orderBy = createOrderByOptionsFromMetrics(metrics);
 
   return (
     <SettingsEditorContainer label={settingsDescription}>

public/app/plugins/datasource/elasticsearch/components/QueryEditor/BucketAggregationsEditor/SettingsEditor/useDescription.ts

-import { describeMetric } from '../../../../utils';
+import { describeMetric, convertOrderByToMetricId } from '../../../../utils';
 import { useQuery } from '../../ElasticsearchQueryContext';
 import { BucketAggregation } from '../aggregations';
 import { bucketAggregationConfig, orderByOptions, orderOptions } from '../utils';
@@ -34,7 +34,7 @@ export const useDescription = (bucketAgg: BucketAggregation): string => {
       if (orderByOption) {
         description += orderByOption.label;
       } else {
-        const metric = metrics?.find(m => m.id === orderBy);
+        const metric = metrics?.find(m => m.id === convertOrderByToMetricId(orderBy));
         if (metric) {
           description += describeMetric(metric);
         } else {

public/app/plugins/datasource/elasticsearch/components/QueryEditor/BucketAggregationsEditor/utils.ts

 import { BucketsConfiguration } from '../../../types';
 import { defaultFilter } from './SettingsEditor/FiltersSettingsEditor/utils';
+import { describeMetric } from '../../../utils';
+import {
+  ExtendedStatMetaType,
+  ExtendedStats,
+  MetricAggregation,
+  Percentiles,
+} from '../MetricAggregationsEditor/aggregations';
+import { SelectableValue } from '@grafana/data';
 
 export const bucketAggregationConfig: BucketsConfiguration = {
   terms: {
@@ -46,7 +54,8 @@ export const bucketAggregationConfig: BucketsConfiguration = {
 };
 
 // TODO: Define better types for the following
-export const orderOptions = [
+type OrderByOption = SelectableValue<string>;
+export const orderOptions: OrderByOption[] = [
   { label: 'Top', value: 'desc' },
   { label: 'Bottom', value: 'asc' },
 ];
@@ -77,3 +86,58 @@ export const intervalOptions = [
   { label: '1h', value: '1h' },
   { label: '1d', value: '1d' },
 ];
+
+/**
+ * This returns the valid options for each of the enabled extended stat
+ */
+function createOrderByOptionsForExtendedStats(metric: ExtendedStats): OrderByOption[] {
+  if (!metric.meta) {
+    return [];
+  }
+  const metaKeys = Object.keys(metric.meta) as ExtendedStatMetaType[];
+  return metaKeys
+    .filter(key => metric.meta?.[key])
+    .map(key => {
+      let method = key as string;
+      // The bucket path for std_deviation_bounds.lower and std_deviation_bounds.upper
+      // is accessed via std_lower and std_upper, respectively.
+      if (key === 'std_deviation_bounds_lower') {
+        method = 'std_lower';
+      }
+      if (key === 'std_deviation_bounds_upper') {
+        method = 'std_upper';
+      }
+      return { label: `${describeMetric(metric)} (${method})`, value: `${metric.id}[${method}]` };
+    });
+}
+
+/**
+ * This returns the valid options for each of the percents listed in the percentile settings
+ */
+function createOrderByOptionsForPercentiles(metric: Percentiles): OrderByOption[] {
+  if (!metric.settings?.percents) {
+    return [];
+  }
+  return metric.settings.percents.map(percent => {
+    // The bucket path for percentile numbers is appended with a `.0` if the number is whole
+    // otherwise you have to use the actual value.
+    const percentString = /^\d+\.\d+/.test(`${percent}`) ? percent : `${percent}.0`;
+    return { label: `${describeMetric(metric)} (${percent})`, value: `${metric.id}[${percentString}]` };
+  });
+}
+
+/**
+ * This creates all the valid order by options based on the metrics
+ */
+export const createOrderByOptionsFromMetrics = (metrics: MetricAggregation[] = []): OrderByOption[] => {
+  const metricOptions = metrics.flatMap(metric => {
+    if (metric.type === 'extended_stats') {
+      return createOrderByOptionsForExtendedStats(metric);
+    } else if (metric.type === 'percentiles') {
+      return createOrderByOptionsForPercentiles(metric);
+    } else {
+      return { label: describeMetric(metric), value: metric.id };
+    }
+  });
+  return [...orderByOptions, ...metricOptions];
+};

public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/aggregations.ts

@@ -114,7 +114,7 @@ export interface ExtendedStats extends MetricAggregationWithField, MetricAggrega
   };
 }
 
-interface Percentiles extends MetricAggregationWithField, MetricAggregationWithInlineScript {
+export interface Percentiles extends MetricAggregationWithField, MetricAggregationWithInlineScript {
   type: 'percentiles';
   settings?: {
     percents?: string[];

public/app/plugins/datasource/elasticsearch/query_builder.ts

@@ -12,6 +12,7 @@ import {
 } from './components/QueryEditor/MetricAggregationsEditor/aggregations';
 import { defaultBucketAgg, defaultMetricAgg, findMetricById } from './query_def';
 import { ElasticsearchQuery } from './types';
+import { convertOrderByToMetricId } from './utils';
 
 export class ElasticQueryBuilder {
   timeField: string;
@@ -34,7 +35,6 @@ export class ElasticQueryBuilder {
   }
 
   buildTermsAgg(aggDef: Terms, queryNode: { terms?: any; aggs?: any }, target: ElasticsearchQuery) {
-    let metricRef;
     queryNode.terms = { field: aggDef.field };
 
     if (!aggDef.settings) {
@@ -54,14 +54,17 @@ export class ElasticQueryBuilder {
       }
 
       // if metric ref, look it up and add it to this agg level
-      metricRef = parseInt(aggDef.settings.orderBy, 10);
-      if (!isNaN(metricRef)) {
+      const metricId = convertOrderByToMetricId(aggDef.settings.orderBy);
+      if (metricId) {
         for (let metric of target.metrics || []) {
-          if (metric.id === aggDef.settings.orderBy) {
+          if (metric.id === metricId) {
+            if (metric.type === 'count') {
+              queryNode.terms.order = { _count: aggDef.settings.order };
+            } else if (isMetricAggregationWithField(metric)) {
               queryNode.aggs = {};
-            queryNode.aggs[metric.id] = {};
-            if (isMetricAggregationWithField(metric)) {
-              queryNode.aggs[metric.id][metric.type] = { field: metric.field };
+              queryNode.aggs[metric.id] = {
+                [metric.type]: { field: metric.field },
+              };
             }
             break;
           }

public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts

@@ -127,6 +127,84 @@ describe('ElasticQueryBuilder', () => {
         expect(secondLevel.aggs['5'].avg.field).toBe('@value');
       });
 
+      it('with term agg and order by count agg', () => {
+        const query = builder.build(
+          {
+            refId: 'A',
+            metrics: [
+              { type: 'count', id: '1' },
+              { type: 'avg', field: '@value', id: '5' },
+            ],
+            bucketAggs: [
+              {
+                type: 'terms',
+                field: '@host',
+                settings: { size: '5', order: 'asc', orderBy: '1' },
+                id: '2',
+              },
+              { type: 'date_histogram', field: '@timestamp', id: '3' },
+            ],
+          },
+          100,
+          '1000'
+        );
+
+        expect(query.aggs['2'].terms.order._count).toEqual('asc');
+        expect(query.aggs['2'].aggs).not.toHaveProperty('1');
+      });
+
+      it('with term agg and order by extended_stats agg', () => {
+        const query = builder.build(
+          {
+            refId: 'A',
+            metrics: [{ type: 'extended_stats', id: '1', field: '@value', meta: { std_deviation: true } }],
+            bucketAggs: [
+              {
+                type: 'terms',
+                field: '@host',
+                settings: { size: '5', order: 'asc', orderBy: '1[std_deviation]' },
+                id: '2',
+              },
+              { type: 'date_histogram', field: '@timestamp', id: '3' },
+            ],
+          },
+          100,
+          '1000'
+        );
+
+        const firstLevel = query.aggs['2'];
+        const secondLevel = firstLevel.aggs['3'];
+
+        expect(firstLevel.aggs['1'].extended_stats.field).toBe('@value');
+        expect(secondLevel.aggs['1'].extended_stats.field).toBe('@value');
+      });
+
+      it('with term agg and order by percentiles agg', () => {
+        const query = builder.build(
+          {
+            refId: 'A',
+            metrics: [{ type: 'percentiles', id: '1', field: '@value', settings: { percents: ['95', '99'] } }],
+            bucketAggs: [
+              {
+                type: 'terms',
+                field: '@host',
+                settings: { size: '5', order: 'asc', orderBy: '1[95.0]' },
+                id: '2',
+              },
+              { type: 'date_histogram', field: '@timestamp', id: '3' },
+            ],
+          },
+          100,
+          '1000'
+        );
+
+        const firstLevel = query.aggs['2'];
+        const secondLevel = firstLevel.aggs['3'];
+
+        expect(firstLevel.aggs['1'].percentiles.field).toBe('@value');
+        expect(secondLevel.aggs['1'].percentiles.field).toBe('@value');
+      });
+
       it('with term agg and valid min_doc_count', () => {
         const query = builder.build(
           {

public/app/plugins/datasource/elasticsearch/utils.ts

@@ -52,3 +52,13 @@ export const removeEmpty = <T>(obj: T): Partial<T> =>
       [key]: value,
     };
   }, {});
+
+/**
+ *  This function converts an order by string to the correct metric id For example,
+ *  if the user uses the standard deviation extended stat for the order by,
+ *  the value would be "1[std_deviation]" and this would return "1"
+ */
+export const convertOrderByToMetricId = (orderBy: string): string | undefined => {
+  const metricIdMatches = orderBy.match(/^(\d+)/);
+  return metricIdMatches ? metricIdMatches[1] : void 0;
+};