feat(elasticsearch): lots of work on elasticsearch metrics processing, handling…

feat(elasticsearch): lots of work on elasticsearch metrics processing, handling grouped responses, etc, #1034

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -172,6 +172,8 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
       return date.getTime();
     };
 
+    // this is quite complex
+    // neeed to recurise down the nested buckets to build series
     ElasticDatasource.prototype._processBuckets = function(buckets, groupByFields, series, level, parentName, parentTime) {
       var points = [];
       var groupBy = groupByFields[level];
@@ -180,13 +182,14 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
         var bucket = buckets[i];
 
         if (groupBy) {
-          var seriesName = "";
+          var seriesName = level > 0 ? bucket.key : '';
           var time = parentTime || bucket.key;
           this._processBuckets(bucket[groupBy.field].buckets, groupByFields, series, level+1, seriesName, time)
         } else {
           var seriesName = parentName;
 
           if (level > 0) {
+            if (seriesName) { seriesName += " "; }
             seriesName += bucket.key;
           } else {
             parentTime = bucket.key;
@@ -202,12 +205,17 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
       var series = [];
 
       for (var i = 0; i < results.responses.length; i++) {
-        var buckets = results.responses[i].aggregations.histogram.buckets;
+        var response = results.responses[i];
+        if (response.error) {
+          throw { message: response.error };
+        }
+
+        var buckets = response.aggregations.histogram.buckets;
         var target = targets[i];
         var points = [];
         var querySeries = {}
 
-        this._processBuckets(buckets, target.groupByFields, querySeries, 0, target.refId);
+        this._processBuckets(buckets, target.groupByFields, querySeries, 0);
 
         _.each(querySeries, function(value) {
           series.push(value);

public/app/plugins/datasource/elasticsearch/partials/query.editor.html

@@ -75,7 +75,7 @@
 <div class="tight-form" ng-hide="target.rawQuery">
 	<ul class="tight-form-list">
 		<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
-			Group By
+			Group by
 		</li>
 		<li ng-repeat="segment in groupBySegments">
 			<metric-segment segment="segment" get-alt-segments="getGroupByFields(segment, $index)" on-value-changed="groupByChanged(segment, $index)"></metric-segment>

public/app/plugins/datasource/elasticsearch/queryBuilder.js

@@ -52,23 +52,23 @@ function () {
             "max": timeTo
           }
         }
-      }
+      },
     };
 
-    var nestedAggs = {};
+    var nestedAggs = query.aggs.histogram;
 
-    if (target.groupByFields && target.groupByFields.length > 0) {
-      var field = target.groupByFields[0].field;
-      nestedAggs[field] = {
-        terms:  {
-          field: field,
-        }
-      }
+    for (var i = 0; i < target.groupByFields.length; i++) {
+      var field = target.groupByFields[i].field;
+      var aggs = {terms: {field: field}};
+
+      nestedAggs.aggs = {};
+      nestedAggs.aggs[field] = aggs;
+      nestedAggs = aggs;
     }
 
-    query.aggs.histogram.aggs = nestedAggs;
+    console.log(angular.toJson(query, true));
 
-    query = JSON.stringify(query);
+    query = angular.toJson(query);
     return query;
   };
 

public/test/specs/elasticsearch-specs.js

@@ -57,14 +57,7 @@ define([
         var result;
 
         beforeEach(function() {
-          result = ctx.ds._processTimeSeries([
-            {
-              refId: 'A',
-              groupByFields: [
-                {field: 'host' }
-              ]
-            }
-          ], {
+          result = ctx.ds._processTimeSeries([{refId: 'A', groupByFields: [{field: 'host' }]}], {
             responses: [{
               aggregations: {
                 histogram: {
@@ -103,6 +96,87 @@ define([
           expect(result.data[1].target).to.be('server2');
         });
       });
+
+      describe('two group by query', function() {
+        var result;
+
+        beforeEach(function() {
+          result = ctx.ds._processTimeSeries([{refId: 'A', groupByFields: [{field: 'host'}, {field: 'site'}]}], {
+            responses: [{
+              aggregations: {
+                histogram: {
+                  buckets: [
+                    {
+                      host: {
+                        buckets: [
+                           {
+                             site: {
+                               buckets: [
+                                 {doc_count: 3, key: 'backend'},
+                                 {doc_count: 1, key: 'frontend'},
+                               ],
+                             },
+                             doc_count: 4, key: 'server1'
+                           },
+                           {
+                             site: {
+                               buckets: [
+                                 {doc_count: 3, key: 'backend'},
+                                 {doc_count: 1, key: 'frontend'},
+                               ],
+                             },
+                             doc_count: 6, key: 'server2'
+                           },
+                        ]
+                      },
+                      doc_count: 10,
+                      key: 1000
+                    },
+                    {
+                      host: {
+                        buckets: [
+                          {
+                            site: {
+                               buckets: [
+                                 {doc_count: 3, key: 'backend'},
+                                 {doc_count: 1, key: 'frontend'},
+                               ],
+                            },
+                            doc_count: 4,
+                            key: 'server1'
+                          },
+                          {
+                            site: {
+                               buckets: [
+                                 {doc_count: 3, key: 'backend'},
+                                 {doc_count: 1, key: 'frontend'},
+                               ],
+                            },
+                            doc_count: 6,
+                            key: 'server2'
+                          },
+                        ]
+                      },
+                      doc_count: 15,
+                      key: 2000
+                    }
+                  ]
+                }
+              }
+            }]
+          })
+        });
+
+        it('should return 2 series', function() {
+          expect(result.data.length).to.be(4);
+          expect(result.data[0].datapoints.length).to.be(2);
+          expect(result.data[0].target).to.be('server1 backend');
+          expect(result.data[1].target).to.be('server1 frontend');
+          expect(result.data[2].target).to.be('server2 backend');
+          expect(result.data[3].target).to.be('server2 frontend');
+        });
+      });
+
     });
   });
 });