Merge branch 'elastic5_support'

docker/blocks/elastic1/elasticsearch.yml

+script.inline: on
+script.indexed: on

docker/blocks/elastic1/fig

+elasticsearch1:
+  image: elasticsearch:1.7.6
+  command: elasticsearch -Des.network.host=0.0.0.0
+  ports:
+    - "11200:9200"
+    - "11300:9300"
+  volumes:
+    - ./blocks/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml

docker/blocks/elastic5/fig

@@ -4,5 +4,5 @@ elasticsearch5:
   image: elasticsearch:5
   image: elasticsearch:5
   command: elasticsearch
   command: elasticsearch
   ports:
   ports:
-    - "9200:9200"
-    - "9300:9300"
+    - "10200:9200"
+    - "10300:9300"

public/app/plugins/datasource/elasticsearch/config_ctrl.ts

@@ -22,8 +22,8 @@ export class ElasticConfigCtrl {
   ];
   ];
 
 
   esVersions = [
   esVersions = [
-    {name: '1.x', value: 1},
     {name: '2.x', value: 2},
     {name: '2.x', value: 2},
+    {name: '5.x', value: 5},
   ];
   ];
 
 
   indexPatternTypeChanged() {
   indexPatternTypeChanged() {

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -81,21 +81,31 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
       range[timeField]= {
       range[timeField]= {
         from: options.range.from.valueOf(),
         from: options.range.from.valueOf(),
         to: options.range.to.valueOf(),
         to: options.range.to.valueOf(),
+        format: "epoch_millis",
       };
       };
 
 
-      if (this.esVersion >= 2) {
-        range[timeField]["format"] = "epoch_millis";
-      }
-
       var queryInterpolated = templateSrv.replace(queryString, {}, 'lucene');
       var queryInterpolated = templateSrv.replace(queryString, {}, 'lucene');
-      var filter = { "bool": { "must": [{ "range": range }] } };
-      var query = { "bool": { "should": [{ "query_string": { "query": queryInterpolated } }] } };
+      var query = {
+        "bool": {
+          "must": [
+            { "range": range },
+            { "query_string": {
+              "query": queryInterpolated }
+            }
+          ]
+        }
+      };
+
       var data = {
       var data = {
-        "fields": [timeField, "_source"],
-        "query" : { "filtered": { "query" : query, "filter": filter } },
+        "query" : query,
         "size": 10000
         "size": 10000
       };
       };
 
 
+      // fields field not supported on ES 5.x
+      if (this.esVersion < 5) {
+        data["fields"] = [timeField, "_source"];
+      }
+
       var header = {search_type: "query_then_fetch", "ignore_unavailable": true};
       var header = {search_type: "query_then_fetch", "ignore_unavailable": true};
 
 
       // old elastic annotations had index specified on them
       // old elastic annotations had index specified on them
@@ -133,11 +143,12 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
 
 
         for (var i = 0; i < hits.length; i++) {
         for (var i = 0; i < hits.length; i++) {
           var source = hits[i]._source;
           var source = hits[i]._source;
-          var fields = hits[i].fields;
           var time = source[timeField];
           var time = source[timeField];
-
-          if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) {
-            time = fields[timeField];
+          if (typeof hits[i].fields !== 'undefined') {
+            var fields = hits[i].fields;
+            if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) {
+              time = fields[timeField];
+            }
           }
           }
 
 
           var event = {
           var event = {
@@ -194,7 +205,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
         luceneQuery = luceneQuery.substr(1, luceneQuery.length - 2);
         luceneQuery = luceneQuery.substr(1, luceneQuery.length - 2);
         esQuery = esQuery.replace("$lucene_query", luceneQuery);
         esQuery = esQuery.replace("$lucene_query", luceneQuery);
 
 
-        var searchType = queryObj.size === 0 ? 'count' : 'query_then_fetch';
+        var searchType = (queryObj.size === 0 && this.esVersion < 5) ? 'count' : 'query_then_fetch';
         var header = this.getQueryHeader(searchType, options.range.from, options.range.to);
         var header = this.getQueryHeader(searchType, options.range.from, options.range.to);
         payload +=  header + '\n';
         payload +=  header + '\n';
 
 
@@ -277,14 +288,15 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
 
 
     this.getTerms = function(queryDef) {
     this.getTerms = function(queryDef) {
       var range = timeSrv.timeRange();
       var range = timeSrv.timeRange();
-      var header = this.getQueryHeader('count', range.from, range.to);
+      var searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count' ;
+      var header = this.getQueryHeader(searchType, range.from, range.to);
       var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef));
       var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef));
 
 
       esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf());
       esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf());
       esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
       esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
       esQuery = header + '\n' + esQuery + '\n';
       esQuery = header + '\n' + esQuery + '\n';
 
 
-      return this._post('_msearch?search_type=count', esQuery).then(function(res) {
+      return this._post('_msearch?search_type=' + searchType, esQuery).then(function(res) {
         if (!res.responses[0].aggregations) {
         if (!res.responses[0].aggregations) {
           return [];
           return [];
         }
         }

public/app/plugins/datasource/elasticsearch/query_builder.js

@@ -11,11 +11,11 @@ function (queryDef) {
 
 
   ElasticQueryBuilder.prototype.getRangeFilter = function() {
   ElasticQueryBuilder.prototype.getRangeFilter = function() {
     var filter = {};
     var filter = {};
-    filter[this.timeField] = {"gte": "$timeFrom", "lte": "$timeTo"};
-
-    if (this.esVersion >= 2) {
-      filter[this.timeField]["format"] = "epoch_millis";
-    }
+    filter[this.timeField] = {
+      gte: "$timeFrom",
+      lte: "$timeTo",
+      format: "epoch_millis",
+    };
 
 
     return filter;
     return filter;
   };
   };
@@ -28,7 +28,8 @@ function (queryDef) {
       return queryNode;
       return queryNode;
     }
     }
 
 
-    queryNode.terms.size = parseInt(aggDef.settings.size, 10);
+    queryNode.terms.size = parseInt(aggDef.settings.size, 10) === 0 ? 500 : parseInt(aggDef.settings.size, 10);
+
     if (aggDef.settings.orderBy !== void 0) {
     if (aggDef.settings.orderBy !== void 0) {
       queryNode.terms.order = {};
       queryNode.terms.order = {};
       queryNode.terms.order[aggDef.settings.orderBy] = aggDef.settings.order;
       queryNode.terms.order[aggDef.settings.orderBy] = aggDef.settings.order;
@@ -62,15 +63,12 @@ function (queryDef) {
     esAgg.field = this.timeField;
     esAgg.field = this.timeField;
     esAgg.min_doc_count = settings.min_doc_count || 0;
     esAgg.min_doc_count = settings.min_doc_count || 0;
     esAgg.extended_bounds = {min: "$timeFrom", max: "$timeTo"};
     esAgg.extended_bounds = {min: "$timeFrom", max: "$timeTo"};
+    esAgg.format = "epoch_millis";
 
 
     if (esAgg.interval === 'auto') {
     if (esAgg.interval === 'auto') {
       esAgg.interval = "$interval";
       esAgg.interval = "$interval";
     }
     }
 
 
-    if (this.esVersion >= 2) {
-      esAgg.format = "epoch_millis";
-    }
-
     if (settings.missing) {
     if (settings.missing) {
       esAgg.missing = settings.missing;
       esAgg.missing = settings.missing;
     }
     }
@@ -80,15 +78,13 @@ function (queryDef) {
 
 
   ElasticQueryBuilder.prototype.getFiltersAgg = function(aggDef) {
   ElasticQueryBuilder.prototype.getFiltersAgg = function(aggDef) {
     var filterObj = {};
     var filterObj = {};
-
     for (var i = 0; i < aggDef.settings.filters.length; i++) {
     for (var i = 0; i < aggDef.settings.filters.length; i++) {
       var query = aggDef.settings.filters[i].query;
       var query = aggDef.settings.filters[i].query;
+
       filterObj[query] = {
       filterObj[query] = {
-        query: {
-          query_string: {
-            query: query,
-            analyze_wildcard: true
-          }
+        query_string: {
+          query: query,
+          analyze_wildcard: true
         }
         }
       };
       };
     }
     }
@@ -100,7 +96,12 @@ function (queryDef) {
     query.size = 500;
     query.size = 500;
     query.sort = {};
     query.sort = {};
     query.sort[this.timeField] = {order: 'desc', unmapped_type: 'boolean'};
     query.sort[this.timeField] = {order: 'desc', unmapped_type: 'boolean'};
-    query.fields = ["*", "_source"];
+
+    // fields field not supported on ES 5.x
+    if (this.esVersion < 5) {
+      query.fields = ["*", "_source"];
+    }
+
     query.script_fields = {},
     query.script_fields = {},
     query.fielddata_fields = [this.timeField];
     query.fielddata_fields = [this.timeField];
     return query;
     return query;
@@ -112,13 +113,11 @@ function (queryDef) {
     }
     }
 
 
     var i, filter, condition;
     var i, filter, condition;
-    var must = query.query.filtered.filter.bool.must;
-
     for (i = 0; i < adhocFilters.length; i++) {
     for (i = 0; i < adhocFilters.length; i++) {
       filter = adhocFilters[i];
       filter = adhocFilters[i];
       condition = {};
       condition = {};
       condition[filter.key] = filter.value;
       condition[filter.key] = filter.value;
-      must.push({"term": condition});
+      query.query.bool.must.push({"term": condition});
     }
     }
   };
   };
 
 
@@ -133,18 +132,15 @@ function (queryDef) {
     var query = {
     var query = {
       "size": 0,
       "size": 0,
       "query": {
       "query": {
-        "filtered": {
-          "query": {
-            "query_string": {
+        "bool": {
+          "must": [
+            {"range": this.getRangeFilter()},
+            {"query_string": {
               "analyze_wildcard": true,
               "analyze_wildcard": true,
-              "query": '$lucene_query',
-            }
-          },
-          "filter": {
-            "bool": {
-              "must": [{"range": this.getRangeFilter()}]
+              "query": '$lucene_query'
+              }
             }
             }
-          }
+          ]
         }
         }
       }
       }
     };
     };
@@ -228,30 +224,26 @@ function (queryDef) {
     var query = {
     var query = {
       "size": 0,
       "size": 0,
       "query": {
       "query": {
-        "filtered": {
-          "filter": {
-            "bool": {
-              "must": [{"range": this.getRangeFilter()}]
-            }
-          }
+        "bool": {
+          "must": [{"range": this.getRangeFilter()}]
         }
         }
       }
       }
     };
     };
 
 
     if (queryDef.query) {
     if (queryDef.query) {
-      query.query.filtered.query = {
+      query.query.bool.must.push({
         "query_string": {
         "query_string": {
           "analyze_wildcard": true,
           "analyze_wildcard": true,
           "query": queryDef.query,
           "query": queryDef.query,
         }
         }
-      };
+      });
     }
     }
 
 
     query.aggs =  {
     query.aggs =  {
       "1": {
       "1": {
         "terms": {
         "terms": {
           "field": queryDef.field,
           "field": queryDef.field,
-          "size": 0,
+          "size": 500,
           "order": {
           "order": {
             "_term": "asc"
             "_term": "asc"
           }
           }

public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts

@@ -28,7 +28,7 @@ describe('ElasticDatasource', function() {
 
 
   describe('When testing datasource with index pattern', function() {
   describe('When testing datasource with index pattern', function() {
     beforeEach(function() {
     beforeEach(function() {
-      createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily'}});
+      createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily', esVersion: '2'}});
     });
     });
 
 
     it('should translate index pattern to current day', function() {
     it('should translate index pattern to current day', function() {
@@ -50,7 +50,7 @@ describe('ElasticDatasource', function() {
     var requestOptions, parts, header;
     var requestOptions, parts, header;
 
 
     beforeEach(function() {
     beforeEach(function() {
-      createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily'}});
+      createDatasource({url: 'http://es.com', index: '[asd-]YYYY.MM.DD', jsonData: {interval: 'Daily', esVersion: '2'}});
 
 
       ctx.backendSrv.datasourceRequest = function(options) {
       ctx.backendSrv.datasourceRequest = function(options) {
         requestOptions = options;
         requestOptions = options;
@@ -77,7 +77,7 @@ describe('ElasticDatasource', function() {
 
 
     it('should json escape lucene query', function() {
     it('should json escape lucene query', function() {
       var body = angular.fromJson(parts[1]);
       var body = angular.fromJson(parts[1]);
-      expect(body.query.filtered.query.query_string.query).to.be('escape\\:test');
+      expect(body.query.bool.must[1].query_string.query).to.be('escape\\:test');
     });
     });
   });
   });
 
 
@@ -85,7 +85,7 @@ describe('ElasticDatasource', function() {
     var requestOptions, parts, header;
     var requestOptions, parts, header;
 
 
     beforeEach(function() {
     beforeEach(function() {
-      createDatasource({url: 'http://es.com', index: 'test'});
+      createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '2'}});
 
 
       ctx.backendSrv.datasourceRequest = function(options) {
       ctx.backendSrv.datasourceRequest = function(options) {
         requestOptions = options;
         requestOptions = options;
@@ -209,4 +209,79 @@ describe('ElasticDatasource', function() {
     });
     });
   });
   });
 
 
+  describe('When issuing aggregation query on es5.x', function() {
+    var requestOptions, parts, header;
+
+    beforeEach(function() {
+      createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}});
+
+      ctx.backendSrv.datasourceRequest = function(options) {
+        requestOptions = options;
+        return ctx.$q.when({data: {responses: []}});
+      };
+
+      ctx.ds.query({
+        range: { from: moment([2015, 4, 30, 10]), to: moment([2015, 5, 1, 10]) },
+        targets: [{
+            bucketAggs: [
+                {type: 'date_histogram', field: '@timestamp', id: '2'}
+            ],
+            metrics: [
+                {type: 'count'}], query: 'test' }
+            ]
+      });
+
+      ctx.$rootScope.$apply();
+      parts = requestOptions.data.split('\n');
+      header = angular.fromJson(parts[0]);
+    });
+
+    it('should not set search type to count', function() {
+      expect(header.search_type).to.not.eql('count');
+    });
+
+    it('should set size to 0', function() {
+      var body = angular.fromJson(parts[1]);
+      expect(body.size).to.be(0);
+    });
+
+  });
+
+  describe('When issuing metricFind query on es5.x', function() {
+    var requestOptions, parts, header, body;
+
+    beforeEach(function() {
+      createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}});
+
+      ctx.backendSrv.datasourceRequest = function(options) {
+        requestOptions = options;
+        return ctx.$q.when({
+            data: {
+                responses: [{aggregations: {"1": [{buckets: {text: 'test', value: '1'}}]}}]
+            }
+        });
+      };
+
+      ctx.ds.metricFindQuery('{"find": "terms", "field": "test"}');
+      ctx.$rootScope.$apply();
+
+      parts = requestOptions.data.split('\n');
+      header = angular.fromJson(parts[0]);
+      body = angular.fromJson(parts[1]);
+    });
+
+    it('should not set search type to count', function() {
+      expect(header.search_type).to.not.eql('count');
+    });
+
+    it('should set size to 0', function() {
+      expect(body.size).to.be(0);
+    });
+
+    it('should not set terms aggregation size to 0', function() {
+      expect(body['aggs']['1']['terms'].size).to.not.be(0);
+    });
+  });
+
 });
 });
+

public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts

@@ -16,7 +16,23 @@ describe('ElasticQueryBuilder', function() {
       bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '1'}],
       bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '1'}],
     });
     });
 
 
-    expect(query.query.filtered.filter.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
+    expect(query.query.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
+    expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom");
+  });
+
+  it('with defaults on es5.x', function() {
+    var builder_5x = new ElasticQueryBuilder({
+      timeField: '@timestamp',
+      esVersion: 5
+    });
+
+    var query = builder_5x.build({
+      metrics: [{type: 'Count', id: '0'}],
+      timeField: '@timestamp',
+      bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '1'}],
+    });
+
+    expect(query.query.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
     expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom");
     expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom");
   });
   });
 
 
@@ -34,39 +50,6 @@ describe('ElasticQueryBuilder', function() {
     expect(query.aggs["2"].aggs["3"].date_histogram.field).to.be("@timestamp");
     expect(query.aggs["2"].aggs["3"].date_histogram.field).to.be("@timestamp");
   });
   });
 
 
-  it('with es1.x and es2.x date histogram queries check time format', function() {
-    var builder_2x = new ElasticQueryBuilder({
-      timeField: '@timestamp',
-      esVersion: 2
-    });
-
-    var query_params = {
-      metrics: [],
-      bucketAggs: [
-        {type: 'date_histogram', field: '@timestamp', id: '1'}
-      ],
-    };
-
-    // format should not be specified in 1.x queries
-    expect("format" in builder.build(query_params)["aggs"]["1"]["date_histogram"]).to.be(false);
-
-    // 2.x query should specify format to be "epoch_millis"
-    expect(builder_2x.build(query_params)["aggs"]["1"]["date_histogram"]["format"]).to.be("epoch_millis");
-  });
-
-  it('with es1.x and es2.x range filter check time format', function() {
-    var builder_2x = new ElasticQueryBuilder({
-      timeField: '@timestamp',
-      esVersion: 2
-    });
-
-    // format should not be specified in 1.x queries
-    expect("format" in builder.getRangeFilter()["@timestamp"]).to.be(false);
-
-    // 2.x query should specify format to be "epoch_millis"
-    expect(builder_2x.getRangeFilter()["@timestamp"]["format"]).to.be("epoch_millis");
-  });
-
   it('with select field', function() {
   it('with select field', function() {
     var query = builder.build({
     var query = builder.build({
       metrics: [{type: 'avg', field: '@value', id: '1'}],
       metrics: [{type: 'avg', field: '@value', id: '1'}],
@@ -138,8 +121,36 @@ describe('ElasticQueryBuilder', function() {
       ],
       ],
     });
     });
 
 
-    expect(query.aggs["2"].filters.filters["@metric:cpu"].query.query_string.query).to.be("@metric:cpu");
-    expect(query.aggs["2"].filters.filters["@metric:logins.count"].query.query_string.query).to.be("@metric:logins.count");
+    expect(query.aggs["2"].filters.filters["@metric:cpu"].query_string.query).to.be("@metric:cpu");
+    expect(query.aggs["2"].filters.filters["@metric:logins.count"].query_string.query).to.be("@metric:logins.count");
+    expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
+  });
+
+  it('with filters aggs on es5.x', function() {
+    var builder_5x = new ElasticQueryBuilder({
+      timeField: '@timestamp',
+      esVersion: 5
+    });
+    var query = builder_5x.build({
+      metrics: [{type: 'count', id: '1'}],
+      timeField: '@timestamp',
+      bucketAggs: [
+        {
+          id: '2',
+          type: 'filters',
+          settings: {
+            filters: [
+              {query: '@metric:cpu' },
+              {query: '@metric:logins.count' },
+            ]
+          }
+        },
+        {type: 'date_histogram', field: '@timestamp', id: '4'}
+      ],
+    });
+
+    expect(query.aggs["2"].filters.filters["@metric:cpu"].query_string.query).to.be("@metric:cpu");
+    expect(query.aggs["2"].filters.filters["@metric:logins.count"].query_string.query).to.be("@metric:logins.count");
     expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
     expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
   });
   });
 
 
@@ -247,7 +258,6 @@ describe('ElasticQueryBuilder', function() {
       {key: 'key1', operator: '=', value: 'value1'}
       {key: 'key1', operator: '=', value: 'value1'}
     ]);
     ]);
 
 
-    expect(query.query.filtered.filter.bool.must[1].term["key1"]).to.be("value1");
+    expect(query.query.bool.must[2].term["key1"]).to.be("value1");
   });
   });
-
 });
 });

public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts

@@ -95,5 +95,11 @@ describe('ElasticQueryDef', function() {
               expect(queryDef.getMetricAggTypes(2).length).to.be(11);
               expect(queryDef.getMetricAggTypes(2).length).to.be(11);
           });
           });
       });
       });
+
+      describe('using esversion 5', function() {
+          it('should get pipeline aggs', function() {
+              expect(queryDef.getMetricAggTypes(5).length).to.be(11);
+          });
+      });
   });
   });
 });
 });