Merge branch 'master' of https://github.com/lpic10/grafana into lpic10-master

public/app/plugins/datasource/elasticsearch/config_ctrl.ts

@@ -24,6 +24,7 @@ export class ElasticConfigCtrl {
   esVersions = [
   esVersions = [
     {name: '1.x', value: 1},
     {name: '1.x', value: 1},
     {name: '2.x', value: 2},
     {name: '2.x', value: 2},
+    {name: '5.x', value: 5},
   ];
   ];
 
 
   indexPatternTypeChanged() {
   indexPatternTypeChanged() {

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -88,14 +88,18 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
       }
       }
 
 
       var queryInterpolated = templateSrv.replace(queryString, {}, 'lucene');
       var queryInterpolated = templateSrv.replace(queryString, {}, 'lucene');
-      var filter = { "bool": { "must": [{ "range": range }] } };
-      var query = { "bool": { "should": [{ "query_string": { "query": queryInterpolated } }] } };
+      var query = { "bool": { "must": [{ "range": range }, { "query_string": { "query": queryInterpolated } }] }};
+
       var data = {
       var data = {
-        "fields": [timeField, "_source"],
-        "query" : { "filtered": { "query" : query, "filter": filter } },
+        "query" : query,
         "size": 10000
         "size": 10000
       };
       };
 
 
+      // fields field not supported on ES 5.x
+      if (this.esVersion < 5) {
+        data["fields"] = [timeField, "_source"];
+      }
+
       var header = {search_type: "query_then_fetch", "ignore_unavailable": true};
       var header = {search_type: "query_then_fetch", "ignore_unavailable": true};
 
 
       // old elastic annotations had index specified on them
       // old elastic annotations had index specified on them
@@ -133,12 +137,13 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
 
 
         for (var i = 0; i < hits.length; i++) {
         for (var i = 0; i < hits.length; i++) {
           var source = hits[i]._source;
           var source = hits[i]._source;
-          var fields = hits[i].fields;
           var time = source[timeField];
           var time = source[timeField];
-
+          if (typeof hits[i].fields !== 'undefined') {
+            var fields = hits[i].fields;
             if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) {
             if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) {
               time = fields[timeField];
               time = fields[timeField];
             }
             }
+          }
 
 
           var event = {
           var event = {
             annotation: annotation,
             annotation: annotation,
@@ -194,7 +199,7 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
         luceneQuery = luceneQuery.substr(1, luceneQuery.length - 2);
         luceneQuery = luceneQuery.substr(1, luceneQuery.length - 2);
         esQuery = esQuery.replace("$lucene_query", luceneQuery);
         esQuery = esQuery.replace("$lucene_query", luceneQuery);
 
 
-        var searchType = queryObj.size === 0 ? 'count' : 'query_then_fetch';
+        var searchType = (queryObj.size === 0 && this.esVersion < 5) ? 'count' : 'query_then_fetch';
         var header = this.getQueryHeader(searchType, options.range.from, options.range.to);
         var header = this.getQueryHeader(searchType, options.range.from, options.range.to);
         payload +=  header + '\n';
         payload +=  header + '\n';
 
 
@@ -277,14 +282,15 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
 
 
     this.getTerms = function(queryDef) {
     this.getTerms = function(queryDef) {
       var range = timeSrv.timeRange();
       var range = timeSrv.timeRange();
-      var header = this.getQueryHeader('count', range.from, range.to);
+      var searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count' ;
+      var header = this.getQueryHeader(searchType, range.from, range.to);
       var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef));
       var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef));
 
 
       esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf());
       esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf());
       esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
       esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
       esQuery = header + '\n' + esQuery + '\n';
       esQuery = header + '\n' + esQuery + '\n';
 
 
-      return this._post('_msearch?search_type=count', esQuery).then(function(res) {
+      return this._post('_msearch?search_type=' + searchType, esQuery).then(function(res) {
         if (!res.responses[0].aggregations) {
         if (!res.responses[0].aggregations) {
           return [];
           return [];
         }
         }

public/app/plugins/datasource/elasticsearch/query_builder.js

@@ -28,7 +28,8 @@ function (queryDef) {
       return queryNode;
       return queryNode;
     }
     }
 
 
-    queryNode.terms.size = parseInt(aggDef.settings.size, 10);
+    queryNode.terms.size = parseInt(aggDef.settings.size, 10) === 0 ? 500 : parseInt(aggDef.settings.size, 10);
+
     if (aggDef.settings.orderBy !== void 0) {
     if (aggDef.settings.orderBy !== void 0) {
       queryNode.terms.order = {};
       queryNode.terms.order = {};
       queryNode.terms.order[aggDef.settings.orderBy] = aggDef.settings.order;
       queryNode.terms.order[aggDef.settings.orderBy] = aggDef.settings.order;
@@ -75,6 +76,14 @@ function (queryDef) {
 
 
     for (var i = 0; i < aggDef.settings.filters.length; i++) {
     for (var i = 0; i < aggDef.settings.filters.length; i++) {
       var query = aggDef.settings.filters[i].query;
       var query = aggDef.settings.filters[i].query;
+      if (this.esVersion >= 5) {
+        filterObj[query] = {
+          query_string: {
+            query: query,
+            analyze_wildcard: true
+          }
+        };
+      } else {
         filterObj[query] = {
         filterObj[query] = {
           query: {
           query: {
             query_string: {
             query_string: {
@@ -84,6 +93,7 @@ function (queryDef) {
           }
           }
         };
         };
       }
       }
+    }
 
 
     return filterObj;
     return filterObj;
   };
   };
@@ -92,7 +102,12 @@ function (queryDef) {
     query.size = 500;
     query.size = 500;
     query.sort = {};
     query.sort = {};
     query.sort[this.timeField] = {order: 'desc', unmapped_type: 'boolean'};
     query.sort[this.timeField] = {order: 'desc', unmapped_type: 'boolean'};
+
+    // fields field not supported on ES 5.x
+    if (this.esVersion < 5) {
       query.fields = ["*", "_source"];
       query.fields = ["*", "_source"];
+    }
+
     query.script_fields = {},
     query.script_fields = {},
     query.fielddata_fields = [this.timeField];
     query.fielddata_fields = [this.timeField];
     return query;
     return query;
@@ -103,8 +118,13 @@ function (queryDef) {
       return;
       return;
     }
     }
 
 
-    var i, filter, condition;
-    var must = query.query.filtered.filter.bool.must;
+    var i, filter, condition, must;
+
+    if (this.esVersion >= 5) {
+      must = query.query.bool.must;
+    } else {
+      must = query.query.filtered.filter.bool.must;
+    }
 
 
     for (i = 0; i < adhocFilters.length; i++) {
     for (i = 0; i < adhocFilters.length; i++) {
       filter = adhocFilters[i];
       filter = adhocFilters[i];
@@ -122,7 +142,25 @@ function (queryDef) {
     target.timeField =  this.timeField;
     target.timeField =  this.timeField;
 
 
     var i, nestedAggs, metric;
     var i, nestedAggs, metric;
-    var query = {
+    var query = {};
+    if (this.esVersion >= 5) {
+      query = {
+        "size": 0,
+        "query": {
+          "bool": {
+            "must": [
+              {"range": this.getRangeFilter()},
+              {"query_string": {
+                "analyze_wildcard": true,
+                "query": '$lucene_query'
+                }
+              }
+            ]
+          }
+        }
+      };
+    } else {
+      query = {
         "size": 0,
         "size": 0,
         "query": {
         "query": {
           "filtered": {
           "filtered": {
@@ -140,6 +178,7 @@ function (queryDef) {
           }
           }
         }
         }
       };
       };
+    }
 
 
     this.addAdhocFilters(query, adhocFilters);
     this.addAdhocFilters(query, adhocFilters);
 
 
@@ -217,7 +256,29 @@ function (queryDef) {
   };
   };
 
 
   ElasticQueryBuilder.prototype.getTermsQuery = function(queryDef) {
   ElasticQueryBuilder.prototype.getTermsQuery = function(queryDef) {
-    var query = {
+    var query;
+
+    if (this.esVersion >= 5) {
+      query = {
+        "size": 0,
+        "query": {
+          "bool": {
+            "must": [{"range": this.getRangeFilter()}]
+          }
+        }
+      };
+
+      if (queryDef.query) {
+        query.query.bool.must.push({
+          "query_string": {
+            "analyze_wildcard": true,
+            "query": queryDef.query,
+          }
+        });
+      }
+
+    } else {
+      query = {
         "size": 0,
         "size": 0,
         "query": {
         "query": {
           "filtered": {
           "filtered": {
@@ -238,12 +299,13 @@ function (queryDef) {
           }
           }
         };
         };
       }
       }
+    }
 
 
     query.aggs =  {
     query.aggs =  {
       "1": {
       "1": {
         "terms": {
         "terms": {
           "field": queryDef.field,
           "field": queryDef.field,
-          "size": 0,
+          "size": 500,
           "order": {
           "order": {
             "_term": "asc"
             "_term": "asc"
           }
           }

public/app/plugins/datasource/elasticsearch/specs/datasource_specs.ts

@@ -209,4 +209,79 @@ describe('ElasticDatasource', function() {
     });
     });
   });
   });
 
 
+  describe('When issuing aggregation query on es5.x', function() {
+    var requestOptions, parts, header;
+
+    beforeEach(function() {
+      createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}});
+
+      ctx.backendSrv.datasourceRequest = function(options) {
+        requestOptions = options;
+        return ctx.$q.when({data: {responses: []}});
+      };
+
+      ctx.ds.query({
+        range: { from: moment([2015, 4, 30, 10]), to: moment([2015, 5, 1, 10]) },
+        targets: [{
+            bucketAggs: [
+                {type: 'date_histogram', field: '@timestamp', id: '2'}
+            ],
+            metrics: [
+                {type: 'count'}], query: 'test' }
+            ]
+      });
+
+      ctx.$rootScope.$apply();
+      parts = requestOptions.data.split('\n');
+      header = angular.fromJson(parts[0]);
+    });
+
+    it('should not set search type to count', function() {
+      expect(header.search_type).to.not.eql('count');
+    });
+
+    it('should set size to 0', function() {
+      var body = angular.fromJson(parts[1]);
+      expect(body.size).to.be(0);
+    });
+
+  });
+
+  describe('When issuing metricFind query on es5.x', function() {
+    var requestOptions, parts, header, body;
+
+    beforeEach(function() {
+      createDatasource({url: 'http://es.com', index: 'test', jsonData: {esVersion: '5'}});
+
+      ctx.backendSrv.datasourceRequest = function(options) {
+        requestOptions = options;
+        return ctx.$q.when({
+            data: {
+                responses: [{aggregations: {"1": [{buckets: {text: 'test', value: '1'}}]}}]
+            }
+        });
+      };
+
+      ctx.ds.metricFindQuery('{"find": "terms", "field": "test"}');
+      ctx.$rootScope.$apply();
+
+      parts = requestOptions.data.split('\n');
+      header = angular.fromJson(parts[0]);
+      body = angular.fromJson(parts[1]);
+    });
+
+    it('should not set search type to count', function() {
+      expect(header.search_type).to.not.eql('count');
+    });
+
+    it('should set size to 0', function() {
+      expect(body.size).to.be(0);
+    });
+
+    it('should not set terms aggregation size to 0', function() {
+      expect(body['aggs']['1']['terms'].size).to.not.be(0);
+    });
+  });
+
 });
 });
+

public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts

@@ -20,6 +20,22 @@ describe('ElasticQueryBuilder', function() {
     expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom");
     expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom");
   });
   });
 
 
+  it('with defaults on es5.x', function() {
+    var builder_5x = new ElasticQueryBuilder({
+      timeField: '@timestamp',
+      esVersion: 5
+    });
+
+    var query = builder_5x.build({
+      metrics: [{type: 'Count', id: '0'}],
+      timeField: '@timestamp',
+      bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '1'}],
+    });
+
+    expect(query.query.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
+    expect(query.aggs["1"].date_histogram.extended_bounds.min).to.be("$timeFrom");
+  });
+
   it('with multiple bucket aggs', function() {
   it('with multiple bucket aggs', function() {
     var query = builder.build({
     var query = builder.build({
       metrics: [{type: 'count', id: '1'}],
       metrics: [{type: 'count', id: '1'}],
@@ -143,6 +159,34 @@ describe('ElasticQueryBuilder', function() {
     expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
     expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
   });
   });
 
 
+  it('with filters aggs on es5.x', function() {
+    var builder_5x = new ElasticQueryBuilder({
+      timeField: '@timestamp',
+      esVersion: 5
+    });
+    var query = builder_5x.build({
+      metrics: [{type: 'count', id: '1'}],
+      timeField: '@timestamp',
+      bucketAggs: [
+        {
+          id: '2',
+          type: 'filters',
+          settings: {
+            filters: [
+              {query: '@metric:cpu' },
+              {query: '@metric:logins.count' },
+            ]
+          }
+        },
+        {type: 'date_histogram', field: '@timestamp', id: '4'}
+      ],
+    });
+
+    expect(query.aggs["2"].filters.filters["@metric:cpu"].query_string.query).to.be("@metric:cpu");
+    expect(query.aggs["2"].filters.filters["@metric:logins.count"].query_string.query).to.be("@metric:logins.count");
+    expect(query.aggs["2"].aggs["4"].date_histogram.field).to.be("@timestamp");
+  });
+
   it('with raw_document metric', function() {
   it('with raw_document metric', function() {
     var query = builder.build({
     var query = builder.build({
       metrics: [{type: 'raw_document', id: '1'}],
       metrics: [{type: 'raw_document', id: '1'}],

public/app/plugins/datasource/elasticsearch/specs/query_def_specs.ts

@@ -95,5 +95,11 @@ describe('ElasticQueryDef', function() {
               expect(queryDef.getMetricAggTypes(2).length).to.be(11);
               expect(queryDef.getMetricAggTypes(2).length).to.be(11);
           });
           });
       });
       });
+
+      describe('using esversion 5', function() {
+          it('should get pipeline aggs', function() {
+              expect(queryDef.getMetricAggTypes(5).length).to.be(11);
+          });
+      });
   });
   });
 });
 });