feat(elasticsearch): added support for index time based patterns, #1034

public/app/features/org/datasourceEditCtrl.js

@@ -12,12 +12,16 @@ function (angular, config) {
 
 
     $scope.httpConfigPartialSrc = 'app/features/org/partials/datasourceHttpConfig.html';
     $scope.httpConfigPartialSrc = 'app/features/org/partials/datasourceHttpConfig.html';
 
 
-    var defaults = {
-      name: '',
-      type: 'graphite',
-      url: '',
-      access: 'proxy'
-    };
+    var defaults = {name: '', type: 'graphite', url: '', access: 'proxy' };
+
+    $scope.indexPatternTypes = [
+      {name: 'No pattern',  value: undefined},
+      {name: 'Hourly',      value: 'Hourly',  example: '[logstash-]YYYY.MM.DD.HH'},
+      {name: 'Daily',       value: 'Daily',   example: '[logstash-]YYYY.MM.DD'},
+      {name: 'Weekly',      value: 'Weekly',  example: '[logstash-]GGGG.WW'},
+      {name: 'Monthly',     value: 'Monthly', example: '[logstash-]YYYY.MM'},
+      {name: 'Yearly',      value: 'Yearly',  example: '[logstash-]YYYY'},
+    ];
 
 
     $scope.init = function() {
     $scope.init = function() {
       $scope.isNew = true;
       $scope.isNew = true;
@@ -117,6 +121,11 @@ function (angular, config) {
       }
       }
     };
     };
 
 
+    $scope.indexPatternTypeChanged = function() {
+      var def = _.findWhere($scope.indexPatternTypes, {value: $scope.current.jsonData.interval});
+      $scope.current.database = def.example || 'es-index-name';
+    };
+
     $scope.init();
     $scope.init();
 
 
   });
   });

public/app/plugins/datasource/elasticsearch/datasource.js

@@ -2,12 +2,13 @@ define([
   'angular',
   'angular',
   'lodash',
   'lodash',
   'moment',
   'moment',
+  'kbn',
   './queryBuilder',
   './queryBuilder',
   './indexPattern',
   './indexPattern',
   './queryCtrl',
   './queryCtrl',
   './directives'
   './directives'
 ],
 ],
-function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
+function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern) {
   'use strict';
   'use strict';
 
 
   var module = angular.module('grafana.services');
   var module = angular.module('grafana.services');
@@ -23,9 +24,9 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
       this.indexPattern = new IndexPattern(datasource.index, datasource.jsonData.interval);
       this.indexPattern = new IndexPattern(datasource.index, datasource.jsonData.interval);
     }
     }
 
 
-    ElasticDatasource.prototype._request = function(method, url, index, data) {
+    ElasticDatasource.prototype._request = function(method, url, data) {
       var options = {
       var options = {
-        url: this.url + "/" + index + url,
+        url: this.url + "/" + url,
         method: method,
         method: method,
         data: data
         data: data
       };
       };
@@ -41,14 +42,14 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
     };
     };
 
 
     ElasticDatasource.prototype._get = function(url) {
     ElasticDatasource.prototype._get = function(url) {
-      return this._request('GET', url, this.indexPattern.getIndexForToday())
+      return this._request('GET', this.indexPattern.getIndexForToday() + url)
         .then(function(results) {
         .then(function(results) {
           return results.data;
           return results.data;
         });
         });
     };
     };
 
 
     ElasticDatasource.prototype._post = function(url, data) {
     ElasticDatasource.prototype._post = function(url, data) {
-      return this._request('POST', url, this.index, data)
+      return this._request('POST', url, data)
         .then(function(results) {
         .then(function(results) {
           return results.data;
           return results.data;
         });
         });
@@ -76,7 +77,7 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
         "size": 10000
         "size": 10000
       };
       };
 
 
-      return this._request('POST', '/_search', annotation.index, data).then(function(results) {
+      return this._request('POST', annotation.index + '/_search', data).then(function(results) {
         var list = [];
         var list = [];
         var hits = results.data.hits.hits;
         var hits = results.data.hits.hits;
 
 
@@ -135,11 +136,20 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
       });
       });
     };
     };
 
 
+    ElasticDatasource.prototype.getQueryHeader = function(timeRange) {
+      var header = {search_type: "count", "ignore_unavailable": true};
+      var from = kbn.parseDate(timeRange.from);
+      var to = kbn.parseDate(timeRange.to);
+      header.index = this.indexPattern.getIndexList(from, to);
+      return angular.toJson(header);
+    };
+
     ElasticDatasource.prototype.query = function(options) {
     ElasticDatasource.prototype.query = function(options) {
       var queryBuilder = new ElasticQueryBuilder();
       var queryBuilder = new ElasticQueryBuilder();
-      var header = '{"index":"' + this.index + '","search_type":"count","ignore_unavailable":true}';
       var payload = "";
       var payload = "";
       var sentTargets = [];
       var sentTargets = [];
+
+      var header = this.getQueryHeader(options.range);
       var timeFrom = this.translateTime(options.range.from);
       var timeFrom = this.translateTime(options.range.from);
       var timeTo = this.translateTime(options.range.to);
       var timeTo = this.translateTime(options.range.to);
 
 

public/app/plugins/datasource/elasticsearch/indexPattern.js

@@ -10,6 +10,14 @@ function (_, moment) {
     this.interval = interval;
     this.interval = interval;
   }
   }
 
 
+  IndexPattern.intervalMap = {
+    "Hours":   { startOf: 'hour',     amount: 'hours'},
+    "Daily":   { startOf: 'day',      amount: 'days'},
+    "Weekly":  { startOf: 'isoWeek',  amount: 'weeks'},
+    "Monthly": { startOf: 'month',    amount: 'months'},
+    "Yearly":  { startOf: 'year',     amount: 'years'},
+  };
+
   IndexPattern.prototype.getIndexForToday = function() {
   IndexPattern.prototype.getIndexForToday = function() {
     if (this.interval) {
     if (this.interval) {
       return moment().format(this.pattern);
       return moment().format(this.pattern);
@@ -19,6 +27,21 @@ function (_, moment) {
   };
   };
 
 
   IndexPattern.prototype.getIndexList = function(from, to) {
   IndexPattern.prototype.getIndexList = function(from, to) {
+    if (!this.interval) {
+      return this.pattern;
+    }
+
+    var intervalInfo = IndexPattern.intervalMap[this.interval];
+    var start = moment(from).utc().startOf(intervalInfo.startOf);
+    var end = moment(to).utc().startOf(intervalInfo.startOf).valueOf();
+    var indexList = [];
+
+    while (start <= end) {
+      indexList.push(start.format(this.pattern));
+      start.add(1, intervalInfo.amount);
+    }
+
+    return indexList;
   };
   };
 
 
   return IndexPattern;
   return IndexPattern;

public/app/plugins/datasource/elasticsearch/partials/config.html

@@ -15,8 +15,7 @@
 			Pattern
 			Pattern
 		</li>
 		</li>
 		<li>
 		<li>
-			<select class="input-medium tight-form-input" ng-model="current.jsonData.interval"
-				ng-options="f.value as f.name for f in [{name: 'No pattern', value: undefined}, {name: 'Hourly', value: 'hourly'}, {name: 'Daily', value: 'daily'}]" ></select>
+			<select class="input-medium tight-form-input" ng-model="current.jsonData.interval" ng-options="f.value as f.name for f in indexPatternTypes" ng-change="indexPatternTypeChanged()" ></select>
 		</li>
 		</li>
 	</ul>
 	</ul>
 	<div class="clearfix"></div>
 	<div class="clearfix"></div>

public/test/specs/elasticsearch-indexPattern-specs.js

@@ -8,7 +8,7 @@ define([
 
 
     describe('when getting index for today', function() {
     describe('when getting index for today', function() {
       it('should return correct index name', function() {
       it('should return correct index name', function() {
-        var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'daily');
+        var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
         var expected = 'asd-' + moment().format('YYYY.MM.DD');
         var expected = 'asd-' + moment().format('YYYY.MM.DD');
 
 
         expect(pattern.getIndexForToday()).to.be(expected);
         expect(pattern.getIndexForToday()).to.be(expected);
@@ -17,16 +17,33 @@ define([
 
 
     describe('when getting index list for time range', function() {
     describe('when getting index list for time range', function() {
 
 
+      describe('no interval', function() {
+        it('should return correct index', function() {
+          var pattern = new IndexPattern('my-metrics');
+          var from = new Date(2015, 4, 30, 1, 2, 3);
+          var to = new Date(2015, 5, 1, 12, 5 , 6);
+          expect(pattern.getIndexList(from, to)).to.eql('my-metrics');
+        });
+      });
+
       describe('daily', function() {
       describe('daily', function() {
 
 
         it('should return correct index list', function() {
         it('should return correct index list', function() {
-          var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'daily');
-          var from = new Date(2015, 4, 29);
-          var to = new Date(2015, 5, 1);
-
-          expect(pattern.getIndexList(from, to)).to.be(['asd', 'asd2']);
+          var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
+          var from = new Date(2015, 4, 30, 1, 2, 3);
+          var to = new Date(2015, 5, 1, 12, 5 , 6);
+
+          var expected =  [
+            'asd-2015.05.29',
+            'asd-2015.05.30',
+            'asd-2015.05.31',
+            'asd-2015.06.01',
+          ];
+
+          expect(pattern.getIndexList(from, to)).to.eql(expected);
         });
         });
-      })
+
+      });
 
 
     });
     });
 
 

public/test/specs/elasticsearch-specs.js

 define([
 define([
   'helpers',
   'helpers',
   'moment',
   'moment',
+  'angular',
   'plugins/datasource/elasticsearch/datasource',
   'plugins/datasource/elasticsearch/datasource',
   'aws-sdk',
   'aws-sdk',
-], function(helpers, moment) {
+], function(helpers, moment, angular) {
   'use strict';
   'use strict';
 
 
   describe('ElasticDatasource', function() {
   describe('ElasticDatasource', function() {
@@ -21,9 +22,9 @@ define([
         ctx.ds = new ctx.service({
         ctx.ds = new ctx.service({
           url: 'http://es.com',
           url: 'http://es.com',
           index: '[asd-]YYYY.MM.DD',
           index: '[asd-]YYYY.MM.DD',
-          jsonData: { interval: 'daily' }
+          jsonData: { interval: 'Daily' }
         });
         });
-      })
+      });
 
 
       it('should translate index pattern to current day', function() {
       it('should translate index pattern to current day', function() {
         var requestOptions;
         var requestOptions;
@@ -38,7 +39,37 @@ define([
         var today = moment().format("YYYY.MM.DD");
         var today = moment().format("YYYY.MM.DD");
         expect(requestOptions.url).to.be("http://es.com/asd-" + today + '/_stats');
         expect(requestOptions.url).to.be("http://es.com/asd-" + today + '/_stats');
       });
       });
+    });
+
+    describe('When issueing metric query with interval pattern', function() {
+      beforeEach(function() {
+        ctx.ds = new ctx.service({
+          url: 'http://es.com',
+          index: '[asd-]YYYY.MM.DD',
+          jsonData: { interval: 'Daily' }
+        });
+      });
+
+      it('should translate index pattern to current day', function() {
+        var requestOptions;
+        ctx.backendSrv.datasourceRequest = function(options) {
+          requestOptions = options;
+          return ctx.$q.when({data: {responses: []}});
+        };
 
 
+        ctx.ds.query({
+          range: {
+            from: new Date(2015, 4, 30, 10),
+            to: new Date(2015, 5, 1, 10)
+          },
+          targets: [{ bucketAggs: [], metrics: [] }]
+        });
+
+        ctx.$rootScope.$apply();
+        var parts = requestOptions.data.split('\n');
+        var header = angular.fromJson(parts[0]);
+        expect(header.index).to.eql(['asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01']);
+      });
     });
     });
 
 
     describe('When processing es response', function() {
     describe('When processing es response', function() {