Commit 0960360b by Torkel Ödegaard

feat(elasticsearch): added support for index time based patterns, #1034

parent 8db47e33
...@@ -12,12 +12,16 @@ function (angular, config) { ...@@ -12,12 +12,16 @@ function (angular, config) {
$scope.httpConfigPartialSrc = 'app/features/org/partials/datasourceHttpConfig.html'; $scope.httpConfigPartialSrc = 'app/features/org/partials/datasourceHttpConfig.html';
var defaults = { var defaults = {name: '', type: 'graphite', url: '', access: 'proxy' };
name: '',
type: 'graphite', $scope.indexPatternTypes = [
url: '', {name: 'No pattern', value: undefined},
access: 'proxy' {name: 'Hourly', value: 'Hourly', example: '[logstash-]YYYY.MM.DD.HH'},
}; {name: 'Daily', value: 'Daily', example: '[logstash-]YYYY.MM.DD'},
{name: 'Weekly', value: 'Weekly', example: '[logstash-]GGGG.WW'},
{name: 'Monthly', value: 'Monthly', example: '[logstash-]YYYY.MM'},
{name: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY'},
];
$scope.init = function() { $scope.init = function() {
$scope.isNew = true; $scope.isNew = true;
...@@ -117,6 +121,11 @@ function (angular, config) { ...@@ -117,6 +121,11 @@ function (angular, config) {
} }
}; };
$scope.indexPatternTypeChanged = function() {
var def = _.findWhere($scope.indexPatternTypes, {value: $scope.current.jsonData.interval});
$scope.current.database = def.example || 'es-index-name';
};
$scope.init(); $scope.init();
}); });
......
...@@ -2,12 +2,13 @@ define([ ...@@ -2,12 +2,13 @@ define([
'angular', 'angular',
'lodash', 'lodash',
'moment', 'moment',
'kbn',
'./queryBuilder', './queryBuilder',
'./indexPattern', './indexPattern',
'./queryCtrl', './queryCtrl',
'./directives' './directives'
], ],
function (angular, _, moment, ElasticQueryBuilder, IndexPattern) { function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern) {
'use strict'; 'use strict';
var module = angular.module('grafana.services'); var module = angular.module('grafana.services');
...@@ -23,9 +24,9 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) { ...@@ -23,9 +24,9 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
this.indexPattern = new IndexPattern(datasource.index, datasource.jsonData.interval); this.indexPattern = new IndexPattern(datasource.index, datasource.jsonData.interval);
} }
ElasticDatasource.prototype._request = function(method, url, index, data) { ElasticDatasource.prototype._request = function(method, url, data) {
var options = { var options = {
url: this.url + "/" + index + url, url: this.url + "/" + url,
method: method, method: method,
data: data data: data
}; };
...@@ -41,14 +42,14 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) { ...@@ -41,14 +42,14 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
}; };
ElasticDatasource.prototype._get = function(url) { ElasticDatasource.prototype._get = function(url) {
return this._request('GET', url, this.indexPattern.getIndexForToday()) return this._request('GET', this.indexPattern.getIndexForToday() + url)
.then(function(results) { .then(function(results) {
return results.data; return results.data;
}); });
}; };
ElasticDatasource.prototype._post = function(url, data) { ElasticDatasource.prototype._post = function(url, data) {
return this._request('POST', url, this.index, data) return this._request('POST', url, data)
.then(function(results) { .then(function(results) {
return results.data; return results.data;
}); });
...@@ -76,7 +77,7 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) { ...@@ -76,7 +77,7 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
"size": 10000 "size": 10000
}; };
return this._request('POST', '/_search', annotation.index, data).then(function(results) { return this._request('POST', annotation.index + '/_search', data).then(function(results) {
var list = []; var list = [];
var hits = results.data.hits.hits; var hits = results.data.hits.hits;
...@@ -135,11 +136,20 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) { ...@@ -135,11 +136,20 @@ function (angular, _, moment, ElasticQueryBuilder, IndexPattern) {
}); });
}; };
ElasticDatasource.prototype.getQueryHeader = function(timeRange) {
var header = {search_type: "count", "ignore_unavailable": true};
var from = kbn.parseDate(timeRange.from);
var to = kbn.parseDate(timeRange.to);
header.index = this.indexPattern.getIndexList(from, to);
return angular.toJson(header);
};
ElasticDatasource.prototype.query = function(options) { ElasticDatasource.prototype.query = function(options) {
var queryBuilder = new ElasticQueryBuilder(); var queryBuilder = new ElasticQueryBuilder();
var header = '{"index":"' + this.index + '","search_type":"count","ignore_unavailable":true}';
var payload = ""; var payload = "";
var sentTargets = []; var sentTargets = [];
var header = this.getQueryHeader(options.range);
var timeFrom = this.translateTime(options.range.from); var timeFrom = this.translateTime(options.range.from);
var timeTo = this.translateTime(options.range.to); var timeTo = this.translateTime(options.range.to);
......
...@@ -10,6 +10,14 @@ function (_, moment) { ...@@ -10,6 +10,14 @@ function (_, moment) {
this.interval = interval; this.interval = interval;
} }
IndexPattern.intervalMap = {
"Hours": { startOf: 'hour', amount: 'hours'},
"Daily": { startOf: 'day', amount: 'days'},
"Weekly": { startOf: 'isoWeek', amount: 'weeks'},
"Monthly": { startOf: 'month', amount: 'months'},
"Yearly": { startOf: 'year', amount: 'years'},
};
IndexPattern.prototype.getIndexForToday = function() { IndexPattern.prototype.getIndexForToday = function() {
if (this.interval) { if (this.interval) {
return moment().format(this.pattern); return moment().format(this.pattern);
...@@ -19,6 +27,21 @@ function (_, moment) { ...@@ -19,6 +27,21 @@ function (_, moment) {
}; };
IndexPattern.prototype.getIndexList = function(from, to) { IndexPattern.prototype.getIndexList = function(from, to) {
if (!this.interval) {
return this.pattern;
}
var intervalInfo = IndexPattern.intervalMap[this.interval];
var start = moment(from).utc().startOf(intervalInfo.startOf);
var end = moment(to).utc().startOf(intervalInfo.startOf).valueOf();
var indexList = [];
while (start <= end) {
indexList.push(start.format(this.pattern));
start.add(1, intervalInfo.amount);
}
return indexList;
}; };
return IndexPattern; return IndexPattern;
......
...@@ -15,8 +15,7 @@ ...@@ -15,8 +15,7 @@
Pattern Pattern
</li> </li>
<li> <li>
<select class="input-medium tight-form-input" ng-model="current.jsonData.interval" <select class="input-medium tight-form-input" ng-model="current.jsonData.interval" ng-options="f.value as f.name for f in indexPatternTypes" ng-change="indexPatternTypeChanged()" ></select>
ng-options="f.value as f.name for f in [{name: 'No pattern', value: undefined}, {name: 'Hourly', value: 'hourly'}, {name: 'Daily', value: 'daily'}]" ></select>
</li> </li>
</ul> </ul>
<div class="clearfix"></div> <div class="clearfix"></div>
......
...@@ -8,7 +8,7 @@ define([ ...@@ -8,7 +8,7 @@ define([
describe('when getting index for today', function() { describe('when getting index for today', function() {
it('should return correct index name', function() { it('should return correct index name', function() {
var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'daily'); var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
var expected = 'asd-' + moment().format('YYYY.MM.DD'); var expected = 'asd-' + moment().format('YYYY.MM.DD');
expect(pattern.getIndexForToday()).to.be(expected); expect(pattern.getIndexForToday()).to.be(expected);
...@@ -17,16 +17,33 @@ define([ ...@@ -17,16 +17,33 @@ define([
describe('when getting index list for time range', function() { describe('when getting index list for time range', function() {
describe('no interval', function() {
it('should return correct index', function() {
var pattern = new IndexPattern('my-metrics');
var from = new Date(2015, 4, 30, 1, 2, 3);
var to = new Date(2015, 5, 1, 12, 5 , 6);
expect(pattern.getIndexList(from, to)).to.eql('my-metrics');
});
});
describe('daily', function() { describe('daily', function() {
it('should return correct index list', function() { it('should return correct index list', function() {
var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'daily'); var pattern = new IndexPattern('[asd-]YYYY.MM.DD', 'Daily');
var from = new Date(2015, 4, 29); var from = new Date(2015, 4, 30, 1, 2, 3);
var to = new Date(2015, 5, 1); var to = new Date(2015, 5, 1, 12, 5 , 6);
var expected = [
'asd-2015.05.29',
'asd-2015.05.30',
'asd-2015.05.31',
'asd-2015.06.01',
];
expect(pattern.getIndexList(from, to)).to.eql(expected);
});
expect(pattern.getIndexList(from, to)).to.be(['asd', 'asd2']);
}); });
})
}); });
......
define([ define([
'helpers', 'helpers',
'moment', 'moment',
'angular',
'plugins/datasource/elasticsearch/datasource', 'plugins/datasource/elasticsearch/datasource',
'aws-sdk', 'aws-sdk',
], function(helpers, moment) { ], function(helpers, moment, angular) {
'use strict'; 'use strict';
describe('ElasticDatasource', function() { describe('ElasticDatasource', function() {
...@@ -21,9 +22,9 @@ define([ ...@@ -21,9 +22,9 @@ define([
ctx.ds = new ctx.service({ ctx.ds = new ctx.service({
url: 'http://es.com', url: 'http://es.com',
index: '[asd-]YYYY.MM.DD', index: '[asd-]YYYY.MM.DD',
jsonData: { interval: 'daily' } jsonData: { interval: 'Daily' }
});
}); });
})
it('should translate index pattern to current day', function() { it('should translate index pattern to current day', function() {
var requestOptions; var requestOptions;
...@@ -38,7 +39,37 @@ define([ ...@@ -38,7 +39,37 @@ define([
var today = moment().format("YYYY.MM.DD"); var today = moment().format("YYYY.MM.DD");
expect(requestOptions.url).to.be("http://es.com/asd-" + today + '/_stats'); expect(requestOptions.url).to.be("http://es.com/asd-" + today + '/_stats');
}); });
});
describe('When issueing metric query with interval pattern', function() {
beforeEach(function() {
ctx.ds = new ctx.service({
url: 'http://es.com',
index: '[asd-]YYYY.MM.DD',
jsonData: { interval: 'Daily' }
});
});
it('should translate index pattern to current day', function() {
var requestOptions;
ctx.backendSrv.datasourceRequest = function(options) {
requestOptions = options;
return ctx.$q.when({data: {responses: []}});
};
ctx.ds.query({
range: {
from: new Date(2015, 4, 30, 10),
to: new Date(2015, 5, 1, 10)
},
targets: [{ bucketAggs: [], metrics: [] }]
});
ctx.$rootScope.$apply();
var parts = requestOptions.data.split('\n');
var header = angular.fromJson(parts[0]);
expect(header.index).to.eql(['asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01']);
});
}); });
describe('When processing es response', function() { describe('When processing es response', function() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment