Commit cc1e3d01 by Torkel Ödegaard

feat(elasticsearch): groundwork for a much more sophisticated elasticsearch query editor

parent 9daa3997
...@@ -174,39 +174,32 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) { ...@@ -174,39 +174,32 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
// This is quite complex // This is quite complex
// neeed to recurise down the nested buckets to build series // neeed to recurise down the nested buckets to build series
ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName, parentTime) { ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName) {
var groupBy = target.groupByFields[level]; var seriesName, value, metric, i, y, bucket, childBucket;
var seriesName, time, value, select, i, y, bucket;
for (i = 0; i < buckets.length; i++) { for (i = 0; i < buckets.length; i++) {
bucket = buckets[i]; bucket = buckets[i];
childBucket = bucket['b' + level];
if (groupBy) { if (childBucket && childBucket.buckets) {
seriesName = level > 0 ? parentName + ' ' + bucket.key : parentName; seriesName = parentName + ' ' + bucket.key;
time = parentTime || bucket.key; this._processBuckets(childBucket.buckets, target, series, level+1, seriesName);
this._processBuckets(bucket[groupBy.field].buckets, target, series, level+1, seriesName, time);
} else { } else {
for (y = 0; y < target.select.length; y++) { for (y = 0; y < target.metrics.length; y++) {
select = target.select[y]; metric = target.metrics[y];
seriesName = parentName; seriesName = parentName;
if (level > 0) { if (metric.field) {
seriesName += ' ' + bucket.key; seriesName += ' ' + metric.field + ' ' + metric.agg;
} else { value = bucket['m' + y.toString()].value;
parentTime = bucket.key;
}
if (select.field) {
seriesName += ' ' + select.field + ' ' + select.agg;
value = bucket[y.toString()].value;
} else { } else {
seriesName += ' count'; seriesName += ' count';
value = bucket.doc_count; value = bucket.doc_count;
} }
var serie = series[seriesName] = series[seriesName] || {target: seriesName, datapoints: []}; var serie = series[seriesName] = series[seriesName] || {target: seriesName, datapoints: []};
serie.datapoints.push([value, parentTime]); serie.datapoints.push([value, bucket.key]);
} }
} }
} }
...@@ -221,11 +214,11 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) { ...@@ -221,11 +214,11 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
throw { message: response.error }; throw { message: response.error };
} }
var buckets = response.aggregations.histogram.buckets; var buckets = response.aggregations["b0"].buckets;
var target = targets[i]; var target = targets[i];
var querySeries = {}; var querySeries = {};
this._processBuckets(buckets, target, querySeries, 0, target.refId); this._processBuckets(buckets, target, querySeries, 1, target.refId);
for (var prop in querySeries) { for (var prop in querySeries) {
if (querySeries.hasOwnProperty(prop)) { if (querySeries.hasOwnProperty(prop)) {
......
define([ define([
'angular', 'angular',
'./queryComponent',
], ],
function (angular) { function (angular) {
'use strict'; 'use strict';
......
...@@ -51,7 +51,7 @@ ...@@ -51,7 +51,7 @@
Time field Time field
</li> </li>
<li> <li>
<metric-segment segment="timeSegment" get-alt-segments="getTimeFields()" on-value-changed="timeFieldChanged()"></metric-segment> <metric-segment segment="timeSegment" get-alt-segments="getFields()" on-value-changed="timeFieldChanged()"></metric-segment>
</li> </li>
</ul> </ul>
...@@ -62,28 +62,39 @@ ...@@ -62,28 +62,39 @@
</div> </div>
</div> </div>
<div class="tight-form" ng-hide="target.rawQuery"> <div ng-hide="target.rawQuery">
<ul class="tight-form-list"> <div class="tight-form">
<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;"> <ul class="tight-form-list">
Select <li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
</li> Metrics
<li ng-repeat="segment in selectSegments"> </li>
<metric-segment segment="segment" get-alt-segments="getSelectSegments(segment, $index)" on-value-changed="selectChanged(segment, $index)"></metric-segment> <li ng-repeat="segment in selectSegments">
</li> <metric-segment segment="segment" get-alt-segments="getSelectSegments(segment, $index)" on-value-changed="selectChanged(segment, $index)"></metric-segment>
</ul> </li>
</ul>
<div class="clearfix"></div> <div class="clearfix"></div>
</div> </div>
<div class="tight-form" ng-hide="target.rawQuery"> <div class="tight-form" ng-repeat="agg in target.bucketAggs">
<ul class="tight-form-list"> <ul class="tight-form-list">
<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;"> <li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
Group by <span ng-show="$first">Group by</span>
</li> <span ng-show="!$first">Then by</span>
<li ng-repeat="segment in groupBySegments"> </li>
<metric-segment segment="segment" get-alt-segments="getGroupByFields(segment, $index)" on-value-changed="groupByChanged(segment, $index)"></metric-segment> <li>
</li> <elastic-query-component model="agg" get-fields="getFields()" on-change="queryUpdated()"></elastic-query-component>
</ul> </li>
<div class="clearfix"></div> </ul>
</div>
<ul class="tight-form-list pull-right">
<li class="tight-form-item" ng-if="$index === 0">
<a class="pointer" ng-click="addBucketAgg()"><i class="fa fa-plus"></i></a>
</li>
<li class="tight-form-item" ng-if="!$last">
<a class="pointer" ng-click="removeBucketAgg(agg, $index)"><i class="fa fa-minus"></i></a>
</li>
</ul>
<div class="clearfix"></div>
</div>
</div>
...@@ -17,6 +17,7 @@ function (angular) { ...@@ -17,6 +17,7 @@ function (angular) {
return angular.fromJson(target.rawQuery); return angular.fromJson(target.rawQuery);
} }
var i, nestedAggs;
var query = { var query = {
"size": 0, "size": 0,
"query": { "query": {
...@@ -36,43 +37,42 @@ function (angular) { ...@@ -36,43 +37,42 @@ function (angular) {
} }
}; };
query.aggs = { nestedAggs = query;
"histogram": {
"date_histogram": {
"interval": target.interval || "$interval",
"field": target.timeField,
"min_doc_count": 0,
"extended_bounds": {
"min": "$timeFrom",
"max": "$timeTo"
}
}
},
};
var nestedAggs = query.aggs.histogram;
var i;
target.groupByFields = target.groupByFields || []; for (i = 0; i < target.bucketAggs.length; i++) {
var aggDef = target.bucketAggs[i];
var esAgg = {};
for (i = 0; i < target.groupByFields.length; i++) { switch(aggDef.type) {
var field = target.groupByFields[i].field; case 'date_histogram': {
var aggs = {terms: {field: field}}; esAgg["date_histogram"] = {
"interval": target.interval || "$interval",
"field": aggDef.field,
"min_doc_count": 0,
"extended_bounds": { "min": "$timeFrom", "max": "$timeTo" }
};
break;
}
case 'terms': {
esAgg["terms"] = { "field": aggDef.field };
break;
}
}
nestedAggs.aggs = {}; nestedAggs.aggs = {};
nestedAggs.aggs[field] = aggs; nestedAggs.aggs['b' + i] = esAgg;
nestedAggs = aggs; nestedAggs = esAgg;
} }
nestedAggs.aggs = {}; nestedAggs.aggs = {};
for (i = 0; i < target.select.length; i++) { for (i = 0; i < target.metrics.length; i++) {
var select = target.select[i]; var metric = target.metrics[i];
if (select.field) { if (metric.field) {
var aggField = {}; var aggField = {};
aggField[select.agg] = {field: select.field}; aggField[metric.agg] = {field: metric.field};
nestedAggs.aggs[i.toString()] = aggField; nestedAggs.aggs['m' + i] = aggField;
} }
} }
......
define([
'angular',
'lodash',
'jquery',
],
function (angular, _, $) {
'use strict';
angular
.module('grafana.directives')
.directive('elasticQueryComponent', function($compile, uiSegmentSrv, $q) {
//var linkTemplate = '<a class="tight-form-item tabindex="1" ng-bind-html="textRep"></a>';
/* jshint maxlen:false */
var template1 = '<metric-segment segment="typeSegment" get-alt-segments="getBucketAggTypes()" on-value-changed="bucketAggTypeChanged()"></metric-segment>';
/* jshint maxlen:false */
var template2 = '<metric-segment segment="fieldSegment" get-alt-segments="getFields()" on-value-changed="fieldChanged()"></metric-segment>';
return {
restrict: 'E',
scope: {
model: "=",
onChange: "&",
getFields: "&",
},
link: function postLink($scope, elem) {
$scope.getBucketAggTypes = function() {
return $q.when([
uiSegmentSrv.newSegment({value: 'terms'}),
uiSegmentSrv.newSegment({value: 'date_histogram'}),
]);
};
$scope.fieldChanged = function() {
$scope.model.field = $scope.fieldSegment.value;
$scope.onChange();
};
$scope.bucketAggTypeChanged = function() {
$scope.model.type = $scope.typeSegment.value;
$scope.onChange();
};
function addElementsAndCompile() {
var $html = $(template1 + template2);
$scope.fieldSegment = uiSegmentSrv.newSegment($scope.model.field);
$scope.typeSegment = uiSegmentSrv.newSegment($scope.model.type);
$html.appendTo(elem);
$compile(elem.contents())($scope);
}
addElementsAndCompile();
}
};
});
});
...@@ -15,17 +15,22 @@ function (angular, _, ElasticQueryBuilder) { ...@@ -15,17 +15,22 @@ function (angular, _, ElasticQueryBuilder) {
if (!target) { return; } if (!target) { return; }
target.timeField = target.timeField || '@timestamp'; target.timeField = target.timeField || '@timestamp';
target.select = target.select || [{ agg: 'count' }]; target.metrics = target.metrics || [{ agg: 'count' }];
target.groupByFields = target.groupByFields || []; target.bucketAggs = target.bucketAggs || [];
target.bucketAggs = [
{
type: 'terms',
field: '@hostname'
},
{
type: 'date_histogram',
field: '@timestamp'
},
];
$scope.timeSegment = uiSegmentSrv.newSegment(target.timeField); $scope.timeSegment = uiSegmentSrv.newSegment(target.timeField);
$scope.groupBySegments = _.map(target.groupByFields, function(group) {
return uiSegmentSrv.newSegment(group.field);
});
$scope.initSelectSegments(); $scope.initSelectSegments();
$scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
$scope.removeSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove select --'}); $scope.removeSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove select --'});
$scope.resetSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- reset --'}); $scope.resetSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- reset --'});
$scope.removeGroupBySegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove group by --'}); $scope.removeGroupBySegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove group by --'});
...@@ -36,7 +41,7 @@ function (angular, _, ElasticQueryBuilder) { ...@@ -36,7 +41,7 @@ function (angular, _, ElasticQueryBuilder) {
$scope.initSelectSegments = function() { $scope.initSelectSegments = function() {
$scope.selectSegments = []; $scope.selectSegments = [];
_.each($scope.target.select, function(select) { _.each($scope.target.metrics, function(select) {
if ($scope.selectSegments.length > 0) { if ($scope.selectSegments.length > 0) {
$scope.selectSegments.push(uiSegmentSrv.newCondition(" and ")); $scope.selectSegments.push(uiSegmentSrv.newCondition(" and "));
} }
...@@ -55,9 +60,10 @@ function (angular, _, ElasticQueryBuilder) { ...@@ -55,9 +60,10 @@ function (angular, _, ElasticQueryBuilder) {
if (segment.type === 'agg' || segment.type === 'plus-button') { if (segment.type === 'agg' || segment.type === 'plus-button') {
var options = [ var options = [
uiSegmentSrv.newSegment({value: 'count', type: 'agg'}), uiSegmentSrv.newSegment({value: 'count', type: 'agg'}),
uiSegmentSrv.newSegment({value: 'avg', type: 'agg', reqField: true}),
uiSegmentSrv.newSegment({value: 'sum', type: 'agg', reqField: true}),
uiSegmentSrv.newSegment({value: 'min', type: 'agg', reqField: true}), uiSegmentSrv.newSegment({value: 'min', type: 'agg', reqField: true}),
uiSegmentSrv.newSegment({value: 'max', type: 'agg', reqField: true}), uiSegmentSrv.newSegment({value: 'max', type: 'agg', reqField: true}),
uiSegmentSrv.newSegment({value: 'avg', type: 'agg', reqField: true}),
]; ];
// if we have other selects and this is not a plus button add remove option // if we have other selects and this is not a plus button add remove option
if (segment.type !== 'plus-button' && $scope.selectSegments.length > 3) { if (segment.type !== 'plus-button' && $scope.selectSegments.length > 3) {
...@@ -78,7 +84,7 @@ function (angular, _, ElasticQueryBuilder) { ...@@ -78,7 +84,7 @@ function (angular, _, ElasticQueryBuilder) {
$scope.selectChanged = function(segment, index) { $scope.selectChanged = function(segment, index) {
// reset // reset
if (segment.value === $scope.resetSelectSegment.value) { if (segment.value === $scope.resetSelectSegment.value) {
$scope.target.select = [{ agg: 'count' }]; $scope.target.metrics = [{ agg: 'count' }];
$scope.initSelectSegments(); $scope.initSelectSegments();
$scope.queryUpdated(); $scope.queryUpdated();
return; return;
...@@ -125,7 +131,7 @@ function (angular, _, ElasticQueryBuilder) { ...@@ -125,7 +131,7 @@ function (angular, _, ElasticQueryBuilder) {
}; };
$scope.rebuildTargetSelects = function() { $scope.rebuildTargetSelects = function() {
$scope.target.select = []; $scope.target.metrics = [];
for (var i = 0; i < $scope.selectSegments.length; i++) { for (var i = 0; i < $scope.selectSegments.length; i++) {
var segment = $scope.selectSegments[i]; var segment = $scope.selectSegments[i];
var select = {agg: segment.value }; var select = {agg: segment.value };
...@@ -138,7 +144,7 @@ function (angular, _, ElasticQueryBuilder) { ...@@ -138,7 +144,7 @@ function (angular, _, ElasticQueryBuilder) {
} }
if (select.field === 'select field') { continue; } if (select.field === 'select field') { continue; }
$scope.target.select.push(select); $scope.target.metrics.push(select);
} }
}; };
...@@ -154,7 +160,7 @@ function (angular, _, ElasticQueryBuilder) { ...@@ -154,7 +160,7 @@ function (angular, _, ElasticQueryBuilder) {
.then(null, $scope.handleQueryError); .then(null, $scope.handleQueryError);
}; };
$scope.getTimeFields = function() { $scope.getFields = function() {
return $scope.datasource.metricFindQuery('fields()') return $scope.datasource.metricFindQuery('fields()')
.then($scope.transformToSegments(false)) .then($scope.transformToSegments(false))
.then(null, $scope.handleQueryError); .then(null, $scope.handleQueryError);
...@@ -165,22 +171,20 @@ function (angular, _, ElasticQueryBuilder) { ...@@ -165,22 +171,20 @@ function (angular, _, ElasticQueryBuilder) {
$scope.queryUpdated(); $scope.queryUpdated();
}; };
$scope.groupByChanged = function(segment, index) { $scope.addBucketAgg = function() {
if (segment.value === $scope.removeGroupBySegment.value) { // if last is date histogram add it before
$scope.target.groupByFields.splice(index, 1); var lastBucket = $scope.target.bucketAggs[$scope.target.bucketAggs.length - 1];
$scope.groupBySegments.splice(index, 1); var addIndex = $scope.target.bucketAggs.length - 1;
$scope.queryUpdated();
return;
}
if (index === $scope.groupBySegments.length-1) { if (lastBucket && lastBucket.type === 'date_histogram') {
$scope.groupBySegments.push(uiSegmentSrv.newPlusButton()); addIndex - 1;
} }
segment.type = 'group-by-key'; $scope.target.bucketAggs.splice(addIndex, 0, {type: "terms", field: "select field" });
segment.fake = false; };
$scope.target.groupByFields[index] = {field: segment.value}; $scope.removeBucketAgg = function(index) {
$scope.target.bucketAggs.splice(index, 1);
$scope.queryUpdated(); $scope.queryUpdated();
}; };
......
...@@ -9,25 +9,42 @@ define([ ...@@ -9,25 +9,42 @@ define([
var builder = new ElasticQueryBuilder(); var builder = new ElasticQueryBuilder();
var query = builder.build({ var query = builder.build({
metrics: [{agg: 'Count'}],
timeField: '@timestamp', timeField: '@timestamp',
select: [{agg: 'Count'}], bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
groupByFields: [],
}); });
expect(query.query.filtered.filter.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom"); expect(query.query.filtered.filter.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
expect(query.aggs.histogram.date_histogram.extended_bounds.min).to.be("$timeFrom"); expect(query.aggs["b0"].date_histogram.extended_bounds.min).to.be("$timeFrom");
}); });
it('with multiple bucket aggs', function() {
var builder = new ElasticQueryBuilder();
var query = builder.build({
metrics: [{agg: 'Count'}],
timeField: '@timestamp',
bucketAggs: [
{type: 'terms', field: '@host'},
{type: 'date_histogram', field: '@timestamp'}
],
});
expect(query.aggs["b0"].terms.field).to.be("@host");
expect(query.aggs["b0"].aggs["b1"].date_histogram.field).to.be("@timestamp");
});
it('with select field', function() { it('with select field', function() {
var builder = new ElasticQueryBuilder(); var builder = new ElasticQueryBuilder();
var query = builder.build({ var query = builder.build({
select: [{agg: 'avg', field: '@value'}], metrics: [{agg: 'avg', field: '@value'}],
groupByFields: [], bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
}, 100, 1000); }, 100, 1000);
var aggs = query.aggs.histogram.aggs; var aggs = query.aggs["b0"].aggs;
expect(aggs["0"].avg.field).to.be("@value"); expect(aggs["m0"].avg.field).to.be("@value");
}); });
......
...@@ -33,7 +33,7 @@ define([ ...@@ -33,7 +33,7 @@ define([
describe('initSelectSegments with 2 selects', function() { describe('initSelectSegments with 2 selects', function() {
it('init selectSegments', function() { it('init selectSegments', function() {
ctx.scope.target.select = [ ctx.scope.target.metrics = [
{agg: 'count'}, {agg: 'count'},
{agg: 'avg', field: 'value'}, {agg: 'avg', field: 'value'},
]; ];
......
...@@ -23,12 +23,12 @@ define([ ...@@ -23,12 +23,12 @@ define([
beforeEach(function() { beforeEach(function() {
result = ctx.ds._processTimeSeries([{ result = ctx.ds._processTimeSeries([{
refId: 'A', refId: 'A',
select: [{agg: 'count'}], metrics: [{agg: 'count'}],
groupByFields: [], bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
}], { }], {
responses: [{ responses: [{
aggregations: { aggregations: {
histogram: { "b0": {
buckets: [ buckets: [
{ {
doc_count: 10, doc_count: 10,
...@@ -60,20 +60,20 @@ define([ ...@@ -60,20 +60,20 @@ define([
beforeEach(function() { beforeEach(function() {
result = ctx.ds._processTimeSeries([{ result = ctx.ds._processTimeSeries([{
refId: 'A', refId: 'A',
select: [{agg: 'count'}, {agg: 'avg', field: 'value'}], metrics: [{agg: 'count'}, {agg: 'avg', field: 'value'}],
groupByFields: [], bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
}], { }], {
responses: [{ responses: [{
aggregations: { aggregations: {
histogram: { "b0": {
buckets: [ buckets: [
{ {
"1": {value: 88}, "m1": {value: 88},
doc_count: 10, doc_count: 10,
key: 1000 key: 1000
}, },
{ {
"1": {value: 99}, "m1": {value: 99},
doc_count: 15, doc_count: 15,
key: 2000 key: 2000
} }
...@@ -103,33 +103,33 @@ define([ ...@@ -103,33 +103,33 @@ define([
beforeEach(function() { beforeEach(function() {
result = ctx.ds._processTimeSeries([{ result = ctx.ds._processTimeSeries([{
refId: 'A', refId: 'A',
select: [{agg: 'count'}], metrics: [{agg: 'count'}],
groupByFields: [{field: 'host' }] bucketAggs: [{type: 'terms', field: 'host'}, {type: 'date_histogram', field: '@timestamp'}],
}], { }], {
responses: [{ responses: [{
aggregations: { aggregations: {
histogram: { "b0": {
buckets: [ buckets: [
{ {
host: { "b1": {
buckets: [ buckets: [
{doc_count: 4, key: 'server1'}, {doc_count: 1, key: 1000},
{doc_count: 6, key: 'server2'}, {doc_count: 3, key: 2000}
] ]
}, },
doc_count: 10, doc_count: 4,
key: 1000 key: 'server1',
}, },
{ {
host: { "b1": {
buckets: [ buckets: [
{doc_count: 4, key: 'server1'}, {doc_count: 2, key: 1000},
{doc_count: 6, key: 'server2'}, {doc_count: 8, key: 2000}
] ]
}, },
doc_count: 15, doc_count: 10,
key: 2000 key: 'server2',
} },
] ]
} }
} }
...@@ -145,90 +145,6 @@ define([ ...@@ -145,90 +145,6 @@ define([
}); });
}); });
describe('group by query 2 fields', function() {
var result;
beforeEach(function() {
result = ctx.ds._processTimeSeries([{
refId: 'A',
select: [{agg: 'count'}],
groupByFields: [{field: 'host'}, {field: 'site'}]
}], {
responses: [{
aggregations: {
histogram: {
buckets: [
{
host: {
buckets: [
{
site: {
buckets: [
{doc_count: 3, key: 'backend'},
{doc_count: 1, key: 'frontend'},
],
},
doc_count: 4, key: 'server1'
},
{
site: {
buckets: [
{doc_count: 3, key: 'backend'},
{doc_count: 1, key: 'frontend'},
],
},
doc_count: 6, key: 'server2'
},
]
},
doc_count: 10,
key: 1000
},
{
host: {
buckets: [
{
site: {
buckets: [
{doc_count: 3, key: 'backend'},
{doc_count: 1, key: 'frontend'},
],
},
doc_count: 4,
key: 'server1'
},
{
site: {
buckets: [
{doc_count: 3, key: 'backend'},
{doc_count: 1, key: 'frontend'},
],
},
doc_count: 6,
key: 'server2'
},
]
},
doc_count: 15,
key: 2000
}
]
}
}
}]
});
});
it('should return 2 series', function() {
expect(result.data.length).to.be(4);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].target).to.be('A server1 backend count');
expect(result.data[1].target).to.be('A server1 frontend count');
expect(result.data[2].target).to.be('A server2 backend count');
expect(result.data[3].target).to.be('A server2 frontend count');
});
});
}); });
}); });
}); });
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment