Skip to content

N
nexpie-grafana-theme
Commit cc1e3d01 by Torkel Ödegaard
Options

feat(elasticsearch): groundwork for a much more sophisticated elasticsearch query editor

parent 9daa3997
Showing with 196 additions and 194 deletions
public/app/plugins/datasource/elasticsearch/datasource.js
......@@ -174,39 +174,32 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
// This is quite complex
// neeed to recurise down the nested buckets to build series
ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName, parentTime) {
var groupBy = target.groupByFields[level];
var seriesName, time, value, select, i, y, bucket;
ElasticDatasource.prototype._processBuckets = function(buckets, target, series, level, parentName) {
var seriesName, value, metric, i, y, bucket, childBucket;
for (i = 0; i < buckets.length; i++) {
bucket = buckets[i];
childBucket = bucket['b' + level];
if (groupBy) {
seriesName = level > 0 ? parentName + ' ' + bucket.key : parentName;
time = parentTime || bucket.key;
this._processBuckets(bucket[groupBy.field].buckets, target, series, level+1, seriesName, time);
if (childBucket && childBucket.buckets) {
seriesName = parentName + ' ' + bucket.key;
this._processBuckets(childBucket.buckets, target, series, level+1, seriesName);
} else {
for (y = 0; y < target.select.length; y++) {
select = target.select[y];
for (y = 0; y < target.metrics.length; y++) {
metric = target.metrics[y];
seriesName = parentName;
if (level > 0) {
seriesName += ' ' + bucket.key;
} else {
parentTime = bucket.key;
}
if (select.field) {
seriesName += ' ' + select.field + ' ' + select.agg;
value = bucket[y.toString()].value;
if (metric.field) {
seriesName += ' ' + metric.field + ' ' + metric.agg;
value = bucket['m' + y.toString()].value;
} else {
seriesName += ' count';
value = bucket.doc_count;
}
var serie = series[seriesName] = series[seriesName] || {target: seriesName, datapoints: []};
serie.datapoints.push([value, parentTime]);
serie.datapoints.push([value, bucket.key]);
}
}
}
......@@ -221,11 +214,11 @@ function (angular, _, config, kbn, moment, ElasticQueryBuilder) {
throw { message: response.error };
}
var buckets = response.aggregations.histogram.buckets;
var buckets = response.aggregations["b0"].buckets;
var target = targets[i];
var querySeries = {};
this._processBuckets(buckets, target, querySeries, 0, target.refId);
this._processBuckets(buckets, target, querySeries, 1, target.refId);
for (var prop in querySeries) {
if (querySeries.hasOwnProperty(prop)) {
......
public/app/plugins/datasource/elasticsearch/directives.js
define([
'angular',
'./queryComponent',
],
function (angular) {
'use strict';
......
public/app/plugins/datasource/elasticsearch/partials/query.editor.html
......@@ -51,7 +51,7 @@
Time field
</li>
<li>
<metric-segment segment="timeSegment" get-alt-segments="getTimeFields()" on-value-changed="timeFieldChanged()"></metric-segment>
<metric-segment segment="timeSegment" get-alt-segments="getFields()" on-value-changed="timeFieldChanged()"></metric-segment>
</li>
</ul>
......@@ -62,10 +62,11 @@
</div>
</div>
<div class="tight-form" ng-hide="target.rawQuery">
<div ng-hide="target.rawQuery">
<div class="tight-form">
<ul class="tight-form-list">
<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
Select
Metrics
</li>
<li ng-repeat="segment in selectSegments">
<metric-segment segment="segment" get-alt-segments="getSelectSegments(segment, $index)" on-value-changed="selectChanged(segment, $index)"></metric-segment>
......@@ -73,17 +74,27 @@
</ul>
<div class="clearfix"></div>
</div>
</div>
<div class="tight-form" ng-hide="target.rawQuery">
<div class="tight-form" ng-repeat="agg in target.bucketAggs">
<ul class="tight-form-list">
<li class="tight-form-item query-keyword tight-form-align" style="width: 75px;">
Group by
<span ng-show="$first">Group by</span>
<span ng-show="!$first">Then by</span>
</li>
<li>
<elastic-query-component model="agg" get-fields="getFields()" on-change="queryUpdated()"></elastic-query-component>
</li>
</ul>
<ul class="tight-form-list pull-right">
<li class="tight-form-item" ng-if="$index === 0">
<a class="pointer" ng-click="addBucketAgg()"><i class="fa fa-plus"></i></a>
</li>
<li ng-repeat="segment in groupBySegments">
<metric-segment segment="segment" get-alt-segments="getGroupByFields(segment, $index)" on-value-changed="groupByChanged(segment, $index)"></metric-segment>
<li class="tight-form-item" ng-if="!$last">
<a class="pointer" ng-click="removeBucketAgg(agg, $index)"><i class="fa fa-minus"></i></a>
</li>
</ul>
<div class="clearfix"></div>
</div>
</div>
public/app/plugins/datasource/elasticsearch/queryBuilder.js
......@@ -17,6 +17,7 @@ function (angular) {
return angular.fromJson(target.rawQuery);
}
var i, nestedAggs;
var query = {
"size": 0,
"query": {
......@@ -36,43 +37,42 @@ function (angular) {
}
};
query.aggs = {
"histogram": {
"date_histogram": {
nestedAggs = query;
for (i = 0; i < target.bucketAggs.length; i++) {
var aggDef = target.bucketAggs[i];
var esAgg = {};
switch(aggDef.type) {
case 'date_histogram': {
esAgg["date_histogram"] = {
"interval": target.interval || "$interval",
"field": target.timeField,
"field": aggDef.field,
"min_doc_count": 0,
"extended_bounds": {
"min": "$timeFrom",
"max": "$timeTo"
"extended_bounds": { "min": "$timeFrom", "max": "$timeTo" }
};
break;
}
case 'terms': {
esAgg["terms"] = { "field": aggDef.field };
break;
}
}
},
};
var nestedAggs = query.aggs.histogram;
var i;
target.groupByFields = target.groupByFields || [];
for (i = 0; i < target.groupByFields.length; i++) {
var field = target.groupByFields[i].field;
var aggs = {terms: {field: field}};
nestedAggs.aggs = {};
nestedAggs.aggs[field] = aggs;
nestedAggs = aggs;
nestedAggs.aggs['b' + i] = esAgg;
nestedAggs = esAgg;
}
nestedAggs.aggs = {};
for (i = 0; i < target.select.length; i++) {
var select = target.select[i];
if (select.field) {
for (i = 0; i < target.metrics.length; i++) {
var metric = target.metrics[i];
if (metric.field) {
var aggField = {};
aggField[select.agg] = {field: select.field};
aggField[metric.agg] = {field: metric.field};
nestedAggs.aggs[i.toString()] = aggField;
nestedAggs.aggs['m' + i] = aggField;
}
}
......
public/app/plugins/datasource/elasticsearch/queryComponent.js 0 → 100644
define([
'angular',
'lodash',
'jquery',
],
function (angular, _, $) {
'use strict';
angular
.module('grafana.directives')
.directive('elasticQueryComponent', function($compile, uiSegmentSrv, $q) {
//var linkTemplate = '<a class="tight-form-item tabindex="1" ng-bind-html="textRep"></a>';
/* jshint maxlen:false */
var template1 = '<metric-segment segment="typeSegment" get-alt-segments="getBucketAggTypes()" on-value-changed="bucketAggTypeChanged()"></metric-segment>';
/* jshint maxlen:false */
var template2 = '<metric-segment segment="fieldSegment" get-alt-segments="getFields()" on-value-changed="fieldChanged()"></metric-segment>';
return {
restrict: 'E',
scope: {
model: "=",
onChange: "&",
getFields: "&",
},
link: function postLink($scope, elem) {
$scope.getBucketAggTypes = function() {
return $q.when([
uiSegmentSrv.newSegment({value: 'terms'}),
uiSegmentSrv.newSegment({value: 'date_histogram'}),
]);
};
$scope.fieldChanged = function() {
$scope.model.field = $scope.fieldSegment.value;
$scope.onChange();
};
$scope.bucketAggTypeChanged = function() {
$scope.model.type = $scope.typeSegment.value;
$scope.onChange();
};
function addElementsAndCompile() {
var $html = $(template1 + template2);
$scope.fieldSegment = uiSegmentSrv.newSegment($scope.model.field);
$scope.typeSegment = uiSegmentSrv.newSegment($scope.model.type);
$html.appendTo(elem);
$compile(elem.contents())($scope);
}
addElementsAndCompile();
}
};
});
});
public/app/plugins/datasource/elasticsearch/queryCtrl.js
......@@ -15,17 +15,22 @@ function (angular, _, ElasticQueryBuilder) {
if (!target) { return; }
target.timeField = target.timeField || '@timestamp';
target.select = target.select || [{ agg: 'count' }];
target.groupByFields = target.groupByFields || [];
target.metrics = target.metrics || [{ agg: 'count' }];
target.bucketAggs = target.bucketAggs || [];
target.bucketAggs = [
{
type: 'terms',
field: '@hostname'
},
{
type: 'date_histogram',
field: '@timestamp'
},
];
$scope.timeSegment = uiSegmentSrv.newSegment(target.timeField);
$scope.groupBySegments = _.map(target.groupByFields, function(group) {
return uiSegmentSrv.newSegment(group.field);
});
$scope.initSelectSegments();
$scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
$scope.removeSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove select --'});
$scope.resetSelectSegment = uiSegmentSrv.newSegment({fake: true, value: '-- reset --'});
$scope.removeGroupBySegment = uiSegmentSrv.newSegment({fake: true, value: '-- remove group by --'});
......@@ -36,7 +41,7 @@ function (angular, _, ElasticQueryBuilder) {
$scope.initSelectSegments = function() {
$scope.selectSegments = [];
_.each($scope.target.select, function(select) {
_.each($scope.target.metrics, function(select) {
if ($scope.selectSegments.length > 0) {
$scope.selectSegments.push(uiSegmentSrv.newCondition(" and "));
}
......@@ -55,9 +60,10 @@ function (angular, _, ElasticQueryBuilder) {
if (segment.type === 'agg' || segment.type === 'plus-button') {
var options = [
uiSegmentSrv.newSegment({value: 'count', type: 'agg'}),
uiSegmentSrv.newSegment({value: 'avg', type: 'agg', reqField: true}),
uiSegmentSrv.newSegment({value: 'sum', type: 'agg', reqField: true}),
uiSegmentSrv.newSegment({value: 'min', type: 'agg', reqField: true}),
uiSegmentSrv.newSegment({value: 'max', type: 'agg', reqField: true}),
uiSegmentSrv.newSegment({value: 'avg', type: 'agg', reqField: true}),
];
// if we have other selects and this is not a plus button add remove option
if (segment.type !== 'plus-button' && $scope.selectSegments.length > 3) {
......@@ -78,7 +84,7 @@ function (angular, _, ElasticQueryBuilder) {
$scope.selectChanged = function(segment, index) {
// reset
if (segment.value === $scope.resetSelectSegment.value) {
$scope.target.select = [{ agg: 'count' }];
$scope.target.metrics = [{ agg: 'count' }];
$scope.initSelectSegments();
$scope.queryUpdated();
return;
......@@ -125,7 +131,7 @@ function (angular, _, ElasticQueryBuilder) {
};
$scope.rebuildTargetSelects = function() {
$scope.target.select = [];
$scope.target.metrics = [];
for (var i = 0; i < $scope.selectSegments.length; i++) {
var segment = $scope.selectSegments[i];
var select = {agg: segment.value };
......@@ -138,7 +144,7 @@ function (angular, _, ElasticQueryBuilder) {
}
if (select.field === 'select field') { continue; }
$scope.target.select.push(select);
$scope.target.metrics.push(select);
}
};
......@@ -154,7 +160,7 @@ function (angular, _, ElasticQueryBuilder) {
.then(null, $scope.handleQueryError);
};
$scope.getTimeFields = function() {
$scope.getFields = function() {
return $scope.datasource.metricFindQuery('fields()')
.then($scope.transformToSegments(false))
.then(null, $scope.handleQueryError);
......@@ -165,22 +171,20 @@ function (angular, _, ElasticQueryBuilder) {
$scope.queryUpdated();
};
$scope.groupByChanged = function(segment, index) {
if (segment.value === $scope.removeGroupBySegment.value) {
$scope.target.groupByFields.splice(index, 1);
$scope.groupBySegments.splice(index, 1);
$scope.queryUpdated();
return;
}
$scope.addBucketAgg = function() {
// if last is date histogram add it before
var lastBucket = $scope.target.bucketAggs[$scope.target.bucketAggs.length - 1];
var addIndex = $scope.target.bucketAggs.length - 1;
if (index === $scope.groupBySegments.length-1) {
$scope.groupBySegments.push(uiSegmentSrv.newPlusButton());
if (lastBucket && lastBucket.type === 'date_histogram') {
addIndex - 1;
}
segment.type = 'group-by-key';
segment.fake = false;
$scope.target.bucketAggs.splice(addIndex, 0, {type: "terms", field: "select field" });
};
$scope.target.groupByFields[index] = {field: segment.value};
$scope.removeBucketAgg = function(index) {
$scope.target.bucketAggs.splice(index, 1);
$scope.queryUpdated();
};
......
public/test/specs/elasticsearch-querybuilder-specs.js
......@@ -9,25 +9,42 @@ define([
var builder = new ElasticQueryBuilder();
var query = builder.build({
metrics: [{agg: 'Count'}],
timeField: '@timestamp',
select: [{agg: 'Count'}],
groupByFields: [],
bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
});
expect(query.query.filtered.filter.bool.must[0].range["@timestamp"].gte).to.be("$timeFrom");
expect(query.aggs.histogram.date_histogram.extended_bounds.min).to.be("$timeFrom");
expect(query.aggs["b0"].date_histogram.extended_bounds.min).to.be("$timeFrom");
});
it('with multiple bucket aggs', function() {
var builder = new ElasticQueryBuilder();
var query = builder.build({
metrics: [{agg: 'Count'}],
timeField: '@timestamp',
bucketAggs: [
{type: 'terms', field: '@host'},
{type: 'date_histogram', field: '@timestamp'}
],
});
expect(query.aggs["b0"].terms.field).to.be("@host");
expect(query.aggs["b0"].aggs["b1"].date_histogram.field).to.be("@timestamp");
});
it('with select field', function() {
var builder = new ElasticQueryBuilder();
var query = builder.build({
select: [{agg: 'avg', field: '@value'}],
groupByFields: [],
metrics: [{agg: 'avg', field: '@value'}],
bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
}, 100, 1000);
var aggs = query.aggs.histogram.aggs;
expect(aggs["0"].avg.field).to.be("@value");
var aggs = query.aggs["b0"].aggs;
expect(aggs["m0"].avg.field).to.be("@value");
});
......
public/test/specs/elasticsearch-queryctrl-specs.js
......@@ -33,7 +33,7 @@ define([
describe('initSelectSegments with 2 selects', function() {
it('init selectSegments', function() {
ctx.scope.target.select = [
ctx.scope.target.metrics = [
{agg: 'count'},
{agg: 'avg', field: 'value'},
];
......
public/test/specs/elasticsearch-specs.js
......@@ -23,12 +23,12 @@ define([
beforeEach(function() {
result = ctx.ds._processTimeSeries([{
refId: 'A',
select: [{agg: 'count'}],
groupByFields: [],
metrics: [{agg: 'count'}],
bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
}], {
responses: [{
aggregations: {
histogram: {
"b0": {
buckets: [
{
doc_count: 10,
......@@ -60,20 +60,20 @@ define([
beforeEach(function() {
result = ctx.ds._processTimeSeries([{
refId: 'A',
select: [{agg: 'count'}, {agg: 'avg', field: 'value'}],
groupByFields: [],
metrics: [{agg: 'count'}, {agg: 'avg', field: 'value'}],
bucketAggs: [{type: 'date_histogram', field: '@timestamp'}],
}], {
responses: [{
aggregations: {
histogram: {
"b0": {
buckets: [
{
"1": {value: 88},
"m1": {value: 88},
doc_count: 10,
key: 1000
},
{
"1": {value: 99},
"m1": {value: 99},
doc_count: 15,
key: 2000
}
......@@ -103,33 +103,33 @@ define([
beforeEach(function() {
result = ctx.ds._processTimeSeries([{
refId: 'A',
select: [{agg: 'count'}],
groupByFields: [{field: 'host' }]
metrics: [{agg: 'count'}],
bucketAggs: [{type: 'terms', field: 'host'}, {type: 'date_histogram', field: '@timestamp'}],
}], {
responses: [{
aggregations: {
histogram: {
"b0": {
buckets: [
{
host: {
"b1": {
buckets: [
{doc_count: 4, key: 'server1'},
{doc_count: 6, key: 'server2'},
{doc_count: 1, key: 1000},
{doc_count: 3, key: 2000}
]
},
doc_count: 10,
key: 1000
doc_count: 4,
key: 'server1',
},
{
host: {
"b1": {
buckets: [
{doc_count: 4, key: 'server1'},
{doc_count: 6, key: 'server2'},
{doc_count: 2, key: 1000},
{doc_count: 8, key: 2000}
]
},
doc_count: 15,
key: 2000
}
doc_count: 10,
key: 'server2',
},
]
}
}
......@@ -145,90 +145,6 @@ define([
});
});
describe('group by query 2 fields', function() {
var result;
beforeEach(function() {
result = ctx.ds._processTimeSeries([{
refId: 'A',
select: [{agg: 'count'}],
groupByFields: [{field: 'host'}, {field: 'site'}]
}], {
responses: [{
aggregations: {
histogram: {
buckets: [
{
host: {
buckets: [
{
site: {
buckets: [
{doc_count: 3, key: 'backend'},
{doc_count: 1, key: 'frontend'},
],
},
doc_count: 4, key: 'server1'
},
{
site: {
buckets: [
{doc_count: 3, key: 'backend'},
{doc_count: 1, key: 'frontend'},
],
},
doc_count: 6, key: 'server2'
},
]
},
doc_count: 10,
key: 1000
},
{
host: {
buckets: [
{
site: {
buckets: [
{doc_count: 3, key: 'backend'},
{doc_count: 1, key: 'frontend'},
],
},
doc_count: 4,
key: 'server1'
},
{
site: {
buckets: [
{doc_count: 3, key: 'backend'},
{doc_count: 1, key: 'frontend'},
],
},
doc_count: 6,
key: 'server2'
},
]
},
doc_count: 15,
key: 2000
}
]
}
}
}]
});
});
it('should return 2 series', function() {
expect(result.data.length).to.be(4);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].target).to.be('A server1 backend count');
expect(result.data[1].target).to.be('A server1 frontend count');
expect(result.data[2].target).to.be('A server2 backend count');
expect(result.data[3].target).to.be('A server2 frontend count');
});
});
});
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment