Skip to content

N
nexpie-grafana-theme
Commit 8087af9c by Archit Sharma Committed by Torkel Ödegaard
Options

ES Nested type bucket aggregation (#4694) (#4694) 

* (elasticsearch): add nested agg (use bucket aggs). fixes #4693

* (elasticsearch): rebased after merge of #6043 refactored from #4527
parent 05952688
Showing with 187 additions and 41 deletions
public/app/plugins/datasource/elasticsearch/bucket_agg.js
......@@ -17,6 +17,7 @@ function (angular, _, queryDef) {
target: "=",
index: "=",
onChange: "&",
// getNestedKeys: "&",
getFields: "&",
}
};
......@@ -52,14 +53,25 @@ function (angular, _, queryDef) {
case 'date_histogram':
case 'terms': {
delete $scope.agg.query;
delete $scope.agg.settings.nested.path;
$scope.agg.field = 'select field';
break;
}
case 'filters': {
delete $scope.agg.field;
delete $scope.agg.settings.nested.path;
$scope.agg.query = '*';
break;
}
case 'nested': {
delete $scope.agg.field;
delete $scope.agg.query;
$scope.agg.settings.nested = {};
$scope.agg.settings.nested.path = 'select field (type: nested)';
$scope.agg.settings.nested.term = 'select nested term path';
$scope.agg.settings.nested.query = 'select query for Nested Term';
break;
}
case 'geohash_grid': {
$scope.agg.settings.precision = 3;
break;
......@@ -79,6 +91,12 @@ function (angular, _, queryDef) {
var settings = $scope.agg.settings || {};
switch($scope.agg.type) {
case 'nested': {
if (settingsLinkText === '') {
settingsLinkText = 'Options';
}
break;
}
case 'terms': {
settings.order = settings.order || "asc";
settings.size = settings.size || "10";
......@@ -170,6 +188,20 @@ function (angular, _, queryDef) {
}
};
$scope.getFieldsNestedPath = function() {
return $scope.getFields({$fieldType: 'nested'});
};
$scope.getFieldsNestedTerm = function() {
return $scope.getFields({$fieldType: 'string'});
};
// FIX THIS: add a method getNestedKeys to be called here
// for getting nested key string ids, based on term/path supplied.
// $scope.getFieldsNestedQuery = function() {
// return $scope.getNestedKeys();
// };
$scope.getIntervalOptions = function() {
return $q.when(uiSegmentSrv.transformToSegments(true, 'interval')(queryDef.intervalOptions));
};
......
public/app/plugins/datasource/elasticsearch/datasource.js
......@@ -290,9 +290,10 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
}
// transform to array
return _.map(fields, function(value) {
var resp = _.map(fields, function(value) {
return value;
});
return resp;
});
};
......
public/app/plugins/datasource/elasticsearch/elastic_response.js
......@@ -97,6 +97,33 @@ function (_, queryDef) {
}
};
ElasticResponse.prototype.processNestedAggregationDocs = function(esAgg, aggDef, target, seriesList, props) {
var metric, y, i, newSeries, bucket, value;
for (y = 0; y < target.metrics.length; y++) {
metric = target.metrics[y];
if (metric.hide) {
continue;
}
newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props};
for (i = 0; i < esAgg.buckets.length; i++) {
bucket = esAgg.buckets[i][aggDef.id]['nested_aggs'];
if (bucket !== undefined) {
value = bucket[metric.id];
if (value !== undefined) {
if (value.normalized_value) {
newSeries.datapoints.push([value.normalized_value, esAgg.buckets[i].key]);
} else {
newSeries.datapoints.push([value.value, esAgg.buckets[i].key]);
}
}
}
}
seriesList.push(newSeries);
}
};
ElasticResponse.prototype.processAggregationDocs = function(esAgg, aggDef, target, docs, props) {
var metric, y, i, bucket, metricName, doc;
......@@ -145,31 +172,39 @@ function (_, queryDef) {
// This is quite complex
// neeed to recurise down the nested buckets to build series
ElasticResponse.prototype.processBuckets = function(aggs, target, seriesList, docs, props, depth) {
var bucket, aggDef, esAgg, aggId;
var bucket, aggDef, aggDefNested, esAgg, aggId;
var maxDepth = target.bucketAggs.length-1;
aggDefNested = _.find(target.bucketAggs, {type: "nested"});
for (aggId in aggs) {
aggDef = _.find(target.bucketAggs, {id: aggId});
esAgg = aggs[aggId];
if (!aggDef) {
continue;
}
if (aggDefNested) {
this.processNestedAggregationDocs(esAgg, aggDefNested, target, seriesList, props);
} else {
aggDef = _.find(target.bucketAggs, {id: aggId});
if (depth === maxDepth) {
if (aggDef.type === 'date_histogram') {
this.processMetrics(esAgg, target, seriesList, props);
} else {
this.processAggregationDocs(esAgg, aggDef, target, docs, props);
if (!aggDef) {
continue;
}
} else {
for (var nameIndex in esAgg.buckets) {
bucket = esAgg.buckets[nameIndex];
props = _.clone(props);
if (bucket.key !== void 0) {
props[aggDef.field] = bucket.key;
if (depth === maxDepth) {
if (aggDef.type === 'date_histogram') {
this.processMetrics(esAgg, target, seriesList, props);
} else {
props["filter"] = nameIndex;
this.processAggregationDocs(esAgg, aggDef, target, docs, props);
}
} else {
for (var nameIndex in esAgg.buckets) {
bucket = esAgg.buckets[nameIndex];
props = _.clone(props);
if (bucket.key) {
props[aggDef.field] = bucket.key;
} else {
props["filter"] = nameIndex;
}
this.processBuckets(bucket, target, seriesList, docs, props, depth+1);
}
if (bucket.key_as_string) {
props[aggDef.field] = bucket.key_as_string;
......@@ -293,7 +328,7 @@ function (_, queryDef) {
if (err.root_cause && err.root_cause.length > 0 && err.root_cause[0].reason) {
result.message = err.root_cause[0].reason;
} else {
result.message = err.reason || 'Unkown elatic error response';
result.message = err.reason || 'Unknown elastic error response';
}
if (response.$$config) {
......@@ -325,7 +360,6 @@ function (_, queryDef) {
this.processBuckets(aggregations, target, tmpSeriesList, docs, {}, 0);
this.trimDatapoints(tmpSeriesList, target);
this.nameSeries(tmpSeriesList, target);
for (var y = 0; y < tmpSeriesList.length; y++) {
seriesList.push(tmpSeriesList[y]);
}
......
public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html
......@@ -7,6 +7,7 @@
<metric-segment-model property="agg.type" options="bucketAggTypes" on-change="onTypeChanged()" custom="false" css-class="width-10"></metric-segment-model>
<metric-segment-model ng-if="agg.field" property="agg.field" get-options="getFieldsInternal()" on-change="onChange()" css-class="width-12"></metric-segment-model>
<metric-segment-model ng-if="agg.settings.nested.path" property="agg.settings.nested.path" get-options="getFieldsNestedPath()" on-change="onChange()" css-class="width-12"></metric-segment-model>
</div>
<div class="gf-form gf-form--grow">
......@@ -52,6 +53,18 @@
</div>
</div>
<div ng-if="agg.type === 'nested'">
<div class="gf-form offset-width-7">
<label class="gf-form-label width-10">Nested Term</label>
<metric-segment-model property="agg.settings.nested.term" get-options="getFieldsNestedTerm()" on-change="onChangeInternal()" css-class="width-12" custom="true"></metric-segment-model>
</div>
<div class="gf-form offset-width-7">
<label class="gf-form-label width-10">Query</label>
<input type="text" class="gf-form-input max-width-12" ng-model="agg.settings.nested.query" spellcheck='false' placeholder="Nested string match query" ng-blur="onChangeInternal()">
</div>
</div>
<div ng-if="agg.type === 'terms'">
<div class="gf-form offset-width-7">
<label class="gf-form-label width-10">Order</label>
......
public/app/plugins/datasource/elasticsearch/query_builder.js
......@@ -59,6 +59,33 @@ function (queryDef) {
return queryNode;
};
ElasticQueryBuilder.prototype.buildNestedAgg = function(aggDef, queryNode, target) {
var metric, y;
if (!aggDef.settings) {
return queryNode;
}
var settings = aggDef.settings;
queryNode.nested = {};
queryNode.nested.path = settings.nested.path;
queryNode.aggs = {
nested_aggs: {
filter: {},
aggs: {}
}
};
queryNode.aggs.nested_aggs.filter.term = {};
queryNode.aggs.nested_aggs.filter.term[settings.nested.term] = settings.nested.query;
queryNode.aggs.nested_aggs.aggs = {};
for (y = 0; y < target.metrics.length; y++) {
metric = target.metrics[y];
queryNode.aggs.nested_aggs.aggs[metric.id] = {};
queryNode.aggs.nested_aggs.aggs[metric.id][metric.type] = {field: metric.field};
}
return queryNode;
};
ElasticQueryBuilder.prototype.getDateHistogramAgg = function(aggDef) {
var esAgg = {};
var settings = aggDef.settings || {};
......@@ -183,6 +210,7 @@ function (queryDef) {
nestedAggs = query;
var foundNested = false;
for (i = 0; i < target.bucketAggs.length; i++) {
var aggDef = target.bucketAggs[i];
var esAgg = {};
......@@ -204,6 +232,10 @@ function (queryDef) {
esAgg['geohash_grid'] = {field: aggDef.field, precision: aggDef.settings.precision};
break;
}
case 'nested': {
foundNested = true;
this.buildNestedAgg(aggDef, esAgg, target);
}
}
nestedAggs.aggs = nestedAggs.aggs || {};
......@@ -211,35 +243,37 @@ function (queryDef) {
nestedAggs = esAgg;
}
nestedAggs.aggs = {};
if (!foundNested) {
nestedAggs.aggs = {};
for (i = 0; i < target.metrics.length; i++) {
metric = target.metrics[i];
if (metric.type === 'count') {
continue;
}
for (i = 0; i < target.metrics.length; i++) {
metric = target.metrics[i];
if (metric.type === 'count') {
continue;
}
var aggField = {};
var metricAgg = null;
var aggField = {};
var metricAgg = null;
if (queryDef.isPipelineAgg(metric.type)) {
if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) {
metricAgg = { buckets_path: metric.pipelineAgg };
if (queryDef.isPipelineAgg(metric.type)) {
if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) {
metricAgg = { buckets_path: metric.pipelineAgg };
} else {
continue;
}
} else {
continue;
metricAgg = {field: metric.field};
}
} else {
metricAgg = {field: metric.field};
}
for (var prop in metric.settings) {
if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) {
metricAgg[prop] = metric.settings[prop];
for (var prop in metric.settings) {
if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) {
metricAgg[prop] = metric.settings[prop];
}
}
}
aggField[metric.type] = metricAgg;
nestedAggs.aggs[metric.id] = aggField;
aggField[metric.type] = metricAgg;
nestedAggs.aggs[metric.id] = aggField;
}
}
return query;
......
public/app/plugins/datasource/elasticsearch/query_def.js
......@@ -22,6 +22,7 @@ function (_) {
bucketAggTypes: [
{text: "Terms", value: 'terms', requiresField: true},
{text: "Filters", value: 'filters' },
{text: "Nested", value: 'nested', requiresField: true},
{text: "Geo Hash Grid", value: 'geohash_grid', requiresField: true},
{text: "Date Histogram", value: 'date_histogram', requiresField: true},
],
......
public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts
......@@ -102,6 +102,37 @@ describe('ElasticQueryBuilder', function() {
expect(firstLevel.aggs["1"].percentiles.percents).to.eql([1,2,3,4]);
});
it('with nested aggs', function(){
var query = builder.build({
metrics: [{type: 'avg', field: 'disk.wr_sec', id: '1'}],
bucketAggs: [
{type: 'date_histogram', field: '@timestamp', id: '2'},
{
id: '3',
type: 'nested',
settings: {
nested: {
path: 'disk' ,
term: 'disk.disk-device',
query: 'dev8-0',
}
}
}
],
});
var firstLevel = query.aggs["2"];
var secondLevel = firstLevel.aggs["3"];
var thirdLevel = secondLevel.aggs["nested_aggs"];
var fourthLevel = thirdLevel.aggs["1"];
expect(firstLevel.date_histogram.field).to.be("@timestamp");
expect(secondLevel.nested.path).to.be("disk");
expect(Object.keys(thirdLevel.filter.term)[0]).to.be("disk.disk-device");
expect(thirdLevel.filter.term["disk.disk-device"]).to.be("dev8-0");
expect(fourthLevel.avg.field).to.be("disk.wr_sec");
});
it('with filters aggs', function() {
var query = builder.build({
metrics: [{type: 'count', id: '1'}],
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment