Skip to content

N
nexpie-grafana-theme
Commit 8087af9c by Archit Sharma Committed by Torkel Ödegaard
Options

ES Nested type bucket aggregation (#4694) (#4694) 

* (elasticsearch): add nested agg (use bucket aggs). fixes #4693

* (elasticsearch): rebased after merge of #6043 refactored from #4527
parent 05952688
Showing with 187 additions and 41 deletions
public/app/plugins/datasource/elasticsearch/bucket_agg.js
...@@ -17,6 +17,7 @@ function (angular, _, queryDef) { ...@@ -17,6 +17,7 @@ function (angular, _, queryDef) {
target: "=", target: "=",
index: "=", index: "=",
onChange: "&", onChange: "&",
// getNestedKeys: "&",
getFields: "&", getFields: "&",
} }
}; };
...@@ -52,14 +53,25 @@ function (angular, _, queryDef) { ...@@ -52,14 +53,25 @@ function (angular, _, queryDef) {
case 'date_histogram': case 'date_histogram':
case 'terms': { case 'terms': {
delete $scope.agg.query; delete $scope.agg.query;
delete $scope.agg.settings.nested.path;
$scope.agg.field = 'select field'; $scope.agg.field = 'select field';
break; break;
} }
case 'filters': { case 'filters': {
delete $scope.agg.field; delete $scope.agg.field;
delete $scope.agg.settings.nested.path;
$scope.agg.query = '*'; $scope.agg.query = '*';
break; break;
} }
case 'nested': {
delete $scope.agg.field;
delete $scope.agg.query;
$scope.agg.settings.nested = {};
$scope.agg.settings.nested.path = 'select field (type: nested)';
$scope.agg.settings.nested.term = 'select nested term path';
$scope.agg.settings.nested.query = 'select query for Nested Term';
break;
}
case 'geohash_grid': { case 'geohash_grid': {
$scope.agg.settings.precision = 3; $scope.agg.settings.precision = 3;
break; break;
...@@ -79,6 +91,12 @@ function (angular, _, queryDef) { ...@@ -79,6 +91,12 @@ function (angular, _, queryDef) {
var settings = $scope.agg.settings || {}; var settings = $scope.agg.settings || {};
switch($scope.agg.type) { switch($scope.agg.type) {
case 'nested': {
if (settingsLinkText === '') {
settingsLinkText = 'Options';
}
break;
}
case 'terms': { case 'terms': {
settings.order = settings.order || "asc"; settings.order = settings.order || "asc";
settings.size = settings.size || "10"; settings.size = settings.size || "10";
...@@ -170,6 +188,20 @@ function (angular, _, queryDef) { ...@@ -170,6 +188,20 @@ function (angular, _, queryDef) {
} }
}; };
$scope.getFieldsNestedPath = function() {
return $scope.getFields({$fieldType: 'nested'});
};
$scope.getFieldsNestedTerm = function() {
return $scope.getFields({$fieldType: 'string'});
};
// FIX THIS: add a method getNestedKeys to be called here
// for getting nested key string ids, based on term/path supplied.
// $scope.getFieldsNestedQuery = function() {
// return $scope.getNestedKeys();
// };
$scope.getIntervalOptions = function() { $scope.getIntervalOptions = function() {
return $q.when(uiSegmentSrv.transformToSegments(true, 'interval')(queryDef.intervalOptions)); return $q.when(uiSegmentSrv.transformToSegments(true, 'interval')(queryDef.intervalOptions));
}; };
......
public/app/plugins/datasource/elasticsearch/datasource.js
...@@ -290,9 +290,10 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes ...@@ -290,9 +290,10 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes
} }
// transform to array // transform to array
return _.map(fields, function(value) { var resp = _.map(fields, function(value) {
return value; return value;
}); });
return resp;
}); });
}; };
......
public/app/plugins/datasource/elasticsearch/elastic_response.js
...@@ -97,6 +97,33 @@ function (_, queryDef) { ...@@ -97,6 +97,33 @@ function (_, queryDef) {
} }
}; };
ElasticResponse.prototype.processNestedAggregationDocs = function(esAgg, aggDef, target, seriesList, props) {
var metric, y, i, newSeries, bucket, value;
for (y = 0; y < target.metrics.length; y++) {
metric = target.metrics[y];
if (metric.hide) {
continue;
}
newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props};
for (i = 0; i < esAgg.buckets.length; i++) {
bucket = esAgg.buckets[i][aggDef.id]['nested_aggs'];
if (bucket !== undefined) {
value = bucket[metric.id];
if (value !== undefined) {
if (value.normalized_value) {
newSeries.datapoints.push([value.normalized_value, esAgg.buckets[i].key]);
} else {
newSeries.datapoints.push([value.value, esAgg.buckets[i].key]);
}
}
}
}
seriesList.push(newSeries);
}
};
ElasticResponse.prototype.processAggregationDocs = function(esAgg, aggDef, target, docs, props) { ElasticResponse.prototype.processAggregationDocs = function(esAgg, aggDef, target, docs, props) {
var metric, y, i, bucket, metricName, doc; var metric, y, i, bucket, metricName, doc;
...@@ -145,31 +172,39 @@ function (_, queryDef) { ...@@ -145,31 +172,39 @@ function (_, queryDef) {
// This is quite complex // This is quite complex
// neeed to recurise down the nested buckets to build series // neeed to recurise down the nested buckets to build series
ElasticResponse.prototype.processBuckets = function(aggs, target, seriesList, docs, props, depth) { ElasticResponse.prototype.processBuckets = function(aggs, target, seriesList, docs, props, depth) {
var bucket, aggDef, esAgg, aggId; var bucket, aggDef, aggDefNested, esAgg, aggId;
var maxDepth = target.bucketAggs.length-1; var maxDepth = target.bucketAggs.length-1;
aggDefNested = _.find(target.bucketAggs, {type: "nested"});
for (aggId in aggs) { for (aggId in aggs) {
aggDef = _.find(target.bucketAggs, {id: aggId});
esAgg = aggs[aggId]; esAgg = aggs[aggId];
if (!aggDef) { if (aggDefNested) {
continue; this.processNestedAggregationDocs(esAgg, aggDefNested, target, seriesList, props);
} } else {
aggDef = _.find(target.bucketAggs, {id: aggId});
if (depth === maxDepth) { if (!aggDef) {
if (aggDef.type === 'date_histogram') { continue;
this.processMetrics(esAgg, target, seriesList, props);
} else {
this.processAggregationDocs(esAgg, aggDef, target, docs, props);
} }
} else {
for (var nameIndex in esAgg.buckets) { if (depth === maxDepth) {
bucket = esAgg.buckets[nameIndex]; if (aggDef.type === 'date_histogram') {
props = _.clone(props); this.processMetrics(esAgg, target, seriesList, props);
if (bucket.key !== void 0) {
props[aggDef.field] = bucket.key;
} else { } else {
props["filter"] = nameIndex; this.processAggregationDocs(esAgg, aggDef, target, docs, props);
}
} else {
for (var nameIndex in esAgg.buckets) {
bucket = esAgg.buckets[nameIndex];
props = _.clone(props);
if (bucket.key) {
props[aggDef.field] = bucket.key;
} else {
props["filter"] = nameIndex;
}
this.processBuckets(bucket, target, seriesList, docs, props, depth+1);
} }
if (bucket.key_as_string) { if (bucket.key_as_string) {
props[aggDef.field] = bucket.key_as_string; props[aggDef.field] = bucket.key_as_string;
...@@ -293,7 +328,7 @@ function (_, queryDef) { ...@@ -293,7 +328,7 @@ function (_, queryDef) {
if (err.root_cause && err.root_cause.length > 0 && err.root_cause[0].reason) { if (err.root_cause && err.root_cause.length > 0 && err.root_cause[0].reason) {
result.message = err.root_cause[0].reason; result.message = err.root_cause[0].reason;
} else { } else {
result.message = err.reason || 'Unkown elatic error response'; result.message = err.reason || 'Unknown elastic error response';
} }
if (response.$$config) { if (response.$$config) {
...@@ -325,7 +360,6 @@ function (_, queryDef) { ...@@ -325,7 +360,6 @@ function (_, queryDef) {
this.processBuckets(aggregations, target, tmpSeriesList, docs, {}, 0); this.processBuckets(aggregations, target, tmpSeriesList, docs, {}, 0);
this.trimDatapoints(tmpSeriesList, target); this.trimDatapoints(tmpSeriesList, target);
this.nameSeries(tmpSeriesList, target); this.nameSeries(tmpSeriesList, target);
for (var y = 0; y < tmpSeriesList.length; y++) { for (var y = 0; y < tmpSeriesList.length; y++) {
seriesList.push(tmpSeriesList[y]); seriesList.push(tmpSeriesList[y]);
} }
......
public/app/plugins/datasource/elasticsearch/partials/bucket_agg.html
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
<metric-segment-model property="agg.type" options="bucketAggTypes" on-change="onTypeChanged()" custom="false" css-class="width-10"></metric-segment-model> <metric-segment-model property="agg.type" options="bucketAggTypes" on-change="onTypeChanged()" custom="false" css-class="width-10"></metric-segment-model>
<metric-segment-model ng-if="agg.field" property="agg.field" get-options="getFieldsInternal()" on-change="onChange()" css-class="width-12"></metric-segment-model> <metric-segment-model ng-if="agg.field" property="agg.field" get-options="getFieldsInternal()" on-change="onChange()" css-class="width-12"></metric-segment-model>
<metric-segment-model ng-if="agg.settings.nested.path" property="agg.settings.nested.path" get-options="getFieldsNestedPath()" on-change="onChange()" css-class="width-12"></metric-segment-model>
</div> </div>
<div class="gf-form gf-form--grow"> <div class="gf-form gf-form--grow">
...@@ -52,6 +53,18 @@ ...@@ -52,6 +53,18 @@
</div> </div>
</div> </div>
<div ng-if="agg.type === 'nested'">
<div class="gf-form offset-width-7">
<label class="gf-form-label width-10">Nested Term</label>
<metric-segment-model property="agg.settings.nested.term" get-options="getFieldsNestedTerm()" on-change="onChangeInternal()" css-class="width-12" custom="true"></metric-segment-model>
</div>
<div class="gf-form offset-width-7">
<label class="gf-form-label width-10">Query</label>
<input type="text" class="gf-form-input max-width-12" ng-model="agg.settings.nested.query" spellcheck='false' placeholder="Nested string match query" ng-blur="onChangeInternal()">
</div>
</div>
<div ng-if="agg.type === 'terms'"> <div ng-if="agg.type === 'terms'">
<div class="gf-form offset-width-7"> <div class="gf-form offset-width-7">
<label class="gf-form-label width-10">Order</label> <label class="gf-form-label width-10">Order</label>
......
public/app/plugins/datasource/elasticsearch/query_builder.js
...@@ -59,6 +59,33 @@ function (queryDef) { ...@@ -59,6 +59,33 @@ function (queryDef) {
return queryNode; return queryNode;
}; };
ElasticQueryBuilder.prototype.buildNestedAgg = function(aggDef, queryNode, target) {
var metric, y;
if (!aggDef.settings) {
return queryNode;
}
var settings = aggDef.settings;
queryNode.nested = {};
queryNode.nested.path = settings.nested.path;
queryNode.aggs = {
nested_aggs: {
filter: {},
aggs: {}
}
};
queryNode.aggs.nested_aggs.filter.term = {};
queryNode.aggs.nested_aggs.filter.term[settings.nested.term] = settings.nested.query;
queryNode.aggs.nested_aggs.aggs = {};
for (y = 0; y < target.metrics.length; y++) {
metric = target.metrics[y];
queryNode.aggs.nested_aggs.aggs[metric.id] = {};
queryNode.aggs.nested_aggs.aggs[metric.id][metric.type] = {field: metric.field};
}
return queryNode;
};
ElasticQueryBuilder.prototype.getDateHistogramAgg = function(aggDef) { ElasticQueryBuilder.prototype.getDateHistogramAgg = function(aggDef) {
var esAgg = {}; var esAgg = {};
var settings = aggDef.settings || {}; var settings = aggDef.settings || {};
...@@ -183,6 +210,7 @@ function (queryDef) { ...@@ -183,6 +210,7 @@ function (queryDef) {
nestedAggs = query; nestedAggs = query;
var foundNested = false;
for (i = 0; i < target.bucketAggs.length; i++) { for (i = 0; i < target.bucketAggs.length; i++) {
var aggDef = target.bucketAggs[i]; var aggDef = target.bucketAggs[i];
var esAgg = {}; var esAgg = {};
...@@ -204,6 +232,10 @@ function (queryDef) { ...@@ -204,6 +232,10 @@ function (queryDef) {
esAgg['geohash_grid'] = {field: aggDef.field, precision: aggDef.settings.precision}; esAgg['geohash_grid'] = {field: aggDef.field, precision: aggDef.settings.precision};
break; break;
} }
case 'nested': {
foundNested = true;
this.buildNestedAgg(aggDef, esAgg, target);
}
} }
nestedAggs.aggs = nestedAggs.aggs || {}; nestedAggs.aggs = nestedAggs.aggs || {};
...@@ -211,35 +243,37 @@ function (queryDef) { ...@@ -211,35 +243,37 @@ function (queryDef) {
nestedAggs = esAgg; nestedAggs = esAgg;
} }
nestedAggs.aggs = {}; if (!foundNested) {
nestedAggs.aggs = {};
for (i = 0; i < target.metrics.length; i++) { for (i = 0; i < target.metrics.length; i++) {
metric = target.metrics[i]; metric = target.metrics[i];
if (metric.type === 'count') { if (metric.type === 'count') {
continue; continue;
} }
var aggField = {}; var aggField = {};
var metricAgg = null; var metricAgg = null;
if (queryDef.isPipelineAgg(metric.type)) { if (queryDef.isPipelineAgg(metric.type)) {
if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) { if (metric.pipelineAgg && /^\d*$/.test(metric.pipelineAgg)) {
metricAgg = { buckets_path: metric.pipelineAgg }; metricAgg = { buckets_path: metric.pipelineAgg };
} else {
continue;
}
} else { } else {
continue; metricAgg = {field: metric.field};
} }
} else {
metricAgg = {field: metric.field};
}
for (var prop in metric.settings) { for (var prop in metric.settings) {
if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) { if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) {
metricAgg[prop] = metric.settings[prop]; metricAgg[prop] = metric.settings[prop];
}
} }
}
aggField[metric.type] = metricAgg; aggField[metric.type] = metricAgg;
nestedAggs.aggs[metric.id] = aggField; nestedAggs.aggs[metric.id] = aggField;
}
} }
return query; return query;
......
public/app/plugins/datasource/elasticsearch/query_def.js
...@@ -22,6 +22,7 @@ function (_) { ...@@ -22,6 +22,7 @@ function (_) {
bucketAggTypes: [ bucketAggTypes: [
{text: "Terms", value: 'terms', requiresField: true}, {text: "Terms", value: 'terms', requiresField: true},
{text: "Filters", value: 'filters' }, {text: "Filters", value: 'filters' },
{text: "Nested", value: 'nested', requiresField: true},
{text: "Geo Hash Grid", value: 'geohash_grid', requiresField: true}, {text: "Geo Hash Grid", value: 'geohash_grid', requiresField: true},
{text: "Date Histogram", value: 'date_histogram', requiresField: true}, {text: "Date Histogram", value: 'date_histogram', requiresField: true},
], ],
......
public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts
...@@ -102,6 +102,37 @@ describe('ElasticQueryBuilder', function() { ...@@ -102,6 +102,37 @@ describe('ElasticQueryBuilder', function() {
expect(firstLevel.aggs["1"].percentiles.percents).to.eql([1,2,3,4]); expect(firstLevel.aggs["1"].percentiles.percents).to.eql([1,2,3,4]);
}); });
it('with nested aggs', function(){
var query = builder.build({
metrics: [{type: 'avg', field: 'disk.wr_sec', id: '1'}],
bucketAggs: [
{type: 'date_histogram', field: '@timestamp', id: '2'},
{
id: '3',
type: 'nested',
settings: {
nested: {
path: 'disk' ,
term: 'disk.disk-device',
query: 'dev8-0',
}
}
}
],
});
var firstLevel = query.aggs["2"];
var secondLevel = firstLevel.aggs["3"];
var thirdLevel = secondLevel.aggs["nested_aggs"];
var fourthLevel = thirdLevel.aggs["1"];
expect(firstLevel.date_histogram.field).to.be("@timestamp");
expect(secondLevel.nested.path).to.be("disk");
expect(Object.keys(thirdLevel.filter.term)[0]).to.be("disk.disk-device");
expect(thirdLevel.filter.term["disk.disk-device"]).to.be("dev8-0");
expect(fourthLevel.avg.field).to.be("disk.wr_sec");
});
it('with filters aggs', function() { it('with filters aggs', function() {
var query = builder.build({ var query = builder.build({
metrics: [{type: 'count', id: '1'}], metrics: [{type: 'count', id: '1'}],
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment