Skip to content

N
nexpie-grafana-theme
Commit 3999a3ca by Torkel Ödegaard
Options

feat(elasticsearch): extended stats like std deviation now works, and sigma… 

feat(elasticsearch): extended stats like std deviation now works, and sigma option as well, added unique count (cardinality as well, #1034
parent efc3def7
Showing with 412 additions and 281 deletions
.jshintrc
......@@ -23,7 +23,7 @@
"laxcomma": true,
"sub": true,
"unused": true,
"maxdepth": 5,
"maxdepth": 6,
"maxlen": 140,
"globals": {
......@@ -32,4 +32,4 @@
"Chromath": false,
"setImmediate": true
}
}
\ No newline at end of file
}
public/app/directives/misc.js
......@@ -63,15 +63,16 @@ function (angular, kbn) {
restrict: 'E',
link: function(scope, elem, attrs) {
var text = $interpolate(attrs.text)(scope);
var model = $interpolate(attrs.model)(scope);
var ngchange = attrs.change ? (' ng-change="' + attrs.change + '"') : '';
var tip = attrs.tip ? (' <tip>' + attrs.tip + '</tip>') : '';
var label = '<label for="' + scope.$id + attrs.model + '" class="checkbox-label">' +
var label = '<label for="' + scope.$id + model + '" class="checkbox-label">' +
text + tip + '</label>';
var template = '<input class="cr1" id="' + scope.$id + attrs.model + '" type="checkbox" ' +
' ng-model="' + attrs.model + '"' + ngchange +
' ng-checked="' + attrs.model + '"></input>' +
' <label for="' + scope.$id + attrs.model + '" class="cr1"></label>';
var template = '<input class="cr1" id="' + scope.$id + model + '" type="checkbox" ' +
' ng-model="' + model + '"' + ngchange +
' ng-checked="' + model + '"></input>' +
' <label for="' + scope.$id + model + '" class="cr1"></label>';
template = label + template;
elem.replaceWith($compile(angular.element(template))(scope));
......
public/app/features/dashboard/partials/settings.html
......@@ -57,7 +57,7 @@
<div class="editor-row">
<div class="tight-form-section">
<h5>Toggles</h5>
<div class="tight-form">
<div class="tight-form last">
<ul class="tight-form-list">
<li class="tight-form-item">
<editor-checkbox text="Editable" model="dashboard.editable"></editor-checkbox>
......@@ -65,7 +65,7 @@
<li class="tight-form-item">
<editor-checkbox text="Hide Controls (CTRL+H)" model="dashboard.hideControls"></editor-checkbox>
</li>
<li class="tight-form-item">
<li class="tight-form-item last">
<editor-checkbox text="Shared Crosshair (CTRL+O)" model="dashboard.sharedCrosshair"></editor-checkbox>
</li>
</ul>
......
public/app/plugins/datasource/elasticsearch/datasource.js
......@@ -5,10 +5,11 @@ define([
'kbn',
'./queryBuilder',
'./indexPattern',
'./elasticResponse',
'./queryCtrl',
'./directives'
],
function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern) {
function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticResponse) {
'use strict';
var module = angular.module('grafana.services');
......@@ -174,8 +175,9 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern) {
payload = payload.replace(/\$maxDataPoints/g, options.maxDataPoints);
payload = templateSrv.replace(payload, options.scopedVars);
var processTimeSeries = _.bind(this._processTimeSeries, this, sentTargets);
return this._post('/_msearch?search_type=count', payload).then(processTimeSeries);
return this._post('/_msearch?search_type=count', payload).then(function(res) {
return new ElasticResponse(sentTargets, res).getTimeSeries();
});
};
ElasticDatasource.prototype.translateTime = function(date) {
......@@ -186,94 +188,6 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern) {
return date.getTime();
};
// This is quite complex
// neeed to recurise down the nested buckets to build series
ElasticDatasource.prototype._processBuckets = function(aggs, target, series, level, parentName) {
var seriesName, value, metric, i, y, bucket, aggDef, esAgg;
function addMetricPoint(seriesName, value, time) {
var current = series[seriesName];
if (!current) {
current = series[seriesName] = {target: seriesName, datapoints: []};
}
current.datapoints.push([value, time]);
}
aggDef = target.bucketAggs[level];
esAgg = aggs[aggDef.id];
for (i = 0; i < esAgg.buckets.length; i++) {
bucket = esAgg.buckets[i];
// if last agg collect series
if (level === target.bucketAggs.length - 1) {
for (y = 0; y < target.metrics.length; y++) {
metric = target.metrics[y];
seriesName = parentName;
switch(metric.type) {
case 'count': {
seriesName += ' count';
value = bucket.doc_count;
addMetricPoint(seriesName, value, bucket.key);
break;
}
case 'percentiles': {
var values = bucket[metric.id].values;
for (var prop in values) {
addMetricPoint(seriesName + ' ' + prop, values[prop], bucket.key);
}
break;
}
case 'extended_stats': {
var stats = bucket[metric.id];
for (var statIndex in metric.stats) {
var statName = metric.stats[statIndex];
addMetricPoint(seriesName + ' ' + statName, stats[statName], bucket.key);
}
break;
}
default: {
seriesName += ' ' + metric.field + ' ' + metric.type;
value = bucket[metric.id].value;
addMetricPoint(seriesName, value, bucket.key);
break;
}
}
}
}
else {
this._processBuckets(bucket, target, series, level+1, parentName + ' ' + bucket.key);
}
}
};
ElasticDatasource.prototype._processTimeSeries = function(targets, results) {
var series = [];
for (var i = 0; i < results.responses.length; i++) {
var response = results.responses[i];
if (response.error) {
throw { message: response.error };
}
var aggregations = response.aggregations;
var target = targets[i];
var querySeries = {};
this._processBuckets(aggregations, target, querySeries, 0, target.refId);
for (var prop in querySeries) {
if (querySeries.hasOwnProperty(prop)) {
series.push(querySeries[prop]);
}
}
}
return { data: series };
};
ElasticDatasource.prototype.metricFindQuery = function() {
return this._get('/_mapping').then(function(res) {
var fields = {};
......
public/app/plugins/datasource/elasticsearch/elasticResponse.js 0 → 100644
define([
],
function () {
'use strict';
function ElasticResponse(targets, response) {
this.targets = targets;
this.response = response;
}
// This is quite complex
// neeed to recurise down the nested buckets to build series
ElasticResponse.prototype.processBuckets = function(aggs, target, series, level, parentName) {
var seriesName, value, metric, i, y, bucket, aggDef, esAgg;
function addMetricPoint(seriesName, value, time) {
var current = series[seriesName];
if (!current) {
current = series[seriesName] = {target: seriesName, datapoints: []};
}
current.datapoints.push([value, time]);
}
aggDef = target.bucketAggs[level];
esAgg = aggs[aggDef.id];
for (i = 0; i < esAgg.buckets.length; i++) {
bucket = esAgg.buckets[i];
// if last agg collect series
if (level === target.bucketAggs.length - 1) {
for (y = 0; y < target.metrics.length; y++) {
metric = target.metrics[y];
seriesName = parentName;
switch(metric.type) {
case 'count': {
seriesName += ' count';
value = bucket.doc_count;
addMetricPoint(seriesName, value, bucket.key);
break;
}
case 'percentiles': {
var values = bucket[metric.id].values;
for (var prop in values) {
addMetricPoint(seriesName + ' ' + prop, values[prop], bucket.key);
}
break;
}
case 'extended_stats': {
var stats = bucket[metric.id];
stats.std_deviation_bounds_upper = stats.std_deviation_bounds.upper;
stats.std_deviation_bounds_lower = stats.std_deviation_bounds.lower;
for (var statName in metric.meta) {
if (metric.meta[statName]) {
addMetricPoint(seriesName + ' ' + statName, stats[statName], bucket.key);
}
}
break;
}
default: {
seriesName += ' ' + metric.field + ' ' + metric.type;
value = bucket[metric.id].value;
addMetricPoint(seriesName, value, bucket.key);
break;
}
}
}
}
else {
this.processBuckets(bucket, target, series, level+1, parentName + ' ' + bucket.key);
}
}
};
ElasticResponse.prototype.getTimeSeries = function() {
var series = [];
for (var i = 0; i < this.response.responses.length; i++) {
var response = this.response.responses[i];
if (response.error) {
throw { message: response.error };
}
var aggregations = response.aggregations;
var target = this.targets[i];
var querySeries = {};
this.processBuckets(aggregations, target, querySeries, 0, target.refId);
for (var prop in querySeries) {
if (querySeries.hasOwnProperty(prop)) {
series.push(querySeries[prop]);
}
}
}
return { data: series };
};
return ElasticResponse;
});
public/app/plugins/datasource/elasticsearch/metricAgg.js
......@@ -12,6 +12,7 @@ function (angular, _, queryDef) {
var metricAggs = $scope.target.metrics;
$scope.metricAggTypes = queryDef.metricAggTypes;
$scope.extendedStats = queryDef.extendedStats;
$scope.init = function() {
$scope.agg = metricAggs[$scope.index];
......@@ -40,8 +41,14 @@ function (angular, _, queryDef) {
break;
}
case 'extended_stats': {
$scope.agg.stats = $scope.agg.stats || ['std_deviation'];
$scope.settingsLinkText = 'Stats: ' + $scope.agg.stats.join(',');
var stats = _.reduce($scope.agg.meta, function(memo, val, key) {
if (val) {
var def = _.findWhere($scope.extendedStats, {value: key});
memo.push(def.text);
}
return memo;
}, []);
$scope.settingsLinkText = 'Stats: ' + stats.join(', ');
}
}
};
......@@ -52,6 +59,9 @@ function (angular, _, queryDef) {
$scope.onTypeChange = function() {
$scope.agg.settings = {};
$scope.agg.meta = {};
$scope.showOptions = false;
$scope.validateModel();
$scope.onChange();
};
......
public/app/plugins/datasource/elasticsearch/partials/bucketAgg.html
......@@ -27,7 +27,7 @@
</div>
<div class="tight-form" ng-if="showOptions">
<div style="tight-form-inner-box" ng-if="agg.type === 'terms'">
<div class="tight-form-inner-box" ng-if="agg.type === 'terms'">
<div class="tight-form">
<ul class="tight-form-list">
<li class="tight-form-item" style="width: 60px">
......
public/app/plugins/datasource/elasticsearch/partials/metricAgg.html
......@@ -26,7 +26,7 @@
</div>
<div class="tight-form" ng-if="showOptions">
<div style="margin: 20px 0 20px 148px;display: inline-block">
<div class="tight-form-inner-box">
<div class="tight-form last" ng-if="agg.type === 'percentiles'">
<ul class="tight-form-list">
<li class="tight-form-item">
......@@ -38,5 +38,29 @@
</ul>
<div class="clearfix"></div>
</div>
<div ng-if="agg.type === 'extended_stats'">
<div class="tight-form" ng-repeat="stat in extendedStats">
<ul class="tight-form-list">
<li class="tight-form-item" style="width: 100px">
{{stat.text}}
</li>
<li class="tight-form-item last">
<editor-checkbox text="" model="agg.meta.{{stat.value}}" change="onChange()"></editor-checkbox>
</li>
</ul>
<div class="clearfix"></div>
</div>
</div>
<div class="tight-form last" ng-if="agg.type === 'extended_stats'">
<ul class="tight-form-list">
<li class="tight-form-item" style="width: 100px">
Sigma
</li>
<li>
<input type="number" class="input-mini tight-form-input last" placeholder="3" ng-model="agg.settings.sigma" ng-blur="onChange()"></input>
</li>
</ul>
<div class="clearfix"></div>
</div>
</div>
</div>
public/app/plugins/datasource/elasticsearch/queryBuilder.js
......@@ -105,7 +105,7 @@ function (angular) {
var metricAgg = {field: metric.field};
for (var prop in metric.settings) {
if (metric.settings.hasOwnProperty(prop)) {
if (metric.settings.hasOwnProperty(prop) && metric.settings[prop] !== null) {
metricAgg[prop] = metric.settings[prop];
}
}
......
public/app/plugins/datasource/elasticsearch/queryDef.js
......@@ -13,6 +13,7 @@ function (_) {
{text: "Min of", value: 'min' },
{text: "Extended Stats", value: 'extended_stats' },
{text: "Percentiles", value: 'percentiles' },
{text: "Unique Count", value: "cardinality" }
],
bucketAggTypes: [
......@@ -41,6 +42,17 @@ function (_) {
{text: "20", value: '20' },
],
extendedStats: [
{text: 'Avg', value: 'avg'},
{text: 'Min', value: 'min'},
{text: 'Max', value: 'max'},
{text: 'Sum', value: 'sum'},
{text: 'Count', value: 'count'},
{text: 'Std Dev', value: 'std_deviation'},
{text: 'Std Dev Upper', value: 'std_deviation_bounds_upper'},
{text: 'Std Dev Lower', value: 'std_deviation_bounds_lower'},
],
getOrderByOptions: function(target) {
var self = this;
var metricRefs = [];
......
public/test/specs/elasticsearch-response-specs.js 0 → 100644
define([
'plugins/datasource/elasticsearch/elasticResponse',
], function(ElasticResponse) {
'use strict';
describe('ElasticResponse', function() {
var targets;
var response;
var result;
describe('simple query and count', function() {
beforeEach(function() {
targets = [{
refId: 'A',
metrics: [{type: 'count', id: '1'}],
bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '2'}],
}];
response = {
responses: [{
aggregations: {
"2": {
buckets: [
{
doc_count: 10,
key: 1000
},
{
doc_count: 15,
key: 2000
}
]
}
}
}]
};
result = new ElasticResponse(targets, response).getTimeSeries();
});
it('should return 1 series', function() {
expect(result.data.length).to.be(1);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].datapoints[0][0]).to.be(10);
expect(result.data[0].datapoints[0][1]).to.be(1000);
});
});
describe('simple query count & avg aggregation', function() {
var result;
beforeEach(function() {
targets = [{
refId: 'A',
metrics: [{type: 'count', id: '1'}, {type: 'avg', field: 'value', id: '2'}],
bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '3'}],
}];
response = {
responses: [{
aggregations: {
"3": {
buckets: [
{
"2": {value: 88},
doc_count: 10,
key: 1000
},
{
"2": {value: 99},
doc_count: 15,
key: 2000
}
]
}
}
}]
};
result = new ElasticResponse(targets, response).getTimeSeries();
});
it('should return 2 series', function() {
expect(result.data.length).to.be(2);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].datapoints[0][0]).to.be(10);
expect(result.data[0].datapoints[0][1]).to.be(1000);
expect(result.data[1].target).to.be("A value avg");
expect(result.data[1].datapoints[0][0]).to.be(88);
expect(result.data[1].datapoints[1][0]).to.be(99);
});
});
describe('single group by query', function() {
var result;
beforeEach(function() {
targets = [{
refId: 'A',
metrics: [{type: 'count', id: '1'}],
bucketAggs: [{type: 'terms', field: 'host', id: '2'}, {type: 'date_histogram', field: '@timestamp', id: '3'}],
}];
response = {
responses: [{
aggregations: {
"2": {
buckets: [
{
"3": {
buckets: [
{doc_count: 1, key: 1000},
{doc_count: 3, key: 2000}
]
},
doc_count: 4,
key: 'server1',
},
{
"3": {
buckets: [
{doc_count: 2, key: 1000},
{doc_count: 8, key: 2000}
]
},
doc_count: 10,
key: 'server2',
},
]
}
}
}]
};
result = new ElasticResponse(targets, response).getTimeSeries();
});
it('should return 2 series', function() {
expect(result.data.length).to.be(2);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].target).to.be('A server1 count');
expect(result.data[1].target).to.be('A server2 count');
});
});
describe('with percentiles ', function() {
var result;
beforeEach(function() {
targets = [{
refId: 'A',
metrics: [{type: 'percentiles', settings: {percents: [75, 90]}, id: '1'}],
bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '3'}],
}];
response = {
responses: [{
aggregations: {
"3": {
buckets: [
{
"1": {values: {"75": 3.3, "90": 5.5}},
doc_count: 10,
key: 1000
},
{
"1": {values: {"75": 2.3, "90": 4.5}},
doc_count: 15,
key: 2000
}
]
}
}
}]
};
result = new ElasticResponse(targets, response).getTimeSeries();
});
it('should return 2 series', function() {
expect(result.data.length).to.be(2);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].target).to.be('A 75');
expect(result.data[1].target).to.be('A 90');
expect(result.data[0].datapoints[0][0]).to.be(3.3);
expect(result.data[0].datapoints[0][1]).to.be(1000);
expect(result.data[1].datapoints[1][0]).to.be(4.5);
});
});
describe('with extended_stats ', function() {
var result;
beforeEach(function() {
targets = [{
refId: 'A',
metrics: [{type: 'extended_stats', meta: {max: true, std_deviation_bounds_upper: true}, id: '1'}],
bucketAggs: [{type: 'date_histogram', id: '3'}],
}];
response = {
responses: [{
aggregations: {
"3": {
buckets: [
{
"1": {max: 10.2, min: 5.5, std_deviation_bounds: {upper: 3, lower: -2}},
doc_count: 10,
key: 1000
},
{
"1": {max: 7.2, min: 3.5, std_deviation_bounds: {upper: 4, lower: -1}},
doc_count: 15,
key: 2000
}
]
}
}
}]
};
result = new ElasticResponse(targets, response).getTimeSeries();
});
it('should return 2 series', function() {
expect(result.data.length).to.be(2);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].target).to.be('A max');
expect(result.data[1].target).to.be('A std_deviation_bounds_upper');
expect(result.data[0].datapoints[0][0]).to.be(10.2);
expect(result.data[0].datapoints[1][0]).to.be(7.2);
expect(result.data[1].datapoints[0][0]).to.be(3);
expect(result.data[1].datapoints[1][0]).to.be(4);
});
});
});
});
public/test/specs/elasticsearch-specs.js
......@@ -3,7 +3,6 @@ define([
'moment',
'angular',
'plugins/datasource/elasticsearch/datasource',
'aws-sdk',
], function(helpers, moment, angular) {
'use strict';
......@@ -18,7 +17,7 @@ define([
});
describe('When testing datasource with index pattern', function() {
beforeEach(function(){
beforeEach(function() {
ctx.ds = new ctx.service({
url: 'http://es.com',
index: '[asd-]YYYY.MM.DD',
......@@ -70,180 +69,9 @@ define([
var header = angular.fromJson(parts[0]);
expect(header.index).to.eql(['asd-2015.05.30', 'asd-2015.05.31', 'asd-2015.06.01']);
});
});
describe('When processing es response', function() {
describe('simple query and count', function() {
var result;
beforeEach(function() {
result = ctx.ds._processTimeSeries([{
refId: 'A',
metrics: [{type: 'count', id: '1'}],
bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '2'}],
}], {
responses: [{
aggregations: {
"2": {
buckets: [
{
doc_count: 10,
key: 1000
},
{
doc_count: 15,
key: 2000
}
]
}
}
}]
});
});
it('should return 1 series', function() {
expect(result.data.length).to.be(1);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].datapoints[0][0]).to.be(10);
expect(result.data[0].datapoints[0][1]).to.be(1000);
});
});
describe('simple query count & avg aggregation', function() {
var result;
beforeEach(function() {
result = ctx.ds._processTimeSeries([{
refId: 'A',
metrics: [{type: 'count', id: '1'}, {type: 'avg', field: 'value', id: '2'}],
bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '3'}],
}], {
responses: [{
aggregations: {
"3": {
buckets: [
{
"2": {value: 88},
doc_count: 10,
key: 1000
},
{
"2": {value: 99},
doc_count: 15,
key: 2000
}
]
}
}
}]
});
});
it('should return 2 series', function() {
expect(result.data.length).to.be(2);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].datapoints[0][0]).to.be(10);
expect(result.data[0].datapoints[0][1]).to.be(1000);
expect(result.data[1].target).to.be("A value avg");
expect(result.data[1].datapoints[0][0]).to.be(88);
expect(result.data[1].datapoints[1][0]).to.be(99);
});
});
describe('single group by query', function() {
var result;
beforeEach(function() {
result = ctx.ds._processTimeSeries([{
refId: 'A',
metrics: [{type: 'count', id: '1'}],
bucketAggs: [{type: 'terms', field: 'host', id: '2'}, {type: 'date_histogram', field: '@timestamp', id: '3'}],
}], {
responses: [{
aggregations: {
"2": {
buckets: [
{
"3": {
buckets: [
{doc_count: 1, key: 1000},
{doc_count: 3, key: 2000}
]
},
doc_count: 4,
key: 'server1',
},
{
"3": {
buckets: [
{doc_count: 2, key: 1000},
{doc_count: 8, key: 2000}
]
},
doc_count: 10,
key: 'server2',
},
]
}
}
}]
});
});
it('should return 2 series', function() {
expect(result.data.length).to.be(2);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].target).to.be('A server1 count');
expect(result.data[1].target).to.be('A server2 count');
});
});
describe('with percentiles ', function() {
var result;
beforeEach(function() {
result = ctx.ds._processTimeSeries([{
refId: 'A',
metrics: [{type: 'percentiles', settings: {percents: [75, 90]}, id: '1'}],
bucketAggs: [{type: 'date_histogram', field: '@timestamp', id: '3'}],
}], {
responses: [{
aggregations: {
"3": {
buckets: [
{
"1": {values: {"75": 3.3, "90": 5.5}},
doc_count: 10,
key: 1000
},
{
"1": {values: {"75": 2.3, "90": 4.5}},
doc_count: 15,
key: 2000
}
]
}
}
}]
});
});
it('should return 2 series', function() {
expect(result.data.length).to.be(2);
expect(result.data[0].datapoints.length).to.be(2);
expect(result.data[0].target).to.be('A 75');
expect(result.data[1].target).to.be('A 90');
expect(result.data[0].datapoints[0][0]).to.be(3.3);
expect(result.data[0].datapoints[0][1]).to.be(1000);
expect(result.data[1].datapoints[1][0]).to.be(4.5);
});
});
});
});
});
public/test/test-main.js
......@@ -156,6 +156,7 @@ require([
'specs/elasticsearch-querybuilder-specs',
'specs/elasticsearch-queryctrl-specs',
'specs/elasticsearch-indexPattern-specs',
'specs/elasticsearch-response-specs',
];
var pluginSpecs = (config.plugins.specs || []).map(function (spec) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment