Skip to content

N
nexpie-grafana-theme
Commit a17e8ea4 by Rashid Khan
Options

A few performance improvements for events with a large number of fields

parent 40e55f0d
Showing with 15 additions and 22 deletions
src/app/components/kbn.js
...@@ -13,15 +13,10 @@ function($, _, moment) { ...@@ -13,15 +13,10 @@ function($, _, moment) {
return field_array.sort(); return field_array.sort();
}; };
kbn.get_all_fields = function(data) { kbn.get_all_fields = function(data,flat) {
var _d = data; return _.uniq(_.without(_.reduce(data,function(memo,hit) {
var fields = []; return flat ? memo.concat(_.keys(kbn.flatten_json(hit._source))) : memo.concat(_.keys(hit._source));
_.each(_d,function(hit) { },[]),'$$hashkey'));
fields = _.uniq(fields.concat(_.keys(kbn.flatten_json(hit._source))));
});
// Remove stupid angular key
fields = _.without(fields,'$$hashKey');
return fields;
}; };
kbn.has_field = function(obj,field) { kbn.has_field = function(obj,field) {
......
src/app/panels/table/module.html
...@@ -19,11 +19,6 @@ ...@@ -19,11 +19,6 @@
<ul class="unstyled" style="{{panel.overflow}}:{{panel.height || row.height}};overflow-y:auto;overflow-x:hidden;"> <ul class="unstyled" style="{{panel.overflow}}:{{panel.height || row.height}};overflow-y:auto;overflow-x:hidden;">
<li ng-style="panel.style" ng-repeat="field in metaFields|filter:fieldFilter|orderBy:identity">
<i class="pointer" ng-class="{'icon-check': _.contains(panel.fields,field),'icon-check-empty': !_.contains(panel.fields,field)}" ng-click="toggle_field(field)"></i>
<a class="pointer" data-unique="1" bs-popover="'app/panels/table/micropanel.html'" data-placement="rightTop" ng-click="toggle_micropanel(field,true)" ng-class="{label: _.contains(panel.fields,field)}">{{field}}</a>
</li>
<li ng-style="panel.style" ng-repeat="field in fields.list|filter:fieldFilter|orderBy:identity" ng-show="panel.all_fields"> <li ng-style="panel.style" ng-repeat="field in fields.list|filter:fieldFilter|orderBy:identity" ng-show="panel.all_fields">
<i class="pointer" ng-class="{'icon-check': _.contains(panel.fields,field),'icon-check-empty': !_.contains(panel.fields,field)}" ng-click="toggle_field(field)"></i> <i class="pointer" ng-class="{'icon-check': _.contains(panel.fields,field),'icon-check-empty': !_.contains(panel.fields,field)}" ng-click="toggle_field(field)"></i>
<a class="pointer" data-unique="1" bs-popover="'app/panels/table/micropanel.html'" data-placement="rightTop" ng-click="toggle_micropanel(field,true)" ng-class="{label: _.contains(panel.fields,field)}">{{field}}</a> <a class="pointer" data-unique="1" bs-popover="'app/panels/table/micropanel.html'" data-placement="rightTop" ng-click="toggle_micropanel(field,true)" ng-class="{label: _.contains(panel.fields,field)}">{{field}}</a>
...@@ -68,7 +63,7 @@ ...@@ -68,7 +63,7 @@
</thead> </thead>
<tbody ng-repeat="event in data| slice:panel.offset:panel.offset+panel.size" ng-class-odd="'odd'"> <tbody ng-repeat="event in data| slice:panel.offset:panel.offset+panel.size" ng-class-odd="'odd'">
<tr ng-click="toggle_details(event)" class="pointer"> <tr ng-click="toggle_details(event)" class="pointer">
<td ng-show="panel.fields.length<1">{{event.kibana._source|stringify|tableTruncate:panel.trimFactor:1}}</td> <td ng-show="panel.fields.length<1">{{event._source|stringify|tableTruncate:panel.trimFactor:1}}</td>
<td ng-show="panel.fields.length>0" ng-repeat="field in panel.fields" ng-bind-html-unsafe="(event.kibana.highlight[field]||event.kibana._source[field]) |tableHighlight | tableTruncate:panel.trimFactor:panel.fields.length"></td> <td ng-show="panel.fields.length>0" ng-repeat="field in panel.fields" ng-bind-html-unsafe="(event.kibana.highlight[field]||event.kibana._source[field]) |tableHighlight | tableTruncate:panel.trimFactor:panel.fields.length"></td>
</tr> </tr>
<tr ng-show="event.kibana.details"> <tr ng-show="event.kibana.details">
......
src/app/panels/table/module.js
...@@ -87,7 +87,6 @@ function (angular, app, _, kbn, moment) { ...@@ -87,7 +87,6 @@ function (angular, app, _, kbn, moment) {
$scope.init = function () { $scope.init = function () {
$scope.Math = Math; $scope.Math = Math;
$scope.metaFields = [];
$scope.identity = angular.identity; $scope.identity = angular.identity;
$scope.$on('refresh',function(){$scope.get_data();}); $scope.$on('refresh',function(){$scope.get_data();});
...@@ -258,6 +257,7 @@ function (angular, app, _, kbn, moment) { ...@@ -258,6 +257,7 @@ function (angular, app, _, kbn, moment) {
if(_segment === 0) { if(_segment === 0) {
$scope.hits = 0; $scope.hits = 0;
$scope.data = []; $scope.data = [];
$scope.current_fields = [];
query_id = $scope.query_id = new Date().getTime(); query_id = $scope.query_id = new Date().getTime();
} }
...@@ -269,21 +269,26 @@ function (angular, app, _, kbn, moment) { ...@@ -269,21 +269,26 @@ function (angular, app, _, kbn, moment) {
// Check that we're still on the same query, if not stop // Check that we're still on the same query, if not stop
if($scope.query_id === query_id) { if($scope.query_id === query_id) {
$scope.data= $scope.data.concat(_.map(results.hits.hits, function(hit) {
// This is exceptionally expensive, especially on events with a large number of fields
$scope.data = $scope.data.concat(_.map(results.hits.hits, function(hit) {
var var
_h = _.clone(hit), _h = _.clone(hit),
_p = _.omit(hit,'_source','sort','_score'); _p = _.omit(hit,'_source','sort','_score');
$scope.metaFields = _.union(_.keys(_p),$scope.metaFields);
// _source is kind of a lie here, never display it, only select values from it // _source is kind of a lie here, never display it, only select values from it
_h.kibana = { _h.kibana = {
_source : _.extend(kbn.flatten_json(hit._source),_p), _source : _.extend(kbn.flatten_json(hit._source),_p),
highlight : kbn.flatten_json(hit.highlight||{}) highlight : kbn.flatten_json(hit.highlight||{})
}; };
// Kind of cheating with the _.map here, but this is faster than kbn.get_all_fields
$scope.current_fields = $scope.current_fields.concat(_.keys(_h.kibana._source));
return _h; return _h;
})); }));
$scope.current_fields = _.uniq($scope.current_fields);
$scope.hits += results.hits.total; $scope.hits += results.hits.total;
// Sort the data // Sort the data
...@@ -303,9 +308,6 @@ function (angular, app, _, kbn, moment) { ...@@ -303,9 +308,6 @@ function (angular, app, _, kbn, moment) {
// Keep only what we need for the set // Keep only what we need for the set
$scope.data = $scope.data.slice(0,$scope.panel.size * $scope.panel.pages); $scope.data = $scope.data.slice(0,$scope.panel.size * $scope.panel.pages);
// Populate current_fields list
$scope.current_fields = kbn.get_all_fields($scope.data);
} else { } else {
return; return;
} }
......
src/app/services/fields.js
...@@ -21,6 +21,7 @@ function (angular, _, config) { ...@@ -21,6 +21,7 @@ function (angular, _, config) {
var indices = _.difference(n,_.keys(self.mapping)); var indices = _.difference(n,_.keys(self.mapping));
// Only get the mapping if there are new indices // Only get the mapping if there are new indices
if(indices.length > 0) { if(indices.length > 0) {
console.log('getting mapping');
self.map(indices).then(function(result) { self.map(indices).then(function(result) {
self.mapping = _.extend(self.mapping,result); self.mapping = _.extend(self.mapping,result);
self.list = mapFields(self.mapping); self.list = mapFields(self.mapping);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment