Efficient sort of arbitrary fields in table panel. Sweet

common/css/main.css

@@ -62,6 +62,10 @@
   font-size: 85%;
 }
 
+.large {
+  font-size: 120%;
+}
+
 .nomargin {
   margin: 0px;
 }

panels/fields/micropanel.html

@@ -3,13 +3,13 @@
   Micro Analysis of {{micropanel.field}} 
   <i class="pointer icon-search" ng-click="build_search('_exists_',micropanel.field);dismiss();"></i>
   <i class="pointer icon-ban-circle" ng-click="build_search('_missing_',micropanel.field);dismiss();"></i>
-  <br><small>{{micropanel.count}} events on this page</small>
+  <br><small>{{micropanel.count}} events in the table set</small>
 </h4>
 <table style="width:500px" class='table table-bordered table-striped table-condensed'>
   <thead>
     <th>{{micropanel.field}}</th>
     <th>Action</th>
-    <th>On Page</th>
+    <th>In set</th>
   </thead>
   <tbody>
     <tr ng-repeat='field in micropanel.values'>

panels/histogram/module.js

@@ -52,7 +52,7 @@ angular.module('kibana.histogram', [])
     if(_.isUndefined($scope.panel.index) || _.isUndefined($scope.time))
       return
 
-    _segment = _.isUndefined(segment) ? 0 : segment
+    var _segment = _.isUndefined(segment) ? 0 : segment
 
     $scope.panel.loading = true;
     var request = $scope.ejs.Request().indices($scope.panel.index[_segment]);

panels/hits/module.html

 <kibana-panel ng-controller='hits' ng-init="init()">
-  <p ng-style="panel.style">{{hits}}</p>
+  <p ng-style="panel.style">&#8805 {{hits}}</p>
 </kibana-panel>         
\ No newline at end of file

panels/table/editor.html

   <div class="row-fluid" ng-controller="table">    
-    <div class="span12">
+    <div style="width:90%">
       <form class="input-append">
         <h6>Query</h6>
         <input type="text" style="width:90%" ng-model="panel.query">
@@ -20,9 +20,10 @@
       <span style="margin-left:3px" ng-click="toggle_field(field)" ng-repeat="field in $parent.panel.fields" class="label remove pointer">{{field}} </span>
     </div>
   </div>
+  <h5>Sorting</h5>
   <div class="row-fluid">
-    <div class="span2"> 
-      <h6>Sortable</h6><input type="checkbox" ng-model="panel.sortable" ng-checked="panel.sortable">
+    <div class="span1"> 
+      <h6>Enable</h6><input type="checkbox" ng-model="panel.sortable" ng-checked="panel.sortable">
     </div>
     <div class="span4" style="white-space:nowrap" ng-show='panel.sortable'>
       <h6>Sort</h6>
@@ -30,12 +31,27 @@
       <select ng-show="all_fields.length>0"style="width:85%" ng-model="panel.sort[0]" ng-options="f for f in all_fields"></select>
       <i ng-click="set_sort(panel.sort[0])" ng-class="{'icon-chevron-up': panel.sort[1] == 'asc','icon-chevron-down': panel.sort[1] == 'desc'}"></i>
     </div>
-    <div class="span3">
-      <h6>Length</h6>
-      <input type="number" class="input-mini" ng-model="panel.size" ng-change="get_data()">
   </div>
+  <h5>Paging and Appearence</h5>
+  <div class="row-fluid">
     <div class="span3"><h6>Font Size</h6> 
-      <select class="input-small" ng-model="panel.style['font-size']" ng-options="f for f in ['6pt','7pt','8pt','10pt','12pt','14pt','16pt','18pt','20pt','24pt','28pt','32pt','36pt','42pt','48pt','52pt','60pt','72pt']"></select></span>
+      <select class="input-small" ng-model="panel.style['font-size']" ng-options="f for f in ['7pt','8pt','9pt','10pt','12pt','14pt','16pt','18pt','20pt','24pt','28pt','32pt','36pt','42pt','48pt','52pt','60pt','72pt']"></select></span>
+    </div>
+    <div class="span2">
+      <h6>Per Page</h6>
+      <input type="number" class="input-mini" ng-model="panel.size" ng-change="get_data()">
+    </div>
+    <div class="span1">
+      <h6>&nbsp;</h6>
+      <center><i class='icon-remove'></i><center>
+    </div>
+    <div class="span2">
+      <h6>Page limit</h6>
+      <input type="number" class="input-mini" ng-model="panel.pages" ng-change="get_data()">
+    </div>
+    <div class="span2 large">
+      <h6>Pageable</h6>
+      <strong>= {{panel.size * panel.pages}}</strong>
     </div>
   </div>
   <!--<div class="row-fluid" ng-show='panel.sortable'>

panels/table/module.html

@@ -7,20 +7,21 @@
   <div style="height:{{panel.height || row.height}};overflow-y:auto;overflow-x:auto">
     <div class="row-fluid">
       <div class="span1 offset3" style="text-align:right">
-        <i ng-click="panel.offset = 0;get_data();" ng-show="panel.offset > 0" class='icon-circle-arrow-left pointer'></i>
-        <i ng-click="panel.offset = (panel.offset - panel.size);get_data();" ng-show="panel.offset > 0" class='icon-arrow-left pointer'></i>
+        <i ng-click="panel.offset = 0" ng-show="panel.offset > 0" class='icon-circle-arrow-left pointer'></i>
+        <i ng-click="panel.offset = (panel.offset - panel.size)" ng-show="panel.offset > 0" class='icon-arrow-left pointer'></i>
       </div>
       <div class="span4" style="text-align:center">
-        <strong>{{panel.offset}}</strong> to <strong>{{panel.offset + data.length}}</strong>
-        <small> of &#8805 {{hits}} hits</small>
+        <strong>{{panel.offset}}</strong> to <strong>{{panel.offset + data.slice(panel.offset,panel.offset+panel.size).length}}</strong>
+        <small> of {{data.length}} available for paging</small>
       </div>
       <div class="span1" style="text-align:left">
-        <i ng-click="panel.offset = (panel.offset + panel.size);get_data();" ng-show="hits > (panel.offset + data.length)" class='icon-arrow-right pointer'></i>
+        <i ng-click="panel.offset = (panel.offset + panel.size)" ng-show="data.length > panel.offset+panel.size" class='icon-arrow-right pointer'></i>
       </div>
     </div>
     <div class="small" ng-show="panel.fields.length == 0">No columns configured. You may want to add a <strong>fields panel</strong>, or click the edit button in the top right of this panel to add some columns</div>
     <table class="table-hover table table-condensed" ng-style="panel.style">
       <thead>
+        <th></th>
         <th style="white-space:nowrap" ng-repeat="field in panel.fields">
           <span  class="pointer" ng-click="set_sort(field)" ng-show='panel.sortable'>
             {{field}} 
@@ -29,9 +30,9 @@
           <span ng-show='!panel.sortable'>{{field}}</span>
         </th>
       </thead>
-      <tbody ng-repeat="row in data" ng-class-odd="'odd'">
+      <tbody ng-repeat="row in data.slice(panel.offset,panel.offset+panel.size)" ng-class-odd="'odd'">
         <tr ng-click="toggle_details(row)">
-          <td ng-repeat="field in panel.fields">{{row[field]}}</td>
+          <td>{{$index}}</td><td ng-repeat="field in panel.fields">{{row[field]}}</td>
         </tr>
         <tr ng-show="row.kibana.details">
           <td colspan=1000>
@@ -56,15 +57,15 @@
     </table>
     <div class="row-fluid">
       <div class="span1 offset3" style="text-align:right">
-        <i ng-click="panel.offset = 0;get_data();" ng-show="panel.offset > 0" class='icon-circle-arrow-left pointer'></i>
-        <i ng-click="panel.offset = (panel.offset - panel.size);get_data();" ng-show="panel.offset > 0" class='icon-arrow-left pointer'></i>
+        <i ng-click="panel.offset = 0" ng-show="panel.offset > 0" class='icon-circle-arrow-left pointer'></i>
+        <i ng-click="panel.offset = (panel.offset - panel.size)" ng-show="panel.offset > 0" class='icon-arrow-left pointer'></i>
       </div>
       <div class="span4" style="text-align:center">
-        <strong>{{panel.offset}}</strong> to <strong>{{panel.offset + data.length}}</strong>
-        <small> of &#8805 {{hits}} hits</small>
+        <strong>{{panel.offset}}</strong> to <strong>{{panel.offset + data.slice(panel.offset,panel.offset+panel.size).length}}</strong>
+        <small> of {{data.length}} available for paging</small>
       </div>
       <div class="span1" style="text-align:left">
-        <i ng-click="panel.offset = (panel.offset + panel.size);get_data();" ng-show="hits > (panel.offset + data.length)" class='icon-arrow-right pointer'></i>
+        <i ng-click="panel.offset = (panel.offset + panel.size)" ng-show="data.length > panel.offset+panel.size" class='icon-arrow-right pointer'></i>
       </div>
     </div>
   </div>

panels/table/module.js

@@ -4,11 +4,12 @@ angular.module('kibana.table', [])
   // Set and populate defaults
   var _d = {
     query   : "*",
-    size    : 100,
+    size    : 100, // Per page
+    pages   : 5,   // Pages available
     offset  : 0,
     sort    : ['@timestamp','desc'],
     group   : "default",
-    style   : {},
+    style   : {'font-size': '9pt'},
     fields  : [],
     sortable: true,
     spyable: true,
@@ -74,14 +75,16 @@ angular.module('kibana.table', [])
     eventBus.broadcast($scope.$id,$scope.panel.group,'query',$scope.panel.query);
   }
 
-  $scope.get_data = function() {
+  $scope.get_data = function(segment,query_id) {
     // Make sure we have everything for the request to complete
     if(_.isUndefined($scope.panel.index) || _.isUndefined($scope.time))
       return
     
     $scope.panel.loading = true;
 
-    var request = $scope.ejs.Request().indices($scope.panel.index)
+    var _segment = _.isUndefined(segment) ? 0 : segment
+
+    var request = $scope.ejs.Request().indices($scope.panel.index[_segment])
       .query(ejs.FilteredQuery(
         ejs.QueryStringQuery($scope.panel.query || '*'),
         ejs.RangeFilter($scope.time.field)
@@ -89,8 +92,7 @@ angular.module('kibana.table', [])
           .to($scope.time.to)
         )
       )
-      .size($scope.panel.size)
-      .from($scope.panel.offset)
+      .size($scope.panel.size*$scope.panel.pages)
       .sort($scope.panel.sort[0],$scope.panel.sort[1]);
 
     $scope.populate_modal(request)
@@ -101,18 +103,57 @@ angular.module('kibana.table', [])
     results.then(function(results) {
       $scope.panel.loading = false;
 
+      if(_segment === 0) {
+        $scope.data = [];
+        query_id = $scope.query_id = new Date().getTime()
+      }
+
       if(_.isUndefined(results)) {
         $scope.panel.error = 'Your query was unsuccessful';
         return;
       }
       $scope.panel.error =  false;
       $scope.hits = results.hits.total;
-      $scope.data = _.map(results.hits.hits, function(hit) {
+
+
+      // Check that we're still on the same query, if not stop
+      if($scope.query_id === query_id) {
+        $scope.data= $scope.data.concat(_.map(results.hits.hits, function(hit) {
           return flatten_json(hit['_source']);
+        }));
+        
+        // Sort the data
+        $scope.data = _.sortBy($scope.data, function(v){
+          return v[$scope.panel.sort[0]]
         });
+
+        // Reverse if needed
+        if($scope.panel.sort[1] == 'desc')
+          $scope.data.reverse();
+        
+        // Keep only what we need for the set
+        $scope.data = $scope.data.slice(0,$scope.panel.size * $scope.panel.pages)
+
+      } else {
+        return;
+      }
+      
+      // This breaks, use $scope.data for this
       $scope.all_fields = get_all_fields(results);
    
       broadcast_results();
+
+      // If we're not sorting in reverse chrono order, query every index for
+      // size*pages results
+      // Otherwise, only get size*pages results then stop querying
+      if(
+          ($scope.data.length < $scope.panel.size*$scope.panel.pages || 
+            !(($scope.panel.sort[0] === $scope.time.field) && $scope.panel.sort[1] === 'desc')) && 
+          _segment+1 < $scope.panel.index.length
+      ) {
+        $scope.get_data(_segment+1,$scope.query_id)
+      }
+
     });
   }