working on metic key indexing

src/app/controllers/metricKeys.js

@@ -8,73 +8,116 @@ function (angular, _, config) {
 
 
   var module = angular.module('kibana.controllers');
   var module = angular.module('kibana.controllers');
 
 
-  module.controller('MetricKeysCtrl', function($scope, $http) {
+  module.controller('MetricKeysCtrl', function($scope, $http, $q) {
+    var elasticSearchUrlForMetricIndex = config.elasticsearch + '/' + config.grafana_index + '/';
+    var loadingDefered;
 
 
     $scope.init = function () {
     $scope.init = function () {
       $scope.metricPath = "prod.apps.api.boobarella.*";
       $scope.metricPath = "prod.apps.api.boobarella.*";
+      $scope.metricCounter = 0;
     };
     };
 
 
-    $scope.loadMetricsFromPath = function () {
-      $scope.infoText = 'loading';
-      loadMetricsRecursive($scope.metricPath)
-
-      /*$http.put(config.elasticsearch + "/grafana-int/", {
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "category_path": {
-          "type": "custom",
-          "tokenizer": "category_path"
-        },
-        "my_ngram_analyzer" : {
-      "tokenizer" : "my_ngram_tokenizer"
+    function deleteIndex()
+    {
+      var deferred = $q.defer();
+      $http.delete(elasticSearchUrlForMetricIndex)
+        .success(function() {
+          deferred.resolve('ok');
+        })
+        .error(function(data, status) {
+          if (status === 404) {
+            deferred.resolve('ok');
+          }
+          else {
+            deferred.reject('elastic search returned unexpected error');
+          }
+        });
+
+      return deferred.promise;
     }
     }
-      },
-      "tokenizer": {
-        "category_path": {
-          "type": "path_hierarchy",
-          "delimiter": "."
+
+    function createIndex()
+    {
+      return $http.put(elasticSearchUrlForMetricIndex, {
+        settings: {
+          analysis: {
+            analyzer: {
+              metric_path_ngram : { tokenizer : "my_ngram_tokenizer" }
+            },
+            tokenizer: {
+              my_ngram_tokenizer : {
+                type : "nGram",
+                min_gram : "3",
+                max_gram : "8",
+                token_chars: [ "letter", "digit", "punctuation", "symbol"]
+              }
+            }
+          }
         },
         },
-        "my_ngram_tokenizer" : {
-          "type" : "nGram",
-          "min_gram" : "4",
-          "max_gram" : "8",
-          "token_chars": [ "letter", "digit" ]
+        mappings: {
+          metricKey: {
+            properties: {
+              metricPath: {
+/*                type: "string",
+                index: "analyzed",
+                index_analyzer: "metric_path_ngram"
+*/
+                type: "multi_field",
+                fields: {
+                  "metricPath": { type: "string", index: "analyzed", index_analyzer: "standard" },
+                  "metricPath_ng": { type: "string", index: "analyzed", index_analyzer: "metric_path_ngram" }
+                }
+              }
+            }
+          }
         }
         }
-      }
-    }
-  },
-  "mappings": {
-    "metricKey": {
-      "properties": {
-        "metricPath": {
-          "type": "string",
-          "index": "analyzed",
-          "index_analyzer": "my_ngram_analyzer"
-    }
-      }
+      });
     }
     }
-  }
-}
-      });*/
+
+    $scope.createIndex = function () {
+      $scope.errorText = null;
+      $scope.infoText = null;
+
+      deleteIndex()
+        .then(createIndex)
+        .then(function () {
+          $scope.infoText = "Index created!";
+        })
+        .then(null, function (err) {
+          $scope.errorText = angular.toJson(err);
+        });
     };
     };
 
 
-    function receiveMetric(data) {
+    $scope.loadMetricsFromPath = function () {
+      $scope.errorText = null;
+      $scope.infoText = null;
+      $scope.metricCounter = 0;
+
+      return loadMetricsRecursive($scope.metricPath)
+        .then(function() {
+          $scope.infoText = "Indexing completed!";
+        }, function(err) {
+          $scope.errorText = "Error: " + err;
+        });
+    };
+
+    function receiveMetric(result) {
+      var data = result.data;
       if (!data || data.length == 0) {
       if (!data || data.length == 0) {
         console.log('no data');
         console.log('no data');
         return;
         return;
       }
       }
 
 
-      _.each(data, function(metric) {
+      var funcs = _.map(data, function(metric) {
         if (metric.expandable) {
         if (metric.expandable) {
-          console.log('Loading children: ' + metric.id);
-          loadMetricsRecursive(metric.id + ".*");
+          return loadMetricsRecursive(metric.id + ".*");
         }
         }
         if (metric.leaf) {
         if (metric.leaf) {
-          saveMetricKey(metric.id);
+          return saveMetricKey(metric.id);
         }
         }
       });
       });
+
+      return $q.all(funcs);
     }
     }
 
 
     function saveMetricKey(metricId) {
     function saveMetricKey(metricId) {
@@ -84,28 +127,27 @@ function (angular, _, config) {
         metricPath: metricId
         metricPath: metricId
       });
       });
 
 
-      request.doIndex(
+      return request.doIndex(
         // Success
         // Success
         function(result) {
         function(result) {
-          console.log('save metric success', result);
+          $scope.infoText = "Indexing " + metricId;
+          $scope.metricCounter = $scope.metricCounter + 1;
         },
         },
         function(error) {
         function(error) {
-          console.log('save metric error', error);
+          $scope.errorText = "failed to save metric " + metricId;
         }
         }
       );
       );
     }
     }
 
 
     function metricLoadError(data, status, headers, config)
     function metricLoadError(data, status, headers, config)
     {
     {
-      console.log('error: ' + status);
-      $scope.error = "failed to get metrics from graphite";
+        $scope.errorText = "failed to get metric";
     }
     }
 
 
-    function loadMetricsRecursive(metricPath, data, callback)
+    function loadMetricsRecursive(metricPath)
     {
     {
-      $http({ method: 'GET', url: config.graphiteUrl + '/metrics/find/?query=' + metricPath} )
-        .success(receiveMetric)
-        .error(metricLoadError);
+      return $http({ method: 'GET', url: config.graphiteUrl + '/metrics/find/?query=' + metricPath} )
+              .then(receiveMetric);
     }
     }
 
 
   });
   });

src/app/controllers/search.js

@@ -21,15 +21,18 @@ function (angular, _, config, $) {
       };
       };
     };
     };
 
 
-    $scope.elasticsearch_dblist = function(query) {
-      var words = query.split(" ");
+    $scope.elasticsearch_dblist = function(queryStr) {
+      var words = queryStr.split(" ");
       var query = $scope.ejs.BoolQuery();
       var query = $scope.ejs.BoolQuery();
       var terms = _.map(words, function(word) {
       var terms = _.map(words, function(word) {
-        return $scope.ejs.MatchQuery("metricPath", word);
+        return $scope.ejs.MatchQuery("metricPath_ng", word).boost(1.2);
       });
       });
 
 
-      console.log("query: ", terms);
-      query.must(terms);
+      var ngramQuery = $scope.ejs.BoolQuery();
+      ngramQuery.must(terms);
+
+      var fieldMatchQuery = $scope.ejs.FieldQuery('metricPath', queryStr + "*").boost(1.2);
+      query.should([ngramQuery, fieldMatchQuery]);
 
 
       var request = $scope.ejs.Request().indices(config.grafana_index).types('metricKey');
       var request = $scope.ejs.Request().indices(config.grafana_index).types('metricKey');
       var results = request.query(query).size(20).doSearch();
       var results = request.query(query).size(20).doSearch();

src/app/panels/graphite/module.js

@@ -416,7 +416,7 @@ function (angular, app, $, _, kbn, moment, timeSeries, graphiteSrv, RQ) {
       restrict: 'A',
       restrict: 'A',
       template: '<div></div>',
       template: '<div></div>',
       link: function(scope, elem) {
       link: function(scope, elem) {
-        var data = [];
+        var data, plot;
 
 
         scope.$on('refresh',function() {
         scope.$on('refresh',function() {
           scope.get_data();
           scope.get_data();
@@ -558,7 +558,7 @@ function (angular, app, $, _, kbn, moment, timeSeries, graphiteSrv, RQ) {
             console.log('Datapoints[0] count:', data[0].data.length);
             console.log('Datapoints[0] count:', data[0].data.length);
             console.log('Datapoints.Total count:', totalDataPoints);*/
             console.log('Datapoints.Total count:', totalDataPoints);*/
 
 
-            scope.plot = $.plot(elem, data, options);
+            plot = $.plot(elem, data, options);
 
 
           } catch(e) {
           } catch(e) {
             console.log(e);
             console.log(e);

src/app/partials/loadmetrics.html

 <div ng-controller="MetricKeysCtrl" ng-init="init()">
 <div ng-controller="MetricKeysCtrl" ng-init="init()">
   <h5>Load metrics keys into elastic search</h5>
   <h5>Load metrics keys into elastic search</h5>
+
   <div class="row-fluid">
   <div class="row-fluid">
-    <div class="span8">
-      <label class="small">Load all metric keys at once (intensive for graphite web)</label>
+    <div class="span12">
+      <label class="small">Load metrics recursive starting from this metric path</label>
+      <input type="text" class="input-xlarge" ng-model="metricPath"> </input>
     </div>
     </div>
-    <div class="span4">
-      <button class="btn btn-danger">Start load all</button>
+  </div>
+
+  <div class="row-fluid" style="margin-top: 15px;">
+    <div class="span12">
+      <button class="btn btn-success" ng-click="createIndex()">Clear/Create index</button>
+      <button class="btn btn-success" ng-click="loadMetricsFromPath()">Load from metric path</button>
+      <button class="btn btn-danger" ng-click="loadAll()">Load all</button>
+      <tip>Load all will fetch all metrics in one call, can be intensive for graphite and for the browser if you have a lot of metrics</tip>
     </div>
     </div>
   </div>
   </div>
-  <div class="row-fluid">
-    <div class="span8">
-      <label class="small">Load metric keys recursively starting at metric key path</label>
-      <input type="text" class="input-xlarge" ng-model="metricPath"> </input>
+
+  <div class="row-fluid" style="margin-top: 15px;">
+    <div class="span12" ng-show="infoText" style="padding-top: 10px;">
+      {{infoText}}
     </div>
     </div>
-    <div class="span4">
-      <button class="btn btn-danger" style="margin-top: 20px" ng-click="loadMetricsFromPath()">Start load from metric path</button>
+    <div class="span12 alert alert-error" ng-show="errorText">
+      {{errorText}}
     </div>
     </div>
   </div>
   </div>
-  <div class="row-fluid">
-    <div class="span12 alert-message block-message info">
-      {{infoText}}
+  <div class="row-fluid" ng-show="metricCounter">
+    <div class="span12" style="padding-top: 10px;">
+      Metrics indexed: {{metricCounter}}
     </div>
     </div>
   </div>
   </div>
-</div>
+</div>
\ No newline at end of file