Skip to content

N
nexpie-grafana-theme
Commit 45ce00c1 by Torkel Ödegaard
Options

working on metic key indexing

parent 58fef5d3
Showing with 122 additions and 69 deletions
src/app/controllers/metricKeys.js
...@@ -8,73 +8,116 @@ function (angular, _, config) { ...@@ -8,73 +8,116 @@ function (angular, _, config) {
var module = angular.module('kibana.controllers'); var module = angular.module('kibana.controllers');
module.controller('MetricKeysCtrl', function($scope, $http) { module.controller('MetricKeysCtrl', function($scope, $http, $q) {
var elasticSearchUrlForMetricIndex = config.elasticsearch + '/' + config.grafana_index + '/';
var loadingDefered;
$scope.init = function () { $scope.init = function () {
$scope.metricPath = "prod.apps.api.boobarella.*"; $scope.metricPath = "prod.apps.api.boobarella.*";
$scope.metricCounter = 0;
}; };
$scope.loadMetricsFromPath = function () { function deleteIndex()
$scope.infoText = 'loading'; {
loadMetricsRecursive($scope.metricPath) var deferred = $q.defer();
$http.delete(elasticSearchUrlForMetricIndex)
/*$http.put(config.elasticsearch + "/grafana-int/", { .success(function() {
{ deferred.resolve('ok');
"settings": { })
"analysis": { .error(function(data, status) {
"analyzer": { if (status === 404) {
"category_path": { deferred.resolve('ok');
"type": "custom", }
"tokenizer": "category_path" else {
}, deferred.reject('elastic search returned unexpected error');
"my_ngram_analyzer" : {
"tokenizer" : "my_ngram_tokenizer"
} }
});
return deferred.promise;
}
function createIndex()
{
return $http.put(elasticSearchUrlForMetricIndex, {
settings: {
analysis: {
analyzer: {
metric_path_ngram : { tokenizer : "my_ngram_tokenizer" }
}, },
"tokenizer": { tokenizer: {
"category_path": { my_ngram_tokenizer : {
"type": "path_hierarchy", type : "nGram",
"delimiter": "." min_gram : "3",
}, max_gram : "8",
"my_ngram_tokenizer" : { token_chars: [ "letter", "digit", "punctuation", "symbol"]
"type" : "nGram",
"min_gram" : "4",
"max_gram" : "8",
"token_chars": [ "letter", "digit" ]
} }
} }
} }
}, },
"mappings": { mappings: {
"metricKey": { metricKey: {
"properties": { properties: {
"metricPath": { metricPath: {
"type": "string", /* type: "string",
"index": "analyzed", index: "analyzed",
"index_analyzer": "my_ngram_analyzer" index_analyzer: "metric_path_ngram"
*/
type: "multi_field",
fields: {
"metricPath": { type: "string", index: "analyzed", index_analyzer: "standard" },
"metricPath_ng": { type: "string", index: "analyzed", index_analyzer: "metric_path_ngram" }
} }
} }
} }
} }
} }
});*/ });
}
$scope.createIndex = function () {
$scope.errorText = null;
$scope.infoText = null;
deleteIndex()
.then(createIndex)
.then(function () {
$scope.infoText = "Index created!";
})
.then(null, function (err) {
$scope.errorText = angular.toJson(err);
});
};
$scope.loadMetricsFromPath = function () {
$scope.errorText = null;
$scope.infoText = null;
$scope.metricCounter = 0;
return loadMetricsRecursive($scope.metricPath)
.then(function() {
$scope.infoText = "Indexing completed!";
}, function(err) {
$scope.errorText = "Error: " + err;
});
}; };
function receiveMetric(data) { function receiveMetric(result) {
var data = result.data;
if (!data || data.length == 0) { if (!data || data.length == 0) {
console.log('no data'); console.log('no data');
return; return;
} }
_.each(data, function(metric) { var funcs = _.map(data, function(metric) {
if (metric.expandable) { if (metric.expandable) {
console.log('Loading children: ' + metric.id); return loadMetricsRecursive(metric.id + ".*");
loadMetricsRecursive(metric.id + ".*");
} }
if (metric.leaf) { if (metric.leaf) {
saveMetricKey(metric.id); return saveMetricKey(metric.id);
} }
}); });
return $q.all(funcs);
} }
function saveMetricKey(metricId) { function saveMetricKey(metricId) {
...@@ -84,28 +127,27 @@ function (angular, _, config) { ...@@ -84,28 +127,27 @@ function (angular, _, config) {
metricPath: metricId metricPath: metricId
}); });
request.doIndex( return request.doIndex(
// Success // Success
function(result) { function(result) {
console.log('save metric success', result); $scope.infoText = "Indexing " + metricId;
$scope.metricCounter = $scope.metricCounter + 1;
}, },
function(error) { function(error) {
console.log('save metric error', error); $scope.errorText = "failed to save metric " + metricId;
} }
); );
} }
function metricLoadError(data, status, headers, config) function metricLoadError(data, status, headers, config)
{ {
console.log('error: ' + status); $scope.errorText = "failed to get metric";
$scope.error = "failed to get metrics from graphite";
} }
function loadMetricsRecursive(metricPath, data, callback) function loadMetricsRecursive(metricPath)
{ {
$http({ method: 'GET', url: config.graphiteUrl + '/metrics/find/?query=' + metricPath} ) return $http({ method: 'GET', url: config.graphiteUrl + '/metrics/find/?query=' + metricPath} )
.success(receiveMetric) .then(receiveMetric);
.error(metricLoadError);
} }
}); });
......
src/app/controllers/search.js
...@@ -21,15 +21,18 @@ function (angular, _, config, $) { ...@@ -21,15 +21,18 @@ function (angular, _, config, $) {
}; };
}; };
$scope.elasticsearch_dblist = function(query) { $scope.elasticsearch_dblist = function(queryStr) {
var words = query.split(" "); var words = queryStr.split(" ");
var query = $scope.ejs.BoolQuery(); var query = $scope.ejs.BoolQuery();
var terms = _.map(words, function(word) { var terms = _.map(words, function(word) {
return $scope.ejs.MatchQuery("metricPath", word); return $scope.ejs.MatchQuery("metricPath_ng", word).boost(1.2);
}); });
console.log("query: ", terms); var ngramQuery = $scope.ejs.BoolQuery();
query.must(terms); ngramQuery.must(terms);
var fieldMatchQuery = $scope.ejs.FieldQuery('metricPath', queryStr + "*").boost(1.2);
query.should([ngramQuery, fieldMatchQuery]);
var request = $scope.ejs.Request().indices(config.grafana_index).types('metricKey'); var request = $scope.ejs.Request().indices(config.grafana_index).types('metricKey');
var results = request.query(query).size(20).doSearch(); var results = request.query(query).size(20).doSearch();
......
src/app/panels/graphite/module.js
...@@ -416,7 +416,7 @@ function (angular, app, $, _, kbn, moment, timeSeries, graphiteSrv, RQ) { ...@@ -416,7 +416,7 @@ function (angular, app, $, _, kbn, moment, timeSeries, graphiteSrv, RQ) {
restrict: 'A', restrict: 'A',
template: '<div></div>', template: '<div></div>',
link: function(scope, elem) { link: function(scope, elem) {
var data = []; var data, plot;
scope.$on('refresh',function() { scope.$on('refresh',function() {
scope.get_data(); scope.get_data();
...@@ -558,7 +558,7 @@ function (angular, app, $, _, kbn, moment, timeSeries, graphiteSrv, RQ) { ...@@ -558,7 +558,7 @@ function (angular, app, $, _, kbn, moment, timeSeries, graphiteSrv, RQ) {
console.log('Datapoints[0] count:', data[0].data.length); console.log('Datapoints[0] count:', data[0].data.length);
console.log('Datapoints.Total count:', totalDataPoints);*/ console.log('Datapoints.Total count:', totalDataPoints);*/
scope.plot = $.plot(elem, data, options); plot = $.plot(elem, data, options);
} catch(e) { } catch(e) {
console.log(e); console.log(e);
......
src/app/partials/loadmetrics.html
<div ng-controller="MetricKeysCtrl" ng-init="init()"> <div ng-controller="MetricKeysCtrl" ng-init="init()">
<h5>Load metrics keys into elastic search</h5> <h5>Load metrics keys into elastic search</h5>
<div class="row-fluid"> <div class="row-fluid">
<div class="span8"> <div class="span12">
<label class="small">Load all metric keys at once (intensive for graphite web)</label> <label class="small">Load metrics recursive starting from this metric path</label>
<input type="text" class="input-xlarge" ng-model="metricPath"> </input>
</div> </div>
<div class="span4">
<button class="btn btn-danger">Start load all</button>
</div> </div>
<div class="row-fluid" style="margin-top: 15px;">
<div class="span12">
<button class="btn btn-success" ng-click="createIndex()">Clear/Create index</button>
<button class="btn btn-success" ng-click="loadMetricsFromPath()">Load from metric path</button>
<button class="btn btn-danger" ng-click="loadAll()">Load all</button>
<tip>Load all will fetch all metrics in one call, can be intensive for graphite and for the browser if you have a lot of metrics</tip>
</div> </div>
<div class="row-fluid">
<div class="span8">
<label class="small">Load metric keys recursively starting at metric key path</label>
<input type="text" class="input-xlarge" ng-model="metricPath"> </input>
</div> </div>
<div class="span4">
<button class="btn btn-danger" style="margin-top: 20px" ng-click="loadMetricsFromPath()">Start load from metric path</button> <div class="row-fluid" style="margin-top: 15px;">
<div class="span12" ng-show="infoText" style="padding-top: 10px;">
{{infoText}}
</div> </div>
<div class="span12 alert alert-error" ng-show="errorText">
{{errorText}}
</div> </div>
<div class="row-fluid"> </div>
<div class="span12 alert-message block-message info"> <div class="row-fluid" ng-show="metricCounter">
{{infoText}} <div class="span12" style="padding-top: 10px;">
Metrics indexed: {{metricCounter}}
</div> </div>
</div> </div>
</div> </div>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment