Skip to content

C
ChickenWeight
Commit 80834e85 by Kornkitt Poolsup
Options

First commit

parent 3db127d9
Showing with 674 additions and 27 deletions
index5.js
......@@ -56,11 +56,11 @@ function query_generator(device_id, range, endtime) {
* Check whether the data is lie in range or not
* @param {double} x checked data
* @param {double} base base data to check range with
* @param {double} range define range of data
*/
function lie_in_range(x, base, range = 0.5) {
var upper = base + (1.25 * base * range);
var lower = base - (0.75 * base * range);
function lie_in_range(x, base) {
if (x - base <= 0.15 && x - base >= 0.75) return true;
var upper = base * 1.5;
var lower = base * 0.75;
if (x <= upper && x >= lower && x > 0) return true;
return false;
}
......@@ -69,16 +69,15 @@ function lie_in_range(x, base, range = 0.5) {
* Check whether data is lie below range or not
* @param {double} x checked data
* @param {double} base base data to check range with
* @param {double} range define range of data
*/
function below_range(x, base, range = 0.5) {
var lower = base - (0.75 * base * range);
function below_range(x, base) {
var lower = base * 0.75;
if (x < lower && x > 0) return true;
return false;
}
function above_range(x, base, range = 0.5) {
var upper = base + (1.25 * base * range);
function above_range(x, base) {
var upper = base * 1.5;
if (x > upper) return true;
return false;
}
......@@ -157,20 +156,21 @@ function skew_remove_out(data) {
}
function currentAverage(data, base_avg) {
data = data.filter(x => {
if (x <= 0.15) return false;
return true;
});
var temp = [];
var below_temp = [];
var lowest = 999999;
if (base_avg <= 0) {
temp = data.filter(x => {
if (x <= 0.01) return false;
return true;
});
// console.log(".." + temp);
temp = skew_remove_out(temp);
// console.log(base_avg + " " + temp);
} else {
data.filter(x => {
if (x <= 0.01) return false;
return true;
}).map(x => {
data.map(x => {
if (lie_in_range(x, base_avg)) {
temp.push(x);
} else if (above_range(x, base_avg)) {
......@@ -178,21 +178,19 @@ function currentAverage(data, base_avg) {
for (var i = 0; i < count; i++) {
temp.push(x / count);
}
} else if (below_range(x, base_avg)) {
below_temp.push(x);
if (x < lowest) lowest = x;
}
});
if (temp.length == 0) {
temp = data.filter(x => {
temp = below_temp.filter(x => {
if (x <= 0.1) return false;
return true;
})
// console.log(".." + temp);
});
temp = skew_remove_out(temp);
// console.log(base_avg + " " + temp);
}
}
// var avg = average(temp);
// if (avg > base_avg || below_range(avg, base_avg)) return avg;
// return base_avg;
return average(temp);
}
......@@ -226,11 +224,9 @@ function currentAverageWeight(bt, data) {
var old_val = avg_arr[avg_arr.length - 2];
avg = currentAverage(x, old_val);
console.log(old_val + " _ " + avg);
if (bt[bt_index] - bt[bt_index - 1] <= ms('1w')) {
// check new average value with latest average value
if (avg > old_val && avg < old_val * 1.25) {
if (avg > old_val) {
// if higher, use the new one
avg_arr[avg_arr.length - 1] = avg;
} else if (avg < old_val && avg > 0 &&
......
index6.js
......@@ -53,6 +53,248 @@ function query_generator(device_id, range, endtime) {
}
/**
* Check whether the data is lie in range or not
* @param {double} x checked data
* @param {double} base base data to check range with
*/
function lie_in_range(x, base) {
// if (x - base <= 0.15 && x - base >= 0.75) return true;
var upper = base * 2;
var lower = base * 0.5;
if (x <= upper && x >= lower && x > 0) return true;
return false;
}
/**
* Check whether data is lie below range or not
* @param {double} x checked data
* @param {double} base base data to check range with
*/
function below_range(x, base) {
var lower = base * 0.5;
if (x < lower && x > 0) return true;
return false;
}
function above_range(x, base) {
var upper = base * 2;
if (x > upper) return true;
return false;
}
/**
* Average the data
* @param {list} data list of array data
*/
function average(data) {
if (data.length <= 0) return 0;
var sum = 0;
data.map(x => sum += x);
return sum / data.length;
}
// find the value at percentile 'range' (0 to 1)
function percentile(data, range) {
var p = (data.length + 1) * range;
var index = Math.floor(p) - 1;
var val = data[index] +
(p - index) * (data[index + 1] - data[index]);
return val;
}
// check skewed of data
function skewed(data) {
data = data.sort((a, b) => a - b); // sort data increasing
var q1 = percentile(data, 0.25);
var q2 = percentile(data, 0.5);
var q3 = percentile(data, 0.75);
var lower_range = q2 - q1;
var upper_range = q3 - q2;
var ratio = lower_range / upper_range;
if (ratio < 0.5) return 'right';
else if (ratio > 1.5) return 'left';
return 'normal';
}
// find the outlier and whether clear it or not - left skewed
function skew_remove_out(data) {
if (data.length <= 0) return [0];
data = data.sort((a, b) => a - b); // sort data increasing
var skew = skewed(data);
var q1 = percentile(data, 0.25);
var q2 = percentile(data, 0.5);
var q3 = percentile(data, 0.75);
var iqr = q3 - q1;
var bound, lower, upper;
console.log(skew + " / " + q1 + " / " + q2 + " / " + q3);
if (skew == 'normal') {
bound = iqr;
lower = q1 - bound;
upper = q3 + bound;
} else if (skew == 'right') {
bound = q2 - q1;
lower = q1 - bound;
upper = q2 + bound;
} else {
bound = q3 - q2;
lower = q2 - bound;
upper = q3 + bound;
}
var outs = []; // outlier
var not_outs = []; // data without outlier
data.map(x => {
if (x < lower || x > upper) outs.push(x);
else not_outs.push(x);
});
return not_outs;
}
function lowest(data) {
var min_data = 999999;
var have_min = false;
data.map(x => {
if (x < min_data) {
min_data = x;
have_min = true;
}
});
// console.log(min_data);
if (have_min && min_data > 0) return min_data;
else return 0;
}
function highest(data) {
var max_data = -999999;
var have_max = false;
data.map(x => {
if (x > max_data) {
max_data = x;
have_max = true;
}
});
if (have_max && max_data > 0) return max_data;
else return 0;
}
function currentAverage(data, base_avg) {
var temp = [];
var below_temp = [];
var lowest = 999999;
if (base_avg <= 0) {
temp = data.filter(x => {
if (x <= 0.01) return false;
return true;
});
temp = skew_remove_out(temp);
} else {
data.map(x => {
if (lie_in_range(x, base_avg)) {
temp.push(x);
} else if (above_range(x, base_avg)) {
var count = Math.round(x / base_avg);
for (var i = 0; i < count; i++) {
temp.push(x / count);
}
} else if (below_range(x, base_avg)) {
below_temp.push(x);
if (x < lowest) lowest = x;
}
});
if (temp.length == 0) {
temp = below_temp.filter(x => {
if (x <= 0.1) return false;
return true;
});
temp = skew_remove_out(temp);
}
}
return average(temp);
}
function currentAverageWeight(bt, data) {
if (data.length <= 0 || bt.length <= 0) return 0;
var bt_index = 0;
var avg_arr = [];
data.map(x => {
x = x.sort((a, b) => a - b); // sort data increasing
var avg;
if (bt_index !== 0 &&
bt[bt_index] - bt[bt_index - 1] !== ms('1d')) {
// in case there are no data passing for some day
if (avg_arr.length > 1 &&
avg_arr[avg_arr.length - 1] !== -1)
avg_arr.push(-1);
// base_time = bt[bt_index];
}
// if no average yet
else if (avg_arr.length <= 0) {
avg = currentAverage(x, 0);
if (avg > 0) avg_arr.push(avg);
}
// check if there is any day skip or previous zero data
else if (avg_arr[avg_arr.length - 1] === -1 &&
avg_arr.length > 1) {
// get latest average value that != -1
var old_val = avg_arr[avg_arr.length - 2];
avg = currentAverage(x, old_val);
if (bt[bt_index] - bt[bt_index - 1] <= ms('1w')) {
// check new average value with latest average value
if (avg > old_val) {
// if higher, use the new one
avg_arr[avg_arr.length - 1] = avg;
} else if (avg < old_val && avg > 0 &&
lie_in_range(avg, old_val)) {
// if lower but still in range, use the latest one
avg_arr[avg_arr.length - 1] = old_val;
} else if (below_range(avg, old_val) && avg > 0) {
// if lower below the lower bound, use new one,
// however, mark with -1 to confirm the new set of data
avg_arr.push(avg);
}
} else {
console.log("check!" + avg);
if (avg > 0) avg_arr.push(avg);
}
}
// if no day skip then continue
else {
avg = currentAverage(x, avg_arr[avg_arr.length - 1]);
if ((avg > avg_arr[avg_arr.length - 1] &&
avg < (avg_arr[avg_arr.length - 1] * 1.25)) ||
(avg > 0 &&
below_range(avg, avg_arr[avg_arr.length - 1]))) {
avg_arr.push(avg);
} else {
avg_arr.push(-1);
}
}
console.log(bt[bt_index] + " / " + bt[bt_index - 1] + " = " +
(bt[bt_index] - bt[bt_index - 1]) + " | " + avg + " / " +
avg_arr[avg_arr.length - 1] + " = " +
avg_arr.length + " | " +
bt_index + " / " + (bt.length - 1));
// base_time += ms('1d');
// bt_index++;
bt_index++;
});
if (avg_arr.length <= 0) return 0;
else if (avg_arr[avg_arr.length - 1] <= 0)
return avg_arr[avg_arr.length - 2];
return avg_arr[avg_arr.length - 1];
}
/**
* Get approximate weight of data
* @param {string} device_id id of retrieved device
* @param {json} response response from REST API
......@@ -78,7 +320,6 @@ function getWeight(device_id, response) {
temp_data_ext = [x[1]];
}
});
bt_ext.push(base_time);
data_ext.push(temp_data_ext);
var avg = currentAverageWeight(bt_ext, data_ext);
......
index7.js 0 → 100644
var rest = require('restler');
var ms = require('ms');
var sc = require('./status_code');
var kairos_usr = 'tsdbadmin';
var kairos_pwd = 'RzuCqG14NrWXMfbj*s6m8v';
var kairos_prt = '32223';
var kairos_url = 'http://' + kairos_usr + ":" + kairos_pwd +
'@nxn1.kube.nexpie.com:' +
kairos_prt + '/';
/**
* Round time to the previous absolube scale time
* @param {integer} t time for scale in milliseconds
* @param {string} scale how rough of time to scale
* @param {string} gmt Greenwich mean time; default is '7h'
*/
function timeRoundDown(t, scale = '1h', gmt = '7h') {
return t - (t % ms(scale)) - ms(gmt);
}
/**
* Generate a query from device_id
* @param {string} device_id id of retrieved device
* @param {string} range how far to get the data
* @param {integer} endtime gets data until when
*/
function query_generator(device_id, range, endtime) {
var mls = ms('1s');
var now = endtime ? endtime : new Date().getTime()
var q_data = {
start_absolute: now - ms(range),
end_absoulte: now,
metrics: [{
name: device_id,
group_by: [{
name: "tag",
tags: ["attr"]
}],
tags: {
attr: ["a12"]
},
aggregators: [{
name: 'avg',
sampling: {
value: mls,
unit: "milliseconds"
}
}]
}]
}
return q_data;
}
/**
* Check whether the data is lie in range or not
* @param {double} x checked data
* @param {double} base base data to check range with
*/
function lie_in_range(x, base) {
// if (x - base <= 0.15 && x - base >= 0.75) return true;
var upper = base * 2;
var lower = base * 0.5;
if (x <= upper && x >= lower && x > 0) return true;
return false;
}
/**
* Check whether data is lie below range or not
* @param {double} x checked data
* @param {double} base base data to check range with
*/
function below_range(x, base) {
var lower = base * 0.5;
if (x < lower && x > 0) return true;
return false;
}
function above_range(x, base) {
var upper = base * 2;
if (x > upper) return true;
return false;
}
/**
* Average the data
* @param {list} data list of array data
*/
function average(data) {
if (data.length <= 0) return 0;
var sum = 0;
data.map(x => sum += x);
return sum / data.length;
}
// find the value at percentile 'range' (0 to 1)
function percentile(data, range) {
var p = (data.length + 1) * range;
var index = Math.floor(p) - 1;
var val = data[index] +
(p - index) * (data[index + 1] - data[index]);
return val;
}
// check skewed of data
function skewed(data) {
data = data.sort((a, b) => a - b); // sort data increasing
var q1 = percentile(data, 0.25);
var q2 = percentile(data, 0.5);
var q3 = percentile(data, 0.75);
var lower_range = q2 - q1;
var upper_range = q3 - q2;
var ratio = lower_range / upper_range;
if (ratio < 0.5) return 'right';
else if (ratio > 1.5) return 'left';
return 'normal';
}
// find the outlier and whether clear it or not - left skewed
function skew_remove_out(data) {
if (data.length <= 0) return [0];
data = data.sort((a, b) => a - b); // sort data increasing
var skew = skewed(data);
var q1 = percentile(data, 0.25);
var q2 = percentile(data, 0.5);
var q3 = percentile(data, 0.75);
var iqr = q3 - q1;
var bound, lower, upper;
// console.log(skew + " / " + q1 + " / " + q2 + " / " + q3);
if (skew == 'normal') {
bound = iqr;
lower = q1 - bound;
upper = q3 + bound;
} else if (skew == 'right') {
bound = q2 - q1;
lower = q1 - bound;
upper = q2 + bound;
} else {
bound = q3 - q2;
lower = q2 - bound;
upper = q3 + bound;
}
var outs = []; // outlier
var not_outs = []; // data without outlier
data.map(x => {
if (x < lower || x > upper) outs.push(x);
else not_outs.push(x);
});
return not_outs;
}
function lowest(data) {
var min_data = 999999;
var have_min = false;
data.map(x => {
if (x < min_data) {
min_data = x;
have_min = true;
}
});
// console.log(min_data);
if (have_min && min_data > 0) return min_data;
else return 0;
}
function highest(data) {
var max_data = -999999;
var have_max = false;
data.map(x => {
if (x > max_data) {
max_data = x;
have_max = true;
}
});
if (have_max && max_data > 0) return max_data;
else return 0;
}
function averageWeight(bt, data) {
var ave = 0;
var bt_index = 0;
data.map(x => {
var ave_arr = [];
// console.log(x.length);'
// console.log(lowest(x));
// x = skew_remove_out(x);
var low = lowest(x);
if (below_range(low, ave)) {
var tmp_arr = [];
x.map(y => {
if (lie_in_range(y, ave)) {
tmp_arr.push(y);
}
});
// low = lowest(tmp_arr);
low = ave;
x = tmp_arr;
}
x.map(tmp => {
if (tmp <= low * 1.75) {
ave_arr.push(tmp);
} else if (tmp > low * 1.75) {
var count = Math.round(tmp / low);
var ave_tmp = tmp / count;
for (var i = 0; i < count; i++) {
ave_arr.push(ave_tmp);
}
}
});
var final_ave = average(ave_arr);
// console.log(bt[bt_index] + " | " +
// ave + " : " + final_ave + " / " + low + " > " +
// x.length + " : " + ave_arr.length);
if (final_ave > ave) {
ave = final_ave;
}
bt_index++;
});
return ave;
}
function allLowestNoise(data) {
var low = lowest(data);
var low_arr = [];
data.map(x => {
if (x / 2 <= low * 1.25) low_arr.push(x);
});
return low_arr;
}
function averageNoise(data, noise) {
var temp = [];
data.map(x => {
if (lie_in_range(x, noise)) {
temp.push(x);
}
});
return average(temp);
}
function clearNoise(data, noise) {
var temp = [];
var ave_noise = averageNoise(data, noise);
data.map(x => {
if (!lie_in_range(x, noise) &&
above_range(x, noise)) {
var noise_reduce = x - ave_noise;
temp.push(noise_reduce);
}
});
// console.log(temp);
return temp;
}
function currentAverageWeight(bt, data) {
if (data.length <= 0 || bt.length <= 0) return 0;
// Step 1: Find all lowest value from each day
var low_arr = [];
var bt_list = [];
var bt_index = 0;
var first_bt = bt_index;
data.map(x => {
x = x.sort((a, b) => a - b); // sort data increasing
var low;
// in case there are no data passing for some day
if (bt_index !== 0 &&
bt[bt_index] - bt[bt_index - 1] !== ms('1d')) {
// check if there already has a no-data-value or not
if (low_arr.length > 1 &&
low_arr[low_arr.length - 1] !== -1)
low_arr.push(-1);
// console.log('in!');
}
// check if there is any day skip or previous zero data
else if (low_arr[low_arr.length - 1] === -1 &&
low_arr.length > 1) {
low = lowest(x);
if (low >= 0) {
low_arr = [low];
bt_list = [bt[bt_index]];
first_bt = bt_index;
}
} else {
low = lowest(x);
if (low >= 0) {
low_arr.push(low);
bt_list.push(bt[bt_index]);
}
}
bt_index++;
});
// Step 2: Find skew of all lowest values
// console.log(low_arr);
var low_arr_tmp = skew_remove_out(low_arr);
var low_arr_len = low_arr.length;
var ave_low = average(low_arr_tmp);
for (var i = 0; i < low_arr_len; i++) {
// check if it include in lowest noise or not
if (!low_arr_tmp.includes(low_arr[i]) && (
below_range(low_arr[i], ave_low) ||
above_range(low_arr[i], ave_low))) {
if (i == 0) { low_arr[i] = 0; }
else if (i == low_arr_len - 1) { low_arr[i] = low_arr[i - 1]; }
else { low_arr[i] = (low_arr[i + 1] + low_arr[i - 1]) / 2; }
}
}
// console.log("first_index: " + first_bt);
// Step 3: Find the noise from all lowest value
var final_data = [];
var final_bt = [];
var fd_index = 0;
var loop_index = 0;
data.map(x => {
if (loop_index >= first_bt) {
x = x.sort((a, b) => a - b); // sort data increasing
var noise_reduce_data = clearNoise(x, low_arr[fd_index]);
final_data.push(noise_reduce_data);
final_bt.push(bt[fd_index + first_bt]);
// console.log(fd_index + " / " + loop_index + " / " +
// bt[fd_index + first_bt] + " / " +
// bt[fd_index]);
fd_index++;
}
loop_index++;
});
// console.log(final_data);
return averageWeight(final_bt, final_data);
}
/**
* Get approximate weight of data
* @param {string} device_id id of retrieved device
* @param {json} response response from REST API
*/
function getWeight(device_id, response) {
var final = [];
response.map(res => {
var data = res.values;
var data_ext = [];
var bt_ext = [];
var base_time = timeRoundDown(data[0][0], '1d');
var temp_data_ext = [];
data.map(x => {
if (timeRoundDown(x[0], '1d') === base_time) {
temp_data_ext.push(x[1]);
} else {
bt_ext.push(base_time);
data_ext.push(temp_data_ext);
base_time = timeRoundDown(x[0], '1d');
temp_data_ext = [x[1]];
}
});
bt_ext.push(base_time);
data_ext.push(temp_data_ext);
var avg = currentAverageWeight(bt_ext, data_ext);
final.push([device_id, avg]);
});
return final;
}
/**
* Retrieve data from device_id to get current approximate weight
* @param {string} device_id id of the retrived device
* @param {string} traceback how far to trace data back
*/
function currentWeight(device_id, traceback = '1y') {
var q_data = query_generator(device_id, traceback);
rest.get(kairos_url +
'api/v1/datapoints/query?query=' +
JSON.stringify(q_data)
).on('complete', function (qres, response) {
if (response) {
console.log('status code (qres): ' +
sc.http_codeToStatus(response.statusCode));
var res = qres.queries[0].results;
var f = getWeight(device_id, res);
console.log(f);
} else {
return;
}
});
}
/**
* For testing and debugging in sequence
*/
function run_all() {
var dev = [
'4c988381-2cd9-4334-914d-971241bcf3c2',
'f592b419-380d-4875-9c99-354a4579d03c'
];
dev.map(x => currentWeight(x));
}
run_all();
package.json
......@@ -4,7 +4,7 @@
"description": "",
"main": "index.js",
"scripts": {
"start": "node index5.js",
"start": "node index6.js",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment