elasticsearch-watchdog
Version:
A watchdog of elasticsearch - cluster nodes' statuses monitor, auto restart, keep PRIMARY node unique.
251 lines (231 loc) • 7.2 kB
JavaScript
var forever = require('forever'),
path = require('path'),
_ = require('lodash'),
fs = require('fs'),
chalk = require('chalk'),
async = require('async'),
VARS = require('./vars'),
helper = require('./helper');
var mon = module.exports = {};
// load forever arguments
var foreverArgs = {root: helper.ROOT};
/**
* Run the web interface, if `port` was undefined, 8088 by default.
* @param port
*/
mon.web = require('../web/index');
/**
* Recall a watchdog by uid then send out.
* @param name
*/
mon.restart = function(name){
forever.load(foreverArgs);
forever.restart(name, false).on('restart', function(processes){
_.delay(mon.list.bind(null, true), 1000);
}).on('error', function(err){
console.log('\n', chalk.red.bold('No puppy is identified as <#' + name + '>'), '\n');
});
};
/**
* Recall all watchdogs and then send out.
*/
mon.restartAll = function(){
forever.load(foreverArgs);
forever.restartAll(false).on('restartAll', function(processes){
_.delay(mon.list.bind(null, true), 1000);
}).on('error', function(err){
console.log('\n', chalk.red.bold('No puppy out door.'), '\n');
});
};
/**
* Stop watching by uid.
* @param name
*/
mon.stop = function(name){
forever.load(foreverArgs);
forever.stop(name, false).on('stop', function(processes){
helper._cleanHouse(processes, true);
_.delay(mon.list.bind(null, true), 1000);
}).on('error', function(err){
console.log('\n', chalk.red.bold('No puppy is identified as <#' + name + '>'), '\n');
});
};
/**
* Recall all watchdogs.
*/
mon.stopAll = function(){
forever.load(foreverArgs);
forever.stopAll(false).on('stopAll', function(processes){
helper._cleanHouse([]);
console.log('\n', chalk.magenta.bold('No puppy out door.'), '\n');
}).on('error', function(err){
console.log('\n', chalk.red.bold('No puppy out door.'), '\n');
});
};
/**
* List the watchdogs.
* @param {Boolean} format
* @param {Function} callback
*/
mon.list = function(format, callback){
forever.load(foreverArgs);
forever.list(false, function(err, processes){
if (processes) {
async.map(processes, mon._formatProcess, format ? mon._printProcesses : function(err, result){
if (err) {
if (callback) {
return callback(err);
}
return console.error('\n', chalk.bold.red('Puppies are going hungry:', err.message), '\n');
}
if (callback) {
callback(null, result);
} else {
console.log(JSON.stringify(result));
}
});
helper._cleanHouse(processes);
}
else {
if (callback) {
return callback(null, []);
}
console.info('\n', chalk.bold.magenta('No puppy out door.'), '\n');
}
});
};
/**
* Start watching cluster
* @param {String} file
* @param {Object} options
*/
mon.start = function(file, options){
var conf = helper.loadConfig(file);
// exec module path.
var execModule = path.resolve(__dirname, helper._DOG);
// load configuration
forever.load(foreverArgs);
// passing YAML and dog name.
var foreverOptions = {
args: [file, conf.watchdog.name, helper._randomNo()]
};
// start forever.
if (options.daemon) {
// try to make sure path exists
var dir = path.resolve(foreverArgs.root, 'logs');
!fs.existsSync(dir) && fs.mkdirSync(dir);
forever.startDaemon(execModule, _.assign(foreverOptions, {
max : options.max || 3,
silent : true,
uid : foreverOptions.args[2].toString(),
logFile: '/dev/null', //path.resolve(dir, conf.watchdog.name + '.log'),
outFile: path.resolve(dir, conf.watchdog.name + '.' + foreverOptions.args[2] + '-out.log'),
errFile: path.resolve(dir, conf.watchdog.name + '.' + foreverOptions.args[2] + '-err.log')
}));
} else {
forever.start(execModule, foreverOptions);
}
};
/**
* Time duration to now.
* @param {Float} tick
* @return {String}
* @private
*/
mon._fromNow = function(tick){
tick = Date.now() - tick;
if (tick < 1e3) {
return tick + 'ms';
}
tick = parseInt(tick / 1e3);
if (tick < 60) {
return tick + 's';
}
var s = tick % 60 + 's';
if (tick < 3600) {
return parseInt(tick / 60) + 'm' + s
}
var m = parseInt((tick % 3600) / 60) + 'm'
return parseInt(tick / 3600) + 'h' + m + s;
};
/**
* Format process data.
* @param {Object} item
* @param {Function} next
* @private
*/
mon._formatProcess = function(item, next){
var process = _(item).pick('ctime', 'uid', 'running', 'restarts')
.assign({name: item.args[1]})
.value();
if (process.name) {
var dataFile = path.resolve(helper.ROOT, 'data', item.args.slice(1).join('.') + '.json');
if (fs.existsSync(dataFile)) {
try {
var data = fs.readFileSync(dataFile, {encoding: 'utf-8'});
process.data = JSON.parse(data);
} catch (err) {
process.data = err.message;
}
}
}
next(null, process);
};
mon._printProcesses = function(err, result){
if (err) {
console.error('\n', intents(1), chalk.bold.red('Puppies are going hungry:', err.message), '\n');
return;
}
console.info('\n', intents(1), chalk.bold.cyan('Puppies are watching:'), '\n');
function intents(i){
return Array(i).join(' ')
};
function hasColor(status){
return !!~['red', 'yellow', 'grey', 'gray', 'green'].indexOf(status);
}
function printNode(health, node, intent, prefix){
var status, _hasColor, _logs = [intents(intent), chalk.dim(prefix)];
if (health && (status = health[node])) {
_hasColor = hasColor(status);
_logs.push(_hasColor ? chalk[status].bold(node) : chalk.red.dim(node));
_logs.push(!_hasColor ? chalk.dim.red('[' + status + ']') : '');
} else {
_logs.push(chalk.bold.dim.red(node.toUpperCase() + ' [missing status]'));
}
console.log.apply(null, _logs);
}
result.forEach(function(p){
var ct = mon._fromNow(p.ctime);
console.log(
intents(2),
(p.name ? chalk.magenta(p.name.toUpperCase()) : chalk.red('LOST')),
'<#' + p.uid + '>',
chalk[p.running ? 'blue' : 'red'](p.running ? 'Alive ' + ct : 'Died ' + ct + ' ago')
);
if (typeof p.data == 'string') {
return console.log(intents(3), chalk.red('Mad puppy:'), chalk.dim(p.data), '\n')
}
if (p.data && p.data.state) {
for (var k in p.data.state) {
printNode(p.data.health, k, 3, '★');
p.data.state[k].forEach(function(node){
printNode(p.data.health, node, 4, '✩');
});
}
} else if (p.data && p.data.health) {
var keys = Object.keys(p.data.health);
if (keys.length == 1) {
printNode(p.data.health, keys[0], 3, '★');
} else {
printNode(null, VARS.NODE_STATUS.UNKNOWN, 3, '★');
keys.forEach(function(k){
printNode(p.data.health, k, 4, '✩');
});
}
} else {
console.log(intents(3), chalk.red('Objective is missing'))
}
console.log(intents(3), chalk.grey((p.data && p.data.timestamp) ? '√ Updated ' + mon._fromNow(p.data.timestamp) + ' ago' : '✘ Unknown uptime'));
console.log('');
});
};