the-shepherd
Version:
Control a herd of wild processes.
766 lines (713 loc) • 25.2 kB
JavaScript
// Generated by CoffeeScript 2.5.1
(function() {
var $, ChildProcess, Group, Groups, Nginx, Path, Proc, SlimProcess, actOnAllGroups, actOnGroup, actOnInstance, addGroup, afterAction, echo, exists, minutes, quoted, removeGroup, saveConfig, simpleAction, verbose, warn;
({$, echo, warn, verbose} = require('../common'));
Path = require('path');
Nginx = null; // placeholder for out of order import
saveConfig = null;
ChildProcess = require('child_process');
SlimProcess = require('../util/process-slim');
({exists} = require('../files'));
({quoted} = require('../common'));
minutes = function(n) {
return n * 60000;
};
// the global herd of processes
Groups = new Map();
// m.clear m.delete m.entries m.forEach m.get m.has m.keys m.set m.size m.values
const DEFAULT_GRACE = 9000;;
Group = (function() {
var createProcess;
class Group extends Array {
constructor(name1, cd1, exec1, n1, port1, grace1 = DEFAULT_GRACE) {
var i, j, ref;
super();
this.name = name1;
this.cd = cd1;
this.exec = exec1;
this.n = n1;
this.port = port1;
this.grace = grace1;
for (i = j = 0, ref = this.n; j < ref; i = j += 1) {
this.push(createProcess(this, i));
}
this.monitors = Object.create(null); // used later by health checks
this.enabled = true;
}
// @public_port, @public_name, @ssl_cert, and @ssl_key can all be set here from src/daemon/nginx
enable(cb) {
var acted;
acted = !this.enabled;
this.enabled = true;
if (acted) {
return this.actOnAllGroups('enable', cb);
} else {
return typeof cb === "function" ? cb() : void 0;
}
}
disable(cb) {
var acted;
acted = this.enabled;
this.enabled = false;
if (acted) {
return this.actOnAllGroups('disable', cb);
} else {
return typeof cb === "function" ? cb() : void 0;
}
}
scale(n, cb) {
var d, dn, j, p, progress, ref;
if (!(isFinite(n) && !isNaN(n) && n >= 0)) {
echo("[scale] Count must be a number >= 0.");
return false;
}
dn = n - this.n;
progress = $.Progress(dn);
progress.then(() => {
return saveConfig(cb);
});
if (dn > 0) {
echo(`[scale(${n})] Adding ${dn} instances...`);
for (d = j = 0, ref = dn; j < ref; d = j += 1) {
p = createProcess(this, this.n++);
if (this.enabled) {
p.start(() => {
return progress.finish(1);
});
} else {
p.disable(() => {
return progress.finish(1);
});
}
this.push(p);
}
} else if (dn < 0) {
echo(`[scale(${n})] Trimming ${dn} instances...`);
while (this.n > n) {
this.pop().stop(() => {
return progress.finish(1);
});
this.n -= 1;
}
} else {
return false;
}
return true;
}
restart(cb) { // Slowly kill and restart one at a time
var oneStep;
(oneStep = (i) => {
if (!(i < this.length)) {
return typeof cb === "function" ? cb() : void 0;
}
return this[i].restart(() => {
return oneStep(i + 1);
});
})(0);
return true;
}
start(cb) {
var oneStep;
(oneStep = (i) => {
if (!(i < this.length)) {
return typeof cb === "function" ? cb(null, true) : void 0;
}
return this[i].start(() => {
return oneStep(i + 1);
});
})(0);
return true;
}
stop(cb) {
var acted, j, len, proc, progress, ref;
acted = false;
progress = $.Progress(this.length + 1).wait(function(err) {
return typeof cb === "function" ? cb(err, acted) : void 0;
});
ref = this;
for (j = 0, len = ref.length; j < len; j++) {
proc = ref[j];
proc.stop(function(err, _acted) {
acted |= _acted;
return progress.finish(1);
});
}
return progress.finish(1);
}
markAsInvalid(reason) {
var j, len, proc, ref;
ref = this;
for (j = 0, len = ref.length; j < len; j++) {
proc = ref[j];
proc.markAsInvalid(reason);
}
return this;
}
actOnAllGroups(method, cb) {
var j, len, progress, ref, x;
progress = $.Progress(this.length);
progress.wait((err) => {
return cb(err, true);
});
ref = this;
for (j = 0, len = ref.length; j < len; j++) {
x = ref[j];
x[method](() => {
return progress.finish(1);
});
}
return progress;
}
toString() {
var i, proc;
return `[group ${this.name}] ` + ((function() {
var j, len, ref, results;
ref = this;
results = [];
for (i = j = 0, len = ref.length; j < len; i = ++j) {
proc = ref[i];
results.push(`${proc.port ? `(${proc.port}) ` : ""}${proc.statusString}`);
}
return results;
}).call(this)).join(", ");
}
toConfig() {
return `add --group ${this.name}` + (this.cd !== "." ? ` --cd ${quoted(this.cd)}` : "") + ` --exec ${quoted(this.exec)} --count ${this.n}` + (this.grace !== DEFAULT_GRACE ? ` --grace ${this.grace}` : "") + (this.port ? ` --port ${this.port}` : "");
}
containsPID(pid) { // returns true if the PID is a child of any group member
var j, len, match, ref, x;
match = false;
ref = this;
for (j = 0, len = ref.length; j < len; j++) {
x = ref[j];
if (match) {
// check all the processes in the tree they created
// and see if any of them is the one we are looking for
break;
}
SlimProcess.visitProcessTree(x, (proc) => {
// this is a synchronous callback
if (proc.pid === owner.pid) {
return match = true;
}
});
} // for all our children
return match;
}
};
createProcess = function(g, i) {
var port;
port = void 0;
if (g.port) {
port = parseInt(g.port) + i;
}
return new Proc(`${g.name}-${i}`, g.cd, g.exec, port, g);
};
return Group;
}).call(this);
Proc = (function() {
class Proc {
constructor(id, cd1, exec1, port1, group1) {
var statusString;
this.id = id;
this.cd = cd1;
this.exec = exec1;
this.port = port1;
this.group = group1;
this.expected = false; // should this proc be up (right now)
this.enabled = true; // should this proc be up (in general)
this.started = false; // is it (when was it) started
this.healthy = void 0; // used later by health checks
this.cooldown = Proc.cooldown; // this increases after each failed restart
// expose uptime
$.defineProperty(this, 'uptime', {
get: () => {
if (this.started) {
return $.now - this.started;
} else {
return 0;
}
}
});
statusString = "unstarted";
$.defineProperty(this, 'statusString', {
get: () => {
return statusString;
},
set: (v) => {
var oldValue;
oldValue = statusString;
statusString = v;
// @log oldValue + " -> " + v
return $.log(this.group.toString());
}
});
$.defineProperty(this, 'pid', {
get: () => {
var ref;
return (ref = this.proc) != null ? ref.pid : void 0;
}
});
}
log(...args) {
var a, i, j, len;
for (i = j = 0, len = args.length; j < len; i = ++j) {
a = args[i];
args[i] = a.replace(/\n([^\n]+)/mg, `\n[${this.id}] $1`);
}
return $.log(`[${this.id}]`, ...args);
}
// Start this process if it isn't already.
start(cb) { // cb called with a 'started' flag that indicates if any work was done
var clearPort, doStart, done, env, retryStart;
done = (err, ret) => {
if (typeof cb === "function") {
cb(err, ret);
}
return ret;
};
if (this.started) {
verbose(`${this.id} already started, skipping.`);
return done(null, false);
}
if (!this.enabled) {
verbose(`${this.id} not enabled, skipping.`);
return done(null, false);
}
this.expected = true;
this.healthy = void 0;
env = Object.assign({}, process.env, {
PORT: this.port
});
// clearPort will try to kill any other process using our port
// but will check some safeties first:
// - dont kill processes owned by other users
// - dont kill processes managed by our same shepherd
clearPort = (cb) => {
if (!(this.expected && this.enabled)) {
verbose(`${this.id} giving up on clearPort`, {expected: this.expected, enabled: this.enabled});
return done(null, false); // stopped while waiting
}
verbose(`${this.id} Checking for owner of port ${this.port}...`);
if (this.port) {
return SlimProcess.getPortOwner(this.port, (err, owner) => {
var invalidPort, this_uid;
if (!owner) {
return cb();
}
// so, the @port is currently owned...
invalidPort = (msg) => {
verbose(`${this.id} Marking port ${this.port} as invalid.`);
return done(this.markAsInvalid("invalid port"));
};
this_uid = process.getuid();
verbose(`${this.id} Checking if owner ${owner.uid} is same as ours ${this_uid}`);
// 1) - dont kill processes owned by other users
if (owner.uid !== this_uid) {
return invalidPort();
}
// 2) - dont kill processes owned by any of our other groups
return SlimProcess.getProcessTable((err, procs) => { // this forces the process table cache to be fresh
var managed;
verbose(`${this.id} Checking if owner pid ${owner.pid} is managed by another group...`);
managed = false;
Groups.forEach(function(group) {
return managed || (managed = group.containsPID(owner.pid));
});
if (!managed) {
verbose(`${this.id} Owner pid ${owner.pid} is not managed by another group, killing it...`);
verbose(owner);
this.statusString = `killing ${owner.pid}`;
return SlimProcess.killProcessTree(owner.pid, 'SIGTERM', (err) => {
if (err) {
warn(`${this.id} failed to kill other owner of port ${this.port}: owner ${owner.pid}`);
warn(err);
}
return setTimeout((() => {
return clearPort(cb);
}), 1000);
});
} else {
warn(`${this.id} asked for PORT ${this.port} but it is in-use by another group in this shepherd`);
return invalidPort();
}
});
});
} else {
return typeof cb === "function" ? cb() : void 0;
}
};
retryStart = () => {
if (this.started || !this.enabled) {
return done(null, false);
}
this.cooldown = Math.min(10000, this.cooldown * 2);
this.statusString = `waiting ${this.cooldown}ms`;
setTimeout(doStart, this.cooldown);
return true;
};
return (doStart = () => {
if (this.started) {
verbose(`${this.id} ignoring start of already started instance.`);
return done(null, false);
}
if (!this.expected) {
verbose(`${this.id} ignoring start of unexpected instance.`);
return done(null, false);
}
return clearPort((err) => {
var _s, checkStarted, finishStarting, opts;
if (err) {
verbose(`${this.id} aborting start while inside clearPort`, {err});
return done(null, false);
}
if (!this.expected) {
verbose(`${this.id} aborting start while inside clearPort`, {expected: this.expected});
return done(null, false);
}
checkStarted = null;
this.statusString = "starting";
try {
this.cd = Path.resolve(process.cwd(), this.cd);
verbose("cd:", this.cd);
} catch (error) {
err = error; // it's actually possible for node's internals to throw an exception here if cwd() is weird
verbose(`${this.id} CWD error: ${err.message}`);
this.markAsInvalid(err.message);
return done(err, false);
}
verbose("exec:", this.exec, "as", this.id);
opts = {
shell: true,
cwd: this.cd,
env: env
};
if (!exists(this.cd)) {
return this.stop(() => {
return this.group.markAsInvalid("invalid dir");
});
}
this.proc = ChildProcess.spawn(this.exec, opts);
this.expected = true; // tell the 'exit' handler to bring us back up if we die
finishStarting = () => {
this.started = $.now;
this.cooldown = Proc.cooldown;
this.statusString = "started";
if (this.port) {
return Nginx.sync(() => {
return done(true);
});
} else {
return done(true);
}
};
if (this.port) {
_s = Date.now();
checkStarted = setTimeout((() => {
var ref;
if (!((this.proc != null) && this.expected)) {
return done(null, false);
}
if (((ref = this.proc) != null ? ref.pid : void 0) == null) {
this.markAsInvalid("exec failed");
return done(null, false);
}
echo(`Waiting for port ${this.port} to be owned by ${this.proc.pid} (will wait ${this.group.grace} ms)`);
return SlimProcess.waitForPortOwner(this.proc, this.port, this.group.grace, (err, owner) => {
if (!(this.expected && this.enabled)) { // stopped while waiting?
echo(`${this.id} abort requested while waiting for port`, {expected: this.expected, enabled: this.enabled});
return this.stop(() => {
return done(null, false);
});
}
switch (err) {
case 'exit':
warn(`${this.id} exited immediately, attempting to resume.`);
this.proc = null;
return this.stop(retryStart);
case 'timeout':
echo(`${this.id} did not listen on port ${this.port} within the timeout: ${this.group.grace}ms`);
return this.stop(retryStart);
default:
if (err != null) {
echo(`${this.id} failed to find port owner after`, (Date.now() - _s) + "ms");
warn(err);
return this.stop(retryStart);
}
}
return finishStarting();
});
}), 50); // if there is no port to wait for then staying up for a few seconds counts as started
} else {
checkStarted = setTimeout(finishStarting, this.group.grace);
}
// Connect the process output to our log writer.
this.proc.stdout.on('data', (data) => {
return this.log(data.toString("utf8"));
});
this.proc.stderr.on('data', (data) => {
return this.log("(stderr)", data.toString("utf8"));
});
return this.proc.on('exit', (code, signal) => {
var ref, ref1;
clearTimeout(checkStarted);
this.started = false;
if ((ref = this.proc) != null ? ref.pid : void 0) {
try {
if ((ref1 = this.proc) != null) {
if (typeof ref1.unref === "function") {
ref1.unref();
}
}
} catch (error) {}
}
this.proc = void 0;
if (this.expected === false) { // it went down and we dont care
this.statusString = `exit(${code}) ok`;
} else {
echo(`${this.id} Unexpected exit, restarting...`);
return retryStart();
}
return true;
});
});
})();
}
stop(cb) {
var err, ref;
this.expected = false;
this.statusString = "stopping";
if (this.checkResumeTimeout !== null) {
verbose("cancelling checkResumeTimeout...");
clearTimeout(this.checkResumeTimeout);
this.checkResumeTimeout = null;
}
if (((ref = this.proc) != null ? ref.pid : void 0) > 1) {
// if the proc is alive, set up a listener for when it exits
this.proc.on('exit', () => {
verbose(`${this.id} event: 'exit'`);
this.started = false;
this.statusString = this.enabled ? "stopped" : "disabled";
if (this.port) {
return Nginx.sync(() => {
return typeof cb === "function" ? cb(null, true) : void 0;
});
} else {
return typeof cb === "function" ? cb(null, true) : void 0;
}
});
try {
// send it a kill signal
SlimProcess.killProcessTree(this.proc.pid, 'SIGTERM', (err) => {
if (err) {
return warn(`${this.id} killProcessTree error: ${err}`);
}
});
} catch (error) {
err = error;
warn(`${this.id} killProcessTree threw exception:`, err);
}
return true;
} else {
this.started = false;
this.proc = null;
this.statusString = this.enabled ? "stopped" : "disabled";
if (typeof cb === "function") {
cb(null, false);
}
return false;
}
}
restart(cb) {
this.statusString = "restarting";
return this.stop(() => {
return this.start(cb);
});
}
enable(cb) {
var acted;
acted = !this.enabled;
this.enabled = true;
if (acted) {
this.start(cb);
} else {
if (typeof cb === "function") {
cb(null, false);
}
}
return acted;
}
markAsInvalid(reason) {
this.enabled = this.expected = this.started = this.healthy = false;
this.statusString = reason;
return false;
}
disable(cb) {
var _end, acted;
acted = this.enabled;
this.enabled = false;
_end = () => {
this.statusString = "disabled";
return typeof cb === "function" ? cb() : void 0;
};
if (this.started) {
this.stop(_end);
} else {
_end();
}
return acted;
}
};
Proc.cooldown = 200;
return Proc;
}).call(this);
actOnInstance = function(method, instanceId, cb) {
var acted, chunks, groupId, index, proc, ref;
if (!(instanceId != null ? instanceId.length : void 0)) {
return false;
}
acted = false;
chunks = instanceId.split('-');
index = chunks[chunks.length - 1];
groupId = chunks.slice(0, chunks.length - 1).join('-');
index = parseInt(index, 10);
proc = (ref = Groups.get(groupId)) != null ? ref[index] : void 0;
if ((!proc) || !(method in proc)) {
return typeof cb === "function" ? cb('invalid method') : void 0;
}
proc[method]((ret) => {
return afterAction(method, ret, cb);
});
return false;
};
actOnGroup = function(method, groupId, cb) {
var group;
group = Groups.get(groupId);
if (!group) {
return typeof cb === "function" ? cb('No such group.') : void 0;
}
if (method in group) {
group[method]((ret) => {
return afterAction(method, ret, cb);
});
} else {
group.actOnAllGroups(method, (ret) => {
return afterAction(method, ret, cb);
});
}
return false;
};
actOnAllGroups = function(method, cb) {
var acted, finishOne, progress;
acted = false;
progress = $.Progress(Groups.size + 1).on("progress", function(cur, max) {
return echo(`actOnAllGroups[${method}] progress:`, progress.inspect());
}).wait(() => {
return afterAction(method, acted, cb);
});
verbose(`actOnAllGroups[${method}] started:`, progress.inspect());
finishOne = (err, act) => {
acted = act || acted;
return progress.finish(1);
};
Groups.forEach(function(group) {
var child, j, len, results;
if (group == null) {
return;
}
if ('function' === typeof group[method]) {
return group[method](finishOne);
} else {
results = [];
for (j = 0, len = group.length; j < len; j++) {
child = group[j];
results.push(typeof child[method] === "function" ? child[method](finishOne) : void 0);
}
return results;
}
});
return progress.finish(1);
};
addGroup = function(name, cd = ".", exec, count = 1, port, grace = DEFAULT_GRACE, cb) {
if (Groups.has(name)) {
warn("Group already exists. Did you mean 'replace'?");
return false;
}
echo("Adding group:", name);
verbose({cd, exec, count, port, grace});
Groups.set(name, new Group(name, cd, exec, count, port, grace));
return afterAction('add', true, cb);
};
removeGroup = function(name, cb) {
var done, g, j, len, proc;
if (!Groups.has(name)) {
echo("No such group:", name);
return false;
}
echo("Removing group:", name);
g = Groups.get(name);
done = $.Progress(g.length).then(() => {
Groups.delete(name);
return afterAction('remove', true, cb);
});
for (j = 0, len = g.length; j < len; j++) {
proc = g[j];
proc.stop(() => {
return done.finish(1);
});
}
return true;
};
afterAction = function(method, ret, cb) {
if (method === 'enable' || method === 'disable' || method === 'add' || method === 'remove' || method === 'replace') {
return saveConfig((err, acted) => {
return typeof cb === "function" ? cb(err, ret) : void 0;
});
} else {
return typeof cb === "function" ? cb(null, ret) : void 0;
}
};
simpleAction = function(method) {
return function(msg, client, cb) {
var _line, targetText;
_line = '';
targetText = msg.g ? `group ${msg.g}` : msg.i ? `instance ${msg.i}` : "everything";
switch (method) {
case 'start':
_line += `Starting ${targetText}...`;
break;
case 'stop':
_line += `Stopping ${targetText}...`;
break;
case 'restart':
_line += `Restarting ${targetText}...`;
break;
case 'enable':
_line += `Enabling ${targetText}...`;
break;
case 'disable':
_line += `Disabling ${targetText}...`;
}
if (_line.length > 0) {
echo(_line);
if (client != null) {
client.write($.TNET.stringify(_line));
}
} else {
echo("simpleAction", method, JSON.stringify(msg));
}
switch (false) {
case !msg.g:
return actOnGroup(method, msg.g, cb);
case !msg.i:
return actOnInstance(method, msg.i, cb);
default:
return actOnAllGroups(method, cb);
}
};
};
Object.assign(module.exports, {Groups, actOnAllGroups, actOnInstance, addGroup, removeGroup, simpleAction});
// fulfill some out-of-order obligations
Nginx = require('./nginx');
({saveConfig} = require('../util/config'));
}).call(this);