smc-hub
Version:
CoCalc: Backend webserver component
325 lines • 15.1 kB
JavaScript
;
/*
* This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
* License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details
*/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __read = (this && this.__read) || function (o, n) {
var m = typeof Symbol === "function" && o[Symbol.iterator];
if (!m) return o;
var i = m.call(o), r, ar = [], e;
try {
while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
}
catch (error) { e = { error: error }; }
finally {
try {
if (r && !r.done && (m = i["return"])) m.call(i);
}
finally { if (e) throw e.error; }
}
return ar;
};
var __spreadArray = (this && this.__spreadArray) || function (to, from) {
for (var i = 0, il = from.length, j = to.length; i < il; i++, j++)
to[j] = from[i];
return to;
};
var __values = (this && this.__values) || function(o) {
var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
if (m) return m.call(o);
if (o && typeof o.length === "number") return {
next: function () {
if (o && i >= o.length) o = void 0;
return { value: o && o[i++], done: !o };
}
};
throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.setup_health_checks = exports.process_alive = exports.set_agent_endpoint = void 0;
// endpoints for various health checks
var debug_1 = __importDefault(require("debug"));
var L = debug_1.default("hub:healthcheck");
var net_1 = require("net");
var validator_1 = require("validator");
var misc_1 = require("smc-util/misc");
var hub_register_1 = require("./hub_register");
// self termination is only activated, if there is a COCALC_HUB_SELF_TERMINATE environment variable
// it's value is an interval in hours, minimum and maximum, for how long it should be alive
// and a drain period in minutes at the end.
// e.g. "24,48,15" for an uptime between 1 and 2 days and 15 minutes of draining
function init_self_terminate() {
var D = L.extend("init_self_terminate");
var startup = Date.now();
var conf = process.env.COCALC_HUB_SELF_TERMINATE;
if (conf == null) {
D("COCALC_HUB_SELF_TERMINATE env var not set, hence no self-termination");
return { startup: startup };
}
var _a = __read(conf.trim().split(","), 3), from_str = _a[0], to_str = _a[1], drain_str = _a[2];
if (!validator_1.isFloat(from_str, { gt: 0 }))
throw new Error("COCALC_HUB_SELF_TERMINATE/from not a positive float");
if (!validator_1.isFloat(to_str, { gt: 0 }))
throw new Error("COCALC_HUB_SELF_TERMINATE/to not a positive float");
if (!validator_1.isFloat(drain_str, { gt: 0 }))
throw new Error("COCALC_HUB_SELF_TERMINATE/drain not a positive float");
var from = parseFloat(from_str);
var to = parseFloat(to_str);
var drain_h = parseFloat(drain_str) / 60; // minutes to hours
D("parsed data:", { from: from, to: to, drain_h: drain_h });
if (from > to)
throw Error("COCALC_HUB_SELF_TERMINATE 'from' must be smaller than 'to', e.g. '24,48,15'");
var uptime = Math.random() * (to - from); // hours
var hours2ms = 1000 * 60 * 60;
var shutdown = startup + (from + uptime) * hours2ms;
var drain = shutdown - drain_h * hours2ms;
if (startup > drain) {
throw new Error("COCALC_HUB_SELF_TERMINATE: startup must be smaller than drain \u2013 " + startup + ">" + drain);
}
D({
startup: new Date(startup).toISOString(),
drain: new Date(drain).toISOString(),
shutdown: new Date(shutdown).toISOString(),
uptime: misc_1.seconds2hms((hours2ms * uptime) / 1000),
draintime: misc_1.seconds2hms((drain_h * hours2ms) / 1000),
});
return { startup: startup, shutdown: shutdown, drain: drain };
}
var _a = init_self_terminate(), startup = _a.startup, shutdown = _a.shutdown, drain = _a.drain;
var agent_port = 0;
var agent_host = "0.0.0.0";
function set_agent_endpoint(port, host) {
L("set_agent_endpoint " + agent_host + ":" + agent_port);
agent_port = port;
agent_host = host;
}
exports.set_agent_endpoint = set_agent_endpoint;
var agent_check_server;
// HAProxy agent-check TCP endpoint
// https://cbonte.github.io/haproxy-dconv/2.0/configuration.html#5.2-agent-check
// for development, set the env var in your startup script or terminal init file
// export COCALC_HUB_SELF_TERMINATE=.1,.2,1
// and then query it like that
// $ telnet 0.0.0.0 $(cat $SMC_ROOT/dev/project/ports/agent-port)
function setup_agent_check() {
if (agent_port == 0 || drain == null) {
L("setup_agent_check: agent_port not set, no agent checks");
return;
}
// TODO this could also return a "weight" for this server, based on load values
// there is also "drain", but we set it to "10%" to avoid a nasty situation, when all endpoints are draining.
// ATTN: weight must be set as well, which is poorly documented here:
// https://cbonte.github.io/haproxy-dconv/2.0/configuration.html#5.2-weight
agent_check_server = net_1.createServer(function (c) {
var msg = Date.now() < drain ? "ready up 100%" : "10%";
c.write(msg + "\r\n");
c.destroy();
});
agent_check_server.listen(agent_port, agent_host);
L("setup_agent_check: listening on " + agent_host + ":" + agent_port);
}
// this could be directly in setup_health_checks, but we also need it in proxy.coffee
// proxy.coffee must be rewritten and restructured first – just wrapping it with a router
// didn't work at all for me
function process_alive() {
var txt = "alive: YES";
var is_dead = true;
if (!hub_register_1.database_is_working()) {
// this will stop haproxy from routing traffic to us
// until db connection starts working again.
txt = "alive: NO – database not working";
}
else if (shutdown != null && Date.now() > shutdown) {
txt = "alive: NO – shutdown initiated";
}
else {
is_dead = false;
}
var code = is_dead ? 404 : 200;
return { txt: txt, code: code };
}
exports.process_alive = process_alive;
function check_concurrent(db) {
var c = db.concurrent();
if (c >= db._concurrent_warn) {
return {
status: "hub not healthy, since concurrent " + c + " >= " + db._concurrent_warn,
abort: true,
};
}
else {
return { status: "concurrent " + c + " < " + db._concurrent_warn };
}
}
function check_uptime() {
var now = Date.now();
var uptime = misc_1.seconds2hms((now - startup) / 1000);
if (shutdown != null && drain != null) {
if (now >= shutdown) {
var msg = "uptime " + uptime + " \u2013 expired, terminating now";
L(msg);
return { status: msg, abort: true };
}
else {
var until = misc_1.seconds2hms((shutdown - now) / 1000);
var drain_str = drain > now
? "draining in " + misc_1.seconds2hms((drain - now) / 1000)
: "draining now";
var msg = "uptime " + uptime + " \u2013 " + drain_str + " \u2013 terminating in " + until;
L(msg);
return { status: msg };
}
}
else {
var msg = "uptime " + uptime + " \u2013 no self-termination";
L(msg);
return { status: msg };
}
}
// same note as above for process_alive()
function process_health_check(db, extra) {
if (extra === void 0) { extra = []; }
return __awaiter(this, void 0, void 0, function () {
var any_abort, txt, _a, _b, test_1, _c, status_1, abort, e_1_1, code;
var e_1, _d;
return __generator(this, function (_e) {
switch (_e.label) {
case 0:
any_abort = false;
txt = "healthchecks:\n";
_e.label = 1;
case 1:
_e.trys.push([1, 6, 7, 8]);
_a = __values(__spreadArray([function () { return check_concurrent(db); }, check_uptime], __read(extra))), _b = _a.next();
_e.label = 2;
case 2:
if (!!_b.done) return [3 /*break*/, 5];
test_1 = _b.value;
return [4 /*yield*/, test_1()];
case 3:
_c = _e.sent(), status_1 = _c.status, abort = _c.abort;
txt += status_1 + " \u2013 " + (abort === true ? "FAIL" : "OK") + "\n";
any_abort = any_abort || abort === true;
_e.label = 4;
case 4:
_b = _a.next();
return [3 /*break*/, 2];
case 5: return [3 /*break*/, 8];
case 6:
e_1_1 = _e.sent();
e_1 = { error: e_1_1 };
return [3 /*break*/, 8];
case 7:
try {
if (_b && !_b.done && (_d = _a.return)) _d.call(_a);
}
finally { if (e_1) throw e_1.error; }
return [7 /*endfinally*/];
case 8:
code = any_abort ? 404 : 200;
return [2 /*return*/, { code: code, txt: txt }];
}
});
});
}
function setup_health_checks(opts) {
return __awaiter(this, void 0, void 0, function () {
var router, db, extra;
var _this = this;
return __generator(this, function (_a) {
router = opts.router, db = opts.db, extra = opts.extra;
setup_agent_check();
// used by HAPROXY for testing that this hub is OK to receive traffic
router.get("/alive", function (_, res) {
var _a = process_alive(), code = _a.code, txt = _a.txt;
res.type("txt");
res.status(code);
res.send(txt);
});
// this is a more general check than concurrent-warn
// additionally to checking the database condition, it also self-terminates
// this hub if it is running for quite some time. beyond that, in the future
// there could be even more checks on top of that.
router.get("/healthcheck", function (_, res) { return __awaiter(_this, void 0, void 0, function () {
var _a, txt, code;
return __generator(this, function (_b) {
switch (_b.label) {
case 0: return [4 /*yield*/, process_health_check(db, extra)];
case 1:
_a = _b.sent(), txt = _a.txt, code = _a.code;
res.status(code);
res.type("txt");
res.send(txt);
return [2 /*return*/];
}
});
}); });
// /concurrent-warn -- could be used by kubernetes to decide whether or not to kill the container; if
// below the warn thresh, returns number of concurrent connection; if hits warn, then
// returns 404 error, meaning hub may be unhealthy. Kubernetes will try a few times before
// killing the container. Will also return 404 if there is no working database connection.
router.get("/concurrent-warn", function (_, res) {
res.type("txt");
if (!hub_register_1.database_is_working()) {
L("/concurrent-warn: not healthy, since database connection not working");
res.status(404).end();
return;
}
var c = db.concurrent();
if (c >= db._concurrent_warn) {
L("/concurrent-warn: not healthy, since concurrent " + c + " >= " + db._concurrent_warn);
res.status(404).end();
return;
}
res.send("" + c);
});
// Return number of concurrent connections (could be useful)
router.get("/concurrent", function (_, res) {
res.type("txt");
res.send("" + db.concurrent());
});
return [2 /*return*/];
});
});
}
exports.setup_health_checks = setup_health_checks;
//# sourceMappingURL=health-checks.js.map