smc-hub
Version:
CoCalc: Backend webserver component
580 lines • 32.9 kB
JavaScript
"use strict";
//########################################################################
// This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
// License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details
//########################################################################
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.program = void 0;
// This is the CoCalc Global HUB. It runs as a daemon, sitting in the
// middle of the action, connected to potentially thousands of clients,
// many Sage sessions, and PostgreSQL database.
/*
TEMPORARY: There is still some code that assumes that process.env.SMC_ROOT is
defined, we for now we define it here on startup using the directory of the
dist/hub.js file typescript creates. This will NOT work if we use cocalc via
the smc-hub npm package, so you better set SMC_ROOT properly in that case.
*/
var path_1 = require("path");
if (!process.env.SMC_ROOT) {
process.env.SMC_ROOT = path_1.resolve(path_1.join(__dirname, "..", ".."));
}
var child_process_1 = require("child_process");
var project_control_1 = require("./servers/project-control");
var blocked_1 = __importDefault(require("blocked"));
var commander_1 = require("commander");
var async_utils_1 = require("smc-util/async-utils");
var awaiting_1 = require("awaiting");
var logger_1 = require("./logger");
var memory_1 = require("smc-util-node/memory");
var base_path_1 = __importDefault(require("smc-util-node/base-path"));
var async_utils_2 = require("smc-util/async-utils");
var COOKIE_OPTIONS = require("./client").COOKIE_OPTIONS; // import { COOKIE_OPTIONS } from "./client";
var auth_1 = require("./auth");
var base_path_2 = __importDefault(require("smc-util-node/base-path"));
var migrate_account_token_1 = require("./postgres/migrate-account-token");
var always_running_1 = require("./postgres/always-running");
var health_checks_1 = require("./health-checks");
var handle_1 = require("./mentions/handle");
var MetricsRecorder = require("./metrics-recorder"); // import * as MetricsRecorder from "./metrics-recorder";
var hub_register_1 = require("./hub_register");
var initZendesk = require("./support").init_support; // import { init_support as initZendesk } from "./support";
var clients_1 = require("./clients");
var sync_1 = require("./stripe/sync");
var stripe_1 = require("./stripe");
var data_1 = require("smc-util-node/data");
var port_1 = __importDefault(require("smc-util-node/port"));
var database_1 = require("./servers/database");
var express_app_1 = __importDefault(require("./servers/express-app"));
var http_1 = __importDefault(require("./servers/http"));
var http_redirect_1 = __importDefault(require("./servers/http-redirect"));
var database_2 = __importDefault(require("./servers/database"));
var project_control_2 = __importDefault(require("./servers/project-control"));
var stop_idle_projects_1 = __importDefault(require("./project-control/stop-idle-projects"));
var version_1 = __importDefault(require("./servers/version"));
var primus_1 = __importDefault(require("./servers/primus"));
var proxy_1 = __importDefault(require("./proxy"));
// Logger tagged with 'hub' for this file.
var winston = logger_1.getLogger("hub");
// program gets populated with the command line options below.
var program = {};
exports.program = program;
// How frequently to register with the database that this hub is up and running,
// and also report number of connected clients.
var REGISTER_INTERVAL_S = 20;
// the jsmap of connected clients
var clients = clients_1.getClients();
function reset_password(email_address) {
return __awaiter(this, void 0, void 0, function () {
var err_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
return [4 /*yield*/, async_utils_1.callback2(database_1.database.reset_password, { email_address: email_address })];
case 1:
_a.sent();
winston.info("Password changed for " + email_address);
return [3 /*break*/, 3];
case 2:
err_1 = _a.sent();
winston.info("Error resetting password -- " + err_1);
return [3 /*break*/, 3];
case 3: return [2 /*return*/];
}
});
});
}
function startLandingService() {
return __awaiter(this, void 0, void 0, function () {
var LandingServer, uncaught_exception_total, landing_server;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
LandingServer = require("@cocalc/landing").LandingServer;
return [4 /*yield*/, initMetrics()];
case 1:
uncaught_exception_total = (_a.sent()).uncaught_exception_total;
landing_server = new LandingServer({
db: database_1.database,
port: port_1.default,
base_url: base_path_1.default,
});
return [4 /*yield*/, landing_server.start()];
case 2:
_a.sent();
addErrorListeners(uncaught_exception_total);
return [2 /*return*/];
}
});
});
}
// This calculates and updates the statistics for the /stats endpoint.
// It's important that we call this periodically, because otherwise the /stats data is outdated.
function init_update_stats() {
return __awaiter(this, void 0, void 0, function () {
var update;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
winston.info("init updating stats periodically");
update = function () { return async_utils_1.callback2(database_1.database.get_stats); };
// Do it every minute:
setInterval(function () { return update(); }, 60000);
// Also do it once now:
return [4 /*yield*/, update()];
case 1:
// Also do it once now:
_a.sent();
return [2 /*return*/];
}
});
});
}
// This calculates and updates the site_license_usage_log.
// It's important that we call this periodically, if we want
// to be able to monitor site license usage. This is enabled
// by default only for dev mode (so for development).
function init_update_site_license_usage_log() {
return __awaiter(this, void 0, void 0, function () {
var update;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
winston.info("init updating site license usage log periodically");
update = function () { return __awaiter(_this, void 0, void 0, function () { return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, database_1.database.update_site_license_usage_log()];
case 1: return [2 /*return*/, _a.sent()];
}
}); }); };
setInterval(update, 31000);
return [4 /*yield*/, update()];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
}
function initMetrics() {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
winston.info("Initializing Metrics Recorder...");
return [4 /*yield*/, awaiting_1.callback(MetricsRecorder.init, winston)];
case 1:
_a.sent();
return [2 /*return*/, {
metric_blocked: MetricsRecorder.new_counter("blocked_ms_total", 'accumulates the "blocked" time in the hub [ms]'),
uncaught_exception_total: MetricsRecorder.new_counter("uncaught_exception_total", 'counts "BUG"s'),
}];
}
});
});
}
function startServer() {
return __awaiter(this, void 0, void 0, function () {
var _a, metric_blocked, uncaught_exception_total, projectControl, _b, router, app, httpServer, msg;
var _this = this;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
winston.info("start_server");
winston.info("dev = " + program.dev);
// Be very sure cookies do NOT work unless over https. IMPORTANT.
if (!COOKIE_OPTIONS.secure) {
throw Error("client cookie options are not secure");
}
winston.info("base_path='" + base_path_2.default + "'");
winston.info("using database \"" + program.keyspace + "\" and database-nodes=\"" + program.databaseNodes + "\"");
return [4 /*yield*/, initMetrics()];
case 1:
_a = _c.sent(), metric_blocked = _a.metric_blocked, uncaught_exception_total = _a.uncaught_exception_total;
// Log anything that blocks the CPU for more than 10ms -- see https://github.com/tj/node-blocked
blocked_1.default(function (ms) {
if (ms > 0) {
metric_blocked.inc(ms);
}
// record that something blocked:
winston.debug("BLOCKED for " + ms + "ms");
});
// Log heap memory usage info
memory_1.init(winston.debug);
// Wait for database connection to work. Everything requires this.
return [4 /*yield*/, async_utils_2.retry_until_success({
f: function () { return __awaiter(_this, void 0, void 0, function () { return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, async_utils_1.callback2(database_1.database.connect)];
case 1: return [2 /*return*/, _a.sent()];
}
}); }); },
start_delay: 1000,
max_delay: 10000,
})];
case 2:
// Wait for database connection to work. Everything requires this.
_c.sent();
winston.info("connected to database.");
if (!program.updateDatabaseSchema) return [3 /*break*/, 4];
winston.info("Update database schema");
return [4 /*yield*/, async_utils_1.callback2(database_1.database.update_schema)];
case 3:
_c.sent();
_c.label = 4;
case 4:
if (program.agentPort) {
winston.info("Configure agent port");
health_checks_1.set_agent_endpoint(program.agentPort, program.hostname);
}
// Handle potentially ancient cocalc installs with old account registration token.
winston.info("Check for all account registration token");
return [4 /*yield*/, migrate_account_token_1.migrate_account_token(database_1.database)];
case 5:
_c.sent();
// Mentions
if (program.mentions) {
winston.info("enabling handling of mentions...");
handle_1.handle_mentions_loop(database_1.database);
}
// Project control
winston.info("initializing project control...");
projectControl = project_control_2.default(program);
if (program.mode != "kucalc" && program.websocketServer) {
// We handle idle timeout of projects.
// This can be disabled via COCALC_NO_IDLE_TIMEOUT.
// This only uses the admin-configurable settings field of projects
// in the database and isn't aware of licenses or upgrades.
stop_idle_projects_1.default(projectControl);
}
if (!program.websocketServer) return [3 /*break*/, 15];
// Initialize the version server -- must happen after updating schema
// (for first ever run).
return [4 /*yield*/, version_1.default()];
case 6:
// Initialize the version server -- must happen after updating schema
// (for first ever run).
_c.sent();
// Stripe
winston.info("initializing stripe support...");
return [4 /*yield*/, stripe_1.init_stripe(database_1.database, winston)];
case 7:
_c.sent();
// Zendesk
winston.info("initializing zendesk support...");
return [4 /*yield*/, awaiting_1.callback(initZendesk)];
case 8:
_c.sent();
if (!(program.mode == "single-user" && process.env.USER == "user")) return [3 /*break*/, 12];
// Definitely in dev mode, probably on cocalc.com in a project, so we kill
// all the running projects when starting the hub:
// Whenever we start the dev server, we just assume
// all projects are stopped, since assuming they are
// running when they are not is bad. Something similar
// is done in cocalc-docker.
winston.info("killing all projects...");
return [4 /*yield*/, async_utils_1.callback2(database_1.database._query, {
safety_check: false,
query: 'update projects set state=\'{"state":"opened"}\'',
})];
case 9:
_c.sent();
return [4 /*yield*/, child_process_1.spawn("pkill", ["-f", "node_modules/.bin/cocalc-project"])];
case 10:
_c.sent();
// Also, unrelated to killing projects, for purposes of developing
// custom software images, we inject a couple of random nonsense entries
// into the table in the DB:
winston.info("inserting random nonsense compute images in database");
return [4 /*yield*/, async_utils_1.callback2(database_1.database.insert_random_compute_images)];
case 11:
_c.sent();
_c.label = 12;
case 12:
if (!(program.mode != "kucalc")) return [3 /*break*/, 15];
return [4 /*yield*/, init_update_stats()];
case 13:
_c.sent();
return [4 /*yield*/, init_update_site_license_usage_log()];
case 14:
_c.sent();
// This is async but runs forever, so don't wait for it.
winston.info("init starting always running projects");
always_running_1.init_start_always_running_projects(database_1.database);
_c.label = 15;
case 15: return [4 /*yield*/, express_app_1.default({
dev: program.dev,
isPersonal: program.personal,
projectControl: projectControl,
landingServer: !!program.landingServer,
shareServer: !!program.shareServer,
})];
case 16:
_b = _c.sent(), router = _b.router, app = _b.app;
// The express app create via initExpressApp above **assumes** that init_passport is done
// or complains a lot. This is obviously not really necessary, but we leave it for now.
return [4 /*yield*/, async_utils_1.callback2(auth_1.init_passport, {
router: router,
database: database_1.database,
host: program.hostname,
})];
case 17:
// The express app create via initExpressApp above **assumes** that init_passport is done
// or complains a lot. This is obviously not really necessary, but we leave it for now.
_c.sent();
httpServer = http_1.default({
cert: program.httpsCert,
key: program.httpsKey,
app: app,
});
winston.info("starting webserver listening on " + program.hostname + ":" + port_1.default);
return [4 /*yield*/, awaiting_1.callback(httpServer.listen.bind(httpServer), port_1.default, program.hostname)];
case 18:
_c.sent();
if (!(port_1.default == 443 && program.httpsCert && program.httpsKey)) return [3 /*break*/, 20];
// also start a redirect from port 80 to port 443.
return [4 /*yield*/, http_redirect_1.default(program.hostname)];
case 19:
// also start a redirect from port 80 to port 443.
_c.sent();
_c.label = 20;
case 20:
if (program.websocketServer) {
winston.info("initializing primus websocket server");
primus_1.default({
httpServer: httpServer,
router: router,
projectControl: projectControl,
clients: clients,
host: program.hostname,
isPersonal: program.personal,
});
}
if (program.proxyServer) {
winston.info("initializing the http proxy server on port " + port_1.default);
proxy_1.default({
projectControl: projectControl,
isPersonal: !!program.personal,
httpServer: httpServer,
app: app,
});
}
if (!(program.websocketServer || program.proxyServer || program.shareServer)) return [3 /*break*/, 22];
winston.info("Starting registering periodically with the database and updating a health check...");
// register the hub with the database periodically, and
// also confirms that database is working.
return [4 /*yield*/, async_utils_1.callback2(hub_register_1.start, {
database: database_1.database,
clients: clients,
host: program.hostname,
port: port_1.default,
interval_s: REGISTER_INTERVAL_S,
})];
case 21:
// register the hub with the database periodically, and
// also confirms that database is working.
_c.sent();
msg = "Started HUB!\n*****\n\n " + (program.httpsKey ? "https" : "http") + "://" + program.hostname + ":" + port_1.default + base_path_1.default + "\n\n*****";
winston.info(msg);
_c.label = 22;
case 22:
addErrorListeners(uncaught_exception_total);
return [2 /*return*/];
}
});
});
}
// addErrorListeners: after successful startup, don't crash on routine errors.
// We don't do this until startup, since we do want to crash on errors on startup.
// TODO: could alternatively be handled via winston (?).
function addErrorListeners(uncaught_exception_total) {
process.addListener("uncaughtException", function (err) {
winston.error("BUG ****************************************************************************");
winston.error("Uncaught exception: " + err);
console.error(err.stack);
winston.error(err.stack);
winston.error("BUG ****************************************************************************");
database_1.database === null || database_1.database === void 0 ? void 0 : database_1.database.uncaught_exception(err);
uncaught_exception_total.inc(1);
});
return process.on("unhandledRejection", function (reason, p) {
winston.error("BUG UNHANDLED REJECTION *********************************************************");
console.error(p, reason); // strangely sometimes winston.error can't actually show the traceback...
winston.error("Unhandled Rejection at:", p, "reason:", reason);
winston.error("BUG UNHANDLED REJECTION *********************************************************");
database_1.database === null || database_1.database === void 0 ? void 0 : database_1.database.uncaught_exception(p);
uncaught_exception_total.inc(1);
});
}
//############################################
// Process command line arguments
//############################################
function main() {
var _a;
return __awaiter(this, void 0, void 0, function () {
var default_db, opts, name_1, err_2;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
default_db = (_a = process.env.PGHOST) !== null && _a !== void 0 ? _a : "localhost";
commander_1.program
.name("cocalc-hub-server")
.usage("options")
.addOption(new commander_1.Option("--mode [string]", "REQUIRED mode in which to run CoCalc (" + project_control_1.COCALC_MODES.join(", ") + ")").choices(project_control_1.COCALC_MODES))
.option("--all", "runs all of the servers: websocket, proxy, share, and landing (so you don't have to pass all those opts separately), and also mentions updator and updates db schema on startup; use this in situations where there is a single hub that serves everything (instead of a microservice situation like kucalc)")
.option("--websocket-server", "run the websocket server")
.option("--proxy-server", "run the proxy server")
.option("--share-server", "run the share server")
.option("--landing-server", "run the closed source landing pages server (requires @cocalc/landing installed)")
.option("--https-key [string]", "serve over https. argument should be a key file (both https-key and https-cert must be specified)")
.option("--https-cert [string]", "serve over https. argument should be a cert file (both https-key and https-cert must be specified)")
.option("--share-path [string]", "describes where the share server finds shared files for each project at (default: " + data_1.projects + "/[project_id])", data_1.projects + "/[project_id]")
.option("--agent-port <n>", "port for HAProxy agent-check (default: 0 -- do not start)", function (n) { return parseInt(n); }, 0)
.option("--hostname [string]", 'host of interface to bind to (default: "127.0.0.1")', "127.0.0.1")
.option("--database-nodes <string,string,...>", "database address (default: '" + default_db + "')", default_db)
.option("--keyspace [string]", 'Database name to use (default: "smc")', "smc")
.option("--passwd [email_address]", "Reset password of given user", "")
.option("--update-database-schema", "If specified, updates database schema on startup (always happens when mode is not kucalc).")
.option("--stripe-sync", "Sync stripe subscriptions to database for all users with stripe id", "yes")
.option("--update-stats", "Calculates the statistics for the /stats endpoint and stores them in the database", "yes")
.option("--delete-expired", "Delete expired data from the database", "yes")
.option("--blob-maintenance", "Do blob-related maintenance (dump to tarballs, offload to gcloud)", "yes")
.option("--mentions", "if given, periodically handle mentions")
.option("--test", "terminate after setting up the hub -- used to test if it starts up properly")
.option("--db-concurrent-warn <n>", "be very unhappy if number of concurrent db requests exceeds this (default: 300)", function (n) { return parseInt(n); }, 300)
.option("--personal", "run VERY UNSAFE: there is only one user and no authentication")
.parse(process.argv);
opts = commander_1.program.opts();
for (name_1 in opts) {
program[name_1] = opts[name_1];
}
if (!program.mode) {
throw Error("the --mode option must be specified");
process.exit(1);
}
if (program.all) {
program.websocketServer =
program.proxyServer =
program.shareServer =
program.landingServer =
program.mentions =
program.updateDatabaseSchema =
true;
}
_b.label = 1;
case 1:
_b.trys.push([1, 16, , 17]);
// Everything we do here requires the database to be initialized. Once
// this is called, require('./postgres/database').default() is a valid db
// instance that can be used.
database_2.default({
host: program.databaseNodes,
database: program.keyspace,
concurrent_warn: program.dbConcurrentWarn,
});
if (!program.passwd) return [3 /*break*/, 3];
winston.debug("Resetting password");
return [4 /*yield*/, reset_password(program.passwd)];
case 2:
_b.sent();
process.exit();
return [3 /*break*/, 15];
case 3:
if (!program.stripeSync) return [3 /*break*/, 5];
winston.debug("Stripe sync");
return [4 /*yield*/, sync_1.stripe_sync({ database: database_1.database, logger: winston })];
case 4:
_b.sent();
process.exit();
return [3 /*break*/, 15];
case 5:
if (!program.deleteExpired) return [3 /*break*/, 7];
return [4 /*yield*/, async_utils_1.callback2(database_1.database.delete_expired, {
count_only: false,
})];
case 6:
_b.sent();
process.exit();
return [3 /*break*/, 15];
case 7:
if (!program.blobMaintenance) return [3 /*break*/, 9];
return [4 /*yield*/, async_utils_1.callback2(database_1.database.blob_maintenance)];
case 8:
_b.sent();
process.exit();
return [3 /*break*/, 15];
case 9:
if (!program.updateStats) return [3 /*break*/, 11];
return [4 /*yield*/, async_utils_1.callback2(database_1.database.get_stats)];
case 10:
_b.sent();
process.exit();
return [3 /*break*/, 15];
case 11:
if (!(program.mode == "kucalc" && program.landingServer)) return [3 /*break*/, 13];
// Kucalc has its own *dedicated* landing server,
// whereas for the other modes startServer (below) just
// enables a landing server when the flag is set.
console.log("LANDING PAGE MODE");
return [4 /*yield*/, startLandingService()];
case 12:
_b.sent();
return [3 /*break*/, 15];
case 13: return [4 /*yield*/, startServer()];
case 14:
_b.sent();
_b.label = 15;
case 15: return [3 /*break*/, 17];
case 16:
err_2 = _b.sent();
console.log(err_2);
winston.error("Error -- ", err_2);
process.exit(1);
return [3 /*break*/, 17];
case 17: return [2 /*return*/];
}
});
});
}
main();
//# sourceMappingURL=hub.js.map