@cocalc/server
Version:
CoCalc server functionality: functions used by either the hub and the next.js server
446 lines • 16.9 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.restartProjectIfRunning = exports.copyPath = exports.ensureConfFilesExists = exports.getStatus = exports.getState = exports.getEnvironment = exports.sanitizedEnv = exports.deleteUser = exports.createUser = exports.launchProjectDaemon = exports.setupDataPath = exports.isProjectRunning = exports.getProjectPID = exports.getUsername = exports.homePath = exports.dataPath = exports.chown = exports.mkdir = void 0;
const util_1 = require("util");
const path_1 = require("path");
const child_process_1 = require("child_process");
const await_spawn_1 = __importDefault(require("await-spawn"));
const fs = __importStar(require("fs"));
const data_1 = require("@cocalc/backend/data");
const misc_1 = require("@cocalc/util/misc");
const async_utils_1 = require("@cocalc/util/async-utils");
const logger_1 = __importDefault(require("@cocalc/backend/logger"));
const misc_2 = require("@cocalc/backend/misc");
const base_path_1 = __importDefault(require("@cocalc/backend/base-path"));
const database_1 = require("@cocalc/database");
const _1 = require(".");
const winston = (0, logger_1.default)("project-control:util");
exports.mkdir = (0, util_1.promisify)(fs.mkdir);
const readFile = (0, util_1.promisify)(fs.readFile);
const stat = (0, util_1.promisify)(fs.stat);
const copyFile = (0, util_1.promisify)(fs.copyFile);
const rm = (0, util_1.promisify)(fs.rm);
async function chown(path, uid) {
await (0, util_1.promisify)(fs.chown)(path, uid, uid);
}
exports.chown = chown;
function dataPath(HOME) {
return (0, path_1.join)(HOME, ".smc");
}
exports.dataPath = dataPath;
function homePath(project_id) {
return data_1.projects.replace("[project_id]", project_id);
}
exports.homePath = homePath;
function getUsername(project_id) {
return project_id.split("-").join("");
}
exports.getUsername = getUsername;
function pidIsRunning(pid) {
try {
process.kill(pid, 0);
return true;
}
catch (_err) {
return false;
}
}
function pidFile(HOME) {
return (0, path_1.join)(dataPath(HOME), "project.pid");
}
// throws error if no such file
async function getProjectPID(HOME) {
return parseInt((await readFile(pidFile(HOME))).toString());
}
exports.getProjectPID = getProjectPID;
async function isProjectRunning(HOME) {
try {
const pid = await getProjectPID(HOME);
//winston.debug(`isProjectRunning(HOME="${HOME}") -- pid=${pid}`);
return pidIsRunning(pid);
}
catch (err) {
//winston.debug(`isProjectRunning(HOME="${HOME}") -- no pid ${err}`);
// err would happen if file doesn't exist, which means nothing to do.
return false;
}
}
exports.isProjectRunning = isProjectRunning;
async function setupDataPath(HOME, uid) {
const data = dataPath(HOME);
winston.debug(`setup "${data}"...`);
await rm(data, { recursive: true, force: true });
await (0, exports.mkdir)(data);
if (uid != null) {
await chown(data, uid);
}
}
exports.setupDataPath = setupDataPath;
async function launchProjectDaemon(env, uid) {
winston.debug(`launching project daemon at "${env.HOME}"...`);
const cwd = (0, path_1.join)(data_1.root, "packages/project");
winston.debug(`"npx cocalc-project --daemon" from "${cwd}" with uid=${uid}`);
await (0, util_1.promisify)((cb) => {
const child = (0, child_process_1.spawn)("npx", ["cocalc-project", "--daemon", "--init", "project_init.sh"], {
env,
cwd,
uid,
gid: uid,
});
child.on("error", (err) => {
winston.debug(`project daemon error ${err}`);
cb(err);
});
child.on("exit", (code) => {
winston.debug(`project daemon exited with code ${code}`);
cb(code);
});
})();
}
exports.launchProjectDaemon = launchProjectDaemon;
async function exec(command, verbose) {
winston.debug(`exec '${command}'`);
const output = await (0, util_1.promisify)(child_process_1.exec)(command);
if (verbose) {
winston.debug(`output: ${JSON.stringify(output)}`);
}
return output;
}
async function createUser(project_id) {
const username = getUsername(project_id);
try {
await exec(`/usr/sbin/userdel ${username}`); // this also deletes the group
}
catch (_) {
// it's fine -- we delete just in case it is left over.
}
const uid = `${(0, misc_2.getUid)(project_id)}`;
winston.debug("createUser: adding group");
await exec(`/usr/sbin/groupadd -g ${uid} -o ${username}`, true);
winston.debug("createUser: adding user");
await exec(`/usr/sbin/useradd -u ${uid} -g ${uid} -o ${username} -m -d ${homePath(project_id)} -s /bin/bash`, true);
}
exports.createUser = createUser;
async function deleteUser(project_id) {
const username = getUsername(project_id);
const uid = `${(0, misc_2.getUid)(project_id)}`;
await exec(`pkill -9 -u ${uid} | true`); // | true since pkill exit 1 if nothing killed.
try {
await exec(`/usr/sbin/userdel ${username}`); // this also deletes the group
}
catch (_) {
// not error if not there...
}
}
exports.deleteUser = deleteUser;
function sanitizedEnv(env) {
const env2 = { ...env };
// Remove some potentially confusing env variables
for (const key of [
"PGDATA",
"PGHOST",
"NODE_ENV",
"NODE_OPTIONS",
"BASE_PATH",
"PORT",
"DATA",
"LOGS",
]) {
delete env2[key];
}
// Comment about stripping things starting with /root:
// These tend to creep in as npm changes, e.g., 'npm_config_userconfig' is
// suddenly /root/.npmrc, and due to permissions this will break starting
// projects with a mysterious "exit code 243" and no further info, which
// is really hard to track down.
for (const key in env2) {
if (key.startsWith("COCALC_") ||
env2[key]?.startsWith("/root") ||
env2[key] == null) {
delete env2[key];
}
}
return env2;
}
exports.sanitizedEnv = sanitizedEnv;
async function getEnvironment(project_id) {
const extra = await (0, async_utils_1.callback2)((0, database_1.db)().get_project_extra_env, { project_id });
const extra_env = Buffer.from(JSON.stringify(extra ?? {})).toString("base64");
const USER = getUsername(project_id);
const HOME = homePath(project_id);
const DATA = dataPath(HOME);
return {
...sanitizedEnv(process.env),
...{
HOME,
BASE_PATH: base_path_1.default,
DATA,
LOGS: (0, path_1.join)(DATA, "logs"),
// important to reset the COCALC_ vars since server env has own in a project
COCALC_PROJECT_ID: project_id,
COCALC_USERNAME: USER,
USER,
COCALC_EXTRA_ENV: extra_env,
PATH: `${HOME}/bin:${HOME}/.local/bin:${process.env.PATH}`,
},
};
}
exports.getEnvironment = getEnvironment;
async function getState(HOME) {
winston.debug(`getState("${HOME}")`);
try {
return {
ip: "localhost",
state: (await isProjectRunning(HOME)) ? "running" : "opened",
time: new Date(),
};
}
catch (err) {
return {
error: `${err}`,
time: new Date(),
state: "opened",
};
}
}
exports.getState = getState;
async function getStatus(HOME) {
winston.debug(`getStatus("${HOME}")`);
const data = dataPath(HOME);
const status = {};
if (!(await isProjectRunning(HOME))) {
return status;
}
for (const path of [
"project.pid",
"hub-server.port",
"browser-server.port",
"sage_server.port",
"sage_server.pid",
"secret_token",
"start-timestamp.txt",
"session-id.txt",
]) {
try {
const val = (await readFile((0, path_1.join)(data, path))).toString().trim();
if (path.endsWith(".pid")) {
const pid = parseInt(val);
if (pidIsRunning(pid)) {
status[path] = pid;
}
}
else if (path == "start-timestamp.txt") {
status.start_ts = parseInt(val);
}
else if (path == "session-id.txt") {
status.session_id = val;
}
else if (path.endsWith(".port")) {
status[path] = parseInt(val);
}
else {
status[path] = val;
}
}
catch (_err) {
//winston.debug(`getStatus: ${_err}`);
}
}
return status;
}
exports.getStatus = getStatus;
async function ensureConfFilesExists(HOME, uid) {
for (const path of ["bashrc", "bash_profile"]) {
const target = (0, path_1.join)(HOME, `.${path}`);
try {
await stat(target);
}
catch (_) {
// file does NOT exist, so create
const source = (0, path_1.join)(data_1.root, "smc_pyutil/smc_pyutil/templates", process.platform, path);
try {
await copyFile(source, target);
if (uid != null) {
await chown(target, uid);
}
}
catch (err) {
winston.error(`ensureConfFilesExists -- ${err}`);
}
}
}
}
exports.ensureConfFilesExists = ensureConfFilesExists;
// Copy a path using rsync and the specified options
// on the local filesystem.
// NOTE: the scheduled CopyOptions
// are not implemented at all here.
async function copyPath(opts, project_id, target_uid) {
winston.info(`copyPath(source="${project_id}"): opts=${JSON.stringify(opts)}`);
const { path, overwrite_newer, delete_missing, backup, timeout, bwlimit } = opts;
if (path == null) {
// typescript already enforces this...
throw Error("path must be specified");
}
const target_project_id = opts.target_project_id ?? project_id;
const target_path = opts.target_path ?? path;
// check that both UUID's are valid
if (!(0, misc_1.is_valid_uuid_string)(project_id)) {
throw Error(`project_id=${project_id} is invalid`);
}
if (!(0, misc_1.is_valid_uuid_string)(target_project_id)) {
throw Error(`target_project_id=${target_project_id} is invalid`);
}
// determine canonical absolute path to source
const sourceHome = homePath(project_id);
const source_abspath = (0, path_1.resolve)((0, path_1.join)(sourceHome, path));
if (!source_abspath.startsWith(sourceHome)) {
throw Error(`source path must be contained in project home dir`);
}
// determine canonical absolute path to target
const targetHome = homePath(target_project_id);
const target_abspath = (0, path_1.resolve)((0, path_1.join)(targetHome, target_path));
if (!target_abspath.startsWith(targetHome)) {
throw Error(`target path must be contained in target project home dir`);
}
// check for trivial special case.
if (source_abspath == target_abspath) {
return;
}
// This can throw an exception if path doesn't exist, which is fine.
const stats = await stat(source_abspath);
// We will use this to decide if we need to add / at end in rsync args.
const isDir = stats.isDirectory();
// Handle args and options to rsync.
// saxz = compressed, archive mode (so leave symlinks, etc.), don't cross filesystem boundaries
// However, omit-link-times -- see http://forums.whirlpool.net.au/archive/2317650 and
// https://github.com/sagemathinc/cocalc/issues/2713
const args = [];
if (process.platform == "darwin") {
// MacOS rsync is pretty cripled, so we omit some very helpful options.
args.push("-zax");
}
else {
args.push(...["-zaxs", "--omit-link-times"]);
}
if (opts.exclude) {
for (const pattern of opts.exclude) {
args.push("--exclude");
args.push(pattern);
}
}
if (!overwrite_newer) {
args.push("--update");
}
if (backup) {
args.push("--backup");
}
if (delete_missing) {
// IMPORTANT: newly created files will be deleted even if overwrite_newer is true
args.push("--delete");
}
if (bwlimit) {
args.push(`--bwlimit=${bwlimit}`);
}
if (timeout) {
args.push(`--timeout=${timeout}`);
}
if (target_uid && target_project_id != project_id) {
// change target ownership on copy; only do this if explicitly requested and needed.
args.push(`--chown=${target_uid}:${target_uid}`);
}
args.push(source_abspath + (isDir ? "/" : ""));
args.push(target_abspath + (isDir ? "/" : ""));
async function make_target_path() {
// note -- uid/gid ignored if target_uid not set.
if (isDir) {
await (0, await_spawn_1.default)("mkdir", ["-p", target_abspath], {
uid: target_uid,
gid: target_uid,
});
}
else {
await (0, await_spawn_1.default)("mkdir", ["-p", (0, path_1.dirname)(target_abspath)], {
uid: target_uid,
gid: target_uid,
});
}
}
// For making the target directory when target_uid is specified,
// we need to use setuid and be the target user, since otherwise
// the permissions are wrong on the containing directory,
// as explained here: https://github.com/sagemathinc/cocalc-docker/issues/146
// However, this will fail if the user hasn't been created, hence
// this code is extra complicated.
try {
await make_target_path();
}
catch (_err) {
// The above probably failed due to the uid/gid not existing.
// In that case, we create the user, then try again.
await createUser(target_project_id);
await make_target_path();
// Assuming the above did work, it's very likely the original
// failing was due to the user not existing, so now we delete
// it again.
await deleteUser(target_project_id);
}
// do the copy!
winston.info(`doing rsync ${args.join(" ")}`);
if (opts.wait_until_done ?? true) {
try {
const stdout = await (0, await_spawn_1.default)("rsync", args, {
timeout: opts.timeout
? 1000 * opts.timeout
: undefined /* spawnAsync has ms units, but rsync has second units */,
});
winston.info(`finished rsync ${stdout}`);
}
catch (err) {
throw Error(`WARNING: copy exited with an error -- ${err.stderr} -- "rsync ${args.join(" ")}"`);
}
}
else {
// TODO/NOTE: this will silently not report any errors.
(0, child_process_1.spawn)("rsync", args, { timeout: opts.timeout });
}
}
exports.copyPath = copyPath;
async function restartProjectIfRunning(project_id) {
// If necessary, restart project to ensure that license gets applied.
// This is not bullet proof in all cases, e.g., for a newly created project,
// and it is better to apply the license when creating the project if possible.
const project = (0, _1.getProject)(project_id);
const { state } = await project.state();
if (state == "starting" || state == "running") {
project.restart(); // don't await this -- it could take a long time and isn't necessary to wait for.
}
}
exports.restartProjectIfRunning = restartProjectIfRunning;
//# sourceMappingURL=util.js.map