smc-hub
Version:
CoCalc: Backend webserver component
649 lines • 31 kB
JavaScript
"use strict";
/*
* This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
* License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details
*/
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== "function" && b !== null)
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __values = (this && this.__values) || function(o) {
var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
if (m) return m.call(o);
if (o && typeof o.length === "number") return {
next: function () {
if (o && i >= o.length) o = void 0;
return { value: o && o[i++], done: !o };
}
};
throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProjectAndUserTracker = void 0;
/*
* decaffeinate suggestions:
* DS001: Remove Babel/TypeScript constructor workaround
* DS102: Remove unnecessary code created because of implicit returns
* DS103: Rewrite code to no longer use __guard__
* DS205: Consider reworking code to avoid use of IIFEs
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
var events_1 = require("events");
var awaiting_1 = require("awaiting");
var async_utils_1 = require("smc-util/async-utils");
var misc_1 = require("smc-util/misc");
var all_results = require("../postgres-base").all_results;
var ProjectAndUserTracker = /** @class */ (function (_super) {
__extends(ProjectAndUserTracker, _super);
function ProjectAndUserTracker(db) {
var _this = _super.call(this) || this;
_this.state = "init";
// by a "set" we mean map to boolean...
// set of accounts we care about
_this.accounts = {};
// map from from project_id to set of users of a given project
_this.users = {};
// map from account_id to set of projects of a given user
_this.projects = {};
// map from account_id to map from account_ids to *number* of
// projects the two users have in common.
_this.collabs = {};
_this.register_todo = {};
// used for a runtime sanity check
_this.do_register_lock = false;
_this.db = db;
return _this;
}
ProjectAndUserTracker.prototype.assert_state = function (state, f) {
if (this.state != state) {
throw Error(f + ": state must be " + state + " but it is " + this.state);
}
};
ProjectAndUserTracker.prototype.init = function () {
return __awaiter(this, void 0, void 0, function () {
var dbg, _a, err_1;
var _this = this;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
this.assert_state("init", "init");
dbg = this.dbg("init");
dbg("Initializing Project and user tracker...");
// every changefeed for a user will result in a listener
// on an event on this one object.
this.setMaxListeners(1000);
_b.label = 1;
case 1:
_b.trys.push([1, 3, , 4]);
// create changefeed listening on changes to projects table
_a = this;
return [4 /*yield*/, async_utils_1.callback2(this.db.changefeed, {
table: "projects",
select: { project_id: "UUID" },
watch: ["users"],
where: {},
})];
case 2:
// create changefeed listening on changes to projects table
_a.feed = _b.sent();
dbg("Success");
return [3 /*break*/, 4];
case 3:
err_1 = _b.sent();
this.handle_error(err_1);
return [2 /*return*/];
case 4:
this.feed.on("change", this.handle_change.bind(this));
this.feed.on("error", this.handle_error.bind(this));
this.feed.on("close", function () { return _this.handle_error("changefeed closed"); });
this.set_state("ready");
return [2 /*return*/];
}
});
});
};
ProjectAndUserTracker.prototype.dbg = function (f) {
return this.db._dbg("Tracker." + f);
};
ProjectAndUserTracker.prototype.handle_error = function (err) {
if (this.state == "closed")
return;
// There was an error in the changefeed.
// Error is totally fatal, so we close up shop.
var dbg = this.dbg("handle_error");
dbg("err='" + err + "'");
this.emit("error", err);
this.close();
};
ProjectAndUserTracker.prototype.set_state = function (state) {
this.state = state;
this.emit(state);
};
ProjectAndUserTracker.prototype.close = function () {
var e_1, _a;
if (this.state == "closed") {
return;
}
this.set_state("closed");
this.removeAllListeners();
if (this.feed != null) {
this.feed.close();
}
if (this.register_todo != null) {
// clear any outstanding callbacks
for (var account_id in this.register_todo) {
var callbacks = this.register_todo[account_id];
if (callbacks != null) {
try {
for (var callbacks_1 = (e_1 = void 0, __values(callbacks)), callbacks_1_1 = callbacks_1.next(); !callbacks_1_1.done; callbacks_1_1 = callbacks_1.next()) {
var cb = callbacks_1_1.value;
cb("closed");
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (callbacks_1_1 && !callbacks_1_1.done && (_a = callbacks_1.return)) _a.call(callbacks_1);
}
finally { if (e_1) throw e_1.error; }
}
}
}
}
misc_1.close(this);
this.state = "closed";
};
ProjectAndUserTracker.prototype.handle_change_delete = function (old_val) {
this.assert_state("ready", "handle_change_delete");
var project_id = old_val.project_id;
if (this.users[project_id] == null) {
// no users, so nothing to worry about.
return;
}
for (var account_id in this.users[project_id]) {
this.remove_user_from_project(account_id, project_id);
}
return;
};
ProjectAndUserTracker.prototype.handle_change = function (x) {
this.assert_state("ready", "handle_change");
if (x.action === "delete") {
if (x.old_val == null)
return; // should never happen
this.handle_change_delete(x.old_val);
}
else {
if (x.new_val == null)
return; // should never happen
this.handle_change_update(x.new_val);
}
};
ProjectAndUserTracker.prototype.handle_change_update = function (new_val) {
return __awaiter(this, void 0, void 0, function () {
var dbg, project_id, users, err_2, any, users_1, users_1_1, account_id, users_now, users_2, users_2_1, account_id, users_before, account_id, account_id;
var e_2, _a, e_3, _b;
return __generator(this, function (_c) {
switch (_c.label) {
case 0:
this.assert_state("ready", "handle_change_update");
dbg = this.dbg("handle_change_update");
dbg(new_val);
project_id = new_val.project_id;
_c.label = 1;
case 1:
_c.trys.push([1, 3, , 4]);
return [4 /*yield*/, query(this.db, {
query: "SELECT jsonb_object_keys(users) AS account_id FROM projects",
where: { "project_id = $::UUID": project_id },
})];
case 2:
users = _c.sent();
return [3 /*break*/, 4];
case 3:
err_2 = _c.sent();
this.handle_error(err_2);
return [2 /*return*/];
case 4:
if (this.users[project_id] == null) {
any = false;
try {
for (users_1 = __values(users), users_1_1 = users_1.next(); !users_1_1.done; users_1_1 = users_1.next()) {
account_id = users_1_1.value.account_id;
if (this.accounts[account_id]) {
any = true;
break;
}
}
}
catch (e_2_1) { e_2 = { error: e_2_1 }; }
finally {
try {
if (users_1_1 && !users_1_1.done && (_a = users_1.return)) _a.call(users_1);
}
finally { if (e_2) throw e_2.error; }
}
if (!any) {
// *and* none of our tracked users are on this project... so don't care
return [2 /*return*/];
}
}
users_now = {};
try {
for (users_2 = __values(users), users_2_1 = users_2.next(); !users_2_1.done; users_2_1 = users_2.next()) {
account_id = users_2_1.value.account_id;
users_now[account_id] = true;
}
}
catch (e_3_1) { e_3 = { error: e_3_1 }; }
finally {
try {
if (users_2_1 && !users_2_1.done && (_b = users_2.return)) _b.call(users_2);
}
finally { if (e_3) throw e_3.error; }
}
users_before = this.users[project_id] != null ? this.users[project_id] : {};
for (account_id in users_now) {
if (!users_before[account_id]) {
this.add_user_to_project(account_id, project_id);
}
}
for (account_id in users_before) {
if (!users_now[account_id]) {
this.remove_user_from_project(account_id, project_id);
}
}
return [2 /*return*/];
}
});
});
};
// add and remove user from a project, maintaining our data structures
ProjectAndUserTracker.prototype.add_user_to_project = function (account_id, project_id) {
this.assert_state("ready", "add_user_to_project");
if (this.projects[account_id] != null &&
this.projects[account_id][project_id]) {
// already added
return;
}
this.emit("add_user_to_project-" + account_id, project_id);
if (this.users[project_id] == null) {
this.users[project_id] = {};
}
var users = this.users[project_id];
users[account_id] = true;
if (this.projects[account_id] == null) {
this.projects[account_id] = {};
}
var projects = this.projects[account_id];
projects[project_id] = true;
if (this.collabs[account_id] == null) {
this.collabs[account_id] = {};
}
var collabs = this.collabs[account_id];
for (var other_account_id in users) {
if (collabs[other_account_id] != null) {
collabs[other_account_id] += 1;
}
else {
collabs[other_account_id] = 1;
this.emit("add_collaborator-" + account_id, other_account_id);
}
var other_collabs = this.collabs[other_account_id];
if (other_collabs[account_id] != null) {
other_collabs[account_id] += 1;
}
else {
other_collabs[account_id] = 1;
this.emit("add_collaborator-" + other_account_id, account_id);
}
}
};
ProjectAndUserTracker.prototype.remove_user_from_project = function (account_id, project_id, no_emit) {
if (no_emit === void 0) { no_emit = false; }
this.assert_state("ready", "remove_user_from_project");
if ((account_id != null ? account_id.length : undefined) !== 36 ||
(project_id != null ? project_id.length : undefined) !== 36) {
throw Error("invalid account_id or project_id");
}
if (!(this.projects[account_id] != null
? this.projects[account_id][project_id]
: undefined)) {
return;
}
if (!no_emit) {
this.emit("remove_user_from_project-" + account_id, project_id);
}
if (this.collabs[account_id] == null) {
this.collabs[account_id] = {};
}
for (var other_account_id in this.users[project_id]) {
this.collabs[account_id][other_account_id] -= 1;
if (this.collabs[account_id][other_account_id] === 0) {
delete this.collabs[account_id][other_account_id];
if (!no_emit) {
this.emit("remove_collaborator-" + account_id, other_account_id);
}
}
this.collabs[other_account_id][account_id] -= 1;
if (this.collabs[other_account_id][account_id] === 0) {
delete this.collabs[other_account_id][account_id];
if (!no_emit) {
this.emit("remove_collaborator-" + other_account_id, account_id);
}
}
}
delete this.users[project_id][account_id];
delete this.projects[account_id][project_id];
};
// Register the given account so that this client watches the database
// in order to be aware of all projects and collaborators of the
// given account.
ProjectAndUserTracker.prototype.register = function (account_id) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, awaiting_1.callback(this.register_cb.bind(this), account_id)];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
};
ProjectAndUserTracker.prototype.register_cb = function (account_id, cb) {
var dbg = this.dbg("register(account_id=\"" + account_id + "\"");
if (this.accounts[account_id] != null) {
dbg("already registered -- listener counts " + JSON.stringify(this.listener_counts(account_id)));
cb();
return;
}
if (misc_1.len(this.register_todo) === 0) {
// no registration is currently happening
this.register_todo[account_id] = [cb];
// kick things off -- this will keep registering accounts
// until everything is done, then this.register_todo will have length 0.
this.do_register();
}
else {
// Accounts are being registered right now. Add to the todo list.
var v = this.register_todo[account_id];
if (v != null) {
v.push(cb);
}
else {
this.register_todo[account_id] = [cb];
}
}
};
// Call do_register_work to completely clear the work
// this.register_todo work queue.
// NOTE: do_register_work does each account, *one after another*,
// rather than doing everything in parallel. WARNING: DO NOT
// rewrite this to do everything in parallel, unless you think you
// thoroughly understand the algorithm, since I think
// doing things in parallel would horribly break!
ProjectAndUserTracker.prototype.do_register = function () {
return __awaiter(this, void 0, void 0, function () {
var account_id, dbg, projects, err_3, e, projects_1, projects_1_1, project, _a, _b, collab_account_id, callbacks, callbacks_2, callbacks_2_1, cb;
var e_4, _c, e_5, _d, e_6, _e;
return __generator(this, function (_f) {
switch (_f.label) {
case 0:
if (this.state != "ready")
return [2 /*return*/]; // maybe shutting down.
account_id = undefined;
for (account_id in this.register_todo)
break;
if (account_id == null)
return [2 /*return*/]; // nothing to do.
dbg = this.dbg("do_register(account_id=\"" + account_id + "\")");
dbg("registering account");
if (this.do_register_lock)
throw Error("do_register MUST NOT be called twice at once!");
this.do_register_lock = true;
_f.label = 1;
case 1:
_f.trys.push([1, , 6, 7]);
projects = void 0;
_f.label = 2;
case 2:
_f.trys.push([2, 4, , 5]);
return [4 /*yield*/, query(this.db, {
query: "SELECT project_id, json_agg(o) as users FROM (SELECT project_id, jsonb_object_keys(users) AS o FROM projects WHERE users ? $1::TEXT ORDER BY last_edited DESC LIMIT 10000) s group by s.project_id",
params: [account_id],
})];
case 3:
// 2021-05-10: one user has a really large number of projects, which causes the hub to crash
// TODO: fix this ORDER BY .. LIMIT .. part properly
projects = _f.sent();
return [3 /*break*/, 5];
case 4:
err_3 = _f.sent();
e = "error registering '" + account_id + "' -- err=" + err_3;
dbg(e);
this.handle_error(e); // it is game over.
return [2 /*return*/];
case 5:
// we care about this account_id
this.accounts[account_id] = true;
dbg("now adding all users to project tracker -- start");
try {
for (projects_1 = __values(projects), projects_1_1 = projects_1.next(); !projects_1_1.done; projects_1_1 = projects_1.next()) {
project = projects_1_1.value;
if (this.users[project.project_id] != null) {
// already have data about this project
continue;
}
else {
try {
for (_a = (e_5 = void 0, __values(project.users)), _b = _a.next(); !_b.done; _b = _a.next()) {
collab_account_id = _b.value;
if (collab_account_id == null) {
continue; // just skip; evidently rarely this isn't defined, maybe due to db error?
}
this.add_user_to_project(collab_account_id, project.project_id);
}
}
catch (e_5_1) { e_5 = { error: e_5_1 }; }
finally {
try {
if (_b && !_b.done && (_d = _a.return)) _d.call(_a);
}
finally { if (e_5) throw e_5.error; }
}
}
}
}
catch (e_4_1) { e_4 = { error: e_4_1 }; }
finally {
try {
if (projects_1_1 && !projects_1_1.done && (_c = projects_1.return)) _c.call(projects_1);
}
finally { if (e_4) throw e_4.error; }
}
dbg("successfully registered -- stop");
callbacks = this.register_todo[account_id];
if (callbacks != null) {
try {
for (callbacks_2 = __values(callbacks), callbacks_2_1 = callbacks_2.next(); !callbacks_2_1.done; callbacks_2_1 = callbacks_2.next()) {
cb = callbacks_2_1.value;
cb();
}
}
catch (e_6_1) { e_6 = { error: e_6_1 }; }
finally {
try {
if (callbacks_2_1 && !callbacks_2_1.done && (_e = callbacks_2.return)) _e.call(callbacks_2);
}
finally { if (e_6) throw e_6.error; }
}
// We are done (trying to) register account_id.
delete this.register_todo[account_id];
}
return [3 /*break*/, 7];
case 6:
this.do_register_lock = false;
return [7 /*endfinally*/];
case 7:
if (misc_1.len(this.register_todo) > 0) {
// Deal with next account that needs to be registered
this.do_register();
}
return [2 /*return*/];
}
});
});
};
// TODO: not actually used by any client yet... but obviously it should
// be since this would be a work/memory leak, right?
ProjectAndUserTracker.prototype.unregister = function (account_id) {
var e_7, _a;
if (!this.accounts[account_id])
return; // nothing to do
var v = [];
for (var project_id in this.projects[account_id]) {
v.push(project_id);
}
delete this.accounts[account_id];
try {
// Forget about any projects they account_id is on that are no longer
// necessary to watch...
for (var v_1 = __values(v), v_1_1 = v_1.next(); !v_1_1.done; v_1_1 = v_1.next()) {
var project_id = v_1_1.value;
var need = false;
for (var other_account_id in this.users[project_id]) {
if (this.accounts[other_account_id] != null) {
need = true;
break;
}
}
if (!need) {
for (var other_account_id in this.users[project_id]) {
this.remove_user_from_project(other_account_id, project_id, true);
}
delete this.users[project_id];
}
}
}
catch (e_7_1) { e_7 = { error: e_7_1 }; }
finally {
try {
if (v_1_1 && !v_1_1.done && (_a = v_1.return)) _a.call(v_1);
}
finally { if (e_7) throw e_7.error; }
}
};
// Return *set* of projects that this user is a collaborator on
ProjectAndUserTracker.prototype.get_projects = function (account_id) {
if (!this.accounts[account_id]) {
// This should never happen, but very rarely it DOES. I do not know why, having studied the
// code. But when it does, just raising an exception blows up the server really badly.
// So for now we just async register the account, return that it is not a collaborator
// on anything. Then some query will fail, get tried again, and work since registration will
// have finished.
//throw Error("account (='#{account_id}') must be registered")
this.register(account_id);
return {};
}
return this.projects[account_id] != null ? this.projects[account_id] : {};
};
// map from collabs of account_id to number of projects they collab
// on (account_id itself counted twice)
ProjectAndUserTracker.prototype.get_collabs = function (account_id) {
if (this.state == "closed")
return {};
return this.collabs[account_id] != null ? this.collabs[account_id] : {};
};
ProjectAndUserTracker.prototype.listener_counts = function (account_id) {
var e_8, _a;
var x = {};
try {
for (var _b = __values([
"add_user_to_project",
"remove_user_from_project",
"add_collaborator",
"remove_collaborator",
]), _c = _b.next(); !_c.done; _c = _b.next()) {
var e = _c.value;
var event_1 = e + "-" + account_id;
x[event_1] = this.listenerCount(event_1);
}
}
catch (e_8_1) { e_8 = { error: e_8_1 }; }
finally {
try {
if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
}
finally { if (e_8) throw e_8.error; }
}
return x;
};
return ProjectAndUserTracker;
}(events_1.EventEmitter));
exports.ProjectAndUserTracker = ProjectAndUserTracker;
function all_query(db, opts, cb) {
if (opts == null) {
throw Error("opts must not be null");
}
opts.cb = all_results(cb);
db._query(opts);
}
function query(db, opts) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, awaiting_1.callback(all_query, db, opts)];
case 1: return [2 /*return*/, _a.sent()];
}
});
});
}
//# sourceMappingURL=project-and-user-tracker.js.map