UNPKG

smc-hub

Version:

CoCalc: Backend webserver component

958 lines (898 loc) 31.4 kB
// Generated by CoffeeScript 2.5.1 (function() { //######################################################################## // This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. // License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details //######################################################################## // Server side synchronized tables built on PostgreSQL, and basic support // for user get query updates. /* INPUT: table -- name of a table select -- map from field names (of table) to their postgres types change -- array of field names (of table) Creates a trigger function that fires whenever any of the given columns changes, and sends the columns in select out as a notification. */ /* Trigger functions */ var Changes, EventEmitter, ProjectAndUserTracker, SCHEMA, SyncTable, all_results, async, defaults, immutable, is_array, misc, misc_node, one_result, pg_type, quote_field, required, trigger_code, trigger_name, underscore, boundMethodCheck = function(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new Error('Bound instance method accessed before binding'); } }, indexOf = [].indexOf; EventEmitter = require('events'); immutable = require('immutable'); async = require('async'); underscore = require('underscore'); ({defaults, is_array} = misc = require('smc-util/misc')); required = defaults.required; misc_node = require('smc-util-node/misc_node'); ({pg_type, one_result, all_results, quote_field} = require('./postgres-base')); ({SCHEMA} = require('smc-util/schema')); ({Changes} = require('./postgres/changefeed')); ({ProjectAndUserTracker} = require('./postgres/project-and-user-tracker')); exports.extend_PostgreSQL = function(ext) { var PostgreSQL; return PostgreSQL = class PostgreSQL extends ext { constructor() { super(...arguments); this._ensure_trigger_exists = this._ensure_trigger_exists.bind(this); this._listen = this._listen.bind(this); this._notification = this._notification.bind(this); this._clear_listening_state = this._clear_listening_state.bind(this); this._stop_listening = this._stop_listening.bind(this); // Server-side changefeed-updated table, which automatically restart changefeed // on error, etc. See SyncTable docs where the class is defined. this.synctable = this.synctable.bind(this); this.changefeed = this.changefeed.bind(this); // Event emitter that changes to users of a project, and collabs of a user. // If it emits 'error' -- which is can and will do sometimes -- then // any client of this tracker must give up on using it! this.project_and_user_tracker = this.project_and_user_tracker.bind(this); } _ensure_trigger_exists(table, select, watch, cb) { var dbg, tgname, trigger_exists; boundMethodCheck(this, PostgreSQL); dbg = this._dbg(`_ensure_trigger_exists(${table})`); dbg(`select=${misc.to_json(select)}`); if (misc.len(select) === 0) { cb('there must be at least one column selected'); return; } tgname = trigger_name(table, select, watch); trigger_exists = void 0; return async.series([ (cb) => { dbg("checking whether or not trigger exists"); return this._query({ query: `SELECT count(*) FROM pg_trigger WHERE tgname = '${tgname}'`, cb: (err, result) => { if (err) { return cb(err); } else { trigger_exists = parseInt(result.rows[0].count) > 0; return cb(); } } }); }, (cb) => { var code; if (trigger_exists) { dbg(`trigger ${tgname} already exists`); cb(); return; } dbg(`creating trigger ${tgname}`); code = trigger_code(table, select, watch); return async.series([ (cb) => { return this._query({ query: code.function, cb: cb }); }, (cb) => { return this._query({ query: code.trigger, cb: cb }); } ], cb); } ], cb); } _listen(table, select, watch, cb) { var dbg, tgname; boundMethodCheck(this, PostgreSQL); dbg = this._dbg(`_listen(${table})`); dbg(`select = ${misc.to_json(select)}`); if (!misc.is_object(select)) { cb('select must be an object'); return; } if (misc.len(select) === 0) { cb('there must be at least one column'); return; } if (!misc.is_array(watch)) { cb('watch must be an array'); return; } if (this._listening == null) { this._listening = {}; } tgname = trigger_name(table, select, watch); if (this._listening[tgname] > 0) { dbg("already listening"); this._listening[tgname] += 1; if (typeof cb === "function") { cb(void 0, tgname); } return; } return async.series([ (cb) => { dbg("ensure trigger exists"); return this._ensure_trigger_exists(table, select, watch, cb); }, (cb) => { dbg("add listener"); return this._query({ query: `LISTEN ${tgname}`, cb: cb }); } ], (err) => { var base; if (err) { dbg(`fail: err = ${err}`); return typeof cb === "function" ? cb(err) : void 0; } else { if ((base = this._listening)[tgname] == null) { base[tgname] = 0; } this._listening[tgname] += 1; dbg("success"); return typeof cb === "function" ? cb(void 0, tgname) : void 0; } }); } _notification(mesg) { boundMethodCheck(this, PostgreSQL); //@_dbg('notification')(misc.to_json(mesg)) # this is way too verbose... return this.emit(mesg.channel, JSON.parse(mesg.payload)); } _clear_listening_state() { boundMethodCheck(this, PostgreSQL); return this._listening = {}; } _stop_listening(table, select, watch, cb) { var tgname; boundMethodCheck(this, PostgreSQL); if (this._listening == null) { this._listening = {}; } tgname = trigger_name(table, select, watch); if ((this._listening[tgname] == null) || this._listening[tgname] === 0) { if (typeof cb === "function") { cb(); } return; } if (this._listening[tgname] > 0) { this._listening[tgname] -= 1; } if (this._listening[tgname] === 0) { return this._query({ query: `UNLISTEN ${tgname}`, cb: cb }); } } synctable(opts) { var err; boundMethodCheck(this, PostgreSQL); opts = defaults(opts, { table: required, columns: void 0, where: void 0, limit: void 0, order_by: void 0, where_function: void 0, // if given; a function of the *primary* key that returns true if and only if it matches the changefeed idle_timeout_s: void 0, // TODO: currently ignored cb: void 0 }); if (this.is_standby) { err = "synctable against standby database not allowed"; if (opts.cb != null) { opts.cb(err); return; } else { throw Error(err); } } return new SyncTable(this, opts.table, opts.columns, opts.where, opts.where_function, opts.limit, opts.order_by, opts.cb); } changefeed(opts) { boundMethodCheck(this, PostgreSQL); opts = defaults(opts, { table: required, // Name of the table select: required, // Map from field names to postgres data types. These must // determine entries of table (e.g., primary key). watch: required, // Array of field names we watch for changes where: required, // Condition involving only the fields in select; or function taking obj with select and returning true or false cb: required }); if (this.is_standby) { if (typeof opts.cb === "function") { opts.cb("changefeed against standby database not allowed"); } return; } new Changes(this, opts.table, opts.select, opts.watch, opts.where, opts.cb); } async project_and_user_tracker(opts) { var cb, err, i, j, len, len1, ref, ref1, results, tracker; boundMethodCheck(this, PostgreSQL); opts = defaults(opts, { cb: required }); if (this._project_and_user_tracker != null) { opts.cb(void 0, this._project_and_user_tracker); return; } if (this._project_and_user_tracker_cbs == null) { this._project_and_user_tracker_cbs = []; } this._project_and_user_tracker_cbs.push(opts.cb); if (this._project_and_user_tracker_cbs.length > 1) { return; } tracker = new ProjectAndUserTracker(this); tracker.once("error", () => { // delete, so that future calls create a new one. return delete this._project_and_user_tracker; }); try { await tracker.init(); this._project_and_user_tracker = tracker; ref = this._project_and_user_tracker_cbs; for (i = 0, len = ref.length; i < len; i++) { cb = ref[i]; cb(void 0, tracker); } return delete this._project_and_user_tracker_cbs; } catch (error) { err = error; ref1 = this._project_and_user_tracker_cbs; results = []; for (j = 0, len1 = ref1.length; j < len1; j++) { cb = ref1[j]; results.push(cb(err)); } return results; } } }; }; SyncTable = class SyncTable extends EventEmitter { constructor(_db, _table, _columns, _where, _where_function, _limit, _order_by, cb) { var e, ref, t, x; super(); this._dbg = this._dbg.bind(this); this._query_opts = this._query_opts.bind(this); this.close = this.close.bind(this); this.connect = this.connect.bind(this); this._notification = this._notification.bind(this); this._init = this._init.bind(this); this._do_init = this._do_init.bind(this); this._reconnect = this._reconnect.bind(this); this._process_results = this._process_results.bind(this); // Remove from synctable anything that no longer matches the where criterion. this._process_deleted = this._process_deleted.bind(this); // Grab any entries from table about which we have been notified of changes. this._update = this._update.bind(this); this.get = this.get.bind(this); this.getIn = this.getIn.bind(this); this.has = this.has.bind(this); // wait until some function of this synctable is truthy this.wait = this.wait.bind(this); this._db = _db; this._table = _table; this._columns = _columns; this._where = _where; this._where_function = _where_function; this._limit = _limit; this._order_by = _order_by; t = SCHEMA[this._table]; if (t == null) { this._state = 'error'; if (typeof cb === "function") { cb(`unknown table ${this._table}`); } return; } try { this._primary_key = this._db._primary_key(this._table); } catch (error) { e = error; if (typeof cb === "function") { cb(e); } return; } this._listen_columns = { [`${this._primary_key}`]: pg_type(t.fields[this._primary_key], this._primary_key) }; // We only trigger an update when one of the columns we care about actually changes. if (this._columns) { this._watch_columns = misc.copy(this._columns); // don't include primary key since it can't change. if (ref = this._primary_key, indexOf.call(this._columns, ref) < 0) { this._columns = this._columns.concat([this._primary_key]); // required } this._select_columns = this._columns; } else { this._watch_columns = []; // means all of them this._select_columns = misc.keys(SCHEMA[this._table].fields); } this._select_query = `SELECT ${(function() { var i, len, ref1, results; ref1 = this._select_columns; results = []; for (i = 0, len = ref1.length; i < len; i++) { x = ref1[i]; results.push(quote_field(x)); } return results; }).call(this)} FROM ${this._table}`; //@_update = underscore.throttle(@_update, 500) this._init((err) => { if (err && (cb == null)) { this.emit("error", err); return; } this.emit('init'); return typeof cb === "function" ? cb(err, this) : void 0; }); } _dbg(f) { boundMethodCheck(this, SyncTable); return this._db._dbg(`SyncTable(table='${this._table}').${f}`); } _query_opts() { var opts; boundMethodCheck(this, SyncTable); opts = {}; opts.query = this._select_query; opts.where = this._where; opts.limit = this._limit; opts.order_by = this._order_by; return opts; } close(cb) { boundMethodCheck(this, SyncTable); this.removeAllListeners(); this._db.removeListener(this._tgname, this._notification); this._db.removeListener('connect', this._reconnect); this._state = 'closed'; delete this._value; return this._db._stop_listening(this._table, this._listen_columns, this._watch_columns, cb); } connect(opts) { boundMethodCheck(this, SyncTable); return opts != null ? typeof opts.cb === "function" ? opts.cb() : void 0 : void 0; } _notification(obj) { var action, k, new_val, old_val; boundMethodCheck(this, SyncTable); //console.log 'notification', obj [action, new_val, old_val] = obj; if (action === 'DELETE' || (new_val == null)) { k = old_val[this._primary_key]; if (this._value.has(k)) { this._value = this._value.delete(k); return process.nextTick(() => { return this.emit('change', k); }); } } else { k = new_val[this._primary_key]; if ((this._where_function != null) && !this._where_function(k)) { return; } // doesn't match -- nothing to do -- ignore this._changed[k] = true; return this._update(); } } _init(cb) { boundMethodCheck(this, SyncTable); return misc.retry_until_success({ f: this._do_init, start_delay: 3000, max_delay: 10000, log: this._dbg("_init"), cb: cb }); } _do_init(cb) { boundMethodCheck(this, SyncTable); this._state = 'init'; // 'init' -> ['error', 'ready'] -> 'closed' this._value = immutable.Map(); this._changed = {}; return async.series([ (cb) => { // ensure database client is listening for primary keys changes to our table return this._db._listen(this._table, this._listen_columns, this._watch_columns, (err, tgname) => { this._tgname = tgname; this._db.on(this._tgname, this._notification); return cb(err); }); }, (cb) => { var opts; opts = this._query_opts(); opts.cb = (err, result) => { if (err) { return cb(err); } else { this._process_results(result.rows); this._db.once('connect', this._reconnect); return cb(); } }; return this._db._query(opts); }, (cb) => { return this._update(cb); } ], (err) => { if (err) { this._state = 'error'; return cb(err); } else { this._state = 'ready'; return cb(); } }); } _reconnect(cb) { var before, dbg; boundMethodCheck(this, SyncTable); dbg = this._dbg("_reconnect"); if (this._state !== 'ready') { dbg("only attempt reconnect if we were already successfully connected at some point."); return; } // Everything was already initialized, but then the connection to the // database was dropped... and then successfully re-connected. Now // we need to (1) setup everything again, and (2) send out notifications // about anything in the table that changed. dbg("Save state from before disconnect"); before = this._value; dbg("Clean up everything."); this._db.removeListener(this._tgname, this._notification); this._db.removeListener('connect', this._reconnect); delete this._value; dbg("connect and initialize"); return this._init((err) => { if (err) { if (typeof cb === "function") { cb(err); } return; } if ((this._value != null) && (before != null)) { // It's highly unlikely that before or @_value would not be defined, but it could happen (see #2527) dbg("notify about anything that changed when we were disconnected"); before.map((v, k) => { if (!v.equals(this._value.get(k))) { return this.emit('change', k); } }); this._value.map((v, k) => { if (!before.has(k)) { return this.emit('change', k); } }); } return typeof cb === "function" ? cb() : void 0; }); } _process_results(rows) { var i, k, len, results, v, x; boundMethodCheck(this, SyncTable); if (this._state === 'closed' || (this._value == null)) { return; } // See https://github.com/sagemathinc/cocalc/issues/4440 // for why the @_value check. Remove this when this is // rewritten in typescript and we can guarantee stuff. results = []; for (i = 0, len = rows.length; i < len; i++) { x = rows[i]; k = x[this._primary_key]; v = immutable.fromJS(misc.map_without_undefined(x)); if (!v.equals(this._value.get(k))) { this._value = this._value.set(k, v); if (this._state === 'ready') { // only send out change notifications after ready. results.push(process.nextTick(() => { return this.emit('change', k); })); } else { results.push(void 0); } } else { results.push(void 0); } } return results; } _process_deleted(rows, changed) { var i, k, kept, len, results, x; boundMethodCheck(this, SyncTable); kept = {}; for (i = 0, len = rows.length; i < len; i++) { x = rows[i]; kept[x[this._primary_key]] = true; } results = []; for (k in changed) { if (!kept[k] && this._value.has(k)) { // The record with primary_key k no longer matches the where criterion // so we delete it from our synctable. this._value = this._value.delete(k); if (this._state === 'ready') { results.push(process.nextTick(() => { return this.emit('change', k); })); } else { results.push(void 0); } } else { results.push(void 0); } } return results; } _update(cb) { var changed, x; boundMethodCheck(this, SyncTable); if (misc.len(this._changed) === 0) { // nothing to do if (typeof cb === "function") { cb(); } return; } changed = this._changed; this._changed = {}; // reset changed set -- could get modified during query below, which is fine. if (this._select_columns.length === 1) { // special case where we don't have to query for more info this._process_results((function() { var i, len, ref, results; ref = misc.keys(changed); results = []; for (i = 0, len = ref.length; i < len; i++) { x = ref[i]; results.push({ [`${this._primary_key}`]: x }); } return results; }).call(this)); if (typeof cb === "function") { cb(); } return; } // Have to query to get actual changed data. return this._db._query({ query: this._select_query, where: [ { [`${this._primary_key} = ANY($)`]: misc.keys(changed) }, this._where ], cb: (err, result) => { var k; if (err) { this._dbg("update")(`error ${err}`); for (k in changed) { this._changed[k] = true; // will try again later } } else { this._process_results(result.rows); this._process_deleted(result.rows, changed); } return typeof cb === "function" ? cb() : void 0; } }); } get(key) { // key = single key or array of keys var i, k, len, r, v; boundMethodCheck(this, SyncTable); if ((key == null) || (this._value == null)) { return this._value; } if (is_array(key)) { // for consistency with smc-util/sync/synctable r = immutable.Map(); for (i = 0, len = key.length; i < len; i++) { k = key[i]; v = this._value.get(k); if (v != null) { r = r.set(k, v); } } return r; } else { return this._value.get(key); } } getIn(x) { var ref; boundMethodCheck(this, SyncTable); return (ref = this._value) != null ? ref.getIn(x) : void 0; } has(key) { var ref; boundMethodCheck(this, SyncTable); return (ref = this._value) != null ? ref.has(key) : void 0; } wait(opts) { var f, fail, fail_timer, x; boundMethodCheck(this, SyncTable); opts = defaults(opts, { until: required, // waits until "until(@)" evaluates to something truthy timeout: 30, // in *seconds* -- set to 0 to disable (sort of DANGEROUS if 0, obviously.) cb: required // cb(undefined, until(@)) on success and cb('timeout') on failure due to timeout }); x = opts.until(this); if (x) { opts.cb(void 0, x); // already true return; } fail_timer = void 0; f = () => { x = opts.until(this); if (x) { this.removeListener('change', f); if (fail_timer != null) { clearTimeout(fail_timer); fail_timer = void 0; } return opts.cb(void 0, x); } }; this.on('change', f); if (opts.timeout) { fail = () => { this.removeListener('change', f); return opts.cb('timeout'); }; fail_timer = setTimeout(fail, 1000 * opts.timeout); } } }; trigger_name = function(table, select, watch) { var c; if (!misc.is_object(select)) { throw Error("trigger_name -- columns must be a map of colname:type"); } c = misc.keys(select); c.sort(); watch = misc.copy(watch); watch.sort(); if (watch.length > 0) { c.push('|'); c = c.concat(watch); } return 'change_' + misc_node.sha1(`${table} ${c.join(' ')}`).slice(0, 16); }; trigger_code = function(table, select, watch) { var _, assign_new, assign_old, build_obj_new, build_obj_old, code, column_decl_new, column_decl_old, field, i, j, k, len, len1, no_change, ref, tgname, type, update_of, x; tgname = trigger_name(table, select, watch); column_decl_old = (function() { var results; results = []; for (field in select) { type = select[field]; results.push(`${field}_old ${type != null ? type : 'text'};`); } return results; })(); column_decl_new = (function() { var results; results = []; for (field in select) { type = select[field]; results.push(`${field}_new ${type != null ? type : 'text'};`); } return results; })(); assign_old = (function() { var results; results = []; for (field in select) { _ = select[field]; results.push(`${field}_old = OLD.${field};`); } return results; })(); assign_new = (function() { var results; results = []; for (field in select) { _ = select[field]; results.push(`${field}_new = NEW.${field};`); } return results; })(); build_obj_old = (function() { var results; results = []; for (field in select) { _ = select[field]; results.push(`'${field}', ${field}_old`); } return results; })(); build_obj_new = (function() { var results; results = []; for (field in select) { _ = select[field]; results.push(`'${field}', ${field}_new`); } return results; })(); if (watch.length > 0) { no_change = ((function() { var i, len, ref, results; ref = watch.concat(misc.keys(select)); results = []; for (i = 0, len = ref.length; i < len; i++) { field = ref[i]; results.push(`OLD.${field} = NEW.${field}`); } return results; })()).join(' AND '); } else { no_change = 'FALSE'; } if (watch.length > 0) { x = {}; for (i = 0, len = watch.length; i < len; i++) { k = watch[i]; x[k] = true; } ref = misc.keys(select); for (j = 0, len1 = ref.length; j < len1; j++) { k = ref[j]; x[k] = true; } update_of = `OF ${((function() { var l, len2, ref1, results; ref1 = misc.keys(x); results = []; for (l = 0, len2 = ref1.length; l < len2; l++) { field = ref1[l]; results.push(quote_field(field)); } return results; })()).join(',')}`; } else { update_of = ""; } code = {}; code.function = `CREATE OR REPLACE FUNCTION ${tgname}() RETURNS TRIGGER AS $$ DECLARE notification json; obj_old json; obj_new json; ${column_decl_old.join('\n')} ${column_decl_new.join('\n')} BEGIN -- TG_OP is 'DELETE', 'INSERT' or 'UPDATE' IF TG_OP = 'DELETE' THEN ${assign_old.join('\n')} obj_old = json_build_object(${build_obj_old.join(',')}); END IF; IF TG_OP = 'INSERT' THEN ${assign_new.join('\n')} obj_new = json_build_object(${build_obj_new.join(',')}); END IF; IF TG_OP = 'UPDATE' THEN IF ${no_change} THEN RETURN NULL; END IF; ${assign_old.join('\n')} obj_old = json_build_object(${build_obj_old.join(',')}); ${assign_new.join('\n')} obj_new = json_build_object(${build_obj_new.join(',')}); END IF; notification = json_build_array(TG_OP, obj_new, obj_old); PERFORM pg_notify('${tgname}', notification::text); RETURN NULL; END; $$ LANGUAGE plpgsql;`; code.trigger = `CREATE TRIGGER ${tgname} AFTER INSERT OR DELETE OR UPDATE ${update_of} ON ${table} FOR EACH ROW EXECUTE PROCEDURE ${tgname}();`; return code; }; /* NOTES: The following is a way to back the changes with a small table. This allows to have changes which are larger than the hard 8000 bytes limit. HSY did this with the idea of having a temporary workaround for a bug related to this. https://github.com/sagemathinc/cocalc/issues/1718 1. Create a table trigger_notifications via the db-schema. For performance reasons, the table itself should be created with "UNLOGGED" see: https://www.postgresql.org/docs/current/static/sql-createtable.html (I've no idea how to specify that in the code here) schema.trigger_notifications = primary_key : 'id' fields: id: type : 'uuid' desc : 'primary key' time: type : 'timestamp' desc : 'time of when the change was created -- used for TTL' notification: type : 'map' desc : "notification payload -- up to 1GB" pg_indexes : [ 'time' ] 2. Modify the trigger function created by trigger_code above such that pg_notifies no longer contains the data structure, but a UUID for an entry in the trigger_notifications table. It creates that UUID on its own and stores the data via a normal insert. notification_id = md5(random()::text || clock_timestamp()::text)::uuid; notification = json_build_array(TG_OP, obj_new, obj_old); INSERT INTO trigger_notifications(id, time, notification) VALUES(notification_id, NOW(), notification); 3. PostgresQL::_notification is modified in such a way, that it looks up that UUID in the trigger_notifications table: @_query query: "SELECT notification FROM trigger_notifications WHERE id ='#{mesg.payload}'" cb : (err, result) => if err dbg("err=#{err}") else payload = result.rows[0].notification * dbg("payload: type=#{typeof(payload)}, data=#{misc.to_json(payload)}") @emit(mesg.channel, payload) Fortunately, there is no string -> json conversion necessary. 4. Below, that function and trigger implement a TTL for the trigger_notifications table. The `date_trunc` is a good idea, because then there is just one lock + delete op per minute, instead of potentially at every write. -- 10 minutes TTL for the trigger_notifications table, deleting only every full minute CREATE FUNCTION delete_old_trigger_notifications() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN DELETE FROM trigger_notifications WHERE time < date_trunc('minute', NOW() - '10 minute'::interval); RETURN NULL; END; $$; -- creating the trigger CREATE TRIGGER trigger_delete_old_trigger_notifications AFTER INSERT ON trigger_notifications EXECUTE PROCEDURE delete_old_trigger_notifications(); */ }).call(this); //# sourceMappingURL=postgres-synctable.js.map