@hocuspocus/server
Version:
plug & play collaboration backend
1,668 lines (1,566 loc) • 92.9 kB
JavaScript
import { writeAuthenticated, writePermissionDenied, WsReadyStates, ResetConnection, ConnectionTimeout, Forbidden, Unauthorized, awarenessStatesToArray } from '@hocuspocus/common';
import * as Y from 'yjs';
import { Doc, applyUpdate, encodeStateAsUpdate } from 'yjs';
import { v4 } from 'uuid';
import { URLSearchParams } from 'node:url';
import { createServer } from 'node:http';
import kleur from 'kleur';
import { WebSocketServer } from 'ws';
/* eslint-env browser */
const BIT8 = 128;
const BITS7 = 127;
/**
* Common Math expressions.
*
* @module math
*/
const floor = Math.floor;
/**
* @function
* @param {number} a
* @param {number} b
* @return {number} The smaller element of a and b
*/
const min = (a, b) => a < b ? a : b;
/**
* @function
* @param {number} a
* @param {number} b
* @return {number} The bigger element of a and b
*/
const max = (a, b) => a > b ? a : b;
/**
* Utility helpers for working with numbers.
*
* @module number
*/
const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER;
/**
* Utility module to work with sets.
*
* @module set
*/
const create$2 = () => new Set();
/**
* Utility module to work with Arrays.
*
* @module array
*/
/**
* Transforms something array-like to an actual Array.
*
* @function
* @template T
* @param {ArrayLike<T>|Iterable<T>} arraylike
* @return {T}
*/
const from = Array.from;
/**
* @param {string} str
* @return {Uint8Array}
*/
const _encodeUtf8Polyfill = str => {
const encodedString = unescape(encodeURIComponent(str));
const len = encodedString.length;
const buf = new Uint8Array(len);
for (let i = 0; i < len; i++) {
buf[i] = /** @type {number} */ (encodedString.codePointAt(i));
}
return buf
};
/* c8 ignore next */
const utf8TextEncoder = /** @type {TextEncoder} */ (typeof TextEncoder !== 'undefined' ? new TextEncoder() : null);
/**
* @param {string} str
* @return {Uint8Array}
*/
const _encodeUtf8Native = str => utf8TextEncoder.encode(str);
/**
* @param {string} str
* @return {Uint8Array}
*/
/* c8 ignore next */
const encodeUtf8 = utf8TextEncoder ? _encodeUtf8Native : _encodeUtf8Polyfill;
/* c8 ignore next */
let utf8TextDecoder = typeof TextDecoder === 'undefined' ? null : new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
/* c8 ignore start */
if (utf8TextDecoder && utf8TextDecoder.decode(new Uint8Array()).length === 1) {
// Safari doesn't handle BOM correctly.
// This fixes a bug in Safari 13.0.5 where it produces a BOM the first time it is called.
// utf8TextDecoder.decode(new Uint8Array()).length === 1 on the first call and
// utf8TextDecoder.decode(new Uint8Array()).length === 1 on the second call
// Another issue is that from then on no BOM chars are recognized anymore
/* c8 ignore next */
utf8TextDecoder = null;
}
/**
* Error helpers.
*
* @module error
*/
/**
* @param {string} s
* @return {Error}
*/
/* c8 ignore next */
const create$1 = s => new Error(s);
/**
* Efficient schema-less binary encoding with support for variable length encoding.
*
* Use [lib0/encoding] with [lib0/decoding]. Every encoding function has a corresponding decoding function.
*
* Encodes numbers in little-endian order (least to most significant byte order)
* and is compatible with Golang's binary encoding (https://golang.org/pkg/encoding/binary/)
* which is also used in Protocol Buffers.
*
* ```js
* // encoding step
* const encoder = encoding.createEncoder()
* encoding.writeVarUint(encoder, 256)
* encoding.writeVarString(encoder, 'Hello world!')
* const buf = encoding.toUint8Array(encoder)
* ```
*
* ```js
* // decoding step
* const decoder = decoding.createDecoder(buf)
* decoding.readVarUint(decoder) // => 256
* decoding.readVarString(decoder) // => 'Hello world!'
* decoding.hasContent(decoder) // => false - all data is read
* ```
*
* @module encoding
*/
/**
* A BinaryEncoder handles the encoding to an Uint8Array.
*/
class Encoder {
constructor () {
this.cpos = 0;
this.cbuf = new Uint8Array(100);
/**
* @type {Array<Uint8Array>}
*/
this.bufs = [];
}
}
/**
* @function
* @return {Encoder}
*/
const createEncoder = () => new Encoder();
/**
* The current length of the encoded data.
*
* @function
* @param {Encoder} encoder
* @return {number}
*/
const length$1 = encoder => {
let len = encoder.cpos;
for (let i = 0; i < encoder.bufs.length; i++) {
len += encoder.bufs[i].length;
}
return len
};
/**
* Transform to Uint8Array.
*
* @function
* @param {Encoder} encoder
* @return {Uint8Array} The created ArrayBuffer.
*/
const toUint8Array = encoder => {
const uint8arr = new Uint8Array(length$1(encoder));
let curPos = 0;
for (let i = 0; i < encoder.bufs.length; i++) {
const d = encoder.bufs[i];
uint8arr.set(d, curPos);
curPos += d.length;
}
uint8arr.set(new Uint8Array(encoder.cbuf.buffer, 0, encoder.cpos), curPos);
return uint8arr
};
/**
* Write one byte to the encoder.
*
* @function
* @param {Encoder} encoder
* @param {number} num The byte that is to be encoded.
*/
const write = (encoder, num) => {
const bufferLen = encoder.cbuf.length;
if (encoder.cpos === bufferLen) {
encoder.bufs.push(encoder.cbuf);
encoder.cbuf = new Uint8Array(bufferLen * 2);
encoder.cpos = 0;
}
encoder.cbuf[encoder.cpos++] = num;
};
/**
* Write a variable length unsigned integer. Max encodable integer is 2^53.
*
* @function
* @param {Encoder} encoder
* @param {number} num The number that is to be encoded.
*/
const writeVarUint = (encoder, num) => {
while (num > BITS7) {
write(encoder, BIT8 | (BITS7 & num));
num = floor(num / 128); // shift >>> 7
}
write(encoder, BITS7 & num);
};
/**
* A cache to store strings temporarily
*/
const _strBuffer = new Uint8Array(30000);
const _maxStrBSize = _strBuffer.length / 3;
/**
* Write a variable length string.
*
* @function
* @param {Encoder} encoder
* @param {String} str The string that is to be encoded.
*/
const _writeVarStringNative = (encoder, str) => {
if (str.length < _maxStrBSize) {
// We can encode the string into the existing buffer
/* c8 ignore next */
const written = utf8TextEncoder.encodeInto(str, _strBuffer).written || 0;
writeVarUint(encoder, written);
for (let i = 0; i < written; i++) {
write(encoder, _strBuffer[i]);
}
} else {
writeVarUint8Array(encoder, encodeUtf8(str));
}
};
/**
* Write a variable length string.
*
* @function
* @param {Encoder} encoder
* @param {String} str The string that is to be encoded.
*/
const _writeVarStringPolyfill = (encoder, str) => {
const encodedString = unescape(encodeURIComponent(str));
const len = encodedString.length;
writeVarUint(encoder, len);
for (let i = 0; i < len; i++) {
write(encoder, /** @type {number} */ (encodedString.codePointAt(i)));
}
};
/**
* Write a variable length string.
*
* @function
* @param {Encoder} encoder
* @param {String} str The string that is to be encoded.
*/
/* c8 ignore next */
const writeVarString = (utf8TextEncoder && /** @type {any} */ (utf8TextEncoder).encodeInto) ? _writeVarStringNative : _writeVarStringPolyfill;
/**
* Append fixed-length Uint8Array to the encoder.
*
* @function
* @param {Encoder} encoder
* @param {Uint8Array} uint8Array
*/
const writeUint8Array = (encoder, uint8Array) => {
const bufferLen = encoder.cbuf.length;
const cpos = encoder.cpos;
const leftCopyLen = min(bufferLen - cpos, uint8Array.length);
const rightCopyLen = uint8Array.length - leftCopyLen;
encoder.cbuf.set(uint8Array.subarray(0, leftCopyLen), cpos);
encoder.cpos += leftCopyLen;
if (rightCopyLen > 0) {
// Still something to write, write right half..
// Append new buffer
encoder.bufs.push(encoder.cbuf);
// must have at least size of remaining buffer
encoder.cbuf = new Uint8Array(max(bufferLen * 2, rightCopyLen));
// copy array
encoder.cbuf.set(uint8Array.subarray(leftCopyLen));
encoder.cpos = rightCopyLen;
}
};
/**
* Append an Uint8Array to Encoder.
*
* @function
* @param {Encoder} encoder
* @param {Uint8Array} uint8Array
*/
const writeVarUint8Array = (encoder, uint8Array) => {
writeVarUint(encoder, uint8Array.byteLength);
writeUint8Array(encoder, uint8Array);
};
/**
* Efficient schema-less binary decoding with support for variable length encoding.
*
* Use [lib0/decoding] with [lib0/encoding]. Every encoding function has a corresponding decoding function.
*
* Encodes numbers in little-endian order (least to most significant byte order)
* and is compatible with Golang's binary encoding (https://golang.org/pkg/encoding/binary/)
* which is also used in Protocol Buffers.
*
* ```js
* // encoding step
* const encoder = encoding.createEncoder()
* encoding.writeVarUint(encoder, 256)
* encoding.writeVarString(encoder, 'Hello world!')
* const buf = encoding.toUint8Array(encoder)
* ```
*
* ```js
* // decoding step
* const decoder = decoding.createDecoder(buf)
* decoding.readVarUint(decoder) // => 256
* decoding.readVarString(decoder) // => 'Hello world!'
* decoding.hasContent(decoder) // => false - all data is read
* ```
*
* @module decoding
*/
const errorUnexpectedEndOfArray = create$1('Unexpected end of array');
const errorIntegerOutOfRange = create$1('Integer out of Range');
/**
* A Decoder handles the decoding of an Uint8Array.
*/
class Decoder {
/**
* @param {Uint8Array} uint8Array Binary data to decode
*/
constructor (uint8Array) {
/**
* Decoding target.
*
* @type {Uint8Array}
*/
this.arr = uint8Array;
/**
* Current decoding position.
*
* @type {number}
*/
this.pos = 0;
}
}
/**
* @function
* @param {Uint8Array} uint8Array
* @return {Decoder}
*/
const createDecoder = uint8Array => new Decoder(uint8Array);
/**
* Create an Uint8Array view of the next `len` bytes and advance the position by `len`.
*
* Important: The Uint8Array still points to the underlying ArrayBuffer. Make sure to discard the result as soon as possible to prevent any memory leaks.
* Use `buffer.copyUint8Array` to copy the result into a new Uint8Array.
*
* @function
* @param {Decoder} decoder The decoder instance
* @param {number} len The length of bytes to read
* @return {Uint8Array}
*/
const readUint8Array = (decoder, len) => {
const view = new Uint8Array(decoder.arr.buffer, decoder.pos + decoder.arr.byteOffset, len);
decoder.pos += len;
return view
};
/**
* Read variable length Uint8Array.
*
* Important: The Uint8Array still points to the underlying ArrayBuffer. Make sure to discard the result as soon as possible to prevent any memory leaks.
* Use `buffer.copyUint8Array` to copy the result into a new Uint8Array.
*
* @function
* @param {Decoder} decoder
* @return {Uint8Array}
*/
const readVarUint8Array = decoder => readUint8Array(decoder, readVarUint(decoder));
/**
* Read one byte as unsigned integer.
* @function
* @param {Decoder} decoder The decoder instance
* @return {number} Unsigned 8-bit integer
*/
const readUint8 = decoder => decoder.arr[decoder.pos++];
/**
* Read unsigned integer (32bit) with variable length.
* 1/8th of the storage is used as encoding overhead.
* * numbers < 2^7 is stored in one bytlength
* * numbers < 2^14 is stored in two bylength
*
* @function
* @param {Decoder} decoder
* @return {number} An unsigned integer.length
*/
const readVarUint = decoder => {
let num = 0;
let mult = 1;
const len = decoder.arr.length;
while (decoder.pos < len) {
const r = decoder.arr[decoder.pos++];
// num = num | ((r & binary.BITS7) << len)
num = num + (r & BITS7) * mult; // shift $r << (7*#iterations) and add it to num
mult *= 128; // next iteration, shift 7 "more" to the left
if (r < BIT8) {
return num
}
/* c8 ignore start */
if (num > MAX_SAFE_INTEGER) {
throw errorIntegerOutOfRange
}
/* c8 ignore stop */
}
throw errorUnexpectedEndOfArray
};
/**
* We don't test this function anymore as we use native decoding/encoding by default now.
* Better not modify this anymore..
*
* Transforming utf8 to a string is pretty expensive. The code performs 10x better
* when String.fromCodePoint is fed with all characters as arguments.
* But most environments have a maximum number of arguments per functions.
* For effiency reasons we apply a maximum of 10000 characters at once.
*
* @function
* @param {Decoder} decoder
* @return {String} The read String.
*/
/* c8 ignore start */
const _readVarStringPolyfill = decoder => {
let remainingLen = readVarUint(decoder);
if (remainingLen === 0) {
return ''
} else {
let encodedString = String.fromCodePoint(readUint8(decoder)); // remember to decrease remainingLen
if (--remainingLen < 100) { // do not create a Uint8Array for small strings
while (remainingLen--) {
encodedString += String.fromCodePoint(readUint8(decoder));
}
} else {
while (remainingLen > 0) {
const nextLen = remainingLen < 10000 ? remainingLen : 10000;
// this is dangerous, we create a fresh array view from the existing buffer
const bytes = decoder.arr.subarray(decoder.pos, decoder.pos + nextLen);
decoder.pos += nextLen;
// Starting with ES5.1 we can supply a generic array-like object as arguments
encodedString += String.fromCodePoint.apply(null, /** @type {any} */ (bytes));
remainingLen -= nextLen;
}
}
return decodeURIComponent(escape(encodedString))
}
};
/* c8 ignore stop */
/**
* @function
* @param {Decoder} decoder
* @return {String} The read String
*/
const _readVarStringNative = decoder =>
/** @type any */ (utf8TextDecoder).decode(readVarUint8Array(decoder));
/**
* Read string of variable length
* * varUint is used to store the length of the string
*
* @function
* @param {Decoder} decoder
* @return {String} The read String
*
*/
/* c8 ignore next */
const readVarString = utf8TextDecoder ? _readVarStringNative : _readVarStringPolyfill;
class IncomingMessage {
constructor(input) {
if (!(input instanceof Uint8Array)) {
input = new Uint8Array(input);
}
this.decoder = createDecoder(input);
}
get encoder() {
if (!this.encoderInternal) {
this.encoderInternal = createEncoder();
}
return this.encoderInternal;
}
readVarUint8Array() {
return readVarUint8Array(this.decoder);
}
peekVarUint8Array() {
const { pos } = this.decoder;
const result = readVarUint8Array(this.decoder);
this.decoder.pos = pos;
return result;
}
readVarUint() {
return readVarUint(this.decoder);
}
readVarString() {
return readVarString(this.decoder);
}
toUint8Array() {
return toUint8Array(this.encoder);
}
writeVarUint(type) {
writeVarUint(this.encoder, type);
}
writeVarString(string) {
writeVarString(this.encoder, string);
}
get length() {
return length$1(this.encoder);
}
}
/**
* Utility module to work with time.
*
* @module time
*/
/**
* Return current unix time.
*
* @return {number}
*/
const getUnixTime = Date.now;
/**
* Utility module to work with key-value stores.
*
* @module map
*/
/**
* Creates a new Map instance.
*
* @function
* @return {Map<any, any>}
*
* @function
*/
const create = () => new Map();
/**
* Get map property. Create T if property is undefined and set T on map.
*
* ```js
* const listeners = map.setIfUndefined(events, 'eventName', set.create)
* listeners.add(listener)
* ```
*
* @function
* @template {Map<any, any>} MAP
* @template {MAP extends Map<any,infer V> ? function():V : unknown} CF
* @param {MAP} map
* @param {MAP extends Map<infer K,any> ? K : unknown} key
* @param {CF} createT
* @return {ReturnType<CF>}
*/
const setIfUndefined = (map, key, createT) => {
let set = map.get(key);
if (set === undefined) {
map.set(key, set = createT());
}
return set
};
/**
* Observable class prototype.
*
* @module observable
*/
/* c8 ignore start */
/**
* Handles named events.
*
* @deprecated
* @template N
*/
class Observable {
constructor () {
/**
* Some desc.
* @type {Map<N, any>}
*/
this._observers = create();
}
/**
* @param {N} name
* @param {function} f
*/
on (name, f) {
setIfUndefined(this._observers, name, create$2).add(f);
}
/**
* @param {N} name
* @param {function} f
*/
once (name, f) {
/**
* @param {...any} args
*/
const _f = (...args) => {
this.off(name, _f);
f(...args);
};
this.on(name, _f);
}
/**
* @param {N} name
* @param {function} f
*/
off (name, f) {
const observers = this._observers.get(name);
if (observers !== undefined) {
observers.delete(f);
if (observers.size === 0) {
this._observers.delete(name);
}
}
}
/**
* Emit a named event. All registered event listeners that listen to the
* specified name will receive the event.
*
* @todo This should catch exceptions
*
* @param {N} name The event name.
* @param {Array<any>} args The arguments that are applied to the event listener.
*/
emit (name, args) {
// copy all listeners to an array first to make sure that no event is emitted to listeners that are subscribed while the event handler is called.
return from((this._observers.get(name) || create()).values()).forEach(f => f(...args))
}
destroy () {
this._observers = create();
}
}
/* c8 ignore end */
/**
* Utility functions for working with EcmaScript objects.
*
* @module object
*/
/**
* @param {Object<string,any>} obj
*/
const keys = Object.keys;
/**
* @deprecated use object.size instead
* @param {Object<string,any>} obj
* @return {number}
*/
const length = obj => keys(obj).length;
/**
* Calls `Object.prototype.hasOwnProperty`.
*
* @param {any} obj
* @param {string|symbol} key
* @return {boolean}
*/
const hasProperty = (obj, key) => Object.prototype.hasOwnProperty.call(obj, key);
/**
* Common functions and function call helpers.
*
* @module function
*/
/**
* @template T
*
* @param {T} a
* @param {T} b
* @return {boolean}
*/
const equalityStrict = (a, b) => a === b;
/* c8 ignore start */
/**
* @param {any} a
* @param {any} b
* @return {boolean}
*/
const equalityDeep = (a, b) => {
if (a == null || b == null) {
return equalityStrict(a, b)
}
if (a.constructor !== b.constructor) {
return false
}
if (a === b) {
return true
}
switch (a.constructor) {
case ArrayBuffer:
a = new Uint8Array(a);
b = new Uint8Array(b);
// eslint-disable-next-line no-fallthrough
case Uint8Array: {
if (a.byteLength !== b.byteLength) {
return false
}
for (let i = 0; i < a.length; i++) {
if (a[i] !== b[i]) {
return false
}
}
break
}
case Set: {
if (a.size !== b.size) {
return false
}
for (const value of a) {
if (!b.has(value)) {
return false
}
}
break
}
case Map: {
if (a.size !== b.size) {
return false
}
for (const key of a.keys()) {
if (!b.has(key) || !equalityDeep(a.get(key), b.get(key))) {
return false
}
}
break
}
case Object:
if (length(a) !== length(b)) {
return false
}
for (const key in a) {
if (!hasProperty(a, key) || !equalityDeep(a[key], b[key])) {
return false
}
}
break
case Array:
if (a.length !== b.length) {
return false
}
for (let i = 0; i < a.length; i++) {
if (!equalityDeep(a[i], b[i])) {
return false
}
}
break
default:
return false
}
return true
};
/**
* @module awareness-protocol
*/
const outdatedTimeout = 30000;
/**
* @typedef {Object} MetaClientState
* @property {number} MetaClientState.clock
* @property {number} MetaClientState.lastUpdated unix timestamp
*/
/**
* The Awareness class implements a simple shared state protocol that can be used for non-persistent data like awareness information
* (cursor, username, status, ..). Each client can update its own local state and listen to state changes of
* remote clients. Every client may set a state of a remote peer to `null` to mark the client as offline.
*
* Each client is identified by a unique client id (something we borrow from `doc.clientID`). A client can override
* its own state by propagating a message with an increasing timestamp (`clock`). If such a message is received, it is
* applied if the known state of that client is older than the new state (`clock < newClock`). If a client thinks that
* a remote client is offline, it may propagate a message with
* `{ clock: currentClientClock, state: null, client: remoteClient }`. If such a
* message is received, and the known clock of that client equals the received clock, it will override the state with `null`.
*
* Before a client disconnects, it should propagate a `null` state with an updated clock.
*
* Awareness states must be updated every 30 seconds. Otherwise the Awareness instance will delete the client state.
*
* @extends {Observable<string>}
*/
class Awareness extends Observable {
/**
* @param {Y.Doc} doc
*/
constructor (doc) {
super();
this.doc = doc;
/**
* @type {number}
*/
this.clientID = doc.clientID;
/**
* Maps from client id to client state
* @type {Map<number, Object<string, any>>}
*/
this.states = new Map();
/**
* @type {Map<number, MetaClientState>}
*/
this.meta = new Map();
this._checkInterval = /** @type {any} */ (setInterval(() => {
const now = getUnixTime();
if (this.getLocalState() !== null && (outdatedTimeout / 2 <= now - /** @type {{lastUpdated:number}} */ (this.meta.get(this.clientID)).lastUpdated)) {
// renew local clock
this.setLocalState(this.getLocalState());
}
/**
* @type {Array<number>}
*/
const remove = [];
this.meta.forEach((meta, clientid) => {
if (clientid !== this.clientID && outdatedTimeout <= now - meta.lastUpdated && this.states.has(clientid)) {
remove.push(clientid);
}
});
if (remove.length > 0) {
removeAwarenessStates(this, remove, 'timeout');
}
}, floor(outdatedTimeout / 10)));
doc.on('destroy', () => {
this.destroy();
});
this.setLocalState({});
}
destroy () {
this.emit('destroy', [this]);
this.setLocalState(null);
super.destroy();
clearInterval(this._checkInterval);
}
/**
* @return {Object<string,any>|null}
*/
getLocalState () {
return this.states.get(this.clientID) || null
}
/**
* @param {Object<string,any>|null} state
*/
setLocalState (state) {
const clientID = this.clientID;
const currLocalMeta = this.meta.get(clientID);
const clock = currLocalMeta === undefined ? 0 : currLocalMeta.clock + 1;
const prevState = this.states.get(clientID);
if (state === null) {
this.states.delete(clientID);
} else {
this.states.set(clientID, state);
}
this.meta.set(clientID, {
clock,
lastUpdated: getUnixTime()
});
const added = [];
const updated = [];
const filteredUpdated = [];
const removed = [];
if (state === null) {
removed.push(clientID);
} else if (prevState == null) {
if (state != null) {
added.push(clientID);
}
} else {
updated.push(clientID);
if (!equalityDeep(prevState, state)) {
filteredUpdated.push(clientID);
}
}
if (added.length > 0 || filteredUpdated.length > 0 || removed.length > 0) {
this.emit('change', [{ added, updated: filteredUpdated, removed }, 'local']);
}
this.emit('update', [{ added, updated, removed }, 'local']);
}
/**
* @param {string} field
* @param {any} value
*/
setLocalStateField (field, value) {
const state = this.getLocalState();
if (state !== null) {
this.setLocalState({
...state,
[field]: value
});
}
}
/**
* @return {Map<number,Object<string,any>>}
*/
getStates () {
return this.states
}
}
/**
* Mark (remote) clients as inactive and remove them from the list of active peers.
* This change will be propagated to remote clients.
*
* @param {Awareness} awareness
* @param {Array<number>} clients
* @param {any} origin
*/
const removeAwarenessStates = (awareness, clients, origin) => {
const removed = [];
for (let i = 0; i < clients.length; i++) {
const clientID = clients[i];
if (awareness.states.has(clientID)) {
awareness.states.delete(clientID);
if (clientID === awareness.clientID) {
const curMeta = /** @type {MetaClientState} */ (awareness.meta.get(clientID));
awareness.meta.set(clientID, {
clock: curMeta.clock + 1,
lastUpdated: getUnixTime()
});
}
removed.push(clientID);
}
}
if (removed.length > 0) {
awareness.emit('change', [{ added: [], updated: [], removed }, origin]);
awareness.emit('update', [{ added: [], updated: [], removed }, origin]);
}
};
/**
* @param {Awareness} awareness
* @param {Array<number>} clients
* @return {Uint8Array}
*/
const encodeAwarenessUpdate = (awareness, clients, states = awareness.states) => {
const len = clients.length;
const encoder = createEncoder();
writeVarUint(encoder, len);
for (let i = 0; i < len; i++) {
const clientID = clients[i];
const state = states.get(clientID) || null;
const clock = /** @type {MetaClientState} */ (awareness.meta.get(clientID)).clock;
writeVarUint(encoder, clientID);
writeVarUint(encoder, clock);
writeVarString(encoder, JSON.stringify(state));
}
return toUint8Array(encoder)
};
/**
* @param {Awareness} awareness
* @param {Uint8Array} update
* @param {any} origin This will be added to the emitted change event
*/
const applyAwarenessUpdate = (awareness, update, origin) => {
const decoder = createDecoder(update);
const timestamp = getUnixTime();
const added = [];
const updated = [];
const filteredUpdated = [];
const removed = [];
const len = readVarUint(decoder);
for (let i = 0; i < len; i++) {
const clientID = readVarUint(decoder);
let clock = readVarUint(decoder);
const state = JSON.parse(readVarString(decoder));
const clientMeta = awareness.meta.get(clientID);
const prevState = awareness.states.get(clientID);
const currClock = clientMeta === undefined ? 0 : clientMeta.clock;
if (currClock < clock || (currClock === clock && state === null && awareness.states.has(clientID))) {
if (state === null) {
// never let a remote client remove this local state
if (clientID === awareness.clientID && awareness.getLocalState() != null) {
// remote client removed the local state. Do not remote state. Broadcast a message indicating
// that this client still exists by increasing the clock
clock++;
} else {
awareness.states.delete(clientID);
}
} else {
awareness.states.set(clientID, state);
}
awareness.meta.set(clientID, {
clock,
lastUpdated: timestamp
});
if (clientMeta === undefined && state !== null) {
added.push(clientID);
} else if (clientMeta !== undefined && state === null) {
removed.push(clientID);
} else if (state !== null) {
if (!equalityDeep(state, prevState)) {
filteredUpdated.push(clientID);
}
updated.push(clientID);
}
}
}
if (added.length > 0 || filteredUpdated.length > 0 || removed.length > 0) {
awareness.emit('change', [{
added, updated: filteredUpdated, removed
}, origin]);
}
if (added.length > 0 || updated.length > 0 || removed.length > 0) {
awareness.emit('update', [{
added, updated, removed
}, origin]);
}
};
/**
* @module sync-protocol
*/
/**
* @typedef {Map<number, number>} StateMap
*/
/**
* Core Yjs defines two message types:
* • YjsSyncStep1: Includes the State Set of the sending client. When received, the client should reply with YjsSyncStep2.
* • YjsSyncStep2: Includes all missing structs and the complete delete set. When received, the client is assured that it
* received all information from the remote client.
*
* In a peer-to-peer network, you may want to introduce a SyncDone message type. Both parties should initiate the connection
* with SyncStep1. When a client received SyncStep2, it should reply with SyncDone. When the local client received both
* SyncStep2 and SyncDone, it is assured that it is synced to the remote client.
*
* In a client-server model, you want to handle this differently: The client should initiate the connection with SyncStep1.
* When the server receives SyncStep1, it should reply with SyncStep2 immediately followed by SyncStep1. The client replies
* with SyncStep2 when it receives SyncStep1. Optionally the server may send a SyncDone after it received SyncStep2, so the
* client knows that the sync is finished. There are two reasons for this more elaborated sync model: 1. This protocol can
* easily be implemented on top of http and websockets. 2. The server should only reply to requests, and not initiate them.
* Therefore it is necessary that the client initiates the sync.
*
* Construction of a message:
* [messageType : varUint, message definition..]
*
* Note: A message does not include information about the room name. This must to be handled by the upper layer protocol!
*
* stringify[messageType] stringifies a message definition (messageType is already read from the bufffer)
*/
const messageYjsSyncStep1 = 0;
const messageYjsSyncStep2 = 1;
const messageYjsUpdate = 2;
/**
* Create a sync step 1 message based on the state of the current shared document.
*
* @param {encoding.Encoder} encoder
* @param {Y.Doc} doc
*/
const writeSyncStep1 = (encoder, doc) => {
writeVarUint(encoder, messageYjsSyncStep1);
const sv = Y.encodeStateVector(doc);
writeVarUint8Array(encoder, sv);
};
/**
* @param {encoding.Encoder} encoder
* @param {Y.Doc} doc
* @param {Uint8Array} [encodedStateVector]
*/
const writeSyncStep2 = (encoder, doc, encodedStateVector) => {
writeVarUint(encoder, messageYjsSyncStep2);
writeVarUint8Array(encoder, Y.encodeStateAsUpdate(doc, encodedStateVector));
};
/**
* Read SyncStep1 message and reply with SyncStep2.
*
* @param {decoding.Decoder} decoder The reply to the received message
* @param {encoding.Encoder} encoder The received message
* @param {Y.Doc} doc
*/
const readSyncStep1 = (decoder, encoder, doc) =>
writeSyncStep2(encoder, doc, readVarUint8Array(decoder));
/**
* Read and apply Structs and then DeleteStore to a y instance.
*
* @param {decoding.Decoder} decoder
* @param {Y.Doc} doc
* @param {any} transactionOrigin
*/
const readSyncStep2 = (decoder, doc, transactionOrigin) => {
try {
Y.applyUpdate(doc, readVarUint8Array(decoder), transactionOrigin);
} catch (error) {
// This catches errors that are thrown by event handlers
console.error('Caught error while handling a Yjs update', error);
}
};
/**
* @param {encoding.Encoder} encoder
* @param {Uint8Array} update
*/
const writeUpdate = (encoder, update) => {
writeVarUint(encoder, messageYjsUpdate);
writeVarUint8Array(encoder, update);
};
/**
* Read and apply Structs and then DeleteStore to a y instance.
*
* @param {decoding.Decoder} decoder
* @param {Y.Doc} doc
* @param {any} transactionOrigin
*/
const readUpdate = readSyncStep2;
var MessageType;
(function (MessageType) {
MessageType[MessageType["Unknown"] = -1] = "Unknown";
MessageType[MessageType["Sync"] = 0] = "Sync";
MessageType[MessageType["Awareness"] = 1] = "Awareness";
MessageType[MessageType["Auth"] = 2] = "Auth";
MessageType[MessageType["QueryAwareness"] = 3] = "QueryAwareness";
MessageType[MessageType["SyncReply"] = 4] = "SyncReply";
MessageType[MessageType["Stateless"] = 5] = "Stateless";
MessageType[MessageType["BroadcastStateless"] = 6] = "BroadcastStateless";
MessageType[MessageType["CLOSE"] = 7] = "CLOSE";
MessageType[MessageType["SyncStatus"] = 8] = "SyncStatus";
})(MessageType || (MessageType = {}));
class OutgoingMessage {
constructor(documentName) {
this.encoder = createEncoder();
writeVarString(this.encoder, documentName);
}
createSyncMessage() {
this.type = MessageType.Sync;
writeVarUint(this.encoder, MessageType.Sync);
return this;
}
createSyncReplyMessage() {
this.type = MessageType.SyncReply;
writeVarUint(this.encoder, MessageType.SyncReply);
return this;
}
createAwarenessUpdateMessage(awareness, changedClients) {
this.type = MessageType.Awareness;
this.category = "Update";
const message = encodeAwarenessUpdate(awareness, changedClients || Array.from(awareness.getStates().keys()));
writeVarUint(this.encoder, MessageType.Awareness);
writeVarUint8Array(this.encoder, message);
return this;
}
writeQueryAwareness() {
this.type = MessageType.QueryAwareness;
this.category = "Update";
writeVarUint(this.encoder, MessageType.QueryAwareness);
return this;
}
writeAuthenticated(readonly) {
this.type = MessageType.Auth;
this.category = "Authenticated";
writeVarUint(this.encoder, MessageType.Auth);
writeAuthenticated(this.encoder, readonly ? "readonly" : "read-write");
return this;
}
writePermissionDenied(reason) {
this.type = MessageType.Auth;
this.category = "PermissionDenied";
writeVarUint(this.encoder, MessageType.Auth);
writePermissionDenied(this.encoder, reason);
return this;
}
writeFirstSyncStepFor(document) {
this.category = "SyncStep1";
writeSyncStep1(this.encoder, document);
return this;
}
writeUpdate(update) {
this.category = "Update";
writeUpdate(this.encoder, update);
return this;
}
writeStateless(payload) {
this.category = "Stateless";
writeVarUint(this.encoder, MessageType.Stateless);
writeVarString(this.encoder, payload);
return this;
}
writeBroadcastStateless(payload) {
this.category = "Stateless";
writeVarUint(this.encoder, MessageType.BroadcastStateless);
writeVarString(this.encoder, payload);
return this;
}
// TODO: should this be write* or create* as method name?
writeSyncStatus(updateSaved) {
this.category = "SyncStatus";
writeVarUint(this.encoder, MessageType.SyncStatus);
writeVarUint(this.encoder, updateSaved ? 1 : 0);
return this;
}
writeCloseMessage(reason) {
this.type = MessageType.CLOSE;
writeVarUint(this.encoder, MessageType.CLOSE);
writeVarString(this.encoder, reason);
return this;
}
toUint8Array() {
return toUint8Array(this.encoder);
}
}
class MessageReceiver {
constructor(message, defaultTransactionOrigin) {
this.message = message;
this.defaultTransactionOrigin = defaultTransactionOrigin;
}
async apply(document, connection, reply) {
const { message } = this;
const type = message.readVarUint();
const emptyMessageLength = message.length;
switch (type) {
case MessageType.Sync:
case MessageType.SyncReply: {
message.writeVarUint(MessageType.Sync);
await this.readSyncMessage(message, document, connection, reply, type !== MessageType.SyncReply);
if (message.length > emptyMessageLength + 1) {
if (reply) {
reply(message.toUint8Array());
}
else if (connection) {
// TODO: We should log this, shouldn’t we?
// this.logger.log({
// direction: 'out',
// type: MessageType.Awareness,
// category: 'Update',
// })
connection.send(message.toUint8Array());
}
}
break;
}
case MessageType.Awareness: {
await applyAwarenessUpdate(document.awareness, message.readVarUint8Array(), connection === null || connection === void 0 ? void 0 : connection.webSocket);
break;
}
case MessageType.QueryAwareness: {
this.applyQueryAwarenessMessage(document, reply);
break;
}
case MessageType.Stateless: {
connection === null || connection === void 0 ? void 0 : connection.callbacks.statelessCallback({
connection,
documentName: document.name,
document,
payload: readVarString(message.decoder),
});
break;
}
case MessageType.BroadcastStateless: {
const msg = message.readVarString();
document.getConnections().forEach((connection) => {
connection.sendStateless(msg);
});
break;
}
case MessageType.CLOSE: {
connection === null || connection === void 0 ? void 0 : connection.close({
code: 1000,
reason: "provider_initiated",
});
break;
}
case MessageType.Auth:
console.error("Received an authentication message on a connection that is already fully authenticated. Probably your provider has been destroyed + recreated really fast.");
break;
default:
console.error(`Unable to handle message of type ${type}: no handler defined! Are your provider/server versions aligned?`);
// Do nothing
}
}
async readSyncMessage(message, document, connection, reply, requestFirstSync = true) {
const type = message.readVarUint();
if (connection) {
await connection.callbacks.beforeSync(connection, {
type,
payload: message.peekVarUint8Array(),
});
}
switch (type) {
case messageYjsSyncStep1: {
readSyncStep1(message.decoder, message.encoder, document);
// When the server receives SyncStep1, it should reply with SyncStep2 immediately followed by SyncStep1.
if (reply && requestFirstSync) {
const syncMessage = new OutgoingMessage(document.name)
.createSyncReplyMessage()
.writeFirstSyncStepFor(document);
reply(syncMessage.toUint8Array());
}
else if (connection) {
const syncMessage = new OutgoingMessage(document.name)
.createSyncMessage()
.writeFirstSyncStepFor(document);
connection.send(syncMessage.toUint8Array());
}
break;
}
case messageYjsSyncStep2:
if (connection === null || connection === void 0 ? void 0 : connection.readOnly) {
// We're in read-only mode, so we can't apply the update.
// Let's use snapshotContainsUpdate to see if the update actually contains changes.
// If not, we can still ack the update
const snapshot = Y.snapshot(document);
const update = readVarUint8Array(message.decoder);
if (Y.snapshotContainsUpdate(snapshot, update)) {
// no new changes in update
const ackMessage = new OutgoingMessage(document.name).writeSyncStatus(true);
connection.send(ackMessage.toUint8Array());
}
else {
// new changes in update that we can't apply, because readOnly
const ackMessage = new OutgoingMessage(document.name).writeSyncStatus(false);
connection.send(ackMessage.toUint8Array());
}
break;
}
readSyncStep2(message.decoder, document, connection !== null && connection !== void 0 ? connection : this.defaultTransactionOrigin);
if (connection) {
connection.send(new OutgoingMessage(document.name)
.writeSyncStatus(true)
.toUint8Array());
}
break;
case messageYjsUpdate:
if (connection === null || connection === void 0 ? void 0 : connection.readOnly) {
connection.send(new OutgoingMessage(document.name)
.writeSyncStatus(false)
.toUint8Array());
break;
}
readUpdate(message.decoder, document, connection);
if (connection) {
connection.send(new OutgoingMessage(document.name)
.writeSyncStatus(true)
.toUint8Array());
}
break;
default:
throw new Error(`Received a message with an unknown type: ${type}`);
}
return type;
}
applyQueryAwarenessMessage(document, reply) {
const message = new OutgoingMessage(document.name).createAwarenessUpdateMessage(document.awareness);
if (reply) {
reply(message.toUint8Array());
}
}
}
class Connection {
/**
* Constructor.
*/
constructor(connection, request, document, socketId, context, readOnly = false) {
this.callbacks = {
onClose: [(document, event) => { }],
beforeHandleMessage: (connection, update) => Promise.resolve(),
beforeSync: (connection, payload) => Promise.resolve(),
statelessCallback: (payload) => Promise.resolve(),
};
this.webSocket = connection;
this.context = context;
this.document = document;
this.request = request;
this.socketId = socketId;
this.readOnly = readOnly;
this.webSocket.binaryType = "nodebuffer";
this.document.addConnection(this);
this.sendCurrentAwareness();
}
/**
* Set a callback that will be triggered when the connection is closed
*/
onClose(callback) {
this.callbacks.onClose.push(callback);
return this;
}
/**
* Set a callback that will be triggered when an stateless message is received
*/
onStatelessCallback(callback) {
this.callbacks.statelessCallback = callback;
return this;
}
/**
* Set a callback that will be triggered before an message is handled
*/
beforeHandleMessage(callback) {
this.callbacks.beforeHandleMessage = callback;
return this;
}
/**
* Set a callback that will be triggered before a sync message is handled
*/
beforeSync(callback) {
this.callbacks.beforeSync = callback;
return this;
}
/**
* Send the given message
*/
send(message) {
if (this.webSocket.readyState === WsReadyStates.Closing ||
this.webSocket.readyState === WsReadyStates.Closed) {
this.close();
return;
}
try {
this.webSocket.send(message, (error) => {
if (error != null)
this.close();
});
}
catch (exception) {
this.close();
}
}
/**
* Send a stateless message with payload
*/
sendStateless(payload) {
const message = new OutgoingMessage(this.document.name).writeStateless(payload);
this.send(message.toUint8Array());
}
/**
* Graceful wrapper around the WebSocket close method.
*/
close(event) {
var _a;
if (this.document.hasConnection(this)) {
this.document.removeConnection(this);
this.callbacks.onClose.forEach((callback) => callback(this.document, event));
const closeMessage = new OutgoingMessage(this.document.name);
closeMessage.writeCloseMessage((_a = event === null || event === void 0 ? void 0 : event.reason) !== null && _a !== void 0 ? _a : "Server closed the connection");
this.send(closeMessage.toUint8Array());
}
}
/**
* Send the current document awareness to the client, if any
* @private
*/
sendCurrentAwareness() {
if (!this.document.hasAwarenessStates()) {
return;
}
const awarenessMessage = new OutgoingMessage(this.document.name).createAwarenessUpdateMessage(this.document.awareness);
this.send(awarenessMessage.toUint8Array());
}
/**
* Handle an incoming message
* @public
*/
handleMessage(data) {
const message = new IncomingMessage(data);
const documentName = message.readVarString();
if (documentName !== this.document.name)
return;
message.writeVarString(documentName);
this.callbacks
.beforeHandleMessage(this, data)
.then(() => {
new MessageReceiver(message).apply(this.document, this).catch((e) => {
console.error("closing connection because of exception", e);
this.close({
code: "code" in e ? e.code : ResetConnection.code,
reason: "reason" in e ? e.reason : ResetConnection.reason,
});
});
})
.catch((e) => {
console.error("closing connection because of exception", e);
this.close({
code: "code" in e ? e.code : ResetConnection.code,
reason: "reason" in e ? e.reason : ResetConnection.reason,
});
});
}
}
class Document extends Doc {
/**
* Constructor.
*/
constructor(name, yDocOptions) {
super(yDocOptions);
this.callbacks = {
// eslint-disable-next-line @typescript-eslint/no-empty-function
onUpdate: (document, connection, update) => { },
beforeBroadcastStateless: (document, stateless) => { },
};
this.connections = new Map();
// The number of direct (non-websocket) connections to this document
this.directConnectionsCount = 0;
this.isDestroyed = false;
this.name = name;
this.awareness = new Awareness(this);
this.awareness.setLocalState(null);
this.awareness.on("update", this.handleAwarenessUpdate.bind(this));
this.on("update", this.handleUpdate.bind(this));
this.isLoading = true;
}
/**
* Check if the Document (XMLFragment or Map) is empty
*/
isEmpty(fieldName) {
// eslint-disable-next-line no-underscore-dangle
return !this.get(fieldName)._start && !this.get(fieldName)._map.size;
}
/**
* Merge the given document(s) into this one
*/
merge(documents) {
(Array.isArray(documents) ? documents : [documents]).forEach((document) => {
applyUpdate(this, encodeStateAsUpdate(document));
});
return this;
}
/**
* Set a callback that will be triggered when the document is updated
*/
onUpdate(callback) {
this.callbacks.onUpdate = callback;
return this;
}
/**
* Set a callback that will be triggered before a stateless message is broadcasted
*/
beforeBroadcastStateless(callback) {
this.callbacks.beforeBroadcastStateless = callback;
return this;
}
/**
* Register a connection and a set of clients on this document keyed by the
* underlying websocket connection
*/
addConnection(connection) {
this.connections.set(connection.webSocket, {
clients: new Set(),
connection,
});
return this;
}
/**
* Is the given connection registered on this document
*/
hasConnection(connection) {
return this.connections.has(connection.webSocket);
}
/**
* Remove the given connection from this document
*/
removeConnection(connection) {
removeAwarenessStates(this.awareness, Array.from(this.getClients(connection.webSocket)), null);
this.connections.delete(connection.webSocket);
return this;
}
addDirectConnection() {
this.directConnectionsCount += 1;
return this;
}
removeDirectConnection() {
if (this.directConnectionsCount > 0) {
this.directConnectionsCount -= 1;
}
return this;
}
/**
* Get the number of active connections for this document
*/
getConnectionsCount() {
return this.connections.size + this.directConnectionsCount;
}
/**
* Get an array of registered connections
*/
getConnections() {
return Array.from(this.connections.values()).map((data) => data.connection);
}
/**
* Get t