recoder-code
Version:
Complete AI-powered development platform with ML model training, plugin registry, real-time collaboration, monitoring, infrastructure automation, and enterprise deployment capabilities
1,625 lines (1,505 loc) • 307 kB
JavaScript
'use strict';
var observable = require('lib0/observable');
var array = require('lib0/array');
var math = require('lib0/math');
var map = require('lib0/map');
var encoding = require('lib0/encoding');
var decoding = require('lib0/decoding');
var random = require('lib0/random');
var promise = require('lib0/promise');
var buffer = require('lib0/buffer');
var error = require('lib0/error');
var binary = require('lib0/binary');
var f = require('lib0/function');
var set = require('lib0/set');
var logging = require('lib0/logging');
var time = require('lib0/time');
var string = require('lib0/string');
var iterator = require('lib0/iterator');
var object = require('lib0/object');
var env = require('lib0/environment');
function _interopNamespaceDefault(e) {
var n = Object.create(null);
if (e) {
Object.keys(e).forEach(function (k) {
if (k !== 'default') {
var d = Object.getOwnPropertyDescriptor(e, k);
Object.defineProperty(n, k, d.get ? d : {
enumerable: true,
get: function () { return e[k]; }
});
}
});
}
n.default = e;
return Object.freeze(n);
}
var array__namespace = /*#__PURE__*/_interopNamespaceDefault(array);
var math__namespace = /*#__PURE__*/_interopNamespaceDefault(math);
var map__namespace = /*#__PURE__*/_interopNamespaceDefault(map);
var encoding__namespace = /*#__PURE__*/_interopNamespaceDefault(encoding);
var decoding__namespace = /*#__PURE__*/_interopNamespaceDefault(decoding);
var random__namespace = /*#__PURE__*/_interopNamespaceDefault(random);
var promise__namespace = /*#__PURE__*/_interopNamespaceDefault(promise);
var buffer__namespace = /*#__PURE__*/_interopNamespaceDefault(buffer);
var error__namespace = /*#__PURE__*/_interopNamespaceDefault(error);
var binary__namespace = /*#__PURE__*/_interopNamespaceDefault(binary);
var f__namespace = /*#__PURE__*/_interopNamespaceDefault(f);
var set__namespace = /*#__PURE__*/_interopNamespaceDefault(set);
var logging__namespace = /*#__PURE__*/_interopNamespaceDefault(logging);
var time__namespace = /*#__PURE__*/_interopNamespaceDefault(time);
var string__namespace = /*#__PURE__*/_interopNamespaceDefault(string);
var iterator__namespace = /*#__PURE__*/_interopNamespaceDefault(iterator);
var object__namespace = /*#__PURE__*/_interopNamespaceDefault(object);
var env__namespace = /*#__PURE__*/_interopNamespaceDefault(env);
/**
* This is an abstract interface that all Connectors should implement to keep them interchangeable.
*
* @note This interface is experimental and it is not advised to actually inherit this class.
* It just serves as typing information.
*
* @extends {ObservableV2<any>}
*/
class AbstractConnector extends observable.ObservableV2 {
/**
* @param {Doc} ydoc
* @param {any} awareness
*/
constructor (ydoc, awareness) {
super();
this.doc = ydoc;
this.awareness = awareness;
}
}
class DeleteItem {
/**
* @param {number} clock
* @param {number} len
*/
constructor (clock, len) {
/**
* @type {number}
*/
this.clock = clock;
/**
* @type {number}
*/
this.len = len;
}
}
/**
* We no longer maintain a DeleteStore. DeleteSet is a temporary object that is created when needed.
* - When created in a transaction, it must only be accessed after sorting, and merging
* - This DeleteSet is send to other clients
* - We do not create a DeleteSet when we send a sync message. The DeleteSet message is created directly from StructStore
* - We read a DeleteSet as part of a sync/update message. In this case the DeleteSet is already sorted and merged.
*/
class DeleteSet {
constructor () {
/**
* @type {Map<number,Array<DeleteItem>>}
*/
this.clients = new Map();
}
}
/**
* Iterate over all structs that the DeleteSet gc's.
*
* @param {Transaction} transaction
* @param {DeleteSet} ds
* @param {function(GC|Item):void} f
*
* @function
*/
const iterateDeletedStructs = (transaction, ds, f) =>
ds.clients.forEach((deletes, clientid) => {
const structs = /** @type {Array<GC|Item>} */ (transaction.doc.store.clients.get(clientid));
if (structs != null) {
const lastStruct = structs[structs.length - 1];
const clockState = lastStruct.id.clock + lastStruct.length;
for (let i = 0, del = deletes[i]; i < deletes.length && del.clock < clockState; del = deletes[++i]) {
iterateStructs(transaction, structs, del.clock, del.len, f);
}
}
});
/**
* @param {Array<DeleteItem>} dis
* @param {number} clock
* @return {number|null}
*
* @private
* @function
*/
const findIndexDS = (dis, clock) => {
let left = 0;
let right = dis.length - 1;
while (left <= right) {
const midindex = math__namespace.floor((left + right) / 2);
const mid = dis[midindex];
const midclock = mid.clock;
if (midclock <= clock) {
if (clock < midclock + mid.len) {
return midindex
}
left = midindex + 1;
} else {
right = midindex - 1;
}
}
return null
};
/**
* @param {DeleteSet} ds
* @param {ID} id
* @return {boolean}
*
* @private
* @function
*/
const isDeleted = (ds, id) => {
const dis = ds.clients.get(id.client);
return dis !== undefined && findIndexDS(dis, id.clock) !== null
};
/**
* @param {DeleteSet} ds
*
* @private
* @function
*/
const sortAndMergeDeleteSet = ds => {
ds.clients.forEach(dels => {
dels.sort((a, b) => a.clock - b.clock);
// merge items without filtering or splicing the array
// i is the current pointer
// j refers to the current insert position for the pointed item
// try to merge dels[i] into dels[j-1] or set dels[j]=dels[i]
let i, j;
for (i = 1, j = 1; i < dels.length; i++) {
const left = dels[j - 1];
const right = dels[i];
if (left.clock + left.len >= right.clock) {
left.len = math__namespace.max(left.len, right.clock + right.len - left.clock);
} else {
if (j < i) {
dels[j] = right;
}
j++;
}
}
dels.length = j;
});
};
/**
* @param {Array<DeleteSet>} dss
* @return {DeleteSet} A fresh DeleteSet
*/
const mergeDeleteSets = dss => {
const merged = new DeleteSet();
for (let dssI = 0; dssI < dss.length; dssI++) {
dss[dssI].clients.forEach((delsLeft, client) => {
if (!merged.clients.has(client)) {
// Write all missing keys from current ds and all following.
// If merged already contains `client` current ds has already been added.
/**
* @type {Array<DeleteItem>}
*/
const dels = delsLeft.slice();
for (let i = dssI + 1; i < dss.length; i++) {
array__namespace.appendTo(dels, dss[i].clients.get(client) || []);
}
merged.clients.set(client, dels);
}
});
}
sortAndMergeDeleteSet(merged);
return merged
};
/**
* @param {DeleteSet} ds
* @param {number} client
* @param {number} clock
* @param {number} length
*
* @private
* @function
*/
const addToDeleteSet = (ds, client, clock, length) => {
map__namespace.setIfUndefined(ds.clients, client, () => /** @type {Array<DeleteItem>} */ ([])).push(new DeleteItem(clock, length));
};
const createDeleteSet = () => new DeleteSet();
/**
* @param {StructStore} ss
* @return {DeleteSet} Merged and sorted DeleteSet
*
* @private
* @function
*/
const createDeleteSetFromStructStore = ss => {
const ds = createDeleteSet();
ss.clients.forEach((structs, client) => {
/**
* @type {Array<DeleteItem>}
*/
const dsitems = [];
for (let i = 0; i < structs.length; i++) {
const struct = structs[i];
if (struct.deleted) {
const clock = struct.id.clock;
let len = struct.length;
if (i + 1 < structs.length) {
for (let next = structs[i + 1]; i + 1 < structs.length && next.deleted; next = structs[++i + 1]) {
len += next.length;
}
}
dsitems.push(new DeleteItem(clock, len));
}
}
if (dsitems.length > 0) {
ds.clients.set(client, dsitems);
}
});
return ds
};
/**
* @param {DSEncoderV1 | DSEncoderV2} encoder
* @param {DeleteSet} ds
*
* @private
* @function
*/
const writeDeleteSet = (encoder, ds) => {
encoding__namespace.writeVarUint(encoder.restEncoder, ds.clients.size);
// Ensure that the delete set is written in a deterministic order
array__namespace.from(ds.clients.entries())
.sort((a, b) => b[0] - a[0])
.forEach(([client, dsitems]) => {
encoder.resetDsCurVal();
encoding__namespace.writeVarUint(encoder.restEncoder, client);
const len = dsitems.length;
encoding__namespace.writeVarUint(encoder.restEncoder, len);
for (let i = 0; i < len; i++) {
const item = dsitems[i];
encoder.writeDsClock(item.clock);
encoder.writeDsLen(item.len);
}
});
};
/**
* @param {DSDecoderV1 | DSDecoderV2} decoder
* @return {DeleteSet}
*
* @private
* @function
*/
const readDeleteSet = decoder => {
const ds = new DeleteSet();
const numClients = decoding__namespace.readVarUint(decoder.restDecoder);
for (let i = 0; i < numClients; i++) {
decoder.resetDsCurVal();
const client = decoding__namespace.readVarUint(decoder.restDecoder);
const numberOfDeletes = decoding__namespace.readVarUint(decoder.restDecoder);
if (numberOfDeletes > 0) {
const dsField = map__namespace.setIfUndefined(ds.clients, client, () => /** @type {Array<DeleteItem>} */ ([]));
for (let i = 0; i < numberOfDeletes; i++) {
dsField.push(new DeleteItem(decoder.readDsClock(), decoder.readDsLen()));
}
}
}
return ds
};
/**
* @todo YDecoder also contains references to String and other Decoders. Would make sense to exchange YDecoder.toUint8Array for YDecoder.DsToUint8Array()..
*/
/**
* @param {DSDecoderV1 | DSDecoderV2} decoder
* @param {Transaction} transaction
* @param {StructStore} store
* @return {Uint8Array|null} Returns a v2 update containing all deletes that couldn't be applied yet; or null if all deletes were applied successfully.
*
* @private
* @function
*/
const readAndApplyDeleteSet = (decoder, transaction, store) => {
const unappliedDS = new DeleteSet();
const numClients = decoding__namespace.readVarUint(decoder.restDecoder);
for (let i = 0; i < numClients; i++) {
decoder.resetDsCurVal();
const client = decoding__namespace.readVarUint(decoder.restDecoder);
const numberOfDeletes = decoding__namespace.readVarUint(decoder.restDecoder);
const structs = store.clients.get(client) || [];
const state = getState(store, client);
for (let i = 0; i < numberOfDeletes; i++) {
const clock = decoder.readDsClock();
const clockEnd = clock + decoder.readDsLen();
if (clock < state) {
if (state < clockEnd) {
addToDeleteSet(unappliedDS, client, state, clockEnd - state);
}
let index = findIndexSS(structs, clock);
/**
* We can ignore the case of GC and Delete structs, because we are going to skip them
* @type {Item}
*/
// @ts-ignore
let struct = structs[index];
// split the first item if necessary
if (!struct.deleted && struct.id.clock < clock) {
structs.splice(index + 1, 0, splitItem(transaction, struct, clock - struct.id.clock));
index++; // increase we now want to use the next struct
}
while (index < structs.length) {
// @ts-ignore
struct = structs[index++];
if (struct.id.clock < clockEnd) {
if (!struct.deleted) {
if (clockEnd < struct.id.clock + struct.length) {
structs.splice(index, 0, splitItem(transaction, struct, clockEnd - struct.id.clock));
}
struct.delete(transaction);
}
} else {
break
}
}
} else {
addToDeleteSet(unappliedDS, client, clock, clockEnd - clock);
}
}
}
if (unappliedDS.clients.size > 0) {
const ds = new UpdateEncoderV2();
encoding__namespace.writeVarUint(ds.restEncoder, 0); // encode 0 structs
writeDeleteSet(ds, unappliedDS);
return ds.toUint8Array()
}
return null
};
/**
* @param {DeleteSet} ds1
* @param {DeleteSet} ds2
*/
const equalDeleteSets = (ds1, ds2) => {
if (ds1.clients.size !== ds2.clients.size) return false
for (const [client, deleteItems1] of ds1.clients.entries()) {
const deleteItems2 = /** @type {Array<import('../internals.js').DeleteItem>} */ (ds2.clients.get(client));
if (deleteItems2 === undefined || deleteItems1.length !== deleteItems2.length) return false
for (let i = 0; i < deleteItems1.length; i++) {
const di1 = deleteItems1[i];
const di2 = deleteItems2[i];
if (di1.clock !== di2.clock || di1.len !== di2.len) {
return false
}
}
}
return true
};
/**
* @module Y
*/
const generateNewClientId = random__namespace.uint32;
/**
* @typedef {Object} DocOpts
* @property {boolean} [DocOpts.gc=true] Disable garbage collection (default: gc=true)
* @property {function(Item):boolean} [DocOpts.gcFilter] Will be called before an Item is garbage collected. Return false to keep the Item.
* @property {string} [DocOpts.guid] Define a globally unique identifier for this document
* @property {string | null} [DocOpts.collectionid] Associate this document with a collection. This only plays a role if your provider has a concept of collection.
* @property {any} [DocOpts.meta] Any kind of meta information you want to associate with this document. If this is a subdocument, remote peers will store the meta information as well.
* @property {boolean} [DocOpts.autoLoad] If a subdocument, automatically load document. If this is a subdocument, remote peers will load the document as well automatically.
* @property {boolean} [DocOpts.shouldLoad] Whether the document should be synced by the provider now. This is toggled to true when you call ydoc.load()
*/
/**
* @typedef {Object} DocEvents
* @property {function(Doc):void} DocEvents.destroy
* @property {function(Doc):void} DocEvents.load
* @property {function(boolean, Doc):void} DocEvents.sync
* @property {function(Uint8Array, any, Doc, Transaction):void} DocEvents.update
* @property {function(Uint8Array, any, Doc, Transaction):void} DocEvents.updateV2
* @property {function(Doc):void} DocEvents.beforeAllTransactions
* @property {function(Transaction, Doc):void} DocEvents.beforeTransaction
* @property {function(Transaction, Doc):void} DocEvents.beforeObserverCalls
* @property {function(Transaction, Doc):void} DocEvents.afterTransaction
* @property {function(Transaction, Doc):void} DocEvents.afterTransactionCleanup
* @property {function(Doc, Array<Transaction>):void} DocEvents.afterAllTransactions
* @property {function({ loaded: Set<Doc>, added: Set<Doc>, removed: Set<Doc> }, Doc, Transaction):void} DocEvents.subdocs
*/
/**
* A Yjs instance handles the state of shared data.
* @extends ObservableV2<DocEvents>
*/
class Doc extends observable.ObservableV2 {
/**
* @param {DocOpts} opts configuration
*/
constructor ({ guid = random__namespace.uuidv4(), collectionid = null, gc = true, gcFilter = () => true, meta = null, autoLoad = false, shouldLoad = true } = {}) {
super();
this.gc = gc;
this.gcFilter = gcFilter;
this.clientID = generateNewClientId();
this.guid = guid;
this.collectionid = collectionid;
/**
* @type {Map<string, AbstractType<YEvent<any>>>}
*/
this.share = new Map();
this.store = new StructStore();
/**
* @type {Transaction | null}
*/
this._transaction = null;
/**
* @type {Array<Transaction>}
*/
this._transactionCleanups = [];
/**
* @type {Set<Doc>}
*/
this.subdocs = new Set();
/**
* If this document is a subdocument - a document integrated into another document - then _item is defined.
* @type {Item?}
*/
this._item = null;
this.shouldLoad = shouldLoad;
this.autoLoad = autoLoad;
this.meta = meta;
/**
* This is set to true when the persistence provider loaded the document from the database or when the `sync` event fires.
* Note that not all providers implement this feature. Provider authors are encouraged to fire the `load` event when the doc content is loaded from the database.
*
* @type {boolean}
*/
this.isLoaded = false;
/**
* This is set to true when the connection provider has successfully synced with a backend.
* Note that when using peer-to-peer providers this event may not provide very useful.
* Also note that not all providers implement this feature. Provider authors are encouraged to fire
* the `sync` event when the doc has been synced (with `true` as a parameter) or if connection is
* lost (with false as a parameter).
*/
this.isSynced = false;
this.isDestroyed = false;
/**
* Promise that resolves once the document has been loaded from a persistence provider.
*/
this.whenLoaded = promise__namespace.create(resolve => {
this.on('load', () => {
this.isLoaded = true;
resolve(this);
});
});
const provideSyncedPromise = () => promise__namespace.create(resolve => {
/**
* @param {boolean} isSynced
*/
const eventHandler = (isSynced) => {
if (isSynced === undefined || isSynced === true) {
this.off('sync', eventHandler);
resolve();
}
};
this.on('sync', eventHandler);
});
this.on('sync', isSynced => {
if (isSynced === false && this.isSynced) {
this.whenSynced = provideSyncedPromise();
}
this.isSynced = isSynced === undefined || isSynced === true;
if (this.isSynced && !this.isLoaded) {
this.emit('load', [this]);
}
});
/**
* Promise that resolves once the document has been synced with a backend.
* This promise is recreated when the connection is lost.
* Note the documentation about the `isSynced` property.
*/
this.whenSynced = provideSyncedPromise();
}
/**
* Notify the parent document that you request to load data into this subdocument (if it is a subdocument).
*
* `load()` might be used in the future to request any provider to load the most current data.
*
* It is safe to call `load()` multiple times.
*/
load () {
const item = this._item;
if (item !== null && !this.shouldLoad) {
transact(/** @type {any} */ (item.parent).doc, transaction => {
transaction.subdocsLoaded.add(this);
}, null, true);
}
this.shouldLoad = true;
}
getSubdocs () {
return this.subdocs
}
getSubdocGuids () {
return new Set(array__namespace.from(this.subdocs).map(doc => doc.guid))
}
/**
* Changes that happen inside of a transaction are bundled. This means that
* the observer fires _after_ the transaction is finished and that all changes
* that happened inside of the transaction are sent as one message to the
* other peers.
*
* @template T
* @param {function(Transaction):T} f The function that should be executed as a transaction
* @param {any} [origin] Origin of who started the transaction. Will be stored on transaction.origin
* @return T
*
* @public
*/
transact (f, origin = null) {
return transact(this, f, origin)
}
/**
* Define a shared data type.
*
* Multiple calls of `ydoc.get(name, TypeConstructor)` yield the same result
* and do not overwrite each other. I.e.
* `ydoc.get(name, Y.Array) === ydoc.get(name, Y.Array)`
*
* After this method is called, the type is also available on `ydoc.share.get(name)`.
*
* *Best Practices:*
* Define all types right after the Y.Doc instance is created and store them in a separate object.
* Also use the typed methods `getText(name)`, `getArray(name)`, ..
*
* @template {typeof AbstractType<any>} Type
* @example
* const ydoc = new Y.Doc(..)
* const appState = {
* document: ydoc.getText('document')
* comments: ydoc.getArray('comments')
* }
*
* @param {string} name
* @param {Type} TypeConstructor The constructor of the type definition. E.g. Y.Text, Y.Array, Y.Map, ...
* @return {InstanceType<Type>} The created type. Constructed with TypeConstructor
*
* @public
*/
get (name, TypeConstructor = /** @type {any} */ (AbstractType)) {
const type = map__namespace.setIfUndefined(this.share, name, () => {
// @ts-ignore
const t = new TypeConstructor();
t._integrate(this, null);
return t
});
const Constr = type.constructor;
if (TypeConstructor !== AbstractType && Constr !== TypeConstructor) {
if (Constr === AbstractType) {
// @ts-ignore
const t = new TypeConstructor();
t._map = type._map;
type._map.forEach(/** @param {Item?} n */ n => {
for (; n !== null; n = n.left) {
// @ts-ignore
n.parent = t;
}
});
t._start = type._start;
for (let n = t._start; n !== null; n = n.right) {
n.parent = t;
}
t._length = type._length;
this.share.set(name, t);
t._integrate(this, null);
return /** @type {InstanceType<Type>} */ (t)
} else {
throw new Error(`Type with the name ${name} has already been defined with a different constructor`)
}
}
return /** @type {InstanceType<Type>} */ (type)
}
/**
* @template T
* @param {string} [name]
* @return {YArray<T>}
*
* @public
*/
getArray (name = '') {
return /** @type {YArray<T>} */ (this.get(name, YArray))
}
/**
* @param {string} [name]
* @return {YText}
*
* @public
*/
getText (name = '') {
return this.get(name, YText)
}
/**
* @template T
* @param {string} [name]
* @return {YMap<T>}
*
* @public
*/
getMap (name = '') {
return /** @type {YMap<T>} */ (this.get(name, YMap))
}
/**
* @param {string} [name]
* @return {YXmlElement}
*
* @public
*/
getXmlElement (name = '') {
return /** @type {YXmlElement<{[key:string]:string}>} */ (this.get(name, YXmlElement))
}
/**
* @param {string} [name]
* @return {YXmlFragment}
*
* @public
*/
getXmlFragment (name = '') {
return this.get(name, YXmlFragment)
}
/**
* Converts the entire document into a js object, recursively traversing each yjs type
* Doesn't log types that have not been defined (using ydoc.getType(..)).
*
* @deprecated Do not use this method and rather call toJSON directly on the shared types.
*
* @return {Object<string, any>}
*/
toJSON () {
/**
* @type {Object<string, any>}
*/
const doc = {};
this.share.forEach((value, key) => {
doc[key] = value.toJSON();
});
return doc
}
/**
* Emit `destroy` event and unregister all event handlers.
*/
destroy () {
this.isDestroyed = true;
array__namespace.from(this.subdocs).forEach(subdoc => subdoc.destroy());
const item = this._item;
if (item !== null) {
this._item = null;
const content = /** @type {ContentDoc} */ (item.content);
content.doc = new Doc({ guid: this.guid, ...content.opts, shouldLoad: false });
content.doc._item = item;
transact(/** @type {any} */ (item).parent.doc, transaction => {
const doc = content.doc;
if (!item.deleted) {
transaction.subdocsAdded.add(doc);
}
transaction.subdocsRemoved.add(this);
}, null, true);
}
// @ts-ignore
this.emit('destroyed', [true]); // DEPRECATED!
this.emit('destroy', [this]);
super.destroy();
}
}
class DSDecoderV1 {
/**
* @param {decoding.Decoder} decoder
*/
constructor (decoder) {
this.restDecoder = decoder;
}
resetDsCurVal () {
// nop
}
/**
* @return {number}
*/
readDsClock () {
return decoding__namespace.readVarUint(this.restDecoder)
}
/**
* @return {number}
*/
readDsLen () {
return decoding__namespace.readVarUint(this.restDecoder)
}
}
class UpdateDecoderV1 extends DSDecoderV1 {
/**
* @return {ID}
*/
readLeftID () {
return createID(decoding__namespace.readVarUint(this.restDecoder), decoding__namespace.readVarUint(this.restDecoder))
}
/**
* @return {ID}
*/
readRightID () {
return createID(decoding__namespace.readVarUint(this.restDecoder), decoding__namespace.readVarUint(this.restDecoder))
}
/**
* Read the next client id.
* Use this in favor of readID whenever possible to reduce the number of objects created.
*/
readClient () {
return decoding__namespace.readVarUint(this.restDecoder)
}
/**
* @return {number} info An unsigned 8-bit integer
*/
readInfo () {
return decoding__namespace.readUint8(this.restDecoder)
}
/**
* @return {string}
*/
readString () {
return decoding__namespace.readVarString(this.restDecoder)
}
/**
* @return {boolean} isKey
*/
readParentInfo () {
return decoding__namespace.readVarUint(this.restDecoder) === 1
}
/**
* @return {number} info An unsigned 8-bit integer
*/
readTypeRef () {
return decoding__namespace.readVarUint(this.restDecoder)
}
/**
* Write len of a struct - well suited for Opt RLE encoder.
*
* @return {number} len
*/
readLen () {
return decoding__namespace.readVarUint(this.restDecoder)
}
/**
* @return {any}
*/
readAny () {
return decoding__namespace.readAny(this.restDecoder)
}
/**
* @return {Uint8Array}
*/
readBuf () {
return buffer__namespace.copyUint8Array(decoding__namespace.readVarUint8Array(this.restDecoder))
}
/**
* Legacy implementation uses JSON parse. We use any-decoding in v2.
*
* @return {any}
*/
readJSON () {
return JSON.parse(decoding__namespace.readVarString(this.restDecoder))
}
/**
* @return {string}
*/
readKey () {
return decoding__namespace.readVarString(this.restDecoder)
}
}
class DSDecoderV2 {
/**
* @param {decoding.Decoder} decoder
*/
constructor (decoder) {
/**
* @private
*/
this.dsCurrVal = 0;
this.restDecoder = decoder;
}
resetDsCurVal () {
this.dsCurrVal = 0;
}
/**
* @return {number}
*/
readDsClock () {
this.dsCurrVal += decoding__namespace.readVarUint(this.restDecoder);
return this.dsCurrVal
}
/**
* @return {number}
*/
readDsLen () {
const diff = decoding__namespace.readVarUint(this.restDecoder) + 1;
this.dsCurrVal += diff;
return diff
}
}
class UpdateDecoderV2 extends DSDecoderV2 {
/**
* @param {decoding.Decoder} decoder
*/
constructor (decoder) {
super(decoder);
/**
* List of cached keys. If the keys[id] does not exist, we read a new key
* from stringEncoder and push it to keys.
*
* @type {Array<string>}
*/
this.keys = [];
decoding__namespace.readVarUint(decoder); // read feature flag - currently unused
this.keyClockDecoder = new decoding__namespace.IntDiffOptRleDecoder(decoding__namespace.readVarUint8Array(decoder));
this.clientDecoder = new decoding__namespace.UintOptRleDecoder(decoding__namespace.readVarUint8Array(decoder));
this.leftClockDecoder = new decoding__namespace.IntDiffOptRleDecoder(decoding__namespace.readVarUint8Array(decoder));
this.rightClockDecoder = new decoding__namespace.IntDiffOptRleDecoder(decoding__namespace.readVarUint8Array(decoder));
this.infoDecoder = new decoding__namespace.RleDecoder(decoding__namespace.readVarUint8Array(decoder), decoding__namespace.readUint8);
this.stringDecoder = new decoding__namespace.StringDecoder(decoding__namespace.readVarUint8Array(decoder));
this.parentInfoDecoder = new decoding__namespace.RleDecoder(decoding__namespace.readVarUint8Array(decoder), decoding__namespace.readUint8);
this.typeRefDecoder = new decoding__namespace.UintOptRleDecoder(decoding__namespace.readVarUint8Array(decoder));
this.lenDecoder = new decoding__namespace.UintOptRleDecoder(decoding__namespace.readVarUint8Array(decoder));
}
/**
* @return {ID}
*/
readLeftID () {
return new ID(this.clientDecoder.read(), this.leftClockDecoder.read())
}
/**
* @return {ID}
*/
readRightID () {
return new ID(this.clientDecoder.read(), this.rightClockDecoder.read())
}
/**
* Read the next client id.
* Use this in favor of readID whenever possible to reduce the number of objects created.
*/
readClient () {
return this.clientDecoder.read()
}
/**
* @return {number} info An unsigned 8-bit integer
*/
readInfo () {
return /** @type {number} */ (this.infoDecoder.read())
}
/**
* @return {string}
*/
readString () {
return this.stringDecoder.read()
}
/**
* @return {boolean}
*/
readParentInfo () {
return this.parentInfoDecoder.read() === 1
}
/**
* @return {number} An unsigned 8-bit integer
*/
readTypeRef () {
return this.typeRefDecoder.read()
}
/**
* Write len of a struct - well suited for Opt RLE encoder.
*
* @return {number}
*/
readLen () {
return this.lenDecoder.read()
}
/**
* @return {any}
*/
readAny () {
return decoding__namespace.readAny(this.restDecoder)
}
/**
* @return {Uint8Array}
*/
readBuf () {
return decoding__namespace.readVarUint8Array(this.restDecoder)
}
/**
* This is mainly here for legacy purposes.
*
* Initial we incoded objects using JSON. Now we use the much faster lib0/any-encoder. This method mainly exists for legacy purposes for the v1 encoder.
*
* @return {any}
*/
readJSON () {
return decoding__namespace.readAny(this.restDecoder)
}
/**
* @return {string}
*/
readKey () {
const keyClock = this.keyClockDecoder.read();
if (keyClock < this.keys.length) {
return this.keys[keyClock]
} else {
const key = this.stringDecoder.read();
this.keys.push(key);
return key
}
}
}
class DSEncoderV1 {
constructor () {
this.restEncoder = encoding__namespace.createEncoder();
}
toUint8Array () {
return encoding__namespace.toUint8Array(this.restEncoder)
}
resetDsCurVal () {
// nop
}
/**
* @param {number} clock
*/
writeDsClock (clock) {
encoding__namespace.writeVarUint(this.restEncoder, clock);
}
/**
* @param {number} len
*/
writeDsLen (len) {
encoding__namespace.writeVarUint(this.restEncoder, len);
}
}
class UpdateEncoderV1 extends DSEncoderV1 {
/**
* @param {ID} id
*/
writeLeftID (id) {
encoding__namespace.writeVarUint(this.restEncoder, id.client);
encoding__namespace.writeVarUint(this.restEncoder, id.clock);
}
/**
* @param {ID} id
*/
writeRightID (id) {
encoding__namespace.writeVarUint(this.restEncoder, id.client);
encoding__namespace.writeVarUint(this.restEncoder, id.clock);
}
/**
* Use writeClient and writeClock instead of writeID if possible.
* @param {number} client
*/
writeClient (client) {
encoding__namespace.writeVarUint(this.restEncoder, client);
}
/**
* @param {number} info An unsigned 8-bit integer
*/
writeInfo (info) {
encoding__namespace.writeUint8(this.restEncoder, info);
}
/**
* @param {string} s
*/
writeString (s) {
encoding__namespace.writeVarString(this.restEncoder, s);
}
/**
* @param {boolean} isYKey
*/
writeParentInfo (isYKey) {
encoding__namespace.writeVarUint(this.restEncoder, isYKey ? 1 : 0);
}
/**
* @param {number} info An unsigned 8-bit integer
*/
writeTypeRef (info) {
encoding__namespace.writeVarUint(this.restEncoder, info);
}
/**
* Write len of a struct - well suited for Opt RLE encoder.
*
* @param {number} len
*/
writeLen (len) {
encoding__namespace.writeVarUint(this.restEncoder, len);
}
/**
* @param {any} any
*/
writeAny (any) {
encoding__namespace.writeAny(this.restEncoder, any);
}
/**
* @param {Uint8Array} buf
*/
writeBuf (buf) {
encoding__namespace.writeVarUint8Array(this.restEncoder, buf);
}
/**
* @param {any} embed
*/
writeJSON (embed) {
encoding__namespace.writeVarString(this.restEncoder, JSON.stringify(embed));
}
/**
* @param {string} key
*/
writeKey (key) {
encoding__namespace.writeVarString(this.restEncoder, key);
}
}
class DSEncoderV2 {
constructor () {
this.restEncoder = encoding__namespace.createEncoder(); // encodes all the rest / non-optimized
this.dsCurrVal = 0;
}
toUint8Array () {
return encoding__namespace.toUint8Array(this.restEncoder)
}
resetDsCurVal () {
this.dsCurrVal = 0;
}
/**
* @param {number} clock
*/
writeDsClock (clock) {
const diff = clock - this.dsCurrVal;
this.dsCurrVal = clock;
encoding__namespace.writeVarUint(this.restEncoder, diff);
}
/**
* @param {number} len
*/
writeDsLen (len) {
if (len === 0) {
error__namespace.unexpectedCase();
}
encoding__namespace.writeVarUint(this.restEncoder, len - 1);
this.dsCurrVal += len;
}
}
class UpdateEncoderV2 extends DSEncoderV2 {
constructor () {
super();
/**
* @type {Map<string,number>}
*/
this.keyMap = new Map();
/**
* Refers to the next unique key-identifier to me used.
* See writeKey method for more information.
*
* @type {number}
*/
this.keyClock = 0;
this.keyClockEncoder = new encoding__namespace.IntDiffOptRleEncoder();
this.clientEncoder = new encoding__namespace.UintOptRleEncoder();
this.leftClockEncoder = new encoding__namespace.IntDiffOptRleEncoder();
this.rightClockEncoder = new encoding__namespace.IntDiffOptRleEncoder();
this.infoEncoder = new encoding__namespace.RleEncoder(encoding__namespace.writeUint8);
this.stringEncoder = new encoding__namespace.StringEncoder();
this.parentInfoEncoder = new encoding__namespace.RleEncoder(encoding__namespace.writeUint8);
this.typeRefEncoder = new encoding__namespace.UintOptRleEncoder();
this.lenEncoder = new encoding__namespace.UintOptRleEncoder();
}
toUint8Array () {
const encoder = encoding__namespace.createEncoder();
encoding__namespace.writeVarUint(encoder, 0); // this is a feature flag that we might use in the future
encoding__namespace.writeVarUint8Array(encoder, this.keyClockEncoder.toUint8Array());
encoding__namespace.writeVarUint8Array(encoder, this.clientEncoder.toUint8Array());
encoding__namespace.writeVarUint8Array(encoder, this.leftClockEncoder.toUint8Array());
encoding__namespace.writeVarUint8Array(encoder, this.rightClockEncoder.toUint8Array());
encoding__namespace.writeVarUint8Array(encoder, encoding__namespace.toUint8Array(this.infoEncoder));
encoding__namespace.writeVarUint8Array(encoder, this.stringEncoder.toUint8Array());
encoding__namespace.writeVarUint8Array(encoder, encoding__namespace.toUint8Array(this.parentInfoEncoder));
encoding__namespace.writeVarUint8Array(encoder, this.typeRefEncoder.toUint8Array());
encoding__namespace.writeVarUint8Array(encoder, this.lenEncoder.toUint8Array());
// @note The rest encoder is appended! (note the missing var)
encoding__namespace.writeUint8Array(encoder, encoding__namespace.toUint8Array(this.restEncoder));
return encoding__namespace.toUint8Array(encoder)
}
/**
* @param {ID} id
*/
writeLeftID (id) {
this.clientEncoder.write(id.client);
this.leftClockEncoder.write(id.clock);
}
/**
* @param {ID} id
*/
writeRightID (id) {
this.clientEncoder.write(id.client);
this.rightClockEncoder.write(id.clock);
}
/**
* @param {number} client
*/
writeClient (client) {
this.clientEncoder.write(client);
}
/**
* @param {number} info An unsigned 8-bit integer
*/
writeInfo (info) {
this.infoEncoder.write(info);
}
/**
* @param {string} s
*/
writeString (s) {
this.stringEncoder.write(s);
}
/**
* @param {boolean} isYKey
*/
writeParentInfo (isYKey) {
this.parentInfoEncoder.write(isYKey ? 1 : 0);
}
/**
* @param {number} info An unsigned 8-bit integer
*/
writeTypeRef (info) {
this.typeRefEncoder.write(info);
}
/**
* Write len of a struct - well suited for Opt RLE encoder.
*
* @param {number} len
*/
writeLen (len) {
this.lenEncoder.write(len);
}
/**
* @param {any} any
*/
writeAny (any) {
encoding__namespace.writeAny(this.restEncoder, any);
}
/**
* @param {Uint8Array} buf
*/
writeBuf (buf) {
encoding__namespace.writeVarUint8Array(this.restEncoder, buf);
}
/**
* This is mainly here for legacy purposes.
*
* Initial we incoded objects using JSON. Now we use the much faster lib0/any-encoder. This method mainly exists for legacy purposes for the v1 encoder.
*
* @param {any} embed
*/
writeJSON (embed) {
encoding__namespace.writeAny(this.restEncoder, embed);
}
/**
* Property keys are often reused. For example, in y-prosemirror the key `bold` might
* occur very often. For a 3d application, the key `position` might occur very often.
*
* We cache these keys in a Map and refer to them via a unique number.
*
* @param {string} key
*/
writeKey (key) {
const clock = this.keyMap.get(key);
if (clock === undefined) {
/**
* @todo uncomment to introduce this feature finally
*
* Background. The ContentFormat object was always encoded using writeKey, but the decoder used to use readString.
* Furthermore, I forgot to set the keyclock. So everything was working fine.
*
* However, this feature here is basically useless as it is not being used (it actually only consumes extra memory).
*
* I don't know yet how to reintroduce this feature..
*
* Older clients won't be able to read updates when we reintroduce this feature. So this should probably be done using a flag.
*
*/
// this.keyMap.set(key, this.keyClock)
this.keyClockEncoder.write(this.keyClock++);
this.stringEncoder.write(key);
} else {
this.keyClockEncoder.write(clock);
}
}
}
/**
* @module encoding
*/
/*
* We use the first five bits in the info flag for determining the type of the struct.
*
* 0: GC
* 1: Item with Deleted content
* 2: Item with JSON content
* 3: Item with Binary content
* 4: Item with String content
* 5: Item with Embed content (for richtext content)
* 6: Item with Format content (a formatting marker for richtext content)
* 7: Item with Type
*/
/**
* @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
* @param {Array<GC|Item>} structs All structs by `client`
* @param {number} client
* @param {number} clock write structs starting with `ID(client,clock)`
*
* @function
*/
const writeStructs = (encoder, structs, client, clock) => {
// write first id
clock = math__namespace.max(clock, structs[0].id.clock); // make sure the first id exists
const startNewStructs = findIndexSS(structs, clock);
// write # encoded structs
encoding__namespace.writeVarUint(encoder.restEncoder, structs.length - startNewStructs);
encoder.writeClient(client);
encoding__namespace.writeVarUint(encoder.restEncoder, clock);
const firstStruct = structs[startNewStructs];
// write first struct with an offset
firstStruct.write(encoder, clock - firstStruct.id.clock);
for (let i = startNewStructs + 1; i < structs.length; i++) {
structs[i].write(encoder, 0);
}
};
/**
* @param {UpdateEncoderV1 | UpdateEncoderV2} encoder
* @param {StructStore} store
* @param {Map<number,number>} _sm
*
* @private
* @function
*/
const writeClientsStructs = (encoder, store, _sm) => {
// we filter all valid _sm entries into sm
const sm = new Map();
_sm.forEach((clock, client) => {
// only write if new structs are available
if (getState(store, client) > clock) {
sm.set(client, clock);
}
});
getStateVector(store).forEach((_clock, client) => {
if (!_sm.has(client)) {
sm.set(client, 0);
}
});
// write # states that were updated
encoding__namespace.writeVarUint(encoder.restEncoder, sm.size);
// Write items with higher client ids first
// This heavily improves the conflict algorithm.
array__namespace.from(sm.entries()).sort((a, b) => b[0] - a[0]).forEach(([client, clock]) => {
writeStructs(encoder, /** @type {Array<GC|Item>} */ (store.clients.get(client)), client, clock);
});
};
/**
* @param {UpdateDecoderV1 | UpdateDecoderV2} decoder The decoder object to read data from.
* @param {Doc} doc
* @return {Map<number, { i: number, refs: Array<Item | GC> }>}
*
* @private
* @function
*/
const readClientsStructRefs = (decoder, doc) => {
/**
* @type {Map<number, { i: number, refs: Array<Item | GC> }>}
*/
const clientRefs = map__namespace.create();
const numOfStateUpdates = decoding__namespace.readVarUint(decoder.restDecoder);
for (let i = 0; i < numOfStateUpdates; i++) {
const numberOfStructs = decoding__namespace.readVarUint(decoder.restDecoder);
/**
* @type {Array<GC|Item>}
*/
const refs = new Array(numberOfStructs);
const client = decoder.readClient();
let clock = decoding__namespace.readVarUint(decoder.restDecoder);
// const start = performance.now()
clientRefs.set(client, { i: 0, refs });
for (let i = 0; i < numberOfStructs; i++) {
const info = decoder.readInfo();
switch (binary__namespace.BITS5 & info) {
case 0: { // GC
const len = decoder.readLen();
refs[i] = new GC(createID(client, clock), len);
clock += len;
break
}
case 10: { // Skip Struct (nothing to apply)
// @todo we could reduce the amount of checks by adding Skip struct to clientRefs so we know that something is missing.
const len = decoding__namespace.readVarUint(decoder.restDecoder);
refs[i] = new Skip(createID(client, clock), len);
clock += len;
break
}
default: { // Item with content
/**
* The optimized implementation doesn't use any variables because inlining variables is faster.
* Below a non-optimized version is shown that implements the basic algorithm with
* a few comments
*/
const cantCopyParentInfo = (info & (binary__namespace.BIT7 | binary__namespace.BIT8)) === 0;
// If parent = null and neither left nor right are defined, then we know that `parent` is child of `y`
// and we read the next string as parentYKey.
// It indicates how we store/retrieve parent from `y.share`
// @type {string|null}
const struct = new Item(
createID(client, clock),
null, // left
(info & binary__namespace.BIT8) === binary__namespace.BIT8 ? decoder.readLeftID() : null, // origin
null, // right
(info & binary__namespace.BIT7) === binary__namespace.BIT7 ? decoder.readRightID() : null, // right origin
cantCopyParentInfo ? (decoder.readParentInfo() ? doc.get(decoder.readString()) : decoder.readLeftID()) : null, // parent
cantCopyParentInfo && (info & binary__namespace.BIT6) === binary__namespace.BIT6 ? decoder.readString() : null, // parentSub
readItemContent(decoder, info) // item content
);
/* A non-optimized implementation of the above algorithm:
// The item that was originally to the left of this item.
const origin = (info & binary.BIT8) === binary.BIT8 ? decoder.readLeftID() : null
// The item that was originally to the right of this item.
const rightOrigin = (info & binary.BIT7) === binary.BIT7 ? decoder.readRightID() : null
const cantCopyParentInfo = (info & (binary.BIT7 | binary.BIT8)) === 0
const hasParentYKey = cantCopyParentInfo ? decoder.readParentInfo() : false
// If parent = null and neither left nor right are defined, then we know that `parent` is child of `y`
// and we read the next string as parentYKey.
// It indicates how we store/retrieve parent from `y.share`
// @type {string|null}
const parentYKey = cantCopyParentInfo && hasParentYKey ? decoder.readString() : null
const struct = new Item(
createID(client, clock),
null, // left
origin, // origin
null, // right
rightOrigin, // right origin
cantCopyParentInfo && !hasParentYKey ? decoder.readLeftID() : (parentYKey !== null ? doc.get(parentYKey) : null), // parent
cantCopyParentInfo && (info & binary.BIT6) === binary.BIT6 ? decoder.readString() : null, // parentSub
readItemContent(decoder, info) // item content
)
*/
refs[i] = struct;
clock += struct.length;
}
}
}
// console.log('time to read: ', performance.now() - start) // @todo remove
}
return clientRefs
};
/**
* Resume computing structs generated by struct readers.
*
* While there is something to do, we integrate structs in this order
* 1. top element on stack, if stack is not empty
* 2. next element from current struct reader (if empty, use next struct reader)
*
* If struct causally depends on another struct (ref.missing), we put next reader of
* `ref.id.client` on top of stack.
*
* At some point we find a struct that has no causal dependencies,
* then we start emptying the stack.
*
* It is not possible to have circles: i.e. struct1 (from client1) depends on struct2 (from client2)
* depends on struct3 (from client1). Therefore the max stack size is equal to `structReaders.length`.
*
* This method is implemented in a way so that we can resume computation if this update
* causally depends on another update.
*
* @param {Transaction} transaction
* @param {StructStore} store
* @param {Map<number, { i: number, refs: (GC | Item)[] }>} clientsStructRefs
* @return { null | { update: Uint8Array, missing: Map<number,number> } }
*
* @private
* @function
*/
const integrateStructs = (transaction, store, clientsStructRefs) => {
/**
* @type {Array<Item | GC>}
*/
const stack = [];
// sort them so that we take the higher id first, in case of conflicts the lower id will probably not conflict with the id from the higher user.
let clientsStructRefsIds = array__namespace.from(clientsStructRefs.keys()).sort((a, b) => a - b);
if (clientsStructRefsIds.length === 0) {
return null
}
const getNextStructTarget = () => {
if (clientsStructRefsIds.length === 0) {
return null
}
let nextStructsTarget = /** @type {{i:number,refs:Array<GC|Item>}} */ (clientsStructRefs.get(clientsStructRefsIds[clientsStructRefsIds.length - 1]));
while (nextStructsTarget.refs.length === nextStructsTarget.i) {
clientsStructRefsIds.pop();
if (clientsStructRefsIds.length > 0) {
nextStructsTarget = /** @type {{i:number,refs:Array<GC|Item>}} */ (clientsStructRefs.get(clientsStructRefsIds[clientsStructRefsIds.length - 1]));
} else {
return null
}
}
return nextStructsTarget
};
let curStructsTarget = getNextStructTarget();
if (curStructsTarget === null) {
return null
}
/**
* @type {StructStore}
*/
const restStructs = new StructStore();
const missingSV = new Map();
/**
* @param {number} client
* @param {number} clock
*/
const updateMissingSv = (client, clock) => {
const mclock = missingSV.get(client);
if (mclock == null || mclock > clock) {
missingSV.set(client, clock);
}
};
/**
* @type {GC|Item}
*/
let stackHead = /** @type {any} */ (curStructsTarget).refs[/** @type {any} */ (curStructsTarget).i++];
// caching the state because it is used very often
const state = new Map();
const addStackToRestSS = () => {
for (const item of stack) {
const client = item.id.client;
const inapplicableItems = clientsStructRefs.get(client);
if (inapplicableItems) {
// decrement because we weren't able to apply previous operation
inapplicableItems.i--;
restStructs.clients.set(client, inapplicableItems.refs.slice(inapplicableItems.i));
clientsStructRefs.delete(client);
inapplicableItems.i = 0;
inapplicableItems.refs = [];
} else {
// item was the last item on clientsStructRefs and the field was already cleared. Add item to restStructs and continue
restStructs.clients.set(client, [item]);
}
// remove client from clientsStructRefsIds to prevent users from applying the same update again
clientsStructRefsIds = clientsStructRefsIds.filter(c => c !== client);
}
stack.length = 0;
};
// iterate over all struct readers until we are done
while (true) {
if (stackHead.constructor !== Skip) {
const localClock = map__namespace.setIfUndefined(state, stackHead.id.client, () => getState(store, stackHead.id.client));
const offset = localClock - stackHead.id.clock;
if (offset < 0) {
// update from the same client is missing
stack.push(stackHead);
updateMissingSv(stackHead.id.client, stackHead.id.clock - 1);
// hid a dead wall, add all items from stack to restSS
addStackToRestSS();
} else {
const missing = stackHead.getMissing(transaction, store);
if (missing !== null) {
stack.push(stackHead);
// get the struct reader that has the missing struct
/**
* @type {{ refs: Array<GC|Item>, i: number }}
*/
const struc