json-crdt-repo
Version:
JSON CRDT server and syncing local-first browser client
1,022 lines (1,021 loc) • 46.5 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.LevelLocalRepo = void 0;
const tslib_1 = require("tslib");
const rxjs_1 = require("rxjs");
const operators_1 = require("rxjs/operators");
const Writer_1 = require("@jsonjoy.com/util/lib/buffers/Writer");
const cbor_1 = require("@jsonjoy.com/json-pack/lib/codecs/cbor");
const json_crdt_1 = require("json-joy/lib/json-crdt");
const deepEqual_1 = require("@jsonjoy.com/util/lib/json-equal/deepEqual");
const constants_1 = require("json-joy/lib/json-crdt-patch/constants");
const once_1 = require("thingies/lib/once");
const timeout_1 = require("thingies/lib/timeout");
const pubsub_1 = require("../../pubsub");
/**
* @todo
*
* 1. Implement pull loop, when WebSocket subscription cannot be established.
*/
var Defaults;
(function (Defaults) {
/**
* The root of the block repository.
*
* ```
* b!<collection>!<id>!
* ```
*/
Defaults["BlockRepoRoot"] = "b";
/**
* The root of the key-space where items are marked as "dirty" and need sync.
*
* ```
* s!b!<collection>!<id>
* ```
*/
// eslint-disable-next-line
Defaults["SyncRoot"] = "s";
/**
* The metadata of the block.
*
* ```
* b!<collection>!<id>!k!x
* ```
*/
Defaults["Metadata"] = "k!x";
/**
* The state of the latest known server-side model.
*
* ```
* b!<collection>!<id>!k!m
* ```
*/
Defaults["Model"] = "k!m";
/**
* List of frontier patches.
*
* ```
* b!<collection>!<id>!f!<time>
* ```
*/
Defaults["Frontier"] = "f";
/**
* List of batches verified by the server.
*
* ```
* b!<collection>!<id>!h!<seq>
* ```
*/
Defaults["Batches"] = "h";
/**
* List of snapshots.
*
* ```
* b!<collection>!<id>!s!<seq>
* ```
*/
// eslint-disable-next-line
Defaults["Snapshots"] = "s";
/**
* The default length of the history, if `hist` metadata property not
* specified.
*/
Defaults[Defaults["HistoryLength"] = 100] = "HistoryLength";
})(Defaults || (Defaults = {}));
let LevelLocalRepo = (() => {
var _a;
let _instanceExtraInitializers = [];
let _stop_decorators;
return _a = class LevelLocalRepo {
constructor(opts) {
this.opts = (tslib_1.__runInitializers(this, _instanceExtraInitializers), opts);
this.codec = new cbor_1.CborJsonValueCodec(new Writer_1.Writer(1024 * 16));
this._conSub = undefined;
this._stopped = false;
this._remoteSyncLoopActive = false;
this._subs = {};
this.kv = opts.kv;
this.locks = opts.locks;
this.sid = opts.sid;
this.connected$ = opts.connected$ ?? new rxjs_1.BehaviorSubject(true);
this.pubsub = opts.pubsub ?? (0, pubsub_1.pubsub)('level-local-repo');
this.cipher = opts.cipher;
this._conSub = this.connected$.subscribe((connected) => {
if (connected && !this._remoteSyncLoopActive)
this.runRemoteSyncLoop();
});
}
stop() {
this._remoteSyncLoopActive = false;
clearTimeout(this._remoteSyncDelayTimer);
this._stopped = true;
this._conSub?.unsubscribe();
}
// eslint-disable-next-line
async encrypt(blob, zip) {
// if (zip) blob = await gzip(blob);
// if (this.cipher) blob = await this.cipher.encrypt(blob);
return blob;
}
// eslint-disable-next-line
async decrypt(blob, zip) {
// if (this.cipher) blob = await this.cipher.decrypt(blob);
// if (zip) blob = await ungzip(blob);
return blob;
}
async encode(value, zip) {
const encoded = this.codec.encoder.encode(value);
const encrypted = await this.encrypt(encoded, zip);
return encrypted;
}
async decode(blob, zip) {
const decrypted = await this.decrypt(blob, zip);
const decoded = this.codec.decoder.decode(decrypted);
return decoded;
}
/** @todo Encrypt collection and key. */
async blockKeyBase(id) {
return Defaults.BlockRepoRoot + '!' + id.join('!') + '!';
}
frontierKeyBase(blockKeyBase) {
return blockKeyBase + Defaults.Frontier + '!';
}
frontierKey(blockKeyBase, time) {
const timeFormatted = time.toString(36).padStart(8, '0');
return this.frontierKeyBase(blockKeyBase) + timeFormatted;
}
batchKeyBase(blockKeyBase) {
return blockKeyBase + Defaults.Batches + '!';
}
batchKey(blockKeyBase, seq) {
const seqFormatted = seq.toString(36).padStart(8, '0');
return this.batchKeyBase(blockKeyBase) + seqFormatted;
}
snapshotKeyBase(blockKeyBase) {
return blockKeyBase + Defaults.Snapshots + '!';
}
snapshotKey(blockKeyBase, seq) {
const seqFormatted = seq.toString(36).padStart(8, '0');
return this.snapshotKeyBase(blockKeyBase) + seqFormatted;
}
syncKey(keyBase) {
return Defaults.SyncRoot + '!' + keyBase;
}
async _exists(keyBase) {
const metaKey = keyBase + Defaults.Metadata;
const exists = (await this.kv.keys({ gte: metaKey, lte: metaKey, limit: 1 }).all()).length > 0;
return exists;
}
_modelWrOp(keyBase, model) {
return this.encode(model, true).then((value) => ({
type: 'put',
key: keyBase + Defaults.Model,
value,
}));
}
_metaWrOp(keyBase, meta) {
return this.encode(meta, false).then((value) => ({
type: 'put',
key: keyBase + Defaults.Metadata,
value,
}));
}
async _modelWrOps(keyBase, model, meta) {
const ops = [this._modelWrOp(keyBase, model)];
if (meta)
ops.push(this._metaWrOp(keyBase, meta));
return Promise.all(ops);
}
async _wrModel(keyBase, model, meta) {
const ops = await this._modelWrOps(keyBase, model, meta);
await this.kv.batch(ops);
}
async load(id) {
const blockId = id.join('/');
const res = await this.opts.rpc.read(blockId);
const block = res.block;
const snapshot = block.snapshot;
const seq = snapshot.seq;
const sid = this.sid;
const model = json_crdt_1.Model.load(snapshot.blob, sid);
for (const batch of block.tip)
for (const patch of batch.patches)
model.applyPatch(json_crdt_1.Patch.fromBinary(patch.blob));
const keyBase = await this.blockKeyBase(id);
const metaKey = keyBase + Defaults.Metadata;
const meta = {
time: 0,
seq,
};
const modelBlob = model.toBinary();
const [metaBlob, modelTupleBlob] = await Promise.all([this.encode(meta, false), this.encode(modelBlob, true)]);
const ops = [
{
type: 'put',
key: metaKey,
value: metaBlob,
},
{
type: 'put',
key: keyBase + Defaults.Model,
value: modelTupleBlob,
},
];
await this.lockBlock(keyBase, async () => {
const exists = await this._exists(keyBase);
if (exists)
throw new Error('EXISTS');
await this.kv.batch(ops);
});
// TODO: Emit something here...
// this.pubsub.pub(['pull', {id, batches: [], snapshot: {seq, blob: modelBlob}}])
return { model, cursor: seq };
}
async readMeta(keyBase) {
const metaKey = keyBase + Defaults.Metadata;
const blob = await this.kv.get(metaKey);
const meta = (await this.decode(blob, false));
return meta;
}
async readModel0(keyBase) {
const modelKey = keyBase + Defaults.Model;
const blob = await this.kv.get(modelKey);
const decoded = (await this.decode(blob, true));
return decoded;
}
async readModel(keyBase) {
try {
const blob = await this.readModel0(keyBase);
const model = json_crdt_1.Model.load(blob, this.sid);
return model;
}
catch (error) {
if (!!error && typeof error === 'object' && error.code === 'LEVEL_NOT_FOUND')
throw new Error('NOT_FOUND');
throw error;
}
}
async *readFrontierBlobs0(keyBase) {
const gt = this.frontierKeyBase(keyBase);
const lt = gt + '~';
for await (const [key, buf] of this.kv.iterator({ gt, lt })) {
/** @todo Remove this conversion once json-pack supports Buffers. */
const uint8 = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
yield [key, uint8];
}
}
async readFrontier0(keyBase) {
const patches = [];
for await (const [, blob] of this.readFrontierBlobs0(keyBase)) {
const patch = json_crdt_1.Patch.fromBinary(blob);
patches.push(patch);
}
return patches;
}
async readFrontierTip(keyBase) {
const frontierBase = this.frontierKeyBase(keyBase);
const lte = frontierBase + '~';
for await (const blob of this.kv.values({ lte, limit: 1, reverse: true }))
return json_crdt_1.Patch.fromBinary(await this.decrypt(blob, false));
return;
}
async lockBlock(keyBase, fn) {
return await this.locks.lock(keyBase, 500, 500)(fn);
}
async lockBlockForSync(keyBase, fn) {
const key = 's!' + keyBase;
return await this.locks.lock(key, 2000, 3000)(fn);
}
isBlockLockedForSync(keyBase) {
const key = 's!' + keyBase;
return this.locks.isLocked(key);
}
// --------------------------------------------------- Remote synchronization
async markDirty(keyBase, id) {
const key = this.syncKey(keyBase);
const record = [id, Date.now()];
const blob = await this.encode(record, false);
await this.kv.put(key, blob);
}
async markDirtyAndSync(keyBase, id) {
if (this._stopped)
return false;
this.markDirty(keyBase, id).catch((error) => {
if (this._stopped)
return;
this.opts.onSyncError?.(error);
});
try {
return await this.push(keyBase, id, true);
}
catch (error) {
if (typeof error === 'object' && error && error.message === 'Not Found')
return false;
else if (error instanceof Error && error.message === 'DISCONNECTED')
return false;
throw error;
}
}
remoteTimeout() {
return this.opts.remoteTimeout ?? 5000;
}
/**
* Pushes to remote.
* @param id Block ID.
* @param pull Whether to pull if there are no patches to push.
*/
async push(keyBase, id, doPull = false) {
if (this._stopped)
return false;
if (!this.connected$.getValue())
throw new Error('DISCONNECTED');
const remote = this.opts.rpc;
const remoteId = id.join('/');
const patches = [];
const ops = [{ type: 'del', key: this.syncKey(keyBase) }];
for await (const [key, blob] of this.readFrontierBlobs0(keyBase)) {
ops.push({ type: 'del', key });
patches.push({ blob });
}
if (!patches.length) {
const meta = await this.readMeta(keyBase);
if (meta.seq === -1) {
remote.create(remoteId).catch((error) => {
this.opts.onSyncError?.(error);
});
}
if (doPull) {
await this.pull(id);
}
return false;
}
// TODO: handle case when this times out, but actually succeeds, so on re-sync it handles the case when the block is already synced.
return await this.lockBlock(keyBase, async () => {
const TIMEOUT = this.remoteTimeout();
const startTime = Date.now();
const assertTimeout = () => {
if (Date.now() - startTime > TIMEOUT)
throw new Error('TIMEOUT');
if (this._stopped)
throw new Error('STOPPED');
};
return await (0, timeout_1.timeout)(TIMEOUT, async () => {
const read = await Promise.all([this.readModel(keyBase), this.readMeta(keyBase)]);
assertTimeout();
let model = read[0];
const meta = read[1];
// TODO: Track some meta to avoid unnecessary syncs.
// if (Date.now() - meta.ts < 1000) return false;
const hist = !!meta.hist;
const lastKnownSeq = meta.seq;
try {
const response = await remote.update(remoteId, { patches }, lastKnownSeq);
assertTimeout();
// TODO: handle case when block is deleted on the server.
// Process pull
const merge = []; // List of merge patches to emit over pubsub.
const pull = response.pull;
if (pull) {
const snapshot = pull.snapshot;
const batches = pull.batches;
if (snapshot) {
model = json_crdt_1.Model.load(snapshot.blob, this.sid);
if (hist) {
ops.push({
type: 'put',
key: this.snapshotKey(keyBase, snapshot.seq),
value: await this.encode(snapshot, true),
});
assertTimeout();
}
}
if (batches) {
for (const b of batches) {
const patches = b.patches;
for (const patch of patches) {
const blob = patch.blob;
merge.push(blob);
model.applyPatch(json_crdt_1.Patch.fromBinary(blob));
}
if (hist) {
ops.push({
type: 'put',
key: this.batchKey(keyBase, b.seq),
value: await this.encode(b, false),
});
assertTimeout();
}
}
}
}
// Process the latest batch
for (const patch of patches) {
const blob = patch.blob;
merge.push(blob);
model.applyPatch(json_crdt_1.Patch.fromBinary(blob));
}
const batch = { ...response.batch, patches };
const seq = batch.seq;
if (hist) {
ops.push({
type: 'put',
key: this.batchKey(keyBase, seq),
value: await this.encode(batch, false),
});
assertTimeout();
}
// Process the model and metadata
delete meta.syncFailures;
meta.syncTs = Date.now();
meta.time = model.clock.time - 1;
meta.seq = seq;
const modelOps = await this._modelWrOps(keyBase, model.toBinary(), meta);
assertTimeout();
ops.push(...modelOps);
await this.kv.batch(ops);
if (merge.length) {
this.pubsub.pub({ type: 'merge', id, patches: merge, seq: seq });
}
return true;
}
catch (error) {
// Store fails and time.
meta.syncTs = Date.now();
meta.syncFailures = (meta.syncFailures ?? 0) + 1;
try {
const op = await this._metaWrOp(keyBase, meta);
await this.kv.batch([op]);
}
catch (err) {
this.opts.onSyncError?.(err);
}
throw error;
}
});
});
}
/**
* Iterates over all blocks marked as dirty.
*/
async *listDirty() {
const gt = Defaults.SyncRoot;
const lt = Defaults.SyncRoot + '~';
for await (const blob of this.kv.values({ gt, lt }))
yield (await this.decode(blob, false));
}
async *listDirtyBatch(batchSize = 3) {
let batch = [];
for await (const record of this.listDirty()) {
batch.push(record);
if (batch.length >= batchSize) {
yield batch;
batch = [];
}
}
if (batch.length)
yield batch;
}
async syncItem(keyBase, id) {
try {
const meta = await this.readMeta(keyBase);
if (meta.syncFailures && meta.syncTs) {
const timeout = Math.min(1000 * Math.pow(2, meta.syncFailures), 30000);
const now = Date.now();
const shouldSkip = now - timeout < meta.syncTs;
if (shouldSkip)
return [id, false];
}
return await this.lockBlockForSync(keyBase, async () => {
const success = await this.push(keyBase, id, true);
return [id, success];
});
}
catch (error) {
return [id, false, error];
}
}
async remoteSyncAll() {
const resultList = [];
for await (const batch of this.listDirtyBatch()) {
if (this._stopped)
return resultList;
if (!this._remoteSyncLoopActive)
return resultList;
if (!this.connected$.getValue())
return resultList;
await Promise.all(batch.map(async (record) => {
const [id] = record;
const keyBase = await this.blockKeyBase(id);
const isLocked = this.isBlockLockedForSync(keyBase);
if (isLocked)
return;
const result = await this.syncItem(keyBase, id);
resultList.push(result);
}));
}
return resultList;
}
runRemoteSyncLoop() {
this._remoteSyncLoopActive = true;
if (!this.connected$.getValue()) {
this._remoteSyncLoopActive = false;
return;
}
this.remoteSyncAll()
.catch((error) => {
this.opts.onSyncError?.(error);
})
.finally(() => {
if (!this._remoteSyncLoopActive)
return;
if (!this.connected$.getValue()) {
this._remoteSyncLoopActive = false;
return;
}
this._remoteSyncDelayTimer = setTimeout(() => {
if (!this._remoteSyncLoopActive)
return;
this.runRemoteSyncLoop();
}, 2000);
});
}
/** ----------------------------------------------------- {@link LocalRepo} */
async create({ id, patches }) {
const keyBase = await this.blockKeyBase(id);
const meta = {
time: 0,
seq: -1,
};
const ops = [];
const model = json_crdt_1.Model.create(void 0, this.sid);
if (patches && patches.length) {
for (const patch of patches) {
const patchId = patch.getId();
if (!patchId)
throw new Error('PATCH_ID_MISSING');
model.applyPatch(patch);
const patchKey = this.frontierKey(keyBase, patchId.time);
const op = {
type: 'put',
key: patchKey,
value: await this.encrypt(patch.toBinary(), false),
};
ops.push(op);
}
}
ops.push(...(await this._modelWrOps(keyBase, model.toBinary(), meta)));
await this.lockBlock(keyBase, async () => {
const exists = await this._exists(keyBase);
if (exists)
throw new Error('EXISTS');
await this.kv.batch(ops);
});
const remote = this.markDirtyAndSync(keyBase, id)
.then(() => { })
.catch((error) => {
if (this._stopped)
return;
this.opts.onSyncError?.(error);
});
remote.catch(() => { });
return { model, remote };
}
async get({ id, remote }) {
try {
const { model, cursor } = await this._syncRead(id);
if (!model)
throw new Error('NOT_FOUND');
return { model, cursor };
}
catch (error) {
if (remote && error instanceof Error && error.message === 'NOT_FOUND')
return await this.load(id);
throw error;
}
}
async getIf(request) {
let get = false;
const keyBase = await this.blockKeyBase(request.id);
const meta = await this.readMeta(keyBase);
if (typeof request.cursor === 'number') {
if (request.cursor < meta.seq)
get = true;
}
if (!get && typeof request.time === 'number') {
if (request.time < meta.time)
get = true;
else {
const tip = await this.readFrontierTip(keyBase);
if (tip) {
const tipTime = tip.getId()?.time ?? 0;
if (request.time < tipTime + tip.span() - 1)
get = true;
}
}
}
if (!get)
return null;
const [model, frontier] = await Promise.all([this.readModel(keyBase), this.readFrontier0(keyBase)]);
model.applyBatch(frontier);
const cursor = meta.seq;
return { model, cursor };
}
async sync(req) {
const cursor = req.cursor;
const { id, patches } = req;
const isNewSession = cursor === undefined;
const isCreate = !!patches;
const isWrite = !!patches && patches.length !== 0;
if (isNewSession) {
if (isWrite) {
try {
return await this._syncCreate(req);
}
catch (error) {
if (error instanceof Error && error.message === 'EXISTS') {
return await this._syncMerge(req);
}
throw error;
}
}
else if (isCreate) {
try {
return await this._syncCreate(req);
}
catch (error) {
if (error instanceof Error && error.message === 'EXISTS')
// TODO: make sure reset does not happen, if models are the same.
// TODO: Check for `req.time` in `_syncRead`.
return await this._syncRead(id);
throw error;
}
}
else {
return await this._syncRead(id);
}
}
else
return await this._syncMerge(req);
}
async _syncCreate(req) {
const { remote } = await this.create(req);
const cursor = -1;
return { cursor, remote };
}
async _syncMerge(req) {
const { id, patches } = req;
let lastKnownTime = 0;
const reqTime = req.time;
if (typeof reqTime === 'number') {
lastKnownTime = reqTime;
const firstPatch = patches?.[0];
if (firstPatch?.getId()?.sid === constants_1.SESSION.GLOBAL)
lastKnownTime = firstPatch.getId().time + firstPatch.span() - 1;
}
else if (patches?.length) {
const firstPatchTime = patches?.[0]?.getId()?.time;
if (typeof firstPatchTime === 'number')
lastKnownTime = firstPatchTime - 1;
}
const keyBase = await this.blockKeyBase(id);
if (!patches || !patches.length)
throw new Error('EMPTY_BATCH');
const writtenPatches = [];
let cursor = -1;
// TODO: Check if `patches` need rebasing, if not, just merge.
// TODO: Return correct response.
// TODO: Check that remote state is in sync, too.
let needsReset = false;
const didPush = await this.lockBlock(keyBase, async () => {
const [tip, meta] = await Promise.all([this.readFrontierTip(keyBase), this.readMeta(keyBase)]);
let nextTick = meta.time + 1;
if (lastKnownTime < meta.time)
needsReset = true;
cursor = meta.seq;
if (tip) {
const tipTime = tip.getId()?.time ?? 0;
nextTick = tipTime + tip.span();
}
const ops = [];
const sid = this.sid;
const length = patches.length;
for (let i = 0; i < length; i++) {
const patch = patches[i];
const patchId = patch.getId();
if (!patchId)
throw new Error('PATCH_ID_MISSING');
const isSchemaPatch = patchId.sid === constants_1.SESSION.GLOBAL && patchId.time === 1;
if (isSchemaPatch) {
const patchAheadOfTip = patchId.time >= nextTick;
if (!patchAheadOfTip)
continue;
}
let rebased = patch;
if (patchId.sid === sid && nextTick > patchId.time) {
needsReset = true;
rebased = patch.rebase(nextTick);
nextTick = rebased.getId().time + rebased.span();
}
const id = rebased.getId();
const time = id.time;
const patchKey = this.frontierKey(keyBase, time);
const uint8 = await this.encrypt(rebased.toBinary(), false);
writtenPatches.push(uint8);
const op = { type: 'put', key: patchKey, value: uint8 };
ops.push(op);
}
if (writtenPatches.length) {
this.pubsub.pub({ type: 'rebase', id, patches: writtenPatches, session: req.session });
}
if (ops.length) {
await this.kv.batch(ops);
return true;
}
return false;
});
if (!didPush && !needsReset) {
const merge = await this.readFrontier0(keyBase);
return { cursor, merge };
}
const remote = this.markDirtyAndSync(keyBase, id)
.then(() => { })
.catch((error) => {
if (this._stopped)
return;
this.opts.onSyncError?.(error);
});
if (needsReset) {
const { cursor, model } = await this._syncRead0(keyBase);
return { cursor, model, remote };
}
return { cursor, remote };
}
async readLocal0(keyBase) {
const [model, meta, frontier] = await Promise.all([
this.readModel(keyBase),
this.readMeta(keyBase),
this.readFrontier0(keyBase),
]);
model.applyBatch(frontier);
const cursor = meta.seq;
return [model, cursor];
}
async _syncRead0(keyBase) {
const [model, cursor] = await this.readLocal0(keyBase);
return {
model,
cursor,
// TODO: `remote` should load the block from the server.
remote: Promise.resolve(),
};
}
async _syncRead(id) {
const keyBase = await this.blockKeyBase(id);
return this._syncRead0(keyBase);
}
async del(id) {
const keyBase = await this.blockKeyBase(id);
const frontierKeyBase = this.frontierKeyBase(keyBase);
const batchKeyBase = this.batchKeyBase(keyBase);
const snapshotKeyBase = this.snapshotKeyBase(keyBase);
const kv = this.kv;
await this.lockBlock(keyBase, async () => {
await kv.batch([
{ type: 'del', key: keyBase + Defaults.Metadata },
{ type: 'del', key: keyBase + Defaults.Model },
]);
this.pubsub.pub({ id, type: 'del' });
await Promise.all([
kv.clear({
gt: frontierKeyBase,
lt: frontierKeyBase + '~',
}),
kv.clear({
gt: batchKeyBase,
lt: batchKeyBase + '~',
}),
kv.clear({
gt: snapshotKeyBase,
lt: snapshotKeyBase + '~',
}),
]);
});
await this.opts.rpc.delete?.(id.join('/'));
}
/**
* Pull from remote.
*/
async pull(id) {
const keyBase = await this.blockKeyBase(id);
try {
const { model, meta } = await this.pullExisting(id, keyBase);
return { model, cursor: meta.seq };
}
catch (error) {
if (!!error && typeof error === 'object' && error.code === 'LEVEL_NOT_FOUND') {
const { model, meta } = await this.pullNew(id, keyBase);
return { model, cursor: meta.seq };
}
else if (error instanceof Error && error.message === 'CONCURRENCY') {
const [model, cursor] = await this.readLocal0(keyBase);
return { model, cursor };
}
throw error;
}
}
async pullNew(id, keyBase) {
const blockId = id.join('/');
const { block } = await this.opts.rpc.read(blockId);
const pubsub = this.pubsub;
return this.lockBlock(keyBase, async () => {
const exists = await this._exists(keyBase);
if (exists)
throw new Error('EXISTS');
const model = json_crdt_1.Model.load(block.snapshot.blob, this.sid);
let seq = block.snapshot.seq;
for (const batch of block.tip) {
if (batch.seq <= seq)
continue;
seq = batch.seq;
for (const patch of batch.patches)
model.applyPatch(json_crdt_1.Patch.fromBinary(patch.blob));
}
const meta = {
time: 0,
seq,
};
const blob = model.toBinary();
await this._wrModel(keyBase, blob, meta);
pubsub.pub({ type: 'reset', id, model: blob });
return { model, meta };
});
}
async pullExisting(id, keyBase) {
// TODO: try catching up using batches, if not possible, reset
// TODO: load batches to catch up with remote
const blockId = id.join('/');
const { seq } = await this.readMeta(keyBase);
const pull = await this.opts.rpc.pull(blockId, seq);
const nextSeq = pull.batches.length ? pull.batches[pull.batches.length - 1].seq : (pull.snapshot?.seq ?? seq);
const pubsub = this.pubsub;
return this.lockBlock(keyBase, async () => {
const [model, meta] = await Promise.all([this.readModel(keyBase), this.readMeta(keyBase)]);
const seq2 = meta.seq;
if (seq2 !== seq)
throw new Error('CONCURRENCY');
if (pull.snapshot) {
if (nextSeq > seq2) {
const model = json_crdt_1.Model.load(pull.snapshot.blob, this.sid);
for (const batch of pull.batches)
for (const patch of batch.patches)
model.applyPatch(json_crdt_1.Patch.fromBinary(patch.blob));
const modelBlob = model.toBinary();
meta.seq = nextSeq;
await this._wrModel(keyBase, modelBlob, meta);
pubsub.pub({ type: 'reset', id, model: modelBlob });
}
return { model, meta };
}
if (!model)
throw new Error('NO_MODEL');
const patches = [];
for (const batch of pull.batches)
for (const patch of batch.patches) {
model.applyPatch(json_crdt_1.Patch.fromBinary(patch.blob));
patches.push(patch.blob);
}
meta.seq = nextSeq;
await this._wrModel(keyBase, model.toBinary(), meta);
pubsub.pub({ type: 'merge', id, patches, seq: meta.seq });
return { model, meta };
});
}
change$(id) {
return (0, rxjs_1.defer)(() => {
const remoteSubscription = this._subRemote(id).subscribe(() => { });
return this.pubsub.bus$.pipe((0, operators_1.map)((msg) => {
switch (msg.type) {
case 'rebase': {
if (!(0, deepEqual_1.deepEqual)(id, msg.id))
return;
const rebase = [];
for (const blob of msg.patches)
rebase.push(json_crdt_1.Patch.fromBinary(blob));
const event = { rebase, session: msg.session };
return event;
}
case 'reset': {
if (!(0, deepEqual_1.deepEqual)(id, msg.id))
return;
const reset = json_crdt_1.Model.load(msg.model, this.sid);
const event = { reset };
return event;
}
case 'merge': {
if (!(0, deepEqual_1.deepEqual)(id, msg.id))
return;
const merge = [];
for (const blob of msg.patches) {
const patch = json_crdt_1.Patch.fromBinary(blob);
if (patch.getId()?.sid === constants_1.SESSION.GLOBAL)
continue;
merge.push(patch);
}
if (!merge.length)
return;
const event = { merge, cursor: msg.seq };
return event;
}
case 'del': {
if (!(0, deepEqual_1.deepEqual)(id, msg.id))
return;
const event = { del: true };
return event;
}
}
}), (0, operators_1.filter)((event) => !!event), (0, operators_1.finalize)(() => {
remoteSubscription.unsubscribe();
}), (0, operators_1.share)());
});
}
_subRemote(id) {
const blockId = id.join('/');
let sub = this._subs[blockId];
if (sub)
return sub;
const source = (0, rxjs_1.defer)(() => this.opts.rpc.listen(blockId).pipe((0, operators_1.switchMap)(async ({ event }) => {
switch (event[0]) {
case 'new':
await this.pull(id);
break;
case 'upd':
await this._onUpd(id, event[1].batch);
break;
case 'del':
await this.del(id);
break;
}
})));
sub = source.pipe((0, operators_1.catchError)(() => source), (0, operators_1.finalize)(() => {
delete this._subs[blockId];
}), (0, operators_1.share)());
this._subs[blockId] = sub;
return sub;
}
async _onUpd(id, batch) {
try {
const keyBase = await this.blockKeyBase(id);
const firstPatch = batch.patches[0];
if (!firstPatch)
return;
const firstPatchSid = json_crdt_1.Patch.fromBinary(firstPatch.blob).getId()?.sid;
if (firstPatchSid === this.sid)
return;
try {
const meta = await this.readMeta(keyBase);
const alreadySynced = meta.seq >= batch.seq;
if (alreadySynced)
return;
const needsPull = meta.seq + 1 < batch.seq;
if (needsPull) {
await this.pull(id);
return;
}
}
catch (error) {
if (!!error && typeof error === 'object' && error.code === 'LEVEL_NOT_FOUND') {
await this.pullNew(id, keyBase);
return;
}
throw error;
}
await this.lockBlock(keyBase, async () => {
const [model, meta] = await Promise.all([this.readModel(keyBase), this.readMeta(keyBase)]);
if (meta.seq + 1 !== batch.seq)
throw new Error('CONFLICT');
const patches = [];
for (const serverPatch of batch.patches) {
const patch = json_crdt_1.Patch.fromBinary(serverPatch.blob);
model.applyPatch(patch);
patches.push(serverPatch.blob);
}
meta.seq = batch.seq;
await this._wrModel(keyBase, model.toBinary(), meta);
this.pubsub.pub({ type: 'merge', id, patches, seq: meta.seq });
});
}
catch (error) {
this.opts.onSyncError?.(error);
}
}
},
(() => {
const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(null) : void 0;
_stop_decorators = [once_1.once];
tslib_1.__esDecorate(_a, null, _stop_decorators, { kind: "method", name: "stop", static: false, private: false, access: { has: obj => "stop" in obj, get: obj => obj.stop }, metadata: _metadata }, null, _instanceExtraInitializers);
if (_metadata) Object.defineProperty(_a, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
})(),
_a;
})();
exports.LevelLocalRepo = LevelLocalRepo;