json-crdt-server
Version:
JSON CRDT server and syncing local-first browser client
237 lines (236 loc) • 8.79 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.BlocksServices = void 0;
const tslib_1 = require("tslib");
const rpc_error_1 = require("rpc-error");
const MemoryStore_1 = require("./store/MemoryStore");
const json_crdt_1 = require("json-joy/lib/json-crdt");
const constants_1 = require("json-joy/lib/json-crdt-patch/constants");
const go_1 = require("thingies/lib/go");
const util_1 = require("./util");
const fs = tslib_1.__importStar(require("fs"));
const rxjs_1 = require("rxjs");
const validateBatch = (batch) => {
if (!batch || typeof batch !== 'object' || Array.isArray(batch))
throw rpc_error_1.RpcError.validation('INVALID_BATCH');
const { patches } = batch;
if (!Array.isArray(patches))
throw rpc_error_1.RpcError.validation('INVALID_PATCHES');
if (patches.length > 100)
throw rpc_error_1.RpcError.validation('TOO_MANY_PATCHES');
if (patches.length < 1)
throw rpc_error_1.RpcError.validation('TOO_FEW_PATCHES');
for (const patch of patches)
if (patch.blob.length > 20000)
throw rpc_error_1.RpcError.validation('PATCH_TOO_LARGE');
};
class BlocksServices {
constructor(services, store = new MemoryStore_1.MemoryStore(), opts = {
historyPerBlock: 10000,
historyCompactionDecision: (seq, pushSize) => pushSize > 250 || !(seq % 100),
}) {
this.services = services;
this.store = store;
this.opts = opts;
this.spaceReclaimDecision = opts.spaceReclaimDecision ?? (0, util_1.storageSpaceReclaimDecision)(fs.promises);
}
async create(id, clientId, batch) {
const now = Date.now();
if (!batch) {
const model = json_crdt_1.Model.create(void 0, constants_1.SESSION.GLOBAL);
const snapshot = {
id,
seq: -1,
blob: model.toBinary(),
ts: now,
};
this.__emitNew(id);
(0, go_1.go)(() => this.gc());
return await this.store.create(snapshot, snapshot);
}
validateBatch(batch);
const model = json_crdt_1.Model.create(void 0, constants_1.SESSION.GLOBAL);
const start = {
id,
seq: -1,
ts: now,
blob: model.toBinary(),
};
for (const patch of batch.patches)
model.applyPatch(json_crdt_1.Patch.fromBinary(patch.blob));
const end = {
id,
seq: 0,
ts: now,
blob: model.toBinary(),
};
const res = await this.store.create(start, end, batch);
this.__emitNew(id);
if (res.batch)
this.__emitUpd(id, res.batch, clientId);
(0, go_1.go)(() => this.gc());
return res;
}
__emitNew(id) {
const msg = ['new'];
this.services.pubsub.publish(`__block:${id}`, msg).catch((error) => {
// tslint:disable-next-line:no-console
console.error('Error publishing new block', error);
});
}
__emitUpd(id, batch, clientId) {
const msg = ['upd', { batch }, clientId];
this.services.pubsub.publish(`__block:${id}`, msg).catch((error) => {
// tslint:disable-next-line:no-console
console.error('Error publishing block patches', error);
});
}
async get(id) {
const { store } = this;
const result = await store.get(id);
if (!result)
throw rpc_error_1.RpcError.notFound();
return result;
}
async view(id) {
const { store } = this;
const result = await store.get(id);
if (!result)
throw rpc_error_1.RpcError.notFound();
const model = json_crdt_1.Model.load(result.block.snapshot.blob);
return model.view();
}
async remove(id) {
const deleted = await this.store.remove(id);
const msg = ['del'];
this.services.pubsub.publish(`__block:${id}`, msg).catch((error) => {
// tslint:disable-next-line:no-console
console.error('Error publishing block deletion', error);
});
return deleted;
}
async scan(id, includeStartSnapshot, offset, limit = 10) {
const { store } = this;
if (typeof offset !== 'number')
offset = await store.seq(id);
if (typeof offset !== 'number')
throw rpc_error_1.RpcError.notFound();
let min = 0, max = 0;
if (!limit || Math.round(limit) !== limit)
throw rpc_error_1.RpcError.badRequest('INVALID_LIMIT');
if (limit > 0) {
min = Number(offset) || 0;
max = min + limit - 1;
}
else {
max = Number(offset) || 0;
min = max - limit + 1;
}
if (min < 0) {
min = 0;
max = Math.abs(limit);
}
const batches = await store.scan(id, min, max);
if (includeStartSnapshot) {
const snap = await store.getSnapshot(id, min - 1);
return {
snapshot: snap.snapshot,
batches: snap.batches.concat(batches),
};
}
return { batches };
}
async pull(id, lastKnownSeq, create = false) {
const { store } = this;
if (typeof lastKnownSeq !== 'number' || lastKnownSeq !== Math.round(lastKnownSeq) || lastKnownSeq < -1)
throw rpc_error_1.RpcError.validation('INVALID_SEQ');
const seq = await store.seq(id);
if (seq === undefined) {
if (create) {
const res = await this.create(id, 0);
return { snapshot: res.block.snapshot, batches: res.batch ? [res.batch] : [] };
}
throw rpc_error_1.RpcError.notFound();
}
if (lastKnownSeq > seq)
return await store.getSnapshot(id, seq);
if (lastKnownSeq === seq)
return { batches: [] };
const delta = seq - lastKnownSeq;
if (lastKnownSeq === -1 || delta > 100)
return await store.getSnapshot(id, seq);
const batches = await store.scan(id, lastKnownSeq + 1, seq);
return { batches };
}
async edit(id, batch, createIfNotExists, clientId) {
if (createIfNotExists) {
const exists = await this.store.exists(id);
if (!exists) {
const res = await this.create(id, 0, batch);
if (!res.batch)
throw rpc_error_1.RpcError.internal('Batch not returned');
return { snapshot: res.block.snapshot, batch: res.batch };
}
}
validateBatch(batch);
const { store } = this;
const get = await store.get(id);
if (!get)
throw rpc_error_1.RpcError.notFound();
const snapshot = get.block.snapshot;
const seq = snapshot.seq + 1;
const model = json_crdt_1.Model.fromBinary(snapshot.blob);
let blobSize = 0;
for (const { blob } of batch.patches) {
blobSize += blob.length;
model.applyPatch(json_crdt_1.Patch.fromBinary(blob));
}
const newSnapshot = {
id,
seq,
blob: model.toBinary(),
};
const res = await store.push(newSnapshot, batch);
const opts = this.opts;
if (seq > opts.historyPerBlock && store.compact && opts.historyCompactionDecision(seq, blobSize)) {
(0, go_1.go)(() => this.compact(id, seq - opts.historyPerBlock));
}
this.__emitUpd(id, res.batch, clientId);
(0, go_1.go)(() => this.gc());
return {
snapshot: res.snapshot,
batch: res.batch,
};
}
async compact(id, to) {
const store = this.store;
if (!store.compact)
return;
await store.compact(id, to, async (blob, iterator) => {
const model = json_crdt_1.Model.fromBinary(blob);
for await (const batch of iterator)
for (const patch of batch.patches)
model.applyPatch(json_crdt_1.Patch.fromBinary(patch.blob));
return model.toBinary();
});
}
listen(id, clientId) {
let obs = this.services.pubsub.listen$(`__block:${id}`);
if (clientId)
obs = obs.pipe((0, rxjs_1.filter)(([, , c]) => c !== clientId));
return obs;
}
stats() {
return this.store.stats();
}
async gc() {
const blocksToDelete = await this.spaceReclaimDecision();
if (blocksToDelete <= 0)
return;
await this.store.removeOldest(blocksToDelete);
}
async stop() {
await this.store.stop?.();
}
}
exports.BlocksServices = BlocksServices;