UNPKG

json-crdt-server

Version:

JSON CRDT server and syncing local-first browser client

284 lines (283 loc) 10 kB
"use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.LevelStore = void 0; const rpc_error_1 = require("rpc-error"); const Writer_1 = require("@jsonjoy.com/util/lib/buffers/Writer"); const cbor_1 = require("@jsonjoy.com/json-pack/lib/codecs/cbor"); const AvlMap_1 = require("sonic-forest/lib/avl/AvlMap"); const Mutex_1 = require("../../../../util/Mutex"); class LevelStore { constructor(kv, codec = new cbor_1.CborJsonValueCodec(new Writer_1.Writer()), mutex = new Mutex_1.Mutex()) { this.kv = kv; this.codec = codec; this.mutex = mutex; } keyBase(id) { return 'b!' + id + '!'; } endKey(id) { return this.keyBase(id) + 'e'; } startKey(id) { return this.keyBase(id) + 's'; } batchBase(id) { return this.keyBase(id) + 'b!'; } batchKey(id, seq) { const seqFormatted = seq.toString(36).padStart(6, '0'); return this.batchBase(id) + seqFormatted; } touchKeyBase() { return 'u!'; } touchKey(id) { return this.touchKeyBase() + id + '!'; } /** @todo Add in-memory cache on read. */ async get(id) { const key = this.endKey(id); try { const blob = await this.kv.get(key); if (!blob) return; const block = this.codec.decoder.decode(blob); return { block }; } catch (error) { if (error && typeof error === 'object' && error.code === 'LEVEL_NOT_FOUND') return; throw error; } } async getSnapshot(id, seq) { const { kv, codec } = this; const { decoder } = codec; const key = this.startKey(id); try { const blob = await kv.get(key); const snapshot = decoder.decode(blob); const batches = []; if (snapshot.seq < seq) { const gte = this.batchKey(id, snapshot.seq + 1); const lte = this.batchKey(id, seq); for await (const blob of kv.values({ gte, lte: lte })) { const batch = decoder.decode(blob); batches.push(batch); } } return { snapshot, batches }; } catch (error) { if (error && typeof error === 'object' && error.code === 'LEVEL_NOT_FOUND') throw rpc_error_1.RpcError.notFound(); throw error; } } async exists(id) { const key = this.endKey(id); const existing = await this.kv.keys({ gte: key, lte: key, limit: 1 }).all(); return existing && existing.length > 0; } async seq(id) { return await this.mutex.acquire(id, async () => { const base = this.batchBase(id); const keys = await this.kv.keys({ lt: base + '~', limit: 1, reverse: true }).all(); if (!keys || keys.length < 1) return; const key = keys[0].slice(base.length); if (!key) return; const seq = Number.parseInt(key, 36); // biome-ignore lint: check for NaN if (seq !== seq) return; return seq; }); } async create(start, end, incomingBatch) { if (incomingBatch) { const { patches } = incomingBatch; if (!Array.isArray(patches) || patches.length < 1) throw new Error('NO_PATCHES'); } const { id } = end; const key = this.endKey(id); const now = end.ts; const encoder = this.codec.encoder; return await this.mutex.acquire(id, async () => { const existing = await this.kv.keys({ gte: key, lte: key, limit: 1 }).all(); if (existing && existing.length > 0) throw rpc_error_1.RpcError.conflict(); const block = { id, snapshot: end, tip: [], ts: now, uts: now }; const ops = [ { type: 'put', key: this.startKey(id), value: encoder.encode(start) }, { type: 'put', key, value: encoder.encode(block) }, { type: 'put', key: this.touchKey(id), value: encoder.encode(now) }, ]; const response = { block }; if (incomingBatch) { const { cts, patches } = incomingBatch; const batch = { seq: 0, ts: end.ts, cts, patches, }; const batchBlob = encoder.encode(batch); const batchKey = this.batchKey(id, 0); ops.push({ type: 'put', key: batchKey, value: batchBlob }); response.batch = batch; } await this.kv.batch(ops); return response; }); } async push(snapshot0, batch0) { const { id, seq } = snapshot0; const { patches } = batch0; if (!Array.isArray(patches) || !patches.length) throw new Error('NO_PATCHES'); return await this.mutex.acquire(id, async () => { const block = await this.get(id); if (!block) throw rpc_error_1.RpcError.notFound(); const blockData = block.block; const snapshot = blockData.snapshot; if (snapshot.seq + 1 !== seq) throw new Error('PATCH_SEQ_INV'); const now = Date.now(); blockData.uts = now; snapshot.seq = seq; snapshot.ts = now; snapshot.blob = snapshot0.blob; const encoder = this.codec.encoder; const batch1 = { seq, ts: now, cts: batch0.cts, patches, }; const ops = [ { type: 'put', key: this.endKey(id), value: encoder.encode(blockData) }, { type: 'put', key: this.batchKey(id, seq), value: encoder.encode(batch1) }, { type: 'put', key: this.touchKey(id), value: encoder.encode(now) }, ]; await this.kv.batch(ops); return { snapshot, batch: batch1 }; }); } async compact(id, to, advance) { const { kv, codec } = this; const { encoder, decoder } = codec; const key = this.startKey(id); await this.mutex.acquire(id + '.trunc', async () => { const start = decoder.decode((await kv.get(key))); if (start.seq >= to) return; const gt = this.batchKey(id, start.seq); const lte = this.batchKey(id, to); const ops = []; async function* iterator() { for await (const [key, blob] of kv.iterator({ gt, lte })) { ops.push({ type: 'del', key }); yield decoder.decode(blob); } } start.blob = await advance(start.blob, iterator()); start.ts = Date.now(); start.seq = to; ops.push({ type: 'put', key, value: encoder.encode(start) }); await kv.batch(ops); }); } async scan(id, min, max) { const from = this.batchKey(id, min); const to = this.batchKey(id, max); const list = []; const decoder = this.codec.decoder; for await (const blob of this.kv.values({ gte: from, lte: to })) { const batch = decoder.decode(blob); list.push(batch); } return list; } async remove(id) { const exists = await this.exists(id); if (!exists) return false; const base = this.keyBase(id); const touchKey = this.touchKey(id); const kv = this.kv; const success = await this.mutex.acquire(id, async () => { await Promise.allSettled([ kv.clear({ gte: base, lte: base + '~', }), kv.del(touchKey), ]); return true; }); return success; } /** @todo Make this method async and return something useful. */ stats() { return { blocks: 0, batches: 0 }; } /** * @todo Need to add GC tests. */ async removeAccessedBefore(ts, limit = 10) { const from = this.touchKey(''); const to = from + '~'; const decoder = this.codec.decoder; let cnt = 0; for await (const [key, blob] of this.kv.iterator({ gte: from, lte: to })) { const value = Number(decoder.decode(blob)); if (ts >= value) continue; cnt++; const id = key.slice(from.length); this.remove(id).catch(() => { }); if (cnt >= limit) return; } } async removeOldest(x) { const heap = new AvlMap_1.AvlMap((a, b) => b - a); const keyBase = this.touchKeyBase(); const gte = keyBase + ''; const lte = keyBase + '~'; const kv = this.kv; const decoder = this.codec.decoder; let first = heap.first(); for await (const [key, value] of kv.iterator({ gte, lte })) { const time = decoder.decode(value); if (heap.size() < x) { heap.set(time, key); continue; } if (!first) first = heap.first(); if (first && time < first.k) { heap.del(first.k); first = undefined; heap.set(time, key); } } if (!heap.size()) return; for (const { v } of heap.entries()) { try { const id = v.slice(keyBase.length, -1); await this.remove(id); } catch { } } } async stop() { await this.kv.close(); } } exports.LevelStore = LevelStore;