json-crdt-server
Version:
JSON CRDT server and syncing local-first browser client
284 lines (283 loc) • 10 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.LevelStore = void 0;
const rpc_error_1 = require("rpc-error");
const Writer_1 = require("@jsonjoy.com/util/lib/buffers/Writer");
const cbor_1 = require("@jsonjoy.com/json-pack/lib/codecs/cbor");
const AvlMap_1 = require("sonic-forest/lib/avl/AvlMap");
const Mutex_1 = require("../../../../util/Mutex");
class LevelStore {
constructor(kv, codec = new cbor_1.CborJsonValueCodec(new Writer_1.Writer()), mutex = new Mutex_1.Mutex()) {
this.kv = kv;
this.codec = codec;
this.mutex = mutex;
}
keyBase(id) {
return 'b!' + id + '!';
}
endKey(id) {
return this.keyBase(id) + 'e';
}
startKey(id) {
return this.keyBase(id) + 's';
}
batchBase(id) {
return this.keyBase(id) + 'b!';
}
batchKey(id, seq) {
const seqFormatted = seq.toString(36).padStart(6, '0');
return this.batchBase(id) + seqFormatted;
}
touchKeyBase() {
return 'u!';
}
touchKey(id) {
return this.touchKeyBase() + id + '!';
}
/** @todo Add in-memory cache on read. */
async get(id) {
const key = this.endKey(id);
try {
const blob = await this.kv.get(key);
if (!blob)
return;
const block = this.codec.decoder.decode(blob);
return { block };
}
catch (error) {
if (error && typeof error === 'object' && error.code === 'LEVEL_NOT_FOUND')
return;
throw error;
}
}
async getSnapshot(id, seq) {
const { kv, codec } = this;
const { decoder } = codec;
const key = this.startKey(id);
try {
const blob = await kv.get(key);
const snapshot = decoder.decode(blob);
const batches = [];
if (snapshot.seq < seq) {
const gte = this.batchKey(id, snapshot.seq + 1);
const lte = this.batchKey(id, seq);
for await (const blob of kv.values({ gte, lte: lte })) {
const batch = decoder.decode(blob);
batches.push(batch);
}
}
return { snapshot, batches };
}
catch (error) {
if (error && typeof error === 'object' && error.code === 'LEVEL_NOT_FOUND')
throw rpc_error_1.RpcError.notFound();
throw error;
}
}
async exists(id) {
const key = this.endKey(id);
const existing = await this.kv.keys({ gte: key, lte: key, limit: 1 }).all();
return existing && existing.length > 0;
}
async seq(id) {
return await this.mutex.acquire(id, async () => {
const base = this.batchBase(id);
const keys = await this.kv.keys({ lt: base + '~', limit: 1, reverse: true }).all();
if (!keys || keys.length < 1)
return;
const key = keys[0].slice(base.length);
if (!key)
return;
const seq = Number.parseInt(key, 36);
// biome-ignore lint: check for NaN
if (seq !== seq)
return;
return seq;
});
}
async create(start, end, incomingBatch) {
if (incomingBatch) {
const { patches } = incomingBatch;
if (!Array.isArray(patches) || patches.length < 1)
throw new Error('NO_PATCHES');
}
const { id } = end;
const key = this.endKey(id);
const now = end.ts;
const encoder = this.codec.encoder;
return await this.mutex.acquire(id, async () => {
const existing = await this.kv.keys({ gte: key, lte: key, limit: 1 }).all();
if (existing && existing.length > 0)
throw rpc_error_1.RpcError.conflict();
const block = { id, snapshot: end, tip: [], ts: now, uts: now };
const ops = [
{ type: 'put', key: this.startKey(id), value: encoder.encode(start) },
{ type: 'put', key, value: encoder.encode(block) },
{ type: 'put', key: this.touchKey(id), value: encoder.encode(now) },
];
const response = { block };
if (incomingBatch) {
const { cts, patches } = incomingBatch;
const batch = {
seq: 0,
ts: end.ts,
cts,
patches,
};
const batchBlob = encoder.encode(batch);
const batchKey = this.batchKey(id, 0);
ops.push({ type: 'put', key: batchKey, value: batchBlob });
response.batch = batch;
}
await this.kv.batch(ops);
return response;
});
}
async push(snapshot0, batch0) {
const { id, seq } = snapshot0;
const { patches } = batch0;
if (!Array.isArray(patches) || !patches.length)
throw new Error('NO_PATCHES');
return await this.mutex.acquire(id, async () => {
const block = await this.get(id);
if (!block)
throw rpc_error_1.RpcError.notFound();
const blockData = block.block;
const snapshot = blockData.snapshot;
if (snapshot.seq + 1 !== seq)
throw new Error('PATCH_SEQ_INV');
const now = Date.now();
blockData.uts = now;
snapshot.seq = seq;
snapshot.ts = now;
snapshot.blob = snapshot0.blob;
const encoder = this.codec.encoder;
const batch1 = {
seq,
ts: now,
cts: batch0.cts,
patches,
};
const ops = [
{ type: 'put', key: this.endKey(id), value: encoder.encode(blockData) },
{ type: 'put', key: this.batchKey(id, seq), value: encoder.encode(batch1) },
{ type: 'put', key: this.touchKey(id), value: encoder.encode(now) },
];
await this.kv.batch(ops);
return { snapshot, batch: batch1 };
});
}
async compact(id, to, advance) {
const { kv, codec } = this;
const { encoder, decoder } = codec;
const key = this.startKey(id);
await this.mutex.acquire(id + '.trunc', async () => {
const start = decoder.decode((await kv.get(key)));
if (start.seq >= to)
return;
const gt = this.batchKey(id, start.seq);
const lte = this.batchKey(id, to);
const ops = [];
async function* iterator() {
for await (const [key, blob] of kv.iterator({ gt, lte })) {
ops.push({ type: 'del', key });
yield decoder.decode(blob);
}
}
start.blob = await advance(start.blob, iterator());
start.ts = Date.now();
start.seq = to;
ops.push({ type: 'put', key, value: encoder.encode(start) });
await kv.batch(ops);
});
}
async scan(id, min, max) {
const from = this.batchKey(id, min);
const to = this.batchKey(id, max);
const list = [];
const decoder = this.codec.decoder;
for await (const blob of this.kv.values({ gte: from, lte: to })) {
const batch = decoder.decode(blob);
list.push(batch);
}
return list;
}
async remove(id) {
const exists = await this.exists(id);
if (!exists)
return false;
const base = this.keyBase(id);
const touchKey = this.touchKey(id);
const kv = this.kv;
const success = await this.mutex.acquire(id, async () => {
await Promise.allSettled([
kv.clear({
gte: base,
lte: base + '~',
}),
kv.del(touchKey),
]);
return true;
});
return success;
}
/** @todo Make this method async and return something useful. */
stats() {
return { blocks: 0, batches: 0 };
}
/**
* @todo Need to add GC tests.
*/
async removeAccessedBefore(ts, limit = 10) {
const from = this.touchKey('');
const to = from + '~';
const decoder = this.codec.decoder;
let cnt = 0;
for await (const [key, blob] of this.kv.iterator({ gte: from, lte: to })) {
const value = Number(decoder.decode(blob));
if (ts >= value)
continue;
cnt++;
const id = key.slice(from.length);
this.remove(id).catch(() => { });
if (cnt >= limit)
return;
}
}
async removeOldest(x) {
const heap = new AvlMap_1.AvlMap((a, b) => b - a);
const keyBase = this.touchKeyBase();
const gte = keyBase + '';
const lte = keyBase + '~';
const kv = this.kv;
const decoder = this.codec.decoder;
let first = heap.first();
for await (const [key, value] of kv.iterator({ gte, lte })) {
const time = decoder.decode(value);
if (heap.size() < x) {
heap.set(time, key);
continue;
}
if (!first)
first = heap.first();
if (first && time < first.k) {
heap.del(first.k);
first = undefined;
heap.set(time, key);
}
}
if (!heap.size())
return;
for (const { v } of heap.entries()) {
try {
const id = v.slice(keyBase.length, -1);
await this.remove(id);
}
catch { }
}
}
async stop() {
await this.kv.close();
}
}
exports.LevelStore = LevelStore;