@fireproof/database
Version:
Live database for the web
1,108 lines (1,093 loc) • 40.4 kB
JavaScript
console.log("Node ESM build");
import { MemoryBlockstore } from '@alanshaw/pail/block';
import { CarReader } from '@ipld/car';
import { encode as encode$1, decode as decode$1, create as create$1 } from 'multiformats/block';
import { sha256 } from 'multiformats/hashes/sha2';
import * as raw from 'multiformats/codecs/raw';
import * as CBW from '@ipld/car/buffer-writer';
import * as codec$1 from '@ipld/dag-cbor';
import { Crypto } from '@peculiar/webcrypto';
import { CID } from 'multiformats';
import { Buffer } from 'buffer';
import { create, load } from 'prolly-trees/cid-set';
import { bf, simpleCompare } from 'prolly-trees/utils';
import { nocache } from 'prolly-trees/cache';
import { put, get, entries } from '@alanshaw/pail/crdt';
import { EventFetcher } from '@alanshaw/pail/clock';
import charwise from 'charwise';
import * as DbIndex from 'prolly-trees/db-index';
function writeQueue(worker, payload = Infinity) {
const queue = [];
let isProcessing = false;
async function process() {
if (isProcessing || queue.length === 0)
return;
isProcessing = true;
const tasksToProcess = queue.splice(0, payload);
const updates = tasksToProcess.map(item => item.task);
const result = await worker(updates);
tasksToProcess.forEach(task => task.resolve(result));
isProcessing = false;
void process();
}
return {
push(task) {
return new Promise((resolve) => {
queue.push({ task, resolve });
void process();
});
}
};
}
async function innerMakeCarFile(fp, t) {
const { cid, bytes } = await encodeCarHeader(fp);
await t.put(cid, bytes);
return encodeCarFile(cid, t);
}
async function encodeCarFile(carHeaderBlockCid, t) {
let size = 0;
// console.log('encodeCarFile', carHeaderBlockCid.bytes.byteLength, { carHeaderBlockCid }, CBW.headerLength)
const headerSize = CBW.headerLength({ roots: [carHeaderBlockCid] });
size += headerSize;
for (const { cid, bytes } of t.entries()) {
size += CBW.blockLength({ cid, bytes });
}
const buffer = new Uint8Array(size);
const writer = CBW.createWriter(buffer, { headerSize });
writer.addRoot(carHeaderBlockCid);
for (const { cid, bytes } of t.entries()) {
writer.write({ cid, bytes });
}
writer.close();
return await encode$1({ value: writer.bytes, hasher: sha256, codec: raw });
}
async function encodeCarHeader(fp) {
return (await encode$1({
value: { fp },
hasher: sha256,
codec: codec$1
}));
}
async function parseCarFile(reader) {
const roots = await reader.getRoots();
const header = await reader.get(roots[0]);
if (!header)
throw new Error('missing header block');
const { value } = await decode$1({ bytes: header.bytes, hasher: sha256, codec: codec$1 });
// @ts-ignore
if (value && value.fp === undefined)
throw new Error('missing fp');
const { fp } = value;
return fp;
}
// from https://github.com/mikeal/encrypted-block
// const crypto = new Crypto()
function getCrypto() {
try {
return new Crypto();
}
catch (e) {
return null;
}
}
const crypto = getCrypto();
function randomBytes(size) {
const bytes = Buffer.allocUnsafe(size);
if (size > 0) {
crypto.getRandomValues(bytes);
}
return bytes;
}
const enc32 = (value) => {
value = +value;
const buff = new Uint8Array(4);
buff[3] = (value >>> 24);
buff[2] = (value >>> 16);
buff[1] = (value >>> 8);
buff[0] = (value & 0xff);
return buff;
};
const readUInt32LE = (buffer) => {
const offset = buffer.byteLength - 4;
return ((buffer[offset]) |
(buffer[offset + 1] << 8) |
(buffer[offset + 2] << 16)) +
(buffer[offset + 3] * 0x1000000);
};
const concat = (buffers) => {
const uint8Arrays = buffers.map(b => b instanceof ArrayBuffer ? new Uint8Array(b) : b);
const totalLength = uint8Arrays.reduce((sum, arr) => sum + arr.length, 0);
const result = new Uint8Array(totalLength);
let offset = 0;
for (const arr of uint8Arrays) {
result.set(arr, offset);
offset += arr.length;
}
return result;
};
const encode = ({ iv, bytes }) => concat([iv, bytes]);
const decode = (bytes) => {
const iv = bytes.subarray(0, 12);
bytes = bytes.slice(12);
return { iv, bytes };
};
const code = 0x300000 + 1337;
async function subtleKey(key) {
return await crypto.subtle.importKey('raw', // raw or jwk
key, // raw data
'AES-GCM', false, // extractable
['encrypt', 'decrypt']);
}
const decrypt$1 = async ({ key, value }) => {
let { bytes, iv } = value;
const cryKey = await subtleKey(key);
const deBytes = await crypto.subtle.decrypt({
name: 'AES-GCM',
iv,
tagLength: 128
}, cryKey, bytes);
bytes = new Uint8Array(deBytes);
const len = readUInt32LE(bytes.subarray(0, 4));
const cid = CID.decode(bytes.subarray(4, 4 + len));
bytes = bytes.subarray(4 + len);
return { cid, bytes };
};
const encrypt$1 = async ({ key, cid, bytes }) => {
const len = enc32(cid.bytes.byteLength);
const iv = randomBytes(12);
const msg = concat([len, cid.bytes, bytes]);
try {
const cryKey = await subtleKey(key);
const deBytes = await crypto.subtle.encrypt({
name: 'AES-GCM',
iv,
tagLength: 128
}, cryKey, msg);
bytes = new Uint8Array(deBytes);
}
catch (e) {
console.log('e', e);
throw e;
}
return { value: { bytes, iv } };
};
const cryptoFn = (key) => {
// @ts-ignore
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
return { encrypt: opts => encrypt$1({ key, ...opts }), decrypt: opts => decrypt$1({ key, ...opts }) };
};
const name = 'jchris@encrypted-block:aes-gcm';
var codec = /*#__PURE__*/Object.freeze({
__proto__: null,
code: code,
crypto: cryptoFn,
decode: decode,
decrypt: decrypt$1,
encode: encode,
encrypt: encrypt$1,
getCrypto: getCrypto,
name: name,
randomBytes: randomBytes
});
const encrypt = async function* ({ get, cids, hasher, key, cache, chunker, root }) {
const set = new Set();
let eroot;
for (const cid of cids) {
const unencrypted = await get(cid);
if (!unencrypted)
throw new Error('missing cid: ' + cid.toString());
const encrypted = await encrypt$1({ ...unencrypted, key });
const block = await encode$1({ ...encrypted, codec, hasher });
yield block;
set.add(block.cid.toString());
if (unencrypted.cid.equals(root))
eroot = block.cid;
}
if (!eroot)
throw new Error('cids does not include root');
const list = [...set].map(s => CID.parse(s));
let last;
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
for await (const node of create({ list, get, cache, chunker, hasher, codec: codec$1 })) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const block = await node.block;
yield block;
last = block;
}
if (!last)
throw new Error('missing last block');
const head = [eroot, last.cid];
const block = await encode$1({ value: head, codec: codec$1, hasher });
yield block;
};
const decrypt = async function* ({ root, get, key, cache, chunker, hasher }) {
const getWithDecode = async (cid) => get(cid).then(async (block) => {
if (!block)
return;
const decoded = await decode$1({ ...block, codec: codec$1, hasher });
return decoded;
});
const getWithDecrypt = async (cid) => get(cid).then(async (block) => {
if (!block)
return;
const decoded = await decode$1({ ...block, codec, hasher });
return decoded;
});
const decodedRoot = await getWithDecode(root);
if (!decodedRoot)
throw new Error('missing root');
if (!decodedRoot.bytes)
throw new Error('missing bytes');
const { value: [eroot, tree] } = decodedRoot;
const rootBlock = await get(eroot);
if (!rootBlock)
throw new Error('missing root block');
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
const cidset = await load({ cid: tree, get: getWithDecode, cache, chunker, codec, hasher });
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
const { result: nodes } = await cidset.getAllEntries();
const unwrap = async (eblock) => {
if (!eblock)
throw new Error('missing block');
if (!eblock.value) {
eblock = await decode$1({ ...eblock, codec, hasher });
}
const { bytes, cid } = await decrypt$1({ ...eblock, key }).catch(e => {
throw e;
});
const block = await create$1({ cid, bytes, hasher, codec });
return block;
};
const promises = [];
for (const { cid } of nodes) {
if (!rootBlock.cid.equals(cid))
promises.push(getWithDecrypt(cid).then(unwrap));
}
yield* promises;
yield unwrap(rootBlock);
};
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
const chunker = bf(30);
async function encryptedMakeCarFile(key, fp, t) {
const { cid, bytes } = await encodeCarHeader(fp);
await t.put(cid, bytes);
return encryptedEncodeCarFile(key, cid, t);
}
async function encryptedEncodeCarFile(key, rootCid, t) {
const encryptionKeyBuffer = Buffer.from(key, 'hex');
const encryptionKey = encryptionKeyBuffer.buffer.slice(encryptionKeyBuffer.byteOffset, encryptionKeyBuffer.byteOffset + encryptionKeyBuffer.byteLength);
const encryptedBlocks = new MemoryBlockstore();
const cidsToEncrypt = [];
for (const { cid } of t.entries()) {
cidsToEncrypt.push(cid);
}
let last = null;
for await (const block of encrypt({
cids: cidsToEncrypt,
get: t.get.bind(t),
key: encryptionKey,
hasher: sha256,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
chunker,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
cache: nocache,
root: rootCid
})) {
await encryptedBlocks.put(block.cid, block.bytes);
last = block;
}
if (!last)
throw new Error('no blocks encrypted');
const encryptedCar = await encodeCarFile(last.cid, encryptedBlocks);
return encryptedCar;
}
async function decodeEncryptedCar(key, reader) {
const roots = await reader.getRoots();
const root = roots[0];
return await decodeCarBlocks(root, reader.get.bind(reader), key);
}
async function decodeCarBlocks(root, get, keyMaterial) {
const decryptionKeyBuffer = Buffer.from(keyMaterial, 'hex');
const decryptionKey = decryptionKeyBuffer.buffer.slice(decryptionKeyBuffer.byteOffset, decryptionKeyBuffer.byteOffset + decryptionKeyBuffer.byteLength);
const decryptedBlocks = new MemoryBlockstore();
let last = null;
for await (const block of decrypt({
root,
get,
key: decryptionKey,
hasher: sha256,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
chunker,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
cache: nocache
})) {
await decryptedBlocks.put(block.cid, block.bytes);
last = block;
}
if (!last)
throw new Error('no blocks decrypted');
return { blocks: decryptedBlocks, root: last.cid };
}
class Loader {
name;
opts = {};
headerStore;
carStore;
carLog = [];
carReaders = new Map();
ready;
key;
keyId;
static defaultHeader;
constructor(name, opts) {
this.name = name;
this.opts = opts || this.opts;
this.ready = this.initializeStores().then(async () => {
if (!this.headerStore || !this.carStore)
throw new Error('stores not initialized');
const meta = await this.headerStore.load('main');
return await this.ingestCarHeadFromMeta(meta);
});
}
async commit(t, done, compact = false) {
await this.ready;
const fp = this.makeCarHeader(done, this.carLog, compact);
const { cid, bytes } = this.key ? await encryptedMakeCarFile(this.key, fp, t) : await innerMakeCarFile(fp, t);
await this.carStore.save({ cid, bytes });
if (compact) {
for (const cid of this.carLog) {
await this.carStore.remove(cid);
}
this.carLog.splice(0, this.carLog.length, cid);
}
else {
this.carLog.push(cid);
}
await this.headerStore.save({ car: cid, key: this.key || null });
return cid;
}
async getBlock(cid) {
await this.ready;
for (const [, reader] of [...this.carReaders]) {
const block = await reader.get(cid);
if (block) {
return block;
}
}
}
async initializeStores() {
const isBrowser = typeof window !== 'undefined';
console.log('is browser?', isBrowser);
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const module = isBrowser ? await require('./store-browser') : await require('./store-fs');
if (module) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
this.headerStore = new module.HeaderStore(this.name);
// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
this.carStore = new module.CarStore(this.name);
}
else {
throw new Error('Failed to initialize stores.');
}
}
async loadCar(cid) {
if (!this.headerStore || !this.carStore)
throw new Error('stores not initialized');
if (this.carReaders.has(cid.toString()))
return this.carReaders.get(cid.toString());
const car = await this.carStore.load(cid);
if (!car)
throw new Error(`missing car file ${cid.toString()}`);
const reader = await this.ensureDecryptedReader(await CarReader.fromBytes(car.bytes));
this.carReaders.set(cid.toString(), reader);
this.carLog.push(cid);
return reader;
}
async ensureDecryptedReader(reader) {
if (!this.key)
return reader;
const { blocks, root } = await decodeEncryptedCar(this.key, reader);
return {
getRoots: () => [root],
get: blocks.get.bind(blocks)
};
}
async setKey(key) {
if (this.key && this.key !== key)
throw new Error('key already set');
this.key = key;
const crypto = getCrypto();
if (!crypto)
throw new Error('missing crypto module');
const subtle = crypto.subtle;
const encoder = new TextEncoder();
const data = encoder.encode(key);
const hashBuffer = await subtle.digest('SHA-256', data);
const hashArray = Array.from(new Uint8Array(hashBuffer));
this.keyId = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
}
async ingestCarHeadFromMeta(meta) {
if (!this.headerStore || !this.carStore)
throw new Error('stores not initialized');
if (!meta) {
// generate a random key
if (!this.opts.public) {
if (getCrypto()) {
await this.setKey(randomBytes(32).toString('hex'));
}
else {
console.warn('missing crypto module, using public mode');
}
}
console.log('no meta, returning default header', this.name, this.keyId);
return this.defaultHeader;
}
const { car: cid, key } = meta;
console.log('ingesting car head from meta', { car: cid, key });
if (key) {
await this.setKey(key);
}
const reader = await this.loadCar(cid);
this.carLog = [cid]; // this.carLog.push(cid)
const carHeader = await parseCarFile(reader);
await this.getMoreReaders(carHeader.cars);
return carHeader;
}
async getMoreReaders(cids) {
await Promise.all(cids.map(cid => this.loadCar(cid)));
}
}
class DbLoader extends Loader {
static defaultHeader = { cars: [], compact: [], head: [] };
defaultHeader = DbLoader.defaultHeader;
makeCarHeader({ head }, cars, compact = false) {
return compact ? { head, cars: [], compact: cars } : { head, cars, compact: [] };
}
}
class IdxLoader extends Loader {
static defaultHeader = { cars: [], compact: [], indexes: new Map() };
defaultHeader = IdxLoader.defaultHeader;
makeCarHeader({ indexes }, cars, compact = false) {
return compact ? { indexes, cars: [], compact: cars } : { indexes, cars, compact: [] };
}
}
class Transaction extends MemoryBlockstore {
parent;
constructor(parent) {
super();
this.parent = parent;
this.parent = parent;
}
async get(cid) {
return this.parent.get(cid);
}
async superGet(cid) {
return super.get(cid);
}
}
class FireproofBlockstore {
ready;
name = null;
loader = null;
opts = {};
transactions = new Set();
constructor(name, LoaderClass, opts) {
this.opts = opts || this.opts;
if (name) {
this.name = name;
this.loader = new LoaderClass(name, this.opts);
this.ready = this.loader.ready;
}
else {
this.ready = Promise.resolve(LoaderClass.defaultHeader);
}
}
// eslint-disable-next-line @typescript-eslint/require-await
async put() {
throw new Error('use a transaction to put');
}
async get(cid) {
for (const f of this.transactions) {
const v = await f.superGet(cid);
if (v)
return v;
}
if (!this.loader)
return;
return await this.loader.getBlock(cid);
}
async commitCompaction(t, head) {
this.transactions.clear();
this.transactions.add(t);
return await this.loader?.commit(t, { head }, true);
}
async *entries() {
const seen = new Set();
for (const t of this.transactions) {
for await (const blk of t.entries()) {
if (seen.has(blk.cid.toString()))
continue;
seen.add(blk.cid.toString());
yield blk;
}
}
}
async executeTransaction(fn, commitHandler) {
const t = new Transaction(this);
this.transactions.add(t);
const done = await fn(t);
const { car, done: result } = await commitHandler(t, done);
return car ? { ...result, car } : result;
}
}
class IndexBlockstore extends FireproofBlockstore {
constructor(name, opts) {
super(name || null, IdxLoader, opts);
}
async transaction(fn, indexes) {
return this.executeTransaction(fn, async (t, done) => {
indexes.set(done.name, done);
const car = await this.loader?.commit(t, { indexes });
return { car, done };
});
}
}
class TransactionBlockstore extends FireproofBlockstore {
constructor(name, opts) {
// todo this will be a map of headers by branch name
super(name || null, DbLoader, opts);
}
async transaction(fn) {
return this.executeTransaction(fn, async (t, done) => {
const car = await this.loader?.commit(t, done);
return { car, done };
});
}
}
async function applyBulkUpdateToCrdt(tblocks, head, updates, options) {
for (const update of updates) {
const link = await makeLinkForDoc(tblocks, update);
const result = await put(tblocks, head, update.key, link, options);
for (const { cid, bytes } of [...result.additions, ...result.removals, result.event]) {
tblocks.putSync(cid, bytes);
}
head = result.head;
}
return { head };
}
async function makeLinkForDoc(blocks, update) {
let value;
if (update.del) {
value = { del: true };
}
else {
value = { doc: update.value };
}
const block = await encode$1({ value, hasher: sha256, codec: codec$1 });
blocks.putSync(block.cid, block.bytes);
return block.cid;
}
async function getValueFromCrdt(blocks, head, key) {
const link = await get(blocks, head, key);
if (!link)
throw new Error(`Missing key ${key}`);
return await getValueFromLink(blocks, link);
}
async function getValueFromLink(blocks, link) {
const block = await blocks.get(link);
if (!block)
throw new Error(`Missing block ${link.toString()}`);
const { value } = (await decode$1({ bytes: block.bytes, hasher: sha256, codec: codec$1 }));
return value;
}
async function clockChangesSince(blocks, head, since) {
const eventsFetcher = new EventFetcher(blocks);
const keys = new Set();
const updates = await gatherUpdates(blocks, eventsFetcher, head, since, [], keys);
return { result: updates.reverse(), head };
}
async function gatherUpdates(blocks, eventsFetcher, head, since, updates = [], keys) {
const sHead = head.map(l => l.toString());
for (const link of since) {
if (sHead.includes(link.toString())) {
return updates;
}
}
for (const link of head) {
const { value: event } = await eventsFetcher.get(link);
const { key, value } = event.data;
if (keys.has(key))
continue;
keys.add(key);
const docValue = await getValueFromLink(blocks, value);
updates.push({ key, value: docValue.doc, del: docValue.del });
if (event.parents) {
updates = await gatherUpdates(blocks, eventsFetcher, event.parents, since, updates, keys);
}
}
return updates;
}
async function doCompact(blocks, head) {
const blockLog = new LoggingFetcher(blocks);
const newBlocks = new Transaction(blocks);
for await (const [, link] of entries(blockLog, head)) {
const bl = await blocks.get(link);
if (!bl)
throw new Error('Missing block: ' + link.toString());
await newBlocks.put(link, bl.bytes);
}
for (const cid of blockLog.cids) {
const bl = await blocks.get(cid);
if (!bl)
throw new Error('Missing block: ' + cid.toString());
await newBlocks.put(cid, bl.bytes);
}
await blocks.commitCompaction(newBlocks, head);
}
class LoggingFetcher {
blocks;
cids = new Set();
constructor(blocks) {
this.blocks = blocks;
}
async get(cid) {
this.cids.add(cid);
return await this.blocks.get(cid);
}
}
class CRDT {
name;
opts = {};
ready;
blocks;
indexBlocks;
indexers = new Map();
_head = [];
constructor(name, opts) {
this.name = name || null;
this.opts = opts || this.opts;
this.blocks = new TransactionBlockstore(name, this.opts);
this.indexBlocks = new IndexBlockstore(name ? name + '.idx' : undefined, this.opts);
this.ready = this.blocks.ready.then((header) => {
// @ts-ignore
if (header.indexes)
throw new Error('cannot have indexes in crdt header');
if (header.head) {
this._head = header.head;
} // todo multi head support here
});
}
async bulk(updates, options) {
await this.ready;
const tResult = await this.blocks.transaction(async (tblocks) => {
const { head } = await applyBulkUpdateToCrdt(tblocks, this._head, updates, options);
this._head = head; // we need multi head support here if allowing calls to bulk in parallel
return { head };
});
return tResult;
}
// async getAll(rootCache: any = null): Promise<{root: any, cids: CIDCounter, clockCIDs: CIDCounter, result: T[]}> {
async get(key) {
await this.ready;
const result = await getValueFromCrdt(this.blocks, this._head, key);
if (result.del)
return null;
return result;
}
async changes(since = []) {
await this.ready;
return await clockChangesSince(this.blocks, this._head, since);
}
async compact() {
await this.ready;
return await doCompact(this.blocks, this._head);
}
}
class Database {
static databases = new Map();
name;
opts = {};
_listeners = new Set();
_crdt;
_writeQueue;
constructor(name, opts) {
this.name = name;
this.opts = opts || this.opts;
this._crdt = new CRDT(name, this.opts);
this._writeQueue = writeQueue(async (updates) => {
const r = await this._crdt.bulk(updates);
await this._notify(updates);
return r;
});
}
async get(id) {
const got = await this._crdt.get(id).catch(e => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
e.message = `Not found: ${id} - ` + e.message;
throw e;
});
if (!got)
throw new Error(`Not found: ${id}`);
const { doc } = got;
return { _id: id, ...doc };
}
async put(doc) {
const { _id, ...value } = doc;
const docId = _id || 'f' + Math.random().toString(36).slice(2); // todo uuid v7
const result = await this._writeQueue.push({ key: docId, value });
return { id: docId, clock: result?.head };
}
async del(id) {
const result = await this._writeQueue.push({ key: id, del: true });
return { id, clock: result?.head };
}
async changes(since = []) {
const { result, head } = await this._crdt.changes(since);
const rows = result.map(({ key, value }) => ({
key, value: { _id: key, ...value }
}));
return { rows, clock: head };
}
subscribe(listener) {
this._listeners.add(listener);
return () => {
this._listeners.delete(listener);
};
}
async _notify(updates) {
if (this._listeners.size) {
const docs = updates.map(({ key, value }) => ({ _id: key, ...value }));
for (const listener of this._listeners) {
await listener(docs);
}
}
}
}
function database(name, opts) {
if (!Database.databases.has(name)) {
Database.databases.set(name, new Database(name, opts));
}
return Database.databases.get(name);
}
class IndexTree {
cid = null;
root = null;
}
const refCompare = (aRef, bRef) => {
if (Number.isNaN(aRef))
return -1;
if (Number.isNaN(bRef))
throw new Error('ref may not be Infinity or NaN');
if (aRef === Infinity)
return 1;
// if (!Number.isFinite(bRef)) throw new Error('ref may not be Infinity or NaN')
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
return simpleCompare(aRef, bRef);
};
const compare = (a, b) => {
const [aKey, aRef] = a;
const [bKey, bRef] = b;
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
const comp = simpleCompare(aKey, bKey);
if (comp !== 0)
return comp;
return refCompare(aRef, bRef);
};
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
const byKeyOpts = { cache: nocache, chunker: bf(30), codec: codec$1, hasher: sha256, compare };
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
const byIdOpts = { cache: nocache, chunker: bf(30), codec: codec$1, hasher: sha256, compare: simpleCompare };
function indexEntriesForChanges(changes, mapFn) {
const indexEntries = [];
changes.forEach(({ key: _id, value, del }) => {
if (del || !value)
return;
let mapCalled = false;
const mapReturn = mapFn({ _id, ...value }, (k, v) => {
mapCalled = true;
if (typeof k === 'undefined')
return;
indexEntries.push({
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
key: [charwise.encode(k), _id],
value: v || null
});
});
if (!mapCalled && mapReturn) {
indexEntries.push({
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
key: [charwise.encode(mapReturn), _id],
value: null
});
}
});
return indexEntries;
}
function makeProllyGetBlock(blocks) {
return async (address) => {
const block = await blocks.get(address);
if (!block)
throw new Error(`Missing block ${address.toString()}`);
const { cid, bytes } = block;
return create$1({ cid, bytes, hasher: sha256, codec: codec$1 });
};
}
async function bulkIndex(tblocks, inIndex, indexEntries, opts) {
if (!indexEntries.length)
return inIndex;
if (!inIndex.root) {
if (!inIndex.cid) {
let returnRootBlock = null;
let returnNode = null;
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
for await (const node of await DbIndex.create({ get: makeProllyGetBlock(tblocks), list: indexEntries, ...opts })) {
const block = await node.block;
await tblocks.put(block.cid, block.bytes);
returnRootBlock = block;
returnNode = node;
}
if (!returnNode || !returnRootBlock)
throw new Error('failed to create index');
return { root: returnNode, cid: returnRootBlock.cid };
}
else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
inIndex.root = await DbIndex.load({ cid: inIndex.cid, get: makeProllyGetBlock(tblocks), ...opts });
}
}
const { root, blocks: newBlocks } = await inIndex.root.bulk(indexEntries);
if (root) {
for await (const block of newBlocks) {
await tblocks.put(block.cid, block.bytes);
}
return { root, cid: (await root.block).cid };
}
else {
return { root: null, cid: null };
}
}
async function loadIndex(tblocks, cid, opts) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
return await DbIndex.load({ cid, get: makeProllyGetBlock(tblocks), ...opts });
}
async function applyQuery(crdt, resp, query) {
if (query.descending) {
resp.result = resp.result.reverse();
}
if (query.limit) {
resp.result = resp.result.slice(0, query.limit);
}
if (query.includeDocs) {
resp.result = await Promise.all(resp.result.map(async (row) => {
const val = await crdt.get(row.id);
const doc = val ? { _id: row.id, ...val.doc } : null;
return { ...row, doc };
}));
}
return {
rows: resp.result.map(row => {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
row.key = charwise.decode(row.key);
return row;
})
};
}
function encodeRange(range) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
return range.map(key => charwise.encode(key));
}
function encodeKey(key) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
return charwise.encode(key);
}
function index({ _crdt }, name, mapFn, meta) {
if (mapFn && meta)
throw new Error('cannot provide both mapFn and meta');
if (mapFn && mapFn.constructor.name !== 'Function')
throw new Error('mapFn must be a function');
if (_crdt.indexers.has(name)) {
const idx = _crdt.indexers.get(name);
idx.applyMapFn(name, mapFn, meta);
}
else {
const idx = new Index(_crdt, name, mapFn, meta);
_crdt.indexers.set(name, idx);
}
return _crdt.indexers.get(name);
}
class Index {
blocks;
crdt;
name = null;
mapFn = null;
mapFnString = '';
byKey = new IndexTree();
byId = new IndexTree();
indexHead = undefined;
includeDocsDefault = false;
initError = null;
ready;
constructor(crdt, name, mapFn, meta) {
this.blocks = crdt.indexBlocks;
this.crdt = crdt;
this.applyMapFn(name, mapFn, meta);
if (!(this.mapFnString || this.initError))
throw new Error('missing mapFnString');
this.ready = this.blocks.ready.then((header) => {
// @ts-ignore
if (header.head)
throw new Error('cannot have head in idx header');
if (header.indexes === undefined)
throw new Error('missing indexes in idx header');
for (const [name, idx] of Object.entries(header.indexes)) {
index({ _crdt: crdt }, name, undefined, idx);
}
});
}
applyMapFn(name, mapFn, meta) {
if (mapFn && meta)
throw new Error('cannot provide both mapFn and meta');
if (this.name && this.name !== name)
throw new Error('cannot change name');
this.name = name;
try {
if (meta) {
// hydrating from header
if (this.indexHead &&
this.indexHead.map(c => c.toString()).join() !== meta.head.map(c => c.toString()).join()) {
throw new Error('cannot apply meta to existing index');
}
this.byId.cid = meta.byId;
this.byKey.cid = meta.byKey;
this.indexHead = meta.head;
if (this.mapFnString) {
// we already initialized from application code
if (this.mapFnString !== meta.map)
throw new Error('cannot apply different mapFn meta');
}
else {
// we are first
this.mapFnString = meta.map;
}
}
else {
if (this.mapFn) {
// we already initialized from application code
if (mapFn) {
if (this.mapFn.toString() !== mapFn.toString())
throw new Error('cannot apply different mapFn app2');
}
}
else {
// application code is creating an index
if (!mapFn) {
mapFn = makeMapFnFromName(name);
}
if (this.mapFnString) {
// we already loaded from a header
if (this.mapFnString !== mapFn.toString())
throw new Error('cannot apply different mapFn app');
}
else {
// we are first
this.mapFnString = mapFn.toString();
}
this.mapFn = mapFn;
}
}
const matches = /=>\s*(.*)/.test(this.mapFnString);
this.includeDocsDefault = matches;
}
catch (e) {
this.initError = e;
}
}
async query(opts = {}) {
await this._updateIndex();
await this._hydrateIndex();
if (!this.byKey.root)
return await applyQuery(this.crdt, { result: [] }, opts);
if (this.includeDocsDefault && opts.includeDocs === undefined)
opts.includeDocs = true;
if (opts.range) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
const { result, ...all } = await this.byKey.root.range(...encodeRange(opts.range));
return await applyQuery(this.crdt, { result, ...all }, opts);
}
if (opts.key) {
const encodedKey = encodeKey(opts.key);
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
return await applyQuery(this.crdt, await this.byKey.root.get(encodedKey), opts);
}
if (opts.prefix) {
if (!Array.isArray(opts.prefix))
opts.prefix = [opts.prefix];
const start = [...opts.prefix, NaN];
const end = [...opts.prefix, Infinity];
const encodedR = encodeRange([start, end]);
return await applyQuery(this.crdt, await this.byKey.root.range(...encodedR), opts);
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
const { result, ...all } = await this.byKey.root.getAllEntries(); // funky return type
return await applyQuery(this.crdt, {
result: result.map(({ key: [k, id], value }) => ({ key: k, id, value })),
...all
}, opts);
}
async _hydrateIndex() {
if (this.byId.root && this.byKey.root)
return;
if (!this.byId.cid || !this.byKey.cid)
return;
this.byId.root = await loadIndex(this.blocks, this.byId.cid, byIdOpts);
this.byKey.root = await loadIndex(this.blocks, this.byKey.cid, byKeyOpts);
}
async _updateIndex() {
await this.ready;
if (this.initError)
throw this.initError;
if (!this.mapFn)
throw new Error('No map function defined');
const { result, head } = await this.crdt.changes(this.indexHead);
if (result.length === 0) {
this.indexHead = head;
return { byId: this.byId, byKey: this.byKey };
}
let staleKeyIndexEntries = [];
let removeIdIndexEntries = [];
if (this.byId.root) {
const removeIds = result.map(({ key }) => key);
const { result: oldChangeEntries } = await this.byId.root.getMany(removeIds);
staleKeyIndexEntries = oldChangeEntries.map(key => ({ key, del: true }));
removeIdIndexEntries = oldChangeEntries.map((key) => ({ key: key[1], del: true }));
}
const indexEntries = indexEntriesForChanges(result, this.mapFn); // use a getter to translate from string
const byIdIndexEntries = indexEntries.map(({ key }) => ({ key: key[1], value: key }));
const indexerMeta = new Map();
for (const [name, indexer] of this.crdt.indexers) {
if (indexer.indexHead) {
indexerMeta.set(name, {
byId: indexer.byId.cid,
byKey: indexer.byKey.cid,
head: indexer.indexHead,
map: indexer.mapFnString,
name: indexer.name
});
}
}
return await this.blocks.transaction(async (tblocks) => {
this.byId = await bulkIndex(tblocks, this.byId, removeIdIndexEntries.concat(byIdIndexEntries), byIdOpts);
this.byKey = await bulkIndex(tblocks, this.byKey, staleKeyIndexEntries.concat(indexEntries), byKeyOpts);
this.indexHead = head;
return { byId: this.byId.cid, byKey: this.byKey.cid, head, map: this.mapFnString, name: this.name };
}, indexerMeta);
}
}
function makeMapFnFromName(name) {
return (doc) => {
if (doc[name])
return doc[name];
};
}
export { Database, Index, database, index };
//# sourceMappingURL=fireproof.esm.js.map