UNPKG

@nostr-dev-kit/ndk-cache-dexie

Version:
785 lines (774 loc) 24.1 kB
// src/index.ts import { NDKEvent as NDKEvent2, deserialize, profileFromEvent } from "@nostr-dev-kit/ndk"; import createDebug2 from "debug"; import { matchFilter } from "nostr-tools"; // src/caches/event-tags.ts async function eventTagsWarmUp(cacheHandler, eventTags) { const array = await eventTags.limit(cacheHandler.maxSize).toArray(); for (const event of array) { cacheHandler.add(event.tagValue, event.eventId, false); } } var eventTagsDump = (eventTags, debug) => { return async (dirtyKeys, cache) => { const entries = []; for (const tagValue of dirtyKeys) { const eventIds = cache.get(tagValue); if (eventIds) { for (const eventId of eventIds) entries.push({ tagValue, eventId }); } } if (entries.length > 0) { debug(`Saving ${entries.length} events cache entries to database`); await eventTags.bulkPut(entries); } dirtyKeys.clear(); }; }; // src/caches/events.ts async function eventsWarmUp(cacheHandler, events) { const array = await events.limit(cacheHandler.maxSize).toArray(); for (const event of array) { cacheHandler.set(event.id, event, false); } } var eventsDump = (events, debug) => { return async (dirtyKeys, cache) => { const entries = []; for (const event of dirtyKeys) { const entry = cache.get(event); if (entry) entries.push(entry); } if (entries.length > 0) { debug(`Saving ${entries.length} events cache entries to database`); await events.bulkPut(entries); } dirtyKeys.clear(); }; }; // src/caches/nip05.ts async function nip05WarmUp(cacheHandler, nip05s) { const array = await nip05s.limit(cacheHandler.maxSize).toArray(); for (const nip05 of array) { cacheHandler.set(nip05.nip05, nip05, false); } } var nip05Dump = (nip05s, debug) => { return async (dirtyKeys, cache) => { const entries = []; for (const nip05 of dirtyKeys) { const entry = cache.get(nip05); if (entry) { entries.push({ nip05, ...entry }); } } if (entries.length) { debug(`Saving ${entries.length} NIP-05 cache entries to database`); await nip05s.bulkPut(entries); } dirtyKeys.clear(); }; }; // src/db.ts import Dexie from "dexie"; var Database = class extends Dexie { profiles; events; eventTags; nip05; lnurl; relayStatus; unpublishedEvents; constructor(name) { super(name); this.version(15).stores({ profiles: "&pubkey", events: "&id, kind", eventTags: "&tagValue", nip05: "&nip05", lnurl: "&pubkey", relayStatus: "&url", unpublishedEvents: "&id" }); } }; var db; function createDatabase(name) { db = new Database(name); } // src/caches/profiles.ts import createDebug from "debug"; var d = createDebug("ndk:dexie-adapter:profiles"); async function profilesWarmUp(cacheHandler, profiles) { const array = await profiles.limit(cacheHandler.maxSize).toArray(); for (const user of array) { const obj = user; cacheHandler.set(user.pubkey, obj, false); } d("Loaded %d profiles from database", cacheHandler.size()); } var profilesDump = (profiles, debug) => { return async (dirtyKeys, cache) => { const entries = []; for (const pubkey of dirtyKeys) { const entry = cache.get(pubkey); if (entry) { entries.push(entry); } } if (entries.length) { debug(`Saving ${entries.length} users to database`); await profiles.bulkPut(entries); } dirtyKeys.clear(); }; }; // src/caches/relay-info.ts async function relayInfoWarmUp(cacheHandler, relayStatus) { const array = await relayStatus.limit(cacheHandler.maxSize).toArray(); for (const entry of array) { cacheHandler.set( entry.url, { url: entry.url, updatedAt: entry.updatedAt, lastConnectedAt: entry.lastConnectedAt, dontConnectBefore: entry.dontConnectBefore }, false ); } } var relayInfoDump = (relayStatus, debug) => { return async (dirtyKeys, cache) => { const entries = []; for (const url of dirtyKeys) { const info = cache.get(url); if (info) { entries.push({ url, updatedAt: info.updatedAt, lastConnectedAt: info.lastConnectedAt, dontConnectBefore: info.dontConnectBefore }); } } if (entries.length > 0) { debug(`Saving ${entries.length} relay status cache entries to database`); await relayStatus.bulkPut(entries); } dirtyKeys.clear(); }; }; // src/caches/unpublished-events.ts import { NDKEvent } from "@nostr-dev-kit/ndk"; var WRITE_STATUS_THRESHOLD = 3; async function unpublishedEventsWarmUp(cacheHandler, unpublishedEvents) { await unpublishedEvents.each((unpublishedEvent) => { cacheHandler.set(unpublishedEvent.event.id, unpublishedEvent, false); }); } function unpublishedEventsDump(unpublishedEvents, debug) { return async (dirtyKeys, cache) => { const entries = []; for (const eventId of dirtyKeys) { const entry = cache.get(eventId); if (entry) { entries.push(entry); } } if (entries.length > 0) { debug(`Saving ${entries.length} unpublished events cache entries to database`); await unpublishedEvents.bulkPut(entries); } dirtyKeys.clear(); }; } async function discardUnpublishedEvent(unpublishedEvents, eventId) { await unpublishedEvents.delete(eventId); } async function getUnpublishedEvents(unpublishedEvents) { const events = []; await unpublishedEvents.each((unpublishedEvent) => { events.push({ event: new NDKEvent(void 0, unpublishedEvent.event), relays: Object.keys(unpublishedEvent.relays), lastTryAt: unpublishedEvent.lastTryAt }); }); return events; } function addUnpublishedEvent(event, relays) { const r = {}; relays.forEach((url) => r[url] = false); this.unpublishedEvents.set(event.id, { id: event.id, event: event.rawEvent(), relays: r }); const onPublished = (relay) => { const url = relay.url; const existingEntry = this.unpublishedEvents.get(event.id); if (!existingEntry) { event.off("publushed", onPublished); return; } existingEntry.relays[url] = true; this.unpublishedEvents.set(event.id, existingEntry); const successWrites = Object.values(existingEntry.relays).filter((v) => v).length; const unsuccessWrites = Object.values(existingEntry.relays).length - successWrites; if (successWrites >= WRITE_STATUS_THRESHOLD || unsuccessWrites === 0) { this.unpublishedEvents.delete(event.id); event.off("published", onPublished); } }; event.on("published", onPublished); } // src/caches/zapper.ts async function zapperWarmUp(cacheHandler, lnurls) { const array = await lnurls.limit(cacheHandler.maxSize).toArray(); for (const lnurl of array) { cacheHandler.set(lnurl.pubkey, { document: lnurl.document, fetchedAt: lnurl.fetchedAt }, false); } } var zapperDump = (lnurls, debug) => { return async (dirtyKeys, cache) => { const entries = []; for (const pubkey of dirtyKeys) { const entry = cache.get(pubkey); if (entry) { entries.push({ pubkey, ...entry }); } } if (entries.length) { debug(`Saving ${entries.length} zapper cache entries to database`); await lnurls.bulkPut(entries); } dirtyKeys.clear(); }; }; // src/lru-cache.ts import { LRUCache } from "typescript-lru-cache"; var CacheHandler = class { cache; dirtyKeys = /* @__PURE__ */ new Set(); options; debug; indexes; isSet = false; maxSize = 0; constructor(options) { this.debug = options.debug; this.options = options; this.maxSize = options.maxSize; if (options.maxSize > 0) { this.cache = new LRUCache({ maxSize: options.maxSize }); setInterval(() => this.dump().catch(console.error), 1e3 * 10); } this.indexes = /* @__PURE__ */ new Map(); } getSet(key) { return this.cache?.get(key); } /** * Get all entries that match the filter. */ getAllWithFilter(filter) { const ret = /* @__PURE__ */ new Map(); this.cache?.forEach((val, key) => { if (filter(key, val)) { ret.set(key, val); } }); return ret; } get(key) { return this.cache?.get(key); } async getWithFallback(key, table) { let entry = this.get(key); if (!entry) { entry = await table.get(key); if (entry) { this.set(key, entry); } } return entry; } async getManyWithFallback(keys, table) { const entries = []; const missingKeys = []; for (const key of keys) { const entry = this.get(key); if (entry) entries.push(entry); else missingKeys.push(key); } if (entries.length > 0) { this.debug(`Cache hit for keys ${entries.length} and miss for ${missingKeys.length} keys`); } if (missingKeys.length > 0) { const startTime = Date.now(); const missingEntries = await table.bulkGet(missingKeys); const endTime = Date.now(); let foundKeys = 0; for (const entry of missingEntries) { if (entry) { this.set(entry.id, entry); entries.push(entry); foundKeys++; } } this.debug( `Time spent querying database: ${endTime - startTime}ms for ${missingKeys.length} keys, which added ${foundKeys} entries to the cache` ); } return entries; } add(key, value, dirty = true) { const existing = this.get(key) ?? /* @__PURE__ */ new Set(); existing.add(value); this.cache?.set(key, existing); if (dirty) this.dirtyKeys.add(key); } set(key, value, dirty = true) { this.cache?.set(key, value); if (dirty) this.dirtyKeys.add(key); for (const [attribute, index] of this.indexes.entries()) { const indexKey = value[attribute]; if (indexKey) { const indexValue = index.get(indexKey) || /* @__PURE__ */ new Set(); indexValue.add(key); index.set(indexKey, indexValue); } } } size() { return this.cache?.size || 0; } delete(key) { this.cache?.delete(key); this.dirtyKeys.add(key); } async dump() { if (this.dirtyKeys.size > 0 && this.cache) { await this.options.dump(this.dirtyKeys, this.cache); this.dirtyKeys.clear(); } } addIndex(attribute) { this.indexes.set(attribute, new LRUCache({ maxSize: this.options.maxSize })); } getFromIndex(index, key) { const ret = /* @__PURE__ */ new Set(); const indexValues = this.indexes.get(index); if (indexValues) { const values = indexValues.get(key); if (values) { for (const key2 of values.values()) { const entry = this.get(key2); if (entry) ret.add(entry); } } } return ret; } }; // src/index.ts var INDEXABLE_TAGS_LIMIT = 10; var NDKCacheAdapterDexie = class { debug; locking = false; ready = false; profiles; zappers; nip05s; events; eventTags; relayInfo; unpublishedEvents; warmedUp = false; warmUpPromise; devMode = false; saveSig; _onReady; constructor(opts = {}) { createDatabase(opts.dbName || "ndk"); this.debug = opts.debug || createDebug2("ndk:dexie-adapter"); this.saveSig = opts.saveSig || false; this.profiles = new CacheHandler({ maxSize: opts.profileCacheSize || 1e5, dump: profilesDump(db.profiles, this.debug), debug: this.debug }); this.zappers = new CacheHandler({ maxSize: opts.zapperCacheSize || 200, dump: zapperDump(db.lnurl, this.debug), debug: this.debug }); this.nip05s = new CacheHandler({ maxSize: opts.nip05CacheSize || 1e3, dump: nip05Dump(db.nip05, this.debug), debug: this.debug }); this.events = new CacheHandler({ maxSize: opts.eventCacheSize || 5e4, dump: eventsDump(db.events, this.debug), debug: this.debug }); this.events.addIndex("pubkey"); this.events.addIndex("kind"); this.eventTags = new CacheHandler({ maxSize: opts.eventTagsCacheSize || 1e5, dump: eventTagsDump(db.eventTags, this.debug), debug: this.debug }); this.relayInfo = new CacheHandler({ maxSize: 500, debug: this.debug, dump: relayInfoDump(db.relayStatus, this.debug) }); this.unpublishedEvents = new CacheHandler({ maxSize: 5e3, debug: this.debug, dump: unpublishedEventsDump(db.unpublishedEvents, this.debug) }); const profile = (label, fn) => { const start = Date.now(); return fn().then(() => { const end = Date.now(); this.debug(label, "took", end - start, "ms"); }); }; const startTime = Date.now(); this.warmUpPromise = Promise.allSettled([ profile("profilesWarmUp", () => profilesWarmUp(this.profiles, db.profiles)), profile("zapperWarmUp", () => zapperWarmUp(this.zappers, db.lnurl)), profile("nip05WarmUp", () => nip05WarmUp(this.nip05s, db.nip05)), profile("relayInfoWarmUp", () => relayInfoWarmUp(this.relayInfo, db.relayStatus)), profile( "unpublishedEventsWarmUp", () => unpublishedEventsWarmUp(this.unpublishedEvents, db.unpublishedEvents) ), profile("eventsWarmUp", () => eventsWarmUp(this.events, db.events)), profile("eventTagsWarmUp", () => eventTagsWarmUp(this.eventTags, db.eventTags)) ]); this.warmUpPromise.then(() => { const endTime = Date.now(); this.warmedUp = true; this.ready = true; this.locking = true; this.debug("Warm up completed, time", endTime - startTime, "ms"); if (this._onReady) this._onReady(); }); } onReady(callback) { this._onReady = callback; } async query(subscription) { if (!this.warmedUp) { const startTime2 = Date.now(); await this.warmUpPromise; this.debug("froze query for", Date.now() - startTime2, "ms", subscription.filters); } const startTime = Date.now(); subscription.filters.map((filter) => this.processFilter(filter, subscription)); const dur = Date.now() - startTime; if (dur > 100) this.debug("query took", dur, "ms", subscription.filter); return []; } async fetchProfile(pubkey) { if (!this.profiles) return null; const user = await this.profiles.getWithFallback(pubkey, db.profiles); return user; } fetchProfileSync(pubkey) { if (!this.profiles) return null; const user = this.profiles.get(pubkey); return user; } async getProfiles(fn) { if (!this.profiles) return; return this.profiles.getAllWithFilter(fn); } saveProfile(pubkey, profile) { const existingValue = this.profiles.get(pubkey); if (existingValue?.created_at && profile.created_at && existingValue.created_at >= profile.created_at) { return; } const cachedAt = Math.floor(Date.now() / 1e3); this.profiles.set(pubkey, { pubkey, ...profile, cachedAt }); this.debug("Saved profile for pubkey", pubkey, profile); } async loadNip05(nip05, maxAgeForMissing = 3600) { const cache = this.nip05s?.get(nip05); if (cache) { if (cache.profile === null) { if (cache.fetchedAt + maxAgeForMissing * 1e3 < Date.now()) return "missing"; return null; } try { return JSON.parse(cache.profile); } catch (_e) { return "missing"; } } const nip = await db.nip05.get({ nip05 }); if (!nip) return "missing"; const now = Date.now(); if (nip.profile === null) { if (nip.fetchedAt + maxAgeForMissing * 1e3 < now) return "missing"; return null; } try { return JSON.parse(nip.profile); } catch (_e) { return "missing"; } } async saveNip05(nip05, profile) { try { const document = profile ? JSON.stringify(profile) : null; this.nip05s.set(nip05, { profile: document, fetchedAt: Date.now() }); } catch (error) { console.error("Failed to save NIP-05 profile for nip05:", nip05, error); } } async loadUsersLNURLDoc(pubkey, maxAgeInSecs = 86400, maxAgeForMissing = 3600) { const cache = this.zappers?.get(pubkey); if (cache) { if (cache.document === null) { if (cache.fetchedAt + maxAgeForMissing * 1e3 < Date.now()) return "missing"; return null; } try { return JSON.parse(cache.document); } catch (_e) { return "missing"; } } const lnurl = await db.lnurl.get({ pubkey }); if (!lnurl) return "missing"; const now = Date.now(); if (lnurl.fetchedAt + maxAgeInSecs * 1e3 < now) return "missing"; if (lnurl.document === null) { if (lnurl.fetchedAt + maxAgeForMissing * 1e3 < now) return "missing"; return null; } try { return JSON.parse(lnurl.document); } catch (_e) { return "missing"; } } async saveUsersLNURLDoc(pubkey, doc) { try { const document = doc ? JSON.stringify(doc) : null; this.zappers?.set(pubkey, { document, fetchedAt: Date.now() }); } catch (error) { console.error("Failed to save LNURL document for pubkey:", pubkey, error); } } processFilter(filter, subscription) { const _filter = { ...filter }; _filter.limit = void 0; const filterKeys = new Set(Object.keys(_filter || {})); filterKeys.delete("since"); filterKeys.delete("limit"); filterKeys.delete("until"); try { if (this.byNip33Query(filterKeys, filter, subscription)) return; if (this.byAuthors(filter, subscription)) return; if (this.byIdsQuery(filter, subscription)) return; if (this.byTags(filter, subscription)) return; if (this.byKinds(filterKeys, filter, subscription)) return; } catch (error) { console.error(error); } } async deleteEventIds(eventIds) { eventIds.forEach((id) => this.events.delete(id)); await db.events.where({ id: eventIds }).delete(); } addUnpublishedEvent = addUnpublishedEvent.bind(this); getUnpublishedEvents = () => getUnpublishedEvents(db.unpublishedEvents); discardUnpublishedEvent = (id) => discardUnpublishedEvent(db.unpublishedEvents, id); async setEvent(event, _filters, relay) { if (event.kind === 0) { if (!this.profiles) return; try { const profile = profileFromEvent(event); this.saveProfile(event.pubkey, profile); } catch { this.debug(`Failed to save profile for pubkey: ${event.pubkey}`); } } let addEvent = true; if (event.isParamReplaceable()) { const existingEvent = this.events.get(event.tagId()); if (existingEvent && event.created_at && existingEvent.createdAt > event.created_at) { addEvent = false; } } if (addEvent) { const eventData = { id: event.tagId(), pubkey: event.pubkey, kind: event.kind, createdAt: event.created_at ?? Date.now(), relay: relay?.url, event: event.serialize(this.saveSig, true) }; if (this.saveSig && event.sig) { eventData.sig = event.sig; } this.events.set(event.tagId(), eventData); const indexableTags = getIndexableTags(event); for (const tag of indexableTags) { this.eventTags.add(tag[0] + tag[1], event.tagId()); } } } updateRelayStatus(url, info) { const val = { url, updatedAt: Date.now(), ...info }; this.relayInfo.set(url, val); } getRelayStatus(url) { const a = this.relayInfo.get(url); if (a) { return { lastConnectedAt: a.lastConnectedAt, dontConnectBefore: a.dontConnectBefore }; } } /** * Searches by authors */ byAuthors(filter, subscription) { if (!filter.authors) return false; let _total = 0; for (const pubkey of filter.authors) { let events = Array.from(this.events.getFromIndex("pubkey", pubkey)); if (filter.kinds) events = events.filter((e) => filter.kinds?.includes(e.kind)); foundEvents(subscription, events, filter); _total += events.length; } return true; } /** * Searches by ids */ byIdsQuery(filter, subscription) { if (filter.ids) { for (const id of filter.ids) { const event = this.events.get(id); if (event) foundEvent(subscription, event, event.relay, filter); } return true; } return false; } /** * Searches by NIP-33 */ byNip33Query(filterKeys, filter, subscription) { const f = ["#d", "authors", "kinds"]; const hasAllKeys = filterKeys.size === f.length && f.every((k) => filterKeys.has(k)); if (hasAllKeys && filter.kinds && filter.authors) { for (const kind of filter.kinds) { const replaceableKind = kind >= 3e4 && kind < 4e4; if (!replaceableKind) continue; for (const author of filter.authors) { for (const dTag of filter["#d"]) { const replaceableId = `${kind}:${author}:${dTag}`; const event = this.events.get(replaceableId); if (event) foundEvent(subscription, event, event.relay, filter); } } } return true; } return false; } /** * Searches by tags and optionally filters by tags */ byTags(filter, subscription) { const tagFilters = Object.entries(filter).filter(([filter2]) => filter2.startsWith("#") && filter2.length === 2).map(([filter2, values]) => [filter2[1], values]); if (tagFilters.length === 0) return false; for (const [tag, values] of tagFilters) { for (const value of values) { const tagValue = tag + value; const eventIds = this.eventTags.getSet(tagValue); if (!eventIds) continue; eventIds.forEach((id) => { const event = this.events.get(id); if (!event) return; if (!filter.kinds || filter.kinds.includes(event.kind)) { foundEvent(subscription, event, event.relay, filter); } }); } } return true; } byKinds(filterKeys, filter, subscription) { if (!filter.kinds || filterKeys.size !== 1 || !filterKeys.has("kinds")) return false; const limit = filter.limit || 500; let totalEvents = 0; const processedEventIds = /* @__PURE__ */ new Set(); const sortedKinds = [...filter.kinds].sort( (a, b) => (this.events.indexes.get("kind")?.get(a)?.size || 0) - (this.events.indexes.get("kind")?.get(b)?.size || 0) ); for (const kind of sortedKinds) { const events = this.events.getFromIndex("kind", kind); for (const event of events) { if (processedEventIds.has(event.id)) continue; processedEventIds.add(event.id); foundEvent(subscription, event, event.relay, filter); totalEvents++; if (totalEvents >= limit) break; } if (totalEvents >= limit) break; } return true; } }; function foundEvents(subscription, events, filter) { if (filter?.limit && events.length > filter.limit) { events = events.sort((a, b) => b.createdAt - a.createdAt).slice(0, filter.limit); } for (const event of events) { foundEvent(subscription, event, event.relay, filter); } } function foundEvent(subscription, event, relayUrl, filter) { try { const deserializedEvent = deserialize(event.event); if (filter && !matchFilter(filter, deserializedEvent)) return; const ndkEvent = new NDKEvent2(void 0, deserializedEvent); const relay = relayUrl ? subscription.pool.getRelay(relayUrl, false) : void 0; ndkEvent.relay = relay; subscription.eventReceived(ndkEvent, relay, true); } catch (e) { console.error("failed to deserialize event", e); } } function getIndexableTags(event) { const indexableTags = []; if (event.kind === 3) return []; for (const tag of event.tags) { if (tag[0].length !== 1) continue; indexableTags.push(tag); if (indexableTags.length >= INDEXABLE_TAGS_LIMIT) return []; } return indexableTags; } export { db, NDKCacheAdapterDexie as default, foundEvent, foundEvents };