@nostr-dev-kit/ndk-cache-dexie
Version:
NDK Dexie Cache Adapter
785 lines (774 loc) • 24.1 kB
JavaScript
// src/index.ts
import { NDKEvent as NDKEvent2, deserialize, profileFromEvent } from "@nostr-dev-kit/ndk";
import createDebug2 from "debug";
import { matchFilter } from "nostr-tools";
// src/caches/event-tags.ts
async function eventTagsWarmUp(cacheHandler, eventTags) {
const array = await eventTags.limit(cacheHandler.maxSize).toArray();
for (const event of array) {
cacheHandler.add(event.tagValue, event.eventId, false);
}
}
var eventTagsDump = (eventTags, debug) => {
return async (dirtyKeys, cache) => {
const entries = [];
for (const tagValue of dirtyKeys) {
const eventIds = cache.get(tagValue);
if (eventIds) {
for (const eventId of eventIds) entries.push({ tagValue, eventId });
}
}
if (entries.length > 0) {
debug(`Saving ${entries.length} events cache entries to database`);
await eventTags.bulkPut(entries);
}
dirtyKeys.clear();
};
};
// src/caches/events.ts
async function eventsWarmUp(cacheHandler, events) {
const array = await events.limit(cacheHandler.maxSize).toArray();
for (const event of array) {
cacheHandler.set(event.id, event, false);
}
}
var eventsDump = (events, debug) => {
return async (dirtyKeys, cache) => {
const entries = [];
for (const event of dirtyKeys) {
const entry = cache.get(event);
if (entry) entries.push(entry);
}
if (entries.length > 0) {
debug(`Saving ${entries.length} events cache entries to database`);
await events.bulkPut(entries);
}
dirtyKeys.clear();
};
};
// src/caches/nip05.ts
async function nip05WarmUp(cacheHandler, nip05s) {
const array = await nip05s.limit(cacheHandler.maxSize).toArray();
for (const nip05 of array) {
cacheHandler.set(nip05.nip05, nip05, false);
}
}
var nip05Dump = (nip05s, debug) => {
return async (dirtyKeys, cache) => {
const entries = [];
for (const nip05 of dirtyKeys) {
const entry = cache.get(nip05);
if (entry) {
entries.push({
nip05,
...entry
});
}
}
if (entries.length) {
debug(`Saving ${entries.length} NIP-05 cache entries to database`);
await nip05s.bulkPut(entries);
}
dirtyKeys.clear();
};
};
// src/db.ts
import Dexie from "dexie";
var Database = class extends Dexie {
profiles;
events;
eventTags;
nip05;
lnurl;
relayStatus;
unpublishedEvents;
constructor(name) {
super(name);
this.version(15).stores({
profiles: "&pubkey",
events: "&id, kind",
eventTags: "&tagValue",
nip05: "&nip05",
lnurl: "&pubkey",
relayStatus: "&url",
unpublishedEvents: "&id"
});
}
};
var db;
function createDatabase(name) {
db = new Database(name);
}
// src/caches/profiles.ts
import createDebug from "debug";
var d = createDebug("ndk:dexie-adapter:profiles");
async function profilesWarmUp(cacheHandler, profiles) {
const array = await profiles.limit(cacheHandler.maxSize).toArray();
for (const user of array) {
const obj = user;
cacheHandler.set(user.pubkey, obj, false);
}
d("Loaded %d profiles from database", cacheHandler.size());
}
var profilesDump = (profiles, debug) => {
return async (dirtyKeys, cache) => {
const entries = [];
for (const pubkey of dirtyKeys) {
const entry = cache.get(pubkey);
if (entry) {
entries.push(entry);
}
}
if (entries.length) {
debug(`Saving ${entries.length} users to database`);
await profiles.bulkPut(entries);
}
dirtyKeys.clear();
};
};
// src/caches/relay-info.ts
async function relayInfoWarmUp(cacheHandler, relayStatus) {
const array = await relayStatus.limit(cacheHandler.maxSize).toArray();
for (const entry of array) {
cacheHandler.set(
entry.url,
{
url: entry.url,
updatedAt: entry.updatedAt,
lastConnectedAt: entry.lastConnectedAt,
dontConnectBefore: entry.dontConnectBefore
},
false
);
}
}
var relayInfoDump = (relayStatus, debug) => {
return async (dirtyKeys, cache) => {
const entries = [];
for (const url of dirtyKeys) {
const info = cache.get(url);
if (info) {
entries.push({
url,
updatedAt: info.updatedAt,
lastConnectedAt: info.lastConnectedAt,
dontConnectBefore: info.dontConnectBefore
});
}
}
if (entries.length > 0) {
debug(`Saving ${entries.length} relay status cache entries to database`);
await relayStatus.bulkPut(entries);
}
dirtyKeys.clear();
};
};
// src/caches/unpublished-events.ts
import { NDKEvent } from "@nostr-dev-kit/ndk";
var WRITE_STATUS_THRESHOLD = 3;
async function unpublishedEventsWarmUp(cacheHandler, unpublishedEvents) {
await unpublishedEvents.each((unpublishedEvent) => {
cacheHandler.set(unpublishedEvent.event.id, unpublishedEvent, false);
});
}
function unpublishedEventsDump(unpublishedEvents, debug) {
return async (dirtyKeys, cache) => {
const entries = [];
for (const eventId of dirtyKeys) {
const entry = cache.get(eventId);
if (entry) {
entries.push(entry);
}
}
if (entries.length > 0) {
debug(`Saving ${entries.length} unpublished events cache entries to database`);
await unpublishedEvents.bulkPut(entries);
}
dirtyKeys.clear();
};
}
async function discardUnpublishedEvent(unpublishedEvents, eventId) {
await unpublishedEvents.delete(eventId);
}
async function getUnpublishedEvents(unpublishedEvents) {
const events = [];
await unpublishedEvents.each((unpublishedEvent) => {
events.push({
event: new NDKEvent(void 0, unpublishedEvent.event),
relays: Object.keys(unpublishedEvent.relays),
lastTryAt: unpublishedEvent.lastTryAt
});
});
return events;
}
function addUnpublishedEvent(event, relays) {
const r = {};
relays.forEach((url) => r[url] = false);
this.unpublishedEvents.set(event.id, { id: event.id, event: event.rawEvent(), relays: r });
const onPublished = (relay) => {
const url = relay.url;
const existingEntry = this.unpublishedEvents.get(event.id);
if (!existingEntry) {
event.off("publushed", onPublished);
return;
}
existingEntry.relays[url] = true;
this.unpublishedEvents.set(event.id, existingEntry);
const successWrites = Object.values(existingEntry.relays).filter((v) => v).length;
const unsuccessWrites = Object.values(existingEntry.relays).length - successWrites;
if (successWrites >= WRITE_STATUS_THRESHOLD || unsuccessWrites === 0) {
this.unpublishedEvents.delete(event.id);
event.off("published", onPublished);
}
};
event.on("published", onPublished);
}
// src/caches/zapper.ts
async function zapperWarmUp(cacheHandler, lnurls) {
const array = await lnurls.limit(cacheHandler.maxSize).toArray();
for (const lnurl of array) {
cacheHandler.set(lnurl.pubkey, { document: lnurl.document, fetchedAt: lnurl.fetchedAt }, false);
}
}
var zapperDump = (lnurls, debug) => {
return async (dirtyKeys, cache) => {
const entries = [];
for (const pubkey of dirtyKeys) {
const entry = cache.get(pubkey);
if (entry) {
entries.push({
pubkey,
...entry
});
}
}
if (entries.length) {
debug(`Saving ${entries.length} zapper cache entries to database`);
await lnurls.bulkPut(entries);
}
dirtyKeys.clear();
};
};
// src/lru-cache.ts
import { LRUCache } from "typescript-lru-cache";
var CacheHandler = class {
cache;
dirtyKeys = /* @__PURE__ */ new Set();
options;
debug;
indexes;
isSet = false;
maxSize = 0;
constructor(options) {
this.debug = options.debug;
this.options = options;
this.maxSize = options.maxSize;
if (options.maxSize > 0) {
this.cache = new LRUCache({ maxSize: options.maxSize });
setInterval(() => this.dump().catch(console.error), 1e3 * 10);
}
this.indexes = /* @__PURE__ */ new Map();
}
getSet(key) {
return this.cache?.get(key);
}
/**
* Get all entries that match the filter.
*/
getAllWithFilter(filter) {
const ret = /* @__PURE__ */ new Map();
this.cache?.forEach((val, key) => {
if (filter(key, val)) {
ret.set(key, val);
}
});
return ret;
}
get(key) {
return this.cache?.get(key);
}
async getWithFallback(key, table) {
let entry = this.get(key);
if (!entry) {
entry = await table.get(key);
if (entry) {
this.set(key, entry);
}
}
return entry;
}
async getManyWithFallback(keys, table) {
const entries = [];
const missingKeys = [];
for (const key of keys) {
const entry = this.get(key);
if (entry) entries.push(entry);
else missingKeys.push(key);
}
if (entries.length > 0) {
this.debug(`Cache hit for keys ${entries.length} and miss for ${missingKeys.length} keys`);
}
if (missingKeys.length > 0) {
const startTime = Date.now();
const missingEntries = await table.bulkGet(missingKeys);
const endTime = Date.now();
let foundKeys = 0;
for (const entry of missingEntries) {
if (entry) {
this.set(entry.id, entry);
entries.push(entry);
foundKeys++;
}
}
this.debug(
`Time spent querying database: ${endTime - startTime}ms for ${missingKeys.length} keys, which added ${foundKeys} entries to the cache`
);
}
return entries;
}
add(key, value, dirty = true) {
const existing = this.get(key) ?? /* @__PURE__ */ new Set();
existing.add(value);
this.cache?.set(key, existing);
if (dirty) this.dirtyKeys.add(key);
}
set(key, value, dirty = true) {
this.cache?.set(key, value);
if (dirty) this.dirtyKeys.add(key);
for (const [attribute, index] of this.indexes.entries()) {
const indexKey = value[attribute];
if (indexKey) {
const indexValue = index.get(indexKey) || /* @__PURE__ */ new Set();
indexValue.add(key);
index.set(indexKey, indexValue);
}
}
}
size() {
return this.cache?.size || 0;
}
delete(key) {
this.cache?.delete(key);
this.dirtyKeys.add(key);
}
async dump() {
if (this.dirtyKeys.size > 0 && this.cache) {
await this.options.dump(this.dirtyKeys, this.cache);
this.dirtyKeys.clear();
}
}
addIndex(attribute) {
this.indexes.set(attribute, new LRUCache({ maxSize: this.options.maxSize }));
}
getFromIndex(index, key) {
const ret = /* @__PURE__ */ new Set();
const indexValues = this.indexes.get(index);
if (indexValues) {
const values = indexValues.get(key);
if (values) {
for (const key2 of values.values()) {
const entry = this.get(key2);
if (entry) ret.add(entry);
}
}
}
return ret;
}
};
// src/index.ts
var INDEXABLE_TAGS_LIMIT = 10;
var NDKCacheAdapterDexie = class {
debug;
locking = false;
ready = false;
profiles;
zappers;
nip05s;
events;
eventTags;
relayInfo;
unpublishedEvents;
warmedUp = false;
warmUpPromise;
devMode = false;
saveSig;
_onReady;
constructor(opts = {}) {
createDatabase(opts.dbName || "ndk");
this.debug = opts.debug || createDebug2("ndk:dexie-adapter");
this.saveSig = opts.saveSig || false;
this.profiles = new CacheHandler({
maxSize: opts.profileCacheSize || 1e5,
dump: profilesDump(db.profiles, this.debug),
debug: this.debug
});
this.zappers = new CacheHandler({
maxSize: opts.zapperCacheSize || 200,
dump: zapperDump(db.lnurl, this.debug),
debug: this.debug
});
this.nip05s = new CacheHandler({
maxSize: opts.nip05CacheSize || 1e3,
dump: nip05Dump(db.nip05, this.debug),
debug: this.debug
});
this.events = new CacheHandler({
maxSize: opts.eventCacheSize || 5e4,
dump: eventsDump(db.events, this.debug),
debug: this.debug
});
this.events.addIndex("pubkey");
this.events.addIndex("kind");
this.eventTags = new CacheHandler({
maxSize: opts.eventTagsCacheSize || 1e5,
dump: eventTagsDump(db.eventTags, this.debug),
debug: this.debug
});
this.relayInfo = new CacheHandler({
maxSize: 500,
debug: this.debug,
dump: relayInfoDump(db.relayStatus, this.debug)
});
this.unpublishedEvents = new CacheHandler({
maxSize: 5e3,
debug: this.debug,
dump: unpublishedEventsDump(db.unpublishedEvents, this.debug)
});
const profile = (label, fn) => {
const start = Date.now();
return fn().then(() => {
const end = Date.now();
this.debug(label, "took", end - start, "ms");
});
};
const startTime = Date.now();
this.warmUpPromise = Promise.allSettled([
profile("profilesWarmUp", () => profilesWarmUp(this.profiles, db.profiles)),
profile("zapperWarmUp", () => zapperWarmUp(this.zappers, db.lnurl)),
profile("nip05WarmUp", () => nip05WarmUp(this.nip05s, db.nip05)),
profile("relayInfoWarmUp", () => relayInfoWarmUp(this.relayInfo, db.relayStatus)),
profile(
"unpublishedEventsWarmUp",
() => unpublishedEventsWarmUp(this.unpublishedEvents, db.unpublishedEvents)
),
profile("eventsWarmUp", () => eventsWarmUp(this.events, db.events)),
profile("eventTagsWarmUp", () => eventTagsWarmUp(this.eventTags, db.eventTags))
]);
this.warmUpPromise.then(() => {
const endTime = Date.now();
this.warmedUp = true;
this.ready = true;
this.locking = true;
this.debug("Warm up completed, time", endTime - startTime, "ms");
if (this._onReady) this._onReady();
});
}
onReady(callback) {
this._onReady = callback;
}
async query(subscription) {
if (!this.warmedUp) {
const startTime2 = Date.now();
await this.warmUpPromise;
this.debug("froze query for", Date.now() - startTime2, "ms", subscription.filters);
}
const startTime = Date.now();
subscription.filters.map((filter) => this.processFilter(filter, subscription));
const dur = Date.now() - startTime;
if (dur > 100) this.debug("query took", dur, "ms", subscription.filter);
return [];
}
async fetchProfile(pubkey) {
if (!this.profiles) return null;
const user = await this.profiles.getWithFallback(pubkey, db.profiles);
return user;
}
fetchProfileSync(pubkey) {
if (!this.profiles) return null;
const user = this.profiles.get(pubkey);
return user;
}
async getProfiles(fn) {
if (!this.profiles) return;
return this.profiles.getAllWithFilter(fn);
}
saveProfile(pubkey, profile) {
const existingValue = this.profiles.get(pubkey);
if (existingValue?.created_at && profile.created_at && existingValue.created_at >= profile.created_at) {
return;
}
const cachedAt = Math.floor(Date.now() / 1e3);
this.profiles.set(pubkey, { pubkey, ...profile, cachedAt });
this.debug("Saved profile for pubkey", pubkey, profile);
}
async loadNip05(nip05, maxAgeForMissing = 3600) {
const cache = this.nip05s?.get(nip05);
if (cache) {
if (cache.profile === null) {
if (cache.fetchedAt + maxAgeForMissing * 1e3 < Date.now()) return "missing";
return null;
}
try {
return JSON.parse(cache.profile);
} catch (_e) {
return "missing";
}
}
const nip = await db.nip05.get({ nip05 });
if (!nip) return "missing";
const now = Date.now();
if (nip.profile === null) {
if (nip.fetchedAt + maxAgeForMissing * 1e3 < now) return "missing";
return null;
}
try {
return JSON.parse(nip.profile);
} catch (_e) {
return "missing";
}
}
async saveNip05(nip05, profile) {
try {
const document = profile ? JSON.stringify(profile) : null;
this.nip05s.set(nip05, { profile: document, fetchedAt: Date.now() });
} catch (error) {
console.error("Failed to save NIP-05 profile for nip05:", nip05, error);
}
}
async loadUsersLNURLDoc(pubkey, maxAgeInSecs = 86400, maxAgeForMissing = 3600) {
const cache = this.zappers?.get(pubkey);
if (cache) {
if (cache.document === null) {
if (cache.fetchedAt + maxAgeForMissing * 1e3 < Date.now()) return "missing";
return null;
}
try {
return JSON.parse(cache.document);
} catch (_e) {
return "missing";
}
}
const lnurl = await db.lnurl.get({ pubkey });
if (!lnurl) return "missing";
const now = Date.now();
if (lnurl.fetchedAt + maxAgeInSecs * 1e3 < now) return "missing";
if (lnurl.document === null) {
if (lnurl.fetchedAt + maxAgeForMissing * 1e3 < now) return "missing";
return null;
}
try {
return JSON.parse(lnurl.document);
} catch (_e) {
return "missing";
}
}
async saveUsersLNURLDoc(pubkey, doc) {
try {
const document = doc ? JSON.stringify(doc) : null;
this.zappers?.set(pubkey, { document, fetchedAt: Date.now() });
} catch (error) {
console.error("Failed to save LNURL document for pubkey:", pubkey, error);
}
}
processFilter(filter, subscription) {
const _filter = { ...filter };
_filter.limit = void 0;
const filterKeys = new Set(Object.keys(_filter || {}));
filterKeys.delete("since");
filterKeys.delete("limit");
filterKeys.delete("until");
try {
if (this.byNip33Query(filterKeys, filter, subscription)) return;
if (this.byAuthors(filter, subscription)) return;
if (this.byIdsQuery(filter, subscription)) return;
if (this.byTags(filter, subscription)) return;
if (this.byKinds(filterKeys, filter, subscription)) return;
} catch (error) {
console.error(error);
}
}
async deleteEventIds(eventIds) {
eventIds.forEach((id) => this.events.delete(id));
await db.events.where({ id: eventIds }).delete();
}
addUnpublishedEvent = addUnpublishedEvent.bind(this);
getUnpublishedEvents = () => getUnpublishedEvents(db.unpublishedEvents);
discardUnpublishedEvent = (id) => discardUnpublishedEvent(db.unpublishedEvents, id);
async setEvent(event, _filters, relay) {
if (event.kind === 0) {
if (!this.profiles) return;
try {
const profile = profileFromEvent(event);
this.saveProfile(event.pubkey, profile);
} catch {
this.debug(`Failed to save profile for pubkey: ${event.pubkey}`);
}
}
let addEvent = true;
if (event.isParamReplaceable()) {
const existingEvent = this.events.get(event.tagId());
if (existingEvent && event.created_at && existingEvent.createdAt > event.created_at) {
addEvent = false;
}
}
if (addEvent) {
const eventData = {
id: event.tagId(),
pubkey: event.pubkey,
kind: event.kind,
createdAt: event.created_at ?? Date.now(),
relay: relay?.url,
event: event.serialize(this.saveSig, true)
};
if (this.saveSig && event.sig) {
eventData.sig = event.sig;
}
this.events.set(event.tagId(), eventData);
const indexableTags = getIndexableTags(event);
for (const tag of indexableTags) {
this.eventTags.add(tag[0] + tag[1], event.tagId());
}
}
}
updateRelayStatus(url, info) {
const val = { url, updatedAt: Date.now(), ...info };
this.relayInfo.set(url, val);
}
getRelayStatus(url) {
const a = this.relayInfo.get(url);
if (a) {
return {
lastConnectedAt: a.lastConnectedAt,
dontConnectBefore: a.dontConnectBefore
};
}
}
/**
* Searches by authors
*/
byAuthors(filter, subscription) {
if (!filter.authors) return false;
let _total = 0;
for (const pubkey of filter.authors) {
let events = Array.from(this.events.getFromIndex("pubkey", pubkey));
if (filter.kinds) events = events.filter((e) => filter.kinds?.includes(e.kind));
foundEvents(subscription, events, filter);
_total += events.length;
}
return true;
}
/**
* Searches by ids
*/
byIdsQuery(filter, subscription) {
if (filter.ids) {
for (const id of filter.ids) {
const event = this.events.get(id);
if (event) foundEvent(subscription, event, event.relay, filter);
}
return true;
}
return false;
}
/**
* Searches by NIP-33
*/
byNip33Query(filterKeys, filter, subscription) {
const f = ["#d", "authors", "kinds"];
const hasAllKeys = filterKeys.size === f.length && f.every((k) => filterKeys.has(k));
if (hasAllKeys && filter.kinds && filter.authors) {
for (const kind of filter.kinds) {
const replaceableKind = kind >= 3e4 && kind < 4e4;
if (!replaceableKind) continue;
for (const author of filter.authors) {
for (const dTag of filter["#d"]) {
const replaceableId = `${kind}:${author}:${dTag}`;
const event = this.events.get(replaceableId);
if (event) foundEvent(subscription, event, event.relay, filter);
}
}
}
return true;
}
return false;
}
/**
* Searches by tags and optionally filters by tags
*/
byTags(filter, subscription) {
const tagFilters = Object.entries(filter).filter(([filter2]) => filter2.startsWith("#") && filter2.length === 2).map(([filter2, values]) => [filter2[1], values]);
if (tagFilters.length === 0) return false;
for (const [tag, values] of tagFilters) {
for (const value of values) {
const tagValue = tag + value;
const eventIds = this.eventTags.getSet(tagValue);
if (!eventIds) continue;
eventIds.forEach((id) => {
const event = this.events.get(id);
if (!event) return;
if (!filter.kinds || filter.kinds.includes(event.kind)) {
foundEvent(subscription, event, event.relay, filter);
}
});
}
}
return true;
}
byKinds(filterKeys, filter, subscription) {
if (!filter.kinds || filterKeys.size !== 1 || !filterKeys.has("kinds")) return false;
const limit = filter.limit || 500;
let totalEvents = 0;
const processedEventIds = /* @__PURE__ */ new Set();
const sortedKinds = [...filter.kinds].sort(
(a, b) => (this.events.indexes.get("kind")?.get(a)?.size || 0) - (this.events.indexes.get("kind")?.get(b)?.size || 0)
);
for (const kind of sortedKinds) {
const events = this.events.getFromIndex("kind", kind);
for (const event of events) {
if (processedEventIds.has(event.id)) continue;
processedEventIds.add(event.id);
foundEvent(subscription, event, event.relay, filter);
totalEvents++;
if (totalEvents >= limit) break;
}
if (totalEvents >= limit) break;
}
return true;
}
};
function foundEvents(subscription, events, filter) {
if (filter?.limit && events.length > filter.limit) {
events = events.sort((a, b) => b.createdAt - a.createdAt).slice(0, filter.limit);
}
for (const event of events) {
foundEvent(subscription, event, event.relay, filter);
}
}
function foundEvent(subscription, event, relayUrl, filter) {
try {
const deserializedEvent = deserialize(event.event);
if (filter && !matchFilter(filter, deserializedEvent)) return;
const ndkEvent = new NDKEvent2(void 0, deserializedEvent);
const relay = relayUrl ? subscription.pool.getRelay(relayUrl, false) : void 0;
ndkEvent.relay = relay;
subscription.eventReceived(ndkEvent, relay, true);
} catch (e) {
console.error("failed to deserialize event", e);
}
}
function getIndexableTags(event) {
const indexableTags = [];
if (event.kind === 3) return [];
for (const tag of event.tags) {
if (tag[0].length !== 1) continue;
indexableTags.push(tag);
if (indexableTags.length >= INDEXABLE_TAGS_LIMIT) return [];
}
return indexableTags;
}
export {
db,
NDKCacheAdapterDexie as default,
foundEvent,
foundEvents
};