@instantdb/core
Version:
Instant's core local abstraction
434 lines • 15 kB
JavaScript
// PersistedObjects save data outside of memory.
//
// When we load a persisted object, it's possible we call `set`
// before we finish loading. To address we handle set in two ways:
//
// 1. Before load
// We simply update currentValue in memory
//
// 2. After load
// We update currentValue in memory and in storage
//
// Each PersistedObject provides it's own `onMerge`
// function to handle the merge of data from storage and memory
// on load
// Uses `requestIdleCallback` if available, otherwise calls the
// callback immediately
import { create } from 'mutative';
function safeIdleCallback(cb, timeout) {
if (typeof requestIdleCallback === 'undefined') {
cb();
}
else {
requestIdleCallback(cb, { timeout });
}
}
export const META_KEY = '__meta';
export class StoreInterface {
constructor(appId, storeName) { }
}
export class PersistedObject {
currentValue;
_subs = [];
_persister;
_merge;
serialize;
parse;
_saveThrottleMs;
_idleCallbackMaxWaitMs;
_nextSave = null;
_nextGc = null;
_pendingSaveKeys = new Set();
_loadedKeys = new Set();
_loadingKeys;
_objectSize;
_log;
onKeyLoaded;
_version = 0;
_meta = {
isLoading: true,
onLoadCbs: [],
value: null,
error: null,
attempts: 0,
};
_gcOpts;
constructor(opts) {
this._persister = opts.persister;
this._merge = opts.merge;
this.serialize = opts.serialize;
this.parse = opts.parse;
this._objectSize = opts.objectSize;
this._log = opts.logger;
this._saveThrottleMs = opts.saveThrottleMs ?? 100;
this._idleCallbackMaxWaitMs = opts.idleCallbackMaxWaitMs ?? 1000;
this._gcOpts = opts.gc;
this.currentValue = {};
this._loadedKeys = new Set();
this._loadingKeys = {};
this._initMeta();
if (opts.preloadEntryCount) {
this._preloadEntries(opts.preloadEntryCount);
}
}
async _initMeta() {
if (this._meta.loadingPromise) {
await this._meta.loadingPromise;
}
try {
const p = this._persister.getItem(META_KEY);
this._meta.loadingPromise = p;
const v = await p;
this._meta.isLoading = false;
this._meta.error = null;
this._meta.loadingPromise = null;
this._meta.attempts = 0;
const existingObjects = this._meta.value?.objects ?? {};
const value = v ?? {};
const objects = value.objects ?? {};
// Merge the values from storage with in-memory values
this._meta.value = {
...value,
objects: { ...existingObjects, ...objects },
};
}
catch (e) {
this._meta.error = e;
this._meta.attempts++;
this._meta.loadingPromise = null;
}
}
async _getMeta() {
if (this._meta.value) {
return this._meta.value;
}
if (this._meta.loadingPromise) {
await this._meta.loadingPromise;
return this._meta.value;
}
this._initMeta();
await this._meta.loadingPromise;
return this._meta.value;
}
async _refreshMeta() {
await this._initMeta();
return this._meta.value;
}
async _preloadEntries(n) {
const meta = await this.waitForMetaToLoad();
if (!meta)
return;
const entries = Object.entries(meta.objects);
entries.sort(([_k_a, a_meta], [_k_b, b_meta]) => {
return b_meta.updatedAt - a_meta.updatedAt;
});
for (const [k] of entries.slice(0, n)) {
this._loadKey(k);
}
}
async _getFromStorage(key) {
try {
const data = await this._persister.getItem(key);
if (!data) {
return data;
}
const parsed = this.parse(key, data);
return parsed;
}
catch (e) {
console.error(`Unable to read from storage for key=${key}`, e);
return null;
}
}
async waitForKeyToLoad(k) {
if (this._loadedKeys.has(k)) {
return this.currentValue[k];
}
await (this._loadingKeys[k] || this._loadKey(k));
return this.currentValue[k];
}
// Used for tests
async waitForMetaToLoad() {
return this._getMeta();
}
// Unloads the key so that it can be garbage collected, but does not
// delete it. Removes the key from currentValue.
unloadKey(k) {
this._loadedKeys.delete(k);
delete this._loadingKeys[k];
delete this.currentValue[k];
}
async _loadKey(k) {
if (this._loadedKeys.has(k) || k in this._loadingKeys)
return;
const p = this._getFromStorage(k);
this._loadingKeys[k] = p;
const value = await p;
delete this._loadingKeys[k];
this._loadedKeys.add(k);
if (value) {
const merged = this._merge(k, value, this.currentValue[k]);
if (merged) {
this.currentValue[k] = merged;
}
}
this.onKeyLoaded && this.onKeyLoaded(k);
}
// Returns a promise with a number so that we can wait for flush
// to finish in the tests. The number is the number of operations
// it performed, but it's mostly there so that typescript will warn
// us if we forget to retun the promise from the function.
_writeToStorage(opts) {
const promises = [];
const skipGc = opts?.skipGc;
if (this._meta.isLoading) {
// Wait for meta to load and try again, give it a delay so that
// we don't spend too much time retrying
const p = new Promise((resolve, reject) => {
setTimeout(() => this._enqueuePersist(opts
? { ...opts, attempts: (opts.attempts || 0) + 1 }
: { attempts: 1 })
.then(resolve)
.catch(reject), 10 + (opts?.attempts ?? 0) * 1000);
});
promises.push(p);
return Promise.all(promises).then((vs) => vs.reduce((acc, x) => acc + x, 0));
}
const metaValue = this._meta.value;
if (!metaValue) {
// If it's not loading and we don't have the data, then there
// must be an error and we're not going to be able to save until
// the error is resolved elsewhere.
return Promise.resolve(0);
}
const keysToDelete = [];
const keysToUpdate = [];
for (const k of this._pendingSaveKeys) {
if (!(k in this.currentValue)) {
keysToDelete.push(k);
delete metaValue.objects[k];
}
else {
keysToUpdate.push(k);
}
}
for (const k of keysToDelete) {
const p = this._persister.removeItem(k);
promises.push(p.then(() => 1));
this._loadedKeys.delete(k);
this._pendingSaveKeys.delete(k);
}
const keysToLoad = [];
const kvPairs = [[META_KEY, metaValue]];
const metaObjects = metaValue.objects ?? {};
metaValue.objects = metaObjects;
for (const k of keysToUpdate) {
if (this._loadedKeys.has(k)) {
const serializedV = this.serialize(k, this.currentValue[k]);
kvPairs.push([k, serializedV]);
const size = this._objectSize(serializedV);
const m = metaObjects[k] ?? {
createdAt: Date.now(),
updatedAt: Date.now(),
size,
};
m.updatedAt = Date.now();
m.size = size;
metaObjects[k] = m;
this._pendingSaveKeys.delete(k);
}
else {
keysToLoad.push(k);
}
}
const p = this._persister.multiSet(kvPairs);
promises.push(p.then(() => 1));
// For the keys that haven't loaded, load the key then try
// persisting again. We don't want to do any async work here
// or else we might end up saving older copies of the data to
// the store.
for (const k of keysToLoad) {
const p = this._loadKey(k).then(() => this._enqueuePersist(opts));
promises.push(p);
}
if (!skipGc) {
this.gc();
}
return Promise.all(promises).then((vs) => {
return vs.reduce((acc, x) => acc + x, 0);
});
}
async flush() {
if (!this._nextSave) {
return;
}
clearTimeout(this._nextSave);
this._nextSave = null;
const p = this._writeToStorage();
return p;
}
async _gc() {
if (!this._gcOpts) {
return;
}
const keys = new Set(await this._persister.getAllKeys());
keys.delete(META_KEY);
// Keys we can't delete
const sacredKeys = new Set(Object.keys(this.currentValue));
for (const k of Object.keys(this._loadingKeys)) {
sacredKeys.add(k);
}
for (const k of this._loadedKeys) {
sacredKeys.add(k);
}
// Refresh meta from the store so that we're less likely to
// clobber data from other tabs
const meta = await this._refreshMeta();
if (!meta) {
this._log.info('Could not gc because we were not able to load meta');
return;
}
const promises = [];
const deets = {
gcOpts: this._gcOpts,
keys,
sacredKeys,
removed: [],
metaRemoved: [],
removedMissingCount: 0,
removedOldCount: 0,
removedThresholdCount: 0,
removedSizeCount: 0,
};
// First, remove all keys we don't know about
for (const key of keys) {
if (sacredKeys.has(key) || key in meta.objects) {
continue;
}
this._log.info('Lost track of key in meta', key);
promises.push(this._persister.removeItem(key));
deets.removed.push(key);
deets.removedMissingCount++;
}
// Remove anything over the max age
const now = Date.now();
for (const [k, m] of Object.entries(meta.objects)) {
if (!sacredKeys.has(k) &&
m.updatedAt < now - this._gcOpts.maxAgeMs) {
promises.push(this._persister.removeItem(k));
delete meta.objects[k];
deets.removed.push(k);
deets.removedOldCount++;
}
}
// Keep queries under max queries
const maxEntries = Object.entries(meta.objects);
maxEntries.sort(([_k_a, a_meta], [_k_b, b_meta]) => {
return a_meta.updatedAt - b_meta.updatedAt;
});
const deletableMaxEntries = maxEntries.filter(([x]) => !sacredKeys.has(x));
if (maxEntries.length > this._gcOpts.maxEntries) {
for (const [k] of deletableMaxEntries.slice(0, maxEntries.length - this._gcOpts.maxEntries)) {
promises.push(this._persister.removeItem(k));
delete meta.objects[k];
deets.removed.push(k);
deets.removedThresholdCount++;
}
}
// Remove oldest entries until we are under max size
const delEntries = Object.entries(meta.objects);
delEntries.sort(([_k_a, a_meta], [_k_b, b_meta]) => {
return a_meta.updatedAt - b_meta.updatedAt;
});
const deletableDelEntries = delEntries.filter(([x]) => !sacredKeys.has(x));
let currentSize = delEntries.reduce((acc, [_k, m]) => {
return acc + m.size;
}, 0);
while (currentSize > 0 &&
currentSize > this._gcOpts.maxSize &&
deletableDelEntries.length) {
const [[k, m]] = deletableDelEntries.splice(0, 1);
currentSize -= m.size;
promises.push(this._persister.removeItem(k));
delete meta.objects[k];
deets.removed.push(k);
deets.removedSizeCount++;
}
// Update meta to remove keys that are no longer in the store
for (const k of Object.keys(meta.objects)) {
if (!keys.has(k) && !sacredKeys.has(k)) {
delete meta.objects[k];
}
}
if (deets.removed.length || deets.metaRemoved.length) {
// Trigger a flush of the meta
promises.push(this._enqueuePersist({ skipGc: true }));
}
this._log.info('Completed GC', deets);
await Promise.all(promises);
return deets;
}
// Schedules a GC to run in one minute (unless it is already scheduled)
gc() {
if (this._nextGc) {
return;
}
this._nextGc = setTimeout(() => {
safeIdleCallback(() => {
this._nextGc = null;
this._gc();
}, 30 * 1000);
},
// 1 minute + some jitter to keep multiple tabs from running at same time
1000 * 60 + Math.random() * 500);
}
_enqueuePersist(opts) {
return new Promise((resolve, reject) => {
if (this._nextSave) {
resolve(0);
return;
}
this._nextSave = setTimeout(() => {
safeIdleCallback(() => {
this._nextSave = null;
this._writeToStorage(opts).then(resolve).catch(reject);
}, this._idleCallbackMaxWaitMs);
}, this._saveThrottleMs);
});
}
version() {
return this._version;
}
// Takes a function that updates the store in place.
// Uses `mutative` to get a list of keys that were changed
// so that we know which entries we need to persist to the store.
updateInPlace(f) {
this._version++;
const [state, patches] = create(this.currentValue, f, {
enablePatches: true,
});
for (const patch of patches) {
const k = patch.path[0];
if (k && typeof k === 'string') {
this._pendingSaveKeys.add(k);
if (!this._loadedKeys.has(k)) {
this._loadKey(k);
}
}
}
this.currentValue = state;
this._enqueuePersist();
for (const cb of this._subs) {
cb(this.currentValue);
}
return state;
}
subscribe(cb) {
this._subs.push(cb);
cb(this.currentValue);
return () => {
this._subs = this._subs.filter((x) => x !== cb);
};
}
}
//# sourceMappingURL=PersistedObject.js.map