@instantdb/core
Version:
Instant's core local abstraction
1,274 lines • 84.1 kB
JavaScript
// @ts-check
import weakHash from "./utils/weakHash.js";
import instaql from "./instaql.js";
import * as instaml from "./instaml.js";
import * as s from "./store.js";
import uuid from "./utils/id.js";
import IndexedDBStorage from "./IndexedDBStorage.js";
import WindowNetworkListener from './WindowNetworkListener.js';
import * as authAPI from "./authAPI.js";
import * as StorageApi from "./StorageAPI.js";
import * as flags from "./utils/flags.js";
import { buildPresenceSlice, hasPresenceResponseChanged } from "./presence.js";
import { Deferred } from "./utils/Deferred.js";
import { PersistedObject } from "./utils/PersistedObject.js";
import { extractTriples } from './model/instaqlResult.js';
import { areObjectsDeepEqual, assocInMutative, dissocInMutative, insertInMutative, } from './utils/object.js';
import { createLinkIndex } from "./utils/linkIndex.js";
import version from "./version.js";
import { create } from 'mutative';
import createLogger from "./utils/log.js";
import { validateQuery } from "./queryValidation.js";
import { validateTransactions } from "./transactionValidation.js";
import { InstantError } from "./InstantError.js";
import { InstantAPIError } from "./utils/fetch.js";
import { validate as validateUUID } from 'uuid';
import { WSConnection, SSEConnection } from "./Connection.js";
import { SyncTable } from "./SyncTable.js";
import { InstantStream } from "./Stream.js";
/** @typedef {import('./utils/log.ts').Logger} Logger */
/** @typedef {import('./Connection.ts').Connection} Connection */
/** @typedef {import('./Connection.ts').TransportType} TransportType */
/** @typedef {import('./Connection.ts').EventSourceConstructor} EventSourceConstructor */
/** @typedef {import('./reactorTypes.ts').QuerySub} QuerySub */
/** @typedef {import('./reactorTypes.ts').QuerySubInStorage} QuerySubInStorage */
/** @typedef {import('./clientTypes.ts').User} User */
/** @typedef {import('./clientTypes.ts').AuthState} AuthState */
/** @typedef {import('./framework.ts').FrameworkClient} FrameworkClient */
export const STATUS = {
CONNECTING: 'connecting',
OPENED: 'opened',
AUTHENTICATED: 'authenticated',
CLOSED: 'closed',
ERRORED: 'errored',
};
const QUERY_ONCE_TIMEOUT = 30_000;
const PENDING_TX_CLEANUP_TIMEOUT = 30_000;
const PENDING_MUTATION_CLEANUP_THRESHOLD = 200;
const ONE_MIN_MS = 1_000 * 60;
const defaultConfig = {
apiURI: 'https://api.instantdb.com',
websocketURI: 'wss://api.instantdb.com/runtime/session',
};
// Param that the backend adds if this is an oauth redirect
const OAUTH_REDIRECT_PARAM = '_instant_oauth_redirect';
const currentUserKey = `currentUser`;
/**
* @param {Object} config
* @param {TransportType} config.transportType
* @param {string} config.appId
* @param {string} config.apiURI
* @param {string} config.wsURI
* @param {EventSourceConstructor} config.EventSourceImpl
* @returns {WSConnection | SSEConnection}
*/
function createTransport({ transportType, appId, apiURI, wsURI, EventSourceImpl, }) {
if (!EventSourceImpl) {
return new WSConnection(`${wsURI}?app_id=${appId}`);
}
switch (transportType) {
case 'ws':
return new WSConnection(`${wsURI}?app_id=${appId}`);
case 'sse':
return new SSEConnection(EventSourceImpl, `${apiURI}/runtime/sse?app_id=${appId}`);
default:
throw new Error('Unknown transport type ' + transportType);
}
}
function isClient() {
const hasWindow = typeof window !== 'undefined';
// this checks if we are running in a chrome extension
// @ts-expect-error
const isChrome = typeof chrome !== 'undefined';
return hasWindow || isChrome;
}
const ignoreLogging = {
'set-presence': true,
'set-presence-ok': true,
'refresh-presence': true,
'patch-presence': true,
};
/**
* @param {QuerySubInStorage} x
* @param {boolean | null} useDateObjects
* @returns {QuerySub}
*/
function querySubFromStorage(x, useDateObjects) {
const v = typeof x === 'string' ? JSON.parse(x) : x;
if (v?.result?.store) {
const attrsStore = s.attrsStoreFromJSON(v.result.attrsStore, v.result.store);
if (attrsStore) {
const storeJSON = v.result.store;
v.result.store = s.fromJSON(attrsStore, {
...storeJSON,
useDateObjects: useDateObjects,
});
v.result.attrsStore = attrsStore;
}
}
return v;
}
/**
*
* @param {string} _key
* @param {QuerySub} sub
* @returns QuerySubInStorage
*/
function querySubToStorage(_key, sub) {
const { result, ...rest } = sub;
const jsonSub = /** @type {import('./reactorTypes.ts').QuerySubInStorage} */ (rest);
if (result) {
/** @type {import('./reactorTypes.ts').QuerySubResultInStorage} */
const jsonResult = {
...result,
store: s.toJSON(result.store),
attrsStore: result.attrsStore.toJSON(),
};
jsonSub.result = jsonResult;
}
return jsonSub;
}
function kvFromStorage(key, x) {
switch (key) {
case 'pendingMutations':
return new Map(typeof x === 'string' ? JSON.parse(x) : x);
default:
return x;
}
}
function kvToStorage(key, x) {
switch (key) {
case 'pendingMutations':
return [...x.entries()];
default:
return x;
}
}
function onMergeQuerySub(_k, storageSub, inMemorySub) {
const storageResult = storageSub?.result;
const memoryResult = inMemorySub?.result;
if (storageResult && !memoryResult && inMemorySub) {
inMemorySub.result = storageResult;
}
return inMemorySub || storageSub;
}
function sortedMutationEntries(entries) {
return [...entries].sort((a, b) => {
const [ka, muta] = a;
const [kb, mutb] = b;
const a_order = muta.order || 0;
const b_order = mutb.order || 0;
if (a_order == b_order) {
return ka < kb ? -1 : ka > kb ? 1 : 0;
}
return a_order - b_order;
});
}
/**
* @template {import('./presence.ts').RoomSchemaShape} [RoomSchema = {}]
*/
export default class Reactor {
/** @type {s.AttrsStore | undefined} */
attrs;
_isOnline = true;
_isShutdown = false;
status = STATUS.CONNECTING;
/** @type {PersistedObject<string, QuerySub, QuerySubInStorage>} */
querySubs;
/** @type {PersistedObject} */
kv;
/** @type {SyncTable} */
_syncTable;
/** @type {InstantStream} */
_instantStream;
/** @type {Record<string, Array<{ q: any, cb: (data: any) => any }>>} */
queryCbs = {};
/** @type {Record<string, Array<{ q: any, eventId: string, dfd: Deferred }>>} */
queryOnceDfds = {};
authCbs = [];
attrsCbs = [];
mutationErrorCbs = [];
connectionStatusCbs = [];
config;
mutationDeferredStore = new Map();
_reconnectTimeoutId = null;
_reconnectTimeoutMs = 0;
/** @type {Connection} */
_transport;
/** @type {TransportType} */
_transportType = 'ws';
/** @type {EventSourceConstructor} */
_EventSource;
/** @type {boolean | null} */
_wsOk = null;
_localIdPromises = {};
_errorMessage = null;
/** @type {Promise<null | {error: {message: string}}> | null}**/
_oauthCallbackResponse = null;
/** @type {null | import('./utils/linkIndex.ts').LinkIndex}} */
_linkIndex = null;
/** @type BroadcastChannel | undefined */
_broadcastChannel;
/** @type {Record<string, {roomType: string; isConnected: boolean; error: any}>} */
_rooms = {};
/** @type {Record<string, boolean>} */
_roomsPendingLeave = {};
_presence = {};
_broadcastQueue = [];
_broadcastSubs = {};
/** @type {{isLoading: boolean; error: any | undefined, user: any | undefined}} */
_currentUserCached = { isLoading: true, error: undefined, user: undefined };
_beforeUnloadCbs = [];
_dataForQueryCache = {};
/** @type {Logger} */
_log;
_pendingTxCleanupTimeout;
_pendingMutationCleanupThreshold;
_inFlightMutationEventIds = new Set();
/** @type FrameworkClient | null */
_frameworkClient = null;
constructor(config, Storage = IndexedDBStorage, NetworkListener = WindowNetworkListener, versions, EventSourceConstructor) {
this._EventSource = EventSourceConstructor;
this.config = { ...defaultConfig, ...config };
this.queryCacheLimit = this.config.queryCacheLimit ?? 10;
this._pendingTxCleanupTimeout =
this.config.pendingTxCleanupTimeout ?? PENDING_TX_CLEANUP_TIMEOUT;
this._pendingMutationCleanupThreshold =
this.config.pendingMutationCleanupThreshold ??
PENDING_MUTATION_CLEANUP_THRESHOLD;
this._log = createLogger(config.verbose || flags.devBackend || flags.instantLogs, () => this._reactorStats());
this.versions = { ...(versions || {}), '@instantdb/core': version };
if (this.config.schema) {
this._linkIndex = createLinkIndex(this.config.schema);
}
// This is to protect us against running
// server-side.
if (!isClient()) {
return;
}
if (!config.appId) {
throw new Error('Instant must be initialized with an appId.');
}
if (!validateUUID(config.appId)) {
throw new Error(`Instant must be initialized with a valid appId. \`${config.appId}\` is not a valid uuid.`);
}
if (typeof BroadcastChannel === 'function') {
this._broadcastChannel = new BroadcastChannel('@instantdb');
this._broadcastChannel.addEventListener('message', async (e) => {
try {
if (e.data?.type === 'auth') {
const res = await this.getCurrentUser({
forceReadFromStorage: true,
});
await this.updateUser(res.user).catch((error) => {
this._log.error('[error] update user', error);
});
}
}
catch (e) {
this._log.error('[error] handle broadcast channel', e);
}
});
}
this._initStorage(Storage);
this._syncTable = new SyncTable(this._trySendAuthed.bind(this), new Storage(this.config.appId, 'syncSubs'), {
useDateObjects: this.config.useDateObjects,
}, this._log, (triples) => {
return s.createStore(this.ensureAttrs(), triples, this.config.enableCardinalityInference, this.config.useDateObjects);
}, () => this.ensureAttrs());
this._instantStream = new InstantStream({
WStream: this.config.WritableStream || WritableStream,
RStream: this.config.ReadableStream || ReadableStream,
trySend: this._trySendAuthed.bind(this),
log: this._log,
});
this._oauthCallbackResponse = this._oauthLoginInit();
// kick off a request to cache it
this.getCurrentUser().then((userInfo) => {
this.syncUserToEndpoint(userInfo.user);
});
setInterval(async () => {
const currentUser = await this.getCurrentUser();
this.syncUserToEndpoint(currentUser.user);
}, ONE_MIN_MS);
NetworkListener.getIsOnline().then((isOnline) => {
this._isOnline = isOnline;
this._startSocket();
NetworkListener.listen((isOnline) => {
// We do this because react native's NetInfo
// fires multiple online events.
// We only want to handle one state change
if (isOnline === this._isOnline) {
return;
}
this._log.info('[network] online =', isOnline);
this._isOnline = isOnline;
if (this._isOnline) {
this._startSocket();
}
else {
this._log.info('Changing status from', this.status, 'to', STATUS.CLOSED);
this._setStatus(STATUS.CLOSED);
}
});
});
if (typeof addEventListener !== 'undefined') {
this._beforeUnload = this._beforeUnload.bind(this);
addEventListener('beforeunload', this._beforeUnload);
}
}
getFrameworkClient() {
return this._frameworkClient;
}
/** @param {FrameworkClient} client */
setFrameworkClient(client) {
this._frameworkClient = client;
}
ensureAttrs() {
if (!this.attrs) {
throw new Error('attrs have not loaded.');
}
return this.attrs;
}
updateSchema(schema) {
this.config = {
...this.config,
schema: schema,
cardinalityInference: Boolean(schema),
};
this._linkIndex = schema ? createLinkIndex(this.config.schema) : null;
}
_reactorStats() {
return {
inFlightMutationCount: this._inFlightMutationEventIds.size,
storedMutationCount: this._pendingMutations().size,
transportType: this._transportType,
};
}
_onQuerySubLoaded(hash) {
this.kv
.waitForKeyToLoad('pendingMutations')
.then(() => this.notifyOne(hash));
}
_initStorage(Storage) {
this.querySubs = new PersistedObject({
persister: new Storage(this.config.appId, 'querySubs'),
merge: onMergeQuerySub,
serialize: querySubToStorage,
parse: (_key, x) => querySubFromStorage(x, this.config.useDateObjects),
// objectSize
objectSize: (x) => x?.result?.store?.triples?.length ?? 0,
logger: this._log,
preloadEntryCount: 10,
gc: {
maxAgeMs: 1000 * 60 * 60 * 24 * 7 * 52, // 1 year
maxEntries: 1000,
// Size of each query is the number of triples
maxSize: 1_000_000, // 1 million triples
},
});
this.querySubs.onKeyLoaded = (k) => this._onQuerySubLoaded(k);
this.kv = new PersistedObject({
persister: new Storage(this.config.appId, 'kv'),
merge: this._onMergeKv,
serialize: kvToStorage,
parse: kvFromStorage,
objectSize: () => 0,
logger: this._log,
saveThrottleMs: 100,
idleCallbackMaxWaitMs: 100,
// Don't GC the kv store
gc: null,
});
this.kv.onKeyLoaded = (k) => {
if (k === 'pendingMutations') {
this.notifyAll();
}
};
// Trigger immediate load for pendingMutations and currentUser
this.kv.waitForKeyToLoad('pendingMutations');
this.kv.waitForKeyToLoad(currentUserKey);
this._beforeUnloadCbs.push(() => {
this.kv.flush();
this.querySubs.flush();
});
}
_beforeUnload() {
for (const cb of this._beforeUnloadCbs) {
cb();
}
this._syncTable.beforeUnload();
}
/**
* @param {'enqueued' | 'pending' | 'synced' | 'timeout' | 'error' } status
* @param {string} eventId
* @param {{message?: string, type?: string, status?: number, hint?: unknown}} [errorMsg]
*/
_finishTransaction(status, eventId, errorMsg) {
const dfd = this.mutationDeferredStore.get(eventId);
this.mutationDeferredStore.delete(eventId);
const ok = status !== 'error' && status !== 'timeout';
if (!dfd && !ok) {
// console.erroring here, as there are no listeners to let know
console.error('Mutation failed', { status, eventId, ...errorMsg });
}
if (!dfd) {
return;
}
if (ok) {
dfd.resolve({ status, eventId });
}
else {
// Check if error comes from server or client
if (errorMsg?.type) {
const { status, ...body } = errorMsg;
dfd.reject(new InstantAPIError({
// @ts-expect-error body.type is not constant typed
body,
status: status ?? 0,
}));
}
else {
dfd.reject(new InstantError(errorMsg?.message || 'Unknown error', errorMsg?.hint));
}
}
}
_setStatus(status, err) {
this.status = status;
this._errorMessage = err;
this.notifyConnectionStatusSubs(status);
this._instantStream.onConnectionStatusChange(status);
}
_onMergeKv = (key, storageV, inMemoryV) => {
switch (key) {
case 'pendingMutations': {
const storageEntries = storageV?.entries() ?? [];
const inMemoryEntries = inMemoryV?.entries() ?? [];
const muts = new Map([...storageEntries, ...inMemoryEntries]);
const rewrittenStorageMuts = storageV
? this._rewriteMutationsSorted(this.attrs, storageV)
: [];
rewrittenStorageMuts.forEach(([k, mut]) => {
if (!inMemoryV?.pendingMutations?.has(k) && !mut['tx-id']) {
this._sendMutation(k, mut);
}
});
return muts;
}
default:
return inMemoryV || storageV;
}
};
_flushEnqueuedRoomData(roomId) {
const enqueuedUserPresence = this._presence[roomId]?.result?.user;
const enqueuedBroadcasts = this._broadcastQueue[roomId];
this._broadcastQueue[roomId] = [];
if (enqueuedUserPresence) {
this._trySetPresence(roomId, enqueuedUserPresence);
}
if (enqueuedBroadcasts) {
for (const item of enqueuedBroadcasts) {
const { topic, roomType, data } = item;
this._tryBroadcast(roomId, roomType, topic, data);
}
}
}
/**
* Does the same thing as add-query-ok
* but called as a result of receiving query info from ssr
* @param {any} q
* @param {{ triples: any; pageInfo: any; }} result
* @param {boolean} enableCardinalityInference
*/
_addQueryData(q, result, enableCardinalityInference) {
if (!this.attrs) {
throw new Error('Attrs in reactor have not been set');
}
const queryHash = weakHash(q);
const attrsStore = this.ensureAttrs();
const store = s.createStore(this.attrs, result.triples, enableCardinalityInference, this.config.useDateObjects);
this.querySubs.updateInPlace((prev) => {
prev[queryHash] = {
result: {
store,
attrsStore,
pageInfo: result.pageInfo,
processedTxId: undefined,
isExternal: true,
},
q,
};
});
this._cleanupPendingMutationsQueries();
this.notifyOne(queryHash);
this.notifyOneQueryOnce(queryHash);
this._cleanupPendingMutationsTimeout();
}
_handleReceive(connId, msg) {
// opt-out, enabled by default if schema
const enableCardinalityInference = Boolean(this.config.schema) &&
('cardinalityInference' in this.config
? Boolean(this.config.cardinalityInference)
: true);
if (!ignoreLogging[msg.op]) {
this._log.info('[receive]', connId, msg.op, msg);
}
switch (msg.op) {
case 'init-ok': {
this._setStatus(STATUS.AUTHENTICATED);
this._reconnectTimeoutMs = 0;
this._setAttrs(msg.attrs);
this._flushPendingMessages();
// (EPH): set session-id, so we know
// which item is us
this._sessionId = msg['session-id'];
for (const roomId of Object.keys(this._rooms)) {
const enqueuedUserPresence = this._presence[roomId]?.result?.user;
const roomType = this._rooms[roomId]?.roomType;
this._tryJoinRoom(roomType, roomId, enqueuedUserPresence);
}
break;
}
case 'add-query-exists': {
this.notifyOneQueryOnce(weakHash(msg.q));
break;
}
case 'add-query-ok': {
const { q, result } = msg;
const hash = weakHash(q);
if (!this._hasQueryListeners() && !this.querySubs.currentValue[hash]) {
break;
}
const pageInfo = result?.[0]?.data?.['page-info'];
const aggregate = result?.[0]?.data?.['aggregate'];
const triples = extractTriples(result);
const attrsStore = this.ensureAttrs();
const store = s.createStore(attrsStore, triples, enableCardinalityInference, this.config.useDateObjects);
this.querySubs.updateInPlace((prev) => {
if (!prev[hash]) {
this._log.info('Missing value in querySubs', { hash, q });
return;
}
prev[hash].result = {
store,
attrsStore,
pageInfo,
aggregate,
processedTxId: msg['processed-tx-id'],
};
});
this._cleanupPendingMutationsQueries();
this.notifyOne(hash);
this.notifyOneQueryOnce(hash);
this._cleanupPendingMutationsTimeout();
break;
}
case 'start-sync-ok': {
this._syncTable.onStartSyncOk(msg);
break;
}
case 'sync-load-batch': {
this._syncTable.onSyncLoadBatch(msg);
break;
}
case 'sync-init-finish': {
this._syncTable.onSyncInitFinish(msg);
break;
}
case 'sync-update-triples': {
this._syncTable.onSyncUpdateTriples(msg);
break;
}
case 'start-stream-ok': {
this._instantStream.onStartStreamOk(msg);
break;
}
case 'stream-flushed': {
this._instantStream.onStreamFlushed(msg);
break;
}
case 'append-failed': {
this._instantStream.onAppendFailed(msg);
break;
}
case 'stream-append': {
this._instantStream.onStreamAppend(msg);
break;
}
case 'refresh-ok': {
const { computations, attrs } = msg;
const processedTxId = msg['processed-tx-id'];
if (attrs) {
this._setAttrs(attrs);
}
this._cleanupPendingMutationsTimeout();
const rewrittenMutations = this._rewriteMutations(this.ensureAttrs(), this._pendingMutations(), processedTxId);
if (rewrittenMutations !== this._pendingMutations()) {
// We know we've changed the mutations to fix the attr ids and removed
// processed attrs, so we'll persist those changes to prevent optimisticAttrs
// from using old attr definitions
this.kv.updateInPlace((prev) => {
prev.pendingMutations = rewrittenMutations;
});
}
const mutations = sortedMutationEntries(rewrittenMutations.entries());
const updates = computations.map((x) => {
const q = x['instaql-query'];
const result = x['instaql-result'];
const hash = weakHash(q);
const triples = extractTriples(result);
const attrsStore = this.ensureAttrs();
const store = s.createStore(attrsStore, triples, enableCardinalityInference, this.config.useDateObjects);
const { store: newStore, attrsStore: newAttrsStore } = this._applyOptimisticUpdates(store, attrsStore, mutations, processedTxId);
const pageInfo = result?.[0]?.data?.['page-info'];
const aggregate = result?.[0]?.data?.['aggregate'];
return {
q,
hash,
store: newStore,
attrsStore: newAttrsStore,
pageInfo,
aggregate,
};
});
updates.forEach(({ hash, q, store, attrsStore, pageInfo, aggregate }) => {
this.querySubs.updateInPlace((prev) => {
if (!prev[hash]) {
this._log.error('Missing value in querySubs', { hash, q });
return;
}
prev[hash].result = {
store,
attrsStore,
pageInfo,
aggregate,
processedTxId,
};
});
});
this._cleanupPendingMutationsQueries();
updates.forEach(({ hash }) => {
this.notifyOne(hash);
});
break;
}
case 'transact-ok': {
const { 'client-event-id': eventId, 'tx-id': txId } = msg;
this._inFlightMutationEventIds.delete(eventId);
const muts = this._rewriteMutations(this.ensureAttrs(), this._pendingMutations());
const prevMutation = muts.get(eventId);
if (!prevMutation) {
break;
}
// update pendingMutation with server-side tx-id
this._updatePendingMutations((prev) => {
prev.set(eventId, {
...prev.get(eventId),
'tx-id': txId,
confirmed: Date.now(),
});
});
const newAttrs = [];
for (const step of prevMutation['tx-steps']) {
if (step[0] === 'add-attr') {
const attr = step[1];
newAttrs.push(attr);
}
}
if (newAttrs.length) {
const existingAttrs = Object.values(this.ensureAttrs().attrs);
this._setAttrs([...existingAttrs, ...newAttrs]);
}
this._finishTransaction('synced', eventId);
this._cleanupPendingMutationsTimeout();
break;
}
case 'patch-presence': {
const roomId = msg['room-id'];
this._trySetRoomConnected(roomId, true);
this._patchPresencePeers(roomId, msg['edits']);
this._notifyPresenceSubs(roomId);
break;
}
case 'refresh-presence': {
const roomId = msg['room-id'];
this._trySetRoomConnected(roomId, true);
this._setPresencePeers(roomId, msg['data']);
this._notifyPresenceSubs(roomId);
break;
}
case 'server-broadcast': {
const room = msg['room-id'];
const topic = msg.topic;
this._trySetRoomConnected(room, true);
this._notifyBroadcastSubs(room, topic, msg);
break;
}
case 'join-room-ok': {
const loadingRoomId = msg['room-id'];
const joinedRoom = this._rooms[loadingRoomId];
if (!joinedRoom) {
if (this._roomsPendingLeave[loadingRoomId]) {
this._tryLeaveRoom(loadingRoomId);
delete this._roomsPendingLeave[loadingRoomId];
}
break;
}
this._trySetRoomConnected(loadingRoomId, true);
this._flushEnqueuedRoomData(loadingRoomId);
break;
}
case 'leave-room-ok': {
const roomId = msg['room-id'];
this._trySetRoomConnected(roomId, false);
break;
}
case 'join-room-error':
const errorRoomId = msg['room-id'];
const errorRoom = this._rooms[errorRoomId];
if (errorRoom) {
errorRoom.error = msg['error'];
}
this._notifyPresenceSubs(errorRoomId);
break;
case 'error':
this._handleReceiveError(msg);
break;
default:
this._log.info('Unknown op', msg.op, msg);
break;
}
}
createWriteStream(opts) {
return this._instantStream.createWriteStream(opts);
}
createReadStream(opts) {
return this._instantStream.createReadStream(opts);
}
_pendingMutations() {
return this.kv.currentValue.pendingMutations ?? new Map();
}
_updatePendingMutations(f) {
this.kv.updateInPlace((prev) => {
const muts = prev.pendingMutations ?? new Map();
prev.pendingMutations = muts;
f(muts);
});
}
/**
* @param {'timeout' | 'error'} status
* @param {string} eventId
* @param {{message?: string, type?: string, status?: number, hint?: unknown}} errorMsg
*/
_handleMutationError(status, eventId, errorMsg) {
const mut = this._pendingMutations().get(eventId);
if (mut && (status !== 'timeout' || !mut['tx-id'])) {
this._updatePendingMutations((prev) => {
prev.delete(eventId);
return prev;
});
this._inFlightMutationEventIds.delete(eventId);
const errDetails = {
message: errorMsg.message,
hint: errorMsg.hint,
};
this.notifyAll();
this.notifyAttrsSubs();
this.notifyMutationErrorSubs(errDetails);
this._finishTransaction(status, eventId, errorMsg);
}
}
_handleReceiveError(msg) {
console.log('error', msg);
const eventId = msg['client-event-id'];
// This might not be a mutation, but it can't hurt to delete it
this._inFlightMutationEventIds.delete(eventId);
const prevMutation = this._pendingMutations().get(eventId);
const errorMessage = {
message: msg.message || 'Uh-oh, something went wrong. Ping Joe & Stopa.',
};
if (msg.hint) {
errorMessage.hint = msg.hint;
}
if (prevMutation) {
this._handleMutationError('error', eventId, msg);
return;
}
if (msg['original-event']?.hasOwnProperty('q') &&
msg['original-event']?.op === 'add-query') {
const q = msg['original-event']?.q;
const hash = weakHash(q);
this.notifyQueryError(weakHash(q), errorMessage);
this.notifyQueryOnceError(q, hash, eventId, errorMessage);
return;
}
const isInitError = msg['original-event']?.op === 'init';
if (isInitError) {
if (msg.type === 'record-not-found' &&
msg.hint?.['record-type'] === 'app-user') {
// User has been logged out
this.changeCurrentUser(null);
return;
}
// We failed to init
this._setStatus(STATUS.ERRORED, errorMessage);
this.notifyAll();
return;
}
switch (msg['original-event']?.op) {
case 'resync-table': {
this._syncTable.onResyncError(msg);
return;
}
case 'start-sync': {
this._syncTable.onStartSyncError(msg);
return;
}
case 'start-stream':
case 'append-stream':
case 'subscribe-stream':
case 'unsubscribe-stream': {
this._instantStream.onRecieveError(msg);
return;
}
}
// We've caught some error which has no corresponding listener.
// Let's console.error to let the user know.
const errorObj = { ...msg };
delete errorObj.message;
delete errorObj.hint;
console.error(msg.message, errorObj);
if (msg.hint) {
console.error('This error comes with some debugging information. Here it is: \n', msg.hint);
}
}
notifyQueryOnceError(q, hash, eventId, e) {
const r = this.queryOnceDfds[hash]?.find((r) => r.eventId === eventId);
if (!r)
return;
r.dfd.reject(e);
this._completeQueryOnce(q, hash, r.dfd);
}
_setAttrs(attrs) {
this.attrs = new s.AttrsStoreClass(attrs.reduce((acc, attr) => {
acc[attr.id] = attr;
return acc;
}, {}), this._linkIndex);
this.notifyAttrsSubs();
}
// ---------------------------
// Queries
getPreviousResult = (q) => {
const hash = weakHash(q);
return this.dataForQuery(hash)?.data;
};
_startQuerySub(q, hash) {
const eventId = uuid();
this.querySubs.updateInPlace((prev) => {
prev[hash] = prev[hash] || { q, result: null, eventId };
prev[hash].lastAccessed = Date.now();
});
this._trySendAuthed(eventId, { op: 'add-query', q });
return eventId;
}
subscribeTable(q, cb) {
return this._syncTable.subscribe(q, cb);
}
/**
* When a user subscribes to a query the following side effects occur:
*
* - We update querySubs to include the new query
* - We update queryCbs to include the new cb
* - If we already have a result for the query we call cb immediately
* - We send the server an `add-query` message
*
* Returns an unsubscribe function
*/
subscribeQuery(q, cb, opts) {
if (!this.config.disableValidation) {
validateQuery(q, this.config.schema);
}
if (opts && 'ruleParams' in opts) {
q = { $$ruleParams: opts['ruleParams'], ...q };
}
const hash = weakHash(q);
const prevResult = this.getPreviousResult(q);
if (prevResult) {
cb(prevResult);
}
this.queryCbs[hash] = this.queryCbs[hash] ?? [];
this.queryCbs[hash].push({ q, cb });
this._startQuerySub(q, hash);
return () => {
this._unsubQuery(q, hash, cb);
};
}
queryOnce(q, opts) {
if (!this.config.disableValidation) {
validateQuery(q, this.config.schema);
}
if (opts && 'ruleParams' in opts) {
q = { $$ruleParams: opts['ruleParams'], ...q };
}
const dfd = new Deferred();
if (!this._isOnline) {
dfd.reject(new Error("We can't run `queryOnce`, because the device is offline."));
return dfd.promise;
}
if (!this.querySubs) {
dfd.reject(new Error("We can't run `queryOnce` on the backend. Use adminAPI.query instead: https://www.instantdb.com/docs/backend#query"));
return dfd.promise;
}
const hash = weakHash(q);
const eventId = this._startQuerySub(q, hash);
this.queryOnceDfds[hash] = this.queryOnceDfds[hash] ?? [];
this.queryOnceDfds[hash].push({ q, dfd, eventId });
setTimeout(() => dfd.reject(new Error('Query timed out')), QUERY_ONCE_TIMEOUT);
return dfd.promise;
}
_completeQueryOnce(q, hash, dfd) {
if (!this.queryOnceDfds[hash])
return;
this.queryOnceDfds[hash] = this.queryOnceDfds[hash].filter((r) => r.dfd !== dfd);
this._cleanupQuery(q, hash);
}
_unsubQuery(q, hash, cb) {
if (!this.queryCbs[hash])
return;
this.queryCbs[hash] = this.queryCbs[hash].filter((r) => r.cb !== cb);
this._cleanupQuery(q, hash);
}
_hasQueryListeners(hash) {
return !!(this.queryCbs[hash]?.length || this.queryOnceDfds[hash]?.length);
}
_cleanupQuery(q, hash) {
const hasListeners = this._hasQueryListeners(hash);
if (hasListeners)
return;
delete this.queryCbs[hash];
delete this.queryOnceDfds[hash];
delete this._dataForQueryCache[hash];
this.querySubs.unloadKey(hash);
this._trySendAuthed(uuid(), { op: 'remove-query', q });
}
// When we `pushTx`, it's possible that we don't yet have `this.attrs`
// This means that `tx-steps` in `pendingMutations` will include `add-attr`
// commands for attrs that already exist.
//
// This will also affect `add-triple` and `retract-triple` which
// reference attr-ids that do not match the server.
//
// We fix this by rewriting `tx-steps` in each `pendingMutation`.
// We remove `add-attr` commands for attrs that already exist.
// We update `add-triple` and `retract-triple` commands to use the
// server attr-ids.
/**
*
* @param {s.AttrsStore} attrs
* @param {any} muts
* @param {number} [processedTxId]
*/
_rewriteMutations(attrs, muts, processedTxId) {
if (!attrs)
return muts;
if (!muts)
return new Map();
const findExistingAttr = (attr) => {
const [_, etype, label] = attr['forward-identity'];
const existing = s.getAttrByFwdIdentName(attrs, etype, label);
return existing;
};
const findReverseAttr = (attr) => {
const [_, etype, label] = attr['forward-identity'];
const revAttr = s.getAttrByReverseIdentName(attrs, etype, label);
return revAttr;
};
const mapping = { attrIdMap: {}, refSwapAttrIds: new Set() };
let mappingChanged = false;
const rewriteTxSteps = (txSteps, txId) => {
const retTxSteps = [];
for (const txStep of txSteps) {
const [action] = txStep;
// Handles add-attr
// If existing, we drop it, and track it
// to update add/retract triples
if (action === 'add-attr') {
const [_action, attr] = txStep;
const existing = findExistingAttr(attr);
if (existing && attr.id !== existing.id) {
mapping.attrIdMap[attr.id] = existing.id;
mappingChanged = true;
continue;
}
if (attr['value-type'] === 'ref') {
const revAttr = findReverseAttr(attr);
if (revAttr) {
mapping.attrIdMap[attr.id] = revAttr.id;
mapping.refSwapAttrIds.add(attr.id);
mappingChanged = true;
continue;
}
}
}
if ((processedTxId &&
txId &&
processedTxId >= txId &&
action === 'add-attr') ||
action === 'update-attr' ||
action === 'delete-attr') {
mappingChanged = true;
// Don't add this step because we already have the newer attrs
continue;
}
// Handles add-triple|retract-triple
// If in mapping, we update the attr-id
const newTxStep = mappingChanged
? instaml.rewriteStep(mapping, txStep)
: txStep;
retTxSteps.push(newTxStep);
}
return mappingChanged ? retTxSteps : txSteps;
};
const rewritten = new Map();
for (const [k, mut] of muts.entries()) {
rewritten.set(k, {
...mut,
'tx-steps': rewriteTxSteps(mut['tx-steps'], mut['tx-id']),
});
}
if (!mappingChanged) {
return muts;
}
return rewritten;
}
_rewriteMutationsSorted(attrs, muts) {
return sortedMutationEntries(this._rewriteMutations(attrs, muts).entries());
}
// ---------------------------
// Transact
/**
* @returns {s.AttrsStore}
*/
optimisticAttrs() {
const pendingMutationSteps = [...this._pendingMutations().values()] // hack due to Map()
.flatMap((x) => x['tx-steps']);
const deletedAttrIds = new Set(pendingMutationSteps
.filter(([action, _attr]) => action === 'delete-attr')
.map(([_action, id]) => id));
const pendingAttrs = [];
for (const [_action, attr] of pendingMutationSteps) {
if (_action === 'add-attr') {
pendingAttrs.push(attr);
}
else if (_action === 'update-attr' &&
attr.id &&
this.attrs?.getAttr(attr.id)) {
const fullAttr = { ...this.attrs.getAttr(attr.id), ...attr };
pendingAttrs.push(fullAttr);
}
}
if (!deletedAttrIds.size && !pendingAttrs.length) {
return this.attrs || new s.AttrsStoreClass({}, this._linkIndex);
}
const attrs = { ...(this.attrs?.attrs || {}) };
for (const attr of pendingAttrs) {
attrs[attr.id] = attr;
}
for (const id of deletedAttrIds) {
delete attrs[id];
}
return new s.AttrsStoreClass(attrs, this._linkIndex);
}
/** Runs instaql on a query and a store */
dataForQuery(hash, applyOptimistic = true) {
const errorMessage = this._errorMessage;
if (errorMessage) {
return { error: errorMessage };
}
if (!this.querySubs)
return;
if (!this.kv.currentValue.pendingMutations)
return;
const querySubVersion = this.querySubs.version();
const querySubs = this.querySubs.currentValue;
const pendingMutationsVersion = this.kv.version();
const pendingMutations = this._pendingMutations();
const { q, result } = querySubs[hash] || {};
if (!result)
return;
const cached = this._dataForQueryCache[hash];
if (cached &&
querySubVersion === cached.querySubVersion &&
pendingMutationsVersion === cached.pendingMutationsVersion) {
return cached;
}
let store = result.store;
let attrsStore = result.attrsStore;
const { pageInfo, aggregate, processedTxId } = result;
const mutations = this._rewriteMutationsSorted(attrsStore, pendingMutations);
if (applyOptimistic) {
const optimisticResult = this._applyOptimisticUpdates(store, attrsStore, mutations, processedTxId);
store = optimisticResult.store;
attrsStore = optimisticResult.attrsStore;
}
const resp = instaql({ store: store, attrsStore: attrsStore, pageInfo, aggregate }, q);
return { data: resp, querySubVersion, pendingMutationsVersion };
}
_applyOptimisticUpdates(store, attrsStore, mutations, processedTxId) {
for (const [_, mut] of mutations) {
if (!mut['tx-id'] || (processedTxId && mut['tx-id'] > processedTxId)) {
const result = s.transact(store, attrsStore, mut['tx-steps']);
store = result.store;
attrsStore = result.attrsStore;
}
}
return { store, attrsStore };
}
/** Re-run instaql and call all callbacks with new data */
notifyOne = (hash) => {
const cbs = this.queryCbs[hash] ?? [];
const prevData = this._dataForQueryCache[hash]?.data;
const resp = this.dataForQuery(hash);
if (!resp?.data)
return;
this._dataForQueryCache[hash] = resp;
if (areObjectsDeepEqual(resp.data, prevData))
return;
cbs.forEach((r) => r.cb(resp.data));
};
notifyOneQueryOnce = (hash) => {
const dfds = this.queryOnceDfds[hash] ?? [];
const data = this.dataForQuery(hash)?.data;
dfds.forEach((r) => {
this._completeQueryOnce(r.q, hash, r.dfd);
r.dfd.resolve(data);
});
};
notifyQueryError = (hash, error) => {
const cbs = this.queryCbs[hash] || [];
cbs.forEach((r) => r.cb({ error }));
};
/** Re-compute all subscriptions */
notifyAll() {
Object.keys(this.queryCbs).forEach((hash) => {
this.querySubs
.waitForKeyToLoad(hash)
.then(() => this.notifyOne(hash))
.catch(() => this.notifyOne(hash));
});
}
loadedNotifyAll() {
this.kv
.waitForKeyToLoad('pendingMutations')
.then(() => this.notifyAll())
.catch(() => this.notifyAll());
}
/** Applies transactions locally and sends transact message to server */
pushTx = (chunks) => {
// Throws if transactions are invalid
if (!this.config.disableValidation) {
validateTransactions(chunks, this.config.schema);
}
try {
const txSteps = instaml.transform({
attrsStore: this.optimisticAttrs(),
schema: this.config.schema,
stores: Object.values(this.querySubs.currentValue).map((sub) => sub?.result?.store),
useDateObjects: this.config.useDateObjects,
}, chunks);
return this.pushOps(txSteps);
}
catch (e) {
return this.pushOps([], e);
}
};
/**
* @param {*} txSteps
* @param {*} [error]
* @returns
*/
pushOps = (txSteps, error) => {
const eventId = uuid();
const mutations = [...this._pendingMutations().values()];
const order = Math.max(0, ...mutations.map((mut) => mut.order || 0)) + 1;
const mutation = {
op: 'transact',
'tx-steps': txSteps,
created: Date.now(),
error,
order,
};
this._updatePendingMutations((prev) => {
prev.set(eventId, mutation);
});
const dfd = new Deferred();
this.mutationDeferredStore.set(eventId, dfd);
this._sendMutation(eventId, mutation);
this.notifyAll();
return dfd.promise;
};
shutdown() {
this._log.info('[shutdown]', this.config.appId);
this._isShutdown = true;
this._transport?.close();
}
/**
* Sends mutation to server and schedules a timeout to cancel it if
* we don't hear back in time.
* Note: If we're offline we don't schedule a timeout, we'll schedule it
* later once we're back online and send the mutation again
*
*/
_sendMutation(eventId, mutation) {
if (mutation.error) {
this._handleMutationError('error', eventId, {
message: mutation.error.message,
});
return;
}
if (this.status !== STATUS.AUTHENTICATED) {
this._finishTransaction('enqueued', eventId);
return;
}
const timeoutMs = Math.max(6000, Math.min(this._inFlightMutationEventIds.size + 1,
// Defensive code in case we don't clean up in flight mutation event ids
this._pendingMutations().size + 1) * 6000);
if (!this._isOnline) {
this._finishTransaction('enqueued', eventId);
}
else {
this._trySend(eventId, mutation);
setTimeout(() => {
if (!this._isOnline) {
return;
}
// If we are here, this means that we have sent this mutation, we are online
// but we have not received a response. If it's this long, something must be wrong,
// so we error with a timeout.
this._handleMutationError('timeout', eventId, {
message: 'transaction timed out',
});
}, timeoutMs);
}
}
// ---------------------------
// Websocket
/** Send messages we accumulated while we were connecting */
_flushPendingMessages() {
const subs = Object.keys(this.queryCbs).map((hash) => {
return this.querySubs.currentValue[hash];
});
// Note: we should not have any nulls in subs, but we're
// doing this defensively just in case.
const safeSubs = subs.filter((x) => x);
safeSubs.forEach(({ eventId, q }) => {
this._trySendAuthed(eventId, { op: 'add-query', q });
});
Object.values(this.queryOnceDfds)
.flat()
.forEach(({ eventId, q }) => {
this._trySendAuthed(eventId, { op: 'add-query', q });
});
const rewrittenMutations = this._rewriteMutations(this.ensureAttrs(), this._pendingMutations());
if (rewrittenMutations !== this._pendingMutations()) {
// Persist rewritten mutations to avoid stale attr ids in future txs