UNPKG

@tanstack/db

Version:

A reactive client store for building super fast apps on sync

772 lines (771 loc) 27.1 kB
import { ensureIndexForExpression } from "../indexes/auto-index.js"; import { and, eq, lt, gte } from "../query/builder/functions.js"; import { Value, PropRef } from "../query/ir.js"; import { EventEmitter } from "../event-emitter.js"; import { compileExpression } from "../query/compiler/evaluators.js"; import { buildCursor } from "../utils/cursor.js"; import { deepEquals } from "../utils.js"; import { createFilteredCallback, createFilterFunctionFromExpression } from "./change-events.js"; class CollectionSubscription extends EventEmitter { constructor(collection, callback, options) { super(); this.collection = collection; this.callback = callback; this.options = options; this.loadedInitialState = false; this.skipFiltering = false; this.snapshotSent = false; this.subsetDemands = []; this.requestedSubsetWhere = /* @__PURE__ */ new WeakMap(); this.sentKeys = /* @__PURE__ */ new Set(); this.publishedRows = /* @__PURE__ */ new Map(); this.stalePublishedRows = /* @__PURE__ */ new Map(); this.limitedSnapshotRowCount = 0; this._status = `ready`; this.pendingLoadSubsetPromises = /* @__PURE__ */ new Set(); if (options.onUnsubscribe) { this.on(`unsubscribed`, options.onUnsubscribe); } if (options.onLoadSubsetError) { this.on(`loadSubset:error`, options.onLoadSubsetError); } if (options.whereExpression) { ensureIndexForExpression(options.whereExpression, this.collection); } const callbackWithSentKeysTracking = (changes) => { this.trackPublishedRows(changes); this.trackSentKeys(changes); callback(changes); }; this.callback = callbackWithSentKeysTracking; this.filteredCallback = options.whereExpression ? createFilteredCallback(this.callback, options) : (changes) => { this.callback(changes); return true; }; this.truncateCleanup = this.collection.on(`truncate`, () => { this.handleTruncate(); }); } get status() { return this._status; } get lastError() { return this._lastError; } /** * Handle collection truncate event by resetting state and re-requesting subsets. * This is called when the sync layer receives a must-refetch and clears all data. * * To prevent a flash of missing content, we buffer all changes (deletes from truncate * and inserts from refetch) until all loadSubset calls succeed, then emit them together. * A failed replay keeps the last published snapshot, resumes ordinary deltas, * and retains subset ownership so a later truncate can retry the replay. */ handleTruncate() { const demandsToReload = [...this.subsetDemands]; const hasLoadSubsetHandler = this.collection._sync.syncLoadSubsetFn !== null; if (demandsToReload.length === 0 || !hasLoadSubsetHandler) { this.snapshotSent = false; this.loadedInitialState = false; this.limitedSnapshotRowCount = 0; this.lastSentKey = void 0; return; } const attempt = { pending: /* @__PURE__ */ new Set(), failed: false, setupComplete: false }; let session = this.truncateReplaySession; if (!session) { session = { publicationState: { loadedInitialState: this.loadedInitialState, snapshotSent: this.snapshotSent, sentKeys: new Set(this.sentKeys), publishedRows: new Map(this.publishedRows), limitedSnapshotRowCount: this.limitedSnapshotRowCount, lastSentKey: this.lastSentKey }, buffer: [], attempts: /* @__PURE__ */ new Set(), currentAttempt: attempt }; this.truncateReplaySession = session; } session.attempts.add(attempt); session.currentAttempt = attempt; for (const demand of demandsToReload) { demand.abortController?.abort(); } this.snapshotSent = false; this.loadedInitialState = false; this.limitedSnapshotRowCount = 0; this.lastSentKey = void 0; queueMicrotask(() => { if (this.truncateReplaySession !== session) return; for (const demand of demandsToReload) { if (!this.subsetDemands.includes(demand)) continue; const isCurrentAttempt = () => this.truncateReplaySession === session && session.currentAttempt === attempt; const nextAcquisition = this.createSubsetAcquisition(demand); let syncResult; try { syncResult = this.loadSubset( nextAcquisition.options, isCurrentAttempt ); } catch { nextAcquisition.abortController.abort(); nextAcquisition.removeRequestAbortListener?.(); attempt.failed = true; continue; } this.observeLoadSubsetResult( syncResult, nextAcquisition.options, true, () => isCurrentAttempt() && !nextAcquisition.options.signal?.aborted ); if (syncResult instanceof Promise) { const pending = { promise: syncResult }; attempt.pending.add(pending); void syncResult.then( () => this.settleTruncateReplay(session, attempt, pending), () => { if (this.subsetDemands.includes(demand) && !nextAcquisition.options.signal?.aborted) { attempt.failed = true; } this.settleTruncateReplay(session, attempt, pending); } ); } try { this.replaceSubsetAcquisition(demand, nextAcquisition); } catch (error) { nextAcquisition.abortController.abort(); nextAcquisition.removeRequestAbortListener?.(); try { this.collection._sync.unloadSubset(nextAcquisition.options); } catch { } this.recordLoadSubsetError(demand.options, error, true); attempt.failed = true; } } attempt.setupComplete = true; this.checkTruncateReplayComplete(session); }); } settleTruncateReplay(session, attempt, pending) { if (this.truncateReplaySession !== session) return; attempt.pending.delete(pending); this.checkTruncateReplayComplete(session); } /** Publish only after every overlapping replay attempt has settled. */ checkTruncateReplayComplete(session) { if (this.truncateReplaySession !== session) return; for (const attempt of session.attempts) { if (!attempt.setupComplete || attempt.pending.size > 0) return; } if (session.currentAttempt.failed) { this.abandonTruncateReplay(session); } else { this.flushTruncateReplay(session); } } /** * Discard an incomplete current replay and restore the last publication. * Rows in that publication remain stale until a later source delta or replay * reconciles them with the source collection. */ abandonTruncateReplay(session) { if (this.truncateReplaySession !== session) return; const publicationState = session.publicationState; this.loadedInitialState = publicationState.loadedInitialState; this.snapshotSent = publicationState.snapshotSent; this.sentKeys = new Set(publicationState.sentKeys); this.publishedRows = new Map(publicationState.publishedRows); this.stalePublishedRows = new Map(publicationState.publishedRows); this.limitedSnapshotRowCount = publicationState.limitedSnapshotRowCount; this.lastSentKey = publicationState.lastSentKey; this.truncateReplaySession = void 0; } /** Publish the complete buffered replacement as one subscriber batch. */ flushTruncateReplay(session) { if (this.truncateReplaySession !== session) return; this.truncateReplaySession = void 0; const retainedDeletes = [...this.stalePublishedRows].map( ([key, value]) => ({ type: `delete`, key, value }) ); this.stalePublishedRows.clear(); const merged = [...session.buffer.flat(), ...retainedDeletes]; const activeDemandFilters = this.subsetDemands.map( (demand) => demand.requestOptions.where ? createFilterFunctionFromExpression(demand.requestOptions.where) : void 0 ); const replacement = this.createPublicationDiff( session.publicationState.publishedRows, merged, (value) => activeDemandFilters.some((filter) => filter?.(value) ?? true) ); if (replacement.length > 0) this.filteredCallback(replacement); this.sentKeys = new Set(this.publishedRows.keys()); if (this.orderByIndex) { this.limitedSnapshotRowCount = this.sentKeys.size; const orderedSentKeys = this.orderByIndex.takeFromStart( this.sentKeys.size, (key) => this.sentKeys.has(key) ); this.lastSentKey = orderedSentKeys.at(-1); } } /** Reduce a replay's raw delete/insert stream to one exact semantic delta. */ createPublicationDiff(baseline, changes, isCoveredByActiveDemand) { const finalRows = new Map(baseline); for (const change of changes) { if (change.type === `delete`) finalRows.delete(change.key); else finalRows.set(change.key, change.value); } for (const [key, value] of finalRows) { if (!isCoveredByActiveDemand(value)) finalRows.delete(key); } const replacement = []; for (const [key, previousValue] of baseline) { const value = finalRows.get(key); if (value === void 0) { replacement.push({ type: `delete`, key, value: previousValue }); } else if (!deepEquals(value, previousValue)) { replacement.push({ type: `update`, key, value, previousValue }); } } for (const [key, value] of finalRows) { if (!baseline.has(key)) replacement.push({ type: `insert`, key, value }); } return replacement; } get isBufferingForTruncate() { return this.truncateReplaySession !== void 0; } setOrderByIndex(index) { this.orderByIndex = index; } /** * Check if an orderBy index has been set for this subscription */ hasOrderByIndex() { return this.orderByIndex !== void 0; } /** * Set subscription status and emit events if changed */ setStatus(newStatus) { if (this._status === newStatus) { return; } const previousStatus = this._status; this._status = newStatus; this.emitInner(`status:change`, { type: `status:change`, subscription: this, previousStatus, status: newStatus }); const eventKey = `status:${newStatus}`; this.emitInner(eventKey, { type: eventKey, subscription: this, previousStatus, status: newStatus }); } /** Observe an asynchronous subset load and restore status on settlement. */ observeLoadSubsetResult(syncResult, options, trackStatus, shouldReportError = () => true) { if (!(syncResult instanceof Promise)) return; if (trackStatus) { this.pendingLoadSubsetPromises.add(syncResult); this.setStatus(`loadingSubset`); } const finish = () => { if (trackStatus) { this.pendingLoadSubsetPromises.delete(syncResult); if (this.pendingLoadSubsetPromises.size === 0) { this.setStatus(`ready`); } } }; void syncResult.then(finish, (error) => { if (shouldReportError()) this.recordLoadSubsetError(options, error); finish(); }); } loadSubset(options, shouldReportError = () => true) { try { return this.collection._sync.loadSubset(options); } catch (error) { if (shouldReportError()) this.recordLoadSubsetError(options, error); throw error; } } /** Create a fresh, abortable adapter acquisition for a replay generation. */ createSubsetAcquisition(demand) { const abortController = new AbortController(); const requestSignal = demand.requestOptions.signal; let removeRequestAbortListener; if (requestSignal?.aborted) { abortController.abort(requestSignal.reason); } else if (requestSignal) { const abort = () => abortController.abort(requestSignal.reason); requestSignal.addEventListener(`abort`, abort, { once: true }); removeRequestAbortListener = () => requestSignal.removeEventListener(`abort`, abort); } return { options: { ...demand.requestOptions, signal: abortController.signal }, abortController, removeRequestAbortListener }; } /** Replace the adapter lease held for one logical subset demand. */ replaceSubsetAcquisition(demand, next) { const previousOptions = demand.options; const removePreviousAbortListener = demand.removeRequestAbortListener; this.collection._sync.unloadSubset(previousOptions); removePreviousAbortListener?.(); demand.options = next.options; demand.abortController = next.abortController; demand.removeRequestAbortListener = next.removeRequestAbortListener; } /** Abort and release one current adapter acquisition. */ releaseSubsetDemand(demand) { demand.abortController?.abort(); try { this.collection._sync.unloadSubset(demand.options); } finally { demand.removeRequestAbortListener?.(); } } /** Start and retain the first acquisition for one logical subset demand. */ startSubsetDemand(requestOptions) { const demand = { requestOptions, options: requestOptions }; const acquisition = this.createSubsetAcquisition(demand); try { const result = this.loadSubset(acquisition.options); demand.options = acquisition.options; demand.abortController = acquisition.abortController; demand.removeRequestAbortListener = acquisition.removeRequestAbortListener; this.subsetDemands.push(demand); return { demand, result }; } catch (error) { acquisition.abortController.abort(); acquisition.removeRequestAbortListener?.(); throw error; } } recordLoadSubsetError(options, error, reportAborted = false) { if (options.signal?.aborted && !reportAborted) return; this._lastError = error; this.emitInner(`loadSubset:error`, { type: `loadSubset:error`, subscription: this, options, error }); } hasLoadedInitialState() { return this.loadedInitialState; } hasSentAtLeastOneSnapshot() { return this.snapshotSent; } emitEvents(changes) { const newChanges = this.filterAndFlipChanges(changes); if (changes.length > 0 && newChanges.length === 0) return false; if (this.isBufferingForTruncate) { if (newChanges.length > 0) { this.truncateReplaySession.buffer.push(newChanges); } return false; } else { return this.filteredCallback(newChanges); } } /** * Sends the snapshot to the callback. * Returns a boolean indicating if it succeeded. * It can only fail if there is no index to fulfill the request * and the optimizedOnly option is set to true, * or, the entire state was already loaded. */ requestSnapshot(opts) { if (this.loadedInitialState) { return false; } const stateOpts = { where: this.options.whereExpression, optimizedOnly: opts?.optimizedOnly ?? false }; if (opts) { if (`where` in opts) { const snapshotWhereExp = opts.where; if (stateOpts.where) { const subWhereExp = stateOpts.where; const combinedWhereExp = and(subWhereExp, snapshotWhereExp); stateOpts.where = combinedWhereExp; } else { stateOpts.where = snapshotWhereExp; } } } else { this.loadedInitialState = true; } const loadOptions = { where: stateOpts.where, signal: opts?.signal, subscription: this, // Include orderBy and limit if provided so sync layer can optimize the query orderBy: opts?.orderBy, limit: opts?.limit }; const { demand, result: syncResult } = this.startSubsetDemand(loadOptions); if (opts?.where) this.requestedSubsetWhere.set(loadOptions, opts.where); opts?.onLoadSubsetResult?.(syncResult); this.observeLoadSubsetResult( syncResult, demand.options, opts?.trackLoadSubsetPromise ?? true ); let snapshot; if (opts?.onUnoptimized) { snapshot = this.collection.currentStateAsChanges({ ...stateOpts, optimizedOnly: true }); if (snapshot === void 0) { opts.onUnoptimized(); snapshot = this.collection.currentStateAsChanges({ ...stateOpts, optimizedOnly: false }); } } else { snapshot = this.collection.currentStateAsChanges(stateOpts); } if (snapshot === void 0) { return false; } const filteredSnapshot = snapshot.filter( (change) => !this.sentKeys.has(change.key) ); for (const change of filteredSnapshot) { this.sentKeys.add(change.key); } this.snapshotSent = true; this.callback(filteredSnapshot); return true; } /** Release one exact subset request while keeping the subscription alive. */ releaseSnapshot(where) { const index = this.subsetDemands.findIndex( (demand2) => demand2.requestOptions.where === where || this.requestedSubsetWhere.get(demand2.requestOptions) === where ); if (index === -1) return; const [demand] = this.subsetDemands.splice(index, 1); if (demand) this.releaseSubsetDemand(demand); } /** * Sends a snapshot that fulfills the `where` clause and all rows are bigger or equal to the cursor. * Requires a range index to be set with `setOrderByIndex` prior to calling this method. * It uses that range index to load the items in the order of the index. * * For multi-column orderBy: * - Uses first value from `minValues` for LOCAL index operations (wide bounds, ensures no missed rows) * - Uses all `minValues` to build a precise composite cursor for SYNC layer loadSubset * * Note 1: it may load more rows than the provided LIMIT because it loads all values equal to the first cursor value + limit values greater. * This is needed to ensure that it does not accidentally skip duplicate values when the limit falls in the middle of some duplicated values. * Note 2: it does not send keys that have already been sent before. */ requestLimitedSnapshot({ orderBy, limit, minValues, offset, trackLoadSubsetPromise: shouldTrackLoadSubsetPromise = true, onLoadSubsetResult }) { if (!limit) throw new Error(`limit is required`); if (!this.orderByIndex) { throw new Error( `Ordered snapshot was requested but no index was found. You have to call setOrderByIndex before requesting an ordered snapshot.` ); } const hasMinValue = minValues !== void 0 && minValues.length > 0; const minValue = minValues?.[0]; const minValueForIndex = minValue; const index = this.orderByIndex; const where = this.options.whereExpression; const whereFilterFn = where ? createFilterFunctionFromExpression(where) : void 0; const filterFn = (key) => { if (key !== void 0 && this.sentKeys.has(key)) { return false; } const value = this.collection.get(key); if (value === void 0) { return false; } return whereFilterFn?.(value) ?? true; }; let biggestObservedValue = minValueForIndex; const changes = []; let keys = []; if (hasMinValue) { const { expression } = orderBy[0]; const allRowsWithMinValue = this.collection.currentStateAsChanges({ where: eq(expression, new Value(minValueForIndex)) }); if (allRowsWithMinValue) { const keysWithMinValue = allRowsWithMinValue.map((change) => change.key).filter((key) => !this.sentKeys.has(key) && filterFn(key)); keys.push(...keysWithMinValue); const keysGreaterThanMin = index.take( limit - keys.length, minValueForIndex, filterFn ); keys.push(...keysGreaterThanMin); } else { keys = index.take(limit, minValueForIndex, filterFn); } } else { keys = index.takeFromStart(limit, filterFn); } const valuesNeeded = () => Math.max(limit - changes.length, 0); const collectionExhausted = () => keys.length === 0; const orderByExpression = orderBy[0].expression; const valueExtractor = orderByExpression.type === `ref` ? compileExpression(new PropRef(orderByExpression.path), true) : null; while (valuesNeeded() > 0 && !collectionExhausted()) { const insertedKeys = /* @__PURE__ */ new Set(); for (const key of keys) { const value = this.collection.get(key); changes.push({ type: `insert`, key, value }); biggestObservedValue = valueExtractor ? valueExtractor(value) : value; insertedKeys.add(key); } keys = index.take(valuesNeeded(), biggestObservedValue, filterFn); } const currentOffset = this.limitedSnapshotRowCount; for (const change of changes) { this.sentKeys.add(change.key); } this.callback(changes); this.limitedSnapshotRowCount = Math.max( this.limitedSnapshotRowCount, currentOffset + changes.length ); if (changes.length > 0) { this.lastSentKey = changes[changes.length - 1].key; } let cursorExpressions; if (minValues !== void 0 && minValues.length > 0) { const whereFromCursor = buildCursor(orderBy, minValues); if (whereFromCursor) { const { expression } = orderBy[0]; const cursorMinValue = minValues[0]; let whereCurrentCursor; if (cursorMinValue instanceof Date) { const cursorMinValuePlus1ms = new Date(cursorMinValue.getTime() + 1); whereCurrentCursor = and( gte(expression, new Value(cursorMinValue)), lt(expression, new Value(cursorMinValuePlus1ms)) ); } else { whereCurrentCursor = eq(expression, new Value(cursorMinValue)); } cursorExpressions = { whereFrom: whereFromCursor, whereCurrent: whereCurrentCursor, lastKey: this.lastSentKey }; } } const loadOptions = { where, // Main filter only, no cursor limit, orderBy, cursor: cursorExpressions, // Cursor expressions passed separately offset: offset ?? currentOffset, // Use provided offset, or auto-tracked offset subscription: this }; const { demand, result: syncResult } = this.startSubsetDemand(loadOptions); onLoadSubsetResult?.(syncResult); this.observeLoadSubsetResult( syncResult, demand.options, shouldTrackLoadSubsetPromise ); } // TODO: also add similar test but that checks that it can also load it from the collection's loadSubset function // and that that also works properly (i.e. does not skip duplicate values) /** * Filters and flips changes for keys that have not been sent yet. * Deletes are filtered out for keys that have not been sent yet. * Updates are flipped into inserts for keys that have not been sent yet. * Duplicate inserts are filtered out to prevent D2 multiplicity > 1. */ filterAndFlipChanges(changes) { changes = this.reconcileStalePublishedChanges(changes); if (this.loadedInitialState || this.skipFiltering) { return changes; } const skipDeleteFilter = this.isBufferingForTruncate; const newChanges = []; for (const change of changes) { let newChange = change; const keyInSentKeys = this.sentKeys.has(change.key); if (!keyInSentKeys) { if (change.type === `update`) { newChange = { ...change, type: `insert`, previousValue: void 0 }; this.sentKeys.add(change.key); } else if (change.type === `delete`) { if (!skipDeleteFilter) { continue; } } else { this.sentKeys.add(change.key); } } else { if (change.type === `insert`) { continue; } else if (change.type === `delete`) { this.sentKeys.delete(change.key); } } newChanges.push(newChange); } return newChanges; } /** * After a failed replay, the source collection is empty but subscribers still * hold the last good publication. Reconcile the first later source delta for * each retained key against that publication instead of treating it as a * duplicate insert. */ reconcileStalePublishedChanges(changes) { if (this.stalePublishedRows.size === 0) return changes; const reconciled = []; for (const change of changes) { const previous = this.stalePublishedRows.get(change.key); if (previous === void 0) { reconciled.push(change); continue; } this.stalePublishedRows.delete(change.key); if (change.type === `delete`) { reconciled.push({ ...change, value: previous, previousValue: void 0 }); } else if (!deepEquals(previous, change.value)) { reconciled.push({ ...change, type: `update`, previousValue: previous }); } } return reconciled; } trackPublishedRows(changes) { for (const change of changes) { if (change.type === `delete`) { this.publishedRows.delete(change.key); } else { this.publishedRows.set(change.key, change.value); } } } trackSentKeys(changes) { if (this.loadedInitialState || this.skipFiltering) { return; } for (const change of changes) { if (change.type === `delete`) { this.sentKeys.delete(change.key); } else { this.sentKeys.add(change.key); } } if (this.orderByIndex) { this.limitedSnapshotRowCount = Math.max( this.limitedSnapshotRowCount, this.sentKeys.size ); } } /** * Mark that the subscription should not filter any changes. * This is used when includeInitialState is explicitly set to false, * meaning the caller doesn't want initial state but does want ALL future changes. */ markAllStateAsSeen() { this.skipFiltering = true; } unsubscribe() { let firstCleanupError; try { this.truncateCleanup?.(); } catch (error) { firstCleanupError = error; } this.truncateCleanup = void 0; this.truncateReplaySession = void 0; this.stalePublishedRows.clear(); for (const demand of this.subsetDemands) { try { this.releaseSubsetDemand(demand); } catch (error) { firstCleanupError ??= error; } } this.subsetDemands = []; try { this.emitInner(`unsubscribed`, { type: `unsubscribed`, subscription: this }); } catch (error) { firstCleanupError ??= error; } finally { this.clearListeners(); } if (firstCleanupError !== void 0) throw firstCleanupError; } } export { CollectionSubscription }; //# sourceMappingURL=subscription.js.map