UNPKG

@tanstack/db

Version:

A reactive client store for building super fast apps on sync

900 lines (899 loc) • 28.9 kB
import { D2, output } from "@tanstack/db-ivm"; import { compileQuery } from "../compiler/index.js"; import { SetWindowRequiresOrderByError, SetWindowReentrancyError, LoadSubsetOperationAbortedError, MissingAliasInputsError } from "../../errors.js"; import { withPublicationContext } from "../../scheduler.js"; import { createDeferred } from "../../deferred.js"; import { deepEquals } from "../../utils.js"; import { runAllCallbacks } from "../../utils/callbacks.js"; import { normalizeError } from "../../utils/error.js"; import { createSourceRecord } from "../../utils/source-record.js"; import { CollectionSubscriber } from "./collection-subscriber.js"; import { getCollectionBuilder } from "./collection-registry.js"; import { LIVE_QUERY_INTERNAL } from "./internal.js"; import { materializeCompilation } from "./materialized-pipeline.js"; import { BucketFacadeAdapter } from "./bucket-facade-adapter.js"; import { scheduleQueryGraphRun } from "./graph-scheduler.js"; import { hasPendingJoinedWork } from "./subset-demand-controller.js"; import { buildQueryFromConfig, extractCollectionsFromQuery, extractCollectionFromSource } from "./utils.js"; import { collectCollectionSources } from "../ir.js"; let liveQueryCollectionCounter = 0; class CollectionConfigBuilder { constructor(config) { this.config = config; this.fz = /* @__PURE__ */ new WeakMap(); this.cN = /* @__PURE__ */ new WeakMap(); this.br = false; this.bq = 0; this.b6 = false; this.cv = false; this.cc = /* @__PURE__ */ new Set(); this.bh = /* @__PURE__ */ new Set(); this.subscriptions = createSourceRecord(); this.lazySourcesCallbacks = createSourceRecord(); this.lazySources = /* @__PURE__ */ new Set(); this.aQ = /* @__PURE__ */ new Map(); this.fp = 0; this.aG = /* @__PURE__ */ new Set(); this.ba = false; this.dq = false; this.syncRunGeneration = 0; this.windowOperationGeneration = 0; this.optimizableOrderByCollections = createSourceRecord(); this.id = config.id || `live-query-${++liveQueryCollectionCounter}`; this.query = buildQueryFromConfig({ query: config.query, requireObjectResult: true }); this.cV = this.query.orderBy?.length ? { offset: this.query.offset ?? 0, limit: this.query.limit ?? Infinity } : void 0; this.bD = this.cV; this.collections = extractCollectionsFromQuery(this.query); this.ai = collectCollectionSources(this.query); if (this.query.orderBy && this.query.orderBy.length > 0) { this.compare = createOrderByComparator(this.cN); } this.queryCompareOptions = extractCollectionFromSource( this.query ).compareOptions; this.compareOptions = this.config.defaultStringCollation ?? this.queryCompareOptions; this.ef(); } /** Direct lexical source Collections, including joins, unions, and includes. */ getSourceCollections() { return this.ai.map(({ collection }) => collection); } /** * Recursively checks if a query or any of its subqueries contains joins */ hasJoins(query) { if (query.join && query.join.length > 0) { return true; } if (query.from.type === `queryRef`) { if (this.hasJoins(query.from.query)) { return true; } } else if (query.from.type === `unionFrom`) { for (const source of query.from.sources) { if (source.type === `queryRef` && this.hasJoins(source.query)) { return true; } } } else if (query.from.type === `unionAll`) { for (const branch of query.from.queries) { if (this.hasJoins(branch)) { return true; } } } return false; } getConfig() { const builder = this; return { id: this.id, getKey: this.config.getKey || ((item) => this.fz.get(item) ?? item.$key), sync: this.fE(), compare: this.compare, defaultStringCollation: this.compareOptions, gcTime: this.config.gcTime ?? 5e3, // 5 seconds by default for live queries schema: this.config.schema, onInsert: this.config.onInsert, onUpdate: this.config.onUpdate, onDelete: this.config.onDelete, startSync: this.config.startSync, singleResult: this.query.singleResult, utils: { get lastSubsetError() { return builder.lastSubsetError; }, setWindow: this.setWindow.bind(this), getWindow: this.getWindow.bind(this), [LIVE_QUERY_INTERNAL]: { getBuilder: () => this, hasCustomGetKey: !!this.config.getKey, hasJoins: this.hasJoins(this.query), hasDistinct: !!this.query.distinct } } }; } setWindow(options) { const windowFn = this.f$; if (!windowFn) { throw new SetWindowRequiresOrderByError(); } if (this.ag || this.br || Object.values(this.optimizableOrderByCollections).some( (info) => info.isRequesting?.() )) { throw new SetWindowReentrancyError(); } const baseWindow = this.L ?? this.bD ?? this.cV; const requestedWindow = { offset: options.offset ?? baseWindow?.offset, limit: options.limit ?? baseWindow?.limit }; const sourceRecovery = this.eV(); if (sourceRecovery) { return sourceRecovery.then(async () => { const settlement2 = this.setWindow(requestedWindow); if (settlement2 !== true) await settlement2; }); } if (this.eF()) { return Promise.reject( this.lastSubsetError ?? new Error(`Source recovery failed`) ); } const windowOperationGeneration = ++this.windowOperationGeneration; const loadOperation = this.liveQueryCollection?._sync.beginLoadSubsetOperation(); for (const info of Object.values(this.optimizableOrderByCollections)) { if (!info.joinedFilterSourceId) continue; for (const plan of this.lazySourcesCallbacks[info.joinedFilterSourceId]?.plans ?? []) { const participant = this.aQ.get(plan.id)?.participant; if (participant?.isPending()) { this.trackSubsetLoadOperationPromise(participant.promise); } } } const previousOperation = this.ag; const operation = { generation: windowOperationGeneration, failed: false }; this.ag = operation; this.dq = false; if (this.aG.size === 0) this.ba = false; try { this.L = requestedWindow; withPublicationContext(() => { windowFn(requestedWindow); this.bx?.(); }); if (operation.failed) throw operation.error; } catch (error) { if (windowOperationGeneration === this.windowOperationGeneration) { this.dq = true; this.L = this.bD; } loadOperation?.cancel(); throw error; } finally { this.ag = previousOperation; } const settlement = loadOperation?.wait() ?? true; if (settlement === true) { this.bD = requestedWindow; return true; } return settlement.then( () => { if (windowOperationGeneration === this.windowOperationGeneration) { this.bD = requestedWindow; } }, (error) => { if (windowOperationGeneration === this.windowOperationGeneration) { this.dq = true; this.L = this.bD; } throw error; } ); } getWindow() { const window = this.bD ?? this.cV; if (!this.f$ || !window) { return void 0; } return { offset: window.offset ?? 0, limit: window.limit ?? 0 }; } isLazySource(sourceId) { return this.lazySources.has(sourceId); } beginDemand(planId, pending = false) { const generation = ++this.fp; const previous = this.aQ.get(planId); const participant = pending ? createDeferred() : void 0; if (participant) { void participant.promise.catch(() => { }); this.trackSubsetLoadOperationPromise(participant.promise); } this.aQ.set(planId, { generation, settled: false, participant }); previous?.participant?.resolve(); return generation; } settleDemand(planId, generation) { const demand = this.aQ.get(planId); if (!demand || demand.generation !== generation || demand.settled) return; demand.settled = true; try { this.bx?.(); demand.participant?.resolve(); } catch (error) { demand.participant?.reject(error); throw error; } } hasPendingJoinedWork(joinedSourceId) { return hasPendingJoinedWork( joinedSourceId, this.lazySourcesCallbacks, this.subscriptions, (planId) => this.aQ.get(planId)?.settled === false ); } failDemand(planId, generation, error) { const demand = this.aQ.get(planId); if (!demand || demand.generation !== generation) return; const normalized = this.recordSubsetError(error); demand.participant?.reject(normalized); this.cI( `Subset demand '${planId}' failed: ${normalized.message}`, normalized ); } recordSubsetError(error, fatalBeforeReady = false) { const normalized = normalizeError(error); this.lastSubsetError = normalized; if (this.ag) { this.ag.failed = true; this.ag.error = normalized; this.ba = true; } if (fatalBeforeReady) { this.cI( `Initial subset load failed: ${normalized.message}`, normalized ); } return normalized; } trackSubsetLoadPromise(promise) { this.liveQueryCollection._sync.trackLoadPromise(promise); } trackSubsetLoadOperationPromise(promise) { this.liveQueryCollection._sync.trackLoadSubsetOperationPromise(promise); } hasActiveWindowOperation() { return this.ag !== void 0; } getActiveWindowOperationGeneration() { return this.ag?.generation; } scheduleGraphRunIfSyncRunCurrent(syncRunGeneration) { if (syncRunGeneration !== this.syncRunGeneration || !this.currentSyncConfig || !this.currentSyncState) { return; } this.scheduleGraphRun(); } trackOrderedLoadPromise(promise, holdPublication = false) { if (!holdPublication && !this.ag && this.liveQueryCollection?.status !== `loading` && this.aG.size === 0) { return; } const syncRunGeneration = this.syncRunGeneration; if (this.aG.size === 0) this.ba = false; this.aG.add(promise); const finish = (succeeded) => { if (syncRunGeneration !== this.syncRunGeneration || !this.aG.delete(promise)) { return; } if (!succeeded) this.ba = true; if (!this.ba && this.aG.size === 0) { this.scheduleGraphRun(); } }; void promise.then( () => finish(true), () => finish(false) ); } retireDemand(planId) { const demand = this.aQ.get(planId); this.aQ.delete(planId); demand?.participant?.resolve(); } hasPendingSourceRecovery() { return Object.values(this.subscriptions).some( (subscription) => subscription.hasPendingTruncateReplacement ); } eV() { const pending = Object.values(this.subscriptions).flatMap( (subscription) => subscription.pendingTruncateReplacement ? [subscription.pendingTruncateReplacement] : [] ); return pending.length > 0 ? Promise.all(pending).then(() => void 0) : void 0; } eF() { return Object.values(this.subscriptions).some( (subscription) => subscription.hasFailedTruncateReplacement ); } getSyncRunGeneration() { return this.syncRunGeneration; } advanceGraphInputRevision() { this.bq++; } getGraphInputRevision() { return this.bq; } // Every graph turn checks ordered demand after D2 reaches a checkpoint. // The source loader owns the fixed-point latches that prevent repeat requests. maybeRunGraph() { if (this.br) { return; } if (!this.currentSyncConfig || !this.currentSyncState) { throw new Error( `maybeRunGraph called without active sync run. This should not happen.` ); } this.br = true; try { const syncRunGeneration = this.syncRunGeneration; const config = this.currentSyncConfig; const { begin, commit } = config; const syncState = this.currentSyncState; const isCurrentSyncRun = () => syncRunGeneration === this.syncRunGeneration && this.currentSyncConfig === config && this.currentSyncState === syncState; if (this.b6) { return; } if (syncState.subscribedToAllCollections) { let checkedLoaders = false; while (syncState.graph.pendingWork() || !checkedLoaders) { if (syncState.graph.pendingWork()) { try { syncState.graph.run(); } catch (error) { if (isCurrentSyncRun()) { this.cI(`Live query graph failed`, error); } throw error; } if (!isCurrentSyncRun()) return; } this.e_?.(); if (!isCurrentSyncRun()) return; checkedLoaders = true; } syncState.flushPendingChanges?.(); if (!isCurrentSyncRun()) return; if (syncState.messagesCount === 0) { begin(); commit(); } this.bS(config); } } finally { this.br = false; } } /** * Schedules a graph run with the transaction-scoped scheduler. * Ensures each builder runs at most once per transaction, with automatic dependency tracking * to run parent queries before child queries. Outside a transaction, runs immediately. * * Multiple calls during a transaction are coalesced into a single execution. * Dependencies are discovered from subscribed live queries. * * Uses the current sync run's config and syncState from instance properties. * * @param options - Optional scheduling configuration * @param options.contextId - Transaction ID to group work; defaults to active transaction */ scheduleGraphRun(options) { if (!this.currentSyncConfig || !this.currentSyncState) { throw new Error( `scheduleGraphRun called without active sync run. This should not happen.` ); } const syncRunGeneration = this.syncRunGeneration; scheduleQueryGraphRun( this, this.bh, () => { if (syncRunGeneration === this.syncRunGeneration && this.currentSyncConfig && this.currentSyncState) { this.maybeRunGraph(); } }, options?.contextId ); } fE() { return { rowUpdateMode: `full`, sync: this.fP.bind(this) }; } fP(config) { const syncRunGeneration = ++this.syncRunGeneration; this.liveQueryCollection = config.collection; this.b6 = false; this.cv = false; this.cc.clear(); this.lastSubsetError = void 0; this.currentSyncConfig = config; const syncState = { messagesCount: 0, subscribedToAllCollections: false, unsubscribeCallbacks: /* @__PURE__ */ new Set() }; let tornDown = false; const teardown = () => { if (tornDown) return; tornDown = true; if (this.syncRunGeneration === syncRunGeneration) { this.syncRunGeneration++; } try { runAllCallbacks(syncState.unsubscribeCallbacks); } finally { syncState.unsubscribeCallbacks.clear(); this.fl(); } }; try { const fullSyncState = this.dC( config, syncState ); this.currentSyncState = fullSyncState; const loadingSubsetUnsubscribe = config.collection.on( `loadingSubset:change`, (event) => { if (!event.isLoadingSubset) { this.bS(config); if (this.hasPendingSourceRecovery()) this.bx?.(); } } ); syncState.unsubscribeCallbacks.add(loadingSubsetUnsubscribe); this.e_ = this.ev(config, fullSyncState); this.bx = () => this.scheduleGraphRun(); this.scheduleGraphRun(); } catch (error) { try { teardown(); } catch { } throw error; } return teardown; } fl() { this.windowOperationGeneration++; this.currentSyncConfig = void 0; this.currentSyncState = void 0; this.bx = void 0; this.e_ = void 0; this.L = void 0; this.bD = this.cV; this.b6 = false; this.cv = false; this.cc.clear(); this.d3 = void 0; this.cY = void 0; this.dL = void 0; this.sourceWhereClausesCache = void 0; this.dg = void 0; this.lazySources.clear(); for (const demand of this.aQ.values()) { demand.participant?.reject(new LoadSubsetOperationAbortedError()); } this.aQ.clear(); this.aG.clear(); this.ba = false; this.dq = false; this.optimizableOrderByCollections = createSourceRecord(); this.lazySourcesCallbacks = createSourceRecord(); Object.keys(this.subscriptions).forEach( (key) => delete this.subscriptions[key] ); } /** * Compiles the query pipeline with all declared aliases. */ ef() { this.d3 = new D2(); const inputs = createSourceRecord(); for (const source of this.ai) { inputs[source.sourceId] = this.d3.newInput(); } this.cY = inputs; const compilation = compileQuery( this.query, this.cY, this.collections, this.subscriptions, this.lazySourcesCallbacks, this.lazySources, this.optimizableOrderByCollections, (windowFn) => { this.f$ = windowFn; if (this.L) { windowFn(this.L); } } ); const materialized = materializeCompilation( compilation, this.config.getKey, this.hasJoins(this.query) ); this.dL = materialized.pipeline; this.sourceWhereClausesCache = compilation.sourceWhereClauses; this.dg = materialized.facades; const missingSources = this.ai.map((source) => source.sourceId).filter((sourceId) => !Object.hasOwn(this.cY, sourceId)); if (missingSources.length > 0) { throw new MissingAliasInputsError(missingSources); } } eA() { if (!this.d3 || !this.cY || !this.dL) { this.ef(); } return { graph: this.d3, inputs: this.cY, pipeline: this.dL }; } dC(config, syncState) { const { begin, commit } = config; const { graph, inputs, pipeline } = this.eA(); let pendingChanges = /* @__PURE__ */ new Map(); pipeline.pipe( output((data) => { const messages = data.getInner(); syncState.messagesCount += messages.length; messages.reduce(accumulateChanges, pendingChanges); }) ); const facadeCompilations = this.dg; const bucketFacades = facadeCompilations ? new BucketFacadeAdapter(this.id, facadeCompilations, (count) => { syncState.messagesCount += count; }) : void 0; if (bucketFacades) { syncState.unsubscribeCallbacks.add(() => bucketFacades.cleanup()); } syncState.flushPendingChanges = () => { const hasParentChanges = pendingChanges.size > 0; const hasChildChanges = bucketFacades?.hasPendingChanges() ?? false; if (!hasParentChanges && !hasChildChanges) { return; } if (this.dq || this.ba || this.hasPendingSourceRecovery() || this.aG.size > 0 || Object.values(this.optimizableOrderByCollections).some( (info) => info.joinedFilterSourceId !== void 0 && this.hasPendingJoinedWork(info.joinedFilterSourceId) )) { return; } for (const [key, { inserts, deletes }] of pendingChanges) { if (Math.abs(inserts - deletes) > 1) { throw new Error( `Live query result key ${String(key)} changed by ${inserts - deletes} rows in one flush; a key has at most one result row.` ); } } let facadePublication; let rootPublication; try { facadePublication = bucketFacades?.flush(); rootPublication = hasParentChanges ? config.collection._deferPublication() : void 0; const changesToApply = bucketFacades ? new Map( [...pendingChanges].map(([key, changes]) => { const resolved = { ...changes, value: bucketFacades.resolve(changes.value) }; if (changes.previousValue !== void 0) { resolved.previousValue = bucketFacades.resolve( changes.previousValue ); } return [key, resolved]; }) ) : pendingChanges; facadePublication?.prepare(); if (hasParentChanges) { begin(); let lookup; const hasSyncedKey = (key) => { lookup ??= config.collection._state.createSyncedKeyLookup(); return lookup(key); }; changesToApply.forEach( this.fG.bind(this, config, hasSyncedKey) ); if (hasOrderOnlyMove(changesToApply)) { markLayoutChange(config.collection); } commit(); } } catch (error) { rootPublication?.discard(); facadePublication?.rollback(); throw error; } pendingChanges = /* @__PURE__ */ new Map(); let publicationError; for (const publish of [ rootPublication?.publish, facadePublication?.publish ]) { if (!publish) continue; try { publish(); } catch (error) { publicationError ??= error; } } if (publicationError !== void 0) throw publicationError; }; graph.finalize(); syncState.graph = graph; syncState.inputs = inputs; syncState.pipeline = pipeline; return syncState; } fG(config, hasSyncedKey, changes, key) { const { write, collection } = config; const { deletes, inserts, value, orderByIndex } = changes; this.fz.set(value, key); if (orderByIndex !== void 0) { this.cN.set(value, orderByIndex); } if (inserts && deletes === 0) { write({ value, type: `insert` }); } else if ( // Insert & update(s) (updates are a delete & insert) inserts > deletes || // A balanced delta updates an existing authoritative row, even if an // optimistic delete hides it or its earlier insert is still queued. inserts === deletes && hasSyncedKey(collection.getKeyFromItem(value)) ) { write({ value, type: `update` }); } else if (deletes > 0) { write({ value, type: `delete` }); } else { throw new Error( `Could not apply changes: ${JSON.stringify(changes)}. This should never happen.` ); } } /** * Handle status changes from source collections */ dj(config, sourceId, collectionId, event) { const { status } = event; if (status === `error`) { this.cc.add(sourceId); this.e8( `Source collection '${collectionId}' entered error state` ); return; } if (status === `cleaned-up`) { this.di(collectionId); return; } if (status === `ready`) { const recovered = this.cc.delete(sourceId); if (recovered && !this.cv && this.cc.size === 0) { this.b6 = false; this.bx?.(); } if (Object.values(this.optimizableOrderByCollections).some( (info) => info.joinedFilterSourceId === sourceId )) { this.bx?.(); } } this.bS(config); } di(collectionId) { if (this.cv) return; const error = new Error( `Source collection '${collectionId}' was manually cleaned up while live query '${this.id}' depends on it. Live queries prevent automatic GC, so this was likely a manual cleanup() call.` ); this.cI(error.message, error); return error; } /** * Update the live query status based on source collection statuses */ bS(config) { const { markReady } = config; if (this.b6) { return; } const subscribedToAll = this.currentSyncState?.subscribedToAllCollections; const allReady = this.eD(); const allDemandsSettled = [...this.aQ.values()].every( (demand) => demand.settled ); const isLoading = this.liveQueryCollection?.isLoadingSubset; if (subscribedToAll && allReady && allDemandsSettled && !isLoading) { markReady(); } } /** * Transition the live query to error state */ cI(message, error) { this.cv = true; this.e8(message, error); } e8(message, error) { this.b6 = true; console.error(`[Live Query Error] ${message}`); this.liveQueryCollection?._lifecycle.markError(error ?? new Error(message)); } eD() { return this.ai.every( (source) => ( // Only on-demand sources settle through route demand. Eager // loadSubset calls return immediately, so they must reach ready. this.lazySources.has(source.sourceId) && source.collection.config.syncMode === `on-demand` || source.collection.isReady() ) ); } /** * Creates one subscription per lexical collection source. * Each source gets independent filters, even when aliases or collections repeat. * Example: `{ employee: col, manager: col }` creates two separate subscriptions. */ ev(config, syncState) { if (this.ai.length === 0) { throw new Error( `Query '${this.id}' has no collection sources. This should not happen; please report.` ); } const loaders = this.ai.map((source) => { const { sourceId, alias, collection } = source; const collectionId = collection.id; const dependencyBuilder = getCollectionBuilder(collection); if (dependencyBuilder && dependencyBuilder !== this) { this.bh.add(dependencyBuilder); } const collectionSubscriber = new CollectionSubscriber( sourceId, alias, collection, this ); const statusUnsubscribe = collection.on(`status:change`, (event) => { this.dj(config, sourceId, collectionId, event); }); let cleanupStartError; const cleanupStartUnsubscribe = collection._onCleanupStart(() => { cleanupStartError = this.di(collectionId); }); if (cleanupStartError) { cleanupStartUnsubscribe(); statusUnsubscribe(); throw cleanupStartError; } syncState.unsubscribeCallbacks.add(statusUnsubscribe); syncState.unsubscribeCallbacks.add(cleanupStartUnsubscribe); if (collection.status === `error`) { this.dj(config, sourceId, collectionId, { type: `status:change`, collection, status: `error`, previousStatus: `error` }); } const subscription = collectionSubscriber.subscribe(); this.subscriptions[sourceId] = subscription; const lazyCallbacks = this.lazySourcesCallbacks[sourceId]; if (lazyCallbacks) { lazyCallbacks.setDemand = (plan, keys) => collectionSubscriber.setDemand(subscription, plan, keys); for (const plan of lazyCallbacks.plans ?? []) { if (plan.initialKeys.size > 0) { lazyCallbacks.setDemand(plan, plan.initialKeys); } } } const loadMore = collectionSubscriber.loadMoreIfNeeded.bind( collectionSubscriber, subscription ); return loadMore; }); syncState.subscribedToAllCollections = true; return () => runAllCallbacks(loaders); } } function createOrderByComparator(orderByIndices) { return (val1, val2) => { const index1 = orderByIndices.get(val1); const index2 = orderByIndices.get(val2); if (index1 && index2) { if (index1 < index2) { return -1; } else if (index1 > index2) { return 1; } else { return 0; } } return 0; }; } function accumulateChanges(acc, [[key, tupleData], multiplicity]) { const [value, orderByIndex] = tupleData; const changes = acc.get(key) || { deletes: 0, inserts: 0, value, orderByIndex }; if (multiplicity < 0) { if (changes.deletes === 0) { changes.previousValue = value; changes.previousOrderByIndex = orderByIndex; } changes.deletes += Math.abs(multiplicity); } else if (multiplicity > 0) { changes.inserts += multiplicity; changes.value = value; if (orderByIndex !== void 0) { changes.orderByIndex = orderByIndex; } } acc.set(key, changes); return acc; } function hasOrderOnlyMove(changesToApply) { for (const changes of changesToApply.values()) { const isUpdate = changes.inserts > 0 && changes.deletes > 0; if (isUpdate && changes.previousValue !== void 0 && deepEquals(changes.previousValue, changes.value) && changes.orderByIndex !== changes.previousOrderByIndex) { return true; } } return false; } function markLayoutChange(collection) { collection._markLayoutChange(); } export { CollectionConfigBuilder }; //# sourceMappingURL=collection-config-builder.js.map