@tanstack/db/dist/cjs/query/live/collection-config-builder.cjs.map

A reactive client store for building super fast apps on sync

{"version":3,"file":"collection-config-builder.cjs","sources":["../../../../src/query/live/collection-config-builder.ts"],"sourcesContent":["import { D2, output, serializeValue } from '@tanstack/db-ivm'\nimport { INCLUDES_ROUTING, compileQuery } from '../compiler/index.js'\nimport { createCollection } from '../../collection/index.js'\nimport {\n  MissingAliasInputsError,\n  SetWindowRequiresOrderByError,\n} from '../../errors.js'\nimport { transactionScopedScheduler } from '../../scheduler.js'\nimport { getActiveTransaction } from '../../transactions.js'\nimport { CollectionSubscriber } from './collection-subscriber.js'\nimport { getCollectionBuilder } from './collection-registry.js'\nimport { LIVE_QUERY_INTERNAL } from './internal.js'\nimport {\n  buildQueryFromConfig,\n  extractCollectionAliases,\n  extractCollectionFromSource,\n  extractCollectionsFromQuery,\n} from './utils.js'\nimport type { LiveQueryInternalUtils } from './internal.js'\nimport type {\n  IncludesCompilationResult,\n  WindowOptions,\n} from '../compiler/index.js'\nimport type { SchedulerContextId } from '../../scheduler.js'\nimport type { CollectionSubscription } from '../../collection/subscription.js'\nimport type { RootStreamBuilder } from '@tanstack/db-ivm'\nimport type { OrderByOptimizationInfo } from '../compiler/order-by.js'\nimport type { Collection } from '../../collection/index.js'\nimport type {\n  ChangeMessage,\n  CollectionConfigSingleRowOption,\n  KeyedStream,\n  ResultStream,\n  StringCollationConfig,\n  SyncConfig,\n  UtilsRecord,\n} from '../../types.js'\nimport type { Context, GetResult } from '../builder/types.js'\nimport type {\n  BasicExpression,\n  IncludesMaterialization,\n  PropRef,\n  QueryIR,\n} from '../ir.js'\nimport type { LazyCollectionCallbacks } from '../compiler/joins.js'\nimport type {\n  Changes,\n  FullSyncState,\n  LiveQueryCollectionConfig,\n  SyncState,\n} from './types.js'\nimport type { AllCollectionEvents } from '../../collection/events.js'\n\nexport type LiveQueryCollectionUtils = UtilsRecord & {\n  getRunCount: () => number\n  /**\n   * Sets the offset and limit of an ordered query.\n   * Is a no-op if the query is not ordered.\n   *\n   * @returns `true` if no subset loading was triggered, or `Promise<void>` that resolves when the subset has been loaded\n   */\n  setWindow: (options: WindowOptions) => true | Promise<void>\n  /**\n   * Gets the current window (offset and limit) for an ordered query.\n   *\n   * @returns The current window settings, or `undefined` if the query is not windowed\n   */\n  getWindow: () => { offset: number; limit: number } | undefined\n  [LIVE_QUERY_INTERNAL]: LiveQueryInternalUtils\n}\n\ntype PendingGraphRun = {\n  loadCallbacks: Set<() => boolean>\n}\n\n// Global counter for auto-generated collection IDs\nlet liveQueryCollectionCounter = 0\n\ntype SyncMethods<TResult extends object> = Parameters<\n  SyncConfig<TResult>[`sync`]\n>[0]\n\nexport class CollectionConfigBuilder<\n  TContext extends Context,\n  TResult extends object = GetResult<TContext>,\n> {\n  private readonly id: string\n  readonly query: QueryIR\n  private readonly collections: Record<string, Collection<any, any, any>>\n  private readonly collectionByAlias: Record<string, Collection<any, any, any>>\n  // Populated during compilation with all aliases (including subquery inner aliases)\n  private compiledAliasToCollectionId: Record<string, string> = {}\n\n  // WeakMap to store the keys of the results\n  // so that we can retrieve them in the getKey function\n  private readonly resultKeys = new WeakMap<object, unknown>()\n\n  // WeakMap to store the orderBy index for each result\n  private readonly orderByIndices = new WeakMap<object, string>()\n\n  private readonly compare?: (val1: TResult, val2: TResult) => number\n  private readonly compareOptions?: StringCollationConfig\n\n  private isGraphRunning = false\n  private runCount = 0\n\n  // Current sync session state (set when sync starts, cleared when it stops)\n  // Public for testing purposes (CollectionConfigBuilder is internal, not public API)\n  public currentSyncConfig:\n    | Parameters<SyncConfig<TResult>[`sync`]>[0]\n    | undefined\n  public currentSyncState: FullSyncState | undefined\n\n  // Error state tracking\n  private isInErrorState = false\n\n  // Reference to the live query collection for error state transitions\n  public liveQueryCollection?: Collection<TResult, any, any>\n\n  private windowFn: ((options: WindowOptions) => void) | undefined\n  private currentWindow: WindowOptions | undefined\n\n  private maybeRunGraphFn: (() => void) | undefined\n\n  private readonly aliasDependencies: Record<\n    string,\n    Array<CollectionConfigBuilder<any, any>>\n  > = {}\n\n  private readonly builderDependencies = new Set<\n    CollectionConfigBuilder<any, any>\n  >()\n\n  // Pending graph runs per scheduler context (e.g., per transaction)\n  // The builder manages its own state; the scheduler just orchestrates execution order\n  // Only stores callbacks - if sync ends, pending jobs gracefully no-op\n  private readonly pendingGraphRuns = new Map<\n    SchedulerContextId,\n    PendingGraphRun\n  >()\n\n  // Unsubscribe function for scheduler's onClear listener\n  // Registered when sync starts, unregistered when sync stops\n  // Prevents memory leaks by releasing the scheduler's reference to this builder\n  private unsubscribeFromSchedulerClears?: () => void\n\n  private graphCache: D2 | undefined\n  private inputsCache: Record<string, RootStreamBuilder<unknown>> | undefined\n  private pipelineCache: ResultStream | undefined\n  public sourceWhereClausesCache:\n    | Map<string, BasicExpression<boolean>>\n    | undefined\n  private includesCache: Array<IncludesCompilationResult> | undefined\n\n  // Map of source alias to subscription\n  readonly subscriptions: Record<string, CollectionSubscription> = {}\n  // Map of source aliases to functions that load keys for that lazy source\n  lazySourcesCallbacks: Record<string, LazyCollectionCallbacks> = {}\n  // Set of source aliases that are lazy (don't load initial state)\n  readonly lazySources = new Set<string>()\n  // Set of collection IDs that include an optimizable ORDER BY clause\n  optimizableOrderByCollections: Record<string, OrderByOptimizationInfo> = {}\n\n  constructor(\n    private readonly config: LiveQueryCollectionConfig<TContext, TResult>,\n  ) {\n    // Generate a unique ID if not provided\n    this.id = config.id || `live-query-${++liveQueryCollectionCounter}`\n\n    this.query = buildQueryFromConfig({\n      query: config.query,\n      requireObjectResult: true,\n    })\n    this.collections = extractCollectionsFromQuery(this.query)\n    const collectionAliasesById = extractCollectionAliases(this.query)\n\n    // Build a reverse lookup map from alias to collection instance.\n    // This enables self-join support where the same collection can be referenced\n    // multiple times with different aliases (e.g., { employee: col, manager: col })\n    this.collectionByAlias = {}\n    for (const [collectionId, aliases] of collectionAliasesById.entries()) {\n      const collection = this.collections[collectionId]\n      if (!collection) continue\n      for (const alias of aliases) {\n        this.collectionByAlias[alias] = collection\n      }\n    }\n\n    // Create compare function for ordering if the query has orderBy\n    if (this.query.orderBy && this.query.orderBy.length > 0) {\n      this.compare = createOrderByComparator<TResult>(this.orderByIndices)\n    }\n\n    // Use explicitly provided compareOptions if available, otherwise inherit from FROM collection\n    this.compareOptions =\n      this.config.defaultStringCollation ??\n      extractCollectionFromSource(this.query).compareOptions\n\n    // Compile the base pipeline once initially\n    // This is done to ensure that any errors are thrown immediately and synchronously\n    this.compileBasePipeline()\n  }\n\n  /**\n   * Recursively checks if a query or any of its subqueries contains joins\n   */\n  private hasJoins(query: QueryIR): boolean {\n    // Check if this query has joins\n    if (query.join && query.join.length > 0) {\n      return true\n    }\n\n    // Recursively check subqueries in the from clause\n    if (query.from.type === `queryRef`) {\n      if (this.hasJoins(query.from.query)) {\n        return true\n      }\n    }\n\n    return false\n  }\n\n  getConfig(): CollectionConfigSingleRowOption<TResult> & {\n    utils: LiveQueryCollectionUtils\n  } {\n    return {\n      id: this.id,\n      getKey:\n        this.config.getKey ||\n        ((item: any) =>\n          (this.resultKeys.get(item) ?? item.$key) as string | number),\n      sync: this.getSyncConfig(),\n      compare: this.compare,\n      defaultStringCollation: this.compareOptions,\n      gcTime: this.config.gcTime || 5000, // 5 seconds by default for live queries\n      schema: this.config.schema,\n      onInsert: this.config.onInsert,\n      onUpdate: this.config.onUpdate,\n      onDelete: this.config.onDelete,\n      startSync: this.config.startSync,\n      singleResult: this.query.singleResult,\n      utils: {\n        getRunCount: this.getRunCount.bind(this),\n        setWindow: this.setWindow.bind(this),\n        getWindow: this.getWindow.bind(this),\n        [LIVE_QUERY_INTERNAL]: {\n          getBuilder: () => this,\n          hasCustomGetKey: !!this.config.getKey,\n          hasJoins: this.hasJoins(this.query),\n          hasDistinct: !!this.query.distinct,\n        },\n      },\n    }\n  }\n\n  setWindow(options: WindowOptions): true | Promise<void> {\n    if (!this.windowFn) {\n      throw new SetWindowRequiresOrderByError()\n    }\n\n    this.currentWindow = options\n    this.windowFn(options)\n    this.maybeRunGraphFn?.()\n\n    // Check if loading a subset was triggered\n    if (this.liveQueryCollection?.isLoadingSubset) {\n      // Loading was triggered, return a promise that resolves when it completes\n      return new Promise<void>((resolve) => {\n        const unsubscribe = this.liveQueryCollection!.on(\n          `loadingSubset:change`,\n          (event) => {\n            if (!event.isLoadingSubset) {\n              unsubscribe()\n              resolve()\n            }\n          },\n        )\n      })\n    }\n\n    // No loading was triggered\n    return true\n  }\n\n  getWindow(): { offset: number; limit: number } | undefined {\n    // Only return window if this is a windowed query (has orderBy and windowFn)\n    if (!this.windowFn || !this.currentWindow) {\n      return undefined\n    }\n    return {\n      offset: this.currentWindow.offset ?? 0,\n      limit: this.currentWindow.limit ?? 0,\n    }\n  }\n\n  /**\n   * Resolves a collection alias to its collection ID.\n   *\n   * Uses a two-tier lookup strategy:\n   * 1. First checks compiled aliases (includes subquery inner aliases)\n   * 2. Falls back to declared aliases from the query's from/join clauses\n   *\n   * @param alias - The alias to resolve (e.g., \"employee\", \"manager\")\n   * @returns The collection ID that the alias references\n   * @throws {Error} If the alias is not found in either lookup\n   */\n  getCollectionIdForAlias(alias: string): string {\n    const compiled = this.compiledAliasToCollectionId[alias]\n    if (compiled) {\n      return compiled\n    }\n    const collection = this.collectionByAlias[alias]\n    if (collection) {\n      return collection.id\n    }\n    throw new Error(`Unknown source alias \"${alias}\"`)\n  }\n\n  isLazyAlias(alias: string): boolean {\n    return this.lazySources.has(alias)\n  }\n\n  // The callback function is called after the graph has run.\n  // This gives the callback a chance to load more data if needed,\n  // that's used to optimize orderBy operators that set a limit,\n  // in order to load some more data if we still don't have enough rows after the pipeline has run.\n  // That can happen because even though we load N rows, the pipeline might filter some of these rows out\n  // causing the orderBy operator to receive less than N rows or even no rows at all.\n  // So this callback would notice that it doesn't have enough rows and load some more.\n  // The callback returns a boolean, when it's true it's done loading data and we can mark the collection as ready.\n  maybeRunGraph(callback?: () => boolean) {\n    if (this.isGraphRunning) {\n      // no nested runs of the graph\n      // which is possible if the `callback`\n      // would call `maybeRunGraph` e.g. after it has loaded some more data\n      return\n    }\n\n    // Should only be called when sync is active\n    if (!this.currentSyncConfig || !this.currentSyncState) {\n      throw new Error(\n        `maybeRunGraph called without active sync session. This should not happen.`,\n      )\n    }\n\n    this.isGraphRunning = true\n\n    try {\n      const { begin, commit } = this.currentSyncConfig\n      const syncState = this.currentSyncState\n\n      // Don't run if the live query is in an error state\n      if (this.isInErrorState) {\n        return\n      }\n\n      // Always run the graph if subscribed (eager execution)\n      if (syncState.subscribedToAllCollections) {\n        let callbackCalled = false\n        while (syncState.graph.pendingWork()) {\n          syncState.graph.run()\n          // Flush accumulated changes after each graph step to commit them as one transaction.\n          // This ensures intermediate join states (like null on one side) don't cause\n          // duplicate key errors when the full join result arrives in the same step.\n          syncState.flushPendingChanges?.()\n          callback?.()\n          callbackCalled = true\n        }\n\n        // Ensure the callback runs at least once even when the graph has no pending work.\n        // This handles lazy loading scenarios where setWindow() increases the limit or\n        // an async loadSubset completes and we need to re-check if more data is needed.\n        if (!callbackCalled) {\n          callback?.()\n        }\n\n        // On the initial run, we may need to do an empty commit to ensure that\n        // the collection is initialized\n        if (syncState.messagesCount === 0) {\n          begin()\n          commit()\n        }\n\n        // After graph processing completes, check if we should mark ready.\n        // This is the canonical place to transition to ready state because:\n        // 1. All data has been processed through the graph\n        // 2. All source collections have had a chance to send their initial data\n        // This prevents marking ready before data is processed (fixes isReady=true with empty data)\n        this.updateLiveQueryStatus(this.currentSyncConfig)\n      }\n    } finally {\n      this.isGraphRunning = false\n    }\n  }\n\n  /**\n   * Schedules a graph run with the transaction-scoped scheduler.\n   * Ensures each builder runs at most once per transaction, with automatic dependency tracking\n   * to run parent queries before child queries. Outside a transaction, runs immediately.\n   *\n   * Multiple calls during a transaction are coalesced into a single execution.\n   * Dependencies are auto-discovered from subscribed live queries, or can be overridden.\n   * Load callbacks are combined when entries merge.\n   *\n   * Uses the current sync session's config and syncState from instance properties.\n   *\n   * @param callback - Optional callback to load more data if needed (returns true when done)\n   * @param options - Optional scheduling configuration\n   * @param options.contextId - Transaction ID to group work; defaults to active transaction\n   * @param options.jobId - Unique identifier for this job; defaults to this builder instance\n   * @param options.alias - Source alias that triggered this schedule; adds alias-specific dependencies\n   * @param options.dependencies - Explicit dependency list; overrides auto-discovered dependencies\n   */\n  scheduleGraphRun(\n    callback?: () => boolean,\n    options?: {\n      contextId?: SchedulerContextId\n      jobId?: unknown\n      alias?: string\n      dependencies?: Array<CollectionConfigBuilder<any, any>>\n    },\n  ) {\n    const contextId = options?.contextId ?? getActiveTransaction()?.id\n    // Use the builder instance as the job ID for deduplication. This is memory-safe\n    // because the scheduler's context Map is deleted after flushing (no long-term retention).\n    const jobId = options?.jobId ?? this\n    const dependentBuilders = (() => {\n      if (options?.dependencies) {\n        return options.dependencies\n      }\n\n      const deps = new Set(this.builderDependencies)\n      if (options?.alias) {\n        const aliasDeps = this.aliasDependencies[options.alias]\n        if (aliasDeps) {\n          for (const dep of aliasDeps) {\n            deps.add(dep)\n          }\n        }\n      }\n\n      deps.delete(this)\n\n      return Array.from(deps)\n    })()\n\n    // Ensure dependent builders are actually scheduled in this context so that\n    // dependency edges always point to a real job (or a deduped no-op if already scheduled).\n    if (contextId) {\n      for (const dep of dependentBuilders) {\n        if (typeof dep.scheduleGraphRun === `function`) {\n          dep.scheduleGraphRun(undefined, { contextId })\n        }\n      }\n    }\n\n    // We intentionally scope deduplication to the builder instance. Each instance\n    // owns caches and compiled pipelines, so sharing work across instances that\n    // merely reuse the same string id would execute the wrong builder's graph.\n\n    if (!this.currentSyncConfig || !this.currentSyncState) {\n      throw new Error(\n        `scheduleGraphRun called without active sync session. This should not happen.`,\n      )\n    }\n\n    // Manage our own state - get or create pending callbacks for this context\n    let pending = contextId ? this.pendingGraphRuns.get(contextId) : undefined\n    if (!pending) {\n      pending = {\n        loadCallbacks: new Set(),\n      }\n      if (contextId) {\n        this.pendingGraphRuns.set(contextId, pending)\n      }\n    }\n\n    // Add callback if provided (this is what accumulates between schedules)\n    if (callback) {\n      pending.loadCallbacks.add(callback)\n    }\n\n    // Schedule execution (scheduler just orchestrates order, we manage state)\n    // For immediate execution (no contextId), pass pending directly since it won't be in the map\n    const pendingToPass = contextId ? undefined : pending\n    transactionScopedScheduler.schedule({\n      contextId,\n      jobId,\n      dependencies: dependentBuilders,\n      run: () => this.executeGraphRun(contextId, pendingToPass),\n    })\n  }\n\n  /**\n   * Clears pending graph run state for a specific context.\n   * Called when the scheduler clears a context (e.g., transaction rollback/abort).\n   */\n  clearPendingGraphRun(contextId: SchedulerContextId): void {\n    this.pendingGraphRuns.delete(contextId)\n  }\n\n  /**\n   * Returns true if this builder has a pending graph run for the given context.\n   */\n  hasPendingGraphRun(contextId: SchedulerContextId): boolean {\n    return this.pendingGraphRuns.has(contextId)\n  }\n\n  /**\n   * Executes a pending graph run. Called by the scheduler when dependencies are satisfied.\n   * Clears the pending state BEFORE execution so that any re-schedules during the run\n   * create fresh state and don't interfere with the current execution.\n   * Uses instance sync state - if sync has ended, gracefully returns without executing.\n   *\n   * @param contextId - Optional context ID to look up pending state\n   * @param pendingParam - For immediate execution (no context), pending state is passed directly\n   */\n  private executeGraphRun(\n    contextId?: SchedulerContextId,\n    pendingParam?: PendingGraphRun,\n  ): void {\n    // Get pending state: either from parameter (no context) or from map (with context)\n    // Remove from map BEFORE checking sync state to prevent leaking entries when sync ends\n    // before the transaction flushes (e.g., unsubscribe during in-flight transaction)\n    const pending =\n      pendingParam ??\n      (contextId ? this.pendingGraphRuns.get(contextId) : undefined)\n    if (contextId) {\n      this.pendingGraphRuns.delete(contextId)\n    }\n\n    // If no pending state, nothing to execute (context was cleared)\n    if (!pending) {\n      return\n    }\n\n    // If sync session has ended, don't execute (graph is finalized, subscriptions cleared)\n    if (!this.currentSyncConfig || !this.currentSyncState) {\n      return\n    }\n\n    this.incrementRunCount()\n\n    const combinedLoader = () => {\n      let allDone = true\n      let firstError: unknown\n      pending.loadCallbacks.forEach((loader) => {\n        try {\n          allDone = loader() && allDone\n        } catch (error) {\n          allDone = false\n          firstError ??= error\n        }\n      })\n      if (firstError) {\n        throw firstError\n      }\n      // Returning false signals that callers should schedule another pass.\n      return allDone\n    }\n\n    this.maybeRunGraph(combinedLoader)\n  }\n\n  private getSyncConfig(): SyncConfig<TResult> {\n    return {\n      rowUpdateMode: `full`,\n      sync: this.syncFn.bind(this),\n    }\n  }\n\n  incrementRunCount() {\n    this.runCount++\n  }\n\n  getRunCount() {\n    return this.runCount\n  }\n\n  private syncFn(config: SyncMethods<TResult>) {\n    // Store reference to the live query collection for error state transitions\n    this.liveQueryCollection = config.collection\n    // Store config and syncState as instance properties for the duration of this sync session\n    this.currentSyncConfig = config\n\n    const syncState: SyncState = {\n      messagesCount: 0,\n      subscribedToAllCollections: false,\n      unsubscribeCallbacks: new Set<() => void>(),\n    }\n\n    // Extend the pipeline such that it applies the incoming changes to the collection\n    const fullSyncState = this.extendPipelineWithChangeProcessing(\n      config,\n      syncState,\n    )\n    this.currentSyncState = fullSyncState\n\n    // Listen for scheduler context clears to clean up our pending state\n    // Re-register on each sync start so the listener is active for the sync session's lifetime\n    this.unsubscribeFromSchedulerClears = transactionScopedScheduler.onClear(\n      (contextId) => {\n        this.clearPendingGraphRun(contextId)\n      },\n    )\n\n    // Listen for loadingSubset changes on the live query collection BEFORE subscribing.\n    // This ensures we don't miss the event if subset loading completes synchronously.\n    // When isLoadingSubset becomes false, we may need to mark the collection as ready\n    // (if all source collections are already ready but we were waiting for subset load to complete)\n    const loadingSubsetUnsubscribe = config.collection.on(\n      `loadingSubset:change`,\n      (event) => {\n        if (!event.isLoadingSubset) {\n          // Subset loading finished, check if we can now mark ready\n          this.updateLiveQueryStatus(config)\n        }\n      },\n    )\n    syncState.unsubscribeCallbacks.add(loadingSubsetUnsubscribe)\n\n    const loadSubsetDataCallbacks = this.subscribeToAllCollections(\n      config,\n      fullSyncState,\n    )\n\n    this.maybeRunGraphFn = () => this.scheduleGraphRun(loadSubsetDataCallbacks)\n\n    // Initial run with callback to load more data if needed\n    this.scheduleGraphRun(loadSubsetDataCallbacks)\n\n    // Return the unsubscribe function\n    return () => {\n      syncState.unsubscribeCallbacks.forEach((unsubscribe) => unsubscribe())\n\n      // Clear current sync session state\n      this.currentSyncConfig = undefined\n      this.currentSyncState = undefined\n\n      // Clear all pending graph runs to prevent memory leaks from in-flight transactions\n      // that may flush after the sync session ends\n      this.pendingGraphRuns.clear()\n\n      // Reset caches so a fresh graph/pipeline is compiled on next start\n      // This avoids reusing a finalized D2 graph across GC restarts\n      this.graphCache = undefined\n      this.inputsCache = undefined\n      this.pipelineCache = undefined\n      this.sourceWhereClausesCache = undefined\n      this.includesCache = undefined\n\n      // Reset lazy source alias state\n      this.lazySources.clear()\n      this.optimizableOrderByCollections = {}\n      this.lazySourcesCallbacks = {}\n\n      // Clear subscription references to prevent memory leaks\n      // Note: Individual subscriptions are already unsubscribed via unsubscribeCallbacks\n      Object.keys(this.subscriptions).forEach(\n        (key) => delete this.subscriptions[key],\n      )\n      this.compiledAliasToCollectionId = {}\n\n      // Unregister from scheduler's onClear listener to prevent memory leaks\n      // The scheduler's listener Set would otherwise keep a strong reference to this builder\n      this.unsubscribeFromSchedulerClears?.()\n      this.unsubscribeFromSchedulerClears = undefined\n    }\n  }\n\n  /**\n   * Compiles the query pipeline with all declared aliases.\n   */\n  private compileBasePipeline() {\n    this.graphCache = new D2()\n    this.inputsCache = Object.fromEntries(\n      Object.keys(this.collectionByAlias).map((alias) => [\n        alias,\n        this.graphCache!.newInput<any>(),\n      ]),\n    )\n\n    const compilation = compileQuery(\n      this.query,\n      this.inputsCache as Record<string, KeyedStream>,\n      this.collections,\n      this.subscriptions,\n      this.lazySourcesCallbacks,\n      this.lazySources,\n      this.optimizableOrderByCollections,\n      (windowFn: (options: WindowOptions) => void) => {\n        this.windowFn = windowFn\n      },\n    )\n\n    this.pipelineCache = compilation.pipeline\n    this.sourceWhereClausesCache = compilation.sourceWhereClauses\n    this.compiledAliasToCollectionId = compilation.aliasToCollectionId\n    this.includesCache = compilation.includes\n\n    // Defensive check: verify all compiled aliases have corresponding inputs\n    // This should never happen since all aliases come from user declarations,\n    // but catch it early if the assumption is violated in the future.\n    const missingAliases = Object.keys(this.compiledAliasToCollectionId).filter(\n      (alias) => !Object.hasOwn(this.inputsCache!, alias),\n    )\n    if (missingAliases.length > 0) {\n      throw new MissingAliasInputsError(missingAliases)\n    }\n  }\n\n  private maybeCompileBasePipeline() {\n    if (!this.graphCache || !this.inputsCache || !this.pipelineCache) {\n      this.compileBasePipeline()\n    }\n    return {\n      graph: this.graphCache!,\n      inputs: this.inputsCache!,\n      pipeline: this.pipelineCache!,\n    }\n  }\n\n  private extendPipelineWithChangeProcessing(\n    config: SyncMethods<TResult>,\n    syncState: SyncState,\n  ): FullSyncState {\n    const { begin, commit } = config\n    const { graph, inputs, pipeline } = this.maybeCompileBasePipeline()\n\n    // Accumulator for changes across all output callbacks within a single graph run.\n    // This allows us to batch all changes from intermediate join states into a single\n    // transaction, avoiding duplicate key errors when joins produce multiple outputs\n    // for the same key (e.g., first output with null, then output with joined data).\n    let pendingChanges: Map<unknown, Changes<TResult>> = new Map()\n\n    pipeline.pipe(\n      output((data) => {\n        const messages = data.getInner()\n        syncState.messagesCount += messages.length\n\n        // Accumulate changes from this output callback into the pending changes map.\n        // Changes for the same key are merged (inserts/deletes are added together).\n        messages.reduce(accumulateChanges<TResult>, pendingChanges)\n      }),\n    )\n\n    // Set up includes output routing and child collection lifecycle\n    const includesState = this.setupIncludesOutput(\n      this.includesCache,\n      syncState,\n    )\n\n    // Flush pending changes and reset the accumulator.\n    // Called at the end of each graph run to commit all accumulated changes.\n    syncState.flushPendingChanges = () => {\n      const hasParentChanges = pendingChanges.size > 0\n      const hasChildChanges = hasPendingIncludesChanges(includesState)\n\n      if (!hasParentChanges && !hasChildChanges) {\n        return\n      }\n\n      let changesToApply = pendingChanges\n\n      // When a custom getKey is provided, multiple D2 internal keys may map\n      // to the same user-visible key. Re-accumulate by custom key so that a\n      // retract + insert for the same logical row merges into an UPDATE\n      // instead of a separate DELETE and INSERT that can race.\n      if (this.config.getKey) {\n        const merged = new Map<unknown, Changes<TResult>>()\n        for (const [, changes] of pendingChanges) {\n          const customKey = this.config.getKey(changes.value)\n          const existing = merged.get(customKey)\n          if (existing) {\n            existing.inserts += changes.inserts\n            existing.deletes += changes.deletes\n            // Keep the value from the insert side (the new value)\n            if (changes.inserts > 0) {\n              existing.value = changes.value\n              if (changes.orderByIndex !== undefined) {\n                existing.orderByIndex = changes.orderByIndex\n              }\n            }\n          } else {\n            merged.set(customKey, { ...changes })\n          }\n        }\n        changesToApply = merged\n      }\n\n      // 1. Flush parent changes\n      if (hasParentChanges) {\n        begin()\n        changesToApply.forEach(this.applyChanges.bind(this, config))\n        commit()\n      }\n      pendingChanges = new Map()\n\n      // 2. Process includes: create/dispose child Collections, route child changes\n      flushIncludesState(\n        includesState,\n        config.collection,\n        this.id,\n        hasParentChanges ? changesToApply : null,\n        config,\n      )\n    }\n\n    graph.finalize()\n\n    // Extend the sync state with the graph, inputs, and pipeline\n    syncState.graph = graph\n    syncState.inputs = inputs\n    syncState.pipeline = pipeline\n\n    return syncState as FullSyncState\n  }\n\n  /**\n   * Sets up output callbacks for includes child pipelines.\n   * Each includes entry gets its own output callback that accumulates child changes,\n   * and a child registry that maps correlation key → child Collection.\n   */\n  private setupIncludesOutput(\n    includesEntries: Array<IncludesCompilationResult> | undefined,\n    syncState: SyncState,\n  ): Array<IncludesOutputState> {\n    if (!includesEntries || includesEntries.length === 0) {\n      return []\n    }\n\n    return includesEntries.map((entry) => {\n      const state: IncludesOutputState = {\n        fieldName: entry.fieldName,\n        childCorrelationField: entry.childCorrelationField,\n        hasOrderBy: entry.hasOrderBy,\n        materialization: entry.materialization,\n        scalarField: entry.scalarField,\n        childRegistry: new Map(),\n        pendingChildChanges: new Map(),\n        correlationToParentKeys: new Map(),\n      }\n\n      // Attach output callback on the child pipeline\n      entry.pipeline.pipe(\n        output((data) => {\n          const messages = data.getInner()\n          syncState.messagesCount += messages.length\n\n          for (const [[childKey, tupleData], multiplicity] of messages) {\n            const [childResult, _orderByIndex, correlationKey, parentContext] =\n              tupleData as unknown as [\n                any,\n                string | undefined,\n                unknown,\n                Record<string, any> | null,\n              ]\n\n            const routingKey = computeRoutingKey(correlationKey, parentContext)\n\n            // Accumulate by [routingKey, childKey]\n            let byChild = state.pendingChildChanges.get(routingKey)\n            if (!byChild) {\n              byChild = new Map()\n              state.pendingChildChanges.set(routingKey, byChild)\n            }\n\n            const existing = byChild.get(childKey) || {\n              deletes: 0,\n              inserts: 0,\n              value: childResult,\n              orderByIndex: _orderByIndex,\n            }\n\n            if (multiplicity < 0) {\n              existing.deletes += Math.abs(multiplicity)\n            } else if (multiplicity > 0) {\n              existing.inserts += multiplicity\n              existing.value = childResult\n            }\n\n            byChild.set(childKey, existing)\n          }\n        }),\n      )\n\n      // Set up shared buffers for nested includes (e.g., comments inside issues)\n      if (entry.childCompilationResult.includes) {\n        state.nestedSetups = setupNestedPipelines(\n          entry.childCompilationResult.includes,\n          syncState,\n        )\n        state.nestedRoutingIndex = new Map()\n        state.nestedRoutingReverseIndex = new Map()\n      }\n\n      return state\n    })\n  }\n\n  private applyChanges(\n    config: SyncMethods<TResult>,\n    changes: {\n      deletes: number\n      inserts: number\n      value: TResult\n      orderByIndex: string | undefined\n    },\n    key: unknown,\n  ) {\n    const { write, collection } = config\n    const { deletes, inserts, value, orderByIndex } = changes\n\n    // Store the key of the result so that we can retrieve it in the\n    // getKey function\n    this.resultKeys.set(value, key)\n\n    // Store the orderBy index if it exists\n    if (orderByIndex !== undefined) {\n      this.orderByIndices.set(value, orderByIndex)\n    }\n\n    // Simple singular insert.\n    if (inserts && deletes === 0) {\n      write({\n        value,\n        type: `insert`,\n      })\n    } else if (\n      // Insert & update(s) (updates are a delete & insert)\n      inserts > deletes ||\n      // Just update(s) but the item is already in the collection (so\n      // was inserted previously).\n      (inserts === deletes && collection.has(collection.getKeyFromItem(value)))\n    ) {\n      write({\n        value,\n        type: `update`,\n      })\n      // Only delete is left as an option\n    } else if (deletes > 0) {\n      write({\n        value,\n        type: `delete`,\n      })\n    } else {\n      throw new Error(\n        `Could not apply changes: ${JSON.stringify(changes)}. This should never happen.`,\n      )\n    }\n  }\n\n  /**\n   * Handle status changes from source collections\n   */\n  private handleSourceStatusChange(\n    config: SyncMethods<TResult>,\n    collectionId: string,\n    event: AllCollectionEvents[`status:change`],\n  ) {\n    const { status } = event\n\n    // Handle error state - any source collection in error puts live query in error\n    if (status === `error`) {\n      this.transitionToError(\n        `Source collection '${collectionId}' entered error state`,\n      )\n      return\n    }\n\n    // Handle manual cleanup - this should not happen due to GC prevention,\n    // but could happen if user manually calls cleanup()\n    if (status === `cleaned-up`) {\n      this.transitionToError(\n        `Source collection '${collectionId}' was manually cleaned up while live query '${this.id}' depends on it. ` +\n          `Live queries prevent automatic GC, so this was likely a manual cleanup() call.`,\n      )\n      return\n    }\n\n    // Update ready status based on all source collections\n    this.updateLiveQueryStatus(config)\n  }\n\n  /**\n   * Update the live query status based on source collection statuses\n   */\n  private updateLiveQueryStatus(config: SyncMethods<TResult>) {\n    const { markReady } = config\n\n    // Don't update status if already in error\n    if (this.isInErrorState) {\n      return\n    }\n\n    const subscribedToAll = this.currentSyncState?.subscribedToAllCollections\n    const allReady = this.allCollectionsReady()\n    const isLoading = this.liveQueryCollection?.isLoadingSubset\n    // Mark ready when:\n    // 1. All subscriptions are set up (subscribedToAllCollections)\n    // 2. All source collections are ready\n    // 3. The live query collection is not loading subset data\n    // This prevents marking the live query ready before its data is processed\n    // (fixes issue where useLiveQuery returns isReady=true with empty data)\n    if (subscribedToAll && allReady && !isLoading) {\n      markReady()\n    }\n  }\n\n  /**\n   * Transition the live query to error state\n   */\n  private transitionToError(message: string) {\n    this.isInErrorState = true\n\n    // Log error to console for debugging\n    console.error(`[Live Query Error] ${message}`)\n\n    // Transition live query collection to error state\n    this.liveQueryCollection?._lifecycle.setStatus(`error`)\n  }\n\n  private allCollectionsReady() {\n    return Object.values(this.collections).every((collection) =>\n      collection.isReady(),\n    )\n  }\n\n  /**\n   * Creates per-alias subscriptions enabling self-join support.\n   * Each alias gets its own subscription with independent filters, even for the same collection.\n   * Example: `{ employee: col, manager: col }` creates two separate subscriptions.\n   */\n  private subscribeToAllCollections(\n    config: SyncMethods<TResult>,\n    syncState: FullSyncState,\n  ) {\n    // Use compiled aliases as the source of truth - these include all aliases from the query\n    // including those from subqueries, which may not be in collectionByAlias\n    const compiledAliases = Object.entries(this.compiledAliasToCollectionId)\n    if (compiledAliases.length === 0) {\n      throw new Error(\n        `Compiler returned no alias metadata for query '${this.id}'. This should not happen; please report.`,\n      )\n    }\n\n    // Create a separate subscription for each alias, enabling self-joins where the same\n    // collection can be used multiple times with different filters and subscriptions\n    const loaders = compiledAliases.map(([alias, collectionId]) => {\n      // Try collectionByAlias first (for declared aliases), fall back to collections (for subquery aliases)\n      const collection =\n        this.collectionByAlias[alias] ?? this.collections[collectionId]!\n\n      const dependencyBuilder = getCollectionBuilder(collection)\n      if (dependencyBuilder && dependencyBuilder !== this) {\n        this.aliasDependencies[alias] = [dependencyBuilder]\n        this.builderDependencies.add(dependencyBuilder)\n      } else {\n        this.aliasDependencies[alias] = []\n      }\n\n      // CollectionSubscriber handles the actual subscription to the source collection\n      // and feeds data into the D2 graph inputs for this specific alias\n      const collectionSubscriber = new CollectionSubscriber(\n        alias,\n        collectionId,\n        collection,\n        this,\n      )\n\n      // Subscribe to status changes for status flow\n      const statusUnsubscribe = collection.on(`status:change`, (event) => {\n        this.handleSourceStatusChange(config, collectionId, event)\n      })\n      syncState.unsubscribeCallbacks.add(statusUnsubscribe)\n\n      const subscription = collectionSubscriber.subscribe()\n      // Store subscription by alias (not collection ID) to support lazy loading\n      // which needs to look up subscriptions by their query alias\n      this.subscriptions[alias] = subscription\n\n      // Create a callback for loading more data if needed (used by OrderBy optimization)\n      const loadMore = collectionSubscriber.loadMoreIfNeeded.bind(\n        collectionSubscriber,\n        subscription,\n      )\n\n      return loadMore\n    })\n\n    // Combine all loaders into a single callback that initiates loading more data\n    // from any source that needs it. Returns true once all loaders have been called,\n    // but the actual async loading may still be in progress.\n    const loadSubsetDataCallbacks = () => {\n      loaders.map((loader) => loader())\n      return true\n    }\n\n    // Mark as subscribed so the graph can start running\n    // (graph only runs when all collections are subscribed)\n    syncState.subscribedToAllCollections = true\n\n    // Note: We intentionally don't call updateLiveQueryStatus() here.\n    // The graph hasn't run yet, so marking ready would be premature.\n    // The canonical place to mark ready is after the graph processes data\n    // in maybeRunGraph(), which ensures data has been processed first.\n\n    return loadSubsetDataCallbacks\n  }\n}\n\nfunction createOrderByComparator<T extends object>(\n  orderByIndices: WeakMap<object, string>,\n) {\n  return (val1: T, val2: T): number => {\n    // Use the orderBy index stored in the WeakMap\n    const index1 = orderByIndices.get(val1)\n    const index2 = orderByIndices.get(val2)\n\n    // Compare fractional indices lexicographically\n    if (index1 && index2) {\n      if (index1 < index2) {\n        return -1\n      } else if (index1 > index2) {\n        return 1\n      } else {\n        return 0\n      }\n    }\n\n    // Fallback to no ordering if indices are missing\n    return 0\n  }\n}\n\n/**\n * Shared buffer setup for a single nested includes level.\n * Pipeline output writes into the buffer; during flush the buffer is drained\n * into per-entry states via the routing index.\n */\ntype NestedIncludesSetup = {\n  compilationResult: IncludesCompilationResult\n  /** Shared buffer: nestedCorrelationKey → Map<childKey, Changes> */\n  buffer: Map<unknown, Map<unknown, Changes<any>>>\n  /** For 3+ levels of nesting */\n  nestedSetups?: Array<NestedIncludesSetup>\n}\n\n/**\n * State tracked per includes entry for output routing and child lifecycle\n */\ntype IncludesOutputState = {\n  fieldName: string\n  childCorrelationField: PropRef\n  /** Whether the child query has an ORDER BY clause */\n  hasOrderBy: boolean\n  /** How the child result is materialized on the parent row */\n  materialization: IncludesMaterialization\n  /** Internal field used to unwrap scalar child selects */\n  scalarField?: string\n  /** Maps correlation key value → child Collection entry */\n  childRegistry: Map<unknown, ChildCollectionEntry>\n  /** Pending child changes: correlationKey → Map<childKey, Changes> */\n  pendingChildChanges: Map<unknown, Map<unknown, Changes<any>>>\n  /** Reverse index: correlation key → Set of parent collection keys */\n  correlationToParentKeys: Map<unknown, Set<unknown>>\n  /** Shared nested pipeline setups (one per nested includes level) */\n  nestedSetups?: Array<NestedIncludesSetup>\n  /** nestedCorrelationKey → parentCorrelationKey */\n  nestedRoutingIndex?: Map<unknown, unknown>\n  /** parentCorrelationKey → Set<nestedCorrelationKeys> */\n  nestedRoutingReverseIndex?: Map<unknown, Set<unknown>>\n}\n\ntype ChildCollectionEntry = {\n  collection: Collection<any, any, any>\n  syncMethods: SyncMethods<any> | null\n  resultKeys: WeakMap<object, unknown>\n  orderByIndices: WeakMap<object, string> | null\n  /** Per-entry nested includes states (one per nested includes level) */\n  includesStates?: Array<IncludesOutputState>\n}\n\nfunction materializesInline(state: IncludesOutputState): boolean {\n  return state.materialization !== `collection`\n}\n\nfunction materializeIncludedValue(\n  state: IncludesOutputState,\n  entry: ChildCollectionEntry | undefined,\n): unknown {\n  if (!entry) {\n    if (state.materialization === `array`) {\n      return []\n    }\n    if (state.materialization === `concat`) {\n      return ``\n    }\n    return undefined\n  }\n\n  if (state.materialization === `collection`) {\n    return entry.collection\n  }\n\n  const rows = [...entry.collection.toArray]\n  const values = state.scalarField\n    ? rows.map((row) => row?.[state.scalarField!])\n    : rows\n\n  if (state.materialization === `array`) {\n    return values\n  }\n\n  return values.map((value) => String(value ?? ``)).join(``)\n}\n\n/**\n * Sets up shared buffers for nested includes pipelines.\n * Instead of writing directly into a single shared IncludesOutputState,\n * each nested pipeline writes into a buffer that is later drained per-entry.\n */\nfunction setupNestedPipelines(\n  includes: Array<IncludesCompilationResult>,\n  syncState: SyncState,\n): Array<NestedIncludesSetup> {\n  return includes.map((entry) => {\n    const buffer: Map<unknown, Map<unknown, Changes<any>>> = new Map()\n\n    // Attach output callback that writes into the shared buffer\n    entry.pipeline.pipe(\n      output((data) => {\n        const messages = data.getInner()\n        syncState.messagesCount += messages.length\n\n        for (const [[childKey, tupleData], multiplicity] of messages) {\n          const [childResult, _orderByIndex, correlationKey, parentContext] =\n            tupleData as unknown as [\n              any,\n              string | undefined,\n              unknown,\n              Record<string, any> | null,\n            ]\n\n          const routingKey = computeRoutingKey(correlationKey, parentContext)\n\n          let byChild = buffer.get(routingKey)\n          if (!byChild) {\n            byChild = new Map()\n            buffer.set(routingKey, byChild)\n          }\n\n          const existing = byChild.get(childKey) || {\n            deletes: 0,\n            inserts: 0,\n            value: childResult,\n            orderByIndex: _orderByIndex,\n          }\n\n          if (multiplicity < 0) {\n            existing.deletes += Math.abs(multiplicity)\n          } else if (multiplicity > 0) {\n            existing.inserts += multiplicity\n            existing.value = childResult\n          }\n\n          byChild.set(childKey, existing)\n        }\n      }),\n    )\n\n    const setup: NestedIncludesSetup = {\n      compilationResult: entry,\n      buffer,\n    }\n\n    // Recursively set up deeper levels\n    if (entry.childCompilationResult.includes) {\n      setup.nestedSetups = setupNestedPipelines(\n        entry.childCompilationResult.includes,\n        syncState,\n      )\n    }\n\n    return setup\n  })\n}\n\n/**\n * Creates fresh per-entry IncludesOutputState array from NestedIncludesSetup array.\n * Each entry gets its own isolated state for nested includes.\n */\nfunction createPerEntryIncludesStates(\n  setups: Array<NestedIncludesSetup>,\n): Array<IncludesOutputState> {\n  return setups.map((setup) => {\n    const state: IncludesOutputState = {\n      fieldName: setup.compilationResult.fieldName,\n      childCorrelationField: setup.compilationResult.childCorrelationField,\n      hasOrderBy: setup.compilationResult.hasOrderBy,\n      materialization: setup.compilationResult.materialization,\n      scalarField: setup.compilationResult.scalarField,\n      childRegistry: new Map(),\n      pendingChildChanges: new Map(),\n      correlationToParentKeys: new Map(),\n    }\n\n    if (setup.nestedSetups) {\n      state.nestedSetups = setup.nestedSetups\n      state.nestedRoutingIndex = new Map()\n      state.nestedRoutingReverseIndex = new Map()\n    }\n\n    return state\n  })\n}\n\n/**\n * Drains shared buffers into per-entry states using the routing index.\n * Returns the set of parent correlation keys that had changes routed to them.\n */\nfunction drainNestedBuffers(state: IncludesOutputState): Set<unknown> {\n  const dirtyCorrelationKeys = new Set<unknown>()\n\n  if (!state.nestedSetups) return dirtyCorrelationKeys\n\n  for (let i = 0; i < state.nestedSetups.length; i++) {\n    const setup = state.nestedSetups[i]!\n    const toDelete: Array<unknown> = []\n\n    for (const [nestedCorrelationKey, childChanges] of setup.buffer) {\n      const parentCorrelationKey =\n        state.nestedRoutingIndex!.get(nestedCorrelationKey)\n      if (parentCorrelationKey === undefined) {\n        // Unroutable — parent not yet seen; keep in buffer\n        continue\n      }\n\n      const entry = state.childRegistry.get(parentCorrelationKey)\n      if (!entry || !entry.includesStates) {\n        continue\n      }\n\n      // Route changes into this entry's per-entry state at position i\n      const entryState = entry.includesStates[i]!\n      for (const [childKey, changes] of childChanges) {\n        let byChild = entryState.pendingChildChanges.get(nestedCorrelationKey)\n        if (!byChild) {\n          byChild = new Map()\n          entryState.pendingChildChanges.set(nestedCorrelationKey, byChild)\n        }\n        const existing = byChild.get(childKey)\n        if (existing) {\n          existing.inserts += changes.inserts\n          existing.deletes += changes.deletes\n          if (changes.inserts > 0) {\n            existing.value = changes.value\n            if (changes.orderByIndex !== undefined) {\n              existing.orderByIndex = changes.orderByIndex\n            }\n          }\n        } else {\n          byChild.set(childKey, { ...changes })\n        }\n      }\n\n      dirtyCorrelationKeys.add(parentCorrelationKey)\n      toDelete.push(nestedCorrelationKey)\n    }\n\n    for (const key of toDelete) {\n      setup.buffer.delete(key)\n    }\n  }\n\n  return dirtyCorrelationKeys\n}\n\n/**\n * Updates the routing index after processing child changes.\n * Maps nested correlation keys to parent correlation keys so that\n * grandchild changes can be routed to the correct per-entry state.\n */\nfunction updateRoutingIndex(\n  state: IncludesOutputState,\n  correlationKey: unknown,\n  childChanges: Map<unknown, Changes<any>>,\n): void {\n  if (!state.nestedSetups) return\n\n  for (const setup of state.nestedSetups) {\n    for (const [, change] of childChanges) {\n      if (change.inserts > 0) {\n        // Read the nested routing key from the INCLUDES_ROUTING stamp.\n        // Must use the composite routing key (not raw correlationKey) to match\n        // how nested buffers are keyed by computeRoutingKey.\n        const nestedRouting =\n          change.value[INCLUDES_ROUTING]?.[setup.compilationResult.fieldName]\n        const nestedCorrelationKey = nestedRouting?.correlationKey\n        const nestedParentContext = nestedRouting?.p