@tanstack/db/src/collection.ts

A reactive client store for building super fast apps on sync

import { withArrayChangeTracking, withChangeTracking } from "./proxy"
import { deepEquals } from "./utils"
import { SortedMap } from "./SortedMap"
import {
  createSingleRowRefProxy,
  toExpression,
} from "./query/builder/ref-proxy"
import { BTreeIndex } from "./indexes/btree-index.js"
import { IndexProxy, LazyIndexWrapper } from "./indexes/lazy-index.js"
import { ensureIndexForExpression } from "./indexes/auto-index.js"
import { createTransaction, getActiveTransaction } from "./transactions"
import {
  CollectionInErrorStateError,
  CollectionIsInErrorStateError,
  CollectionRequiresConfigError,
  CollectionRequiresSyncConfigError,
  DeleteKeyNotFoundError,
  DuplicateKeyError,
  DuplicateKeySyncError,
  InvalidCollectionStatusTransitionError,
  InvalidSchemaError,
  KeyUpdateNotAllowedError,
  MissingDeleteHandlerError,
  MissingInsertHandlerError,
  MissingUpdateArgumentError,
  MissingUpdateHandlerError,
  NegativeActiveSubscribersError,
  NoKeysPassedToDeleteError,
  NoKeysPassedToUpdateError,
  NoPendingSyncTransactionCommitError,
  NoPendingSyncTransactionWriteError,
  SchemaMustBeSynchronousError,
  SchemaValidationError,
  SyncCleanupError,
  SyncTransactionAlreadyCommittedError,
  SyncTransactionAlreadyCommittedWriteError,
  UndefinedKeyError,
  UpdateKeyNotFoundError,
} from "./errors"
import { createFilteredCallback, currentStateAsChanges } from "./change-events"
import { CollectionEvents } from "./collection-events.js"
import type {
  AllCollectionEvents,
  CollectionEventHandler,
} from "./collection-events.js"
import type { Transaction } from "./transactions"
import type { StandardSchemaV1 } from "@standard-schema/spec"
import type { SingleRowRefProxy } from "./query/builder/ref-proxy"
import type {
  ChangeListener,
  ChangeMessage,
  CollectionConfig,
  CollectionStatus,
  CurrentStateAsChangesOptions,
  Fn,
  InferSchemaInput,
  InferSchemaOutput,
  InsertConfig,
  OperationConfig,
  OptimisticChangeMessage,
  PendingMutation,
  StandardSchema,
  SubscribeChangesOptions,
  Transaction as TransactionType,
  TransactionWithMutations,
  UtilsRecord,
  WritableDeep,
} from "./types"
import type { IndexOptions } from "./indexes/index-options.js"
import type { BaseIndex, IndexResolver } from "./indexes/base-index.js"

interface PendingSyncedTransaction<T extends object = Record<string, unknown>> {
  committed: boolean
  operations: Array<OptimisticChangeMessage<T>>
  truncate?: boolean
  deletedKeys: Set<string | number>
}

/**
 * Enhanced Collection interface that includes both data type T and utilities TUtils
 * @template T - The type of items in the collection
 * @template TKey - The type of the key for the collection
 * @template TUtils - The utilities record type
 * @template TInsertInput - The type for insert operations (can be different from T for schemas with defaults)
 */
export interface Collection<
  T extends object = Record<string, unknown>,
  TKey extends string | number = string | number,
  TUtils extends UtilsRecord = {},
  TSchema extends StandardSchemaV1 = StandardSchemaV1,
  TInsertInput extends object = T,
> extends CollectionImpl<T, TKey, TUtils, TSchema, TInsertInput> {
  readonly utils: TUtils
}

/**
 * Creates a new Collection instance with the given configuration
 *
 * @template T - The schema type if a schema is provided, otherwise the type of items in the collection
 * @template TKey - The type of the key for the collection
 * @template TUtils - The utilities record type
 * @param options - Collection options with optional utilities
 * @returns A new Collection with utilities exposed both at top level and under .utils
 *
 * @example
 * // Pattern 1: With operation handlers (direct collection calls)
 * const todos = createCollection({
 *   id: "todos",
 *   getKey: (todo) => todo.id,
 *   schema,
 *   onInsert: async ({ transaction, collection }) => {
 *     // Send to API
 *     await api.createTodo(transaction.mutations[0].modified)
 *   },
 *   onUpdate: async ({ transaction, collection }) => {
 *     await api.updateTodo(transaction.mutations[0].modified)
 *   },
 *   onDelete: async ({ transaction, collection }) => {
 *     await api.deleteTodo(transaction.mutations[0].key)
 *   },
 *   sync: { sync: () => {} }
 * })
 *
 * // Direct usage (handlers manage transactions)
 * const tx = todos.insert({ id: "1", text: "Buy milk", completed: false })
 * await tx.isPersisted.promise
 *
 * @example
 * // Pattern 2: Manual transaction management
 * const todos = createCollection({
 *   getKey: (todo) => todo.id,
 *   schema: todoSchema,
 *   sync: { sync: () => {} }
 * })
 *
 * // Explicit transaction usage
 * const tx = createTransaction({
 *   mutationFn: async ({ transaction }) => {
 *     // Handle all mutations in transaction
 *     await api.saveChanges(transaction.mutations)
 *   }
 * })
 *
 * tx.mutate(() => {
 *   todos.insert({ id: "1", text: "Buy milk" })
 *   todos.update("2", draft => { draft.completed = true })
 * })
 *
 * await tx.isPersisted.promise
 *
 * @example
 * // Using schema for type inference (preferred as it also gives you client side validation)
 * const todoSchema = z.object({
 *   id: z.string(),
 *   title: z.string(),
 *   completed: z.boolean()
 * })
 *
 * const todos = createCollection({
 *   schema: todoSchema,
 *   getKey: (todo) => todo.id,
 *   sync: { sync: () => {} }
 * })
 *
 */

// Overload for when schema is provided
export function createCollection<
  T extends StandardSchemaV1,
  TKey extends string | number = string | number,
  TUtils extends UtilsRecord = {},
>(
  options: CollectionConfig<InferSchemaOutput<T>, TKey, T> & {
    schema: T
    utils?: TUtils
  }
): Collection<InferSchemaOutput<T>, TKey, TUtils, T, InferSchemaInput<T>>

// Overload for when no schema is provided
// the type T needs to be passed explicitly unless it can be inferred from the getKey function in the config
export function createCollection<
  T extends object,
  TKey extends string | number = string | number,
  TUtils extends UtilsRecord = {},
>(
  options: CollectionConfig<T, TKey, never> & {
    schema?: never // prohibit schema if an explicit type is provided
    utils?: TUtils
  }
): Collection<T, TKey, TUtils, never, T>

// Implementation
export function createCollection(
  options: CollectionConfig<any, string | number, any> & {
    schema?: StandardSchemaV1
    utils?: UtilsRecord
  }
): Collection<any, string | number, UtilsRecord, any, any> {
  const collection = new CollectionImpl<any, string | number, any, any, any>(
    options
  )

  // Copy utils to both top level and .utils namespace
  if (options.utils) {
    collection.utils = { ...options.utils }
  } else {
    collection.utils = {}
  }

  return collection
}

export class CollectionImpl<
  TOutput extends object = Record<string, unknown>,
  TKey extends string | number = string | number,
  TUtils extends UtilsRecord = {},
  TSchema extends StandardSchemaV1 = StandardSchemaV1,
  TInput extends object = TOutput,
> {
  public config: CollectionConfig<TOutput, TKey, TSchema>

  // Core state - make public for testing
  public transactions: SortedMap<string, Transaction<any>>
  public pendingSyncedTransactions: Array<PendingSyncedTransaction<TOutput>> =
    []
  public syncedData: Map<TKey, TOutput> | SortedMap<TKey, TOutput>
  public syncedMetadata = new Map<TKey, unknown>()

  // Optimistic state tracking - make public for testing
  public optimisticUpserts = new Map<TKey, TOutput>()
  public optimisticDeletes = new Set<TKey>()

  // Cached size for performance
  private _size = 0

  // Index storage
  private lazyIndexes = new Map<number, LazyIndexWrapper<TKey>>()
  private resolvedIndexes = new Map<number, BaseIndex<TKey>>()
  private isIndexesResolved = false
  private indexCounter = 0

  // Event system
  private changeListeners = new Set<ChangeListener<TOutput, TKey>>()
  private changeKeyListeners = new Map<
    TKey,
    Set<ChangeListener<TOutput, TKey>>
  >()

  // Utilities namespace
  // This is populated by createCollection
  public utils: Record<string, Fn> = {}

  // State used for computing the change events
  private syncedKeys = new Set<TKey>()
  private preSyncVisibleState = new Map<TKey, TOutput>()
  private recentlySyncedKeys = new Set<TKey>()
  private hasReceivedFirstCommit = false
  private isCommittingSyncTransactions = false

  // Array to store one-time ready listeners
  private onFirstReadyCallbacks: Array<() => void> = []
  private hasBeenReady = false

  // Event batching for preventing duplicate emissions during transaction flows
  private batchedEvents: Array<ChangeMessage<TOutput, TKey>> = []
  private shouldBatchEvents = false

  // Lifecycle management
  private _status: CollectionStatus = `idle`
  private activeSubscribersCount = 0
  private gcTimeoutId: ReturnType<typeof setTimeout> | null = null
  private preloadPromise: Promise<void> | null = null
  private syncCleanupFn: (() => void) | null = null

  // Event system
  private events: CollectionEvents

  /**
   * Register a callback to be executed when the collection first becomes ready
   * Useful for preloading collections
   * @param callback Function to call when the collection first becomes ready
   * @example
   * collection.onFirstReady(() => {
   *   console.log('Collection is ready for the first time')
   *   // Safe to access collection.state now
   * })
   */
  public onFirstReady(callback: () => void): void {
    // If already ready, call immediately
    if (this.hasBeenReady) {
      callback()
      return
    }

    this.onFirstReadyCallbacks.push(callback)
  }

  /**
   * Check if the collection is ready for use
   * Returns true if the collection has been marked as ready by its sync implementation
   * @returns true if the collection is ready, false otherwise
   * @example
   * if (collection.isReady()) {
   *   console.log('Collection is ready, data is available')
   *   // Safe to access collection.state
   * } else {
   *   console.log('Collection is still loading')
   * }
   */
  public isReady(): boolean {
    return this._status === `ready`
  }

  /**
   * Mark the collection as ready for use
   * This is called by sync implementations to explicitly signal that the collection is ready,
   * providing a more intuitive alternative to using commits for readiness signaling
   * @private - Should only be called by sync implementations
   */
  private markReady(): void {
    // Can transition to ready from loading or initialCommit states
    if (this._status === `loading` || this._status === `initialCommit`) {
      this.setStatus(`ready`)

      // Call any registered first ready callbacks (only on first time becoming ready)
      if (!this.hasBeenReady) {
        this.hasBeenReady = true

        // Also mark as having received first commit for backwards compatibility
        if (!this.hasReceivedFirstCommit) {
          this.hasReceivedFirstCommit = true
        }

        const callbacks = [...this.onFirstReadyCallbacks]
        this.onFirstReadyCallbacks = []
        callbacks.forEach((callback) => callback())
      }
    }

    // Always notify dependents when markReady is called, after status is set
    // This ensures live queries get notified when their dependencies become ready
    if (this.changeListeners.size > 0) {
      this.emitEmptyReadyEvent()
    }
  }

  public id = ``

  /**
   * Gets the current status of the collection
   */
  public get status(): CollectionStatus {
    return this._status
  }

  /**
   * Get the number of subscribers to the collection
   */
  public get subscriberCount(): number {
    return this.activeSubscribersCount
  }

  /**
   * Validates that the collection is in a usable state for data operations
   * @private
   */
  private validateCollectionUsable(operation: string): void {
    switch (this._status) {
      case `error`:
        throw new CollectionInErrorStateError(operation, this.id)
      case `cleaned-up`:
        // Automatically restart the collection when operations are called on cleaned-up collections
        this.startSync()
        break
    }
  }

  /**
   * Validates state transitions to prevent invalid status changes
   * @private
   */
  private validateStatusTransition(
    from: CollectionStatus,
    to: CollectionStatus
  ): void {
    if (from === to) {
      // Allow same state transitions
      return
    }
    const validTransitions: Record<
      CollectionStatus,
      Array<CollectionStatus>
    > = {
      idle: [`loading`, `error`, `cleaned-up`],
      loading: [`initialCommit`, `ready`, `error`, `cleaned-up`],
      initialCommit: [`ready`, `error`, `cleaned-up`],
      ready: [`cleaned-up`, `error`],
      error: [`cleaned-up`, `idle`],
      "cleaned-up": [`loading`, `error`],
    }

    if (!validTransitions[from].includes(to)) {
      throw new InvalidCollectionStatusTransitionError(from, to, this.id)
    }
  }

  /**
   * Safely update the collection status with validation
   * @private
   */
  private setStatus(newStatus: CollectionStatus): void {
    this.validateStatusTransition(this._status, newStatus)
    const previousStatus = this._status
    this._status = newStatus

    // Resolve indexes when collection becomes ready
    if (newStatus === `ready` && !this.isIndexesResolved) {
      // Resolve indexes asynchronously without blocking
      this.resolveAllIndexes().catch((error) => {
        console.warn(`Failed to resolve indexes:`, error)
      })
    }

    // Emit event
    this.events.emitStatusChange(newStatus, previousStatus)
  }

  /**
   * Creates a new Collection instance
   *
   * @param config - Configuration object for the collection
   * @throws Error if sync config is missing
   */
  constructor(config: CollectionConfig<TOutput, TKey, TSchema>) {
    // eslint-disable-next-line
    if (!config) {
      throw new CollectionRequiresConfigError()
    }
    if (config.id) {
      this.id = config.id
    } else {
      this.id = crypto.randomUUID()
    }

    // eslint-disable-next-line
    if (!config.sync) {
      throw new CollectionRequiresSyncConfigError()
    }

    this.transactions = new SortedMap<string, Transaction<any>>((a, b) =>
      a.compareCreatedAt(b)
    )

    // Set default values for optional config properties
    this.config = {
      ...config,
      autoIndex: config.autoIndex ?? `eager`,
    }

    // Set up data storage with optional comparison function
    if (this.config.compare) {
      this.syncedData = new SortedMap<TKey, TOutput>(this.config.compare)
    } else {
      this.syncedData = new Map<TKey, TOutput>()
    }

    // Set up event system
    this.events = new CollectionEvents(this)

    // Only start sync immediately if explicitly enabled
    if (config.startSync === true) {
      this.startSync()
    }
  }

  /**
   * Start sync immediately - internal method for compiled queries
   * This bypasses lazy loading for special cases like live query results
   */
  public startSyncImmediate(): void {
    this.startSync()
  }

  /**
   * Start the sync process for this collection
   * This is called when the collection is first accessed or preloaded
   */
  private startSync(): void {
    if (this._status !== `idle` && this._status !== `cleaned-up`) {
      return // Already started or in progress
    }

    this.setStatus(`loading`)

    try {
      const cleanupFn = this.config.sync.sync({
        collection: this,
        begin: () => {
          this.pendingSyncedTransactions.push({
            committed: false,
            operations: [],
            deletedKeys: new Set(),
          })
        },
        write: (messageWithoutKey: Omit<ChangeMessage<TOutput>, `key`>) => {
          const pendingTransaction =
            this.pendingSyncedTransactions[
              this.pendingSyncedTransactions.length - 1
            ]
          if (!pendingTransaction) {
            throw new NoPendingSyncTransactionWriteError()
          }
          if (pendingTransaction.committed) {
            throw new SyncTransactionAlreadyCommittedWriteError()
          }
          const key = this.getKeyFromItem(messageWithoutKey.value)

          // Check if an item with this key already exists when inserting
          if (messageWithoutKey.type === `insert`) {
            const insertingIntoExistingSynced = this.syncedData.has(key)
            const hasPendingDeleteForKey =
              pendingTransaction.deletedKeys.has(key)
            const isTruncateTransaction = pendingTransaction.truncate === true
            // Allow insert after truncate in the same transaction even if it existed in syncedData
            if (
              insertingIntoExistingSynced &&
              !hasPendingDeleteForKey &&
              !isTruncateTransaction
            ) {
              throw new DuplicateKeySyncError(key, this.id)
            }
          }

          const message: ChangeMessage<TOutput> = {
            ...messageWithoutKey,
            key,
          }
          pendingTransaction.operations.push(message)

          if (messageWithoutKey.type === `delete`) {
            pendingTransaction.deletedKeys.add(key)
          }
        },
        commit: () => {
          const pendingTransaction =
            this.pendingSyncedTransactions[
              this.pendingSyncedTransactions.length - 1
            ]
          if (!pendingTransaction) {
            throw new NoPendingSyncTransactionCommitError()
          }
          if (pendingTransaction.committed) {
            throw new SyncTransactionAlreadyCommittedError()
          }

          pendingTransaction.committed = true

          // Update status to initialCommit when transitioning from loading
          // This indicates we're in the process of committing the first transaction
          if (this._status === `loading`) {
            this.setStatus(`initialCommit`)
          }

          this.commitPendingTransactions()
        },
        markReady: () => {
          this.markReady()
        },
        truncate: () => {
          const pendingTransaction =
            this.pendingSyncedTransactions[
              this.pendingSyncedTransactions.length - 1
            ]
          if (!pendingTransaction) {
            throw new NoPendingSyncTransactionWriteError()
          }
          if (pendingTransaction.committed) {
            throw new SyncTransactionAlreadyCommittedWriteError()
          }

          // Clear all operations from the current transaction
          pendingTransaction.operations = []
          pendingTransaction.deletedKeys.clear()

          // Mark the transaction as a truncate operation. During commit, this triggers:
          // - Delete events for all previously synced keys (excluding optimistic-deleted keys)
          // - Clearing of syncedData/syncedMetadata
          // - Subsequent synced ops applied on the fresh base
          // - Finally, optimistic mutations re-applied on top (single batch)
          pendingTransaction.truncate = true
        },
      })

      // Store cleanup function if provided
      this.syncCleanupFn = typeof cleanupFn === `function` ? cleanupFn : null
    } catch (error) {
      this.setStatus(`error`)
      throw error
    }
  }

  /**
   * Preload the collection data by starting sync if not already started
   * Multiple concurrent calls will share the same promise
   */
  public preload(): Promise<void> {
    if (this.preloadPromise) {
      return this.preloadPromise
    }

    this.preloadPromise = new Promise<void>((resolve, reject) => {
      if (this._status === `ready`) {
        resolve()
        return
      }

      if (this._status === `error`) {
        reject(new CollectionIsInErrorStateError())
        return
      }

      // Register callback BEFORE starting sync to avoid race condition
      this.onFirstReady(() => {
        resolve()
      })

      // Start sync if collection hasn't started yet or was cleaned up
      if (this._status === `idle` || this._status === `cleaned-up`) {
        try {
          this.startSync()
        } catch (error) {
          reject(error)
          return
        }
      }
    })

    return this.preloadPromise
  }

  /**
   * Clean up the collection by stopping sync and clearing data
   * This can be called manually or automatically by garbage collection
   */
  public async cleanup(): Promise<void> {
    // Clear GC timeout
    if (this.gcTimeoutId) {
      clearTimeout(this.gcTimeoutId)
      this.gcTimeoutId = null
    }

    // Stop sync - wrap in try/catch since it's user-provided code
    try {
      if (this.syncCleanupFn) {
        this.syncCleanupFn()
        this.syncCleanupFn = null
      }
    } catch (error) {
      // Re-throw in a microtask to surface the error after cleanup completes
      queueMicrotask(() => {
        if (error instanceof Error) {
          // Preserve the original error and stack trace
          const wrappedError = new SyncCleanupError(this.id, error)
          wrappedError.cause = error
          wrappedError.stack = error.stack
          throw wrappedError
        } else {
          throw new SyncCleanupError(this.id, error as Error | string)
        }
      })
    }

    // Clear data
    this.syncedData.clear()
    this.syncedMetadata.clear()
    this.optimisticUpserts.clear()
    this.optimisticDeletes.clear()
    this._size = 0
    this.pendingSyncedTransactions = []
    this.syncedKeys.clear()
    this.hasReceivedFirstCommit = false
    this.hasBeenReady = false
    this.onFirstReadyCallbacks = []
    this.preloadPromise = null
    this.batchedEvents = []
    this.shouldBatchEvents = false

    this.events.cleanup()

    // Update status
    this.setStatus(`cleaned-up`)

    return Promise.resolve()
  }

  /**
   * Start the garbage collection timer
   * Called when the collection becomes inactive (no subscribers)
   */
  private startGCTimer(): void {
    if (this.gcTimeoutId) {
      clearTimeout(this.gcTimeoutId)
    }

    const gcTime = this.config.gcTime ?? 300000 // 5 minutes default

    // If gcTime is 0, GC is disabled
    if (gcTime === 0) {
      return
    }

    this.gcTimeoutId = setTimeout(() => {
      if (this.activeSubscribersCount === 0) {
        this.cleanup()
      }
    }, gcTime)
  }

  /**
   * Cancel the garbage collection timer
   * Called when the collection becomes active again
   */
  private cancelGCTimer(): void {
    if (this.gcTimeoutId) {
      clearTimeout(this.gcTimeoutId)
      this.gcTimeoutId = null
    }
  }

  /**
   * Increment the active subscribers count and start sync if needed
   */
  private addSubscriber(): void {
    const previousSubscriberCount = this.activeSubscribersCount
    this.activeSubscribersCount++
    this.cancelGCTimer()

    // Start sync if collection was cleaned up
    if (this._status === `cleaned-up` || this._status === `idle`) {
      this.startSync()
    }

    this.events.emitSubscribersChange(
      this.activeSubscribersCount,
      previousSubscriberCount
    )
  }

  /**
   * Decrement the active subscribers count and start GC timer if needed
   */
  private removeSubscriber(): void {
    const previousSubscriberCount = this.activeSubscribersCount
    this.activeSubscribersCount--

    if (this.activeSubscribersCount === 0) {
      this.startGCTimer()
    } else if (this.activeSubscribersCount < 0) {
      throw new NegativeActiveSubscribersError()
    }

    this.events.emitSubscribersChange(
      this.activeSubscribersCount,
      previousSubscriberCount
    )
  }

  /**
   * Recompute optimistic state from active transactions
   */
  private recomputeOptimisticState(
    triggeredByUserAction: boolean = false
  ): void {
    // Skip redundant recalculations when we're in the middle of committing sync transactions
    if (this.isCommittingSyncTransactions) {
      return
    }

    const previousState = new Map(this.optimisticUpserts)
    const previousDeletes = new Set(this.optimisticDeletes)

    // Clear current optimistic state
    this.optimisticUpserts.clear()
    this.optimisticDeletes.clear()

    const activeTransactions: Array<Transaction<any>> = []

    for (const transaction of this.transactions.values()) {
      if (![`completed`, `failed`].includes(transaction.state)) {
        activeTransactions.push(transaction)
      }
    }

    // Apply active transactions only (completed transactions are handled by sync operations)
    for (const transaction of activeTransactions) {
      for (const mutation of transaction.mutations) {
        if (mutation.collection === this && mutation.optimistic) {
          switch (mutation.type) {
            case `insert`:
            case `update`:
              this.optimisticUpserts.set(
                mutation.key,
                mutation.modified as TOutput
              )
              this.optimisticDeletes.delete(mutation.key)
              break
            case `delete`:
              this.optimisticUpserts.delete(mutation.key)
              this.optimisticDeletes.add(mutation.key)
              break
          }
        }
      }
    }

    // Update cached size
    this._size = this.calculateSize()

    // Collect events for changes
    const events: Array<ChangeMessage<TOutput, TKey>> = []
    this.collectOptimisticChanges(previousState, previousDeletes, events)

    // Filter out events for recently synced keys to prevent duplicates
    // BUT: Only filter out events that are actually from sync operations
    // New user transactions should NOT be filtered even if the key was recently synced
    const filteredEventsBySyncStatus = events.filter((event) => {
      if (!this.recentlySyncedKeys.has(event.key)) {
        return true // Key not recently synced, allow event through
      }

      // Key was recently synced - allow if this is a user-triggered action
      if (triggeredByUserAction) {
        return true
      }

      // Otherwise filter out duplicate sync events
      return false
    })

    // Filter out redundant delete events if there are pending sync transactions
    // that will immediately restore the same data, but only for completed transactions
    // IMPORTANT: Skip complex filtering for user-triggered actions to prevent UI blocking
    if (this.pendingSyncedTransactions.length > 0 && !triggeredByUserAction) {
      const pendingSyncKeys = new Set<TKey>()

      // Collect keys from pending sync operations
      for (const transaction of this.pendingSyncedTransactions) {
        for (const operation of transaction.operations) {
          pendingSyncKeys.add(operation.key as TKey)
        }
      }

      // Only filter out delete events for keys that:
      // 1. Have pending sync operations AND
      // 2. Are from completed transactions (being cleaned up)
      const filteredEvents = filteredEventsBySyncStatus.filter((event) => {
        if (event.type === `delete` && pendingSyncKeys.has(event.key)) {
          // Check if this delete is from clearing optimistic state of completed transactions
          // We can infer this by checking if we have no remaining optimistic mutations for this key
          const hasActiveOptimisticMutation = activeTransactions.some((tx) =>
            tx.mutations.some(
              (m) => m.collection === this && m.key === event.key
            )
          )

          if (!hasActiveOptimisticMutation) {
            return false // Skip this delete event as sync will restore the data
          }
        }
        return true
      })

      // Update indexes for the filtered events
      if (filteredEvents.length > 0) {
        this.updateIndexes(filteredEvents)
      }
      this.emitEvents(filteredEvents, triggeredByUserAction)
    } else {
      // Update indexes for all events
      if (filteredEventsBySyncStatus.length > 0) {
        this.updateIndexes(filteredEventsBySyncStatus)
      }
      // Emit all events if no pending sync transactions
      this.emitEvents(filteredEventsBySyncStatus, triggeredByUserAction)
    }
  }

  /**
   * Calculate the current size based on synced data and optimistic changes
   */
  private calculateSize(): number {
    const syncedSize = this.syncedData.size
    const deletesFromSynced = Array.from(this.optimisticDeletes).filter(
      (key) => this.syncedData.has(key) && !this.optimisticUpserts.has(key)
    ).length
    const upsertsNotInSynced = Array.from(this.optimisticUpserts.keys()).filter(
      (key) => !this.syncedData.has(key)
    ).length

    return syncedSize - deletesFromSynced + upsertsNotInSynced
  }

  /**
   * Collect events for optimistic changes
   */
  private collectOptimisticChanges(
    previousUpserts: Map<TKey, TOutput>,
    previousDeletes: Set<TKey>,
    events: Array<ChangeMessage<TOutput, TKey>>
  ): void {
    const allKeys = new Set([
      ...previousUpserts.keys(),
      ...this.optimisticUpserts.keys(),
      ...previousDeletes,
      ...this.optimisticDeletes,
    ])

    for (const key of allKeys) {
      const currentValue = this.get(key)
      const previousValue = this.getPreviousValue(
        key,
        previousUpserts,
        previousDeletes
      )

      if (previousValue !== undefined && currentValue === undefined) {
        events.push({ type: `delete`, key, value: previousValue })
      } else if (previousValue === undefined && currentValue !== undefined) {
        events.push({ type: `insert`, key, value: currentValue })
      } else if (
        previousValue !== undefined &&
        currentValue !== undefined &&
        previousValue !== currentValue
      ) {
        events.push({
          type: `update`,
          key,
          value: currentValue,
          previousValue,
        })
      }
    }
  }

  /**
   * Get the previous value for a key given previous optimistic state
   */
  private getPreviousValue(
    key: TKey,
    previousUpserts: Map<TKey, TOutput>,
    previousDeletes: Set<TKey>
  ): TOutput | undefined {
    if (previousDeletes.has(key)) {
      return undefined
    }
    if (previousUpserts.has(key)) {
      return previousUpserts.get(key)
    }
    return this.syncedData.get(key)
  }

  /**
   * Emit an empty ready event to notify subscribers that the collection is ready
   * This bypasses the normal empty array check in emitEvents
   */
  private emitEmptyReadyEvent(): void {
    // Emit empty array directly to all listeners
    for (const listener of this.changeListeners) {
      listener([])
    }
    // Emit to key-specific listeners
    for (const [_key, keyListeners] of this.changeKeyListeners) {
      for (const listener of keyListeners) {
        listener([])
      }
    }
  }

  /**
   * Emit events either immediately or batch them for later emission
   */
  private emitEvents(
    changes: Array<ChangeMessage<TOutput, TKey>>,
    forceEmit = false
  ): void {
    // Skip batching for user actions (forceEmit=true) to keep UI responsive
    if (this.shouldBatchEvents && !forceEmit) {
      // Add events to the batch
      this.batchedEvents.push(...changes)
      return
    }

    // Either we're not batching, or we're forcing emission (user action or ending batch cycle)
    let eventsToEmit = changes

    // If we have batched events and this is a forced emit, combine them
    if (this.batchedEvents.length > 0 && forceEmit) {
      eventsToEmit = [...this.batchedEvents, ...changes]
      this.batchedEvents = []
      this.shouldBatchEvents = false
    }

    if (eventsToEmit.length === 0) return

    // Emit to all listeners
    for (const listener of this.changeListeners) {
      listener(eventsToEmit)
    }

    // Emit to key-specific listeners
    if (this.changeKeyListeners.size > 0) {
      // Group changes by key, but only for keys that have listeners
      const changesByKey = new Map<TKey, Array<ChangeMessage<TOutput, TKey>>>()
      for (const change of eventsToEmit) {
        if (this.changeKeyListeners.has(change.key)) {
          if (!changesByKey.has(change.key)) {
            changesByKey.set(change.key, [])
          }
          changesByKey.get(change.key)!.push(change)
        }
      }

      // Emit batched changes to each key's listeners
      for (const [key, keyChanges] of changesByKey) {
        const keyListeners = this.changeKeyListeners.get(key)!
        for (const listener of keyListeners) {
          listener(keyChanges)
        }
      }
    }
  }

  /**
   * Get the current value for a key (virtual derived state)
   */
  public get(key: TKey): TOutput | undefined {
    // Check if optimistically deleted
    if (this.optimisticDeletes.has(key)) {
      return undefined
    }

    // Check optimistic upserts first
    if (this.optimisticUpserts.has(key)) {
      return this.optimisticUpserts.get(key)
    }

    // Fall back to synced data
    return this.syncedData.get(key)
  }

  /**
   * Check if a key exists in the collection (virtual derived state)
   */
  public has(key: TKey): boolean {
    // Check if optimistically deleted
    if (this.optimisticDeletes.has(key)) {
      return false
    }

    // Check optimistic upserts first
    if (this.optimisticUpserts.has(key)) {
      return true
    }

    // Fall back to synced data
    return this.syncedData.has(key)
  }

  /**
   * Get the current size of the collection (cached)
   */
  public get size(): number {
    return this._size
  }

  /**
   * Get all keys (virtual derived state)
   */
  public *keys(): IterableIterator<TKey> {
    // Yield keys from synced data, skipping any that are deleted.
    for (const key of this.syncedData.keys()) {
      if (!this.optimisticDeletes.has(key)) {
        yield key
      }
    }
    // Yield keys from upserts that were not already in synced data.
    for (const key of this.optimisticUpserts.keys()) {
      if (!this.syncedData.has(key) && !this.optimisticDeletes.has(key)) {
        // The optimisticDeletes check is technically redundant if inserts/updates always remove from deletes,
        // but it's safer to keep it.
        yield key
      }
    }
  }

  /**
   * Get all values (virtual derived state)
   */
  public *values(): IterableIterator<TOutput> {
    for (const key of this.keys()) {
      const value = this.get(key)
      if (value !== undefined) {
        yield value
      }
    }
  }

  /**
   * Get all entries (virtual derived state)
   */
  public *entries(): IterableIterator<[TKey, TOutput]> {
    for (const key of this.keys()) {
      const value = this.get(key)
      if (value !== undefined) {
        yield [key, value]
      }
    }
  }

  /**
   * Get all entries (virtual derived state)
   */
  public *[Symbol.iterator](): IterableIterator<[TKey, TOutput]> {
    for (const [key, value] of this.entries()) {
      yield [key, value]
    }
  }

  /**
   * Execute a callback for each entry in the collection
   */
  public forEach(
    callbackfn: (value: TOutput, key: TKey, index: number) => void
  ): void {
    let index = 0
    for (const [key, value] of this.entries()) {
      callbackfn(value, key, index++)
    }
  }

  /**
   * Create a new array with the results of calling a function for each entry in the collection
   */
  public map<U>(
    callbackfn: (value: TOutput, key: TKey, index: number) => U
  ): Array<U> {
    const result: Array<U> = []
    let index = 0
    for (const [key, value] of this.entries()) {
      result.push(callbackfn(value, key, index++))
    }
    return result
  }

  /**
   * Attempts to commit pending synced transactions if there are no active transactions
   * This method processes operations from pending transactions and applies them to the synced data
   */
  commitPendingTransactions = () => {
    // Check if there are any persisting transaction
    let hasPersistingTransaction = false
    for (const transaction of this.transactions.values()) {
      if (transaction.state === `persisting`) {
        hasPersistingTransaction = true
        break
      }
    }

    // pending synced transactions could be either `committed` or still open.
    // we only want to process `committed` transactions here
    const {
      committedSyncedTransactions,
      uncommittedSyncedTransactions,
      hasTruncateSync,
    } = this.pendingSyncedTransactions.reduce(
      (acc, t) => {
        if (t.committed) {
          acc.committedSyncedTransactions.push(t)
          if (t.truncate === true) {
            acc.hasTruncateSync = true
          }
        } else {
          acc.uncommittedSyncedTransactions.push(t)
        }
        return acc
      },
      {
        committedSyncedTransactions: [] as Array<
          PendingSyncedTransaction<TOutput>
        >,
        uncommittedSyncedTransactions: [] as Array<
          PendingSyncedTransaction<TOutput>
        >,
        hasTruncateSync: false,
      }
    )

    if (!hasPersistingTransaction || hasTruncateSync) {
      // Set flag to prevent redundant optimistic state recalculations
      this.isCommittingSyncTransactions = true

      // First collect all keys that will be affected by sync operations
      const changedKeys = new Set<TKey>()
      for (const transaction of committedSyncedTransactions) {
        for (const operation of transaction.operations) {
          changedKeys.add(operation.key as TKey)
        }
      }

      // Use pre-captured state if available (from optimistic scenarios),
      // otherwise capture current state (for pure sync scenarios)
      let currentVisibleState = this.preSyncVisibleState
      if (currentVisibleState.size === 0) {
        // No pre-captured state, capture it now for pure sync operations
        currentVisibleState = new Map<TKey, TOutput>()
        for (const key of changedKeys) {
          const currentValue = this.get(key)
          if (currentValue !== undefined) {
            currentVisibleState.set(key, currentValue)
          }
        }
      }

      const events: Array<ChangeMessage<TOutput, TKey>> = []
      const rowUpdateMode = this.config.sync.rowUpdateMode || `partial`

      for (const transaction of committedSyncedTransactions) {
        // Handle truncate operations first
        if (transaction.truncate) {
          // TRUNCATE PHASE
          // 1) Emit a delete for every currently-synced key so downstream listeners/indexes
          //    observe a clear-before-rebuild. We intentionally skip keys already in
          //    optimisticDeletes because their delete was previously emitted by the user.
          for (const key of this.syncedData.keys()) {
            if (this.optimisticDeletes.has(key)) continue
            const previousValue =
              this.optimisticUpserts.get(key) || this.syncedData.get(key)
            if (previousValue !== undefined) {
              events.push({ type: `delete`, key, value: previousValue })
            }
          }

          // 2) Clear the authoritative synced base. Subsequent server ops in this
          //    same commit will rebuild the base atomically.
          this.syncedData.clear()
          this.syncedMetadata.clear()
          this.syncedKeys.clear()

          // 3) Clear currentVisibleState for truncated keys to ensure subsequent operations
          //    are compared against the post-truncate state (undefined) rather than pre-truncate state
          //    This ensures that re-inserted keys are emitted as INSERT events, not UPDATE events
          for (const key of changedKeys) {
            currentVisibleState.delete(key)
          }
        }

        for (const operation of transaction.operations) {
          const key = operation.key as TKey
          this.syncedKeys.add(key)

          // Update metadata
          switch (operation.type) {
            case `insert`:
              this.syncedMetadata.set(key, operation.metadata)
              break
            case `update`:
              this.syncedMetadata.set(
                key,
                Object.assign(
                  {},
                  this.syncedMetadata.get(key),
                  operation.metadata
                )
              )
              break
            case `delete`:
              this.syncedMetadata.delete(key)
              break
          }

          // Update synced data
          switch (operation.type) {
            case `insert`:
              this.syncedData.set(key, operation.value)
              break
            case `update`: {
              if (rowUpdateMode === `partial`) {
                const updatedValue = Object.assign(
                  {},
                  this.syncedData.get(key),
                  operation.value
                )
                this.syncedData.set(key, updatedValue)
              } else {
                this.syncedData.set(key, operation.value)
              }
              break
            }
            case `delete`:
              this.syncedData.delete(key)
              break
          }
        }
      }

      // After applying synced operations, if this commit included a truncate,
      // re-apply optimistic mutations on top of the fresh synced base. This ensures
      // the UI preserves local intent while respecting server rebuild semantics.
      // Ordering: deletes (above) -> server ops (just applied) -> optimistic upserts.
      if (hasTruncateSync) {
        // Avoid duplicating keys that were inserted/updated by synced operations in this commit
        const syncedInsertedOrUpdatedKeys = new Set<TKey>()
        for (const t of committedSyncedTransactions) {
          for (const op of t.operations) {
            if (op.type === `insert` || op.type === `update`) {
              syncedInsertedOrUpdatedKeys.add(op.key as TKey)
            }
          }
        }

        // Build re-apply sets from ACTIVE optimistic transactions against the new synced base
        // We do not copy maps; we compute intent directly from transactions to avoid drift.
        const reapplyUpserts = new Map<TKey, TOutput>()
        const reapplyDeletes = new Set<TKey>()

        for (const tx of this.transactions.values()) {
          if ([`completed`, `failed`].includes(tx.state)) continue
          for (const mutation of tx.mutations) {
            if (mutation.collection !== this || !mutation.optimistic) continue
            const key = mutation.key as TKey
            switch (mutation.type) {
              case `insert`:
                reapplyUpserts.set(key, mutation.modified as TOutput)
                reapplyDeletes.delete(key)
                break
              case `update`: {
                const base = this.syncedData.get(key)
                const next = base
                  ? (Object.assign({}, base, mutation.changes) as TOutput)
                  : (mutation.modified as TOutput)
                reapplyUpserts.set(key, next)
                reapplyDeletes.delete(key)
                break
              }
              case `delete`:
                reapplyUpserts.delete(key)
                reapplyDeletes.add(key)
                break
            }
          }
        }

        // Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete.
        // If the server also inserted/updated the same key in this batch, override that value
        // with the optimistic value to preserve local intent.
        for (const [key, value] of reapplyUpserts) {
          if (reapplyDeletes.has(key)) continue
          if (syncedInsertedOrUpdatedKeys.has(key)) {
            let foundInsert = false
            for (let i = events.length - 1; i >= 0; i--) {
              const evt = events[i]!
              if (evt.key === key && evt.type === `insert`) {
                evt.value = value
                foundInsert = true
                break
              }
            }
            if (!foundInsert) {
              events.push({ type: `insert`, key, value })
            }
          } else {
            events.push({ type: `insert`, key, value })
          }
        }

        // Finally, ensure we do NOT insert keys that have an outstanding optimistic delete.
        if (events.length > 0 && reapplyDeletes.size > 0) {
          const filtered: Array<ChangeMessage<TOutput, TKey>> = []
          for (const evt of events) {
            if (evt.type === `insert` && reapplyDeletes.has(evt.key)) {
              continue
            }
            filtered.push(evt)
          }
          events.length = 0
          events.push(...filtered)
        }

        // Ensure listeners are active before emitting this critical batch
        if (!this.isReady()) {
          this.setStatus(`ready`)
        }
      }

      // Maintain optimistic state appropriately
      // Clear optimistic state since sync operations will now provide the authoritative data.
      // Any still-active user transactions will be re-applied below in recompute.
      this.optimisticUpserts.clear()
      this.optimisticDeletes.clear()

      // Reset flag and recompute optimistic state for any remaining active transactions
      this.isCommittingSyncTransactions = false
      for (const transaction of this.transactions.values()) {
        if (![`completed`, `failed`].includes(transaction.state)) {
          for (const mutation of transaction.mutations) {
            if (mutation.collection === this && mutation.optimistic) {
              switch (mutation.type) {
                case `insert`:
                case `update`:
                  this.optimisticUpserts.set(
                    mutation.key,
                    mutation.modified as TOutput
                  )
                  this.optimisticDeletes.delete(mutation.key)
                  break
                case `delete`:
                  this.optimisticUpserts.delete(mutation.key)
                  this.optimisticDeletes.add(mutation.key)
                  break
              }
            }
          }
        }
      }

      // Check for redundant sync operations that match completed optimistic operations
      const completedOptimisticOps = new Map<TKey, any>()

      for (const transaction of this.transactions.values()) {
        if (transaction.state === `completed`) {
          for (const mutation of transaction.mutations) {
            if (mutation.collection === this && changedKeys.has(mutation.key)) {
              completedOptimisticOps.set(mutation.key, {
                type: mutation.type,
                value: mutation.modified,
              })
            }
          }
        }
      }

      // Now check what actually changed in the final visible state
      for (const key of changedKeys) {
        const previousVisibleValue = currentVisibleState.get(key)
        const newVisibleValue = this.get(key) // This returns the new derived state

        // Check if this sync operation is redundant with a completed optimistic operation
        const completedOp = completedOptimisticOps.get(key)
        const isRedundantSync =
          completedOp &&
          newVisibleValue !== undefined &&
          deepEquals(completedOp.value, newVisibleValue)

        if (!isRedundantSync) {
          if (
            previousVisibleValue === undefined &&
            newVisibleValue !== undefined
          ) {
            events.push({
              type: `insert`,
              key,
              value: newVisibleValue,
            })
          } else if (
            previousVisibleValue !== undefined &&
            newVisibleValue === undefined
          ) {
            events.push({
              type: `delete`,
              key,
              value: previousVisibleValue,
            })
          } else if (
            previousVisibleValue !== undefined &&
            newVisibleValue !== undefined &&
            !deepEquals(previousVisibleValue, newVisibleValue)
          ) {
            events.push({
              type: `update`,
              key,
              value: newVisibleValue,
              previousValue: previousVisibleValue,
            })
          }
        }
      }

      // Update cached size after synced data changes
      this._size = this.calculateSize()

      // Update indexes for all events before emitting
      if (events.length > 0) {
        this.updateIndexes(events)
      }

      // End batching and emit all events (combines any batched events with sync events)
      this.emitEvents(events, true)

      this.pendingSyncedTransactions = uncommittedSyncedTransactions

      // Clear the pre-sync state since sync operations are complete
      this.preSyncVisibleState.clear()

      // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them
      Promise.resolve().then(() => {
        this.recentlySyncedKeys.clear()
      })

      // Call any registered one-time commit listeners
      if (!this.hasReceivedFirstCommit) {
        this.hasReceivedFirstCommit = true
        const callbacks = [...this.onFirstReadyCallbacks]
        this.onFirstReadyCallbacks = []
        callbacks.forEach((callback) => callback())
      }
    }
  }

  /**
   * Schedule cleanup of a transaction when it completes
   * @private
   */
  private scheduleTransactionCleanup(transaction: Transaction<any>): void {
    // Only schedule cleanup for transactions that aren't already completed
    if (transaction.state === `completed`) {
      this.transactions.delete(transaction.id)
      return
    }

    // Schedule cleanup when the transaction completes
    transaction.isPersisted.promise
      .then(() => {
        // Transaction completed successfully, remove it immediately
        this.transactions.delete(transaction.id)
      })
      .catch(() => {
        // Transaction failed, but we want to keep failed transactions for reference
        // so don't remove it.
        // This empty catch block is necessary to prevent unhandled promise rejections.
      })
  }

  private ensureStandardSchema(schema: unknown): StandardSchema<TOutput> {
    // If the schema already implements the standard-schema interface, return it
    if (schema && `~standard` in (schema as {})) {
      return schema as StandardSchema<TOutput>
    }

    throw new InvalidSchemaError()
  }

  public getKeyFromItem(item: TOutput): TKey {
    re