@tanstack/db/dist/esm/query/effect.js

A reactive client store for building super fast apps on sync

import { D2, output } from "@tanstack/db-ivm";
import { createDeferred } from "../deferred.js";
import { runAllCallbacks } from "../utils/callbacks.js";
import { normalizeError } from "../utils/error.js";
import { deepEquals } from "../utils.js";
import { compileQuery } from "./compiler/index.js";
import { normalizeExpressionPaths } from "./compiler/expressions.js";
import { getCollectionBuilder } from "./live/collection-registry.js";
import { scheduleQueryGraphRun } from "./live/graph-scheduler.js";
import { SubsetDemandController, hasPendingJoinedWork } from "./live/subset-demand-controller.js";
import { OrderedSourceLoader } from "./live/ordered-source-loader.js";
import { buildQueryFromConfig, extractCollectionFromSource, extractCollectionsFromQuery, splitUpdates, reconcileChangesForD2, sendChangesToInput, computeSubscriptionOrderByHints } from "./live/utils.js";
import { collectCollectionSources } from "./ir.js";
let effectCounter = 0;
function createEffect(config) {
  const id = config.id ?? `live-query-effect-${++effectCounter}`;
  const abortController = new AbortController();
  const ctx = {
    effectId: id,
    signal: abortController.signal
  };
  const inFlightHandlers = /* @__PURE__ */ new Set();
  let disposed = false;
  const onBatchProcessed = (events) => {
    if (disposed) return;
    if (events.length === 0) return;
    if (config.onBatch) {
      try {
        const result = config.onBatch(events, ctx);
        if (result instanceof Promise) {
          const tracked = result.catch((error) => {
            reportError(error, events[0], config.onError);
          });
          trackPromise(tracked, inFlightHandlers);
        }
      } catch (error) {
        reportError(error, events[0], config.onError);
      }
    }
    for (const event of events) {
      if (abortController.signal.aborted) break;
      const handler = getHandlerForEvent(event, config);
      if (!handler) continue;
      try {
        const result = handler(event, ctx);
        if (result instanceof Promise) {
          const tracked = result.catch((error) => {
            reportError(error, event, config.onError);
          });
          trackPromise(tracked, inFlightHandlers);
        }
      } catch (error) {
        reportError(error, event, config.onError);
      }
    }
  };
  let disposalPromise;
  const dispose = () => {
    if (disposalPromise) return disposalPromise;
    const completion = createDeferred();
    const attempt = completion.promise;
    disposalPromise = attempt;
    disposed = true;
    abortController.abort();
    void (async () => {
      let cleanupFailed = false;
      let cleanupError;
      try {
        runner.dispose();
      } catch (error) {
        cleanupFailed = true;
        cleanupError = error;
      }
      if (inFlightHandlers.size > 0) {
        await Promise.allSettled([...inFlightHandlers]);
      }
      if (cleanupFailed) throw cleanupError;
    })().then(completion.resolve, completion.reject);
    void attempt.then(
      () => {
      },
      () => {
        if (disposalPromise === attempt) disposalPromise = void 0;
      }
    );
    return attempt;
  };
  const runner = new EffectPipelineRunner({
    query: config.query,
    skipInitial: config.skipInitial ?? false,
    onBatchProcessed,
    onSourceError: (error) => {
      if (disposed) return;
      if (config.onSourceError) {
        try {
          config.onSourceError(error);
        } catch (callbackError) {
          console.error(
            `[Effect '${id}'] onSourceError callback threw:`,
            callbackError
          );
        }
      } else {
        console.error(`[Effect '${id}'] ${error.message}. Disposing effect.`);
      }
      void dispose().catch((cleanupError) => {
        console.error(
          `[Effect '${id}'] failed to dispose after a source error:`,
          cleanupError
        );
      });
    }
  });
  try {
    runner.start();
  } catch (error) {
    try {
      runner.dispose();
    } catch (cleanupError) {
      console.error(
        `[Effect '${id}'] failed to dispose after a startup error:`,
        cleanupError
      );
    }
    throw error;
  }
  return {
    dispose,
    get disposed() {
      return disposed;
    }
  };
}
class EffectPipelineRunner {
  constructor(config) {
    this.subscriptions = {};
    this.lazySourcesCallbacks = {};
    this.lazySources = /* @__PURE__ */ new Set();
    this.demand = new SubsetDemandController();
    this.optimizableOrderByCollections = {};
    this.orderedLoaders = /* @__PURE__ */ new Map();
    this.pendingOrderedPublications = /* @__PURE__ */ new Set();
    this.orderedPublicationFailed = false;
    this.unsubscribeCallbacks = /* @__PURE__ */ new Set();
    this.sentToD2RowsBySource = /* @__PURE__ */ new Map();
    this.pendingChanges = /* @__PURE__ */ new Map();
    this.publishedRows = /* @__PURE__ */ new Map();
    this.initialLoadComplete = false;
    this.subscribedToAllCollections = false;
    this.builderDependencies = /* @__PURE__ */ new Set();
    this.isGraphRunning = false;
    this.starting = false;
    this.disposed = false;
    this.skipInitial = config.skipInitial;
    this.onBatchProcessed = config.onBatchProcessed;
    this.onSourceError = config.onSourceError;
    this.query = buildQueryFromConfig({ query: config.query });
    this.queryCompareOptions = extractCollectionFromSource(
      this.query
    ).compareOptions;
    this.collections = extractCollectionsFromQuery(this.query);
    this.collectionSources = collectCollectionSources(this.query);
    this.compilePipeline();
  }
  /** Compile the D2 graph and query pipeline */
  compilePipeline() {
    this.graph = new D2();
    this.inputs = Object.fromEntries(
      this.collectionSources.map((source) => [
        source.sourceId,
        this.graph.newInput()
      ])
    );
    const compilation = compileQuery(
      this.query,
      this.inputs,
      this.collections,
      // These mutable objects are captured by reference. The join compiler
      // reads them later when the graph runs, so they must be populated
      // (in start()) before the first graph run.
      this.subscriptions,
      this.lazySourcesCallbacks,
      this.lazySources,
      this.optimizableOrderByCollections,
      () => {
      }
      // setWindowFn (no-op — effects don't paginate)
    );
    this.sourceWhereClauses = compilation.sourceWhereClauses;
    compilation.pipeline.pipe(
      output((data) => {
        const messages = data.getInner();
        messages.reduce(accumulateEffectChanges, this.pendingChanges);
      })
    );
    this.graph.finalize();
  }
  isDisposed() {
    return this.disposed;
  }
  /** Subscribe to source collections and start processing */
  start() {
    this.starting = true;
    if (this.collectionSources.length === 0) {
      this.starting = false;
      return;
    }
    if (!this.skipInitial) {
      this.initialLoadComplete = true;
    }
    const pendingBuffers = /* @__PURE__ */ new Map();
    for (const source of this.collectionSources) {
      if (this.isDisposed()) {
        this.starting = false;
        return;
      }
      const { sourceId, alias, collection } = source;
      const collectionId = collection.id;
      this.sentToD2RowsBySource.set(sourceId, /* @__PURE__ */ new Map());
      const dependencyBuilder = getCollectionBuilder(collection);
      if (dependencyBuilder) {
        this.builderDependencies.add(dependencyBuilder);
      }
      const whereClause = this.sourceWhereClauses?.get(sourceId);
      const whereExpression = whereClause ? normalizeExpressionPaths(whereClause, alias) : void 0;
      const buffer = [];
      pendingBuffers.set(sourceId, buffer);
      const isLazy = this.lazySources.has(sourceId);
      const orderByInfo = this.optimizableOrderByCollections[sourceId];
      const changeCallback = (changes) => {
        const bufferedChanges = pendingBuffers.get(sourceId);
        if (bufferedChanges) bufferedChanges.push(changes);
        else this.handleSourceChanges(sourceId, changes);
      };
      const subscription = collection.subscribeChanges(changeCallback, {
        ...this.buildSubscriptionOptions(
          alias,
          isLazy,
          orderByInfo,
          whereExpression
        ),
        onLoadSubsetError: ({ error }) => {
          this.onSourceError(normalizeError(error));
        }
      });
      this.subscriptions[sourceId] = subscription;
      const unsubscribe = () => {
        delete this.subscriptions[sourceId];
        subscription.unsubscribe();
      };
      if (this.isDisposed()) {
        unsubscribe();
        this.starting = false;
        return;
      }
      this.unsubscribeCallbacks.add(unsubscribe);
      if (Object.values(this.optimizableOrderByCollections).some(
        (info) => info.joinedFilterSourceId === sourceId
      )) {
        this.unsubscribeCallbacks.add(
          subscription.on(`status:ready`, () => {
            if (!this.disposed) this.scheduleGraphRun();
          })
        );
      }
      const lazyCallbacks = this.lazySourcesCallbacks[sourceId];
      if (lazyCallbacks) {
        lazyCallbacks.setDemand = (plan, keys) => this.setDemand(subscription, plan, keys);
        for (const plan of lazyCallbacks.plans ?? []) {
          if (plan.initialKeys.size > 0) {
            lazyCallbacks.setDemand(plan, plan.initialKeys);
          }
        }
      }
      if (orderByInfo) {
        const loader = new OrderedSourceLoader(
          orderByInfo,
          subscription,
          alias,
          (result, holdPublication) => {
            const holdsInitialPublication = this.skipInitial && !this.initialLoadComplete;
            if (holdPublication || holdsInitialPublication) {
              this.trackOrderedPublication(
                Promise.resolve(result),
                () => subscription.pendingTruncateReplacement
              );
            }
          },
          () => false,
          void 0,
          () => orderByInfo.joinedFilterSourceId !== void 0 && this.hasPendingJoinedWork(orderByInfo.joinedFilterSourceId)
        );
        this.orderedLoaders.set(sourceId, loader);
        loader.start();
      }
      const handleSourceCleanup = () => {
        if (this.disposed) return;
        this.onSourceError(
          new Error(
            `Source collection '${collectionId}' was cleaned up while effect depends on it`
          )
        );
      };
      const statusUnsubscribe = collection.on(`status:change`, (event) => {
        if (this.disposed) return;
        const { status } = event;
        if (status === `error`) {
          this.onSourceError(
            new Error(
              `Source collection '${collectionId}' entered error state`
            )
          );
          return;
        }
        if (status === `cleaned-up`) {
          handleSourceCleanup();
          return;
        }
        if (this.skipInitial && !this.initialLoadComplete && this.canCompleteInitialLoad()) {
          this.initialLoadComplete = true;
        }
      });
      this.unsubscribeCallbacks.add(statusUnsubscribe);
      const cleanupStartUnsubscribe = collection._onCleanupStart(handleSourceCleanup);
      if (this.isDisposed()) {
        cleanupStartUnsubscribe();
        this.starting = false;
        return;
      }
      this.unsubscribeCallbacks.add(cleanupStartUnsubscribe);
    }
    this.subscribedToAllCollections = true;
    for (const [sourceId, buffer] of pendingBuffers) {
      pendingBuffers.delete(sourceId);
      for (const changes of buffer) {
        this.handleSourceChanges(sourceId, changes, false);
      }
    }
    this.runGraph();
    if (this.skipInitial && !this.initialLoadComplete && this.canCompleteInitialLoad()) {
      this.initialLoadComplete = true;
    }
    this.starting = false;
  }
  /** Handle incoming changes from a source collection */
  handleSourceChanges(sourceId, changes, scheduleGraph = true) {
    if (this.optimizableOrderByCollections[sourceId]) {
      this.orderedLoaders.get(sourceId)?.onSourceChanges(changes, this.sentToD2RowsBySource.get(sourceId));
      changes = [...splitUpdates(changes)];
    }
    this.sendChangesToD2(sourceId, changes);
    if (scheduleGraph) this.scheduleGraphRun();
  }
  setDemand(subscription, plan, keys) {
    let update;
    try {
      update = this.demand.setDemand(subscription, plan, keys);
    } catch (error) {
      if (!Object.is(subscription.lastError, error)) throw error;
      if (this.starting) throw error;
      return;
    }
    if (update.ready instanceof Promise) {
      void update.ready.then(
        () => {
          if (!this.disposed) this.scheduleGraphRun();
        },
        () => {
        }
      );
    }
  }
  /**
   * Schedule a graph run via the transaction-scoped scheduler.
   *
   * When called within a transaction, the run is deferred until the
   * transaction flushes, coalescing multiple changes into a single graph
   * execution. Without a transaction, the graph runs immediately.
   *
   * Dependencies are discovered from source collections that are themselves
   * live query collections, ensuring parent queries run before effects.
   */
  scheduleGraphRun() {
    scheduleQueryGraphRun(this, this.builderDependencies, () => {
      if (!this.disposed && this.subscribedToAllCollections) this.runGraph();
    });
  }
  /** Hold Effect callback publication across one authoritative repair chain. */
  trackOrderedPublication(promise, replacementAfterFailure) {
    if (this.disposed) return;
    if (this.pendingOrderedPublications.size === 0) {
      this.orderedPublicationFailed = false;
    }
    this.pendingOrderedPublications.add(promise);
    const finish = (succeeded) => {
      if (!this.pendingOrderedPublications.delete(promise)) return;
      if (!succeeded) this.orderedPublicationFailed = true;
      if (succeeded && !this.orderedPublicationFailed && this.pendingOrderedPublications.size === 0 && !this.disposed) {
        this.scheduleGraphRun();
      }
    };
    void promise.then(
      () => finish(true),
      () => {
        const replacement = replacementAfterFailure();
        if (replacement && this.pendingOrderedPublications.delete(promise) && !this.disposed) {
          this.trackOrderedPublication(replacement, replacementAfterFailure);
          return;
        }
        finish(false);
      }
    );
  }
  /**
   * Send changes to the D2 input for the given lexical source.
   * Returns the number of multiset entries sent.
   */
  sendChangesToD2(sourceId, changes) {
    if (this.disposed || !this.inputs || !this.graph) return 0;
    const input = this.inputs[sourceId];
    if (!input) return 0;
    const sentRows = this.sentToD2RowsBySource.get(sourceId);
    const reconciled = reconcileChangesForD2(changes, sentRows);
    return sendChangesToInput(input, reconciled);
  }
  /**
   * Run the D2 graph until quiescence, then emit accumulated events once.
   *
   * All output across the entire while-loop is accumulated into a single
   * batch so that users see one `onBatchProcessed` invocation per scheduler
   * run, even when ordered loading causes multiple graph steps.
   */
  runGraph() {
    if (this.isGraphRunning || this.disposed || !this.graph) return;
    this.isGraphRunning = true;
    try {
      if (!this.graph.pendingWork()) this.loadMoreIfNeeded();
      while (!this.isDisposed() && this.graph.pendingWork()) {
        this.graph.run();
        if (this.disposed) break;
        this.loadMoreIfNeeded();
      }
      this.flushPendingChanges();
      if (this.skipInitial && !this.initialLoadComplete && this.canCompleteInitialLoad()) {
        this.initialLoadComplete = true;
      }
    } finally {
      this.isGraphRunning = false;
    }
  }
  /** Classify accumulated changes into DeltaEvents and invoke the callback */
  flushPendingChanges() {
    if (this.pendingChanges.size === 0) return;
    if (this.orderedPublicationFailed || this.pendingOrderedPublications.size > 0) {
      return;
    }
    if (this.hasPendingJoinedFilterWork()) {
      return;
    }
    const shouldPublish = !this.skipInitial || this.initialLoadComplete;
    const tracksPublishedRows = this.orderedLoaders.size > 0;
    const events = [];
    for (const [key, changes] of this.pendingChanges) {
      const event = classifyDelta(
        key,
        changes,
        tracksPublishedRows ? this.publishedRows : void 0
      );
      if (event) {
        if (tracksPublishedRows) {
          if (event.type === `exit`) this.publishedRows.delete(key);
          else this.publishedRows.set(key, event.value);
        }
        if (shouldPublish) events.push(event);
      }
    }
    this.pendingChanges = /* @__PURE__ */ new Map();
    if (events.length > 0) {
      this.onBatchProcessed(events);
    }
  }
  /** Check if all source collections are in the ready state */
  checkAllCollectionsReady() {
    return Object.values(this.collections).every(
      (collection) => collection.isReady()
    );
  }
  /** Initial callbacks start only after source readiness and ordered loading. */
  canCompleteInitialLoad() {
    return !this.orderedPublicationFailed && this.pendingOrderedPublications.size === 0 && !this.hasPendingJoinedFilterWork() && this.checkAllCollectionsReady();
  }
  hasPendingJoinedFilterWork() {
    return Object.values(this.optimizableOrderByCollections).some(
      (info) => info.joinedFilterSourceId !== void 0 && this.hasPendingJoinedWork(info.joinedFilterSourceId)
    );
  }
  /**
   * Build subscription options for an alias based on whether it uses ordered
   * loading, is lazy, or should pass orderBy/limit hints.
   */
  buildSubscriptionOptions(alias, isLazy, orderByInfo, whereExpression) {
    if (this.query.limit === 0) {
      return { includeInitialState: false, whereExpression };
    }
    if (orderByInfo) {
      return { includeInitialState: false, whereExpression };
    }
    const includeInitialState = !isLazy;
    const hints = computeSubscriptionOrderByHints(
      this.query,
      alias,
      this.queryCompareOptions
    );
    return {
      includeInitialState,
      whereExpression,
      ...hints.orderBy ? { orderBy: hints.orderBy } : {},
      ...hints.limit !== void 0 ? { limit: hints.limit } : {}
    };
  }
  /**
   * After each graph run step, check if any ordered query's topK operator
   * needs more data. If so, load more rows via requestLimitedSnapshot.
   */
  loadMoreIfNeeded() {
    for (const loader of this.orderedLoaders.values()) {
      try {
        loader.loadMore();
      } catch (error) {
        if (!this.disposed && !Object.values(this.subscriptions).some(
          (subscription) => Object.is(subscription.lastError, error)
        ))
          throw error;
      }
    }
  }
  hasPendingJoinedWork(joinedSourceId) {
    return hasPendingJoinedWork(
      joinedSourceId,
      this.lazySourcesCallbacks,
      this.subscriptions,
      (planId) => this.demand.hasPendingDemand(planId)
    );
  }
  /** Tear down subscriptions and clear state */
  dispose() {
    if (this.disposed) return;
    this.disposed = true;
    this.subscribedToAllCollections = false;
    try {
      runAllCallbacks(this.unsubscribeCallbacks);
    } finally {
      this.unsubscribeCallbacks.clear();
      this.clearPipelineState();
    }
  }
  clearPipelineState() {
    this.sentToD2RowsBySource.clear();
    this.pendingChanges.clear();
    this.publishedRows.clear();
    this.pendingOrderedPublications.clear();
    this.orderedPublicationFailed = false;
    this.lazySources.clear();
    this.demand.clear();
    this.builderDependencies.clear();
    for (const loader of this.orderedLoaders.values()) loader.dispose();
    this.orderedLoaders.clear();
    for (const key of Object.keys(this.lazySourcesCallbacks)) {
      delete this.lazySourcesCallbacks[key];
    }
    for (const key of Object.keys(this.optimizableOrderByCollections)) {
      delete this.optimizableOrderByCollections[key];
    }
    this.graph = void 0;
    this.inputs = void 0;
    this.sourceWhereClauses = void 0;
  }
}
function getHandlerForEvent(event, config) {
  switch (event.type) {
    case `enter`:
      return config.onEnter;
    case `exit`:
      return config.onExit;
    case `update`:
      return config.onUpdate;
  }
}
function accumulateEffectChanges(acc, [[key, tupleData], multiplicity]) {
  const [value] = tupleData;
  const changes = acc.get(key) || {
    deletes: 0,
    inserts: 0
  };
  if (multiplicity < 0) {
    changes.deletes += Math.abs(multiplicity);
    changes.deleteValue ??= value;
  } else if (multiplicity > 0) {
    changes.inserts += multiplicity;
    changes.insertValue = value;
  }
  acc.set(key, changes);
  return acc;
}
function classifyDelta(key, changes, publishedRows) {
  const { inserts, deletes, insertValue, deleteValue } = changes;
  const wasPresent = publishedRows ? publishedRows.has(key) : deletes > 0;
  const previousValue = publishedRows?.get(key) ?? deleteValue;
  const isPresent = publishedRows ? Number(wasPresent) + inserts - deletes > 0 : inserts > 0;
  if (!wasPresent && isPresent) {
    return { type: `enter`, key, value: insertValue };
  }
  if (wasPresent && !isPresent) {
    return { type: `exit`, key, value: previousValue ?? deleteValue };
  }
  if (wasPresent && isPresent && insertValue !== void 0 && (!publishedRows || !deepEquals(previousValue, insertValue))) {
    return {
      type: `update`,
      key,
      value: insertValue,
      previousValue
    };
  }
  return void 0;
}
function trackPromise(promise, inFlightHandlers) {
  inFlightHandlers.add(promise);
  const finish = () => {
    inFlightHandlers.delete(promise);
  };
  void promise.then(finish, finish);
}
function reportError(error, event, onError) {
  const normalised = normalizeError(error);
  if (onError) {
    try {
      onError(normalised, event);
    } catch (onErrorError) {
      console.error(`[Effect] Error in onError handler:`, onErrorError);
      console.error(`[Effect] Original error:`, normalised);
    }
  } else {
    console.error(`[Effect] Unhandled error in handler:`, normalised);
  }
}
export {
  createEffect
};
//# sourceMappingURL=effect.js.map