@tanstack/db
Version:
A reactive client store for building super fast apps on sync
603 lines (602 loc) • 20.2 kB
JavaScript
import { D2, output } from "@tanstack/db-ivm";
import { transactionScopedScheduler } from "../scheduler.js";
import { getActiveTransaction } from "../transactions.js";
import { compileQuery } from "./compiler/index.js";
import { normalizeExpressionPaths, normalizeOrderByPaths } from "./compiler/expressions.js";
import { getCollectionBuilder } from "./live/collection-registry.js";
import { buildQueryFromConfig, extractCollectionsFromQuery, extractCollectionAliases, splitUpdates, filterDuplicateInserts, sendChangesToInput, computeSubscriptionOrderByHints, computeOrderedLoadCursor, trackBiggestSentValue } from "./live/utils.js";
let effectCounter = 0;
function createEffect(config) {
const id = config.id ?? `live-query-effect-${++effectCounter}`;
const abortController = new AbortController();
const ctx = {
effectId: id,
signal: abortController.signal
};
const inFlightHandlers = /* @__PURE__ */ new Set();
let disposed = false;
const onBatchProcessed = (events) => {
if (disposed) return;
if (events.length === 0) return;
if (config.onBatch) {
try {
const result = config.onBatch(events, ctx);
if (result instanceof Promise) {
const tracked = result.catch((error) => {
reportError(error, events[0], config.onError);
});
trackPromise(tracked, inFlightHandlers);
}
} catch (error) {
reportError(error, events[0], config.onError);
}
}
for (const event of events) {
if (abortController.signal.aborted) break;
const handler = getHandlerForEvent(event, config);
if (!handler) continue;
try {
const result = handler(event, ctx);
if (result instanceof Promise) {
const tracked = result.catch((error) => {
reportError(error, event, config.onError);
});
trackPromise(tracked, inFlightHandlers);
}
} catch (error) {
reportError(error, event, config.onError);
}
}
};
const dispose = async () => {
if (disposed) return;
disposed = true;
abortController.abort();
runner.dispose();
if (inFlightHandlers.size > 0) {
await Promise.allSettled([...inFlightHandlers]);
}
};
const runner = new EffectPipelineRunner({
query: config.query,
skipInitial: config.skipInitial ?? false,
onBatchProcessed,
onSourceError: (error) => {
if (disposed) return;
if (config.onSourceError) {
try {
config.onSourceError(error);
} catch (callbackError) {
console.error(
`[Effect '${id}'] onSourceError callback threw:`,
callbackError
);
}
} else {
console.error(`[Effect '${id}'] ${error.message}. Disposing effect.`);
}
dispose();
}
});
runner.start();
return {
dispose,
get disposed() {
return disposed;
}
};
}
class EffectPipelineRunner {
constructor(config) {
this.compiledAliasToCollectionId = {};
this.subscriptions = {};
this.lazySourcesCallbacks = {};
this.lazySources = /* @__PURE__ */ new Set();
this.optimizableOrderByCollections = {};
this.biggestSentValue = /* @__PURE__ */ new Map();
this.lastLoadRequestKey = /* @__PURE__ */ new Map();
this.unsubscribeCallbacks = /* @__PURE__ */ new Set();
this.sentToD2KeysByAlias = /* @__PURE__ */ new Map();
this.pendingChanges = /* @__PURE__ */ new Map();
this.initialLoadComplete = false;
this.subscribedToAllCollections = false;
this.builderDependencies = /* @__PURE__ */ new Set();
this.aliasDependencies = {};
this.isGraphRunning = false;
this.disposed = false;
this.deferredCleanup = false;
this.skipInitial = config.skipInitial;
this.onBatchProcessed = config.onBatchProcessed;
this.onSourceError = config.onSourceError;
this.query = buildQueryFromConfig({ query: config.query });
this.collections = extractCollectionsFromQuery(this.query);
const aliasesById = extractCollectionAliases(this.query);
this.collectionByAlias = {};
for (const [collectionId, aliases] of aliasesById.entries()) {
const collection = this.collections[collectionId];
if (!collection) continue;
for (const alias of aliases) {
this.collectionByAlias[alias] = collection;
}
}
this.compilePipeline();
}
/** Compile the D2 graph and query pipeline */
compilePipeline() {
this.graph = new D2();
this.inputs = Object.fromEntries(
Object.keys(this.collectionByAlias).map((alias) => [
alias,
this.graph.newInput()
])
);
const compilation = compileQuery(
this.query,
this.inputs,
this.collections,
// These mutable objects are captured by reference. The join compiler
// reads them later when the graph runs, so they must be populated
// (in start()) before the first graph run.
this.subscriptions,
this.lazySourcesCallbacks,
this.lazySources,
this.optimizableOrderByCollections,
() => {
}
// setWindowFn (no-op — effects don't paginate)
);
this.pipeline = compilation.pipeline;
this.sourceWhereClauses = compilation.sourceWhereClauses;
this.compiledAliasToCollectionId = compilation.aliasToCollectionId;
this.pipeline.pipe(
output((data) => {
const messages = data.getInner();
messages.reduce(accumulateEffectChanges, this.pendingChanges);
})
);
this.graph.finalize();
}
/** Subscribe to source collections and start processing */
start() {
const compiledAliases = Object.entries(this.compiledAliasToCollectionId);
if (compiledAliases.length === 0) {
return;
}
if (!this.skipInitial) {
this.initialLoadComplete = true;
}
const pendingBuffers = /* @__PURE__ */ new Map();
for (const [alias, collectionId] of compiledAliases) {
const collection = this.collectionByAlias[alias] ?? this.collections[collectionId];
this.sentToD2KeysByAlias.set(alias, /* @__PURE__ */ new Set());
const dependencyBuilder = getCollectionBuilder(collection);
if (dependencyBuilder) {
this.aliasDependencies[alias] = [dependencyBuilder];
this.builderDependencies.add(dependencyBuilder);
} else {
this.aliasDependencies[alias] = [];
}
const whereClause = this.sourceWhereClauses?.get(alias);
const whereExpression = whereClause ? normalizeExpressionPaths(whereClause, alias) : void 0;
const buffer = [];
pendingBuffers.set(alias, buffer);
const isLazy = this.lazySources.has(alias);
const orderByInfo = this.getOrderByInfoForAlias(alias);
const changeCallback = orderByInfo ? (changes) => {
if (pendingBuffers.has(alias)) {
pendingBuffers.get(alias).push(changes);
} else {
this.trackSentValues(alias, changes, orderByInfo.comparator);
const split = [...splitUpdates(changes)];
this.handleSourceChanges(alias, split);
}
} : (changes) => {
if (pendingBuffers.has(alias)) {
pendingBuffers.get(alias).push(changes);
} else {
this.handleSourceChanges(alias, changes);
}
};
const subscriptionOptions = this.buildSubscriptionOptions(
alias,
isLazy,
orderByInfo,
whereExpression
);
const subscription = collection.subscribeChanges(
changeCallback,
subscriptionOptions
);
this.subscriptions[alias] = subscription;
if (orderByInfo) {
this.requestInitialOrderedSnapshot(alias, orderByInfo, subscription);
}
this.unsubscribeCallbacks.add(() => {
subscription.unsubscribe();
delete this.subscriptions[alias];
});
const statusUnsubscribe = collection.on(`status:change`, (event) => {
if (this.disposed) return;
const { status } = event;
if (status === `error`) {
this.onSourceError(
new Error(
`Source collection '${collectionId}' entered error state`
)
);
return;
}
if (status === `cleaned-up`) {
this.onSourceError(
new Error(
`Source collection '${collectionId}' was cleaned up while effect depends on it`
)
);
return;
}
if (this.skipInitial && !this.initialLoadComplete && this.checkAllCollectionsReady()) {
this.initialLoadComplete = true;
}
});
this.unsubscribeCallbacks.add(statusUnsubscribe);
}
this.subscribedToAllCollections = true;
for (const [alias] of pendingBuffers) {
const buffer = pendingBuffers.get(alias);
pendingBuffers.delete(alias);
const orderByInfo = this.getOrderByInfoForAlias(alias);
for (const changes of buffer) {
if (orderByInfo) {
this.trackSentValues(alias, changes, orderByInfo.comparator);
const split = [...splitUpdates(changes)];
this.sendChangesToD2(alias, split);
} else {
this.sendChangesToD2(alias, changes);
}
}
}
this.runGraph();
if (this.skipInitial && !this.initialLoadComplete) {
if (this.checkAllCollectionsReady()) {
this.initialLoadComplete = true;
}
}
}
/** Handle incoming changes from a source collection */
handleSourceChanges(alias, changes) {
this.sendChangesToD2(alias, changes);
this.scheduleGraphRun(alias);
}
/**
* Schedule a graph run via the transaction-scoped scheduler.
*
* When called within a transaction, the run is deferred until the
* transaction flushes, coalescing multiple changes into a single graph
* execution. Without a transaction, the graph runs immediately.
*
* Dependencies are discovered from source collections that are themselves
* live query collections, ensuring parent queries run before effects.
*/
scheduleGraphRun(alias) {
const contextId = getActiveTransaction()?.id;
const deps = new Set(this.builderDependencies);
if (alias) {
const aliasDeps = this.aliasDependencies[alias];
if (aliasDeps) {
for (const dep of aliasDeps) {
deps.add(dep);
}
}
}
if (contextId) {
for (const dep of deps) {
if (typeof dep === `object` && dep !== null && `scheduleGraphRun` in dep && typeof dep.scheduleGraphRun === `function`) {
dep.scheduleGraphRun(void 0, { contextId });
}
}
}
transactionScopedScheduler.schedule({
contextId,
jobId: this,
dependencies: deps,
run: () => this.executeScheduledGraphRun()
});
}
/**
* Called by the scheduler when dependencies are satisfied.
* Checks that the effect is still active before running.
*/
executeScheduledGraphRun() {
if (this.disposed || !this.subscribedToAllCollections) return;
this.runGraph();
}
/**
* Send changes to the D2 input for the given alias.
* Returns the number of multiset entries sent.
*/
sendChangesToD2(alias, changes) {
if (this.disposed || !this.inputs || !this.graph) return 0;
const input = this.inputs[alias];
if (!input) return 0;
const collection = this.collectionByAlias[alias];
if (!collection) return 0;
const sentKeys = this.sentToD2KeysByAlias.get(alias);
const filtered = filterDuplicateInserts(changes, sentKeys);
return sendChangesToInput(input, filtered, collection.config.getKey);
}
/**
* Run the D2 graph until quiescence, then emit accumulated events once.
*
* All output across the entire while-loop is accumulated into a single
* batch so that users see one `onBatchProcessed` invocation per scheduler
* run, even when ordered loading causes multiple graph steps.
*/
runGraph() {
if (this.isGraphRunning || this.disposed || !this.graph) return;
this.isGraphRunning = true;
try {
while (this.graph.pendingWork()) {
this.graph.run();
if (this.disposed) break;
this.loadMoreIfNeeded();
}
this.flushPendingChanges();
} finally {
this.isGraphRunning = false;
if (this.deferredCleanup) {
this.deferredCleanup = false;
this.finalCleanup();
}
}
}
/** Classify accumulated changes into DeltaEvents and invoke the callback */
flushPendingChanges() {
if (this.pendingChanges.size === 0) return;
if (this.skipInitial && !this.initialLoadComplete) {
this.pendingChanges = /* @__PURE__ */ new Map();
return;
}
const events = [];
for (const [key, changes] of this.pendingChanges) {
const event = classifyDelta(key, changes);
if (event) {
events.push(event);
}
}
this.pendingChanges = /* @__PURE__ */ new Map();
if (events.length > 0) {
this.onBatchProcessed(events);
}
}
/** Check if all source collections are in the ready state */
checkAllCollectionsReady() {
return Object.values(this.collections).every(
(collection) => collection.isReady()
);
}
/**
* Build subscription options for an alias based on whether it uses ordered
* loading, is lazy, or should pass orderBy/limit hints.
*/
buildSubscriptionOptions(alias, isLazy, orderByInfo, whereExpression) {
if (orderByInfo) {
return { includeInitialState: false, whereExpression };
}
const includeInitialState = !isLazy;
const hints = computeSubscriptionOrderByHints(this.query, alias);
return {
includeInitialState,
whereExpression,
...hints.orderBy ? { orderBy: hints.orderBy } : {},
...hints.limit !== void 0 ? { limit: hints.limit } : {}
};
}
/**
* Request the initial ordered snapshot for an alias.
* Uses requestLimitedSnapshot (index-based cursor) or requestSnapshot
* (full load with limit) depending on whether an index is available.
*/
requestInitialOrderedSnapshot(alias, orderByInfo, subscription) {
const { orderBy, offset, limit, index } = orderByInfo;
const normalizedOrderBy = normalizeOrderByPaths(orderBy, alias);
if (index) {
subscription.setOrderByIndex(index);
subscription.requestLimitedSnapshot({
limit: offset + limit,
orderBy: normalizedOrderBy,
trackLoadSubsetPromise: false
});
} else {
subscription.requestSnapshot({
orderBy: normalizedOrderBy,
limit: offset + limit,
trackLoadSubsetPromise: false
});
}
}
/**
* Get orderBy optimization info for a given alias.
* Returns undefined if no optimization exists for this alias.
*/
getOrderByInfoForAlias(alias) {
const collectionId = this.compiledAliasToCollectionId[alias];
if (!collectionId) return void 0;
const info = this.optimizableOrderByCollections[collectionId];
if (info && info.alias === alias) {
return info;
}
return void 0;
}
/**
* After each graph run step, check if any ordered query's topK operator
* needs more data. If so, load more rows via requestLimitedSnapshot.
*/
loadMoreIfNeeded() {
for (const [, orderByInfo] of Object.entries(
this.optimizableOrderByCollections
)) {
if (!orderByInfo.dataNeeded) continue;
if (this.pendingOrderedLoadPromise) {
continue;
}
const n = orderByInfo.dataNeeded();
if (n > 0) {
this.loadNextItems(orderByInfo, n);
}
}
}
/**
* Load n more items from the source collection, starting from the cursor
* position (the biggest value sent so far).
*/
loadNextItems(orderByInfo, n) {
const { alias } = orderByInfo;
const subscription = this.subscriptions[alias];
if (!subscription) return;
const cursor = computeOrderedLoadCursor(
orderByInfo,
this.biggestSentValue.get(alias),
this.lastLoadRequestKey.get(alias),
alias,
n
);
if (!cursor) return;
this.lastLoadRequestKey.set(alias, cursor.loadRequestKey);
subscription.requestLimitedSnapshot({
orderBy: cursor.normalizedOrderBy,
limit: n,
minValues: cursor.minValues,
trackLoadSubsetPromise: false,
onLoadSubsetResult: (loadResult) => {
if (loadResult instanceof Promise) {
this.pendingOrderedLoadPromise = loadResult;
loadResult.finally(() => {
if (this.pendingOrderedLoadPromise === loadResult) {
this.pendingOrderedLoadPromise = void 0;
}
});
}
}
});
}
/**
* Track the biggest value sent for a given ordered alias.
* Used for cursor-based pagination in loadNextItems.
*/
trackSentValues(alias, changes, comparator) {
const sentKeys = this.sentToD2KeysByAlias.get(alias) ?? /* @__PURE__ */ new Set();
const result = trackBiggestSentValue(
changes,
this.biggestSentValue.get(alias),
sentKeys,
comparator
);
this.biggestSentValue.set(alias, result.biggest);
if (result.shouldResetLoadKey) {
this.lastLoadRequestKey.delete(alias);
}
}
/** Tear down subscriptions and clear state */
dispose() {
if (this.disposed) return;
this.disposed = true;
this.subscribedToAllCollections = false;
this.unsubscribeCallbacks.forEach((fn) => fn());
this.unsubscribeCallbacks.clear();
this.sentToD2KeysByAlias.clear();
this.pendingChanges.clear();
this.lazySources.clear();
this.builderDependencies.clear();
this.biggestSentValue.clear();
this.lastLoadRequestKey.clear();
this.pendingOrderedLoadPromise = void 0;
for (const key of Object.keys(this.lazySourcesCallbacks)) {
delete this.lazySourcesCallbacks[key];
}
for (const key of Object.keys(this.aliasDependencies)) {
delete this.aliasDependencies[key];
}
for (const key of Object.keys(this.optimizableOrderByCollections)) {
delete this.optimizableOrderByCollections[key];
}
if (this.isGraphRunning) {
this.deferredCleanup = true;
} else {
this.finalCleanup();
}
}
/** Clear graph references — called after graph run completes or immediately from dispose */
finalCleanup() {
this.graph = void 0;
this.inputs = void 0;
this.pipeline = void 0;
this.sourceWhereClauses = void 0;
}
}
function getHandlerForEvent(event, config) {
switch (event.type) {
case `enter`:
return config.onEnter;
case `exit`:
return config.onExit;
case `update`:
return config.onUpdate;
}
}
function accumulateEffectChanges(acc, [[key, tupleData], multiplicity]) {
const [value] = tupleData;
const changes = acc.get(key) || {
deletes: 0,
inserts: 0
};
if (multiplicity < 0) {
changes.deletes += Math.abs(multiplicity);
changes.deleteValue ??= value;
} else if (multiplicity > 0) {
changes.inserts += multiplicity;
changes.insertValue = value;
}
acc.set(key, changes);
return acc;
}
function classifyDelta(key, changes) {
const { inserts, deletes, insertValue, deleteValue } = changes;
if (inserts > 0 && deletes === 0) {
return { type: `enter`, key, value: insertValue };
}
if (deletes > 0 && inserts === 0) {
return { type: `exit`, key, value: deleteValue };
}
if (inserts > 0 && deletes > 0) {
return {
type: `update`,
key,
value: insertValue,
previousValue: deleteValue
};
}
return void 0;
}
function trackPromise(promise, inFlightHandlers) {
inFlightHandlers.add(promise);
promise.finally(() => {
inFlightHandlers.delete(promise);
});
}
function reportError(error, event, onError) {
const normalised = error instanceof Error ? error : new Error(String(error));
if (onError) {
try {
onError(normalised, event);
} catch (onErrorError) {
console.error(`[Effect] Error in onError handler:`, onErrorError);
console.error(`[Effect] Original error:`, normalised);
}
} else {
console.error(`[Effect] Unhandled error in handler:`, normalised);
}
}
export {
createEffect
};
//# sourceMappingURL=effect.js.map