@tanstack/db
Version:
A reactive client store for building super fast apps on sync
772 lines (771 loc) • 27.3 kB
JavaScript
"use strict";
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
const autoIndex = require("../indexes/auto-index.cjs");
const functions = require("../query/builder/functions.cjs");
const ir = require("../query/ir.cjs");
const eventEmitter = require("../event-emitter.cjs");
const evaluators = require("../query/compiler/evaluators.cjs");
const cursor = require("../utils/cursor.cjs");
const utils = require("../utils.cjs");
const changeEvents = require("./change-events.cjs");
class CollectionSubscription extends eventEmitter.EventEmitter {
constructor(collection, callback, options) {
super();
this.collection = collection;
this.callback = callback;
this.options = options;
this.loadedInitialState = false;
this.skipFiltering = false;
this.snapshotSent = false;
this.subsetDemands = [];
this.requestedSubsetWhere = /* @__PURE__ */ new WeakMap();
this.sentKeys = /* @__PURE__ */ new Set();
this.publishedRows = /* @__PURE__ */ new Map();
this.stalePublishedRows = /* @__PURE__ */ new Map();
this.limitedSnapshotRowCount = 0;
this._status = `ready`;
this.pendingLoadSubsetPromises = /* @__PURE__ */ new Set();
if (options.onUnsubscribe) {
this.on(`unsubscribed`, options.onUnsubscribe);
}
if (options.onLoadSubsetError) {
this.on(`loadSubset:error`, options.onLoadSubsetError);
}
if (options.whereExpression) {
autoIndex.ensureIndexForExpression(options.whereExpression, this.collection);
}
const callbackWithSentKeysTracking = (changes) => {
this.trackPublishedRows(changes);
this.trackSentKeys(changes);
callback(changes);
};
this.callback = callbackWithSentKeysTracking;
this.filteredCallback = options.whereExpression ? changeEvents.createFilteredCallback(this.callback, options) : (changes) => {
this.callback(changes);
return true;
};
this.truncateCleanup = this.collection.on(`truncate`, () => {
this.handleTruncate();
});
}
get status() {
return this._status;
}
get lastError() {
return this._lastError;
}
/**
* Handle collection truncate event by resetting state and re-requesting subsets.
* This is called when the sync layer receives a must-refetch and clears all data.
*
* To prevent a flash of missing content, we buffer all changes (deletes from truncate
* and inserts from refetch) until all loadSubset calls succeed, then emit them together.
* A failed replay keeps the last published snapshot, resumes ordinary deltas,
* and retains subset ownership so a later truncate can retry the replay.
*/
handleTruncate() {
const demandsToReload = [...this.subsetDemands];
const hasLoadSubsetHandler = this.collection._sync.syncLoadSubsetFn !== null;
if (demandsToReload.length === 0 || !hasLoadSubsetHandler) {
this.snapshotSent = false;
this.loadedInitialState = false;
this.limitedSnapshotRowCount = 0;
this.lastSentKey = void 0;
return;
}
const attempt = {
pending: /* @__PURE__ */ new Set(),
failed: false,
setupComplete: false
};
let session = this.truncateReplaySession;
if (!session) {
session = {
publicationState: {
loadedInitialState: this.loadedInitialState,
snapshotSent: this.snapshotSent,
sentKeys: new Set(this.sentKeys),
publishedRows: new Map(this.publishedRows),
limitedSnapshotRowCount: this.limitedSnapshotRowCount,
lastSentKey: this.lastSentKey
},
buffer: [],
attempts: /* @__PURE__ */ new Set(),
currentAttempt: attempt
};
this.truncateReplaySession = session;
}
session.attempts.add(attempt);
session.currentAttempt = attempt;
for (const demand of demandsToReload) {
demand.abortController?.abort();
}
this.snapshotSent = false;
this.loadedInitialState = false;
this.limitedSnapshotRowCount = 0;
this.lastSentKey = void 0;
queueMicrotask(() => {
if (this.truncateReplaySession !== session) return;
for (const demand of demandsToReload) {
if (!this.subsetDemands.includes(demand)) continue;
const isCurrentAttempt = () => this.truncateReplaySession === session && session.currentAttempt === attempt;
const nextAcquisition = this.createSubsetAcquisition(demand);
let syncResult;
try {
syncResult = this.loadSubset(
nextAcquisition.options,
isCurrentAttempt
);
} catch {
nextAcquisition.abortController.abort();
nextAcquisition.removeRequestAbortListener?.();
attempt.failed = true;
continue;
}
this.observeLoadSubsetResult(
syncResult,
nextAcquisition.options,
true,
() => isCurrentAttempt() && !nextAcquisition.options.signal?.aborted
);
if (syncResult instanceof Promise) {
const pending = { promise: syncResult };
attempt.pending.add(pending);
void syncResult.then(
() => this.settleTruncateReplay(session, attempt, pending),
() => {
if (this.subsetDemands.includes(demand) && !nextAcquisition.options.signal?.aborted) {
attempt.failed = true;
}
this.settleTruncateReplay(session, attempt, pending);
}
);
}
try {
this.replaceSubsetAcquisition(demand, nextAcquisition);
} catch (error) {
nextAcquisition.abortController.abort();
nextAcquisition.removeRequestAbortListener?.();
try {
this.collection._sync.unloadSubset(nextAcquisition.options);
} catch {
}
this.recordLoadSubsetError(demand.options, error, true);
attempt.failed = true;
}
}
attempt.setupComplete = true;
this.checkTruncateReplayComplete(session);
});
}
settleTruncateReplay(session, attempt, pending) {
if (this.truncateReplaySession !== session) return;
attempt.pending.delete(pending);
this.checkTruncateReplayComplete(session);
}
/** Publish only after every overlapping replay attempt has settled. */
checkTruncateReplayComplete(session) {
if (this.truncateReplaySession !== session) return;
for (const attempt of session.attempts) {
if (!attempt.setupComplete || attempt.pending.size > 0) return;
}
if (session.currentAttempt.failed) {
this.abandonTruncateReplay(session);
} else {
this.flushTruncateReplay(session);
}
}
/**
* Discard an incomplete current replay and restore the last publication.
* Rows in that publication remain stale until a later source delta or replay
* reconciles them with the source collection.
*/
abandonTruncateReplay(session) {
if (this.truncateReplaySession !== session) return;
const publicationState = session.publicationState;
this.loadedInitialState = publicationState.loadedInitialState;
this.snapshotSent = publicationState.snapshotSent;
this.sentKeys = new Set(publicationState.sentKeys);
this.publishedRows = new Map(publicationState.publishedRows);
this.stalePublishedRows = new Map(publicationState.publishedRows);
this.limitedSnapshotRowCount = publicationState.limitedSnapshotRowCount;
this.lastSentKey = publicationState.lastSentKey;
this.truncateReplaySession = void 0;
}
/** Publish the complete buffered replacement as one subscriber batch. */
flushTruncateReplay(session) {
if (this.truncateReplaySession !== session) return;
this.truncateReplaySession = void 0;
const retainedDeletes = [...this.stalePublishedRows].map(
([key, value]) => ({
type: `delete`,
key,
value
})
);
this.stalePublishedRows.clear();
const merged = [...session.buffer.flat(), ...retainedDeletes];
const activeDemandFilters = this.subsetDemands.map(
(demand) => demand.requestOptions.where ? changeEvents.createFilterFunctionFromExpression(demand.requestOptions.where) : void 0
);
const replacement = this.createPublicationDiff(
session.publicationState.publishedRows,
merged,
(value) => activeDemandFilters.some((filter) => filter?.(value) ?? true)
);
if (replacement.length > 0) this.filteredCallback(replacement);
this.sentKeys = new Set(this.publishedRows.keys());
if (this.orderByIndex) {
this.limitedSnapshotRowCount = this.sentKeys.size;
const orderedSentKeys = this.orderByIndex.takeFromStart(
this.sentKeys.size,
(key) => this.sentKeys.has(key)
);
this.lastSentKey = orderedSentKeys.at(-1);
}
}
/** Reduce a replay's raw delete/insert stream to one exact semantic delta. */
createPublicationDiff(baseline, changes, isCoveredByActiveDemand) {
const finalRows = new Map(baseline);
for (const change of changes) {
if (change.type === `delete`) finalRows.delete(change.key);
else finalRows.set(change.key, change.value);
}
for (const [key, value] of finalRows) {
if (!isCoveredByActiveDemand(value)) finalRows.delete(key);
}
const replacement = [];
for (const [key, previousValue] of baseline) {
const value = finalRows.get(key);
if (value === void 0) {
replacement.push({
type: `delete`,
key,
value: previousValue
});
} else if (!utils.deepEquals(value, previousValue)) {
replacement.push({
type: `update`,
key,
value,
previousValue
});
}
}
for (const [key, value] of finalRows) {
if (!baseline.has(key)) replacement.push({ type: `insert`, key, value });
}
return replacement;
}
get isBufferingForTruncate() {
return this.truncateReplaySession !== void 0;
}
setOrderByIndex(index) {
this.orderByIndex = index;
}
/**
* Check if an orderBy index has been set for this subscription
*/
hasOrderByIndex() {
return this.orderByIndex !== void 0;
}
/**
* Set subscription status and emit events if changed
*/
setStatus(newStatus) {
if (this._status === newStatus) {
return;
}
const previousStatus = this._status;
this._status = newStatus;
this.emitInner(`status:change`, {
type: `status:change`,
subscription: this,
previousStatus,
status: newStatus
});
const eventKey = `status:${newStatus}`;
this.emitInner(eventKey, {
type: eventKey,
subscription: this,
previousStatus,
status: newStatus
});
}
/** Observe an asynchronous subset load and restore status on settlement. */
observeLoadSubsetResult(syncResult, options, trackStatus, shouldReportError = () => true) {
if (!(syncResult instanceof Promise)) return;
if (trackStatus) {
this.pendingLoadSubsetPromises.add(syncResult);
this.setStatus(`loadingSubset`);
}
const finish = () => {
if (trackStatus) {
this.pendingLoadSubsetPromises.delete(syncResult);
if (this.pendingLoadSubsetPromises.size === 0) {
this.setStatus(`ready`);
}
}
};
void syncResult.then(finish, (error) => {
if (shouldReportError()) this.recordLoadSubsetError(options, error);
finish();
});
}
loadSubset(options, shouldReportError = () => true) {
try {
return this.collection._sync.loadSubset(options);
} catch (error) {
if (shouldReportError()) this.recordLoadSubsetError(options, error);
throw error;
}
}
/** Create a fresh, abortable adapter acquisition for a replay generation. */
createSubsetAcquisition(demand) {
const abortController = new AbortController();
const requestSignal = demand.requestOptions.signal;
let removeRequestAbortListener;
if (requestSignal?.aborted) {
abortController.abort(requestSignal.reason);
} else if (requestSignal) {
const abort = () => abortController.abort(requestSignal.reason);
requestSignal.addEventListener(`abort`, abort, { once: true });
removeRequestAbortListener = () => requestSignal.removeEventListener(`abort`, abort);
}
return {
options: {
...demand.requestOptions,
signal: abortController.signal
},
abortController,
removeRequestAbortListener
};
}
/** Replace the adapter lease held for one logical subset demand. */
replaceSubsetAcquisition(demand, next) {
const previousOptions = demand.options;
const removePreviousAbortListener = demand.removeRequestAbortListener;
this.collection._sync.unloadSubset(previousOptions);
removePreviousAbortListener?.();
demand.options = next.options;
demand.abortController = next.abortController;
demand.removeRequestAbortListener = next.removeRequestAbortListener;
}
/** Abort and release one current adapter acquisition. */
releaseSubsetDemand(demand) {
demand.abortController?.abort();
try {
this.collection._sync.unloadSubset(demand.options);
} finally {
demand.removeRequestAbortListener?.();
}
}
/** Start and retain the first acquisition for one logical subset demand. */
startSubsetDemand(requestOptions) {
const demand = {
requestOptions,
options: requestOptions
};
const acquisition = this.createSubsetAcquisition(demand);
try {
const result = this.loadSubset(acquisition.options);
demand.options = acquisition.options;
demand.abortController = acquisition.abortController;
demand.removeRequestAbortListener = acquisition.removeRequestAbortListener;
this.subsetDemands.push(demand);
return { demand, result };
} catch (error) {
acquisition.abortController.abort();
acquisition.removeRequestAbortListener?.();
throw error;
}
}
recordLoadSubsetError(options, error, reportAborted = false) {
if (options.signal?.aborted && !reportAborted) return;
this._lastError = error;
this.emitInner(`loadSubset:error`, {
type: `loadSubset:error`,
subscription: this,
options,
error
});
}
hasLoadedInitialState() {
return this.loadedInitialState;
}
hasSentAtLeastOneSnapshot() {
return this.snapshotSent;
}
emitEvents(changes) {
const newChanges = this.filterAndFlipChanges(changes);
if (changes.length > 0 && newChanges.length === 0) return false;
if (this.isBufferingForTruncate) {
if (newChanges.length > 0) {
this.truncateReplaySession.buffer.push(newChanges);
}
return false;
} else {
return this.filteredCallback(newChanges);
}
}
/**
* Sends the snapshot to the callback.
* Returns a boolean indicating if it succeeded.
* It can only fail if there is no index to fulfill the request
* and the optimizedOnly option is set to true,
* or, the entire state was already loaded.
*/
requestSnapshot(opts) {
if (this.loadedInitialState) {
return false;
}
const stateOpts = {
where: this.options.whereExpression,
optimizedOnly: opts?.optimizedOnly ?? false
};
if (opts) {
if (`where` in opts) {
const snapshotWhereExp = opts.where;
if (stateOpts.where) {
const subWhereExp = stateOpts.where;
const combinedWhereExp = functions.and(subWhereExp, snapshotWhereExp);
stateOpts.where = combinedWhereExp;
} else {
stateOpts.where = snapshotWhereExp;
}
}
} else {
this.loadedInitialState = true;
}
const loadOptions = {
where: stateOpts.where,
signal: opts?.signal,
subscription: this,
// Include orderBy and limit if provided so sync layer can optimize the query
orderBy: opts?.orderBy,
limit: opts?.limit
};
const { demand, result: syncResult } = this.startSubsetDemand(loadOptions);
if (opts?.where) this.requestedSubsetWhere.set(loadOptions, opts.where);
opts?.onLoadSubsetResult?.(syncResult);
this.observeLoadSubsetResult(
syncResult,
demand.options,
opts?.trackLoadSubsetPromise ?? true
);
let snapshot;
if (opts?.onUnoptimized) {
snapshot = this.collection.currentStateAsChanges({
...stateOpts,
optimizedOnly: true
});
if (snapshot === void 0) {
opts.onUnoptimized();
snapshot = this.collection.currentStateAsChanges({
...stateOpts,
optimizedOnly: false
});
}
} else {
snapshot = this.collection.currentStateAsChanges(stateOpts);
}
if (snapshot === void 0) {
return false;
}
const filteredSnapshot = snapshot.filter(
(change) => !this.sentKeys.has(change.key)
);
for (const change of filteredSnapshot) {
this.sentKeys.add(change.key);
}
this.snapshotSent = true;
this.callback(filteredSnapshot);
return true;
}
/** Release one exact subset request while keeping the subscription alive. */
releaseSnapshot(where) {
const index = this.subsetDemands.findIndex(
(demand2) => demand2.requestOptions.where === where || this.requestedSubsetWhere.get(demand2.requestOptions) === where
);
if (index === -1) return;
const [demand] = this.subsetDemands.splice(index, 1);
if (demand) this.releaseSubsetDemand(demand);
}
/**
* Sends a snapshot that fulfills the `where` clause and all rows are bigger or equal to the cursor.
* Requires a range index to be set with `setOrderByIndex` prior to calling this method.
* It uses that range index to load the items in the order of the index.
*
* For multi-column orderBy:
* - Uses first value from `minValues` for LOCAL index operations (wide bounds, ensures no missed rows)
* - Uses all `minValues` to build a precise composite cursor for SYNC layer loadSubset
*
* Note 1: it may load more rows than the provided LIMIT because it loads all values equal to the first cursor value + limit values greater.
* This is needed to ensure that it does not accidentally skip duplicate values when the limit falls in the middle of some duplicated values.
* Note 2: it does not send keys that have already been sent before.
*/
requestLimitedSnapshot({
orderBy,
limit,
minValues,
offset,
trackLoadSubsetPromise: shouldTrackLoadSubsetPromise = true,
onLoadSubsetResult
}) {
if (!limit) throw new Error(`limit is required`);
if (!this.orderByIndex) {
throw new Error(
`Ordered snapshot was requested but no index was found. You have to call setOrderByIndex before requesting an ordered snapshot.`
);
}
const hasMinValue = minValues !== void 0 && minValues.length > 0;
const minValue = minValues?.[0];
const minValueForIndex = minValue;
const index = this.orderByIndex;
const where = this.options.whereExpression;
const whereFilterFn = where ? changeEvents.createFilterFunctionFromExpression(where) : void 0;
const filterFn = (key) => {
if (key !== void 0 && this.sentKeys.has(key)) {
return false;
}
const value = this.collection.get(key);
if (value === void 0) {
return false;
}
return whereFilterFn?.(value) ?? true;
};
let biggestObservedValue = minValueForIndex;
const changes = [];
let keys = [];
if (hasMinValue) {
const { expression } = orderBy[0];
const allRowsWithMinValue = this.collection.currentStateAsChanges({
where: functions.eq(expression, new ir.Value(minValueForIndex))
});
if (allRowsWithMinValue) {
const keysWithMinValue = allRowsWithMinValue.map((change) => change.key).filter((key) => !this.sentKeys.has(key) && filterFn(key));
keys.push(...keysWithMinValue);
const keysGreaterThanMin = index.take(
limit - keys.length,
minValueForIndex,
filterFn
);
keys.push(...keysGreaterThanMin);
} else {
keys = index.take(limit, minValueForIndex, filterFn);
}
} else {
keys = index.takeFromStart(limit, filterFn);
}
const valuesNeeded = () => Math.max(limit - changes.length, 0);
const collectionExhausted = () => keys.length === 0;
const orderByExpression = orderBy[0].expression;
const valueExtractor = orderByExpression.type === `ref` ? evaluators.compileExpression(new ir.PropRef(orderByExpression.path), true) : null;
while (valuesNeeded() > 0 && !collectionExhausted()) {
const insertedKeys = /* @__PURE__ */ new Set();
for (const key of keys) {
const value = this.collection.get(key);
changes.push({
type: `insert`,
key,
value
});
biggestObservedValue = valueExtractor ? valueExtractor(value) : value;
insertedKeys.add(key);
}
keys = index.take(valuesNeeded(), biggestObservedValue, filterFn);
}
const currentOffset = this.limitedSnapshotRowCount;
for (const change of changes) {
this.sentKeys.add(change.key);
}
this.callback(changes);
this.limitedSnapshotRowCount = Math.max(
this.limitedSnapshotRowCount,
currentOffset + changes.length
);
if (changes.length > 0) {
this.lastSentKey = changes[changes.length - 1].key;
}
let cursorExpressions;
if (minValues !== void 0 && minValues.length > 0) {
const whereFromCursor = cursor.buildCursor(orderBy, minValues);
if (whereFromCursor) {
const { expression } = orderBy[0];
const cursorMinValue = minValues[0];
let whereCurrentCursor;
if (cursorMinValue instanceof Date) {
const cursorMinValuePlus1ms = new Date(cursorMinValue.getTime() + 1);
whereCurrentCursor = functions.and(
functions.gte(expression, new ir.Value(cursorMinValue)),
functions.lt(expression, new ir.Value(cursorMinValuePlus1ms))
);
} else {
whereCurrentCursor = functions.eq(expression, new ir.Value(cursorMinValue));
}
cursorExpressions = {
whereFrom: whereFromCursor,
whereCurrent: whereCurrentCursor,
lastKey: this.lastSentKey
};
}
}
const loadOptions = {
where,
// Main filter only, no cursor
limit,
orderBy,
cursor: cursorExpressions,
// Cursor expressions passed separately
offset: offset ?? currentOffset,
// Use provided offset, or auto-tracked offset
subscription: this
};
const { demand, result: syncResult } = this.startSubsetDemand(loadOptions);
onLoadSubsetResult?.(syncResult);
this.observeLoadSubsetResult(
syncResult,
demand.options,
shouldTrackLoadSubsetPromise
);
}
// TODO: also add similar test but that checks that it can also load it from the collection's loadSubset function
// and that that also works properly (i.e. does not skip duplicate values)
/**
* Filters and flips changes for keys that have not been sent yet.
* Deletes are filtered out for keys that have not been sent yet.
* Updates are flipped into inserts for keys that have not been sent yet.
* Duplicate inserts are filtered out to prevent D2 multiplicity > 1.
*/
filterAndFlipChanges(changes) {
changes = this.reconcileStalePublishedChanges(changes);
if (this.loadedInitialState || this.skipFiltering) {
return changes;
}
const skipDeleteFilter = this.isBufferingForTruncate;
const newChanges = [];
for (const change of changes) {
let newChange = change;
const keyInSentKeys = this.sentKeys.has(change.key);
if (!keyInSentKeys) {
if (change.type === `update`) {
newChange = { ...change, type: `insert`, previousValue: void 0 };
this.sentKeys.add(change.key);
} else if (change.type === `delete`) {
if (!skipDeleteFilter) {
continue;
}
} else {
this.sentKeys.add(change.key);
}
} else {
if (change.type === `insert`) {
continue;
} else if (change.type === `delete`) {
this.sentKeys.delete(change.key);
}
}
newChanges.push(newChange);
}
return newChanges;
}
/**
* After a failed replay, the source collection is empty but subscribers still
* hold the last good publication. Reconcile the first later source delta for
* each retained key against that publication instead of treating it as a
* duplicate insert.
*/
reconcileStalePublishedChanges(changes) {
if (this.stalePublishedRows.size === 0) return changes;
const reconciled = [];
for (const change of changes) {
const previous = this.stalePublishedRows.get(change.key);
if (previous === void 0) {
reconciled.push(change);
continue;
}
this.stalePublishedRows.delete(change.key);
if (change.type === `delete`) {
reconciled.push({
...change,
value: previous,
previousValue: void 0
});
} else if (!utils.deepEquals(previous, change.value)) {
reconciled.push({
...change,
type: `update`,
previousValue: previous
});
}
}
return reconciled;
}
trackPublishedRows(changes) {
for (const change of changes) {
if (change.type === `delete`) {
this.publishedRows.delete(change.key);
} else {
this.publishedRows.set(change.key, change.value);
}
}
}
trackSentKeys(changes) {
if (this.loadedInitialState || this.skipFiltering) {
return;
}
for (const change of changes) {
if (change.type === `delete`) {
this.sentKeys.delete(change.key);
} else {
this.sentKeys.add(change.key);
}
}
if (this.orderByIndex) {
this.limitedSnapshotRowCount = Math.max(
this.limitedSnapshotRowCount,
this.sentKeys.size
);
}
}
/**
* Mark that the subscription should not filter any changes.
* This is used when includeInitialState is explicitly set to false,
* meaning the caller doesn't want initial state but does want ALL future changes.
*/
markAllStateAsSeen() {
this.skipFiltering = true;
}
unsubscribe() {
let firstCleanupError;
try {
this.truncateCleanup?.();
} catch (error) {
firstCleanupError = error;
}
this.truncateCleanup = void 0;
this.truncateReplaySession = void 0;
this.stalePublishedRows.clear();
for (const demand of this.subsetDemands) {
try {
this.releaseSubsetDemand(demand);
} catch (error) {
firstCleanupError ??= error;
}
}
this.subsetDemands = [];
try {
this.emitInner(`unsubscribed`, {
type: `unsubscribed`,
subscription: this
});
} catch (error) {
firstCleanupError ??= error;
} finally {
this.clearListeners();
}
if (firstCleanupError !== void 0) throw firstCleanupError;
}
}
exports.CollectionSubscription = CollectionSubscription;
//# sourceMappingURL=subscription.cjs.map