donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
158 lines • 6.35 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.createCompositePageToken = createCompositePageToken;
exports.parseCompositePageToken = parseCompositePageToken;
exports.federatedList = federatedList;
function createCompositePageToken(state) {
return Buffer.from(JSON.stringify(state)).toString('base64');
}
function parseCompositePageToken(token) {
if (!token) {
return { sources: [] };
}
try {
const decoded = JSON.parse(Buffer.from(token, 'base64').toString());
if (decoded && Array.isArray(decoded.sources)) {
return decoded;
}
}
catch {
// fall through
}
return { sources: [] };
}
/**
* Federate a paginated listing across multiple persistence layers
* (resume-by-id).
*
* Each layer holds an opaque cursor and an optional `resumeAfterId`. On
* each page request, federation fetches from each non-exhausted layer
* (advancing through batches as needed until it has at least `limit`
* candidates per layer), stable-sorts the merged contributions with the
* caller's `comparator`, and trims to `limit`. For each layer, the
* **last kept item in that layer's order** becomes the new
* `resumeAfterId`, with `cursor` set to the layer-token used to fetch
* that item's batch. Layers whose contribution is wholly dropped by the
* trim retain their pre-call state so their items remain reachable on
* later pages.
*
* **Critical contract: the comparator must NOT tiebreak by id.** Ties
* on the sort key must return 0 from `comparator`. `Array.sort` is
* stable since ES2019, and federation feeds the merge in
* `[layer0..., layer1..., ...]` insertion order, so ties resolve by
* `(layerIndex, layer-internal position)` — exactly the property that
* keeps each layer's kept items as a *prefix* of its own order, which
* is what makes a single `resumeAfterId` representable. An id
* tiebreaker would silently break this and lose items.
*
* **Layer contract.** Layers must return the same items in the same
* order for the same `(query, pageToken)` pair across calls. Their
* internal ordering does not need to match federation's tiebreaker —
* just be deterministic.
*/
async function federatedList(layers, query, comparator) {
const requestedLimit = Math.min(Math.max(1, query.limit ?? 100), 100);
const incomingState = parseCompositePageToken(query.pageToken);
const initialFor = (i) => {
const s = incomingState.sources[i];
return s ? { ...s } : { exhausted: false };
};
const work = [];
for (let i = 0; i < layers.length; i++) {
const init = initialFor(i);
const w = { contribution: [], layerExhausted: false };
work.push(w);
if (init.exhausted) {
w.layerExhausted = true;
continue;
}
let cursor = init.cursor;
let resume = init.resumeAfterId;
while (w.contribution.length < requestedLimit) {
const layerQuery = {
...query,
limit: requestedLimit,
pageToken: cursor,
};
const batch = await layers[i].getItems(layerQuery);
let items = batch.items;
if (resume !== undefined) {
const idx = items.findIndex((it) => it.id === resume);
if (idx >= 0) {
items = items.slice(idx + 1);
}
// The skip is one-shot: clear `resume` regardless of whether we
// found it. If found, we've passed it; if not, it was in a
// previously-fetched batch and won't reappear in subsequent
// batches we fetch in this loop.
resume = undefined;
}
for (const it of items) {
w.contribution.push({
item: it,
layerIndex: i,
fetchCursor: cursor,
});
}
if (!batch.nextPageToken) {
w.layerExhausted = true;
break;
}
cursor = batch.nextPageToken;
}
}
// Stable sort: ties resolve by insertion order = (layerIndex, layer
// position). Do NOT add an id tiebreaker (see header comment).
const merged = [];
for (const w of work) {
merged.push(...w.contribution);
}
merged.sort((a, b) => comparator(a.item, b.item));
const page = merged.slice(0, requestedLimit);
const newSources = [];
for (let i = 0; i < layers.length; i++) {
const init = initialFor(i);
if (init.exhausted) {
newSources.push({ exhausted: true });
continue;
}
const keptFromLayer = page.filter((w) => w.layerIndex === i);
if (keptFromLayer.length === 0) {
if (work[i].contribution.length === 0 && work[i].layerExhausted) {
// Layer truly has nothing — empty stream. Mark exhausted so
// we don't re-query it forever.
newSources.push({ exhausted: true });
}
else {
// Layer's items were all outranked by other layers. Roll back
// to the incoming state so they remain reachable on later
// pages, when the outranking layers have moved on.
newSources.push(init);
}
continue;
}
const lastKept = keptFromLayer[keptFromLayer.length - 1];
const allFromLayer = work[i].contribution;
const allKept = allFromLayer.length === keptFromLayer.length;
if (work[i].layerExhausted && allKept) {
newSources.push({ exhausted: true });
}
else {
newSources.push({
cursor: lastKept.fetchCursor,
resumeAfterId: lastKept.item.id,
exhausted: false,
});
}
}
const someUnconsumed = merged.length > requestedLimit;
const someNotExhausted = newSources.some((s) => !s.exhausted);
const hasMore = someUnconsumed || someNotExhausted;
return {
items: page.map((w) => w.item),
nextPageToken: hasMore
? createCompositePageToken({ sources: newSources })
: undefined,
};
}
//# sourceMappingURL=FederatedPagination.js.map