@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
366 lines (344 loc) • 15.5 kB
text/typescript
import type { AdapterEvent } from "../types";
type QueueReader = (result: IteratorResult<AdapterEvent>) => void;
export const PREFLIGHT_HEARTBEAT_RETAIN_LIMIT = 16;
/**
* Coalescing threshold for adjacent text/thinking deltas buffered with no
* waiting reader (UTF-16 code units). This is a merge-size ceiling; the two
* retention budgets below are what bound memory.
*/
export const COALESCE_MAX_CHUNK_LENGTH = 64 * 1024;
/**
* Retained-string budget for the WHOLE queue, in UTF-16 code units.
*
* This bounds what the queue is holding at one moment, not how much a turn
* streams: every dequeue gives its charge back, so a long healthy stream with a
* consumer attached never accumulates and is never capped by total length.
*
* 32 MiB is deliberately far above any single legitimate burst. A synchronous
* producer can legally fill the queue before its consumer is scheduled — the
* image loop does exactly that with over a million one-character deltas, which
* coalesce into roughly 1.2 MB of retained text — so a budget near that size
* aborts healthy turns rather than stalled ones. The previous effective bound
* was the 1024-event cap times the 64 KiB merge ceiling, so 64 MiB; this halves
* it while leaving that legitimate burst an order of magnitude of headroom.
*/
export const DEFAULT_MAX_BACKLOG_CODE_UNITS = 32 * 1024 * 1024;
/**
* Retained-string budget for ONE queued event, in UTF-16 code units.
*
* Separate from the aggregate on purpose, because the two describe different
* failures. Passing the aggregate means the consumer is not keeping up. Passing
* this one means a single event is malformed or unbounded, which stays true
* however empty the queue is, so it must be refused even with the whole
* aggregate free. They also report different terminal messages, so an operator
* reading the turn's error learns which happened.
*
* Like the aggregate, this governs what the queue RETAINS. An event handed
* straight to a waiting consumer is never held here, so neither budget applies
* to it: refusing it would abort a turn over memory this queue does not own,
* and the consumer's own per-event bound governs that payload instead.
*/
export const DEFAULT_MAX_EVENT_CODE_UNITS = 8 * 1024 * 1024;
const BACKLOG_EXCEEDED_MESSAGE = "consumer stalled: adapter event backlog exceeded — turn aborted";
const EVENT_TOO_LARGE_MESSAGE = "adapter event exceeds the single-event retained-string budget — turn aborted";
/**
* Bound on how far the retention measure walks into one event. AdapterEvent is
* a plain-data union, but two of its members carry open provider-shaped bags
* (`providerState`, `usage.rawUsage`) whose depth no type here controls. The
* ceilings keep a single push O(1)-ish rather than O(whatever an adapter
* attached), and under-counting a pathological object is the safe direction:
* the event-count cap still bounds how many of them can be retained.
*/
const RETENTION_MAX_DEPTH = 8;
const RETENTION_MAX_NODES = 4096;
/**
* Retained UTF-16 code units carried by one event's string payload.
*
* Measured by walking own enumerable properties rather than by naming each
* variant's string fields: a hand-written per-variant table is exhaustive over
* a union, so adding an event type on one branch while a consumer lands on
* another produces a measure that silently stops counting the new payload.
* The walk is the derived answer and needs no update when the union grows.
*
* `type` is skipped because it is the discriminant, identical for every event
* of a kind and not payload anyone is buffering.
*/
export function retainedEventCodeUnits(event: AdapterEvent): number {
// Defensive: this measures values an adapter produced. A malformed emission
// has to become a terminal event, not a TypeError thrown out of push() with
// the queue half-updated.
if (!event || typeof event !== "object") return 0;
let total = 0;
let nodes = 0;
const seen = new Set<object>();
const visit = (value: unknown, depth: number): void => {
if (typeof value === "string") {
total += value.length;
return;
}
if (!value || typeof value !== "object" || depth >= RETENTION_MAX_DEPTH || seen.has(value)) return;
seen.add(value);
for (const nested of Object.values(value)) {
if (nodes++ >= RETENTION_MAX_NODES) return;
visit(nested, depth + 1);
}
};
for (const [key, value] of Object.entries(event)) {
if (key === "type") continue;
if (nodes++ >= RETENTION_MAX_NODES) break;
visit(value, 1);
}
return total;
}
function positiveBudget(value: number | undefined, fallback: number, name: string): number {
const resolved = value ?? fallback;
if (!Number.isSafeInteger(resolved) || resolved <= 0) {
throw new RangeError(`${name} must be a positive safe integer`);
}
return resolved;
}
export interface AdapterEventQueue {
/**
* Returns true when the event was merged into the buffered tail instead of
* becoming its own retained item. A caller that charges a memory budget for
* what the queue holds needs that distinction: a merged delta costs only its
* appended payload, while a new item costs a whole serialized event.
*/
push(event: AdapterEvent): boolean;
close(): void;
stream(): AsyncIterable<AdapterEvent>;
collect(): Promise<AdapterEvent[]>;
/**
* Retained string payload the queue is currently holding, in UTF-16 code
* units. Exposed so a caller — and a regression — can assert the counter
* returns to zero on every terminal path instead of inferring it from an
* abort that happened to fire.
*/
retainedCodeUnits(): number;
}
export interface AdapterEventPreflight {
stream: AsyncIterable<AdapterEvent>;
error?: Extract<AdapterEvent, { type: "error" }>;
empty: boolean;
replayUnsafe: boolean;
ready?: boolean;
timedOut?: boolean;
}
async function* replay(
buffered: readonly AdapterEvent[],
iterator: AsyncIterator<AdapterEvent>,
pendingNext?: Promise<IteratorResult<AdapterEvent>>,
): AsyncGenerator<AdapterEvent> {
try {
for (const event of buffered) yield event;
if (pendingNext) {
const next = await pendingNext;
if (next.done) return;
yield next.value;
}
while (true) {
const next = await iterator.next();
if (next.done) return;
yield next.value;
}
} finally {
await iterator.return?.();
}
}
export async function preflightAdapterEvents(
source: AsyncIterable<AdapterEvent>,
classifyFirstEvent?: (event: AdapterEvent) => Extract<AdapterEvent, { type: "error" }> | undefined,
options?: { maxWaitMs?: number; honorReady?: boolean },
): Promise<AdapterEventPreflight> {
const iterator = source[Symbol.asyncIterator]();
const buffered: AdapterEvent[] = [];
let replayUnsafe = false;
const maxWaitMs = options?.maxWaitMs;
if (maxWaitMs !== undefined && maxWaitMs <= 0) {
return { stream: replay(buffered, iterator), empty: false, replayUnsafe, timedOut: true };
}
let timeout: ReturnType<typeof setTimeout> | undefined;
let timeoutPromise: Promise<"timeout"> | undefined;
if (maxWaitMs !== undefined) {
timeoutPromise = new Promise(resolve => {
timeout = setTimeout(() => resolve("timeout"), maxWaitMs);
});
}
try {
while (true) {
const pendingNext = iterator.next();
const raced = timeoutPromise ? await Promise.race([pendingNext, timeoutPromise]) : await pendingNext;
if (raced === "timeout") {
return {
stream: replay(buffered, iterator, pendingNext),
empty: false,
replayUnsafe,
timedOut: true,
};
}
const next = raced;
if (next.done) return { stream: replay(buffered, iterator), empty: true, replayUnsafe };
if (next.value.type === "heartbeat") {
replayUnsafe ||= next.value.replayUnsafe === true;
// Preserve the latch in replay even after the original unsafe heartbeat is evicted.
buffered.push(replayUnsafe ? { ...next.value, replayUnsafe: true } : next.value);
if (next.value.preflightReady === true && options?.honorReady !== false) {
return { stream: replay(buffered, iterator), empty: false, replayUnsafe, ready: true };
}
if (buffered.length > PREFLIGHT_HEARTBEAT_RETAIN_LIMIT) buffered.shift();
continue;
}
const classifiedError = replayUnsafe ? undefined : classifyFirstEvent?.(next.value);
if (classifiedError) {
buffered.push(classifiedError);
await iterator.return?.();
return { stream: replay(buffered, iterator), error: classifiedError, empty: false, replayUnsafe };
}
buffered.push(next.value);
if (next.value.type === "error") {
await iterator.return?.();
return { stream: replay(buffered, iterator), error: next.value, empty: false, replayUnsafe };
}
return { stream: replay(buffered, iterator), empty: false, replayUnsafe };
}
} finally {
if (timeout !== undefined) clearTimeout(timeout);
}
}
export function createAdapterEventQueue(opts?: {
maxBacklog?: number;
maxBacklogCodeUnits?: number;
maxEventCodeUnits?: number;
onBacklogExceeded?: () => void;
}): AdapterEventQueue {
const queued: AdapterEvent[] = [];
/**
* What each queued item was charged, in lockstep with `queued`. Releasing the
* recorded charge rather than re-measuring is what makes the accounting exact
* on every path: a merge, a terminal record admitted past the budget and a
* plain event all give back precisely what they took, so the counter cannot
* drift positive (a leak) or negative (a budget the next turn gets for free).
*/
const charged: number[] = [];
const readers: QueueReader[] = [];
const maxBacklog = opts?.maxBacklog ?? 1_024;
const maxBacklogCodeUnits = positiveBudget(opts?.maxBacklogCodeUnits, DEFAULT_MAX_BACKLOG_CODE_UNITS, "maxBacklogCodeUnits");
const maxEventCodeUnits = positiveBudget(opts?.maxEventCodeUnits, DEFAULT_MAX_EVENT_CODE_UNITS, "maxEventCodeUnits");
let retained = 0;
let closed = false;
// Merge an incoming delta into the buffered tail when no reader is waiting.
// The event cap counts events, not tokens, so a detached or briefly stalled
// consumer (e.g. a Codex app mid-reconnect whose disconnect Bun has not yet
// delivered) used to hit it within seconds of token-granular streaming and
// abort a healthy turn. Adjacent same-phase text deltas, adjacent thinking
// deltas, and consecutive heartbeats carry no ordering information between
// themselves, so merging them preserves every consumer contract while making
// the cap approximate buffered items again.
// Pushed objects may be retained by adapters, so the tail is REPLACED with
// a fresh object — never mutated (alias safety). Returning the replacement
// instead of installing it lets push price the merge before committing to it.
const planTailMerge = (tail: AdapterEvent, event: AdapterEvent): AdapterEvent | null => {
if (event.type === "heartbeat") {
if (tail.type !== "heartbeat") return null;
// Heartbeats carry no ordering between themselves, but the replay-unsafe
// marker is not ordering — it is a latch. Dropping the incoming event
// would discard the only record that Cursor already performed a local
// side effect, and preflight would then permit an OAuth replay of it.
if (event.replayUnsafe !== true && event.preflightReady !== true) return tail;
return {
type: "heartbeat",
...(tail.replayUnsafe === true || event.replayUnsafe === true ? { replayUnsafe: true as const } : {}),
...(tail.preflightReady === true || event.preflightReady === true ? { preflightReady: true as const } : {}),
};
}
if (event.type === "text_delta" && tail.type === "text_delta" && tail.phase === event.phase) {
if (tail.text.length + event.text.length > COALESCE_MAX_CHUNK_LENGTH) return null;
return { type: "text_delta", text: tail.text + event.text, phase: tail.phase };
}
if (event.type === "thinking_delta" && tail.type === "thinking_delta") {
if (tail.thinking.length + event.thinking.length > COALESCE_MAX_CHUNK_LENGTH) return null;
return { type: "thinking_delta", thinking: tail.thinking + event.thinking };
}
return null;
};
/**
* Record why the turn is ending and close. The terminal error is admitted
* past both budgets — refusing to retain the explanation of a refusal would
* leave the consumer with a silent truncation — but it is charged like any
* other item so the counter stays exact through the final drain.
*/
const abortWith = (message: string): false => {
opts?.onBacklogExceeded?.();
const terminal: AdapterEvent = { type: "error", message };
const cost = retainedEventCodeUnits(terminal);
queued.push(terminal);
charged.push(cost);
retained += cost;
close();
return false;
};
const push = (event: AdapterEvent): boolean => {
if (closed) return false;
const reader = readers.shift();
if (reader) {
// Handed straight to a waiting consumer, so the queue retains nothing and
// charges nothing. Neither budget applies to an event it never holds.
reader({ done: false, value: event });
return false;
}
const tail = queued[queued.length - 1];
const merged = tail ? planTailMerge(tail, event) : null;
if (merged && tail) {
const replacement = retainedEventCodeUnits(merged);
if (replacement > maxEventCodeUnits) return abortWith(EVENT_TOO_LARGE_MESSAGE);
// Charge only what the backlog actually gains. A merge keeps the tail's
// own fields, so the incoming event's duplicated phase is never retained
// twice and an unchanged tail costs nothing at all.
const delta = replacement - charged[charged.length - 1]!;
if (delta > maxBacklogCodeUnits - retained) return abortWith(BACKLOG_EXCEEDED_MESSAGE);
queued[queued.length - 1] = merged;
charged[charged.length - 1] = replacement;
retained += delta;
return true;
}
const cost = retainedEventCodeUnits(event);
if (cost > maxEventCodeUnits) return abortWith(EVENT_TOO_LARGE_MESSAGE);
// Both refusals are priced before anything is retained, so an event that is
// turned away is never charged for.
if (queued.length >= maxBacklog) return abortWith(BACKLOG_EXCEEDED_MESSAGE);
if (cost > maxBacklogCodeUnits - retained) return abortWith(BACKLOG_EXCEEDED_MESSAGE);
queued.push(event);
charged.push(cost);
retained += cost;
return false;
};
const close = (): void => {
if (closed) return;
closed = true;
while (readers.length > 0) {
readers.shift()?.({ done: true, value: undefined as never });
}
};
async function* stream(): AsyncIterable<AdapterEvent> {
while (true) {
const next = queued.shift();
if (next) {
retained -= charged.shift() ?? 0;
yield next;
continue;
}
if (closed) return;
const result = await new Promise<IteratorResult<AdapterEvent>>(resolve => {
readers.push(resolve);
});
if (result.done) return;
yield result.value;
}
}
const collect = async (): Promise<AdapterEvent[]> => {
const events: AdapterEvent[] = [];
for await (const event of stream()) events.push(event);
return events;
};
return { push, close, stream, collect, retainedCodeUnits: () => retained };
}