@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
500 lines (472 loc) • 21.6 kB
text/typescript
import type { AdapterEvent } from "../types";
import { truncationReasonFor } from "../responses/truncated-stop-reason";
type QueueReader = (result: IteratorResult<AdapterEvent>) => void;
export const PREFLIGHT_HEARTBEAT_RETAIN_LIMIT = 16;
/**
* Coalescing threshold for adjacent text/thinking deltas buffered with no
* waiting reader (UTF-16 code units). This is a merge-size ceiling; the two
* retention budgets below are what bound memory.
*/
export const COALESCE_MAX_CHUNK_LENGTH = 64 * 1024;
/**
* Retained-string budget for the WHOLE queue, in UTF-16 code units.
*
* This bounds what the queue is holding at one moment, not how much a turn
* streams: every dequeue gives its charge back, so a long healthy stream with a
* consumer attached never accumulates and is never capped by total length.
*
* 32 MiB is deliberately far above any single legitimate burst. A synchronous
* producer can legally fill the queue before its consumer is scheduled — the
* image loop does exactly that with over a million one-character deltas, which
* coalesce into roughly 1.2 MB of retained text — so a budget near that size
* aborts healthy turns rather than stalled ones. The previous effective bound
* was the 1024-event cap times the 64 KiB merge ceiling, so 64 MiB; this halves
* it while leaving that legitimate burst an order of magnitude of headroom.
*/
export const DEFAULT_MAX_BACKLOG_CODE_UNITS = 32 * 1024 * 1024;
/**
* Retained-string budget for ONE queued event, in UTF-16 code units.
*
* Separate from the aggregate on purpose, because the two describe different
* failures. Passing the aggregate means the consumer is not keeping up. Passing
* this one means a single event is malformed or unbounded, which stays true
* however empty the queue is, so it must be refused even with the whole
* aggregate free. They also report different terminal messages, so an operator
* reading the turn's error learns which happened.
*
* Like the aggregate, this governs what the queue RETAINS. An event handed
* straight to a waiting consumer is never held here, so neither budget applies
* to it: refusing it would abort a turn over memory this queue does not own,
* and the consumer's own per-event bound governs that payload instead.
*/
export const DEFAULT_MAX_EVENT_CODE_UNITS = 8 * 1024 * 1024;
const BACKLOG_EXCEEDED_MESSAGE = "consumer stalled: adapter event backlog exceeded — turn aborted";
const EVENT_TOO_LARGE_MESSAGE = "adapter event exceeds the single-event retained-string budget — turn aborted";
/**
* Bound on how far the retention measure walks into one event. AdapterEvent is
* a plain-data union, but two of its members carry open provider-shaped bags
* (`providerState`, `usage.rawUsage`) whose depth no type here controls. The
* ceilings keep a single push O(1)-ish rather than O(whatever an adapter
* attached), and under-counting a pathological object is the safe direction:
* the event-count cap still bounds how many of them can be retained.
*/
const RETENTION_MAX_DEPTH = 8;
const RETENTION_MAX_NODES = 4096;
/**
* Retained UTF-16 code units carried by one event's string payload.
*
* Measured by walking own enumerable properties rather than by naming each
* variant's string fields: a hand-written per-variant table is exhaustive over
* a union, so adding an event type on one branch while a consumer lands on
* another produces a measure that silently stops counting the new payload.
* The walk is the derived answer and needs no update when the union grows.
*
* `type` is skipped because it is the discriminant, identical for every event
* of a kind and not payload anyone is buffering.
*/
export function retainedEventCodeUnits(event: AdapterEvent): number {
// Defensive: this measures values an adapter produced. A malformed emission
// has to become a terminal event, not a TypeError thrown out of push() with
// the queue half-updated.
if (!event || typeof event !== "object") return 0;
let total = 0;
let nodes = 0;
const seen = new Set<object>();
const visit = (value: unknown, depth: number): void => {
if (typeof value === "string") {
total += value.length;
return;
}
if (!value || typeof value !== "object" || depth >= RETENTION_MAX_DEPTH || seen.has(value)) return;
seen.add(value);
for (const nested of Object.values(value)) {
if (nodes++ >= RETENTION_MAX_NODES) return;
visit(nested, depth + 1);
}
};
for (const [key, value] of Object.entries(event)) {
if (key === "type") continue;
if (nodes++ >= RETENTION_MAX_NODES) break;
visit(value, 1);
}
return total;
}
function positiveBudget(value: number | undefined, fallback: number, name: string): number {
const resolved = value ?? fallback;
if (!Number.isSafeInteger(resolved) || resolved <= 0) {
throw new RangeError(`${name} must be a positive safe integer`);
}
return resolved;
}
export interface AdapterEventQueue {
/**
* Returns true when the event was merged into the buffered tail instead of
* becoming its own retained item. A caller that charges a memory budget for
* what the queue holds needs that distinction: a merged delta costs only its
* appended payload, while a new item costs a whole serialized event.
*/
push(event: AdapterEvent): boolean;
close(): void;
stream(): AsyncIterable<AdapterEvent>;
collect(): Promise<AdapterEvent[]>;
/**
* Retained string payload the queue is currently holding, in UTF-16 code
* units. Exposed so a caller — and a regression — can assert the counter
* returns to zero on every terminal path instead of inferring it from an
* abort that happened to fire.
*/
retainedCodeUnits(): number;
}
export interface AdapterEventPreflight {
stream: AsyncIterable<AdapterEvent>;
error?: Extract<AdapterEvent, { type: "error" }>;
empty: boolean;
replayUnsafe: boolean;
ready?: boolean;
timedOut?: boolean;
}
async function* replay(
buffered: readonly AdapterEvent[],
iterator: AsyncIterator<AdapterEvent>,
pendingNext?: Promise<IteratorResult<AdapterEvent>>,
): AsyncGenerator<AdapterEvent> {
try {
for (const event of buffered) yield event;
if (pendingNext) {
const next = await pendingNext;
if (next.done) return;
yield next.value;
}
while (true) {
const next = await iterator.next();
if (next.done) return;
yield next.value;
}
} finally {
await iterator.return?.();
}
}
export const DEFAULT_MAX_REASONING_EVENTS = 65_536;
/**
* Required-tool reasoning and zero-output token-limit preflight may hold buffered reasoning,
* but never for an unbounded/trickling prelude. Bound the reasoning hold to commit and replay.
*/
export const DEFAULT_MAX_REASONING_HOLD_MS = 1_000;
/**
* Classifies an adapter terminal event that reached an output token limit before producing output.
*
* When an upstream candidate exhausts its max output token budget without emitting any text or tool
* calls, this classifies the event as an HTTP 502 `zero_output_token_limit` error so combo failover
* can attempt a backup target. Candidate cooling is request-local (`comboFailureCooldownScope` returns
* "none") because the failure is specific to the requested token limit rather than provider unhealthiness.
*/
export function classifyZeroOutputTokenLimit(
event: AdapterEvent,
options?: { allowCompactionWithContent?: boolean },
): Extract<AdapterEvent, { type: "error" }> | undefined {
const zeroOutputLimit = event.type === "incomplete"
? event.reason === "max_output_tokens" && event.retryable !== false && event.endTurn !== false && !event.providerState
: event.type === "done"
&& event.stopReason?.trim().toLowerCase() !== "pause_turn"
&& (["max_tokens", "length", "max_output_tokens"].includes(event.stopReason ?? "") || truncationReasonFor(event.stopReason) === "max_output_tokens")
&& event.endTurn !== false && !event.providerState && (options?.allowCompactionWithContent || !event.compactionEncryptedContent);
if (!zeroOutputLimit) return undefined;
// With bufferCompaction, a compaction summary may have been produced in-memory without streaming.
// When that run terminates on max_tokens, classify it under the same code so failover can advance,
// since zero output was committed/streamed to the client.
const message = options?.allowCompactionWithContent
? "Remote compaction reached output token limit before completing"
: "Provider reached its output token limit before producing output";
return {
type: "error", status: 502, errorType: "upstream_error", code: "zero_output_token_limit",
message,
...((event.type === "done" || event.type === "incomplete") && event.usage ? { usage: event.usage } : {}),
};
}
export function isReasoningEvent(event: AdapterEvent): boolean {
return event.type === "thinking_delta"
|| event.type === "thinking_signature"
|| event.type === "redacted_thinking"
|| event.type === "kiro_redacted_reasoning"
|| event.type === "reasoning_raw_delta";
}
export function isContentEvent(event: AdapterEvent): boolean {
switch (event.type) {
case "text_delta":
return event.text.length > 0;
case "tool_call_start":
case "tool_call_delta":
case "tool_call_end":
case "web_search_call_begin":
case "web_search_call_end":
return true;
default:
return false;
}
}
export async function preflightAdapterEvents(
source: AsyncIterable<AdapterEvent>,
classifyFirstEvent?: (event: AdapterEvent) => Extract<AdapterEvent, { type: "error" }> | undefined,
options?: {
maxWaitMs?: number;
honorReady?: boolean;
bufferCompaction?: boolean;
maxReasoningHoldMs?: number;
},
): Promise<AdapterEventPreflight> {
const iterator = source[Symbol.asyncIterator]();
const buffered: AdapterEvent[] = [];
let replayUnsafe = false;
let bufferedReasoningCodeUnits = 0;
const maxWaitMs = options?.maxWaitMs;
if (maxWaitMs !== undefined && maxWaitMs <= 0) {
return { stream: replay(buffered, iterator), empty: false, replayUnsafe, timedOut: true };
}
let timeout: ReturnType<typeof setTimeout> | undefined;
let timeoutPromise: Promise<"timeout"> | undefined;
let retainedHeartbeats = 0;
if (maxWaitMs !== undefined) {
timeoutPromise = new Promise(resolve => {
timeout = setTimeout(() => resolve("timeout"), maxWaitMs);
});
}
const maxReasoningHoldMs = options?.maxReasoningHoldMs !== undefined
? options.maxReasoningHoldMs
: DEFAULT_MAX_REASONING_HOLD_MS;
let reasoningTimeout: ReturnType<typeof setTimeout> | undefined;
let reasoningPromise: Promise<"reasoning-hold-expired"> | undefined;
try {
while (true) {
const pendingNext = iterator.next();
const raceCandidates: Promise<IteratorResult<AdapterEvent> | "timeout" | "reasoning-hold-expired">[] = [pendingNext];
if (timeoutPromise) raceCandidates.push(timeoutPromise);
if (reasoningPromise) raceCandidates.push(reasoningPromise);
const raced = await Promise.race(raceCandidates);
if (raced === "timeout") {
return {
stream: replay(buffered, iterator, pendingNext),
empty: false,
replayUnsafe,
timedOut: true,
};
}
if (raced === "reasoning-hold-expired") {
return {
stream: replay(buffered, iterator, pendingNext),
empty: false,
replayUnsafe,
};
}
const next = raced;
if (next.done) {
const hasMeaningful = buffered.some(event => event.type !== "heartbeat");
return { stream: replay(buffered, iterator), empty: !hasMeaningful, replayUnsafe };
}
if (next.value.type === "heartbeat") {
const wasUnsafe = replayUnsafe;
replayUnsafe ||= next.value.replayUnsafe === true;
// Preserve the latch in replay even after the original unsafe heartbeat is evicted.
buffered.push(replayUnsafe ? { ...next.value, replayUnsafe: true } : next.value);
retainedHeartbeats += 1;
if (replayUnsafe && !wasUnsafe) {
for (let i = 0; i < buffered.length; i++) {
const ev = buffered[i];
if (ev.type === "heartbeat" && !ev.replayUnsafe) {
buffered[i] = { ...ev, replayUnsafe: true };
}
}
}
if (next.value.preflightReady === true && options?.honorReady !== false) {
return { stream: replay(buffered, iterator), empty: false, replayUnsafe, ready: true };
}
if (retainedHeartbeats > PREFLIGHT_HEARTBEAT_RETAIN_LIMIT) {
const firstHeartbeatIndex = buffered.findIndex(e => e.type === "heartbeat");
if (firstHeartbeatIndex !== -1) {
buffered.splice(firstHeartbeatIndex, 1);
retainedHeartbeats -= 1;
}
}
continue;
}
const shouldBuffer = classifyFirstEvent !== undefined || options?.bufferCompaction === true;
const emptyDelta = next.value.type === "text_delta" && next.value.text.length === 0;
const isBufferingEvent = isReasoningEvent(next.value)
|| emptyDelta
|| (options?.bufferCompaction === true && next.value.type === "text_delta");
if (isBufferingEvent) {
if (!shouldBuffer || maxReasoningHoldMs <= 0) {
buffered.push(next.value);
return { stream: replay(buffered, iterator), empty: false, replayUnsafe };
}
if (!reasoningPromise && maxReasoningHoldMs > 0) {
reasoningPromise = new Promise(resolve => {
reasoningTimeout = setTimeout(() => resolve("reasoning-hold-expired"), maxReasoningHoldMs);
});
}
buffered.push(next.value);
bufferedReasoningCodeUnits += retainedEventCodeUnits(next.value);
if (buffered.length > DEFAULT_MAX_REASONING_EVENTS || bufferedReasoningCodeUnits > DEFAULT_MAX_EVENT_CODE_UNITS) {
return { stream: replay(buffered, iterator), empty: false, replayUnsafe };
}
continue;
}
const hasPriorMeaningful = buffered.some(e => e.type !== "heartbeat");
const classifiedError = replayUnsafe
? undefined
: hasPriorMeaningful
? (next.value.type === "done" || next.value.type === "incomplete" ? classifyFirstEvent?.(next.value) : undefined)
: classifyFirstEvent?.(next.value);
if (classifiedError) {
buffered.push(classifiedError);
await iterator.return?.();
return { stream: replay(buffered, iterator), error: classifiedError, empty: false, replayUnsafe };
}
buffered.push(next.value);
if (next.value.type === "error") {
await iterator.return?.();
return { stream: replay(buffered, iterator), error: next.value, empty: false, replayUnsafe };
}
return { stream: replay(buffered, iterator), empty: false, replayUnsafe };
}
} finally {
if (timeout !== undefined) clearTimeout(timeout);
if (reasoningTimeout !== undefined) clearTimeout(reasoningTimeout);
}
}
export function createAdapterEventQueue(opts?: {
maxBacklog?: number;
maxBacklogCodeUnits?: number;
maxEventCodeUnits?: number;
onBacklogExceeded?: () => void;
}): AdapterEventQueue {
const queued: AdapterEvent[] = [];
/**
* What each queued item was charged, in lockstep with `queued`. Releasing the
* recorded charge rather than re-measuring is what makes the accounting exact
* on every path: a merge, a terminal record admitted past the budget and a
* plain event all give back precisely what they took, so the counter cannot
* drift positive (a leak) or negative (a budget the next turn gets for free).
*/
const charged: number[] = [];
const readers: QueueReader[] = [];
const maxBacklog = opts?.maxBacklog ?? 1_024;
const maxBacklogCodeUnits = positiveBudget(opts?.maxBacklogCodeUnits, DEFAULT_MAX_BACKLOG_CODE_UNITS, "maxBacklogCodeUnits");
const maxEventCodeUnits = positiveBudget(opts?.maxEventCodeUnits, DEFAULT_MAX_EVENT_CODE_UNITS, "maxEventCodeUnits");
let retained = 0;
let closed = false;
// Merge an incoming delta into the buffered tail when no reader is waiting.
// The event cap counts events, not tokens, so a detached or briefly stalled
// consumer (e.g. a Codex app mid-reconnect whose disconnect Bun has not yet
// delivered) used to hit it within seconds of token-granular streaming and
// abort a healthy turn. Adjacent same-phase text deltas, adjacent thinking
// deltas, and consecutive heartbeats carry no ordering information between
// themselves, so merging them preserves every consumer contract while making
// the cap approximate buffered items again.
// Pushed objects may be retained by adapters, so the tail is REPLACED with
// a fresh object — never mutated (alias safety). Returning the replacement
// instead of installing it lets push price the merge before committing to it.
const planTailMerge = (tail: AdapterEvent, event: AdapterEvent): AdapterEvent | null => {
if (event.type === "heartbeat") {
if (tail.type !== "heartbeat") return null;
// Heartbeats carry no ordering between themselves, but the replay-unsafe
// marker is not ordering — it is a latch. Dropping the incoming event
// would discard the only record that Cursor already performed a local
// side effect, and preflight would then permit an OAuth replay of it.
if (event.replayUnsafe !== true && event.preflightReady !== true) return tail;
return {
type: "heartbeat",
...(tail.replayUnsafe === true || event.replayUnsafe === true ? { replayUnsafe: true as const } : {}),
...(tail.preflightReady === true || event.preflightReady === true ? { preflightReady: true as const } : {}),
};
}
if (event.type === "text_delta" && tail.type === "text_delta" && tail.phase === event.phase) {
if (tail.text.length + event.text.length > COALESCE_MAX_CHUNK_LENGTH) return null;
return { type: "text_delta", text: tail.text + event.text, phase: tail.phase };
}
if (event.type === "thinking_delta" && tail.type === "thinking_delta") {
if (tail.thinking.length + event.thinking.length > COALESCE_MAX_CHUNK_LENGTH) return null;
return { type: "thinking_delta", thinking: tail.thinking + event.thinking };
}
return null;
};
/**
* Record why the turn is ending and close. The terminal error is admitted
* past both budgets — refusing to retain the explanation of a refusal would
* leave the consumer with a silent truncation — but it is charged like any
* other item so the counter stays exact through the final drain.
*/
const abortWith = (message: string): false => {
opts?.onBacklogExceeded?.();
const terminal: AdapterEvent = { type: "error", message };
const cost = retainedEventCodeUnits(terminal);
queued.push(terminal);
charged.push(cost);
retained += cost;
close();
return false;
};
const push = (event: AdapterEvent): boolean => {
if (closed) return false;
const reader = readers.shift();
if (reader) {
// Handed straight to a waiting consumer, so the queue retains nothing and
// charges nothing. Neither budget applies to an event it never holds.
reader({ done: false, value: event });
return false;
}
const tail = queued[queued.length - 1];
const merged = tail ? planTailMerge(tail, event) : null;
if (merged && tail) {
const replacement = retainedEventCodeUnits(merged);
if (replacement > maxEventCodeUnits) return abortWith(EVENT_TOO_LARGE_MESSAGE);
// Charge only what the backlog actually gains. A merge keeps the tail's
// own fields, so the incoming event's duplicated phase is never retained
// twice and an unchanged tail costs nothing at all.
const delta = replacement - charged[charged.length - 1]!;
if (delta > maxBacklogCodeUnits - retained) return abortWith(BACKLOG_EXCEEDED_MESSAGE);
queued[queued.length - 1] = merged;
charged[charged.length - 1] = replacement;
retained += delta;
return true;
}
const cost = retainedEventCodeUnits(event);
if (cost > maxEventCodeUnits) return abortWith(EVENT_TOO_LARGE_MESSAGE);
// Both refusals are priced before anything is retained, so an event that is
// turned away is never charged for.
if (queued.length >= maxBacklog) return abortWith(BACKLOG_EXCEEDED_MESSAGE);
if (cost > maxBacklogCodeUnits - retained) return abortWith(BACKLOG_EXCEEDED_MESSAGE);
queued.push(event);
charged.push(cost);
retained += cost;
return false;
};
const close = (): void => {
if (closed) return;
closed = true;
while (readers.length > 0) {
readers.shift()?.({ done: true, value: undefined as never });
}
};
async function* stream(): AsyncIterable<AdapterEvent> {
while (true) {
const next = queued.shift();
if (next) {
retained -= charged.shift() ?? 0;
yield next;
continue;
}
if (closed) return;
const result = await new Promise<IteratorResult<AdapterEvent>>(resolve => {
readers.push(resolve);
});
if (result.done) return;
yield result.value;
}
}
const collect = async (): Promise<AdapterEvent[]> => {
const events: AdapterEvent[] = [];
for await (const event of stream()) events.push(event);
return events;
};
return { push, close, stream, collect, retainedCodeUnits: () => retained };
}