@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
422 lines (393 loc) • 21.5 kB
text/typescript
import { clientFacingModelText, type ClientFacingModelLabel } from "../../lib/error-destination";
/**
* Pre-dispatch input admission (#1412).
*
* Refuses a turn whose estimated input cannot plausibly fit the model context window,
* BEFORE auth resolution, circuit admission, or any upstream I/O. #1412 reported ~127k of
* real context compounding to 1.3M-1.6M tokens and crashing the proxy; the provider would
* reject such a turn anyway, so paying for the round trip buys nothing.
*
* Deliberately narrow. This is not a context manager and not a compaction trigger: it
* catches the pathological case and stays out of the way otherwise. Every uncertainty
* resolves toward admitting.
*/
import { collapseDevinModelUid, DEVIN_MODEL_CONTEXT_WINDOWS } from "../../adapters/devin/live-models";
import { openAIChatSerializesThinking } from "../../adapters/openai-chat/messages";
import {
nativeOpenAiContextWindow,
nativeOpenAiMaxInputTokens,
nativeOpenAiMaxOutputTokens,
type NativeContextLimitsInput,
} from "../../codex/catalog/metadata";
import { getModelMetadata } from "../../generated/model-metadata";
import { estimateTokens } from "../../lib/token-estimate";
import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers";
import { modelRecordValue } from "../../reasoning-effort";
import type { OcxContentPart, OcxParsedRequest, OcxProviderConfig } from "../../types";
/**
* Multiplier applied to the ceiling before refusing.
*
* 2.5, not something tighter, because the estimate is a heuristic and the cost of being wrong
* is asymmetric: a false refusal fails a turn the provider would have answered, while an
* over-admission merely pays for one round trip the provider then rejects itself.
*
* The original margin was sized against a sampling artifact — `cjkRatio` read every stride-th
* character, so a payload of fixed-width records could sample as 100% CJK while being 1.6% CJK
* and inflate the estimate by 4.0/2.5 = 1.6x. That sampler is gone: CJK characters are now
* counted exactly, so that particular 1.6x divergence cannot occur and the headroom it bought is
* no longer spent on it.
*
* The margin is still 2.5 because the estimator it guards got LARGER, not smaller. Counting the
* two scripts separately raises a pure-Latin estimate by 1.25x and a Korean one by up to 1.67x
* against the previous model, which consumes real headroom: measured against the old estimator's
* scale, 2.5 now behaves like roughly 2.0x for Latin and 1.5x for Korean-dominant input. That is
* the intended direction — the estimates are closer to what providers actually charge, so the
* same multiplier is a tighter and more honest bound — and it still refuses the #1412 shape
* (10x compounding) several times over.
*/
export const ADMISSION_TOLERANCE = 2.5;
/**
* Token cost charged for a remote image URL. The bytes are not in this request — the
* provider fetches them — so the URL own length is not the cost. A small flat charge
* acknowledges the tiles the image will occupy without pretending to know its dimensions.
*/
const REMOTE_IMAGE_TOKENS = 850;
/** Decoded image bytes per token. Coarse tile-count proxy, not a provider formula. */
const IMAGE_BYTES_PER_TOKEN = 750;
/** Compressed-audio bytes per token. More conservative than the image proxy. */
const AUDIO_BYTES_PER_TOKEN = 256;
export interface InputAdmissionResult {
admitted: boolean;
estimatedTokens: number;
/** Resolved ceiling, or null when nothing could be resolved (=> always admitted). */
ceiling: number | null;
/** Actual shared context window when an input-only ceiling differs. */
contextWindow?: number | null;
/** Output space reserved by the combo preflight; absent on the loose direct gate. */
requiredOutputHeadroom?: number;
/**
* The margin-adjusted estimate the combo gate actually decided on. Present only when the
* strict gate ran, so refusal reporting can name the limit that bound without re-deriving
* the margin. `estimatedTokens` stays the raw figure.
*/
admissionEstimateTokens?: number;
}
function positive(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null;
}
/**
* Charge a `data:` URL by its DECODED size rather than its character length: base64 inflates
* by 4/3, so charging the string would overcount by a third. A remote URL is charged flat.
*/
function imageTokens(imageUrl: string): number {
if (!imageUrl.startsWith("data:")) return REMOTE_IMAGE_TOKENS;
const comma = imageUrl.indexOf(",");
if (comma < 0) return REMOTE_IMAGE_TOKENS;
const payload = imageUrl.length - comma - 1;
if (payload <= 0) return 0;
const decoded = Math.floor((payload * 3) / 4);
return Math.max(1, Math.ceil(decoded / IMAGE_BYTES_PER_TOKEN));
}
function contentPartTokens(part: OcxContentPart, modelId: string): number {
if (part.type === "image") return imageTokens(part.imageUrl);
if (part.type === "video") return imageTokens(part.videoUrl);
if (part.type === "audio") return base64PayloadTokens(part.data, AUDIO_BYTES_PER_TOKEN);
// An inline document is a base64 payload, not a sentence: estimating it from its marker would
// admit a request whose real input is orders of magnitude larger. Counted arithmetically —
// rebuilding the data URL here would materialize a second request-sized string just to measure it.
if (part.type === "document") return base64PayloadTokens(part.data);
return estimateTokens(part.text, modelId);
}
function base64PayloadTokens(base64: string, bytesPerToken = IMAGE_BYTES_PER_TOKEN): number {
if (base64.length === 0) return 0;
const decoded = Math.floor((base64.length * 3) / 4);
return Math.max(1, Math.ceil(decoded / bytesPerToken));
}
function contentTokens(content: string | readonly OcxContentPart[], modelId: string): number {
if (typeof content === "string") return estimateTokens(content, modelId);
let total = 0;
for (const part of content) total += contentPartTokens(part, modelId);
return total;
}
/**
* Estimate the input tokens of a parsed request.
*
* Walks the whole `OcxMessage` union rather than user text alone. Assistant turns carry
* their content as `OcxAssistantContentPart[]` — text, thinking blocks, and tool calls whose
* JSON arguments are frequently the largest single item in an agent conversation. A walk
* that counted only `{type:"text"}` would undercount exactly the turns that trigger this
* gate. The routed provider determines whether replayed thinking reaches the wire.
*/
export function estimateInputTokens(parsed: OcxParsedRequest, modelId: string, provider?: OcxProviderConfig): number {
const { context } = parsed;
const countThinking = provider?.adapter !== "openai-chat"
|| openAIChatSerializesThinking(provider, parsed.modelId).text;
let total = 0;
for (const prompt of context.systemPrompt ?? []) total += estimateTokens(prompt, modelId);
for (const message of context.messages) {
if (message.role === "assistant") {
for (const part of message.content) {
if (part.type === "text") total += estimateTokens(part.text, modelId);
else if (part.type === "thinking") {
if (countThinking) total += estimateTokens(part.thinking, modelId);
}
else total += estimateTokens(part.name, modelId) + estimateTokens(JSON.stringify(part.arguments), modelId);
}
// Opaque provider blob replayed verbatim upstream, so it costs real input tokens.
if (message.kiroRedactedReasoning) total += estimateTokens(message.kiroRedactedReasoning, modelId);
continue;
}
total += contentTokens(message.content, modelId);
}
// Tool schemas ride every turn: name, description, and the JSON parameter schema all
// reach the upstream, and a large MCP catalog can dominate a short conversation.
for (const tool of context.tools ?? []) {
total += estimateTokens(tool.name, modelId)
+ estimateTokens(tool.description, modelId)
+ estimateTokens(JSON.stringify(tool.parameters), modelId);
}
return total;
}
/**
* Resolve the admission ceiling. Pure: no filesystem, no catalog, no registry scan.
*
* `provider` must be the ROUTED config (`route.provider`), which `routedProviderConfig`
* has already transport-guarded and merged. Re-deriving from `config.providers[name]` would
* reject a user-defined provider that merely shares a built-in name using limits that
* belong to a different service.
*/
interface ResolvedContextLimits {
/** The target's total context window: input and output share it. */
window: number | null;
/** Largest admissible input, which input-only caps may tighten below the window. */
ceiling: number | null;
}
/**
* Devin spells one model dotted in catalog families (`swe-1.7`) and hyphenated in uids
* (`swe-1-7`), with optional effort suffixes (`swe-2-flash-medium`). A per-model record is
* matched on the exact id first, then the effort-collapsed id, then both hyphenated forms, and
* finally on record keys folded the same way (`swe-1.7` serves a `swe-1-7-medium` request), so
* an operator override on any of those spellings binds before a provider-wide default.
*/
function devinRecordValue(record: Record<string, number> | undefined, modelId: string): number | null {
const normalized = modelId.replace(/\./g, "-");
// Lower-case before collapsing so an upper-case effort suffix (SWE-1-7-MEDIUM) is removed too.
const collapsed = collapseDevinModelUid(normalized.toLowerCase());
for (const key of new Set([modelId, collapseDevinModelUid(modelId), normalized, collapsed])) {
const value = positive(modelRecordValue(record, key));
if (value !== null) return value;
}
if (!record) return null;
// Exact spellings missed; fold dotted record keys only now so an exact key always wins.
for (const target of new Set([normalized, collapsed])) {
// Fold case too, like modelRecordValue's exact-spelling lookup.
const folded = target.toLowerCase();
for (const [key, raw] of Object.entries(record)) {
if (key.replace(/\./g, "-").toLowerCase() !== folded) continue;
const value = positive(raw);
if (value !== null) return value;
}
}
return null;
}
function resolveContextLimits(
provider: OcxProviderConfig,
providerName: string,
modelId: string,
nativeContextCap?: NativeContextLimitsInput,
): ResolvedContextLimits {
// `modelRecordValue`, not a bare lookup: the catalog resolves these same two maps that
// way, so a `gpt-oss` entry covers `gpt-oss:120b`. Reading raw here made the gate fall
// back to the provider-wide window and refuse turns the model can plainly hold.
const devin = provider.adapter === "devin";
const configured = (devin
? devinRecordValue(provider.modelContextWindows, modelId)
: positive(modelRecordValue(provider.modelContextWindows, modelId)))
?? positive(provider.contextWindow);
// The canonical `openai` registry entry declares no context fields, so without this the
// gate would be inert on the default Codex route. All three clauses are load-bearing: a
// transport-mismatched custom provider named "openai" is preserved verbatim by routing
// and must not inherit built-in native limits, and a routed `provider/model` id is not a
// native slug. Static maps only — no catalog read.
const canonicalNativeBare = providerName === OPENAI_CODEX_PROVIDER_ID
&& isCanonicalOpenAiForwardProvider(provider)
&& !modelId.includes("/");
const nativeLimits = canonicalNativeBare && configured !== null
? {
...(typeof nativeContextCap === "number" ? { cap: nativeContextCap } : (nativeContextCap ?? {})),
modelWindows: { [modelId]: configured },
}
: nativeContextCap;
const native = canonicalNativeBare
? positive(nativeOpenAiContextWindow(modelId, nativeLimits))
: null;
const nativeMaxInput = canonicalNativeBare ? positive(nativeOpenAiMaxInputTokens(modelId, nativeLimits)) : null;
// Devin's live catalog windows reach the Codex model list but never `provider.modelContextWindows`;
// without the adapter table every devin combo target resolved no window and the strict gate stayed open.
const window = canonicalNativeBare ? (native ?? generatedNativeWindow(modelId, configured, nativeContextCap))
: (configured ?? (devin ? devinRecordValue(DEVIN_MODEL_CONTEXT_WINDOWS, modelId) : null));
// modelMaxInputTokens is an input-only cap, so it can only tighten the window.
const configuredMaxInput = devin
? devinRecordValue(provider.modelMaxInputTokens, modelId)
: positive(modelRecordValue(provider.modelMaxInputTokens, modelId));
const limits = [window, configuredMaxInput, nativeMaxInput].filter((v): v is number => v !== null);
return { window, ceiling: limits.length === 0 ? null : Math.min(...limits) };
}
/**
* Generated-catalog keys, not routing provider names. `OPENAI_CODEX_PROVIDER_ID` is the string
* `"openai"` -- the canonical Codex forward route -- so using it to index the generated bundle
* would silently skip the native Codex rows and read the public API rows instead.
*/
const NATIVE_METADATA_CATALOGS = ["openai-codex", "openai"] as const;
/**
* Static in-tree metadata for a canonical native slug the narrower override and pinned-native
* tables do not carry. Falling through to null made input admission completely blind for
* exactly those models, which is how a 128k target accepted a turn it could not finish.
*
* This deliberately covers slugs that are no longer offered in the picker: a retired slug is
* still dispatchable when an operator names it explicitly in a combo target, and that is the
* configuration where the gate was inert. This is a generated bundle compiled into the binary,
* not a live catalog read, so it adds no I/O. Explicit provider and operator caps may only
* narrow the result, never widen it.
*/
function generatedNativeWindow(
modelId: string,
configured: number | null,
nativeContextCap: NativeContextLimitsInput | undefined,
): number | null {
let generated: number | null = null;
for (const catalog of NATIVE_METADATA_CATALOGS) {
generated = positive(getModelMetadata(catalog, modelId)?.contextWindow);
if (generated !== null) break;
}
if (generated === null) return null;
const cap = typeof nativeContextCap === "number"
? positive(nativeContextCap)
: positive(nativeContextCap?.cap);
return Math.min(generated, configured ?? generated, cap ?? generated);
}
export function resolveInputCeiling(
provider: OcxProviderConfig,
providerName: string,
modelId: string,
// Operator cap for the canonical native provider. Passed in rather than read from a
// config here so this stays pure: no filesystem, no catalog, no registry scan.
nativeContextCap?: NativeContextLimitsInput,
): number | null {
return resolveContextLimits(provider, providerName, modelId, nativeContextCap).ceiling;
}
/**
* Largest output the concrete target can emit. Used only to avoid reserving MORE than the
* target could ever produce when a client asks for a bigger allowance than the model has.
* Unknown stays unknown rather than inventing a capability.
*/
export function resolveOutputCeiling(
provider: OcxProviderConfig,
providerName: string,
modelId: string,
): number | null {
const configured = positive(modelRecordValue(provider.modelMaxOutputTokens, modelId))
?? positive(provider.defaultMaxOutputTokens);
const canonicalNativeBare = providerName === OPENAI_CODEX_PROVIDER_ID
&& isCanonicalOpenAiForwardProvider(provider)
&& !modelId.includes("/");
const native = canonicalNativeBare ? positive(nativeOpenAiMaxOutputTokens(modelId)) : null;
const limits = [configured, native].filter((v): v is number => v !== null);
return limits.length === 0 ? null : Math.min(...limits);
}
/**
* Output room a combo child must retain when the caller declares no `max_output_tokens`. The
* loose direct-request gate tolerates pathological input because a solo request has nowhere
* else to go; a combo child does -- input admitted at 99.9% of the window leaves a few hundred
* tokens, so the target answers `incomplete`/`max_output_tokens` AFTER selection, when no
* sibling remains to retry. Bounded so a small window keeps a usable fraction and a large one
* cannot reserve a whole answer's worth.
*/
const COMBO_DEFAULT_OUTPUT_RESERVE = { fraction: 0.1, min: 1_024, max: 32_768 } as const;
/**
* Combo-only admission. A fallback must be able to satisfy the caller's declared output
* allowance inside its OWN context window. Otherwise it returns 200, emits a few hundred
* tokens, and terminates on `finish_reason: length` — which the Anthropic surface renders as
* "response exceeded the output token maximum" even though the real cause was the total
* window. By then the next target cannot be tried, because output has already committed.
*
* Two budgets are checked separately so the reserve is counted exactly once. `ceiling` is an
* input-only budget once `modelMaxInputTokens` tightens it below the window, so the output
* reserve belongs against `window`, not against `ceiling`.
*
* Direct and single-target requests keep the deliberately loose 2.5x pathological-input gate.
* This stricter rule applies only to synthetic combo children, where skipping one known-small
* target is safe and the ladder continues before any upstream bytes are sent. Unknown context
* falls back to the direct check, and an undeclared output allowance uses a default reserve.
*/
export function checkComboTargetInputAdmission(
parsed: OcxParsedRequest,
provider: OcxProviderConfig,
providerName: string,
modelId: string,
nativeContextCap?: NativeContextLimitsInput,
): InputAdmissionResult {
const { window, ceiling } = resolveContextLimits(provider, providerName, modelId, nativeContextCap);
if (window === null || ceiling === null) {
return checkInputAdmission(parsed, provider, providerName, modelId, nativeContextCap);
}
const requestedOutput = positive(parsed.options.maxOutputTokens);
const targetOutput = resolveOutputCeiling(provider, providerName, modelId);
const defaultReserve = Math.min(
COMBO_DEFAULT_OUTPUT_RESERVE.max,
Math.max(COMBO_DEFAULT_OUTPUT_RESERVE.min, Math.floor(window * COMBO_DEFAULT_OUTPUT_RESERVE.fraction)),
);
const requiredOutputHeadroom = requestedOutput === null
? (targetOutput === null ? defaultReserve : Math.min(defaultReserve, targetOutput))
: (targetOutput === null ? requestedOutput : Math.min(requestedOutput, targetOutput));
const estimatedTokens = estimateInputTokens(parsed, modelId, provider);
// Apply the 1.1x estimate margin without rounding exact fits up by one token.
const admissionEstimate = Math.ceil((estimatedTokens * 11) / 10);
return {
admitted: admissionEstimate <= ceiling && admissionEstimate + requiredOutputHeadroom <= window,
estimatedTokens,
admissionEstimateTokens: admissionEstimate,
ceiling,
requiredOutputHeadroom,
...(window !== ceiling ? { contextWindow: window } : {}),
};
}
/**
* Fail-open when no ceiling is known; refuse only past `ceiling * ADMISSION_TOLERANCE`.
*
* The caller is responsible for skipping compaction turns — see the call site in core.ts.
*/
export function checkInputAdmission(
parsed: OcxParsedRequest,
provider: OcxProviderConfig,
providerName: string,
modelId: string,
nativeContextCap?: NativeContextLimitsInput,
): InputAdmissionResult {
const { window, ceiling } = resolveContextLimits(provider, providerName, modelId, nativeContextCap);
if (ceiling === null) return { admitted: true, estimatedTokens: 0, ceiling: null };
const estimatedTokens = estimateInputTokens(parsed, modelId, provider);
return { admitted: estimatedTokens <= ceiling * ADMISSION_TOLERANCE, estimatedTokens, ceiling,
...(window !== ceiling ? { contextWindow: window } : {}) };
}
export function inputAdmissionRefusalMessage(input: InputAdmissionResult, model: ClientFacingModelLabel): string {
const count = (value: number | null) => value !== null && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : 0;
const label = clientFacingModelText(model);
const decisionEstimate = input.admissionEstimateTokens ?? input.estimatedTokens;
const estimate = count(decisionEstimate);
const ceiling = count(input.ceiling);
if (input.contextWindow !== undefined && decisionEstimate > ceiling) {
const window = input.contextWindow === null ? "" : `; context window: ${count(input.contextWindow)} tokens`;
return `Estimated input (~${estimate} tokens) exceeds the input limit of ${label} (${ceiling} tokens${window}). `
+ "Start a new session or choose a model with a larger input limit.";
}
const window = count(input.contextWindow ?? input.ceiling);
const inputLimit = input.contextWindow !== undefined ? `; input limit: ${ceiling} tokens` : "";
return input.requiredOutputHeadroom !== undefined
? `Estimated input (~${estimate} tokens) plus ${count(input.requiredOutputHeadroom)} `
+ `tokens of requested output headroom cannot fit the context window of ${label} `
+ `(${window} tokens${inputLimit}).`
: `Estimated input (~${estimate} tokens) is far past the context window `
+ `of ${label} (${window} tokens${inputLimit}). Start a new session or choose a `
+ `model with a larger context window.`;
}