UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

373 lines (365 loc) • 19.7 kB
import type { ResponsesRequestContext } from "./core-options"; import { createRequestExecutionBudget, isRequestExecutionBudget } from "../../lib/request-execution-budget"; import { chargeWorkflowSends, workflowSendCeilingReached, workflowSpendCeilingReached, } from "../../lib/workflow-budget"; import { workflowRefusalResponse } from "../workflow-refusal"; import type { AttemptRecoveryKind, AttemptRecoveryWithheld } from "../../usage/log"; import { noteAttemptRecoveryWithheld, noteAttemptSend } from "../request-log"; import { TRANSIENT_RETRY_MAX_ATTEMPTS } from "../../lib/upstream-retry"; import type { DispatchDecision, DispatchIntent, RequestExecutionBudget, SendClass, SingleUseDispatchPermit, } from "../../lib/request-execution-budget"; /** * The transient-5xx ladder cap for ONE leg, given the provider's configured request total. * * `remainingTransientSendBudget(cap)` treats `cap` as a ceiling on what REMAINS, which is the * right shape for the fixed constant: every leg may ask for up to three, and the request-wide * base allowance is what actually bounds the total. A configured `transientRetryOn5xx.attempts` * is documented as the total for one request including the first send, so it has to be reduced * by what the request already sent before it is intersected with that allowance. Passing it * straight through would make it a per-leg ceiling instead, and a request configured at one send * could still reach upstream again on a recovery leg (#4893). * * An absent policy returns the constant unchanged, so a provider that configures nothing behaves * exactly as it does today at every call site. */ export function transientSendCapFor( configuredAttempts: number | undefined, sendsUsed: number, ): number { if (configuredAttempts === undefined) return TRANSIENT_RETRY_MAX_ATTEMPTS; return Math.max(0, configuredAttempts - Math.max(0, sendsUsed)); } /** Owns the shared request send counter and recovery permits. */ export function createResponsesSendBudget( requestContext: Pick<ResponsesRequestContext, "options" | "req" | "logCtx">, ) { const { options, req, logCtx } = requestContext; // One transient-retry budget for the whole LOGICAL request, read ABOVE the passthrough branch // so that branch shares it too. It used to be a local declared below, which put it in the // temporal dead zone for the passthrough sends and left each recovery leg taking the helper's // fresh default of 3. It is now a holder carried on options, so a combo child inherits the // parent's spend instead of starting over per target -- both halves of the measured // amplification in #4546. const sendBudget = options.sendBudget ?? createRequestExecutionBudget(); // The root workflow is the user-visible task. A per-request cap cannot bound a fan-out that // sends once per child seven hundred times, so every send charged to the request is charged // to the root as well (#4546). const workflowRootId = req.headers.get("x-codex-parent-thread-id")?.trim() || undefined; const noteTransientSends = (used: number): void => { const charged = Math.max(0, used); sendBudget.used += charged; options.onCompactionRecoverySendsReported?.(charged); chargeWorkflowSends(workflowRootId, charged); }; // Refused before any dispatch, and deliberately not by evicting the root's ledger entry: // dropping the record to make room would hand the fan-out a fresh allowance, which is the // laundering this ceiling exists to stop. The client is told the task needs a new grant // rather than being given a synthetic upstream error. if (workflowSendCeilingReached(workflowRootId)) { // A log context exists here, unlike at HTTP admission, so the row this request writes is // marked synthetic rather than reading as a request that vanished with zero sends. return workflowRefusalResponse("workflow-sends-exhausted", logCtx, undefined, workflowRootId); } // The token ceiling asked at the same seam, for the same reason the count one is asked here. // Without it a spent root reaches the dispatch ladder, is refused by the ledger at the first // physical send, and answers with the generic send-budget error every exhausted request // returns -- a refusal an operator cannot tell from an ordinary budget exhaustion, on a // ceiling they configured themselves. Asked before dispatch, it names the scope and the // number instead. Returns undefined and touches no ledger when no ceiling is configured. const spentCeiling = workflowSpendCeilingReached(workflowRootId); if (spentCeiling) { return workflowRefusalResponse( "workflow-spend-exhausted", logCtx, undefined, workflowRootId, spentCeiling, ); } // No floor. Math.max(1, ...) meant an exhausted request still funded one send on every // recovery leg, so a bounded per-leg allowance never became a bounded per-request one. const remainingTransientSendBudget = (budget: number): number => isRequestExecutionBudget(sendBudget) ? sendBudget.remainingBaseSends(budget) : Math.max(0, budget - sendBudget.used); // The adapter contract needs the full budget, not just the counter. options.sendBudget is // typed as the narrow holder so a caller that predates this can still pass one, so narrow it // once here rather than asserting at each adapter call site. const adapterSendBudget = isRequestExecutionBudget(sendBudget) ? sendBudget : undefined; /** * Records an adapter's OWN inner retries against this attempt. * * Ordinal 1 is the send each call site already recorded through `noteRoutedAttemptSend`, so only * the extra physical sends are added here and an adapter that does not retry internally * leaves its log byte-for-byte as it was. Kiro reaches roughly eighteen sends per call and * Cursor re-sends a whole turn, and both reported one; a count that cannot be observed * cannot be pinned by a regression, which is why the instrumentation precedes the cap. */ const noteAdapterPhysicalSend = ( inputTokens: number | undefined, send: { ordinal: number; recovery?: AttemptRecoveryKind }, options: { readonly includeFirst?: boolean } = {}, ): void => { // Ordinal 1 is skipped because the caller normally records it before dispatch. An adapter // that reports every send asks for it to be counted here instead, so that the first send is // logged where it actually happens rather than before admission could still refuse it. if (send.ordinal <= 1 && options.includeFirst !== true) return; noteAttemptSend(logCtx.activeAttempt, inputTokens, send.recovery); }; /** * Records a recovery an adapter was ready to make and the budget refused. * * No send happened, so this deliberately does not touch `sendCount`. It is the other half of * the pair that makes a one-send log readable: no recovery kind AND no withheld reason means * nothing was eligible; a withheld reason means something was (#5044). */ const noteAdapterRecoveryWithheld = (withheld: { reason: AttemptRecoveryWithheld }): void => { noteAttemptRecoveryWithheld(logCtx.activeAttempt, withheld.reason); }; /** * Whether this request has any base send left under `cap`. * * The cap is a parameter because a lane that reads a provider's configured * `transientRetryOn5xx` ladder has to ask this question at the SAME cap its sends use. * Asking at the constant while dispatching at a configured value lets a provider with * headroom be told it is exhausted, and lets one configured below the constant pass this * check and then be refused at the send (#4893). Defaulted, so every existing caller keeps * the constant it already used. */ const sendBudgetExhausted = (cap: number = TRANSIENT_RETRY_MAX_ATTEMPTS): boolean => remainingTransientSendBudget(cap) === 0; /** * Spend one operator-granted replacement for an ambiguous failure of THIS logical request. * * The counter is the execution budget's, so a combo child that derives its own scope draws on * the same grant. A budget that predates it -- a stub, or a caller that passed the narrow * holder -- cannot grant anything, and refusing is the fail-closed answer for a send whose * upstream state is unknown. */ const claimAmbiguousResend = (limit: number): boolean => isRequestExecutionBudget(sendBudget) && sendBudget.claimAmbiguousResend?.(limit) === true; /** * A credential hop reserves the send its own replay will make, and that replay is a recovery * leg. The leg must SPEND the hop's reservation instead of taking a second one: the * final-recovery reserve is single, so a rebuild that reserved on top of a hop would be * refused and the request would answer with a synthetic 502 in place of the real 429 the hop * was recovering from. */ let pendingHopPermit: SingleUseDispatchPermit | undefined = options.compactionRecoveryPermit; /** * The budget an adapter's OWN dispatch ladder reserves against. * * Kiro and Cursor reserve once per physical send, and that is right: their ladders are the * layer that actually sends, and counting one adapter call as one send hid up to eighteen * upstream requests. But a credential hop has already booked the replay it is about to make, * and a reservation IS the charge, so an adapter that reserves again turns one physical send * into two charges -- and once the base allowance is spent, into a refusal that answers with * a synthetic error in place of the 429 the hop was recovering from (#4709). * * The hop hands its reservation down through `pendingHopPermit`, the same seam the * passthrough ladder already uses, and this view spends it on the adapter's FIRST * reservation. Every later send in that ladder is a new physical send and is charged * normally. A permit the adapter takes but never sends under is released through the same * call it would have used for a reservation of its own, so an abandoned replay is refunded * rather than left charged. */ const adapterDispatchBudget: RequestExecutionBudget | undefined = adapterSendBudget === undefined ? undefined : adapterDispatchBudgetView(adapterSendBudget, { claimHopPermit: () => { const permit = pendingHopPermit; pendingHopPermit = undefined; return permit; }, }); /** * How many sends a recovery leg may make, and the permit that authorises the last one. * * The base allowance is spent first. Once it is gone a recovery class may still draw the * single shared final-recovery reserve -- which is what keeps the validated sanitized rebuild * after a 5xx streak alive at four total sends -- but an account move and a rebuild cannot * each take one. A caller with an exact provider total can suppress that reserve. The * `countedExternally` flag exists because these legs run through the retry helper, which reports * the same send again through `onSendsConsumed`. */ const recoverySendAllowance = ( cap: number, sendClass: SendClass, targetKey: string, options: { allowFinalRecoveryReserve?: boolean } = {}, ): { attempts: number; permit?: SingleUseDispatchPermit } => { const base = remainingTransientSendBudget(cap); // The hop already paid for this leg's first send. Include it in the helper's total // attempts without charging it again, or the final account loses one transient attempt. if (base > 0) return { attempts: Math.min(cap, base + (pendingHopPermit ? 1 : 0)) }; // A provider-configured transient total is an exact physical-send ceiling. Once it is // exhausted, the request-wide recovery reserve must not silently widen it. The default stays // permissive so unconfigured providers retain the guarded profile's fourth recovery send. if (options.allowFinalRecoveryReserve === false) return { attempts: 0 }; if (pendingHopPermit) { const hopPermit = pendingHopPermit; pendingHopPermit = undefined; return { attempts: 1, permit: hopPermit }; } if (!isRequestExecutionBudget(sendBudget)) return { attempts: 0 }; const decision = sendBudget.reserveDispatch({ sendClass, targetKey, countedExternally: true }); return decision.allowed ? { attempts: 1, permit: decision.permit } : { attempts: 0 }; }; /** * One credential hop of this logical request, admitted by the INTERSECTION of two bounds. * * The snapshotted generic OAuth roster cap and `ANTHROPIC_POOL_MAX_FAILOVERS_PER_REQUEST` * bound rotation within one credential roster. What neither * can see is everything else this request already sent, so three hops layered on a spent * budget still reached upstream three more times. A hop now happens only when its own layer * cap AND the shared budget both permit it, and the smaller of the two wins. * * `countedExternally` is for the hops whose replay goes out through the retry helper, which * reports the same physical send through `onSendsConsumed`; the others are charged here and * nowhere else. A refusal is not an error: the caller keeps the real upstream response -- * status, `Retry-After`, quota body -- because return-the-last-answer is the exhaustion * contract this unit settled on. */ /** * A credential rotation inside ONE provider's roster is "auth-recovery", not * "account-failover". The distinction is load-bearing: "account-failover" sets * `isAlternateTarget` unconditionally, so under `maxAlternateTargetSends: 1` the first * rotation would refuse every later one AND consume the single slot a genuine cross-pool * move needs -- a roster whose first two accounts are both 429'd would return the 429 * while a free third account sat unused. The roster cap bounds how far rotation walks; * the shared total bounds how many sends the request makes. Reserve "account-failover" * for a real move between pools. */ const reserveCredentialHop = ( sendClass: SendClass, targetKey: string, countedExternally = false, ): { allowed: boolean; permit?: SingleUseDispatchPermit } => { if (!isRequestExecutionBudget(sendBudget)) return { allowed: true }; const decision = sendBudget.reserveDispatch({ sendClass, // A same-provider credential hop is not a model/endpoint transition. Its diagnostic // label must not replace the physical target used by the adapter's next retry. targetKey: sendClass === "auth-recovery" ? sendBudget.lastTargetKey ?? targetKey : targetKey, countedExternally, }); return decision.allowed ? { allowed: true, permit: decision.permit } : { allowed: false }; }; /** * Both classes share the one reserve, so this only changes what the decision is called -- * but a recovery event that says "repair" when a credential refresh drove it is the kind of * mislabelled evidence #4592 existed to stop. */ const recoveryClassFor = (recovery: AttemptRecoveryKind): SendClass => /401|429|oauth|rate-limit|key/.test(recovery) ? "auth-recovery" : "repair"; return { workflowRootId, noteTransientSends, remainingTransientSendBudget, /** * Physical sends this logical request has already made. * * A live getter, not a snapshot: it is read once per dispatch leg to resolve a configured * ladder, and a value frozen at construction would answer for a request that had sent * nothing. */ get sendsUsed(): number { return sendBudget.used; }, adapterSendBudget, adapterDispatchBudget, noteAdapterPhysicalSend, noteAdapterRecoveryWithheld, sendBudgetExhausted, claimAmbiguousResend, get ambiguousResendSpent(): boolean { return isRequestExecutionBudget(sendBudget) && sendBudget.ambiguousResendSpent === true; }, get pendingHopPermit(): SingleUseDispatchPermit | undefined { return pendingHopPermit; }, set pendingHopPermit(value: SingleUseDispatchPermit | undefined) { pendingHopPermit = value; }, recoverySendAllowance, reserveCredentialHop, recoveryClassFor, }; } export type ResponsesSendBudget = Exclude<ReturnType<typeof createResponsesSendBudget>, Response>; /** * A LIVE delegating view of one request's execution budget, with a credential hop's * reservation spendable through it. * * Every member forwards rather than copying. A spread of the budget would freeze `used`, * `reserveSpent` and the target counters at construction time, handing the adapter a budget * that can never read as exhausted -- the same class of defect as the fresh per-layer * allowances #4546 removed. */ function adapterDispatchBudgetView( budget: RequestExecutionBudget, hop: { claimHopPermit: () => SingleUseDispatchPermit | undefined }, ): RequestExecutionBudget { return { get used(): number { return budget.used; }, set used(next: number) { budget.used = next; }, logicalRequestId: budget.logicalRequestId, policyVersion: budget.policyVersion, policy: budget.policy, get reserveSpent(): boolean { return budget.reserveSpent; }, get alternateTargetSends(): number { return budget.alternateTargetSends; }, get targetTransitions(): number { return budget.targetTransitions; }, get lastTargetKey(): string | undefined { return budget.lastTargetKey; }, remainingBaseSends: (cap: number): number => budget.remainingBaseSends(cap), claimAmbiguousResend: (limit: number): boolean => budget.claimAmbiguousResend?.(limit) === true, get ambiguousResendSpent(): boolean { return budget.ambiguousResendSpent === true; }, reserveDispatch(intent: DispatchIntent): DispatchDecision { // A dispatch whose upstream state is unknown is refused on its own merits. A hop that // already paid does not make an unsafe replay safe, so that check stays with the budget. if (intent.replaySafe !== false) { const hopPermit = hop.claimHopPermit(); if (hopPermit && intent.targetKey !== budget.lastTargetKey) { // The rotated credential can select a different regional endpoint. Replace the // provisional booking synchronously so the actual destination obeys transition // limits, while the physical send is still charged only once. hopPermit.release(); return budget.reserveDispatch({ ...intent, sendClass: hopPermit.sendClass }); } // Confirmed here rather than in `use()`: the adapter reserves immediately before it // opens the transport, which is the same boundary the hop's own confirmation uses. // A permit some other leg already settled returns false, and this falls through to a // real reservation rather than handing the adapter a dead permit -- an adapter whose // `use()` fails treats the request as exhausted and stops sending entirely. if (hopPermit !== undefined && hopPermit.assumeCharge()) { let spent = false; return { allowed: true, permit: { sendClass: hopPermit.sendClass, use: (): boolean => { if (spent) return false; spent = true; return true; }, assumeCharge: (): boolean => { if (spent) return false; spent = true; return true; }, // The hop's charge is already settled and belongs to the leg that asked for it, // so there is nothing here to refund. release: (): void => {}, }, }; } } return budget.reserveDispatch(intent); }, }; }