UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

486 lines (458 loc) • 23.2 kB
/** * One logical request, one execution budget (#4546). * * The amplification behind #4546 was never a single missing limit. Every layer that can * re-send a request -- transport retry, adapter retry, auth recovery, account failover, combo * failover, repair -- counted its own allowance, so a per-layer 3 composed into a per-request * 12. #4605 and #4608 gave the transient layers one shared counter; this module is the policy * that counter answers to. * * The policy is an INTERSECTION of constraints, not four independent counters. A request that * still has total allowance left is not thereby entitled to a second account move, and a * request that changed credentials does not get its target-transition allowance back. The * default profile keeps the recovery shape that actually works today -- three same-account * sends plus one alternate -- by funding the alternate from a reserve that a validated * sanitized repair can spend instead, but never both. */ import type { TransientSendBudget } from "./upstream-retry"; export type SendClass = | "initial" | "transient" | "auth-recovery" | "repair" | "account-failover" | "combo-failover" | "prewarm"; export interface RequestExecutionBudgetPolicy { /** Every model send of one logical request, including the reserve. */ readonly maxTotalModelSends: number; /** Shared by the initial send, same-target transient retries, and refresh/repair legs. */ readonly baseSendAllowance: number; /** ONE final recovery, shared by an account move and a validated rebuild. Not one each. */ readonly finalRecoveryAllowance: number; readonly maxAlternateTargetSends: number; readonly maxTargetTransitions: number; } /** * Text Codex guarded profile. Three same-account sends plus one alternate is the recovery * shape that live traffic depends on, so a flat ceiling of 3 would break a working path. */ export const CODEX_TEXT_GUARDED_BUDGET_POLICY: RequestExecutionBudgetPolicy = { maxTotalModelSends: 4, baseSendAllowance: 3, finalRecoveryAllowance: 1, maxAlternateTargetSends: 1, maxTargetTransitions: 1, }; export const REQUEST_BUDGET_POLICY_VERSION = "guarded-v1"; export type BudgetDenial = | "total-exhausted" | "base-allowance-exhausted" | "final-recovery-spent" | "alternate-target-exhausted" | "target-transition-exhausted" | "spend-exhausted" | "not-replay-safe"; export interface DispatchIntent { readonly sendClass: SendClass; /** * (provider route, endpoint, model lane, upstream credential identity). A quota domain is a * different thing and must not be folded in here. */ readonly targetKey: string; /** * False refuses the dispatch outright. A request whose execution state upstream is unknown * is not replayable just because budget remains (RFC 9110 9.2.2). */ readonly replaySafe?: boolean; /** * True when the physical send is already reported through another counter -- the retry * helpers' `onSendsConsumed` hook. The send is still booked at reservation time, because an * advisory reservation cannot stop a concurrent leg; what changes is that the booking is * PENDING, and the first send the external reporter names settles it instead of adding a * second charge. Charging both is how a four-send cap silently becomes a two-send cap. */ readonly countedExternally?: boolean; } export interface SingleUseDispatchPermit { readonly sendClass: SendClass; /** * Confirm the dispatch this permit already paid for. The reservation is the charge, so this * charges nothing; it is how a leg proves it is the one that sent. A second call returns * false, which is what keeps a retry thunk from sending twice on one permit. */ use(): boolean; /** * Hand back a reservation that never dispatched -- a credential move that found no alternate, * a rebuild abandoned before the send. Idempotent, and a no-op once the permit was used or * once an external send reporter already settled it. */ release(): void; /** * Take over an externally counted booking, because the layer holding this permit is the one * that physically sends. * * `countedExternally` promises that a retry helper will name this send through * `onSendsConsumed`. An adapter that owns its own dispatch ladder -- Kiro's reset loop, * Cursor's transport loop -- reserves per physical send instead, so no reporter ever arrives * and the pending booking would sit there until it silently swallowed an unrelated later * report. Confirming through this method settles the permit AND closes the booking, so the * send stays charged exactly once (#4709). Returns false once the permit is settled, which is * what keeps one permit from admitting two sends. */ assumeCharge(): boolean; } export type DispatchDecision = | { allowed: true; permit: SingleUseDispatchPermit } | { allowed: false; reason: BudgetDenial }; /** * Notified when this request's physical-send count moves. * * `spent` is the only number here that counts SENDS rather than intentions: a reservation * increments it, a refund decrements it, and an externally reported send settles against a * booking that was already counted. Anything that books one entry per increment therefore * books exactly one entry per physical send -- which is what lets the durable spend ledger * have a production caller without every dispatch site in the tree remembering to call it. * * `charge` may refuse, and a refusal denies the dispatch. That is deliberate: the ledger is * the only bound here that survives a restart, so a limit it enforces has to be able to stop a * send rather than merely describe one. */ export interface RequestSendObserver { /** * Book one physical send. False refuses the dispatch before the budget charges it. * * `alreadySent` marks a send that has already left, which the reporting transports below * do after the fact. Its answer is not a decision -- nothing can un-send it -- and the * observer must RECORD it rather than drop it. Dropping it is a fixpoint: the send that * would cross a ceiling never joins the total, the total stays just under, and the ceiling * never fires for any later request either. */ charge(options?: { alreadySent?: boolean }): boolean; /** Give back a booking whose send never happened. */ refund(): void; } /** * Carried on HandleResponsesOptions so a combo child, a rebuild and an alternate-account leg * all decrement the same holder. `used` is the existing #4605 counter and still counts every * model send; the reserve is what the fourth send draws on once the base allowance is gone. */ export interface RequestExecutionBudget extends TransientSendBudget { readonly logicalRequestId: string; readonly policyVersion: string; readonly policy: RequestExecutionBudgetPolicy; reserveDispatch(intent: DispatchIntent): DispatchDecision; /** * Sends still available from the base allowance, capped by a layer's own maximum. * Returns 0 when the allowance is gone -- it never floors to 1, because a floor of 1 is * what let every recovery leg send one more time forever. * * A reserved-but-unconfirmed send is spent for this purpose. The alternative -- counting only * confirmed sends -- is what let two legs read the same remainder and both dispatch. */ remainingBaseSends(cap: number): number; readonly reserveSpent: boolean; readonly alternateTargetSends: number; readonly targetTransitions: number; readonly lastTargetKey: string | undefined; /** * Spend one operator-granted replacement for an AMBIGUOUS failure of this logical request, * up to `limit`. False once the request has none left. * * It lives on the budget rather than beside the policy that grants it because it has to be * shared exactly where the physical-send ledger is shared. A combo child derives its own * budget from the parent's ledger, and two counters would let a request whose parent leg * reset before the head and whose child leg reset after it replace an unknown-state send * twice. It is NOT a send budget: an authorised replacement still has to fit inside * `remainingBaseSends` like every other send. * * `limit` is the ceiling the ASKING leg is authorised to present, and the request keeps the * smallest one any leg has presented. A leg reads it from `route.provider`, which credential * rotation, OAuth refresh, transport resolution and a combo target all reassign mid-request, * so a per-call ceiling meant the number of duplicate inferences a request could make * depended on which row happened to ask last: a row granting one, then a row granting two, * bought a second replacement of a turn that may already have run. * * Optional so a hand-written stub that satisfies the shape test keeps typechecking; a caller * that cannot reach it has no operator override, which is the fail-closed answer. */ claimAmbiguousResend?(limit: number): boolean; /** True once any scope has claimed a replacement for this logical request. */ readonly ambiguousResendSpent?: boolean; } const RESERVE_FUNDED_CLASSES: ReadonlySet<SendClass> = new Set<SendClass>([ "account-failover", "combo-failover", "repair", "auth-recovery", ]); let logicalRequestSeq = 0; /** * One request's physical-send ledger, held apart from the budget object so a derived policy * scope can share the exact same one. * * `spent` and `pendingExternalSends` belong together: a pending booking is a send that is * already counted in `spent` and awaiting its reporter, so a scope that shared one without the * other would either charge that send twice or never charge it at all. * * The durable-spend observer belongs here for the same reason. It books one entry per physical * send by watching this counter move, so a derived scope that spent the counter without * carrying the observer would move it without booking, and a combo child's sends would go * missing from the ledger (#4707). */ interface SharedSendLedger { spent: number; pendingExternalSends: number; /** * Spend one of this logical request's replacements for an ambiguous failure. Beside `spent` * for the same reason `pendingExternalSends` is: a derived scope that shared one without the * other would hand the request a second grant. * * A function rather than the raw count, because the count is not the whole state. The * ceiling belongs to the request too, and a bridged scope has no counter of its own to keep * it in -- it has to ask whoever holds the request's grant. */ claimAmbiguousResend(limit: number): boolean; readonly ambiguousResendSpent: boolean; readonly observer?: RequestSendObserver; } const sharedSendLedgers = new WeakMap<RequestExecutionBudget, SharedSendLedger>(); /** * One logical request's replacement grant: how many it has spent, and the ceiling it is held * to. * * The ceiling is the SMALLEST any leg has presented rather than whatever the current leg * presents. Each leg reads its number from the provider row it is running against, and that * row changes inside one request -- credential rotation, OAuth refresh, transport resolution * and each combo target reassign it. Taking the asking leg's number let a request that had * already spent the one replacement a strict row granted buy another as soon as a more * permissive row asked, which is a second duplicate inference of one turn. */ function createAmbiguousResendGrant(): Pick<SharedSendLedger, "claimAmbiguousResend" | "ambiguousResendSpent"> { let claimed = 0; let ceiling: number | undefined; return { get ambiguousResendSpent(): boolean { return claimed > 0; }, claimAmbiguousResend(limit: number): boolean { const presented = Number.isFinite(limit) ? Math.trunc(limit) : 0; // A zero or nonsense ceiling refuses on its own and leaves the request's alone. It is a // caller that cannot state a grant, not an operator narrowing this request: a leg with no // policy is refused before it ever claims, so binding the request to a malformed number // would only let such a caller cancel a grant an opted-in row really made. if (presented <= 0) return false; ceiling = ceiling === undefined ? presented : Math.min(ceiling, presented); if (claimed >= ceiling) return false; claimed += 1; return true; }, }; } function createRequestExecutionBudgetWithLedger( policy: RequestExecutionBudgetPolicy, logicalRequestId: string | undefined, counter: SharedSendLedger, ): RequestExecutionBudget { const observer = counter.observer; let reserveSpent = false; let alternateTargetSends = 0; let targetTransitions = 0; let lastTargetKey: string | undefined; const budget: RequestExecutionBudget = { get used(): number { return counter.spent; }, set used(next: number) { // The retry helpers report their real send count by assigning through this field. A // reservation taken with `countedExternally` has already booked one of those sends, so // the report settles the pending booking first and only the surplus is charged. const delta = next - counter.spent; if (delta <= 0) { counter.spent = Math.max(0, next); return; } const settled = Math.min(delta, counter.pendingExternalSends); counter.pendingExternalSends -= settled; const charged = delta - settled; counter.spent += charged; // These sends have already left. The ledger records them even past a ceiling it would // have refused, because refusing after the fact only hides spend that was really // incurred -- the refusal has to happen at the reservation below, or not at all. for (let index = 0; index < charged; index += 1) observer?.charge({ alreadySent: true }); }, logicalRequestId: logicalRequestId ?? `lr-${Date.now().toString(36)}-${(logicalRequestSeq += 1).toString(36)}`, policyVersion: REQUEST_BUDGET_POLICY_VERSION, policy, get reserveSpent() { return reserveSpent; }, get alternateTargetSends() { return alternateTargetSends; }, get targetTransitions() { return targetTransitions; }, get lastTargetKey() { return lastTargetKey; }, remainingBaseSends(cap: number): number { const capped = Number.isFinite(cap) ? Math.trunc(cap) : 0; return Math.max(0, Math.min(capped, policy.baseSendAllowance - counter.spent)); }, claimAmbiguousResend(limit: number): boolean { return counter.claimAmbiguousResend(limit); }, get ambiguousResendSpent(): boolean { return counter.ambiguousResendSpent; }, reserveDispatch(intent: DispatchIntent): DispatchDecision { if (intent.replaySafe === false) return { allowed: false, reason: "not-replay-safe" }; if (counter.spent >= policy.maxTotalModelSends) return { allowed: false, reason: "total-exhausted" }; const changesTarget = lastTargetKey !== undefined && lastTargetKey !== intent.targetKey; const isAlternateTarget = changesTarget || intent.sendClass === "account-failover" || intent.sendClass === "combo-failover"; if (isAlternateTarget && changesTarget && targetTransitions >= policy.maxTargetTransitions) { return { allowed: false, reason: "target-transition-exhausted" }; } if (isAlternateTarget && alternateTargetSends >= policy.maxAlternateTargetSends) { return { allowed: false, reason: "alternate-target-exhausted" }; } // The base allowance is spent first. Only once it is gone does a recovery class reach // for the single shared reserve -- an account move and a validated rebuild cannot each // take one. const drawsReserve = policy.baseSendAllowance - counter.spent <= 0; if (drawsReserve) { if (!RESERVE_FUNDED_CLASSES.has(intent.sendClass)) { return { allowed: false, reason: "base-allowance-exhausted" }; } if (reserveSpent || policy.finalRecoveryAllowance <= 0) { return { allowed: false, reason: "final-recovery-spent" }; } } // Consulted last, because it is the only bound here that WRITES. A ledger entry booked // for a dispatch a cheaper check above would have refused is spend this request never // makes, and it would hold those tokens against the scope until retention expired. if (observer && !observer.charge()) return { allowed: false, reason: "spend-exhausted" }; // THE RESERVATION IS THE CHARGE. Deciding here and charging in `use()` left a window in // which two legs read the same remainder, both received a permit, and both dispatched: // one remaining send admitted two physical sends, which is the per-request multiplication // this budget exists to stop. Everything is booked now; `release()` is the way back. const previousTargetKey = lastTargetKey; counter.spent += 1; if (intent.countedExternally === true) counter.pendingExternalSends += 1; if (drawsReserve) reserveSpent = true; if (isAlternateTarget) alternateTargetSends += 1; if (changesTarget) targetTransitions += 1; lastTargetKey = intent.targetKey; let settled: "open" | "used" | "released" = "open"; return { allowed: true, permit: { sendClass: intent.sendClass, use(): boolean { if (settled !== "open") return false; settled = "used"; return true; }, assumeCharge(): boolean { if (settled !== "open") return false; settled = "used"; // The booking this reservation made for an external reporter is now owned by the // caller. Leaving it pending is not harmless: the next `used` report of this request // would settle against it and one real send would go uncharged. if (intent.countedExternally === true && counter.pendingExternalSends > 0) { counter.pendingExternalSends -= 1; } return true; }, release(): void { if (settled !== "open") return; settled = "released"; // An externally counted reservation the reporter already settled paid for a send // that physically happened. Refunding it would hand the request a free send back. if (intent.countedExternally === true) { if (counter.pendingExternalSends === 0) return; counter.pendingExternalSends -= 1; } counter.spent -= 1; observer?.refund(); if (drawsReserve) reserveSpent = false; if (isAlternateTarget) alternateTargetSends -= 1; if (changesTarget) targetTransitions -= 1; lastTargetKey = previousTargetKey; }, }, }; }, }; sharedSendLedgers.set(budget, counter); return budget; } export function createRequestExecutionBudget( policy: RequestExecutionBudgetPolicy = CODEX_TEXT_GUARDED_BUDGET_POLICY, logicalRequestId?: string, observer?: RequestSendObserver, ): RequestExecutionBudget { const grant = createAmbiguousResendGrant(); return createRequestExecutionBudgetWithLedger(policy, logicalRequestId, { spent: 0, pendingExternalSends: 0, claimAmbiguousResend: grant.claimAmbiguousResend, get ambiguousResendSpent(): boolean { return grant.ambiguousResendSpent; }, ...(observer ? { observer } : {}), }); } /** * A budget that applies its own policy and keeps its own recovery ledgers while spending the * parent's exact physical-send ledger. * * Aliasing the public `used` property was not enough, and that is the whole defect. The factory * reads its own private counter back in `remainingBaseSends`, in the total check, and in the * reserve test, so an aliased scope answered every admission question from a counter that only * ever saw its own reservations. A combo's per-target holdback is computed from * `maxTotalModelSends` and is therefore unenforceable unless the scope actually observes what * the request has already spent. */ export function deriveRequestExecutionBudget( parent: RequestExecutionBudget, policy: RequestExecutionBudgetPolicy, ): RequestExecutionBudget { return createRequestExecutionBudgetWithLedger(policy, parent.logicalRequestId, ledgerFor(parent)); } /** * A budget that did not come from this factory still honors the public `used` contract, so * bridge onto it rather than failing the request. `isRequestExecutionBudget` is a shape test, * so a stub can reach here; turning that into a thrown error would convert a routing request * into a 500 to report a condition production never produces. Only a factory-backed parent can * share pending external bookings and a durable-spend observer, which are private by * construction; a bridged scope keeps the parent's spend accurate and books nothing of its own. */ /** * Grant claims made THROUGH a bridge, keyed by the bridged parent so every scope derived from it * sees them. A parent that predates `ambiguousResendSpent` can still grant through * `claimAmbiguousResend`; reading only its missing flag would report "not spent" after a derived * scope spent the grant, and a combo would then hop on a zero-output 200 from the replacement. */ const bridgedGrantClaims = new WeakMap<RequestExecutionBudget, { claimed: boolean }>(); function ledgerFor(parent: RequestExecutionBudget): SharedSendLedger { const existing = sharedSendLedgers.get(parent); if (existing) return existing; let pendingExternalSends = 0; let bridged = bridgedGrantClaims.get(parent); if (!bridged) { bridged = { claimed: false }; bridgedGrantClaims.set(parent, bridged); } const claims = bridged; return { get spent(): number { return parent.used; }, set spent(next: number) { parent.used = next; }, get pendingExternalSends(): number { return pendingExternalSends; }, set pendingExternalSends(next: number) { pendingExternalSends = next; }, // Asked of the parent rather than counted here. A local counter is a SECOND grant: two // scopes derived from one bridged parent, or one scope beside the parent it was derived // from, each replaced an unknown-state send once. Pending bookings and the durable-spend // observer genuinely cannot cross this boundary because they are private to the factory, // but the grant can -- `claimAmbiguousResend` is public on the parent. A parent that does // not implement it grants nothing, which is the fail-closed answer for a send whose // upstream state is unknown. claimAmbiguousResend: (limit: number): boolean => { const granted = parent.claimAmbiguousResend?.(limit) === true; if (granted) claims.claimed = true; return granted; }, get ambiguousResendSpent(): boolean { return claims.claimed || parent.ambiguousResendSpent === true; }, }; } export function isRequestExecutionBudget( value: TransientSendBudget | undefined, ): value is RequestExecutionBudget { return typeof (value as RequestExecutionBudget | undefined)?.reserveDispatch === "function"; }