UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

896 lines (846 loc) • 39.8 kB
/** * Root-workflow admission: a finite budget above the logical request (#4546). * * The per-request send budget bounds how many times ONE request reaches upstream. It cannot * bound how many requests a fan-out makes. A worker that spawns seven hundred children, each * of which sends exactly once, never violates a per-request cap and still spends the account. * That is the second half of the #4546 incident and it needs a ceiling of its own. * * The unit is the root workflow -- the user-visible task -- identified by the parent thread * header when the client supplies one. A retry is not a new user task and gets no new * allowance; a genuinely new top-level request does. * * Two caps intersect here. The COUNT caps (concurrency, distinct children, physical sends) * are process-local and in-memory. The TOKEN cap is the durable spend-reservation ledger in * spend-reservation-ledger.ts: when the caller supplies a spend request, admission also * reserves input + enforceable output ceiling against the root, identity and pool scopes, * and that accounting survives a restart. The count caps alone remain the guarantee for a * second process sharing the pool; the durable ledger's single-process topology is stated * in that module's header and applies here unchanged. */ import { sharedSpendLedger, spendCeilingsConfigured, type SpendAdmissionPolicy, type SpendReservationLedger, type SpendReservationProof, type SpendScope, type SpendUsage, } from "./spend-reservation-ledger"; /** * What a token-ceiling refusal has to be able to say. * * "Budget exhausted" on its own is the failure this repository keeps re-learning: a policy * rejection wearing another error's clothing sends an operator to look at the provider. The * scope says WHICH ceiling fired -- one task, one account, or the whole pool -- and the limit * is the number they would otherwise have to read the journal to recover. The scope ID is * deliberately not here: root ids are client thread headers and identity ids are credentials, * and the ledger's rule is that neither is written down in the clear. */ export interface WorkflowSpendDenialDetail { readonly scope: SpendScope; readonly limit: number; /** Tokens the refused reservation would have taken the scope to, where that is known. */ readonly projected?: number; /** A pool ceiling also includes unknown-owner history conservatively charged to every pool. */ readonly includesUnboundPoolHistory?: boolean; } /** Operator-facing name for each scope. What an operator calls it, not what the type calls it. */ const SPEND_SCOPE_LABEL: Record<SpendScope, string> = { root: "task", identity: "account", pool: "provider pool", }; /** * Thousands separators, done here rather than by `toLocaleString`. * * A ceiling is an eight- or nine-digit number and an unseparated one is genuinely hard to read * against the figure beside it. `toLocaleString` would do this too, but its output depends on * the ICU data the runtime happens to carry, and a message a test pins must not differ between * a developer's machine and a CI image. */ const formatTokenCount = (tokens: number): string => Math.trunc(tokens).toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); export interface WorkflowBudgetPolicy { /** Children admitted concurrently under one root. */ readonly maxConcurrentChildren: number; /** Physical model sends charged to one root INSIDE {@link WorkflowBudgetPolicy.windowMs}. */ readonly maxPhysicalSends: number; /** Distinct children one root may have inside the same window. */ readonly maxDistinctChildren: number; /** * The interval both counts are measured over. * * These were lifetime totals, and a lifetime total is the wrong instrument. The cap was * written against a fan-out that sends once per child seven hundred times, which is a RATE; * a running total cannot tell that from an ordinary session spread over an afternoon and * refuses both. Because the root id is the caller thread, for Codex that made the ceiling a * session expiry: a session reaching it was refused for the rest of the process even after * going idle for hours, and the only cure was restarting the proxy. * * Omitted means {@link WORKFLOW_DEFAULT_WINDOW_MS}. A count inside a window is never larger * than the same count over a lifetime, so windowing can only ever admit more for identical * traffic -- no install sees a refusal it would not have seen before. */ readonly windowMs?: number; /** * Concurrency slots a fan-out may never take. An interactive turn arriving into a saturated * root still gets admitted; without this a worker burst starves the conversation it serves. */ readonly interactiveReserve: number; /** * Roots tracked at once, as a hard bound rather than a hint. At the ceiling one idle root * with no unexpired count history or exhausted spend is evicted; when none is safe the new * root is REFUSED with `workflow-tracking-exhausted`. Admitting it anyway is what made a * caller minting new ids able to grow this map past the number written here. */ readonly maxTrackedRoots: number; } /** * Ten minutes. Long enough that the burst this ceiling was written against -- seven hundred * sends in a minute -- is still refused several times over, and short enough that an ordinary * session, which averages far less than a send every two seconds, never approaches it. */ export const WORKFLOW_DEFAULT_WINDOW_MS = 10 * 60_000; export const DEFAULT_WORKFLOW_BUDGET_POLICY: WorkflowBudgetPolicy = { maxConcurrentChildren: 8, maxPhysicalSends: 256, maxDistinctChildren: 64, interactiveReserve: 1, maxTrackedRoots: 512, windowMs: WORKFLOW_DEFAULT_WINDOW_MS, }; /** Fixed ring size. Ten minutes over twelve slots gives fifty-second granularity. */ const WORKFLOW_WINDOW_SLOTS = 12; function workflowWindowMs(policy: WorkflowBudgetPolicy): number { const declared = policy.windowMs; return declared !== undefined && Number.isFinite(declared) && declared > 0 ? declared : WORKFLOW_DEFAULT_WINDOW_MS; } /** * Slot size for one root's own window. * * The geometry is read off the state rather than off whatever policy the current caller * happens to hold. Two callers may legitimately pass different policies for the same root -- * the ceiling numbers are the caller's business -- but if they also disagreed about * `windowMs`, the slot ids one of them wrote would be on a scale the other cannot read, and * charging with a long window while reading with a short one makes every stored slot look * ancient and the ceiling never fire at all. */ function windowSlotMs(windowMs: number): number { return Math.max(1, Math.ceil(windowMs / WORKFLOW_WINDOW_SLOTS)); } /** * Add sends to the ring, resetting a slot whose turn has come round again. * * A ring rather than a list of timestamps because the storage has to be bounded: a root that * sends forever would otherwise grow forever, and this ledger exists to bound a fan-out. */ function recordWindowedSends(state: WorkflowState, now: number, sends: number): void { const slotMs = windowSlotMs(state.windowMs); const slot = Math.floor(now / slotMs); const index = ((slot % WORKFLOW_WINDOW_SLOTS) + WORKFLOW_WINDOW_SLOTS) % WORKFLOW_WINDOW_SLOTS; if (state.sendSlotAt[index] !== slot) { state.sendSlotAt[index] = slot; state.sendSlotCount[index] = 0; } state.sendSlotCount[index] = (state.sendSlotCount[index] ?? 0) + sends; } /** Sends inside the window. A slot older than the window contributes nothing. */ function windowedSends(state: WorkflowState, now: number): number { const slotMs = windowSlotMs(state.windowMs); const oldest = Math.floor(now / slotMs) - (WORKFLOW_WINDOW_SLOTS - 1); let total = 0; for (let index = 0; index < WORKFLOW_WINDOW_SLOTS; index += 1) { if ((state.sendSlotAt[index] ?? Number.NEGATIVE_INFINITY) >= oldest) { total += state.sendSlotCount[index] ?? 0; } } return total; } /** * Forget children last seen before the window opened, and report how many remain. * * Pruning on read keeps the map bounded without a timer: every admission pays for the children * it can still see, and a root that goes quiet is cleaned up the next time it speaks. */ function windowedChildren(state: WorkflowState, now: number): number { const cutoff = now - state.windowMs; for (const [childId, lastSeenMs] of state.children) { if (lastSeenMs <= cutoff) state.children.delete(childId); } return state.children.size; } export type WorkflowDenial = | "workflow-concurrency-exhausted" | "workflow-sends-exhausted" | "workflow-children-exhausted" | "workflow-spend-exhausted" /** * The root table is full and every entry retains activity, unexpired counts or exhausted * spend, so admitting this root would discard accounting. Refusing is the honest answer: * `maxTrackedRoots` is a bound, and inserting anyway made it a suggestion. */ | "workflow-tracking-exhausted" /** This send id was already reserved once; a repeat buys no second dispatch. */ | "workflow-send-replayed" /** The reservation could not be made durable, and a configured ceiling requires it. */ | "workflow-spend-undurable" | "workflow-pool-history-unresolved"; /** * The sentence an operator reads, plus the machine-readable name of the ceiling that fired. * * All four count denials used to share one sentence about a "concurrent-work limit", which was * accurate for exactly one of them. Worse, the wire cannot carry the distinction on its own: * `classifyError` rewrites every 429 to `rate_limit_error` / `rate_limit_exceeded`, so the body * of a refusal this proxy made is shaped exactly like a provider rate limit. Each sentence * therefore names which ceiling fired; send refusals describe the dispatch that was stopped * without making claims about earlier attempts in the same request. */ export function workflowDenialSummary( reason: WorkflowDenial, spend?: WorkflowSpendDenialDetail, ): { code: string; message: string } { switch (reason) { case "workflow-sends-exhausted": return { code: "workflow_sends_exhausted", message: "This proxy refused the request locally: the task reached its send ceiling for" + " the current window, so no provider was contacted. The window rolls forward on its" + " own; work already in flight settles as it finishes.", }; case "workflow-children-exhausted": return { code: "workflow_children_exhausted", message: "This proxy refused the request locally: the task reached its ceiling on" + " distinct child threads for the current window, so no provider was contacted." + " A child that goes quiet ages out of the count.", }; case "workflow-concurrency-exhausted": return { code: "workflow_concurrency_exhausted", message: "This proxy refused the request locally: the task has no free concurrency slot," + " so no provider was contacted. Slots are released as the turns holding them finish.", }; case "workflow-spend-exhausted": return { code: "workflow_spend_exhausted", // With the denial in hand the sentence names the ceiling that fired and its number, // because the alternative is an operator who can see that something refused and has // no way to find out what. Without one -- a caller that knows only the reason -- the // original sentence is kept unchanged. message: spend && Object.hasOwn(SPEND_SCOPE_LABEL, spend.scope) && Number.isSafeInteger(spend.limit) && spend.limit >= 0 && (spend.projected === undefined || (Number.isSafeInteger(spend.projected) && spend.projected >= 0)) ? "This proxy refused the request locally: the configured " + SPEND_SCOPE_LABEL[spend.scope] + " token ceiling of " + formatTokenCount(spend.limit) + " is spent" + (spend.projected !== undefined ? " (this send would have taken it to " + formatTokenCount(spend.projected) + ")" : "") + (spend.includesUnboundPoolHistory ? ". The total includes unassigned historical provider-pool balances, conservatively counted against every provider pool until verified mappings assign them; this can overrestrict otherwise unused pools" : "") + ", so this send was refused before contacting a provider. Spend is durable, so it does not roll forward" + " with the send window: raise or remove spend." + spend.scope + ".maxTokens in config.json to grant more." : "This proxy refused the request locally: the task reached a configured token" + " ceiling, so no provider was contacted.", }; case "workflow-tracking-exhausted": return { code: "workflow_tracking_exhausted", message: "This proxy refused the request locally: it is already tracking as many tasks as" + " it may, and every one retains activity, unexpired counts or exhausted spend, so no provider was" + " contacted.", }; case "workflow-send-replayed": return { code: "workflow_send_replayed", message: "This proxy refused the request locally: this send was already reserved once, and" + " a repeat buys no second dispatch.", }; case "workflow-pool-history-unresolved": return { code: "workflow_pool_history_unresolved", message: "Historical provider-pool spend needs verified alias mappings before dispatch. No provider was contacted." }; case "workflow-spend-undurable": return { code: "workflow_spend_undurable", message: "This proxy refused the request locally: the token reservation could not be made" + " durable and a configured ceiling requires it, so no provider was contacted.", }; } } /** * Response header naming the ceiling that refused, on a refusal this proxy made itself. * * It exists because the body cannot carry it: `classifyError` rewrites every 429 to * `rate_limit_error` / `rate_limit_exceeded`, so a local refusal and a provider rate limit are * byte-identical in shape. Changing that classification would change how every client retries, * so the name goes beside the body instead. Response projections require the in-process proxy * marker before carrying this header, so an upstream response cannot forge local provenance. */ export const WORKFLOW_LOCAL_REFUSAL_HEADER = "x-opencodex-local-refusal"; export type WorkflowBudgetEventKind = "refused" | "cleared"; export interface WorkflowBudgetEvent { readonly at: number; readonly kind: WorkflowBudgetEventKind; readonly rootId: string; /** The ceiling that fired. Present for `refused`, absent for `cleared`. */ readonly reason?: WorkflowDenial; /** Which token scope refused, on a spend denial. Absent on every count denial. */ readonly spendScope?: SpendScope; /** That scope's ceiling, so the event is readable without the config open beside it. */ readonly spendLimit?: number; /** The pool ceiling conservatively included unidentified historical pool balances. */ readonly spendIncludesUnboundPoolHistory?: boolean; /** Windowed sends at the moment of the event. */ readonly sends: number; /** Windowed distinct children at the moment of the event. */ readonly children: number; } /** * How many events are kept. Small on purpose: this is an operator's recent-history view, not an * audit log, and it lives in the same process memory the ceilings do. */ export const WORKFLOW_EVENT_CAPACITY = 64; const budgetEvents: WorkflowBudgetEvent[] = []; /** * Record a local budget decision. * * This exists because the refusal has nowhere else to go. The HTTP admission check runs before * the body is parsed, so there is no model, no provider and no request-log context to attach to; * writing a usage row there would mean inventing both. Every entry here is by construction a * decision this proxy made without contacting anyone, which is a stronger statement than a flag * on a row shared with upstream results. */ function recordBudgetEvent(event: WorkflowBudgetEvent): void { budgetEvents.push(event); while (budgetEvents.length > WORKFLOW_EVENT_CAPACITY) budgetEvents.shift(); } /** Newest first. `limit` is clamped to what is actually kept. */ export function listWorkflowBudgetEvents(limit: number = WORKFLOW_EVENT_CAPACITY): WorkflowBudgetEvent[] { const wanted = Number.isFinite(limit) && limit > 0 ? Math.min(Math.floor(limit), WORKFLOW_EVENT_CAPACITY) : 0; if (wanted === 0) return []; return budgetEvents.slice(-wanted).reverse(); } /** * Record a refusal decided outside `admitWorkflowTurn`. * * The pre-dispatch ceiling check in the responses path is a second refusal, taken after * admission already succeeded, so nothing in this module sees it. Without this it was the one * refusal an operator could hit that left no event behind. */ export function recordWorkflowRefusalEvent( rootId: string | undefined, reason: WorkflowDenial, now: number = Date.now(), spend?: WorkflowSpendDenialDetail, ): void { if (!rootId) return; const state = roots.get(rootId); recordBudgetEvent({ at: now, kind: "refused", rootId, reason, ...(spend ? { spendScope: spend.scope, spendLimit: spend.limit } : {}), ...(spend?.includesUnboundPoolHistory ? { spendIncludesUnboundPoolHistory: true } : {}), sends: state ? windowedSends(state, now) : 0, children: state ? windowedChildren(state, now) : 0, }); } export type WorkflowLane = "interactive" | "worker"; export interface WorkflowAdmission { readonly rootId: string; /** * The request is about to leave for upstream. Call this at the dispatch boundary: until it * runs, releasing the lease costs nothing, and after it a missing usage frame is booked as * unresolved spend. */ markDispatched(): void; release(): void; } export type WorkflowDecision = | { admitted: true; lease: WorkflowAdmission } | { admitted: false; reason: WorkflowDenial; rootId: string; /** Which spend scope refused, when the denial came from the token ledger. */ spendScope?: SpendScope; /** That scope's configured ceiling, so a caller can say what it was. */ spendLimit?: number; /** Tokens the refused reservation would have taken the scope to, where known. */ spendProjected?: number; /** A pool ceiling includes unbound historical balances for every candidate pool. */ spendIncludesUnboundPoolHistory?: boolean; }; /** * Token reservation attached to an admission. `outputCeilingTokens` is the ENFORCEABLE * ceiling -- the caller's max_output_tokens or the model's documented cap, never an * optimistic estimate and never shrunk by a cache-hit expectation. Omitting `spend` * entirely keeps the historical count-only admission, which is also what an unconfigured * install gets: token accounting is observed by default and refuses nothing until an * operator sets real limits. */ export interface WorkflowSpendRequest { /** Stable id of the physical send; settlement is idempotent on this key. */ readonly sendId: string; readonly identityId?: string; readonly poolId?: string; readonly inputTokens: number; readonly outputCeilingTokens: number; } interface WorkflowState { active: number; /** Lifetime total, kept for diagnostics only. The ceiling reads the window instead. */ sends: number; /** Ring of per-slot send counts; sendSlotAt[i] names the slot that bucket holds. */ sendSlotCount: number[]; sendSlotAt: number[]; /** Child id to the last time it was admitted, so a child that stops ages out of the count. */ children: Map<string, number>; lastSeenMs: number; /** Window this root's ring and child map are measured over, fixed when the root appeared. */ windowMs: number; } function newWorkflowState(now: number, policy: WorkflowBudgetPolicy): WorkflowState { return { active: 0, sends: 0, sendSlotCount: new Array<number>(WORKFLOW_WINDOW_SLOTS).fill(0), sendSlotAt: new Array<number>(WORKFLOW_WINDOW_SLOTS).fill(Number.NEGATIVE_INFINITY), children: new Map<string, number>(), lastSeenMs: now, windowMs: workflowWindowMs(policy), }; } const roots = new Map<string, WorkflowState>(); /** * Evict the oldest root that is safe to forget, and report whether one was found. * * The return value is the point. An earlier version returned void and the caller inserted * the new root regardless, so `maxTrackedRoots` bounded nothing whenever every candidate * was active or exhausted -- which is precisely the fan-out this file exists to bound. */ function evictOneRoot( spendLedger?: SpendReservationLedger, now: number = Date.now(), ): boolean { let oldestKey: string | undefined; let oldestAt = Number.POSITIVE_INFINITY; for (const [key, state] of roots) { // Dropping active or unexpired count state would hand the same root fresh allowances, // even when it used only part of a ceiling. Durable spend remains independently protected. if (state.active > 0) continue; if (windowedSends(state, now) > 0 || windowedChildren(state, now) > 0) continue; if (spendLedger?.exhausted("root", key) === true) continue; if (state.lastSeenMs < oldestAt) { oldestAt = state.lastSeenMs; oldestKey = key; } } if (oldestKey === undefined) return false; roots.delete(oldestKey); return true; } /** * Admit one turn under a root workflow. * * `childId` distinguishes the members of a fan-out; omit it for the root's own turns. * An interactive lane may use the reserved slots a worker lane may not. * * When `spend` is given, admission also reserves its tokens on the spend ledger -- at the * root, identity and pool scopes at once -- before a concurrency slot is taken. A turn * released without settlement is resolved by whether it was ever DISPATCHED: an undispatched * turn gives its tokens back, and a dispatched one keeps them as unresolved spend, because a * send whose usage never arrived may still have been billed. Call `lease.markDispatched()` * at the point the request leaves for upstream; without it, admission followed by a local * validation or routing failure would book spend that never happened. */ export function admitWorkflowTurn( rootId: string | undefined, lane: WorkflowLane, policy: WorkflowBudgetPolicy = DEFAULT_WORKFLOW_BUDGET_POLICY, childId?: string, now: number = Date.now(), spend?: WorkflowSpendRequest, spendLedger?: SpendReservationLedger, ): WorkflowDecision | undefined { if (!rootId) return undefined; // An explicit ledger is consulted even without a spend request, so root eviction can // still see spend-exhausted entries. The shared one is resolved whenever a ceiling is // CONFIGURED, which is what lets admission refuse an already-spent scope before a body is // parsed. An install that configured nothing resolves no ledger, opens no journal, and runs // this function exactly as it did before -- the unconfigured path has to stay byte-identical // because the ledger is on and journalling by default. const ledger = spendLedger ?? (spend || spendCeilingsConfigured() ? sharedSpendLedger() : undefined); let state = roots.get(rootId); // Every refusal below goes on the record through this one seam. Recording at each return // site instead of at the HTTP caller is what makes the record complete: the spend denials // are decided inside the ledger branch and never surface as a distinct reason to the caller // that formats the response. const refuse = (reason: WorkflowDenial, denial?: WorkflowSpendDenialDetail): WorkflowDecision => { const current = roots.get(rootId); recordBudgetEvent({ at: now, kind: "refused", rootId, reason, ...(denial ? { spendScope: denial.scope, spendLimit: denial.limit } : {}), sends: current ? windowedSends(current, now) : 0, children: current ? windowedChildren(current, now) : 0, }); return { admitted: false, reason, rootId, ...(denial ? { spendScope: denial.scope, spendLimit: denial.limit, ...(denial.projected !== undefined ? { spendProjected: denial.projected } : {}), ...(denial.includesUnboundPoolHistory ? { spendIncludesUnboundPoolHistory: true } : {}), } : {}), }; }; if (!state) { if (roots.size >= policy.maxTrackedRoots && !evictOneRoot(ledger, now)) { // Nothing may be forgotten, so the new root is refused instead of admitted over the // bound. Evicting unexpired history would reset an existing allowance, and a caller // minting fresh ids could repeatedly recover budget from the forgotten counts. return refuse("workflow-tracking-exhausted"); } state = newWorkflowState(now, policy); roots.set(rootId, state); } state.lastSeenMs = now; if (windowedSends(state, now) >= policy.maxPhysicalSends) { return refuse("workflow-sends-exhausted"); } if (childId !== undefined && !state.children.has(childId) && windowedChildren(state, now) >= policy.maxDistinctChildren) { return refuse("workflow-children-exhausted"); } const ceiling = lane === "worker" ? Math.max(0, policy.maxConcurrentChildren - policy.interactiveReserve) : policy.maxConcurrentChildren; if (state.active >= ceiling) { return refuse("workflow-concurrency-exhausted"); } // The counts are checked first and the token ceiling second, and the order is deliberate // rather than emergent. A count check reads two integers this process already holds; a token // check may have to build the ledger and replay its journal. Checking the cheap bound first // means the expensive one is never reached for a request the cheap one already refused. // // The two therefore CAN disagree, and the intersection is what is enforced: a request passes // only when every count cap and every token ceiling admits it. A token denial happens before // any count is charged, and a count denial happens before any reservation is booked, so // neither leaves the other's accounting to unwind. Whichever refuses first is reported as // itself -- one refusal is never relabelled as the other, because "sends exhausted" and // "spend exhausted" send an operator to two different remedies. // // A scope whose ceiling is ALREADY spent is refused here rather than at the reservation. The // reservation needs a token count, which is not known until the body is parsed and a route // resolved; an exhausted scope needs neither and is the cheapest refusal available. if (!spend && ledger) { const reached = spentRootCeiling(rootId, ledger); if (reached) return refuse("workflow-spend-exhausted", reached); } if (spend && ledger) { const decision = ledger.reserve({ sendId: spend.sendId, scopes: { rootId, identityId: spend.identityId, poolId: spend.poolId }, inputTokens: spend.inputTokens, outputCeilingTokens: spend.outputCeilingTokens, at: now, }); if (!decision.reserved) { const denial = decision.denial; // Every ledger refusal denies a DISPATCH. A duplicate send id and an undurable // reservation are reported as themselves rather than folded into "exhausted", because // an operator reading a 429 needs to know which of the three happened. const reason: WorkflowDenial = denial.reason === "duplicate-send-id" ? "workflow-send-replayed" : denial.reason === "pool-history-unresolved" ? "workflow-pool-history-unresolved" : denial.reason === "reserve-not-durable" || denial.reason === "journal-corrupt" ? "workflow-spend-undurable" : denial.reason === "tracking-capacity-exhausted" ? "workflow-tracking-exhausted" : "workflow-spend-exhausted"; return refuse( reason, denial.reason === "spend-limit-exceeded" ? { scope: denial.scope, limit: denial.limit, projected: denial.projected, ...(denial.includesUnboundPoolHistory ? { includesUnboundPoolHistory: true } : {}), } : undefined, ); } } state.active += 1; if (childId !== undefined) state.children.set(childId, now); let released = false; return { admitted: true, lease: { rootId, markDispatched(): void { if (spend && ledger) ledger.markDispatched(spend.sendId); }, release(): void { if (released) return; released = true; const current = roots.get(rootId); if (current) { current.active = Math.max(0, current.active - 1); // Eviction ordering only; no ceiling reads lastSeenMs, so the wall clock is the // right source here and a caller does not need to inject one. current.lastSeenMs = Date.now(); } // Which of the two applies depends on whether the send ever left this process. // `abandon` succeeds only while the reservation is undispatched -- a turn refused by // local validation or routing releases its tokens and books nothing, because // inventing debt the account never incurred breaks the budget in the other // direction. Once dispatched, abandon refuses and markLost keeps the cost as // unresolved spend, since a send whose usage frame never arrived may still have been // billed. Both are no-ops once settleWorkflowSpend already ran. if (spend && ledger && !ledger.abandon(spend.sendId)) ledger.markLost(spend.sendId); }, }, }; } /** * Charge physical sends to a root. Called from the send budget's own accounting so a retry * inside one request counts toward the workflow total, not only the request total. */ export function chargeWorkflowSends( rootId: string | undefined, sends: number, now: number = Date.now(), ): void { if (!rootId || sends <= 0) return; const state = roots.get(rootId); if (!state) return; state.sends += sends; // Geometry comes off the root itself, so no caller can charge on one scale and read on // another. This function does not take a policy at all any more: it has no ceiling to // compare, and the only thing a policy could have supplied here was that scale. recordWindowedSends(state, now, sends); state.lastSeenMs = now; } /** * Atomically reserve one physical send under a root's live count ceiling. * * JavaScript runs this read-and-increment without an await, so concurrent request and retry * ladders cannot all observe the same remaining slot. Call only at physical admission, after * unused dispatch permits can be released; a claimed physical start remains counted. */ export function reserveWorkflowSend( rootId: string | undefined, policy: WorkflowBudgetPolicy = DEFAULT_WORKFLOW_BUDGET_POLICY, now: number = Date.now(), ): boolean { if (!rootId) return true; const state = roots.get(rootId); if (!state) return true; if (windowedSends(state, now) >= policy.maxPhysicalSends) return false; state.sends += 1; recordWindowedSends(state, now, 1); state.lastSeenMs = now; return true; } /** * Settle a send's reservation with the usage the response actually reported. Idempotent * per send id -- a second call returns false and books nothing. When the usage frame was * lost, call this never and let the lease's release move the reservation to unresolved * spend, or call the ledger's markLost directly. */ export function settleWorkflowSpend( sendId: string, usage: SpendUsage, spendLedger?: SpendReservationLedger, ): boolean { return (spendLedger ?? sharedSpendLedger()).settle(sendId, usage); } /** * Record that the send left for upstream. * * This is the line between "may be released for free" and "may have been billed". Admission * alone is not dispatch: a turn can be admitted and then fail request validation, provider * routing, or a local guard without a single byte reaching a model. Booking those as spend * invents debt the account never incurred, so the reservation only becomes unresolvable * after this call. */ export function dispatchWorkflowSpend(sendId: string, spendLedger?: SpendReservationLedger): boolean { return (spendLedger ?? sharedSpendLedger()).markDispatched(sendId); } /** * Give a reservation back because the send never happened. Refused once dispatched, where * settle or markLost is the only honest outcome. */ export function abandonWorkflowSpend(sendId: string, spendLedger?: SpendReservationLedger): boolean { return (spendLedger ?? sharedSpendLedger()).abandon(sendId); } /** * Whether this root has already spent its whole physical-send ceiling. * * Separate from `admitWorkflowTurn` so a caller can refuse before dispatch without taking a * concurrency slot it would have to remember to release. */ export function workflowSendCeilingReached( rootId: string | undefined, policy: WorkflowBudgetPolicy = DEFAULT_WORKFLOW_BUDGET_POLICY, now: number = Date.now(), ): boolean { if (!rootId) return false; const state = roots.get(rootId); return state !== undefined && windowedSends(state, now) >= policy.maxPhysicalSends; } /** * The root scope's ceiling when that scope is already spent, or undefined. * * Root only: identity and pool are not known until routing has picked an account, so those two * refuse at the reservation itself. `exhausted` sums settled spend, open reservations and * unresolved spend, which is the same total the reservation compares, so this answers the same * question the reservation would -- just without needing the request's token count. */ function spentRootCeiling( rootId: string, ledger: SpendReservationLedger, excludingSendId?: string, admissionPolicy?: SpendAdmissionPolicy, ): WorkflowSpendDenialDetail | undefined { const limit = (admissionPolicy ?? ledger.policy).root.maxTokens; if (limit === undefined) return undefined; return ledger.exhausted("root", rootId, excludingSendId, admissionPolicy) ? { scope: "root", limit } : undefined; } /** * A root or routed pool ceiling already spent, or undefined when neither is exhausted. * * The count-side twin of {@link workflowSendCeilingReached}, and the responses path calls both * at the same seam for the same reason: a refusal decided before dispatch can be reported as * ITSELF -- a named ceiling, a synthetic log row, a machine-readable header -- instead of * surfacing later as a generic send-budget error from whichever leg happened to run out first. * * Returns undefined when no ceiling is configured, without resolving a ledger, so an install * that never opted in neither pays for this check nor opens a journal because of it. */ export function workflowSpendCeilingReached( rootId: string | undefined, spendLedger?: SpendReservationLedger, poolId?: string, reservation?: SpendReservationProof, admissionPolicy?: SpendAdmissionPolicy, ): WorkflowSpendDenialDetail | undefined { if (!rootId && !poolId) return undefined; // An explicit empty snapshot belongs to an observe-only request, even if current config // enables a ceiling. Conversely, old enforced requests still read totals after disable. if (admissionPolicy && admissionPolicy.root.maxTokens === undefined && admissionPolicy.pool.maxTokens === undefined) return undefined; const ledger = spendLedger ?? (admissionPolicy || spendCeilingsConfigured() ? sharedSpendLedger() : undefined); if (!ledger) return undefined; // This reads the current alias partition; it never publishes continuity metadata. const excludingSendId = reservation?.ledger === ledger ? reservation.sendId : undefined; const root = rootId ? spentRootCeiling(rootId, ledger, excludingSendId, admissionPolicy) : undefined; if (root) return root; const limit = (admissionPolicy ?? ledger.policy).pool.maxTokens; return poolId && limit !== undefined && ledger.exhausted("pool", poolId, excludingSendId, admissionPolicy) ? { scope: "pool", limit, ...(ledger.hasUnboundPositivePoolHistory() ? { includesUnboundPoolHistory: true } : {}), } : undefined; } export interface WorkflowBudgetSnapshot { active: number; /** Sends inside the window. This is the number the ceiling compares. */ sends: number; /** Children inside the window, which is likewise what the ceiling compares. */ children: number; /** Everything the root has ever sent, for diagnostics; no ceiling reads it. */ lifetimeSends: number; windowMs: number; maxPhysicalSends: number; maxDistinctChildren: number; } export function workflowBudgetSnapshot( rootId: string, policy: WorkflowBudgetPolicy = DEFAULT_WORKFLOW_BUDGET_POLICY, now: number = Date.now(), ): WorkflowBudgetSnapshot | undefined { const state = roots.get(rootId); if (!state) return undefined; return { active: state.active, sends: windowedSends(state, now), children: windowedChildren(state, now), lifetimeSends: state.sends, windowMs: state.windowMs, maxPhysicalSends: policy.maxPhysicalSends, maxDistinctChildren: policy.maxDistinctChildren, }; } /** * Roots this process is currently tracking, most recently active first. * * Bounded by `limit` because `maxTrackedRoots` is 512 and an operator asking what is going on * wants the busy end of that, not a dump. */ export function listTrackedWorkflowRoots( limit = 64, policy: WorkflowBudgetPolicy = DEFAULT_WORKFLOW_BUDGET_POLICY, now: number = Date.now(), ): Array<{ rootId: string } & WorkflowBudgetSnapshot> { const wanted = Number.isFinite(limit) && limit > 0 ? Math.floor(limit) : 0; if (wanted === 0) return []; return [...roots.entries()] .sort((left, right) => right[1].lastSeenMs - left[1].lastSeenMs) .slice(0, wanted) .flatMap(([rootId]) => { const snapshot = workflowBudgetSnapshot(rootId, policy, now); return snapshot ? [{ rootId, ...snapshot }] : []; }); } /** * Clear ONE root's windowed count ceilings, and report what they were. * * Three things are deliberately left alone. `active` belongs to turns still in flight, and * zeroing it would let their releases drive the count negative and hand out concurrency slots * that are already taken. The spend ledger is a token budget an operator did not ask to * forgive, and a count ceiling is not a licence to reset it. `sends` -- the lifetime total -- * survives too, so the record of what this root actually did cannot be laundered by clearing * it; only the ceilings move. * * Returns the snapshot taken immediately before the clear, so the caller can put on the record * what it forgave, or `undefined` when the root is not tracked at all. */ export function clearWorkflowBudgetForRoot( rootId: string, policy: WorkflowBudgetPolicy = DEFAULT_WORKFLOW_BUDGET_POLICY, now: number = Date.now(), ): WorkflowBudgetSnapshot | undefined { const state = roots.get(rootId); if (!state) return undefined; const before = workflowBudgetSnapshot(rootId, policy, now); state.sendSlotCount.fill(0); state.sendSlotAt.fill(Number.NEGATIVE_INFINITY); state.children.clear(); state.lastSeenMs = now; recordBudgetEvent({ at: now, kind: "cleared", rootId, sends: before?.sends ?? 0, children: before?.children ?? 0, }); return before; } /** Test seam. Production never clears a live ledger: that would reset a spent budget. */ export function resetWorkflowBudgetsForTest(): void { roots.clear(); budgetEvents.length = 0; }