@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
359 lines (336 loc) • 15.7 kB
text/typescript
import { randomUUID } from "node:crypto";
import { isCodexAccountGenerationLive, readCodexAccountRecord, type CodexRefreshProvenance } from "../account-store";
import { isCodexAccountPaused } from "../account-pause";
import { isSelectableCodexPoolAccount } from "../account-id";
import { isAccountNeedsReauth } from "../account-runtime-state";
import { MAIN_CODEX_ACCOUNT_ID } from "../main-account";
import type { OcxConfig } from "../../types";
import { CODEX_QUOTA_PROBE_INTERVAL_MS, type CodexUpstreamOutcomeMeta } from "./cooldown-math";
import {
deleteScopedHealth,
getAccountHealth,
listScopedHealthEntries,
scopedHealthFor,
setAccountHealth,
setScopedHealth,
type CodexQuotaScope,
type CodexUpstreamHealth,
} from "./health-store";
export type CodexQuotaRecoveryProbeClaim = {
accountId: string;
scope?: CodexQuotaScope;
leaseId: string;
cooldownGeneration: number;
credentialGeneration: number;
/** Claim-time `replacedAt`; unchanged after a probe-owned refresh, stamped on external replacement. */
credentialReplacedAt?: number;
};
export type CodexQuotaRecoveryProbeProof = {
credentialGeneration?: number;
};
/**
* Grant at most one probe lease per interval for a cooled-down account.
*
* A cooled-down account is short-circuited locally, so it never sends traffic and
* no organic 2xx can prove that upstream quota recovered — the cooldown can only
* end by expiry or a proxy restart (#433). Releasing a single probe breaks that
* deadlock. Explicit Retry-After cooldowns are excluded: those are literal retry
* directives, not window announcements.
*
* Returns the lease id, or null when no probe may go out right now.
*/
export function tryAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): string | null {
if (!canAcquireCodexQuotaProbeLease(accountId, now)) return null;
const health = getAccountHealth(accountId)!;
const probeLeaseId = randomUUID();
setAccountHealth(accountId, {
...health,
probeLeaseId,
probeLeaseGeneration: health.cooldownGeneration ?? 0,
lastProbeAt: now,
});
return probeLeaseId;
}
/** Side-effect-free check mirroring {@link tryAcquireCodexQuotaProbeLease} eligibility. */
export function canAcquireCodexQuotaProbeLease(accountId: string, now = Date.now()): boolean {
return canAcquireQuotaProbeLease(getAccountHealth(accountId), now);
}
function canAcquireQuotaProbeLease(health: CodexUpstreamHealth | undefined, now: number): boolean {
if (!health) return false;
const cooldownUntil = health.cooldownUntil;
if (typeof cooldownUntil !== "number" || !Number.isFinite(cooldownUntil) || cooldownUntil <= now) return false;
if (health.cooldownSource === "retry-after") return false;
if (health.probeLeaseId !== undefined) return false;
const origin = health.lastProbeAt ?? health.cooldownSince ?? cooldownUntil;
return now - origin >= CODEX_QUOTA_PROBE_INTERVAL_MS;
}
/**
* Claim due reset-derived cooldown probes without consulting account selection.
* Added Pool credentials only; owned main usage recovery is handled separately.
*/
export function claimDueCodexQuotaRecoveryProbes(
config: OcxConfig,
limit: number,
now = Date.now(),
): CodexQuotaRecoveryProbeClaim[] {
const boundedLimit = Math.max(0, Math.floor(limit));
if (boundedLimit === 0) return [];
const candidates: Array<{
accountId: string;
scope?: CodexQuotaScope;
health: CodexUpstreamHealth;
credentialGeneration: number;
credentialReplacedAt?: number;
order: number;
}> = [];
for (const [order, account] of (config.codexAccounts ?? []).entries()) {
if (!isSelectableCodexPoolAccount(account)
|| isCodexAccountPaused(config, account.id)
|| isAccountNeedsReauth(account.id)) continue;
const record = readCodexAccountRecord(account.id);
if (!record?.credential || record.deletedAt != null) continue;
const due = [
{ scope: undefined, health: getAccountHealth(account.id) },
...[...(listScopedHealthEntries(account.id))].map(([scope, health]) => ({ scope, health })),
].filter((entry): entry is { scope?: CodexQuotaScope; health: CodexUpstreamHealth } =>
// Generic WHAM evidence can recover only ordinary quota, never Reserve.
// Do not spend this account's one claim per pass on an independent scope and
// delay the shared scope that the response can actually recover.
(entry.scope === undefined || entry.scope === "shared")
&& entry.health?.cooldownSource === "reset-derived"
&& canAcquireQuotaProbeLease(entry.health, now))
.sort((a, b) =>
(a.health.lastProbeAt ?? a.health.cooldownSince ?? 0)
- (b.health.lastProbeAt ?? b.health.cooldownSince ?? 0));
const candidate = due[0];
if (candidate) candidates.push({
accountId: account.id,
...(candidate.scope ? { scope: candidate.scope } : {}),
health: candidate.health,
credentialGeneration: record.generation,
...(record.replacedAt !== undefined ? { credentialReplacedAt: record.replacedAt } : {}),
order,
});
}
candidates.sort((a, b) => {
const age = (a.health.lastProbeAt ?? a.health.cooldownSince ?? 0)
- (b.health.lastProbeAt ?? b.health.cooldownSince ?? 0);
return age || a.order - b.order;
});
return candidates.slice(0, boundedLimit).map(candidate => {
const leaseId = randomUUID();
const next = {
...candidate.health,
probeLeaseId: leaseId,
probeLeaseGeneration: candidate.health.cooldownGeneration ?? 0,
lastProbeAt: now,
};
if (candidate.scope) setScopedHealth(candidate.accountId, candidate.scope, next);
else setAccountHealth(candidate.accountId, next);
return {
accountId: candidate.accountId,
...(candidate.scope ? { scope: candidate.scope } : {}),
leaseId,
cooldownGeneration: candidate.health.cooldownGeneration ?? 0,
credentialGeneration: candidate.credentialGeneration,
...(candidate.credentialReplacedAt !== undefined
? { credentialReplacedAt: candidate.credentialReplacedAt }
: {}),
};
});
}
type CooldownRecoveryLease = Pick<CodexQuotaRecoveryProbeClaim,
"accountId" | "scope" | "leaseId" | "cooldownGeneration">;
export type ManualResetCooldownClaim =
| { kind: "pool"; probe: CodexQuotaRecoveryProbeClaim }
| { kind: "main"; probe: CooldownRecoveryLease };
function manualResetAccountEligible(config: OcxConfig, accountId: string): boolean {
return !isCodexAccountPaused(config, accountId) && !isAccountNeedsReauth(accountId)
&& (accountId === MAIN_CODEX_ACCOUNT_ID
|| (config.codexAccounts ?? []).some(account => account.id === accountId && isSelectableCodexPoolAccount(account)));
}
/** Explicit reset bypasses probe pacing, never another owner's lease or quota scope. */
export function claimManualResetCooldowns(
config: OcxConfig,
accountId: string,
now = Date.now(),
expectedPoolGeneration?: number,
): ManualResetCooldownClaim[] {
if (!manualResetAccountEligible(config, accountId)) return [];
const record = accountId === MAIN_CODEX_ACCOUNT_ID ? undefined : readCodexAccountRecord(accountId);
if (accountId !== MAIN_CODEX_ACCOUNT_ID && (!record?.credential || record.deletedAt != null)) return [];
if (record && expectedPoolGeneration !== undefined && record.generation !== expectedPoolGeneration) return [];
const claims: ManualResetCooldownClaim[] = [];
for (const scope of [undefined, "shared"] as const) {
const health = scope ? scopedHealthFor(accountId, scope) : getAccountHealth(accountId);
if (!health || health.cooldownSource !== "reset-derived" || health.probeLeaseId !== undefined
|| !Number.isFinite(health.cooldownUntil) || !(health.cooldownUntil! > now)) continue;
const leaseId = randomUUID();
const cooldownGeneration = health.cooldownGeneration ?? 0;
const next = { ...health, probeLeaseId: leaseId, probeLeaseGeneration: cooldownGeneration, lastProbeAt: now };
if (scope) setScopedHealth(accountId, scope, next);
else setAccountHealth(accountId, next);
const probe = { accountId, scope, leaseId, cooldownGeneration };
claims.push(record ? { kind: "pool", probe: {
...probe, credentialGeneration: record.generation, credentialReplacedAt: record.replacedAt,
} } : { kind: "main", probe });
}
return claims;
}
export type ManualResetRefreshLineage = Readonly<{
fromGeneration: number;
toGeneration: number;
provenance: CodexRefreshProvenance;
}>;
type ManualResetQuotaProof = CodexQuotaRecoveryProbeProof & {
refreshLineage?: ManualResetRefreshLineage;
};
/** Main proof is checked by the already-owned auth operation, never by a Pool record. */
export function settleManualResetCooldown(
config: OcxConfig,
claim: ManualResetCooldownClaim,
recovered: boolean,
proof: ManualResetQuotaProof = {},
now = Date.now(),
): boolean {
if (!recovered) return settleCooldownRecoveryLease(claim.probe, false, now);
const eligible = manualResetAccountEligible(config, claim.probe.accountId);
if (claim.kind === "main") return settleCooldownRecoveryLease(claim.probe, eligible, now);
const lineage = proof.refreshLineage;
// Equal wall-clock replacement stamps do not establish ancestry. Manual +1
// recovery additionally needs the actual forced-refresh result for this edge.
const ownedGeneration = proof.credentialGeneration === claim.probe.credentialGeneration
|| (proof.credentialGeneration === claim.probe.credentialGeneration + 1
&& lineage?.fromGeneration === claim.probe.credentialGeneration
&& lineage.toGeneration === proof.credentialGeneration
&& (lineage.provenance === "self-refresh" || lineage.provenance === "joined-lineage"));
return settleCodexQuotaRecoveryProbe(claim.probe, eligible && ownedGeneration, proof, now);
}
/** Settle one background recovery claim without mutating account-wide outcome state. */
export function settleCodexQuotaRecoveryProbe(
claim: CodexQuotaRecoveryProbeClaim,
recovered: boolean,
proof: CodexQuotaRecoveryProbeProof,
now = Date.now(),
): boolean {
const health = claim.scope
? scopedHealthFor(claim.accountId, claim.scope)
: getAccountHealth(claim.accountId);
if (!health || health.probeLeaseId !== claim.leaseId) return false;
const currentRecord = readCodexAccountRecord(claim.accountId);
const proofGeneration = proof.credentialGeneration;
// A probe-owned token refresh (getValidCodexToken) advances the credential generation by
// exactly one while preserving `replacedAt`; an external credential replacement bumps the
// generation too but stamps a fresh `replacedAt`. Accept the +1 transition only when the
// claim-time lineage is intact AND the generation the fresh quota was proven under is live.
const generationFenced = proofGeneration !== undefined
&& (proofGeneration === claim.credentialGeneration
? isCodexAccountGenerationLive(claim.accountId, proofGeneration)
: proofGeneration === claim.credentialGeneration + 1
&& currentRecord?.replacedAt === claim.credentialReplacedAt
&& isCodexAccountGenerationLive(claim.accountId, proofGeneration));
return settleCooldownRecoveryLease(claim, recovered && generationFenced, now);
}
function settleCooldownRecoveryLease(claim: CooldownRecoveryLease, recovered: boolean, now: number): boolean {
const health = claim.scope ? scopedHealthFor(claim.accountId, claim.scope) : getAccountHealth(claim.accountId);
if (!health || health.probeLeaseId !== claim.leaseId) return false;
const fenced = (claim.scope === undefined || claim.scope === "shared")
&& health.cooldownSource === "reset-derived"
&& (health.cooldownGeneration ?? 0) === claim.cooldownGeneration
&& (health.probeLeaseGeneration ?? 0) === claim.cooldownGeneration;
if (!recovered || !fenced) {
const released = withProbeLeaseReleased(health, now);
if (claim.scope) setScopedHealth(claim.accountId, claim.scope, released);
else setAccountHealth(claim.accountId, released);
return false;
}
if (claim.scope) {
deleteScopedHealth(claim.accountId, claim.scope);
} else {
const {
cooldownUntil: _until,
cooldownSince: _since,
cooldownSource: _source,
probeLeaseId: _leaseId,
probeLeaseGeneration: _leaseGeneration,
// "The quota window moved" is a statement about the whole refusal, so the avoidance it
// announced goes with the block it produced. Leaving it would make this escape hatch stop
// escaping: the account would still be passed over by every selection it is meant to win.
quotaAvoidUntil: _avoid,
...rest
} = health;
setAccountHealth(claim.accountId, {
...rest,
cooldownGeneration: claim.cooldownGeneration + 1,
lastProbeAt: now,
});
}
return true;
}
/** Acquire the recovery probe for one confirmed model-specific quota group. */
export function tryAcquireCodexQuotaScopeProbeLease(
accountId: string,
scope: CodexQuotaScope,
now = Date.now(),
): string | null {
const health = scopedHealthFor(accountId, scope);
if (!canAcquireQuotaProbeLease(health, now)) return null;
const probeLeaseId = randomUUID();
setScopedHealth(accountId, scope, {
...health!,
probeLeaseId,
probeLeaseGeneration: health!.cooldownGeneration ?? 0,
lastProbeAt: now,
});
return probeLeaseId;
}
/** Side-effect-free check for a confirmed model-specific quota probe. */
export function canAcquireCodexQuotaScopeProbeLease(
accountId: string,
scope: CodexQuotaScope,
now = Date.now(),
): boolean {
return canAcquireQuotaProbeLease(scopedHealthFor(accountId, scope), now);
}
/**
* Hand a probe lease back without recording an upstream outcome. Used by paths
* that take a lease and then fail before any request reaches upstream.
*/
export function releaseCodexQuotaProbeLease(accountId: string, leaseId: string, now = Date.now()): void {
const health = getAccountHealth(accountId);
if (!health || health.probeLeaseId !== leaseId) return;
setAccountHealth(accountId, withProbeLeaseReleased(health, now));
}
/** Release a model-specific quota probe when the request never reaches upstream. */
export function releaseCodexQuotaScopeProbeLease(
accountId: string,
scope: CodexQuotaScope,
leaseId: string,
now = Date.now(),
): void {
const health = scopedHealthFor(accountId, scope);
if (!health || health.probeLeaseId !== leaseId) return;
setScopedHealth(accountId, scope, withProbeLeaseReleased(health, now));
}
/**
* True when this outcome belongs to the account's in-flight probe. The
* undefined-id guard matters: without it an outcome carrying no lease would match
* an account holding no lease and be mistaken for the probe owner.
*/
export function ownsProbeLease(health: CodexUpstreamHealth | undefined, meta: CodexUpstreamOutcomeMeta): boolean {
return meta.probeLeaseId !== undefined && meta.probeLeaseId === health?.probeLeaseId;
}
/**
* True when the owning probe may still clear the cooldown. A later 429 bumps the
* generation, so a probe that started under an older cooldown must not erase the
* newer restriction (which may carry an explicit Retry-After).
*/
export function probeMayClearCooldown(health: CodexUpstreamHealth | undefined, meta: CodexUpstreamOutcomeMeta): boolean {
return ownsProbeLease(health, meta)
&& (health!.probeLeaseGeneration ?? 0) === (health!.cooldownGeneration ?? 0);
}
/** Strip the in-flight lease while preserving every hard-cooldown field. */
export function withProbeLeaseReleased(health: CodexUpstreamHealth, now: number): CodexUpstreamHealth {
const { probeLeaseId: _id, probeLeaseGeneration: _gen, ...rest } = health;
return { ...rest, lastProbeAt: now };
}