UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

374 lines (339 loc) • 15.7 kB
/** * In-memory, per-provider TTL cache for live `/models` results. * * Ported in spirit from jawcode's packages/ai/src/model-manager.ts (the "always load the latest * model list" resolver): live fetch when the cache is stale, serve the cache while it is fresh, * and fall back to the last-known-good list when a live fetch fails. opencodex's proxy is a single * long-running process and the on-disk Codex catalog already persists the last sync across * restarts, so an in-memory cache is sufficient here (no SQLite layer needed). */ import type { CatalogModel } from "./catalog"; import type { GenerationContext } from "../lib/state-store-sweeper"; import { enforceAppOwnedMemoryBudget, type RetainedStoreSnapshot } from "../lib/app-owned-memory"; import { clearLiveCursorRosterState } from "../adapters/cursor/catalog"; /** Default freshness window. Matches Codex's own 5-min models cache so the two stay in step. */ export const DEFAULT_MODEL_CACHE_TTL_MS = 5 * 60 * 1000; interface CacheEntry { models: CatalogModel[]; fetchedAt: number; sizeBytes: number; /** Irreversible credential/account identity for entitlement-sensitive catalogs. */ authorityIdentity?: string; } export type ProviderModelDiscoveryFailureReason = | "http" | "blocked" | "invalid_response" | "network" | "provider"; export type ProviderModelDiscoveryStatus = | { status: "ok" } | { status: "failed"; reason: "http"; httpStatus: number } | { status: "failed"; reason: Exclude<ProviderModelDiscoveryFailureReason, "http">; httpStatus?: never; }; export type ProviderModelDiscoveryFailure = ProviderModelDiscoveryStatus extends infer Status ? Status extends { status: "failed" } ? Omit<Status, "status"> : never : never; /** Whether clearing cache rows also revokes in-flight discovery authority. */ export type ModelCacheClearReason = "authority" | "eviction"; const cache = new Map<string, CacheEntry>(); let globalCacheGeneration = 0; export const providerCacheGenerations = new Map<string, number>(); let cacheBytes = 0; let oldestCachedProvider: string | undefined; let oldestCachedAt: number | null = null; const modelCacheEncoder = new TextEncoder(); function recomputeOldestCachedProvider(): void { oldestCachedProvider = undefined; oldestCachedAt = null; for (const [provider, entry] of cache) { if (oldestCachedAt !== null && entry.fetchedAt >= oldestCachedAt) continue; oldestCachedProvider = provider; oldestCachedAt = entry.fetchedAt; } } function deleteCachedProvider(provider: string): number { const entry = cache.get(provider); if (!entry) return 0; cache.delete(provider); cacheBytes = Math.max(0, cacheBytes - entry.sizeBytes); if (oldestCachedProvider === provider) recomputeOldestCachedProvider(); // A removal changes this provider's content as surely as a publication does. bumpProviderCacheRevision(provider); return entry.sizeBytes; } /** Cooldown after a failed live `/models` fetch, so a dead/unreachable provider doesn't re-pay * the full fetch timeout on every catalog poll (issue #54: UI stalls behind corporate proxies). */ export const MODELS_FETCH_FAILURE_COOLDOWN_MS = 30_000; interface DiscoveryFailure { at: number; /** * Credential the failure was observed under, for entitlement-specific rosters. Absent means * the failure is credential-agnostic (a plain `/models` endpoint) and suppresses every * caller, which is the original #54 behaviour. */ authorityIdentity?: string; } const failureAt = new Map<string, DiscoveryFailure>(); const discoveryStatus = new Map<string, ProviderModelDiscoveryStatus>(); /** * How many models the last successful discovery actually returned, before any configured-alias * augmentation or custom-model merge. Consumers cannot recover this by subtracting known custom * ids: a custom id that later also appears upstream would make a real live catalog look * custom-only, and the provider's configured fallback would wrongly stay authoritative. */ const liveModelCounts = new Map<string, number>(); let lastReconciledGeneration = 0; export function markModelsFetchFailure( provider: string, now = Date.now(), authorityIdentity?: string, ): void { failureAt.set(provider, { at: now, ...(authorityIdentity ? { authorityIdentity } : {}) }); } /** `liveModelCount` is required so a caller that forgets to pass it fails typecheck instead of * silently recording zero and misclassifying the provider as having no live catalog. */ export function markProviderDiscoveryOk(provider: string, liveModelCount: number): void { discoveryStatus.set(provider, { status: "ok" }); liveModelCounts.set(provider, Math.max(0, Math.floor(liveModelCount))); } export function markProviderDiscoveryFailed( provider: string, failure: ProviderModelDiscoveryFailure, ): void { discoveryStatus.set(provider, { status: "failed", ...failure }); } /** * Decide whether a discovery FAILURE should be logged, to avoid flooding the log with an identical * warning on every poll (#395: an anthropic-adapter baseUrl without `/v1/models`, e.g. Azure AI * Foundry, returns HTTP 404 forever; the 30s cooldown re-probes and previously re-logged each time). * * Returns true only when the failure SIGNATURE changed since the last observed status — i.e. the * previous state was ok/undefined, or a different reason/httpStatus. Repeated identical failures * stay observable through `getProviderDiscoveryStatus()` / the providers API without log spam. * Call this BEFORE `markProviderDiscoveryFailed` so it can see the prior state. */ export function shouldLogDiscoveryFailure( provider: string, failure: ProviderModelDiscoveryFailure, ): boolean { const prev = discoveryStatus.get(provider); if (!prev || prev.status !== "failed") return true; if (prev.reason !== failure.reason) return true; if (prev.reason === "http" && failure.reason === "http") { return prev.httpStatus !== failure.httpStatus; } return false; } export function clearProviderDiscoveryStatus(provider: string): void { discoveryStatus.delete(provider); liveModelCounts.delete(provider); } export function getProviderDiscoveryStatus(provider: string): ProviderModelDiscoveryStatus | undefined { return discoveryStatus.get(provider); } /** Undefined when discovery has never succeeded for this provider. A failed refresh keeps the * last successful count so a stale catalog is still recognised as live-origin. */ export function getProviderLiveModelCount(provider: string): number | undefined { return liveModelCounts.get(provider); } /** * Whether a failed discovery still suppresses the next one. * * `authorityIdentity` scopes the suppression to the credential that actually observed the * failure. A roster that upstream filters per account is evidence about that account, and * one account's 401 or 404 must not decide that a different account has no catalog. A * failure recorded without an identity stays credential-agnostic and suppresses everyone, * so the plain-endpoint providers keep the timeout protection #54 added. */ export function isModelsFetchCoolingDown( provider: string, cooldownMs = MODELS_FETCH_FAILURE_COOLDOWN_MS, now = Date.now(), authorityIdentity?: string, ): boolean { const failure = failureAt.get(provider); if (failure === undefined || now - failure.at >= cooldownMs) return false; if (failure.authorityIdentity === undefined || authorityIdentity === undefined) return true; return failure.authorityIdentity === authorityIdentity; } /** Fresh cached models for a provider, or null when absent/stale (caller should re-fetch). */ export function getFreshCached(provider: string, ttlMs: number, now = Date.now(), authorityIdentity?: string): CatalogModel[] | null { const entry = cache.get(provider); if (!entry) return null; if (authorityIdentity !== undefined && entry.authorityIdentity !== authorityIdentity) return null; return now - entry.fetchedAt < ttlMs ? entry.models : null; } /** Last-known-good models regardless of age — the fallback when a live fetch fails. */ export function getStaleCached(provider: string, authorityIdentity?: string): CatalogModel[] | null { const entry = cache.get(provider); if (!entry) return null; if (authorityIdentity !== undefined && entry.authorityIdentity !== authorityIdentity) return null; return entry.models; } /** Selector decoding may use unscoped rows, but scoped rows need current authority. */ export function getRoutingCached( provider: string, resolveAuthority: () => string | undefined, ): CatalogModel[] | null { const entry = cache.get(provider); if (!entry) return null; if (entry.authorityIdentity !== undefined) { try { if (resolveAuthority() !== entry.authorityIdentity) return null; } catch { return null; } } return entry.models; } /** Capture the cache generation before an asynchronous provider discovery starts. */ export function captureModelCacheGeneration(provider: string): string { if (!providerCacheGenerations.has(provider)) providerCacheGenerations.set(provider, 0); return `${globalCacheGeneration}:${providerCacheGenerations.get(provider)!}`; } /** Whether a discovery started under {@link captureModelCacheGeneration} may still write. */ export function isModelCacheGenerationCurrent(provider: string, generation: string): boolean { return generation === captureModelCacheGeneration(provider); } /** * How many times this provider's cached content has actually changed. * * Deliberately separate from the generation. A generation revokes an in-flight discovery's right * to publish, so it advances on an authority clear and must not be repurposed: moving it on a * successful publication would cancel writes that are still legitimate. This counts accepted * publications and removals instead, which is what a reader holding a derived roster needs to * know, and it advances on exactly the event a generation does not: a discovery that succeeded * and changed the rows. * * Reading is passive. An unseen provider reads as `0` rather than seeding an entry, so observing * one cannot alter what a later capture or publication sees. */ const providerCacheRevisions = new Map<string, number>(); let globalContentRevision = 0; function bumpProviderCacheRevision(provider: string): void { providerCacheRevisions.set(provider, (providerCacheRevisions.get(provider) ?? 0) + 1); } export function observeModelCacheRevision(provider: string): string { // The global term covers a clear that empties the map wholesale, which per-provider counters // cannot express: without it, wiping every entry and republishing identical-looking rows would // read as unchanged. It also survives pruning, so a retired provider cannot come back with a // counter that matches a roster built before it left. return `${globalContentRevision}:${providerCacheRevisions.get(provider) ?? 0}`; } /** * Store a live result unless the cache was cleared while that asynchronous discovery was running. * The optional generation keeps existing direct cache writers unchanged while discovery callers can * prevent a previous OAuth account from repopulating the current account's cache. */ export function setCached( provider: string, models: CatalogModel[], now = Date.now(), generation?: string, authorityIdentity?: string, ): boolean { if (generation !== undefined && !isModelCacheGenerationCurrent(provider, generation)) return false; deleteCachedProvider(provider); const sizeBytes = modelCacheEncoder.encode(provider).byteLength + modelCacheEncoder.encode(JSON.stringify(models)).byteLength; cache.set(provider, { models, fetchedAt: now, sizeBytes, ...(authorityIdentity ? { authorityIdentity } : {}) }); cacheBytes += sizeBytes; if (oldestCachedAt === null || now < oldestCachedAt) { oldestCachedProvider = provider; oldestCachedAt = now; } enforceAppOwnedMemoryBudget(); // Published and accepted, so anything derived from this provider's rows is now out of date. bumpProviderCacheRevision(provider); return true; } /** Drop one provider's cache (or all) so the next resolve forces a live re-fetch. */ export function clearModelCache( provider?: string, reason: ModelCacheClearReason = "authority", ): void { clearLiveCursorRosterState(provider); const revokesInFlightDiscovery = reason === "authority"; if (provider) { if (revokesInFlightDiscovery) { providerCacheGenerations.set(provider, (providerCacheGenerations.get(provider) ?? 0) + 1); } deleteCachedProvider(provider); failureAt.delete(provider); discoveryStatus.delete(provider); liveModelCounts.delete(provider); } else { if (revokesInFlightDiscovery) globalCacheGeneration += 1; cache.clear(); // A wholesale clear changes every provider's content at once, and clearing the map means no // per-provider counter can record it. Advancing the global term retires every derived roster // and lets the per-provider entries be dropped without an ABA on the way back. globalContentRevision += 1; providerCacheRevisions.clear(); cacheBytes = 0; oldestCachedProvider = undefined; oldestCachedAt = null; failureAt.clear(); discoveryStatus.clear(); liveModelCounts.clear(); } } export function reconcileModelCacheProviders( validProviders: ReadonlySet<string>, generation = lastReconciledGeneration + 1, ): number { if (generation <= lastReconciledGeneration) return 0; const removedProviders = new Set<string>(); const trackedProviders = new Set([ ...providerCacheGenerations.keys(), ...providerCacheRevisions.keys(), ...failureAt.keys(), ...discoveryStatus.keys(), ...liveModelCounts.keys(), ...cache.keys(), ]); let revokedRemovedProviderAuthority = false; for (const provider of trackedProviders) { if (validProviders.has(provider)) continue; if (!revokedRemovedProviderAuthority) { globalCacheGeneration += 1; // Advanced BEFORE any revision entry is dropped, so a provider that comes back cannot // present the counter a roster was built against. Without this the delete below is an ABA: // the entry returns at zero and an old stamp matches again. globalContentRevision += 1; revokedRemovedProviderAuthority = true; } providerCacheGenerations.set(provider, (providerCacheGenerations.get(provider) ?? 0) + 1); providerCacheGenerations.delete(provider); deleteCachedProvider(provider); failureAt.delete(provider); discoveryStatus.delete(provider); liveModelCounts.delete(provider); // AFTER the cache deletion, which bumps this provider's revision and would otherwise recreate // the entry we just removed. Dropped rather than left behind: a provider the configuration no // longer has must not keep an entry alive for the life of the process merely because nothing // cleared the whole cache. The global epoch advanced above, so the removal is not an ABA. providerCacheRevisions.delete(provider); removedProviders.add(provider); } lastReconciledGeneration = generation; return removedProviders.size; } export function reconcileModelCacheGeneration(context: GenerationContext): number { return reconcileModelCacheProviders(context.providerNames, context.generation); } export function modelCacheRetainedStoreSnapshot(): RetainedStoreSnapshot { return { count: cache.size, bytes: cacheBytes, evictableBytes: cacheBytes, pinnedBytes: 0, oldestAt: oldestCachedAt, }; } export function evictOldestModelCacheForBudget(): number { return oldestCachedProvider === undefined ? 0 : deleteCachedProvider(oldestCachedProvider); }