@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
257 lines (238 loc) • 10.7 kB
text/typescript
/**
* Provider cache affinity: keep a conversation on the key-pool credential that
* served its last turn, so that credential's upstream prompt cache is read again
* instead of rebuilt on another key.
*
* Modeled on Magpie's gateway affinity design, expressed in this codebase's
* terms: conversations are session lanes (server/request-log-conversation),
* credentials are key-pool entries identified by their pool entry id (never key
* material), and outcomes are recorded from attempt finalization, where every
* adapter's cache usage is already normalized into cache-read / cache-creation
* token counts.
*
* Affinity is a preference inside eligibility, never an override: this module
* only answers "which entry served this lane last, and is that worth
* preferring?". Whether that entry may serve right now (disabled, cooling,
* pacing) is decided by the caller in key-failover, which owns those rules.
*
* The store is process-local, following the combo session-recall precedent:
* a restart forgets every binding and the first post-restart turn of a
* conversation is placed by ordinary selection again.
*
* Lane ids are client-supplied, so a client that churns them can fill the
* store and evict other conversations' bindings. The blast radius is lost
* cache locality only — no correctness or credential exposure — and the
* count/byte bounds below are the mitigation.
*/
import type { CacheAffinityMode, OcxProviderConfig } from "../types";
import type { GenerationContext } from "../lib/state-store-sweeper";
import { clearCacheAffinityIdentitiesForTests, currentCacheAffinityIdentity, reconcileCacheAffinityIdentities } from "./cache-affinity-identity";
/** Tokens read from or written to an upstream cache that make a binding worth keeping across turns. */
export const CACHE_AFFINITY_WORTH_TOKENS = 1024;
/** How long an ordinary (5-minute-retention) cache stays warm after the turn that built it. */
export const CACHE_AFFINITY_COLD_MS = 5 * 60_000;
/** How long a long-retention (1-hour) cache write stays warm. */
export const CACHE_AFFINITY_COLD_LONG_MS = 60 * 60_000;
/** Hard expiry for a binding, however warm: past this the conversation is placed fresh. */
export const CACHE_AFFINITY_TTL_MS = 24 * 60 * 60_000;
const CAPACITY = 512;
/** Per-entry byte cap over the retained strings, mirroring combo session recall. */
const ENTRY_BYTES_MAX = 1024;
/** Aggregate byte cap across every retained binding. */
const TOTAL_BYTES_MAX = 64 * 1024;
export type CacheAffinityReason =
| "cache" // kept: measured cache worth keeping and still warm (auto)
| "session" // kept: session mode prefers the bound credential while eligible
| "turn" // kept: turn mode, detected continuation of the in-progress turn
| "continuation"; // kept: detected continuation of the in-progress turn (auto)
export interface CacheAffinityHit {
entryId: string;
reason: CacheAffinityReason;
}
interface Binding {
providerName: string;
entryId: string;
identity: string;
model: string;
cacheRead: number;
cacheWrite: number;
longExpiresAt?: number;
at: number;
}
const bindings = new Map<string, Binding>();
let lastReconciledGeneration = 0;
let bindingsBytes = 0;
function bindingBytes(lane: string, binding: Binding): number {
return Buffer.byteLength(lane) + Buffer.byteLength(binding.providerName)
+ Buffer.byteLength(binding.entryId) + Buffer.byteLength(binding.model) + Buffer.byteLength(binding.identity);
}
function deleteBinding(lane: string): void {
const binding = bindings.get(lane);
if (!binding) return;
bindings.delete(lane);
bindingsBytes -= bindingBytes(lane, binding);
}
export function cacheAffinityMode(provider: Pick<OcxProviderConfig, "cacheAffinity">): CacheAffinityMode {
return provider.cacheAffinity ?? "auto";
}
export function cacheAffinityApplies(provider: OcxProviderConfig): boolean {
if (cacheAffinityMode(provider) === "off") return false;
if (provider.authMode === "oauth" || provider.authMode === "forward") return false;
return (provider.apiKeyPool?.length ?? 0) >= 2;
}
export interface ConsultProviderCacheAffinityInput {
lane: string | undefined;
providerName: string;
provider: OcxProviderConfig;
model: string;
/** The request continues the lane's in-progress turn (tool results being returned). */
continuation?: boolean;
now?: number;
}
/**
* The bound credential for this lane, when affinity says to prefer it.
* Purely a preference: the caller must still validate the entry against the
* live pool (present, enabled, not cooling) before sending with it.
*/
export function consultProviderCacheAffinity(input: ConsultProviderCacheAffinityInput): CacheAffinityHit | null {
const { lane, providerName, provider, model } = input;
if (!lane || !cacheAffinityApplies(provider)) return null;
const binding = bindings.get(lane);
if (!binding) return null;
const now = input.now ?? Date.now();
if (now - binding.at >= CACHE_AFFINITY_TTL_MS) {
deleteBinding(lane);
return null;
}
if (binding.providerName !== providerName || binding.model !== model) return null;
if (binding.identity !== currentCacheAffinityIdentity(providerName, provider, binding.entryId)) {
deleteBinding(lane);
return null;
}
const mode = cacheAffinityMode(provider);
const warmUntil = Math.max(binding.at + CACHE_AFFINITY_COLD_MS, binding.longExpiresAt ?? 0);
if (mode === "session") return { entryId: binding.entryId, reason: "session" };
if (input.continuation === true) {
if (now > warmUntil) return null;
return { entryId: binding.entryId, reason: mode === "turn" ? "turn" : "continuation" };
}
if (mode === "turn") return null;
// auto, across turns: only measured-worth bindings that are still warm.
if (now > warmUntil) return null;
const worth = binding.cacheRead + binding.cacheWrite;
return worth >= CACHE_AFFINITY_WORTH_TOKENS ? { entryId: binding.entryId, reason: "cache" } : null;
}
export interface RecordProviderCacheAffinityInput {
lane: string | undefined;
providerName: string;
entryId: string | undefined;
identity: string | undefined;
model: string;
cacheRead?: number;
cacheWrite?: number;
/** Portion of the cache write kept at long (1-hour) retention, when the adapter reports it. */
cacheWriteLong?: number;
inputTokens?: number;
now?: number;
}
/**
* Remember who served a completed turn. A turn with no cache telemetry at all
* counts its sent prompt as written (silent-cache vendors such as Ollama
* Cloud): the prompt the vendor cached is the prompt that was sent.
*/
export function recordProviderCacheAffinity(input: RecordProviderCacheAffinityInput): void {
const { lane, providerName, entryId, model, identity } = input;
if (!lane || !entryId || !identity) return;
const cacheRead = input.cacheRead ?? 0;
// No write counter reported — reads-only reporters (OpenAI Chat reports
// reads, never writes) and fully silent vendors alike: the prompt that was
// sent is what the vendor cached, so it counts as written. An explicitly
// reported write count, zero included, is believed as-is.
const cacheWrite = input.cacheWrite ?? input.inputTokens ?? 0;
const now = input.now ?? Date.now();
const previous = bindings.get(lane);
const matchingLongRead = cacheRead > 0 && previous?.longExpiresAt !== undefined
&& previous.providerName === providerName
&& previous.entryId === entryId && previous.identity === identity && previous.model === model
&& now >= previous.at && now <= previous.longExpiresAt;
const binding: Binding = {
providerName,
entryId,
identity,
model,
cacheRead,
cacheWrite,
longExpiresAt: (input.cacheWriteLong ?? 0) > 0 ? now + CACHE_AFFINITY_COLD_LONG_MS
: matchingLongRead ? previous.longExpiresAt : undefined,
at: now,
};
if (bindingBytes(lane, binding) > ENTRY_BYTES_MAX) return;
// Re-insert so eviction drops the least recently recorded lane first.
deleteBinding(lane);
bindings.set(lane, binding);
bindingsBytes += bindingBytes(lane, binding);
while (bindings.size > CAPACITY || bindingsBytes > TOTAL_BYTES_MAX) {
const oldest = bindings.keys().next().value;
if (oldest === undefined || oldest === lane) break;
deleteBinding(oldest);
}
}
export function releaseProviderCacheAffinity(
lane: string | undefined,
providerName?: string,
entryId?: string,
identity?: string,
): boolean {
if (!lane) return false;
const binding = bindings.get(lane);
if (!binding) return false;
if ((providerName !== undefined && binding.providerName !== providerName)
|| (entryId !== undefined && binding.entryId !== entryId)
|| (identity !== undefined && binding.identity !== identity)) return false;
deleteBinding(lane);
return true;
}
/** The entry a lane is currently bound to, if any (finalization log attribution). */
export function boundEntryIdForLane(lane: string | undefined): string | undefined {
return lane ? bindings.get(lane)?.entryId : undefined;
}
/** Periodic expiry for the state-store sweeper; expiry is also evaluated on consult. */
export function sweepExpiredProviderCacheAffinity(now: number): number {
let removed = 0;
for (const [lane, binding] of bindings) {
if (now - binding.at >= CACHE_AFFINITY_TTL_MS) {
deleteBinding(lane);
removed += 1;
}
}
return removed;
}
/**
* Drop bindings whose provider left the live configuration, was disabled, or
* lost the bound credential, on a configuration generation change. Consult
* re-validates against the live pool on every request regardless; this keeps
* unconsulted lanes from lingering.
*/
export function reconcileProviderCacheAffinity(context: GenerationContext, providers?: Readonly<Record<string, OcxProviderConfig>>): number {
if (context.generation <= lastReconciledGeneration) return 0;
lastReconciledGeneration = context.generation;
if (providers) reconcileCacheAffinityIdentities(providers);
let removed = 0;
for (const [lane, binding] of bindings) {
const entryIds = context.providerKeyEntryIds?.get(binding.providerName);
if (!context.providerNames.has(binding.providerName)
|| context.disabledProviderNames?.has(binding.providerName)
|| (entryIds !== undefined && !entryIds.has(binding.entryId))
|| (providers !== undefined && binding.identity !== currentCacheAffinityIdentity(binding.providerName, providers[binding.providerName], binding.entryId))) {
deleteBinding(lane);
removed += 1;
}
}
return removed;
}
export function clearProviderCacheAffinityForTests(): void {
bindings.clear();
bindingsBytes = 0;
lastReconciledGeneration = 0;
clearCacheAffinityIdentitiesForTests();
}