@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
392 lines (367 loc) • 19.6 kB
text/typescript
/**
* Which models may serve AS the vision sidecar (the describer), as opposed to the
* models the sidecar describes FOR.
*
* Three rules make this non-obvious, and all are load-bearing:
*
* 1. `provider.noVisionModels` marks models the proxy describes images for, and
* `applyProviderConfigHints` deliberately ADDS "image" to their advertised
* input modalities so the Codex app does not block the attachment client-side.
* A blind model therefore advertises image input. Membership in that list is a
* hard disqualifier here, checked BEFORE the modality list it rewrote.
* 2. The catalog enriches configured providers from the registry before it
* applies that rule. Eligibility must use the same effective provider policy,
* including a `noVisionModels` list learned after the config was persisted.
* 3. Catalog rows frequently omit `inputModalities` entirely (the live
* `/api/models` response carries none for either openai or anthropic rows).
* Unknown is not zero: when no source can speak, the model stays eligible
* rather than silently vanishing from the picker.
* Native OpenAI metadata is likewise authoritative only for a native row or
* the canonical OpenAI provider; a routed row with the same id owns its own
* explicit modalities.
*/
import { modelInList, type OcxConfig, type OcxProviderConfig } from "../types";
import { modelRecordValue } from "../reasoning-effort";
import { getModelMetadataCaseInsensitive, resolveMetadataProvider } from "../generated/model-metadata";
import { nativeInputModalities } from "../codex/catalog/metadata";
import { SUPPORTED_NATIVE_OPENAI_SLUGS } from "../codex/catalog/native-models";
import { enrichProviderFromRegistry } from "../providers/derive";
import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers-destination";
/**
* The wire protocols `planVisionSidecar` can dispatch to (#2188 roadmap 170
* REVISED). "routed" describes through the proxy's OWN router via loopback —
* one executor for every non-forward, non-OAuth-Anthropic provider row.
*/
export type VisionSidecarBackend = "openai" | "anthropic" | "routed";
/**
* Input modalities an explicit custom row declares for one routed model.
*
* A custom row is the operator's own definition of that model, so its declaration outranks the
* provider-level hints the predicates below read. Without this the two halves of one config
* disagreed in production: the catalog overlay in `src/codex/catalog/routed-gather.ts` copies
* `customModels[].inputModalities` onto the row and the dashboard showed "text, image", while
* the request path consulted only `providers[].noVisionModels` / `modelInputModalities` and
* stripped the image before dispatch. A model the operator had declared image-capable therefore
* received an omission marker instead of its attachment.
*
* Precedence, highest first: `modelCapabilities` (the documented per-model capability
* declaration), then this custom-row declaration, then `noVisionModels`, then
* `modelInputModalities`, then registry/vendor metadata. `modelCapabilities` stays on top
* because it is the dedicated capability axis and the CLI `--text-only` flag writes it; when the
* two explicit forms contradict each other the more specific axis wins.
*
* Matching is exact on the routed identity — provider name and native model id, the pair the
* custom-model API keys rows by. A row that declares no modalities returns `undefined` rather
* than `["text"]`, so it stays silent instead of turning into a text-only claim.
*/
export function customRowInputModalities(
config: Pick<OcxConfig, "customModels">,
providerName: string,
modelId: string,
): string[] | undefined {
for (const row of config.customModels ?? []) {
if (row.provider !== providerName || row.modelId !== modelId) continue;
if (Array.isArray(row.inputModalities) && row.inputModalities.length > 0) {
return [...row.inputModalities];
}
}
return undefined;
}
/**
* The custom row's verdict for one model, in the shape the predicates need:
* `true`/`false` when the row declares modalities, `undefined` when it declares none.
*/
function customRowAcceptsImageInput(
config: Pick<OcxConfig, "customModels">,
providerName: string,
modelId: string,
): boolean | undefined {
const declared = customRowInputModalities(config, providerName, modelId);
return declared === undefined ? undefined : declared.includes("image");
}
/** The two sides every deployment has; also the empty-auth fallback set. */
export type UniversalVisionBackend = "openai" | "anthropic";
/**
* Default entry per backend: cheap, image-capable, and present in every deployment. Offered
* whenever its side is enabled, and withheld only when that provider explicitly lists it as a
* model the sidecar describes FOR — never merely because a metadata table stayed silent.
*
* Keyed by the UNIVERSAL subset on purpose (roadmap 170, audit blocker A):
* xai/gemini are auth-gated sides whose catalogs are present whenever the side
* is, so they carry no baseline, and a narrow-key total record documents that
* without sprinkling non-null assertions at the consumers.
*/
export const BASELINE_VISION_MODELS: Record<UniversalVisionBackend, string> = {
openai: "gpt-5.6-luna",
anthropic: "claude-haiku-4-5",
};
export interface VisionCandidateModel {
provider: string;
id: string;
inputModalities?: string[];
native?: boolean;
}
/**
* The config slice the capability predicates need. `customModels` is optional so provider-only
* callers and unit tests keep compiling; a caller that omits it is simply silent about custom
* rows rather than wrong about them.
*/
export type VisionCapabilityConfig =
Pick<OcxConfig, "providers"> & { customModels?: OcxConfig["customModels"] };
export interface VisionModelOption {
value: string;
label: string;
backend: VisionSidecarBackend;
/** True when the row is a guaranteed baseline rather than a catalog discovery. */
baseline?: boolean;
}
type EnrichedProviderCache = Map<string, OcxProviderConfig>;
/**
* Whether the proxy must describe images for this model before dispatching its main request.
*
* `noVisionModels` is an explicit override. A modality declaration is only evidence for this
* path when it describes a text model that excludes image input: an audio-only declaration is
* not a text-only model and must not be widened to image through the vision sidecar.
*/
export function isModelVisionSidecarConsumer(
provider: Pick<OcxProviderConfig, "noVisionModels" | "modelInputModalities" | "modelCapabilities">,
modelId: string,
): boolean {
const declared = Object.hasOwn(provider.modelCapabilities ?? {}, modelId)
? provider.modelCapabilities?.[modelId]?.inputModalities : undefined;
if (declared !== undefined) return declared.includes("text") && !declared.includes("image");
if (modelInList(provider.noVisionModels, modelId)) return true;
const modalities = modelRecordValue(provider.modelInputModalities, modelId);
return Array.isArray(modalities) && modalities.includes("text") && !modalities.includes("image");
}
function advertisesImageInput(modalities: readonly string[] | undefined): boolean | undefined {
if (!modalities || modalities.length === 0) return undefined;
return modalities.includes("image");
}
/** Vendor-table modalities for a routed row, or undefined when the table has no opinion. */
function metadataImageInput(provider: string, modelId: string): boolean | undefined {
const resolved = resolveMetadataProvider(provider) ?? provider;
const meta = getModelMetadataCaseInsensitive(resolved, modelId);
return advertisesImageInput(meta?.input);
}
/**
* Mirrors the catalog's registry enrichment without mutating saved configuration. The picker
* checks many rows from one provider, so its caller shares this small call-local cache.
*/
function enrichedProviderForVision(
config: Pick<OcxConfig, "providers">,
providerName: string,
cache: EnrichedProviderCache,
): OcxProviderConfig | undefined {
const cached = cache.get(providerName);
if (cached) return cached;
const configured = config.providers?.[providerName];
if (!configured) return undefined;
// Runtime providers may carry non-cloneable hooks (for example a provider-scoped fetch).
// Enrichment mutates top-level fields but does not mutate nested provider values in place, so
// a shallow copy plus private copies of the vision-capability containers is sufficient and
// avoids both config mutation and structuredClone(DataCloneError) on runtime functions.
const enriched: OcxProviderConfig = {
...configured,
...(configured.noVisionModels ? { noVisionModels: [...configured.noVisionModels] } : {}),
...(configured.modelInputModalities ? {
modelInputModalities: Object.fromEntries(
Object.entries(configured.modelInputModalities).map(([id, modalities]) => [id, [...modalities]]),
),
} : {}),
...(configured.modelCapabilities ? {
modelCapabilities: Object.fromEntries(Object.entries(configured.modelCapabilities).map(([id, capability]) => [
id,
{
...capability,
...(capability.inputModalities ? { inputModalities: [...capability.inputModalities] } : {}),
},
])),
} : {}),
};
enrichProviderFromRegistry(providerName, enriched);
cache.set(providerName, enriched);
return enriched;
}
function isVisionSidecarConsumerWithCache(
config: VisionCapabilityConfig,
providerName: string,
modelId: string,
cache: EnrichedProviderCache,
): boolean {
const provider = enrichedProviderForVision(config, providerName, cache);
if (provider === undefined) return false;
// Documented precedence, highest first: the dedicated per-model capability axis, then the
// operator's own custom row, then the provider-level hints. Reading the custom row ahead of
// `modelCapabilities` would let it override the more specific axis.
const capabilityDeclared = Object.hasOwn(provider.modelCapabilities ?? {}, modelId)
? provider.modelCapabilities?.[modelId]?.inputModalities : undefined;
if (capabilityDeclared !== undefined) {
return capabilityDeclared.includes("text") && !capabilityDeclared.includes("image");
}
const customDeclared = customRowInputModalities(config, providerName, modelId);
if (customDeclared !== undefined) return customDeclared.includes("text") && !customDeclared.includes("image");
return isModelVisionSidecarConsumer(provider, modelId);
}
/**
* Is this model listed as one the sidecar describes FOR? Such a model cannot be
* the describer, and its advertised modalities are untrustworthy.
*/
export function isVisionSidecarConsumer(config: VisionCapabilityConfig, providerName: string, modelId: string): boolean {
return isVisionSidecarConsumerWithCache(config, providerName, modelId, new Map());
}
/**
* Can this model accept an image on the wire? Sources are consulted in descending
* trustworthiness; the first that speaks wins. `undefined` means nothing knows,
* which callers treat as eligible.
*/
export function modelAcceptsImageInput(
config: VisionCapabilityConfig,
candidate: VisionCandidateModel,
): boolean | undefined {
return modelAcceptsImageInputWithCache(config, candidate, new Map());
}
function modelAcceptsImageInputWithCache(
config: VisionCapabilityConfig,
candidate: VisionCandidateModel,
cache: EnrichedProviderCache,
): boolean | undefined {
if (candidate.native === true || (candidate.provider === "openai" && SUPPORTED_NATIVE_OPENAI_SLUGS.has(candidate.id))) {
const nativeProvider = enrichedProviderForVision(config, candidate.provider, cache);
const declared = Object.hasOwn(nativeProvider?.modelCapabilities ?? {}, candidate.id)
? nativeProvider?.modelCapabilities?.[candidate.id]?.inputModalities : undefined;
if (declared !== undefined) return declared.includes("image");
// The catalog already lets an explicit custom row replace the native modality list for the
// same slug; consult it here too or the row would advertise what the request path ignores.
const fromCustomRow = customRowAcceptsImageInput(config, candidate.provider, candidate.id);
if (fromCustomRow !== undefined) return fromCustomRow;
// The sidecar hints come after both explicit declarations, matching the documented order.
// Ahead of them a `noVisionModels` membership would short-circuit to text-only before the
// operator's own row was read.
if (nativeProvider && isModelVisionSidecarConsumer(nativeProvider, candidate.id)) return false;
return advertisesImageInput(nativeInputModalities(candidate.id)) ?? true;
}
if (isVisionSidecarConsumerWithCache(config, candidate.provider, candidate.id, cache)) return false;
const provider = enrichedProviderForVision(config, candidate.provider, cache);
const declared = Object.hasOwn(provider?.modelCapabilities ?? {}, candidate.id)
? provider?.modelCapabilities?.[candidate.id]?.inputModalities : undefined;
if (declared !== undefined) return declared.includes("image");
// Ahead of the provider's own hints: this row is the operator's definition of this exact model,
// and the catalog overlay reads the same field. Behind modelCapabilities, which is the dedicated
// capability axis the CLI `--text-only` writes.
const fromCustomRow = customRowAcceptsImageInput(config, candidate.provider, candidate.id);
if (fromCustomRow !== undefined) return fromCustomRow;
const configuredModalities = provider ? modelRecordValue(provider.modelInputModalities, candidate.id) : undefined;
const fromConfiguredModalities = advertisesImageInput(configuredModalities);
if (fromConfiguredModalities !== undefined) return fromConfiguredModalities;
const canonicalCodex = candidate.provider === "openai"
&& provider !== undefined
&& isCanonicalOpenAiForwardProvider(provider);
if (canonicalCodex) {
const fromCodexBackend = metadataImageInput("openai-codex", candidate.id);
if (fromCodexBackend !== undefined) return fromCodexBackend;
}
const fromRow = advertisesImageInput(candidate.inputModalities);
if (fromRow !== undefined) return fromRow;
return metadataImageInput(candidate.provider, candidate.id);
}
/** Eligible = not a sidecar consumer, and not positively known to be text-only. */
export function isVisionEligibleModel(
config: VisionCapabilityConfig,
candidate: VisionCandidateModel,
): boolean {
return isVisionEligibleModelWithCache(config, candidate, new Map());
}
function isVisionEligibleModelWithCache(
config: VisionCapabilityConfig,
candidate: VisionCandidateModel,
cache: EnrichedProviderCache,
): boolean {
return modelAcceptsImageInputWithCache(config, candidate, cache) !== false;
}
/** Which executor can describe through this row. */
export function visionBackendForCandidate(
config: Pick<OcxConfig, "providers">,
candidate: VisionCandidateModel,
anthropicProviderName?: string,
): VisionSidecarBackend | undefined {
if (candidate.native || candidate.provider === "openai") return "openai";
// The runtime dispatches through exactly ONE Anthropic provider — the enabled OAuth row
// with a healthy active account that `findAnthropicVisionProvider` picks. Speaking the
// Messages wire is not enough: a key-auth row of the same adapter is unreachable, and an
// option that cannot be dispatched is worse than a missing one, because selecting it fails
// at describe time rather than at pick time.
if (anthropicProviderName !== undefined && candidate.provider === anthropicProviderName) {
return "anthropic";
}
// EVERY other provider row describes through the proxy's own router
// (roadmap 170 revised): the loopback executor covers all provider wires,
// so no row is left without an executor.
return "routed";
}
function baselineCandidate(
backend: UniversalVisionBackend,
anthropicProviderName: string | undefined,
): VisionCandidateModel {
return {
provider: backend === "openai" ? "openai" : anthropicProviderName ?? "anthropic",
id: BASELINE_VISION_MODELS[backend],
// Both baselines are known image-capable. This makes their only exclusion path the
// provider's explicit consumer list, never a silent or stale metadata table.
inputModalities: ["text", "image"],
};
}
/**
* The picker's option list: every eligible row reachable by an enabled
* executor, plus each enabled universal side's baseline unless that baseline
* is explicitly excluded, de-duplicated and stably ordered (side rank order,
* baselines first within a side). Anthropic rows must belong to the OAuth
* provider that would actually execute them, so `anthropicProviderName` is
* what makes that side's catalog rows eligible at all; xai/gemini rows map by
* provider identity and appear only when the caller enabled those backends.
*
* This is the SUGGESTION list (narrow): it emits only rows an executor can reach
* and some source has heard of. It is deliberately NOT the same set as the write
* gate. `modelAcceptsImageInput` answers "can we prove this model cannot see?"
* and is the only input to rejection. Absence from this list must never imply
* rejection — an unknown id stays eligible via the undefined → eligible fallback.
*
* De-duplication is by BARE model id, first eligible row wins. Two providers of
* the same adapter family can expose the same id; they resolve to the same
* backend, and only `value` reaches the client, so first-wins costs nothing.
*/
export function visionEligibleModelOptions(
config: VisionCapabilityConfig,
candidates: readonly VisionCandidateModel[],
enabledBackends: readonly VisionSidecarBackend[],
anthropicProviderName?: string,
): VisionModelOption[] {
const enabled = new Set(enabledBackends);
const byValue = new Map<string, VisionModelOption>();
const enrichedProviders: EnrichedProviderCache = new Map();
for (const backend of Object.keys(BASELINE_VISION_MODELS) as UniversalVisionBackend[]) {
if (!enabled.has(backend)) continue;
const candidate = baselineCandidate(backend, anthropicProviderName);
if (!isVisionEligibleModelWithCache(config, candidate, enrichedProviders)) continue;
const id = candidate.id;
byValue.set(id, { value: id, label: id, backend, baseline: true });
}
for (const candidate of candidates) {
const backend = visionBackendForCandidate(config, candidate, anthropicProviderName);
if (!backend || !enabled.has(backend)) continue;
if (!isVisionEligibleModelWithCache(config, candidate, enrichedProviders)) continue;
// Routed rows carry NAMESPACED values ("provider/model") so the loopback
// dispatch is unambiguous under routeModel; the legacy sides keep bare ids
// (GUI current-value compatibility, and the forward/OAuth executors POST
// the string verbatim). De-dup stays keyed by the emitted value.
const value = backend === "routed" ? `${candidate.provider}/${candidate.id}` : candidate.id;
if (byValue.has(value)) continue;
byValue.set(value, { value, label: value, backend });
}
// Two slots per side (baseline first), ranked openai < anthropic < routed
// so widening the union appends rather than interleaves (roadmap 170).
const sideRank: Record<VisionSidecarBackend, number> = { openai: 0, anthropic: 2, routed: 4 };
const order = (option: VisionModelOption) =>
sideRank[option.backend] + (option.baseline ? 0 : 1);
return [...byValue.values()].sort((a, b) => order(a) - order(b) || a.value.localeCompare(b.value));
}