@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
172 lines (154 loc) • 7.95 kB
text/typescript
/**
* Measured capability per task domain, so role auto-assign can tier candidates by what they score
* instead of what they cost. Artificial Analysis is read live when configured:
*
* - Artificial Analysis (https://artificialanalysis.ai/data-api/docs), when the user configures
* their own API key. Its terms allow that user's internal use with attribution and forbid
* redistributing the raw data, so its scores never leave this process.
*
* Every domain reads the Intelligence Index: Artificial Analysis no longer publishes a Coding or
* an Agentic Index (both are always null) and its free model list carries no math index. The
* source names a model by display name with one row per reasoning setting ("Claude Opus 5.5
* (High)"); capabilityModelKey drops the setting and folds the rest to a slug, which matches a
* catalog id once its provider prefix is gone ("cpa/claude-opus-5-5"). The best setting's score
* stands for the model, since a role can be given that setting. A candidate with no row is
* unmeasured, never guessed.
*/
import { resolveProviderApiKey } from "../providers/api-key-resolve";
import type { OcxConfig } from "../types";
import { SIZING_DOMAINS, type SizingDomain } from "./role-sizing";
export interface CapabilitySource {
readonly name: string;
readonly url: string;
readonly license: string;
/** Whether the terms let a response carry the raw scores. */
readonly redistributable: boolean;
}
export const ARTIFICIAL_ANALYSIS_SOURCE: CapabilitySource = {
name: "Artificial Analysis",
url: "https://artificialanalysis.ai",
license: "Artificial Analysis API terms: internal use, attribution required, no redistribution",
redistributable: false,
};
/**
* The free-tier language model list, which any key may read. It replaced
* /api/v2/data/llms/models, retired at 2026-11-04 23:59 UTC with 410 Gone afterwards
* (https://artificialanalysis.ai/data-api/migrate-v2-data).
*/
export const ARTIFICIAL_ANALYSIS_URL = "https://artificialanalysis.ai/api/v2/language/models/free";
const INTELLIGENCE_INDEX = "artificial_analysis_intelligence_index";
/**
* Pages read per refresh. The list is paginated (200 rows a page in the published example, two
* pages); ten pages bound one daily refresh to a tenth of the free tier's 100 requests a day.
*/
export const MAX_TABLE_PAGES = 10;
const FETCH_TIMEOUT_MS = 30_000;
/** The bytes all pages of one refresh may total; the list is under 1 MB today, 16 MB still bounds memory. */
export const MAX_TABLE_BYTES = 16 * 1024 * 1024;
export interface CapabilityTable {
readonly source: CapabilitySource | null;
readonly publishedAt: string | null;
/** Per domain: model key to its best published score. */
readonly domains: Partial<Record<SizingDomain, ReadonlyMap<string, number>>>;
/** Why a configured source was passed over for this one. */
readonly fallbackReason?: string;
}
export type LoadCapability = (config: OcxConfig) => Promise<CapabilityTable>;
export function capabilityModelKey(name: string): string {
return name.toLowerCase().replace(/\([^)]*\)/g, " ").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
}
export function candidateModelKey(slug: string): string {
return capabilityModelKey(slug.slice(slug.lastIndexOf("/") + 1));
}
function keepBest(scores: Map<string, number>, key: string, value: number) {
if (key) scores.set(key, Math.max(value, scores.get(key) ?? -Infinity));
}
export function capabilityTableFromArtificialAnalysis(body: unknown): CapabilityTable {
const rows = (body as { data?: unknown } | null)?.data;
if (!Array.isArray(rows)) throw new Error("Artificial Analysis returned no model list");
const scores = new Map<string, number>();
for (const row of rows as Record<string, unknown>[]) {
const value = (row?.evaluations as Record<string, unknown> | undefined)?.[INTELLIGENCE_INDEX];
if (typeof value !== "number" || !Number.isFinite(value)) continue;
if (typeof row.name === "string") keepBest(scores, capabilityModelKey(row.name), value);
if (typeof row.slug === "string") keepBest(scores, capabilityModelKey(row.slug), value);
}
const domains: Partial<Record<SizingDomain, ReadonlyMap<string, number>>> = {};
if (scores.size > 0) for (const domain of SIZING_DOMAINS) domains[domain] = scores;
return { source: ARTIFICIAL_ANALYSIS_SOURCE, publishedAt: null, domains };
}
/** The body's bytes, refused once they pass maxBytes, whatever the transport declared. */
export async function readBoundedBytes(response: Response, maxBytes: number): Promise<ArrayBuffer> {
const chunks: Uint8Array[] = [];
let length = 0;
const reader = response.body?.getReader();
if (!reader) return new ArrayBuffer(0);
for (;;) {
const { done, value } = await reader.read();
if (done) break;
if (length + value.byteLength > maxBytes) {
await reader.cancel();
throw new Error("response exceeded byte bound");
}
chunks.push(value);
length += value.byteLength;
}
const out = new Uint8Array(length);
let offset = 0;
for (const chunk of chunks) {
out.set(chunk, offset);
offset += chunk.byteLength;
}
return out.buffer;
}
/** A list past either bound fails whole, so price ranking takes over rather than tiering on part of it. */
export async function fetchArtificialAnalysisTable(apiKey: string, fetchImpl: typeof fetch = fetch): Promise<CapabilityTable> {
const rows: unknown[] = [];
let budget = MAX_TABLE_BYTES;
for (let page = 1; page <= MAX_TABLE_PAGES; page++) {
const url = new URL(ARTIFICIAL_ANALYSIS_URL);
url.searchParams.set("page", String(page));
const response = await fetchImpl(url.href, {
redirect: "error",
headers: { "x-api-key": apiKey },
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
});
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const bytes = await readBoundedBytes(response, budget);
budget -= bytes.byteLength;
const body = JSON.parse(new TextDecoder().decode(bytes)) as { data?: unknown; pagination?: { has_more?: unknown } } | null;
if (!Array.isArray(body?.data)) throw new Error("Artificial Analysis returned no model list");
const hasMore = body.pagination?.has_more;
if (typeof hasMore !== "boolean") throw new Error("Artificial Analysis returned no pagination.has_more");
rows.push(...body.data);
if (!hasMore) return capabilityTableFromArtificialAnalysis({ data: rows });
}
throw new Error(`Artificial Analysis listed more than ${MAX_TABLE_PAGES} pages`);
}
/** One fetch per period key; a failed fetch is forgotten so the next request retries. */
function periodCache() {
let slot: { key: string; table: Promise<CapabilityTable> } | null = null;
return (key: string, fetchTable: () => Promise<CapabilityTable>): Promise<CapabilityTable> => {
if (slot?.key === key) return slot.table;
const table = fetchTable();
slot = { key, table };
table.catch(() => { if (slot?.table === table) slot = null; });
return table;
};
}
const artificialAnalysisCache = periodCache();
/**
* Artificial Analysis when the user's key resolves, read once per UTC day to stay inside the
* free tier's daily request quota. Absent or unavailable scores leave price ranking in charge.
*/
export const loadCapabilityTable: LoadCapability = async config => {
const now = new Date().toISOString();
const unavailable = (fallbackReason: string): CapabilityTable => ({ source: null, publishedAt: null, domains: {}, fallbackReason });
const apiKey = resolveProviderApiKey(config.codexRoleCapability?.artificialAnalysis?.apiKey)?.trim();
if (!apiKey) return unavailable("benchmark data was unavailable: no Artificial Analysis key is configured");
try {
return await artificialAnalysisCache(`${now.slice(0, 10)} ${apiKey}`, () => fetchArtificialAnalysisTable(apiKey));
} catch {
return unavailable("Artificial Analysis could not be read");
}
};