UNPKG

@bitkyc08/opencodex

Version:

Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code

425 lines (413 loc) • 18 kB
import { chatCompletionsErrorBody } from "../chat/outbound"; import { classifyError, cyberPolicyErrorType, CYBER_POLICY_ERROR_CODE, isCyberPolicyCode } from "../lib/errors"; import { redactSecretString } from "../lib/redact"; import { isTranslatorBudgetExceededError, TRANSLATOR_MAX_SSE_EVENT_BYTES, type TranslatorBudget, } from "../lib/translator-budget"; import type { OcxUsage } from "../types"; import { resolveStallTimeoutSec } from "../stall-timeout"; import { createSseBlockBuffer, replaceSseDataPayload, sseDataPayload } from "./sse-payload-rewrite"; type Rec = Record<string, unknown>; function isRec(value: unknown): value is Rec { return value !== null && typeof value === "object" && !Array.isArray(value); } export function usageFromChat(value: unknown): OcxUsage | undefined { if (!isRec(value)) return undefined; const prompt = typeof value.prompt_tokens === "number" ? value.prompt_tokens : undefined; const completion = typeof value.completion_tokens === "number" ? value.completion_tokens : undefined; if (prompt === undefined && completion === undefined) return undefined; const promptDetails = isRec(value.prompt_tokens_details) ? value.prompt_tokens_details : undefined; const completionDetails = isRec(value.completion_tokens_details) ? value.completion_tokens_details : undefined; return { inputTokens: prompt ?? 0, outputTokens: completion ?? 0, ...(typeof promptDetails?.cached_tokens === "number" ? { cachedInputTokens: promptDetails.cached_tokens } : {}), ...(typeof completionDetails?.reasoning_tokens === "number" ? { reasoningOutputTokens: completionDetails.reasoning_tokens } : {}), }; } export function structuredError(value: unknown): { message: string; type?: string; code?: string | null; status?: number; } | null { if (!isRec(value) || value.error === undefined || value.error === null) return null; const error = value.error; if (typeof error === "string") return { message: redactSecretString(error) }; if (!isRec(error)) return { message: "upstream error" }; const rawMessage = typeof error.message === "string" ? error.message : "upstream error"; return { message: redactSecretString(rawMessage), ...(typeof error.type === "string" ? { type: error.type } : {}), ...(error.code === null || typeof error.code === "string" ? { code: error.code } : {}), ...(typeof error.status === "number" && Number.isInteger(error.status) ? { status: error.status } : {}), }; } function normalizedChunk(value: Rec, requestedModel: string): Rec { return { ...value, id: typeof value.id === "string" ? value.id : `chatcmpl-${crypto.randomUUID().replace(/-/g, "").slice(0, 24)}`, object: "chat.completion.chunk", created: typeof value.created === "number" ? value.created : Math.floor(Date.now() / 1000), model: requestedModel, }; } export function jsonCompletionSse(value: Rec, requestedModel: string, budget?: TranslatorBudget): string { const id = typeof value.id === "string" ? value.id : `chatcmpl-${crypto.randomUUID().replace(/-/g, "").slice(0, 24)}`; const created = typeof value.created === "number" ? value.created : Math.floor(Date.now() / 1000); const model = requestedModel; // Relay the upstream service-tier echo on every chunk, matching how OpenAI/xAI // annotate streamed chat chunks, so streaming callers see the same confirmation // the non-streaming body carries. const serviceTier = typeof value.service_tier === "string" ? { service_tier: value.service_tier } : {}; const choices = Array.isArray(value.choices) ? value.choices : []; const choice = isRec(choices[0]) ? choices[0] : {}; const message = isRec(choice.message) ? choice.message : {}; const frames: Rec[] = [{ id, object: "chat.completion.chunk", created, model, ...serviceTier, choices: [{ index: 0, delta: { role: "assistant", content: "" }, finish_reason: null }], }]; const delta: Rec = {}; if (typeof message.content === "string" && message.content.length > 0) delta.content = message.content; if (typeof message.refusal === "string") delta.refusal = message.refusal; if (typeof message.reasoning_content === "string" && message.reasoning_content.length > 0) { delta.reasoning_content = message.reasoning_content; } if (Array.isArray(message.tool_calls) && message.tool_calls.length > 0) { delta.tool_calls = message.tool_calls.filter(isRec).map((tool, index) => ({ ...tool, index })); } if (Object.keys(delta).length > 0) { frames.push({ id, object: "chat.completion.chunk", created, model, ...serviceTier, choices: [{ index: 0, delta, finish_reason: null }] }); } frames.push({ id, object: "chat.completion.chunk", created, model, ...serviceTier, choices: [{ index: 0, delta: {}, finish_reason: typeof choice.finish_reason === "string" ? choice.finish_reason : "stop" }], ...(value.usage !== undefined ? { usage: value.usage } : {}), }); // Keep the frame strings charged while the joined body is allocated. The final // string and Response's UTF-8 body coexist until response ownership ends. const scope = { kind: "live_transient" as const }; let frameBytes = 0; const serialized: string[] = []; try { for (const frame of frames) { const text = `data: ${JSON.stringify(frame)}\n\n`; const bytes = Buffer.byteLength(text); budget?.chargeRetained(bytes, scope); frameBytes += bytes; serialized.push(text); } const done = "data: [DONE]\n\n"; const outputBytes = frameBytes + Buffer.byteLength(done); budget?.chargeRetained(outputBytes * 2, scope); return serialized.join("") + done; } finally { budget?.releaseRetained(frameBytes, scope); } } interface NativeChatSseOptions { requestedModel: string; translatorBudget: TranslatorBudget; signal: AbortSignal; stallTimeoutSec?: number; /** * The upstream is local infrastructure (loopback / private / `.local` / `.lan`); an unset * `stallTimeoutSec` then resolves to disabled so a slow local model is not cut mid-turn. */ localUpstream?: boolean; onFirstOutput?: () => void; onUsage: (usage: OcxUsage) => void; onTerminal?: (status: number, message?: string) => void; onCancel?: () => void; } export function nativeChatSse( body: ReadableStream<Uint8Array>, options: NativeChatSseOptions, ): ReadableStream<Uint8Array> { const reader = body.getReader(); const decoder = new TextDecoder(); const encoder = new TextEncoder(); const scope = { kind: "live_transient" as const }; const buffer = createSseBlockBuffer(options.translatorBudget, nextBytes => { if (nextBytes > TRANSLATOR_MAX_SSE_EVENT_BYTES) { throw new Error("upstream SSE event exceeded the safe limit", { cause: { code: "translation_buffer_limit" } }); } }); let queuedBytes = 0; let sawFinish = false; let sawDone = false; let firstOutput = false; let sawTextOutput = false; let sawToolCalls = false; let settled = false; let cancelled = false; let cancelledBySignal = false; const stallSec = resolveStallTimeoutSec(options.stallTimeoutSec, { localUpstream: options.localUpstream }); // 0 is an explicit disabled budget (local upstream, or the operator's stallTimeoutSec: 0). Map // it to an Infinity deadline so every `now >= pullDeadline` / `remaining <= 0` check stays // false and the read race is skipped, instead of arming an immediate kill on the first pull. const stallEnabled = stallSec > 0; const stallMs = stallEnabled ? stallSec * 1_000 : Number.POSITIVE_INFINITY; // Count only active upstream consumption; downstream backpressure must not // consume the provider's inactivity allowance. Comments and empty deltas do. let remainingStallMs = stallMs; let pullDeadline = 0; const stalled = new Error("upstream stream stalled without meaningful progress"); const releaseQueued = () => { if (queuedBytes === 0) return; options.translatorBudget.releaseRetained(queuedBytes, scope); queuedBytes = 0; }; const enqueue = (controller: ReadableStreamDefaultController<Uint8Array>, text: string) => { const byteLength = Buffer.byteLength(text); const reservation = options.translatorBudget.reserveTransient(byteLength, scope); try { controller.enqueue(encoder.encode(text)); reservation.commitRetained(); queuedBytes += byteLength; } catch (error) { reservation.release(); throw error; } }; const releaseBuffer = () => buffer.clear(); const settle = (status: number, message?: string) => { if (settled) return; settled = true; options.signal.removeEventListener("abort", onAbort); options.onTerminal?.(status, message); }; const fail = ( controller: ReadableStreamDefaultController<Uint8Array>, message: string, code: string, status = 502, type = "upstream_error", ) => { const safeMessage = redactSecretString(message); try { const frame = `data: ${JSON.stringify(chatCompletionsErrorBody(status, safeMessage, type, code))}\n\n`; if (code === "translation_buffer_limit") controller.enqueue(encoder.encode(frame)); else enqueue(controller, frame); } catch { /* controller may already be closed */ } releaseBuffer(); settle(status, safeMessage); try { void reader.cancel(new Error(safeMessage)).catch(() => {}); } catch { /* already closed */ } try { controller.close(); } catch { /* already closed */ } }; const processBlock = ( controller: ReadableStreamDefaultController<Uint8Array>, block: string, delimiter: string, ): void => { const payload = sseDataPayload(block); if (payload === null) { if (performance.now() >= pullDeadline) throw stalled; enqueue(controller, block + delimiter); return; } const trimmed = payload.trim(); if (trimmed === "") { if (performance.now() >= pullDeadline) throw stalled; enqueue(controller, block + delimiter); return; } if (trimmed === "[DONE]") { sawDone = true; enqueue(controller, replaceSseDataPayload(block, "[DONE]") + delimiter); releaseBuffer(); settle(200); try { void reader.cancel().catch(() => {}); } catch { /* already closed */ } controller.close(); return; } let parsed: unknown; try { parsed = JSON.parse(payload); } catch { fail(controller, "upstream returned malformed SSE JSON", "upstream_sse_invalid"); return; } if (!isRec(parsed)) { fail(controller, "upstream returned an invalid SSE event", "upstream_sse_invalid"); return; } const error = structuredError(parsed); if (error) { const status = error.status ?? 502; const classified = classifyError(status, error.type ?? "upstream_error", error.message); if (isCyberPolicyCode(error.code) || classified.code === CYBER_POLICY_ERROR_CODE) { classified.code = CYBER_POLICY_ERROR_CODE; classified.type = cyberPolicyErrorType(error.type); } else if (error.code !== undefined && error.code !== null) { classified.code = error.code; } const safe = chatCompletionsErrorBody(status, classified.message, classified.type, classified.code); enqueue(controller, replaceSseDataPayload(block, JSON.stringify(safe)) + delimiter); releaseBuffer(); settle(isCyberPolicyCode(classified.code) ? 400 : status, classified.message); try { void reader.cancel(new Error(classified.message)).catch(() => {}); } catch { /* already closed */ } controller.close(); return; } const usage = usageFromChat(parsed.usage); if (usage) options.onUsage(usage); const choices = Array.isArray(parsed.choices) ? parsed.choices : []; const finished = choices.some(choice => isRec(choice) && typeof choice.finish_reason === "string" && choice.finish_reason.length > 0); if (finished) sawFinish = true; const hasText = choices.some(choice => { if (!isRec(choice) || !isRec(choice.delta)) return false; const delta = choice.delta; return (typeof delta.content === "string" && delta.content.length > 0) || (typeof delta.reasoning_content === "string" && delta.reasoning_content.length > 0) || (typeof delta.reasoning === "string" && delta.reasoning.length > 0) || (Array.isArray(delta.reasoning_details) && delta.reasoning_details.some(detail => isRec(detail) && typeof detail.text === "string" && detail.text.length > 0)) || (typeof delta.refusal === "string" && delta.refusal.length > 0); }); const hasTool = choices.some(choice => { if (!isRec(choice) || !isRec(choice.delta)) return false; const delta = choice.delta; return Array.isArray(delta.tool_calls) && delta.tool_calls.some(tool => { if (!isRec(tool)) return false; const fn = isRec(tool.function) ? tool.function : {}; return (typeof tool.id === "string" && tool.id.length > 0) || (typeof fn.name === "string" && fn.name.length > 0) || (typeof fn.arguments === "string" && fn.arguments.length > 0); }); }); if (hasText) sawTextOutput = true; if (hasTool) sawToolCalls = true; const hasProgress = hasText || hasTool; if (hasProgress || finished) pullDeadline = performance.now() + stallMs; else if (performance.now() >= pullDeadline) throw stalled; if (!firstOutput && hasProgress) { firstOutput = true; options.onFirstOutput?.(); } enqueue(controller, replaceSseDataPayload(block, JSON.stringify(normalizedChunk(parsed, options.requestedModel))) + delimiter); }; function onAbort() { cancelledBySignal = true; releaseBuffer(); if (!settled) { settled = true; options.onCancel?.(); } try { void reader.cancel(options.signal.reason).catch(() => {}); } catch { /* already closed */ } } if (options.signal.aborted) onAbort(); else options.signal.addEventListener("abort", onAbort, { once: true }); return new ReadableStream<Uint8Array>({ async pull(controller) { releaseQueued(); pullDeadline = performance.now() + remainingStallMs; try { for (;;) { if (cancelledBySignal) { releaseBuffer(); controller.close(); return; } const next = buffer.next(); if (next) { buffer.compact(); processBlock(controller, next.block, next.delimiter); return; } if (performance.now() >= pullDeadline) throw stalled; let timeout: ReturnType<typeof setTimeout> | undefined; let read: Awaited<ReturnType<typeof reader.read>>; try { read = stallEnabled ? await Promise.race([ reader.read(), new Promise<never>((_, reject) => { const check = () => { const remaining = pullDeadline - performance.now(); if (remaining <= 0) reject(stalled); else timeout = setTimeout(check, Math.min(remaining, 2_147_483_647)); }; check(); }), ]) : await reader.read(); } finally { if (timeout !== undefined) clearTimeout(timeout); } const { done, value } = read; if (cancelled) return; if (cancelledBySignal) { releaseBuffer(); controller.close(); return; } if (!done) { buffer.append(decoder.decode(value, { stream: true })); continue; } buffer.append(decoder.decode()); if (buffer.tail().trim().length > 0) { fail(controller, "upstream SSE ended with an unterminated event", "upstream_sse_unterminated"); return; } releaseBuffer(); if (sawDone || sawFinish) { if (!sawDone) enqueue(controller, "data: [DONE]\n\n"); settle(200); controller.close(); } else if (sawToolCalls) { // Mid tool-call truncation must fail closed to prevent downstream executing partial arguments. fail(controller, "upstream SSE ended mid tool call without a terminal signal — possible truncation", "upstream_sse_truncated"); } else if (sawTextOutput) { // Upstream produced text or reasoning output but missed the terminal frame; recover cleanly with [DONE]. enqueue(controller, "data: [DONE]\n\n"); settle(200); controller.close(); } else { fail(controller, "upstream SSE ended before a terminal event", "upstream_sse_truncated"); } return; } } catch (error) { if (cancelled || cancelledBySignal) { releaseBuffer(); try { controller.close(); } catch { /* already closed */ } return; } const overflow = isTranslatorBudgetExceededError(error) || (error instanceof Error && (error.cause as { code?: unknown } | undefined)?.code === "translation_buffer_limit"); fail( controller, overflow ? "upstream SSE event exceeded the safe limit" : error instanceof Error ? error.message : String(error), overflow ? "translation_buffer_limit" : error === stalled ? "upstream_stall_timeout" : "upstream_sse_error", ); } finally { remainingStallMs = Math.max(0, pullDeadline - performance.now()); } }, cancel(reason) { cancelled = true; releaseQueued(); releaseBuffer(); options.signal.removeEventListener("abort", onAbort); if (!settled) { settled = true; options.onCancel?.(); } try { return reader.cancel(reason).then(() => undefined, () => undefined); } catch { return Promise.resolve(); } }, }); }