@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
434 lines (411 loc) • 17.4 kB
text/typescript
import { freeformFallbackKeys } from "./apply-patch-envelope";
import {
JSON_ESCAPES,
JSON_WHITESPACE,
MAX_FREEFORM_WRAPPER_SCAN_CHARS,
scanFreeformWrapper,
} from "./freeform-wrapper-scan";
/**
* Whether a body could still grow into one complete outer Markdown fence.
*
* `stripMarkdownCodeFence` removes such a fence at completion for exactly the two tools that
* own the grammar, so a fenced body's streamed bytes and its completed input disagree unless
* the stream holds. A buffer that does not open with a fence can never acquire one, so
* ordinary bodies are unaffected; a buffer that does keeps its preview suppressed for the
* whole call, because a closing fence can still be followed by more text that withdraws it.
*/
function mayBecomeFencedBody(text: string, toolName: string): boolean {
if (toolName !== "exec" && toolName !== "apply_patch") return false;
const head = text.trimStart();
if (head === "") return true;
return head.startsWith("```") || "```".startsWith(head);
}
const LOW_SURROGATE_ESCAPE = /^\\u[dD][c-fC-F][0-9a-fA-F]{2}$/;
const HEX_DIGIT = /^[0-9a-fA-F]$/;
const HEX_QUAD = /^[0-9a-fA-F]{4}$/;
const JSON_NUMBER_CHAR = /^[-+0-9.eE]$/;
const JSON_NUMBER = /^-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?$/;
const JSON_LITERALS = ["true", "false", "null"] as const;
const CANONICAL_KEY = "input";
const STRING_END = Symbol("string end");
const ownsBodyRepair = (toolName: string): boolean => toolName === "exec" || toolName === "apply_patch";
type NestedState =
| "start" | "keyOpen" | "key" | "colon" | "valueOpen" | "string"
| "scalar" | "container" | "afterValue" | "trailing" | "never";
/**
* Whether a decoded executable body can still grow into the one extra `{"input":string}`
* wrapper that completion removes, answered one character at a time so a streamed body is walked
* once rather than once per delta.
*
* Ordinary object-leading JavaScript such as `{ let x` or `({` fails within a few characters and
* streams from there; only a prefix that is still a possible wrapper holds. A false answer never
* turns true again as the body grows, so the held prefix is flushed once and streaming continues.
*
* Every member name must be `input`, because completion unwraps only a single-key object, and
* duplicates are followed because `JSON.parse` keeps the last. A non-string value is skipped to its
* end so a later duplicate can still replace it, and an object that closes on a non-string last
* value is ruled out. Scalars are checked against the JSON grammar; containers are skipped by
* bracket and string tracking alone. That skip is looser than `JSON.parse` but never unsound: it
* finds the true end of every valid container, and any text it misreads is invalid JSON, which
* completion would not unwrap either way.
*/
class NestedWrapperMatcher {
scanned = 0;
private state: NestedState = "start";
private key = "";
/** Pending escape inside the current string: "", a backslash, or `\\u` and the hex read so far. */
private escape = "";
private scalar = "";
private readonly closers: string[] = [];
private inContainerString = false;
private containerEscape = false;
private lastValueString = false;
get possible(): boolean {
return this.state !== "never";
}
feed(text: string): boolean {
for (let i = 0; i < text.length && this.state !== "never"; i++) {
this.scanned++;
this.step(text[i]!);
}
return this.possible;
}
private step(c: string): void {
switch (this.state) {
case "start":
if (!JSON_WHITESPACE.has(c)) this.state = c === "{" ? "keyOpen" : "never";
return;
case "keyOpen":
if (JSON_WHITESPACE.has(c)) return;
this.state = c === '"' ? "key" : "never";
this.key = "";
this.escape = "";
return;
case "key": {
const part = this.stringStep(c);
if (part === null) this.state = "never";
else if (part === STRING_END) this.state = this.key === CANONICAL_KEY ? "colon" : "never";
else if (!CANONICAL_KEY.startsWith(this.key += part)) this.state = "never";
return;
}
case "colon":
if (!JSON_WHITESPACE.has(c)) this.state = c === ":" ? "valueOpen" : "never";
return;
case "valueOpen":
if (JSON_WHITESPACE.has(c)) return;
this.lastValueString = c === '"';
this.escape = "";
if (c === '"') this.state = "string";
else this.startOtherValue(c);
return;
case "string": {
const part = this.stringStep(c);
if (part === null) this.state = "never";
else if (part === STRING_END) this.state = "afterValue";
return;
}
case "scalar":
this.stepScalar(c);
return;
case "container":
this.stepContainer(c);
return;
case "afterValue":
if (JSON_WHITESPACE.has(c)) return;
if (c === ",") this.state = "keyOpen";
else this.state = c === "}" && this.lastValueString ? "trailing" : "never";
return;
case "trailing":
if (!JSON_WHITESPACE.has(c)) this.state = "never";
return;
case "never":
return;
}
}
/**
* One character of a JSON string: the text it decodes to ("" while an escape is pending),
* `STRING_END` at the closing quote, or `null` when no continuation can parse. Only key
* text needs the decoded value, so escapes other than `\\u` keep a marker.
*/
private stringStep(c: string): string | typeof STRING_END | null {
if (this.escape === "") {
if (c === '"') return STRING_END;
if (c.charCodeAt(0) <= 0x1f) return null;
if (c !== "\\") return c;
this.escape = "\\";
return "";
}
if (this.escape === "\\") {
if (c === "u") {
this.escape = "\\u";
return "";
}
if (!JSON_ESCAPES.has(c)) return null;
this.escape = "";
return "\\" + c;
}
if (!HEX_DIGIT.test(c)) return null;
this.escape += c;
if (this.escape.length < 6) return "";
const code = parseInt(this.escape.slice(2), 16);
this.escape = "";
return String.fromCharCode(code);
}
private startOtherValue(c: string): void {
if (c === "{" || c === "[") {
this.closers.length = 0;
this.closers.push(c === "{" ? "}" : "]");
this.inContainerString = false;
this.containerEscape = false;
this.state = "container";
return;
}
if (JSON_NUMBER_CHAR.test(c) || c === "t" || c === "f" || c === "n") {
this.scalar = c;
this.state = "scalar";
return;
}
this.state = "never";
}
private stepScalar(c: string): void {
if (JSON_WHITESPACE.has(c) || c === "," || c === "}") {
const complete = (JSON_LITERALS as readonly string[]).includes(this.scalar) || JSON_NUMBER.test(this.scalar);
if (!complete) {
this.state = "never";
return;
}
this.state = "afterValue";
this.step(c);
return;
}
this.scalar += c;
const literal = JSON_LITERALS.find(word => word[0] === this.scalar[0]);
const viable = literal ? literal.startsWith(this.scalar) : JSON_NUMBER_CHAR.test(c);
if (!viable) this.state = "never";
}
private stepContainer(c: string): void {
if (this.inContainerString) {
if (this.containerEscape) this.containerEscape = false;
else if (c === "\\") this.containerEscape = true;
else if (c === '"') this.inContainerString = false;
return;
}
if (c === '"') this.inContainerString = true;
else if (c === "{" || c === "[") this.closers.push(c === "{" ? "}" : "]");
else if (c === "}" || c === "]") {
if (this.closers.pop() !== c) this.state = "never";
else if (this.closers.length === 0) this.state = "afterValue";
}
}
}
type DecodeStop = "open" | "closed" | "stalled" | "invalid";
/**
* Decode the JSON string body in `text` from `from`, stopping at the first byte it cannot
* resolve. `next` is where decoding resumes once more text arrives; `scanned` counts the characters
* read, including a split escape that will be read again.
*
* Every stop is a hold rather than a guess, because `JSON.parse` decides the completed value
* and anything invented here would be retracted. Three of them are not obvious:
*
* An escape JSON does not define makes the whole wrapper unparseable no matter what arrives
* next, so completion falls back to the raw text. `invalid` stops the preview rather than
* continuing to decode a value the completed item will never carry. `\\b` and `\\f` are defined,
* and were previously decoded to the letters b and f. A literal control character is not legal
* inside a JSON string either, and stops the same way.
*
* A lone high surrogate is not a character. Emitting one alone puts an unpaired code unit in a
* delta the client has to decode by itself, so the pair is emitted together or not at all; a
* complete escape that is not its low half leaves the preview `stalled` for the rest of the call.
*/
function decodeJsonStringChunk(
text: string,
from: number,
): { chunk: string; next: number; stop: DecodeStop; scanned: number } {
let chunk = "";
let scanned = 0;
let i = from;
let stop: DecodeStop = "open";
while (i < text.length) {
const c = text[i]!;
scanned++;
if (c === '"') {
stop = "closed";
break;
}
if (c !== "\\") {
if (c.charCodeAt(0) <= 0x1f) {
stop = "invalid";
break;
}
chunk += c;
i++;
continue;
}
const n = text[i + 1];
if (n === undefined) break; // escape split across chunks: wait for more
scanned++;
if (n === "u") {
const hex = text.slice(i + 2, i + 6);
scanned += hex.length;
if (hex.length < 4) break; // split across chunks: wait for more
if (!HEX_QUAD.test(hex)) {
stop = "invalid";
break;
}
const code = parseInt(hex, 16);
if (code >= 0xd800 && code <= 0xdbff) {
const low = text.slice(i + 6, i + 12);
scanned += low.length;
if (!LOW_SURROGATE_ESCAPE.test(low)) {
if (low.length === 6) stop = "stalled";
break;
}
chunk += String.fromCharCode(code, parseInt(low.slice(2), 16));
i += 12;
continue;
}
chunk += String.fromCharCode(code);
i += 6;
continue;
}
const escaped = JSON_ESCAPES.get(n);
if (escaped === undefined) {
stop = "invalid";
break;
}
chunk += escaped;
i += 2;
}
return { chunk, next: i, stop, scanned };
}
/**
* The value to stream so far for one call's growing arguments, or `null` to HOLD because nothing
* can be decided yet.
*
* A reader belongs to one call and is fed that call's whole accumulated arguments on each delta.
* They only grow, so it resumes where the previous delta stopped: the canonical value is decoded
* and checked for a nested wrapper once overall, which keeps a long body streamed in small deltas
* linear instead of quadratic. A shorter buffer or a different tool name starts over.
*
*
* `input` is decidable from its prefix: `unwrapFreeformToolInput` returns it whenever the
* key is present, whatever else the object carries, so its value can be unescaped
* progressively and never retracted, except for executable bodies that may contain one
* extra canonical wrapper. Such a body holds only while its prefix can still become that
* wrapper (`mayBecomeNestedWrapper`); the first byte that rules it out flushes the held text.
*
* One case escapes that claim and is accepted rather than fixed: a duplicate `input` key.
* `JSON.parse` keeps the last one, so `{"input":"a","input":"b"}` streams a and completes with
* b. Closing it means holding every canonical wrapper until its object parses, which is the
* progressive streaming this path exists to provide. The completed item stays authoritative,
* and no model emits a duplicate key in practice.
*
* A wrapper that turns invalid AFTER streaming has committed has the same shape and the same
* answer. `{"input":"a` followed by `\\qb"}` has already published a when the undefined escape
* arrives, and completion returns the raw text because nothing parses. The preview stops
* there: no rewind, and no decoded text the completed item does not contain. Bounding the
* damage is what is available without giving up progressive streaming, and the args are
* unusable in that case whichever representation wins.
*
* A fallback key decides only at the object's close. It unwraps as the SINGLE string field,
* and a second key can still arrive while the object is open — so a value emitted early would
* have to be taken back. The scan therefore keeps a member table and consults it at the
* close, publishing the wrapped value (or the raw text) without ever reparsing the buffer.
* The routed passthrough in `responses-custom-tool-repair.ts` already holds any object prefix
* for the same reason (#5047).
*
* An object that has not reached a canonical key YET is in exactly that position, and used to
* be treated as raw because it did not match the literal `{"input":"`. It holds now: `input`
* can still arrive after other properties, or wearing an escaped spelling, and completion would
* then unwrap a body whose wrapper syntax had already been published as deltas (#5151). The
* cost of holding is preview on an object body that turns out not to be a wrapper; the cost of
* not holding was publishing bytes the completed item removes.
*/
export class ProgressiveFreeformInputReader {
private toolName: string | undefined;
private seen = 0;
private mode: "scan" | "raw" | "hold" | "input" = "scan";
private next = 0;
private decoded = "";
private stop: DecodeStop = "open";
private nested: NestedWrapperMatcher | null = null;
private fenceRuledOut = false;
private decodeScanned = 0;
/** Characters this reader has walked so far; a test seam for the linear-work bound. */
get scannedChars(): number {
return this.decodeScanned + (this.nested?.scanned ?? 0);
}
update(args: string, toolName: string): string | null {
if (toolName !== this.toolName || args.length < this.seen) this.reset(toolName);
this.seen = args.length;
if (this.mode === "scan") {
const scan = scanFreeformWrapper(args, freeformFallbackKeys(toolName));
if (scan.kind === "raw") {
// Prefix-decided: no extension of this text becomes a wrapper again.
this.mode = "raw";
} else if (scan.kind === "hold") {
// Past the bound the scan only ever sees the same clamped prefix, so it holds for good.
if (args.length > MAX_FREEFORM_WRAPPER_SCAN_CHARS) this.mode = "hold";
return null;
} else if (scan.canonical) {
this.mode = "input";
this.next = scan.valueStart;
this.nested = ownsBodyRepair(toolName) ? new NestedWrapperMatcher() : null;
} else {
// A closed fallback wrapper can still be withdrawn by trailing text, so it is decided
// afresh on each delta. Its object fits inside the scan bound, which bounds this too.
return this.closedFallbackValue(args, scan.valueStart, toolName);
}
}
if (this.mode === "hold") return null;
if (this.mode === "raw") return this.mayBecomeFenced(args) ? null : args;
return this.canonicalValue(args);
}
private reset(toolName: string): void {
this.toolName = toolName;
this.seen = 0;
this.mode = "scan";
this.next = 0;
this.decoded = "";
this.stop = "open";
this.nested = null;
this.fenceRuledOut = false;
}
private canonicalValue(args: string): string | null {
if (this.stop === "open") {
const step = decodeJsonStringChunk(args, this.next);
this.decodeScanned += step.scanned;
this.next = step.next;
this.stop = step.stop;
this.decoded += step.chunk;
if (this.nested?.possible) this.nested.feed(step.chunk);
}
if (this.stop === "invalid") return null;
// Completion may remove one extra {input:string} body wrapper, so hold while it can.
if (this.nested?.possible) return null;
return this.mayBecomeFenced(this.decoded) ? null : this.decoded;
}
private closedFallbackValue(args: string, valueStart: number, toolName: string): string | null {
const step = decodeJsonStringChunk(args, valueStart);
this.decodeScanned += step.scanned;
if (step.stop === "invalid") return null;
if (ownsBodyRepair(toolName)) {
const nested = new NestedWrapperMatcher();
const possible = nested.feed(step.chunk);
this.decodeScanned += nested.scanned;
if (possible) return null;
}
return mayBecomeFencedBody(step.chunk, toolName) ? null : step.chunk;
}
/** `mayBecomeFencedBody`, remembering a false answer: a ruled-out fence never returns. */
private mayBecomeFenced(text: string): boolean {
if (this.fenceRuledOut) return false;
if (mayBecomeFencedBody(text, this.toolName ?? "")) return true;
this.fenceRuledOut = true;
return false;
}
}
/** One-shot form of `ProgressiveFreeformInputReader` for a buffer read once. */
export function progressiveFreeformInput(args: string, toolName: string): string | null {
return new ProgressiveFreeformInputReader().update(args, toolName);
}