@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
264 lines (244 loc) • 10.1 kB
text/typescript
/** Lossless physical TOML lines with lexical structural boundaries. */
export interface SourceLine {
text: string;
eol: "\r\n" | "\n" | "";
/** False when this physical line began inside a multiline string or collection. */
structural: boolean;
}
export function splitSourceLines(content: string): SourceLine[] {
const lines: SourceLine[] = [];
let offset = 0;
while (offset < content.length) {
const lf = content.indexOf("\n", offset);
if (lf === -1) {
lines.push({ text: content.slice(offset), eol: "", structural: true });
break;
}
const crlf = lf > offset && content[lf - 1] === "\r";
lines.push({
text: content.slice(offset, crlf ? lf - 1 : lf),
eol: crlf ? "\r\n" : "\n",
structural: true,
});
offset = lf + 1;
}
markStructuralLines(lines);
return lines;
}
/** Root boundaries exclude headers and assignments that are merely multiline/array data. */
export function rootSourceLines(content: string): { bom: string; lines: SourceLine[]; rootEnd: number } {
const bom = content.startsWith("\uFEFF") ? "\uFEFF" : "";
const lines = splitSourceLines(content.slice(bom.length));
const firstTable = lines.findIndex(line => line.structural && /^\s*\[/.test(line.text));
return { bom, lines, rootEnd: firstTable === -1 ? lines.length : firstTable };
}
export function sourceText(lines: readonly SourceLine[]): string {
return lines.map(line => line.text + line.eol).join("");
}
const assignmentKeyToken = /^[ \t]*([A-Za-z0-9_-]+|"(?:[^"\\]|\\.)*"|'[^']*')[ \t]*=/;
/** Decode only a single key token, without parsing unrelated values or dotted paths. */
export function rootAssignmentKey(text: string): string | null {
const match = assignmentKeyToken.exec(text);
if (!match) return null;
try {
return Object.keys(Bun.TOML.parse(`${match[1]} = true\n`))[0] ?? null;
} catch {
return null;
}
}
interface AssignmentSpan { key: string; start: number; end: number; text: string }
function lexicalAssignmentSpan(lines: readonly SourceLine[], start: number, limit: number): AssignmentSpan | null {
const line = lines[start];
if (!line?.structural) return null;
const key = rootAssignmentKey(line.text);
if (key === null) return null;
let end = start + 1;
while (end < limit && !lines[end]!.structural) end += 1;
// Store the original assignment bytes, excluding only its final document separator.
const text = sourceText(lines.slice(start, end)).replace(/\r?\n$/, "");
return { key, start, end, text };
}
/** A complete assignment whose value can be decoded without losing precision. */
export function sourceAssignment(lines: readonly SourceLine[], start: number, limit = lines.length):
(AssignmentSpan & { value: unknown }) | null {
const span = lexicalAssignmentSpan(lines, start, limit);
if (!span) return null;
try {
const parsed = Bun.TOML.parse(span.text) as Record<string, unknown>;
return Object.keys(parsed).length === 1 && Object.hasOwn(parsed, span.key)
? { ...span, value: parsed[span.key] } : null;
} catch { return null; }
}
/** Validate complete syntax for removal/capture without requiring a JavaScript value. */
export function sourceAssignmentSpan(lines: readonly SourceLine[], start: number, limit = lines.length): AssignmentSpan | null {
const decoded = sourceAssignment(lines, start, limit);
if (decoded) return decoded;
const span = lexicalAssignmentSpan(lines, start, limit);
if (!span) return null;
// Bun rejects valid TOML integers outside the JS safe-integer range, as scalars and
// inside arrays or inline tables alike. Validate a copy in which each such integer that
// fits signed 64 bits reads `0`; the parser stays responsible for every other piece of
// grammar, and the returned span keeps the original bytes untouched.
const masked = maskWideIntegers(span.text);
if (masked === null) return null;
try {
const parsed = Bun.TOML.parse(masked);
return Object.keys(parsed).length === 1 && Object.hasOwn(parsed, span.key) ? span : null;
} catch { return null; }
}
const integerToken = /^(?:[+-]?(?:0|[1-9](?:_?[0-9])*)|0x[0-9A-Fa-f](?:_?[0-9A-Fa-f])*|0o[0-7](?:_?[0-7])*|0b[01](?:_?[01])*)$/;
const bareToken = /[A-Za-z0-9_+.:-]/;
/**
* The assignment with every integer outside the JS safe range replaced by `0`, skipping
* strings and comments. Null when an integer does not fit signed 64 bits (TOML requires a
* reader to reject it) or when nothing needed masking, so the original parse failure stands.
*/
function maskWideIntegers(text: string): string | null {
let out = "";
let masked = false;
let index = 0;
const copyUntil = (end: number) => { out += text.slice(index, end); index = end; };
while (index < text.length) {
const char = text[index]!;
if (text.startsWith('"""', index) || text.startsWith("'''", index)) {
const delimiter = text.slice(index, index + 3);
let cursor = index + 3;
while (cursor < text.length && !text.startsWith(delimiter, cursor)) cursor += delimiter === '"""' && text[cursor] === "\\" ? 2 : 1;
cursor = Math.min(text.length, cursor + 3);
// A closing delimiter may carry up to two more quote characters of content.
for (let extra = 0; extra < 2 && text[cursor] === delimiter[0]; extra += 1) cursor += 1;
copyUntil(cursor);
} else if (char === '"' || char === "'") {
let cursor = index + 1;
while (cursor < text.length && text[cursor] !== char && text[cursor] !== "\n") cursor += char === '"' && text[cursor] === "\\" ? 2 : 1;
copyUntil(Math.min(text.length, cursor + 1));
} else if (char === "#") {
const newline = text.indexOf("\n", index);
copyUntil(newline === -1 ? text.length : newline);
} else if (bareToken.test(char)) {
let cursor = index + 1;
while (cursor < text.length && bareToken.test(text[cursor]!)) cursor += 1;
const token = text.slice(index, cursor);
// A digit-only bare key (`{ 9223372036854775808 = 1 }`) is a name, not a number.
const isKey = /^[ \t]*[=.]/.test(text.slice(cursor));
if (!isKey && integerToken.test(token)) {
const integer = BigInt(token.replaceAll("_", ""));
if (integer < -(1n << 63n) || integer >= (1n << 63n)) return null;
if (integer > BigInt(Number.MAX_SAFE_INTEGER) || integer < -BigInt(Number.MAX_SAFE_INTEGER)) {
out += "0";
index = cursor;
masked = true;
continue;
}
}
copyUntil(cursor);
} else {
copyUntil(index + 1);
}
}
return masked ? out : null;
}
/** Cosmetic document whitespace only; opaque value lines retain every byte. */
export function normalizeStructuralWhitespace(content: string): string {
const { bom, lines } = rootSourceLines(content);
const out: SourceLine[] = [];
let blankRun = 0;
for (const line of lines) {
if (line.structural && line.text === "") {
blankRun += 1;
if (blankRun > 1) continue;
} else {
blankRun = 0;
}
out.push(line);
}
while (out.at(-1)?.structural && out.at(-1)!.text.trim() === "") out.pop();
const text = bom + sourceText(out);
const eol = lines.find(line => line.eol)?.eol || "\n";
return text.endsWith("\n") ? text : text + eol;
}
type MultilineStringKind = "basic" | "literal" | null;
/** Length of a closing multiline delimiter: TOML allows a run of three to five quotes. */
function quoteRunLength(text: string, index: number, quote: string): number {
let end = index;
while (end < text.length && text[end] === quote && end - index < 5) end += 1;
return end - index;
}
/**
* TOML table-looking text inside a multiline string is data, not syntax. Keep
* a deliberately small lexical state machine so the format-preserving editor
* never treats those physical lines as headers, keys, or ownership markers.
*/
function markStructuralLines(lines: SourceLine[]): void {
let multiline: MultilineStringKind = null;
let squareDepth = 0;
let curlyDepth = 0;
for (const line of lines) {
line.structural = multiline === null && squareDepth === 0 && curlyDepth === 0;
let single: "basic" | "literal" | null = null;
for (let index = 0; index < line.text.length;) {
if (multiline === "basic") {
if (line.text.startsWith('"""', index)) {
// A closing delimiter may carry up to two content quotes before it.
multiline = null;
index += quoteRunLength(line.text, index, '"');
} else if (line.text[index] === "\\") {
index += 2;
} else {
index += 1;
}
continue;
}
if (multiline === "literal") {
if (line.text.startsWith("'''", index)) {
multiline = null;
index += quoteRunLength(line.text, index, "'");
} else {
index += 1;
}
continue;
}
if (single === "basic") {
if (line.text[index] === "\\") index += 2;
else if (line.text[index] === '"') {
single = null;
index += 1;
} else index += 1;
continue;
}
if (single === "literal") {
if (line.text[index] === "'") single = null;
index += 1;
continue;
}
if (line.text[index] === "#") break;
if (line.text.startsWith('"""', index)) {
multiline = "basic";
index += 3;
} else if (line.text.startsWith("'''", index)) {
multiline = "literal";
index += 3;
} else if (line.text[index] === '"') {
single = "basic";
index += 1;
} else if (line.text[index] === "'") {
single = "literal";
index += 1;
} else if (line.text[index] === "[") {
squareDepth += 1;
index += 1;
} else if (line.text[index] === "]") {
squareDepth = Math.max(0, squareDepth - 1);
index += 1;
} else if (line.text[index] === "{") {
curlyDepth += 1;
index += 1;
} else if (line.text[index] === "}") {
curlyDepth = Math.max(0, curlyDepth - 1);
index += 1;
} else {
index += 1;
}
}
}
}