@bitkyc08/opencodex
Version:
Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code
435 lines (415 loc) • 19.1 kB
text/typescript
/**
* ChatGPT-backend citation markers.
*
* The ChatGPT backend delimits inline citations with Unicode private-use characters:
*
* \uE200 cite \uE202 turn1view0 \uE202 turn1view1 \uE201
*
* The desktop client renders that as source chips. The Codex TUI does not: it prints the
* codepoints literally, so the user sees "citeturn1view0turn1view1" in the answer and in
* the saved transcript (#3150).
*
* OpenCodex neither produces nor understands this grammar — it arrives as ordinary
* assistant text from a ChatGPT-derived backend (GitHub Copilot in the report). The proxy
* is the last place that can remove it before a client that cannot render it.
*
* Strip, do not translate. The `turnNviewN` ids are turn-scoped and opaque, and the
* response carries no mapping from them to a URL, so there is nothing to convert them
* into. Structured `url_citation` annotations are a separate path and are untouched.
*
* Only citation spans are stripped. The same delimiters carry other inline directives
* that the Codex App renders, such as `\uE200visualize\uE202{"path":…}\uE201` from the
* bundled visualize plugin (#6039). A span is a citation only when the keyword between
* START and its first SEPARATOR or END is one of `CITATION_KEYWORDS`; any other span
* passes through byte for byte.
*/
/** Opens a citation span. */
export const CITATION_MARKER_START = "\uE200";
/** Separates the `cite` keyword and each source reference inside a span. */
export const CITATION_MARKER_SEPARATOR = "\uE202";
/** Closes a citation span. */
export const CITATION_MARKER_END = "\uE201";
/**
* Keywords that mark a span as a ChatGPT citation. `filecite` is the uploaded-file variant
* of `cite` and uses the same `turnN…` reference grammar.
*/
const CITATION_KEYWORDS: readonly string[] = ["cite", "filecite"];
/** True when the text contains any of the three delimiters. Cheap pre-check. */
export function hasCitationMarker(text: string): boolean {
return text.includes(CITATION_MARKER_START)
|| text.includes(CITATION_MARKER_SEPARATOR)
|| text.includes(CITATION_MARKER_END);
}
/**
* Upper bound on the length of a span (START through END inclusive).
*
* A real citation is `cite` plus a few turn-scoped ids, so it is far under this. Without a
* bound, a backend that emits a START and never terminates it makes the streaming filter
* withhold the rest of the response. A non-citation directive keeps later STARTs literal
* only within the same bound, so one unterminated directive cannot hide a real citation for
* the rest of the message.
*/
const MAX_CITATION_SPAN_LENGTH = 4_096;
export interface CitationMarkerFilter {
/** Feed one streaming delta; returns the portion safe to emit now. */
push(delta: string): string;
/** Release anything still held when the message closes. */
flush(): string;
}
/**
* The memory-citation tags the Codex client parses as hidden markup (openai/codex
* `codex-rs/utils/stream-parser/src/citation.rs`): matching is literal and non-nested, the text
* from an opening tag to the FIRST closing tag is hidden, and an opening tag that never closes is
* auto-closed at the end of the stream, hiding everything after it. A model that quotes the tag
* ("output the `...` block at the end") therefore appears to truncate at the quote. Backticks do
* not protect it, because the client parses before any markdown rendering.
*/
const MEMORY_CITATION_OPEN = "<oai-mem-citation>";
const MEMORY_CITATION_CLOSE = "</oai-mem-citation>";
const MEMORY_CITATION_MENTION = /<(\/?oai-mem-citation>)/g;
/**
* Body shape of a REAL memory-citation block: it opens with a complete `<citation_entries>` list, or
* with a complete ids list (`<rollout_ids>` or `<thread_ids>`) when the entries are absent. That matches
* the client's `parse_memory_citation` (`codex-rs/memories/read/src/citations.rs`), which accepts a block
* carrying either list and rejects one carrying neither.
*
* This is what separates a block the client should render from a mention of the tag name inside
* ordinary prose. Escaping both would break the real feature: the client would print the tags
* instead of the citation list, and the turn's memory references would silently disappear.
*/
const MEMORY_CITATION_BODY_ELEMENTS: ReadonlyArray<readonly [open: string, close: string]> = [
["<citation_entries>", "</citation_entries>"],
["<rollout_ids>", "</rollout_ids>"],
["<thread_ids>", "</thread_ids>"],
];
/** First index at or after `from` (below `limit`) that is not whitespace. */
function skipWhitespace(text: string, from: number, limit: number): number {
let at = from;
while (at < limit && /\s/.test(text[at]!)) at++;
return at;
}
/**
* Longest candidate block, in UTF-16 code units (JavaScript string length), that the streaming
* escaper holds while waiting for its closing tag; the whole-string form applies the same limit.
* Callers charge the held bytes to their translator budget (`live_transient`) as the hold grows.
*/
const MAX_MEMORY_CITATION_HOLD = 16_384;
/**
* Whether the text right after an opening tag still could begin a real block.
*
* A real block puts `<citation_entries>` (usually on the next line) immediately after the open
* tag. Ordinary prose that quotes the tag puts a backtick, punctuation, or words there instead,
* so one short lookahead separates the two without buffering the whole block.
*/
function bodyCouldBeReal(afterOpen: string): boolean {
const trimmed = afterOpen.replace(/^\s+/, "");
return MEMORY_CITATION_BODY_ELEMENTS.some(([open]) => open.startsWith(trimmed) || trimmed.startsWith(open));
}
function isMemoryCitationBlock(body: string): boolean {
const trimmed = body.replace(/^\s+/, "");
return MEMORY_CITATION_BODY_ELEMENTS.some(([open, close]) => trimmed.startsWith(open) && trimmed.includes(close, open.length));
}
/**
* Whether an opening tag begins a real block: the entries list has to follow it directly.
*
* `isMemoryCitationBlock` over a whole body cannot answer this, because prose before a real block
* also spans a `<citation_entries>…</citation_entries>…<rollout_ids>` stretch when the search is
* anchored at the mention rather than at the block. The adjacency check is what makes the mention
* provably a mention.
*/
function opensRealBlock(text: string, openAt: number, closeCache?: { at: number }): number | undefined {
const afterOpen = openAt + MEMORY_CITATION_OPEN.length;
const bodyStart = skipWhitespace(text, afterOpen, Math.min(text.length, openAt + MAX_MEMORY_CITATION_HOLD));
if (!MEMORY_CITATION_BODY_ELEMENTS.some(([open]) => text.startsWith(open, bodyStart))) return undefined;
// Callers scanning one string for several opening tags share the next closing tag, so prose with
// many unclosed candidates costs one forward search instead of one per tag (-2: not searched yet).
let close: number;
if (closeCache) {
if (closeCache.at === -2 || (closeCache.at !== -1 && closeCache.at < bodyStart)) {
closeCache.at = text.indexOf(MEMORY_CITATION_CLOSE, bodyStart);
}
close = closeCache.at;
} else {
close = text.indexOf(MEMORY_CITATION_CLOSE, bodyStart);
}
if (close === -1) return undefined;
// The streaming escaper holds at most MAX_MEMORY_CITATION_HOLD of a candidate, so a block larger
// than that is a mention in BOTH forms; one size rule keeps deltas and `output_text.done` equal.
if (close + MEMORY_CITATION_CLOSE.length - openAt > MAX_MEMORY_CITATION_HOLD) return undefined;
return isMemoryCitationBlock(text.slice(afterOpen, close)) ? close + MEMORY_CITATION_CLOSE.length : undefined;
}
/**
* Neutralize a literal memory-citation tag so the client renders it as text.
*
* The client parses the tag as markup before markdown, so wrapping it in backticks does not
* protect it. The leading `<` becomes `<`. In ordinary markdown text that renders as `<` again;
* inside an inline code span or a fenced block CommonMark does not decode entities, so the reader
* sees `<oai-mem-citation>` literally. That visible difference is the trade-off for keeping the
* rest of the answer: the alternative is the client hiding everything after the quote.
*
* Only the Responses wire needs this: Chat Completions and Messages clients never parse the tag,
* so the bridges apply it behind `escapeMemoryCitationTags`.
*/
export function escapeMemoryCitationMentions(text: string): string {
if (!text.includes("oai-mem-citation")) return text;
// Claim real blocks FIRST, so a mention earlier in the prose cannot pair with a real block's
// closing tag and leave the block half-escaped. Everything outside a claimed block is a
// mention and gets escaped, tags included. Matching runs over the ORIGINAL string as one pass
// (appendReplacement), because `String.replace` would re-scan the text we just escaped.
const blocks: Array<{ start: number; end: number }> = [];
const closeCache = { at: -2 };
let searchFrom = 0;
for (;;) {
const open = text.indexOf(MEMORY_CITATION_OPEN, searchFrom);
if (open === -1) break;
const end = opensRealBlock(text, open, closeCache);
if (end !== undefined) {
blocks.push({ start: open, end });
searchFrom = end;
} else {
searchFrom = open + MEMORY_CITATION_OPEN.length;
}
}
// Outside claimed blocks every tag is a mention; one linear regex pass escapes them all.
const escapedMentions = (segment: string): string => segment.replace(MEMORY_CITATION_MENTION, "<$1");
if (blocks.length === 0) return escapedMentions(text);
let out = "";
let cursor = 0;
for (const block of blocks) {
out += escapedMentions(text.slice(cursor, block.start));
out += text.slice(block.start, block.end);
cursor = block.end;
}
out += escapedMentions(text.slice(cursor));
return out;
}
export interface MemoryCitationEscaper {
/** Feed one streaming delta; returns the portion safe to emit now. */
push(delta: string): string;
/** Text still withheld at the end of the stream; resolve it with `escapeMemoryCitationMentions`. */
pending(): string;
}
/** Escaper for deliveries that must not rewrite text (Chat Completions, Messages). */
export const passthroughMemoryCitationEscaper: MemoryCitationEscaper = {
push: (delta: string) => delta,
pending: () => "",
};
/**
* Streaming form of `escapeMemoryCitationMentions`.
*
* A tag can straddle delta boundaries, so a per-delta regex would miss the mention that matters
* most: the model writing the tag name one token at a time. The escaper buffers until it can tell
* a mention from a real block. A real block is emitted verbatim, tags included, once its closing
* tag and body prove it valid — the client keeps it, which is the whole point of the block.
* Everything else is a mention and gets `<`.
*/
export function createMemoryCitationEscaper(): MemoryCitationEscaper {
/**
* Unresolved tail. It starts at a `<` and is either a partial tag, or an opening tag whose block
* has not closed yet. A candidate block is held WHOLE until its closing tag arrives, because only
* the complete body (entries, then rollout ids) proves it is a real block rather than a mention.
*/
let held = "";
const escapeOpen = "<" + MEMORY_CITATION_OPEN.slice(1);
const escapeClose = "<" + MEMORY_CITATION_CLOSE.slice(1);
return {
push(delta: string): string {
if (!held && !delta.includes("<")) return delta;
const input = held + delta;
held = "";
let out = "";
let i = 0;
// Ordinary text stays one input span, even when it contains many literal '<' characters.
// Flush that span only at a rewrite or hold boundary, rather than building one rope per '<'.
let literalStart = 0;
while (i < input.length) {
const next = input.indexOf("<", i);
if (next === -1) break;
const remaining = input.length - next;
if (input.startsWith(MEMORY_CITATION_CLOSE, next)) {
// A close tag outside a claimed block is prose.
out += input.slice(literalStart, next) + escapeClose;
i = next + MEMORY_CITATION_CLOSE.length;
literalStart = i;
continue;
}
if (!input.startsWith(MEMORY_CITATION_OPEN, next)) {
if (remaining < MEMORY_CITATION_CLOSE.length) {
const tail = input.slice(next);
if (MEMORY_CITATION_OPEN.startsWith(tail) || MEMORY_CITATION_CLOSE.startsWith(tail)) {
// A tag is still arriving.
held = tail;
return out + input.slice(literalStart, next);
}
}
i = next + 1;
continue;
}
// Only an opening-tag candidate needs a bounded body lookahead. Ordinary '<' characters
// above use offsets in the original string, without allocating a tail for every match.
const tail = input.slice(next, next + MAX_MEMORY_CITATION_HOLD + 1);
const afterOpen = tail.slice(MEMORY_CITATION_OPEN.length);
if (!bodyCouldBeReal(afterOpen)) {
out += input.slice(literalStart, next) + escapeOpen;
i = next + MEMORY_CITATION_OPEN.length;
literalStart = i;
continue;
}
const end = opensRealBlock(tail, 0);
if (end !== undefined) {
// A complete real block is released verbatim, both tags included, so the delta stream
// equals what `escapeMemoryCitationMentions` keeps in the final text.
i = next + end;
continue;
}
if (!tail.includes(MEMORY_CITATION_CLOSE, MEMORY_CITATION_OPEN.length) && remaining <= MAX_MEMORY_CITATION_HOLD) {
// The block may still close; hold it whole.
held = tail;
return out + input.slice(literalStart, next);
}
// Closed without a valid body, or too large to keep holding: a mention.
out += input.slice(literalStart, next) + escapeOpen;
i = next + MEMORY_CITATION_OPEN.length;
literalStart = i;
}
return out + input.slice(literalStart);
},
pending(): string {
return held;
},
};
}
/**
* Remove every complete citation span from a whole string.
*
* This runs the streaming filter over the whole text, so the bridge's re-stripped
* `output_text.done` always equals the concatenated deltas it already emitted (#3843).
* Anything that is not a complete citation span stays verbatim: an unterminated START, a
* stray SEPARATOR or END, and every non-citation directive.
*/
export function stripCitationMarkers(text: string): string {
if (!text.includes(CITATION_MARKER_START)) return text;
const filter = createCitationMarkerFilter();
return filter.push(text) + filter.flush();
}
/**
* Streaming filter, as a per-character state machine so the result never depends on how the
* text was split into deltas.
*
* - `text`: ordinary text, emitted at once. A START opens `keyword`.
* - `keyword`: the START and keyword so far are withheld while the keyword can still become
* a citation keyword. A SEPARATOR after a citation keyword enters `citation`; an END right
* after one removes the span. Any other keyword releases the text and enters `other`.
* - `citation`: withheld until its END, then removed. A new START means this one was
* malformed, so it is released verbatim; so is a span that reaches the length bound.
* - `other`: the keyword of a non-citation span, emitted at once. A START here means the
* earlier one was malformed and opens a new span; a SEPARATOR enters `opaque`.
* - `opaque`: the body of a non-citation directive such as `visualize` (#6039), emitted at
* once. STARTs inside it are payload, not new spans, until its END or the length bound.
*
* A stream that ends while text is withheld releases it verbatim, so nothing the model
* actually said is lost.
*/
export function createCitationMarkerFilter(): CitationMarkerFilter {
let mode: "text" | "keyword" | "citation" | "other" | "opaque" = "text";
// The withheld START, keyword, and body in `keyword` and `citation` modes.
let held = "";
// Characters from START so far in `other` and `opaque` modes.
let spanLength = 0;
const isKeywordPrefix = (keyword: string): boolean => CITATION_KEYWORDS.some(k => k.startsWith(keyword));
return {
push(delta: string): string {
let out = "";
let i = 0;
while (i < delta.length) {
if (mode === "text") {
const next = delta.indexOf(CITATION_MARKER_START, i);
if (next === -1) {
out += delta.slice(i);
break;
}
out += delta.slice(i, next);
held = CITATION_MARKER_START;
mode = "keyword";
i = next + 1;
continue;
}
const ch = delta[i]!;
if (mode === "other" || mode === "opaque") {
if (spanLength + 1 > MAX_CITATION_SPAN_LENGTH) {
// Past the bound this is ordinary text again; handle the character in `text`.
mode = "text";
continue;
}
if (ch === CITATION_MARKER_START && mode === "other") {
held = CITATION_MARKER_START;
mode = "keyword";
} else {
out += ch;
spanLength += 1;
if (ch === CITATION_MARKER_END) mode = "text";
else if (ch === CITATION_MARKER_SEPARATOR) mode = "opaque";
}
i += 1;
continue;
}
i += 1;
if (ch === CITATION_MARKER_START) {
// The withheld START was malformed: release it and open a new span here.
out += held;
held = CITATION_MARKER_START;
mode = "keyword";
continue;
}
if (mode === "citation") {
if (ch === CITATION_MARKER_END) {
held = "";
mode = "text";
continue;
}
held += ch;
// One more character plus an END would exceed the bound, so this is not a span.
if (held.length >= MAX_CITATION_SPAN_LENGTH) {
out += held;
held = "";
mode = "text";
}
continue;
}
// mode === "keyword"
if (ch === CITATION_MARKER_SEPARATOR || ch === CITATION_MARKER_END) {
if (CITATION_KEYWORDS.includes(held.slice(1))) {
if (ch === CITATION_MARKER_END) {
held = "";
mode = "text";
} else {
held += ch;
mode = "citation";
}
} else {
out += held + ch;
spanLength = held.length + 1;
held = "";
mode = ch === CITATION_MARKER_END ? "text" : "opaque";
}
continue;
}
held += ch;
if (!isKeywordPrefix(held.slice(1))) {
out += held;
spanLength = held.length;
held = "";
mode = "other";
}
}
return out;
},
flush(): string {
const rest = held;
held = "";
spanLength = 0;
mode = "text";
return rest;
},
};
}