@juspay/neurolink/dist/utils/anthropicCacheBreakpoints.js

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

/**
 * Anthropic prompt-cache breakpoint placement for the native Vertex+Claude
 * request path.
 *
 * Vertex does NOT support automatic prompt caching — the only way the
 * conversation prefix gets cached across turns is explicit `cache_control`
 * markers in the request. Anthropic renders a request as `tools → system →
 * messages` and caches the prefix up to each marker, with a hard ceiling of
 * four markers.
 *
 * Without a marker on the message history, the entire (growing, often
 * tool-result-heavy) conversation falls *after* the last breakpoint and is
 * re-billed at full input price on every turn. That is the regression this
 * fixes: it gives the history a rolling breakpoint so the stable prefix is
 * cached at ~0.1x and only the newest turn is fresh.
 */
const EPHEMERAL = { type: "ephemeral" };
/** Anthropic allows at most four `cache_control` breakpoints per request. */
export const ANTHROPIC_MAX_CACHE_BREAKPOINTS = 4;
const MAX_BREAKPOINTS = ANTHROPIC_MAX_CACHE_BREAKPOINTS;
/**
 * Annotate a native Vertex+Claude request with prompt-cache breakpoints.
 *
 * Budget allocation (max 4 markers):
 *   1. The stable prefix — the last system block when a system prompt is
 *      present (this single marker caches `tools + system`, since system
 *      renders after tools); otherwise the last tool definition.
 *   2-4. A rolling breakpoint on the last few messages, so the
 *      growing-but-now-stable conversation history is cached and only the
 *      newest turn is billed as fresh input.
 *
 * Pure: the inputs are cloned, never mutated.
 */
export function applyVertexAnthropicCacheBreakpoints(input) {
    let budget = MAX_BREAKPOINTS;
    // 1. Stable prefix. A bare string `system` cannot carry cache_control, so
    // convert it to block form. Marking the last system block caches the whole
    // tools + system prefix; only fall back to marking the last tool when there
    // is no system prompt to mark.
    let system = input.system;
    let tools = input.tools;
    const hasSystem = !!input.system && input.system.trim().length > 0;
    if (hasSystem) {
        system = [
            { type: "text", text: input.system, cache_control: EPHEMERAL },
        ];
        budget--;
    }
    else if (input.tools && input.tools.length > 0) {
        const lastIndex = input.tools.length - 1;
        tools = input.tools.map((tool, i) => i === lastIndex ? { ...tool, cache_control: EPHEMERAL } : tool);
        budget--;
    }
    // 2. Rolling history breakpoints over the tail of the conversation.
    const messages = input.messages.map((m) => ({ ...m }));
    let remaining = Math.min(budget, input.maxHistoryBreakpoints ?? MAX_BREAKPOINTS);
    for (let i = messages.length - 1; i >= 0 && remaining > 0; i--) {
        if (markLastContentBlock(messages, i)) {
            remaining--;
        }
    }
    return { system, tools, messages };
}
/**
 * Count the `cache_control` markers already present on a request. The direct
 * Anthropic path arrives with markers the upstream layers placed —
 * MessageBuilder tags the system prompt, the client tags the last tool
 * definition, and message content blocks may carry their own. Anthropic
 * rejects requests with more than four markers, so any additional history
 * breakpoints must fit in the remaining budget.
 */
export function countAnthropicCacheMarkers(input) {
    let count = 0;
    if (Array.isArray(input.system)) {
        count += input.system.filter((b) => b.cache_control).length;
    }
    if (input.tools) {
        count += input.tools.filter((t) => t.cache_control).length;
    }
    for (const message of input.messages) {
        if (Array.isArray(message.content)) {
            count += message.content.filter((b) => b.cache_control).length;
        }
    }
    return count;
}
/**
 * Rolling history breakpoints for request paths whose stable-prefix markers
 * are managed upstream (direct Anthropic: system via MessageBuilder, last
 * tool via the provider client). Marks the last content block of up to `budget`
 * tail messages; a tail block that already carries a marker is left as-is
 * without consuming budget (it already serves as that breakpoint). Pure —
 * the input array is cloned, never mutated.
 */
export function applyAnthropicHistoryCacheBreakpoints(input, budget) {
    const messages = input.map((m) => ({ ...m }));
    let remaining = Math.max(0, Math.min(budget, MAX_BREAKPOINTS));
    for (let i = messages.length - 1; i >= 0 && remaining > 0; i--) {
        if (lastContentBlockHasMarker(messages[i])) {
            continue;
        }
        if (markLastContentBlock(messages, i)) {
            remaining--;
        }
    }
    return messages;
}
/** True when the last content block of a message already carries `cache_control`. */
function lastContentBlockHasMarker(message) {
    if (!Array.isArray(message.content) || message.content.length === 0) {
        return false;
    }
    const last = message.content[message.content.length - 1];
    return !!last.cache_control;
}
/**
 * Place a cache breakpoint on the last content block of `messages[i]`.
 * Anthropic attaches `cache_control` to a content block, not the message
 * envelope, so a string content body is first converted to block form.
 * Returns false when the message has no markable block (caller then walks to
 * an earlier message), so an empty turn never silently consumes a breakpoint.
 */
function markLastContentBlock(messages, i) {
    const message = messages[i];
    if (typeof message.content === "string") {
        if (message.content.length === 0) {
            return false;
        }
        messages[i] = {
            ...message,
            content: [
                { type: "text", text: message.content, cache_control: EPHEMERAL },
            ],
        };
        return true;
    }
    if (!Array.isArray(message.content) || message.content.length === 0) {
        return false;
    }
    // Shallow clone is sufficient: we only ever add a top-level `cache_control`
    // field to the last block and never mutate nested members (e.g. an image's
    // `source`). Those nested objects stay shared with the input by reference,
    // which is safe because they are never written to. If a future block shape
    // requires mutating nested members, deep-clone that block instead.
    const content = message.content.map((block) => ({ ...block }));
    const lastIndex = content.length - 1;
    content[lastIndex] = { ...content[lastIndex], cache_control: EPHEMERAL };
    messages[i] = { ...message, content };
    return true;
}
/**
 * Breakpoint placement for a Claude Messages request the proxy built itself,
 * rather than one a client authored.
 *
 * A request translated from another wire format arrives with no markers at
 * all, because the format it came from has no way to express one: OpenAI-shaped
 * callers rely on automatic prefix caching, which Anthropic does not do. A
 * faithful translation is therefore cached at 0% on every turn, and a caller
 * that fell back here from an exhausted pool burns the replacement quota faster
 * than the one it was spared.
 *
 * Placement follows Anthropic's `tools → system → messages` rendering: one
 * marker on the last system block covers tools *and* system in a single
 * breakpoint, and the rest roll along the tail of the history so a growing
 * conversation keeps its settled prefix cached. Markers the caller supplied are
 * counted first and never displaced — this only spends what is left of the
 * four.
 *
 * Pure: inputs are cloned, never mutated.
 */
export function applyClaudeRequestCacheBreakpoints(request, options) {
    const existing = countAnthropicCacheMarkers({
        system: request.system,
        tools: request.tools,
        messages: [],
    });
    const inMessages = request.messages.reduce((n, m) => n +
        (Array.isArray(m.content)
            ? m.content.filter((b) => "cache_control" in b && b.cache_control)
                .length
            : 0), 0);
    let budget = MAX_BREAKPOINTS - existing - inMessages;
    if (budget <= 0) {
        return request;
    }
    const out = { ...request };
    const breakpointOneMarker = ephemeral(options?.ttl);
    // One marker on the stable head. The last system block is the better anchor
    // than the last tool, because system renders after tools and so a single
    // marker there caches both.
    if (Array.isArray(out.system) && out.system.length > 0 && existing === 0) {
        const system = out.system.map((b) => ({ ...b }));
        system[system.length - 1] = {
            ...system[system.length - 1],
            cache_control: breakpointOneMarker,
        };
        out.system = system;
        budget--;
    }
    else if ((!out.system || (Array.isArray(out.system) && out.system.length === 0)) &&
        out.tools &&
        out.tools.length > 0 &&
        existing === 0) {
        const tools = out.tools.map((t) => ({ ...t }));
        tools[tools.length - 1] = {
            ...tools[tools.length - 1],
            cache_control: breakpointOneMarker,
        };
        out.tools = tools;
        budget--;
    }
    if (budget > 0) {
        out.messages = applyClaudeHistoryBreakpoints(out.messages, budget);
    }
    return out;
}
const CLAUDE_EPHEMERAL = { type: "ephemeral" };
/**
 * Builds the breakpoint-1 marker, honoring a caller-supplied TTL override.
 * Breakpoints 2-4 (`applyClaudeHistoryBreakpoints`) always use the bare
 * `CLAUDE_EPHEMERAL` constant instead of this — they roll forward every turn
 * or two, so paying the 1-hour (2x) rate on a marker about to be superseded
 * is pure waste; only the stable head benefits from the longer window.
 */
function ephemeral(ttl) {
    return ttl ? { type: "ephemeral", ttl } : CLAUDE_EPHEMERAL;
}
/**
 * Roll `budget` markers backwards along the history, marking the last content
 * block of each message that can carry one. Only text and tool_result blocks
 * accept `cache_control`; a message ending in anything else is skipped without
 * spending budget.
 *
 * A message whose content is a plain string is promoted to the single text
 * block the marker has to live on. Skipping strings instead would have left an
 * ordinary conversation with no history breakpoint at all, since the OpenAI
 * bridge emits every plain-text turn as a string — which is precisely the
 * growing history this is here to keep cached.
 */
function applyClaudeHistoryBreakpoints(input, budget) {
    const messages = input.map((m) => ({ ...m }));
    let remaining = Math.max(0, Math.min(budget, MAX_BREAKPOINTS));
    for (let i = messages.length - 1; i >= 0 && remaining > 0; i--) {
        const content = messages[i].content;
        if (typeof content === "string") {
            if (content.length === 0) {
                continue;
            }
            messages[i] = {
                ...messages[i],
                content: [
                    { type: "text", text: content, cache_control: CLAUDE_EPHEMERAL },
                ],
            };
            remaining--;
            continue;
        }
        if (!Array.isArray(content) || content.length === 0) {
            continue;
        }
        const last = content[content.length - 1];
        if (last.type !== "text" && last.type !== "tool_result") {
            continue;
        }
        if (last.cache_control) {
            continue;
        }
        const blocks = content.map((b) => ({ ...b }));
        blocks[blocks.length - 1] = { ...last, cache_control: CLAUDE_EPHEMERAL };
        messages[i] = { ...messages[i], content: blocks };
        remaining--;
    }
    return messages;
}