@sema-agent/core/dist/engine/compaction/utils.js

Stateless, task-oriented AI agent core

import { normalizeLlmMessageContent } from "../harness/messages.js";
import { sliceHeadSafe } from "../../core/surrogate-safe-slice.js";
import { cutEnginePrefix, cutEngineSegments, flattenableUserText } from "../../core/untrusted-text.js";
import { fileArgPath } from "../../tools/fs/safety.js";
import { fileOpAccessOf } from "../../core/tool-registry.js";
export const DURABLE_HISTORY_NAMES = new Map([
    ["read", "read"],
    ["write", "create"],
    ["edit", "edit"],
    ["read_file", "read"],
    ["write_file", "create"],
    ["edit_file", "edit"],
]);
export function createFileOps() {
    return {
        read: new Set(),
        written: new Set(),
        edited: new Set(),
        modifiedOrder: new Set(),
    };
}
export function extractFileOpsFromMessage(message, fileOps) {
    if (message.role !== "assistant") {
        return;
    }
    if (!("content" in message) || !Array.isArray(message.content)) {
        return;
    }
    for (const block of message.content) {
        if (typeof block !== "object" || block === null) {
            continue;
        }
        if (!("type" in block) || block.type !== "toolCall") {
            continue;
        }
        if (!("arguments" in block) || !("name" in block)) {
            continue;
        }
        const args = block.arguments;
        if (!args) {
            continue;
        }
        const path = fileArgPath(args);
        if (!path) {
            continue;
        }
        const access = fileOpAccessOf(block.name) ?? DURABLE_HISTORY_NAMES.get(block.name);
        if (access === "read")
            fileOps.read.add(path);
        else if (access === "create") {
            fileOps.written.add(path);
            touchModified(fileOps, path);
        }
        else if (access === "edit") {
            fileOps.edited.add(path);
            touchModified(fileOps, path);
        }
    }
}
export const SKILL_RETENTION_PER_SKILL_MAX_CHARS = 20_000;
export const SKILL_RETENTION_TOTAL_MAX_CHARS = 100_000;
export const SKILL_RETENTION_TRUNCATION_MARKER = "[... skill content truncated for compaction; invoke the Skill tool again if you need the full text]";
const SKILL_TOOL_NAME_LITERAL = "Skill";
const SKILL_INPUT_SECTION_SEPARATOR = "\n\n## Input\n";
export function stripSkillInputSection(text) {
    const at = text.indexOf(SKILL_INPUT_SECTION_SEPARATOR);
    return at >= 0 ? text.slice(0, at) : text;
}
export function extractInvokedSkills(droppedMessages, prevRetained) {
    const byName = new Map();
    if (prevRetained) {
        for (const s of prevRetained.toReversed()) {
            byName.set(s.name, s.content);
        }
    }
    const callIdToSkillName = new Map();
    for (const msg of droppedMessages) {
        if (msg.role === "assistant" && "content" in msg && Array.isArray(msg.content)) {
            for (const block of msg.content) {
                if (typeof block === "object" && block !== null &&
                    "type" in block && block.type === "toolCall" &&
                    "name" in block && block.name === SKILL_TOOL_NAME_LITERAL &&
                    "id" in block && typeof block.id === "string") {
                    const args = ("arguments" in block ? block.arguments : undefined);
                    const skillName = args?.skill ?? args?.name;
                    if (typeof skillName === "string" && skillName.length > 0) {
                        callIdToSkillName.set(block.id, skillName);
                    }
                }
            }
        }
        else if (msg.role === "toolResult" && msg.toolName === SKILL_TOOL_NAME_LITERAL && !msg.isError) {
            const skillName = callIdToSkillName.get(msg.toolCallId);
            if (skillName === undefined)
                continue;
            const text = stripSkillInputSection(msg.content
                .filter((c) => c.type === "text")
                .map((c) => c.text)
                .join(""));
            if (!text)
                continue;
            byName.delete(skillName);
            byName.set(skillName, text);
        }
    }
    return [...byName.entries()].toReversed().map(([name, content]) => ({ name, content }));
}
export function budgetInvokedSkillsRetention(newestFirst, maxTotalChars) {
    const totalCap = Math.min(SKILL_RETENTION_TOTAL_MAX_CHARS, Math.max(0, maxTotalChars ?? SKILL_RETENTION_TOTAL_MAX_CHARS));
    const perSkillCap = Math.min(SKILL_RETENTION_PER_SKILL_MAX_CHARS, totalCap);
    if (perSkillCap <= SKILL_RETENTION_TRUNCATION_MARKER.length) {
        return [];
    }
    const out = [];
    let budget = totalCap;
    for (const s of newestFirst) {
        let content = s.content;
        if (content.length > perSkillCap) {
            content =
                content.slice(0, perSkillCap - SKILL_RETENTION_TRUNCATION_MARKER.length) +
                    SKILL_RETENTION_TRUNCATION_MARKER;
        }
        if (content.length > budget) {
            continue;
        }
        budget -= content.length;
        out.push({ name: s.name, content });
    }
    return out;
}
export function replaceInvokedSkillBodiesForSummary(groups, retainedNames) {
    const callIdToSkillName = new Map();
    for (const msg of groups.flat()) {
        if (msg.role !== "assistant" || !("content" in msg) || !Array.isArray(msg.content))
            continue;
        for (const block of msg.content) {
            if (typeof block === "object" && block !== null &&
                "type" in block && block.type === "toolCall" &&
                "name" in block && block.name === SKILL_TOOL_NAME_LITERAL &&
                "id" in block && typeof block.id === "string") {
                const args = ("arguments" in block ? block.arguments : undefined);
                const skillName = args?.skill ?? args?.name;
                if (typeof skillName === "string" && skillName.length > 0) {
                    callIdToSkillName.set(block.id, skillName);
                }
            }
        }
    }
    return groups.map((messages) => messages.map((msg) => {
        if (msg.role !== "toolResult" || msg.toolName !== SKILL_TOOL_NAME_LITERAL || msg.isError)
            return msg;
        const skillName = callIdToSkillName.get(msg.toolCallId);
        if (skillName === undefined)
            return msg;
        const chars = msg.content
            .filter((c) => c.type === "text")
            .reduce((sum, c) => sum + c.text.length, 0);
        if (chars === 0)
            return msg;
        const note = retainedNames.has(skillName)
            ? `[Skill "${skillName}" instructions (${chars} chars) are carried in the invoked-skills area of the compaction summary — omitted from this summarization input to avoid double-carrying them.]`
            : `[Skill "${skillName}" instructions (${chars} chars) were omitted here and did not fit the invoked-skills retention budget — re-invoke the Skill tool if they are needed again.]`;
        return { ...msg, content: [{ type: "text", text: note }] };
    }));
}
export function readRetainedInvokedSkills(details) {
    if (details === null || typeof details !== "object")
        return [];
    const raw = details.invokedSkills;
    if (!Array.isArray(raw))
        return [];
    return raw.filter((s) => s !== null &&
        typeof s === "object" &&
        typeof s.name === "string" &&
        typeof s.content === "string");
}
export function renderInvokedSkillsRetention(skills) {
    if (skills.length === 0)
        return "";
    const sections = skills.map((s) => `### Skill: ${s.name}\n\n${s.content}`);
    return (`\n\n<invoked-skills>\n` +
        `Skill instructions loaded earlier in this session, preserved across compaction (most recently invoked first). ` +
        `If a skill's content was truncated, invoke the Skill tool again for the full text.\n\n` +
        `${sections.join("\n\n")}\n</invoked-skills>`);
}
const PERSISTED_OUTPUT_REF_RE = /<persisted-output ref="([^"]+)"/g;
const OFFLOAD_TOOL_NAME_LITERAL = "ReadToolResult";
export const PERSISTED_OUTPUT_REFS_MAX_ENTRIES = 50;
export function extractPersistedOutputRefs(messages, into) {
    for (const msg of messages) {
        if (msg.role !== "toolResult" || !Array.isArray(msg.content))
            continue;
        for (const block of msg.content) {
            if (block.type !== "text" || typeof block.text !== "string")
                continue;
            for (const m of block.text.matchAll(PERSISTED_OUTPUT_REF_RE)) {
                into.add(m[1]);
            }
        }
    }
}
export function readElidedMessages(details) {
    if (details === null || typeof details !== "object")
        return undefined;
    const raw = details.elidedMessages;
    return typeof raw === "number" && Number.isSafeInteger(raw) && raw >= 0 ? raw : undefined;
}
export function readUnsummarizedMessages(details) {
    if (details === null || typeof details !== "object")
        return undefined;
    const raw = details.unsummarizedMessages;
    return typeof raw === "number" && Number.isSafeInteger(raw) && raw >= 0 ? raw : undefined;
}
export function readCompactionActiveTools(details) {
    if (details === null || typeof details !== "object")
        return [];
    const raw = details.activeTools;
    if (!Array.isArray(raw))
        return [];
    return raw.filter((n) => typeof n === "string" && n.length > 0);
}
export function readPersistedOutputRefs(details) {
    if (details === null || typeof details !== "object")
        return [];
    const raw = details.persistedOutputRefs;
    if (!Array.isArray(raw))
        return [];
    return raw.filter((r) => typeof r === "string" && r.length > 0);
}
export function formatPersistedOutputRefs(refs) {
    if (refs.length === 0)
        return "";
    return (`\n\n<persisted-tool-outputs>\n` +
        `Tool outputs elided by compaction were persisted in full and remain retrievable — call ` +
        `${OFFLOAD_TOOL_NAME_LITERAL} with a ref to read one back:\n` +
        `${capList(refs)}\n</persisted-tool-outputs>`);
}
function touchModified(fileOps, path) {
    fileOps.modifiedOrder.delete(path);
    fileOps.modifiedOrder.add(path);
}
export function computeFileLists(fileOps) {
    const modified = new Set([...fileOps.edited, ...fileOps.written]);
    const readOnly = [...fileOps.read].filter((f) => !modified.has(f)).toSorted();
    const modifiedFiles = [...modified].toSorted();
    const ordered = [...fileOps.modifiedOrder].filter((f) => modified.has(f));
    const unordered = modifiedFiles.filter((f) => !fileOps.modifiedOrder.has(f));
    const modifiedFilesByRecency = [...ordered.toReversed(), ...unordered];
    return { readFiles: readOnly, modifiedFiles, modifiedFilesByRecency };
}
const FILE_LIST_MAX_ENTRIES = 50;
function capList(files) {
    if (files.length <= FILE_LIST_MAX_ENTRIES) {
        return files.join("\n");
    }
    const shown = files.slice(0, FILE_LIST_MAX_ENTRIES);
    return `${shown.join("\n")}\n[... ${files.length - FILE_LIST_MAX_ENTRIES} more files]`;
}
export function formatFileOperations(readFiles, modifiedFiles) {
    const sections = [];
    if (readFiles.length > 0) {
        sections.push(`<read-files>\n${capList(readFiles)}\n</read-files>`);
    }
    if (modifiedFiles.length > 0) {
        sections.push(`<modified-files>\n${capList(modifiedFiles)}\n</modified-files>`);
    }
    if (sections.length === 0) {
        return "";
    }
    return `\n\n${sections.join("\n\n")}`;
}
export function stripFileOperationsFooter(summary) {
    let out = summary;
    for (;;) {
        const next = out.replace(/\n\n<(read-files|modified-files|persisted-tool-outputs)>\n[\s\S]*?\n<\/\1>\s*$/, "");
        if (next === out)
            return out;
        out = next;
    }
}
const TOOL_RESULT_MAX_CHARS = 2000;
function safeJsonStringify(value) {
    try {
        return JSON.stringify(value) ?? "undefined";
    }
    catch {
        return "[unserializable]";
    }
}
function truncateForSummary(text, maxChars) {
    if (text.length <= maxChars) {
        return text;
    }
    let keep = maxChars;
    let out = "";
    for (let i = 0; i < 4; i++) {
        const kept = sliceHeadSafe(text, keep);
        out = `${kept}\n\n[... ${text.length - kept.length} more characters truncated]`;
        if (out.length <= maxChars)
            break;
        if (keep === 0)
            return maxChars >= 1 ? `${sliceHeadSafe(text, maxChars - 1)}…` : "";
        keep = Math.max(0, keep - (out.length - maxChars));
    }
    return out.length < text.length ? out : text;
}
export function serializeConversation(messages) {
    const parts = [];
    for (const rawMsg of messages) {
        const msg = normalizeLlmMessageContent(rawMsg);
        if (msg.role === "user") {
            const raw = typeof msg.content === "string"
                ? msg.content
                : msg.content
                    .filter((c) => c.type === "text")
                    .map((c) => c.text)
                    .join("");
            if (msg.provenance === "engine-note") {
                if (raw) {
                    parts.push(`[Engine]: ${raw}`);
                }
                continue;
            }
            if (msg.engineMinted === true) {
                continue;
            }
            const flat = flattenableUserText(msg.content);
            let content;
            if (flat === undefined) {
                content = raw;
            }
            else {
                const bySegments = cutEngineSegments(flat, msg.engineSegments);
                content = bySegments.malformed ? flat : bySegments.cut ? bySegments.text : cutEnginePrefix(flat, msg.enginePrefixChars).text;
            }
            if (content) {
                parts.push(`[User]: ${content}`);
            }
        }
        else if (msg.role === "assistant") {
            const textParts = [];
            const thinkingParts = [];
            const toolCalls = [];
            for (const block of msg.content) {
                if (block.type === "text") {
                    textParts.push(block.text);
                }
                else if (block.type === "thinking") {
                    thinkingParts.push(block.thinking);
                }
                else if (block.type === "toolCall") {
                    const args = block.arguments;
                    const argsStr = Object.entries(args)
                        .map(([k, v]) => `${k}=${safeJsonStringify(v)}`)
                        .join(", ");
                    toolCalls.push(`${block.name}(${argsStr})`);
                }
            }
            if (thinkingParts.length > 0) {
                parts.push(`[Assistant thinking]: ${thinkingParts.join("\n")}`);
            }
            if (textParts.length > 0) {
                parts.push(`[Assistant]: ${textParts.join("\n")}`);
            }
            if (toolCalls.length > 0) {
                parts.push(`[Assistant tool calls]: ${toolCalls.join("; ")}`);
            }
        }
        else if (msg.role === "toolResult") {
            const content = msg.content
                .filter((c) => c.type === "text")
                .map((c) => c.text)
                .join("");
            if (content) {
                parts.push(`[Tool result]: ${truncateForSummary(content, TOOL_RESULT_MAX_CHARS)}`);
            }
        }
    }
    return parts.join("\n\n");
}