@phren/cli/dist/bridge/changes.js

Knowledge layer for AI agents - CLI, MCP server, and data layer

import { nonInteractiveGitEnv } from "../utils-helpers.js";
import { execFile } from "node:child_process";
import { appendFile, copyFile, mkdir, mkdtemp, readdir, readFile, realpath, rm, stat, unlink, utimes } from "node:fs/promises";
import { homeDir } from "../home-paths.js";
import path from "node:path";
import { promisify } from "node:util";
import { bridgeRoot, object } from "./protocol.js";
import { z } from "zod";
import { ProcessPool } from "./limits.js";
import { phrenStoreRoot } from "./transcripts.js";
import { countGit, countTick } from "./metrics.js";
const exec = promisify(execFile);
/** One file a shell command changed, with the patch between the working
 * tree before the call and after it. `root` is the repository; `path` is
 * relative to it. */
export const changedFileSchema = z.object({ root: z.string(), path: z.string(), status: z.string(), patch: z.string(),
    added: z.number().int().nonnegative(), removed: z.number().int().nonnegative(), redacted: z.boolean().optional() });
const changeRowSchema = z.object({ toolUseId: z.string(), files: z.array(changedFileSchema) });
export const secretName = (file) => /^(?:\.env(?:\..*)?|.*\.(?:pem|key|p12|pfx|keychain-db|tfstate)|id_rsa.*|id_ed25519.*|(?:.*\.)?credentials\.json|\.netrc|\.npmrc|\.pypirc)$/i.test(path.basename(file));
const gitPool = new ProcessPool(2);
/** Tools whose filesystem changes are captured around the lifecycle callback. */
export const SHELL_TOOLS = new Set(["Bash", "bash", "shell", "Shell", "exec_command", "shell_command", "local_shell", "write_stdin"]);
const NAMED = /(?<![\w@:/])(?:~\/|\.\/|\/)[\w.@+~-]+(?:\/[\w.@+~-]+)*/g;
/** The places a command names — the same rule the phone applies. */
const FILE_TOOLS = new Set(["write", "edit", "multiedit", "notebookedit", "apply_patch", "str_replace_editor", "create_file", "replace_string_in_file", "multi_replace_string_in_file"]);
export function capturesChanges(tool, input) {
    return SHELL_TOOLS.has(tool) || FILE_TOOLS.has(tool.split(".").at(-1).toLowerCase()) || typeof input.command === "string" || typeof input.cmd === "string";
}
export function namedPaths(command, input = {}) {
    const found = new Set();
    for (const match of command.slice(0, 65_536).matchAll(NAMED)) {
        if (match[0].length > 2)
            found.add(match[0]);
        if (found.size === 24)
            break;
    }
    for (const value of [input.file_path, input.path, input.notebook_path,
        ...((Array.isArray(input.replacements) ? input.replacements : []).map(v => object(v).filePath))]) {
        if (typeof value === "string" && value.length > 0 && value.length <= 4096 && !value.includes("\0"))
            found.add(value);
    }
    // Patch headers are paths, never execute their content or scan new file text.
    const patch = [input.patch, input.input, input.command, command].filter((v) => typeof v === "string").join("\n").slice(0, 262144);
    for (const match of patch.matchAll(/^\*\*\* (?:Update|Add|Delete) File: (.+)$|^\*\*\* Move to: (.+)$/gm)) {
        const value = (match[1] ?? match[2]).trim();
        if (value && value.length <= 4096 && !value.includes("\0"))
            found.add(value);
        if (found.size >= 48)
            break;
    }
    return [...found].slice(0, 48);
}
async function git(cwd, args, extra = {}, signal) {
    return gitPool.run(signal, async () => (countGit("changes"), await exec("git", ["-C", cwd, "--no-pager", ...args], {
        signal, timeout: 10_000, maxBuffer: 8_388_608, env: nonInteractiveGitEnv({ ...process.env, GIT_OPTIONAL_LOCKS: "0", GIT_CONFIG_NOSYSTEM: "1", ...extra }),
    })).stdout);
}
/** The repository holding `target` (a file, a folder, or something not yet
 * created), confined to the user's home; undefined otherwise. */
async function repositoryOf(target, cwd, home = homeDir(), signal) {
    const absolute = target === "~" || target.startsWith("~/") ? path.join(home, target.slice(1)) : path.resolve(cwd, target);
    let existing = absolute;
    while (!(await stat(existing).catch(() => undefined))) {
        const parent = path.dirname(existing);
        if (parent === existing)
            return undefined;
        existing = parent;
    }
    const real = await realpath(existing);
    home = await realpath(home);
    if (real !== home && !real.startsWith(home + path.sep))
        return undefined;
    const dir = (await stat(real)).isDirectory() ? real : path.dirname(real);
    try {
        return await realpath((await git(dir, ["rev-parse", "--show-toplevel"], {}, signal)).trim());
    }
    catch {
        return undefined;
    }
}
async function scratchTree(root, signal) {
    const dir = path.join(bridgeRoot(), "changes-scratch");
    await mkdir(dir, { recursive: true, mode: 0o700 });
    const scratch = await mkdtemp(path.join(dir, "repo-"));
    try {
        const objects = await realpath(path.resolve(root, (await git(root, ["rev-parse", "--git-path", "objects"], {}, signal)).trim()));
        await mkdir(path.join(scratch, "objects"), { mode: 0o700 });
        const env = { GIT_OBJECT_DIRECTORY: path.join(scratch, "objects"), GIT_ALTERNATE_OBJECT_DIRECTORIES: JSON.stringify(objects), GIT_INDEX_FILE: path.join(scratch, "index") };
        return { hash: await treeHash(root, env, signal), scratch, env };
    }
    catch (error) {
        await rm(scratch, { recursive: true, force: true });
        throw error;
    }
}
/** Both the index and all new blobs/trees stay in scratch storage. The real
 * object store is a read-only alternate, including for linked worktrees. */
async function treeHash(root, env, signal) {
    const index = path.resolve(root, (await git(root, ["rev-parse", "--git-path", "index"], {}, signal)).trim());
    const temp = env.GIT_INDEX_FILE;
    await unlink(temp).catch(() => undefined);
    await copyFile(index, temp).catch(() => undefined);
    // Force content checks for files rewritten to the same size in one instant.
    await utimes(temp, 1, 1).catch(() => undefined);
    await git(root, ["add", "-A", "--ignore-errors", "--", "."], env, signal);
    return (await git(root, ["write-tree"], env, signal)).trim();
}
const NO_DIFF = ["diff", "--no-ext-diff", "--no-textconv", "--no-color"];
async function treeDiff(root, before, after, env, signal) {
    if (before === after)
        return [];
    const records = (await git(root, [...NO_DIFF, "--name-status", "-z", before, after], env, signal)).split("\0");
    const files = [];
    for (let i = 0; i + 1 < records.length && files.length < 40; i += 2) {
        const status = records[i].slice(0, 1), original = records[i + 1];
        let file = original;
        if (/[RC]/.test(status))
            file = records[++i + 1]; // the new name follows the old
        if (!file)
            continue;
        const spec = `:(literal)${file}`;
        const counts = (await git(root, [...NO_DIFF, "--numstat", before, after, "--", spec], env, signal)).split("\t");
        const redacted = secretName(original) || secretName(file) || counts[0] === "-" || counts[1] === "-";
        let patch = redacted ? "" : await git(root, [...NO_DIFF, before, after, "--", spec], env, signal);
        if (patch.length > 200_000)
            patch = patch.slice(0, 200_000) + "\n… (truncated)\n";
        const added = Number(counts[0]) || 0, removed = Number(counts[1]) || 0;
        files.push({ root, path: file, status, patch, added, removed, ...(redacted ? { redacted: true } : {}) });
    }
    return files;
}
const CLAIM_RETENTION = 10 * 60_000;
/** The real path of a file that may not exist yet: its folder's real path plus its name. */
async function realPathOf(file) {
    return realpath(file).catch(async () => path.join(await realpath(path.dirname(file)).catch(() => path.dirname(file)), path.basename(file)));
}
/** The files a call names in its structured input, never the words of a command line. */
export function claimedPaths(input, cwd, home = homeDir()) {
    const found = new Set();
    for (const value of [input.file_path, input.path, input.notebook_path,
        ...((Array.isArray(input.replacements) ? input.replacements : []).map(v => object(v).filePath))]) {
        if (typeof value === "string" && value.length > 0 && value.length <= 4096 && !value.includes("\0"))
            found.add(value);
    }
    const patch = [input.patch, input.input].filter((v) => typeof v === "string").join("\n").slice(0, 262144);
    for (const match of patch.matchAll(/^\*\*\* (?:Update|Add|Delete) File: (.+)$|^\*\*\* Move to: (.+)$/gm)) {
        const value = (match[1] ?? match[2]).trim();
        if (value && value.length <= 4096 && !value.includes("\0"))
            found.add(value);
        if (found.size >= 48)
            break;
    }
    return [...found].map(value => value === "~" || value.startsWith("~/") ? path.join(home, value.slice(1)) : path.resolve(cwd, value));
}
const PENDING_FOR = 6_000;
const BUDGET = 2_500;
const DAY = 86_400_000;
/** Remove old records first, then oldest records until within the disk budget. */
export async function pruneChanges(now = Date.now()) {
    const dir = path.join(bridgeRoot(), "changes");
    const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
    const files = [];
    for (const entry of entries) {
        if (!entry.isFile() || !entry.name.endsWith(".jsonl"))
            continue;
        const file = path.join(dir, entry.name), info = await stat(file).catch(() => undefined);
        if (info)
            files.push({ file, size: info.size, at: info.mtimeMs });
    }
    files.sort((a, b) => b.at - a.at);
    let size = 0;
    for (const file of files) {
        if (now - file.at > 30 * DAY || size + file.size > 268_435_456)
            await unlink(file.file).catch(() => undefined);
        else
            size += file.size;
    }
}
export async function startChangeRetention() {
    await pruneChanges();
    const timer = setInterval(async () => { countTick("change-retention"); await pruneChanges().catch(() => { }); }, DAY);
    timer.unref();
    return () => clearInterval(timer);
}
/** Snapshots live only around a shell call. Persisted results are validated
 * when loaded; the LRU retains at most 16 conversations, including loads. */
export class ToolChanges {
    /** Called after a non-empty change event is recorded, so modules that follow
     * file changes (the code index) can react without polling. */
    onRecord;
    snapshots = new Map();
    results = new Map();
    controllers = new Set();
    claims = [];
    /** A hook callback's wall-clock cap on Git work; past it the call records
     * no change. Tests about what is captured raise it, since a loaded Windows
     * runner's Git can take longer than a person's machine ever does. */
    budgetMs;
    constructor(options = {}) { this.budgetMs = options.budgetMs ?? BUDGET; }
    file(conversation) { return path.join(bridgeRoot(), "changes", conversation.replace(/[^A-Za-z0-9._-]/g, "_") + ".jsonl"); }
    async budget(work) {
        const controller = new AbortController();
        this.controllers.add(controller);
        const timer = setTimeout(() => controller.abort(), this.budgetMs);
        try {
            return await work(controller.signal);
        }
        finally {
            clearTimeout(timer);
            this.controllers.delete(controller);
        }
    }
    async discard(snapshot) {
        clearTimeout(snapshot.expiry);
        await Promise.all([...snapshot.trees.values()].map(tree => rm(tree.scratch, { recursive: true, force: true })));
    }
    async close() {
        for (const controller of this.controllers)
            controller.abort();
        for (const snapshot of this.snapshots.values()) {
            await snapshot.result?.catch(() => { });
            await this.discard(snapshot);
        }
        this.snapshots.clear();
    }
    async before(conversation, toolUseId, cwd, command, input = {}) {
        const key = `${conversation}\0${toolUseId}`;
        if (!toolUseId)
            return;
        // Real paths, like the repository roots the diff reports (/var → /private/var).
        const claimed = await Promise.all(claimedPaths(input, cwd).map(realPathOf));
        this.claim(conversation, toolUseId, claimed);
        if (this.snapshots.size >= 64 || this.snapshots.has(key))
            return;
        const snapshot = { at: Date.now(), conversation, own: new Set(claimed), trees: new Map(), taking: true };
        this.snapshots.set(key, snapshot);
        try {
            await this.budget(async (signal) => {
                const roots = new Set();
                for (const target of [cwd, phrenStoreRoot(), ...namedPaths(command, input)]) {
                    signal.throwIfAborted();
                    if (roots.size >= 6)
                        break;
                    const root = await repositoryOf(target, cwd, homeDir(), signal);
                    if (root)
                        roots.add(root);
                }
                for (const root of roots) {
                    try {
                        snapshot.trees.set(root, await scratchTree(root, signal));
                    }
                    catch {
                        signal.throwIfAborted();
                    }
                }
                signal.throwIfAborted();
            });
            snapshot.taking = false;
            if (snapshot.dropped) {
                await this.discard(snapshot);
                return;
            }
            if (!snapshot.trees.size) {
                this.snapshots.delete(key);
                return;
            }
            // A missing PostToolUse/reader cannot retain untracked contents forever.
            snapshot.expiry = setTimeout(() => {
                if (!snapshot.result) {
                    this.snapshots.delete(key);
                    void this.discard(snapshot).catch(() => { });
                }
            }, 30 * 60_000);
            snapshot.expiry.unref();
        }
        catch {
            if (this.snapshots.get(key) === snapshot)
                this.snapshots.delete(key);
            await this.discard(snapshot);
        }
    }
    /** Forget the snapshot of a tool call whose PreToolUse callback gave up
     * before the snapshot was taken: the tool may have run while it was read,
     * so no diff is shown for that call rather than a wrong one. */
    drop(conversation, toolUseId) {
        const key = `${conversation}\0${toolUseId}`, snapshot = this.snapshots.get(key);
        if (!snapshot || snapshot.result)
            return;
        this.snapshots.delete(key);
        if (snapshot.taking)
            snapshot.dropped = true;
        else
            void this.discard(snapshot).catch(() => { });
    }
    claim(conversation, toolUseId, paths) {
        const now = Date.now();
        this.claims = this.claims.filter(claim => claim.end === undefined ? now - claim.start < 30 * 60_000 : now - claim.end < CLAIM_RETENTION);
        if (paths.length && this.claims.length < 512)
            this.claims.push({ conversation, toolUseId, paths: new Set(paths), start: now });
    }
    /** Whether another conversation's call named this file while `snapshot` ran. */
    claimedElsewhere(snapshot, file, until) {
        return this.claims.some(claim => claim.conversation !== snapshot.conversation && claim.paths.has(file)
            && claim.start <= until && (claim.end ?? until) >= snapshot.at);
    }
    async after(conversation, toolUseId) {
        const key = `${conversation}\0${toolUseId}`, snapshot = this.snapshots.get(key);
        for (const claim of this.claims)
            if (claim.conversation === conversation && claim.toolUseId === toolUseId)
                claim.end ??= Date.now();
        if (!snapshot)
            return;
        if (snapshot.taking) {
            this.drop(conversation, toolUseId);
            return;
        }
        snapshot.result ??= this.compute(snapshot);
        const files = await snapshot.result;
        if (this.snapshots.get(key) === snapshot) {
            this.snapshots.delete(key);
            await this.record(conversation, toolUseId, files);
        }
    }
    async compute(snapshot) {
        try {
            return await this.budget(async (signal) => {
                const files = [];
                for (const [root, before] of snapshot.trees) {
                    try {
                        files.push(...await treeDiff(root, before.hash, await treeHash(root, before.env, signal), before.env, signal));
                    }
                    catch {
                        signal.throwIfAborted();
                    }
                }
                // A file another agent's edit named during this call is its change,
                // not this call's, even though it landed inside the same window.
                // A call that names its files (an Edit, a Write, a patch) changed
                // only those, whatever else moved in the repository meanwhile.
                const until = Date.now();
                const kept = [];
                for (const file of files) {
                    // Both sides as real paths: a root reported through a symlinked
                    // folder (/var on macOS) still matches the call's own claim.
                    const absolute = await realPathOf(path.resolve(file.root, file.path));
                    if (snapshot.own.size ? snapshot.own.has(absolute) : !this.claimedElsewhere(snapshot, absolute, until))
                        kept.push(file);
                }
                return kept;
            });
        }
        catch {
            return [];
        }
        finally {
            await this.discard(snapshot);
        }
    }
    cache(conversation) {
        const entry = this.results.get(conversation) ?? { files: new Map() };
        this.results.delete(conversation);
        this.results.set(conversation, entry);
        while (this.results.size > 16)
            this.results.delete(this.results.keys().next().value);
        return entry;
    }
    async load(conversation) {
        const entry = this.cache(conversation);
        entry.loaded ??= (async () => {
            const file = this.file(conversation);
            const size = (await stat(file).catch(() => undefined))?.size ?? 0;
            const text = size <= 16_777_216 ? await readFile(file, "utf8").catch(() => "") : "";
            for (const line of text.split("\n")) {
                if (!line)
                    continue;
                try {
                    const row = changeRowSchema.parse(JSON.parse(line));
                    // Also sanitize old records made before patch redaction existed.
                    entry.files.set(row.toolUseId, row.files.map(file => secretName(file.path) || file.redacted || /^(?:Binary files |GIT binary patch)/m.test(file.patch)
                        ? { ...file, patch: "", redacted: true } : file));
                }
                catch { /* Torn or invalid rows are not transcript events. */ }
            }
        })();
        await entry.loaded;
        return entry;
    }
    async record(conversation, toolUseId, files) {
        (await this.load(conversation)).files.set(toolUseId, files);
        if (!files.length)
            return;
        const file = this.file(conversation);
        await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
        const size = (await stat(file).catch(() => undefined))?.size ?? 0;
        const line = JSON.stringify({ toolUseId, files }) + "\n";
        if (size + Buffer.byteLength(line) <= 16_777_216)
            await appendFile(file, line, { mode: 0o600 });
        this.onRecord?.(files);
    }
    async recordedPaths(conversation) {
        return [...new Set([...(await this.load(conversation)).files.values()].flatMap(files => files.flatMap(file => [file.root, path.resolve(file.root, file.path)])))];
    }
    view(conversation) {
        return {
            pending: toolUseId => {
                const snapshot = this.snapshots.get(`${conversation}\0${toolUseId}`);
                return !!snapshot && !snapshot.result && Date.now() - snapshot.at < PENDING_FOR;
            },
            changes: async (toolUseId) => {
                await this.after(conversation, toolUseId);
                const known = (await this.load(conversation)).files.get(toolUseId);
                return known?.length ? known : undefined;
            },
        };
    }
}
/** The tool calls whose output a transcript row carries. */
export function outputCallIds(raw, source) {
    const ids = (value) => typeof value === "string" && value ? [value] : [];
    if (source === "codex") {
        const p = (raw.payload ?? {});
        return ["function_call_output", "custom_tool_call_output"].includes(String(p.type)) ? ids(p.call_id) : [];
    }
    if (source === "copilot")
        return raw.type === "tool.execution_complete" ? ids((raw.data ?? {}).toolCallId) : [];
    const message = (source === "phren" || source === "opencode" ? (raw.data ?? {}).message : raw.message);
    const content = message?.content;
    if (!Array.isArray(content))
        return [];
    return content.flatMap(block => block && typeof block === "object" && block.type === "tool_result" ? ids(block.tool_use_id) : []);
}