@phren/cli/dist/bridge/memory-store.js

Knowledge layer for AI agents - CLI, MCP server, and data layer

import { execFile, spawn } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
import { lstat, mkdir, realpath } from "node:fs/promises";
import path from "node:path";
import { promisify } from "node:util";
import { z } from "zod";
import { withFileLock } from "../governance/locks.js";
import { BridgeError, bridgeRoot } from "./protocol.js";
// The phone reads memory from a paired computer through these routes, in the
// same git terms its sync engine uses with GitHub: a head, a recursive tree,
// blobs by sha, and compare-and-swap file writes. The head is the store's
// working tree (committed or not), so the phone sees what agents on this
// computer see, and .gitignore keeps machine-local files off the phone.
const exec = promisify(execFile);
const SHA = /^[0-9a-f]{40}$/;
const MAX_FILE = 4 * 1024 * 1024;
let queue = Promise.resolve();
/** Serialize git work on the snapshot index; two phone polls share one index. */
function serialized(work) {
    const next = queue.then(work, work);
    queue = next.catch(() => undefined);
    return next;
}
async function git(store, args, env = {}) {
    try {
        const { stdout } = await exec("git", ["-C", store, ...args], {
            env: { ...process.env, ...env, GIT_TERMINAL_PROMPT: "0" }, maxBuffer: 64 * 1024 * 1024, timeout: 60_000,
        });
        return stdout;
    }
    catch (error) {
        const stderr = String(error.stderr ?? "").trim();
        throw new BridgeError(500, `git ${args[0]} failed in the store${stderr ? `: ${stderr.split("\n")[0]}` : "."}`);
    }
}
/** The sha git gives these bytes as a blob; GitHub reports the same one. */
export function blobSha(content) {
    return createHash("sha1").update(`blob ${content.length}\0`).update(content).digest("hex");
}
async function isRepository(store) {
    return (await lstat(path.join(store, ".git")).catch(() => undefined)) !== undefined;
}
/** Only the public repository identity reaches the phone, never credentials
 * embedded in a remote URL or a local filesystem path. */
export function githubStoreRepository(remote) {
    let host, pathname;
    const scp = /^(?:[^@/\s]+@)?github\.com:([^\s]+)$/i.exec(remote.trim());
    if (scp) {
        host = "github.com";
        pathname = scp[1];
    }
    else {
        try {
            const url = new URL(remote.trim());
            if (!["https:", "ssh:"].includes(url.protocol) || url.search || url.hash)
                return undefined;
            host = url.hostname;
            pathname = url.pathname.replace(/^\//, "");
        }
        catch {
            return undefined;
        }
    }
    if (host.toLowerCase() !== "github.com")
        return undefined;
    const repository = pathname.replace(/\/$/, "").replace(/\.git$/i, "");
    return /^[A-Za-z0-9][A-Za-z0-9-]*\/[A-Za-z0-9_.-]+$/.test(repository) ? repository.toLowerCase() : undefined;
}
async function storeRepositoryIdentity(store) {
    const [remote, branch] = await Promise.all([
        git(store, ["config", "--get", "remote.origin.url"]).catch(() => ""),
        git(store, ["symbolic-ref", "--quiet", "--short", "HEAD"]).catch(() => ""),
    ]);
    const repository = githubStoreRepository(remote);
    return repository && branch.trim() ? { repository, branch: branch.trim() } : undefined;
}
/** A tree sha for the store's current working tree, written through a private index. */
export function storeHead(store, indexFile = path.join(bridgeRoot(), "store-snapshot.index")) {
    return serialized(async () => {
        if (!await isRepository(store))
            throw new BridgeError(409, "This computer's phren store is not a git repository. Run phren init.");
        await mkdir(path.dirname(indexFile), { recursive: true, mode: 0o700 });
        const env = { GIT_INDEX_FILE: indexFile };
        if (!await lstat(indexFile).catch(() => undefined)) {
            const head = await git(store, ["rev-parse", "--verify", "--quiet", "HEAD"]).catch(() => "");
            await git(store, head.trim() ? ["read-tree", "HEAD"] : ["read-tree", "--empty"], env);
        }
        await git(store, ["add", "--all", "--", "."], env);
        const sha = (await git(store, ["write-tree"], env)).trim();
        const repositoryIdentity = await storeRepositoryIdentity(store);
        return { sha, ...(repositoryIdentity ? { repositoryIdentity } : {}) };
    });
}
/** The recursive tree for a head, with the fields the phone's GitTree decodes. */
export async function storeTree(store, sha) {
    if (!SHA.test(sha))
        throw new BridgeError(400, "Invalid tree sha.");
    if ((await git(store, ["cat-file", "-t", sha]).catch(() => "")).trim() !== "tree")
        throw new BridgeError(404, "Unknown tree.");
    const tree = (await git(store, ["ls-tree", "-r", "-l", "-z", sha])).split("\0").filter(Boolean).flatMap(line => {
        const match = /^(\d+) (\w+) ([0-9a-f]{40}) +(\d+|-)\t(.+)$/s.exec(line);
        // Symlinks and submodules never reach the phone.
        if (!match || match[2] !== "blob" || match[1] === "120000")
            return [];
        return [{ path: match[5], type: "blob", sha: match[3], size: Number(match[4]) }];
    });
    return { sha, truncated: false, tree };
}
export async function storeBlob(store, sha) {
    if (!SHA.test(sha))
        throw new BridgeError(400, "Invalid blob sha.");
    if ((await git(store, ["cat-file", "-t", sha]).catch(() => "")).trim() !== "blob")
        throw new BridgeError(404, "Unknown blob.");
    const size = Number((await git(store, ["cat-file", "-s", sha])).trim());
    if (size > MAX_FILE)
        throw new BridgeError(413, "This store file exceeds the phone's 4 MiB limit. Archive or shorten it on the computer, then sync again.", { code: "store-file-too-large", size, limit: MAX_FILE });
    const { stdout } = await exec("git", ["-C", store, "cat-file", "blob", sha], { encoding: "buffer", maxBuffer: MAX_FILE, timeout: 60_000 });
    return { sha, encoding: "base64", content: stdout.toString("base64") };
}
/** At most this many blobs and this much content (before base64) per batch answer. */
const BATCH_COUNT = 256;
const BATCH_BYTES = 9 * 1024 * 1024;
/**
 * Many blobs in one answer (Phren desktop's store mirror): one
 * `git cat-file --batch` for the whole request instead of three git processes
 * per blob. Blobs past the byte budget, over the 4 MiB file limit or unknown
 * come back as `{ sha, error }` for the client to fetch one by one or skip.
 */
export async function storeBlobs(store, data) {
    const shas = z.array(z.string().regex(SHA)).min(1).max(BATCH_COUNT).parse(data.shas);
    const unique = [...new Set(shas)];
    // Sizes first (no content), so the budget is applied before anything is read.
    const sizes = new Map();
    for (const line of (await catFile(store, "--batch-check", unique)).toString("utf8").split("\n")) {
        const [sha, type, size] = line.split(" ");
        if (sha && type && type !== "missing")
            sizes.set(sha, { type, size: Number(size) });
    }
    const blobs = [];
    const wanted = [];
    let budget = BATCH_BYTES;
    for (const sha of unique) {
        const info = sizes.get(sha);
        if (!info || info.type !== "blob") {
            blobs.push({ sha, error: "unknown" });
            continue;
        }
        if (info.size > MAX_FILE) {
            blobs.push({ sha, error: "too-large", size: info.size });
            continue;
        }
        if (info.size > budget) {
            blobs.push({ sha, error: "later" });
            continue;
        }
        budget -= info.size;
        wanted.push(sha);
    }
    if (wanted.length) {
        const out = await catFile(store, "--batch", wanted);
        let offset = 0;
        for (const sha of wanted) {
            const newline = out.indexOf(10, offset);
            if (newline < 0) {
                blobs.push({ sha, error: "later" });
                continue;
            }
            const size = Number(out.subarray(offset, newline).toString("utf8").split(" ")[2]);
            offset = newline + 1;
            blobs.push({ sha, encoding: "base64", content: out.subarray(offset, offset + size).toString("base64") });
            offset += size + 1;
        }
    }
    return { blobs };
}
/** Run one `git cat-file --batch[-check]` over these ids; output capped near the batch budget. */
function catFile(store, mode, shas) {
    return new Promise((resolve, reject) => {
        const child = spawn("git", ["-C", store, "cat-file", mode], { stdio: ["pipe", "pipe", "ignore"] });
        const chunks = [];
        let total = 0;
        child.stdout.on("data", (chunk) => {
            total += chunk.length;
            if (total > BATCH_BYTES + BATCH_COUNT * 128) {
                child.kill();
                reject(new BridgeError(413, "The batch is too large."));
                return;
            }
            chunks.push(chunk);
        });
        child.on("error", reject);
        child.on("close", () => resolve(Buffer.concat(chunks)));
        child.stdin.end(shas.map(sha => `${sha}\n`).join(""));
    });
}
/** Resolve a store-relative path, refusing escapes, .git and symlinked parents. */
export async function storeFile(store, relative) {
    const normalized = path.posix.normalize(relative);
    if (!relative || relative !== normalized || normalized.startsWith("../") || normalized === ".." || path.posix.isAbsolute(normalized)
        || normalized.split("/").some(part => part === ".git" || part === "" || part === ".") || /[\0\r\n]/.test(normalized)) {
        throw new BridgeError(400, "Invalid store path.");
    }
    const root = await realpath(store);
    let parent = path.dirname(path.join(root, normalized));
    // Missing parents are created later; the nearest existing one must stay inside the store.
    for (;;) {
        const real = await realpath(parent).catch(() => undefined);
        if (real) {
            if (real !== root && !real.startsWith(root + path.sep))
                throw new BridgeError(400, "Invalid store path.");
            break;
        }
        parent = path.dirname(parent);
    }
    const file = path.join(root, normalized);
    const info = await lstat(file).catch(() => undefined);
    if (info && !info.isFile())
        throw new BridgeError(400, "Invalid store path.");
    return file;
}
/** `withFileLock` on a store file, a held lock answered as a 409 to retry. */
function lockedStoreFile(file, rel, work) {
    try {
        withFileLock(file, work);
    }
    catch (error) {
        if (error instanceof BridgeError || !/could not acquire lock/.test(String(error?.message)))
            throw error;
        throw new BridgeError(409, `${rel} is being written on this computer (its lock is held). Try again.`);
    }
}
function currentShaSync(file) {
    try {
        return blobSha(readFileSync(file));
    }
    catch (error) {
        if (error.code === "ENOENT")
            return null;
        throw error;
    }
}
const putSchema = z.object({ path: z.string().max(1024), content: z.string(), sha: z.string().regex(SHA).nullable().optional() });
const deleteSchema = z.object({ path: z.string().max(1024), sha: z.string().regex(SHA) });
/** Write one file if it still has the sha the phone last saw (null: must not exist). */
export function putStoreFile(store, data) {
    const input = putSchema.parse(data);
    const content = Buffer.from(input.content, "base64");
    if (content.length > MAX_FILE)
        throw new BridgeError(413, "This file is too large for the phone to write.");
    return serialized(async () => {
        const file = await storeFile(store, input.path);
        // Under the lock the CLI's writers take (`<file>.lock`), so a write of
        // theirs cannot land between the sha check and the swap. withFileLock
        // waits synchronously, so the work inside it is synchronous too.
        lockedStoreFile(file, input.path, () => {
            if (currentShaSync(file) !== (input.sha ?? null))
                throw new BridgeError(409, `${input.path} changed on this computer.`);
            mkdirSync(path.dirname(file), { recursive: true });
            const temporary = `${file}.phren-${process.pid}-${Date.now()}`;
            writeFileSync(temporary, content, { flag: "wx" });
            renameSync(temporary, file);
        });
        return { content: { sha: blobSha(content), path: input.path }, commit: { sha: blobSha(content) } };
    });
}
export function deleteStoreFile(store, data) {
    const input = deleteSchema.parse(data);
    return serialized(async () => {
        const file = await storeFile(store, input.path);
        // The writers' lock across check and delete, as in putStoreFile.
        lockedStoreFile(file, input.path, () => {
            if (currentShaSync(file) !== input.sha)
                throw new BridgeError(409, `${input.path} changed on this computer.`);
            unlinkSync(file);
        });
        return { ok: true };
    });
}
export const STORE_ROUTES = { head: "/v1/store/head", tree: "/v1/store/tree", blob: "/v1/store/blob", blobs: "/v1/store/blobs", file: "/v1/store/file", delete: "/v1/store/delete" };
export async function storeRoute(store, method, url, data) {
    if (method === "GET" && url.pathname === STORE_ROUTES.head)
        return storeHead(store);
    if (method === "GET" && url.pathname === STORE_ROUTES.tree)
        return storeTree(store, url.searchParams.get("sha") ?? "");
    if (method === "GET" && url.pathname === STORE_ROUTES.blob)
        return storeBlob(store, url.searchParams.get("sha") ?? "");
    if (method === "POST" && url.pathname === STORE_ROUTES.blobs)
        return storeBlobs(store, data ?? {});
    if (method === "POST" && url.pathname === STORE_ROUTES.file)
        return putStoreFile(store, data ?? {});
    if (method === "POST" && url.pathname === STORE_ROUTES.delete)
        return deleteStoreFile(store, data ?? {});
    throw new BridgeError(404, "Unknown Phren Hook route.");
}