@sema-agent/core/dist/core/memory-engine/data-plane.js

Stateless, task-oriented AI agent core

import { uuidv7 } from "../../internal/harness.js";
import { firstSentence } from "../memory.js";
import { computeEntryRev, parseEntryFile, serializeEntryFile } from "./frontmatter.js";
import { scanMemoryFileName, scanMemoryWrite } from "./scan.js";
import { MEMORY_INDEX_FILENAME } from "./entry-scan.js";
import { scopeDirName } from "./layout.js";
export const REMOTE_HARVEST_PER_FILE_BYTES = 100 * 1024;
export const REMOTE_MASS_DELETION_FUSE_RATIO = 0.5;
export const REMOTE_HARVEST_MAX_FILES = 500;
export const REMOTE_HARVEST_MAX_DEPTH = 3;
export async function materializeEntriesToFiles(backend, scopes, writeScope) {
    const all = [...new Set(writeScope !== null ? [...scopes, writeScope] : [...scopes])];
    const headers = await backend.listHeaders(all);
    const entries = await backend.getByIds(headers.map((h) => h.id));
    const files = [];
    const readonlyDirs = [...new Set(all.filter((s) => s !== writeScope).map((s) => scopeDirName(s)))];
    const baseline = { writeScope, files: [], readonlyDirs };
    for (const e of entries) {
        const readonly = e.scope !== writeScope;
        const relPath = readonly ? `${scopeDirName(e.scope)}/${e.slug}.md` : `${e.slug}.md`;
        const content = serializeEntryFile(e);
        files.push({ relPath, content, readonly });
        baseline.files.push({ relPath, id: e.id, rev: e.rev, scope: e.scope, readonly });
    }
    return { files, baseline };
}
export function harvestFilesToPatches(baseline, files, opts = {}) {
    const perFileBytes = opts.perFileBytes ?? REMOTE_HARVEST_PER_FILE_BYTES;
    const fuseRatio = opts.massDeletionFuseRatio ?? REMOTE_MASS_DELETION_FUSE_RATIO;
    const maxFiles = opts.maxFiles ?? REMOTE_HARVEST_MAX_FILES;
    const maxDepth = opts.maxDepth ?? REMOTE_HARVEST_MAX_DEPTH;
    const out = { patches: [], rejections: [], missing: [], projections: [] };
    const writeScope = baseline.writeScope;
    if (writeScope === null)
        return out;
    const byRel = new Map(files.map((f) => [f.relPath, f]));
    const baseByRel = new Map(baseline.files.map((b) => [b.relPath, b]));
    const basePathById = new Map(baseline.files.filter((b) => !b.readonly).map((b) => [b.id, b.relPath]));
    const readonlyPrefixes = new Set();
    for (const b of baseline.files) {
        if (!b.readonly)
            continue;
        const top = b.relPath.split("/")[0];
        if (top !== undefined && b.relPath.includes("/"))
            readonlyPrefixes.add(`${top}/`);
    }
    for (const d of baseline.readonlyDirs ?? []) {
        if (d !== "")
            readonlyPrefixes.add(`${d}/`);
    }
    const inReadonlyDir = (relPath) => {
        for (const p of readonlyPrefixes)
            if (relPath.startsWith(p))
                return true;
        return false;
    };
    for (const b of baseline.files.filter((x) => x.readonly)) {
        const f = byRel.get(b.relPath);
        if (f === undefined) {
            out.missing.push(b.relPath);
            continue;
        }
        if (revOf(f.content, b.id) !== b.rev) {
            out.rejections.push({
                path: b.relPath,
                code: "readonly_layer",
                reason: "a read-only inherited memory layer was modified; changes to inherited layers are not writable from this session",
            });
        }
    }
    const presentIds = new Set();
    const parsedByRel = new Map();
    for (const f of files) {
        if (baseByRel.get(f.relPath)?.readonly)
            continue;
        const p = parseEntryFile(f.content);
        parsedByRel.set(f.relPath, p);
        if (p.id !== undefined)
            presentIds.add(p.id);
    }
    const writable = baseline.files.filter((b) => !b.readonly);
    const missingWritable = writable.filter((b) => !byRel.has(b.relPath) && !presentIds.has(b.id));
    out.missing.push(...missingWritable.map((b) => b.relPath));
    if (writable.length > 0 && missingWritable.length / writable.length > fuseRatio) {
        out.incident = {
            kind: "mass_deletion",
            detail: `${missingWritable.length}/${writable.length} materialized memory files are missing from the read-back set — judged an accident, harvest refused (no patches; explicit frontmatter tombstones are the only deletion channel)`,
        };
        out.patches = [];
        return out;
    }
    let kept = 0;
    for (const f of files) {
        const base = baseByRel.get(f.relPath);
        if (base?.readonly)
            continue;
        if (f.relPath.startsWith("/") || f.relPath.split("/").some((seg) => seg === ".." || seg === "." || seg === "")) {
            out.rejections.push({ path: f.relPath, code: "outside_root", reason: "file path escapes the memory root (containment gate, fail-closed)" });
            continue;
        }
        const baseName = f.relPath.split("/").pop() ?? f.relPath;
        if (baseName === MEMORY_INDEX_FILENAME || !baseName.endsWith(".md") || baseName.startsWith("."))
            continue;
        if (inReadonlyDir(f.relPath)) {
            if (base === undefined) {
                out.rejections.push({ path: f.relPath, code: "readonly_layer", reason: "a file inside a read-only inherited layer's directory is not writable from this session" });
            }
            continue;
        }
        if (f.relPath.split("/").length - 1 > maxDepth) {
            out.rejections.push({ path: f.relPath, code: "nested_too_deep", reason: `memory files nested deeper than ${maxDepth} directory levels are not harvested` });
            continue;
        }
        if (kept >= maxFiles) {
            out.rejections.push({ path: f.relPath, code: "file_cap", reason: `memory file count exceeds the cap (${maxFiles}); consolidate before adding more` });
            continue;
        }
        kept++;
        const baseUnchanged = base !== undefined && revOf(f.content, base.id) === base.rev;
        if (baseUnchanged)
            continue;
        const sizeBytes = Buffer.byteLength(f.content, "utf8");
        if (sizeBytes > perFileBytes) {
            out.rejections.push({ path: f.relPath, code: "too_large", reason: `memory file is ${sizeBytes} bytes — over the ${perFileBytes}-byte cap; split or trim it (rejected, NOT truncated)` });
            continue;
        }
        const findings = [];
        const nameFinding = scanMemoryFileName(f.relPath);
        if (nameFinding !== undefined)
            findings.push(nameFinding);
        findings.push(...scanMemoryWrite(f.content));
        const finding = findings[0];
        if (finding !== undefined) {
            out.rejections.push({ path: f.relPath, code: finding.code, reason: `memory write blocked: ${finding.reason} (remote lane: the deployment owns quarantine I/O)` });
            continue;
        }
        const parsed = parsedByRel.get(f.relPath) ?? parseEntryFile(f.content);
        const baseId = base?.id;
        let id = baseId ?? parsed.id;
        if (id === undefined)
            id = uuidv7();
        const fm = { ...parsed.frontmatter };
        const slug = f.relPath.replace(/\.md$/i, "");
        if (fm.name === undefined)
            fm.name = slug.split("/").pop();
        if (fm.description === undefined && !fm.deleted)
            fm.description = firstSentence(parsed.body) || undefined;
        const entry = { id, slug, frontmatter: fm, body: parsed.body, rev: "", scope: writeScope };
        entry.rev = computeEntryRev(entry);
        if (fm.deleted === true) {
            if (baseId !== undefined || (parsed.id !== undefined && basePathById.has(parsed.id))) {
                const basePath = baseId !== undefined ? f.relPath : basePathById.get(parsed.id);
                const baseRev = baseByRel.get(basePath)?.rev;
                out.patches.push({ op: "delete", id, ...(baseRev !== undefined ? { baseRev } : {}) });
            }
            continue;
        }
        if (baseId !== undefined) {
            out.projections.push({ relPath: f.relPath, entry, content: serializeEntryFile(entry) });
            out.patches.push({ op: "update", id, entry, ...(base.rev !== undefined ? { baseRev: base.rev } : {}) });
            continue;
        }
        const renamedFrom = parsed.id !== undefined ? basePathById.get(parsed.id) : undefined;
        if (renamedFrom !== undefined && !byRel.has(renamedFrom)) {
            const moved = { ...entry, id: parsed.id, rev: computeEntryRev({ id: parsed.id, frontmatter: fm, body: parsed.body }) };
            out.projections.push({ relPath: f.relPath, entry: moved, content: serializeEntryFile(moved) });
            const baseRev = baseByRel.get(renamedFrom)?.rev;
            out.patches.push({ op: "update", id: parsed.id, entry: moved, ...(baseRev !== undefined ? { baseRev } : {}) });
            continue;
        }
        out.projections.push({ relPath: f.relPath, entry, content: serializeEntryFile(entry) });
        out.patches.push({ op: "add", id: entry.id, entry });
    }
    return out;
}
export function projectionsToWriteBack(result, report) {
    const appliedIds = new Set(report.applied.filter((a) => a.op !== "delete").map((a) => a.id));
    return result.projections.filter((p) => appliedIds.has(p.entry.id)).map((p) => ({ relPath: p.relPath, content: p.content }));
}
export function screenInboundEntries(entries, opts = {}) {
    const perFileBytes = opts.perFileBytes ?? REMOTE_HARVEST_PER_FILE_BYTES;
    const out = [];
    for (const e of entries) {
        const findings = [];
        const nameFinding = scanMemoryFileName(`${e.slug}.md`);
        if (nameFinding !== undefined)
            findings.push(nameFinding);
        const serialized = serializeEntryFile(e);
        const size = Buffer.byteLength(serialized, "utf8");
        if (size > perFileBytes)
            findings.push({ code: "too_large", reason: `serialized entry is ${size} bytes — over the ${perFileBytes}-byte cap` });
        findings.push(...scanMemoryWrite(serialized));
        if (findings.length > 0)
            out.push({ id: e.id, findings });
    }
    return out;
}
function revOf(text, id) {
    const parsed = parseEntryFile(text);
    return computeEntryRev({ id, frontmatter: parsed.frontmatter, body: parsed.body });
}