@sema-agent/core/dist/core/memory-engine/frontmatter.js

Stateless, task-oriented AI agent core

import { createHash } from "node:crypto";
import { FRONTMATTER_FENCE, isFrontmatterFence, UNCLOSED_FRONTMATTER_REASON } from "../frontmatter-fence.js";
import { MEMORY_ORIGIN_CAUSES } from "./types.js";
const ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{7,63}$/;
export function isValidEntryId(value) {
    return ID_RE.test(value);
}
const ORIGIN_AT_RE = /^-?\d+(\.\d+)?$/;
function atSerializationStable(at) {
    return Number.isFinite(at) && ORIGIN_AT_RE.test(String(at));
}
function locateFrontmatter(text) {
    const firstNewline = text.indexOf("\n");
    if (!isFrontmatterFence(firstNewline === -1 ? text : text.slice(0, firstNewline)))
        return undefined;
    const lines = text.split("\n");
    for (let i = 1; i < lines.length; i++) {
        if (isFrontmatterFence(lines[i]))
            return { lines, end: i };
    }
    return { lines, end: -1 };
}
export const UNCLOSED_MEMORY_FILE_REASON = `memory file ${UNCLOSED_FRONTMATTER_REASON} (rejected, NOT harvested as a file without frontmatter); close the block with a line that is exactly ---`;
export function parseEntryFile(text) {
    const fm = {};
    const located = locateFrontmatter(text);
    if (located === undefined)
        return { frontmatter: fm, body: text, hadFrontmatter: false };
    if (located.end === -1)
        return { frontmatter: fm, body: text, hadFrontmatter: false, unclosed: true };
    const { lines, end } = located;
    let id;
    const extra = [];
    let inMetadata = false;
    let inProvenance = false;
    const provRaw = [];
    const provUnknown = [];
    const prov = {};
    let inOrigin = false;
    const originRaw = [];
    let originPristine = true;
    const orig = {};
    let inDistilled = false;
    const distilledRaw = [];
    let distilledPristine = true;
    const dist = { inputs: [] };
    for (let i = 1; i < end; i++) {
        const line = lines[i] ?? "";
        const trimmed = line.trim();
        if (trimmed === "")
            continue;
        const indented = /^\s/.test(line);
        if (!indented) {
            inMetadata = false;
            inProvenance = false;
            inOrigin = false;
            inDistilled = false;
        }
        if (inMetadata) {
            const m = /^\s+type:\s*(.+?)\s*$/.exec(line);
            if (m) {
                fm.type = m[1];
                continue;
            }
            extra.push(line);
            continue;
        }
        if (inProvenance) {
            provRaw.push(line);
            const m = /^\s+([A-Za-z][A-Za-z0-9_-]*):\s*(.*?)\s*$/.exec(line);
            if (m) {
                const [, k, v] = m;
                if (k === "kind")
                    prov.kind = v;
                else if (k === "path")
                    prov.path = v;
                else if (k === "contentHash")
                    prov.contentHash = v;
                else if (k === "ingestedAt")
                    prov.ingestedAt = /^-?\d+(\.\d+)?$/.test(v) ? Number(v) : Number.NaN;
                else
                    provUnknown.push(line);
            }
            else {
                provUnknown.push(line);
            }
            continue;
        }
        if (/^metadata:\s*$/.test(trimmed)) {
            inMetadata = true;
            continue;
        }
        if (inOrigin) {
            originRaw.push(line);
            const m = /^\s+([A-Za-z][A-Za-z0-9_-]*):\s*(.*?)\s*$/.exec(line);
            if (m) {
                const [, k = "", v = ""] = m;
                if (k === "taint" && orig.taint === undefined)
                    orig.taint = v;
                else if (k === "cause" && orig.cause === undefined)
                    orig.cause = v;
                else if (k === "at" && orig.at === undefined)
                    orig.at = ORIGIN_AT_RE.test(v) ? Number(v) : Number.NaN;
                else
                    originPristine = false;
            }
            else {
                originPristine = false;
            }
            continue;
        }
        if (inDistilled) {
            distilledRaw.push(line);
            const m = /^\s+([A-Za-z][A-Za-z0-9_-]*):\s*(.*?)\s*$/.exec(line);
            if (m) {
                const [, k = "", v = ""] = m;
                if (k === "planId" && dist.planId === undefined && /^\S+$/.test(v))
                    dist.planId = v;
                else if (k === "at" && dist.at === undefined && ORIGIN_AT_RE.test(v))
                    dist.at = Number(v);
                else if (k === "carrierRev" && dist.carrierRev === undefined && /^\S+$/.test(v))
                    dist.carrierRev = v;
                else if (k === "input") {
                    const parts = v.split(/\s+/);
                    const [inId, inRev, flag, ...rest] = parts;
                    if (inId !== undefined && isValidEntryId(inId) && inRev !== undefined && /^\S+$/.test(inRev) && rest.length === 0 && (flag === undefined || flag === "superseded")) {
                        dist.inputs.push({ id: inId, rev: inRev, ...(flag === "superseded" ? { superseded: true } : {}) });
                    }
                    else {
                        distilledPristine = false;
                    }
                }
                else
                    distilledPristine = false;
            }
            else {
                distilledPristine = false;
            }
            continue;
        }
        if (/^provenance:\s*$/.test(trimmed)) {
            inProvenance = true;
            provRaw.push(line);
            continue;
        }
        if (/^origin:\s*$/.test(trimmed) && originRaw.length === 0) {
            inOrigin = true;
            originRaw.push(line);
            continue;
        }
        if (/^distilled:\s*$/.test(trimmed) && distilledRaw.length === 0) {
            inDistilled = true;
            distilledRaw.push(line);
            continue;
        }
        const kv = /^([A-Za-z_][A-Za-z0-9_-]*):\s*(.*?)\s*$/.exec(trimmed);
        if (!kv || indented) {
            extra.push(line);
            continue;
        }
        const [, key, value] = kv;
        switch (key) {
            case "id":
                if (isValidEntryId(value))
                    id = value;
                break;
            case "name":
                fm.name = value;
                break;
            case "description":
                fm.description = value;
                break;
            case "type":
                fm.type = value;
                break;
            case "deleted":
                fm.deleted = value === "true" || value === "yes";
                break;
            case "trust":
                if (value === "untrusted")
                    fm.trust = "untrusted";
                else
                    extra.push(line);
                break;
            default:
                extra.push(line);
                break;
        }
    }
    if (provRaw.length > 0) {
        if (prov.kind === "repo_file" &&
            typeof prov.path === "string" &&
            prov.path.length > 0 &&
            typeof prov.contentHash === "string" &&
            prov.contentHash.length > 0 &&
            typeof prov.ingestedAt === "number" &&
            Number.isFinite(prov.ingestedAt)) {
            fm.provenance = { kind: "repo_file", path: prov.path, contentHash: prov.contentHash, ingestedAt: prov.ingestedAt };
            extra.push(...provUnknown);
        }
        else {
            extra.push(...provRaw);
        }
    }
    if (originRaw.length > 0) {
        if (originPristine &&
            orig.taint === "external" &&
            typeof orig.at === "number" &&
            atSerializationStable(orig.at) &&
            (orig.cause === undefined || MEMORY_ORIGIN_CAUSES.includes(orig.cause))) {
            fm.origin = { taint: "external", ...(orig.cause !== undefined ? { cause: orig.cause } : {}), at: orig.at };
        }
        else {
            extra.push(...originRaw);
        }
    }
    if (distilledRaw.length > 0) {
        if (distilledPristine && dist.planId !== undefined && typeof dist.at === "number" && atSerializationStable(dist.at) && dist.carrierRev !== undefined && dist.inputs.length > 0) {
            fm.distilled = { planId: dist.planId, at: dist.at, carrierRev: dist.carrierRev, inputs: dist.inputs };
        }
        else {
            extra.push(...distilledRaw);
        }
    }
    if (extra.length > 0)
        fm.extra = extra;
    let body = lines.slice(end + 1).join("\n");
    if (body.startsWith("\n"))
        body = body.slice(1);
    return { id, frontmatter: fm, body, hadFrontmatter: true };
}
export function serializeEntryFile(entry) {
    const fm = entry.frontmatter;
    const lines = [FRONTMATTER_FENCE, `id: ${entry.id}`];
    if (fm.name !== undefined)
        lines.push(`name: ${fm.name}`);
    if (fm.description !== undefined)
        lines.push(`description: ${fm.description}`);
    if (fm.type !== undefined)
        lines.push("metadata:", `  type: ${fm.type}`);
    if (fm.deleted)
        lines.push("deleted: true");
    if (fm.provenance !== undefined) {
        lines.push("provenance:", `  kind: ${fm.provenance.kind}`, `  path: ${fm.provenance.path}`, `  contentHash: ${fm.provenance.contentHash}`, `  ingestedAt: ${fm.provenance.ingestedAt}`);
    }
    if (fm.trust !== undefined)
        lines.push(`trust: ${fm.trust}`);
    if (fm.extra)
        lines.push(...fm.extra);
    if (fm.origin !== undefined) {
        lines.push("origin:", `  taint: ${fm.origin.taint}`, ...(fm.origin.cause !== undefined ? [`  cause: ${fm.origin.cause}`] : []), `  at: ${fm.origin.at}`);
    }
    if (fm.distilled !== undefined) {
        lines.push("distilled:", `  planId: ${fm.distilled.planId}`, `  at: ${fm.distilled.at}`, `  carrierRev: ${fm.distilled.carrierRev}`, ...fm.distilled.inputs.map((i) => `  input: ${i.id} ${i.rev}${i.superseded === true ? " superseded" : ""}`));
    }
    lines.push(FRONTMATTER_FENCE, "");
    const body = entry.body.replace(/\s+$/, "");
    return `${lines.join("\n")}${body}${body ? "\n" : ""}`;
}
export function computeEntryRev(entry) {
    const fm = entry.frontmatter;
    const hasCarrierExtra = fm.extra !== undefined && fm.extra.some((line) => ORIGIN_FORM_LINE_RE.test(line));
    const canonicalOrigin = fm.origin !== undefined ? fm.origin : hasCarrierExtra ? committedOriginOf(fm) : undefined;
    const canonical = JSON.stringify([
        entry.id,
        fm.name ?? null,
        fm.description ?? null,
        fm.type ?? null,
        fm.deleted === true,
        hasCarrierExtra ? partitionOriginLines(fm.extra).kept : (fm.extra ?? []),
        entry.body.replace(/\s+$/, ""),
        ...(fm.provenance !== undefined || fm.trust !== undefined
            ? [fm.trust ?? null, fm.provenance !== undefined ? [fm.provenance.kind, fm.provenance.path, fm.provenance.contentHash, fm.provenance.ingestedAt] : null]
            : []),
        ...(canonicalOrigin !== undefined ? [["origin", canonicalOrigin.taint, canonicalOrigin.cause ?? null, canonicalOrigin.at]] : []),
        ...(fm.distilled !== undefined ? [["distilled", fm.distilled.planId, fm.distilled.at, fm.distilled.inputs.map((i) => [i.id, i.rev, i.superseded === true ? 1 : 0])]] : []),
    ]);
    return createHash("sha256").update(canonical, "utf8").digest("hex").slice(0, 16);
}
const ORIGIN_FORM_LINE_RE = /^\s*origin\s*:/;
export function hasOriginFormExtra(fm) {
    return fm.extra !== undefined && fm.extra.some((line) => ORIGIN_FORM_LINE_RE.test(line));
}
export function extraOriginCarriers(extra) {
    if (extra === undefined)
        return [];
    const carriers = [];
    let block;
    const closeBlock = () => {
        if (block === undefined)
            return;
        const { taint, cause, at } = block;
        block = undefined;
        if (taint === "external" && typeof at === "number" && atSerializationStable(at) && (cause === undefined || MEMORY_ORIGIN_CAUSES.includes(cause))) {
            carriers.push({ taint: "external", ...(cause !== undefined ? { cause: cause } : {}), at });
        }
        else {
            carriers.push({ taint: "external", at: 0 });
        }
    };
    for (const line of extra) {
        if (/^\s*origin\s*:\s*$/.test(line)) {
            closeBlock();
            block = {};
            continue;
        }
        if (block !== undefined && /^\s/.test(line)) {
            const m = /^\s+([A-Za-z][A-Za-z0-9_-]*):\s*(.*?)\s*$/.exec(line);
            if (m !== null) {
                const [, k = "", v = ""] = m;
                if (k === "taint" && block.taint === undefined)
                    block.taint = v;
                else if (k === "cause" && block.cause === undefined)
                    block.cause = v;
                else if (k === "at" && block.at === undefined)
                    block.at = ORIGIN_AT_RE.test(v) ? Number(v) : Number.NaN;
            }
            continue;
        }
        closeBlock();
        if (ORIGIN_FORM_LINE_RE.test(line))
            carriers.push({ taint: "external", at: 0 });
    }
    closeBlock();
    return carriers;
}
export function committedOriginOf(fm) {
    if (fm.origin !== undefined)
        return { taint: fm.origin.taint, ...(fm.origin.cause !== undefined ? { cause: fm.origin.cause } : {}), at: fm.origin.at };
    return extraOriginCarriers(fm.extra)[0];
}
export function consolidationExposedFrontmatter(fm) {
    return committedOriginOf(fm) !== undefined || fm.trust === "untrusted";
}
export function originEquals(a, b) {
    if (a === undefined || b === undefined)
        return a === b;
    return a.taint === b.taint && a.cause === b.cause && a.at === b.at;
}
export function ambiguousOriginRepresentation(fm) {
    if (fm.origin !== undefined && !atSerializationStable(fm.origin.at))
        return true;
    const values = extraOriginCarriers(fm.extra);
    if (fm.origin !== undefined)
        values.push({ taint: fm.origin.taint, ...(fm.origin.cause !== undefined ? { cause: fm.origin.cause } : {}), at: fm.origin.at });
    return values.some((v) => !originEquals(v, values[0]));
}
function partitionOriginLines(extra) {
    const kept = [];
    let removedAny = false;
    let inBlock = false;
    for (const line of extra) {
        if (ORIGIN_FORM_LINE_RE.test(line)) {
            inBlock = /^\s*origin\s*:\s*$/.test(line);
            removedAny = true;
            continue;
        }
        if (inBlock && /^\s/.test(line)) {
            removedAny = true;
            continue;
        }
        inBlock = false;
        kept.push(line);
    }
    return { kept, removedAny };
}
export function stripModelWrittenOrigin(fm) {
    let stripped = false;
    if (fm.origin !== undefined) {
        delete fm.origin;
        stripped = true;
    }
    if (fm.extra !== undefined) {
        const { kept, removedAny } = partitionOriginLines(fm.extra);
        if (removedAny) {
            stripped = true;
            if (kept.length > 0)
                fm.extra = kept;
            else
                delete fm.extra;
        }
    }
    return stripped;
}
const DISTILLED_FORM_LINE_RE = /^\s*distilled\s*:/;
export function hasDistilledFormExtra(fm) {
    return fm.extra !== undefined && fm.extra.some((line) => DISTILLED_FORM_LINE_RE.test(line));
}
export function committedDistilledOf(fm) {
    const d = fm.distilled;
    if (d === undefined)
        return undefined;
    return { planId: d.planId, at: d.at, carrierRev: d.carrierRev, inputs: d.inputs.map((i) => ({ id: i.id, rev: i.rev, ...(i.superseded === true ? { superseded: true } : {}) })) };
}
export function distilledEquals(a, b) {
    if (a === undefined || b === undefined)
        return a === b;
    if (a.planId !== b.planId || a.at !== b.at || a.carrierRev !== b.carrierRev || a.inputs.length !== b.inputs.length)
        return false;
    for (let i = 0; i < a.inputs.length; i++) {
        const x = a.inputs[i];
        const y = b.inputs[i];
        if (x.id !== y.id || x.rev !== y.rev || (x.superseded === true) !== (y.superseded === true))
            return false;
    }
    return true;
}
function partitionDistilledLines(extra) {
    const kept = [];
    let removedAny = false;
    let inBlock = false;
    for (const line of extra) {
        if (DISTILLED_FORM_LINE_RE.test(line)) {
            inBlock = /^\s*distilled\s*:\s*$/.test(line);
            removedAny = true;
            continue;
        }
        if (inBlock && /^\s/.test(line)) {
            removedAny = true;
            continue;
        }
        inBlock = false;
        kept.push(line);
    }
    return { kept, removedAny };
}
export function stripModelWrittenDistilled(fm) {
    let stripped = false;
    if (fm.distilled !== undefined) {
        delete fm.distilled;
        stripped = true;
    }
    if (fm.extra !== undefined) {
        const { kept, removedAny } = partitionDistilledLines(fm.extra);
        if (removedAny) {
            stripped = true;
            if (kept.length > 0)
                fm.extra = kept;
            else
                delete fm.extra;
        }
    }
    return stripped;
}
export function entryFromFile(text, id, slug, scope) {
    const parsed = parseEntryFile(text);
    const entry = { id, slug, frontmatter: parsed.frontmatter, body: parsed.body, rev: "", scope };
    entry.rev = computeEntryRev(entry);
    return entry;
}