@sema-agent/core/dist/core/memory-engine/sync-client.js

Stateless, task-oriented AI agent core

import { ambiguousOriginRepresentation, computeEntryRev, serializeEntryFile } from "./frontmatter.js";
import { screenInboundEntries } from "./data-plane.js";
import { scanMemoryFileName, scanMemoryWrite } from "./scan.js";
import { MAX_MEMORY_BYTES } from "../memory.js";
import { reconcileMemoryEntries } from "./sync.js";
import { MEMORY_ORIGIN_CAUSES } from "./types.js";
function memorySyncPath(scope) {
    return `/v1/memory/sync/${encodeURIComponent(scope)}`;
}
function baselineStub(id, rev, scope) {
    return { id, rev, scope, slug: `sync-baseline-stub/${id}`, frontmatter: {}, body: "" };
}
const isRecord = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
const isNonEmptyString = (v) => typeof v === "string" && v.length > 0;
function parseMemorySyncResponse(raw, scope, peer) {
    const fail = (why) => {
        throw new Error(`memory sync: transport response is not a valid sync response (${why})`);
    };
    if (!isRecord(raw))
        fail("body must be a JSON object { applied, conflicts, serverEntries, serverDeletes, cursor }");
    const b = raw;
    const applied = [];
    if (!Array.isArray(b["applied"]))
        fail("applied must be an array");
    for (const a of b["applied"]) {
        if (!isRecord(a) || (a["op"] !== "add" && a["op"] !== "update" && a["op"] !== "delete") || !isNonEmptyString(a["id"])) {
            fail("each applied item must be { op: add|update|delete, id }");
        }
        const rec = a;
        applied.push({ op: rec["op"], id: rec["id"], ...(isNonEmptyString(rec["slug"]) ? { slug: rec["slug"] } : {}) });
    }
    const conflicts = [];
    if (!Array.isArray(b["conflicts"]))
        fail("conflicts must be an array");
    for (const c of b["conflicts"]) {
        if (!isRecord(c) || !isNonEmptyString(c["id"]) || typeof c["reason"] !== "string")
            fail("each conflict must be { id, reason }");
        const rec = c;
        conflicts.push({
            id: rec["id"],
            reason: rec["reason"],
            ...(isNonEmptyString(rec["baseRev"]) ? { baseRev: rec["baseRev"] } : {}),
            ...(isNonEmptyString(rec["currentRev"]) ? { currentRev: rec["currentRev"] } : {}),
        });
    }
    const serverEntries = [];
    if (!Array.isArray(b["serverEntries"]))
        fail("serverEntries must be an array");
    for (const raw2 of b["serverEntries"]) {
        if (!isRecord(raw2))
            fail("each server entry must be an object { id, slug, frontmatter, body, rev, scope }");
        const e = raw2;
        if (!isNonEmptyString(e["id"]) || !isNonEmptyString(e["slug"]) || typeof e["body"] !== "string" || !isNonEmptyString(e["rev"])) {
            fail("each server entry must carry non-empty id/slug/rev and a string body");
        }
        if (e["scope"] !== scope)
            fail(`server entry ${JSON.stringify(e["id"])} has scope ${JSON.stringify(e["scope"])} — must equal the synced scope`);
        const fmRaw = e["frontmatter"] ?? {};
        if (!isRecord(fmRaw))
            fail("entry.frontmatter must be an object");
        const fm = fmRaw;
        for (const k of ["name", "description", "type"]) {
            if (fm[k] !== undefined && typeof fm[k] !== "string")
                fail(`entry.frontmatter.${k} must be a string when present`);
        }
        if (fm["deleted"] !== undefined && typeof fm["deleted"] !== "boolean")
            fail("entry.frontmatter.deleted must be a boolean when present");
        if (fm["extra"] !== undefined && !(Array.isArray(fm["extra"]) && fm["extra"].every((s) => typeof s === "string"))) {
            fail("entry.frontmatter.extra must be an array of strings when present");
        }
        if (fm["trust"] !== undefined && fm["trust"] !== "untrusted")
            fail('entry.frontmatter.trust must be "untrusted" when present');
        const provWire = fm["provenance"];
        if (provWire !== undefined &&
            (!isRecord(provWire) ||
                provWire["kind"] !== "repo_file" ||
                !isNonEmptyString(provWire["path"]) ||
                !isNonEmptyString(provWire["contentHash"]) ||
                typeof provWire["ingestedAt"] !== "number" ||
                !Number.isFinite(provWire["ingestedAt"]))) {
            fail('entry.frontmatter.provenance must be { kind: "repo_file", path, contentHash, ingestedAt } when present');
        }
        const originWire = fm["origin"];
        if (originWire !== undefined &&
            (!isRecord(originWire) ||
                originWire["taint"] !== "external" ||
                typeof originWire["at"] !== "number" ||
                !Number.isFinite(originWire["at"]) ||
                (originWire["cause"] !== undefined && !(typeof originWire["cause"] === "string" && MEMORY_ORIGIN_CAUSES.includes(originWire["cause"]))))) {
            fail('entry.frontmatter.origin must be { taint: "external", cause?, at } with a known cause when present');
        }
        const distilledWire = fm["distilled"];
        if (distilledWire !== undefined &&
            (!isRecord(distilledWire) ||
                !isNonEmptyString(distilledWire["planId"]) ||
                typeof distilledWire["at"] !== "number" ||
                !Number.isFinite(distilledWire["at"]) ||
                !isNonEmptyString(distilledWire["carrierRev"]) ||
                !(Array.isArray(distilledWire["inputs"]) &&
                    distilledWire["inputs"].length > 0 &&
                    distilledWire["inputs"].every((row) => isRecord(row) && isNonEmptyString(row["id"]) && isNonEmptyString(row["rev"]) && (row["superseded"] === undefined || row["superseded"] === true))))) {
            fail("entry.frontmatter.distilled must be { planId, at, carrierRev, inputs: [{ id, rev, superseded? }, …] } when present");
        }
        const extraRaw = fm["extra"];
        if (Array.isArray(extraRaw) && extraRaw.every((s) => typeof s === "string")) {
            const pickedOrigin = originWire !== undefined
                ? {
                    taint: "external",
                    ...(originWire["cause"] !== undefined ? { cause: originWire["cause"] } : {}),
                    at: originWire["at"],
                }
                : undefined;
            if (ambiguousOriginRepresentation({ ...(pickedOrigin !== undefined ? { origin: pickedOrigin } : {}), extra: fm["extra"] })) {
                fail("entry.frontmatter carries conflicting or duplicated origin representations (ambiguous marker representation refused)");
            }
        }
        serverEntries.push({
            id: e["id"],
            slug: e["slug"],
            body: e["body"],
            rev: e["rev"],
            scope,
            frontmatter: {
                ...(fm["name"] !== undefined ? { name: fm["name"] } : {}),
                ...(fm["description"] !== undefined ? { description: fm["description"] } : {}),
                ...(fm["type"] !== undefined ? { type: fm["type"] } : {}),
                ...(fm["deleted"] !== undefined ? { deleted: fm["deleted"] } : {}),
                ...(provWire !== undefined
                    ? {
                        provenance: {
                            kind: "repo_file",
                            path: provWire["path"],
                            contentHash: provWire["contentHash"],
                            ingestedAt: provWire["ingestedAt"],
                        },
                    }
                    : {}),
                ...(fm["trust"] !== undefined ? { trust: "untrusted" } : {}),
                ...(originWire !== undefined
                    ? {
                        origin: {
                            taint: "external",
                            ...(originWire["cause"] !== undefined ? { cause: originWire["cause"] } : {}),
                            at: originWire["at"],
                        },
                    }
                    : {}),
                ...(distilledWire !== undefined
                    ? {
                        distilled: {
                            planId: distilledWire["planId"],
                            at: distilledWire["at"],
                            carrierRev: distilledWire["carrierRev"],
                            inputs: distilledWire["inputs"].map((row) => ({
                                id: row["id"],
                                rev: row["rev"],
                                ...(row["superseded"] === true ? { superseded: true } : {}),
                            })),
                        },
                    }
                    : {}),
                ...(fm["extra"] !== undefined ? { extra: [...fm["extra"]] } : {}),
            },
        });
    }
    const serverDeletes = [];
    if (!Array.isArray(b["serverDeletes"]))
        fail("serverDeletes must be an array");
    for (const d of b["serverDeletes"]) {
        if (!isRecord(d) || !isNonEmptyString(d["id"]) || !isNonEmptyString(d["baseRev"]))
            fail("each server delete must be { id, baseRev }");
        const rec = d;
        serverDeletes.push({ id: rec["id"], baseRev: rec["baseRev"] });
    }
    const cur = b["cursor"];
    if (!isRecord(cur))
        fail("cursor must be an object { peer, baseRevs, updatedAtMs }");
    const curRec = cur;
    if (curRec["peer"] !== peer)
        fail(`cursor.peer ${JSON.stringify(curRec["peer"])} does not match the requested peer ${JSON.stringify(peer)}`);
    if (!isRecord(curRec["baseRevs"]))
        fail("cursor.baseRevs must be an object mapping id → rev");
    const baseRevs = {};
    for (const [id, rev] of Object.entries(curRec["baseRevs"])) {
        if (!isNonEmptyString(rev))
            fail(`cursor.baseRevs[${JSON.stringify(id)}] must be a non-empty rev string`);
        baseRevs[id] = rev;
    }
    const updatedAtMs = curRec["updatedAtMs"];
    if (typeof updatedAtMs !== "number" || !Number.isFinite(updatedAtMs))
        fail("cursor.updatedAtMs must be a finite number");
    const pullTruncatedRaw = b["pullTruncated"];
    if (pullTruncatedRaw !== undefined && typeof pullTruncatedRaw !== "boolean")
        fail("pullTruncated must be a boolean when present");
    return {
        applied,
        conflicts,
        serverEntries,
        serverDeletes,
        cursor: { peer, baseRevs, updatedAtMs: updatedAtMs },
        ...(pullTruncatedRaw === true ? { pullTruncated: true } : {}),
    };
}
export async function syncMemoryScope(opts) {
    const { backend, scope, transport, cursor } = opts;
    if (!isNonEmptyString(scope))
        throw new Error("memory sync: scope must be a non-empty string");
    const peer = opts.peer ?? cursor?.peer ?? "central";
    const now = opts.now ?? Date.now;
    if (cursor !== undefined && cursor.scope !== scope) {
        throw new Error(`memory sync: cursor is for scope ${JSON.stringify(cursor.scope)}, not ${JSON.stringify(scope)}`);
    }
    if (cursor !== undefined && cursor.peer !== peer) {
        throw new Error(`memory sync: cursor is for peer ${JSON.stringify(cursor.peer)}, not ${JSON.stringify(peer)}`);
    }
    for (const [name, v] of [["maxPushEntries", opts.maxPushEntries], ["maxPullEntries", opts.maxPullEntries], ["maxEntryBytes", opts.maxEntryBytes]]) {
        if (v !== undefined && (!Number.isSafeInteger(v) || v < 1)) {
            throw new Error(`memory sync: ${name} must be a positive integer (>= 1) when set, got ${String(v)}`);
        }
    }
    const baseRevs = { ...(cursor?.baseRevs ?? {}) };
    const headers = (await backend.listHeaders([scope])).filter((h) => h.scope === scope);
    const fetched = (await backend.getByIds(headers.map((h) => h.id))).filter((e) => e.scope === scope);
    const local = fetched.filter((e) => e.frontmatter.deleted !== true);
    const stubs = Object.entries(baseRevs).map(([id, rev]) => baselineStub(id, rev, scope));
    const plan = reconcileMemoryEntries(baseRevs, local, stubs);
    if (plan.pull.length > 0 || plan.deleteLocal.length > 0 || plan.conflicts.length > 0 || plan.cleared.length > 0) {
        throw new Error("memory sync: internal invariant violated — a baseline-stub peer produced pull/deleteLocal/conflict/cleared legs");
    }
    const pushGateConflicts = [];
    const gatedPush = plan.push.filter((e) => {
        const findings = [];
        const nameFinding = scanMemoryFileName(`${e.slug}.md`);
        if (nameFinding !== undefined)
            findings.push(nameFinding);
        findings.push(...scanMemoryWrite(serializeEntryFile(e), { maxBytes: opts.maxEntryBytes ?? MAX_MEMORY_BYTES }));
        if (findings.length === 0)
            return true;
        pushGateConflicts.push({
            side: "local",
            id: e.id,
            reason: `push_gate: ${findings.map((f) => `${f.code}: ${f.reason}`).join("; ")} — the entry was NOT pushed (it stays local; fix or remove it to stop this report)`,
        });
        return false;
    });
    let pushEntries = gatedPush;
    let pushTruncated = false;
    if (opts.maxPushEntries !== undefined) {
        pushEntries = [...gatedPush].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
        if (pushEntries.length > opts.maxPushEntries) {
            pushEntries = pushEntries.slice(0, opts.maxPushEntries);
            pushTruncated = true;
        }
    }
    const request = {
        peer,
        baseRevs,
        entries: pushEntries,
        deletes: plan.deleteRemote.map((d) => d.id),
        ...(opts.maxPullEntries !== undefined ? { pull: { limit: opts.maxPullEntries } } : {}),
    };
    const resp = parseMemorySyncResponse(await transport(memorySyncPath(scope), request), scope, peer);
    const conflicts = [...pushGateConflicts, ...resp.conflicts.map((c) => ({ side: "server", ...c }))];
    const rejected = new Set();
    const localIds = new Set(local.map((e) => e.id));
    const serverEntryIds = new Set(resp.serverEntries.map((e) => e.id));
    for (const e of resp.serverEntries) {
        const computed = computeEntryRev(e);
        if (computed !== e.rev) {
            rejected.add(e.id);
            conflicts.push({
                side: "local",
                id: e.id,
                reason: `rev_mismatch: server claimed ${JSON.stringify(e.rev)} but the content hashes to ${JSON.stringify(computed)} (rev is core's computeEntryRev content-hash contract)`,
                ...(baseRevs[e.id] !== undefined ? { baseRev: baseRevs[e.id] } : {}),
            });
            continue;
        }
        if (baseRevs[e.id] === undefined && localIds.has(e.id)) {
            rejected.add(e.id);
            conflicts.push({
                side: "local",
                id: e.id,
                reason: `untracked_local_exists: server offers ${JSON.stringify(e.id)} as a first-round add, but an entry with that id already exists locally (concurrent create the server had not observed) — not overwritten`,
            });
        }
    }
    for (const f of screenInboundEntries(resp.serverEntries.filter((e) => !rejected.has(e.id)))) {
        rejected.add(f.id);
        conflicts.push({
            side: "local",
            id: f.id,
            reason: `inbound_gate: ${f.findings.map((x) => `${x.code}: ${x.reason}`).join("; ")}`,
            ...(baseRevs[f.id] !== undefined ? { baseRev: baseRevs[f.id] } : {}),
        });
    }
    const deletePatches = [];
    for (const d of resp.serverDeletes) {
        if (serverEntryIds.has(d.id)) {
            rejected.add(d.id);
            conflicts.push({
                side: "local",
                id: d.id,
                reason: `server_delete_and_entry: ${JSON.stringify(d.id)} appears in both serverEntries and serverDeletes of one response — the server cannot both push and delete it in one round`,
                baseRev: d.baseRev,
            });
            continue;
        }
        const base = baseRevs[d.id];
        if (base === undefined) {
            rejected.add(d.id);
            conflicts.push({
                side: "local",
                id: d.id,
                reason: `server_delete_not_in_baseline: server asked to delete ${JSON.stringify(d.id)} but it is absent from our input baseline (never synced here — a delete may not cross onto an id the server has never seen from us)`,
                baseRev: d.baseRev,
            });
            continue;
        }
        if (base !== d.baseRev) {
            rejected.add(d.id);
            conflicts.push({
                side: "local",
                id: d.id,
                reason: `server_delete_baserev_mismatch: server delete claims baseRev ${JSON.stringify(d.baseRev)} but our baseline holds ${JSON.stringify(base)} — refused (concurrent local change since the baseline)`,
                baseRev: base,
                currentRev: d.baseRev,
            });
            continue;
        }
        deletePatches.push({ op: "delete", id: d.id, baseRev: base });
    }
    const patches = [
        ...resp.serverEntries
            .filter((e) => !rejected.has(e.id))
            .map((e) => {
            const base = baseRevs[e.id];
            return base !== undefined
                ? { op: "update", id: e.id, entry: e, baseRev: base }
                : { op: "add", id: e.id, entry: e, guard: "absent" };
        }),
        ...deletePatches,
    ];
    const report = patches.length > 0 ? await backend.applyPatches(patches) : { applied: [], conflicts: [] };
    for (const c of report.conflicts) {
        conflicts.push({
            side: "local",
            id: c.id,
            reason: c.reason,
            ...(c.baseRev !== undefined ? { baseRev: c.baseRev } : {}),
            ...(c.currentRev !== undefined ? { currentRev: c.currentRev } : {}),
        });
    }
    const cursorAdvanced = rejected.size === 0 && report.conflicts.length === 0;
    let nextCursor = cursor;
    const warnings = [];
    if (cursorAdvanced) {
        const pulledRevById = new Map(resp.serverEntries.map((e) => [e.id, e.rev]));
        const pushedRevById = new Map(request.entries.map((e) => [e.id, e.rev]));
        const serverAppliedIds = new Set(resp.applied.filter((a) => a.op !== "delete").map((a) => a.id));
        const disputedIds = new Set(resp.conflicts.map((c) => c.id));
        const pushAcked = (id, rev) => rev === pushedRevById.get(id) && (serverAppliedIds.has(id) || !disputedIds.has(id));
        const nextBaseRevs = {};
        for (const [id, rev] of Object.entries(resp.cursor.baseRevs)) {
            if (rev === baseRevs[id] || pushAcked(id, rev) || rev === pulledRevById.get(id)) {
                nextBaseRevs[id] = rev;
                continue;
            }
            const retained = baseRevs[id];
            const conflictedPush = rev === pushedRevById.get(id);
            if (retained !== undefined) {
                nextBaseRevs[id] = retained;
                warnings.push(conflictedPush
                    ? `cursor_prefill_rejected: server cursor advanced ${JSON.stringify(id)} to our pushed rev ${JSON.stringify(rev)} while reporting that push rejected/conflicted — kept the old baseline ${JSON.stringify(retained)} (the local version stays re-pushed / ladder-escalated next round instead of being silently overwritten)`
                    : `cursor_prefill_rejected: server cursor advanced ${JSON.stringify(id)} to ${JSON.stringify(rev)} without delivering it this round — kept the old baseline ${JSON.stringify(retained)} (the entry stays pullable next round)`);
            }
            else {
                warnings.push(`cursor_prefill_rejected: server cursor introduced ${JSON.stringify(id)} at ${JSON.stringify(rev)} but never delivered it — dropped from the baseline (accepting it would spuriously delete-propagate an entry never seen here)`);
            }
        }
        for (const op of report.applied) {
            if (op.op === "delete") {
                delete nextBaseRevs[op.id];
            }
            else {
                const landedRev = pulledRevById.get(op.id);
                if (landedRev !== undefined)
                    nextBaseRevs[op.id] = landedRev;
            }
        }
        nextCursor = { scope, peer, baseRevs: nextBaseRevs, updatedAtMs: now() };
    }
    return {
        ok: cursorAdvanced && conflicts.length === 0,
        pushed: { entries: request.entries.map((e) => e.id), deletes: [...request.deletes] },
        serverApplied: resp.applied,
        localApplied: report.applied,
        conflicts,
        cursorAdvanced,
        cursor: nextCursor,
        ...(pushTruncated ? { pushTruncated: true } : {}),
        ...(resp.pullTruncated === true ? { pullTruncated: true } : {}),
        ...(warnings.length > 0 ? { warnings } : {}),
    };
}