@sema-agent/core/dist/stores/file/mailbox-store.js

Stateless, task-oriented AI agent core

import { join, resolve, sep } from "node:path";
import { randomBytes } from "node:crypto";
import { assertRetentionPolicy } from "../../core/retention-policy.js";
import { existsSync, linkSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs";
import { detachMailboxMessage, newestSentAt, readMailboxPeerMeta, } from "../../core/mailbox-store.js";
import { AppendLog, atomicWriteFile, canonicalStoreKey, ensureDir, readJsonlRecords, sanitizeScope, sanitizePathComponent } from "./fs-atomic.js";
import { guardCorruptSink } from "./corrupt-read-sink.js";
import { physicalPathOf } from "../../core/physical-path.js";
import { assertAdoptionBootGate } from "./adoption/marker.js";
function diskMarkOf(path) {
    try {
        const st = statSync(path);
        return { ino: st.ino, size: st.size, mtimeMs: st.mtimeMs };
    }
    catch (err) {
        if (err.code === "ENOENT")
            return undefined;
        throw err;
    }
}
const UNKNOWN_MARK = { ino: -1, size: -1, mtimeMs: -1 };
function markAfterWrite(path) {
    try {
        return diskMarkOf(path);
    }
    catch {
        return UNKNOWN_MARK;
    }
}
function sameDiskMark(a, b) {
    if (a === undefined || b === undefined)
        return a === b;
    return a.ino === b.ino && a.size === b.size && a.mtimeMs === b.mtimeMs;
}
const sharedBoxes = new Map();
function listBoxFiles(scopeDir) {
    try {
        return readdirSync(scopeDir, { withFileTypes: true })
            .filter((e) => e.isFile() && e.name.endsWith(".jsonl"))
            .map((e) => e.name);
    }
    catch {
        return [];
    }
}
const pathLocks = new Map();
function withPathLock(key, fn) {
    const prev = pathLocks.get(key) ?? Promise.resolve();
    const run = prev.then(() => fn(), () => fn());
    const tail = run.then(() => { }, () => { });
    pathLocks.set(key, tail);
    void tail.then(() => {
        if (pathLocks.get(key) === tail)
            pathLocks.delete(key);
    });
    return run;
}
export const MAILBOX_BOX_LOCK_TIMEOUT_CODE = "mailbox.box_lock_timeout";
export const MAILBOX_BOX_LOCK_COMPROMISED_CODE = "mailbox.box_lock_compromised";
const BOX_LOCK_RECLAIM_GRACE_MS = 5_000;
const BOX_LOCK_BACKOFF_MIN_MS = 1;
const BOX_LOCK_BACKOFF_MAX_MS = 16;
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const abandonedLockNonces = new Set();
function ownerAlive(owner) {
    if (owner.pid === process.pid)
        return !abandonedLockNonces.has(owner.nonce);
    return isProcessAlive(owner.pid);
}
function isProcessAlive(pid) {
    try {
        process.kill(pid, 0);
        return true;
    }
    catch (err) {
        return err.code !== "ESRCH";
    }
}
function readLockOwner(lockPath) {
    let raw;
    try {
        raw = readFileSync(lockPath, "utf8");
    }
    catch (err) {
        if (err.code === "ENOENT")
            return undefined;
        return "unobservable";
    }
    const [pidText, nonce, rest] = raw.split("\n");
    if (pidText === undefined || !/^[1-9][0-9]*$/.test(pidText) || nonce === undefined || !/^[0-9a-f]{16}$/.test(nonce) || rest !== "")
        return "unreadable";
    return { pid: Number.parseInt(pidText, 10), nonce };
}
function publishLockFile(lockPath, owner) {
    const tmp = `${lockPath}.pub.${owner.pid}.${owner.nonce}`;
    writeFileSync(tmp, `${owner.pid}\n${owner.nonce}\n`, { mode: 0o600 });
    try {
        linkSync(tmp, lockPath);
        return true;
    }
    catch (err) {
        const back = readLockOwner(lockPath);
        if (back !== undefined && back !== "unreadable" && back !== "unobservable" && back.pid === owner.pid && back.nonce === owner.nonce)
            return true;
        if (back === "unobservable")
            abandonedLockNonces.add(owner.nonce);
        if (err.code === "EEXIST")
            return false;
        throw err;
    }
    finally {
        try {
            unlinkSync(tmp);
        }
        catch {
        }
    }
}
export async function acquireMailboxBoxLock(lockPath, waitMs) {
    const mine = { pid: process.pid, nonce: randomBytes(8).toString("hex") };
    const deadline = Date.now() + waitMs;
    let backoff = BOX_LOCK_BACKOFF_MIN_MS;
    let holderNote = "unknown holder";
    let swept = false;
    for (;;) {
        let published;
        try {
            published = publishLockFile(lockPath, mine);
        }
        catch (err) {
            if (err.code !== "ENOENT")
                throw err;
            ensureDir(join(lockPath, ".."));
            continue;
        }
        if (published) {
            return () => {
                let now = readLockOwner(lockPath);
                if (now === "unobservable")
                    now = readLockOwner(lockPath);
                if (now === "unobservable") {
                    abandonedLockNonces.add(mine.nonce);
                    console.warn(`[sema] file mailbox: the box lock ${lockPath} could not be read back at release (a read fault); it is kept and still names this live process (pid ${process.pid}). ` +
                        `This process reclaims it at its next use of the box; another process waits on it and times out naming this pid until then.`);
                    return;
                }
                if (now === undefined || now === "unreadable" || now.nonce !== mine.nonce) {
                    const e = new Error(`FileMailboxStore: the box lock changed hands while this process held it (a stale reclaim overlapped) — the operation just performed may have raced another process; do not trust it. The lock file's location is on this error's \`lockPath\`, not in this message.`);
                    e.code = MAILBOX_BOX_LOCK_COMPROMISED_CODE;
                    e.lockPath = lockPath;
                    throw e;
                }
                try {
                    unlinkSync(lockPath);
                }
                catch (first) {
                    const again = readLockOwner(lockPath);
                    if (again === undefined || again === "unreadable" || (again !== "unobservable" && again.nonce !== mine.nonce))
                        return;
                    try {
                        if (again === "unobservable")
                            throw first;
                        unlinkSync(lockPath);
                    }
                    catch {
                        abandonedLockNonces.add(mine.nonce);
                        console.warn(`[sema] file mailbox: the box lock ${lockPath} could not be handed back (${first.message}); it still names this live process (pid ${process.pid}). ` +
                            `This process reclaims it at its next use of the box; another process waits on it and times out naming this pid until then.`);
                    }
                }
            };
        }
        const owner = readLockOwner(lockPath);
        if (owner === undefined)
            continue;
        let reclaim = "blocked";
        if (owner === "unreadable") {
            holderNote = "unreadable lock file — remove it by hand if no process holds it";
        }
        else if (owner === "unobservable") {
            holderNote = "the lock file could not be read (a read fault)";
        }
        else if (ownerAlive(owner)) {
            holderNote = owner.pid === process.pid ? `pid ${owner.pid} (this process — another isolate or thread of it)` : `pid ${owner.pid}`;
        }
        else {
            const r = reclaimStaleBoxLock(lockPath, owner, mine);
            reclaim = r.outcome;
            holderNote = r.note;
        }
        if (reclaim === "done")
            continue;
        if (!swept) {
            swept = true;
            sweepLockResidue(join(lockPath, ".."), lockPath.slice(join(lockPath, "..").length + 1));
        }
        if (Date.now() >= deadline) {
            const e = new Error(`FileMailboxStore: the box lock was not obtainable within ${waitMs}ms (${holderNote}) — refusing to touch the box without it. The lock file's location is on this error's \`lockPath\`, not in this message.`);
            e.code = MAILBOX_BOX_LOCK_TIMEOUT_CODE;
            e.lockPath = lockPath;
            if (owner !== "unreadable" && owner !== "unobservable")
                e.holderPid = owner.pid;
            throw e;
        }
        await sleep(backoff / 2 + Math.random() * (backoff / 2));
        backoff = Math.min(BOX_LOCK_BACKOFF_MAX_MS, backoff * 2);
    }
}
function ageOf(path) {
    try {
        return Date.now() - statSync(path).mtimeMs;
    }
    catch {
        return undefined;
    }
}
function reclaimStaleBoxLock(lockPath, stale, mine) {
    const gate = `${lockPath}.reclaim.${stale.nonce}`;
    let gateOwned;
    try {
        gateOwned = publishLockFile(gate, mine);
    }
    catch (err) {
        return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid}; the reclaim gate could not be published: ${err.message}` };
    }
    if (!gateOwned) {
        const g = readLockOwner(gate);
        if (g === "unobservable")
            return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid}; its reclaim gate could not be read (a read fault)` };
        const gateLive = g !== undefined && g !== "unreadable" && ownerAlive(g);
        if (gateLive)
            return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid}; pid ${g.pid} is reclaiming it` };
        if (g === undefined || g !== "unreadable" || (ageOf(gate) ?? 0) > BOX_LOCK_RECLAIM_GRACE_MS) {
            try {
                unlinkSync(gate);
            }
            catch {
            }
        }
        return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid}; a previous reclaim's gate is being cleared` };
    }
    try {
        const again = readLockOwner(lockPath);
        if (again === undefined)
            return { outcome: "done", note: "" };
        if (again === "unobservable")
            return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid} could not be re-read under the gate (a read fault)` };
        if (again === "unreadable" || again.pid !== stale.pid || again.nonce !== stale.nonce) {
            return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid} was reclaimed meanwhile; re-judging the current holder` };
        }
        const aside = `${lockPath}.stale.${mine.pid}.${mine.nonce}`;
        try {
            renameSync(lockPath, aside);
        }
        catch (err) {
            if (err.code === "ENOENT")
                return { outcome: "done", note: "" };
            return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid} cannot be moved aside: ${err.message}` };
        }
        try {
            unlinkSync(aside);
        }
        catch {
        }
        sweepLockResidue(join(lockPath, ".."), lockPath.slice(join(lockPath, "..").length + 1));
        return { outcome: "done", note: "" };
    }
    finally {
        try {
            unlinkSync(gate);
        }
        catch {
        }
    }
}
const RESIDUE_NAME_RE = /\.jsonl\.lock\.(?:pub|stale)\.([1-9][0-9]*)\.([0-9a-f]{16})$/;
const GATE_NAME_RE = /\.jsonl\.lock\.reclaim\.[0-9a-f]{16}$/;
function sweepLockResidue(dir, lockName) {
    let names;
    try {
        names = readdirSync(dir);
    }
    catch {
        return;
    }
    const now = Date.now();
    for (const name of names) {
        if (lockName !== undefined && !name.startsWith(`${lockName}.`))
            continue;
        const isGate = GATE_NAME_RE.test(name);
        const named = RESIDUE_NAME_RE.exec(name);
        if (!isGate && named === null)
            continue;
        const p = join(dir, name);
        let owner;
        if (isGate) {
            const g = readLockOwner(p);
            if (g === "unobservable")
                continue;
            owner = g !== undefined && g !== "unreadable" ? g : undefined;
        }
        else {
            owner = { pid: Number.parseInt(named[1], 10), nonce: named[2] };
        }
        if (owner !== undefined && ownerAlive(owner))
            continue;
        if (owner === undefined) {
            let age;
            try {
                age = now - statSync(p).mtimeMs;
            }
            catch {
                continue;
            }
            if (age <= BOX_LOCK_RECLAIM_GRACE_MS)
                continue;
        }
        try {
            unlinkSync(p);
        }
        catch {
        }
    }
}
export class FileMailboxStore {
    crossProcessSafe = true;
    dir;
    tmpDir;
    fsyncEnabled;
    compactEvery;
    lockWaitMs;
    touched = new Map();
    discloseCorrupt;
    constructor(root, opts = {}) {
        assertAdoptionBootGate(root, "FileMailboxStore");
        this.fsyncEnabled = opts.fsync !== false;
        this.compactEvery = opts.compactEvery ?? 500;
        const lockWaitMs = opts.lockWaitMs ?? 10_000;
        if (typeof lockWaitMs !== "number" || !Number.isFinite(lockWaitMs) || lockWaitMs <= 0) {
            throw new Error(`FileMailboxStore: lockWaitMs must be a finite number > 0, got ${JSON.stringify(opts.lockWaitMs)}`);
        }
        this.lockWaitMs = lockWaitMs;
        this.discloseCorrupt = guardCorruptSink(opts.onCorruptRead);
        const lexicalDir = resolve(join(root, "mailboxes"));
        const lexicalTmp = resolve(join(root, "tmp"));
        ensureDir(lexicalDir);
        ensureDir(lexicalTmp);
        this.dir = physicalPathOf(lexicalDir);
        this.tmpDir = physicalPathOf(lexicalTmp);
    }
    boxPath(scope, handle) {
        return join(this.dir, sanitizeScope(scope), `${sanitizePathComponent(handle).toLowerCase()}.jsonl`);
    }
    lockKey(scope, handle) {
        return canonicalStoreKey(this.boxPath(scope, handle));
    }
    withBox(key, path, fn) {
        return withPathLock(key, async () => {
            const release = await acquireMailboxBoxLock(`${path}.lock`, this.lockWaitMs);
            let result;
            try {
                result = await fn();
            }
            catch (err) {
                try {
                    release();
                }
                catch {
                }
                throw err;
            }
            release();
            return result;
        });
    }
    reconcile(b) {
        const now = diskMarkOf(b.path);
        if (sameDiskMark(now, b.mark))
            return;
        if (now === undefined || b.mark === undefined || now.ino !== b.mark.ino)
            b.log.closeForSwap();
        const events = readJsonlRecords(b.path, this.discloseCorrupt, MAILBOX_EVENT_JUDGE);
        b.messages = [];
        b.nextSeq = 1;
        delete b.lease;
        for (const ev of events)
            applyEvent(b, ev);
        b.events = Math.max(0, events.length - (b.messages.length + 1 + (b.lease !== undefined ? 1 : 0)));
        b.mark = now;
    }
    load(scope, handle) {
        const path = this.boxPath(scope, handle);
        const key = canonicalStoreKey(path);
        let b = sharedBoxes.get(key);
        if (b) {
            if (this.touched.get(key) !== b) {
                b.refs++;
                this.touched.set(key, b);
            }
            this.reconcile(b);
            return b;
        }
        ensureDir(join(this.dir, sanitizeScope(scope)));
        const events = readJsonlRecords(path, this.discloseCorrupt, MAILBOX_EVENT_JUDGE);
        const state = { messages: [], nextSeq: 1, log: new AppendLog(path), events: events.length, refs: 1, path, mark: undefined };
        for (const ev of events)
            applyEvent(state, ev);
        state.mark = diskMarkOf(path);
        sharedBoxes.set(key, state);
        this.touched.set(key, state);
        return state;
    }
    commit(b, path, ev) {
        b.log.append(ev, this.fsyncEnabled);
        if (MAILBOX_EVENT_JUDGE.accepts(ev))
            applyEvent(b, ev);
        b.events++;
        if (b.events >= this.compactEvery) {
            try {
                this.compact(b, path);
            }
            catch {
            }
        }
        b.mark = markAfterWrite(path);
    }
    compact(b, path) {
        const lines = [];
        lines.push(JSON.stringify({ t: "lease", owner: "", expiresAt: 0, maxSeq: b.nextSeq - 1 }));
        for (const m of b.messages)
            lines.push(JSON.stringify({ t: "append", m }));
        if (b.lease !== undefined)
            lines.push(JSON.stringify({ t: "lease", ...b.lease }));
        b.log.closeForSwap();
        atomicWriteFile(this.tmpDir, path, lines.length ? `${lines.join("\n")}\n` : "");
        b.events = 0;
        b.mark = markAfterWrite(path);
    }
    async append(scope, handle, msg) {
        if (scope === undefined || scope === "")
            throw new Error("MailboxStore.append: refusing a message without a scope");
        const peerMeta = readMailboxPeerMeta(msg.peerMeta);
        return this.withBox(this.lockKey(scope, handle), this.boxPath(scope, handle), () => {
            const b = this.load(scope, handle);
            const seq = b.nextSeq;
            const m = {
                seq,
                ...(msg.from !== undefined ? { from: msg.from } : {}),
                content: msg.content,
                sentAt: msg.sentAt,
                ...(msg.hopChain !== undefined ? { hopChain: [...msg.hopChain] } : {}),
                ...(peerMeta !== undefined ? { peerMeta } : {}),
            };
            this.commit(b, this.boxPath(scope, handle), { t: "append", m });
            return seq;
        });
    }
    async claimLease(scope, handle, owner, ttlMs, now = Date.now()) {
        return this.withBox(this.lockKey(scope, handle), this.boxPath(scope, handle), () => {
            const b = this.load(scope, handle);
            if (b.messages.length === 0)
                return null;
            if (b.lease !== undefined && b.lease.expiresAt > now && b.lease.owner !== owner)
                return null;
            const maxSeq = b.messages[b.messages.length - 1].seq;
            this.commit(b, this.boxPath(scope, handle), { t: "lease", owner, expiresAt: now + ttlMs, maxSeq });
            return { messages: b.messages.map((m) => detachMailboxMessage(m)), maxSeq };
        });
    }
    async ack(scope, handle, owner, upToSeq) {
        return this.withBox(this.lockKey(scope, handle), this.boxPath(scope, handle), () => {
            const b = this.load(scope, handle);
            if (b.lease === undefined || b.lease.owner !== owner)
                return { acked: false };
            this.commit(b, this.boxPath(scope, handle), { t: "ack", upToSeq });
            return { acked: true };
        });
    }
    async releaseLease(scope, handle, owner) {
        return this.withBox(this.lockKey(scope, handle), this.boxPath(scope, handle), () => {
            const b = this.load(scope, handle);
            if (b.lease?.owner === owner)
                this.commit(b, this.boxPath(scope, handle), { t: "release", owner });
        });
    }
    async peekCount(scope, handle) {
        const path = this.boxPath(scope, handle);
        const loaded = sharedBoxes.get(canonicalStoreKey(path));
        if (loaded !== undefined && sameDiskMark(diskMarkOf(path), loaded.mark)) {
            return withPathLock(this.lockKey(scope, handle), () => loaded.messages.length);
        }
        if (!existsSync(path))
            return 0;
        return withPathLock(this.lockKey(scope, handle), () => {
            const b = { messages: [], nextSeq: 1 };
            for (const ev of readJsonlRecords(path, this.discloseCorrupt, MAILBOX_EVENT_JUDGE))
                applyEvent(b, ev);
            return b.messages.length;
        });
    }
    async drop(scope, handle) {
        const path = this.boxPath(scope, handle);
        const key = canonicalStoreKey(path);
        return this.withBox(key, path, () => {
            const b = sharedBoxes.get(key);
            if (b) {
                b.log.close();
                sharedBoxes.delete(key);
            }
            this.touched.delete(key);
            atomicWriteFile(this.tmpDir, path, "");
        });
    }
    async reap(scope, now, opts) {
        assertRetentionPolicy("MailboxStore.reap", opts);
        sweepLockResidue(join(this.dir, sanitizeScope(scope)));
        if (opts?.maxAgeMs === undefined)
            return 0;
        let dropped = 0;
        const scopeDir = join(this.dir, sanitizeScope(scope));
        const prefix = scopeDir + sep;
        const maxAgeMs = opts.maxAgeMs;
        for (const [key, b] of [...sharedBoxes]) {
            if (!b.path.startsWith(prefix))
                continue;
            if (sameDiskMark(diskMarkOf(b.path), b.mark)) {
                const seen = newestSentAt(b.messages);
                if (seen === undefined || seen >= now - maxAgeMs)
                    continue;
            }
            const swept = await this.withBox(key, b.path, () => {
                const live = sharedBoxes.get(key);
                if (live === undefined || live !== b)
                    return false;
                this.reconcile(b);
                const newest = newestSentAt(b.messages);
                if (newest === undefined || newest >= now - maxAgeMs)
                    return false;
                b.messages = [];
                delete b.lease;
                this.compact(b, b.path);
                return true;
            });
            if (swept)
                dropped++;
        }
        for (const name of listBoxFiles(scopeDir)) {
            const path = join(scopeDir, name);
            const key = canonicalStoreKey(path);
            if (sharedBoxes.has(key))
                continue;
            const swept = await this.withBox(key, path, () => {
                if (sharedBoxes.has(key))
                    return false;
                const b = { messages: [], nextSeq: 1 };
                for (const ev of readJsonlRecords(path, this.discloseCorrupt, MAILBOX_EVENT_JUDGE))
                    applyEvent(b, ev);
                const newest = newestSentAt(b.messages);
                if (newest === undefined || newest >= now - maxAgeMs)
                    return false;
                atomicWriteFile(this.tmpDir, path, `${JSON.stringify({ t: "lease", owner: "", expiresAt: 0, maxSeq: b.nextSeq - 1 })}\n`);
                return true;
            });
            if (swept)
                dropped++;
        }
        return dropped;
    }
    close() {
        for (const [path, mine] of this.touched) {
            const b = sharedBoxes.get(path);
            if (!b || b !== mine)
                continue;
            b.refs--;
            if (b.refs <= 0) {
                b.log.close();
                sharedBoxes.delete(path);
            }
        }
        this.touched.clear();
    }
}
const num = (v) => typeof v === "number" && Number.isFinite(v);
const MAILBOX_EVENT_JUDGE = {
    accepts: (r) => {
        const ev = r;
        if (ev.t === "append")
            return ev.m !== null && typeof ev.m === "object" && num(ev.m.seq);
        if (ev.t === "lease")
            return typeof ev.owner === "string" && num(ev.maxSeq) && num(ev.expiresAt);
        if (ev.t === "ack")
            return num(ev.upToSeq);
        return ev.t === "release" && typeof ev.owner === "string";
    },
    what: "a mailbox record is not one of this box's events — an unknown tag, or a known tag whose payload is missing",
};
function applyEvent(b, ev) {
    if (ev.t === "append") {
        b.messages.push(ev.m);
        if (ev.m.seq >= b.nextSeq)
            b.nextSeq = ev.m.seq + 1;
    }
    else if (ev.t === "lease") {
        if (ev.maxSeq >= b.nextSeq)
            b.nextSeq = ev.maxSeq + 1;
        if (ev.owner !== "")
            b.lease = { owner: ev.owner, expiresAt: ev.expiresAt, maxSeq: ev.maxSeq };
    }
    else if (ev.t === "ack") {
        b.messages = b.messages.filter((m) => m.seq > ev.upToSeq);
        if (b.lease !== undefined && b.lease.maxSeq <= ev.upToSeq)
            b.lease = undefined;
    }
    else if (b.lease?.owner === ev.owner) {
        b.lease = undefined;
    }
}