@sema-agent/core/dist/tools/artifact/local-stub.js

Stateless, task-oriented AI agent core

import { randomBytes } from "node:crypto";
import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
import { dirname, join, resolve, sep } from "node:path";
import { ARTIFACT_HOST_FAILURE_CODES, } from "../../core/artifact-host.js";
import { physicalPathOf } from "../../core/physical-path.js";
import { acquireStoreDirLock } from "../../stores/file/fs-atomic.js";
import { normalizePublishedPath } from "./artifact-tool.js";
export const EVAL_STUB_URL_PREFIX = "eval-stub://artifact/";
export const LOCAL_STUB_MAX_FILE_BYTES = 16 * 1024 * 1024;
export const LOCAL_STUB_MAX_TOTAL_BYTES = 64 * 1024 * 1024;
const LEDGER_FILE = "ledger.json";
const TMP_DIR = ".tmp";
const LOCK_FILE = "LOCK";
const RESERVED_NAMES = new Set([LEDGER_FILE, TMP_DIR, LOCK_FILE].map((n) => n.toLowerCase()));
const SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
const ATTEMPT_WINDOW = 512;
const emptyLedger = () => ({ v: 1, artifacts: {}, attempts: {} });
const isRow = (x, key) => {
    if (x === null || typeof x !== "object")
        return false;
    const r = x;
    if (typeof r.slug !== "string" || r.slug !== key || !SLUG_RE.test(r.slug) || r.slug.startsWith(".") || RESERVED_NAMES.has(r.slug.toLowerCase()))
        return false;
    if (typeof r.url !== "string" || !Number.isSafeInteger(r.version) || r.version < 1 || !Array.isArray(r.files))
        return false;
    return r.files.every((f) => f !== null && typeof f === "object" && typeof f.path === "string" && normalizePublishedPath(f.path).ok && Number.isSafeInteger(f.bytes));
};
const isAttempt = (x) => {
    if (x === null || typeof x !== "object" || Array.isArray(x))
        return false;
    const a = x;
    if ("ref" in a) {
        const r = a.ref;
        return r !== null && typeof r === "object" && typeof r.url === "string" && typeof r.slug === "string" && typeof r.version === "string";
    }
    if (typeof a.code !== "string" || !ARTIFACT_HOST_FAILURE_CODES.includes(a.code))
        return false;
    return a.code === "conflict" ? typeof a.slug === "string" && typeof a.currentVersion === "string" : typeof a.message === "string";
};
function parseLedger(text, path) {
    let parsed;
    try {
        parsed = JSON.parse(text);
    }
    catch {
        throw new Error(`artifact stub ledger is corrupt (not JSON): ${path}`);
    }
    const l = parsed;
    if (l === null || typeof l !== "object" || l.v !== 1 || l.artifacts === null || typeof l.artifacts !== "object" || l.attempts === null || typeof l.attempts !== "object") {
        throw new Error(`artifact stub ledger is corrupt (unexpected shape): ${path}`);
    }
    for (const [key, row] of Object.entries(l.artifacts))
        if (!isRow(row, key))
            throw new Error(`artifact stub ledger is corrupt (bad artifact row): ${path}`);
    for (const attempt of Object.values(l.attempts))
        if (!isAttempt(attempt))
            throw new Error(`artifact stub ledger is corrupt (bad attempt record): ${path}`);
    return { v: 1, artifacts: { ...l.artifacts }, attempts: { ...l.attempts } };
}
function fsyncDirBestEffort(dir) {
    try {
        const fd = openSync(dir, "r");
        try {
            fsyncSync(fd);
        }
        finally {
            closeSync(fd);
        }
    }
    catch {
    }
}
function fsyncTreeBestEffort(dir) {
    for (const entry of readdirSync(dir, { withFileTypes: true }))
        if (entry.isDirectory())
            fsyncTreeBestEffort(join(dir, entry.name));
    fsyncDirBestEffort(dir);
}
function writeFsync(path, bytes) {
    const fd = openSync(path, "w", 0o600);
    try {
        writeFileSync(fd, bytes);
        fsyncSync(fd);
    }
    finally {
        closeSync(fd);
    }
}
function slugBase(fileName) {
    const dot = fileName.lastIndexOf(".");
    const stem = (dot > 0 ? fileName.slice(0, dot) : fileName).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40);
    return stem.length > 0 ? stem : "artifact";
}
export class LocalArtifactStub {
    dir;
    #mint = new Map();
    #chain = Promise.resolve();
    constructor(opts) {
        if (typeof opts?.dir !== "string" || opts.dir.length === 0)
            throw new Error("localArtifactStub: `dir` is required");
        this.dir = resolve(opts.dir);
    }
    capabilities() {
        return { plane: "local-stub", actions: ["publish"], commit: "throw_means_uncommitted", defaultVisibility: "private", maxFileBytes: LOCAL_STUB_MAX_FILE_BYTES, maxTotalBytes: LOCAL_STUB_MAX_TOTAL_BYTES, multiFile: false, storageDir: this.dir };
    }
    onMount(runId) {
        this.#mint.set(runId, new Set());
    }
    ownMinted(runId) {
        return new Set(this.#mint.get(runId) ?? []);
    }
    publish(req, ctx) {
        const run = this.#chain.then(() => this.#publishLocked(req, ctx));
        this.#chain = run.then(() => undefined, () => undefined);
        return run;
    }
    #ledgerPath() {
        return join(this.dir, LEDGER_FILE);
    }
    #loadLedger() {
        const p = this.#ledgerPath();
        if (!existsSync(p))
            return emptyLedger();
        return parseLedger(readFileSync(p, "utf8"), p);
    }
    #versionDir(slug, version) {
        return join(this.dir, slug, `v${version}`);
    }
    #snapshotOf(row) {
        const base = this.#versionDir(row.slug, row.version);
        const realBase = physicalPathOf(base);
        const files = row.files.map((f) => {
            const target = physicalPathOf(resolve(base, ...f.path.split("/")));
            if (!target.startsWith(realBase + sep))
                throw new Error(`artifact stub ledger is corrupt (a file resolves outside its version directory): ${this.#ledgerPath()}`);
            return { path: f.path, content: new Uint8Array(readFileSync(target)), ...(f.mediaType !== undefined ? { mediaType: f.mediaType } : {}) };
        });
        return {
            files,
            ...(row.title !== undefined ? { title: row.title } : {}),
            ...(row.description !== undefined ? { description: row.description } : {}),
            ...(row.favicon !== undefined ? { favicon: row.favicon } : {}),
            ...(row.label !== undefined ? { label: row.label } : {}),
        };
    }
    #mintSlug(ledger, requested, firstFile) {
        const fold = requested?.toLowerCase();
        if (requested !== undefined && fold !== undefined && SLUG_RE.test(requested) && !RESERVED_NAMES.has(fold) && !requested.startsWith(".") && !Object.keys(ledger.artifacts).some((s) => s.toLowerCase() === fold))
            return requested;
        const base = slugBase(firstFile);
        const taken = new Set(Object.keys(ledger.artifacts).map((s) => s.toLowerCase()));
        for (;;) {
            const slug = `${base}-${randomBytes(3).toString("hex")}`;
            if (!taken.has(slug))
                return slug;
        }
    }
    #commitLedger(next) {
        const ledgerTmp = join(this.dir, `${LEDGER_FILE}.${randomBytes(4).toString("hex")}.tmp`);
        try {
            const text = JSON.stringify(next, null, 2);
            writeFsync(ledgerTmp, text);
            const verified = parseLedger(readFileSync(ledgerTmp, "utf8"), ledgerTmp);
            if (JSON.stringify(verified) !== JSON.stringify(next))
                throw new Error("artifact stub ledger did not read back as written");
            renameSync(ledgerTmp, this.#ledgerPath());
        }
        catch (err) {
            try {
                rmSync(ledgerTmp, { force: true });
            }
            catch {
            }
            throw err;
        }
        fsyncDirBestEffort(this.dir);
    }
    #withAttempt(ledger, attemptId, attempt) {
        const entries = Object.entries(ledger.attempts).filter(([k]) => k !== attemptId);
        while (entries.length >= ATTEMPT_WINDOW)
            entries.shift();
        entries.push([attemptId, attempt]);
        return Object.fromEntries(entries);
    }
    #replayOf(ledger, attempt) {
        if ("ref" in attempt)
            return { ok: true, ref: attempt.ref };
        if (attempt.code === "conflict") {
            const row = ledger.artifacts[attempt.slug];
            return { ok: false, code: "conflict", currentVersion: row !== undefined ? String(row.version) : attempt.currentVersion, current: row !== undefined ? this.#snapshotOf(row) : "reread" };
        }
        return { ok: false, code: attempt.code, message: attempt.message };
    }
    async #publishLocked(req, ctx) {
        mkdirSync(this.dir, { recursive: true, mode: 0o700 });
        const lease = acquireStoreDirLock(this.dir, { label: "artifact stub", fileName: LOCK_FILE });
        try {
            const ledger = this.#loadLedger();
            const replay = ledger.attempts[ctx.attemptId];
            if (replay !== undefined)
                return this.#replayOf(ledger, replay);
            const decided = this.#decide(ledger, req, ctx);
            if (!decided.ok) {
                const attempt = decided.outcome.code === "conflict" ? { code: "conflict", slug: decided.slug ?? "", currentVersion: decided.outcome.currentVersion } : { code: decided.outcome.code, message: decided.outcome.message };
                this.#commitLedger({ ...ledger, attempts: this.#withAttempt(ledger, ctx.attemptId, attempt) });
                return decided.outcome;
            }
            const { slug, version, existing, files, mint } = decided;
            const versionDir = this.#versionDir(slug, version);
            const tmpDir = join(this.dir, TMP_DIR, `${slug}-v${version}-${randomBytes(4).toString("hex")}`);
            let staged = false;
            try {
                if (existsSync(versionDir))
                    rmSync(versionDir, { recursive: true, force: true });
                mkdirSync(tmpDir, { recursive: true, mode: 0o700 });
                for (const f of files) {
                    const target = join(tmpDir, ...f.path.split("/"));
                    mkdirSync(dirname(target), { recursive: true, mode: 0o700 });
                    writeFsync(target, f.content);
                }
                fsyncTreeBestEffort(tmpDir);
                mkdirSync(dirname(versionDir), { recursive: true, mode: 0o700 });
                renameSync(tmpDir, versionDir);
                staged = true;
                fsyncDirBestEffort(dirname(versionDir));
                const now = new Date().toISOString();
                const url = `${EVAL_STUB_URL_PREFIX}${slug}`;
                const row = {
                    slug,
                    url,
                    version,
                    ...(req.title !== undefined ? { title: req.title } : existing?.title !== undefined ? { title: existing.title } : {}),
                    ...(req.description !== undefined ? { description: req.description } : existing?.description !== undefined ? { description: existing.description } : {}),
                    ...(req.favicon !== undefined ? { favicon: req.favicon } : existing?.favicon !== undefined ? { favicon: existing.favicon } : {}),
                    ...(req.label !== undefined ? { label: req.label } : {}),
                    files: files.map((f) => ({ path: f.path, ...(f.mediaType !== undefined ? { mediaType: f.mediaType } : {}), bytes: f.content.byteLength })),
                    createdAt: existing?.createdAt ?? now,
                    updatedAt: now,
                };
                const ref = { url, slug, version: String(version), ...(row.title !== undefined ? { title: row.title } : {}) };
                this.#commitLedger({ v: 1, artifacts: { ...ledger.artifacts, [slug]: row }, attempts: this.#withAttempt(ledger, ctx.attemptId, { ref }) });
                mint.add(slug);
                return { ok: true, ref };
            }
            catch (err) {
                for (const orphan of [tmpDir, ...(staged ? [versionDir] : [])]) {
                    try {
                        rmSync(orphan, { recursive: true, force: true });
                    }
                    catch {
                    }
                }
                throw err;
            }
        }
        finally {
            lease.release();
        }
    }
    #decide(ledger, req, ctx) {
        const refuse = (code, message, slug) => ({ ok: false, outcome: { ok: false, code, message }, ...(slug !== undefined ? { slug } : {}) });
        if (req.removeFiles !== undefined && req.removeFiles.length > 0)
            return refuse("refused", "the local stub does not apply removeFiles");
        if (req.files.length === 0)
            return refuse("refused", "a publish needs at least one file");
        const seen = new Set();
        const files = [];
        let total = 0;
        for (const f of req.files) {
            const norm = normalizePublishedPath(f.path);
            if (!norm.ok)
                return refuse("refused", `published path ${JSON.stringify(f.path).slice(0, 120)} ${norm.reason}`);
            const fold = norm.path.toLowerCase();
            if (seen.has(fold))
                return refuse("refused", `published path ${JSON.stringify(norm.path).slice(0, 120)} collides with another entry`);
            seen.add(fold);
            total += f.content.byteLength;
            if (f.content.byteLength > LOCAL_STUB_MAX_FILE_BYTES)
                return refuse("payload_too_large", `${norm.path.slice(0, 120)} is ${f.content.byteLength} bytes; the limit is ${LOCAL_STUB_MAX_FILE_BYTES} bytes per file`);
            files.push({ path: norm.path, content: f.content, ...(f.mediaType !== undefined ? { mediaType: f.mediaType } : {}) });
        }
        if (total > LOCAL_STUB_MAX_TOTAL_BYTES)
            return refuse("payload_too_large", `the files total ${total} bytes; the limit is ${LOCAL_STUB_MAX_TOTAL_BYTES} bytes per version`);
        const mint = this.#mint.get(ctx.runId) ?? new Set();
        this.#mint.set(ctx.runId, mint);
        if (req.url !== undefined) {
            if (!req.url.startsWith(EVAL_STUB_URL_PREFIX))
                return refuse("url_foreign", "not an eval-stub:// link this run published");
            const slug = req.url.slice(EVAL_STUB_URL_PREFIX.length);
            if (!mint.has(slug))
                return refuse("url_foreign", "this run did not publish that artifact", slug);
            const existing = ledger.artifacts[slug];
            if (existing === undefined)
                return refuse("not_found", "the artifact is no longer in the ledger", slug);
            if (req.baseVersion !== undefined && req.baseVersion !== String(existing.version) && req.force !== true) {
                return { ok: false, outcome: { ok: false, code: "conflict", currentVersion: String(existing.version), current: this.#snapshotOf(existing) }, slug };
            }
            return { ok: true, slug, version: existing.version + 1, existing, files, mint };
        }
        return { ok: true, slug: this.#mintSlug(ledger, req.slug, files[0].path), version: 1, existing: undefined, files, mint };
    }
    rows() {
        return Object.values(this.#loadLedger().artifacts).map((r) => ({ url: r.url, slug: r.slug, version: String(r.version), ...(r.title !== undefined ? { title: r.title } : {}) }));
    }
    versionFiles(slug, version) {
        const dir = this.#versionDir(slug, version);
        return existsSync(dir) ? readdirSync(dir, { recursive: true }).map(String).sort() : [];
    }
}
export function localArtifactStub(opts) {
    return new LocalArtifactStub(opts);
}