@sema-agent/core/dist/stores/file/workflow-journal-store.js

Stateless, task-oriented AI agent core

import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, unlinkSync } from "node:fs";
import { createHash } from "node:crypto";
import { join } from "node:path";
import { callKeyOrdinal } from "../../core/workflow-journal-store.js";
import { AppendLog, readJsonlRecords } from "./fs-atomic.js";
import { guardCorruptSink } from "./corrupt-read-sink.js";
import { canonicalStoreKey, sanitizePathComponent, writeThenLink } from "./fs-atomic.js";
import { assertAdoptionBootGate } from "./adoption/marker.js";
import { assertJournalEntryFits } from "../../core/workflow-journal-store.js";
export { MAX_JOURNAL_RESULT_BYTES, oversizeJournalResult } from "../../core/workflow-journal-store.js";
function entryOf(rec) {
    if (rec.parked !== undefined)
        return { callKey: rec.callKey, parked: rec.parked };
    return { callKey: rec.callKey, result: rec.result };
}
export const RESUME_CLAIM_TTL_MS = 60 * 60 * 1000;
export const RESUME_CLAIM_RELEASE_GRACE_MS = 5_000;
export const RESUME_CLAIM_SWEEP_INTERVAL_MS = 60_000;
export const RESUME_CLAIM_REAP_GRACE_MS = 5 * 60_000;
const sharedJournalDirs = new Map();
const MAX_OPEN_JOURNAL_LOGS = 64;
function isJournalLine(r) {
    const rec = r;
    return typeof rec.callKey === "string" && typeof rec.scope === "string" && Number.isFinite(rec.ordinal);
}
export class FileWorkflowJournalStore {
    fsyncEnabled;
    dir;
    claimsDir;
    shared;
    sharedKey;
    closed = false;
    lastClaimSweepAt = 0;
    get logs() {
        return this.shared.logs;
    }
    get scopes() {
        return this.shared.scopes;
    }
    discloseCorrupt;
    constructor(root, fsyncEnabled = true, opts) {
        this.fsyncEnabled = fsyncEnabled;
        this.discloseCorrupt = guardCorruptSink(opts?.onCorruptRead);
        assertAdoptionBootGate(root, "FileWorkflowJournalStore");
        this.dir = join(root, "workflow-journal");
        mkdirSync(this.dir, { recursive: true, mode: 0o700 });
        this.claimsDir = join(this.dir, "claims");
        this.sharedKey = canonicalStoreKey(this.dir);
        const existing = sharedJournalDirs.get(this.sharedKey);
        if (existing !== undefined) {
            existing.refs += 1;
            this.shared = existing;
        }
        else {
            this.shared = { logs: new Map(), scopes: new Map(), refs: 1 };
            sharedJournalDirs.set(this.sharedKey, this.shared);
        }
    }
    pathFor(runId) {
        return join(this.dir, `${sanitizePathComponent(runId)}.jsonl`);
    }
    locator(runId) {
        try {
            return this.pathFor(runId);
        }
        catch {
            return undefined;
        }
    }
    replay(runId) {
        let path;
        try {
            path = this.pathFor(runId);
        }
        catch {
            return undefined;
        }
        if (!existsSync(path))
            return undefined;
        let scope;
        const byOrdinal = new Map();
        const judge = {
            accepts: (r) => {
                if (!isJournalLine(r))
                    return false;
                scope ??= r.scope;
                return r.scope === scope;
            },
            what: "a journal line carries no callKey/scope pair, or a scope other than its run's first line",
        };
        for (const rec of readJsonlRecords(path, this.discloseCorrupt, judge))
            byOrdinal.set(rec.ordinal, entryOf(rec));
        return scope === undefined ? undefined : { scope, byOrdinal };
    }
    async append(runId, scope, entry) {
        assertJournalEntryFits(entry);
        const committed = this.scopes.get(runId) ?? this.replay(runId)?.scope;
        if (committed !== undefined && committed !== scope) {
            throw new Error(`workflow journal: runId ${runId} already recorded under a different scope`);
        }
        this.scopes.set(runId, committed ?? scope);
        let log = this.logs.get(runId);
        if (log === undefined) {
            log = new AppendLog(this.pathFor(runId));
        }
        else {
            this.logs.delete(runId);
        }
        this.logs.set(runId, log);
        while (this.logs.size > MAX_OPEN_JOURNAL_LOGS) {
            const coldest = this.logs.keys().next().value;
            if (coldest === undefined || coldest === runId)
                break;
            const stale = this.logs.get(coldest);
            this.logs.delete(coldest);
            try {
                stale?.close();
            }
            catch {
            }
        }
        const line = entry.parked !== undefined
            ? { scope, ordinal: callKeyOrdinal(entry.callKey), callKey: entry.callKey, parked: entry.parked }
            : { scope, ordinal: callKeyOrdinal(entry.callKey), callKey: entry.callKey, result: entry.result };
        log.append(line, this.fsyncEnabled);
    }
    async load(runId, scope) {
        const rec = this.replay(runId);
        if (!rec || rec.scope !== scope)
            return [];
        return [...rec.byOrdinal.entries()].sort((a, b) => a[0] - b[0]).map(([, e]) => e);
    }
    claimPathFor(sourceRunId, scope) {
        const scopeTag = createHash("sha256").update(scope).digest("hex").slice(0, 16);
        return join(this.claimsDir, `${sanitizePathComponent(sourceRunId)}.${scopeTag}.json`);
    }
    readClaimRaw(path) {
        let raw;
        try {
            raw = readFileSync(path, "utf8");
        }
        catch {
            return undefined;
        }
        try {
            const rec = JSON.parse(raw);
            if (typeof rec.holder !== "string" || typeof rec.expiresAt !== "number" || !Number.isFinite(rec.expiresAt)) {
                return undefined;
            }
            return { raw, rec };
        }
        catch {
            return undefined;
        }
    }
    unlinkIfUnchanged(path, expectedRaw) {
        try {
            if (expectedRaw !== undefined && readFileSync(path, "utf8") !== expectedRaw)
                return;
            unlinkSync(path);
        }
        catch {
        }
    }
    sweepExpiredClaims() {
        const now = Date.now();
        if (now - this.lastClaimSweepAt < RESUME_CLAIM_SWEEP_INTERVAL_MS)
            return;
        this.lastClaimSweepAt = now;
        let names;
        try {
            names = readdirSync(this.claimsDir);
        }
        catch {
            return;
        }
        for (const name of names) {
            const path = join(this.claimsDir, name);
            try {
                const existing = this.readClaimRaw(path);
                if (existing === undefined) {
                    if (now - statSync(path).mtimeMs > RESUME_CLAIM_TTL_MS)
                        unlinkSync(path);
                    continue;
                }
                if (existing.rec.expiresAt + RESUME_CLAIM_REAP_GRACE_MS > now)
                    continue;
                this.unlinkIfUnchanged(path, existing.raw);
            }
            catch {
            }
        }
    }
    async resumeClaim(input) {
        const { sourceRunId, newRunId, scope } = input;
        let path;
        try {
            path = this.claimPathFor(sourceRunId, scope);
        }
        catch {
            return { granted: true };
        }
        mkdirSync(this.claimsDir, { recursive: true, mode: 0o700 });
        this.sweepExpiredClaims();
        const record = { sourceRunId, holder: newRunId, expiresAt: Date.now() + RESUME_CLAIM_TTL_MS };
        const payload = JSON.stringify(record);
        for (let attempt = 0; attempt < 2; attempt++) {
            try {
                writeThenLink(path, payload);
                return { granted: true };
            }
            catch (err) {
                if (err.code !== "EEXIST")
                    throw err;
            }
            const existing = this.readClaimRaw(path);
            if (existing !== undefined && existing.rec.holder === newRunId) {
                return { granted: true };
            }
            if (existing === undefined || existing.rec.expiresAt <= Date.now()) {
                this.unlinkIfUnchanged(path, existing?.raw);
                continue;
            }
            return { granted: false, holder: existing.rec.holder };
        }
        const winner = this.readClaimRaw(path);
        if (winner !== undefined && winner.rec.holder === newRunId)
            return { granted: true };
        return { granted: false, holder: winner?.rec.holder };
    }
    async releaseResumeClaim(input) {
        let path;
        try {
            path = this.claimPathFor(input.sourceRunId, input.scope);
        }
        catch {
            return;
        }
        const existing = this.readClaimRaw(path);
        if (existing === undefined || existing.rec.holder !== input.newRunId)
            return;
        if (existing.rec.expiresAt - Date.now() <= RESUME_CLAIM_RELEASE_GRACE_MS)
            return;
        this.unlinkIfUnchanged(path, existing.raw);
    }
    async deleteByRun(runId) {
        const log = this.logs.get(runId);
        if (log !== undefined) {
            try {
                log.close();
            }
            catch { }
            this.logs.delete(runId);
        }
        this.scopes.delete(runId);
        const path = this.pathFor(runId);
        if (!existsSync(path))
            return 0;
        unlinkSync(path);
        return 1;
    }
    dispose() {
        if (this.closed)
            return;
        this.closed = true;
        this.shared.refs -= 1;
        if (this.shared.refs > 0)
            return;
        for (const log of this.shared.logs.values()) {
            try {
                log.close();
            }
            catch { }
        }
        this.shared.logs.clear();
        this.shared.scopes.clear();
        sharedJournalDirs.delete(this.sharedKey);
    }
}