@sema-agent/core/dist/core/runner/prepare-suspend-saga.js

Stateless, task-oriented AI agent core

import { defaultTaskRegistry } from "../task-registry.js";
import { isRemoteExecutionEnv, isSuspendable, RemoteExecutionError } from "../remote-env.js";
import { constraintChainDigest, constraintChainEntryOfLayer } from "../tool-policy.js";
import { raceSettlementAgainstSignal } from "./abort-race.js";
import { placementValueOrAbsent } from "./checkpoint-scope.js";
import { serializeAnnouncedListings } from "./announce-once-ledger.js";
import { remoteEnvFailureNote, restoreWorkspaceWithRetry } from "./remote-env-retry.js";
import { thrownText } from "../thrown-value.js";
const PARK_COMPENSATION_TIMEOUT_MS = 30_000;
export async function compensateUnparkedPause(remoteEnv, snapshotId, io) {
    const boundCtl = new AbortController();
    const boundTimer = setTimeout(() => boundCtl.abort(), io.boundMs);
    const bound = boundCtl.signal;
    let phase = "resumeVM";
    let restoreAttempts = 1;
    let vmRunning = false;
    const work = (async () => {
        const { outcome: back, attempts: backAttempts } = await restoreWorkspaceWithRetry(remoteEnv, snapshotId, { abortSignal: bound }, (attempt) => (restoreAttempts = attempt));
        if (!back.ok) {
            io.noteFailure(remoteEnvFailureNote("resumeVM", back.error, backAttempts));
            io.disclose(back.error);
            return { ok: false, reason: `resumeVM failed (${back.error.code}) while compensating an unparked pause` };
        }
        vmRunning = true;
        if (bound.aborted) {
            return { ok: false, reason: "resumeVM settled only after the decision bound" };
        }
        phase = "postResumeInit";
        const init = await remoteEnv.postResumeInit({ abortSignal: bound });
        if (!init.ok) {
            io.noteFailure(remoteEnvFailureNote("postResumeInit", init.error, 1));
            io.disclose(init.error);
            return { ok: false, reason: `postResumeInit failed (${init.error.code}) while compensating an unparked pause` };
        }
        return { ok: true };
    })();
    const raced = await raceSettlementAgainstSignal(work, bound);
    clearTimeout(boundTimer);
    if (raced.tag === "value")
        return raced.value;
    if (raced.tag === "threw") {
        io.noteFailure(remoteEnvFailureNote(phase, raced.error instanceof RemoteExecutionError
            ? raced.error
            : new RemoteExecutionError("unknown", `the pause compensation's ${phase} call threw: ${thrownText(raced.error)}`), phase === "resumeVM" ? restoreAttempts : 1));
        io.disclose(raced.error);
        return {
            ok: false,
            reason: "the pause compensation threw (adapter contract violation; the deployment's error face carries the exception)",
        };
    }
    const timedOut = new RemoteExecutionError("timeout", `the pause compensation's ${phase} call did not settle within the ${io.boundMs}ms decision bound`);
    io.noteFailure(remoteEnvFailureNote(phase, timedOut, phase === "resumeVM" ? restoreAttempts : 1));
    io.disclose(timedOut);
    const destroyOwnerlessLate = () => {
        io.disclose(new Error(`park compensation settled after its ${io.boundMs}ms decision bound: the VM is running with no run ` +
            `left to own it — destroying the env (an ownerless running VM must not linger; the run already took ` +
            `the fail-closed arm when the bound fired)`));
        void Promise.resolve()
            .then(() => remoteEnv.destroy())
            .catch((destroyErr) => io.disclose(destroyErr));
    };
    void work.then((late) => {
        if (vmRunning) {
            destroyOwnerlessLate();
            return;
        }
        void late;
    }, (lateErr) => {
        io.disclose(lateErr);
        if (vmRunning)
            destroyOwnerlessLate();
    });
    return {
        ok: false,
        reason: `the pause compensation did not settle within ${io.boundMs}ms — treated as failed (a late-restored VM is destroyed; a late restore failure is disclosed)`,
    };
}
function stampPlacementRootSessionId(placementRootResolved) {
    return placementValueOrAbsent(placementRootResolved);
}
export function prepareSuspendSaga(input) {
    const { gateMachineryActive, abortController, harness, checkpointStore, deps, sessionId, hostTaskId, taskScope, liveSpendRef, resume, nestedStats, internals, activeTools, outputRef, readFileStateForCheckpoint, faceCheckpointSection, writeFaceCheckpointSection, skillGrantCheckpointSection, reminderMark, handsCwdRef, worktreeSessionRef, undeliveredContinuationRef, inheritedParentConstraints, seedInheritedGate, ancestorRulesNow, inheritedShellGate, permissionMode, inheritedAdmittedOrgScopes, ownOrgVerdictRef, orgGovernedProvenance, announcedListingsRef, gitStatusRef, hookIdentity, placementRootResolved, externalContentTargetActive, remoteEnvFailures, memoryEngineSession, suspendLoopRef, ownedEnv, incompleteSuspendAdapter } = input;
    if (!gateMachineryActive)
        return { saga: undefined };
    const inFlightSpendMicroUsd = () => {
        const own = liveSpendRef.get?.().costMicroUsd ?? 0;
        const seededNested = resume?.seed.nestedStats.costMicroUsd;
        const nestedDelta = nestedStats.costMicroUsd - (typeof seededNested === "number" && Number.isFinite(seededNested) ? seededNested : 0);
        return (Number.isFinite(own) ? own : 0) + Math.max(0, Number.isFinite(nestedDelta) ? nestedDelta : 0);
    };
    const repairBundleForCheckpoint = (parkedSpendMicroUsd) => {
        const carried = internals?.repairBundle !== undefined
            ? structuredClone(internals.repairBundle)
            : resume?.seed.repairBundle !== undefined
                ? structuredClone(resume.seed.repairBundle)
                : undefined;
        if (carried === undefined)
            return undefined;
        if (parkedSpendMicroUsd === undefined || !Number.isFinite(parkedSpendMicroUsd) || parkedSpendMicroUsd <= 0)
            return carried;
        const prior = typeof carried.spentMicroUsd === "number" && Number.isFinite(carried.spentMicroUsd) ? Math.max(0, carried.spentMicroUsd) : 0;
        carried.spentMicroUsd = prior + parkedSpendMicroUsd;
        return carried;
    };
    const screeningParkDisclosedRef = { done: false };
    const serializeCheckpointState = (workspaceHandle, parkedSpendMicroUsd) => ({
        activeTools: [...activeTools],
        outputRef: { value: outputRef.value, set: outputRef.set },
        nestedStats: { ...nestedStats },
        readFileState: readFileStateForCheckpoint ? [...readFileStateForCheckpoint.entries()] : undefined,
        readFace: faceCheckpointSection(),
        writeFace: writeFaceCheckpointSection(),
        skillGrants: skillGrantCheckpointSection(),
        reminderMark,
        repairBundle: repairBundleForCheckpoint(parkedSpendMicroUsd),
        workspaceHandle,
        handsCwd: handsCwdRef?.current,
        activeWorktree: worktreeSessionRef?.current ? { ...worktreeSessionRef.current } : undefined,
        runningBackgroundTasks: (() => {
            const live = defaultTaskRegistry
                .list({ owner: hostTaskId, scope: taskScope, sessionId })
                .filter((t) => t.status === "pending" || t.status === "running")
                .map((t) => ({ id: t.task_id, ...(t.description !== undefined ? { description: t.description } : {}) }));
            const owed = undeliveredContinuationRef.current.orphanTasks.filter((o) => !live.some((t) => t.id === o.id));
            const all = [...owed, ...live];
            return all.length > 0 ? all : undefined;
        })(),
        pendingSteerQueue: undefined,
        inheritedGate: (() => {
            const requiresParentConstraint = (inheritedParentConstraints?.length ?? 0) > 0 || seedInheritedGate?.requiresParentConstraint === true;
            const parentConstraintCount = (inheritedParentConstraints?.length ?? 0) > 0 ? inheritedParentConstraints.length : seedInheritedGate?.parentConstraintCount;
            const constraintChain = (inheritedParentConstraints?.length ?? 0) > 0
                ? inheritedParentConstraints.map((pc) => constraintChainEntryOfLayer(pc))
                : seedInheritedGate?.constraintChain;
            const constraintDigest = (inheritedParentConstraints?.length ?? 0) > 0
                ? constraintChainDigest(constraintChain)
                : seedInheritedGate?.constraintDigest;
            const declaredMode = permissionMode.declared;
            const modeRecorded = declaredMode !== "default" || permissionMode.requested !== undefined || seedInheritedGate?.permissionMode !== undefined;
            const ancestorRulesAtMint = ancestorRulesNow();
            return ancestorRulesAtMint !== undefined ||
                inheritedShellGate !== undefined ||
                modeRecorded ||
                inheritedAdmittedOrgScopes !== undefined ||
                ownOrgVerdictRef.current !== undefined ||
                orgGovernedProvenance ||
                requiresParentConstraint
                ? {
                    ...(ancestorRulesAtMint !== undefined ? { ancestorRules: structuredClone(ancestorRulesAtMint) } : {}),
                    ...(inheritedShellGate !== undefined ? { shellGate: inheritedShellGate } : {}),
                    ...(modeRecorded ? { permissionMode: declaredMode } : {}),
                    ...(permissionMode.prePlanMode !== undefined ? { prePlanMode: permissionMode.prePlanMode } : {}),
                    ...(inheritedAdmittedOrgScopes !== undefined ? { admittedOrgScopes: [...inheritedAdmittedOrgScopes] } : {}),
                    ...(ownOrgVerdictRef.current !== undefined
                        ? { ownAdmittedOrgScopes: [...ownOrgVerdictRef.current.scopes], ownAdmittedOrgWriteScope: ownOrgVerdictRef.current.writeScope }
                        : {}),
                    ...(orgGovernedProvenance ? { orgAdmissionGoverned: true } : {}),
                    requiresParentConstraint,
                    ...(requiresParentConstraint && parentConstraintCount !== undefined ? { parentConstraintCount } : {}),
                    ...(requiresParentConstraint && constraintChain !== undefined
                        ? { constraintChain: structuredClone(constraintChain), ...(constraintDigest !== undefined ? { constraintDigest } : {}) }
                        : {}),
                }
                : undefined;
        })(),
        announcedListings: serializeAnnouncedListings(announcedListingsRef),
        gitAnnouncement: gitStatusRef.announced !== undefined ? { ...gitStatusRef.announced } : undefined,
        delegationProvenance: internals?.delegationProvenance !== undefined ? { ...internals.delegationProvenance.ref.current } : undefined,
        isDelegatedChild: hookIdentity.isDelegatedChild ? true : undefined,
        placementRootSessionId: stampPlacementRootSessionId(placementRootResolved),
        externalContentTarget: externalContentTargetActive ? true : undefined,
    });
    const compensatePausedVM = (remoteEnv, snapshotId) => compensateUnparkedPause(remoteEnv, snapshotId, {
        boundMs: PARK_COMPENSATION_TIMEOUT_MS,
        noteFailure: (note) => remoteEnvFailures.push(note),
        disclose: (err) => {
            try {
                deps.onError?.(err, { phase: "config", sessionId });
            }
            catch {
            }
        },
    });
    const commitSuspendSaga = async (token, cp, remoteEnv, remoteHandle, cutSignal) => {
        if (!checkpointStore)
            return { tag: "absent" };
        if (memoryEngineSession)
            await memoryEngineSession.harvest("checkpoint");
        if (cutSignal?.aborted) {
            if (remoteEnv !== undefined && remoteHandle?.snapshotId !== undefined) {
                const back = await compensatePausedVM(remoteEnv, remoteHandle.snapshotId);
                if (!back.ok) {
                    abortController.abort();
                    void harness.abort();
                    return { tag: "compensation_failed", reason: back.reason };
                }
            }
            return { tag: "cut" };
        }
        const announceCommittedScreeningPark = () => {
            const committedCount = cp.state.inheritedGate?.parentConstraintCount;
            if (cp.state.inheritedGate?.requiresParentConstraint === true &&
                !screeningParkDisclosedRef.done &&
                (inheritedParentConstraints ?? []).some((pc) => pc.preToolUse !== undefined)) {
                screeningParkDisclosedRef.done = true;
                try {
                    deps.onError?.(new Error(`durable park under an inherited PreToolUse SCREENING constraint: this checkpoint records ` +
                        `requiresParentConstraint with parentConstraintCount=${committedCount ?? "(unrecorded)"}, and the live ` +
                        `closures cannot be persisted. A resume on THIS Runner re-supplies them automatically; a resume on a fresh ` +
                        `Runner (restart / another replica) must hand back the whole chain via resumeStream(..., internals), ` +
                        `rebuilding the screening entry with createPreToolUseConstraintPolicy(hook, env) — otherwise the row stays ` +
                        `pending. If the face only observes, declare Hooks.preToolUseObservational and it stops entering the chain.`), { phase: "degraded", sessionId, classification: "screening-constraint-in-durable-chain" });
                }
                catch {
                }
            }
        };
        const confirmPutOutcome = async (putErr) => {
            if (putErr?.code === "checkpoint.already_exists")
                return "absent";
            try {
                const row = await checkpointStore.get(token);
                if (row == null)
                    return "absent";
                const sameCall = (a, b) => a.kind === b.kind && (a.kind !== "tool_approval" || b.kind !== "tool_approval" || a.toolCallId === b.toolCallId);
                const isOurs = row.scope === cp.scope && row.sessionId === cp.sessionId && row.leafId === cp.leafId && sameCall(row.pendingAction, cp.pendingAction);
                return isOurs ? "committed" : "absent";
            }
            catch (readErr) {
                deps.onError?.(readErr, { phase: "config", sessionId });
                return "unknown";
            }
        };
        try {
            await checkpointStore.put(token, cp);
            announceCommittedScreeningPark();
            return { tag: "committed" };
        }
        catch (putErr) {
            deps.onError?.(putErr, { phase: "config", sessionId });
            const confirmed = await confirmPutOutcome(putErr);
            if (confirmed === "committed") {
                try {
                    deps.onError?.(new Error(`durable approval checkpoint: the store's write reported a failure but the row EXISTS — the commit ` +
                        `landed and its acknowledgement was lost (a network backend can complete the INSERT and drop the ` +
                        `connection before replying). The run suspends on that row rather than continuing, which would leave ` +
                        `a redeemable pending checkpoint behind. This backend's acknowledgement path is lossy — the operator ` +
                        `face carries the store's own message.`), { phase: "degraded", sessionId, classification: "checkpoint-put-confirmation-lost" });
                }
                catch {
                }
                announceCommittedScreeningPark();
                return { tag: "committed" };
            }
            if (confirmed === "unknown") {
                const unknownReason = "the approval checkpoint's state cannot be established (the store rejected the write and then could not be read back; a committed-but-unacknowledged row may exist) — the run is stopped rather than continued past an approval whose durable record is unknown";
                try {
                    deps.onError?.(new Error(`durable approval checkpoint: ${unknownReason}`), {
                        phase: "degraded",
                        sessionId,
                        classification: "checkpoint-put-outcome-unknown",
                    });
                }
                catch {
                }
                abortController.abort();
                void harness.abort();
                return { tag: "unknown", reason: unknownReason };
            }
            const reason = "the approval checkpoint could not be persisted (the store rejected the write; the deployment's error face carries the store's own message)";
            if (remoteEnv !== undefined && remoteHandle?.snapshotId !== undefined) {
                const back = await compensatePausedVM(remoteEnv, remoteHandle.snapshotId);
                if (back.ok)
                    return { tag: "absent", reason };
                abortController.abort();
                void harness.abort();
                return { tag: "compensation_failed", reason };
            }
            return { tag: "absent", reason };
        }
    };
    const suspendLoopCapHit = (count, cap, detail) => {
        if (cap === undefined)
            return false;
        if (count + 1 <= cap)
            return false;
        suspendLoopRef.hit = true;
        abortController.abort();
        void harness.abort();
        try {
            deps.onError?.(new Error(`suspend loop: task already suspended ${count} time(s) (max ${cap}); refusing to suspend again${detail}`), { phase: "config", sessionId });
        }
        catch {
        }
        return true;
    };
    const suspendableEnv = ownedEnv !== undefined && isSuspendable(ownedEnv) ? ownedEnv : undefined;
    const parkOnlyRemoteEnv = suspendableEnv === undefined && ownedEnv !== undefined && isRemoteExecutionEnv(ownedEnv) && incompleteSuspendAdapter === undefined
        ? ownedEnv
        : undefined;
    const parkOnlyHandle = (env) => {
        const { snapshotId: _lineage, ...identity } = env.workspaceHandle();
        return { ...identity, restoreMode: "park_only" };
    };
    return { saga: { inFlightSpendMicroUsd, serializeCheckpointState, compensatePausedVM, commitSuspendSaga, suspendLoopCapHit, suspendableEnv, parkOnlyRemoteEnv, parkOnlyHandle } };
}