UNPKG

@sema-agent/core

Version:

Stateless, task-oriented AI agent core

423 lines (422 loc) • 26.6 kB
import { MAX_CONSECUTIVE_COMPACTION_FAILURES } from "./compaction-knobs.js"; import { isSyntheticApiErrorMessage } from "../../internal/harness.js"; import { adjudicateDerivedRoute, fallbackToPrimaryNotice, sameRouteIdentity } from "../../brain/route-adjudicator.js"; import { isDegenerateCutMessage } from "../../brain/terminal-cause.js"; import { hasBackgroundShell, sweepRunBoundaryBackgroundShells } from "../background-shell.js"; import { resolveCheckpointStore } from "../checkpoint-store.js"; import { hookSeatExpiredError, runHookSeat } from "../hooks.js"; import { modelCostToPricing } from "../pricing.js"; import { hasDestroy, isIsolated } from "../remote-env.js"; import { resolveTaskModel } from "../roles.js"; import { defaultTaskRegistry } from "../task-registry.js"; import { emitTrace } from "../trace.js"; import { deliverEngineNotice, parentLegIds, settleOwedUserInput } from "../types.js"; import { assembleResult } from "./assemble-result.js"; import { discloseUnconsumedHalt } from "./halt-attribution.js"; import { forwardSafely } from "./forward-sink.js"; import { TIMER_LATENESS_REPORT_MS } from "./clock-and-limits.js"; import { resumeDecisionWasNegative } from "./decide-continuation.js"; import { drainForwardedFramesBeforeDone, forwardsSubagentEvents } from "./prepare-run-refs.js"; import { generatePromptSuggestions } from "./prompt-suggestions.js"; import { settleTeardownLeg } from "./teardown-bounded.js"; import { owedDateAnchor } from "./announce-once-ledger.js"; import { autoRunConsolidationFromRecommendation, autoRunOnRecommendationEffective, scopeHasUnconsumedRecommendation } from "./memory-consolidation-driver.js"; import { amendTerminal, terminalProjection } from "./terminal-projection.js"; import { mintCompactedEvent } from "../auto-compaction.js"; import { asThrownError } from "../thrown-value.js"; export async function runSettleAndTeardown(input) { const { spec, queue, prepared, resume, internals, rs, stats, ident, parentToolCallId, subagentName, loopLatch, timeout, ownCommittedTailRef, emitDelegationLifecycle, taskIdRef, manualCompactRef, drainManualCompact, final, threw, abortedLive, userInterruptedLive, answerSettled, strandedHumanAnswers, reasoningResolution, comp, compactionBreaker, owedUserInput, setResult, onSuggestions, onSuspend, runner, sessions } = input; try { if (comp?.compacted) { queue.push(mintCompactedEvent(comp, "auto", ident())); if (comp.phaseDurations !== undefined && comp.durationMs !== undefined) { const pd = comp.phaseDurations; const pdDur = comp.durationMs; emitTrace(rs.telemetry.tracer, () => ({ kind: "compaction.phase_timings", version: 1, taskId: rs.telemetry.taskId, ...pd, durationMs: pdDur, ts: Date.now() })); } prepared.cacheBreakDetector?.notifyCompaction(); } if (prepared.nestedStats.tasks > 0) { stats.nested = { tokens: prepared.nestedStats.tokens, turns: prepared.nestedStats.turns, tasks: prepared.nestedStats.tasks, ...(prepared.nestedStats.anyUnpriced ? {} : { costMicroUsd: prepared.nestedStats.costMicroUsd }), }; } if (stats.totalInputTokens > 0) { const rawHit = stats.cachedTokens / stats.totalInputTokens; if (stats.cachedTokens > stats.totalInputTokens) { runner.deps.onError?.(new Error(`prompt-cache: usage/api mismatch — cachedTokens (${stats.cachedTokens}) > totalInputTokens (${stats.totalInputTokens}); the cache family is likely wrong. Does model.api match the brain serving it? Override with model.params.promptCacheFamily = "input-includes-cached" | "input-excludes-cached".`), { phase: "prompt-cache", sessionId: prepared.sessionId }); } stats.cacheHitRate = Math.min(1, rawHit); if (!rs.telemetry.cacheBreakReported && stats.turns >= 2 && stats.totalInputTokens >= 8000 && stats.cacheHitRate < 0.15) { const cacheWritten = stats.cacheWriteTokens + stats.cacheWriteTokensLong; const cause = rs.degrade.degraded ? `a mid-task model switch (${rs.degrade.degraded.from} → ${rs.degrade.degraded.to}, degradation) reset the prefix cache — this is the likely cause` : cacheWritten > 0 ? `this task reported ${cacheWritten} cache-write tokens across its calls (compaction included) but served almost none back as reads — consistent with a prompt prefix that CHANGES between turns (client-side: volatile content up front, or per-turn tool churn/reorder) and, less often, with server-side eviction. Keep volatile content (memory/timestamps/ids) out of the prefix and the tool list stable in membership AND order` : `no call in this task reported any cache-write tokens — and this API family may not report them at all (an openai-shaped usage row carries cached READS only), so the write side is no evidence here; check that the prompt prefix (system prompt + tool list, membership AND order) is byte-stable across turns and that this route caches this model`; runner.deps.onError?.(new Error(`prompt-cache: low prefix-cache hit rate ${(stats.cacheHitRate * 100).toFixed(0)}% over ${stats.turns} turns (${stats.totalInputTokens} prompt tokens) — ${cause}. See design/09.`), { phase: "prompt-cache", sessionId: prepared.sessionId }); } } if (prepared.resourceLedger && (resume?.cp.gate.kind === "resource_limit" || prepared.pausedRef.current?.cause.gate.kind === "resource_limit")) { stats.costMicroUsd += prepared.resourceLedger.spentMicroUsd; stats.tokens += prepared.resourceLedger.spentTokens; stats.turns += prepared.resourceLedger.spentTurns; } if (prepared.humanReviewRef.count > 0) { stats.humanReview = { count: prepared.humanReviewRef.count, totalWaitMs: prepared.humanReviewRef.totalWaitMs, gates: [...prepared.humanReviewRef.gates], }; } const reminderDisclosuresActive = Object.keys(prepared.reminderDisclosureCounts).length > 0; if (rs.counters.finalVerifyInjections > 0 || rs.attach.attachmentsInjected > 0 || rs.counters.repetitionCuts > 0 || rs.counters.repetitionSpared > 0 || rs.counters.approachNoticesSent > 0 || reminderDisclosuresActive) { stats.mechanisms = { ...(rs.counters.finalVerifyInjections > 0 ? { finalVerifyInjected: true } : {}), ...(rs.counters.finalVerifyInjections > 0 ? { finalVerifyInjections: rs.counters.finalVerifyInjections } : {}), ...(rs.attach.attachmentsInjected > 0 ? { attachmentsInjected: rs.attach.attachmentsInjected } : {}), ...(rs.counters.approachNoticesSent > 0 ? { approachNoticesSent: rs.counters.approachNoticesSent } : {}), ...(rs.counters.repetitionCuts > 0 ? { repetitionCuts: rs.counters.repetitionCuts } : {}), ...(rs.counters.repetitionSpared > 0 ? { repetitionSpared: rs.counters.repetitionSpared } : {}), ...(rs.counters.repetitionEvents.length > 0 ? { repetitionEvents: rs.counters.repetitionEvents } : {}), ...(reminderDisclosuresActive ? { reminderDisclosures: { ...prepared.reminderDisclosureCounts } } : {}), }; } const result = assembleResult(spec, prepared.sessionId, final, stats, { threw, runId: prepared.runId, model: prepared.model.id, unpricedSpend: rs.telemetry.unpricedSpend, rewindNotes: prepared.rewindNotes, editedFiles: prepared.editedFilesSnapshot(), haltedOnUserRejection: prepared.batchHaltRef.current !== undefined, haltConsumption: prepared.harness.haltConsumption(), strandedHumanAnswers, remoteEnvFailures: prepared.remoteEnvFailures, effectiveReadFace: prepared.effectiveReadFace, effectiveWriteFace: prepared.effectiveWriteFace, effectiveReadDenyPatterns: prepared.effectiveReadDenyPatterns, effectiveMemoryScopes: prepared.effectiveMemoryScopes, effectiveReasoning: reasoningResolution.current, ...(rs.counters.compactionPasses > 0 || compactionBreaker.failures >= MAX_CONSECUTIVE_COMPACTION_FAILURES ? { compaction: { passes: rs.counters.compactionPasses, breakerOpen: compactionBreaker.failures >= MAX_CONSECUTIVE_COMPACTION_FAILURES, ...(rs.counters.compactionKind !== undefined ? { kind: rs.counters.compactionKind } : {}) } } : {}), retryAfterMs: rs.limits.platformTerminal?.retryAfterMs, abortedForTimeout: timeout.fired, abortedForTurns: rs.limits.turnsExceeded, abortedLive, answerSettled, budgetHit: rs.limits.budgetHit, budgetAxis: rs.limits.budgetAxis, blockedReason: prepared.blockedRef.reason, blockedDetail: prepared.blockedRef.detail, conflict: prepared.conflictRef.hit, gitCoreOverBudget: prepared.gitStatusRef.terminalCode === "irreducible_core_over_budget", outputInvalid: rs.degrade.outputInvalid, suspendLoop: prepared.suspendLoopRef.hit, ...(prepared.pausedRef.current !== undefined ? { paused: prepared.pausedRef.current.cause } : {}), }); if (abortedLive && answerSettled && result.terminal.kind === "completed") { deliverEngineNotice(runner.deps.onNotice, { code: "task.interrupt_unconsumed", message: "a stop arrived after the run had already given its answer: nothing was cut or stopped by it — the answer stands " + "as the result, the run ends completed, and no interruption marker or limit code records the stop.", detail: { sessionId: prepared.sessionId, runId: rs.telemetry.runId, ...(spec.taskId !== undefined ? { taskId: spec.taskId } : {}), origin: userInterruptedLive ? "interrupt" : timeout.fired ? "walltime" : rs.limits.turnsExceeded ? "turns" : "signal", }, }); } discloseUnconsumedHalt({ accepted: loopLatch.userHalted, consumption: prepared.harness.haltConsumption(), onNotice: runner.deps.onNotice, sessionId: prepared.sessionId, runId: rs.telemetry.runId, taskId: spec.taskId }); let rowReturned = false; if (resume !== undefined && resume.pendingActionStarted !== true && result.terminal.kind !== "completed" && result.terminal.kind !== "paused" && !(resumeDecisionWasNegative(resume) && resume.decisionDelivered === true) && resume.onEnvRestoreFailed !== undefined) { try { await resume.onEnvRestoreFailed("env_failed"); rowReturned = true; amendTerminal(result, "resume.reopened_unstarted", (m) => `${m ?? "the resume leg did not complete"} — the approved action never started, so the checkpoint was REOPENED (pending): the same approval can be resumed again, do not ask for a fresh one`); } catch { amendTerminal(result, "resume.reopen_failed_unstarted", (m) => `${m ?? "the resume leg did not complete"} — the approved action never started, and reopening the checkpoint FAILED: its state is unprovable from here. Re-read it before deciding; do NOT issue a fresh approval on the assumption the old one is dead`); } } settleOwedUserInput(owedUserInput, rowReturned, { taskId: spec.taskId ?? prepared.sessionId, sessionId: prepared.sessionId, runId: prepared.runId, parent: parentLegIds(internals) }, runner.deps.onNotice); prepared.sealReadStateSeat?.(ownCommittedTailRef.current); { const wss = prepared.workspaceStateSettle; if (wss !== undefined && result.terminal.kind !== "paused") { const curCwd = prepared.cwdRef?.current !== undefined && prepared.cwdRef.current !== wss.rootCanonical ? prepared.cwdRef.current : undefined; const curWt = prepared.worktreeSessionRef?.current; const changed = curCwd !== wss.restoredHandsCwd || curWt?.worktreeDir !== wss.restoredWorktreeDir; if (curCwd !== undefined || curWt !== undefined || changed || wss.baselineUnknown === true) { await settleTeardownLeg(() => prepared.session.appendWorkspaceState({ taskRoot: wss.rootCanonical, ...(curCwd !== undefined ? { handsCwd: curCwd } : {}), ...(curWt !== undefined ? { activeWorktree: { ...curWt } } : {}), }), "session.appendWorkspaceState (settle leg)", (e) => runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId })); } } } const stopFailureHook = (spec.hooks ?? runner.deps.hooks)?.stopFailure; if (stopFailureHook !== undefined && result.terminal.kind === "failed" && final?.stopReason === "error" && !isDegenerateCutMessage(final) && threw === undefined && rs.limits.budgetHit === undefined && !rs.degrade.outputInvalid && !prepared.suspendLoopRef.hit && !abortedLive && result.terminal.code !== "conflict") { const failedCause = result.terminal; try { const failureSeat = await runHookSeat("stopFailure", { timeoutMs: prepared.hookTimeoutMs }, (sig) => stopFailureHook({ identity: prepared.hookIdentity, error: failedCause.message ?? "model error", ...(failedCause.code !== undefined ? { errorKind: failedCause.code } : {}), turns: stats.turns, signal: sig, })); if (failureSeat.expired) { try { runner.deps.onError?.(hookSeatExpiredError("stopFailure", prepared.hookTimeoutMs, failureSeat.cause, "the terminal observation was abandoned; the assembled TaskResult is unchanged"), { phase: "hook", sessionId: prepared.sessionId }); } catch { } } } catch (err) { try { runner.deps.onError?.(asThrownError(err), { phase: "hook", sessionId: prepared.sessionId }); } catch { } } } const terminalFace = terminalProjection(result.terminal); emitDelegationLifecycle({ phase: "terminal", identity: prepared.hookIdentity, status: terminalFace.status, turns: stats.turns, ...(terminalFace.errorCode !== undefined ? { errorCode: terminalFace.errorCode } : {}), }); if (taskIdRef !== undefined) taskIdRef.delegationTerminalOwed = undefined; if (rs.degrade.degraded) result.degraded = rs.degrade.degraded; if (prepared.outputRef.set) result.structuredOutput = prepared.outputRef.value; const committedToken = prepared.pausedRef.current?.cause.token; const committedScope = prepared.pausedRef.current?.scope; if (committedToken !== undefined) { const store = resolveCheckpointStore(spec, runner.deps); if (store && committedScope !== undefined) { onSuspend({ env: prepared.ownedEnv, token: committedToken, scope: committedScope, store, sessionId: prepared.sessionId, }); } } if (resume && committedToken === undefined) { await settleTeardownLeg(() => sessions.unpin?.(resume.cp.sessionId), "sessions.unpin (resumed-terminal leg)", (e) => runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId })); } if (prepared.memoryEngineSession) { const mes = prepared.memoryEngineSession; await settleTeardownLeg(() => mes.harvest(), "memoryEngineSession.harvest (settle leg)", (e) => runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId })); const scope = mes.handle.writeScope; if (scope !== null && autoRunOnRecommendationEffective(runner.deps) && scopeHasUnconsumedRecommendation(mes.engine, scope)) { void autoRunConsolidationFromRecommendation({ deps: runner.deps, engine: mes.engine, scope, sessionId: prepared.sessionId, onNotice: runner.deps.onNotice, onError: runner.deps.onError }); } } emitTrace(rs.telemetry.tracer, () => ({ kind: "task.end", version: 1, taskId: rs.telemetry.taskId, runId: rs.telemetry.runId, status: terminalFace.status, errorCode: terminalFace.errorCode, turns: stats.turns, tokens: stats.tokens, ...(rs.telemetry.unpricedSpend ? {} : { costMicroUsd: stats.costMicroUsd }), durationMs: Date.now() - rs.telemetry.taskStart, ...(prepared.outputRef.set ? { hasStructuredOutput: true } : {}), ...(stats.mechanisms !== undefined ? { mechanisms: (({ repetitionEvents: _events, ...scalars }) => scalars)(stats.mechanisms) } : {}), ...(timeout.latenessMs !== undefined && timeout.latenessMs > TIMER_LATENESS_REPORT_MS ? { timerLatenessMs: timeout.latenessMs } : {}), ...(rs.counters.walltimeSyncBackstopFired ? { walltimeSyncBackstop: true } : {}), ts: Date.now(), })); setResult(result); if (parentToolCallId !== undefined && result.terminal.kind !== "paused") { const terminalTick = { type: "task_progress", taskId: rs.telemetry.taskId, ...(internals?.delegationTaskType !== undefined ? { taskType: internals.delegationTaskType } : {}), ...(internals?.cycleSeq !== undefined ? { seq: internals.cycleSeq } : {}), ...(internals?.parentTaskId !== undefined ? { parentTaskId: internals.parentTaskId } : {}), ...(subagentName ? { name: subagentName } : {}), ...(internals?.explicitAgentName !== undefined ? { spawnName: internals.explicitAgentName } : {}), model: prepared.model.id, usage: { totalTokens: stats.tokens, toolUses: stats.toolCalls, durationMs: Date.now() - rs.telemetry.taskStart }, status: result.terminal.kind === "completed" ? "completed" : "failed", ...ident(), }; queue.push(terminalTick); forwardSafely(internals, terminalTick); } if (manualCompactRef.waiters.length > 0) { queue.push({ type: "compaction_outcome", outcome: "mooted", trigger: "manual", reason: "task_ending", ...ident() }); drainManualCompact("mooted"); } manualCompactRef.emitMooted = undefined; if (forwardsSubagentEvents(spec)) await drainForwardedFramesBeforeDone(); queue.push({ type: "done", result }); queue.close(); await prepared.fileHistoryBoundary?.settle(); if (spec.suggestNextPrompts && result.terminal.kind === "completed" && !abortedLive) { onSuggestions(suggestNextPrompts(spec, prepared, result, runner)); } } finally { settleOwedUserInput(owedUserInput, !prepared.suspendProgressRef.executedApproved, { taskId: spec.taskId ?? prepared.sessionId, sessionId: prepared.sessionId, runId: prepared.runId, parent: parentLegIds(internals) }, runner.deps.onNotice); const writeDateAnchor = owedDateAnchor(prepared.session, prepared.dateChange, rs.attach.dateState); if (writeDateAnchor !== undefined) await settleTeardownLeg(writeDateAnchor, "date anchor (settle leg)", (e) => runner.deps.onError?.(e, { phase: "degraded", sessionId: prepared.sessionId })); const ownedEnvDying = prepared.ownedEnv !== undefined && hasDestroy(prepared.ownedEnv) && isIsolated(prepared.ownedEnv) && prepared.pausedRef.current === undefined; await settleTeardownLeg(() => defaultTaskRegistry.settleKilledForOwner({ ...prepared.registryAccess, sessionId: prepared.sessionId }, { source: abortedLive && !answerSettled && !timeout.fired && !rs.limits.turnsExceeded && rs.limits.budgetHit === undefined && prepared.pausedRef.current === undefined ? "user" : "parent", skipSessionScoped: true, envDying: ownedEnvDying, retainProcesses: spec.retainBackgroundProcesses === true, }), "taskRegistry.settleKilledForOwner", (e) => runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId })); const bgEnv = prepared.ownedEnv ?? runner.deps.executionEnv; if (bgEnv && hasBackgroundShell(bgEnv)) { try { await settleTeardownLeg(() => sweepRunBoundaryBackgroundShells(bgEnv, defaultTaskRegistry, { task: spec.retainBackgroundProcesses === true, envDying: ownedEnvDying }), "sweepBackgroundShells", (e) => runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId })); } catch (e) { runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId }); } } defaultTaskRegistry.clearBackgroundForOwner(prepared.registryAccess); defaultTaskRegistry.abortBackgroundAgentsForOwner(prepared.registryAccess, { skipSessionScoped: true }); if (prepared.subagentRetain) { await settleTeardownLeg(() => prepared.subagentRetain.disposeAll(), "subagentRetain.disposeAll", (e) => runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId })); } await settleTeardownLeg(() => teardownOwnedEnv(prepared, runner), "teardownOwnedEnv", (e) => runner.deps.onError?.(e, { phase: "config", sessionId: prepared.sessionId })); } return {}; } const SUGGESTIONS_DEFAULT_COUNT = 3; const SUGGESTIONS_MAX_COUNT = 8; const SUGGESTIONS_TIMEOUT_MS = 30_000; const SUGGESTIONS_TRANSCRIPT_MESSAGES = 12; async function suggestNextPrompts(spec, prepared, result, runner) { const cfg = typeof spec.suggestNextPrompts === "object" ? spec.suggestNextPrompts : {}; const count = cfg.count && cfg.count > 0 ? Math.min(cfg.count, SUGGESTIONS_MAX_COUNT) : SUGGESTIONS_DEFAULT_COUNT; const ac = new AbortController(); let timer; const timeout = new Promise((resolve) => { timer = setTimeout(() => { ac.abort(); resolve([]); }, SUGGESTIONS_TIMEOUT_MS); }); const work = (async () => { let model = prepared.model; let thinking = prepared.thinking; try { const resolved = resolveTaskModel({ modelRole: cfg.role ?? "summarize", roles: spec.roles }, runner.deps); model = resolved.model; thinking = resolved.thinking; } catch (roleErr) { if (cfg.role !== undefined) { let asked; try { asked = String(cfg.role); } catch { asked = `<unrenderable ${typeof cfg.role}>`; } try { runner.deps.onError?.(new Error(`suggestNextPrompts.role ${JSON.stringify(asked.length > 80 ? `${asked.slice(0, 80)}…` : asked)} did not resolve to a model — the prompt-suggestion pass ran on the task's own model (${prepared.model.id}) instead. Configure that role in RunnerDeps.roles / TaskSpec.roles, or omit the field to use the "summarize" role.`, { cause: roleErr }), { phase: "suggestions", sessionId: prepared.sessionId }); } catch { } } } if (!sameRouteIdentity(model, prepared.model)) { const verdict = await adjudicateDerivedRoute({ brain: runner.deps.brain, model, getApiKeyAndHeaders: spec.getApiKeyAndHeaders }); if (verdict !== undefined && !verdict.ok) { deliverEngineNotice(runner.deps.onNotice, fallbackToPrimaryNotice({ seat: "prompt-suggestions", from: model.id, to: prepared.model.id, verdict, sessionId: prepared.sessionId, runId: prepared.runId })); model = prepared.model; thinking = prepared.thinking; } } const pricing = runner.deps.pricing?.[model.id] ?? modelCostToPricing(model.cost); const ctx = await prepared.session.buildContext(); const transcript = ctx.messages.filter((m) => !isSyntheticApiErrorMessage(m)).slice(-SUGGESTIONS_TRANSCRIPT_MESSAGES); const out = await generatePromptSuggestions({ brain: runner.deps.brain, model, pricing, thinking, getApiKeyAndHeaders: spec.getApiKeyAndHeaders, signal: ac.signal }, transcript, count); if (out.tokens > 0 || out.costMicroUsd > 0) { result.stats.suggestions = { tokens: out.tokens, costMicroUsd: out.costMicroUsd }; if (result.stats.costBreakdown) { result.stats.costBreakdown.suggestionsMicroUsd = out.costMicroUsd; } } return out.suggestions; })(); work.catch(() => { }); try { return await Promise.race([work, timeout]); } catch (err) { try { runner.deps.onError?.(err, { phase: "suggestions", sessionId: prepared.sessionId }); } catch { } return []; } finally { clearTimeout(timer); } } async function teardownOwnedEnv(prepared, runner) { if (prepared.ownedEnv && hasDestroy(prepared.ownedEnv) && prepared.pausedRef.current === undefined) { try { await prepared.ownedEnv.destroy(); } catch (err) { runner.deps.onError?.(err, { phase: "config", sessionId: prepared.sessionId }); } } }