@sema-agent/core
Version:
Stateless, task-oriented AI agent core
475 lines (474 loc) • 24.6 kB
JavaScript
import { delimitUntrusted } from "./untrusted-text.js";
import { envDeclaresRetain, retainsBackgroundProcesses } from "./background-shell.js";
import { assertOwnership, defaultMonitorTimers, MONITOR_MAX_TIMEOUT_MS, MONITOR_DEFAULT_TIMEOUT_MS, MONITOR_BATCH_WINDOW_MS, MONITOR_MAX_BATCHES_PER_MINUTE, MONITOR_STORM_BURST, MONITOR_STORM_KILL_AFTER_MS, MONITOR_LINE_BUF_CAP, MONITOR_SPILL_CAP_CHARS, TASK_OUTPUT_MAX_CHARS, mintCompletionId, clipMonitorEvent, clipMonitorLine, terminalTaskSummary, accountDroppedBytes, renderSpoolBody, SPOOL_REPLAY, rollSpoolText, statusFromBackground, droppedGapNote, firstDropNote, alreadyTerminalStopNote, clipTaskOutput, sleepPollStep, } from "./task-registry-shared.js";
import { buildToolResultRef, OFFLOAD_TOOL_NAME, toolResultProvenanceOf } from "./tool-result-store.js";
import { thrownText } from "./thrown-value.js";
export function registerMonitorLane(core, input) {
assertOwnership(input, "registerMonitor");
const id = core.mintTaskId("monitor");
const now = input.now ?? Date.now();
const timers = input.timers ?? defaultMonitorTimers;
const persistent = input.persistent === true;
const timeoutMs = persistent
? undefined
: Math.min(MONITOR_MAX_TIMEOUT_MS, Math.max(1, Math.floor(input.timeoutMs ?? MONITOR_DEFAULT_TIMEOUT_MS)));
const maxBatchesPerMinute = Math.max(1, Math.floor(input.maxBatchesPerMinute ?? MONITOR_MAX_BATCHES_PER_MINUTE));
const stormBurst = Math.max(1, Math.floor(input.stormBurst ?? MONITOR_STORM_BURST));
const handle = {
id,
type: "monitor",
status: "running",
...(input.description !== undefined ? { description: input.description } : {}),
...(input.owner !== undefined ? { owner: input.owner } : {}),
...(input.scope !== undefined ? { scope: input.scope } : {}),
...(input.toolUseId !== undefined ? { toolUseId: input.toolUseId } : {}),
...(persistent ? { persistent: true } : {}),
...(input.sessionScoped ? { sessionScoped: true } : {}),
createdAt: now,
updatedAt: now,
env: input.env,
shellId: input.shellId,
spool: { stdout: "", stderr: "", rolledChars: 0 },
lineBuf: "",
seq: 0,
stormTokens: stormBurst,
stormBurst,
stormRefillIntervalMs: Math.max(1, Math.round(60_000 / maxBatchesPerMinute)),
stormRefilledAt: timers.now(),
batchWindowMs: Math.max(1, Math.floor(input.batchWindowMs ?? MONITOR_BATCH_WINDOW_MS)),
maxBatchesPerMinute,
...(timeoutMs !== undefined ? { timeoutMs, deadlineAt: timers.now() + timeoutMs } : {}),
timers,
...(input.onEvent !== undefined ? { onEvent: input.onEvent } : {}),
...(input.toolResultStore !== undefined ? { toolResultStore: input.toolResultStore } : {}),
...(input.sessionId !== undefined ? { spillSessionId: input.sessionId } : {}),
...(retainsBackgroundProcesses({ env: envDeclaresRetain(input.env), task: input.retainBackgroundProcesses === true }) ? { retained: true } : {}),
...(input.retainBackgroundProcesses !== undefined ? { retainStatedAtLaunch: true } : {}),
};
core.handles.set(id, handle);
try {
startMonitorWatcherLane(core, id);
}
catch (e) {
core.handles.delete(id);
throw e;
}
return id;
}
function spillRolledMonitorChunk(handle, stream, dropped) {
const store = handle.toolResultStore;
if (store === undefined)
return;
const used = handle.spillCharsUsed ?? 0;
if (used >= MONITOR_SPILL_CAP_CHARS) {
handle.spillCapped = true;
return;
}
const n = stream === "out" ? (handle.spillSegCount ?? 0) : (handle.spillErrSegCount ?? 0);
const ref = buildToolResultRef(handle.spillSessionId ?? "no-session", handle.id, `${stream}_seg${n}`);
handle.spillCharsUsed = used + dropped.length;
if (stream === "out")
handle.spillSegCount = n + 1;
else
handle.spillErrSegCount = n + 1;
try {
const provenance = handle.spillSessionId === undefined ? undefined : toolResultProvenanceOf(handle.spillSessionId);
void Promise.resolve(store.put(ref, dropped, provenance)).catch(() => {
handle.spillFailed = true;
});
}
catch {
handle.spillFailed = true;
}
}
function monitorSpillNote(handle) {
const outN = handle.spillSegCount ?? 0;
const errN = handle.spillErrSegCount ?? 0;
if (outN === 0 && errN === 0)
return "";
const sid = handle.spillSessionId ?? "no-session";
const segLabel = (stream, n) => {
const first = buildToolResultRef(sid, handle.id, `${stream}_seg0`);
return n <= 1 ? `ref "${first}"` : `refs "${first}" .. "${buildToolResultRef(sid, handle.id, `${stream}_seg${n - 1}`)}"`;
};
const clauses = [];
if (outN > 0)
clauses.push(`stdout ${segLabel("out", outN)}`);
if (errN > 0)
clauses.push(`stderr ${segLabel("err", errN)}`);
const coverage = handle.spillFailed === true
? " — a write failed partway through; the ref chain may be INCOMPLETE, read what is there"
: handle.spillCapped === true
? ` — spill cap (${MONITOR_SPILL_CAP_CHARS} chars) reached; earlier segments retained, later rolls were not spilled`
: "";
return `; spilled to ${clauses.join(", ")} (read back via ${OFFLOAD_TOOL_NAME})${coverage}`;
}
export function absorbMonitorPollLane(handle, v) {
const spool = handle.spool;
accountDroppedBytes(spool, v);
const ROLL_CAP = 2 * TASK_OUTPUT_MAX_CHARS;
spool.stdout = rollSpoolText(spool, spool.stdout + v.stdout, ROLL_CAP, (dropped) => spillRolledMonitorChunk(handle, "out", dropped));
spool.stderr = rollSpoolText(spool, spool.stderr + v.stderr, ROLL_CAP, (dropped) => spillRolledMonitorChunk(handle, "err", dropped));
handle.lineBuf += v.stdout;
if (handle.lineBuf.length > MONITOR_LINE_BUF_CAP)
handle.lineBuf = handle.lineBuf.slice(-MONITOR_LINE_BUF_CAP);
const parts = handle.lineBuf.split(/\r?\n/);
handle.lineBuf = parts.pop() ?? "";
return parts.filter((l) => l !== "");
}
function consumeStormToken(handle, now) {
const gained = Math.floor((now - handle.stormRefilledAt) / handle.stormRefillIntervalMs);
if (gained > 0) {
handle.stormTokens = Math.min(handle.stormBurst, handle.stormTokens + gained);
handle.stormRefilledAt += gained * handle.stormRefillIntervalMs;
}
if (handle.stormTokens <= 0)
return false;
handle.stormTokens -= 1;
return true;
}
export function emitMonitorEventLane(handle, n) {
try {
handle.onEvent?.(n);
}
catch {
}
}
export function startMonitorWatcherLane(core, id) {
const handle = core.handles.get(id);
if (!handle || handle.type !== "monitor")
return;
const timers = handle.timers;
let ticking = false;
const MAX_RETRYABLE_POLL_FAILURES = 3;
let pollFailures = 0;
const stop = () => {
if (handle.watcher !== undefined) {
timers.clearInterval(handle.watcher);
handle.watcher = undefined;
}
};
const emit = (n) => emitMonitorEventLane(handle, n);
const pendingFinalLines = (lines) => {
const out = handle.lineBuf !== "" ? [...lines, handle.lineBuf] : lines;
handle.lineBuf = "";
return out;
};
const pendingSuppressionNote = () => consumePendingSuppressionNote(handle);
const terminal = (status, detail, finalLines, exitCode) => {
handle.updatedAt = Date.now();
mintCompletionId(handle);
stop();
core.notifyTerminalOnce(handle, () => emit({
task_id: id,
task_type: "monitor",
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
status,
...(status === "killed" && handle.stoppedBy !== undefined ? { stoppedBy: handle.stoppedBy } : {}),
summary: `${terminalTaskSummary("monitor", (handle.description ?? "monitor").slice(0, 200), status, exitCode)} — ${detail}${droppedGapNote(handle.spool)}`,
...(finalLines.length > 0 ? { lines: finalLines.map(clipMonitorLine), result: clipMonitorEvent(finalLines.map(clipMonitorLine).join("\n")), ...(status === "killed" ? { partial: true } : {}) } : {}),
...(handle.completionId !== undefined ? { completionId: handle.completionId } : {}),
}));
};
const tick = async () => {
if (ticking)
return;
ticking = true;
handle.tickInFlight = true;
try {
if (core.handles.get(id) !== handle)
return stop();
if (handle.status !== "running")
return stop();
const r = await handle.env.pollBackground(handle.shellId);
if (core.handles.get(id) !== handle || handle.status !== "running")
return stop();
if (!r.ok) {
if ((r.error.code === "timeout" || r.error.code === "io") && ++pollFailures <= MAX_RETRYABLE_POLL_FAILURES)
return;
handle.status = "failed";
return terminal("failed", `monitor lost its process (${r.error.message}); watch ended.${pendingSuppressionNote()}`, pendingFinalLines([]));
}
pollFailures = 0;
const spool = handle.spool;
const lines = absorbMonitorPollLane(handle, r.value);
handle.status = statusFromBackground(r.value.status, r.value.exitCode);
if (r.value.status !== "running") {
spool.exitCode = r.value.exitCode;
if (handle.status === "killed" && handle.stoppedBy === undefined)
handle.stoppedBy = handle.stopSource ?? (r.value.timedOut === true ? "timeout" : "system");
if (r.value.timedOut === true) {
spool.timedOut = true;
if (r.value.timeoutSec !== undefined)
spool.timeoutSec = r.value.timeoutSec;
}
const label = r.value.status === "exited"
? `exited(code ${r.value.exitCode}); watch ended.`
: r.value.timedOut === true
? `timed out after ${r.value.timeoutSec !== undefined ? `${r.value.timeoutSec}s` : "its background time budget"} (the execution environment's background time budget); the process was killed and the watch ended.`
: `${r.value.status}; watch ended.`;
return terminal(handle.status === "completed" ? "completed" : handle.status === "killed" ? "killed" : "failed", `${label}${pendingSuppressionNote()}`, pendingFinalLines(lines), r.value.status === "exited" ? r.value.exitCode : undefined);
}
const now = timers.now();
if (handle.deadlineAt !== undefined && now >= handle.deadlineAt) {
core.markStopSource(id, "timeout");
const k = await handle.env.killBackground(handle.shellId);
if (core.handles.get(id) !== handle || handle.status !== "running")
return stop();
if (!k.ok && k.error.code !== "not_found") {
handle.status = "failed";
return terminal("failed", `timed out after ${handle.timeoutMs}ms, but the kill failed (${k.error.message}); the process may still be running. Watch ended.${pendingSuppressionNote()}`, pendingFinalLines(lines));
}
handle.status = "killed";
handle.stoppedBy = handle.stopSource ?? "timeout";
return terminal("killed", `timed out after ${handle.timeoutMs}ms; the process was killed and the watch ended.${pendingSuppressionNote()}`, pendingFinalLines(lines));
}
if (lines.length === 0) {
if (r.value.stdout.length > 0 || r.value.stderr.length > 0)
handle.updatedAt = Date.now();
return;
}
if (!consumeStormToken(handle, now)) {
const previousSuppressedAt = handle.lastSuppressedAt;
handle.suppressedBatches = (handle.suppressedBatches ?? 0) + 1;
handle.lastSuppressedAt = now;
if (previousSuppressedAt !== undefined && now - previousSuppressedAt > handle.stormRefillIntervalMs * 3) {
handle.overloadSince = now;
}
else {
handle.overloadSince ??= now;
}
handle.updatedAt = Date.now();
if (now - handle.overloadSince <= MONITOR_STORM_KILL_AFTER_MS)
return;
const suppressed = handle.suppressedBatches;
const overloadSecs = Math.round((now - handle.overloadSince) / 1000);
core.markStopSource(id, "system");
const k = await handle.env.killBackground(handle.shellId);
if (core.handles.get(id) !== handle || handle.status !== "running")
return stop();
if (!k.ok && k.error.code !== "not_found") {
handle.status = "failed";
return terminal("failed", `auto-stop triggered (${suppressed} event batch(es) suppressed over ${overloadSecs}s of sustained overload), but the kill failed (${k.error.message}); the process may still be running. Watch ended.`, pendingFinalLines(lines));
}
handle.status = "killed";
handle.stoppedBy = handle.stopSource ?? "system";
return terminal("killed", `auto-stopped: too much output (${suppressed} event batch(es) suppressed over ${overloadSecs}s of sustained overload; sustained budget is ${handle.maxBatchesPerMinute} batches/minute). ` +
`The process was killed; its full output remains readable via TaskOutput. Restart it with a quieter command (filter its output) if you still need the watch.`, pendingFinalLines(lines));
}
if ((handle.suppressedBatches ?? 0) > 0) {
const suppressed = handle.suppressedBatches;
handle.suppressedBatches = 0;
if (handle.lastSuppressedAt !== undefined && now - handle.lastSuppressedAt > handle.stormRefillIntervalMs * 3)
handle.overloadSince = undefined;
handle.seq += 1;
const note = `[${suppressed} events suppressed — output rate too high. Consider using TaskStop to restart this monitor with a more selective filter.]`;
emit({
task_id: id,
task_type: "monitor",
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
status: "event",
seq: handle.seq,
summary: `${(handle.description ?? "monitor").slice(0, 200)} — ${suppressed} event batch(es) suppressed (output rate too high); the suppressed output is still readable via TaskOutput`,
result: note,
});
}
handle.seq += 1;
handle.updatedAt = Date.now();
const shown = lines.map(clipMonitorLine);
emit({
task_id: id,
task_type: "monitor",
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
status: "event",
seq: handle.seq,
lines: shown,
summary: `${(handle.description ?? "monitor").slice(0, 200)} — ${lines.length} new output line(s)${firstDropNote(spool)}`,
result: clipMonitorEvent(shown.join("\n")),
});
}
catch (e) {
if (core.handles.get(id) !== handle || handle.status !== "running")
return stop();
if (++pollFailures <= MAX_RETRYABLE_POLL_FAILURES)
return;
handle.status = "failed";
terminal("failed", `monitor watcher crashed (${thrownText(e)}); watch ended.${pendingSuppressionNote()}`, pendingFinalLines([]));
}
finally {
ticking = false;
handle.tickInFlight = undefined;
}
};
handle.watcher = timers.setInterval(tick, handle.batchWindowMs);
}
export async function pollMonitorLane(handle, filter, deadline, signal) {
let rx;
if (filter !== undefined) {
try {
rx = new RegExp(filter);
}
catch {
return {
content: `Error (TaskOutput): invalid filter regex: ${filter}`,
details: { task_id: handle.id, type: "monitor", status: handle.status, retrieval_status: "not_ready" },
isError: true,
};
}
}
while (handle.status === "running" && deadline !== undefined && Date.now() < deadline && !signal?.aborted) {
await sleepPollStep(deadline, signal);
}
const spool = handle.spool;
const shape = (text) => clipTaskOutput(rx ? text.split(/\r?\n/).filter((l) => rx.test(l)).join("\n") : text, handle.outputFile);
const running = handle.status === "running";
const retrieval = running && deadline !== undefined && Date.now() >= deadline ? "timeout" : "success";
const body = renderSpoolBody(spool, {
running,
handleStatus: handle.status,
shape,
rolledExtraClause: monitorSpillNote(handle),
drainNote: spool.drainFailed === true ? "[!] the final output drain at stop failed — the tail of the output may be missing\n" : "",
});
return {
content: delimitUntrusted(`TaskOutput ${handle.id}`, body),
details: {
task_id: handle.id,
type: "monitor",
status: handle.status,
retrieval_status: retrieval,
...(handle.status === "killed" && handle.stoppedBy !== undefined ? { stoppedBy: handle.stoppedBy } : {}),
...(handle.completionId !== undefined ? { completionId: handle.completionId } : {}),
details: {
status: running ? "running" : handle.status,
exitCode: spool.exitCode,
replay: SPOOL_REPLAY,
...(spool.droppedBytes !== undefined ? { bytesDroppedBeforeCursor: spool.droppedBytes } : {}),
...(spool.timedOut === true ? { timedOut: true, ...(spool.timeoutSec !== undefined ? { timeoutSec: spool.timeoutSec } : {}) } : {}),
},
},
};
}
function consumePendingSuppressionNote(handle) {
const suppressed = handle.suppressedBatches ?? 0;
if (suppressed <= 0)
return "";
handle.suppressedBatches = 0;
return ` [${suppressed} event batch(es) suppressed before the watch ended — output rate was too high; the suppressed output is still readable via TaskOutput.]`;
}
function explicitStopTerminalNote(handle, alreadyGone) {
const label = (handle.description ?? "monitor").slice(0, 200);
const detail = alreadyGone
? "killed before completion (stopped via TaskStop; the process was already gone); watch ended."
: "killed before completion (stopped via TaskStop); watch ended.";
emitMonitorEventLane(handle, {
task_id: handle.id,
task_type: "monitor",
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
status: "killed",
...(handle.stoppedBy !== undefined ? { stoppedBy: handle.stoppedBy } : {}),
summary: `${terminalTaskSummary("monitor", label, "killed")} — ${detail}${droppedGapNote(handle.spool)}`,
...(handle.completionId !== undefined ? { completionId: handle.completionId } : {}),
});
}
export async function stopMonitorLane(core, handle) {
const wasRunning = handle.status === "running";
if (handle.watcher !== undefined) {
handle.timers.clearInterval(handle.watcher);
handle.watcher = undefined;
}
if (wasRunning && handle.tickInFlight !== true) {
try {
const drained = await handle.env.pollBackground(handle.shellId);
if (drained.ok) {
const lines = absorbMonitorPollLane(handle, drained.value);
const finalLines = handle.lineBuf !== "" ? [...lines, handle.lineBuf] : lines;
handle.lineBuf = "";
if (finalLines.length > 0 && handle.status === "running") {
handle.seq += 1;
handle.updatedAt = Date.now();
emitMonitorEventLane(handle, {
task_id: handle.id,
task_type: "monitor",
...(handle.toolUseId !== undefined ? { toolUseId: handle.toolUseId } : {}),
status: "event",
seq: handle.seq,
lines: finalLines.map(clipMonitorLine),
summary: `${(handle.description ?? "monitor").slice(0, 200)} — ${finalLines.length} final output line(s) drained at stop${firstDropNote(handle.spool)}`,
result: clipMonitorEvent(finalLines.map(clipMonitorLine).join("\n")),
});
}
}
else {
handle.spool.drainFailed = true;
}
}
catch {
handle.spool.drainFailed = true;
}
}
const drainNote = handle.spool.drainFailed === true ? " (the final output drain failed — TaskOutput may be missing the last window)" : "";
core.markStopSource(handle.id, "parent");
const r = await handle.env.killBackground(handle.shellId);
if (handle.status !== "running") {
return {
content: wasRunning ? `Terminated ${handle.id}.${drainNote}${consumePendingSuppressionNote(handle)}` : alreadyTerminalStopNote(handle.id, handle.status),
details: {
task_id: handle.id,
type: "monitor",
status: handle.status,
retrieval_status: "success",
...(handle.stoppedBy !== undefined ? { stoppedBy: handle.stoppedBy } : {}),
...(handle.completionId !== undefined ? { completionId: handle.completionId } : {}),
},
};
}
if (!r.ok) {
if (r.error.code === "not_found") {
if (handle.stoppedBy === undefined)
handle.stoppedBy = handle.stopSource ?? "parent";
handle.status = "killed";
handle.updatedAt = Date.now();
mintCompletionId(handle);
core.notifyTerminalOnce(handle, () => explicitStopTerminalNote(handle, true));
return {
content: `Terminated ${handle.id} (the process was already gone).${drainNote}${consumePendingSuppressionNote(handle)}`,
details: {
task_id: handle.id,
type: "monitor",
status: "killed",
retrieval_status: "success",
stoppedBy: handle.stoppedBy,
...(handle.completionId !== undefined ? { completionId: handle.completionId } : {}),
},
};
}
if (handle.watcher === undefined)
startMonitorWatcherLane(core, handle.id);
const retrieval_status = r.error.code === "timeout" ? "timeout" : "not_ready";
return {
content: `Error (TaskStop): ${r.error.message} — the kill did not land; the process may still be running. The watch keeps running: events and the terminal notification still fire.`,
isError: true,
details: { task_id: handle.id, type: "monitor", status: handle.status, retrieval_status, error: r.error.code },
};
}
if (handle.status === "running") {
if (handle.stoppedBy === undefined)
handle.stoppedBy = handle.stopSource ?? "parent";
handle.status = "killed";
handle.updatedAt = Date.now();
mintCompletionId(handle);
core.notifyTerminalOnce(handle, () => explicitStopTerminalNote(handle, false));
}
return {
content: `Terminated ${handle.id}.${drainNote}${consumePendingSuppressionNote(handle)}`,
details: {
task_id: handle.id,
type: "monitor",
status: handle.status,
retrieval_status: "success",
...(handle.stoppedBy !== undefined ? { stoppedBy: handle.stoppedBy } : {}),
...(handle.completionId !== undefined ? { completionId: handle.completionId } : {}),
},
};
}