comfyui-mcp
Version:
Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),
1,515 lines • 84.8 kB
JavaScript
// Panel agent — one persistent Claude Agent SDK streaming session per ComfyUI
// panel tab. This is the autonomous background driver for the sidebar panel: the
// orchestrator (src/orchestrator/index.ts) owns the UI bridge and feeds each
// tab's user messages into that tab's session here; the agent's replies flow
// back out to the panel chat.
//
// Why the Agent SDK (not --sdk-url / CCR-v2): we need a persistent background
// agent with a live "channel in" (push messages over time), interrupt/inject,
// and SUBSCRIPTION auth with no API key — without patching the claude binary.
// `query({ prompt: <async generator> })` is exactly that: the generator stays
// open (the channel in), `Query.interrupt()` stops a live turn, and with
// ANTHROPIC_API_KEY unset the SDK reads the on-disk claude.ai OAuth login
// (verified: the session reports apiKeySource=none on this machine).
//
// The spawned agent runs THIS comfyui-mcp build as its MCP server in normal
// mode, so it talks to the live ComfyUI over COMFYUI_URL and never contends for
// the bridge port the orchestrator owns.
import { logger } from "../utils/logger.js";
import { errorText, promptText } from "./error-text.js";
import { ClaudeBackend, fetchSupportedModels, fetchSupportedCommands, } from "./claude-backend.js";
// The provider-specific Claude probes live in claude-backend.ts now; re-export
// them so the orchestrator (index.ts) keeps importing them from here.
export { fetchSupportedModels, fetchSupportedCommands };
function msgOf(err) {
return errorText(err);
}
/** Idle window for the per-turn freeze watchdog: if a turn that's in flight
* receives NO events at all for this long, treat it as stalled. Generous (legit
* tool work is slow but still streams progress) and overridable for tests via
* COMFYUI_MCP_TURN_IDLE_MS. Default 3.5 min. */
const TURN_IDLE_MS = Number(process.env.COMFYUI_MCP_TURN_IDLE_MS) || 210_000;
/** Longest a single tool call may hold the idle watchdog off before it's treated
* as genuinely stuck rather than legitimately slow. An MCP tool call streams NO
* progress notification between its start and end (e.g. install_custom_node
* awaiting ComfyUI-Manager's 600s cap), so while one is in flight the turn is
* working even though the app-server is silent — the watchdog must not trip. But
* a tool open past THIS is more likely wedged than slow, so the watchdog is
* allowed to fire. Generous (> Manager's 600s). Read live (not a load-time
* const) so it stays overridable per test. */
function toolBusyMaxMs() {
return Number(process.env.COMFYUI_MCP_TOOL_BUSY_MAX_MS) || 900_000;
}
/** How long after an interrupt to wait for the aborted turn's `result` (which
* releases the turn gate at the correct moment) before force-releasing it as a
* fallback. Short enough to feel immediate if a result somehow never arrives;
* in the normal case the result lands well within this and the gate opens then.
* Overridable (tuning / tests) via COMFYUI_MCP_INTERRUPT_RELEASE_MS. */
const INTERRUPT_RELEASE_FALLBACK_MS = Number(process.env.COMFYUI_MCP_INTERRUPT_RELEASE_MS) || 1500;
/** The full neutral set, ordered low→high (for nearest-level mapping). */
export const EFFORTS = ["none", "minimal", "low", "medium", "high", "xhigh", "max"];
export function isEffort(v) {
return typeof v === "string" && EFFORTS.includes(v);
}
/**
* One persistent streaming session for a single panel tab. Messages typed in
* the panel are queued via `send()` and yielded into the live `query()` session
* as user turns; the session never closes until `stop()`.
*/
export class PanelAgent {
tabId;
deps;
/** The injected provider adapter (Claude today). PanelAgent owns the queue,
* turn-gate, rewind tracking and self-restart; the backend owns the SDK call,
* option building, and SDKMessage→AgentEvent normalization. */
backend;
queue = [];
waiting = null;
closed = false;
/** The user message(s) of the turn currently in flight — captured at dispatch so
* an INTERRUPT (panel "send now") can RE-QUEUE the interrupted text ahead of the
* new message, instead of dropping the work the agent was mid-reply on. Cleared
* on a clean turn result (nothing to re-queue) and on stall/rewind (abandoned on
* purpose). Null whenever no turn is in flight. */
inFlight = null;
/** True while a turn is in flight (working→done). Lets the manager defer a
* session-restarting option change (effort) until the turn finishes, instead
* of interrupting and silently dropping the in-flight reply. */
busy = false;
/** True once THIS turn's failure has been surfaced to the user — set by the
* `error` event case so an error-subtype `result` right behind it doesn't
* paint a second failure line. Reset when a turn starts (busy = true). */
errorSurfaced = false;
/** True between a USER-initiated interrupt (Stop / send-now) and the aborted
* turn's terminal result. The Claude SDK reports an interrupted turn as an
* error-subtype result (error_during_execution), which the never-end-in-
* silence guard below would otherwise paint as "⚠️ That turn failed" — a
* scary failure line for something the user did on purpose. Cleared on the
* next result and at the next turn dispatch, so a REAL later failure still
* surfaces. */
interruptRequested = false;
// ---- turn idle watchdog (freeze safety net) ----
// A stalled turn (the backend stops emitting ANY events — e.g. a wedged Codex
// app-server) would otherwise leave the panel "working" forever. This is an
// IDLE timer (reset on every received event), NOT a hard turn cap: legit tool
// work is slow but still streams progress/tool events, so only a TRUE stall
// (no events at all for the whole window) trips it. On trip we surface a clear
// terminal error, advance the turn-gate (so the next queued batch runs), and
// best-effort interrupt the backend. Composes with the backend's own terminal
// result: completeTurn() is capped at yieldedTurns, so a late real result after
// a trip can't double-advance the gate.
idleTimer = null;
/** Guards against a trip firing twice / racing a real result for one turn. */
idleTripped = false;
/** Tool calls started (item/started) but not yet ended (item/completed). A tool
* in flight is legitimate work even when the app-server sends nothing, so the
* watchdog defers while this is > 0 — the fix for the #307-review finding that
* narrowing raw-notification liveness could false-trip a long silent tool call.
* Reset per turn so a leaked start can't wedge the next turn's watchdog. */
openToolCalls = 0;
/** When the current run of open tool calls began (0 → 1 transition), so the
* defer can be bounded by TOOL_BUSY_MAX_MS. */
toolBusySince = 0;
// ---- turn gate (race-free) ----
// The channel releases ONE batch per turn so the SDK can't read ahead (which
// prematurely "read" queued messages and lost them on interrupt). Implemented
// with MONOTONIC COUNTERS, not a resolver: after yielding batch N the channel
// waits until completedTurns >= yieldedTurns. This is deadlock-proof — if the
// turn's result fires BEFORE the channel parks, the counter has already caught
// up and the channel never blocks (the resolver version deadlocked on that
// race, stranding every later message).
yieldedTurns = 0;
completedTurns = 0;
turnWaiter = null;
/** After an interrupt we DON'T force the turn gate open synchronously — that
* fed the next batch into the backend before the aborted turn had settled, so
* Claude took the message into the session but started no turn on it (wedged
* until the slow idle watchdog or the next message). Instead the aborted
* turn's `result` event drives completeTurn() at the right moment; this is a
* short fallback that opens the gate anyway if no result ever arrives, so an
* interrupt can never stop cold. */
interruptReleaseTimer = null;
/** Mutable so the model/effort picker can change them at runtime. */
model;
effort;
/** UUID of the last assistant message seen — reported as the turn anchor on
* result, and used as the resume point when forking (rewind). */
lastAssistantUuid = null;
/** Set by requestRewind() to fork the session on the next (re)start. `anchor`
* is the assistant UUID to resume up to (drop everything after); null = fresh. */
pendingRewind = null;
/** Captured from the session's init message; enables resume across restarts. */
sessionId = null;
/** Id of the assistant message currently streaming (from message_start), so
* stream deltas and the final committed `say` share one bubble id. */
streamMsgId = null;
title;
/** Usage from the most recent assistant API response — the CURRENT context
* size (input + cache), as opposed to result.usage which sums every internal
* call in the turn and wildly overstates context fill. */
lastUsage = null;
/** Context window for the active model, cached from result.modelUsage. */
contextWindow = 0;
/** Last status pushed — re-sent on reconnect so the meter isn't blank. */
lastStatus = null;
/** Set true when start()'s bounded self-restart loop GAVE UP (the session kept
* dropping immediately) — as opposed to an intentional stop(). The manager
* reads this to distinguish a fatal "agent backend is dead" settle (which must
* bubble up so the orchestrator can self-exit + let the pack respawn a clean
* one) from an ordinary retire. */
gaveUp = false;
constructor(tabId, deps, backend) {
this.tabId = tabId;
this.deps = deps;
this.model = deps.model;
this.effort = deps.effort;
// Default to the Claude adapter; injectable so a future toggle can swap it.
this.backend =
backend ??
new ClaudeBackend({
mcpServers: deps.mcpServers,
comfyuiUrl: deps.comfyuiUrl,
systemAppend: deps.systemAppend,
panelServer: deps.panelServer,
pluginPath: deps.pluginPath,
});
}
short() {
return this.tabId.slice(0, 8);
}
/** Queue a panel message and wake the streaming generator (the "channel in").
* `images` are ComfyUI refs delivered inline as image blocks (vision). */
send(text, opts) {
if (opts?.title)
this.title = opts.title;
this.queue.push({ text, images: opts?.images, mid: opts?.mid });
const wake = this.waiting;
this.waiting = null;
wake?.();
}
/** Rewind the CONVERSATION: fork the session at `anchor` (an assistant UUID
* reported via onTurnAnchor) so everything after it is dropped from the agent's
* memory, then restart. `anchor` null forks to a fresh session. The edited
* message arrives separately as the next user_message (queued + drained by the
* forked channel). The graph (code) scope is handled panel-side. */
requestRewind(anchor) {
this.pendingRewind = { anchor };
if (anchor === null)
this.sessionId = null; // fresh fork → don't resume
// A rewind deliberately DROPS everything after the anchor (the edited message
// arrives separately), so the interrupted turn's text must NOT be re-queued.
this.inFlight = null;
// Break the current stream so start()'s loop re-enters and forks.
void this.backend.interrupt().catch(() => { });
const wake = this.waiting;
this.waiting = null;
wake?.();
}
/** Drop a still-queued message (the user cancelled/edited it before the agent
* got to it). Returns true if it was found and removed; false if it was
* already dequeued (the turn started — too late to cancel). */
cancelQueued(mid) {
const i = this.queue.findIndex((item) => item.mid === mid);
if (i < 0)
return false;
this.queue.splice(i, 1);
return true;
}
/** Reorder still-queued messages to match the panel's desired flush order.
* `order` is a list of mids; queued items are sorted by their index in it
* (items not named keep their relative order, after the named ones). Only the
* not-yet-dequeued queue is touched — a turn already in flight is unaffected. */
reorderQueue(order) {
if (!Array.isArray(order) || this.queue.length < 2)
return;
const rank = new Map(order.map((mid, i) => [mid, i]));
const at = (mid) => (mid && rank.has(mid) ? rank.get(mid) : Number.MAX_SAFE_INTEGER);
// Stable sort by desired rank (JS Array.sort is stable), so unnamed items
// keep their relative order and trail the explicitly-ordered ones.
this.queue.sort((a, b) => at(a.mid) - at(b.mid));
}
/**
* Inject a ComfyUI execution event (run finished / errored) as a turn, so the
* agent learns its render landed and can comment — solving "the asset never
* reached the agent." Only meaningful when a session is live (the manager only
* calls this for an existing agent, so we never spawn one just for an event).
*/
injectEvent(ev) {
let text = null;
let images;
if (ev.kind === "executed") {
const imgs = ev.images ?? [];
const names = imgs.map((i) => i.filename).filter(Boolean).join(", ") || "(unnamed)";
// A custom `note` (e.g. the panel's video-storyboard summary) replaces the
// default image-acknowledgement wording so the agent is told accurately
// what it's looking at (a contact sheet of a video, not a still image).
const note = typeof ev.note === "string" && ev.note.trim() ? ev.note.trim() : null;
text =
`[panel event] ` +
(note
? `${note} `
: `A run on the user's canvas just finished and produced ${imgs.length} output image(s): ${names}. `) +
// Only claim images are attached when some actually are (a note-only event —
// e.g. a video that produced no storyboard — has none), and only when this
// backend can actually see them — a text-only backend told "attached below"
// would confabulate having viewed the render.
(imgs.length
? this.backend.capabilities.vision
? `The image(s) are attached below and already shown to the user in the panel. `
: `You cannot view images on this provider, but they are already shown to the user in the panel. `
: ``) +
`Reply with ONE short sentence acknowledging the result and suggesting a sensible next step — you do NOT need to call any tools. Don't repeat an earlier comment.`;
// Attach the outputs inline so the agent SEES the render (no fetch needed).
if (this.backend.capabilities.vision) {
images = imgs.filter((i) => i.filename).map((i) => ({ ...i, type: i.type ?? "output" }));
}
}
else if (ev.kind === "run_error") {
text =
`[panel event] The user's workflow run just ERRORED: ${ev.error ?? "unknown error"}. ` +
`If it relates to what you were doing, diagnose it (panel_get_errors has the details) and offer a fix.`;
}
if (!text)
return;
this.busy = true;
this.deps.onTurn?.(this.tabId, "working"); // event triggers a turn — show working
this.queue.push({ text, images });
const wake = this.waiting;
this.waiting = null;
wake?.();
}
/** Push a ComfyUI EXECUTION error into the session with urgency — the "hey,
* look at me" path. Renders fail ASYNC (minutes after the agent queued them via
* panel_run), so without this the agent never learns and carries on as if the
* run succeeded. INTERRUPT any live turn (re-queued so it resumes AFTER the
* error), then put the error at the FRONT of the queue so the agent addresses
* it before anything else. */
async injectRunError(error) {
if (this.closed)
return;
const text = `[panel event] ⚠️ The workflow run you just queued ERRORED on the user's canvas: ${error}. ` +
`STOP — do not carry on as if it succeeded. If it relates to what you were doing, diagnose it ` +
`(panel_get_errors has the full node-level details) and fix it; otherwise tell the user briefly.`;
if (this.inFlight) {
// Stop the live turn and re-queue it so the agent handles the error FIRST,
// then resumes whatever it was doing.
await this.interrupt({ requeueInFlight: true });
}
this.busy = true;
this.deps.onTurn?.(this.tabId, "working");
this.queue.unshift({ text }); // front: ahead of any re-queued interrupted turn
const wake = this.waiting;
this.waiting = null;
wake?.();
}
/** Switch the model live (the SDK applies it to the next turn). */
async setModel(model) {
if (model === this.model)
return;
this.model = model;
try {
// setModel is live: no session restart, the next turn uses it.
await this.backend.setModel?.(model);
logger.info(`[panel-agent ${this.short()}] model → ${model}`);
}
catch (err) {
logger.debug(`[panel-agent ${this.short()}] setModel: ${msgOf(err)}`);
}
}
/**
* Record a new effort. The SDK takes effort as a session option, so this only
* affects the live session if the caller restarts it (the manager recreates
* the agent with resume so the conversation continues). Returns true if it
* changed.
*/
setEffortPending(effort) {
if (effort === this.effort)
return false;
this.effort = effort;
return true;
}
/** True once stop() was called — distinguishes an intentional shutdown from
* an SDK session that ended on its own (so the manager can self-heal). */
get isStopped() {
return this.closed;
}
/** True while a turn is actively running (between working and done). */
get isBusy() {
return this.busy;
}
/** True when messages are queued but not yet consumed (a turn is about to
* start). The manager waits for both !busy and !hasPending before a restart. */
get hasPending() {
return this.queue.length > 0;
}
/** Remove and return any unsent queued messages — so a session restart can hand
* them to the replacement agent instead of dropping them. Items keep their
* panel `mid`, so re-delivery still flips the right bubble on dequeue (seen). */
takePending() {
const items = this.queue;
this.queue = [];
return items;
}
get currentModel() {
return this.model;
}
get currentEffort() {
return this.effort;
}
/** Stop the current turn without ending the session (a "stop" button, or the
* panel "send now" which interrupts then sends). The turn ends → release the
* next queued message so an interrupt ADVANCES to the next pending turn (and
* only stops cold when nothing is queued).
*
* SEND-NOW PARITY: an interrupt mid-reply was DROPPING the message the agent
* was working on (only the new "send now" message got answered). So if a turn
* is in flight when we interrupt, RE-QUEUE its user text at the FRONT of the
* queue. The new message (sent right after the interrupt) lands behind it, so
* the next turn drains BOTH into one batch — interrupted-first — and the agent
* addresses both. Cleared so a later clean result can't re-queue it again. */
async interrupt(opts = {}) {
// Capture + clear BEFORE the async backend call so a result racing in can't
// both clear it and have us re-queue a stale copy.
const interrupted = this.inFlight;
this.inFlight = null;
// The aborted turn's result is EXPECTED (and error-subtyped on the Claude
// SDK) — don't let the result case report it as a turn failure.
this.interruptRequested = true;
// The turn that's in flight RIGHT NOW — the one we're aborting. Captured
// before the await so the release fallback below targets exactly this turn's
// gate (not a later one the channel may legitimately advance to if the
// aborted turn's result lands during the await).
const interruptedTurn = this.yieldedTurns;
// Re-queue the interrupted turn ONLY for "send now" (requeueInFlight) — there
// the user wants BOTH the interrupted message and the new one answered. A plain
// Stop / Ctrl+C / Esc (requeueInFlight=false) must NOT re-queue, or it would
// silently re-run the turn the user just stopped (double tool actions).
if (interrupted && opts.requeueInFlight) {
// Front of the queue: the interrupted work is addressed before whatever the
// user sends next (which is appended after this interrupt is handled).
this.queue.unshift({ text: interrupted.text, images: interrupted.images });
}
try {
await this.backend.interrupt();
}
catch (err) {
logger.debug(`[panel-agent ${this.short()}] interrupt: ${msgOf(err)}`);
}
finally {
// Do NOT force the gate open here. Synchronously releasing it fed the next
// batch (the re-queued turn + the "send now" message) into Claude before the
// aborted turn had settled — the SDK took the message into the session but
// started no turn on it, so it sat wedged until the slow idle watchdog (or
// the user's next message) nudged it. Instead let the aborted turn's
// `result` event drive completeTurn() — that fires once the SDK has finished
// tearing the turn down and is ready for the next prompt, so the re-queued
// batch is fed at the right moment and answered immediately. The fallback
// guarantees we still advance if no result ever arrives.
this.armInterruptReleaseFallback(interruptedTurn);
}
}
/** Bounded safety net for interrupt(): if the aborted turn's `result` (which
* opens the gate via completeTurn) hasn't arrived shortly, force the gate so an
* interrupt can't stop cold. `guardTurn` is the interrupted turn's number —
* we only force-release while THAT turn is still the one the gate waits on, so
* a result that lands during the await (advancing the channel to a new, legit
* in-flight turn) can't be wrongly cut short. Cancelled by completeTurn(). */
armInterruptReleaseFallback(guardTurn) {
if (this.interruptReleaseTimer)
clearTimeout(this.interruptReleaseTimer);
this.interruptReleaseTimer = setTimeout(() => {
this.interruptReleaseTimer = null;
if (this.closed)
return;
// Fire only if the SAME interrupted turn still hasn't completed (no result
// arrived). If a result landed, completedTurns has reached guardTurn (and
// completeTurn already cleared this timer), so this is a no-op.
if (this.completedTurns < guardTurn) {
logger.debug(`[panel-agent ${this.short()}] interrupt: no result within ${INTERRUPT_RELEASE_FALLBACK_MS}ms — releasing the gate`);
this.releaseTurns();
}
}, INTERRUPT_RELEASE_FALLBACK_MS);
this.interruptReleaseTimer.unref?.();
}
clearInterruptReleaseFallback() {
if (this.interruptReleaseTimer) {
clearTimeout(this.interruptReleaseTimer);
this.interruptReleaseTimer = null;
}
}
/** End the session and release the agent (tab closed / orchestrator shutdown). */
async stop() {
this.closed = true;
this.inFlight = null; // teardown must not leave a turn that could be re-queued
this.clearIdleWatchdog(); // don't let a turn watchdog fire after teardown
const wake = this.waiting;
this.waiting = null;
wake?.(); // let the generator observe `closed` and return
this.releaseTurns(); // and unblock it if it's parked at the turn gate
try {
await this.backend.interrupt();
}
catch {
// already winding down
}
// Permanently dispose of the backend's resources (kill any child process
// tree, drop the live connection). interrupt() alone is a no-op when idle, so
// a backend that owns a child process (Codex app-server) would otherwise be
// orphaned across stop/reset/effort-restart/stopAll/shutdown. Idempotent.
try {
await this.backend.close?.();
}
catch {
// best-effort teardown
}
}
/** A turn finished (result) → let the channel release the next batch. Capped at
* yieldedTurns so an interrupt + a late result for the same turn can't double-
* count and let the gate run ahead. */
completeTurn() {
// A real result settled the (interrupted or clean) turn — the post-interrupt
// fallback is no longer needed.
this.clearInterruptReleaseFallback();
this.completedTurns = Math.min(this.completedTurns + 1, this.yieldedTurns);
const w = this.turnWaiter;
this.turnWaiter = null;
w?.();
}
/** Force the gate open regardless of results (interrupt fallback / shutdown) so
* an interrupt advances to the next pending batch instead of stopping cold. */
releaseTurns() {
this.clearInterruptReleaseFallback();
this.completedTurns = this.yieldedTurns;
const w = this.turnWaiter;
this.turnWaiter = null;
w?.();
}
// The streaming "channel in": an async generator that stays open and yields a
// user turn whenever the panel sends one. The session idles between messages
// and wakes the moment send() pushes — solving "can't wake an idle session".
// ONE batch is released per turn (counter gate) so the backend can't read ahead.
// Yields PROVIDER-NEUTRAL turns ({text, images}); the backend shapes them into
// its native user message (Claude resolves the image refs to inline blocks).
async *channel() {
while (!this.closed) {
if (this.queue.length === 0) {
// Idle & settled (we only reach here after the prior turn's gate opened):
// reset the counters to 0. Keeps them small and SELF-HEALS any drift a
// post-interrupt stray result may have introduced during a busy burst.
this.yieldedTurns = 0;
this.completedTurns = 0;
await new Promise((resolve) => {
this.waiting = resolve;
});
}
if (this.closed)
return;
// Drain the WHOLE queue into ONE turn — rapid-fire follow-ups are handled
// together (I see them all and reply once) rather than one-per-turn. Each
// message is now actually being taken off the queue: fire onSeen for every
// mid so all their bubbles flip from queued/muted to read at once.
const batch = this.queue.splice(0, this.queue.length);
if (batch.length === 0)
continue;
for (const it of batch) {
if (it.mid)
this.deps.onSeen?.(this.tabId, it.mid);
}
// Coerce each part before joining: a structured payload that slipped past
// ingress would otherwise stringify to "[object Object]" here (#175).
let text = batch.map((it) => promptText(it.text)).join("\n\n");
let images = batch.flatMap((it) => it.images ?? []);
if (images.length && !this.backend.capabilities.vision) {
// Text-only backend: every non-vision adapter silently ignores image
// refs, which reads to the user as "it ignored my screenshot". Say so
// visibly, and tell the MODEL too so it can't pretend it saw them.
this.deps.onSay(this.tabId, `📎 This provider is text-only, so I can't see the ${images.length > 1 ? "images" : "image"} you attached. ` +
`Describe what's in it, or switch to a vision-capable provider (Claude, Codex, or Gemini) and re-send.`);
text +=
`\n\n[panel note: the user attached ${images.length} image(s), but this provider is text-only and you CANNOT see them. ` +
`Do not claim or imply you saw them — if the content matters, ask the user to describe it or switch to a vision-capable provider.]`;
images = [];
}
if (this.closed)
return;
// Remember the in-flight turn's user text so an interrupt mid-reply can
// re-queue it (send-now must address BOTH the interrupted and new message).
this.inFlight = { text, ...(images.length ? { images } : {}) };
this.yieldedTurns += 1; // this batch is turn N
// Mark the turn in flight AT DISPATCH (not on the first event). Without this
// the watchdog's `busy` guard would be false for the exact zero-event freeze
// it's meant to catch, so onTurnStalled() would no-op. (handleEvent's later
// `busy = true` becomes a harmless no-op.) Also shows "working" immediately.
this.errorSurfaced = false; // fresh turn → its failure (if any) is unreported
this.interruptRequested = false; // a stale interrupt can't mute THIS turn's failure
this.busy = true;
this.deps.onTurn?.(this.tabId, "working");
// Arm the freeze watchdog AT DISPATCH: a turn that produces NO events at all
// (the exact ROOT CAUSE B freeze — turn/start sent, no notifications ever
// returned) never reaches handleEvent, so arming here is what catches it.
// Subsequent events re-arm it (handleEvent → bumpIdleWatchdog); a clean
// result disarms it.
this.bumpIdleWatchdog();
yield { text, ...(images.length ? { images } : {}) };
// Hold the next batch until THIS turn completes. Race-free: if the result
// already fired (completedTurns caught up) we don't park at all, so the
// channel can never deadlock and strand later messages.
while (!this.closed && this.completedTurns < this.yieldedTurns) {
await new Promise((resolve) => {
this.turnWaiter = resolve;
});
}
}
}
/**
* Run the persistent session, SELF-RESTARTING when the SDK session ends on its
* own (idle/error). The input queue (`this.queue`) and `sessionId` survive
* across restarts, so pending messages aren't lost and context resumes — this
* is the durable fix for the "connected but dead" wedge (previously a session
* that ended left a dead agent and every later message queued into a channel
* that was never read). Resolves only on stop() or after repeated immediate
* failures (gives up + tells the user). Safe to call once.
*/
async start(resumeSessionId) {
let quickRestarts = 0;
// Preflight the backend (e.g. lazy-load the optional SDK) OUTSIDE the restart
// loop so a HARD startup failure (missing dependency, bad runtime) surfaces
// immediately as a clear reject — instead of being caught as a "dropped
// session", retried four times, and reported as "session keeps ending".
await this.backend.prepare?.();
while (!this.closed) {
// A pending rewind (one-shot) forks the session at its anchor; otherwise
// resume normally.
const rewind = this.pendingRewind;
this.pendingRewind = null;
// A fresh fork (anchor === null) must NOT resume anything — not even the
// resumeSessionId start() was called with — or it silently continues the old
// session instead of starting clean. (requestRewind(null) clears sessionId;
// this also suppresses the resumeSessionId fallback.)
const resume = rewind?.anchor === null ? undefined : (this.sessionId ?? resumeSessionId);
const startedAt = Date.now();
// Set below if the backend fails because the resume target is gone; keeps a
// recoverable resume-miss from counting toward the give-up threshold.
let resumeMiss = false;
// Fresh channel → reset the turn-gate counters so a restart/resume never
// inherits a stale offset that would mis-gate the first batch.
this.yieldedTurns = 0;
this.completedTurns = 0;
this.turnWaiter = null;
// Drop the prior session's last assistant UUID so a fork can't report a
// stale (pre-fork) anchor for the first turn of the new session.
this.lastAssistantUuid = null;
// A fresh channel won't have an in-flight turn — clear any stale capture so
// a post-restart interrupt can't re-queue a dead session's message.
this.inFlight = null;
try {
// Drive the injected backend: it builds the provider session (resume/fork),
// shapes the neutral channel into native turns, and yields canonical events.
for await (const ev of this.backend.run({
channel: this.channel(),
model: this.model,
...(this.effort ? { effort: this.effort } : {}),
...(resume ? { resume } : {}),
sessionId: this.sessionId,
rewindAnchor: rewind?.anchor ?? null,
// LIVENESS: re-arm the freeze watchdog on ANY sign the backend is alive —
// not just translated AgentEvents. A long Codex tool call (panel_run →
// a multi-minute ComfyUI generation) emits raw app-server notifications
// throughout but may translate to NO AgentEvents during the wait; without
// this the watchdog would falsely trip on a HEALTHY generation and
// interrupt the turn. handleEvent() still bumps on real events; this
// covers the silent-but-working gap between them. A genuine zero-event
// freeze fires neither path, so the real-stall catch is preserved.
onActivity: () => this.bumpIdleWatchdog(),
})) {
this.handleEvent(ev);
}
}
catch (err) {
if (this.closed)
break;
const emsg = msgOf(err);
logger.error(`[panel-agent ${this.short()}] stream error: ${emsg}`);
// A resume whose target session no longer exists — e.g. the orchestrator was
// relaunched from a different cwd, or the session transcript was pruned —
// fails with "No conversation found with session ID: <id>". Current Codex
// can instead report "no rollout found for thread id <uuid>" for the same
// pruned/missing-target condition (#277). Retrying the SAME resume just
// loops until the give-up threshold trips and self-exits the whole
// orchestrator (a live bridge left serving a permanently-dead agent). Drop
// the dead resume target so the NEXT iteration starts a FRESH session and
// self-heals; queued messages (this.queue) survive and replay.
// GUARD (#278): only treat this as a recoverable resume-MISS when this
// start actually RESUMED something. A FRESH start (no resume) that emits
// the same text is a real, non-recoverable failure and MUST count toward
// the rapid-restart give-up bound below — otherwise resumeMiss would reset
// the counter forever and spin an infinite restart loop.
if (resume && /(?:No conversation found with session ID|no rollout found for thread id)/i.test(emsg)) {
logger.warn(`[panel-agent ${this.short()}] resume target is gone — starting a fresh session`);
this.sessionId = null;
resumeSessionId = undefined;
resumeMiss = true;
}
// SELF-HEAL: a crash mid-turn (the SDK process dying, e.g. code
// 4294967295) leaves the triggering message unprocessed. Resuming the
// session finds the turn already recorded as ended, so it produces empty
// "success" turns and the user's request is silently EATEN. Re-queue the
// in-flight message so the restarted/fresh session actually re-runs it.
// Idempotent enough: a duplicate render beats a lost request, and the
// quickRestarts give-up guard still bounds a message that crash-loops.
if (this.inFlight) {
const interrupted = this.inFlight;
this.inFlight = null;
this.queue.unshift(interrupted);
logger.warn(`[panel-agent ${this.short()}] crash mid-turn — re-queued the interrupted message so it isn't lost`);
}
}
// Session ended (cleanly or via error) — disarm any armed watchdog AND the
// interrupt-release fallback so a stale timer from the dead session can't fire
// into the restarted one (the restart resets the gate counters to 0, so a
// leftover fallback armed for old turn N would force-release the new session's
// first turn — gate run-ahead). The gate counters are reset next iteration.
this.clearIdleWatchdog();
this.clearInterruptReleaseFallback();
if (this.closed)
break;
// Session ended on its own — bound rapid failure loops so a persistently
// broken SDK doesn't spin forever or black-hole each message.
// A recoverable resume-miss (handled above by dropping the dead session) must
// NOT count toward the give-up threshold — the next iteration starts fresh.
quickRestarts = !resumeMiss && Date.now() - startedAt < 5000 ? quickRestarts + 1 : 0;
if (quickRestarts >= 4) {
logger.error(`[panel-agent ${this.short()}] session keeps ending immediately — giving up`);
this.sessionId = null; // don't resume a session that won't stay up
// Flag the fatal give-up so the manager → orchestrator can self-exit and
// let the pack respawn a clean orchestrator (root-cause fix for the
// "bridge open but no panel agent responded" wedge: a live orchestrator
// serving a permanently-dead agent). We DON'T onSay the old "Disconnect →
// Connect" nudge here — the orchestrator is about to exit and the panel's
// sticky reconnect respawns automatically.
this.gaveUp = true;
break;
}
logger.warn(`[panel-agent ${this.short()}] session ended — restarting${this.sessionId ? " (resume)" : ""}`);
await new Promise((r) => setTimeout(r, 250));
}
logger.info(`[panel-agent ${this.short()}] stopped`);
}
/** (Re)arm the per-turn idle watchdog. Called on every event received while a
* turn could be in flight, so the timer only fires after a FULL idle window
* with no events — a true stall. A no-op once a turn has already tripped (until
* the next turn re-arms via a fresh event after clearIdleWatchdog). */
bumpIdleWatchdog() {
if (this.closed || this.idleTripped)
return;
if (this.idleTimer)
clearTimeout(this.idleTimer);
this.idleTimer = setTimeout(() => this.onTurnStalled(), TURN_IDLE_MS);
this.idleTimer.unref?.();
}
/** Disarm the idle watchdog (turn ended cleanly, or session restarting). */
clearIdleWatchdog() {
if (this.idleTimer) {
clearTimeout(this.idleTimer);
this.idleTimer = null;
}
this.idleTripped = false;
// The turn ended — drop any tool-busy state so a start whose matching end was
// never seen (errored/interrupted turn) can't defer the NEXT turn's watchdog.
this.openToolCalls = 0;
this.toolBusySince = 0;
}
/** The current turn produced NO events for the whole idle window → it's frozen.
* Surface a clear error, clear the "working" indicator, advance the turn-gate
* so the next queued batch can run, and best-effort interrupt the backend so a
* wedged child stops. Idempotent per turn via idleTripped; completeTurn() is
* capped at yieldedTurns so a late real result can't double-advance the gate. */
onTurnStalled() {
if (this.closed || this.idleTripped || !this.busy)
return;
// A tool call in flight is legitimate work even with a silent app-server (an
// MCP tool call has no progress notification between its start and end). Defer
// and re-arm rather than kill a healthy long tool call — UNLESS a tool has
// been open past TOOL_BUSY_MAX_MS, in which case it's more likely wedged than
// slow and we fall through to the real trip. (#307 review finding.)
if (this.openToolCalls > 0 && Date.now() - this.toolBusySince < toolBusyMaxMs()) {
this.bumpIdleWatchdog();
return;
}
this.idleTripped = true;
this.idleTimer = null;
logger.error(`[panel-agent ${this.short()}] turn stalled — no events for ${Math.round(TURN_IDLE_MS / 1000)}s; surfacing error and releasing the gate`);
this.deps.onSay(this.tabId, "⚠️ The agent stopped responding (the turn stalled with no activity). I've cleared it — please try again.");
this.busy = false;
// The stalled turn is abandoned + surfaced as an error — don't re-queue its
// text (a wedged message could otherwise loop on every interrupt).
this.inFlight = null;
this.completeTurn(); // release the next queued batch instead of hanging
this.deps.onTurn?.(this.tabId, "done");
// Best-effort: stop the wedged turn so the backend doesn't keep a dead child
// half-alive. The self-restart loop in start() recovers the session.
void this.backend.interrupt().catch(() => { });
}
// Handle a canonical AgentEvent from the backend. This is the provider-agnostic
// half of what used to be route(SDKMessage): all the panel orchestration (turn
// gate, busy/working indicator, anchor tracking, usage meter, onSay commit)
// lives here; the backend already normalized the provider's native messages.
handleEvent(ev) {
// Any event means the turn is alive — reset the idle watchdog. The `result`
// case below disarms it entirely (turn ended). Placed before the switch so it
// covers every event type without per-case bumps.
this.bumpIdleWatchdog();
switch (ev.type) {
case "session": {
this.sessionId = ev.sessionId;
if (ev.model)
this.model = ev.model;
this.deps.onSession?.(this.tabId, ev.sessionId);
logger.info(`[panel-agent ${this.short()}] init model=${ev.model} session=${ev.sessionId.slice(0, 8)} effort=${this.effort ?? "default"}`);
break;
}
case "thinking": {
// Live extended-thinking token count → drives a "thinking… (N)" meter
// so the user can see the agent reasoning (not stuck) before any text.
this.busy = true;
this.deps.onTurn?.(this.tabId, "working");
this.deps.onThinking?.(this.tabId, ev.tokens);
break;
}
case "stream_start": {
// Live partial output (includePartialMessages). The backend already
// decoded the raw stream events; group deltas + the final commit by id.
if (!this.deps.onStream)
break;
this.streamMsgId = ev.id;
break;
}
case "assistant_delta": {
if (!this.deps.onStream)
break;
const id = this.streamMsgId;
if (!id)
break;
this.deps.onTurn?.(this.tabId, "working");
this.deps.onStream(this.tabId, { phase: ev.thinking ? "think" : "text", id, delta: ev.text });
break;
}
case "stream_end": {
if (!this.deps.onStream)
break;
if (this.streamMsgId)
this.deps.onStream(this.tabId, { phase: "end", id: this.streamMsgId });
this.streamMsgId = null;
break;
}
case "assistant": {
// Still working — keep the panel's indicator alive through the turn.
this.busy = true;
this.deps.onTurn?.(this.tabId, "working");
// Remember this message's UUID — it's the rewind anchor for the turn.
if (typeof ev.uuid === "string")
this.lastAssistantUuid = ev.uuid;
// Each assistant API response carries the CURRENT context size — report
// it live so the meter updates throughout the turn, not just at the end.
if (ev.usage) {
this.lastUsage = ev.usage;
this.reportStatus(ev.usage);
}
// Commit the authoritative reply text as ONE message. With streaming on,
// the panel already showed a live preview (matched by this message id);
// the commit replaces it with the final text. Without streaming (or for
// injected events), it just renders a normal bubble.
if (ev.text) {
this.deps.onSay(this.tabId, ev.text, { id: ev.id, streamed: true });
}
break;
}
case "tool_call": {
// Tool visibility — surface the agent's actions (a canvas-less mobile
// client otherwise sees only a spinner between reply bubbles). Only the
// START phase is forwarded as a compact activity line.
this.busy = true;
this.deps.onTurn?.(this.tabId, "working");
if (ev.phase === "start") {
// Enter "tool busy": while a tool runs the app-server can be silent for
// longer than the idle window, so onTurnStalled() defers rather than
// trips (bounded by TOOL_BUSY_MAX_MS). Stamp the start of the run on the
// 0 → 1 edge only, so several overlapping tools share one deadline.
if (this.openToolCalls === 0)
this.toolBusySince = Date.now();
this.openToolCalls += 1;
this.deps.onToolCall?.(this.tabId, ev.name);
}
else {
this.openToolCalls = Math.max(0, this.openToolCalls - 1);
}
break;
}
case "error": {
// A backend-reported turn error (codex/gemini/grok emit these before
// their error result). Without this case the message fell through
// `default` and the user watched a turn end in TOTAL silence — the
// exact "three Hellos into the void" failure from the support thread.
// Surface it as a visible chat line; the follow-up `result` event still
// advances the turn gate normally.
const detail = typeof ev.message === "string" && ev.message.trim() ? ev.message.trim() : "unknown error";
this.errorSurfaced = true;
this.deps.onSay(this.tabId, `⚠️ The ${this.model} turn failed: ${detail}\n\nNothing was lost — try again, switch models from the composer picker, or check the terminal running the orchestrator for more detail.`);
logger.warn(`[panel-agent ${this.short()}] backend error: ${detail}`);
break;
}
case "result": {
// Cache the context window + cost from the result, then re-report using
// the last assistant usage (the true current context).
if (ev.contextWindow && ev.contextWindow > this.contextWindow) {
this.contextWindow = ev.contextWindow;
}
if (this.lastUsage)
this.reportStatus(this.lastUsage, ev.costUsd);
this.busy = false;
// Turn completed → nothing to re-queue on a later interrupt. (A clean
// completion must NOT have its message re-queued.)
this.inFlight = null;
// Turn ended cleanly → disarm the freeze watchdog. (If it already tripped,
// completeTurn() is a capped no-op, so the gate can't double-advance.)
this.clearIdleWatchdog();
this.completeTurn(); // turn finished → release the next queued batch
// Report this turn's anchor (last assistant UUID) so the panel can later
// fork the conversation here for a rewind.
if (this.lastAssistantUuid)
this.deps.onTurnAnchor?.(this.tabId, this.lastAssistantUuid);
this.deps.onTurn?.(this.tabId, "done");
// Failed turn that never produced a visible error (the claude SDK path
// reports failures only via the result subtype; the `error` event case
// covers codex/gemini/grok) → say SOMETHING. A turn must never end in
// silence, error turns included. EXCEPT a turn the user just interrupted
// (Stop / send-now): its error-subtype result is the interrupt landing,
// not a failure.
const wasInterrupted = this.interruptRequested;
this.interruptRequested = false; // one result consumes the flag
if ((ev.ok === false || /error/i.test(ev.subtype ?? "")) && !this.errorSurfaced && !wasInterrupted) {
this.errorSurfaced = true;
this.deps.onSay(this.tabId, `⚠️ That turn failed (${ev.subtype ?? "error"}) without a reply. ` +
`Nothing was lost — try again, switch models from the composer picker, or check the orchestrator terminal for detail.`);
}
logger.info(`[panel-agent ${this.short()}] turn done (subtype=${ev.subtype})`);
break;
}
default:
break;
}
}
/** Push a context/usage snapshot derived from a single API response's usage.
* `used` is that response's PROMPT size (fresh + cached input) = the current
* context fill — NOT cumulative, and NOT including output tokens. */
reportStatus(usage, costUsd) {
if (!this.deps.onStatus)
return;
try {
const used = (usage.input_tokens ?? 0) +
(usage.cache_read_input_tokens ?? 0) +
(usage.cache_creation_input_tokens ?? 0);
const status = {
used,
model: this.model,
...(this.contextWindow
? { contextWindow: this.contextWindow, contextPct: used / this.contextWindow }
: {}),
...(typeof costUsd === "number" ? { costUsd } : {}),
};
this.lastStatus = status;
this.deps.onStatus(this.tabId, status);
}
catch (err) {
logger.debug(`[panel-agent ${this.short()}] usage report failed: ${msgOf(err)}`);
}
}
}
/** Owns one PanelAgent per tab id, spawned lazily on the tab's first message. */
export class PanelAgentManager {
agents = new Map();
opts;
/** Per-tab session id to resume on the next spawn (reload restore). */
pendingResume = new Map();
/** Held mail (issue #256): messages queued into an agent that then FAILED to
* start (prepare() rejected before the channel was ever consumed) are
* captured here at settle(err) instead of dying with the agent — the next
* spawn on the same composite key re-delivers them, so nothing sent into the
* spawn → prepare()-reject window is silently dropped. */
heldMessages = new Map();
/** Tabs whose effort changed mid-turn — the session restart is deferred to the
* next idle moment so we never interrupt (and silently drop) a live reply. */
pendingEffortRestart = new Set();
/** Tabs awaiting a comfyui-MCP-env respawn (a tool secret was saved). Value is
* an optional nudge to enqueue after the resumed agent comes back (e.g. "retry
* the download"). Applied at the next idle so the saving turn finishes first. */
pendingMcpRestart = new Map();
/** Default model/effort for newly-spawned agents (the env/config defaults). */
model;
effort;
/** Per-key model/effort OVERRIDE set by the picker (set_options). Keyed by the
* COMPOSITE agent key `tabId::backend`, so a model/effort chosen for one
* provider NEVER bleeds into another: a Codex "gpt-5.5" pick must not become the
* Claude spawn's model (which errors "model gpt-5.5 may not exist"). A provider
* switch calls reset(oldKey), which drops that key's override, so the new
* backend falls back to its OWN default. A same-provider reconnect reuses the
* same key, so the user's pick persists. */
modelByKey = new Map();
effortByKey = new Map();
constructor(opts) {
this.opts = opts;
this.model = opts.model;
this.effort = opts.effort;
}
/** The model a newly-spawned agent for `tabId` should use: the per-key override
* (picker) when set for THIS key, else the shared default. */
modelFor(tabId) {
return this.modelByKey.get(tabId) ?? this.model;
}
/** The effort a newly-spawned agent for `tabId` should use: the per-key override
* (picker) when set for THIS key, else the shared default. */
effortFor(tabId) {
return this.effortByKey.has(tabId) ? this.effortByKey.get(tabId) : this.effort;
}
/** The picker's model OVERRIDE for this composite key (`tabId::backend`), if
* any — undefined when the key runs the shared default. Lets the models
* push report the model this tab will ACTUALLY spawn with as `current`,
* instead of the backend default the override supersedes. */
modelOverrideFor(tabId) {
return this.modelByKey.get(tabId);
}
makeAgent(tabId) {
// Inject the toggle-selected backend (Codex) when provided; otherwise the
// PanelAgent constructor defaults to ClaudeBackend (existing behavior).
const backend = this.opts.makeBackend?.(tabId);
return new PanelAgent(tabId, {
// The factory (fresh per spawn — re-reads closures AND per-tab state like
// the Blind gate) wins over the static set (kept for tests/back-compat).
mcpServers: this.opts.makeMcpServers?.(tabId) ?? this.opts.mcpServers,
comfyuiUrl: this.opts.comfyuiUrl,
systemAppend: this.opts.systemAppend,
model: this.modelFor(tabId),
effort: this.effortFor(tabId),
onSay: this.opts.onSay,
onStream: this.opts.onStream,
onStatus: this.opts.onStatus,
// Persist the session id to our durable store (resume-after-restart) BEFORE
// forwarding it to the panel — so it's on disk the moment the SDK reports it.
onSession: (id, sid) => {
this.opts.sessionStore?.set(id, sid);
this.opts.onSession?.(id, sid);
},
onTurnAnchor: this.opts.onTurnAnchor,
// Wrap onTurn so the manager learns when a turn ends — the safe point to
// apply a deferred, session-restarting effort change.
onTurn: (id, state) => {
this.opts.onTurn?.(id, state);
// The safe point to apply any deferred session-restart (effort change
// and/or comfyui-MCP-env respawn). COALESCED into a single replacement so
// an agent is never restarted twice in a row (which would lose the resume
// id of the just-spawned, not-yet-session'd agent).
if (state === "done")
this.applyPendingRestarts(id);
},
onThinking: this.opts.onThinking,
onToolCall: this.opts.onToolCall,
onSeen: this.opts.onSeen,
panelServer: this.opts.makePanelServer?.(tabId),
pluginPath: this.opts.pluginPath,
}, backend);
}
/** Update the system-prompt append used for NEWLY-spawned agents. Lets the
* orchestrator refresh the live ENVIRONMENT-CAPABILITIES block (e.g. after a
* ComfyUI restart where Triton/SageAttention may now be installed) without
* rebuilding the manager. Already-running agents keep their original prompt
* until they next respawn (a soft reload / new session). */
setSystemAppend(systemAppend) {
this.opts.systemAppend = systemAppend;
}
/** Update the MCP server set used for NEWLY-spawned agents. The orchestrator
* calls this after a tool secret is saved so the rebuilt comfyui server env
* (now carrying the secret) is what the next spawn passes. Already-running
* agents keep their current env until they respawn — drive that with
* restartAllForMcpEnv() so the live comfyui MCP subprocess is recreated. */
setMcpServers(mcpServers) {
this.opts.mcpServers = mcpServers;
}
/** Update the ComfyUI URL used for NEWLY-spawned agents (image-byte fetching).
* Codex/Gemini backends read the URL via the orchestrator's per-spawn env, so a
* restartAllForMcpEnv() after this points every provider at the new target. */
setComfyuiUrl(comfyuiUrl) {
this.opts.comfyuiUrl = comfyuiUrl;
}
/** Whether a live agent (session) exists for this composite key — used to flag
* the mobile mirror picker's "session attached" dot. */
hasLiveAgent(key) {
return this.agents.has(key);
}
/** Respawn every active tab's agent (resume + carry-over) so the live comfyui
* MCP subprocess is recreated with the updated env. Deferred to each tab's
* next idle so the turn that SAVED the secret finishes first (we never
* interrupt a live reply). `nudge`, if given, is enqueued to each resumed
* agent so it auto-continues (e.g. retries the download the secret unblocked). */
restartAllForMcpEnv(nudge) {
for (const tabId of this.agents.keys()) {
this.pendingMcpRestart.set(tabId, nudge ?? null);
// Apply immediately when the tab is already idle; otherwise it fires on the
// next turn-done via applyPendingRestarts().
this.applyPendingRestarts(tabId);
}
}
/** Single-tab variant of restartAllForMcpEnv — used when ONE tab's tool-server
* spawn env changed (e.g. the Blind toggle, issue #90). Same coalesced
* at-idle replacement; no-op when the tab has no live agent. */
restartForMcpEnv(key, nudge) {
if (!this.agents.has(key))
return;
this.pendingMcpRestart.set(key, nudge ?? null);
this.applyPendingRestarts(key);
}
/** Rebuild a live agent because its PROVIDER credential rotated (#278): keyed
* backends (OpenRouter/Custom/GLM/Kimi/Moonshot) capture the API key at
* construction, so a rotated/revoked key only takes effect on a fresh spawn.
* Reuses the coalesced at-idle restart (a fresh makeBackend reads the new
* env key) WITHOUT a retry nudge — this is a silent key refresh, not the
* download-401 flow. No-op when the tab has no live agent. */
restartForProviderKey(key) {
this.restartForMcpEnv(key);
}
/**
* Apply any deferred session-restart for a tab once it's idle — COALESCING a
* pending effort change and a pending comfyui-MCP-env respawn into ONE
* replacement. Both are session-construction changes (effort + mcpServers) that
* the manager has already stored on itself, so a single spawn picks up both.
*
* Doing them separately would restart the agent twice in a row: the first spawn
* resumes from the OLD agent's session id, but the SECOND would resume from the
* just-spawned agent whose session id hasn't been emitted yet (null) — silently
* dropping the conversation. Coalescing replaces the original agent exactly once
* with the correct resume id and fires the retry nudge a single time.
*
* No-op unless something is pending and the agent has fully settled (idle).
*/
applyPendingRestarts(tabId) {
const wantEffort = this.pendingEffortRestart.has(tabId);
const wantMcp = this.pendingMcpRestart.has(tabId);
if (!wantEffort && !wantMcp)
return;
const agent = this.agents.get(tabId);
if (!agent || agent.isStopped) {
this.pendingEffortRestart.delete(tabId);
this.pendingMcpRestart.delete(tabId);
return;
}
// Still mid-work (a queued message will start the next turn) — wait for the
// next idle so we don't restart between back-to-back turns.
if (agent.isBusy || agent.hasPending)
return;
// Only the MCP respawn carries a retry nudge.
const nudge = wantMcp ? (this.pendingMcpRestart.get(tabId) ?? undefined) : undefined;
this.pendingEffortRestart.delete(tabId);
this.pendingMcpRestart.delete(tabId);
const carried = this.restartAgentResume(tabId, agent, nudge);
const reasons = [wantEffort ? "effort" : null, wantMcp ? "comfyui-mcp-env" : null]
.filter(Boolean)
.join("+");
logger.info(`[panel-orchestrator] tab ${tabId.slice(0, 8)} restart applied (idle, reason=${reasons}, ${carried} queued carried over${nudge ? " + retry nudge" : ""})`);
}
/** Cancel a still-queued message for a tab (user edited/deleted it before the
* agent read it). Returns true if it was removed from the queue. Also reaches
* HELD mail (issue #256 follow-up): after a failed start parks messages in
* heldMessages there is no live agent, but the user can still delete/edit the
* bubble — without this the cancelled prompt would execute on the next spawn
* (possibly alongside its replacement). */
cancelQueued(tabId, mid) {
if (this.agents.get(tabId)?.cancelQueued(mid))
return true;
const held = this.heldMessages.get(tabId);
if (held) {
const i = held.findIndex((item) => item.mid === mid);
if (i >= 0) {
held.splice(i, 1);
if (held.length === 0)
this.heldMessages.delete(tabId);
return true;
}
}
return false;
}
/** Replace a tab's agent with a fresh one (picks up the manager's current
* model/effort/mcpServers), resuming the conversation and carrying over any
* unsent queued messages. `nudge`, if given, is enqueued after the carried-over
* messages so the resumed agent auto-continues. Returns how many were carried. */
restartAgentResume(tabId, oldAgent, nudge) {
const resume = oldAgent.sessionId ?? undefined;
const pending = oldAgent.takePending();
const fresh = this.spawn(tabId, resume); // new agent owns the tab now
for (const item of pending)
fresh.send(item.text, { images: item.images, mid: item.mid });
if (nudge)
fresh.send(nudge);
void oldAgent.stop(); // retire the old one; it's no longer mapped
return pending.length;
}
/** Last usage snapshot for a tab's agent (for re-pushing the meter on connect). */
lastStatusFor(tabId) {
return this.agents.get(tabId)?.lastStatus ?? null;
}
/** Feed a ComfyUI execution event to an EXISTING agent (no-op if none — we
* never spawn an agent just to react to an event). Returns whether delivered. */
injectEvent(tabId, ev) {
const agent = this.agents.get(tabId);
if (!agent || agent.isStopped)
return false; // best-effort; don't enqueue into a closed agent
agent.injectEvent(ev);
return true;
}
/** Push a ComfyUI execution error to a tab's agent — interrupt the live turn
* and front-queue the error so the agent stops and addresses it. */
async injectRunError(tabId, error) {
const agent = this.agents.get(tabId);
if (!agent || agent.isStopped)
return false;
await agent.injectRunError(error);
return true;
}
spawn(tabId, resume) {
const agent = this.makeAgent(tabId);
this.agents.set(tabId, agent);
logger.info(`[panel-orchestrator] spawning agent for tab ${tabId.slice(0, 8)}${resume ? " (resume)" : ""} (${this.agents.size} active)`);
// Re-deliver mail orphaned by a previous FAILED start on this key (issue
// #256): the doomed agent's still-queued messages were captured at
// settle(err) below; queue them into the fresh agent FIRST so they run
// ahead of whatever message triggered this spawn (chronological order).
// If this start fails too, settle(err) captures them right back — no loss.
const held = this.heldMessages.get(tabId);
if (held?.length) {
this.heldMessages.delete(tabId);
for (const item of held)
agent.send(item.text, { images: item.images, mid: item.mid });
logger.info(`[panel-orchestrator] tab ${tabId.slice(0, 8)} re-delivering ${held.length} message(s) held from the previous failed start`);
}
// start() now SELF-RESTARTS internally on session-end, so it only settles on
// an intentional stop() or after it gives up (repeated immediate failures),
// or it rejects on a hard start failure. In the give-up / reject cases, drop
// the dead agent (if still mapped and not stopped on purpose) so the next
// user message spawns a fresh one.
const settle = (err) => {
if (agent.isStopped)
return;
// Locate the agent by IDENTITY, not by its spawn-time key (issue #255):
// rebindAgent() may have moved a still-starting agent to a NEW key (panel
// tab-id migration), and the old key-based guard (agents.get(tabId) !==
// agent) early-returned in that case — a prepare()-rejected agent then
// stayed mapped under the new key forever: hasLiveAgent true, queue never
// drained, no warning, tab wedged until reset. The failure must clean up
// (and report onStartFailure for) whatever key CURRENTLY maps this agent.
let key;
for (const [k, a] of this.agents) {
if (a === agent) {
key = k;
break;
}
}
if (key === undefined)
return; // already replaced/dropped — nothing to settle
const gaveUp = agent.gaveUp;
this.agents.delete(key);
if (err) {
const m = msgOf(err);
logger.error(`[panel-agent ${key.slice(0, 8)}] failed to start: ${m}`);
// Held mail (issue #256): prepare() rejected before the channel was
// ever consumed, so EVERY message sent since the spawn is still in the
// agent's queue — including the one that triggered it. Capture them
// (BEFORE stop(), which closes the agent) for re-delivery by the next
// spawn on this key, so nothing dies silently with the doomed agent.
const orphaned = agent.takePending();
if (orphaned.length) {
this.heldMessages.set(key, [...(this.heldMessages.get(key) ?? []), ...orphaned]);
logger.warn(`[panel-orchestrator] tab ${key.slice(0, 8)} holding ${orphaned.length} undelivered message(s) — re-delivered on the next successful start`);
}
// PER-TAB degradation (issue #250): a hard start failure is almost
// always a tab-local configuration error — an invalid API key (the
// endpoint 401s in prepare()), an unreachable base URL, a missing CLI
// login. It must degrade THIS tab only. The old escalation
// (onAgentFatal → orchestrator self-exit) killed every OTHER tab too,
// including healthy sessions on different providers. The agent slot was
// cleared above, so after the user fixes the key, the next message /
// Disconnect → Connect spawns a fresh agent on the same tab.
// Report under the CURRENT key (not the spawn-time tabId) so the
// orchestrator's composite-key → panel-tab split reaches the tab that
// owns the agent NOW, even after a rebind (issue #255).
if (this.opts.onStartFailure)
this.opts.onStartFailure(key, m);
else
this.opts.onSay(key, `⚠️ The panel agent could not start: ${m}`);
// Insurance, not correctness: every current backend self-cleans when
// prepare() throws, but that's per-backend convention — stop() makes
// backend.close?.() a guarantee now that the process SURVIVES this
// path (it used to exit, which mooted cleanup).
void agent.stop().catch(() => { });
}
else if (gaveUp) {
// The bounded self-restart loop gave up — the session keeps dropping. Same
// fatal signal: let the orchestrator self-exit + respawn.
this.opts.onAgentFatal?.(key, "agent session kept dropping (self-restart gave up)");
}
};
void agent.start(resume).then(() => settle(), (err) => settle(err));
return agent;
}
/** Rewind a tab's conversation: fork the live session at `anchor` (dropping
* everything after). The edited message follows as the next user_message.
* Returns false if no live agent. */
rewind(tabId, anchor) {
const agent = this.agents.get(tabId);
if (!agent || agent.isStopped)
return false;
agent.requestRewind(anchor);
return true;
}
/** Reorder a tab's still-queued messages to the panel's desired flush order.
* Also applies to HELD mail (a failed start parked the queue — the panel can
* still drag bubbles while the tab is degraded), with the same stable-sort
* semantics as PanelAgent.reorderQueue. */
reorderQueue(tabId, order) {
let applied = false;
const held = this.heldMessages.get(tabId);
if (Array.isArray(order) && held && held.length >= 2) {
const rank = new Map(order.map((mid, i) => [mid, i]));
const at = (mid) => (mid && rank.has(mid) ? rank.get(mid) : Number.MAX_SAFE_INTEGER);
held.sort((a, b) => at(a.mid) - at(b.mid));
applied = true;
}
const agent = this.agents.get(tabId);
if (!agent || agent.isStopped)
return applied;
agent.reorderQueue(order);
return true;
}
/** Record a session id to resume when this tab next spawns (reload restore). */
setResume(tabId, sessionId) {
if (this.agents.has(tabId))
return; // a live agent already owns the session
this.pendingResume.set(tabId, sessionId);
}
/**
* Rebind an agent from its current (stale) tabId to a new tabId — recovers
* from a panel tab-id scheme migration (e.g. random UUID → deterministic
* tmp:/wf: prefixed ids) without losing the conversation. The agent's panel
* tools still carry the old tabId; the UiBridge's tabMigrations map (see
* ui-bridge.ts resolveTarget) redirects those calls to the new connection.
* Returns true if an agent was rebound.
*/
rebindAgent(oldKey, newKey) {
if (oldKey === newKey)
return false;
if (this.agents.has(newKey))
return false;
// DURABLE state migrates even when no live agent exists (codex review:
// the agent may have been reaped while its persisted session survives —
// skipping this meant the rebound tab started a fresh conversation).
const moveDurable = () => {
if (this.pendingResume.has(oldKey)) {
this.pendingResume.set(newKey, this.pendingResume.get(oldKey));
this.pendingResume.delete(oldKey);
}
const persisted = this.opts.sessionStore?.get(oldKey);
if (persisted)
this.opts.sessionStore?.set(newKey, persisted);
this.opts.sessionStore?.clear(oldKey);
// Held mail from a failed start migrates too (issue #256) — it exists
// precisely when NO live agent does, so it must move in the durable pass
// for the rebound tab's next spawn to re-deliver it. Old-key mail is
// older, so it goes ahead of anything already held under the new key.
const heldOld = this.heldMessages.get(oldKey);
if (heldOld?.length) {
this.heldMessages.set(newKey, [...heldOld, ...(this.heldMessages.get(newKey) ?? [])]);
}
this.heldMessages.delete(oldKey);
};
const agent = this.agents.get(oldKey);
if (!agent || agent.isStopped) {
moveDurable();
return false;
}
this.agents.delete(oldKey);
this.agents.set(newKey, agent);
// has()-based moves throughout (codex review): truthy checks dropped
// legitimate null/undefined sentinels (e.g. restart-without-nudge, an
// explicit effort-override clear).
if (this.pendingEffortRestart.has(oldKey)) {
this.pendingEffortRestart.delete(oldKey);
this.pendingEffortRestart.add(newKey);
}
if (this.pendingMcpRestart.has(oldKey)) {
this.pendingMcpRestart.set(newKey, this.pendingMcpRestart.get(oldKey));
this.pendingMcpRestart.delete(oldKey);
}
if (this.modelByKey.has(oldKey)) {
this.modelByKey.set(newKey, this.modelByKey.get(oldKey));
this.modelByKey.delete(oldKey);
}
if (this.effortByKey.has(oldKey)) {
this.effortByKey.set(newKey, this.effortByKey.get(oldKey));
this.effortByKey.delete(oldKey);
}
moveDurable();
return true;
}
/** Route a panel message to its tab's agent, creating the agent if needed.
* Never routes into a stopped agent (whose channel is closed) — respawns so
* the message reaches a live session. */
send(tabId, text, meta) {
let agent = this.agents.get(tabId);
if (agent?.isStopped) {
this.agents.delete(tabId);
agent = undefined;
}
if (!agent) {
// Resume order: the orchestrator's OWN durable copy wins — it is the source
// of truth and the only survivor when the process was killed and respawned
// (a wedge restart). The panel's `hello.resume` is only a HINT, used when we
// have no record (e.g. a brand-new orchestrator whose disk was wiped while a
// panel still holds a session id). Making the panel authoritative is what
// caused the reconnect "flip-flop": a stale/duplicate panel claim could
// override the session the orchestrator is actually holding. The store is
// keyed per (tab, backend), so a provider switch finds no entry here and
// correctly starts fresh (the panel replays the transcript to seed it).
const resume = this.opts.sessionStore?.get(tabId) ?? this.pendingResume.get(tabId);
this.pendingResume.delete(tabId);
agent = this.spawn(tabId, resume);
}
agent.send(text, { title: meta?.title, images: meta?.images, mid: meta?.mid });
}
/**
* Apply a model/effort change for a tab. Model switches live (SDK setModel).
* Effort has no live setter, so it needs a fresh resumed session — but we NEVER
* do that mid-turn (it would interrupt and silently drop the in-flight reply,
* which read as "the agent stopped responding"). If a turn is running, the
* restart is deferred to the next idle moment (applyPendingRestarts); if idle,
* it happens now. Either way the model change is applied live immediately.
* `restarted` is true only when the session was actually recreated in this call.
*/
async setOptions(tabId, next) {
const changes = [];
let restarted = false;
let deferred = false;
if (typeof next.model === "string" && next.model && next.model !== this.modelFor(tabId)) {
// Per-KEY override (tabId::backend) so this pick can't poison a different
// provider's spawn — the switch-then-error bug (Codex gpt-5.5 → Claude).
this.modelByKey.set(tabId, next.model);
changes.push(`model=${next.model}`);
}
// null clears effort back to the SDK default; undefined leaves it untouched.
let effortChanged = false;
if (next.effort !== undefined) {
const nextEffort = next.effort ?? undefined;
if (nextEffort !== this.effortFor(tabId)) {
this.effortByKey.set(tabId, nextEffort);
effortChanged = true;
changes.push(`effort=${nextEffort ?? "default"}`);
}
}
const agent = this.agents.get(tabId);
if (agent) {
// Apply a model change live regardless of the effort path (so a deferred
// effort restart doesn't hold up the model switch).
if (typeof next.model === "string" && next.model) {
await agent.setModel(next.model);
}
if (effortChanged) {
// Mark the restart pending, then let the COALESCING applier decide: if the
// agent is idle it restarts now (folding in any pending comfyui-MCP-env
// respawn + its nudge as a single replacement); if mid-turn it defers to
// the next turn-done. This guarantees the agent is never restarted twice.
this.pendingEffortRestart.add(tabId);
const busy = agent.isBusy || agent.hasPending;
this.applyPendingRestarts(tabId);
if (busy) {
deferred = true;
}
else {
restarted = true;
}
}
}
if (changes.length) {
logger.info(`[panel-orchestrator] tab ${tabId.slice(0, 8)} options: ${changes.join(" ")}${deferred ? " (effort restart deferred to idle)" : ""}`);
}
return { model: this.modelFor(tabId), effort: this.effortFor(tabId), restarted, deferred };
}
/** Forget a tab's agent so the next message starts a brand-new session. The
* map mutation is synchronous and the old agent is stopped fire-and-forget,
* so the caller (e.g. resume_session) can set a new pendingResume right after
* without a concurrent send() spawning a non-resumed agent in an await gap. */
reset(tabId) {
const agent = this.agents.get(tabId);
this.agents.delete(tabId);
this.pendingResume.delete(tabId);
// Forget the durable session too — a NEW chat must start fresh, so the disk
// fallback in send() can't resurrect the conversation the user just cleared.
// (resume_session calls reset() then setResume() with the chosen id, so the
// historical session is re-armed right after and re-persisted on next onSession.)
this.opts.sessionStore?.clear(tabId);
this.pendingEffortRestart.delete(tabId); // a reset supersedes any deferred restart
this.pendingMcpRestart.delete(tabId);
this.heldMessages.delete(tabId); // a reset is an explicit fresh start — drop held mail
// Drop this key's picker override so a provider switch (which reset()s the old
// key) can't carry the old provider's model/effort into the new backend's spawn.
this.modelByKey.delete(tabId);
this.effortByKey.delete(tabId);
if (agent) {
logger.info(`[panel-orchestrator] tab ${tabId.slice(0, 8)} reset — new session next message`);
void agent.stop();
}
}
async interrupt(tabId, opts = {}) {
await this.agents.get(tabId)?.interrupt(opts);
}
async stopAll() {
this.pendingEffortRestart.clear();
this.pendingMcpRestart.clear();
this.heldMessages.clear();
await Promise.all([...this.agents.values()].map((a) => a.stop()));
this.agents.clear();
}
count() {
return this.agents.size;
}
/** True when any message is parked in the held-mail map (a start failure
* captured a doomed agent's queue for re-delivery, issue #256). Surfaced so
* the self-restart gate can refuse to restart while mail is parked —
* teardown (stopAll) ERASES held mail, so an auto-restart while it exists
* would silently drop the very messages the hold protects. */
hasHeldMail() {
for (const msgs of this.heldMessages.values()) {
if (msgs.length > 0)
return true;
}
return false;
}
/** True when this key's agent has a turn in flight (or messages queued to
* start one imminently) — i.e. the panel's working spinner belongs to a REAL
* turn that will push its own turn:"done". The orchestrator's held-during-gen
* branch checks this before clearing the spinner, so a tab-wide turn:"done"
* can never hide an ACTIVE turn's spinner or disarm its resume nudge. */
isTurnActive(key) {
const agent = this.agents.get(key);
return !!agent && !agent.isStopped && (agent.isBusy || agent.hasPending);
}
/** True when NO agent is mid-turn or holding queued messages AND no failed-
* start mail is parked for re-delivery — the only moment a self-restart may
* replace the process without eating a reply (or erasing held mail). */
allIdle() {
for (const a of this.agents.values()) {
if (a.isBusy || a.hasPending)
return false;
}
return !this.hasHeldMail();
}
get defaults() {
return { model: this.model, effort: this.effort };
}
}
//# sourceMappingURL=panel-agent.js.map