UNPKG

comfyui-mcp

Version:

Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),

580 lines • 27 kB
// Passive ComfyUI render watchdog for the panel orchestrator. // // The orchestrator never sees live render progress on its own: panel_run queues // through the user's BROWSER, and the per-agent comfyui MCP only opens its WS for // its own generate calls. So a render that wedges (a single sampler step running // for minutes at high resolution) is invisible here — which is how a stalled job // once let the agent stack three more behind it before anyone noticed. // // This service opens its OWN lightweight WebSocket to COMFYUI_URL, and — on // modern ComfyUI — supplements it with a cheap HTTP poll (see poll() below). // The WS handlers keep the full event vocabulary for older servers, but on // ComfyUI 0.28 a passive (non-originating) client receives ONLY `status` // frames: execution_start / executing / execution_* AND progress / // progress_state are all sid-scoped to the QUEUING client (verified wire-level // on a live 0.28.0 — a foreign run delivers nothing but queue_remaining // transitions here). So run ATTRIBUTION for foreign jobs must come from HTTP: // • GET /queue — queue_running entries are [number, prompt_id, ...], // which names the running prompt (issue #258); // • GET /history tail — a run that starts AND finishes between polls still // lands in history, so diffing the newest ids yields a // completion event with success/error status even when // no live signal was ever observed (issue #259). // It holds the last-known run state and derives a stall/backlog report the // orchestrator surfaces to the agent as a turn-start note (the same channel as // the crash-dump injector). // // Everything here is BEST-EFFORT: if the socket can't open or drops, the report // is simply "inactive" and nothing in the orchestrator changes. It must never // throw into the main path. import WebSocket from "ws"; import { logger } from "../utils/logger.js"; import { getComfyUIAuthHeaders } from "../config.js"; import { comfyuiFetch } from "../comfyui/fetch.js"; const RECONNECT_MS = 5000; // /history tail entries fetched per poll. Wide enough that a realistic burst // of sub-second runs between 1 Hz polls stays inside the window; when a diff // still saturates it (every entry new), we log the potential gap instead of // silently claiming coverage. const HISTORY_TAIL_ITEMS = 32; class QueueMonitorImpl { ws = null; url = null; stopped = false; reconnectTimer = null; // Generation start/end transition hooks (for the Ollama VRAM pause). Fired on // the idle→running edge and the running→idle edge, best-effort (a throwing // handler must never break the monitor). `busy` is our own edge-tracking flag, // distinct from runningPromptId (which flips null between backlogged prompts). busy = false; onRunStart = null; onRunEnd = null; state = { connected: false, runningPromptId: null, currentNode: null, progressValue: null, progressMax: null, queueRemaining: 0, lastActivityTs: null, lastCompleted: null, }; // ---- HTTP-poll bookkeeping (the broadcast-safe channel on modern ComfyUI) ---- pollInFlight = false; // Bumped on every start()/stop(). A poll captures it before fetching and // abandons its (now stale) responses if a retarget happened while awaiting — // otherwise an in-flight /queue answer from the OLD ComfyUI could mutate // state for the NEW one. pollGeneration = 0; // When THIS monitor came up (ms epoch) — the priming cutoff for the history // diff: tail entries that completed before this predate us and are swallowed; // ones that completed after (a run finishing during startup) are reported. monitorStartTs = Date.now(); // /history tail diff: the ids seen on the previous poll. historyPrimed = false; historySeen = new Set(); // Prompt ids already reported as completed — dedupes the WS event vs. the // history diff observing the same finish. Bounded FIFO. completedReported = new Set(); // Completions not yet drained by the broadcaster. Bounded. pendingCompletions = []; /** Open the watchdog WS to ComfyUI. Idempotent per-URL; best-effort (never * throws). A retarget (new URL) or a prior stop() must re-open the socket: * the orchestrator calls stop()+start(newUrl) when ComfyUI is retargeted * (e.g. 127.0.0.1→localhost from a panel hello), so a stale `this.url` must * NOT early-return — that left the watchdog permanently disconnected. */ start(comfyuiUrl) { if (this.url === comfyuiUrl && !this.stopped) return; // already live on this URL this.stop(); // tear down any prior socket/reconnect timer (also on URL change) this.url = comfyuiUrl; this.stopped = false; // A (re)start may target a DIFFERENT ComfyUI whose history tail is all new // to us — invalidate any in-flight poll (generation bump), re-prime the // diff, and drop completion state that belonged to the old target so its // backlog can neither replay nor leak across the retarget. this.pollGeneration++; this.monitorStartTs = Date.now(); this.historyPrimed = false; this.historySeen.clear(); this.completedReported.clear(); this.pendingCompletions.length = 0; this.state.lastCompleted = null; this.connect(); } /** Register generation-transition handlers (idempotent overwrite). Called by * the orchestrator to unload/warm the local Ollama model around renders. */ setTransitionHandlers(h) { this.onRunStart = h.onRunStart ?? null; this.onRunEnd = h.onRunEnd ?? null; } emitStart() { if (this.busy) return; this.busy = true; try { this.onRunStart?.(); } catch (err) { logger.debug(`[queue-monitor] onRunStart threw: ${err instanceof Error ? err.message : String(err)}`); } } emitEndIfIdle() { // Only truly idle when nothing is running AND the queue is drained — between // backlogged prompts runningPromptId briefly clears but queueRemaining stays // positive, and we must NOT warm the model just to unload it again. if (!this.busy) return; if (this.state.runningPromptId !== null) return; if (this.state.queueRemaining > 0) return; this.busy = false; try { this.onRunEnd?.(); } catch (err) { logger.debug(`[queue-monitor] onRunEnd threw: ${err instanceof Error ? err.message : String(err)}`); } } /** Is a generation currently in flight (edge-tracked)? */ isBusy() { return this.busy; } stop() { this.stopped = true; this.pollGeneration++; // strand any in-flight poll's pending responses if (this.reconnectTimer) clearTimeout(this.reconnectTimer); this.reconnectTimer = null; try { this.ws?.close(); } catch { /* ignore */ } this.ws = null; // Clear the flag here rather than relying on the old socket's `close`: once // we null `this.ws`, that socket's now-superseded close handler early-returns // (this.ws !== ws) and would otherwise leave `connected` stuck true — through // a retarget's stop()+start() gap, or indefinitely if the reconnect fails. this.state.connected = false; } wsUrl() { // http(s)://host:port → ws(s)://host:port/ws?clientId=... const base = (this.url ?? "http://127.0.0.1:8188").replace(/^http/, "ws").replace(/\/+$/, ""); return `${base}/ws?clientId=comfyui-mcp-watchdog`; } connect() { if (this.stopped) return; let ws; try { // Ride the same auth as HTTP (COMFYUI_AUTH_* + Cloudflare Access service // token) on the WS handshake, so the watchdog reaches a ComfyUI behind a // proxy / CF Access. undefined when unauth'd → identical to `new WebSocket(url)`. const authHeaders = getComfyUIAuthHeaders(); ws = new WebSocket(this.wsUrl(), Object.keys(authHeaders).length ? { headers: authHeaders } : undefined); } catch (err) { logger.debug(`[queue-monitor] WS construct failed: ${err instanceof Error ? err.message : String(err)}`); this.scheduleReconnect(); return; } this.ws = ws; // Guard every handler against a superseded socket: on retarget, stop()+start() // opens a new socket while the old one is still async-closing. Without the // `this.ws !== ws` check the old socket's late `close` would null out the NEW // socket and schedule a spurious reconnect. ws.on("open", () => { if (this.ws !== ws) return; this.state.connected = true; logger.debug("[queue-monitor] watchdog WS connected"); }); ws.on("message", (raw, isBinary) => { if (this.ws !== ws) return; if (isBinary) return; // preview image frames — ignore this.onMessage(raw.toString()); }); ws.on("close", () => { if (this.ws !== ws) return; // a superseded socket closing — ignore this.state.connected = false; this.ws = null; this.scheduleReconnect(); }); ws.on("error", (err) => { logger.debug(`[queue-monitor] WS error: ${err.message}`); try { ws.close(); } catch { /* close handler schedules the reconnect */ } }); } scheduleReconnect() { if (this.stopped || this.reconnectTimer) return; this.reconnectTimer = setTimeout(() => { this.reconnectTimer = null; this.connect(); }, RECONNECT_MS); // Don't keep the process alive solely for the watchdog reconnect. this.reconnectTimer.unref?.(); } touchActivity() { this.state.lastActivityTs = Date.now(); } /** Adopt [promptId] as the running prompt when it's new — the broadcast-safe * substitute for the sid-scoped execution_start this client never receives * on modern ComfyUI. Fires the start transition exactly once per run. */ adoptRunningPrompt(promptId) { if (typeof promptId !== "string" || promptId === this.state.runningPromptId) return; this.state.runningPromptId = promptId; this.touchActivity(); this.emitStart(); } clearRunning() { this.state.runningPromptId = null; this.state.currentNode = null; this.state.progressValue = null; this.state.progressMax = null; this.state.lastActivityTs = null; } onMessage(text) { let msg; try { msg = JSON.parse(text); } catch { return; } const data = (msg.data ?? {}); switch (msg.type) { case "status": { const status = data.status; const execInfo = status?.exec_info; const qr = execInfo?.queue_remaining; if (typeof qr === "number") { this.state.queueRemaining = qr; // A status frame with an empty queue is ComfyUI's authoritative // "fully idle" signal. On modern ComfyUI (0.2x) the sid-scoped // executing/execution_success events never reach this passive // watchdog (see the progress_state case), so a run learned from // progress frames would otherwise never clear — drain it here. if (qr === 0) { if (this.state.runningPromptId !== null) this.clearRunning(); this.emitEndIfIdle(); } } break; } case "execution_start": { this.state.runningPromptId = typeof data.prompt_id === "string" ? data.prompt_id : null; this.state.currentNode = null; this.state.progressValue = null; this.state.progressMax = null; this.touchActivity(); this.emitStart(); break; } case "executing": { const node = data.node; if (node === null || node === undefined) { // ComfyUI sends node:null at the end of a prompt's execution. this.clearRunning(); this.emitEndIfIdle(); } else { const n = String(node); if (n !== this.state.currentNode) this.touchActivity(); // a new node = real progress this.state.currentNode = n; if (typeof data.prompt_id === "string") this.state.runningPromptId = data.prompt_id; } break; } case "progress": { const value = typeof data.value === "number" ? data.value : null; const max = typeof data.max === "number" ? data.max : null; // ONLY treat an advancing value as activity — a wedged step re-emits the // same value, and that must keep the stall clock running. if (value !== null && value !== this.state.progressValue) this.touchActivity(); this.state.progressValue = value; this.state.progressMax = max; if (typeof data.node === "string") this.state.currentNode = data.node; // progress IS broadcast to every client and carries the prompt_id — // adopt it, since the sid-scoped execution_start may never have arrived // (see the progress_state case below). this.adoptRunningPrompt(data.prompt_id); break; } case "progress_state": { // Modern ComfyUI (verified live on 0.28): execution_start / executing / // execution_success are sent ONLY to the client that queued the prompt, // so this passive watchdog never sees them — but progress_state IS // broadcast, fires from the first node on, and names the running // prompt + node. Derive the run state from it so browser-/agent-queued // renders stay visible here (running flag, prompt_id, current node). this.adoptRunningPrompt(data.prompt_id); const nodes = data.nodes; if (nodes && typeof nodes === "object") { for (const entry of Object.values(nodes)) { if (!entry || typeof entry !== "object") continue; const n = entry; if (n.state === "running" && typeof n.node_id === "string") { if (n.node_id !== this.state.currentNode) this.touchActivity(); // node advanced this.state.currentNode = n.node_id; } } } break; } case "execution_success": case "execution_error": case "execution_interrupted": { // Older ComfyUI broadcasts these to every client; modern 0.28 scopes // them to the originator (own runs still pass through here when the // orchestrator queued them). recordCompletion dedupes against the // /history diff seeing the same finish. if (typeof data.prompt_id === "string") { this.recordCompletion(data.prompt_id, msg.type === "execution_success" ? "success" : msg.type === "execution_interrupted" ? "interrupted" : "error"); } this.clearRunning(); this.emitEndIfIdle(); break; } default: break; } } /** Record one finished run exactly once (WS event and /history diff can both * observe the same finish). Completing the tracked running prompt also * clears the run state. Never throws. */ recordCompletion(promptId, status) { if (this.completedReported.has(promptId)) return; this.completedReported.add(promptId); // Bounded FIFO — Set iterates in insertion order, so drop the oldest. while (this.completedReported.size > 200) { const oldest = this.completedReported.values().next().value; if (oldest === undefined) break; this.completedReported.delete(oldest); } const ev = { promptId, status, at: Date.now() }; this.state.lastCompleted = ev; this.pendingCompletions.push(ev); // Bound must exceed the history tail (HISTORY_TAIL_ITEMS) so one saturated // diff still reaches the broadcaster whole; beyond that, drop the oldest. while (this.pendingCompletions.length > 2 * HISTORY_TAIL_ITEMS) this.pendingCompletions.shift(); if (this.state.runningPromptId === promptId) { this.clearRunning(); this.emitEndIfIdle(); } } /** Hand the not-yet-broadcast completions to the queue_status broadcaster * (each drains exactly once). */ drainCompletions() { if (this.pendingCompletions.length === 0) return []; return this.pendingCompletions.splice(0); } /** GET a JSON endpoint on the monitored ComfyUI. Best-effort: null on any * failure, hard 2.5s timeout so a wedged server can't pile up polls. */ async fetchJson(path) { if (!this.url) return null; const ctrl = new AbortController(); const timer = setTimeout(() => ctrl.abort(), 2500); timer.unref?.(); try { const res = await comfyuiFetch(`${this.url.replace(/\/+$/, "")}${path}`, { signal: ctrl.signal }); if (!res.ok) return null; return (await res.json()); } catch { return null; } finally { clearTimeout(timer); } } /** One HTTP poll tick — the broadcast-safe channel on modern ComfyUI, where * the passive WS carries no attribution (see header). Reads GET /queue for * the running prompt_id + true depth (#258) and diffs GET /history's tail * for runs that finished since the last poll — including runs that started * AND finished entirely between polls (#259). Best-effort, never rejects, * self-guards against overlap. Both fetches run in PARALLEL so the * worst-case poll is one timeout, not two — overlapping ticks bail at * pollInFlight, so a serial 5s worst case would stall attribution. */ async poll() { if (this.stopped || !this.url || this.pollInFlight) return; this.pollInFlight = true; const gen = this.pollGeneration; const fetchStart = Date.now(); try { const [q, h] = await Promise.all([ this.fetchJson("/queue"), this.fetchJson(`/history?max_items=${HISTORY_TAIL_ITEMS}`), ]); // A stop()/start() (retarget) happened while we were awaiting — these // responses belong to the OLD target; writing them would corrupt the // fresh state (and re-seed the old server's completions). if (gen !== this.pollGeneration || this.stopped) return; this.applyQueue(q, fetchStart); this.applyHistory(h); } catch (err) { logger.debug(`[queue-monitor] poll failed: ${err instanceof Error ? err.message : String(err)}`); } finally { this.pollInFlight = false; } } applyQueue(raw, fetchStart) { const q = raw; if (!q || typeof q !== "object") return; const running = Array.isArray(q.queue_running) ? q.queue_running : []; const pending = Array.isArray(q.queue_pending) ? q.queue_pending : []; this.state.queueRemaining = running.length + pending.length; // queue_running entries are [number, prompt_id, prompt, extra, outputs] — // the ONLY place a passive observer learns WHICH prompt runs on 0.28. const first = running[0]; if (Array.isArray(first) && typeof first[1] === "string") { this.adoptRunningPrompt(first[1]); } else if (running.length === 0 && this.state.runningPromptId !== null) { // Empty queue → the tracked run is over. Skip the clear if a run was // adopted AFTER this fetch began (the response would be stale for it). if ((this.state.lastActivityTs ?? 0) <= fetchStart) { this.clearRunning(); this.emitEndIfIdle(); } } else if (running.length === 0) { this.emitEndIfIdle(); } } /** Read one /history entry into diff-able facts. `queueNum` is ComfyUI's * monotonic queue counter (prompt[0]) — /history object order is NOT * chronological (see history-select.ts), so this is the only real order * key. `completedTs` is the newest server-side message timestamp (the end * event is always last), null when the entry carries none. */ parseHistoryEntry(id, raw) { const entry = raw; const st = entry && typeof entry === "object" ? entry.status : undefined; const messages = Array.isArray(st?.messages) ? st.messages : []; let status; if (st?.completed === true || st?.status_str === "success" || st === undefined) { status = "success"; } else if (messages.some((m) => Array.isArray(m) && m[0] === "execution_interrupted")) { status = "interrupted"; } else { status = "error"; } let completedTs = null; for (const m of messages) { if (!Array.isArray(m)) continue; const ts = m[1]?.timestamp; if (typeof ts === "number" && (completedTs === null || ts > completedTs)) completedTs = ts; } const prompt = entry && typeof entry === "object" ? entry.prompt : undefined; const queueNum = Array.isArray(prompt) && typeof prompt[0] === "number" ? prompt[0] : Number.MAX_SAFE_INTEGER; return { id, queueNum, status, completedTs }; } applyHistory(raw) { const h = raw; if (!h || typeof h !== "object" || Array.isArray(h)) return; const ids = Object.keys(h); if (!this.historyPrimed) { // First look after (re)start. Entries whose completion predates the // monitor are swallowed — but a run that finished DURING startup // (server-side timestamp after monitorStartTs) is a real completion the // subscribers must still see, not priming noise. (Timestamps are the // server's clock; against a remote host with heavy skew this degrades to // at worst a replayed or swallowed tail entry at startup — best-effort.) this.historyPrimed = true; this.historySeen = new Set(ids); const fresh = ids .map((id) => this.parseHistoryEntry(id, h[id])) .filter((e) => e.completedTs !== null && e.completedTs > this.monitorStartTs) .sort((a, b) => a.queueNum - b.queueNum); for (const e of fresh) this.recordCompletion(e.id, e.status); return; } const unseen = ids .filter((id) => !this.historySeen.has(id)) .map((id) => this.parseHistoryEntry(id, h[id])) // Oldest-first by ComfyUI's monotonic queue number, so the burst replays // in true order and lastCompleted lands on the genuinely newest run. .sort((a, b) => a.queueNum - b.queueNum); // Saturated window: EVERY entry of a full tail is new since the last // successful diff — completions may have scrolled past unobserved. Say so // instead of silently claiming full coverage. if (unseen.length >= HISTORY_TAIL_ITEMS && unseen.length === ids.length) { logger.warn(`[queue-monitor] history tail saturated (${unseen.length} new entries in one diff) — some run completions may have been missed between polls`); } for (const e of unseen) this.recordCompletion(e.id, e.status); this.historySeen = new Set(ids); } /** Cheap snapshot for backpressure (panel_run) and the live `queue_status` * broadcast (queue-status-broadcast.ts): is anything in flight, and where? */ snapshot() { return { connected: this.state.connected, running: this.state.runningPromptId !== null, runningPromptId: this.state.runningPromptId, queueDepth: Math.max(0, this.state.queueRemaining), currentNode: this.state.currentNode, progressValue: this.state.progressValue, progressMax: this.state.progressMax, lastCompleted: this.state.lastCompleted, }; } /** Stall/backlog report for the turn-start injector. */ report(stallMs) { const running = this.state.runningPromptId !== null; const queueDepth = Math.max(running ? 1 : 0, this.state.queueRemaining); const idleFor = running && this.state.lastActivityTs ? Date.now() - this.state.lastActivityTs : 0; const stalled = running && idleFor >= stallMs; const progress = this.state.progressValue !== null && this.state.progressMax !== null ? `${this.state.progressValue}/${this.state.progressMax}` : null; return { stalled, backlog: queueDepth > 1, runningPromptId: this.state.runningPromptId, currentNode: this.state.currentNode, queueDepth, stalledForMs: stalled ? idleFor : 0, progress, }; } } /** Process-wide singleton (one ComfyUI per orchestrator). */ export const QueueMonitor = new QueueMonitorImpl(); //# sourceMappingURL=queue-monitor.js.map