comfyui-mcp
Version:
Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),
497 lines • 21.6 kB
JavaScript
// Claude Agent SDK backend — the provider-specific adapter behind the
// AgentBackend port. It owns everything that knows about
// `@anthropic-ai/claude-agent-sdk`: the lazy SDK import, the `query()` call,
// Claude `Options` building (including forkSession/resumeSessionAt/resume),
// model/command enumeration, the live model setter, interrupt, and the
// normalization of `SDKMessage` → canonical `AgentEvent`.
//
// PanelAgent keeps all provider-agnostic orchestration (queue, turn-gate, bridge
// push, rewind-anchor tracking, self-restart) and drives this backend via
// `for await (const ev of backend.run({...}))`. See
// docs/design/agent-backend-injection.md.
import { logger } from "../utils/logger.js";
import { errorText } from "./error-text.js";
import { buildAgentSpawnEnv } from "../services/panel-secrets.js";
import { CLAUDE_CAPABILITIES, } from "./agent-backend.js";
// ---- reasoning effort mapping ----
// Effort is now a provider-neutral union (it must survive a provider switch — see
// panel-agent.ts Effort). The Agent SDK only accepts the Claude scale
// (low|medium|high|xhigh|max), so map any off-scale neutral value (Codex's
// "none"/"minimal") to the nearest valid Claude level. Shared levels pass through.
const CLAUDE_EFFORTS = new Set(["low", "medium", "high", "xhigh", "max"]);
function toClaudeEffort(effort) {
if (!effort)
return undefined;
const e = effort.toLowerCase();
if (CLAUDE_EFFORTS.has(e))
return e;
if (e === "none" || e === "minimal")
return "low"; // Codex's sub-low → Claude's floor
return undefined; // unknown → SDK default
}
// The Agent SDK is an OPTIONAL dependency (it pulls in ~100 packages and is only
// needed for the panel orchestrator), so load it lazily and fail with a clear
// message rather than at import time for everyone.
let queryFn = null;
export async function loadQuery() {
if (queryFn)
return queryFn;
try {
const mod = await import("@anthropic-ai/claude-agent-sdk");
queryFn = mod.query;
}
catch {
throw new Error("The panel orchestrator requires the optional dependency @anthropic-ai/claude-agent-sdk. Install it with: npm i @anthropic-ai/claude-agent-sdk");
}
return queryFn;
}
function msgOf(err) {
return errorText(err);
}
/**
* Ask the SDK which models the current account can actually use — so the panel's
* picker reflects the live subscription instead of a hardcoded list. This is the
* only model-enumeration path that works on the subscription/OAuth lane:
* `query.supportedModels()` (the public Models API and `claude` CLI both require
* an API key, which we deliberately don't have here).
*
* Runs a minimal throwaway session — no MCP servers, no plugins/skills — so it's
* cheap; the control request resolves right after init, then we tear it down.
* Returns [] on any failure so the panel can fall back gracefully.
*/
export async function fetchSupportedModels(model) {
const query = await loadQuery();
let stop = false;
let wake = null;
// An idle channel: keeps the control connection open without ever consuming a
// turn, until we tear it down.
async function* idle() {
while (!stop) {
await new Promise((resolve) => {
wake = resolve;
});
}
}
const q = query({
prompt: idle(),
options: {
model,
permissionMode: "bypassPermissions",
allowDangerouslySkipPermissions: true,
strictMcpConfig: true,
mcpServers: {},
// SECURITY: the SDK spawns a Claude Code subprocess; `env` REPLACES its
// environment entirely. Pass the agent env (process.env MINUS tool-only
// secrets) so RunPod/HF/CivitAI tokens never reach the LLM subprocess.
// Claude auth is OAuth-file-based (ANTHROPIC_API_KEY is deliberately unset).
env: buildAgentSpawnEnv(),
},
});
// The transport is pumped by iterating the query — do it in the background so
// the control request (supportedModels) gets its response.
const drain = (async () => {
try {
for await (const _ of q) {
void _;
}
}
catch {
// torn down below
}
})();
let timer;
try {
// Never hang forever: a stuck probe would leave a permanently-pending
// cached promise and the panel would never get its model list.
const models = await Promise.race([
q.supportedModels(),
new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error("supportedModels timed out")), 20000);
}),
]);
logger.info(`[panel-orchestrator] supportedModels: ${models.map((m) => m.value).join(", ") || "(none)"}`);
return models;
}
catch (err) {
logger.warn(`[panel-orchestrator] supportedModels probe failed: ${msgOf(err)}`);
return [];
}
finally {
if (timer)
clearTimeout(timer);
stop = true;
// Cast re-widens: TS narrows the closure-mutated local back to its init value.
wake?.();
try {
await q.interrupt();
}
catch {
// already winding down
}
void drain;
}
}
/**
* Probe the SDK for the slash commands this account/session exposes (built-ins
* like /compact, plus any loaded skills) so the panel can surface them in the
* composer's completion menu. Same throwaway-session pattern as
* fetchSupportedModels; returns [] on any failure so the panel degrades cleanly.
*/
export async function fetchSupportedCommands(model) {
const query = await loadQuery();
let stop = false;
let wake = null;
async function* idle() {
while (!stop) {
await new Promise((resolve) => {
wake = resolve;
});
}
}
const q = query({
prompt: idle(),
options: {
model,
permissionMode: "bypassPermissions",
allowDangerouslySkipPermissions: true,
strictMcpConfig: true,
mcpServers: {},
// SECURITY: agent env only (no tool-only secrets) — see fetchSupportedModels.
env: buildAgentSpawnEnv(),
},
});
const drain = (async () => {
try {
for await (const _ of q) {
void _;
}
}
catch {
// torn down below
}
})();
let timer;
try {
const commands = await Promise.race([
q.supportedCommands(),
new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error("supportedCommands timed out")), 20000);
}),
]);
logger.info(`[panel-orchestrator] supportedCommands: ${commands.map((c) => "/" + c.name).join(", ") || "(none)"}`);
return commands;
}
catch (err) {
logger.warn(`[panel-orchestrator] supportedCommands probe failed: ${msgOf(err)}`);
return [];
}
finally {
if (timer)
clearTimeout(timer);
stop = true;
wake?.();
try {
await q.interrupt();
}
catch {
// already winding down
}
void drain;
}
}
/**
* The Claude Agent SDK adapter. One instance per PanelAgent; it holds the live
* `Query` for the current session and re-creates it on each `run()`.
*/
export class ClaudeBackend {
id = "claude";
capabilities = CLAUDE_CAPABILITIES;
deps;
q = null;
constructor(deps) {
this.deps = deps;
}
/** Preflight: lazy-load the Agent SDK once so a missing optional dependency
* fails fast (a clear "install @anthropic-ai/claude-agent-sdk" reject) rather
* than being retried as a dropped session inside the restart loop. Idempotent. */
async prepare() {
await loadQuery();
}
/** Fetch a ComfyUI image and wrap it as an Anthropic base64 image block, or
* null on any failure (the text reference still names it as a fallback). */
async fetchImageBlock(ref) {
if (!this.deps.comfyuiUrl || !ref?.filename)
return null;
try {
const u = new URL("/view", this.deps.comfyuiUrl);
u.searchParams.set("filename", ref.filename);
u.searchParams.set("type", ref.type || "input");
if (ref.subfolder)
u.searchParams.set("subfolder", ref.subfolder);
const res = await fetch(u, { signal: AbortSignal.timeout(15000) });
if (!res.ok)
return null;
let mt = (res.headers.get("content-type") || "").split(";")[0].trim().toLowerCase();
if (!["image/png", "image/jpeg", "image/gif", "image/webp"].includes(mt)) {
mt = "image/png"; // Anthropic-supported set; ComfyUI outputs are PNG by default
}
const buf = Buffer.from(await res.arrayBuffer());
if (buf.length > 12 * 1024 * 1024)
return null; // keep context sane
return { type: "image", source: { type: "base64", media_type: mt, data: buf.toString("base64") } };
}
catch {
return null;
}
}
/** Shape one provider-neutral turn into a Claude `SDKUserMessage`, resolving
* image refs to inline base64 blocks so the agent SEES them in this turn (no
* view_image/get_image round-trip). */
async shapeTurn(turn) {
const text = turn.text;
const images = turn.images ?? [];
let content = text;
if (images.length) {
const blocks = [];
for (const ref of images) {
const b = await this.fetchImageBlock(ref);
if (b)
blocks.push(b);
}
if (blocks.length)
content = [{ type: "text", text }, ...blocks];
}
return {
type: "user",
message: { role: "user", content },
parent_tool_use_id: null,
};
}
buildOptions(opts) {
const model = opts.model;
const effort = toClaudeEffort(opts.effort);
const resume = opts.resume;
const rewindAnchor = opts.rewindAnchor;
// Rewind: fork the conversation at the anchor (resume up to that message, then
// branch into a new session id) so everything after it is dropped. A null
// anchor means "fresh session" (handled by clearing resume below).
const rewindOpts = rewindAnchor
? { resume: opts.sessionId ?? resume, resumeSessionAt: rewindAnchor, forkSession: true }
: null;
return {
model,
permissionMode: "bypassPermissions",
// Required alongside bypassPermissions (intentional, isolated background agent).
allowDangerouslySkipPermissions: true,
// Stream partial assistant messages so the panel can show thinking + reply
// text live (token-by-token) instead of only the final block. route() turns
// these into onStream deltas; the final assistant message still commits the
// authoritative text via onSay (reconciled by message id).
includePartialMessages: true,
mcpServers: {
...this.deps.mcpServers,
// Live-graph control of THIS tab's open workflow (in-process; talks to
// the bridge). Lets the agent build on what the user sees.
...(this.deps.panelServer ? { panel: this.deps.panelServer } : {}),
},
// Only our comfyui MCP — never inherit the user's project/user MCP config
// (which may run a second comfyui that grabs the bridge port).
strictMcpConfig: true,
systemPrompt: {
type: "preset",
preset: "claude_code",
append: this.deps.systemAppend,
},
// Reasoning effort, when the picker has set one.
...(effort ? { effort } : {}),
// Load the bundled comfyui-mcp plugin so the agent has model expertise
// (IDEOGRAM/WAN/LTX/Qwen/… skills) out of the box — "install the package
// = expert agent". Omitted if the plugin dir can't be found.
...(this.deps.pluginPath
? {
plugins: [{ type: "local", path: this.deps.pluginPath }],
skills: "all",
}
: {}),
...(rewindOpts ?? (resume ? { resume } : {})),
// SECURITY: the SDK spawns a Claude Code subprocess; `env` REPLACES its
// environment entirely. Pass the agent env (process.env MINUS tool-only
// secrets — RunPod/HF/CivitAI tokens) so tool credentials never reach the
// LLM subprocess. Claude auth is OAuth-file-based (no env key needed).
env: buildAgentSpawnEnv(),
};
}
/**
* Open the Claude session and yield canonical AgentEvents. The user channel
* (PanelAgent's gated queue) is shaped turn-by-turn into `SDKUserMessage`s by
* `shapeTurn`; the SDK's `SDKMessage`s are normalized into `AgentEvent`s.
*/
async *run(opts) {
const query = await loadQuery();
const self = this;
// Wrap the neutral channel: shape each turn into the SDK's native form right
// before it's read, so PanelAgent never deals in SDKUserMessage.
async function* prompt() {
for await (const turn of opts.channel) {
yield await self.shapeTurn(turn);
}
}
const q = query({ prompt: prompt(), options: this.buildOptions(opts) });
this.q = q;
for await (const message of q) {
// LIVENESS: every SDKMessage — including the ones route() doesn't translate
// (tool-progress, rate-limit, etc.) — is a sign the session is alive, so
// re-arm PanelAgent's idle watchdog. Claude already streams continuously, so
// this is effectively a no-op for behavior here; it keeps the watchdog's
// re-arm source uniform across both backends via the port.
opts.onActivity?.();
yield* this.route(message);
}
}
/** Switch the model live (the SDK applies it to the next turn). */
async setModel(model) {
try {
// setModel is live: no session restart, the next turn uses it.
await this.q?.setModel?.(model);
}
catch (err) {
logger.debug(`[claude-backend] setModel: ${msgOf(err)}`);
}
}
async interrupt() {
await this.q?.interrupt();
}
/** Permanently dispose of the live SDK query. The Agent SDK has no explicit
* "dispose" beyond interrupt(), which both stops the in-flight turn and lets
* the underlying transport wind down once the prompt generator is no longer
* iterated (PanelAgent.stop() closes the channel before calling this). We then
* drop our reference so the query can be GC'd. Idempotent + safe when never
* started (q is null) — and a true no-op for Claude's behavior: stop() already
* called interrupt(), so this only releases the reference. */
async close() {
const q = this.q;
this.q = null;
if (!q)
return;
try {
await q.interrupt();
}
catch {
// already winding down / never fully started
}
}
async listModels() {
return [];
}
/** Normalize one SDK message into zero or more canonical AgentEvents. */
*route(message) {
switch (message.type) {
case "system":
if (message.subtype === "init") {
yield {
type: "session",
sessionId: message.session_id,
...(message.model ? { model: message.model } : {}),
};
logger.info(`[panel-agent] init model=${message.model} session=${message.session_id.slice(0, 8)} apiKeySource=${message.apiKeySource} skills=${message.skills?.length ?? 0}`);
}
else if (message.subtype === "thinking_tokens") {
// Live extended-thinking token count → drives a "thinking… (N)" meter
// so the user can see the agent reasoning (not stuck) before any text.
const t = message.estimated_tokens;
if (typeof t === "number") {
yield { type: "thinking", tokens: t };
}
}
break;
case "stream_event": {
// Live partial output (includePartialMessages). Turn the raw Anthropic
// stream events into thinking/reply deltas the panel renders token-by-
// token. The authoritative text still commits via the `assistant` case.
const ev = message.event;
if (!ev)
break;
const evType = ev.type;
if (evType === "message_start") {
const mid = ev.message?.id;
yield { type: "stream_start", id: typeof mid === "string" ? mid : null };
}
else if (evType === "content_block_delta") {
const d = ev.delta;
if (!d)
break;
if (d.type === "thinking_delta" && typeof d.thinking === "string" && d.thinking) {
yield { type: "assistant_delta", text: d.thinking, thinking: true };
}
else if (d.type === "text_delta" && typeof d.text === "string" && d.text) {
yield { type: "assistant_delta", text: d.text };
}
}
else if (evType === "message_stop") {
yield { type: "stream_end" };
}
break;
}
case "assistant": {
// Remember this message's UUID — it's the rewind anchor for the turn.
const auid = message.uuid;
// Each assistant API response carries the CURRENT context size — report
// it live so the meter updates throughout the turn, not just at the end.
const u = message.message?.usage;
// Commit the authoritative reply text as ONE message. With streaming on,
// the panel already showed a live preview (matched by this message id);
// the commit replaces it with the final text. Without streaming (or for
// injected events), it just renders a normal bubble.
const content = (message.message?.content ?? []);
const text = content
.filter((b) => b.type === "text" && typeof b.text === "string")
.map((b) => b.text)
.join("\n\n")
.trim();
const id = message.message?.id;
yield {
type: "assistant",
text,
...(typeof auid === "string" ? { uuid: auid } : {}),
...(id ? { id } : {}),
...(u ? { usage: u } : {}),
};
// TOOL VISIBILITY: surface each tool the agent invoked as a tool_call
// event, so a canvas-less mobile client can show the agent's actions
// (not just a spinner). The panel infers these from its live canvas.
for (const b of content) {
if (b.type === "tool_use" && typeof b.name === "string") {
yield { type: "tool_call", name: b.name, phase: "start" };
}
}
break;
}
case "result": {
// Cache the context window + cost from the result, then re-report using
// the last assistant usage (the true current context).
const m = message;
let contextWindow;
for (const mu of Object.values(m.modelUsage ?? {})) {
if (mu?.contextWindow && (contextWindow === undefined || mu.contextWindow > contextWindow)) {
contextWindow = mu.contextWindow;
}
}
yield {
type: "result",
ok: message.subtype === "success",
subtype: message.subtype,
...(contextWindow !== undefined ? { contextWindow } : {}),
...(typeof m.total_cost_usd === "number" ? { costUsd: m.total_cost_usd } : {}),
};
break;
}
default:
// Parity with the original route(): other SDK message types (incl. SDK
// rate-limit and tool-progress events) are not surfaced — the panel never
// consumed them. The canonical AgentEvent union still declares `rate_limit`
// and `tool_call` for other backends / a future Claude enhancement to emit.
break;
}
}
}
//# sourceMappingURL=claude-backend.js.map