comfyui-mcp
Version:
Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),
208 lines • 12.3 kB
JavaScript
// Agent backend port — the provider-neutral seam that lets the panel orchestrator
// run on different agent providers (Claude Agent SDK today, OpenAI Codex next) via
// dependency injection. See design/agent-backend-injection.md.
//
// PanelAgent keeps the orchestration (queue, turn-gate, bridge push, rewind-anchor
// tracking, self-restart) and delegates the provider-specific bits — opening a
// session, normalizing the provider's message stream to canonical AgentEvents,
// interrupt, model enumeration, session resume/fork — to an injected AgentBackend.
/** Stamp every event of a per-turn stream with the backend-minted `turn` marker
* (see AgentEvent.turn). Backends mint one incrementing marker per turn read
* from the channel and wrap their per-turn generator: events inside a turn are
* stamped; a `session` event (outside any turn) passes through unstamped. */
export async function* stampTurn(stream, turn) {
for await (const ev of stream) {
yield ev.type === "session" ? ev : { ...ev, turn };
}
}
/** Capability descriptor for the Claude Agent SDK backend. */
export const CLAUDE_CAPABILITIES = {
persistentChannel: true,
streamingDeltas: true,
interruptMidTurn: true,
forkAtAnchor: true,
inProcessMcp: true,
modelEnumeration: true,
slashCommands: true,
hooks: true,
vision: true, // resolves image refs to inline base64 blocks (shapeTurn)
audio: false, // the Anthropic Messages API has no audio input block (#790)
turnMarkers: true, // claude-backend stamps every event from the #745 per-turn trace FIFO
};
/** Capability descriptor for the Codex app-server backend (Phase 2). */
export const CODEX_CAPABILITIES = {
persistentChannel: true, // thread + turn/start (resume by threadId)
streamingDeltas: true,
interruptMidTurn: true, // turn/interrupt
forkAtAnchor: false, // thread/resume is whole-thread only (for now)
inProcessMcp: false, // config-declared MCP servers only
modelEnumeration: true, // config/read
slashCommands: false,
hooks: false,
vision: true, // gpt-5.5 sees images; delivered as `localImage` turn input items
audio: false, // codex app-server turn input accepts text + localImage only (#790)
turnMarkers: true, // stampTurn() wraps each per-turn stream
};
/** Capability descriptor for the Gemini CLI ACP backend (Agent Client Protocol).
* Mirrors the Codex posture: a persistent session over a JSON-RPC-over-stdio
* client, streaming deltas + tool calls, interrupt via `session/cancel`, and
* config-declared MCP servers (no in-process SDK MCP). forkAtAnchor is false —
* ACP `session/load` is whole-session only, with no per-turn rewind anchor. */
export const GEMINI_CAPABILITIES = {
persistentChannel: true, // session/new + repeated session/prompt
streamingDeltas: true, // session/update agent_message_chunk / agent_thought_chunk
interruptMidTurn: true, // session/cancel
forkAtAnchor: false, // session/load is whole-session only (no anchor fork)
inProcessMcp: false, // ACP session/new declares config MCP servers only
modelEnumeration: true, // static catalog (gemini-2.5-pro / -flash) — ACP exposes no catalog
slashCommands: false,
hooks: false,
vision: true, // gemini-2.5 sees images; delivered as inline base64 image ContentBlocks
// ACP defines an `audio` ContentBlock and requires the agent to advertise the
// `audio` prompt capability before a client may send one. Neither CLI has been
// observed advertising it, so the send path could never be exercised — and its
// failure mode (session/prompt rejecting after the bytes were attached) would
// surface as a generic error, i.e. an attachment the user is never told did not
// arrive. Shipping that is the overclaim #790 exists to remove, so this is false
// and the attachment is refused centrally, naming a path that works.
audio: false,
turnMarkers: true, // stampTurn() wraps each per-turn stream
};
/** Capability descriptor for the Antigravity CLI backend (`agy`, issue #262) —
* Google's official replacement for the individual-tier Gemini CLI subscription
* path (Google AI Pro/Ultra). `agy` exposes no machine-readable event protocol,
* only plain-text `-p` print mode + `--continue` conversation continuity — so
* this is a spawn-per-turn adapter with honestly reduced capabilities:
* streamingDeltas=true is the CLI's own progressive stdout (final answer text
* only, no structured tool events), and vision=false (no documented -p image
* input). Models come LIVE from `agy models` (no static catalog). */
export const ANTIGRAVITY_CAPABILITIES = {
persistentChannel: true, // spawn-per-turn, continuity via `agy --continue`
streamingDeltas: true, // progressive stdout of the final answer
interruptMidTurn: true, // kill the in-flight child tree; next turn continues
forkAtAnchor: false, // agy owns conversation storage; no per-turn anchor
inProcessMcp: false, // workspace .agents/mcp_config.json only
modelEnumeration: true, // `agy models` (live account catalog)
slashCommands: false,
hooks: false,
vision: false, // no documented image input in -p mode
audio: false, // `agy -p` has no documented media input at all (#790)
turnMarkers: true, // stampTurn() wraps each per-turn stream
};
/** Capability descriptor for the pi.dev CLI backend (`pi`, issue #491) — the
* open-source multi-provider coding agent from earendil-works. Unlike agy, pi
* exposes a DOCUMENTED machine-readable JSON-lines event stream (`--mode json`)
* with real text deltas, per-tool events, and a resumable session id — so this
* is a spawn-per-turn adapter that parses that stream. inProcessMcp is false and
* there is NO config-MCP path either: pi has no MCP client at all (its tools are
* built-ins + TS extensions), so a pi turn does NOT get the ComfyUI panel_* /
* comfyui tools — a real limitation surfaced in the ready banner. vision=false
* (no documented headless image input). Models come from `pi --list-models`. */
export const PI_CAPABILITIES = {
persistentChannel: true, // spawn-per-turn, continuity via `pi --session <id>`
streamingDeltas: true, // `--mode json` text_delta events
interruptMidTurn: true, // kill the in-flight child tree; next turn continues
forkAtAnchor: false, // whole-session resume only (no per-turn anchor wired)
inProcessMcp: false, // pi has no MCP client at all (built-in tools + extensions)
modelEnumeration: true, // `pi --list-models`
slashCommands: false,
hooks: false,
vision: false, // no documented headless image input
audio: false, // pi's JSON mode has no documented media input (#790)
turnMarkers: true, // stampTurn() wraps each per-turn stream
};
/** Capability descriptor for the Grok CLI ACP backend (xAI / Grok Build).
* Same ACP posture as Gemini: persistent session, streaming deltas, interrupt,
* config-declared MCP servers, static model catalog at spawn. */
export const GROK_CAPABILITIES = {
persistentChannel: true,
streamingDeltas: true,
interruptMidTurn: true,
forkAtAnchor: false,
inProcessMcp: false,
modelEnumeration: true, // grok-composer-2.5-fast / grok-build — ACP exposes no catalog
slashCommands: false,
hooks: false,
vision: true,
// ACP defines an `audio` ContentBlock and requires the agent to advertise the
// `audio` prompt capability before a client may send one. Neither CLI has been
// observed advertising it, so the send path could never be exercised — and its
// failure mode (session/prompt rejecting after the bytes were attached) would
// surface as a generic error, i.e. an attachment the user is never told did not
// arrive. Shipping that is the overclaim #790 exists to remove, so this is false
// and the attachment is refused centrally, naming a path that works.
audio: false,
turnMarkers: true, // stampTurn() wraps each per-turn stream
};
/** Capability descriptor for the Qwen Code CLI ACP backend (`qwen --acp`,
* issue #1417). Same ACP posture as Gemini: persistent session, streaming
* deltas, interrupt via session/cancel, config-declared MCP servers, static
* model catalog pinned at spawn. */
export const QWEN_CAPABILITIES = {
persistentChannel: true, // session/new + repeated session/prompt
streamingDeltas: true, // session/update agent_message_chunk / agent_thought_chunk
interruptMidTurn: true, // session/cancel
forkAtAnchor: false, // session/load is whole-session only (no anchor fork)
inProcessMcp: false, // ACP session/new declares config MCP servers only
modelEnumeration: true, // static catalog — ACP exposes no catalog
slashCommands: false,
hooks: false,
vision: true, // delivered as inline base64 image ContentBlocks (when the agent advertises image input)
audio: false, // same overclaim rationale as GEMINI_CAPABILITIES.audio (#790)
turnMarkers: true, // stampTurn() wraps each per-turn stream
};
/** Capability descriptor for the Ollama local-LLM backend (issue #97's panel
* phase). Ollama is a stateless HTTP daemon, so the backend owns the whole
* agentic loop itself (stream /api/chat, dispatch tool calls, repeat) and
* keeps the conversation history in-memory — persistentChannel is true from
* the panel's perspective, but forkAtAnchor is out. Vision is true at the
* BACKEND level: whether images are actually understood is a per-MODEL
* property (gemma4 sees them, qwen3 doesn't; DeepSeek's API rejects them), so
* the backend always attempts delivery and degrades gracefully — an endpoint
* that rejects image input gets one retry with images stripped and an honest
* note to both the user and the model. */
export const OLLAMA_CAPABILITIES = {
persistentChannel: true, // in-memory history + repeated /api/chat
streamingDeltas: true, // NDJSON chunk stream
interruptMidTurn: true, // AbortController on the in-flight fetch
forkAtAnchor: false,
inProcessMcp: false, // MCP clients internally; the model sees the 6-tool router
modelEnumeration: true, // GET /api/tags (locally pulled models)
slashCommands: false,
hooks: false,
vision: true, // attempted for every model; graceful strip-and-retry on rejection
audio: true, // native /api/chat images[] and openai input_audio, both live-verified; per-model gate via /api/show (#790) plus verified-tag allowlist on the native image slot (#1972)
turnMarkers: true, // stampTurn() wraps each per-turn stream
};
/** ChatGPT subscription via direct Codex OAuth (~/.codex/auth.json) — Codex Responses
* HTTP backend (not the codex app-server CLI). Same 6-tool router as Ollama. */
export const CHATGPT_CAPABILITIES = {
persistentChannel: true,
streamingDeltas: true,
interruptMidTurn: true,
forkAtAnchor: false,
inProcessMcp: false,
modelEnumeration: true, // ~/.codex/models_cache.json + configured default
slashCommands: false,
hooks: false,
vision: true, // Responses input_image data URLs; strip-and-retry on rejection (#218)
audio: false, // the Codex Responses models take input_text/input_image only (#790)
turnMarkers: true, // stampTurn() wraps each per-turn stream
};
/** Kimi Code subscription OAuth or KIMI_API_KEY — OpenAI-compatible coding API. */
export const KIMI_CAPABILITIES = {
...OLLAMA_CAPABILITIES,
};
// NOTE: glm/moonshot no longer need dedicated *_CAPABILITIES consts — the generic
// api-key factory (services/openai-provider-registry + makeOpenAiKeyBackend in
// orchestrator/index.ts) builds a plain OllamaBackend, which already reports
// OLLAMA_CAPABILITIES (the value the old GLM_CAPABILITIES/MOONSHOT_CAPABILITIES
// spreads carried). Their BackendId union members are retained above.
/** GitHub Copilot chat via in-panel device-code OAuth — EXPERIMENTAL (ToS risk,
* see OAUTH_PROVIDERS.copilot). OpenAI-compatible chat/completions + the same
* 6-tool router as Ollama/GLM/Kimi. The exact contract (endpoints, headers) is
* UNVERIFIED offline — see copilot-backend.ts's module doc. */
export const COPILOT_CAPABILITIES = {
...OLLAMA_CAPABILITIES,
};
//# sourceMappingURL=agent-backend.js.map