UNPKG

comfyui-mcp

Version:

Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),

477 lines • 20.7 kB
import { getObjectInfo } from "../comfyui/client.js"; import { enqueueWorkflow } from "./workflow-executor.js"; import { config } from "../config.js"; import { ValidationError } from "../utils/errors.js"; /** * API / partner-node support, mirroring `comfy-cli generate` (list / schema / run). * * MECHANISM & ASSUMPTIONS (confirmed against ComfyUI + comfy-cli source): * - These nodes run entirely on the *connected* ComfyUI server via its normal * HTTP API (/object_info + /prompt), so this works against remote servers too. * We reuse the existing client (getObjectInfo) and enqueue helper rather than * re-implementing anything. * - ComfyUI marks hosted partner/API nodes in /object_info with `api_node: true` * (server.py emits this from the node class's `API_NODE` attribute). Their * `category` also conventionally starts with "api node/" (e.g. * "api node/image/BFL", "api node/video/Kling"). We treat EITHER signal as a * match so older/newer ComfyUI builds both work. * Ref: comfyanonymous/ComfyUI server.py node_info() and comfy_api_nodes/*.py. * - AUTH: API nodes require a Comfy account / API key, but that credential is * supplied to the *ComfyUI server* out-of-band — it is injected by the server * into the node's HIDDEN inputs (`auth_token_comfy_org` / `api_key_comfy_org`) * from the logged-in session. The MCP client does NOT (and should not) place * the key in the workflow JSON. If the server isn't authenticated, the job * will be enqueued but fail server-side; we surface that as guidance rather * than trying to pass credentials ourselves. (Lower-confidence area — see * docs.comfy.org/tutorials/api-nodes.) */ const API_CATEGORY_PREFIX = "api node"; const defaultDeps = { getObjectInfo, enqueue: enqueueWorkflow, }; /** True if a node definition is a hosted partner/API node. */ export function isApiNode(def) { if (def.api_node === true) return true; const category = (def.category ?? "").toLowerCase(); return category === API_CATEGORY_PREFIX || category.startsWith(`${API_CATEGORY_PREFIX}/`); } /** * Discover hosted partner/API nodes available on the connected ComfyUI. * Optionally narrow by a case-insensitive substring matched against the * class_type, display name, or category (e.g. "image", "kling", "video"). */ export async function listApiNodes(filter, deps = defaultDeps) { const objectInfo = await deps.getObjectInfo(); const needle = filter?.trim().toLowerCase(); const results = []; for (const [classType, def] of Object.entries(objectInfo)) { if (!def || !isApiNode(def)) continue; const summary = { class_type: classType, display_name: def.display_name || classType, category: def.category ?? "", description: def.description ?? "", output: Array.isArray(def.output) ? def.output : [], deprecated: def.deprecated === true, experimental: def.experimental === true, }; if (needle) { const haystack = `${classType} ${summary.display_name} ${summary.category}`.toLowerCase(); if (!haystack.includes(needle)) continue; } results.push(summary); } results.sort((a, b) => a.class_type.localeCompare(b.class_type)); return results; } /** Collect the ids of every subgraph DEFINITION in a UI-format workflow. A * top-level node whose `type` equals one of these ids is a subgraph INSTANCE — a * local structural grouping, not a real ComfyUI node type — so it must NOT be * classified (it would otherwise look "unknown"/paid). The instance's real nodes * live in the definition and are walked separately. Returns the def map so the * caller can both skip instances and recurse into the definitions. */ function subgraphDefMap(g) { const m = new Map(); const defs = g.definitions?.subgraphs; if (Array.isArray(defs)) { for (const sg of defs) { const id = sg?.id; if (typeof id === "string" && id.length > 0) m.set(id, sg); } } return m; } /** Count of subgraph definitions in a (UI-format) workflow — 0 if none/unknown. */ export function countSubgraphs(graph) { if (!graph || typeof graph !== "object") return 0; return subgraphDefMap(graph).size; } /** * Extract the set of REAL node class_types referenced by a workflow, accepting * BOTH UI/litegraph format (top-level `nodes` array, each with a `type`) and * API/prompt format (top-level numeric keys, each an object with `class_type`). * * SUBGRAPH-AWARE: subgraph instances (a node whose `type` is a subgraph * definition id, i.e. a UUID) are NOT real node types — they're skipped, and the * instance's real inner nodes (from `definitions.subgraphs[].nodes`) are walked * instead (nested subgraphs included, since all definitions live in one flat * list). This keeps subgraph-heavy workflows from being mis-flagged "unknown" * while still classifying any API node nested inside a subgraph. */ export function extractWorkflowClassTypes(graph) { const out = new Set(); if (!graph || typeof graph !== "object") return []; const g = graph; // UI/litegraph format. if (Array.isArray(g.nodes)) { const defs = subgraphDefMap(g); const walk = (nodes) => { if (!Array.isArray(nodes)) return; for (const n of nodes) { const t = n?.type; if (typeof t !== "string" || t.length === 0) continue; if (defs.has(t)) continue; // subgraph instance — structural; its def is walked below out.add(t); } }; walk(g.nodes); for (const sg of defs.values()) walk(sg.nodes); // real nodes inside every subgraph def return [...out]; } // API/prompt format (no subgraph definitions). for (const v of Object.values(g)) { const ct = v?.class_type; if (typeof ct === "string" && ct.length > 0) out.add(ct); } return [...out]; } /** * Classify a workflow's runtime (local-GPU vs paid API nodes) by scanning its * node class_types against the connected ComfyUI's /object_info (the same signal * isApiNode uses). Works on UI or API/prompt format graphs. */ export async function checkWorkflowRuntime(graph, deps = defaultDeps) { const classTypes = extractWorkflowClassTypes(graph); const objectInfo = await deps.getObjectInfo(); const apiNodes = []; const unknownNodes = []; for (const ct of classTypes) { const def = objectInfo[ct]; if (!def) { unknownNodes.push(ct); continue; } if (isApiNode(def)) apiNodes.push(ct); } const hasApiNodes = apiNodes.length > 0; // "api" only if EVERY classifiable node is an API node; "mixed" if some are. const classifiable = classTypes.length - unknownNodes.length; let runtime; let usesApiNodes; if (hasApiNodes) { runtime = apiNodes.length >= classifiable && classifiable > 0 ? "api" : "mixed"; usesApiNodes = true; } else if (unknownNodes.length > 0) { // No recognized API nodes, but some class_types aren't in /object_info — they // COULD be paid API/partner nodes the server doesn't expose. Don't claim free. runtime = "unknown"; usesApiNodes = null; } else { runtime = "local"; usesApiNodes = false; } return { runtime, usesApiNodes, apiNodes, classTypes, unknownNodes, subgraphCount: countSubgraphs(graph) }; } function describeInputs(specs, required) { if (!specs) return []; return Object.entries(specs).map(([name, spec]) => { // Spec shape: [type, config?] where type is a string (e.g. "STRING") or a // string[] of enum options (e.g. ["fast", "quality"]). const type = Array.isArray(spec) ? spec[0] : spec; const config = (Array.isArray(spec) ? spec[1] : undefined) ?? {}; return { name, type: type, required, config: config, }; }); } /** Return the input schema for a given API node (from its /object_info entry). */ export async function getApiNodeSchema(classType, deps = defaultDeps) { const objectInfo = await deps.getObjectInfo(); const def = objectInfo[classType]; if (!def) { throw new ValidationError(`Node "${classType}" was not found on the connected ComfyUI. ` + `Use list_api_nodes to see available API nodes.`); } if (!isApiNode(def)) { throw new ValidationError(`Node "${classType}" exists but is not an API/partner node ` + `(category "${def.category ?? ""}"). Use list_api_nodes to find API nodes.`); } return { class_type: classType, display_name: def.display_name || classType, category: def.category ?? "", description: def.description ?? "", is_api_node: true, is_output_node: def.output_node === true, inputs: [ ...describeInputs(def.input?.required, true), ...describeInputs(def.input?.optional, false), ], hidden_inputs: def.input?.hidden ? Object.keys(def.input.hidden) : [], output: Array.isArray(def.output) ? def.output : [], output_name: Array.isArray(def.output_name) ? def.output_name : [], }; } const SIMPLE_WIDGET_TYPES = new Set(["INT", "FLOAT", "STRING", "BOOLEAN", "COMBO"]); /** * If a node input is a v3 dynamic combo, return its option list (each option has * a `key` plus the nested `inputs` it reveals). Returns null otherwise. */ function dynamicComboOptions(descriptor) { const type = descriptor.type; const opts = descriptor.config.options; if (!Array.isArray(opts)) return null; const typeIsDyn = typeof type === "string" && type.toUpperCase().includes("DYNAMICCOMBO"); // Also detect by shape: options that carry nested `inputs` (not plain strings). const shapeIsDyn = opts.some((o) => o != null && typeof o === "object" && "inputs" in o); if (!typeIsDyn && !shapeIsDyn) return null; return opts; } /** * Classify a nested input spec (from a dynamic combo option) as a POSITIONAL * widget (one that carries a scalar value the caller supplies) and resolve its * default. Non-widget nested inputs (AUTOGROW lists, IMAGE/link types) are not * emitted unless the caller explicitly provides a value. */ function classifyNestedSpec(spec) { if (!Array.isArray(spec)) return { isWidget: false, dflt: undefined }; const [type, cfg] = spec; if (Array.isArray(type)) { // Inline combo: ["1K","2K",...] return { isWidget: true, dflt: cfg?.default ?? type[0] }; } const t = String(type).toUpperCase(); if (SIMPLE_WIDGET_TYPES.has(t)) { let dflt = cfg?.default; if (dflt === undefined && Array.isArray(cfg?.options) && cfg.options.length) { dflt = cfg.options[0]; } return { isWidget: true, dflt }; } return { isWidget: false, dflt: undefined }; } /** * Build the `inputs` map for an API-node prompt from caller-provided values, * serializing any v3 dynamic-combo input into its dotted `model.<nested>` form. * * The caller may supply nested values in any of three natural shapes — already * dotted (`"model.aspect_ratio"`), flat (`"aspect_ratio"`), or as a nested object * (`model: { key, aspect_ratio }`). Required nested widgets the caller omits are * filled from their schema default so the server doesn't 400. Hidden inputs * (server-injected auth) are dropped. Exported for unit testing. */ export function buildApiNodeInputs(schema, provided) { const inputs = {}; const consumed = new Set(); // provided keys absorbed by combo expansion // Pass 1 — expand dynamic combos into dotted keys. for (const desc of schema.inputs) { const options = dynamicComboOptions(desc); if (!options) continue; const name = desc.name; // The selected option key may arrive as a string or inside a nested object. let selected = provided[name]; let selObj; if (selected != null && typeof selected === "object" && !Array.isArray(selected)) { selObj = selected; selected = selObj.key ?? selObj.value; } const cfgDefault = desc.config.default; if (selected === undefined || selected === null) { selected = cfgDefault ?? options[0]?.key; } const option = options.find((o) => o.key === selected) ?? options[0]; if (!option) continue; consumed.add(name); inputs[name] = option.key ?? selected; const emitNested = (nName, nSpec, requiredFill) => { const dottedKey = `${name}.${nName}`; let val; if (dottedKey in provided) { val = provided[dottedKey]; consumed.add(dottedKey); } else if (selObj && nName in selObj) { val = selObj[nName]; } else if (nName in provided) { val = provided[nName]; consumed.add(nName); } if (val === undefined) { if (!requiredFill) return; // optional / non-widget — omit when absent const info = classifyNestedSpec(nSpec); if (!info.isWidget || info.dflt === undefined) return; val = info.dflt; } inputs[dottedKey] = val; }; for (const [nName, nSpec] of Object.entries(option.inputs?.required ?? {})) { const info = classifyNestedSpec(nSpec); emitNested(nName, nSpec, info.isWidget); } for (const [nName, nSpec] of Object.entries(option.inputs?.optional ?? {})) { emitNested(nName, nSpec, false); } } // Pass 2 — copy remaining provided inputs (skip consumed + hidden). for (const [key, value] of Object.entries(provided)) { if (consumed.has(key)) continue; if (schema.hidden_inputs.includes(key)) continue; if (key in inputs) continue; inputs[key] = value; } // Pass 3 — fill any required top-level WIDGET input the caller omitted with its // schema default, so /prompt validation doesn't reject a missing required combo // /scalar (link-type inputs like IMAGE have no default and are left absent). for (const desc of schema.inputs) { if (!desc.required) continue; if (desc.name in inputs) continue; if (dynamicComboOptions(desc)) continue; // already emitted above const cfg = desc.config; const type = desc.type; let dflt; if (Array.isArray(type)) { dflt = cfg.default ?? type[0]; } else if (SIMPLE_WIDGET_TYPES.has(String(type).toUpperCase())) { dflt = cfg.default ?? (Array.isArray(cfg.options) ? cfg.options[0] : undefined); } if (dflt !== undefined) inputs[desc.name] = dflt; } return { inputs, consumed }; } /** * Build a minimal single-node workflow that runs a chosen API node with the * provided inputs and enqueue it. Returns the prompt_id (reuses the existing * enqueue path). The API node is typically an output node, so it can stand * alone as a complete graph. */ export async function generateWithApiNode(args, deps = defaultDeps) { const schema = await getApiNodeSchema(args.class_type, deps); const notes = []; const provided = args.inputs ?? {}; // Build the prompt inputs, serializing any v3 dynamic-combo input into its // dotted `model.<nested>` form (the canvas does this; a flat form 400s). const { inputs, consumed } = buildApiNodeInputs(schema, provided); // Acceptable input names: visible schema names, plus the nested names a dynamic // combo reveals (in both flat and dotted form) — so we don't warn about them. const knownNames = new Set(schema.inputs.map((i) => i.name)); for (const desc of schema.inputs) { const options = dynamicComboOptions(desc); if (!options) continue; for (const opt of options) { for (const nName of Object.keys(opt.inputs?.required ?? {})) { knownNames.add(nName); knownNames.add(`${desc.name}.${nName}`); } for (const nName of Object.keys(opt.inputs?.optional ?? {})) { knownNames.add(nName); knownNames.add(`${desc.name}.${nName}`); } } } // Warn about inputs that aren't part of the node's visible schema (typos, // or hidden auth fields the server should fill itself). for (const key of Object.keys(provided)) { if (knownNames.has(key) || consumed.has(key)) continue; if (schema.hidden_inputs.includes(key)) { notes.push(`Ignoring "${key}": it is a hidden input filled by the ComfyUI server, not the client.`); } else { notes.push(`Unknown input "${key}" for ${args.class_type} (passing through anyway).`); } } // Check required inputs are present in the built prompt (after combo expansion // and default-filling), and flag any genuinely missing ones. const missingRequired = schema.inputs .filter((i) => i.required && !(i.name in inputs)) .map((i) => i.name); if (missingRequired.length > 0) { notes.push(`Missing required input(s): ${missingRequired.join(", ")}. ` + `The server may reject the job. Use get_api_node_schema for details.`); } const workflow = { "1": { class_type: args.class_type, inputs, _meta: { title: schema.display_name }, }, }; // Merge caller-supplied supporting nodes (e.g. LoadImage → IMAGE link input). if (args.extra_nodes) { for (const [id, node] of Object.entries(args.extra_nodes)) { if (id === "1" || id === "2") { throw new ValidationError(`extra_nodes id "${id}" is reserved (the API node is "1"; "2" may be used for an auto-added output node). Use a different id.`); } workflow[id] = node; } } // ComfyUI only executes graphs that reach a terminal OUTPUT_NODE; a bare // non-output API node fails validation with "prompt_no_outputs". If the API // node isn't itself an output node, wire its IMAGE output into a SaveImage. if (!schema.is_output_node) { const imageIdx = schema.output.findIndex((o) => String(o).toUpperCase() === "IMAGE"); if (imageIdx >= 0) { workflow["2"] = { class_type: "SaveImage", inputs: { images: ["1", imageIdx], filename_prefix: "ComfyUI" }, _meta: { title: "Save Image" }, }; notes.push("Added a SaveImage output node — ComfyUI requires a terminal output node " + "for the prompt to execute."); } else { notes.push(`"${args.class_type}" is not an output node and has no IMAGE output, so no ` + "output node was auto-added; the prompt may fail with 'prompt_no_outputs'. " + "Wire a terminal output node yourself if needed."); } } // comfy.org API-node credentials travel in /prompt extra_data (the server // injects them into the node's hidden inputs). Supply the configured key so // API nodes work even when the ComfyUI server isn't logged in itself. const extraData = {}; if (config.comfyApiKey) { extraData.api_key_comfy_org = config.comfyApiKey; notes.push("Attached COMFY_API_KEY to the request (extra_data.api_key_comfy_org)."); } else { notes.push("No COMFY_API_KEY configured — relying on the ComfyUI server's own logged-in " + "comfy.org session for API-node auth. Set COMFY_API_KEY if the server isn't logged in."); } const result = await deps.enqueue(workflow, { disable_random_seed: args.disable_random_seed, extra_data: Object.keys(extraData).length > 0 ? extraData : undefined, }); return { prompt_id: result.prompt_id, queue_remaining: result.queue_remaining, workflow, notes, }; } //# sourceMappingURL=api-nodes.js.map