comfyui-mcp
Version:
Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),
530 lines • 24.8 kB
JavaScript
import { readFileSync } from "node:fs";
import { getComfyUIBaseUrl, getComfyUIAuthHeaders } from "../config.js";
import { getObjectInfo } from "../comfyui/client.js";
import { bodyPrefixOf, classifyNonJson, isNonJsonResponseError, readComfyJson, } from "../comfyui/json-guard.js";
import { convertUiToApi, isApiFormat, isUiFormat } from "../services/workflow-converter.js";
import { enumeratePacks, resolvePackWorkflowFile } from "./skills-access.js";
// Roles that make a slot "primary" (surfaced in `slots`); everything else is a
// still-overridable but structural/secondary widget (surfaced in `other_slots`).
const PRIMARY_ROLES = new Set([
"prompt", "seed", "steps", "cfg", "guidance", "sampler", "scheduler",
"denoise", "width", "height", "batch_size", "checkpoint", "lora", "model",
"strength", "input_image", "input_media", "length", "frame_rate", "shift",
]);
/** Classify a widget into a semantic role from its name + owning class type. */
export function classifyWidget(classType, widget) {
const w = widget.toLowerCase();
const c = classType.toLowerCase();
if (w === "text" || w === "prompt" || w === "positive_prompt" || w === "negative_prompt") {
if (/textencode|prompt|conditioning/.test(c) || w !== "text")
return "prompt";
}
if (w === "seed" || w === "noise_seed" || w === "rand_seed")
return "seed";
if (w === "steps")
return "steps";
if (w === "cfg" || w === "cfg_scale" || w === "guidance_scale")
return "cfg";
if (w === "guidance")
return "guidance";
if (w === "sampler_name")
return "sampler";
if (w === "scheduler")
return "scheduler";
if (w === "denoise")
return "denoise";
if (w === "width")
return "width";
if (w === "height")
return "height";
if (w === "batch_size")
return "batch_size";
if (w === "ckpt_name")
return "checkpoint";
if (w === "lora_name" || /^lora_\d+$/.test(w))
return "lora";
if (w === "unet_name" || w === "model_name" || w === "clip_name" ||
/^clip_name\d+$/.test(w) || w === "vae_name" || w === "style_model_name" ||
w === "control_net_name" || w === "ipadapter_file" || w === "gguf_name") {
return "model";
}
if (w === "strength_model" || w === "strength_clip" || (w === "strength" && /lora/.test(c))) {
return "strength";
}
if (w === "image" && /loadimage/.test(c))
return "input_image";
if ((w === "video" || w === "audio" || w === "file") && /^load/.test(c))
return "input_media";
if (w === "length" || w === "num_frames" || w === "frames" || w === "video_frames")
return "length";
if (w === "frame_rate" || w === "fps")
return "frame_rate";
if (w === "shift" || w === "max_shift")
return "shift";
return "other";
}
/** An API-format connection reference: [nodeId, outputSlot] — only when the
* referenced node actually exists (a literal like [512, 512] is a widget value). */
function isLinkValue(v, nodeKeys) {
return (Array.isArray(v) &&
v.length === 2 &&
(typeof v[0] === "string" || typeof v[0] === "number") &&
typeof v[1] === "number" &&
nodeKeys.has(String(v[0])));
}
function specFor(def, widget) {
if (!def)
return undefined;
// V3 dynamic-combo nested inputs are keyed "<combo>.<nested>" — no direct
// spec lookup; treat as schema-less (value-inferred type).
return def.input?.required?.[widget] ?? def.input?.optional?.[widget];
}
const MAX_OPTIONS = 24;
/**
* Extract override slots from an API/prompt-format graph. Pure + offline:
* `objectInfo` (live /object_info or the built-in fallback) only ENRICHES
* (type/min/max/options) — extraction itself needs nothing but the graph.
*/
export function extractTemplateSlots(api, objectInfo) {
const slots = [];
const other = [];
const nodeKeys = new Set(Object.keys(api));
const ids = Object.keys(api).sort((a, b) => {
const na = Number(a);
const nb = Number(b);
if (Number.isFinite(na) && Number.isFinite(nb))
return na - nb;
return a.localeCompare(b);
});
for (const nodeId of ids) {
const node = api[nodeId];
if (!node || typeof node !== "object" || typeof node.class_type !== "string")
continue;
const inputs = node.inputs;
// An inputs ARRAY is malformed (Object.entries would emit fake "<id>.0"
// index keys) — only a name→value record is a valid API-format inputs map.
if (!inputs || typeof inputs !== "object" || Array.isArray(inputs))
continue;
const def = objectInfo?.[node.class_type];
const title = node._meta?.title;
for (const [widget, value] of Object.entries(inputs)) {
if (isLinkValue(value, nodeKeys))
continue; // connection, not an overridable widget
const slot = {
key: `${nodeId}.${widget}`,
node_id: nodeId,
class_type: node.class_type,
widget,
role: classifyWidget(node.class_type, widget),
type: "STRING",
value,
};
if (title && title !== node.class_type)
slot.node_title = title;
const spec = specFor(def, widget);
if (spec) {
const [typeSpec, cfg] = spec;
if (Array.isArray(typeSpec)) {
slot.type = "COMBO";
slot.options_count = typeSpec.length;
slot.options = typeSpec.slice(0, MAX_OPTIONS);
}
else {
slot.type = String(typeSpec);
}
if (cfg && typeof cfg === "object") {
if (typeof cfg.min === "number")
slot.min = cfg.min;
if (typeof cfg.max === "number")
slot.max = cfg.max;
if (typeof cfg.step === "number")
slot.step = cfg.step;
}
}
else {
// No schema for this node/widget — infer the type from the value.
slot.type =
typeof value === "number"
? Number.isInteger(value) ? "INT" : "FLOAT"
: typeof value === "boolean"
? "BOOLEAN"
: "STRING";
}
(PRIMARY_ROLES.has(slot.role) ? slots : other).push(slot);
}
}
return { slots, other_slots: other };
}
// ── Fallback node schema ─────────────────────────────────────────────────────
// A minimal object_info for the common core nodes, used when the live server
// is unreachable so UI-format templates can still be mapped from positional
// widgets_values to NAMED widgets (the converter needs a def per class type).
// Live /object_info always wins when available.
function def(required) {
return {
input: { required },
input_order: { required: Object.keys(required) },
output: [],
output_is_list: [],
output_name: [],
name: "",
display_name: "",
description: "",
category: "",
output_node: false,
};
}
const INT = ["INT", {}];
const FLOAT = ["FLOAT", {}];
const STR = ["STRING", {}];
const COMBO = ["COMBO", {}];
export const FALLBACK_OBJECT_INFO = {
CLIPTextEncode: def({ text: ["STRING", { multiline: true }] }),
KSampler: def({
seed: ["INT", { min: 0, control_after_generate: true }],
steps: ["INT", { min: 1, max: 10000 }],
cfg: ["FLOAT", { min: 0, max: 100 }],
sampler_name: COMBO,
scheduler: COMBO,
denoise: ["FLOAT", { min: 0, max: 1 }],
}),
KSamplerAdvanced: def({
add_noise: COMBO,
noise_seed: ["INT", { min: 0, control_after_generate: true }],
steps: ["INT", { min: 1, max: 10000 }],
cfg: ["FLOAT", { min: 0, max: 100 }],
sampler_name: COMBO,
scheduler: COMBO,
start_at_step: INT,
end_at_step: INT,
return_with_leftover_noise: COMBO,
}),
EmptyLatentImage: def({ width: INT, height: INT, batch_size: INT }),
EmptySD3LatentImage: def({ width: INT, height: INT, batch_size: INT }),
CheckpointLoaderSimple: def({ ckpt_name: COMBO }),
LoraLoader: def({ lora_name: COMBO, strength_model: FLOAT, strength_clip: FLOAT }),
LoraLoaderModelOnly: def({ lora_name: COMBO, strength_model: FLOAT }),
LoadImage: def({ image: COMBO }),
SaveImage: def({ filename_prefix: STR }),
UNETLoader: def({ unet_name: COMBO, weight_dtype: COMBO }),
VAELoader: def({ vae_name: COMBO }),
CLIPLoader: def({ clip_name: COMBO, type: COMBO }),
DualCLIPLoader: def({ clip_name1: COMBO, clip_name2: COMBO, type: COMBO }),
RandomNoise: def({ noise_seed: ["INT", { min: 0, control_after_generate: true }] }),
KSamplerSelect: def({ sampler_name: COMBO }),
BasicScheduler: def({ scheduler: COMBO, steps: INT, denoise: FLOAT }),
FluxGuidance: def({ guidance: FLOAT }),
CFGGuider: def({ cfg: FLOAT }),
ModelSamplingSD3: def({ shift: FLOAT }),
};
/**
* Normalize a loaded template graph (UI or API format) to API/prompt format so
* every widget carries a NAME. UI graphs need node defs — live object_info when
* reachable, else FALLBACK_OBJECT_INFO (unknown node types are skipped with a
* warning rather than crashing). Returns null when the JSON is neither format.
*/
export function templateGraphToApi(graph, objectInfo) {
if (isUiFormat(graph)) {
const merged = { ...FALLBACK_OBJECT_INFO, ...(objectInfo ?? {}) };
// isUiFormat only checks that nodes/links are arrays — drop malformed
// entries (null nodes, missing id/type) so the converter can't crash on
// odd shapes; report what was dropped.
const ui = graph;
const warnings = [];
const goodNodes = (ui.nodes ?? [])
.filter((n) => !!n && typeof n === "object" && typeof n.id === "number" &&
typeof n.type === "string")
// The converter iterates node.inputs/outputs as arrays of objects and
// dereferences entry fields (input.link, input.name) — normalize any
// non-array nested field AND drop non-object entries so a malformed node
// can't crash it. (Non-array widgets_values is LEGITIMATE — e.g. VHS
// nodes use a name→value object — the converter handles it; leave alone.)
.map((n) => {
const slotArr = (v) => Array.isArray(v)
? v.filter((e) => !!e && typeof e === "object")
: undefined;
const inputs = slotArr(n.inputs);
const outputs = slotArr(n.outputs);
// widgets_values may be a positional array OR a name→value record (VHS
// nodes) — both are valid. A scalar (string/number/…) would crash the
// converter's `name in wv` check, so normalize it away.
const wv = n.widgets_values;
const widgets_values = Array.isArray(wv) || (wv != null && typeof wv === "object") ? wv : undefined;
if (widgets_values === wv &&
inputs?.length === (Array.isArray(n.inputs) ? n.inputs.length : -1) &&
outputs?.length === (Array.isArray(n.outputs) ? n.outputs.length : -1)) {
return n; // fully well-shaped, keep as-is
}
return { ...n, inputs, outputs, widgets_values };
});
if (goodNodes.length !== (ui.nodes ?? []).length) {
warnings.push(`Dropped ${(ui.nodes ?? []).length - goodNodes.length} malformed node entr(y/ies) (missing id/type).`);
}
const goodLinks = (ui.links ?? []).filter((l) => Array.isArray(l) && l.length >= 6);
const sanitized = { ...ui, nodes: goodNodes, links: goodLinks };
const { workflow, warnings: convWarnings } = convertUiToApi(sanitized, merged);
return { api: workflow, warnings: [...warnings, ...convWarnings], objectInfo: merged };
}
if (isApiFormat(graph)) {
return {
api: graph,
warnings: [],
objectInfo: { ...FALLBACK_OBJECT_INFO, ...(objectInfo ?? {}) },
};
}
return null;
}
/**
* Resolve a template query against the /api/workflow_templates index — the SAME
* index list_packs (action:"list_templates") returns. Pure + offline so it is unit-testable
* independent of the live server.
*
* The query may be a bare template name ("i2mv_sdxl_ldm_view_selector") OR a
* source-qualified id ("ComfyUI-MVAdapter/i2mv_sdxl_ldm_view_selector") — the
* qualified form (module/name, matching how action:"list_templates" groups
* entries) disambiguates when the same name is provided by multiple modules.
*/
export function resolveTemplateFromIndex(index, query) {
// A source-qualified id splits on the FIRST slash into module + name. Module
// names don't contain slashes; template names in practice don't either, but
// splitting on the first slash keeps the (module) prefix authoritative.
const slash = query.indexOf("/");
const qModule = slash > 0 ? query.slice(0, slash) : null;
const qName = slash > 0 ? query.slice(slash + 1) : query;
const all = [];
const matches = [];
// Dedupe by module/name: ComfyUI's index builder can list the same basename
// twice within one module (multiple recognized workflow dirs). That is NOT
// cross-source ambiguity, so collapse duplicates before the ambiguity check.
const seen = new Set();
for (const [mod, names] of Object.entries(index)) {
if (!Array.isArray(names))
continue;
for (const n of names) {
const nm = typeof n === "string" ? n : n?.name;
if (typeof nm !== "string")
continue;
all.push(`${mod}/${nm}`);
if (nm === qName && (qModule == null || mod === qModule)) {
const dedupeKey = `${mod}/${nm}`;
if (seen.has(dedupeKey))
continue;
seen.add(dedupeKey);
matches.push({ module: mod, name: nm });
}
}
}
if (matches.length === 0)
return { error: "not-found", all };
if (matches.length > 1)
return { error: "ambiguous", candidates: matches };
return { match: matches[0] };
}
/** Fetch an official workflow template's graph from the live ComfyUI. */
async function loadServerTemplate(name) {
// Use the SAME canonical base URL + auth headers as the connected ComfyUI
// client (getObjectInfo/getClient). A bare protocol://host:port fetch drops
// the reverse-proxy base path and any gateway auth headers, so a proxied or
// authed remote that list_packs (action:"list_templates") (now) reaches would otherwise
// look "unreachable" here — the inconsistency this issue reported.
const base = getComfyUIBaseUrl();
const authHeaders = getComfyUIAuthHeaders();
const url = `${base}/api/workflow_templates`;
let index = {};
try {
const res = await fetch(url, {
headers: authHeaders,
signal: AbortSignal.timeout(8000),
});
// `.json()` on an HTML body threw into the catch below, which reported the
// server UNREACHABLE — for a server that answered (#828). readComfyJson
// names what actually answered instead, and that message is surfaced rather
// than collapsed into the unreachable text.
if (!res.ok) {
// An HTTP ERROR is not an empty index. Falling through with `index = {}`
// made a 502 from a proxy, a 403 from a sign-in gate, or a gateway's JSON
// error envelope read as "there is no template by that name" — a
// confident wrong verdict about the template, for an endpoint that
// answered and never told us anything about templates at all (codex gate,
// round 9, finding 1).
const contentType = res.headers.get("content-type") ?? "";
// unknown-ok: "" is interpolated into an ERROR MESSAGE and nothing else — the
// HTTP status is reported either way, so an unreadable body costs detail in the
// text, never a wrong conclusion. Verified there is no branch on this value.
const body = await res.text().catch(() => "");
let parsedOk = false;
try {
JSON.parse(body);
parsedOk = true;
}
catch {
parsedOk = false;
}
return {
error: parsedOk
? `Not a bundled pack, and the template index could not be read: ${url} returned ${res.status} with this JSON body: ${bodyPrefixOf(body)}. Whether the requested template exists is UNKNOWN.`
: `Not a bundled pack, and the template index could not be read: ${classifyNonJson({ url, status: res.status, contentType, body }).message} Whether the requested template exists is UNKNOWN.`,
available: [],
};
}
index = await readComfyJson(res, {
url,
expectShape: (v) => !!v && typeof v === "object" && !Array.isArray(v),
shapeHint: "the /api/workflow_templates index (an object keyed by source)",
});
}
catch (err) {
if (isNonJsonResponseError(err)) {
return {
error: `Not a bundled pack, and the template index could not be read: ${err.message}`,
available: [],
};
}
return {
error: "Not a bundled pack, and the ComfyUI server is unreachable to look up custom-node-contributed workflow templates.",
available: [],
};
}
const resolved = resolveTemplateFromIndex(index, name);
if ("error" in resolved) {
if (resolved.error === "ambiguous") {
return {
error: `Template name "${name}" is provided by multiple sources — qualify it as "<module>/${name}".`,
available: resolved.candidates.map((c) => `${c.module}/${c.name}`),
};
}
const needle = name.toLowerCase();
const near = resolved.all.filter((n) => n.toLowerCase().includes(needle)).slice(0, 10);
return {
error: `No custom-node-contributed workflow template named "${name}" (core templates from the comfyui-workflow-templates package are not in this /api/workflow_templates index — check the ComfyUI frontend's Templates browser for those).`,
available: near.length ? near : resolved.all.slice(0, 20),
// #809: `available` is a PREVIEW, and a caller who reads it as the full index
// concludes their template does not exist. State the real total and the tool that
// lists all of them.
available_count: resolved.all.length,
...((near.length ? near.length : Math.min(resolved.all.length, 20)) < resolved.all.length
? {
available_truncated: `Showing ${near.length ? near.length : Math.min(resolved.all.length, 20)} of ${resolved.all.length} templates (fixed preview cap — no parameter raises it); list_packs (action:\"list_templates\") returns every one.`,
}
: {}),
};
}
const { module, name: tmpl } = resolved.match;
// Core templates ship in the comfyui-workflow-templates package served under
// /templates/; custom-node templates under /api/workflow_templates/<module>/
// (older servers: /extensions/<module>/example_workflows/). Try in order.
const enc = encodeURIComponent(tmpl);
const candidates = [
`${base}/templates/${enc}.json`,
`${base}/api/workflow_templates/${encodeURIComponent(module)}/${enc}.json`,
`${base}/extensions/${encodeURIComponent(module)}/example_workflows/${enc}.json`,
];
for (const url of candidates) {
try {
const res = await fetch(url, { headers: authHeaders, signal: AbortSignal.timeout(8000) });
if (!res.ok)
continue;
return { graph: await res.json(), source: `server-template:${module}` };
}
catch {
// try next candidate
}
}
return {
error: `Template "${tmpl}" is indexed (module "${module}") but its workflow JSON could not be fetched from the server.`,
available: [],
};
}
/**
* `enqueue_workflow (action:"template_schema")` — the handler the standalone
* template-schema tool used to carry (0.50.0 slice 16), unchanged apart from
* losing its own registration.
*
* Same resolution order (bundled pack, then the live server's template index),
* same slot extraction, same JSON payload, and the same two `isError` RETURNS
* for an unresolvable name and an unrecognised graph — those are returns, not
* throws, so they survive the move untouched. The try/catch moved OUT to the
* dispatcher in workflow-execute.ts, which applies the identical
* `errorToToolResult`.
*/
export async function templateSchemaAction(args) {
const name = args.template.trim();
let graph;
let source;
const packFile = resolvePackWorkflowFile(name);
if (packFile) {
graph = JSON.parse(readFileSync(packFile, "utf8"));
source = `pack:${name}`;
}
else {
const res = await loadServerTemplate(name);
if ("error" in res) {
const packs = enumeratePacks().map((p) => String(p.name));
const near = packs.filter((p) => p.toLowerCase().includes(name.toLowerCase()));
return {
isError: true,
content: [
{
type: "text",
text: JSON.stringify({
error: `Could not resolve template "${name}". ${res.error}`,
near_matches: [...near, ...res.available].slice(0, 20),
hint: "Use list_packs (action:\"list\") for bundled packs or list_packs (action:\"list_templates\") for custom-node-contributed templates (core templates only appear in the ComfyUI frontend's own Templates browser).",
}, null, 2),
},
],
};
}
graph = res.graph;
source = res.source;
}
// Live node schema enriches types/min/max/options and names UI widgets;
// fall back to the built-in core-node table offline.
let live = null;
try {
live = await getObjectInfo();
}
catch {
live = null;
}
const normalized = templateGraphToApi(graph, live);
if (!normalized) {
return {
isError: true,
content: [
{
type: "text",
text: `Template "${name}" resolved (${source}) but its JSON is neither a UI-format nor an API-format ComfyUI workflow.`,
},
],
};
}
const { slots, other_slots } = extractTemplateSlots(normalized.api, normalized.objectInfo);
return {
content: [
{
type: "text",
text: JSON.stringify({
template: name,
source,
node_schema: live ? "live-object_info" : "builtin-fallback (server unreachable — types/options are best-effort)",
slot_key_format: '<nodeId>.<widget_name> — pass these keys as enqueue_workflow (action:"run_template") overrides',
slot_count: slots.length,
slots,
other_slot_count: other_slots.length,
other_slots,
// #809: a warnings array silently clipped at 20 reads as "these are
// all the warnings". Report the true count alongside the preview.
warning_count: normalized.warnings.length,
warnings: normalized.warnings.slice(0, 20),
...(normalized.warnings.length > 20
? {
warnings_truncated: `Showing 20 of ${normalized.warnings.length} warnings (fixed preview cap — no parameter raises it); fix these and re-run to surface the rest.`,
}
: {}),
}, null, 2),
},
],
};
}
//# sourceMappingURL=template-schema.js.map