UNPKG

comfyui-mcp

Version:

Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),

203 lines (189 loc) • 10.9 kB
// CODEX KNOWLEDGE-PARITY SMOKE — proves the Codex backend has the SAME bundled // family expertise Claude gets natively. Claude loads all plugin skills; Codex // loads none, but now reaches the identical knowledge through the comfyui MCP's // list_skills / read_skill / list_packs / read_pack_workflow tools. // // Starts a codex-mode orchestrator + a headless mock panel (graph executors), // points the headless comfyui MCP's COMFYUI_MCP_TOOL_TRACE at a temp JSONL file // so we can observe the skill/pack tool calls (they ride the stdio MCP, not the // panel bridge), then prompts "set up a krea2 workflow on my canvas" and asserts: // 1) Codex DISCOVERED the krea2 family via list_skills/read_skill (or the pack // equivalents list_packs/read_pack_workflow) — NOT pure from-scratch guessing. // 2) Codex APPLIED the pack / loaded the pack's ready workflow (read_pack_workflow // and/or apply_manifest, then built nodes on the canvas) — NOT a generic graph. // // node scripts/codex-knowledge-parity-smoke.mjs // // Env: TEST_PORT (default 9151), SCENARIO_CAP_MS (default 240000 — Codex + reading // a full SKILL.md is slow). import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { tmpdir } from "node:os"; import { join } from "node:path"; import fs from "node:fs"; import net from "node:net"; import { WebSocket } from "ws"; const PORT = Number(process.env.TEST_PORT || 9151); const MCP_ENTRY = fileURLToPath(new URL("../dist/index.js", import.meta.url)); const COMFY_PATH = fileURLToPath(new URL("..", import.meta.url)); // a real dir w/ packs/ + plugin/ const DEAD_COMFY = "http://127.0.0.1:9"; const CAP_MS = Number(process.env.SCENARIO_CAP_MS || 240000); const TRACE = join(tmpdir(), `comfyui-mcp-tooltrace-${PORT}-${Date.now()}.jsonl`); function makeGraph() { let seq = 0; const nodes = new Map(); const add = (type, title) => { const id = ++seq; nodes.set(id, { id, type, title: title || type, is_subgraph: false, widgets: { value: 0, text: "" }, inputs: [{ name: "in0", type: "*", link: null }, { name: "model", type: "MODEL", link: null }], outputs: [{ name: "out0", type: "*", links: [] }, { name: "MODEL", type: "MODEL", links: [] }], }); return nodes.get(id); }; const brief = (n) => ({ id: n.id, type: n.type, title: n.title, is_subgraph: !!n.is_subgraph, widgets: n.widgets, inputs: n.inputs, outputs: n.outputs }); const need = (id) => { const n = nodes.get(Number(id)); if (!n) throw new Error(`No node ${id}`); return n; }; const EXEC = { graph_get_state: () => ({ viewing: { scope: "root" }, node_count: nodes.size, truncated: false, nodes: [...nodes.values()].map(brief) }), graph_add_node: ({ class_type, title }) => { if (!class_type) throw new Error("class_type required"); return { added: brief(add(class_type, title)) }; }, graph_remove_node: ({ node_id }) => { const n = need(node_id); nodes.delete(Number(node_id)); return { removed: brief(n) }; }, graph_clear: () => { const c = nodes.size; nodes.clear(); return { cleared: c }; }, graph_connect: ({ from_node_id, to_node_id }) => ({ connected: { from: { node_id: from_node_id }, to: { node_id: to_node_id } } }), graph_disconnect: ({ node_id }) => ({ disconnected: { node_id } }), graph_set_widget: ({ node_id, widget, value }) => { const n = need(node_id); const p = n.widgets[widget]; n.widgets[widget] = value; return { set: { node_id, widget, previous: p, value } }; }, graph_set_title: ({ node_id, title }) => { const n = need(node_id); const p = n.title; n.title = title; return { node_id, previous: p, title }; }, graph_move_node: ({ node_id, pos }) => ({ moved: { node_id, to: pos } }), graph_canvas: ({ action }) => ({ canvas: { action } }), graph_run: ({ batch_count }) => ({ queued: true, batch_count: batch_count ?? 1 }), graph_get_errors: () => ({ last_execution_error: null, node_errors: null, note: "no errors" }), workflow_save: () => ({ saved: true }), workflow_save_as: ({ name }) => ({ saved_as: `workflows/${name}.json` }), workflow_new: () => ({ created: true }), workflow_open: ({ path }) => ({ opened: { path } }), workflow_list: () => ({ active: { path: "workflows/current.json", filename: "current.json", key: "cur" }, open: [] }), graph_select_nodes: ({ node_ids }) => ({ selected: node_ids }), set_todo: ({ items }) => ({ ok: true, count: (items || []).length }), ask_user: (m) => (m.options && m.options[0] && m.options[0].label) || "yes", }; return { nodes, EXEC }; } function waitForPort(port, timeoutMs = 30000) { const start = Date.now(); return new Promise((resolve, reject) => { const tick = () => { const s = net.connect(port, "127.0.0.1"); s.on("connect", () => { s.destroy(); resolve(true); }); s.on("error", () => { s.destroy(); if (Date.now() - start > timeoutMs) reject(new Error("port timeout")); else setTimeout(tick, 300); }); }; tick(); }); } function runScenario(task) { return new Promise((resolve) => { const { nodes, EXEC } = makeGraph(); const commands = []; const says = []; let done = false, idleTimer = null; const ws = new WebSocket(`ws://127.0.0.1:${PORT}`); const hard = setTimeout(finish, CAP_MS); function finish() { if (done) return; done = true; clearTimeout(hard); if (idleTimer) clearTimeout(idleTimer); try { ws.close(); } catch {} const counts = {}; for (const c of commands) counts[c] = (counts[c] || 0) + 1; resolve({ counts, says, finalNodes: nodes.size }); } ws.on("open", () => { ws.send(JSON.stringify({ type: "hello", tab_id: `codex-kp-smoke`, title: "smoke" })); setTimeout(() => { console.log(` -> TASK: ${task}`); ws.send(JSON.stringify({ type: "user_message", text: task })); }, 1500); }); ws.on("message", (buf) => { let m; try { m = JSON.parse(buf.toString()); } catch { return; } if (typeof m.rid === "string" && typeof m.cmd === "string") { commands.push(m.cmd); let reply; try { const fn = EXEC[m.cmd]; if (!fn) throw new Error(`unknown ${m.cmd}`); reply = { rid: m.rid, ok: true, result: fn(m) }; } catch (e) { reply = { rid: m.rid, ok: false, error: e.message }; } console.log(` <cmd ${m.cmd}>`); try { ws.send(JSON.stringify(reply)); } catch {} return; } if (m.type === "say") { says.push(m.text); console.log(` << say: ${String(m.text).slice(0, 160)}`); } else if (m.type === "ack" && m.kind === "ready") console.log(` << ready (${m.agent}, backend=${m.backend})`); else if (m.type === "ack" && m.kind === "degraded") console.log(` << DEGRADED ack — backend not healthy`); if (m.type === "turn" && m.state === "done") { if (idleTimer) clearTimeout(idleTimer); idleTimer = setTimeout(finish, 6000); } }); ws.on("error", () => {}); }); } function readTrace() { try { return fs.readFileSync(TRACE, "utf8").trim().split(/\n+/).filter(Boolean).map((l) => JSON.parse(l)); } catch { return []; } } async function main() { console.log(`[kp-smoke] starting CODEX orchestrator on :${PORT} (COMFYUI_URL=${DEAD_COMFY}, COMFYUI_PATH=${COMFY_PATH})`); console.log(`[kp-smoke] tool trace → ${TRACE}`); const env = { ...process.env, PANEL_AGENT_BACKEND: "codex", COMFYUI_MCP_BRIDGE_PORT: String(PORT), COMFYUI_URL: DEAD_COMFY, // Local mode so apply_manifest is available AND so the comfyui MCP resolves // packs/ + plugin/ against this real repo dir. COMFYUI_PATH: COMFY_PATH, COMFYUI_MCP_TOOL_TRACE: TRACE, }; delete env.COMFYUI_MCP_PARENT_PID; const logPath = fileURLToPath(new URL("../codex-kp-smoke-orch.log", import.meta.url)); const logFd = fs.openSync(logPath, "w"); const orch = spawn(process.execPath, [MCP_ENTRY, "--panel-orchestrator"], { env, stdio: ["ignore", logFd, logFd] }); let exitCode = 1; try { await waitForPort(PORT); console.log("[kp-smoke] orchestrator up.\n"); console.log("• Scenario: set up a krea2 workflow on my canvas"); const r = await runScenario("Clear my workflow and set up a krea2 text-to-image workflow on my canvas. Use whatever ready expertise comfyui-mcp already ships if there is any — don't reinvent the graph."); const trace = readTrace(); const tools = trace.map((t) => t.tool); const readSkillArgs = trace.filter((t) => t.tool === "read_skill").map((t) => t.args?.name); const readPackArgs = trace.filter((t) => t.tool === "read_pack_workflow").map((t) => t.args?.name); const calledListSkills = tools.includes("list_skills"); const calledReadSkill = tools.includes("read_skill"); const calledListPacks = tools.includes("list_packs"); const calledReadPack = tools.includes("read_pack_workflow"); const discoveredKrea2 = readSkillArgs.some((n) => String(n || "").includes("krea2")) || readPackArgs.some((n) => String(n || "").includes("krea2")); // Discovery = consulted the bundled knowledge (skills and/or packs) for the family. const discovery = calledListSkills || calledReadSkill || calledListPacks || calledReadPack; // Applied ready expertise = read the pack workflow (the expert graph) and/or // built nodes on the canvas from it. const builtOnCanvas = (r.counts.graph_add_node || 0) >= 1; const appliedPack = calledReadPack || discoveredKrea2; console.log(`\n===== CODEX KNOWLEDGE-PARITY SMOKE =====`); console.log(`tool trace (skill/pack calls): ${JSON.stringify(tools)}`); console.log(`read_skill names: ${JSON.stringify(readSkillArgs)}`); console.log(`read_pack_workflow names: ${JSON.stringify(readPackArgs)}`); console.log(`all bridge commands: ${JSON.stringify(r.counts)}`); console.log(`---`); console.log(`Codex consulted bundled skills/packs (list_skills/read_skill/list_packs/read_pack_workflow): ${discovery ? "YES" : "NO"}`); console.log(`Codex discovered the krea2 family (krea2-* skill or pack): ${discoveredKrea2 ? "YES" : "NO"}`); console.log(`Codex applied the pack / read its ready workflow: ${appliedPack ? "YES" : "NO"}`); console.log(`Codex built nodes on the live canvas: ${builtOnCanvas ? "YES" : "NO"}`); // PASS = it discovered the family knowledge AND used the ready pack expertise. exitCode = discovery && discoveredKrea2 ? 0 : 1; console.log(`\n${exitCode === 0 ? "PASS" : "FAIL"} — knowledge parity ${exitCode === 0 ? "achieved" : "NOT achieved"}.`); } catch (e) { console.error("[kp-smoke] ERROR:", e.message); } finally { try { orch.kill("SIGTERM"); } catch {} setTimeout(() => { try { orch.kill("SIGKILL"); } catch {}; console.log(`[kp-smoke] orchestrator log: ${logPath}`); process.exit(exitCode); }, 1500); } } main();