UNPKG

comfyui-mcp

Version:

Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),

227 lines (226 loc) • 11.5 kB
// Shared arena scenario set — task prompts + harness-side ground-truth // verification, used by BOTH scripts/llm-arena.mjs (compact 3-tool mode) and // scripts/llm-arena-full.mjs (full 113-tool surface for the fine-tune // pipeline). verify(harnessCall, t) receives a direct (toolName, args) → text // callback in either mode and NEVER trusts the model's claims. /** * Each scenario: task prompt, the underlying tools that count as the "right" * primary move, and an optional verify(harnessCall, transcript) ground-truth * check the HARNESS runs against ComfyUI itself (never trusts the model). */ export const SCENARIOS = [ { id: "health", title: "Server health & GPU report", task: "Check whether the ComfyUI server is healthy and tell me the GPU name and how much free VRAM it has.", primary: ["health_check", "get_system_stats"], verify: async (_call, t) => /(cuda|nvidia|rtx|gtx|radeon|vram)/i.test(t.finalAnswer), }, { id: "models", title: "Installed checkpoint discovery", task: "Find out which checkpoint models are installed on the ComfyUI server and tell me the name of one of them.", primary: ["list_local_models"], verify: async (call, t) => { const res = await call("list_local_models", { model_type: "checkpoints" }); const names = [...res.matchAll(/([\w.-]+)\.(safetensors|ckpt|sft|gguf)/gi)].map((m) => m[1].toLowerCase(), ); const answer = t.finalAnswer.toLowerCase(); return names.some((n) => answer.includes(n) || answer.includes(n.slice(0, 12))); }, }, { id: "registry", title: "Custom-node registry search", task: "Find a tool that can search for ComfyUI custom node packs, then use it to search for 'controlnet' and tell me the name of one node pack from its results.", primary: ["search_custom_nodes", "search_models"], verify: async (_call, t) => t.finalAnswer.trim().length > 0, }, { id: "queue", title: "Queue inspection", task: "How many jobs are currently running or pending in the ComfyUI queue? Answer with the numbers.", primary: ["get_queue", "health_check", "get_system_stats"], verify: async (_call, t) => /\d/.test(t.finalAnswer), }, { id: "generate", title: "Text-to-image generation + async polling", task: "Generate a 512x512 image of a red apple on a wooden table. The generation runs asynchronously — " + "after starting it, check its job status until it has finished, then tell me the output filename or asset id.", primary: ["generate_image", "enqueue_workflow"], // right family but incomplete execution (built a workflow, never enqueued) partial: ["create_workflow", "dsl_to_workflow"], followup: ["get_job_status", "get_history", "list_output_images", "view_image", "list_assets", "get_queue", "generation_stats"], verify: async (call, t) => { // ground truth: the prompt_id the model started must be done with outputs const ids = [...t.toolText.matchAll(/"prompt_id":\s*"([0-9a-f-]{8,})"/g)].map((m) => m[1]); if (!ids.length) return false; for (let attempt = 0; attempt < 30; attempt++) { const status = await call("get_job_status", { prompt_id: ids[ids.length - 1] }); if (/"done":\s*true/.test(status) && !/"error"/.test(status)) return true; if (/"error":/.test(status)) return false; await new Promise((r) => setTimeout(r, 2000)); } return false; }, }, // ── The GAUNTLET — added when the whole SoTA tier tied 10/10 on the base // set. Same server-side verification discipline, but these stress parameter // fidelity, error recovery, and multi-render state tracking. { id: "precision", title: "Parameter-exact build + render", task: "Render a txt2img image with the checkpoint v1-5-pruned-emaonly-fp16.safetensors, EXACTLY 12 sampling steps, " + "EXACTLY 384x384 pixels, positive prompt 'a green pear on a table'. Wait until it finishes, then report the prompt_id.", primary: ["generate_image", "enqueue_workflow"], partial: ["create_workflow", "dsl_to_workflow"], verify: async (_call, t) => { // ground truth from ComfyUI itself: the EXECUTED graph must carry the // exact parameters, and the job must have completed with outputs. const ids = [...t.toolText.matchAll(/"prompt_id":\s*"([0-9a-f-]{8,})"/g)].map((m) => m[1]); for (const id of ids.reverse()) { try { const res = await fetch(`${process.env.COMFYUI_URL ?? "http://127.0.0.1:8188"}/history/${id}`); const hist = (await res.json())[id]; if (!hist?.status?.completed) continue; const nodes = Object.values(hist.prompt?.[2] ?? {}); const steps = nodes.some((n) => n.inputs?.steps === 12); const size = nodes.some((n) => n.inputs?.width === 384 && n.inputs?.height === 384); if (steps && size) return true; } catch { /* try the next id */ } } return false; }, }, { id: "breakfix", title: "Deliberate failure → diagnose → recover", task: "First, try to render an image using a checkpoint named exactly 'nonexistent-model.safetensors'. That will fail — " + "read the error and explain in ONE sentence why. Then recover: render 'a blue cube' at 512x512 with a checkpoint " + "that IS installed, wait for it to complete, and report its prompt_id.", primary: ["generate_image", "enqueue_workflow"], partial: ["create_workflow"], verify: async (call, t) => { // (a) the failure actually happened (the bogus name shows up in an error), // (b) a real render then completed. const sawFailure = /nonexistent-model\.safetensors/.test(t.toolText) && /error|invalid|not (?:found|in)/i.test(t.toolText); if (!sawFailure) return false; const ids = [...t.toolText.matchAll(/"prompt_id":\s*"([0-9a-f-]{8,})"/g)].map((m) => m[1]); for (const id of ids.reverse()) { const status = await call("get_job_status", { prompt_id: id }); if (/"done":\s*true/.test(status) && !/"error"/.test(status)) return true; } return false; }, }, { id: "provenance", title: "Generate → find asset → regenerate with override", task: "Render a 512x512 image of 'a red bicycle' and wait for it to complete. Then find the ASSET it produced " + "(the asset registry lists recent assets), and regenerate that asset with a steps=8 override, waiting for the " + "second render to complete too. Report both prompt_ids.", primary: ["generate_image", "enqueue_workflow"], partial: ["create_workflow", "list_assets", "get_asset_metadata"], verify: async (call, t) => { // regenerate must have actually run, and there must be two DISTINCT // completed prompts. if (!t.calls.some((c) => c.tool === "regenerate" && c.ok)) return false; const ids = [...new Set([...t.toolText.matchAll(/"prompt_id":\s*"([0-9a-f-]{8,})"/g)].map((m) => m[1]))]; if (ids.length < 2) return false; let done = 0; for (const id of ids) { const status = await call("get_job_status", { prompt_id: id }); if (/"done":\s*true/.test(status) && !/"error"/.test(status)) done++; } return done >= 2; }, }, // ── The CRUCIBLE — round 3, added when four models tied the gauntlet at // 16/16. Custom graph COMPOSITION (no template covers these): one graph // with two piped outputs, and a two-stage pipeline chained through the // staging tool. All still SD1.5-only and server-verified. { id: "multiout", title: "One graph, two piped outputs", task: "Build and enqueue ONE single workflow that renders 'a lighthouse at dusk' at 512x512 AND, in the same graph, " + "pipes that image through a 2x upscale so the SAME run saves TWO outputs: the 512x512 original and a 1024x1024 " + "version. No template does this — compose the graph yourself. Wait for completion and report the prompt_id.", primary: ["enqueue_workflow"], partial: ["create_workflow", "dsl_to_workflow", "generate_image", "modify_workflow"], verify: async (_call, t) => { const base = process.env.COMFYUI_URL ?? "http://127.0.0.1:8188"; const ids = [...new Set([...t.toolText.matchAll(/"prompt_id":\s*"([0-9a-f-]{8,})"/g)].map((m) => m[1]))]; for (const id of ids.reverse()) { try { const hist = (await (await fetch(`${base}/history/${id}`)).json())[id]; if (!hist?.status?.completed) continue; // gather every output image of THIS single prompt and read its real // pixel size from the PNG header — need two distinct sizes, 2x apart. const dims = new Set(); for (const out of Object.values(hist.outputs ?? {})) { for (const img of out.images ?? []) { if (img.type !== "output") continue; const u = new URL("/view", base); u.searchParams.set("filename", img.filename); u.searchParams.set("type", "output"); if (img.subfolder) u.searchParams.set("subfolder", img.subfolder); const buf = Buffer.from(await (await fetch(u)).arrayBuffer()); if (buf.length > 24 && buf.toString("ascii", 1, 4) === "PNG") { dims.add(`${buf.readUInt32BE(16)}x${buf.readUInt32BE(20)}`); } } } if (dims.has("512x512") && dims.has("1024x1024")) return true; } catch { /* try next id */ } } return false; }, }, { id: "pipeline", title: "Two-stage pipe via output staging", task: "Two-stage pipeline. Stage 1: render 'a plain wooden mask' at 512x512 and wait for it to finish. Stage 2: run an " + "img2img pass over stage 1's ACTUAL output image with the prompt 'an ornate golden mask' and denoise about 0.55, " + "and wait for it too. Do NOT guess file paths — use the staging tool that feeds a previous output into the next " + "stage's loader. Report both prompt_ids.", primary: ["enqueue_workflow", "generate_image"], partial: ["create_workflow", "stage_output_as_input"], verify: async (call, t) => { // the staging tool must have actually run… if (!t.calls.some((c) => c.tool === "stage_output_as_input" && c.ok)) return false; const base = process.env.COMFYUI_URL ?? "http://127.0.0.1:8188"; const ids = [...new Set([...t.toolText.matchAll(/"prompt_id":\s*"([0-9a-f-]{8,})"/g)].map((m) => m[1]))]; if (ids.length < 2) return false; // …and the LAST completed prompt must be a real img2img graph: a // LoadImage feeding it and a KSampler with partial denoise. for (const id of ids.reverse()) { try { const hist = (await (await fetch(`${base}/history/${id}`)).json())[id]; if (!hist?.status?.completed) continue; const nodes = Object.values(hist.prompt?.[2] ?? {}); const hasLoad = nodes.some((n) => n.class_type === "LoadImage"); const partialDenoise = nodes.some( (n) => typeof n.inputs?.denoise === "number" && n.inputs.denoise > 0.2 && n.inputs.denoise < 0.9, ); if (hasLoad && partialDenoise) return true; break; // only judge the newest completed prompt as stage 2 } catch { /* try next id */ } } return false; }, }, ];