comfyui-mcp
Version:
Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),
333 lines (311 loc) • 13 kB
JavaScript
// ComfyUI LLM Arena — run a field of local/hosted models through the same
// real-ComfyUI task set over compact-mode MCP, and produce a leaderboard.
//
// node scripts/llm-arena.mjs # default local field via Ollama
// ARENA_MODELS=gemma4:e4b,qwen3:4b node scripts/llm-arena.mjs
// OLLAMA_HOST=http://127.0.0.1:11434 # endpoint override
//
// Hosted models (any OpenAI-compatible API — DeepSeek, GLM, MiniMax, MiMo,
// or all of them through OpenRouter):
// ARENA_API=openai ARENA_BASE_URL=https://openrouter.ai/api/v1 \
// ARENA_API_KEY=sk-... ARENA_MODELS="deepseek/deepseek-chat,z-ai/glm-4.7" \
// node scripts/llm-arena.mjs
// ARENA_OUT=<dir> redirects output (default ./arena-results, results merge).
//
// Requirements: `npm run build`, a running ComfyUI with at least one txt2img
// checkpoint, Ollama with the models pulled. Results land in arena-results/
// (JSON + share-ready markdown report).
//
// TIP: generate_image auto-selects the FIRST local checkpoint when none is
// set. If your checkpoints folder leads with a non-txt2img model (video/SAM),
// pin one for the whole arena: COMFYUI_DEFAULT_CHECKPOINT=<file>.safetensors
//
// Scoring per scenario: PASS = 2 (task done, verified against ComfyUI),
// PARTIAL = 1 (right tool ran, outcome incomplete), FAIL = 0.
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
import { SCENARIOS } from "./arena-scenarios.mjs";
const OLLAMA = process.env.OLLAMA_HOST ?? "http://127.0.0.1:11434";
const MODELS = (process.env.ARENA_MODELS ?? "gemma4:e4b,gemma4:e2b,qwen3:8b,qwen3:4b,llama3.1:8b")
.split(",")
.map((m) => m.trim())
.filter(Boolean);
const MAX_ROUNDS = Number(process.env.ARENA_MAX_ROUNDS ?? 22);
/** Tier label recorded on this run's models (e.g. "SoTA", "B-tier", "local")
* so the merged leaderboard can compare classes. */
const TIER = process.env.ARENA_TIER ?? "local";
const SCENARIO_TIMEOUT_MS = Number(process.env.ARENA_SCENARIO_TIMEOUT_MS ?? 360_000);
const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..");
const OUT_DIR = process.env.ARENA_OUT ?? join(process.cwd(), "arena-results");
const SYSTEM =
"You control a ComfyUI MCP server through exactly three tools: list_tools (catalog), " +
"describe_tool (one tool's parameters), call_tool (run a tool by name with args). " +
"Always look up a tool with describe_tool before running it with call_tool. " +
"Catalog entries are tool NAMES, not data — complete every task by actually running tools with call_tool.";
/**
* Two API dialects, one arena:
* - ARENA_API=ollama (default): Ollama-native /api/chat — lets us pin num_ctx.
* - ARENA_API=openai: any OpenAI-compatible /v1/chat/completions — hosted
* models (DeepSeek, GLM, MiniMax, MiMo, OpenRouter, ...). Set ARENA_BASE_URL
* and ARENA_API_KEY. Tool results carry tool_call_id per the OpenAI shape.
*/
const API = process.env.ARENA_API ?? "ollama";
const BASE_URL = process.env.ARENA_BASE_URL ?? `${OLLAMA}/v1`;
const API_KEY = process.env.ARENA_API_KEY ?? "";
async function chat(model, messages, tools) {
if (API === "openai") {
const res = await fetch(`${BASE_URL.replace(/\/$/, "")}/chat/completions`, {
method: "POST",
headers: {
"content-type": "application/json",
...(API_KEY ? { authorization: `Bearer ${API_KEY}` } : {}),
},
body: JSON.stringify({ model, messages, tools, tool_choice: "auto", temperature: 0 }),
});
if (!res.ok) throw new Error(`${BASE_URL} http ${res.status}: ${await res.text()}`);
const msg = (await res.json()).choices?.[0]?.message;
if (!msg) throw new Error("openai-compatible response had no choices[0].message");
return msg;
}
const res = await fetch(`${OLLAMA}/api/chat`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
model,
messages,
tools,
stream: false,
options: { num_ctx: 16384, temperature: 0 },
}),
});
if (!res.ok) throw new Error(`ollama http ${res.status}: ${await res.text()}`);
return (await res.json()).message;
}
/** Tool-result message in the active dialect. */
function toolResultMessage(toolCall, toolName, content) {
if (API === "openai") {
return { role: "tool", tool_call_id: toolCall.id ?? "call_0", content };
}
return { role: "tool", tool_name: toolName, content };
}
async function connectMcp() {
const transport = new StdioClientTransport({
command: process.execPath,
args: [join(ROOT, "dist", "index.js")],
env: {
...process.env,
COMFYUI_MCP_TOOL_MODE: "compact",
COMFYUI_MCP_PANEL_AUTOINSTALL: "0",
COMFYUI_MCP_AUTOUPDATE: "0",
LOG_LEVEL: "error",
},
});
const mcp = new Client({ name: "llm-arena", version: "0.0.0" });
await mcp.connect(transport);
return mcp;
}
function textOf(result) {
return (result.content ?? [])
.filter((c) => c.type === "text")
.map((c) => c.text)
.join("\n");
}
async function runScenario(mcp, ollamaTools, model, scenario) {
const messages = [
{ role: "system", content: SYSTEM },
{ role: "user", content: scenario.task },
];
const t = { calls: [], toolText: "", finalAnswer: "", rounds: 0, nudges: 0 };
const started = Date.now();
for (let round = 0; round < MAX_ROUNDS; round++) {
if (Date.now() - started > SCENARIO_TIMEOUT_MS) break;
t.rounds = round + 1;
let msg;
try {
msg = await chat(model, messages, ollamaTools);
} catch (err) {
t.finalAnswer = `(harness error: ${err.message})`;
break;
}
messages.push(msg);
if (!msg.tool_calls?.length) {
if (!t.calls.some((c) => c.ok) && t.nudges < 2) {
t.nudges++;
messages.push({
role: "user",
content:
"You have not successfully run a tool yet. Use describe_tool then call_tool to actually do the task.",
});
continue;
}
t.finalAnswer = msg.content ?? "";
break;
}
for (const tc of msg.tool_calls) {
const name = tc.function.name;
let args = tc.function.arguments;
if (typeof args === "string") {
try {
args = JSON.parse(args);
} catch {
args = {};
}
}
let text = "";
let ok = false;
try {
const result = await mcp.callTool({ name, arguments: args });
text = textOf(result);
ok = !result.isError;
} catch (err) {
text = `MCP error: ${err.message}`;
}
if (name === "call_tool") {
const inner = args?.name ?? args?.tool_name ?? "?";
t.calls.push({ tool: inner, ok });
console.log(` ${inner} ${ok ? "ok" : "ERR"}`);
}
t.toolText += `\n${text}`;
messages.push(toolResultMessage(tc, name, text.slice(0, 12000)));
}
}
const okTools = [...new Set(t.calls.filter((c) => c.ok).map((c) => c.tool))];
const primaryOk = scenario.primary.some((p) => okTools.includes(p));
const followupOk = !scenario.followup || scenario.followup.some((p) => okTools.includes(p));
// harness-side ground truth, via a direct call_tool (never trusts the model)
const harnessCall = async (toolName, toolArgs) => {
const res = await mcp.callTool({ name: "call_tool", arguments: { name: toolName, args: toolArgs } });
return textOf(res);
};
let verified = false;
if (primaryOk) {
try {
verified = await scenario.verify(harnessCall, t);
} catch {
verified = false;
}
}
const partialOk = primaryOk || (scenario.partial ?? []).some((p) => okTools.includes(p));
const score = primaryOk && followupOk && verified ? 2 : partialOk ? 1 : 0;
return {
scenario: scenario.id,
score,
verdict: score === 2 ? "PASS" : score === 1 ? "PARTIAL" : "FAIL",
rounds: t.rounds,
nudges: t.nudges,
seconds: Math.round((Date.now() - started) / 1000),
okTools,
finalAnswer: t.finalAnswer.slice(0, 400),
transcript: messages,
};
}
// ---------------------------------------------------------------------------
console.log(`ComfyUI LLM Arena — models: ${MODELS.join(", ")}`);
const mcp = await connectMcp();
const { tools } = await mcp.listTools();
if (tools.length !== 3) {
console.error(`expected 3 compact meta-tools, got ${tools.length}`);
process.exit(1);
}
const ollamaTools = tools.map((t) => ({
type: "function",
function: { name: t.name, description: t.description, parameters: t.inputSchema },
}));
// preflight: ComfyUI reachable?
const preflight = await mcp.callTool({ name: "call_tool", arguments: { name: "get_system_stats", args: {} } });
if (preflight.isError) {
console.error(`ComfyUI is not reachable: ${textOf(preflight).slice(0, 300)}`);
process.exit(1);
}
let gpu = "unknown GPU";
try {
const stats = JSON.parse(textOf(preflight));
gpu = stats?.devices?.[0]?.name ?? gpu;
} catch {
// non-JSON stats — leave the placeholder
}
const all = [];
for (const model of MODELS) {
console.log(`\n════════ ${model} ════════`);
const results = [];
for (const scenario of SCENARIOS) {
console.log(` ▸ ${scenario.id}`);
const r = await runScenario(mcp, ollamaTools, model, scenario);
console.log(` ${r.verdict} (${r.seconds}s, ${r.rounds} rounds, nudges=${r.nudges})`);
results.push(r);
}
const total = results.reduce((s, r) => s + r.score, 0);
// full transcripts go to their own files; keep the leaderboard JSON light
mkdirSync(join(OUT_DIR, "transcripts"), { recursive: true });
for (const r of results) {
writeFileSync(
join(OUT_DIR, "transcripts", `${model.replace(/[:/]/g, "_")}-${r.scenario}.json`),
JSON.stringify(r.transcript, null, 2),
);
delete r.transcript;
}
all.push({ model, tier: TIER, total, max: SCENARIOS.length * 2, results });
console.log(` Σ ${model}: ${total}/${SCENARIOS.length * 2}`);
}
await mcp.close();
await new Promise((r) => setTimeout(r, 250));
// ---------------------------------------------------------------------------
// merge with prior runs so the field can be run one model at a time
mkdirSync(OUT_DIR, { recursive: true });
const resultsPath = join(OUT_DIR, "arena-results.json");
if (existsSync(resultsPath)) {
try {
const prior = JSON.parse(readFileSync(resultsPath, "utf8"));
for (const p of prior.leaderboard ?? []) {
if (!all.some((m) => m.model === p.model)) all.push({ tier: "local", ...p });
}
} catch {
// corrupt prior results — overwrite
}
}
// Tie-breakers, in order: score, fewer nudges (didn't need correcting),
// fewer rounds (efficient tool use), less wall time. All already recorded.
const nudgesOf = (m) => m.results.reduce((s, r) => s + (r.nudges ?? 0), 0);
const roundsOf = (m) => m.results.reduce((s, r) => s + (r.rounds ?? 0), 0);
const secondsOf = (m) => m.results.reduce((s, r) => s + (r.seconds ?? 0), 0);
all.sort(
(a, b) =>
b.total - a.total ||
nudgesOf(a) - nudgesOf(b) ||
roundsOf(a) - roundsOf(b) ||
secondsOf(a) - secondsOf(b),
);
writeFileSync(resultsPath, JSON.stringify({ gpu, scenarios: SCENARIOS.map((s) => ({ id: s.id, title: s.title, task: s.task })), leaderboard: all }, null, 2));
const md = [];
md.push(`# ComfyUI LLM Arena`);
md.push("");
md.push(
`Local models driving a real ComfyUI (${gpu}) through [comfyui-mcp](https://github.com/artokun/comfyui-mcp)'s ` +
`compact tool mode — 3 meta-tools instead of ~200 schemas, so even small models can play. ` +
`Each model gets the identical task set; results are verified against the ComfyUI server, not the model's claims.`,
);
md.push("");
const header = ["Model", "Tier", ...SCENARIOS.map((s) => s.id), "Score", "Nudges", "Rounds", "Time"];
md.push(`| ${header.join(" | ")} |`);
md.push(`|${header.map(() => "---").join("|")}|`);
const icon = { 2: "✅", 1: "🟡", 0: "❌" };
for (const m of all) {
const cells = m.results.map((r) => icon[r.score]);
md.push(
`| \`${m.model}\` | ${m.tier ?? "local"} | ${cells.join(" | ")} | **${m.total}/${m.max}** | ${nudgesOf(m)} | ${roundsOf(m)} | ${secondsOf(m)}s |`,
);
}
md.push("");
md.push(`Scenarios: ${SCENARIOS.map((s) => `**${s.id}** (${s.title.toLowerCase()})`).join(" · ")}`);
md.push("");
md.push(`✅ task completed & verified · 🟡 right tool, incomplete outcome · ❌ failed`);
md.push("");
md.push("Reproduce: `npm run build && node scripts/llm-arena.mjs` — bring your own models via `ARENA_MODELS=...`");
writeFileSync(join(OUT_DIR, "arena-report.md"), `${md.join("\n")}\n`);
console.log(`\n══════ LEADERBOARD ══════`);
for (const m of all) console.log(`${String(m.total).padStart(2)}/${m.max} ${m.model}`);
console.log(`\nreport: ${join(OUT_DIR, "arena-report.md")}`);