comfyui-mcp
Version:
Local-first, agent-native control plane for ComfyUI — MCP server + autonomous sidebar agent that drives your live graph in natural language on ANY LLM: Claude/ChatGPT/Gemini on your subscription (no API key), free local models via Ollama (fully offline),
441 lines • 22.7 kB
JavaScript
// Audio attachments on an agent turn (issue #790).
//
// Before this module the agent could DRIVE ComfyUI's audio tools but never HEAR
// anything: every backend attached only image content parts, so an audio file
// either never left the orchestrator or (worse) was handed to a vision part and
// silently ignored. This module owns the two things that must never be guessed:
//
// 1. WHICH bytes are audio we can actually deliver (mime + non-empty + size).
// 2. WHAT the user is told when we cannot deliver them.
//
// The governing rule for (2): a SILENTLY dropped attachment is the worst
// outcome. The user asks about a sound, the model answers from the text alone,
// and nothing in the transcript reveals it never heard it. So every path here
// ends in either "delivered" or a refusal the user reads verbatim, and every
// refusal names something that WOULD work from where the caller is standing.
//
// Wire formats (both live-verified against Ollama on 2026-08-04, gemma4:e2b —
// see docs/local-llms.mdx):
// • Ollama native POST /api/chat → message.images[] (raw base64).
// Ollama's own /v1/audio/transcriptions middleware does exactly this
// (openai/openai.go FromTranscriptionRequest puts AudioData in Images), so
// `images` is really "binary multimodal attachments", not strictly images.
// • OpenAI dialect POST /v1/chat/completions → {type:"input_audio",
// input_audio:{data,format}}. NOTE an audio data-URL in an `image_url` part
// is a hard 400 ("invalid image input") — audio must never ride that part.
//
// NOT shipped: ACP (Gemini/Grok CLI) session/prompt does define an audio
// ContentBlock, but it requires the agent to advertise an `audio` prompt
// capability first and neither CLI has been observed doing so — an unexercisable
// send path whose failure mode is an unheard attachment is the overclaim this
// module exists to remove. Those backends declare `audio: false` instead.
/**
* Extension → mime for audio we are willing to put on the wire. Keep this the
* SINGLE source of truth: the accepted-format list quoted back to the user in a
* refusal is derived from these keys, so the two can never drift.
*/
export const AUDIO_MIME_BY_EXT = Object.freeze({
wav: "audio/wav",
wave: "audio/wav",
mp3: "audio/mpeg",
ogg: "audio/ogg",
oga: "audio/ogg",
opus: "audio/opus",
flac: "audio/flac",
m4a: "audio/mp4",
m4b: "audio/mp4",
aac: "audio/aac",
mpga: "audio/mpeg",
mp2: "audio/mpeg",
webm: "audio/webm",
weba: "audio/webm",
});
/**
* Extensions that are unmistakably AUDIO but that we cannot encode.
*
* These exist so classification and encodability stay separate questions. A
* `song.wma` routed onto the image path would be handed to a vision content
* part — a hard 400 on the OpenAI dialect, an image-slot mis-encode on the
* native one, and in neither case an explanation the user can act on. Claiming
* it here means it reaches the audio path and gets the real answer: "that isn't
* a format I can encode; convert it to one of …".
*/
export const AUDIO_EXT_NOT_ENCODABLE = Object.freeze([
"wma",
"mid",
"midi",
"aiff",
"aif",
"aifc",
"alac",
"amr",
"au",
"snd",
"ra",
"rm",
"ape",
"wv",
"dsf",
"dff",
"caf",
"voc",
"8svx",
"mka",
"m4p",
"spx",
"awb",
"aa",
"aax",
"qcp",
]);
/** True when a filename is audio-SHAPED, whether or not we can encode it. Used
* to decide which path an attachment belongs on — never whether it can be sent. */
export function looksLikeAudioFilename(filename) {
const dot = filename.lastIndexOf(".");
if (dot < 0 || dot === filename.length - 1)
return false;
const ext = filename.slice(dot + 1).toLowerCase();
return ext in AUDIO_MIME_BY_EXT || AUDIO_EXT_NOT_ENCODABLE.includes(ext);
}
/** The formats a refusal message offers, derived from AUDIO_MIME_BY_EXT so the
* prose can never claim support the encoder table doesn't actually have. */
export function supportedAudioFormats() {
return [...new Set(Object.keys(AUDIO_MIME_BY_EXT))].join(", ");
}
/** True for any mime we can encode as an audio part. */
export function isDeliverableAudioMime(mime) {
const m = mime.split(";")[0].trim().toLowerCase();
return Object.values(AUDIO_MIME_BY_EXT).includes(m);
}
/** Mime for a filename, or null when the extension is not audio we deliver.
* Used both to CLASSIFY (is this attachment audio at all?) and to fill in a
* mime when the server's Content-Type is missing or generic. */
export function audioMimeForFilename(filename) {
const dot = filename.lastIndexOf(".");
if (dot < 0 || dot === filename.length - 1)
return null;
const ext = filename.slice(dot + 1).toLowerCase();
return AUDIO_MIME_BY_EXT[ext] ?? null;
}
/** OpenAI `input_audio.format` token for a mime. The OpenAI content part takes a
* bare format word, not a mime — a wrong token is rejected by strict endpoints,
* so this is an explicit table rather than a mime.split("/") guess. */
export function openAiAudioFormat(mime) {
switch (mime.split(";")[0].trim().toLowerCase()) {
case "audio/wav":
return "wav";
case "audio/mpeg":
return "mp3";
case "audio/ogg":
return "ogg";
case "audio/opus":
return "opus";
case "audio/flac":
return "flac";
case "audio/mp4":
return "m4a";
case "audio/aac":
return "aac";
case "audio/webm":
return "webm";
default:
return "wav";
}
}
/**
* Largest audio payload we will inline. A base64 body is ~1.33x the file, and a
* local endpoint has to hold the whole request in memory before it decodes, so
* this is a real limit rather than a formality. 24 MB comfortably covers a
* full-length mp3/ogg song (the reporter's case in #790) while refusing an
* uncompressed multi-minute WAV, which the refusal tells the user how to fix.
*/
export const MAX_AUDIO_BYTES = 24 * 1024 * 1024;
/** Model-facing note appended to the user text, so the model cannot answer as if
* it had heard something it never received. Mirrors the image note's contract. */
export function audioModelNote(refusals) {
const refused = refusals.filter((r) => r.status === "refused");
if (!refused.length)
return "";
return (`\n\n[panel note: the user attached ${refused.length} audio file(s) that did NOT reach you — ` +
refused.map((r) => `${r.filename} (${r.reason})`).join("; ") +
`. You did NOT hear them. Do not describe, transcribe, analyse or imply anything about their contents; ` +
`say plainly that the audio did not reach you.]`);
}
/**
* Model-facing note for audio we ESTABLISHED this model can take and did put on
* the request.
*
* Not decoration. Measured live on 2026-08-04 against gemma4:e2b: the WAV was
* demonstrably in context (555 prompt tokens, `/api/show` reporting the `audio`
* capability), and the model still answered *"I do not have the capability to
* transcribe audio; my functions are limited to operating ComfyUI"* — the panel
* system prompt casts it as a graph operator, and it reasons itself out of a
* sense it actually has. Delivering the bytes and letting that stand would be a
* different flavour of the same silent failure: the audio arrives, the user is
* told nothing, and the answer is composed as if it hadn't.
*
* Every claim here is backed by something observed: the server reported the
* capability, and the bytes are on the request.
*/
export function audioDeliveredModelNote(count, model) {
return (`\n\n[panel note: ${count} audio file(s) are attached to THIS message and you can hear them — the server ` +
`reports that ${model} has the audio capability, and the audio is in your context. Listen to it and answer ` +
`from what you actually hear. Do NOT reply that you cannot process audio or that no audio was provided; ` +
`if you genuinely perceive nothing, say that you received audio but could not make anything out.]`);
}
/** Model-facing note for an attempt we could not confirm landed. Deliberately
* weaker than the refusal note: the audio probably DID arrive. */
export function audioUnverifiedModelNote(count) {
return (`\n\n[panel note: ${count} audio file(s) were attached to this turn, but this endpoint exposes no ` +
`capability probe so the delivery is UNCONFIRMED. If you cannot actually perceive the audio, say so ` +
`plainly instead of guessing at its contents.]`);
}
/** User-facing summary line for a turn that carried audio. Returns null when
* there is nothing worth saying (everything delivered on an established path). */
export function audioUserNotice(outcomes, confidence, model) {
const refused = outcomes.filter((o) => o.status === "refused");
const delivered = outcomes.filter((o) => o.status === "delivered");
const parts = [];
if (refused.length) {
// One line per refusal: they can have DIFFERENT causes and therefore
// different remedies, and collapsing them would hide the actionable one.
for (const r of refused)
if (r.status === "refused")
parts.push(`🔇 ${r.text}`);
}
if (delivered.length && confidence === "unverified") {
// Deliberately phrased as what is ON the outgoing request, not as a
// completed delivery: at the moment this is said the bytes are attached but
// the response has not come back, and claiming otherwise would be exactly
// the "reported as sent when it was not" failure this feature exists to
// prevent. If the endpoint then rejects it, the caller's strip-and-retry
// issues the correction.
parts.push(`🎧 Attaching ${delivered.length} audio file(s) to ${model}. This endpoint exposes no capability ` +
`probe, so I cannot confirm the model actually receives them — if the reply does not reflect what ` +
`is in the file, it did not hear it. A local Ollama model reports its own audio capability, so ` +
`that path can be checked (\`ollama pull ${AUDIO_CAPABLE_OLLAMA_MODELS[0]}\`).`);
}
return parts.length ? parts.join("\n\n") : null;
}
/** Audio-capable local models we have a concrete, checkable pull command for.
* Ollama's own release integration tests pin these as the audio-tested set
* (integration/reg_release_test.go: releaseAudioModels), which is why they are
* named here rather than a general "try a multimodal model" hand-wave.
*
* This is also the native-Ollama SEND allowlist (#1972). `/api/show` reporting
* `audio` is an architecture flag, not a guarantee these weights can hear: a
* namespaced Gemma 4 fork accepts a WAV in `message.images[]` and returns a
* fluent fabricated transcript. Only these tags may ride that carrier. */
export const AUDIO_CAPABLE_OLLAMA_MODELS = ["gemma4:e2b", "gemma4:e4b", "nemotron3:33b"];
function normalizeOllamaAudioModelId(model) {
let s = model.trim().toLowerCase();
if (s.startsWith("library/"))
s = s.slice("library/".length);
return s;
}
/**
* True when `model` is one of the Ollama tags we have actually verified can
* hear — not merely a tag whose architecture reports an `audio` capability.
*
* Native Ollama has no separate audio field; bytes ride `message.images[]`.
* "The endpoint took it" is therefore not "the model heard it" (#1972).
* A user/org namespace is a different weight set even when `/api/show`
* inherits the flag, so `library/` is the only prefix that still matches.
*/
export function isKnownAudioCapableOllamaModel(model) {
const normalized = normalizeOllamaAudioModelId(model);
return AUDIO_CAPABLE_OLLAMA_MODELS.some((known) => normalizeOllamaAudioModelId(known) === normalized);
}
/** Refusal text: this backend has no audio content part at all. Names the
* providers that DO, so the remedy is reachable from where the user is. */
export function noAudioPartText(backendId, filename) {
// The remedy names the ONE provider where audio is capability-checked and
// verified. Listing the OpenAI-compatible providers as equals here would send
// the user to a path that only *attempts* delivery — a remedy is no use if it
// is itself an overclaim.
return (`I can't send ${filename} to the model: the ${backendId} provider has no audio input at all in this ` +
`build, so the file would be dropped without either of us noticing. Switch to the local Ollama provider ` +
`with a model that reports audio support (\`ollama pull ${AUDIO_CAPABLE_OLLAMA_MODELS[0]}\`) and ` +
`re-send — that path checks the model's capability before sending. Or ask me to run a ComfyUI ` +
`audio-analysis node over the file instead and read you the numbers.`);
}
/** Refusal text: the endpoint answered our capability probe, and this model
* cannot take audio. Quotes what the server actually reported. */
export function modelLacksAudioText(model, capabilities, filename) {
const caps = capabilities.length ? capabilities.join(", ") : "none reported";
return (`I can't send ${filename} to ${model}: the server reports its capabilities as [${caps}] — no audio. ` +
`Pull a model that can hear and select it, e.g. \`ollama pull ${AUDIO_CAPABLE_OLLAMA_MODELS[0]}\` ` +
`(others: ${AUDIO_CAPABLE_OLLAMA_MODELS.slice(1).join(", ")}). Until then I have not heard this file ` +
`and won't pretend otherwise.`);
}
/** Refusal text: native Ollama would put audio in the image slot, and this
* model is not one we have verified can hear.
*
* Distinct from `modelLacksAudioText`: the server may well list an `audio`
* capability (architecture flag) even when the weights invent a transcript.
* Quoting that flag as "no audio" would be a lie; this names the actual
* reason we are not sending. */
export function modelNotVerifiedAudioText(model, filename) {
return (`I can't send ${filename} to ${model}: native Ollama carries audio in the image slot, and a model ` +
`that merely ACCEPTS that payload can still invent a transcript. ${model} is not in the verified-audio ` +
`set, so I am not sending the bytes. Pull a model that can hear, e.g. \`ollama pull ${AUDIO_CAPABLE_OLLAMA_MODELS[0]}\` ` +
`(others: ${AUDIO_CAPABLE_OLLAMA_MODELS.slice(1).join(", ")}), and re-send. Until then I have not heard this file ` +
`and won't pretend otherwise.`);
}
/** Refusal text: not a format we can encode. */
export function unsupportedFormatText(filename, mime) {
return (`I can't send ${filename} to the model: ${mime ? `its type is ${mime}, which` : "it has no audio file extension I recognise, so it"} ` +
`is not an audio format I can encode. Convert it to one of: ${supportedAudioFormats()} — and re-attach.`);
}
/** Refusal text: the attachment exists but has no bytes. */
export function emptyAudioText(filename) {
return (`I can't send ${filename} to the model: the file is 0 bytes, so there is nothing to hear. ` +
`Re-export or re-upload it and attach it again.`);
}
/** Refusal text: too big to inline. */
export function tooLargeAudioText(filename, bytes) {
const mb = (bytes / (1024 * 1024)).toFixed(1);
const capMb = (MAX_AUDIO_BYTES / (1024 * 1024)).toFixed(0);
return (`I can't send ${filename} to the model: it is ${mb} MB and the inline limit is ${capMb} MB. ` +
`Re-encode it as mp3 or ogg, or trim it to the section you want me to listen to, and re-attach.`);
}
/**
* Most audio attachments carried on ONE turn.
*
* Low on purpose: audio is far more token-dense than an image — measured live
* against gemma4:e2b, 1.5 s of 16 kHz WAV cost ~40 prompt tokens (~27 tok/s), so
* a single 3-minute song is already ~5k tokens of a small model's window. The
* excess is REFUSED OUT LOUD rather than truncated away, because a quietly
* dropped third attachment is the same silent failure as a dropped first one.
*/
export const MAX_AUDIO_ATTACHMENTS = 2;
/** Refusal text: more attachments than one turn will carry. */
export function tooManyAudioText(filename, limit) {
return (`I can't send ${filename} to the model: only ${limit} audio file(s) fit on one turn ` +
`(audio is token-expensive — roughly 27 prompt tokens per second of sound). ` +
`Send it on its own follow-up message and I'll listen to it then.`);
}
/** Refusal text: we could not read the bytes out of ComfyUI. */
export function fetchFailedAudioText(filename, detail) {
return (`I can't send ${filename} to the model: fetching it from ComfyUI failed (${detail}). ` +
`Check the file still exists in the ComfyUI input/output folder and re-attach it.`);
}
/**
* Fetch one audio attachment out of ComfyUI's /view and classify it.
*
* Shared by every backend that can carry audio, so the four failure shapes the
* user can hit — unreadable extension, non-audio Content-Type, zero bytes, over
* the inline cap — produce the SAME actionable refusal wherever they occur.
* There is deliberately no `null` return: "couldn't read it" must never be
* indistinguishable from "there was nothing to attach".
*/
export async function fetchAudioAttachment(comfyuiUrl, ref, fetchImpl = fetch) {
const refuse = (reason, text) => ({
ok: false,
outcome: { status: "refused", filename: ref.filename, reason, text },
});
if (!ref?.filename)
return refuse("fetch-failed", fetchFailedAudioText("(unnamed)", "the attachment carried no filename"));
const nameMime = audioMimeForFilename(ref.filename);
if (!nameMime)
return refuse("unsupported-format", unsupportedFormatText(ref.filename, null));
if (!comfyuiUrl) {
return refuse("fetch-failed", fetchFailedAudioText(ref.filename, "no ComfyUI URL is configured for this agent"));
}
try {
const u = new URL("/view", comfyuiUrl);
u.searchParams.set("filename", ref.filename);
u.searchParams.set("type", ref.type || "input");
if (ref.subfolder)
u.searchParams.set("subfolder", ref.subfolder);
const res = await fetchImpl(u, { signal: AbortSignal.timeout(30000) });
if (!res.ok)
return refuse("fetch-failed", fetchFailedAudioText(ref.filename, `http ${res.status}`));
const ct = (res.headers.get("content-type") || "").split(";")[0].trim().toLowerCase();
// The server's Content-Type outranks the filename, and only three answers
// are acceptable:
// audio/* we can encode -> use it
// audio/* we cannot encode -> refuse, naming the type
// generic binary or absent -> fall back to the extension
// Anything ELSE is refused rather than trusted to the extension. A 200 from
// a proxy or a mis-served file can be text/html or image/png under a .wav
// name, and attaching those bytes as audio would report a delivery that,
// from the model's side, contains no sound at all — the silent failure this
// whole path exists to prevent, wearing a success message.
const GENERIC_BINARY = new Set(["", "application/octet-stream", "binary/octet-stream", "application/binary"]);
let mime = nameMime;
if (ct.startsWith("audio/")) {
if (!isDeliverableAudioMime(ct))
return refuse("unsupported-format", unsupportedFormatText(ref.filename, ct));
mime = ct;
}
else if (!GENERIC_BINARY.has(ct)) {
return refuse("unsupported-format", unsupportedFormatText(ref.filename, ct));
}
const buf = Buffer.from(await res.arrayBuffer());
if (buf.length === 0)
return refuse("empty-file", emptyAudioText(ref.filename));
if (buf.length > MAX_AUDIO_BYTES)
return refuse("too-large", tooLargeAudioText(ref.filename, buf.length));
return { ok: true, b64: buf.toString("base64"), mime, bytes: buf.length };
}
catch (err) {
return refuse("fetch-failed", fetchFailedAudioText(ref.filename, err instanceof Error ? err.message : String(err)));
}
}
/**
* Split a mixed attachment list into images and audio by FILENAME EXTENSION.
*
* A panel that only knows about `images` can still attach a .wav, and routing it
* to an image content part is not a harmless no-op: on the OpenAI dialect it is
* a hard 400, and on the native dialect it lands in an image slot. Splitting
* here means the audio path claims it (and can refuse it honestly) instead.
*
* Classification uses `looksLikeAudioFilename`, NOT "can we encode it" — a
* `song.wma` is audio the user is trying to share, and it belongs on the path
* that can tell them what to convert it to.
*/
/**
* Collapse refs that name the SAME ComfyUI file to one attachment.
*
* The panel has two carriers for a sound — the dedicated `audio` array and the
* legacy `images` array a older/compatible composer still uses — and a panel
* that populates both sends the same file twice. Without this, one attachment
* counts twice against MAX_AUDIO_ATTACHMENTS: the second copy is REFUSED, and
* the user is told out loud that a file they attached once did not fit, while
* the identical bytes are also sitting on the request. A duplicate of a file we
* are already sending is not a second attachment.
*
* Identity is the tuple ComfyUI's /view is addressed by (filename + subfolder +
* type), normalised the same way `fetchAudioAttachment` builds that URL, so two
* refs are merged only when they would fetch byte-for-byte the same file. A
* missing `type` really does mean "input" on both sides — that is the fetcher's
* default, not a guess made here.
*
* Order is preserved and the FIRST occurrence wins, so the earlier carrier
* decides nothing beyond position.
*/
export function dedupeAudioRefs(refs) {
const seen = new Set();
const out = [];
for (const ref of refs) {
const key = JSON.stringify([ref.filename, ref.subfolder ?? "", ref.type || "input"]);
if (seen.has(key))
continue;
seen.add(key);
out.push(ref);
}
return out;
}
export function splitAudioAttachments(refs) {
const images = [];
const audio = [];
for (const ref of refs ?? []) {
if (looksLikeAudioFilename(ref.filename))
audio.push(ref);
else
images.push(ref);
}
return { images, audio };
}
//# sourceMappingURL=audio-attachment.js.map