@kolbo/mcp
Version:
Kolbo AI MCP Server - Generate images, videos, music, speech, and sound effects from Claude Code
2,739 lines • 229 kB
JavaScript
/* ⛔ BACKWARD COMPATIBILITY: Tool names and arg names below are a PUBLIC
* CONTRACT. Never rename, remove, or break an existing tool/arg — old cached
* `npx @kolbo/mcp` installs in the wild will break silently. Add new tools or
* new OPTIONAL args only. Full rules: ../index.js top-of-file and CLAUDE.md. */
const { z } = require('zod');
const FormData = require('form-data');
const { pollUntilDone, waitWindowMs } = require('../polling');
const { resolveToBuffer, pollOrTimedOut, creditFields, projectIdField, sessionIdField, inlineImageBlocks, linkFields, uiGenerating, asyncGenerating, uiCompleted, appsEnabled } = require('./_shared');
const { ownedUrl } = require('./owned-url');
const { UI, uiResult, canonicalModelId, assertModelSupportsType, modelInfo, voiceInfo, resolveCatalogAspectRatio } = require('../apps');
const { modelTypeForEditOperation, assertExecutableEditModel } = require('./editModelCatalog');
const { withLocalRehost, rehostLocalPaths } = require('./local-rehost');
const { validateVideoPrompt } = require('./video-prompt');
// ─── Cinematic Dimensions schema (shared by generate_image + generate_image_edit) ───
// Kolbo's "Cinema mode": eight independent photographic dimensions, each an OPTIONAL
// preset id fetched from list_cinematic_presets. Any dimension left unset = "Auto" (the
// enhancer decides). Applies ONLY when the user explicitly wants a deliberate cinematic
// look — omit the whole object for a normal generation. Ids are validated server-side
// against their dimension (passing a lighting id in the camera slot is a 400). The
// selected fragments are woven into the prompt before enhancement.
// NOTE: dimensions are data-driven — always call list_cinematic_presets for the live set
// and valid ids; the keys below mirror the current catalog (Genre was removed).
const cinematicDim = (label) => z.string().nullable().optional()
.describe(`${label} preset id from list_cinematic_presets. Omit or null for Auto.`);
const CINEMATIC_SCHEMA = z.object({
camera: cinematicDim('Camera body/format (e.g. ARRI Alexa, 16mm film, smartphone)'),
lens: cinematicDim('Lens character (e.g. anamorphic, vintage prime, macro)'),
focal_length: cinematicDim('Focal length / field of view (e.g. 24mm wide, 85mm portrait)'),
aperture: cinematicDim('Aperture / depth of field (e.g. f/1.4 shallow, f/8 deep)'),
angle: cinematicDim('Camera angle (e.g. low angle, birds-eye, eye level)'),
shot_type: cinematicDim('Shot type / framing (e.g. close-up, wide shot, full shot)'),
color_palette: cinematicDim('Color grade (e.g. teal & orange, technicolor, classic B&W)'),
lighting: cinematicDim('Lighting technique (e.g. rim light, golden hour, low-key)'),
}).optional().describe(
'Cinema mode — an optional deliberate photographic treatment. Pass preset ids obtained from ' +
'list_cinematic_presets (at most one per dimension). Only include a dimension the user actually ' +
'wants; every omitted/null dimension is Auto (the enhancer completes the look in the spirit of the ' +
'ones you set). Omit the whole object entirely for an ordinary, non-cinematic generation. Ids are ' +
'validated against their dimension server-side. Dimensions are data-driven — never hardcode ids.'
);
// ─── Batch fan-out (prompts[] / items[]) ────────────────────────────────────
// One tool call, N DIFFERENT inputs → N generations tracked by ONE widget.
// The manual-control twin of generate_creative_director: no orchestration pass,
// the user's exact prompts verbatim. Submit failures never sink the batch —
// successful ids proceed, failed prompts are reported alongside.
// Over the cap is a hard rejection, never a truncation: this used to
// `.slice(0, MAX_BATCH_PROMPTS)`, so a 9-prompt call silently generated 8 and
// the caller had no way to know which prompt vanished. `promptsField` also caps
// the array in the schema (so hosts see `maxItems` before calling); the guard
// below is the choke point EVERY batch tool routes through, and names the count.
//
// An item is EITHER a bare prompt string (generate_image / generate_video, where
// only the prompt varies) OR an object carrying its own per-item inputs
// alongside the prompt (generate_video_from_image: `{ image_url, prompt }` — the
// image is what varies, and that is the whole point). Either way the widget
// captions each tile with the item's prompt, so that is the label we carry.
// ElevenLabs Music v2.5 composition-plan chunk — one entry is EITHER a generation chunk
// (text/duration_ms/positive_styles, optionally conditioning_ref to steer its style off a
// stored song) OR an audio-reference chunk (song_id/range, splices that slice in unchanged).
// All fields optional at the schema level; the backend validates the real constraints
// (3-120s per generation chunk, <=30s conditioning reference, <=30 chunks per plan).
const MUSIC_TIME_RANGE_SCHEMA = z.object({
start_ms: z.number().int().min(0),
end_ms: z.number().int().min(0),
}).describe('{ start_ms, end_ms } — a slice of a stored song, in milliseconds.');
const MUSIC_CHUNK_SCHEMA = z.object({
text: z.string().optional().describe('Generation chunk only: section name in [brackets], lyric lines, {inline directions}. E.g. "[Chorus]\\nWe rise tonight".'),
duration_ms: z.number().int().min(3000).max(120000).optional().describe('Generation chunk only: length in ms, 3000-120000.'),
positive_styles: z.array(z.string()).optional().describe('Generation chunk only: styles/directions to include (max 50). The first chunk\'s styles set the whole song\'s tone.'),
negative_styles: z.array(z.string()).optional().describe('Generation chunk only: styles/directions to avoid.'),
context_adherence: z.enum(['low', 'medium', 'high']).optional().describe('Generation chunk only: how closely it follows neighboring chunks. Default high.'),
conditioning_ref: z.object({ song_id: z.string(), range: MUSIC_TIME_RANGE_SCHEMA }).optional().describe('Generation chunk only: condition this chunk\'s STYLE on <=30s of a stored song (does not splice audio in — it steers a fresh generation).'),
condition_strength: z.enum(['low', 'medium', 'high', 'xhigh']).optional().describe('Generation chunk only: how strongly it follows conditioning_ref. Default medium.'),
song_id: z.string().optional().describe('Audio-reference chunk only: id of a stored song (from a prior generate_music result\'s song_id, or create_music_reference_audio) to splice in unchanged.'),
range: MUSIC_TIME_RANGE_SCHEMA.optional().describe('Audio-reference chunk only: the slice of song_id to insert unchanged.'),
});
const MAX_BATCH_PROMPTS = 8;
const MAX_STATUS_IDS = 20;
async function submitBatch(rawItems, submitOne) {
if (rawItems.length > MAX_BATCH_PROMPTS) {
throw new Error(
`Too many prompts: ${rawItems.length} received, max ${MAX_BATCH_PROMPTS} per call. ` +
`Split them across ${Math.ceil(rawItems.length / MAX_BATCH_PROMPTS)} calls of at most ${MAX_BATCH_PROMPTS}.`
);
}
const items = rawItems
.map((it) => (typeof it === 'object' && it !== null
? { input: it, label: String(it.prompt || '').trim() }
: { input: String(it).trim(), label: String(it).trim() }))
.filter((x) => x.label);
const settled = await Promise.allSettled(items.map((x) => submitOne(x.input)));
const ok = [], failed = [];
settled.forEach((s, i) => {
if (s.status === 'fulfilled' && s.value && s.value.generation_id) ok.push({ prompt: items[i].label, gen: s.value });
else failed.push({ prompt: items[i].label, error: (s.reason && s.reason.message) || 'submit failed' });
});
if (!ok.length) throw new Error(`All ${items.length} batch submissions failed: ${failed[0].error}`);
return { ok, failed, ids: ok.map((o) => o.gen.generation_id) };
}
// Blocking path for text-only hosts: wait for every batch member, aggregate.
// This is the exact path Kolbo Code takes for a `prompts`-array batch call
// (ui() is false there, so the uiGenerating branch above never runs) — it was
// text-only, no structuredContent, so a batch of N prompts rendered as a bare
// JSON dump instead of N cards. Same class of bug as get_generation_status's
// multi-id branch, same fix: always ship structuredContent via the shared
// kind:'status' grid, which already renders any mix of completed/processing
// items correctly.
async function pollBatch(client, batch, { interval, timeout }, toolName, submittedModel) {
const polls = await Promise.all(batch.ids.map((id) => pollOrTimedOut(client, id, { interval, timeout })));
const generations = polls.map((p, i) => p.timedOut
? { prompt: batch.ok[i].prompt, generation_id: batch.ids[i], status: 'processing', note: 'Still running — call get_generation_status with wait=true to collect it.' }
: { prompt: batch.ok[i].prompt, generation_id: batch.ids[i], status: 'completed', ...creditFields(polls[i].result), urls: p.result.result.urls, playback_urls: p.result.result.playback_urls });
const text = JSON.stringify({
batch: true,
session_id: batch.ok[0].gen.session_id,
generations,
failed_submissions: batch.failed.length ? batch.failed : undefined
}, null, 2);
// Name the model that actually ran — every id in one batch ran the same one.
const modelsRan = [...new Set(polls.filter((p) => !p.timedOut).map((p) => p.result.result && p.result.result.model).filter(Boolean))];
return uiCompleted({
tool: toolName, kind: 'status', client,
model: modelsRan.length === 1 ? modelsRan[0] : (submittedModel || 'Generations'),
gen: { generation_id: batch.ids[0], session_id: batch.ok[0].gen.session_id },
settings: {},
items: generations.map(g => ({
id: g.generation_id,
state: g.status,
title: g.prompt,
url: Array.isArray(g.urls) ? g.urls[0] : undefined,
urls: Array.isArray(g.urls) ? g.urls : undefined,
credits_used: g.credits_used,
})),
}, text);
}
// Same extension sniff the generation widget already does (kindFromTool in
// apps/widgets/generation.js) — a status check knows a url came back, not what
// kind of media it is, and the card needs the media kind to pick a renderer.
function mediaKind(url) {
const u = String(url || '').split('?')[0].toLowerCase();
if (/\.(mp4|mov|webm|mkv)$/.test(u)) return 'video';
if (/video-elements-results|generated-videos|\/videos?\//i.test(u)) return 'video';
if (/\.(mp3|wav|m4a|aac|ogg|flac)$/.test(u)) return 'audio';
if (/\.(glb|gltf|fbx|obj|usdz)$/.test(u)) return '3d';
return 'image';
}
function preferOwned(urls) {
const list = (urls || []).filter((item) => typeof item === 'string' && item);
const ours = list.filter(ownedUrl);
return ours.length ? ours : list;
}
const isUrlSource = (source) => typeof source === 'string' && /^https?:\/\//i.test(source);
// Which multipart kind a LOCAL file should upload as. mediaKind answers for the
// card; the upload routes only know these three, and the kind decides the
// content type a receiving controller dispatches on (kolbo-api's elements
// controller buckets req.files by `file.mimetype`), so guessing 'image' for
// everything filed a local .mp4 as a reference image.
const uploadKind = (source) => {
const kind = mediaKind(source);
return kind === 'video' || kind === 'audio' ? kind : 'image';
};
// The elements route's own multer ceiling (kolbo-api sdk/index.js). Cap here so
// an oversized file fails with a readable message instead of after the upload.
const ELEMENTS_MAX_UPLOAD_BYTES = 200 * 1024 * 1024;
// ─── Widget settings block ──────────────────────────────────────────────────
// What the CALLER actually asked for, for the generation card AND for the model
// reading the tool result. Undefined/false keys are dropped by JSON.stringify, so
// only values that were really supplied ever surface. This used to be
// `{ resolution, aspect_ratio }` only — `quality` (and every knob below it) was
// silently dropped, so three calls at low/medium/high rendered identical cards.
// `visual_dna`/`moodboard`/`preset` used to collapse to a count and two
// booleans, which told nobody WHICH asset was applied — not the user reading
// the card, and not the agent reading the tool result, so a follow-up call
// could not reuse the same DNA or preset without re-listing. Carry the ids.
const refSettings = (a = {}) => ({
font_ids: a.font_ids?.length ? a.font_ids : undefined,
enhance_prompt: a.enhance_prompt || undefined,
web_search: a.enable_web_search || undefined,
visual_dna_ids: (a.visual_dna_ids && a.visual_dna_ids.length) ? a.visual_dna_ids : undefined,
moodboard_id: a.moodboard_id || undefined,
moodboard_ids: (a.moodboard_ids && a.moodboard_ids.length) ? a.moodboard_ids : undefined,
preset_id: a.preset_id || undefined,
cinematic: a.cinematic ? true : undefined,
});
const aspectRatioDescribe = (fallback) =>
'Aspect ratio (e.g. "16:9", "9:16", "1:1"). MCP always snaps unsupported values '
+ 'to the closest ratio this model accepts — the card and the request both use '
+ 'the snapped value. Prefer list_models `supported_aspect_ratios`.'
+ (fallback ? ` Default: "${fallback}".` : '');
const imageSettings = (a = {}) => ({
resolution: a.resolution,
aspect_ratio: a.aspect_ratio,
quality: a.quality,
background: a.background,
output_format: a.output_format,
...refSettings(a),
});
// Same block for the video tools, which carried only { duration, resolution,
// aspect_ratio } — a DNA-anchored video card showed no sign a DNA was in play.
const videoSettings = (a = {}) => ({
// Multi-shot elements calls usually set shot COUNT and let the model take the
// default per-shot length, so `duration` came through undefined and the card
// showed model/resolution/ratio but never how long the video actually is —
// the one number the user picked and is paying for. Fall back to what the API
// echoed back on start, then to per-shot × shots.
duration: a.duration ?? a.reported_duration ?? (a.shot_duration && a.shots ? a.shot_duration * a.shots : undefined),
...(a.shots > 1 ? { shots: a.shots } : {}),
// Multishot with no explicit count: the provider decides how many, so there
// is no number to show — but the card still has to say the toggle applied.
...(a.multi_shot ? { multi_shot: true } : {}),
resolution: a.resolution,
...(typeof a.draft === 'boolean' ? { is_draft: a.draft } : {}),
aspect_ratio: a.aspect_ratio,
...(a.mode ? { mode: a.mode } : {}),
...refSettings(a),
});
const promptsField = (what) => z.array(z.string()).max(MAX_BATCH_PROMPTS).optional().describe(
`BATCH MODE — several DIFFERENT prompts (2–${MAX_BATCH_PROMPTS}) generated concurrently in ONE call and rendered together in ONE combined widget. **Hard cap: ${MAX_BATCH_PROMPTS} prompts per call — more than that is REJECTED with an error (never silently truncated), so split a longer list across several calls of at most ${MAX_BATCH_PROMPTS}.** Whenever the user wants multiple distinct ${what} with their own prompts, ALWAYS pass them all here instead of making several separate calls — separate calls clutter the chat with stacked widgets. All prompts share the same model/settings. When set, \`prompt\` is ignored. For N variations of a SINGLE prompt use num_images (image tools); for an AI-planned coherent scene set use generate_creative_director.`
);
// Native workflows: exactly one source video, prompt optional (the server analyzes the media).
const MORPHIOUS_MODELS = new Set(['kolbo-morphious-motion', 'kolbo-morphious-swap', 'kolbo-morphious-reframe', 'kolbo-morphious-lite-motion', 'kolbo-morphious-lite-swap']);
const PROMPTLESS_NATIVE = new Set([...MORPHIOUS_MODELS, 'higgsfield-genjutsu-motion-transfer', 'higgsfield-genjutsu-object-swap']);
const MORPHIOUS_GUIDE = ' MORPHIOUS / GENJUTSU (exactly ONE source video, omit duration - it comes from the source): higgsfield-genjutsu-motion-transfer and higgsfield-genjutsu-object-swap accept an omitted prompt, and so do the kolbo-morphious-* models (Morphious only needs the source video and references; it analyzes them itself). '
+ 'kolbo-morphious-motion 1-30 images, 4-30s; kolbo-morphious-swap 0-30 images, 4-30s; both 480p/720p/1080p, output keeps the source shape. '
+ 'Lite (kolbo-morphious-lite-motion 1-10 images / kolbo-morphious-lite-swap 0-10 images) caps at 15s. '
+ 'kolbo-morphious-reframe: NO images/DNA, 1-300s, aspect_ratio REQUIRED (16:9, 9:16, 1:1, 4:3, 3:4, 21:9), 540p or 720p. '
+ 'higgsfield-genjutsu-motion-transfer / -object-swap: 1-8 images or Visual DNA, 4-30s, 480p/720p/1080p. '
+ 'STYLES: to restyle a whole video into an animated look (2D toon, painterly 3D, gouache, comic, clay, felt...), pick an id from list_morphious_styles and pass it as style_id with model kolbo-morphious-motion; no reference image or prompt is needed.';
function registerGenerateTools(server, client, options = {}) {
// Every JSON POST from these tools rehosts local file paths into the media
// library first (see local-rehost.js) — the API only understands URLs.
client = withLocalRehost(client, { allowLocalFiles: !options.remote });
// Only enabled by hosts that explicitly opt in (the remote HTTP connector).
// stdio hosts (Kolbo Code, Claude Desktop, Cursor) leave this false, so their
// tool output is unchanged: a text block with the image URL.
const inlineImages = !!options.inlineImages;
const resolveInput = (source, kind, opts = {}) => resolveToBuffer(source, kind, {
...opts,
allowLocalFiles: !options.remote,
});
// JSON-body tools (edit_image) cannot carry a file, so a local path is
// uploaded to the media library first and its CDN URL is sent instead.
// URLs pass through untouched.
const rehostLocal = async (source, kind, project_id) => {
if (!source || isUrlSource(source)) return source;
const file = await resolveInput(source, kind);
const form = new FormData();
form.append('file', file.buffer, { filename: file.filename, contentType: file.contentType });
if (project_id) form.append('project_id', project_id);
const uploaded = await client.postMultipart('/v1/media/upload', form);
const url = uploaded?.media?.url || uploaded?.url;
if (!url) throw new Error(`Upload of ${source} returned no URL`);
return url;
};
// MCP Apps hosts (claude.ai remote connector, Claude Desktop) get an instant
// "submitted" response + a live ui://kolbo/generation.html widget that keeps
// one wait=true status call in flight. Text-only hosts never take this branch.
const ui = () => appsEnabled(server, options);
const returnsImmediately = () => ui() || !!options.asyncGenerations;
const submittedResult = (params) => ui() ? uiGenerating(params) : asyncGenerating(params);
// How long the STATUS tools may block inside one tool call before handing
// back a non-terminal result the caller re-issues. Bounded by the transport,
// not by the generation — full reasoning and the measured numbers live next
// to the constants in ../polling.js.
const WAIT_WINDOW_MS = waitWindowMs(options);
const WAIT_WINDOW_S = Math.round(WAIT_WINDOW_MS / 1000);
// What to tell a caller holding a still-running generation. Never "don't call
// again" — for anything longer than the window, calling again IS the protocol.
const stillRunningHint = (idsPhrase) =>
`Still running — this is NOT a failure and no credits were lost. Each wait=true call blocks for at most ~${WAIT_WINDOW_S}s and then returns whatever the state is, so a long job (music ~3 min, video can be longer) legitimately needs SEVERAL wait=true calls in a row. Call get_generation_status again with wait=true${idsPhrase}. Do not spin with wait=false, and do not re-run the generation tool.`;
// ─── generate_image ────────────────────────────────────────
server.tool(
'generate_image',
'Generate image(s) from a text prompt using Kolbo AI. Supports Visual DNA profiles (for character/style/product consistency), moodboards (for style direction), Kolbo image presets, reference images (for composition guidance), batch generation (num_images for variations of ONE prompt, `prompts` for SEVERAL different prompts in one combined widget), and web-search grounding. PRESET CONTRACT: resolve with list_presets type="image" AND search="<name>" (headless, bible, character sheet, or a user preset), then pass the exact id as `preset_id`. Custom instructions live on the preset — prefer this over generate_character_sheet. Never list the whole catalog. When the user wants multiple distinct images, pass all their prompts in `prompts` in ONE call — never a series of separate generate_image calls. For EDITING an existing image, use generate_image_edit instead. For a coordinated multi-scene set planned by AI from a single brief (storyboard, ad campaign), use generate_creative_director. Returns the final image URL(s) when complete.',
{
prompt: z.string().optional().describe('Text description of the image to generate. Required unless `prompts` is provided.'),
prompts: promptsField('images'),
font_ids: z.array(z.string().regex(/^[a-f0-9]{24}$/i)).max(3).optional().describe('Ready family IDs from My Fonts (list_fonts/upload_font). Requires supports_custom_fonts=true. State exact text/language/weight in prompt; server prepares private references. Applies to every prompt in a batch.'),
model: z.string().optional().describe('Model identifier — REQUIRED in practice: pick a specific model, do NOT omit (omitting = Smart Select auto-pick, which we avoid). Strong current defaults: "nano-banana-2" (versatile, text rendering, multilingual) or "gpt-image-2" (photoreal, infographics). Call list_models type="text_to_img" to see all options and pick per the user\'s intent.'),
aspect_ratio: z.string().optional().describe(aspectRatioDescribe('1:1')),
enhance_prompt: z.boolean().optional().describe('Enhance the prompt for better results. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
num_images: z.number().optional().describe('Number of images to generate in one call. Default: 1. Note: some models (Midjourney etc.) have a fixed `images_per_request` and ignore this — check list_models.'),
reference_images: z.array(z.string()).optional().describe('STYLE/COMPOSITION inspiration only — does NOT embed reference pixels. Array of image URLs used to guide the look-and-feel of a brand-new generation. The model interprets the references and regenerates approximations conditioned on them. It will NOT copy pixels from these images into the output. **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model — exceeding it is a deterministic 400.** To embed a specific logo, icon, watermark, or asset pixel-accurately, use generate_image_edit with the asset in source_images. To EDIT an existing image, also use generate_image_edit.'),
visual_dna_ids: z.array(z.string()).optional().describe('Visual DNA profile IDs (from create_visual_dna / list_visual_dnas) for character / style / product / scene consistency. **Cap: pass at most `max_visual_dna` IDs from list_models — if the field is null/0 or `supports_visual_dna: false`, the model rejects DNA entirely (silently ignored in some paths).** How DNA works: the server fetches the DNA\'s reference images AND always injects its `description` field into the prompt as plaintext (by design — independent of enhance_prompt). Practical implication: do NOT also write physical descriptors of the same subject in your own prompt — they will compete with the DNA description text. For pixel-accurate face anchoring of a specific person, prefer passing the DNA\'s reference image directly via source_images on generate_image_edit and OMIT visual_dna_ids. visual_dna_ids is best for style / scene / product DNAs and for soft consistency across a set.'),
moodboard_id: z.string().optional().describe('Moodboard ID (from list_moodboards / get_moodboard) whose master_prompt and style_guide should be applied to this generation.'),
enable_web_search: z.boolean().optional().describe('Enable web-search grounding for the prompt (useful for current events, brand references, real-world accuracy). Default: false'),
resolution: z.string().optional().describe('Image resolution tier: "1K" (~1024px), "2K" (Full HD), "3K" (QHD), or "4K" (UHD). Model-dependent — call list_models and read supported_resolutions on the chosen model. Read resolution_multipliers on the same model to predict credit cost. Omit to use the model default.'),
quality: z.string().optional().describe('Quality tier for models that support it (e.g. "low", "medium", "high", "xhigh", "max", "auto"). Check list_models → supported_qualities on the chosen model. "auto" is normalised to "medium" on gpt-image-2. Omit to use the model default.'),
background: z.enum(['auto', 'opaque', 'transparent']).optional().describe('Output background. Before passing "transparent", call list_models for the chosen image model and require supports_transparent_background=true. Use PNG or WebP (PNG is selected by default when omitted). Kolbo forwards the setting and appends the exact phrase "no background" once to the effective prompt; prompt wording alone is not sufficient.'),
output_format: z.enum(['png', 'jpeg', 'webp']).optional().describe('Output file format on supported OpenAI image models. Default: png.'),
output_compression: z.number().int().min(0).max(100).optional().describe('JPEG/WebP output compression quality (0-100); not applicable to PNG.'),
moderation: z.enum(['auto', 'low']).optional().describe('Moderation setting on supported OpenAI image models. Default: auto.'),
mask_image_url: z.string().url().optional().describe('Optional public PNG mask URL for supported OpenAI edits with source/reference images. Transparent regions indicate areas to edit.'),
preset_id: z.string().optional().describe('Exact preset ID from list_presets type="image". If the user requests any image preset, resolve it with list_presets and pass it here; do not omit it.'),
cinematic: CINEMATIC_SCHEMA,
skip_color_palette: z.boolean().optional().describe('Opt this single call OUT of the account\'s active Color DNA palette (see list_color_palettes / activate_color_palette). By default, if the user has an active palette it strict-grades every generation automatically — pass true only when the user explicitly wants this one image ungraded.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ prompt, prompts, model, aspect_ratio, enhance_prompt = false, num_images, reference_images, visual_dna_ids, moodboard_id, enable_web_search, resolution, quality, background, output_format, output_compression, moderation, mask_image_url, preset_id, cinematic, skip_color_palette, project_id, session_id, font_ids }) => {
if (!prompt && !(prompts && prompts.length)) throw new Error('Provide prompt or prompts');
model = await canonicalModelId(client, model, 'text_to_img'); // lenient id resolution ("z-image" → "z-image/turbo")
aspect_ratio = await resolveCatalogAspectRatio(client, model, aspect_ratio, 'text_to_img');
const shared = {
model, aspect_ratio, enhance_prompt, font_ids,
reference_images, visual_dna_ids, moodboard_id, enable_web_search, resolution, quality, background, output_format, output_compression, moderation, mask_image_url, preset_id, cinematic, skip_color_palette, project_id, session_id
};
// Batch mode: N different prompts, one widget owning all generation ids.
if (prompts && prompts.length) {
const batch = await submitBatch(prompts, (p) => client.post('/v1/generate/image', { ...shared, prompt: p }));
if (returnsImmediately()) return submittedResult({
tool: 'generate_image', kind: 'image', gen: batch.ok[0].gen, client, model,
count: batch.ids.length, settings: imageSettings(shared),
generation_ids: batch.ids, prompts: batch.ok.map((o) => o.prompt),
failed_submissions: batch.failed,
status_args: { generation_ids: batch.ids, wait: true },
reference_images
});
return pollBatch(client, batch, { interval: (batch.ok[0].gen.poll_interval_hint || 3) * 1000, timeout: 150000 }, 'generate_image', model);
}
const gen = await client.post('/v1/generate/image', { ...shared, prompt, num_images });
if (returnsImmediately()) return submittedResult({
tool: 'generate_image', kind: 'image', gen, client, model, prompt,
count: num_images, settings: imageSettings(shared),
reference_images
});
const poll = await pollOrTimedOut(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 3) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
const images = await inlineImageBlocks(result.result.urls, { enabled: inlineImages });
return uiCompleted({
tool: 'generate_image', kind: 'image', gen, client, model, prompt,
count: num_images, settings: imageSettings(shared),
// The status result reports what the server ACTUALLY conditioned on
// (reference_details: DNA stills, merged sources, auto-routing) — the
// input arg is only what the caller happened to name.
reference_images: result.result.reference_images || reference_images,
urls: result.result.urls,
playback_urls: result.result.playback_urls,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: gen.session_id,
urls: result.result.urls,
playback_urls: result.result.playback_urls,
model: result.result.model,
prompt_used: result.result.prompt_used,
_followup_hint: 'If the user asks to edit/change/modify this image next (scene, lighting, objects, style, color — any content edit), pass urls[0] to generate_image_edit. Use edit_image ONLY for mechanical ops (upscale/reframe/removebg/enhance_skin). Do NOT call generate_image again.'
}, null, 2), images);
}
);
// ─── generate_image_edit ──────────────────────────────────
server.tool(
'generate_image_edit',
'THE tool for ANY prompt-driven / content edit of an existing image — changing the scene ("make it night", "change the sky to sunset"), adding/removing/replacing objects, restyling, recoloring, compositing, or any "edit this image to…" request. This is the image-editing equivalent of generate_image and runs on strong dedicated editing models (nano-banana-2, gpt-image-2). Provide the source image URL(s) in `source_images` and the instruction in `prompt`. Supports Visual DNA profiles, moodboards, and Kolbo image-editing presets. PRESET CONTRACT: if the user requests a preset, call list_presets type="image_edit" and pass its exact id as `preset_id`; never silently omit it. Do NOT use `edit_image` for these — that tool is only for mechanical enhancements (upscale/reframe/remove-background/skin). For a brand-new image from scratch, use generate_image. Returns the edited image URL(s) when complete.',
{
prompt: z.string().optional().describe('Description of the edit to apply (e.g., "remove the background", "change the sky to sunset"). Required unless `prompts` is provided.'),
prompts: promptsField('edits of the SAME source images'),
font_ids: z.array(z.string().regex(/^[a-f0-9]{24}$/i)).max(3).optional().describe('Ready My Fonts family IDs. Requires supports_custom_fonts=true. Describe requested typography changes; unrelated text is preserved. Applies to every batch edit.'),
model: z.string().optional().describe('Model identifier — REQUIRED in practice: pick a specific model, do NOT omit (omitting = Smart Select auto-pick, which we avoid). Many text-to-image ids double as editors: the server auto-routes a base id to its editing variant when source_images is present (e.g. "gpt-image-2" → gpt-image-2/edit, "nano-banana-2" → nano-banana-2-image-editing) — passing the bare id is fine, no need to hunt for the "/edit" suffix yourself. BUT this only works for models that actually have a registered edit variant. For prompt-driven photoreal photo edits (object removal, keep-this-person/remove-the-rest, crowd cleanup, inpainting) the ONLY auto-pick defaults are "nano-banana-2" or "gpt-image-2" (use GPT Image 2 when the image needs readable text). Do NOT auto-pick Flux 2 / flux-2/edit / Flux Klein — those are generate-from-scratch / style models; use them only if the user names Flux. If unsure, confirm the model appears in `list_models type="image_editing"` and choose by the strengths summary — Flux edit variants are named-only.'),
source_images: z.array(z.string()).describe('PIXEL-ACCURATE compositing. Array of source images (URLs or absolute local paths) whose pixel content is composited into the output. **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model — exceeding it is a deterministic 400.** Three modes the model auto-detects from input shape: (1) Single image → edit/transform that image. (2) Multiple images, one base + others → composite the others into the base. (3) Multiple images with no clear base → generate a new scene that pixel-accurately embeds the supplied images at positions described in the prompt. Mode 3 is the canonical pattern for thumbnails / branded compositions where exact-pixel logo + face fidelity matter. Refer to source images in the prompt by ordinal position ("FIRST source image", "SECOND source image") or use @image1/@image2 tags. Add "composite AS-IS, do not redraw or restyle" to lock pixels.'),
reference_images: z.array(z.string()).optional().describe('STYLE/COMPOSITION inspiration, alongside `source_images` on the same call — does NOT embed reference pixels. Use when the edit should follow a look sampled from other images ("re-light this shot like these references"). The pixels that must survive the edit go in `source_images`; these only steer the look. **Cap: `source_images` + `reference_images` together must not exceed `max_reference_images` from list_models for the chosen model.**'),
aspect_ratio: z.string().optional().describe(aspectRatioDescribe('1:1')),
enhance_prompt: z.boolean().optional().describe('Enhance the prompt for better results. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
num_images: z.number().optional().describe('Number of output images. Default: 1'),
visual_dna_ids: z.array(z.string()).optional().describe('Visual DNA profile IDs for character / style / product consistency. **Cap: pass at most `max_visual_dna` IDs from list_models for the chosen model.** How DNA works: the server fetches the DNA\'s reference images AND always injects its `description` field into the prompt as plaintext (by design — independent of enhance_prompt). For pixel-accurate face anchoring of a specific person on this tool, the PREFERRED pattern is to pass the face photo directly via source_images and OMIT visual_dna_ids — that way the face pixels anchor the output and no description text competes. Do NOT pass visual_dna_ids if source_images already contains the same person\'s face (face averaging). visual_dna_ids is best here for style / product DNAs.'),
moodboard_id: z.string().optional().describe('Moodboard ID whose master_prompt and style_guide should be applied.'),
enable_web_search: z.boolean().optional().describe('Enable web-search grounding. Default: false'),
resolution: z.string().optional().describe('Image resolution tier: "1K" / "2K" / "3K" / "4K". Model-dependent — call list_models and read supported_resolutions. Default: "1K" for most edit models.'),
quality: z.string().optional().describe('Quality tier for edit models that support it (e.g. "low", "medium", "high", "xhigh", "max", "auto"). Check list_models → supported_qualities on the chosen model. "auto" is normalised to "medium" on gpt-image-2. Omit to use the model default.'),
background: z.enum(['auto', 'opaque', 'transparent']).optional().describe('Output background. Before passing "transparent", call list_models for the chosen image-edit model and require supports_transparent_background=true. Use PNG or WebP (PNG is selected by default when omitted). Kolbo forwards the setting and appends the exact phrase "no background" once to the effective prompt; prompt wording alone is not sufficient.'),
output_format: z.enum(['png', 'jpeg', 'webp']).optional().describe('Output file format on supported OpenAI image models. Default: png.'),
output_compression: z.number().int().min(0).max(100).optional().describe('JPEG/WebP output compression quality (0-100); not applicable to PNG.'),
moderation: z.enum(['auto', 'low']).optional().describe('Moderation setting on supported OpenAI image models. Default: auto.'),
mask_image_url: z.string().url().optional().describe('Optional public PNG mask URL for supported OpenAI edits with source/reference images. Transparent regions indicate areas to edit.'),
preset_id: z.string().optional().describe('Exact preset ID from list_presets type="image_edit" to apply an image-editing preset. If the user requests a preset, resolve and pass it; do not silently omit it.'),
cinematic: CINEMATIC_SCHEMA,
skip_color_palette: z.boolean().optional().describe('Opt this single call OUT of the account\'s active Color DNA palette (see list_color_palettes / activate_color_palette). By default, if the user has an active palette it strict-grades every generation automatically — pass true only when the user explicitly wants this one edit ungraded.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ prompt, prompts, model, source_images, reference_images, aspect_ratio, enhance_prompt = false, num_images, visual_dna_ids, moodboard_id, enable_web_search, resolution, quality, background, output_format, output_compression, moderation, mask_image_url, preset_id, cinematic, skip_color_palette, project_id, session_id, font_ids }) => {
if (!prompt && !(prompts && prompts.length)) throw new Error('Provide prompt or prompts');
model = await canonicalModelId(client, model, 'image_editing'); // lenient id resolution ("z-image" → "z-image/turbo")
aspect_ratio = await resolveCatalogAspectRatio(client, model, aspect_ratio, 'image_editing');
const shared = {
model, source_images, reference_images, aspect_ratio, enhance_prompt, font_ids,
visual_dna_ids, moodboard_id, enable_web_search, resolution, quality, background, output_format, output_compression, moderation, mask_image_url, preset_id, cinematic, skip_color_palette, project_id, session_id
};
const settings = imageSettings(shared);
// Batch mode: N different edit instructions against the SAME source
// images, one widget owning every generation id. Same contract as
// generate_image — without it, "give me 4 variations of this edit" came
// back as four stacked cards.
if (prompts && prompts.length) {
const batch = await submitBatch(prompts, (p) => client.post('/v1/generate/image-edit', { ...shared, prompt: p }));
if (returnsImmediately()) return submittedResult({
tool: 'generate_image_edit', kind: 'image', gen: batch.ok[0].gen, client, model,
count: batch.ids.length, settings,
generation_ids: batch.ids, prompts: batch.ok.map((o) => o.prompt),
failed_submissions: batch.failed,
status_args: { generation_ids: batch.ids, wait: true },
reference_images: [...(source_images || []), ...(reference_images || [])]
});
return pollBatch(client, batch, { interval: (batch.ok[0].gen.poll_interval_hint || 3) * 1000, timeout: 150000 }, 'generate_image_edit', model);
}
const gen = await client.post('/v1/generate/image-edit', { ...shared, prompt, num_images });
if (returnsImmediately()) return submittedResult({
tool: 'generate_image_edit', kind: 'image', gen, client, model, prompt,
count: num_images,
settings,
reference_images: [...(source_images || []), ...(reference_images || [])]
});
// Multi-source compositing or DNA-anchored edits routinely exceed 120s
// server-side. But we still need to return before the 180s host cutoff,
// so use 150s for all cases. The LLM can call get_generation_status
// with wait=true to continue polling.
const poll = await pollOrTimedOut(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 3) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
const images = await inlineImageBlocks(result.result.urls, { enabled: inlineImages });
return uiCompleted({
tool: 'generate_image_edit', kind: 'image', gen, client, model, prompt,
count: num_images, settings: imageSettings(shared),
reference_images: result.result.reference_images
|| [...(source_images || []), ...(reference_images || [])],
urls: result.result.urls,
playback_urls: result.result.playback_urls,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: gen.session_id,
urls: result.result.urls,
playback_urls: result.result.playback_urls,
model: result.result.model,
prompt_used: result.result.prompt_used,
_followup_hint: 'If the user asks for another edit on this output, pass urls[0] back into generate_image_edit as source_images. For targeted ops (upscale/reframe/removebg/enhance_skin) use edit_image instead. Do NOT call generate_image from scratch.'
}, null, 2), images);
}
);
// ─── generate_creative_director ─────────────────────────────
server.tool(
'generate_creative_director',
'Generate 2–8 related images or videos as one coherent set from a single creative brief. Use scene_count (NOT num_images) to set the number of scenes (1–8, default 4). Use this when the user gives a general brief ("make 4 product shots", "create a storyboard") and you are planning the scenes — it handles style consistency and runs scenes in parallel. If the user explicitly provides separate prompts for each image, use ONE generate_image call with the `prompts` array instead (never parallel single-prompt calls). Supports image and video modes (workflow_type). Visual DNA and moodboard references keep character/style consistent across every scene.',
{
prompt: z.string().describe('Creative brief or concept describing the full set of scenes to generate'),
scene_count: z.number().optional().describe('Number of scenes/images to generate, 1–8. Default: 4. Use this — NOT num_images — to control how many outputs are created.'),
model: z.string().optional().describe('Model identifier applied to every scene. Pick a SPECIFIC model — do NOT omit (omitting = Smart Select auto-pick, which we avoid); call list_models for this type and choose the model that best fits the user\'s intent.'),
aspect_ratio: z.string().optional().describe(aspectRatioDescribe('1:1')),
workflow_type: z.string().optional().describe('"image" (default) or "video"'),
duration: z.number().optional().describe('Duration in seconds per scene (video mode only). Must be a value in `supported_durations` from list_models, OR within `min_output_duration`-`max_output_duration`. E.g., 5 or 10.'),
enhance_prompt: z.boolean().optional().describe('Enhance prompts per scene. Default: false — only pass true if the user explicitly asks to enhance/improve the prompts.'),
reference_images: z.array(z.string()).optional().describe('Array of reference images (URLs or absolute local paths) to guide style/composition of every scene. **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model.**'),
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply consistently across every scene. **Cap: pass at most `max_visual_dna` IDs from list_models for the chosen model.** This is the ideal way to keep a character or product looking the same in all scenes of a campaign.'),
moodboard_id: z.string().optional().describe('A single moodboard ID whose master_prompt and style_guide should shape every scene.'),
moodboard_ids: z.array(z.string()).optional().describe('Multiple moodboard IDs when blending styles. Prefer `moodboard_id` for single moodboards.'),
resolution: z.string().optional().describe('Resolution tier applied to every scene. Images: "1K" / "2K" / "3K" / "4K". Videos: "720p" / "1080p" / "1440p" / "2160p". Values are model-dependent — call list_models and read supported_resolutions on the target model. Multiplied across every scene.'),
font_ids: z.array(z.string().regex(/^[a-f0-9]{24}$/i)).max(3).optional().describe('Image workflows only: ready My Fonts family IDs applied to each scene. Requires supports_custom_fonts=true; do not create specimen references yourself.'),
project_id: projectIdField
},
async ({ prompt, scene_count, model, aspect_ratio, workflow_type, duration, enhance_prompt = false, reference_images, visual_dna_ids, moodboard_id, moodboard_ids, resolution, project_id, font_ids }) => {
model = await canonicalModelId(client, model, workflow_type === 'video' ? 'text_to_video' : 'text_to_img'); // lenient id resolution ("z-image" → "z-image/turbo")
aspect_ratio = await resolveCatalogAspectRatio(client, model, aspect_ratio, workflow_type === 'video' ? 'text_to_video' : 'text_to_img');
const gen = await client.post('/v1/generate/creative-director', {
prompt, scene_count, model, aspect_ratio, workflow_type, duration, font_ids,
enhance_prompt, reference_images, visual_dna_ids, moodboard_id, moodboard_ids, resolution, project_id
});
const cdStatusUrl = `/v1/generate/creative-director/${gen.generation_id}/status`;
if (returnsImmediately()) return submittedResult({
tool: 'generate_creative_director', kind: 'scenes', gen, client, model, prompt,
count: scene_count || 4,
settings: videoSettings({ duration, resolution, aspect_ratio, mode: workflow_type || 'image', enhance_prompt, visual_dna_ids, moodboard_id, moodboard_ids }),
reference_images,
poll_tool: 'get_creative_director_status',
status_args: { generation_id: gen.generation_id, wait: true }
});
// Video mode runs N scenes as parallel video generations, each of which
// can take several minutes — a whole batch routinely exceeds the 10-min
// image window. Give video batches 30 min (the server watchdog finalizes
// stuck batches around 15-30 min, so this aligns client + server).
const cdTimeout = workflow_type === 'video' ? 1800000 : 600000;
let result;
try {
result = await pollUntilDone(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 5) * 1000,
timeout: cdTimeout,
statusUrl: cdStatusUrl
});
} catch (err) {
// On a client-side poll timeout the batch is almost always STILL
// running (or already finished) on the server. Don't lose the work:
// return whatever scenes have landed plus the exact re-check path,
// instead of throwing an opaque timeout at the agent.
if (err && err.timedOut) {
let partial = null;
try { partial = await client.get(cdStatusUrl); } catch (_) { /* ignore */ }
const doneScenes = (partial?.scenes || []).filter(s => s.status === 'completed').map(s => ({
scene_number: s.scene_number, title: s.title, image_urls: s.image_urls, video_urls: s.video_urls
}));
return { content: [{ type: 'text', text: JSON.stringify({
state: partial?.state || 'processing',
generation_id: gen.generation_id,
scenes: doneScenes,
total_scenes: partial?.scenes?.length || 0,
completed_scenes: doneScenes.length,
_timed_out: true,
_hint: `Still running after the poll window. Call get_creative_director_status with generation_id="${gen.generation_id}" to keep checking until state="completed" — do NOT re-run generate_creative_director.`
}, null, 2) }] };
}
throw err;
}
const scenes = (result.scenes || [])
.filter(s => s.status === 'completed')
.map(s => ({
scene_number: s.scene_number,
title: s.title,
image_urls: s.image_urls,
video_urls: s.video_urls
}));
const cdText = JSON.stringify({
...creditFields(result),
scenes,
total_scenes: result.scenes?.length || 0,
completed_scenes: scenes.length,
_followup_hint: 'Each scene is a separate asset. If the user asks to edit one scene, find that scene by scene_number/title and pass its image_urls[0] (or video_urls[0]) to generate_image_edit / edit_image / edit_video / generate_video_from_video. Do NOT re-run generate_creative_director unless the user explicitly wants a brand-new set.'
}, null, 2);
return { content: [{ type: 'text', text: cdText }] };
}
);
// ─── get_creative_director_status ──────────────────────────
// Creative Director uses a DEDICATED status route (per-scene state + all
// scene image/video URLs). The generic get_generation_status hits
// /v1/generate/:id/status and CANNOT read a CD batch — so a long or
// backgrounded CD run (especially parallel VIDEO scenes that exceed the
// blocking poll window) needs this tool to be re-checked until done.
server.tool(
'get_creative_director_status',
'Check the status of a Creative Director batch (from generate_creative_director) by its generation_id. Returns overall state ("processing" until EVERY scene is terminal, then "completed"/"failed") plus each scene\'s number, title, per-scene status, and image_urls/video_urls. Set wait=true to block until the batch is terminal or the wait window closes, whichever comes first — a batch longer than one window returns state="processing" and you simply call again with wait=true. Prefer this over the generic get_generation_status for Creative Director ids — the generic tool now returns the same scene data (it delegates here), but this one is the direct route.',
{
generation_id: z.string().describe('The Creative Director generation_id returned by generate_creative_director.'),
wait: z.boolean().optional().describe(`If true, block until the batch is terminal, for at most ~${WAIT_WINDOW_S}s per call. A batch that outlives one window comes back state="processing" (not an error) — call again with wait=true until it is terminal. Always prefer this over polling with wait=false.`)
},
async ({ generation_id, wait }) => {
const statusUrl = `/v1/generate/creative-director/${encodeURIComponent(generation_id)}/status`;
let status;
if (wait) {
try {
status = await pollUntilDone(client, generation_id, {
interval: 15000,
timeout: WAIT_WINDOW_MS,
statusUrl
});
} catch (err) {
// A tracking timeout is not a failed paid generation. Return the
// latest batch state so the widget can start one new long wait.
if (!err?.timedOut) throw err;
status = await client.get(statusUrl);
}
} else {
status = await client.get(statusUrl);
}
const scenes = (status.scenes || []).map(s => ({
scene_number: s.scene_number,
status: s.status,
title: s.title,
image_urls: s.image_urls || null,
video_urls: s.video_urls || null
}));
const completed = scenes.filter(s => s.status === 'completed').length;
const text = JSON.stringify({
...creditFields(status),
state: status.state,
generation_id,
progress: status.progress,
scenes,
total_scenes: scenes.length,
completed_scenes: completed,
_hint: status.state === 'completed'
? 'All scenes terminal. Every completed scene\'s image_urls/video_urls are final.'
: `Still running — not a failure. Each wait=true call blocks for at most ~${WAIT_WINDOW_S}s, and a video batch routinely outlasts several windows, so call get_creative_director_status again with wait=true and keep going until state is terminal. Scenes that already carry image_urls/video_urls are done; never re-run generate_creative_director.`
}, null, 2);
// Same text-only gap as get_generation_status/pollBatch, called out in
// this tool's own comment above ("especially parallel VIDEO scenes that
// exceed the blocking poll window") — a CD batch checked here after
// outliving one wait window rendered as a bare JSON dump, no scene
// thumbnails. Always ship the same kind:'status' grid, one item per scene.
return uiCompleted({
tool: 'get_creative_director_status', kind: 'status', client,
model: 'Generations', gen: { generation_id },
settings: {},
items: scenes.map(s => ({
id: generation_id + '-' + s.scene_number,
state: s.status,
title: s.title || ('Scene ' + s.scene_number),
url: (Array.isArray(s.image_urls) && s.image_urls[0]) || (Array.isArray(s.video_urls) && s.video_urls[0]) || undefined,
})),
}, text);
}
);
// ─── generate_video ────────────────────────────────────────
// Text-to-video is now a MODE inside videoGeneration/controller.js (sdk/controller.js
// posts mode:'text_to_video'), and that controller DOES read visualDnaIds — so Visual
// DNA works here. The old "text-to-video never reads visualDnaIds" note described the
// retired textToVideoGeneration path and was stale.
server.tool(
'generate_video',
'Generate a video from a text prompt using Kolbo AI. For SEVERAL different videos, pass all their prompts in `prompts` in ONE call (one combined widget) — never a series of separate calls. For animating an existing still image into motion, use generate_video_from_image instead. For a coordinated multi-scene video campaign, use generate_creative_director with workflow_type="video". Supports reference images (for style/composition guidance) and Visual DNA for character consistency. Seedance 2/2.5 PERFORM quoted dialogue natively (synced voice, lip movement, room tone) — do not route scene dialogue to generate_speech or generate_lipsync; write it in ENGLISH or Latin transliteration of Hebrew ("shalom"), never Hebrew script — Seedance 2/2.5 do not speak Hebrew; prefer Gemini Omni Flash 1.1 or Gemini Omni 1 for native Hebrew. Resolution is a credit MULTIPLIER (vs 720p: 480p x0.44, 1080p x2.25, 4k x4.95), so use a lower resolution for previews. For actual Seedance 2.5 Draft, explicitly pass resolution="480p-draft" with model="seedance-2-5", then use edit_video draft_quote/draft_enhance for paid finalization. Plain "480p" is a regular generation, not Draft. ROUTE BEFORE CALLING: when reference images anchor IDENTITY (specific characters, a specific product, a location that must match) — especially 2+ of them — that is generate_elements, not this tool; reference_images here are loose style/composition hints. Decide the right tool FIRST: a mis-routed call still starts a PAID generation, and switching tools afterwards without cancel_generation leaves the user paying for both. Returns the final video URL when complete.',
{
prompt: z.string().optional().describe('Text description of the video to generate. Required unless `prompts` is provided.'),
prompts: promptsField('videos'),
model: z.string().optional().describe('Model identifier — pick a SPECIFIC model, do NOT omit (omitting = Smart Select auto-pick, which we avoid). If the user already named a model/family (this turn or earlier), pass that name — do not substitute a cheaper default. Only when no model was named: "seedance-2" (versatile) or "veo3" (cinematic + native audio) are reasonable auto-picks; Kling is strongest for motion (list_models type="text_to_video" for exact ids). Call list_models for supported_durations / supported_aspect_ratios.'),
aspect_ratio: z.string().optional().describe(aspectRatioDescribe('16:9')),
duration: z.number().optional().describe('Duration in seconds. Must be a value in `supported_durations` from list_models, OR within `min_output_duration`-`max_output_duration` (whichever the model exposes). Default: 5'),
enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
reference_images: z.array(z.string()).optional().describe('Array of images (URLs or absolute local paths) used as visual references (style / composition / subject). **Cap: pass at most `max_reference_images` URLs from list_models for the chosen model — exceeding it is a deterministic 400.**'),
draft: z.boolean().optional().describe('Special Draft mode. True forces 480p Draft even if resolution says 1080p; false selects regular video. Only supported when list_models advertises draft capabilities. For Seedance 2.5 editing use model seedance-2-5-video-to-video with generate_video_from_video; the source duration and aspect ratio are preserved. Uses ordinary credits; finalize through draft_quote then draft_enhance.'),
resolution: z.string().optional().describe('Video resolution or named tier. Read supported_resolutions and resolution_multipliers from list_models. Seedance 2.5 offers "480p-draft" on the same model: ordinary credits, then optional paid draft_quote/draft_enhance finalization. Regular "480p" remains a normal generation.'),
preset_id: z.string().optional().describe('Preset ID from list_presets type="video" to apply a saved motion/style preset to this generation.'),
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply for character/style consistency. Every DNA passed here MUST also be tagged in the prompt as @ExactDNAName. **Cap: pass at most `max_visual_dna` IDs from list_models for the chosen model; if `supports_visual_dna: false`, DNA is silently ignored.**'),
sound_enabled: z.boolean().optional().describe('Enable (`true`) or disable (`false`) AI-generated synced audio on the output video. Honored by `sound_generation_type: "native"` models (Veo 3.1, Kling V3/2.6, PixVerse V6). Seedance 2.x reports type "none" (no toggle) but `sound_baked_in: true` — those still emit real audio; do not tell the user the model is silent. Omit to use `sound_enabled_by_default`. Pass `false` only when the user asks for silent AND the model is native (not baked-in). Enabling sound may apply `sound_credit_multiplier` to cost.'),
skip_color_palette: z.boolean().optional().describe('Opt this single call OUT of the account\'s active Color DNA palette (see list_color_palettes / activate_color_palette). By default, if the user has an active palette it strict-grades every generation automatically — pass true only when the user explicitly wants this one video ungraded.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ prompt, prompts, model, aspect_ratio, duration, enhance_prompt = false, reference_images, resolution, draft, preset_id, visual_dna_ids, sound_enabled, skip_color_palette, project_id, session_id }) => {
if (!prompt && !(prompts && prompts.length)) throw new Error('Provide prompt or prompts');
if (draft === true) resolution = '480p-draft';
else if (draft === false && resolution?.endsWith('-draft')) resolution = resolution.slice(0, -6);
model = await canonicalModelId(client, model, 'text_to_video'); // lenient id resolution ("z-image" → "z-image/turbo")
aspect_ratio = await resolveCatalogAspectRatio(client, model, aspect_ratio, 'text_to_video');
const shared = {
model, aspect_ratio, duration, enhance_prompt, reference_images, resolution, draft, preset_id, visual_dna_ids, sound_enabled, skip_color_palette, project_id, session_id
};
// Batch mode: N different prompts, one widget owning all generation ids.
if (prompts && prompts.length) {
const batch = await submitBatch(prompts, (p) => client.post('/v1/generate/video', { ...shared, prompt: p }));
if (returnsImmediately()) return submittedResult({
tool: 'generate_video', kind: 'video', gen: batch.ok[0].gen, client, model,
count: batch.ids.length, settings: videoSettings(shared),
generation_ids: batch.ids, prompts: batch.ok.map((o) => o.prompt),
failed_submissions: batch.failed,
status_args: { generation_ids: batch.ids, wait: true },
reference_images
});
return pollBatch(client, batch, { interval: (batch.ok[0].gen.poll_interval_hint || 8) * 1000, timeout: 150000 }, 'generate_video', model);
}
const gen = await client.post('/v1/generate/video', { ...shared, prompt });
if (returnsImmediately()) return submittedResult({
tool: 'generate_video', kind: 'video', gen, client, model, prompt,
settings: videoSettings(shared),
reference_images
});
// 15 min — Kling O3 4K, Veo 3.1 at higher durations/resolutions, Hailuo 2.3,
// and Seedance 2 routinely run 5-12 min under provider queue load; the old
// 5-min window forced routine calls into the timeout-and-recover path.
const poll = await pollOrTimedOut(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 8) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'generate_video', kind: 'video', gen, client, model, prompt,
settings: videoSettings(shared),
reference_images,
urls: result.result.urls,
playback_urls: result.result.playback_urls,
thumbnail_url: result.result.thumbnail_url,
duration: result.result.duration,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: gen.session_id,
urls: result.result.urls,
playback_urls: result.result.playback_urls,
model: result.result.model,
duration: result.result.duration,
thumbnail_url: result.result.thumbnail_url,
prompt_used: result.result.prompt_used,
_followup_hint: 'If the user asks to edit/restyle/extend this video next, pass urls[0] to edit_video (upscale/reframe/face_swap/extend/generate_audio/lipsync/magic_edit) or generate_video_from_video (restyle). Do NOT call generate_video from scratch.'
}, null, 2));
}
);
// ─── generate_video_from_image ─────────────────────────────
server.tool(
'generate_video_from_image',
'Animate an existing still image into a video using Kolbo AI. The image comes from `image_url`; `prompt` describes the motion (not the subject — the subject is already in the image). For generating a video from scratch, use generate_video. FOR A MULTI-SHOT FILM with recurring characters or locations, prefer ONE generate_elements call carrying the whole DNA-anchored sequence over a series of separate image-to-video clips. ANIMATING SEVERAL SHOTS OF THE SAME SEQUENCE? Pass them ALL in `items` in ONE call (one combined widget) — never a series of separate calls, which buries the chat under one widget per clip. Across calls (a sequence longer than the batch cap), make the first call without `session_id`, take the `session_id` from its result, and pass that same `session_id` on every following call — otherwise each clip becomes its own session and the user gets a stack of near-identical single-clip sessions in the Kolbo sidebar. Returns the final video URL(s) when complete.',
{
image_url: z.string().optional().describe('URL or absolute local path of the source image to animate. Required unless `items` is provided.'),
prompt: z.string().optional().describe('Text description of the desired MOTION (e.g., "camera slowly pans right while the character walks forward"). Required unless `items` is provided.'),
items: z.array(z.object({
image_url: z.string().describe('URL or absolute local path of the source image to animate for THIS clip.'),
prompt: z.string().describe('Text description of the desired MOTION for THIS clip.'),
})).max(MAX_BATCH_PROMPTS).optional().describe(
`BATCH MODE — several DIFFERENT stills (2–${MAX_BATCH_PROMPTS}) animated concurrently in ONE call and rendered together in ONE combined widget. Unlike the \`prompts\` array on generate_image / generate_video, each entry pairs its OWN \`image_url\` with its OWN motion \`prompt\` — the image is what varies, and that is the point. **Hard cap: ${MAX_BATCH_PROMPTS} items per call — more than that is REJECTED with an error (never silently truncated), so split a longer sequence across several calls of at most ${MAX_BATCH_PROMPTS}.** Whenever the user wants several stills animated (a shot sequence, a storyboard, an animatic), ALWAYS pass them all here instead of making several separate calls — separate calls clutter the chat with stacked widgets. All items share the same model / duration / resolution / aspect_ratio / sound_enabled / project_id / session_id. When set, \`image_url\` and \`prompt\` are ignored.`
),
model: z.string().optional().describe('Model identifier — pick a SPECIFIC model, do NOT omit (omitting = Smart Select auto-pick, which we avoid). If the user already named a model/family (this turn or earlier), pass that name — a text-to-video id remaps to the family\'s image-to-video sibling. Do not substitute a cheaper default (named Grok Imagine → not Seedance). Only when no model was named: "seedance-2" or "veo3" are reasonable auto-picks; Kling is strongest for motion (list_models type="img_to_video" for exact ids).'),
aspect_ratio: z.string().optional().describe(aspectRatioDescribe('16:9')),
duration: z.number().optional().describe('Duration in seconds. Must be in `supported_durations` from list_models, OR within `min_output_duration`-`max_output_duration`. Default: 5'),
enhance_prompt: z.boolean().optional().describe('Enhance the motion prompt. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to maintain consistency with prior characters / styles. **Cap: pass at most `max_visual_dna` IDs from list_models for the chosen model; if `supports_visual_dna: false` the model ignores DNA entirely.**'),
draft: z.boolean().optional().describe('Special Draft mode. True forces 480p Draft even if resolution says 1080p; false selects regular video. Only supported when list_models advertises draft capabilities. For Seedance 2.5 editing use model seedance-2-5-video-to-video with generate_video_from_video; the source duration and aspect ratio are preserved. Uses ordinary credits; finalize through draft_quote then draft_enhance.'),
resolution: z.string().optional().describe('Video resolution or named tier. Read supported_resolutions from list_models. Seedance 2.5 offers "480p-draft" on the same model, with optional paid finalization through draft_quote/draft_enhance. Regular "480p" is not draft.'),
sound_enabled: z.boolean().optional().describe('Enable (`true`) or disable (`false`) AI-generated synced audio on the output video. Honored by `sound_generation_type: "native"` models (Veo 3.1 Lite, Kling V3 4K, PixVerse V6). Seedance 2.x reports type "none" (no toggle) but `sound_baked_in: true` — those still emit real audio; do not tell the user the model is silent. Omit to use `sound_enabled_by_default`. Pass `false` only when the user asks for silent AND the model is native (not baked-in). Enabling sound may apply `sound_credit_multiplier` to cost.'),
skip_color_palette: z.boolean().optional().describe('Opt this single call OUT of the account\'s active Color DNA palette (see list_color_palettes / activate_color_palette). By default, if the user has an active palette it strict-grades every generation automatically — pass true only when the user explicitly wants this one video ungraded.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ image_url, prompt, items, model, aspect_ratio, duration, enhance_prompt = false, visual_dna_ids, resolution, draft, sound_enabled, skip_color_palette, project_id, session_id }) => {
if (!(items && items.length) && !(image_url && prompt)) throw new Error('Provide image_url + prompt, or items');
if (draft === true) resolution = '480p-draft';
else if (draft === false && resolution?.endsWith('-draft')) resolution = resolution.slice(0, -6);
model = await canonicalModelId(client, model, 'img_to_video'); // lenient id resolution ("z-image" → "z-image/turbo")
aspect_ratio = await resolveCatalogAspectRatio(client, model, aspect_ratio, 'img_to_video');
const shared = {
model, aspect_ratio, duration, enhance_prompt, visual_dna_ids, resolution, draft, sound_enabled, skip_color_palette, project_id, session_id
};
// Batch mode: N different stills, one widget owning all generation ids.
// Same fan-out as generate_video's prompts[], except the varying part is
// the (image_url, prompt) PAIR — submitBatch carries the prompt as the
// per-tile caption either way.
if (items && items.length) {
const batch = await submitBatch(items, (it) => client.post('/v1/generate/video/from-image', { ...shared, image_url: it.image_url, prompt: it.prompt }));
if (returnsImmediately()) return submittedResult({
tool: 'generate_video_from_image', kind: 'video', gen: batch.ok[0].gen, client, model,
count: batch.ids.length, settings: videoSettings({ duration, resolution, draft, aspect_ratio, enhance_prompt, visual_dna_ids }),
generation_ids: batch.ids, prompts: batch.ok.map((o) => o.prompt),
failed_submissions: batch.failed,
status_args: { generation_ids: batch.ids, wait: true },
reference_images: items.map((item) => item.image_url)
});
return pollBatch(client, batch, { interval: (batch.ok[0].gen.poll_interval_hint || 8) * 1000, timeout: 900000 }, 'generate_video_from_image', model);
}
const gen = await client.post('/v1/generate/video/from-image', { ...shared, image_url, prompt });
if (returnsImmediately()) return submittedResult({
tool: 'generate_video_from_image', kind: 'video', gen, client, model, prompt,
settings: videoSettings({ duration, resolution, draft, aspect_ratio, enhance_prompt, visual_dna_ids }),
reference_images: [image_url]
});
// 15 min — same real-world generation times as generate_video (Kling O3 4K,
// Veo 3.1, Hailuo 2.3, Seedance 2 at higher durations/resolutions).
const poll = await pollOrTimedOut(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 8) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'generate_video_from_image', kind: 'video', gen, client, model, prompt,
settings: videoSettings({ duration, resolution, draft, aspect_ratio, enhance_prompt, visual_dna_ids }),
reference_images: [image_url],
urls: result.result.urls,
playback_urls: result.result.playback_urls,
thumbnail_url: result.result.thumbnail_url,
duration: result.result.duration,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: gen.session_id,
urls: result.result.urls,
playback_urls: result.result.playback_urls,
model: result.result.model,
duration: result.result.duration,
thumbnail_url: result.result.thumbnail_url,
_followup_hint: 'If the user asks to edit/restyle/extend this video next, pass urls[0] to edit_video or generate_video_from_video. Do NOT re-run generate_video_from_image unless they want a fresh animation from a different source image. Animating more shots of THIS same sequence? Pass the session_id above back on each of those calls so they all land in one session.'
}, null, 2));
}
);
// ─── generate_music ────────────────────────────────────────
server.tool(
'generate_music',
'Generate music from a text description using Kolbo AI. Supports instrumental mode, custom lyrics, style direction, vocal gender, negative tags, song length, and Suno fine-controls (style weight, weirdness, audio weight, persona/singing voice). Default model is Suno. Some controls are Suno-only; the engine ignores controls that do not apply to the chosen model. Returns the final audio URL when complete.',
{
prompt: z.string().optional().describe('Text description of the music to generate (e.g., "upbeat electronic dance track with synthesizers"). Required unless composition_plan is passed (ElevenLabs Music v2.5 structured mode, which cannot be combined with prompt).'),
model: z.string().optional().describe('Model identifier. Use list_models type="music_gen" to see options. Omit for Suno (default).'),
style: z.string().optional().describe('Music style / genre (e.g., "pop", "rock", "lo-fi", "electronic", "jazz")'),
title: z.string().optional().describe('Song title. If omitted, one is generated.'),
instrumental: z.boolean().optional().describe('Generate instrumental only, no vocals. Default: false'),
lyrics: z.string().optional().describe('Custom lyrics for the song. If omitted, lyrics are generated automatically from the prompt unless instrumental is true.'),
vocal_gender: z.string().optional().describe('Preferred vocal gender: "male" or "female". Only applies when instrumental is false.'),
negative_tags: z.string().optional().describe('Styles / sounds to EXCLUDE, comma-separated (e.g. "heavy metal, screaming, distortion"). Suno.'),
duration_seconds: z.number().optional().describe('Target song length in seconds. Length-capable models: ElevenLabs Music (up to 600s) and Suno v6 / v6 Wild (10-360s, Custom Mode only). Suno v6 Mini does NOT honour it - it returns ~200s whatever you ask, so the control is not offered there. Omit for the model default (Suno picks its own natural length).'),
enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
preset_id: z.string().optional().describe('Preset ID from list_presets type="music" to apply a saved music style preset.'),
// ── Suno fine controls ──
style_weight: z.number().optional().describe('Suno: how strongly the style/genre is applied, 0–1.'),
weirdness: z.number().optional().describe('Suno: creativity / weirdness constraint, 0–1. Higher = more experimental.'),
audio_weight: z.number().optional().describe('Suno: influence of an audio/persona reference, 0–1.'),
persona_id: z.string().optional().describe('Suno persona id — reuse a saved singing voice/persona.'),
use_composition_plan: z.boolean().optional().describe('Suno: enable structured composition planning (verse/chorus structure).'),
singing_dna_id: z.string().optional().describe('Visual DNA character id whose singing voice to use (must be owned by the caller).'),
singing_voice_id: z.string().optional().describe('Custom cloned singing-voice id (must be owned by the caller).'),
// ── ElevenLabs Music v2.5 structured mode (mutually exclusive with `prompt`) ──
composition_plan: z.array(MUSIC_CHUNK_SCHEMA).optional().describe('ElevenLabs Music v2.5 ONLY, model must resolve to the ElevenLabs music model: an ordered list of chunks instead of a prompt. A GENERATION chunk has text/duration_ms/positive_styles (optionally conditioning_ref+condition_strength to steer it off a stored song). An AUDIO-REFERENCE chunk has song_id/range and splices that stored slice in unchanged (used to keep part of an existing track — see edit_music_section for the common case of regenerating one section). Get a starter plan from create_music_composition_plan, or build one from a prior generate_music result\'s `song_id`. Cannot be combined with `prompt`.'),
seed: z.number().optional().describe('ElevenLabs Music v2.5: random seed for reproducibility. Only valid together with composition_plan, never with prompt.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ prompt, model, style, title, instrumental, lyrics, vocal_gender, negative_tags, duration_seconds, enhance_prompt = false, preset_id, style_weight, weirdness, audio_weight, persona_id, use_composition_plan, singing_dna_id, singing_voice_id, composition_plan, seed, project_id, session_id }) => {
if (!prompt && !composition_plan) throw new Error('prompt is required (or pass composition_plan for ElevenLabs Music v2.5)');
model = await canonicalModelId(client, model, 'music_gen'); // lenient id resolution ("z-image" → "z-image/turbo")
const gen = await client.post('/v1/generate/music', {
prompt, model, style, title, instrumental, lyrics, vocal_gender, negative_tags,
duration_seconds, enhance_prompt, preset_id,
style_weight, weirdness, audio_weight, persona_id, use_composition_plan,
singing_dna_id, singing_voice_id,
...(composition_plan ? { composition_plan: { chunks: composition_plan }, seed } : {}),
project_id, session_id
});
if (returnsImmediately()) return submittedResult({
tool: 'generate_music', kind: 'audio', gen, client, model: model || 'Suno', prompt,
settings: { mode: instrumental ? 'instrumental' : (style || undefined), preset_id },
});
const poll = await pollOrTimedOut(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 8) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'generate_music', kind: 'audio', gen, client, model: model || 'Suno', prompt,
settings: { mode: instrumental ? 'instrumental' : (style || undefined), preset_id },
urls: result.result.urls,
playback_urls: result.result.playback_urls,
title: result.result.title,
duration: result.result.duration,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: gen.session_id,
urls: result.result.urls,
playback_urls: result.result.playback_urls,
title: result.result.title,
duration: result.result.duration,
lyrics: result.result.lyrics,
// ElevenLabs Music v2.5 only — reuse in a later composition_plan / edit_music_section.
...(result.result.song_id ? { song_id: result.result.song_id } : {}),
}, null, 2));
}
);
// ─── create_music_composition_plan ─────────────────────────
// FREE (no credits, no generation created) — ElevenLabs Music v2.5's POST /v1/music/plan.
server.tool(
'create_music_composition_plan',
'Generate a starter ElevenLabs Music v2.5 structured composition plan (an ordered list of chunks with section text, styles, and durations) from a plain-text prompt. FREE — creates no generation and charges no credits. Edit the returned chunks and pass them as `composition_plan` to generate_music to actually create the track.',
{
prompt: z.string().describe('Text description of the song to plan (e.g. "an upbeat pop song with verse and chorus about summer").'),
duration_seconds: z.number().optional().describe('Target total length in seconds, 3-600. Omit to let the model choose.'),
project_id: projectIdField,
session_id: sessionIdField,
},
async ({ prompt, duration_seconds, project_id, session_id }) => {
const res = await client.post('/v1/generate/music/composition-plan', { prompt, duration_seconds, project_id, session_id });
return { content: [{ type: 'text', text: JSON.stringify(res, null, 2) }] };
}
);
// ─── create_music_reference_audio ───────────────────────────
// BILLED at the same rate as a song generation of the clip's length (ElevenLabs charges
// the upload itself, even half-price if it flags the clip for copyright) — this is NOT a
// free utility call, unlike create_music_composition_plan above.
server.tool(
'create_music_reference_audio',
'Upload an existing audio clip you already have in Kolbo so it can be used as an audio reference in a later generate_music composition_plan (as a song_id in a conditioning_ref, to steer style, or in an audio-reference chunk, to splice a slice in unchanged) or as the source for edit_music_section. BILLED the same as generating a track of the clip\'s length — ElevenLabs charges for this upload itself. LOCAL FILE? Call upload_media first and pass the returned media id here.',
{
media_id: z.string().describe('A media-library item id for an audio file you own — from upload_media, or from list_media (mediaType audio).'),
project_id: projectIdField,
session_id: sessionIdField,
},
async ({ media_id, project_id, session_id }) => {
const res = await client.post('/v1/generate/music/reference-audio', { media_id, project_id, session_id });
return { content: [{ type: 'text', text: JSON.stringify(res, null, 2) }] };
}
);
// ─── edit_music_section ─────────────────────────────────────
// ElevenLabs Music v2.5 inpainting — regenerate one time range of an existing track,
// keeping the rest unchanged. BILLED like an ordinary generation (creates a new track).
server.tool(
'edit_music_section',
'Regenerate one section (time range) of an ElevenLabs Music v2.5 track you already generated, keeping the rest of the song unchanged — e.g. "redo the outro with different lyrics". Only works on tracks made with the ElevenLabs Music model (check the earlier generate_music result for a `song_id`; a track without one cannot be edited this way). Creates a NEW generation in the same session; the source track is untouched. Returns a generation_id to poll like any other generate_music call.',
{
source_generation_id: z.string().describe('The generation_id of the ElevenLabs Music v2.5 track to edit (must have returned a song_id).'),
start_ms: z.number().int().min(0).describe('Start of the region to regenerate, in milliseconds.'),
end_ms: z.number().int().min(0).describe('End of the region to regenerate, in milliseconds. (end_ms - start_ms) must be 3000-120000ms.'),
text: z.string().optional().describe('New section text — [Section Name], lyric lines, {inline directions}. Omit to keep it instrumental/unlabeled.'),
positive_styles: z.array(z.string()).optional().describe('Styles/directions for the regenerated section.'),
negative_styles: z.array(z.string()).optional().describe('Styles/directions to avoid in the regenerated section.'),
context_adherence: z.enum(['low', 'medium', 'high']).optional().describe('How closely the new section follows the kept audio around it. Default high.'),
project_id: projectIdField,
session_id: sessionIdField,
},
async ({ source_generation_id, start_ms, end_ms, text, positive_styles, negative_styles, context_adherence, project_id, session_id }) => {
const gen = await client.post('/v1/generate/music/edit-section', {
source_generation_id, start_ms, end_ms, text,
positive_styles, negative_styles, context_adherence,
project_id, session_id,
});
if (returnsImmediately()) return submittedResult({
tool: 'edit_music_section', kind: 'audio', gen, client, model: 'ElevenLabs Music', prompt: text || '(section edit)',
});
const poll = await pollOrTimedOut(client, gen.generation_id, { interval: (gen.poll_interval_hint || 8) * 1000, timeout: 150000 });
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'edit_music_section', kind: 'audio', gen, client, model: 'ElevenLabs Music', prompt: text || '(section edit)',
urls: result.result.urls,
playback_urls: result.result.playback_urls,
title: result.result.title,
duration: result.result.duration,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: gen.session_id,
urls: result.result.urls,
playback_urls: result.result.playback_urls,
duration: result.result.duration,
...(result.result.song_id ? { song_id: result.result.song_id } : {}),
}, null, 2));
}
);
// ─── music import / extend / cover ─────────────────────────
// Three tools over the SDK's /v1/generate/music/{import,extend,cover}. The web app has
// had upload-extend and upload-cover for a long time; agents could not reach either
// until those routes existed.
const musicSourceFields = {
audio_url: z.string().optional().describe('PUBLIC URL of the source audio. Either this (with rights_confirmed) or upload_id is required. Local file? Call upload_media first and pass the returned https:// URL.'),
upload_id: z.string().optional().describe('Upload id from import_music_audio. Use this to reuse one imported file across several calls instead of re-importing it.'),
rights_confirmed: z.boolean().optional().describe('REQUIRED with audio_url: confirms the caller holds the rights to the source audio. Not needed when passing upload_id (it was confirmed at import).'),
prompt: z.string().optional().describe('What the new material should sound like.'),
style: z.string().optional().describe('Music style / genre for the new material.'),
title: z.string().optional().describe('Title for the result. Generated if omitted.'),
instrumental: z.boolean().optional().describe('Produce instrumental only, no vocals. Default: false'),
lyrics: z.string().optional().describe('Custom lyrics. Omit to have them generated, or set instrumental.'),
model: z.string().optional().describe('Model identifier. Omit for the Suno default.'),
project_id: projectIdField,
session_id: sessionIdField
};
server.tool(
'import_music_audio',
'Import a PUBLIC audio URL into Kolbo as a reusable music source, returning an upload_id for extend_music / cover_music. Use it when the same track feeds several calls — both of those tools also accept audio_url directly for a one-shot. LOCAL FILE? Call upload_media first and pass the returned https:// URL; this tool cannot read the caller\'s disk. You must set rights_confirmed: true — Kolbo requires the caller to hold the rights to audio they upload.',
{
audio_url: z.string().describe('PUBLIC URL of the audio file to import.'),
upload_type: z.string().optional().describe('What the import is for: "extend" or "cover". Must match the tool you later call with it. Default: "extend".'),
rights_confirmed: z.boolean().describe('REQUIRED true: confirms the caller holds the rights to this audio.'),
confirmation_text: z.string().optional().describe('Optional free-text rights confirmation recorded with the upload.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ audio_url, upload_type, rights_confirmed, confirmation_text, project_id, session_id }) => {
const r = await client.post('/v1/generate/music/import', {
audio_url, upload_type: upload_type || 'extend', rights_confirmed, confirmation_text, project_id, session_id
});
return result(r);
}
);
// Both generation tools share one runner — the only difference is the route and the label.
const runMusicSourceOp = async (route, toolName, args) => {
const { model: rawModel, continue_at, ...rest } = args;
const model = await canonicalModelId(client, rawModel, 'music_gen');
const gen = await client.post(route, { ...rest, model, ...(continue_at !== undefined ? { continue_at } : {}) });
if (returnsImmediately()) return submittedResult({
tool: toolName, kind: 'audio', gen, client, model: model || 'Suno', prompt: rest.prompt,
settings: { mode: rest.instrumental ? 'instrumental' : (rest.style || undefined) },
});
const poll = await pollOrTimedOut(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 8) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const res = poll.result;
return uiCompleted({
tool: toolName, kind: 'audio', gen, client, model: model || 'Suno', prompt: rest.prompt,
settings: { mode: rest.instrumental ? 'instrumental' : (rest.style || undefined) },
urls: res.result.urls,
playback_urls: res.result.playback_urls,
tracks: res.result.tracks,
title: res.result.title,
duration: res.result.duration,
credits_used: creditFields(res).credits_used,
}, JSON.stringify({
...creditFields(res),
session_id: gen.session_id,
urls: res.result.urls,
tracks: res.result.tracks,
title: res.result.title,
duration: res.result.duration,
lyrics: res.result.lyrics
}, null, 2));
};
server.tool(
'extend_music',
'Continue an existing track — Kolbo generates new music that carries on from a point in the source audio. Pass audio_url (with rights_confirmed) or an upload_id from import_music_audio. There is NO target-length control here: the length follows the source and continue_at, which is why duration_seconds is not accepted. Returns the final audio URL when complete.',
{ ...musicSourceFields, continue_at: z.number().optional().describe('Seconds into the source audio to continue FROM. Omit to continue from the end.') },
async (args) => runMusicSourceOp('/v1/generate/music/extend', 'extend_music', args)
);
server.tool(
'cover_music',
'Re-record an existing track in a new style, keeping its musical identity — a lo-fi cover of a rock song, an acoustic take on an electronic track. Pass audio_url (with rights_confirmed) or an upload_id from import_music_audio. There is NO target-length control here: a cover follows the length of its source, which is why duration_seconds is not accepted. Returns the final audio URL when complete.',
{ ...musicSourceFields },
async (args) => runMusicSourceOp('/v1/generate/music/cover', 'cover_music', args)
);
// ─── generate_speech ───────────────────────────────────────
server.tool(
'generate_speech',
'Convert text to speech using Kolbo AI. SCOPE: narration, voiceover, and standalone audio (ads, explainers, audiobooks, announcements). This is NOT the route for character dialogue in a film you are generating with Seedance 2/2.5 — those models PERFORM quoted dialogue natively (synced voices, lip movement, room tone), so sending scene dialogue here just adds a second, unsynced voice track. Default provider is ElevenLabs. To pick a specific voice by language/gender, call list_voices first and pass the returned voice_id (or a voice display name — both work). Every voice belongs to a provider (ElevenLabs, DeepDub, MiniMax, Google/Gemini, OpenAI, Zonos) and each provider exposes its own expressive/style controls below — the engine ignores any control that does not apply to the chosen voice\'s provider, so it is safe to pass only what you need. Returns the final audio URL when complete.',
{
text: z.string().describe('The text to convert to speech'),
voice: z.string().optional().describe('Voice ID or display name — MUST come from a `list_voices` result, never constructed. Google/Gemini ids in particular are not validated provider-side: an id that is not in the catalog is silently mapped to another voice (or a default one) and the audio comes back in a voice nobody asked for. Do not pattern-match a locale onto an id you saw for another language. Default: "Rachel"'),
model: z.string().optional().describe('Model identifier. Use list_models type="text_to_speech" to see options. Default: eleven_v4 (ElevenLabs V4). ElevenLabs V4/V3 voices follow inline audio tags in the text: delivery ([whispers], [shouting], [sarcastic]), reactions ([laughs], [sighs], [clears throat]), pauses ([pause], [long pause]) and, on V4, free-form directions like [Low, steady voice, restrained urgency]. Tags are always English, even in Hebrew text. SSML such as <break> is NOT supported.'),
turbo: z.boolean().optional().describe('ElevenLabs V4 voices ONLY. Turbo: faster generation at half the credits, slightly lower quality. Rejected (TURBO_UNSUPPORTED) for any voice not on ElevenLabs V4. Default false.'),
language: z.string().optional().describe('Language / accent code (e.g., "en-US", "he-IL", "es-ES"). For Google/Gemini voices this is the Accent control (does not translate the text). Default: "en-US"'),
// ── Expressive style / emotion (provider-specific) ──
// Google/Gemini: prefer a named preset id from the Kolbo UI picker; free-form
// style_instructions is the escape hatch for custom direction.
style_instructions_preset_id: z.enum([
'warm', 'dramatic', 'whisper', 'excited', 'calm', 'cheerful',
'serious', 'storyteller', 'sad', 'intimate', 'british', 'commercial',
'custom', 'none',
]).optional().describe('Google/Gemini voices ONLY. Named voice-direction preset from the Kolbo TTS picker (warm, dramatic, whisper, excited, calm, cheerful, serious, storyteller, sad, intimate, british, commercial). Use "custom" with style_instructions for free-form direction, or "none" for default delivery. Preferred over raw style_instructions when a preset fits — the API expands it to the English directive fal receives and stores the label for history/reuse.'),
style_instructions: z.string().optional().describe('Google/Gemini voices ONLY. Free-form natural-language voice direction, e.g. "whisper conspiratorially, slightly amused" or "excited sports announcer". Max 500 chars. When set without a preset_id, treated as custom. Ignored by other providers.'),
style_instructions_label: z.string().optional().describe('Google/Gemini voices ONLY. Friendly label stored on the generation for history cards (defaults to the preset name or the custom text). Usually omit — the API fills it.'),
selected_style: z.string().optional().describe('DeepDub, MiniMax & Cartesia voices. Preset expressive style/emotion. DeepDub: reading, conversational, angry, breathy, panic, amused, sad, whisper, singing, shout, scream, mumbling, excited. MiniMax/Cartesia: happy, sad, angry, fearful, disgusted, surprised, calm, fluent, whisper (or provider emotion names). Ignored by other providers.'),
emotion: z.string().optional().describe('MiniMax / Cartesia voices. Emotion: happy, sad, angry, fearful, disgusted, surprised, calm, fluent, whisper. Prefer this OR selected_style (both map to the same delivery control).'),
speaking_speed: z.number().optional().describe('Speech speed 0.5 (slow) – 2.0 (fast); Cartesia clamped 0.6–1.5. Default 1.0. Applies to ElevenLabs / OpenAI / Google / MiniMax / Cartesia / DeepDub (as tempo when tempo omitted).'),
// ── ElevenLabs voice settings ──
similarity_boost: z.number().optional().describe('ElevenLabs voice similarity, 0–1. Default 0.75. Higher hews closer to the original voice.'),
style: z.number().optional().describe('ElevenLabs style exaggeration, 0–1. Default 0.5. Higher = more expressive/dramatic.'),
use_speaker_boost: z.boolean().optional().describe('ElevenLabs speaker boost. Default true.'),
// ── DeepDub controls ──
variance: z.number().optional().describe('DeepDub voice variance, 0–1. Default 0.2. Higher = more takes/variation.'),
tempo: z.number().optional().describe('DeepDub tempo multiplier, 0–2. Default 1.0.'),
promptBoost: z.boolean().optional().describe('DeepDub prompt-fidelity boost. Default true.'),
seed: z.number().optional().describe('Reproducibility seed (DeepDub / Zonos). Same seed + inputs → same output.'),
accentControl: z.object({
accentBaseLocale: z.string().describe('Base accent locale, e.g. "en-US".'),
accentLocale: z.string().describe('Target accent locale, e.g. "en-GB".'),
accentRatio: z.number().optional().describe('Blend ratio 0–1. Default 0.5.')
}).optional().describe('DeepDub accent steering. Provide both base and target locale to blend an accent.'),
voiceTitle: z.string().optional().describe('DeepDub display title for a custom/cloned voice.'),
// ── MiniMax fine controls ──
minimax_pitch: z.number().optional().describe('MiniMax pitch, −12 to 12. Default 0.'),
minimax_vol: z.number().optional().describe('MiniMax volume, 0–10. Default 1.'),
minimax_intensity: z.number().optional().describe('MiniMax voice intensity.'),
minimax_timbre: z.number().optional().describe('MiniMax voice timbre.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ text, voice, model, turbo, language, style_instructions_preset_id, style_instructions, style_instructions_label, selected_style, emotion, speaking_speed, similarity_boost, style, use_speaker_boost, variance, tempo, promptBoost, seed, accentControl, voiceTitle, minimax_pitch, minimax_vol, minimax_intensity, minimax_timbre, project_id, session_id }) => {
model = await canonicalModelId(client, model, 'text_to_speech'); // lenient id resolution ("z-image" → "z-image/turbo")
// Resolve the requested voice against the REAL catalog (cached) so the card
// can show its display name + portrait instead of a raw id, and so an id
// that does not exist is reported instead of rendering silently: Google
// voice ids collapse to their last segment provider-side, so a made-up
// "en-US-Chirp3-HD-<Name>" either speaks as some other catalog entry's
// voice or falls back to a default one — with the card naming neither.
const voiceRecord = await voiceInfo(client, voice).catch(() => null);
const unknownVoice = voice && !voiceRecord && !/^custom_/i.test(voice)
? `Voice "${voice}" is not in the Kolbo voice catalog. Call list_voices and pass a voice_id it returns — an unrecognised id is NOT rejected, it is silently mapped to a different voice, so the audio will not be the voice you named.`
: null;
const gen = await client.post('/v1/generate/speech', {
text, voice, model, turbo, language,
style_instructions_preset_id, style_instructions, style_instructions_label,
selected_style, emotion, speaking_speed,
similarity_boost, style, use_speaker_boost,
variance, tempo, promptBoost, seed, accentControl, voiceTitle,
minimax_pitch, minimax_vol, minimax_intensity, minimax_timbre,
project_id, session_id
});
if (returnsImmediately()) return submittedResult({
tool: 'generate_speech', kind: 'audio', gen, client, model, prompt: text,
voice: voiceRecord,
settings: {
voice: voice || 'Rachel',
style: selected_style || emotion || style_instructions_preset_id || style_instructions,
speaking_speed,
language,
},
warning: unknownVoice
});
const poll = await pollOrTimedOut(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 5) * 1000,
timeout: 120000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
// What ACTUALLY ran, not what was asked for. The status result reports the
// engine's own choice (`model: "google_tts"`, `voice: "he-IL-Chirp3-HD-Kore"`)
// and for an omitted model that is the ONLY place the real answer appears —
// the card was labelling those "Smart Select", which is not even a text-to-
// speech option, and naming the voice the caller typed rather than the one
// that spoke. Same resolution addDisplayNames does for the polling path.
const ranVoice = (await voiceInfo(client, result.result?.voice).catch(() => null)) || voiceRecord;
return uiCompleted({
tool: 'generate_speech', kind: 'audio', gen, client, prompt: text,
model: result.result?.model || model,
voice: ranVoice,
settings: {
voice: (ranVoice && ranVoice.name) || voice || 'Rachel',
style: selected_style || emotion || style_instructions_preset_id || style_instructions,
speaking_speed,
language,
},
urls: result.result.urls,
playback_urls: result.result.playback_urls,
duration: result.result.duration,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: gen.session_id,
urls: result.result.urls,
playback_urls: result.result.playback_urls,
voice: result.result.voice,
duration: result.result.duration,
...(unknownVoice ? { _warning: unknownVoice } : {})
}, null, 2));
}
);
// ─── generate_sound ────────────────────────────────────────
server.tool(
'generate_sound',
'Generate sound effects (not music, not speech) from a text description using Kolbo AI. Use this for ambient sounds, foley, impacts, atmospheres, UI sounds, etc. For music use generate_music; for voice use generate_speech. Beyond the core prompt/duration, per-provider controls are available (Stable Audio guidance, Kie loop/tempo/key, Seed-Audio voice/speed/volume/pitch + reference audio/image); the engine ignores controls that do not apply to the chosen model. Returns the final audio URL when complete.',
{
prompt: z.string().describe('Text description of the sound effect (e.g., "thunder clap with rain", "door creaking open", "futuristic UI beep")'),
model: z.string().optional().describe('Model identifier. Use list_models type="text_to_sound" to see options. Default: elevenlabs-sound-effects-v1'),
duration: z.number().optional().describe('Duration in seconds. Omit for automatic duration.'),
prompt_influence: z.number().optional().describe('ElevenLabs: how strongly the prompt guides the generation (0–1). Default: 0.5. Lower = more creative freedom; higher = more literal.'),
// ── FAL Stable Audio / mmaudio ──
cfg_strength: z.number().optional().describe('FAL (Stable Audio 3 / mmaudio): classifier-free guidance strength. Higher hews closer to the prompt.'),
// ── Kie ──
sound_loop: z.boolean().optional().describe('Kie: generate a seamlessly looping sound.'),
sound_tempo: z.number().optional().describe('Kie: tempo control.'),
sound_key: z.string().optional().describe('Kie: musical key / scale.'),
// ── FAL Seed Audio ──
seed_voice: z.string().optional().describe('FAL Seed-Audio: voice to use.'),
seed_speed: z.number().optional().describe('FAL Seed-Audio: speed multiplier, 0.5–2.0.'),
seed_volume: z.number().optional().describe('FAL Seed-Audio: volume, 0–1.'),
seed_pitch: z.number().optional().describe('FAL Seed-Audio: pitch shift in semitones.'),
seed_reference_audio_urls: z.array(z.string()).optional().describe('FAL Seed-Audio: up to 3 reference audio URLs to condition the sound.'),
seed_reference_image_url: z.string().optional().describe('FAL Seed-Audio: a reference image URL to condition the sound.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ prompt, model, duration, prompt_influence, cfg_strength, sound_loop, sound_tempo, sound_key, seed_voice, seed_speed, seed_volume, seed_pitch, seed_reference_audio_urls, seed_reference_image_url, project_id, session_id }) => {
model = await canonicalModelId(client, model, 'text_to_sound'); // lenient id resolution ("z-image" → "z-image/turbo")
const gen = await client.post('/v1/generate/sound', {
prompt, model, duration, prompt_influence,
cfg_strength, sound_loop, sound_tempo, sound_key,
seed_voice, seed_speed, seed_volume, seed_pitch,
seed_reference_audio_urls, seed_reference_image_url, project_id, session_id
});
if (returnsImmediately()) return submittedResult({
tool: 'generate_sound', kind: 'audio', gen, client, model, prompt,
settings: { duration },
reference_images: seed_reference_image_url ? [seed_reference_image_url] : []
});
const poll = await pollOrTimedOut(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 5) * 1000,
timeout: 120000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'generate_sound', kind: 'audio', gen, client, model, prompt,
settings: { duration },
reference_images: seed_reference_image_url ? [seed_reference_image_url] : [],
urls: result.result.urls,
playback_urls: result.result.playback_urls,
duration: result.result.duration,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: gen.session_id,
urls: result.result.urls,
playback_urls: result.result.playback_urls,
duration: result.result.duration
}, null, 2));
}
);
// The status endpoint reports RAW ids — `model: "google_tts"`, `voice:
// "he-IL-Chirp3-HD-Kore"`. That is the model/voice that ACTUALLY ran (for
// Smart Select it is the only place the choice surfaces), so the card must
// show it — but with the clean name + icon everything else uses. Resolved
// here, off the same cached catalogs `list_models` / `list_voices` serve, so
// the widget adds no round trip per generation. Written INTO `result` because
// that is the object the widget merges over its generating-phase state.
async function addDisplayNames(status) {
const r = status && status.result;
if (!r || typeof r !== 'object') return;
// Always WRITE both keys, even on a catalog miss: the widget merges this
// over its generating-phase state, so a missing key silently keeps the
// pre-submit guess ("Smart Select") for a model that is not what ran. An
// unresolvable id is at least honest.
if (r.model) {
const info = await modelInfo(client, r.model).catch(() => null);
r.model_name = (info && info.name) || r.model;
r.model_icon = (info && info.icon) || null;
}
if (r.voice) {
const v = await voiceInfo(client, r.voice).catch(() => null);
r.voice_name = (v && v.name) || r.voice;
r.voice_thumbnail = (v && v.thumbnail) || null;
}
}
// ─── get_generation_status ─────────────────────────────────
server.tool(
'get_generation_status',
`Check the status of one or more generations. Use after a generation tool returned "submitted" (widget hosts) or timed out. Tracking SEVERAL concurrent generations? Pass them ALL in generation_ids — one call returns an all_done summary. Need the final result? Set wait=true and the server blocks until every generation finishes, for at most ~${WAIT_WINDOW_S}s per call. A job that outlives one window (music is ~3 min, video longer) comes back state="processing" — that is a normal result, not an error: call again with wait=true and keep going until every id is terminal. Never poll with wait=false in a loop.`,
{
generation_id: z.string().optional().describe('A single generation ID to check'),
generation_ids: z.array(z.string()).max(MAX_STATUS_IDS).optional().describe(`Multiple generation IDs to check in ONE call (max ${MAX_STATUS_IDS}). Returns { all_done, pending, generations[] } — always prefer this over checking IDs one by one.`),
wait: z.boolean().optional().describe(`If true, block until every generation reaches a terminal state (completed/failed), for at most ~${WAIT_WINDOW_S}s per call, then return whatever state they are in. Anything still processing is reported, not errored — re-issue with wait=true and only the still-pending ids. This is always better than polling with wait=false.`)
},
async ({ generation_id, generation_ids, wait }) => {
const ids = (generation_ids && generation_ids.length > 0)
? [...new Set(generation_ids)]
: (generation_id ? [generation_id] : []);
if (ids.length === 0) throw new Error('Provide generation_id or generation_ids');
if (ids.length > MAX_STATUS_IDS) throw new Error(`At most ${MAX_STATUS_IDS} generation IDs can be checked at once.`);
// One status check (or blocking poll) per id. Never let one bad id
// reject the whole batch — surface it as a failed entry instead.
const checkOne = async (id) => {
try {
if (wait) {
// Widgets use this long-wait path. A 15s API check cadence keeps
// completion responsive without multiplying backend traffic for
// every card left open in a host conversation. The window itself is
// bounded by the transport — see WAIT_WINDOW_MS above.
const result = await pollUntilDone(client, id, { interval: 15000, timeout: WAIT_WINDOW_MS });
return { generation_id: id, ...result };
}
const result = await client.get(`/v1/generate/${encodeURIComponent(id)}/status`);
return { generation_id: id, ...result };
} catch (err) {
if (err.timedOut) {
return { generation_id: id, state: 'processing', _timed_out: true, note: `Still running after this ~${WAIT_WINDOW_S}s wait window — call get_generation_status again with wait=true.` };
}
if (err.name === 'GenerationFailedError') {
return { generation_id: id, state: 'failed', error: err.message };
}
return { generation_id: id, state: 'unknown', error: err.message };
}
};
const results = await Promise.all(ids.map(checkOne));
await Promise.all(results.map(addDisplayNames));
const pending = results.filter(r => r.state !== 'completed' && r.state !== 'failed' && r.state !== 'cancelled');
const unresolved = results.filter(r => r.state === 'unknown');
const doneHint = 'ALL generations are in a final state — do NOT poll again. Report the results to the user.';
// The old wait=false hint said "call it ONCE with wait=true ... to block
// until they finish". That is the advice that broke: one wait=true call
// cannot outlast a 185s music job, and a caller that obeyed it got a
// transport error instead of a result. Say what actually works.
const pendingIds = pending.map(r => r.generation_id);
const idsPhrase = pendingIds.length > 1
? ` and ONLY the still-pending ids: ${JSON.stringify(pendingIds)}`
: '';
const pendingHint = unresolved.length
? `Status could not be verified for ${unresolved.map(r => r.generation_id).join(', ')}. Check the ID and connection, then retry once later; do not poll in a loop or submit a replacement generation.`
: stillRunningHint(idsPhrase);
// Single-id calls keep the original flat TEXT shape — a live uiGenerating
// widget (ui()==true, still open from the original generate_* call) polls
// this tool directly from its own iframe JS and reads state/result at the
// top level, so `text` must stay exactly what it always was.
//
// But a single id checked here often has NO live widget behind it at all:
// Kolbo Code never opens one (ui() is false there), and even on hosts that
// do, the original call can have returned poll.timedOut before uiGenerating
// ever ran (see pollOrTimedOut) — a parallel batch is exactly when that
// happens, since every concurrent generation competes for the same
// per-call poll window. Either way this is then a FRESH, standalone tool
// response that needs its own renderable card, same reasoning as the
// multi-id fix just above. Reuse the same kind:'status' single-item grid
// rather than inventing per-media-type rendering here.
if (!generation_ids || generation_ids.length === 0) {
const single = results[0];
single._hint = pending.length === 0
? 'This generation is in a FINAL state — do not poll it again.'
: pendingHint;
const singleText = JSON.stringify(single, null, 2);
const res = single.result || {};
// A live generation card polls THIS branch from inside its own iframe,
// and it reads structuredContent — never the text (apps/html.js
// structured() returns structuredContent first and only falls back to
// parsing content[].text). So preserving the flat TEXT shape, as the
// comment above intended, protected nothing: the card does
// `r = st.result || st` then `r.urls`, found only `items`, and painted
// "No output received / Failed" over a generation that had completed
// and billed. Every card rendered before v1.74 keeps that old iframe JS
// forever, so the flat fields have to live in structuredContent too.
const urls = preferOwned(Array.isArray(res.urls) ? res.urls : []);
const done = single.state === 'completed' && urls.length > 0;
const extraContent = options.asyncGenerations && done && mediaKind(urls[0]) === 'image'
? await inlineImageBlocks(urls, { enabled: inlineImages, maxCount: 1, maxBytes: 512 * 1024, fetchTimeoutMs: 3000 })
: undefined;
return uiCompleted({
tool: 'get_generation_status', kind: done ? mediaKind(urls[0]) : 'status', client,
// The model that ACTUALLY ran. A single-id check resolves one
// generation, so 'Generations' here overwrote the real model name on
// the finished card the moment the live widget merged this payload in.
model: res.model || 'Generations',
gen: {
generation_id: single.generation_id,
session_id: single.session_id,
project_id: single.project_id,
type: single.type,
},
state: single.state,
// The failure reason. The SDK always sends it, but this card dropped it, so a
// failed generation surfaced as a bare "failed" with no way to act on it.
error: single.state === 'failed' ? single.error : undefined,
urls: done ? urls : undefined,
thumbnail_url: res.thumbnail_url,
// Transcription results ride the same status tool as media
// generations; without these the transcript widget merges a payload
// with no transcript in it and renders "(empty transcript)".
text: typeof res.text === 'string' ? res.text : undefined,
srt_url: res.srt_url || undefined,
word_by_word_srt_url: res.word_by_word_srt_url || undefined,
txt_url: res.txt_url || undefined,
audio_url: res.audio_url || undefined,
// The refs the server actually conditioned on (reference_details) —
// the live card merges this payload over its submit-time state, so
// the finished card shows every reference, including server-side
// additions (DNA stills, merged sources) the tool call never named.
reference_images: res.reference_images,
// Transcription results ride the same status tool as media
// generations; without these the transcript widget merges a payload
// with no transcript in it.
text: typeof res.text === 'string' ? res.text : undefined,
srt_url: res.srt_url || undefined,
word_by_word_srt_url: res.word_by_word_srt_url || undefined,
txt_url: res.txt_url || undefined,
audio_url: res.audio_url || undefined,
prompt: res.prompt_used || res.prompt || undefined,
credits_used: creditFields(single).credits_used,
items: [{
id: single.generation_id,
state: single.state,
error: single.state === 'failed' ? single.error : undefined,
title: res.prompt_used || res.prompt || undefined,
url: Array.isArray(res.urls) ? res.urls[0] : undefined,
}],
}, singleText, extraContent);
}
const text = JSON.stringify({
all_done: pending.length === 0,
completed: results.filter(r => r.state === 'completed').length,
failed: results.filter(r => r.state === 'failed' || r.state === 'cancelled').length,
still_processing: pending.map(r => r.generation_id),
_hint: pending.length === 0 ? doneHint : pendingHint,
generations: results
}, null, 2);
// Checking SEVERAL ids in one call — the "run N generations in parallel,
// then check on all of them" pattern — is the one shape hasGeneratedOutput
// (the host's card-selection check) cannot see into: media here sits one
// level under EACH ARRAY ITEM's own `result`, and nothing walks arrays
// looking for it. Without structuredContent the card degrades to a plain
// status list — ids and state, no thumbnails, even for completed items
// whose media is sitting right there. uiCompleted gives it the same rich
// per-item render every generate_* tool already gets; kind:'status' picks
// renderStatusGrid, the one path here that mixes independent
// pending/completed/failed items in one grid instead of assuming they all
// finished together.
//
// Always ship structuredContent, unconditionally — Kolbo Code does NOT
// advertise MCP Apps, so gating this behind ui()/appsEnabled() (as this
// used to) left that host with text only, and its own generic fallback
// renderer built a plain status list with no thumbnails, exactly the
// "black tile" class of bug already fixed the same way in media.js /
// moodboards.js / visual_dna.js / listResult(). uiResult always embeds
// both text and structuredContent, so hosts without a UI-app renderer
// are unaffected — only the fallback quality changes.
// A live batch card (prompts[] fan-out) merges this payload over its
// submit-time state, so a hardcoded 'Generations' model here painted
// over the real model chip on every finished batch. When all results
// ran the same model (the batch case), name it; only a genuinely mixed
// multi-id check falls back to the generic label. Same for the
// references actually used — one shared set per batch.
const modelsRan = [...new Set(results.map(r => r.result && r.result.model).filter(Boolean))];
const actualRefs = results
.map(r => r.result && r.result.reference_images)
.find(refs => Array.isArray(refs) && refs.length);
const completedImageUrls = results.flatMap((r) => {
const urls = r?.result?.urls;
return Array.isArray(urls) && urls.length && mediaKind(urls[0]) === 'image' ? urls : [];
});
const extraContent = options.asyncGenerations
? await inlineImageBlocks(completedImageUrls, { enabled: inlineImages, maxCount: 1, maxBytes: 512 * 1024, fetchTimeoutMs: 3000 })
: undefined;
return uiCompleted({
tool: 'get_generation_status', kind: 'status', client,
model: modelsRan.length === 1 ? modelsRan[0] : 'Generations',
gen: {
generation_id: ids[0],
session_id: results[0] && results[0].session_id,
project_id: results[0] && results[0].project_id,
type: results[0] && results[0].type,
},
settings: {},
reference_images: actualRefs,
items: results.map(r => {
const res = r.result || {};
return {
id: r.generation_id,
state: r.state,
error: r.state === 'failed' ? r.error : undefined,
title: res.prompt_used || res.prompt || undefined,
url: Array.isArray(res.urls) ? res.urls[0] : undefined,
// A live batch card reads these from structuredContent (its only
// view of this response) to fill each cell and repaint its chips.
urls: Array.isArray(res.urls) ? res.urls : undefined,
credits_used: creditFields(r).credits_used,
model: res.model, model_name: res.model_name, model_icon: res.model_icon,
reference_images: res.reference_images,
};
}),
}, text, extraContent);
}
);
// ═════════════════════════════════════════════════════════════
// ─── cancel_generation ─────────────────────────────────────
server.tool(
'cancel_generation',
'Cancel a running generation and release/refund its credits. Works for every ' +
'generation type the SDK tracks (image, image edit, video, image-to-video, ' +
'video-to-video, elements, first/last frame, lipsync, music, speech, sound, 3D, ' +
'motion graphic, global image/video edit, creative director). Use when the user says ' +
'"stop", "cancel that", "never mind" while something is still running, or to clear a ' +
'stuck generation. Returns how many credits came back. A generation that already ' +
'finished or failed cannot be cancelled — you get a 409 with its current status, which ' +
'is not an error worth retrying. For Shorts Creator jobs use shorts_cancel instead.',
{
generation_id: z.string().describe('The generation id to cancel, as returned by any generate_* tool or get_generation_status.')
},
async ({ generation_id }) => {
try {
const result = await client.post(`/v1/generate/${encodeURIComponent(generation_id)}/cancel`, {});
return {
content: [{
type: 'text',
text: JSON.stringify({
generation_id,
cancelled: true,
...(result && typeof result === 'object' ? result : {}),
_hint: 'Generation cancelled. Do NOT call get_generation_status on it — the state is final.'
}, null, 2)
}]
};
} catch (err) {
// 409 = already terminal. That is an answer, not a failure — report it
// plainly so the model does not retry or apologise for a non-problem.
const conflict = err.status === 409 || /cannot cancel/i.test(err.message || '');
return {
content: [{
type: 'text',
text: JSON.stringify({
generation_id,
cancelled: false,
reason: err.message,
_hint: conflict
? 'This generation already reached a final state, so there was nothing to cancel. Tell the user it had already finished — do not retry.'
: 'Cancel failed. Check the generation_id is correct via get_generation_status.'
}, null, 2)
}]
};
}
}
);
// ═════════════════════════════════════════════════════════════
// ─── 2026-04 SDK Expansion Batch ─────────────────────────────
// ═════════════════════════════════════════════════════════════
// ─── generate_elements ─────────────────────────────────────
server.tool(
'generate_elements',
'Generate a video from reference elements (images, videos, and/or audio) + a text prompt. SPECIAL MODEL EXCEPTION: seedance-2-5-multilingual accepts an ordinary prompt in any language with the ORIGINAL target-language dialogue in double quotes, including Hebrew script. Use this tool even without references for that model, duration 8 or 12, resolution 480p/720p/1080p (no Draft), up to 3 still references or Visual DNAs, no video/audio references. It performs translation, distinct designed voices and segmented lip sync internally; never transliterate or pre-translate its quoted dialogue. The following regular Seedance instructions apply only to other model IDs. Use when the user wants to animate specific uploaded/referenced assets — e.g. "animate this product", "put these 3 characters into a scene". PRIMARY ROUTE FOR A DNA-ANCHORED MULTI-SHOT FILM: one call can carry the whole sequence — seedance-2-5 takes 4-30s, up to 30 shots and 20 Visual DNAs in a SINGLE generation (seedance-2: 4-15s, 9 DNAs) — instead of a stack of separate clips. DIALOGUE IS PERFORMED NATIVELY: quoted dialogue in the prompt comes back as synced voices with lip movement, room tone and the SFX named in the AUDIO block — never route scene dialogue to generate_speech or generate_lipsync. Write dialogue in ENGLISH or Latin transliteration of Hebrew ("shalom"), never Hebrew script — Seedance does not speak Hebrew; prefer Gemini Omni Flash 1.1 or Gemini Omni 1 for native Hebrew. COST: resolution is a multiplier. When list_models publishes `video_input_credit` and this call carries videos, charge that rate against nominal input seconds + nominal output seconds; MP4 padding within 0.15s of an integer snaps to that integer and larger fractions round up. Otherwise use the normal output-second rate. PROMPT CONTRACT (Seedance / Elements): Locked Intro only — Total line, then [GLOBAL LOOK] / [CAST] / [LOCATION] / SHOT N. Do NOT write SCENE CONTEXT / OPTICS / ACTION department packs. Every Visual DNA in visual_dna_ids MUST also appear in the prompt as @ExactDNAName (e.g. "@Zohar walks…") — never "Zohar\'s" or "the man on the left" as a substitute. IMPORTANT: different models accept different numbers and durations of inputs — call list_models type="elements" and read elements_max_images / elements_max_videos / elements_max_audio plus min_video_duration / max_video_duration before generating. For text-only → video use generate_video instead. For animating a single still image use generate_video_from_image. Returns the final video URL when complete.' + MORPHIOUS_GUIDE,
{
prompt: z.string().optional().describe('Optional ONLY for kolbo-morphious-* and higgsfield-genjutsu-* models (the server analyzes the media). For seedance-2-5-multilingual: ordinary scene description with original target-language dialogue in double quotes; no manual translation or transliteration. For other models, Locked Intro prompt (Seedance/Elements): Total line, [GLOBAL LOOK], [CAST] with @ExactDNAName for every visual_dna_ids entry, [LOCATION], then SHOT N. Not SCENE CONTEXT/OPTICS/ACTION packs. Never substitute "the left man" or "Zohar\'s" for @Name. EVERY attached reference must also be tagged by its 1-based array position — `@Image 1`/`@Image 2` (reference_images), `@Video 1` (reference_videos), `@Audio 1` (reference_audio_urls) — and its job stated ("@Image 1 defines the character\'s face and wardrobe", "@Video 1 defines the camera move"). An untagged attachment is ignored by the engine even though it was uploaded and billed.'),
model: z.string().optional().describe('Model identifier. If the user already named a family (Grok / Kling / Veo / Seedance / …), pass THAT family — never default to Seedance because Elements often uses it. Use list_models type="elements" for exact ids and elements_max_* caps. Do NOT omit (omitting = Smart Select).'),
reference_images: z.array(z.string()).optional().describe('Array of image references (product shots, character references, etc.). Tag each one in the prompt text by its 1-based position here — item 1 is `@Image 1`, item 2 is `@Image 2` — or the engine ignores it. Accepts a public URL (forwarded as-is; if the API rejects an external URL as untrusted, it is auto-rehosted into the media library and retried once) OR an absolute local path, which is uploaded for you. **Cap: pass at most `elements_max_images` URLs from list_models for the chosen model — exceeding it is a deterministic 400.**'),
reference_videos: z.array(z.string()).optional().describe('Array of reference videos for models that accept video inputs. Tag each one in the prompt text by its 1-based position here — `@Video 1`, `@Video 2` — stating what it defines (motion, camera move, pacing), or the engine ignores it. Accepts a public URL (forwarded as-is; if the API rejects an external URL as untrusted, it is auto-rehosted into the media library and retried once) OR an absolute local path, which is uploaded for you. **Cap: pass at most `elements_max_videos` URLs and keep every clip within `min_video_duration`-`max_video_duration` from list_models.** If `video_input_credit` is present, every attached video contributes its nominal duration to combined-second billing: encoder padding within 0.15s of an integer snaps to it; larger fractions round up.'),
reference_audio_urls: z.array(z.string()).optional().describe('Array of reference audio tracks for models that accept audio inputs. Tag each one in the prompt text by its 1-based position here — `@Audio 1`, `@Audio 2` — stating what it carries (dialogue, ambience, music), or the engine ignores it. Accepts a public URL (forwarded as-is; if the API rejects an external URL as untrusted, it is auto-rehosted into the media library and retried once) OR an absolute local path, which is uploaded for you. **Cap: pass at most `elements_max_audio` URLs from list_models.** `audio_url` remains supported as the legacy single-track form.'),
audio_url: z.string().optional().describe('A single reference audio track — legacy form of reference_audio_urls. Accepts a public URL (forwarded as-is; if the API rejects an external URL as untrusted, it is auto-rehosted into the media library and retried once) OR an absolute local path, which is uploaded for you. **Audio constraints: `elements_max_audio` from list_models gates whether audio is accepted at all; audio duration must fall within `min_audio_duration`-`max_audio_duration`; format must be in `supported_audio_formats` (if specified).**'),
files: z.array(z.string()).optional().describe('Untyped catch-all for mixed media — images, videos AND audio, each a URL or an absolute local path. The kind is detected from the file extension and the item is routed to the matching reference list, so a local .mp4 is sent as a video and a local .mp3 as audio. Prefer the typed lists (reference_images / reference_videos / reference_audio_urls) when you already know the kind; they accept local paths too. URLs given here are forwarded as URLs, never re-uploaded. **Caps still apply per kind: `elements_max_images` / `elements_max_videos` / `elements_max_audio` from list_models. Local uploads are capped at 200MB each.**'),
duration: z.number().optional().describe('Output duration in seconds. Must be in `supported_durations` from list_models, OR within `min_output_duration`-`max_output_duration`. Default: 5'),
aspect_ratio: z.string().optional().describe(aspectRatioDescribe('16:9')),
motion: z.string().optional().describe('Motion style / intensity hint (optional)'),
preset_id: z.string().optional().describe('Preset ID from list_presets type="video" (optional)'),
enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply for character/style consistency across outputs. **Cap: pass at most `max_visual_dna` IDs from list_models for the chosen model.**'),
style_id: z.string().optional().describe('Morphious Styles only (kolbo-morphious-motion or kolbo-morphious-swap; Standard only, the Lite modes refuse a style): id of a look from list_morphious_styles (Kolbo catalog or a custom style the user made). Restyles the WHOLE video in that look; the style counts as reference image 1, so no reference_images are needed. Prompt optional.'),
draft: z.boolean().optional().describe('Special Draft mode. True forces 480p Draft even if resolution says 1080p; false selects regular video. Only supported when list_models advertises draft capabilities. For Seedance 2.5 editing use model seedance-2-5-video-to-video with generate_video_from_video; the source duration and aspect ratio are preserved. Uses ordinary credits; finalize through draft_quote then draft_enhance.'),
resolution: z.string().optional().describe('Video resolution or named tier: read supported_resolutions from list_models. For Seedance 2.5 Draft use resolution="480p-draft" on the matching public model: seedance-2-5 for creation, seedance-2-5-video-to-video for editing. Plain "480p" generates regular video, NOT Draft. Finalize an actual draft with edit_video operation="draft_quote" then "draft_enhance".'),
sound_enabled: z.boolean().optional().describe('Enable (`true`) or disable (`false`) AI-generated synced audio on the output video. Honored by `sound_generation_type: "native"` models (Kling O3/V3, Veo 3.1, PixVerse V6). Omit to use `sound_enabled_by_default`. Enabling sound may apply `sound_credit_multiplier` to cost.'),
keyframes: z.array(z.object({
image_url: z.string().describe('Public URL of the keyframe image'),
timestamp_seconds: z.number().describe('Moment on the OUTPUT timeline (seconds, 0 = first frame) where this image is pinned')
})).optional().describe('Timeline-pinned keyframes for multi-keyframe models (e.g. "flux-3-keyframes", "luma-ray-3-2-storyboard"): the model generates the motion BETWEEN the pinned images. Only models with `supports_keyframes: true` in list_models accept this; cap = `max_keyframes` (FLUX 3: 10). Requires an explicit `duration` — timestamps beyond it are clamped. Ignored by ordinary elements models. OPTIONAL for flux-3-keyframes: if omitted, pass the images via reference_images instead — they are played through IN ORDER, timed from timing language in the prompt or spaced evenly. Pass explicit keyframes only when you need exact control.'),
multi_shots: z.boolean().optional().describe('Enable Multishot metadata/UI badge (and native provider flag when the model supportsMultiShot). Never rewrites or reshapes the prompt — craft shot structure yourself (`Multishot ON`, Total/SHOT lines). MCP default enhance_prompt=false keeps the prompt raw; multi_shots alone is badge/flag only. For crafted demos: enhance_prompt:false + multi_shots:true.'),
multi_shot_count: z.number().optional().describe('Explicit shot count when multi_shots=true (e.g. 2). Clamped to ~2 shots/sec of duration.'),
session_name: z.string().optional().describe('Short sidebar session title (max ~60 chars), e.g. "UGC — Hana bathroom light". When set, skips AI auto-naming from the prompt boilerplate.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ prompt = '', model, reference_images, reference_videos, reference_audio_urls, audio_url, files, duration, aspect_ratio, motion, preset_id, enhance_prompt = false, visual_dna_ids, style_id, resolution, draft, sound_enabled, keyframes, multi_shots, multi_shot_count, session_name, project_id, session_id }) => {
validateVideoPrompt({ prompt, duration, aspect_ratio, multi_shots, multi_shot_count });
if (draft === true) resolution = '480p-draft';
else if (draft === false && resolution?.endsWith('-draft')) resolution = resolution.slice(0, -6);
model = await canonicalModelId(client, model, 'elements'); // lenient id resolution ("z-image" → "z-image/turbo")
aspect_ratio = await resolveCatalogAspectRatio(client, model, aspect_ratio, 'elements');
validateVideoPrompt({ prompt, duration, aspect_ratio, multi_shots, multi_shot_count });
if (!prompt && !PROMPTLESS_NATIVE.has(model)) throw new Error('prompt is required');
// Elements is the one tool that takes all three modalities, and either a
// URL or a local path for any of them. Bucket every input by the kind the
// API expects, then split by transport:
// URL → forwarded AS a url. Downloading one only to re-upload it
// costs the round-trip, risks the 200MB multer ceiling, and
// duplicates an asset that already lives on the CDN. `files`
// used to do exactly that to every URL handed to it.
// local → the only thing that becomes a multipart part, uploaded under
// its real kind. `files` forced 'image' on everything, so a
// local .mp4/.mp3 arrived with the wrong content type and the
// elements controller — which buckets req.files by mimetype —
// never saw it as video or audio.
const media = { image: [], video: [], audio: [] };
const collect = (list, forced) => {
for (const src of Array.isArray(list) ? list : (list ? [list] : [])) {
if (typeof src !== 'string' || !src.trim()) continue;
// The typed lists declare their own kind; `files` is the untyped
// catch-all, so it is classified by extension.
const bucket = media[forced || uploadKind(src)];
if (!bucket.includes(src)) bucket.push(src);
}
};
collect(reference_images, 'image');
collect(reference_videos, 'video');
collect(reference_audio_urls, 'audio');
collect(audio_url, 'audio'); // legacy single-track form; the API merges it into audioUrls anyway
collect(files);
const urlsOf = (kind) => media[kind].filter(isUrlSource);
const locals = ['image', 'video', 'audio']
.flatMap((kind) => media[kind].filter((src) => !isUrlSource(src)).map((src) => ({ src, kind })));
const some = (list) => (list.length ? list : undefined);
const body = {
prompt, model,
reference_images: some(urlsOf('image')),
reference_videos: some(urlsOf('video')),
reference_audio_urls: some(urlsOf('audio')),
duration, aspect_ratio, motion, preset_id, enhance_prompt, visual_dna_ids, style_id,
resolution, draft, sound_enabled, keyframes, multi_shots, multi_shot_count,
session_name, project_id, session_id
};
const startElements = async () => {
if (!locals.length) {
return client.post('/v1/generate/elements', body);
}
const resolved = await Promise.all(locals.map(({ src, kind }) =>
resolveInput(src, kind, { maxBytes: ELEMENTS_MAX_UPLOAD_BYTES })));
const form = new FormData();
for (const [key, value] of Object.entries(body)) {
if (value === undefined || value === null) continue;
// The two URL lists are read by DIFFERENT parsers on the API side and
// only one shape satisfies both. reference_videos/_audio_urls go
// through parseJsonArray, which needs a JSON string (a bare url makes
// JSON.parse throw and the list is dropped). reference_images goes
// through collectHttpMediaUrls, which walks strings and arrays but
// never JSON-decodes — so a stringified array fails its
// `^https?://` test and every URL reference is silently lost. Repeated
// form fields arrive as a string (one) or an array (several), and that
// walker handles both.
if (key === 'reference_images') {
for (const url of value) form.append('reference_images', url);
continue;
}
form.append(key, typeof value === 'object' ? JSON.stringify(value) : String(value));
}
for (const f of resolved) {
form.append('files', f.buffer, { filename: f.filename, contentType: f.contentType });
}
return client.postMultipart('/v1/generate/elements', form);
};
let startResponse;
try {
startResponse = await startElements();
} catch (err) {
// The elements trust gate only accepts remote references that are BOTH
// Kolbo-hosted AND registered to this caller's library/project — any
// other public URL 400s with this code, even though every schema here
// says "public URL, forwarded as-is". Agents used to recover by hand
// (upload_media, then resend). Do that detour for them: rehost every
// remote reference into the caller's library and retry ONCE. The gate
// fires before any charge, so the failed first attempt costs nothing.
if (err?.code !== 'UNTRUSTED_REFERENCE_MEDIA_URL') throw err;
for (const kind of ['image', 'video', 'audio']) {
media[kind] = await Promise.all(media[kind].map(async (src) => {
if (!isUrlSource(src)) return src;
const file = await resolveInput(src, kind, { maxBytes: ELEMENTS_MAX_UPLOAD_BYTES });
const form = new FormData();
form.append('file', file.buffer, { filename: file.filename, contentType: file.contentType });
if (project_id) form.append('project_id', project_id);
const uploaded = await client.postMultipart('/v1/media/upload', form);
return uploaded?.media?.url || uploaded?.url || src;
}));
}
body.reference_images = some(urlsOf('image'));
body.reference_videos = some(urlsOf('video'));
body.reference_audio_urls = some(urlsOf('audio'));
startResponse = await startElements();
}
if (returnsImmediately()) return submittedResult({
tool: 'generate_elements', kind: 'video', gen: startResponse, client, model, prompt,
settings: videoSettings({
duration,
reported_duration: startResponse?.duration ?? startResponse?.result?.duration,
// `multi_shots` is a boolean (z.boolean()); the old
// `Array.isArray(multi_shots) ? multi_shots.length : undefined`
// branch could never be true, so enabling multishot WITHOUT an
// explicit count left shots undefined and the card silently showed
// no multishot indicator at all. Carry the flag so the chip can
// render even when the shot count is the provider's to decide.
shots: multi_shot_count,
multi_shot: multi_shots === true || undefined,
resolution, draft, aspect_ratio, enhance_prompt, visual_dna_ids, preset_id,
}),
// Already bucketed and deduped above, so a URL handed to `files`
// renders under its real kind instead of being dropped or double-listed.
reference_images: [...urlsOf('image'), ...(keyframes || []).map((keyframe) => keyframe.image_url)],
reference_videos: urlsOf('video'),
reference_audio: urlsOf('audio')
});
const poll = await pollOrTimedOut(client, startResponse.generation_id, {
interval: (startResponse.poll_interval_hint || 8) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'generate_elements', kind: 'video', gen: startResponse, client, model, prompt,
settings: videoSettings({
duration,
reported_duration: result.result?.duration,
// `multi_shots` is a boolean (z.boolean()); the old
// `Array.isArray(multi_shots) ? multi_shots.length : undefined`
// branch could never be true, so enabling multishot WITHOUT an
// explicit count left shots undefined and the card silently showed
// no multishot indicator at all. Carry the flag so the chip can
// render even when the shot count is the provider's to decide.
shots: multi_shot_count,
multi_shot: multi_shots === true || undefined,
resolution, draft, aspect_ratio, enhance_prompt, visual_dna_ids, preset_id,
}),
reference_images: [...urlsOf('image'), ...(keyframes || []).map((keyframe) => keyframe.image_url)],
reference_videos: urlsOf('video'),
reference_audio: urlsOf('audio'),
urls: result.result?.urls || [],
thumbnail_url: result.result?.thumbnail_url || null,
duration: result.result?.duration || null,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: startResponse.session_id,
urls: result.result?.urls || [],
thumbnail_url: result.result?.thumbnail_url || null,
duration: result.result?.duration || null,
model: result.result?.model || null
}, null, 2));
}
);
// ─── generate_first_last_frame ─────────────────────────────
server.tool(
'generate_first_last_frame',
'Generate a video that morphs / interpolates from a FIRST frame to a LAST frame. Provide the two frames as URLs (first_frame_url + last_frame_url) or as URLs/absolute local paths (first_frame + last_frame) — the pairs are interchangeable and may be mixed. Optional prompt describes the desired motion/transition. Returns the final video URL when complete.',
{
first_frame_url: z.string().optional().describe('Public URL of the first frame image. Interchangeable with first_frame.'),
last_frame_url: z.string().optional().describe('Public URL of the last frame image. Interchangeable with last_frame.'),
first_frame: z.string().optional().describe('URL or absolute local path to the first frame — alternative to first_frame_url. A URL here is forwarded as a URL, not re-uploaded.'),
last_frame: z.string().optional().describe('URL or absolute local path to the last frame — alternative to last_frame_url. A URL here is forwarded as a URL, not re-uploaded.'),
prompt: z.string().optional().describe('Optional description of the desired motion between the two frames (e.g. "smooth camera dolly in")'),
model: z.string().optional().describe('Model identifier. Use list_models type="firstlastgenerations" to see options. Pick a SPECIFIC model — do NOT omit (omitting = Smart Select auto-pick, which we avoid); call list_models for this type and choose the model that best fits the user\'s intent.'),
duration: z.number().optional().describe('Duration in seconds. Must be in `supported_durations` from list_models, OR within `min_output_duration`-`max_output_duration`. Default: 5'),
aspect_ratio: z.string().optional().describe(aspectRatioDescribe('16:9') + ' Auto-detected from the first frame if omitted.'),
enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply. **Cap: pass at most `max_visual_dna` IDs from list_models for the chosen model; if `supports_visual_dna: false`, DNA is silently ignored.**'),
draft: z.boolean().optional().describe('Special Draft mode. True forces 480p Draft even if resolution says 1080p; false selects regular video. Only supported when list_models advertises draft capabilities. For Seedance 2.5 editing use model seedance-2-5-video-to-video with generate_video_from_video; the source duration and aspect ratio are preserved. Uses ordinary credits; finalize through draft_quote then draft_enhance.'),
resolution: z.string().optional().describe('Video resolution or named tier: read supported_resolutions from list_models. For Seedance 2.5 Draft use resolution="480p-draft" on the matching public model: seedance-2-5 for creation, seedance-2-5-video-to-video for editing. Plain "480p" generates regular video, NOT Draft. Finalize an actual draft with edit_video operation="draft_quote" then "draft_enhance".'),
sound_enabled: z.boolean().optional().describe('Enable (`true`) or disable (`false`) AI-generated synced audio on the output video. Honored by `sound_generation_type: "native"` models (Kling O3/V3, Veo 3.1, PixVerse V6). Omit to use `sound_enabled_by_default`. Enabling sound may apply `sound_credit_multiplier` to cost.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ first_frame_url, last_frame_url, first_frame, last_frame, prompt, model, duration, aspect_ratio, enhance_prompt = false, visual_dna_ids, resolution, draft, sound_enabled, project_id, session_id }) => {
if (draft === true) resolution = '480p-draft';
else if (draft === false && resolution?.endsWith('-draft')) resolution = resolution.slice(0, -6);
model = await canonicalModelId(client, model, 'firstlastgenerations'); // lenient id resolution ("z-image" → "z-image/turbo")
aspect_ratio = await resolveCatalogAspectRatio(client, model, aspect_ratio, 'firstlastgenerations');
// One frame per position, whichever arg carried it. The two arg pairs were
// treated as two exclusive MODES, so a URL handed to first_frame/last_frame
// (which their own descriptions invite) took the multipart path and got
// downloaded and re-uploaded — a pointless round-trip that also had to fit
// the route's 10MB upload ceiling a plain URL never touches. Transport is
// now decided per frame by what the value IS, not which arg it arrived in,
// which also makes one-URL-one-local work instead of erroring out.
const firstSource = first_frame_url || first_frame;
const lastSource = last_frame_url || last_frame;
if (!firstSource || !lastSource) {
throw new Error('Provide both frames: first_frame_url + last_frame_url (URLs) or first_frame + last_frame (URL or absolute local path).');
}
let startResponse;
if (!isUrlSource(firstSource) || !isUrlSource(lastSource)) {
const [firstResolved, lastResolved] = await Promise.all([
resolveInput(firstSource, 'image'),
resolveInput(lastSource, 'image')
]);
const form = new FormData();
form.append('files', firstResolved.buffer, { filename: firstResolved.filename, contentType: firstResolved.contentType });
form.append('files', lastResolved.buffer, { filename: lastResolved.filename, contentType: lastResolved.contentType });
if (prompt) form.append('prompt', prompt);
if (model) form.append('model', model);
if (duration !== undefined) form.append('duration', String(duration));
if (aspect_ratio) form.append('aspect_ratio', aspect_ratio);
if (enhance_prompt !== undefined) form.append('enhance_prompt', String(enhance_prompt));
if (visual_dna_ids) form.append('visual_dna_ids', JSON.stringify(visual_dna_ids));
if (resolution) form.append('resolution', resolution);
if (draft !== undefined) form.append('draft', String(draft));
if (sound_enabled !== undefined) form.append('sound_enabled', String(sound_enabled));
if (project_id) form.append('project_id', project_id);
if (session_id) form.append('session_id', session_id);
startResponse = await client.postMultipart('/v1/generate/first-last-frame', form);
} else {
startResponse = await client.post('/v1/generate/first-last-frame', {
first_frame_url: firstSource, last_frame_url: lastSource,
prompt, model, duration, aspect_ratio, enhance_prompt, visual_dna_ids, resolution, draft, sound_enabled, project_id, session_id
});
}
if (returnsImmediately()) return submittedResult({
tool: 'generate_first_last_frame', kind: 'video', gen: startResponse, client, model, prompt,
settings: videoSettings({ duration, resolution, draft, aspect_ratio, enhance_prompt, visual_dna_ids }),
reference_images: [firstSource, lastSource].filter(isUrlSource)
});
const poll = await pollOrTimedOut(client, startResponse.generation_id, {
interval: (startResponse.poll_interval_hint || 8) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'generate_first_last_frame', kind: 'video', gen: startResponse, client, model, prompt,
settings: videoSettings({ duration, resolution, draft, aspect_ratio, enhance_prompt, visual_dna_ids }),
reference_images: [firstSource, lastSource].filter(isUrlSource),
urls: result.result?.urls || [],
thumbnail_url: result.result?.thumbnail_url || null,
duration: result.result?.duration || null,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: startResponse.session_id,
urls: result.result?.urls || [],
thumbnail_url: result.result?.thumbnail_url || null,
duration: result.result?.duration || null,
model: result.result?.model || null
}, null, 2));
}
);
// ─── generate_lipsync ──────────────────────────────────────
server.tool(
'generate_lipsync',
'Lipsync an audio track to a source image or video. SCOPE: it syncs an EXISTING audio waveform to a face in footage you already have (dubbing, translation, fixing a delivered take). It is NOT how you get characters to speak in a film you are generating — Seedance 2/2.5 perform quoted dialogue natively with synced lip movement, so a scene generated there needs no lipsync pass. Both `source` (image or video) and `audio` can be provided as URLs or as absolute local file paths. Pass a text_prompt only if the model supports it (some lipsync models do character performance from a prompt). **Validate before submitting: for `lipsync-video` sources, the input video duration must fall within `min_video_duration`-`max_video_duration` from list_models; audio duration must fall within `min_audio_duration`-`max_audio_duration` (and if `audio_max_follows_video_duration: true`, audio is also capped at the video duration); audio format must be in `supported_audio_formats` when specified.** Returns a lipsynced video URL.',
{
source: z.string().describe('URL or absolute local path to the source image or video (the face to animate). For lipsync-video: duration must fall within `min_video_duration`-`max_video_duration` from list_models.'),
audio: z.string().describe('URL or absolute local path to the audio track (the voice to sync to). Duration must fall within `min_audio_duration`-`max_audio_duration` from list_models; format must be in `supported_audio_formats` (when set).'),
text_prompt: z.string().optional().describe('Optional text prompt (for performance-capable models). For Sync-3 this is the free-text emotion/acting prompt, e.g. "speaking with excitement, calm and serious".'),
model: z.string().optional().describe('Model identifier. Use list_models type="lipsync-image" or type="lipsync-video" to see options. Pick a SPECIFIC model — do NOT omit (omitting = Smart Select auto-pick, which we avoid); call list_models for this type and choose the model that best fits the user\'s intent.'),
bounding_box_target: z.array(z.number()).optional().describe('Optional bounding box [x, y, w, h] for multi-face inputs (Hedra Character3 style). Leave empty for single-face.'),
// Sync-3 (fal-ai/sync-lipsync/v3) only; ignored by other models.
sync_mode: z.enum(['cut_off', 'loop', 'bounce', 'silence', 'remap']).optional().describe('Sync-3 / sync-lipsync family: how to reconcile an audio/video length mismatch. Default cut_off.'),
model_mode: z.enum(['lips', 'face', 'head', 'lipsync', 'emotion', 'talking_head']).optional().describe('Sync-3 only: which region drives the sync.'),
emotion: z.enum(['neutral', 'happy', 'sad', 'angry', 'disgusted', 'surprised']).optional().describe('Sync-3 only: quick emotion shortcut. A free-text text_prompt overrides this and gives finer control.'),
temperature: z.number().min(0).max(1).optional().describe('Sync-3 only: expressiveness 0 (subtle) .. 1 (energetic).'),
occlusion_detection_enabled: z.boolean().optional().describe('Sync-3 only: handle objects passing in front of the face.'),
active_speaker_detection: z.object({
auto_detect: z.boolean().optional().describe('Auto-detect and sync the active speaker.'),
v3: z.boolean().optional().describe('Use Sync.so v3 detection engine.'),
frame_number: z.number().int().min(0).optional().describe('Frame index the coordinates refer to.'),
coordinates: z.array(z.number().int()).length(2).optional().describe('[x, y] PIXEL point on the speaker face (source-video resolution).'),
bounding_boxes: z.array(z.array(z.number().int())).optional().describe('Per-frame face boxes [x1,y1,x2,y2].'),
bounding_boxes_url: z.string().optional().describe('URL to a JSON file with per-frame boxes.'),
face_image: z.string().optional().describe('Base64-encoded reference face image.')
}).optional().describe('Sync-3 only: choose which speaker gets synced in a multi-person video. Use auto_detect:true for automatic, or coordinates + frame_number to pin a specific face.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ source, audio, text_prompt, model, bounding_box_target, sync_mode, model_mode, emotion, temperature, occlusion_detection_enabled, active_speaker_detection, project_id, session_id }) => {
model = await canonicalModelId(client, model, ['lipsync-image', 'lipsync-video']); // lenient id resolution ("z-image" → "z-image/turbo")
if (!source) throw new Error('source is required (URL or absolute local path to image/video)');
if (!audio) throw new Error('audio is required (URL or absolute local path to audio file)');
const sourceIsUrl = typeof source === 'string' && /^https?:\/\//i.test(source);
const audioIsUrl = typeof audio === 'string' && /^https?:\/\//i.test(audio);
let startResponse;
if (sourceIsUrl && audioIsUrl) {
// URL mode
startResponse = await client.post('/v1/generate/lipsync', {
source_url: source,
audio_url: audio,
prompt: text_prompt,
model,
bounding_box_target,
// Sync-3 advanced options (additive; ignored by other models)
sync_mode,
model_mode,
emotion,
temperature,
occlusion_detection_enabled,
active_speaker_detection,
project_id, session_id
});
} else {
// File mode (or mixed — resolve any local paths, pass URLs through as body fields)
const form = new FormData();
if (!sourceIsUrl) {
const resolved = await resolveInput(source, /\.(mp4|mov|webm|mkv)$/i.test(source) ? 'video' : 'image');
// Decide field name by kind — lipsync controller uses .fields() with image/video/audio.
const isVideo = /\.(mp4|mov|webm|mkv|avi|m4v)$/i.test(resolved.filename);
form.append(isVideo ? 'video' : 'image', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
} else {
form.append('source_url', source);
}
if (!audioIsUrl) {
const resolved = await resolveInput(audio, 'audio');
form.append('audio', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
} else {
form.append('audio_url', audio);
}
if (text_prompt) form.append('prompt', text_prompt);
if (model) form.append('model', model);
if (bounding_box_target) form.append('bounding_box_target', JSON.stringify(bounding_box_target));
// Sync-3 advanced options (additive — ignored by other models)
if (sync_mode) form.append('sync_mode', sync_mode);
if (model_mode) form.append('model_mode', model_mode);
if (emotion) form.append('emotion', emotion);
if (temperature !== undefined) form.append('temperature', String(temperature));
if (occlusion_detection_enabled !== undefined) form.append('occlusion_detection_enabled', String(occlusion_detection_enabled));
if (active_speaker_detection) form.append('active_speaker_detection', JSON.stringify(active_speaker_detection));
if (project_id) form.append('project_id', project_id);
if (session_id) form.append('session_id', session_id);
startResponse = await client.postMultipart('/v1/generate/lipsync', form);
}
if (returnsImmediately()) return submittedResult({
tool: 'generate_lipsync', kind: 'video', gen: startResponse, client, model,
prompt: text_prompt, settings: { mode: 'lipsync' },
reference_images: sourceIsUrl && !/\.(mp4|mov|webm|mkv|avi|m4v)(\?|$)/i.test(source) ? [source] : [],
});
const poll = await pollOrTimedOut(client, startResponse.generation_id, {
interval: (startResponse.poll_interval_hint || 8) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'generate_lipsync', kind: 'video', gen: startResponse, client, model,
prompt: text_prompt, settings: { mode: 'lipsync' },
reference_images: sourceIsUrl && !/\.(mp4|mov|webm|mkv|avi|m4v)(\?|$)/i.test(source) ? [source] : [],
reference_videos: sourceIsUrl && /\.(mp4|mov|webm|mkv|avi|m4v)(\?|$)/i.test(source) ? [source] : [],
urls: result.result?.urls || [],
thumbnail_url: result.result?.thumbnail_url || null,
duration: result.result?.duration || null,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: startResponse.session_id,
urls: result.result?.urls || [],
thumbnail_url: result.result?.thumbnail_url || null,
duration: result.result?.duration || null,
model: result.result?.model || null
}, null, 2));
}
);
// ─── generate_video_from_video ─────────────────────────────
server.tool(
'generate_video_from_video',
'Restyle / transform an existing video (video-to-video). Use for style transfer, scene restyling, subject swap, motion transfer, character replacement, or burning in styled subtitles (VEED Subtitles). Source video can be a URL or absolute local path. `prompt` is OPTIONAL: most models need it, but prompt-less models (VEED Subtitles, Act Two, Wan Animate, Kling Motion Control) ignore it. For VEED Subtitles, pass a `preset` style and optional `source_language` / `translation_language` instead of a prompt. IMPORTANT: different models support different extra inputs — call list_models type="video_to_video" and read max_images / max_videos / max_elements on the chosen model before generating. Pass reference_images for models with max_images > 0 (e.g. Kling O1/O3, Aleph, WAN VACE), reference_videos for models with max_videos > 1 (e.g. WAN 2.6 reference-to-video accepts up to 3), and elements for models with max_elements > 0. REPAIR / RETIME (not restyling, no prompt needed — these keep the footage and fix or retime it): model "topaz/deblur/video" removes lens, motion and compression blur; "topaz/colorize/video" colorizes black-and-white footage; "topaz/interpolate/video" is SLOW MOTION and frame-rate conversion (see slowdown_factor / target_fps); "topaz/sdr-to-hdr/video" masters SDR footage to HDR (see output_format). Reach for these when the user says blurry, shaky-detail, black-and-white, slow motion, smoother frame rate, or HDR — a restyle model would repaint the video instead of repairing it. For animating a still image use generate_video_from_image instead. For text-only → video use generate_video.' + MORPHIOUS_GUIDE,
{
source_video: z.string().describe('URL or absolute local path to the primary source video to restyle. **Source duration must fall within `min_video_duration`-`max_video_duration` from list_models for the chosen model** — videos outside that range are rejected (or silently truncated by some upstream providers). For models that use reference_videos as their primary input (e.g. WAN 2.6 reference-to-video), pass the first reference video here and also include it in reference_videos.'),
prompt: z.string().optional().describe('Text description of the desired restyle / transformation. Required by most video-to-video models; omit for prompt-less models (VEED Subtitles, Act Two, Wan Animate, Kling Motion Control, kolbo-morphious-*, higgsfield-genjutsu-motion-transfer, higgsfield-genjutsu-object-swap).'),
model: z.string().optional().describe('Model identifier. Use list_models type="video_to_video" to see options and check max_images / max_videos / max_elements / max_video_duration per model. Pick a SPECIFIC model — do NOT omit (omitting = Smart Select auto-pick, which we avoid); call list_models for this type and choose the model that best fits the user\'s intent.'),
aspect_ratio: z.string().optional().describe(aspectRatioDescribe() + ' Default: matches source.'),
duration: z.number().optional().describe('Output duration in seconds. Must be in `supported_durations` from list_models, OR within `min_output_duration`-`max_output_duration`. Default: matches source'),
enhance_prompt: z.boolean().optional().describe('Enhance the prompt. Default: false — only pass true if the user explicitly asks to enhance/improve the prompt.'),
visual_dna_ids: z.array(z.string()).optional().describe('Array of Visual DNA profile IDs to apply for character/style consistency. **Cap: pass at most `max_visual_dna` IDs from list_models for the chosen model; if `supports_visual_dna: false`, DNA is silently ignored.**'),
style_id: z.string().optional().describe('Morphious Styles only (kolbo-morphious-motion or kolbo-morphious-swap; Standard only, the Lite modes refuse a style): id of a look from list_morphious_styles (Kolbo catalog or a custom style the user made). Restyles the WHOLE video in that look; the style counts as reference image 1, so no reference_images are needed. Prompt optional.'),
draft: z.boolean().optional().describe('Special Draft mode. True forces 480p Draft even if resolution says 1080p; false selects regular video. Only supported when list_models advertises draft capabilities. For Seedance 2.5 editing use model seedance-2-5-video-to-video with generate_video_from_video; the source duration and aspect ratio are preserved. Uses ordinary credits; finalize through draft_quote then draft_enhance.'),
resolution: z.string().optional().describe('Video resolution or named tier: read supported_resolutions from list_models. For Seedance 2.5 Draft use resolution="480p-draft" on the matching public model: seedance-2-5 for creation, seedance-2-5-video-to-video for editing. Plain "480p" generates regular video, NOT Draft. Finalize an actual draft with edit_video operation="draft_quote" then "draft_enhance".'),
reference_images: z.array(z.string()).optional().describe('Array of reference image URLs for models that support additional image inputs. **Cap: pass at most `max_images` URLs from list_models — if `max_images === 0` the model does not accept image refs.** Examples: character reference images for Kling O1/O3, style reference for Aleph/gen4_aleph, character image for WAN VACE video-edit.'),
reference_videos: z.array(z.string()).optional().describe('Array of additional reference video URLs for models that support multiple video inputs. **Cap: pass at most `max_videos` URLs from list_models — if `max_videos <= 1` only the source_video is accepted.** Example: WAN 2.6 reference-to-video accepts 1–3 reference videos.'),
elements: z.array(z.string()).optional().describe('Array of element image URLs. **Cap: pass at most `max_elements` URLs from list_models — if `max_elements === 0` the model does not accept elements.** Elements are style or character reference assets alongside the main video.'),
sound_enabled: z.boolean().optional().describe('Enable (`true`) or disable (`false`) AI-generated synced audio on the output video. Honored by `sound_generation_type: "native"` models (Kling O3/V3, Veo 3.1, PixVerse V6). Omit to use `sound_enabled_by_default`. Enabling sound may apply `sound_credit_multiplier` to cost.'),
// VEED Subtitles (model: veed/subtitles) — burns styled subtitles into the video
preset: z.string().optional().describe('VEED Subtitles only: caption style preset (e.g. "glass", "whisper", "fusion", "simple", "vegas"). Call list_models type="video_to_video" for the veed/subtitles model. Ignored by other models.'),
source_language: z.string().optional().describe('VEED Subtitles only: BCP-47 code of the spoken language to improve transcription accuracy (e.g. "en-US", "es-ES", "he-IL"). Omit to auto-detect.'),
translation_language: z.string().optional().describe('VEED Subtitles only: BCP-47 code to translate the subtitles into (e.g. "en-US", "fr-FR"). Omit to keep the original spoken language.'),
srt_content: z.string().optional().describe('VEED Subtitles only: raw .srt subtitle text to burn in. When set, auto-transcription is skipped.'),
srt_file_url: z.string().optional().describe('VEED Subtitles only: URL to a .srt subtitle file. Alternative to srt_content. When set, auto-transcription is skipped.'),
vocabulary: z.array(z.object({
word: z.string().describe('Correct spelling to enforce'),
replaces: z.array(z.string()).describe('Mis-transcriptions to replace with `word`'),
})).optional().describe('VEED Subtitles only: brand names / jargon to help transcription (e.g. [{"word":"Kolbo","replaces":["colbo","kolboo"]}]). Ignored when srt_content / srt_file_url is set.'),
customization: z.object({
position: z.enum(['top', 'center', 'bottom']).optional().describe('Caption vertical position. Ignored by complex animated presets.'),
shadow: z.enum(['none', 'min', 'mid', 'max']).optional().describe('Text shadow intensity.'),
text_customizations: z.object({
baseline: z.object({ font: z.string().optional(), weight: z.number().int().min(100).max(900).optional(), color: z.string().optional() }).optional().describe('All words: Google font name, weight 100-900, hex colour.'),
highlighted: z.object({ font: z.string().optional(), weight: z.number().int().min(100).max(900).optional(), color: z.string().optional() }).optional().describe('Highlighted word tier styling.'),
}).optional(),
}).optional().describe('VEED Subtitles only: style overrides. Any omitted field keeps the preset default. Best supported by Basic presets.'),
enhancement_model: z.string().optional()
.describe('Topaz Slow Motion only (model "topaz/interpolate/video"): retiming engine — "Apollo" (smooth motion, default), "Chronos" (complex motion and occlusion), or "Aion" (highest quality for extreme slow motion, and the most expensive).'),
target_fps: z.number().optional()
.describe('Topaz Slow Motion only: frames per second of the OUTPUT (16-120, default 60). Higher rates generate more frames and cost proportionally more.'),
slowdown_factor: z.number().optional()
.describe('Topaz Slow Motion only: how many times longer the output runs (1-8, default 1). 4 turns a 5s clip into 20s of slow motion. Billing is on the OUTPUT length, so an 8x pass costs 8x a 1x pass.'),
output_format: z.string().optional()
.describe('Topaz HDR only (model "topaz/sdr-to-hdr/video"): "mp4" for 10-bit H.265 HDR10 (default) or "prores" for 10-bit ProRes 422 HQ. Both are HDR masters — the in-app player shows a tone-mapped SDR preview and the HDR file is the download.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ source_video, prompt, model, aspect_ratio, duration, enhance_prompt = false, visual_dna_ids, style_id, resolution, draft, sound_enabled, reference_images, reference_videos, elements, preset, source_language, translation_language, srt_content, srt_file_url, vocabulary, customization, enhancement_model, target_fps, slowdown_factor, output_format, project_id, session_id }) => {
if (draft === true) resolution = '480p-draft';
else if (draft === false && resolution?.endsWith('-draft')) resolution = resolution.slice(0, -6);
model = await canonicalModelId(client, model, 'video_to_video'); // lenient id resolution ("z-image" → "z-image/turbo")
aspect_ratio = await resolveCatalogAspectRatio(client, model, aspect_ratio, 'video_to_video');
if (!source_video) throw new Error('source_video is required');
const isUrl = /^https?:\/\//i.test(source_video);
let startResponse;
if (isUrl) {
startResponse = await client.post('/v1/generate/video-from-video', {
video_url: source_video, prompt, model, aspect_ratio, duration, enhance_prompt, visual_dna_ids, style_id, resolution, draft, sound_enabled,
reference_images, reference_videos, elements, preset, source_language, translation_language,
srt_content, srt_file_url, vocabulary, customization,
enhancement_model, target_fps, slowdown_factor, output_format,
project_id, session_id
});
} else {
const resolved = await resolveInput(source_video, 'video');
const form = new FormData();
form.append('files', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
if (prompt) form.append('prompt', prompt);
if (preset) form.append('preset', preset);
if (source_language) form.append('source_language', source_language);
if (translation_language) form.append('translation_language', translation_language);
if (srt_content) form.append('srt_content', srt_content);
if (srt_file_url) form.append('srt_file_url', srt_file_url);
if (vocabulary) form.append('vocabulary', JSON.stringify(vocabulary));
if (customization) form.append('customization', JSON.stringify(customization));
if (model) form.append('model', model);
if (aspect_ratio) form.append('aspect_ratio', aspect_ratio);
if (duration !== undefined) form.append('duration', String(duration));
if (enhance_prompt !== undefined) form.append('enhance_prompt', String(enhance_prompt));
if (visual_dna_ids) form.append('visual_dna_ids', JSON.stringify(visual_dna_ids));
if (style_id) form.append('style_id', style_id);
if (resolution) form.append('resolution', resolution);
if (draft !== undefined) form.append('draft', String(draft));
if (sound_enabled !== undefined) form.append('sound_enabled', String(sound_enabled));
if (reference_images) form.append('reference_images', JSON.stringify(reference_images));
if (reference_videos) form.append('reference_videos', JSON.stringify(reference_videos));
if (elements) form.append('elements', JSON.stringify(elements));
if (enhancement_model) form.append('enhancement_model', enhancement_model);
if (target_fps !== undefined) form.append('target_fps', String(target_fps));
if (slowdown_factor !== undefined) form.append('slowdown_factor', String(slowdown_factor));
if (output_format) form.append('output_format', output_format);
if (project_id) form.append('project_id', project_id);
if (session_id) form.append('session_id', session_id);
startResponse = await client.postMultipart('/v1/generate/video-from-video', form);
}
if (returnsImmediately()) return submittedResult({
tool: 'generate_video_from_video', kind: 'video', gen: startResponse, client, model,
prompt: prompt || (preset ? `Subtitles preset: ${preset}` : undefined),
settings: videoSettings({ duration, resolution, draft, aspect_ratio, mode: preset ? 'subtitles' : 'restyle', enhance_prompt, visual_dna_ids }),
reference_images: [...(reference_images || []), ...(elements || [])],
// The source clip IS the primary reference for a restyle — showing the
// extra references while hiding the video being transformed was backwards.
reference_videos: [
...(isUrl ? [source_video] : []),
...(reference_videos || [])
]
});
const poll = await pollOrTimedOut(client, startResponse.generation_id, {
interval: (startResponse.poll_interval_hint || 8) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'generate_video_from_video', kind: 'video', gen: startResponse, client, model,
prompt: prompt || (preset ? `Subtitles preset: ${preset}` : undefined),
settings: videoSettings({ duration, resolution, draft, aspect_ratio, mode: preset ? 'subtitles' : 'restyle', enhance_prompt, visual_dna_ids }),
reference_images: [...(reference_images || []), ...(elements || [])],
reference_videos: [
...(isUrl ? [source_video] : []),
...(reference_videos || [])
],
urls: result.result?.urls || [],
thumbnail_url: result.result?.thumbnail_url || null,
duration: result.result?.duration || null,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: startResponse.session_id,
urls: result.result?.urls || [],
thumbnail_url: result.result?.thumbnail_url || null,
duration: result.result?.duration || null,
model: result.result?.model || null
}, null, 2));
}
);
// ─── transcribe_audio ──────────────────────────────────────
server.tool(
'transcribe_audio',
'Transcribe audio or video into text + SRT subtitles. Source can be a URL or an absolute local file path. Returns the full text, SRT content, duration, and download URLs for .srt/.txt files. Works on both audio-only files (mp3, wav, m4a) and videos with audio tracks (mp4, mov, webm). Supports language selection, speaker diarization, audio-event tagging, and SRT subtitle formatting controls.',
{
source: z.string().describe('URL or absolute local path to the audio / video file to transcribe'),
language: z.string().optional().describe('Language code of the speech (e.g. "en", "he", "es"). Omit to auto-detect.'),
diarize: z.boolean().optional().describe('Detect and label distinct speakers. Default: false.'),
tag_audio_events: z.boolean().optional().describe('Tag non-speech audio events (laughter, applause, music) in the transcript. Default: false.'),
remove_punctuation: z.boolean().optional().describe('Strip punctuation from the transcript. Default: false.'),
generate_srt: z.boolean().optional().describe('Produce SRT + word-by-word SRT subtitle files. Default: true.'),
words_per_line: z.number().optional().describe('SRT: max words per subtitle line, 1–18. Default: 12.'),
lines_per_subtitle: z.number().optional().describe('SRT: max lines per subtitle cue, 1–4. Default: 2.'),
stretch_captions: z.boolean().optional().describe('SRT: extend each cue\'s end time to the next cue\'s start (gap-free subtitles). Default: true.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({ source, language, diarize, tag_audio_events, remove_punctuation, generate_srt, words_per_line, lines_per_subtitle, stretch_captions, project_id, session_id }) => {
if (!source) throw new Error('source is required (URL or absolute local path)');
// Advanced transcription controls forwarded when provided (undefined keys are dropped by the client).
const opts = {
language, diarize, tag_audio_events, remove_punctuation,
generate_srt, words_per_line, lines_per_subtitle, stretch_captions, project_id, session_id
};
const isUrl = /^https?:\/\//i.test(source);
let startResponse;
if (isUrl) {
startResponse = await client.post('/v1/transcribe', { audio_url: source, ...opts });
} else {
const resolved = await resolveInput(source, 'audio');
const form = new FormData();
form.append('file', resolved.buffer, { filename: resolved.filename, contentType: resolved.contentType });
for (const [k, v] of Object.entries(opts)) {
if (v !== undefined && v !== null) form.append(k, typeof v === 'boolean' ? String(v) : v);
}
startResponse = await client.postMultipart('/v1/transcribe', form);
}
if (ui()) {
return uiResult(UI.transcript, JSON.stringify({
status: 'submitted',
generation_id: startResponse.generation_id,
_widget_note: 'A live Kolbo transcription widget is rendering above — it shows progress, the transcript text, and SRT/TXT download buttons. Tell the user it is transcribing. If you need the transcript text for a follow-up step, call get_generation_status ONCE with this generation_id and wait=true — it blocks until done; do NOT poll in a loop.',
}, null, 2), {
widget: 'transcript', phase: 'generating',
generation_id: startResponse.generation_id,
poll_tool: 'get_generation_status',
status_args: { generation_id: startResponse.generation_id, wait: true },
audio_url: isUrl ? source : undefined,
...linkFields('transcribe_audio', startResponse),
});
}
if (options.asyncGenerations) {
return asyncGenerating({
tool: 'transcribe_audio',
kind: 'audio',
gen: startResponse,
});
}
const poll = await pollOrTimedOut(client, startResponse.generation_id, {
interval: (startResponse.poll_interval_hint || 5) * 1000,
timeout: 150000 // Return before host cutoff; LLM can call get_generation_status with wait=true for long podcasts
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return {
content: [{
type: 'text',
text: JSON.stringify({
...creditFields(result),
session_id: startResponse.session_id,
text: result.result?.text || '',
srt_url: result.result?.srt_url || null,
word_by_word_srt_url: result.result?.word_by_word_srt_url || null,
txt_url: result.result?.txt_url || null,
duration: result.result?.duration || null
}, null, 2)
}]
};
}
);
// ─── generate_3d ───────────────────────────────────────────
server.tool(
'generate_3d',
'Generate a 3D model from a text prompt, a single reference image, or multiple reference images (for multi-view reconstruction). Returns model URLs in multiple formats (GLB, FBX, OBJ, USDZ). Modes: "text" (prompt-only), "single" (one image), "multi" (multiple images for better quality). The mode is auto-detected from the inputs if not specified. Three model families with family-scoped settings: Meshy V7 (mesh/texture controls should_remesh/should_texture/texture_image_url/symmetry_mode/enable_safety_checker, plus auto-rigging + animation: enable_rigging rigs a humanoid character with basic walk/run animation, enable_animation applies one of ~697 presets on top — requires enable_rigging; both cost extra credits), Trellis v1 (texture_size, guidance/sampling steps, mesh_simplify, multiimage_algo), Trellis 2 (resolution, t2_texture_size, decimation_target, remesh, tex_sampling_steps). Params for a different family than the selected model are ignored.',
{
prompt: z.string().optional().describe('Text description of the 3D object to generate (used in text mode and also as a hint in image modes)'),
reference_images: z.array(z.string()).optional().describe('Array of image URLs or absolute local paths. 1 image → single mode, 2+ → multi mode.'),
mode: z.string().optional().describe('Explicitly set mode: "text" | "single" | "multi". Auto-detected from reference_images if omitted.'),
texture_prompt: z.string().optional().describe('Optional prompt to guide texture generation'),
model: z.string().optional().describe('Model identifier. Use list_models type="three_d" to see all 3D options, or filter by sub-type: "3d_text_to_model", "3d_image_to_model", "3d_multi_image_to_model", "3d_world".'),
topology: z.string().optional().describe('Topology preset (optional, model-specific)'),
target_polycount: z.number().optional().describe('Target polygon count (optional, model-specific)'),
enable_tpose: z.boolean().optional().describe('Force T-pose for character models (optional)'),
enable_pbr: z.boolean().optional().describe('Enable PBR textures (optional). Requires should_texture.'),
symmetry_mode: z.string().optional().describe('Symmetry control: "off" | "auto" | "on" (Meshy V7 only, default "auto")'),
should_remesh: z.boolean().optional().describe('Rebuild the mesh with clean game-ready topology (Meshy V7 only, default true). False keeps the raw reconstructed mesh.'),
should_texture: z.boolean().optional().describe('Generate textures (Meshy V7 only, default true). False produces an untextured mesh at a lower credit cost; disables enable_pbr/texture_prompt/texture_image_url.'),
texture_image_url: z.string().optional().describe('URL of a 2D image to guide texturing (Meshy V7 only). Requires should_texture.'),
enable_safety_checker: z.boolean().optional().describe('Screen input images before generation (Meshy V7 only, default true)'),
enable_rigging: z.boolean().optional().describe('Auto-rig the model as a humanoid character with basic walk/run animations (Meshy V7 only, default false). Best results with clearly defined limbs. Costs extra credits.'),
rigging_height_meters: z.number().optional().describe('Approximate character height in meters (Meshy V7 only, default 1.7). Only used when enable_rigging is true.'),
enable_animation: z.boolean().optional().describe('Apply an animation preset to the rigged model (Meshy V7 only, default false). Requires enable_rigging. Costs extra credits on top of rigging.'),
animation_action_id: z.number().optional().describe('Animation preset ID, 0-696, default 92 ("Idle") (Meshy V7 only). Only used when enable_animation is true. See docs.meshy.ai/en/api/animation-library for the preset catalog.'),
art_style: z.string().optional().describe('"realistic" | "sculpture" (Meshy text mode only). Sculpture disables PBR.'),
enable_prompt_expansion: z.boolean().optional().describe('AI-expand the prompt before generation (Meshy text mode only)'),
seed: z.number().optional().describe('Seed for reproducibility (Trellis models only)'),
texture_size: z.string().optional().describe('Texture resolution "512" | "1024" | "2048" (Trellis v1 only)'),
resolution: z.string().optional().describe('Generation resolution "512" | "1024" | "1536" (Trellis 2 only)'),
ss_guidance_strength: z.number().optional().describe('Sparse-structure guidance strength (Trellis only)'),
ss_sampling_steps: z.number().optional().describe('Sparse-structure sampling steps (Trellis only) — more steps = higher quality, slower'),
slat_guidance_strength: z.number().optional().describe('Structured-latent guidance strength (Trellis v1 only)'),
slat_sampling_steps: z.number().optional().describe('Structured-latent sampling steps (Trellis only)'),
mesh_simplify: z.number().optional().describe('Mesh simplification ratio (Trellis v1 only)'),
multiimage_algo: z.string().optional().describe('"stochastic" | "multidiffusion" — multi-image fusion algorithm (Trellis v1 multi mode only)'),
decimation_target: z.number().optional().describe('Target polygon count after decimation (Trellis 2 only)'),
remesh: z.boolean().optional().describe('Remesh with projection (Trellis 2 only, default true)'),
tex_sampling_steps: z.number().optional().describe('Texture sampling steps (Trellis 2 only)'),
t2_texture_size: z.string().optional().describe('Texture resolution "1024" | "2048" | "4096" (Trellis 2 only)'),
project_id: projectIdField
},
async ({ prompt, reference_images, mode, texture_prompt, model, topology, target_polycount, enable_tpose, enable_pbr, symmetry_mode, should_remesh, should_texture, texture_image_url, enable_safety_checker, enable_rigging, rigging_height_meters, enable_animation, animation_action_id, art_style, enable_prompt_expansion, seed, texture_size, resolution, ss_guidance_strength, ss_sampling_steps, slat_guidance_strength, slat_sampling_steps, mesh_simplify, multiimage_algo, decimation_target, remesh, tex_sampling_steps, t2_texture_size, project_id }) => {
model = await canonicalModelId(client, model, ['3d_text_to_model', '3d_image_to_model', '3d_multi_image_to_model', '3d_world']); // lenient id resolution ("z-image" → "z-image/turbo")
if (!prompt && !(reference_images && reference_images.length > 0)) {
throw new Error('Provide prompt (text mode) or reference_images (single/multi mode)');
}
const startResponse = await client.post('/v1/generate/3d', {
mode,
prompt,
reference_images,
texture_prompt,
model,
topology,
target_polycount,
enable_tpose,
enable_pbr,
symmetry_mode,
should_remesh,
should_texture,
texture_image_url,
enable_safety_checker,
enable_rigging,
rigging_height_meters,
enable_animation,
animation_action_id,
art_style,
enable_prompt_expansion,
seed,
texture_size,
resolution,
ss_guidance_strength,
ss_sampling_steps,
slat_guidance_strength,
slat_sampling_steps,
mesh_simplify,
multiimage_algo,
decimation_target,
remesh,
tex_sampling_steps,
t2_texture_size,
project_id
});
if (returnsImmediately()) return submittedResult({
tool: 'generate_3d', kind: '3d', gen: startResponse, client, model, prompt,
settings: { mode: mode || (reference_images?.length > 1 ? 'multi' : reference_images?.length === 1 ? 'single' : 'text') },
reference_images
});
const poll = await pollOrTimedOut(client, startResponse.generation_id, {
interval: (startResponse.poll_interval_hint || 8) * 1000,
timeout: 150000 // Return before host cutoff; LLM can call get_generation_status with wait=true for slow 3D jobs
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'generate_3d', kind: '3d', gen: startResponse, client, model, prompt,
settings: { mode: mode || (reference_images?.length > 1 ? 'multi' : reference_images?.length === 1 ? 'single' : 'text') },
reference_images,
urls: result.result?.urls || [],
thumbnail_url: result.result?.thumbnail_url || null,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
urls: result.result?.urls || [],
thumbnail_url: result.result?.thumbnail_url || null,
mode: result.result?.mode || null,
prompt_used: result.result?.prompt_used || null
}, null, 2));
}
);
// ─── edit_image ────────────────────────────────────────────
server.tool(
'edit_image',
'Apply a targeted AI edit to an existing image. Covers mechanical enhancements (upscale, remove background, skin retouching) AND creative operations (expand/outpaint, reframe, inpaint, erase, face swap, background replace, camera angle, multi-shot grid, split/upscale). ⚠️ To EXPAND an image / add space / widen it to a new aspect ratio while keeping the existing artwork intact, use operation="zoom_out" — NOT "reframe" (reframe re-generates the whole picture). ⚠️ For open-ended PROMPT-DRIVEN content edits — "make it night", restyling, adding/removing objects — use `generate_image_edit` instead; it runs on stronger dedicated editing models and produces better results.',
{
image_url: z.string().describe('URL or absolute local path of the primary source image to edit.'),
operation: z.enum([
'upscale', 'clarity_upscale',
'reframe', 'zoom_out',
'removebg', 'background_replace',
'enhance_skin',
'inpaint', 'erase',
'face_swap',
'camera_angle',
'split', 'split_upscale',
'multi_shot',
'magic_edit',
'enhance'
]).describe([
'Edit operation:',
'"upscale" — increase resolution by 2×, 3×, or 4× (use `scale`). "clarity_upscale" — AI-powered clarity upscale with detail enhancement (use `resolution`).',
'"zoom_out" — THE default way to expand / extend / outpaint / uncrop / widen an image, or to grow it into a wider or taller frame. The original pixels are PRESERVED and only the new area around them is generated. Expand uniformly with `zoom_out_percentage`, or in one direction with `expand_left` / `expand_right` / `expand_top` / `expand_bottom` (pixels — this is how you hit a target aspect ratio: read the source width/height, then add the missing pixels on the side you want). `prompt` describes what should appear in the NEW space only. Note: `aspect_ratio` is ignored by this operation.',
'"reframe" — RE-GENERATES the entire picture at a new aspect ratio (Luma Photon). The subject is re-imagined, not preserved — expect a different-looking image. Use ONLY when the user explicitly wants the shot re-taken in another format. If they said "expand", "extend", "widen", "uncrop", "add space", "fill the sides", or "keep it the same but 16:9", they want "zoom_out" instead. Requires `aspect_ratio`.',
'"removebg" — remove the image background, output is transparent PNG.',
'"background_replace" — remove background and replace it with AI-generated content from `prompt`.',
'"enhance_skin" — portrait skin retouching (use `skin_strength`: "subtle" | "realistic" | "pimple" | "freckle").',
'"enhance" — Topaz photo correction at the SOURCE resolution (no resizing). Pick the tool with `model`: "topaz/adjust/image" (exposure, white balance, or colorizing a black-and-white photo), "topaz/sharpen/image" (lens / motion / portrait / wildlife blur, or Super Focus for severely blurred shots), "topaz/denoise/image" (high-ISO and night noise), "topaz/restore/image" (old or damaged photos, dust and scratches). Choose the specific engine with `enhancement_model` — call list_models type="graphics_enhance" to see each model\'s engines. To make an image BIGGER use "upscale" instead.',
'"inpaint" — paint over a masked area using `mask_image_url` (B&W mask, white = fill area) and optional `prompt`. Add reference images via `additional_images`.',
'"erase" — erase an object defined by `mask_image_url` (white = erase area).',
'"face_swap" — swap the face in `image_url` with the face from `mask_image_url` (required).',
'"camera_angle" — generate the image from a different camera angle. Set `generate_all_angles=true` for a full set. Use `prompt` to guide the angle.',
'"split" — split the image into a 3×3 grid of tiles. "split_upscale" — split into a grid and upscale each tile.',
'"multi_shot" — generate a 3×3 multi-shot grid of scenes (uses `additional_images` as reference shots). Use `resolution` for output quality.',
'"magic_edit" (DEPRECATED) — prompt-driven content edit. Prefer `generate_image_edit` for better results.',
].join(' ')),
model: z.string().optional()
.describe('Concrete model identifier for this operation. Dynamically discover it with list_models using the operation-specific type: upscale/clarity_upscale/split_upscale → "image_upscale"; reframe → "image_reframe"; zoom_out → "image_zoom_out"; inpaint → "inpaint"; erase → "erase"; face_swap → "face_swap"; removebg → "background_remove"; background_replace → "background_replace"; enhance_skin → "skin_enhancer"; enhance → "graphics_enhance"; magic_edit/camera_angle → "image_editing". multi_shot and split use pinned/non-selectable processing, so omit model for those operations. Omit elsewhere to use the platform default. Never pass a kolbo_gateway_* navigation alias.'),
// ── upscale ────────────────────────────────────────────
scale: z.number().optional()
.describe('Upscale factor: 1, 2 or 4. Used with operation="upscale". Default: 2. There is NO 8 — every Topaz split endpoint declares upscale_factor max 4 in its OpenAPI, so an 8 is a guaranteed provider error. Output size is otherwise unbounded: 4x on a 4K source is a 16K result and is priced pro-rata, not refused.'),
enhancement_model: z.string().optional()
.describe('Topaz engine. With operation="upscale" on model "topaz/upscale/image" it selects the engine family: "Standard V2" / "High Fidelity V3" / "CGI" / "Text Refine" (faithful), "Wonder 3.5" (rebuilds natural detail), "Bloom 2" (reinvents detail — most expensive), "Transparent" (keeps the alpha channel). With operation="enhance" it selects the correction engine for the chosen model. Omit for the model default. Call list_models to see the engines a model offers.'),
output_format: z.string().optional()
.describe('Output image format: "jpeg" or "png". Used with Topaz "upscale" and "enhance". Defaults to jpeg (the transparent upscaler always returns png).'),
resolution: z.string().optional()
.describe('Target output resolution (e.g. "4k", "2k", "1080p"). Used with "clarity_upscale", "split_upscale", "multi_shot".'),
// ── reframe ────────────────────────────────────────────
aspect_ratio: z.string().optional()
.describe(aspectRatioDescribe() + ' Required for operation="reframe". Ignored by "zoom_out" — size that expansion with `zoom_out_percentage` or the `expand_*` pixel args.'),
// ── zoom_out (outpaint / expand) ───────────────────────
zoom_out_percentage: z.number().optional()
.describe('How much to expand outward on ALL sides, as a percentage (0-90). Used with operation="zoom_out". Default: 20. Ignored when any `expand_*` arg is set.'),
expand_left: z.number().optional()
.describe('Pixels of new AI-generated content to add on the LEFT. Used with operation="zoom_out" (0-700). This is how you widen an image asymmetrically, e.g. to turn a 16:9 frame into 21:9 without touching the subject.'),
expand_right: z.number().optional()
.describe('Pixels of new AI-generated content to add on the RIGHT. Used with operation="zoom_out" (0-700).'),
expand_top: z.number().optional()
.describe('Pixels of new AI-generated content to add on TOP. Used with operation="zoom_out" (0-700).'),
expand_bottom: z.number().optional()
.describe('Pixels of new AI-generated content to add on the BOTTOM. Used with operation="zoom_out" (0-700).'),
// ── enhance_skin ───────────────────────────────────────
skin_strength: z.enum(['subtle', 'realistic', 'pimple', 'freckle']).optional()
.describe('Skin enhancement preset. Used with operation="enhance_skin". Default: "realistic".'),
// ── inpaint / erase / face_swap / background_replace / zoom_out / camera_angle / magic_edit ──
prompt: z.string().optional()
.describe('Text instruction guiding the edit. Required for "background_replace". Used with "inpaint", "zoom_out", "camera_angle", and the deprecated "magic_edit". For "zoom_out" it describes what fills the NEW space only — the original image is left as-is.'),
mask_image_url: z.string().optional()
.describe('URL or absolute local path of a mask image (black & white; white = affected area). Required for "inpaint" and "erase". For "face_swap", this is the face reference image.'),
additional_images: z.array(z.string()).optional()
.describe('Extra reference images — URLs or absolute local paths (up to 8). For "inpaint": reference images that guide style/content. For "multi_shot": the set of scene reference shots. For "magic_edit": additional source images for composite edits.'),
// ── camera_angle ───────────────────────────────────────
generate_all_angles: z.boolean().optional()
.describe('When true, generates a full set of camera angles instead of just one. Only used with operation="camera_angle".'),
// ── quality / prompt enhancement ───────────────────────
quality: z.string().optional()
.describe('Output quality preset (e.g. "high", "standard"). Applies where the underlying model supports quality tiers.'),
ai_optimize: z.boolean().optional()
.describe('Whether to let Kolbo AI enhance your prompt before sending to the model. Default: false — your prompt reaches the model exactly as written. Only pass true if the user explicitly asks to enhance/improve the prompt.'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({
image_url, operation, model, scale, aspect_ratio, skin_strength, prompt,
mask_image_url, additional_images, generate_all_angles, resolution, quality, ai_optimize = false,
zoom_out_percentage, expand_left, expand_right, expand_top, expand_bottom,
enhancement_model, output_format,
project_id, session_id
}) => {
const editModelType = modelTypeForEditOperation('image', operation);
if ((operation === 'multi_shot' || operation === 'split') && model) {
throw new Error(`model is not configurable for ${operation}; omit it to use the operation's pinned processing path`);
}
assertExecutableEditModel(model, 'image', operation);
model = await canonicalModelId(client, model, editModelType || undefined);
assertExecutableEditModel(model, 'image', operation);
await assertModelSupportsType(client, model, editModelType || undefined);
aspect_ratio = await resolveCatalogAspectRatio(client, model, aspect_ratio, editModelType || undefined);
// Basic validation
if (operation === 'reframe' && !aspect_ratio) throw new Error('aspect_ratio is required for reframe');
if (operation === 'enhance' && !model) {
throw new Error('model is required for enhance — pick one of: topaz/adjust/image, topaz/sharpen/image, topaz/denoise/image, topaz/restore/image');
}
if (operation === 'background_replace' && !prompt) throw new Error('prompt is required for background_replace');
if (operation === 'face_swap' && !mask_image_url && !(additional_images && additional_images.length > 0)) {
throw new Error('mask_image_url (face reference) is required for face_swap');
}
// Local paths → CDN URLs (image_url / mask_image_url / additional_images).
[image_url, mask_image_url, additional_images] = await Promise.all([
rehostLocal(image_url, 'image', project_id),
rehostLocal(mask_image_url, 'image', project_id),
additional_images ? Promise.all(additional_images.map((s) => rehostLocal(s, 'image', project_id))) : additional_images,
]);
const gen = await client.post('/v1/edit/image', {
image_url, operation, model, scale, aspect_ratio, skin_strength, prompt,
mask_image_url, additional_images, generate_all_angles, resolution, quality, ai_optimize,
zoom_out_percentage, expand_left, expand_right, expand_top, expand_bottom,
enhancement_model, output_format,
project_id, session_id
});
if (returnsImmediately()) return submittedResult({
tool: 'edit_image', kind: 'image', gen, client, model,
prompt: prompt || operation,
settings: { mode: operation, aspect_ratio, scale, resolution },
reference_images: [image_url, mask_image_url, ...(additional_images || [])].filter(Boolean)
});
const poll = await pollOrTimedOut(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 5) * 1000,
timeout: 150000 // Return before host cutoff; LLM can call get_generation_status with wait=true for split_upscale and multi_shot
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'edit_image', kind: 'image', gen, client, model,
prompt: prompt || operation,
settings: { mode: operation, aspect_ratio, scale, resolution },
reference_images: [image_url, mask_image_url, ...(additional_images || [])].filter(Boolean),
urls: result.result?.urls || [],
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: gen.session_id,
urls: result.result?.urls || [],
edit_type: result.result?.edit_type || null,
model: result.result?.model || null
}, null, 2));
}
);
// ─── edit_video ────────────────────────────────────────────
server.tool(
'edit_video',
'Apply a targeted AI edit to an existing video. Covers both mechanical edits (upscale, reframe, remove watermark, remove background) and creative operations (generate audio, face swap, extend, lipsync, inpaint, retake, magic edit). Returns the edited video URL when complete.',
{
video_url: z.string().describe('URL of the source video to edit.'),
operation: z.enum([
'upscale', 'reframe',
'generate_audio', 'remove_watermark',
'face_swap', 'extend', 'magic_edit',
'lipsync', 'remove_background',
'inpaint', 'retake', 'draft_enhance', 'draft_quote'
]).describe([
'Edit operation:',
'"draft_enhance" — render an owned saved draft at full quality in its original project. Pass `resolution` from the draft model capabilities; the server selects its finalization engine. No prompt or provider task ID is needed.',
'"draft_quote" — check a saved draft’s supported final resolutions, expiry and exact credit cost without starting a render. Use this before draft_enhance.',
'"upscale" — boost to 4K/2K resolution (use `scale` for factor, `resolution` for target, `target_fps` for frame rate). On model "blackforestlabs/flux-video-upscale" (Flux Video Upscale): `scale` is 1.5-3x (no named `resolution` target — billed on the OUTPUT tier it lands in), `mode` is "precise" (source-faithful, default) or "creative" (reimagines detail — a pricier tier), and `prompt` optionally guides the creative-mode enhancement.',
'"reframe" — change aspect ratio (requires `aspect_ratio`; use `grid_position_x`/`grid_position_y` to control where the original sits).',
'"generate_audio" — add AI-generated audio from `prompt`. Optionally split into `sound_effect_prompt` and `background_music_prompt`. Set `original_sound=true` to keep original audio alongside.',
'"remove_watermark" — AI-powered watermark removal.',
'"face_swap" — replace the face in the video with the face from `image_url` (required).',
'"extend" — lengthen the video at `mode` ("start" or "end") by `duration` seconds. Optional `prompt` and `context` guide the generated content.',
'"magic_edit" — restyle or transform the video with `prompt` (required).',
'"lipsync" — sync a face to audio. Provide `audio_url` (audio file) OR `text_prompt` (will synthesize speech).',
'"remove_background" — remove or greenscreen the video background. Use `refine_edges=true` for cleaner cutouts, `subject_is_person=true` for portrait-optimized mode.',
'"inpaint" — replace a region of the video. Provide `mask_video_url` (B&W mask, white=fill area), `prompt` for what to generate, and optionally `object_prompt` (what to replace) and `video_strength` (0–1, how strongly to follow the original).',
'"retake" — regenerate a segment of the video. Use `start_time` (seconds) + `duration` + optional `prompt`.',
].join(' ')),
model: z.string().optional()
.describe('Concrete model identifier for this operation. Dynamically discover it with list_models using the operation-specific type: upscale → "video_upscale"; reframe → "video_reframe"; generate_audio → "video_to_sound"; remove_watermark → "video_watermark_removal"; face_swap → "video_face_swap"; extend → "video_extend"; magic_edit → "video_to_video"; lipsync → "lipsync-video"; remove_background → "video_background_removal"; inpaint → "video_inpaint"; retake → "video_retake". Compare the returned credit, supported resolutions/aspect ratios, duration limits, resolution multipliers, and params, then pass a concrete identifier. Omit to use the platform default. Never pass a kolbo_gateway_* navigation alias.'),
// ── upscale ────────────────────────────────────────────
scale: z.number().optional()
.describe('Upscale factor (e.g. 2, 4). Used with operation="upscale". On "blackforestlabs/flux-video-upscale" the valid range is 1.5-3 (default 2).'),
resolution: z.string().optional()
.describe('Target resolution (e.g. "4k", "2k", "1080p"). Used with "upscale" and "reframe". Not used by "blackforestlabs/flux-video-upscale" — use `scale` instead.'),
target_fps: z.number().optional()
.describe('Target frame rate (e.g. 24, 30, 60). Used with operation="upscale".'),
enhancement_model: z.string().optional()
.describe('Topaz enhancement engine for operation="upscale" on model "topaz/upscale/video". Families: Precision (faithful — "Proteus", "Artemis High Quality", "Iris", "Dione TV", "Gaia CG", "Gaia 2"), Denoise ("Nyx", "Nyx Fast"), Generative (rebuilds detail that is not in the source — "Starlight Fast 2", "Starlight HQ", "Starlight Precise 2.6"), Creative ("Astra 2", always renders 4K). Generative and Creative engines cost several times the Precision rate. Omit for the default.'),
// ── reframe ────────────────────────────────────────────
aspect_ratio: z.string().optional()
.describe(aspectRatioDescribe() + ' Required for operation="reframe".'),
grid_position_x: z.number().optional()
.describe('Horizontal position (0.0–1.0) of the original content within the reframed canvas. Used with "reframe". Default: 0.5 (center).'),
grid_position_y: z.number().optional()
.describe('Vertical position (0.0–1.0) of the original content within the reframed canvas. Used with "reframe". Default: 0.5 (center).'),
// ── generate_audio ─────────────────────────────────────
prompt: z.string().optional()
.describe('Text prompt. Required for "magic_edit". Used with "generate_audio", "extend", "inpaint", "retake", optionally "lipsync" (for text-to-speech instead of audio_url), and optionally "upscale" on model "blackforestlabs/flux-video-upscale" (guides creative-mode detail enhancement).'),
sound_effect_prompt: z.string().optional()
.describe('Separate prompt for sound effects layer. Used with operation="generate_audio" when you want to specify SFX and music independently.'),
background_music_prompt: z.string().optional()
.describe('Separate prompt for background music layer. Used with operation="generate_audio" alongside `sound_effect_prompt`.'),
original_sound: z.boolean().optional()
.describe('When true, keeps the original video audio and mixes in the generated audio. Used with "generate_audio". Default: false.'),
cfg_strength: z.number().optional()
.describe('Guidance strength for audio generation (higher = follows prompt more strictly). Used with "generate_audio".'),
audio_format: z.enum(['wav', 'mp3', 'aac', 'flac']).optional()
.describe('Separate generated-audio format for Sonilo sound-effects models. Read output_audio_formats/default_output_audio_format from list_models; default is "aac".'),
segments: z.array(z.object({
start: z.number().nonnegative(),
end: z.number().positive(),
prompt: z.string()
})).optional()
.describe('Optional contiguous Sonilo sound-design ranges. The first start must be 0, each end must equal the next start, and the final end must not exceed the video duration. Omit to let Sonilo detect scenes automatically.'),
// ── face_swap ──────────────────────────────────────────
image_url: z.string().optional()
.describe('URL of the reference face image. Required for operation="face_swap".'),
// ── extend ─────────────────────────────────────────────
duration: z.number().optional()
.describe('Seconds of content to generate. Used with "extend" (required) and "retake" (optional). Typical range: 1–20.'),
mode: z.string().optional()
.describe('Extension direction: "start" or "end". Used with "extend". For "retake": replacement mode (default: "replace_audio_and_video"). For "upscale" on model "blackforestlabs/flux-video-upscale": "precise" (source-faithful, default) or "creative" (reimagines detail — pricier). Default: "end".'),
context: z.string().optional()
.describe('Additional context to guide what gets generated in the extended segment. Used with "extend".'),
// ── lipsync ────────────────────────────────────────────
audio_url: z.string().optional()
.describe('URL of the audio track to sync to the face. Required for "lipsync" unless `text_prompt` is provided.'),
text_prompt: z.string().optional()
.describe('Text to synthesize as speech and sync to the face. Used with "lipsync" as an alternative to `audio_url`.'),
// ── remove_background ──────────────────────────────────
refine_edges: z.boolean().optional()
.describe('Apply edge refinement for cleaner background removal. Used with "remove_background". Default: false.'),
subject_is_person: z.boolean().optional()
.describe('Optimize background removal for a human subject (portrait mode). Used with "remove_background". Default: false.'),
// ── inpaint ────────────────────────────────────────────
mask_video_url: z.string().optional()
.describe('URL of a mask video (B&W; white = area to fill). Used with operation="inpaint".'),
object_prompt: z.string().optional()
.describe('Description of the object being replaced (helps the model understand what to remove). Used with "inpaint".'),
video_strength: z.number().optional()
.describe('How closely to follow the original video (0.0–1.0; higher = closer to original). Used with "inpaint".'),
// ── retake ─────────────────────────────────────────────
start_time: z.number().optional()
.describe('Start time in seconds of the segment to retake. Used with operation="retake".'),
project_id: projectIdField,
session_id: sessionIdField
},
async ({
video_url, operation, model, aspect_ratio, scale, prompt,
image_url, audio_url, duration, mode,
target_fps, resolution, enhancement_model,
grid_position_x, grid_position_y,
sound_effect_prompt, background_music_prompt, original_sound, cfg_strength,
audio_format, segments,
refine_edges, subject_is_person,
text_prompt, context,
mask_video_url, object_prompt, video_strength,
start_time,
project_id, session_id
}) => {
const editModelType = modelTypeForEditOperation('video', operation);
assertExecutableEditModel(model, 'video', operation);
model = await canonicalModelId(client, model, editModelType || undefined);
assertExecutableEditModel(model, 'video', operation);
await assertModelSupportsType(client, model, editModelType || undefined);
aspect_ratio = await resolveCatalogAspectRatio(client, model, aspect_ratio, editModelType || undefined);
// Validation
if (operation === 'magic_edit' && !prompt) throw new Error('prompt is required for magic_edit');
if (operation === 'generate_audio' && !prompt && !sound_effect_prompt && !background_music_prompt
&& !(typeof model === 'string' && model.includes('sonilo'))) {
throw new Error('prompt is required for generate_audio unless dedicated sound/music prompts are provided or the selected model supports automatic captioning');
}
if (operation === 'reframe' && !aspect_ratio)throw new Error('aspect_ratio is required for reframe');
if (operation === 'face_swap' && !image_url) throw new Error('image_url (reference face) is required for face_swap');
if (operation === 'lipsync' && !audio_url && !text_prompt) throw new Error('audio_url or text_prompt is required for lipsync');
if (operation === 'extend' && !duration) throw new Error('duration is required for extend');
// Keep the resolved inputs so the widget can load local-file references
// from the same CDN URLs submitted to the API.
const editBody = await rehostLocalPaths(client, {
video_url, operation, model, aspect_ratio, scale, prompt, enhancement_model,
image_url, audio_url, duration, mode,
target_fps, resolution,
grid_position_x, grid_position_y,
sound_effect_prompt, background_music_prompt, original_sound, cfg_strength,
audio_format, segments,
refine_edges, subject_is_person,
text_prompt, context,
mask_video_url, object_prompt, video_strength,
start_time,
project_id, session_id
}, { allowLocalFiles: !options.remote });
const gen = await client.post('/v1/edit/video', editBody);
const editRefs = {
reference_images: [editBody.image_url].filter(Boolean),
reference_videos: [editBody.video_url, editBody.mask_video_url].filter(Boolean),
reference_audio: [editBody.audio_url].filter(Boolean),
};
if (operation === 'draft_quote') return { content: [{ type: 'text', text: JSON.stringify(gen, null, 2) }] };
if (returnsImmediately()) return submittedResult({
tool: 'edit_video', kind: 'video', gen, client, model,
prompt: prompt || operation,
settings: { mode: operation, duration, aspect_ratio, resolution },
...editRefs
});
const poll = await pollOrTimedOut(client, gen.generation_id, {
interval: (gen.poll_interval_hint || 8) * 1000,
timeout: 150000
});
if (poll.timedOut) return poll.timedOut;
const result = poll.result;
return uiCompleted({
tool: 'edit_video', kind: 'video', gen, client, model,
prompt: prompt || operation,
settings: { mode: operation, duration, aspect_ratio, resolution },
...editRefs,
urls: result.result?.urls || [],
duration: result.result?.duration || null,
credits_used: creditFields(result).credits_used,
}, JSON.stringify({
...creditFields(result),
session_id: gen.session_id,
urls: result.result?.urls || [],
download_url: result.result?.download_url || null,
edit_type: result.result?.edit_type || null,
duration: result.result?.duration || null,
model: result.result?.model || null
}, null, 2));
}
);
// ─── trim_video ────────────────────────────────────────────
server.tool(
'trim_video',
'Cut a section out of a video by start/end time (frame-accurate server-side trim, no credits for AI generation — pure processing). Source must be a Kolbo-hosted video URL (generated output or upload_media result). The tool waits for the job to finish (usually seconds) and returns the trimmed video URL.',
{
video_url: z.string().describe('Kolbo-hosted URL of the source video.'),
start_time: z.number().describe('Trim start, in seconds (>= 0).'),
end_time: z.number().describe('Trim end, in seconds (> start_time, max 3600).'),
project_id: z.string().optional().describe('Project to file the trimmed video into (from list_projects).')
},
async ({ video_url, start_time, end_time, project_id }) => {
const body = { video_url, start_time, end_time };
if (project_id) body.project_id = project_id;
const submit = await client.post('/v1/video/trim', body);
const jobId = submit.jobId || submit.generationId;
if (!jobId) return { content: [{ type: 'text', text: JSON.stringify(submit, null, 2) }] };
// Poll the dedicated trim progress endpoint until done (~seconds).
const deadline = Date.now() + 180000;
let last = submit;
while (Date.now() < deadline) {
await new Promise(r => setTimeout(r, 4000));
last = await client.get(`/v1/video/trim/${encodeURIComponent(jobId)}`);
if (last.status === 'completed' || last.url || last.videoUrl) break;
if (last.status === 'failed') break;
}
return {
content: [{
type: 'text',
text: JSON.stringify({
job_id: jobId,
status: last.status || 'processing',
video_url: last.url || last.videoUrl || null
}, null, 2)
}]
};
}
);
}
module.exports = { registerGenerateTools };