@sogni-ai/sogni-creative-agent-skill
Version:
Sogni Creative Agent Skill: agent skill and CLI for Sogni AI image, 3D, speech, video, and music generation.
491 lines (480 loc) • 26.1 kB
JavaScript
// desktop-extension/server/tools.mjs
// MCP tool registry: JSON Schemas shown to Claude Desktop plus pure
// buildArgs() functions that translate tool input into sogni-agent argv.
// buildArgs never spawns anything — index.mjs owns process execution.
const GEN_BASE = ['--json', '-q', '--no-update-check'];
// Past the CLI's longest default wait (30 minutes, for any render on Relaxed), so
// the CLI can report the project id before this process-level safety valve fires.
const MIN_HARD_KILL_MS = 31 * 60 * 1000;
const HARD_KILL_GRACE_MS = 60 * 1000;
const DEFAULT_TOOL_TIMEOUT_SECONDS = {
generate_video: 1800,
generate_music: 600,
};
export function hardKillTimeoutMs(toolName, input) {
const explicitTimeoutSeconds = Number(input?.timeout_seconds);
const timeoutSeconds = Number.isFinite(explicitTimeoutSeconds) && explicitTimeoutSeconds > 0
? explicitTimeoutSeconds
: DEFAULT_TOOL_TIMEOUT_SECONDS[toolName];
if (!timeoutSeconds) return MIN_HARD_KILL_MS;
// Let the CLI's own timeout run first so it can await the normal Sogni
// project cancellation protocol before this process-level safety valve.
return Math.max(MIN_HARD_KILL_MS, (timeoutSeconds * 1000) + HARD_KILL_GRACE_MS);
}
function push(args, flag, value) {
if (value !== undefined && value !== null && value !== '') args.push(flag, String(value));
}
function required(input, field, hint) {
const v = input[field];
if (v === undefined || v === null || v === '') {
throw new Error(`${field} is required${hint ? ` — ${hint}` : ''}.`);
}
return v;
}
const str = (description) => ({ type: 'string', description });
const num = (description) => ({ type: 'number', description });
const RENDER_OPTIONS = {
network: { type: 'string', enum: ['fast', 'relaxed'], description: 'Supernet for this render; omit to keep the configured default' },
priority: { type: 'boolean', description: 'Only when the user chooses Priority: paid Unlimited Fast worker generations consume fair-use capacity at 1.5x for earlier queue placement' },
};
function pushRenderOptions(args, input) {
push(args, '--network', input.network);
if (input.priority === true) args.push('--priority');
}
// Chat clients (Claude Desktop etc.) never expose chat-attached files to this
// server's filesystem; without this warning the model wastes turns discovering
// that boundary by trial and error.
const CHAT_ATTACHMENT_NOTE =
'Files attached to the chat are NOT visible to Sogni tools — import them with import_media first ' +
'(returns a local path), or pass a local absolute path, URL, or saved persona.';
export const TOOLS = [
{
name: 'generate_image',
description:
'Generate one or more images from a text prompt on the Sogni GPU network. ' +
'Pass output_path (absolute, .png/.jpg) to save locally; otherwise a hosted URL is returned. ' +
'Use context_images for image editing ("make the background a beach") and persona to include a saved person. ' +
'Image results are attached inline in the tool result. ' +
'The model reads your prompt with no model-specific rewrite. Before writing the prompt, call prompt_guide for the model you chose, ' +
'and write it in that model family\'s own form: ' +
'https://raw.githubusercontent.com/Sogni-AI/sogni-creative-agent-skill/main/references/image-prompting.md. ' +
CHAT_ATTACHMENT_NOTE,
inputSchema: {
type: 'object',
properties: {
prompt: str('What to generate'),
...RENDER_OPTIONS,
output_format: { type: 'string', enum: ['png', 'jpg', 'webp', 'gif'], description: 'GIF requires GPT Image 2.6' },
image_quality: { type: 'string', enum: ['low', 'medium', 'high', 'xhigh', 'max'] },
image_complexity: { type: 'string', enum: ['low', 'medium', 'high'], description: 'GPT Image 2.6 planning effort' },
image_frame_count: num('GPT Image 2.6 GIF frames, 2-16; default 9'),
image_frame_duration_ms: num('GIF frame duration, 20-2000 milliseconds; default 200'),
image_markup_url: str('Transparent annotation PNG for a GPT Image 2.6 still edit, aligned to the first context image'),
output_path: str('Absolute file path to save the image (optional)'),
quality: { type: 'string', enum: ['fast', 'hq', 'pro'], description: 'Quality preset' },
model: str('Model id (optional; overrides quality preset)'),
width: num('Width in px (default 512)'),
height: num('Height in px (default 512)'),
count: num('Number of images (default 1)'),
seed: num('Fixed seed for reproducibility'),
context_images: { type: 'array', items: { type: 'string' }, description: 'Absolute paths of context/edit images' },
persona: str('Saved persona name to use as reference'),
timeout_seconds: num('Generation timeout override'),
},
required: ['prompt'],
},
buildArgs(input) {
const prompt = required(input, 'prompt');
const args = [...GEN_BASE];
pushRenderOptions(args, input);
push(args, '--output-format', input.output_format);
push(args, '--image-quality', input.image_quality);
push(args, '--image-complexity', input.image_complexity);
push(args, '--image-frame-count', input.image_frame_count);
push(args, '--image-frame-duration-ms', input.image_frame_duration_ms);
push(args, '--image-markup-url', input.image_markup_url);
push(args, '-o', input.output_path);
push(args, '-Q', input.quality);
push(args, '-m', input.model);
push(args, '-w', input.width);
push(args, '-h', input.height);
push(args, '-n', input.count);
push(args, '-s', input.seed);
for (const c of input.context_images ?? []) push(args, '-c', c);
push(args, '--persona', input.persona);
push(args, '-t', input.timeout_seconds);
args.push(prompt);
return args;
},
},
{
name: 'generate_video',
description:
'Generate a video from a text prompt, optionally driven by reference media. ' +
'ref = start frame image (alone defaults to wan_v2.2-14b-fp8_i2v_lightx2v), ' +
'ref_end = end frame (with ref, defaults to ltx25-22b-int8_i2v_distilled and the standard first/last-frame template; no transition LoRA), ' +
'MiniMax H3 supports t2v, i2v, first/last-frame, and r2v; H3 r2v accepts up to 9 images, 3 videos, ' +
'and 3 audios (12 files total) through the repeatable reference arrays and uses <Picture 1>/<Video 1>/<Audio 1> prompt tags. ' +
'MiniMax H3 FastH3 audio-to-video (model minimax-h3-fasth3-ia2v-turbo with ref + ref_audio, ' +
'minimax-h3-fasth3-flfa2v-turbo with ref + ref_end + ref_audio, minimax-h3-fasth3-a2v-turbo with ref_audio only) ' +
'drives the clip with the uploaded audio and keeps it as the soundtrack; no LoRAs. ' +
'MiniMax H3 i2v, first/last-frame, FastH3 audio-to-video and r2v also take keyframes: extra images pinned at chosen times inside the clip. ' +
'ref_audio = soundtrack/lip-sync audio, ref_video = motion reference. ' +
'Seedance 2.5 draft mode: seedance_draft renders a 480p draft at the 480p price, and the result carries its jobId and seedanceDraftExpiresAt; ' +
'seedance_finalize with that jobId (same model, no prompt, media, duration or size) renders the same video at 1080p before it expires (about three hours). ' +
'Iterate on drafts and finalize only the take you keep. ' +
'The model reads your prompt with no model-specific rewrite (Wan 3 aside). Before writing the prompt, call prompt_guide for the model you chose, ' +
'and read the section for that model (LTX, MiniMax H3, Seedance, WAN 2.2, Wan 3) in ' +
'https://raw.githubusercontent.com/Sogni-AI/sogni-creative-agent-skill/main/references/video-prompting.md. ' +
'Rendering takes minutes; prefer output_path (absolute .mp4). ' + CHAT_ATTACHMENT_NOTE,
inputSchema: {
type: 'object',
properties: {
prompt: str('The prompt, in the chosen model\'s own grammar (see the tool description): what is seen and heard'),
...RENDER_OPTIONS,
output_format: { type: 'string', enum: ['mp4', 'mov'] },
return_last_frame: { type: 'boolean', description: 'Export the last frame for a supported video model' },
output_path: str('Absolute .mp4 path to save (optional)'),
model: str('Video model id (optional)'),
workflow: { type: 'string', enum: ['t2v', 'i2v', 'r2v', 's2v', 'ia2v', 'a2v', 'v2v', 'animate-move', 'animate-replace'] },
duration: num('Duration in seconds (default 5)'),
ref: str('Absolute path or URL of the start-frame image; loose Picture 1 for MiniMax H3 r2v'),
ref_end: str('Absolute path or URL of the end-frame image'),
ref_audio: str('Absolute path or URL of reference audio'),
ref_video: str('Absolute path or URL of reference video'),
reference_images: { type: 'array', items: { type: 'string' }, description: 'Loose image reference paths/URLs (-c); repeatable for MiniMax H3 r2v' },
reference_audios: { type: 'array', items: { type: 'string' }, description: 'Additional reference audio paths/URLs; MiniMax H3 r2v supports up to 3 total' },
reference_videos: { type: 'array', items: { type: 'string' }, description: 'Additional reference video paths/URLs; MiniMax H3 r2v supports up to 3 total' },
keyframes: {
type: 'array',
maxItems: 8,
description:
'MiniMax H3 only. Pin up to 8 images at exact moments inside the video, in addition to the first/last frame (ref/ref_end). ' +
'Each item is {image, at_seconds}: image is an absolute path or URL; at_seconds is when the video should land on that image. ' +
'Keyframes must fall strictly inside the clip (not on the first or last frame) and at distinct times. ' +
'Describe what each keyframe shows in the prompt at its time; a keyframe with a new angle, place or light starts a new shot. ' +
'Two keyframes are included in the price; each additional keyframe adds a little.',
items: {
type: 'object',
properties: {
image: str('Absolute path or URL of the keyframe image'),
at_seconds: num('When the video lands on this image, in seconds from the start'),
},
required: ['image', 'at_seconds'],
additionalProperties: false,
},
},
generate_audio: { type: 'boolean', description: 'For MiniMax H3, keep generated audio (true) or strip it from the returned video (false)' },
persona: str('Saved persona name (reference frame)'),
target_resolution: num('Short-side target in px, preserves aspect'),
seedance_draft: { type: 'boolean', description: 'Seedance 2.5: render a 480p draft that seedance_finalize can render at 1080p' },
seedance_finalize: str('Seedance 2.5: the jobId of a draft to render at 1080p with its own prompt, media, length, ratio, seed and audio (send no prompt)'),
timeout_seconds: num('Generation timeout override (default 1800)'),
},
required: [],
},
buildArgs(input) {
// A finalize renders its draft's own prompt; every other render needs one.
const finalize = input.seedance_finalize;
const prompt = finalize ? input.prompt : required(input, 'prompt');
const args = [...GEN_BASE, '--video'];
pushRenderOptions(args, input);
push(args, '--output-format', input.output_format);
if (input.return_last_frame === true) args.push('--return-last-frame');
push(args, '-o', input.output_path);
push(args, '-m', input.model);
push(args, '--workflow', input.workflow);
push(args, '--duration', input.duration);
push(args, '--ref', input.ref);
push(args, '--ref-end', input.ref_end);
push(args, '--ref-audio', input.ref_audio);
for (const refAudio of input.reference_audios ?? []) push(args, '--ref-audio', refAudio);
push(args, '--ref-video', input.ref_video);
for (const refVideo of input.reference_videos ?? []) push(args, '--ref-video', refVideo);
for (const refImage of input.reference_images ?? []) push(args, '-c', refImage);
for (const keyframe of input.keyframes ?? []) {
const image = keyframe?.image;
const atSeconds = keyframe?.at_seconds;
if (typeof image !== 'string' || !image || typeof atSeconds !== 'number' || !Number.isFinite(atSeconds)) {
throw new Error('keyframes items need image (a path or URL) and at_seconds (a number of seconds).');
}
args.push('--keyframe', `${image}@${atSeconds}`);
}
if (input.generate_audio === true) args.push('--generate-audio');
if (input.generate_audio === false) args.push('--no-generate-audio');
push(args, '--persona', input.persona);
push(args, '--target-resolution', input.target_resolution);
if (input.seedance_draft === true) args.push('--seedance-draft');
push(args, '--seedance-finalize', finalize);
push(args, '-t', input.timeout_seconds);
if (prompt) args.push(prompt);
return args;
},
},
{
name: 'prompt_guide',
description:
'The prompt rules Sogni\'s own prompt writers use for one image or video model and mode. Call it before writing a prompt ' +
'for generate_image or generate_video: those tools send your words to the model with no model-specific rewrite, ' +
'so these rules are the shaping the prompt gets. Free and offline: no credits, no API call. ' +
'Pass the model id you will generate with and the mode you will run; omit model to list every family, mode and model id.',
inputSchema: {
type: 'object',
properties: {
model: str('The model id or alias you will pass as model, e.g. seedance2-5, ltx25, minimax-h3-r2v-balanced, wan22-i2v, krea2_identity_edit_v1_2'),
mode: str(
'The mode you will run: t2v, i2v, l2v (last frame only), flf (first and last frame), r2v, v2v, a2v, ia2v, flfa2v, s2v, ' +
'extend, i2v-keyframes, a2v-keyframes, animate-move, animate-replace, generate or edit. ' +
'Default: the mode the model id names, else t2v for video.',
),
},
},
// MiniMax H3 guides run past 20,000 characters (r2v is about 20,100).
maxResultChars: 64_000,
buildArgs(input) {
const args = ['prompt-guide'];
push(args, '--model', input.model);
push(args, '--mode', input.mode);
args.push('--no-update-check');
return args;
},
},
{
name: 'generate_music',
description:
'Generate music/audio from a text prompt with MiniMax Music 3 by default. Omit lyrics for an instrumental. ' +
'Music 3 plays 10-300 seconds (default 60; the length is a ceiling, so a song may end early). ' +
'Put tempo and key in the prompt; with no music_model, bpm and keyscale are added to the prompt for you. ' +
'Prefer output_path (absolute .mp3/.wav/.flac).',
inputSchema: {
type: 'object',
properties: {
prompt: str('Style/mood description; include tempo and key, e.g. "warm acoustic ballad, 92 BPM, A minor"'),
...RENDER_OPTIONS,
output_path: str('Absolute audio file path to save (optional)'),
lyrics: str('Song lyrics (optional). Music 3: plain section tags on their own lines ([Verse], [Chorus]), no modifiers inside brackets'),
duration: num('Seconds. Music 3 (default): 10-300, default 60. ACE-Step turbo/sft: 10-600, default 30'),
bpm: num('Beats per minute (30-300). An exact control only on ACE-Step (music_model turbo or sft)'),
keyscale: str('Key/scale, e.g. "C major". An exact control only on ACE-Step (music_model turbo or sft)'),
music_model: str(
'music3 (default): MiniMax Music 3. turbo: ACE-Step 1.5 XL Turbo, a cheap draft with exact BPM/key and up to 600 s. ' +
'sft: ACE-Step 1.5 XL SFT. Full model IDs (minimax_music3, ace_step_1.5_xl_turbo, ...) also work. ' +
'Leave unset unless the user asks for ACE-Step, a draft, or a track over 300 s.',
),
audio_format: { type: 'string', enum: ['mp3', 'flac', 'wav'] },
timeout_seconds: num('Generation timeout override (default 600)'),
},
required: ['prompt'],
},
buildArgs(input) {
const prompt = required(input, 'prompt');
const args = [...GEN_BASE, '--music'];
pushRenderOptions(args, input);
push(args, '-o', input.output_path);
push(args, '--music-model', input.music_model);
push(args, '--lyrics', input.lyrics);
push(args, '--duration', input.duration);
push(args, '--bpm', input.bpm);
push(args, '--keyscale', input.keyscale);
push(args, '--output-format', input.audio_format);
push(args, '-t', input.timeout_seconds);
args.push(prompt);
return args;
},
},
{
name: 'photobooth',
description:
'Face-transfer portrait generation: renders the person in the ref photo into a new scene ' +
'(e.g. "LinkedIn professional headshot", "80s fashion portrait"). ' +
'Image results are attached inline in the tool result. ' + CHAT_ATTACHMENT_NOTE,
inputSchema: {
type: 'object',
properties: {
prompt: str('Scene/style description'),
ref: str('Absolute path or URL of the face photo'),
output_path: str('Absolute image path to save (optional)'),
count: num('Number of variations (default 1)'),
},
required: ['prompt', 'ref'],
},
buildArgs(input) {
const prompt = required(input, 'prompt');
const ref = required(input, 'ref', 'a face photo path or URL');
const args = [...GEN_BASE, '--photobooth', '--ref', String(ref)];
push(args, '-n', input.count);
push(args, '-o', input.output_path);
args.push(prompt);
return args;
},
},
{
name: 'edit_video',
description:
'Safe local video utilities (ffmpeg wrappers): extract the first/last frame, ' +
'concatenate clips (with optional soundtrack), or remix the audio of an existing video. ' +
'Extracted frames are attached inline in the tool result.',
inputSchema: {
type: 'object',
properties: {
action: { type: 'string', enum: ['extract_first_frame', 'extract_last_frame', 'concat_videos', 'remix_audio'] },
input: str('Input video path (extract/remix actions)'),
output: str('Output file path'),
clips: { type: 'array', items: { type: 'string' }, description: 'Clip paths for concat_videos (min 2)' },
audio_path: str('Soundtrack to mux over concat_videos output'),
bed_audio: str('Audio bed for remix_audio'),
audio_loop: { type: 'boolean', description: 'Loop the bed to cover the video (remix_audio)' },
},
required: ['action', 'output'],
},
buildArgs(input) {
const action = required(input, 'action');
if (action === 'extract_first_frame' || action === 'extract_last_frame') {
const src = required(input, 'input', 'the source video');
const output = required(input, 'output');
return ['--no-update-check', `--${action.replaceAll('_', '-')}`, String(src), String(output)];
}
if (action === 'concat_videos') {
const output = required(input, 'output');
const clips = input.clips ?? [];
if (clips.length < 2) throw new Error('concat_videos needs at least two clips.');
const args = ['--no-update-check', '--concat-videos', String(output), ...clips.map(String)];
push(args, '--concat-audio', input.audio_path);
return args;
}
if (action === 'remix_audio') {
const src = required(input, 'input', 'the source video');
const output = required(input, 'output');
const args = ['--no-update-check', '--remix-audio', String(src), String(output)];
push(args, '--bed-audio', input.bed_audio);
if (input.audio_loop) args.push('--audio-loop');
return args;
}
throw new Error(`Unknown action "${action}".`);
},
},
{
name: 'import_media',
description:
'Save base64-encoded media bytes to the local Sogni media inbox and return the absolute file path, ' +
'ready to pass as ref / context_images / persona ref. Use this to hand chat-attached files to the ' +
'generation tools: read the attachment bytes (e.g. via code execution), downscale large images to ' +
'~1024px JPEG first, base64-encode, and send. For anything over ~40KB, send multiple calls: the first ' +
'with filename, each following chunk with append_to_path set to the returned path.',
inputSchema: {
type: 'object',
properties: {
data: str('Base64-encoded file bytes for this chunk (keep each chunk under ~64KB of base64)'),
filename: str('Filename with a media extension for a NEW import, e.g. "photo.jpg"'),
append_to_path: str('Path returned by the previous call — appends this chunk to that file instead of creating a new one'),
},
required: ['data'],
},
local: true, // handled inside the server (import-media.mjs); never spawns the CLI
},
{
name: 'list_media',
description:
'List recent files in the Sogni media inbox: media the user sent via connected messaging apps ' +
'plus files saved by import_media. Chat attachments do NOT appear here unless imported first.',
inputSchema: {
type: 'object',
properties: { type: { type: 'string', enum: ['images', 'audio', 'all'] } },
},
buildArgs(input) {
return ['--json', '--no-update-check', '--list-media', input.type ?? 'images'];
},
},
{
name: 'manage_personas',
description:
'Manage saved personas (named people with reference photos and optional voice clips). ' +
'Actions: list, add (needs name + ref photo), remove, resolve (show details), list_account, import_account (explicitly copy a saved account Persona and original references into the local library; never replaces a local name). ' + CHAT_ATTACHMENT_NOTE,
inputSchema: {
type: 'object',
properties: {
action: { type: 'string', enum: ['list', 'add', 'remove', 'resolve', 'list_account', 'import_account'] },
account_persona_id: str('Saved account Persona id, from list_account (import_account)'),
after: str('Next-page cursor returned by list_account'),
name: str('Persona name (add/remove/resolve)'),
ref: str('Reference photo path or URL (add)'),
relationship: { type: 'string', enum: ['self', 'partner', 'child', 'friend', 'pet'] },
description: str('Appearance description (add)'),
voice_clip: str('Voice clip audio path (add)'),
},
required: ['action'],
},
buildArgs(input) {
const action = required(input, 'action');
if (action === 'list') return ['--json', '--no-update-check', '--persona-list'];
if (action === 'list_account') {
const args = ['--json', '--no-update-check', '--persona-list-account'];
push(args, '--persona-account-after', input.after);
return args;
}
if (action === 'import_account') return ['--json', '--no-update-check', '--persona-import-account', String(required(input, 'account_persona_id'))];
const name = required(input, 'name');
if (action === 'remove') return ['--no-update-check', '--persona-remove', String(name)];
if (action === 'resolve') return ['--json', '--no-update-check', '--persona-resolve', String(name)];
if (action === 'add') {
const ref = required(input, 'ref', 'a reference photo for the persona');
const args = ['--no-update-check', '--persona-add', String(name), '--ref', String(ref)];
push(args, '--relationship', input.relationship);
push(args, '--description', input.description);
push(args, '--voice-clip', input.voice_clip);
return args;
}
throw new Error(`Unknown action "${action}".`);
},
},
{
name: 'manage_memories',
description:
'Manage persistent user preferences (e.g. preferred_style). Actions: list, get, set, remove.',
inputSchema: {
type: 'object',
properties: {
action: { type: 'string', enum: ['list', 'get', 'set', 'remove'] },
key: str('Memory key (get/set/remove)'),
value: str('Memory value (set)'),
category: { type: 'string', enum: ['preference', 'fact', 'context'] },
},
required: ['action'],
},
buildArgs(input) {
const action = required(input, 'action');
if (action === 'list') return ['--json', '--no-update-check', '--memory-list'];
const key = required(input, 'key');
if (action === 'get') return ['--json', '--no-update-check', '--memory-get', String(key)];
if (action === 'remove') return ['--no-update-check', '--memory-remove', String(key)];
if (action === 'set') {
const value = required(input, 'value');
const args = ['--no-update-check', '--memory-set', String(key), String(value)];
push(args, '--memory-category', input.category);
return args;
}
throw new Error(`Unknown action "${action}".`);
},
},
{
name: 'sogni_doctor',
description:
'Health check for the Sogni install: Node, credentials, ffmpeg, live auth, version. ' +
'Run only after another tool fails — not as a routine preflight.',
inputSchema: { type: 'object', properties: {} },
buildArgs() {
return ['--doctor', '--json', '--no-update-check'];
},
},
{
name: 'account_balance',
description: 'Show the SPARK/SOGNI token balances for the configured Sogni account.',
inputSchema: { type: 'object', properties: {} },
buildArgs() {
return ['--balances', '--json', '--no-update-check'];
},
},
];
export function getTool(name) {
return TOOLS.find((t) => t.name === name);
}