cerevox
Version:
TypeScript SDK for browser automation and secure command execution in highly available and scalable micro computer environments
1,273 lines (1,262 loc) • 192 kB
JavaScript
#!/usr/bin/env node
"use strict";
/*
* MCP Server
* CallTool 有些工具要求的时间较长，可能会60秒超时，callTool 的时候最好设置一下 timeout
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.run = run;
const mcp_js_1 = require("@modelcontextprotocol/sdk/server/mcp.js");
const stdio_js_1 = require("@modelcontextprotocol/sdk/server/stdio.js");
const zod_1 = require("zod");
const index_1 = __importDefault(require("../../index"));
const constants_1 = require("../../utils/constants");
const videokit_1 = require("../../utils/videokit");
const promises_1 = require("node:fs/promises");
const node_path_1 = __importStar(require("node:path"));
const doubao_voices_full_1 = require("./helper/doubao_voices_full");
const node_fs_1 = require("node:fs");
const coze_1 = require("../../utils/coze");
const mp3_duration_1 = __importDefault(require("mp3-duration"));
const image_size_1 = __importDefault(require("image-size"));
function createErrorResponse(error, operation) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.error(`[${operation}] Error:`, error);
return {
content: [
{
type: 'text',
text: JSON.stringify({
success: false,
error: errorMessage,
operation,
timestamp: new Date().toISOString(),
}),
},
],
};
}
// Session 状态检查
async function validateSession(operation) {
if (closeSessionTimerId) {
clearTimeout(closeSessionTimerId);
closeSessionTimerId = null;
}
if (!session || !(await session.isRunning())) {
session = null;
throw new Error(`Session not initialized. Please call 'project-open' first before using ${operation}.`);
}
const projectRulesFile = (0, node_path_1.resolve)(projectLocalDir, '.trae', 'rules', `project_rules.md`);
if (!(0, node_fs_1.existsSync)(projectRulesFile)) {
throw new Error(`Project rules file not found: ${projectRulesFile}. Please call 'retrieve-rules-context' first.`);
}
return session;
}
// 文件名验证
function validateFileName(fileName) {
if (!fileName || fileName.trim() === '') {
throw new Error('File name cannot be empty');
}
if (fileName.includes('..') ||
fileName.includes('/') ||
fileName.includes('\\')) {
throw new Error('Invalid file name: contains illegal characters');
}
return fileName.trim();
}
function getMaterialUri(session, fileName) {
return session.sandbox.getUrl(`/zerocut/${session.terminal.id}/materials/${(0, node_path_1.basename)(fileName)}`);
}
/* Configuration
{
"mcpServers": {
"cerevox-zerocut": {
"command": "npx",
"args": ["-y", "cerevox-zerocut"],
"env": {
"CEREVOX_API_KEY": "ck-**************"
}
},
}
}
*/
async function initProject(session) {
const terminal = session.terminal;
const workDir = `/home/user/cerevox-zerocut/projects/${terminal.id}`;
await session.files.mkdir(workDir);
await (await terminal.run(`cd ${workDir}`)).end();
// 创建素材目录和成品目录 materials、output
await (await terminal.run('mkdir materials output')).end();
// 文件包括（大部分不需要此时创建）
// storyboard.json 故事版
// draft_content.json 草稿内容，Agent在创作过程中，会根据项目规范，自动生成和修改该文件
return workDir;
}
async function saveMaterial(session, url, saveToFileName) {
const terminal = session.terminal;
const saveToPath = `/home/user/cerevox-zerocut/projects/${terminal.id}/materials/${saveToFileName}`;
const saveLocalPath = (0, node_path_1.resolve)(projectLocalDir, 'materials', saveToFileName);
// 先下载到本地，再上传 sandbox，比直接 sandbox 更好，也以免下载超时
// 通过 fetch 下载到本地
const res = await fetch(url);
const arrayBuffer = await res.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
await (0, promises_1.writeFile)(saveLocalPath, buffer);
// 保存后上传，这样更靠谱
const files = session.files;
await files.upload(saveLocalPath, saveToPath);
return saveToPath;
}
async function runFfmpeg(session, compiled) {
const files = session.files;
if (compiled.extraFiles?.length) {
for (const f of compiled.extraFiles) {
const writePath = f.path.replace(/[\\]+/g, '/'); // 因为有可能在不同操作系统下远程运行，所以需要 normalize
await files.write(writePath, f.content);
// await files.write(f.path, f.content);
}
}
const terminal = session.terminal;
const response = await terminal.run(compiled.cmd);
response.stdout.pipe(process.stdout);
response.stderr.pipe(process.stderr);
return response.json();
}
async function sendProgress(context, progress, total, message) {
const token = context._meta?.progressToken;
if (!token)
return; // 客户端没要进度就不用发
await context.sendNotification({
method: 'notifications/progress',
params: { progressToken: token, progress, total, message },
});
}
async function updateMediaLogs(session, fileName, result, mediaType = 'video') {
try {
const mediaLogsPath = node_path_1.default.resolve(projectLocalDir, 'media_logs.json');
let mediaLogs = [];
// 尝试读取现有文件
try {
const existingContent = await (0, promises_1.readFile)(mediaLogsPath, 'utf-8');
mediaLogs = JSON.parse(existingContent);
}
catch (error) {
// 文件不存在或格式错误，使用空数组
mediaLogs = [];
}
// 检查是否已存在相同文件名的记录
const existingIndex = mediaLogs.findIndex(item => item.fileName === fileName);
const logEntry = {
fileName,
mediaType,
timestamp: new Date().toISOString(),
result,
};
if (existingIndex >= 0) {
// 更新现有记录
mediaLogs[existingIndex] = logEntry;
}
else {
// 添加新记录
mediaLogs.push(logEntry);
}
// 写入文件
await (0, promises_1.writeFile)(mediaLogsPath, JSON.stringify(mediaLogs, null, 2));
console.log(`Updated media_logs.json: ${fileName} -> ${JSON.stringify(result)} (${mediaType})`);
}
catch (error) {
console.warn(`Failed to update media_logs.json for ${fileName}:`, error);
}
}
async function filterMaterialsForUpload(materials, projectLocalDir) {
const filesToUpload = [];
const skippedFiles = [];
// 检查是否存在 media_logs.json
const mediaLogsPath = node_path_1.default.resolve(projectLocalDir, 'media_logs.json');
let mediaLogs = [];
try {
const existingContent = await (0, promises_1.readFile)(mediaLogsPath, 'utf-8');
mediaLogs = JSON.parse(existingContent);
}
catch (error) {
// 文件不存在或格式错误，使用空数组
mediaLogs = [];
}
// 检查文件名是否符合 scXX_ 前缀规则
const isSceneFile = (fileName) => {
const baseName = (0, node_path_1.basename)(fileName);
return /^sc\d+_/.test(baseName);
};
// 获取文件的前缀和扩展名
const getFilePrefix = (fileName) => {
const baseName = (0, node_path_1.basename)(fileName);
const match = baseName.match(/^(sc\d+_)/);
return match ? match[1] : '';
};
const getFileExtension = (fileName) => {
return node_path_1.default.extname(fileName);
};
// 分组处理文件
const sceneFiles = [];
const regularFiles = [];
for (const material of materials) {
// 跳过 .DS_Store 文件
if ((0, node_path_1.basename)(material) === '.DS_Store') {
skippedFiles.push(material);
continue;
}
// 跳过 .local.xxx 文件（文件名中包含 .local.）
if ((0, node_path_1.basename)(material).includes('.local.')) {
skippedFiles.push(material);
continue;
}
if (isSceneFile(material)) {
sceneFiles.push(material);
}
else {
regularFiles.push(material);
}
}
// 处理场景文件（scXX_ 前缀）
if (sceneFiles.length > 0 && mediaLogs.length > 0) {
// 按前缀和扩展名分组
const groupedFiles = new Map();
for (const file of sceneFiles) {
const prefix = getFilePrefix(file);
const ext = getFileExtension(file);
const key = `${prefix}${ext}`;
if (!groupedFiles.has(key)) {
groupedFiles.set(key, []);
}
groupedFiles.get(key).push(file);
}
// 对每个组，只保留最新的文件
for (const [key, files] of groupedFiles) {
if (files.length === 1) {
filesToUpload.push(files[0]);
}
else {
// 找到在 media_logs 中有记录的文件
const filesWithLogs = [];
const filesWithoutLogs = [];
for (const file of files) {
const fileName = (0, node_path_1.basename)(file);
const logEntry = mediaLogs.find(log => log.fileName === fileName);
if (logEntry) {
filesWithLogs.push({ file, timestamp: logEntry.timestamp });
}
else {
filesWithoutLogs.push(file);
}
}
// 如果有文件在 media_logs 中没有记录，这些文件必须上传
filesToUpload.push(...filesWithoutLogs);
// 对于有记录的文件，只保留最新的一个
if (filesWithLogs.length > 0) {
filesWithLogs.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
const latestFileWithLog = filesWithLogs[0].file;
filesToUpload.push(latestFileWithLog);
// 其他有记录的文件标记为跳过
for (let i = 1; i < filesWithLogs.length; i++) {
skippedFiles.push(filesWithLogs[i].file);
}
}
}
}
}
else {
// 如果没有 media_logs.json 或没有场景文件，全部上传
filesToUpload.push(...sceneFiles);
}
// 处理普通文件（全量上传）
filesToUpload.push(...regularFiles);
return { filesToUpload, skippedFiles };
}
async function listFiles(dir) {
const entries = await (0, promises_1.readdir)(dir, { withFileTypes: true });
return entries.filter(e => e.isFile()).map(e => (0, node_path_1.resolve)(dir, e.name));
}
// Create an MCP server
const server = new mcp_js_1.McpServer({
name: 'Cerevox Server',
version: constants_1.VERSION,
});
const cerevox = new index_1.default({
apiKey: process.env.CEREVOX_API_KEY || '',
logLevel: 'error',
});
let session = null;
let projectLocalDir = process.env.ZEROCUT_PROJECT_CWD || process.cwd() || '.';
let checkStoryboardFlag = false;
let checkAudioVideoDurationFlag = false;
let checkStoryboardSubtitlesFlag = false;
let closeSessionTimerId = null;
// 注册 ZeroCut 指导规范 Prompt
server.registerPrompt('zerocut-guideline', {
title: 'ZeroCut 短视频创作指导规范',
description: '专业的短视频创作 Agent 指导规范，包含完整的工作流程、工具说明和质量建议',
}, async () => {
try {
const promptPath = (0, node_path_1.resolve)(__dirname, './prompts/zerocut-core.md');
const promptContent = await (0, promises_1.readFile)(promptPath, 'utf-8');
return {
messages: [
{
role: 'user',
content: {
type: 'text',
text: promptContent,
},
},
],
};
}
catch (error) {
console.error('Failed to load zerocut-guideline prompt:', error);
throw new Error(`Failed to load zerocut-guideline prompt: ${error}`);
}
});
server.registerTool('retrieve-rules-context', {
title: 'ZeroCut Basic Rules',
description: `ZeroCut-${constants_1.VERSION} Basic Rules`,
inputSchema: {
purpose: zod_1.z
.enum([
'general-video',
'music-video',
'stage-play',
'story-telling',
'creative-ad',
'expert',
'material-creation',
'freeform',
'custom',
])
.default('general-video')
.describe(`The purpose of the rules context to retrieve.
- general-video 创建通用视频
- music-video 创建音乐视频
- stage-play 创建舞台播放视频
- story-telling 创建故事讲述视频
- creative-ad 创建创意广告视频
- expert 以专家模式创建视频，必须用户主动要求才触发
- material-creation 素材创作模式，必须用户主动要求才触发
- freeform 自由创作模式，必须用户主动要求才触发
- custom 自定义模式`),
},
}, async ({ purpose }) => {
const projectRulesFile = (0, node_path_1.resolve)(projectLocalDir, '.trae', 'rules', `project_rules.md`);
const customRulesFile = (0, node_path_1.resolve)(projectLocalDir, '.trae', 'rules', `custom_rules.md`);
let promptContent = '';
if ((0, node_fs_1.existsSync)(projectRulesFile)) {
promptContent = await (0, promises_1.readFile)(projectRulesFile, 'utf-8');
if ((0, node_fs_1.existsSync)(customRulesFile)) {
promptContent +=
'\n\n---\n\n' + (await (0, promises_1.readFile)(customRulesFile, 'utf-8'));
}
}
else {
// 当 projectRulesFile 不存在时，设置 checkStoryboardFlag 为 false
checkStoryboardFlag = false;
// 当 projectRulesFile 不存在时，设置 checkAudioVideoDurationFlag 为 false
checkAudioVideoDurationFlag = false;
// 当 projectRulesFile 不存在时，设置 checkStoryboardSubtitlesFlag 为 false
checkStoryboardSubtitlesFlag = false;
if (purpose !== 'general-video' &&
purpose !== 'music-video' &&
purpose !== 'stage-play' &&
purpose !== 'story-telling' &&
purpose !== 'creative-ad' &&
purpose !== 'expert' &&
purpose !== 'material-creation' &&
purpose !== 'freeform') {
return createErrorResponse(`Project rules file not found: ${projectRulesFile}`, 'retrieve-rules-context');
}
}
try {
if (!promptContent) {
const promptPath = (0, node_path_1.resolve)(__dirname, `./prompts/rules/${purpose}.md`);
promptContent = await (0, promises_1.readFile)(promptPath, 'utf-8');
// 确保目录存在
await (0, promises_1.mkdir)((0, node_path_1.dirname)(projectRulesFile), { recursive: true });
await (0, promises_1.writeFile)(projectRulesFile, promptContent);
}
// 如果 purpose 为 freeform，复制 skills 和 knowledge 目录
if (purpose === 'freeform') {
// 递归复制目录的通用函数
const copyDirectory = async (src, dest) => {
const entries = await (0, promises_1.readdir)(src, { withFileTypes: true });
for (const entry of entries) {
const srcPath = (0, node_path_1.resolve)(src, entry.name);
const destPath = (0, node_path_1.resolve)(dest, entry.name);
if (entry.isDirectory()) {
await (0, promises_1.mkdir)(destPath, { recursive: true });
await copyDirectory(srcPath, destPath);
}
else {
const content = await (0, promises_1.readFile)(srcPath, 'utf-8');
await (0, promises_1.writeFile)(destPath, content);
}
}
};
// 复制 skills 目录
const sourceSkillsDir = (0, node_path_1.resolve)(__dirname, './prompts/skills');
const targetSkillsDir = (0, node_path_1.resolve)(projectLocalDir, '.trae', 'skills');
try {
await (0, promises_1.mkdir)(targetSkillsDir, { recursive: true });
await copyDirectory(sourceSkillsDir, targetSkillsDir);
console.log(`Skills directory copied to ${targetSkillsDir}`);
}
catch (skillsError) {
console.warn(`Failed to copy skills directory: ${skillsError}`);
}
// 复制 knowledge 目录
const sourceKnowledgeDir = (0, node_path_1.resolve)(__dirname, './prompts/knowledge');
const targetKnowledgeDir = (0, node_path_1.resolve)(projectLocalDir, '.trae', 'knowledge');
try {
await (0, promises_1.mkdir)(targetKnowledgeDir, { recursive: true });
await copyDirectory(sourceKnowledgeDir, targetKnowledgeDir);
console.log(`Knowledge directory copied to ${targetKnowledgeDir}`);
}
catch (knowledgeError) {
console.warn(`Failed to copy knowledge directory: ${knowledgeError}`);
}
}
return {
content: [
{
type: 'text',
text: JSON.stringify({
type: 'project_rules',
content: promptContent,
projectRulesFile,
}),
},
],
};
}
catch (error) {
console.error(`Failed to load rules context prompt for ${purpose}:`, error);
return createErrorResponse(`Failed to load rules context prompt for ${purpose}: ${error}`, 'retrieve-rules-context');
}
});
server.registerTool('project-open', {
title: 'Open Project',
description: 'Launch a new Cerevox session with a Chromium browser instance and open a new project context. Supports smart file filtering to optimize upload performance.',
inputSchema: {
localDir: zod_1.z
.string()
.optional()
.default('.')
.describe('The path of the file to upload.'),
tosFiles: zod_1.z
.array(zod_1.z.string())
.optional()
.default([])
.describe('对象存储系统中的持久化文件，不通过本地直接下载到项目目录，可选参数'),
uploadAllFiles: zod_1.z
.boolean()
.optional()
.default(false)
.describe('Whether to upload all files without filtering. If true, skips the smart filtering logic.'),
},
}, async ({ localDir, uploadAllFiles, tosFiles }, context) => {
try {
if (closeSessionTimerId) {
clearTimeout(closeSessionTimerId);
closeSessionTimerId = null;
}
// 检查是否已有活跃session
if (session) {
console.warn('Session already exists, closing previous session');
// try {
// await session.close();
// } catch (closeError) {
// console.warn('Failed to close previous session:', closeError);
// }
const result = {
success: true,
sessionId: session.id,
workDir: `/home/user/cerevox-zerocut/projects/${session.terminal.id}`,
projectLocalDir,
};
return {
content: [
{
type: 'text',
text: JSON.stringify(result),
},
],
};
}
const apiKey = process.env.CEREVOX_API_KEY;
// 验证API密钥
if (!apiKey) {
throw new Error('CEREVOX_API_KEY environment variable is required');
}
if (!apiKey.startsWith('ck-') || apiKey.length < 102) {
throw new Error('不正确的apiKey，请登录ZeroCut后台（https://workspace.zerocut.cn/apikey）获取apiKey，进行配置');
}
console.log('Launching new Cerevox session...');
session = await cerevox.launch({
timeout: 60,
});
if (!session) {
throw new Error('Failed to create Cerevox session');
}
console.log('Initializing project...');
const workDir = await initProject(session);
projectLocalDir = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), localDir || '.');
const syncDir = (0, node_path_1.resolve)(projectLocalDir, 'materials');
try {
await (0, promises_1.mkdir)(syncDir, { recursive: true });
}
catch (mkdirError) {
console.warn('Failed to create materials directory:', mkdirError);
// 继续执行，可能目录已存在
}
let materials = [];
// 文件过滤逻辑
let filesToUpload = [];
let skippedFiles = [];
if (localDir) {
try {
materials = await listFiles(syncDir);
}
catch (listError) {
console.warn('Failed to list materials:', listError);
materials = [];
}
if (uploadAllFiles) {
// 如果 uploadAllFiles 为 true，跳过智能过滤，上传所有文件
filesToUpload = materials;
skippedFiles = [];
}
else {
// 智能文件过滤逻辑
const filterResult = await filterMaterialsForUpload(materials, projectLocalDir);
filesToUpload = filterResult.filesToUpload;
skippedFiles = filterResult.skippedFiles;
}
}
const files = session.files;
let progress = 0;
const uploadErrors = [];
const totalFiles = filesToUpload.length + tosFiles.length;
for (const material of filesToUpload) {
try {
await files.upload(material, `${workDir}/materials/${(0, node_path_1.basename)(material)}`);
await sendProgress(context, ++progress, totalFiles, material);
}
catch (uploadError) {
const errorMsg = `Failed to upload ${material}: ${uploadError}`;
console.error(errorMsg);
uploadErrors.push(errorMsg);
}
}
for (const tosFile of tosFiles) {
try {
const url = new URL(tosFile);
await session.terminal.run(`wget -O ${workDir}/materials/${(0, node_path_1.basename)(url.pathname)} ${tosFile}`);
await sendProgress(context, ++progress, totalFiles, tosFile);
}
catch (uploadError) {
const errorMsg = `Failed to upload ${tosFile}: ${uploadError}`;
console.error(errorMsg);
uploadErrors.push(errorMsg);
}
}
const result = {
success: true,
nextActionSuggest: '检查规则上下文是否已召回，若未召回，调用 retrieve_rules 工具召回规则上下文',
sessionId: session.id,
workDir,
projectLocalDir,
materials,
uploadedFiles: filesToUpload.map(file => (0, node_path_1.basename)(file)),
skippedFiles: skippedFiles.map(file => (0, node_path_1.basename)(file)),
uploadErrors: uploadErrors.length > 0 ? uploadErrors : undefined,
};
return {
content: [
{
type: 'text',
text: JSON.stringify(result),
},
],
};
}
catch (error) {
// 不自动关闭session，让agent根据异常信息自行处理
return createErrorResponse(error, 'project-open');
}
});
server.registerTool('project-close', {
title: 'Close Project',
description: 'Close the current Cerevox session and release all resources.',
inputSchema: {
inMinutes: zod_1.z
.number()
.int()
.min(0)
.max(20)
.default(5)
.describe('Close the session after the specified number of minutes. Default is 5 minutes. 如果用户要求立即关闭会话，请将该参数设置为0！'),
},
}, async ({ inMinutes }) => {
try {
if (session) {
closeSessionTimerId = setTimeout(() => {
console.log('Closing Cerevox session...');
session?.close();
session = null;
}, inMinutes * 60 * 1000);
console.log('Session closed successfully');
}
else {
console.warn('No active session to close');
}
return {
content: [
{
type: 'text',
text: JSON.stringify({
success: true,
message: 'Project closed successfully.',
timestamp: new Date().toISOString(),
}),
},
],
};
}
catch (error) {
// 即使关闭失败，也要清理session引用
session = null;
return createErrorResponse(error, 'project-close');
}
});
// 列出项目下的所有文件
server.registerTool('list-project-files', {
title: 'List Project Files',
description: 'List all files in the materials directory.',
inputSchema: {},
}, async () => {
try {
// 验证session状态
const currentSession = await validateSession('list-project-files');
console.log('Listing project files...');
const terminal = currentSession.terminal;
if (!terminal) {
throw new Error('Terminal not available in current session');
}
let cwd;
try {
cwd = await terminal.getCwd();
}
catch (cwdError) {
console.error('Failed to get current working directory:', cwdError);
throw new Error('Failed to get current working directory');
}
console.log(`Current working directory: ${cwd}`);
// 安全地列出各目录文件，失败时返回空数组
const listFilesWithFallback = async (path, dirName) => {
try {
const files = await currentSession.files.listFiles(path);
console.log(`Found ${files?.length || 0} files in ${dirName}`);
return files || [];
}
catch (error) {
console.warn(`Failed to list files in ${dirName} (${path}):`, error);
return [];
}
};
const [rootFiles, materialsFiles, outputFiles] = await Promise.all([
listFilesWithFallback(cwd, 'root'),
listFilesWithFallback(`${cwd}/materials`, 'materials'),
listFilesWithFallback(`${cwd}/output`, 'output'),
]);
const result = {
success: true,
cwd,
root: rootFiles,
materials: materialsFiles,
output: outputFiles,
totalFiles: rootFiles.length + materialsFiles.length + outputFiles.length,
timestamp: new Date().toISOString(),
};
console.log(`Total files found: ${result.totalFiles}`);
return {
content: [
{
type: 'text',
text: JSON.stringify(result),
},
],
};
}
catch (error) {
return createErrorResponse(error, 'list-project-files');
}
});
server.registerTool('generate-character-image', {
title: 'Generate Character Image',
description: 'Generate a turnaround image or portrait for any character.',
inputSchema: {
type: zod_1.z.enum(['banana', 'seedream']).optional().default('banana'),
name: zod_1.z.string().describe('The name of the character.'),
gender: zod_1.z
.enum(['male', 'female'])
.describe('The gender of the character.'),
age: zod_1.z
.enum(['儿童', '少年', '青年', '中年', '老年'])
.describe('The age of the character.'),
appearance: zod_1.z.string().describe('The appearance of the character.'),
clothing: zod_1.z.string().describe('The clothing of the character.'),
personality: zod_1.z.string().describe('The personality of the character.'),
detail_features: zod_1.z
.string()
.default('无具体描述')
.describe('Detailed features and traits of the character.'),
style: zod_1.z
.string()
.default('写实')
.describe('构图风格，如写实、卡通、水墨画等'),
referenceImage: zod_1.z.string().optional().describe('形象参考图.'),
referenceImagePrompt: zod_1.z
.string()
.default('形象参考[图1]的人物形象\n')
.describe('形象参考图的提示文本.'),
isTurnaround: zod_1.z
.boolean()
.default(true)
.describe('是否生成三视图。true: 生成4096x3072的三视图，false: 生成2304x4096的竖版人物正视图'),
saveToFileName: zod_1.z.string().describe('The filename to save.'),
},
}, async ({ type, name, gender, age, appearance, clothing, personality, detail_features, style, saveToFileName, referenceImage, referenceImagePrompt, isTurnaround, }) => {
try {
// 验证session状态
const currentSession = await validateSession('generate-character-image');
const validatedFileName = validateFileName(saveToFileName);
// 根据 isTurnaround 参数生成不同的提示词和尺寸
let prompt;
let size;
let roleDescriptionPrompt = `角色名称：${name}
角色性别：${gender}
角色年龄：${age}
角色外观：${appearance}
角色服装：${clothing}
角色性格：${personality}
角色细节特征：${detail_features}
构图风格：${style}
`;
const ai = currentSession.ai;
try {
const promptOptimizer = await (0, promises_1.readFile)((0, node_path_1.resolve)(__dirname, './prompts/character-prompt-optimizer.md'), 'utf8');
const completion = await ai.getCompletions({
model: 'Doubao-Seed-1.6-flash',
messages: [
{
role: 'system',
content: promptOptimizer,
},
{
role: 'user',
content: roleDescriptionPrompt,
},
],
});
const optimizedPrompt = completion.choices[0]?.message?.content.trim();
if (optimizedPrompt) {
roleDescriptionPrompt = `${optimizedPrompt} 8K,超高细节,逼真质感。`;
}
}
catch (error) {
console.warn('Failed to optimize character prompt');
}
if (referenceImage) {
roleDescriptionPrompt = `${referenceImagePrompt}${roleDescriptionPrompt}`;
}
if (isTurnaround) {
// 生成三视图
prompt = `
你是一个专业的角色设计师，请根据角色设定生成角色全身三视图，画面左1/3部分是人物侧视图，中间1/3部分是人物正视图，右侧1/3部分是人物背视图，三部分都必须包括人物全身（从头到脚），构图远近、大小均相同，平面排布无透视，外表、服饰、形象保持完全一致，确保是由三部相机分别**同时**从同一角色侧面、正面和背面进行拍摄的画面。除了这三个人物形象构图外，不再有任何其他元素。图片为白底，图中不带任何文字。
## 核心约束
- 标准角色全身三视图（含正面 / 侧面 / 背面，三者形态统一）
- 构图风格: ${style}
- 背景：纯白色
- 角色表情正常（自然平视，嘴角平直），无剧情特征（仅展示角色设计，无动作叙事）
## 基础参数
- 超高清分辨率
## 三视图排列方式
- 横向排列（正面居左 / 侧面居中 / 背面居右），单视图中角色完整展示从头到脚，并占据各自画面大部分区域
## 角色主体：
- 角色性别：${gender}
- 角色年龄：${age}
- 体型：普通，体态自然挺直（正面挺胸平视，侧面肩线水平，背面脊柱端正，无歪斜）
## 服饰外观
- 角色外观：${appearance}
- 角色服饰：${clothing}
## 细节特征（定辨识度 + 场景融合）
- ${roleDescriptionPrompt}
- ${detail_features}
## 参考图
${referenceImage ? referenceImagePrompt : '无参考图'}
## 技术要求：
三视图角色整体（从头到脚）形态完全统一（正背面服装长度、配饰位置完全对应，无尺寸或形态差异），符合真实空间逻辑（正面可见的配饰，背面因遮挡不显示），线条清晰无杂乱笔触，色彩均匀，背景纯净无杂色或多余元素。
`.trim();
size = '4000x3000';
}
else {
// 生成竖版人物正视图
prompt = `
你是一个专业的角色设计师，请根据角色设定生成完整的全身正视图，角色面向观众，展现完整的身体比例和服装细节。图片为白底，图中不带任何文字。设定为：
${roleDescriptionPrompt}
`.trim();
size = '2160x3840';
}
let imageBase64Array;
if (referenceImage) {
try {
const imagePath = (0, node_path_1.dirname)(referenceImage) !== '.'
? referenceImage
: `./materials/${referenceImage}`;
// 需要得到当前项目的绝对路径
const imageFilePath = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), projectLocalDir, imagePath);
// 读取图片文件内容
const imageBuffer = await (0, promises_1.readFile)(imageFilePath);
const fileName = (0, node_path_1.basename)(imagePath);
const mimeType = fileName.toLowerCase().endsWith('.png')
? 'image/png'
: fileName.toLowerCase().endsWith('.jpg') ||
fileName.toLowerCase().endsWith('.jpeg')
? 'image/jpeg'
: 'image/png';
// 转换为base64编码
const base64String = `data:${mimeType};base64,${imageBuffer.toString('base64')}`;
imageBase64Array = [base64String];
}
catch (error) {
console.error(`Failed to load reference image ${referenceImage}：`, error);
return createErrorResponse(`Failed to load reference image ${referenceImage}: ${error}`, 'generate-image');
}
}
const res = await ai.generateImage({
type,
prompt,
size,
image: imageBase64Array,
});
if (!res) {
throw new Error('Failed to generate image: no response from AI service');
}
if (res.url) {
console.log('Image generated successfully, saving to materials...');
const uri = await saveMaterial(currentSession, res.url, validatedFileName);
const result = {
success: true,
// source: res.url,
uri,
prompt,
isTurnaround,
size,
timestamp: new Date().toISOString(),
};
return {
content: [
{
type: 'text',
text: JSON.stringify(result),
},
],
};
}
else {
console.warn('Image generation completed but no URL returned');
return {
content: [
{
type: 'text',
text: JSON.stringify({
success: false,
error: 'No image URL returned from AI service',
response: res,
timestamp: new Date().toISOString(),
}),
},
],
};
}
}
catch (error) {
return createErrorResponse(error, 'generate-character-image');
}
});
server.registerTool('generate-line-sketch', {
title: 'Generate Line Sketch',
description: 'Generate line sketch material based on user prompt.',
inputSchema: {
prompt: zod_1.z.string().describe('The prompt to generate line sketch.'),
saveToFileName: zod_1.z
.string()
.describe('The filename to save the generated line sketch.'),
},
}, async ({ prompt, saveToFileName }) => {
try {
// 验证session状态
await validateSession('generate-line-sketch');
// 验证文件名
validateFileName(saveToFileName);
// 调用AI生成线稿
const res = await session.ai.generateLineSketch({ prompt });
if (res && res.url) {
// 保存到本地
await saveMaterial(session, res.url, saveToFileName);
const result = {
success: true,
url: res.url,
localPath: getMaterialUri(session, saveToFileName),
timestamp: new Date().toISOString(),
};
return {
content: [
{
type: 'text',
text: JSON.stringify(result),
},
],
};
}
else {
throw new Error('No URL returned from AI service');
}
}
catch (error) {
return createErrorResponse(error, 'generate-line-sketch');
}
});
server.registerTool('upload-custom-material', {
title: 'Upload Custom Material',
description: 'Upload material files (images: jpeg/png, videos: mp4, audio: mp3) from the local filesystem to the materials directory. For video and audio files, duration information will be included in the response.',
inputSchema: {
localFileName: zod_1.z
.string()
.describe('The file name of the file to upload. Supported formats: jpeg, png, mp4, mp3'),
},
}, async ({ localFileName }) => {
try {
// 验证session状态
const currentSession = await validateSession('upload-custom-material');
// 构建本地文件路径，使用 ZEROCUT_PROJECT_CWD 环境变量
const validatedPath = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), projectLocalDir, 'materials', localFileName.trim());
// 验证本地文件存在性
if (!(0, node_fs_1.existsSync)(validatedPath)) {
throw new Error(`File not found: ${validatedPath}`);
}
const fileName = (0, node_path_1.basename)(validatedPath);
const validatedFileName = validateFileName(fileName);
// 检查文件格式
const fileExtension = fileName.toLowerCase().split('.').pop();
const allowedFormats = ['jpeg', 'jpg', 'png', 'mp4', 'mp3'];
if (!fileExtension || !allowedFormats.includes(fileExtension)) {
throw new Error(`Unsupported file format: ${fileExtension}. Allowed formats: ${allowedFormats.join(', ')}`);
}
console.log(`Uploading material: ${validatedPath} -> ${validatedFileName}`);
const files = currentSession.files;
// 直接上传到 sandbox 的 cerevox-zerocut 项目 materials 目录
const terminal = currentSession.terminal;
const materialsDir = `/home/user/cerevox-zerocut/projects/${terminal.id}/materials`;
await files.upload(validatedPath, `${materialsDir}/${validatedFileName}`);
// 通过 getMaterialUri 获取材料 URI
const materialUri = getMaterialUri(currentSession, validatedFileName);
// 检测媒体文件时长
let durationMs;
const isVideoOrAudio = ['mp4', 'mp3'].includes(fileExtension);
if (isVideoOrAudio) {
try {
// 使用现有的 getMediaDuration 函数获取时长（秒）
let durationSeconds = null;
if (fileExtension === 'mp4') {
durationSeconds = await (0, videokit_1.getMediaDuration)(validatedPath);
}
else if (fileExtension === 'mp3') {
durationSeconds = await (0, mp3_duration_1.default)(validatedPath);
}
if (durationSeconds !== null) {
durationMs = Math.round(durationSeconds * 1000); // 转换为毫秒
}
}
catch (error) {
console.warn(`Failed to get duration for ${validatedFileName}:`, error);
// 时长检测失败不影响上传，继续处理
}
}
const result = {
success: true,
uri: materialUri,
filename: validatedFileName,
timestamp: new Date().toISOString(),
};
// 如果是视频或音频文件且成功获取时长，添加到结果中
if (durationMs !== undefined) {
result.durationMs = durationMs;
}
console.log('Material uploaded successfully:', result);
try {
await updateMediaLogs(currentSession, validatedFileName, result);
}
catch (error) {
console.warn(`Failed to update media_logs.json for ${validatedFileName}:`, error);
}
return {
content: [
{
type: 'text',
text: JSON.stringify(result),
},
],
};
}
catch (error) {
return createErrorResponse(error, 'upload-custom-material');
}
});
server.registerTool('generate-image', {
title: 'Generate Image',
description: `生成图片`,
inputSchema: {
type: zod_1.z.enum(['banana', 'seedream']).optional().default('seedream'),
prompt: zod_1.z
.string()
.describe('The prompt to generate. 一般要严格对应 storyboard 中当前场景的 start_frame 或 end_frame 中的字段描述'),
sceneIndex: zod_1.z
.number()
.min(1)
.optional()
.describe('场景索引，从1开始的下标，如果非场景对应素材，则可不传，场景素材必传'),
storyBoardFile: zod_1.z
.string()
.optional()
.default('storyboard.json')
.describe('故事板文件路径'),
skipConsistencyCheck: zod_1.z
.boolean()
.optional()
.default(false)
.describe('是否跳过一致性检查，默认为false（即默认进行一致性检查）'),
skipCheckWithSceneReason: zod_1.z
.string()
.optional()
.describe('跳过校验的理由，如果skipConsistencyCheck设为true，必须要传这个参数'),
size: zod_1.z
.enum([
'1024x1024',
'864x1152',
'1152x864',
'1280x720',
'720x1280',
'832x1248',
'1248x832',
'1512x648',
// 2K
'2048x2048',
'1728x2304',
'2304x1728',
'2560x1440',
'1440x2560',
'1664x2496',
'2496x1664',
'3024x1456',
// 4K
'3840x2160',
'2160x3840',
'4000x3000',
'3000x4000',
'4096x4096',
'3072x4096',
'4096x3072',
'4096x2304',
'2304x4096',
'2731x4096',
'4096x2731',
'4096x1968',
])
.default('720x1280')
.describe('The size of the image.'),
saveToFileName: zod_1.z.string().describe('The filename to save.'),
watermark: zod_1.z
.boolean()
.optional()
.default(false)
.describe('Whether to add watermark to the image.'),
optimizePrompt: zod_1.z
.boolean()
.optional()
.default(false)
.describe('Whether to optimize the prompt.'),
referenceImages: zod_1.z
.array(zod_1.z.object({
image: zod_1.z.string().describe('Local image file path'),
type: zod_1.z
.enum(['character', 'object', 'background', 'linesketch'])
.describe('Type of the reference image. 必须传，如果是参考角色三视图，传character，如果是参考背景图，传background，如果是参考线稿，传linesketch，否则传object'),
name: zod_1.z.string().describe('Name for this reference image'),
description: zod_1.z
.string()
.optional()
.describe('Description for this reference image.如果是linesketch类型，这个参数是参考线稿的描述，建议传'),
isTurnaround: zod_1.z
.boolean()
.describe('Whether this is a turnaround image.如果是三视图，这个参数务必传true'),
}))
.optional()
.describe(`Array of reference images with character or object names.如果stage_atmosphere中有角色apply_reference_image，那么必须要传这个参数生成分镜图片
传参示例
\`\`\`
{
"image": "latiao.jpeg",
"type": "object",
"name": "卫龙辣条",
}
\`\`\`
`),
},
}, async ({ type = 'seedream', prompt, sceneIndex, storyBoardFile = 'storyboard.json', skipConsistencyCheck = false, size = '720x1280', saveToFileName, watermark, referenceImages, optimizePrompt, }) => {
try {
// 验证session状态
const currentSession = await validateSession('generate-image');
const storyBoardPath = (0, node_path_1.resolve)(process.env.ZEROCUT_PROJECT_CWD || process.cwd(), projectLocalDir, storyBoardFile);
// 检查 storyboard 标志
if (!checkStoryboardFlag && (0, node_fs_1.existsSync)(storyBoardPath)) {
checkStoryboardFlag = true;
return createErrorResponse('必须先审查生成的 storyboard.json 内容，确保每个场景中的stage_atmosphere内容按照规则被正确融合到start_frame和video_prompt中，不得遗漏，检查完成后先汇报，如果有问题，应当先修改 storyboard.json 内容，然后再调用 generate-image 生成图片。注意修改 storyboard 内容时，仅修改相应字段的字符串值，不要破坏JSON格式！', 'generate-image');
}
const validatedFileName = validateFileName(saveToFileName);
// 校验 prompt 与 storyboard.json 中场景设定的一致性
if (sceneIndex && !skipConsistencyCheck) {
try {
if ((0, node_fs_1.existsSync)(storyBoardPath)) {
const storyBoardContent = await (0, promises_1.readFile)(storyBoardPath, 'utf8');
// 检查 storyBoard JSON 语法合法性
let storyBoard;
try {
storyBoard = JSON.parse(storyBoardContent);
}
catch (jsonError) {
return createErrorResponse(`storyBoard 文件 ${storyBoardFile} 存在 JSON 语法错误，请修复后重试。错误详情: ${jsonError instanceof Error ? jsonError.message : String(jsonError)}`, 'generate-image');
}
if (storyBoard.scenes && Array.isArray(storyBoard.scenes)) {
const scene = storyBoard.scenes[sceneIndex - 1]; // sceneIndex 从1开始，数组从0开始
if (scene) {
const startFrame = scene.start_frame;
const endFrame = scene.end_frame;
// 检查 prompt 是否严格等于 start_frame 或 end_frame
if (prompt !== startFrame && prompt !== endFrame) {
return createErrorResponse('图片提示词必须严格遵照storyboard的设定，如果用户明确指出不需要遵守，请将skipConsistencyCheck设置为true后再次调用', 'generate-image');
}
// 校验 size 参数与 storyboard 的 orientation 属性一致性
if (size && storyBoard.orientation) {
const isLandscapeSize = [
'1152x864',
'1280x720',
'1248x832',
'1512x648',
'2304x1728',
'2560x1440',
'2496x1664',
'3024x1456',
'4096x3072',
'4096x2304',
'4096x2731',
'4096x1968',
].includes(size);
const isPortraitSize = [
'864x1152',
'720x1280',
'832x1248',
'1728x2304',
'1440x2560',
'1664x2496',
'3072x4096',
'2304x4096',
'2731x4096',
].includes(size);
const isSquareSize = [
'1024x1024',
'2048x2048',
'4096x4096',
].includes(size);
if (storyBoard.orientation === 'landscape' &&
!isLandscapeSize &&
!isSquareSize) {
return createErrorResponse(`故事板设定为横屏模式(orientation: landscape)，但生图尺寸 ${size} 为竖屏格式，请使用横屏尺寸如 1280x720、2560x1440、4096x2304 等`, 'generate-image');
}
if (storyBoard.orientation === 'portrait' &&
!isPortraitSize &&
!isSquareSize) {
return createErrorResponse(`故事板设定为竖屏模式(orientation: portrait)，但生图尺寸 ${size} 为横屏格式，请使用竖屏尺寸如 720x1280、1440x2560、2304x4096 等`, 'generate-image');
}
}
}
else {
console.warn(`Scene index ${sceneIndex} not found in storyboard.json`);
}
}
}
else {
console.warn(`Story board file not found: ${storyBoardPath}`);
}
}
catch (error) {
console.error('Failed to validate prompt with story board:', error);
// 如果读取或解析 storyboard.json 失败，继续执行但记录警告
}
}
// 检查并替换英文单引号包裹的中文内容为中文双引号
// 这样才能让 seedream 生成更好的中文文字
let processedPrompt = prompt.replace(/'([^']*[\u4e00-\u9fff][^']*)'/g, '“$1”');
if (optimizePrompt) {
try {
const ai = currentSession.ai;
const promptOptimizer = await (0, promises_1.readFile)((0, node_path_1.resolve)(__dirname, './prompts/image-prompt-optimizer.md'), 'utf8');
const completion = await ai.getCompletions({
model: 'Doubao-Seed-1.6-flash',
messages: [
{
role: 'system',
content: promptOptimizer,