cerevox/dist/core/ai.js

TypeScript SDK for browser automation and secure command execution in highly available and scalable micro computer environments

"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AI = void 0;
const constants_1 = require("../utils/constants");
const base_1 = require("./base");
const session_1 = require("./session");
const path_1 = __importDefault(require("path"));
const coze_1 = require("../utils/coze");
const uuid_1 = require("uuid");
async function waitForWorkflowFinish(taskUrl, apiKey = process.env.CEREVOX_API_KEY, logger, onProgress, interval = 1500) {
    while (true) {
        const resData = await (0, coze_1.queryWorkflowRunHistory)(taskUrl, apiKey);
        logger.debug(`get workflow status: ${JSON.stringify(resData)}`);
        console.debug(`get workflow status: ${JSON.stringify(resData)}`);
        onProgress?.(resData);
        if (resData.status === 'Fail') {
            const errMsg = JSON.stringify({
                ...resData,
                debug_url: undefined,
            });
            throw new Error(`Video generation failed: ${errMsg}`);
        }
        if (resData.status === 'Success') {
            return {
                ...JSON.parse(resData.data.Output),
                // debug_url: resData.debug_url,
            };
        }
        await new Promise(resolve => setTimeout(resolve, interval || 1500));
    }
}
async function waitForVideoTaskComplete(sandbox, taskUrl, logger, onProgress, interval = 1500) {
    const headers = {
        'Content-Type': 'application/json',
    };
    while (true) {
        const response = await sandbox.request(taskUrl, {
            method: 'GET',
            headers: headers,
        });
        if (!response.ok) {
            throw new Error(`Video generation failed: ${response.statusText}`);
        }
        const data = await response.json();
        logger.debug(`get video task status: ${JSON.stringify(data)}`);
        onProgress?.(data);
        if (data.status === 'succeeded' || data.status === 'succeed') {
            return data;
        }
        if (data.status === 'failed' || data.status === 'Fail' || !response.ok) {
            throw new Error(`Video generation failed: ${JSON.stringify(data)}`);
        }
        if (data.status === 'canceled') {
            throw new Error(`Video generation canceled: ${JSON.stringify(data)}`);
        }
        await new Promise(resolve => setTimeout(resolve, interval || 1500));
    }
}
async function waitForMusicTaskComplete(sandbox, taskUrl, logger, onProgress, interval = 1500) {
    const headers = {
        'Content-Type': 'application/json',
    };
    while (true) {
        const response = await sandbox.request(taskUrl, {
            method: 'GET',
            headers: headers,
        });
        if (!response.ok) {
            throw new Error(`Video generation failed: ${response.statusText}`);
        }
        const data = await response.json();
        logger.debug(`get music task status: ${JSON.stringify(data)}`);
        onProgress?.(data);
        const result = data.Result;
        if (result?.Status === 2) {
            return result.SongDetail;
        }
        if (result?.Status >= 3) {
            throw new Error(`BGM generation failed: ${JSON.stringify(data)}`);
        }
        await new Promise(resolve => setTimeout(resolve, interval || 1500));
    }
}
let AI = class AI extends base_1.BaseClass {
    constructor(session) {
        super(session.getLogger().level);
        this.session = session;
    }
    async getModels() {
        const res = await this.session.sandbox.request(`/ai/models`, {
            method: 'GET',
            headers: {
                'Content-Type': 'application/json',
            },
        });
        const data = await res.json();
        return data;
    }
    async generateImageSeries(options) {
        try {
            if (Array.isArray(options.prompts)) {
                const count = options.prompts.length;
                options.prompts = options.prompts
                    .map((p, i) => `图片${i + 1}: ${p.trim()}`)
                    .join('\n');
                options.prompts = `生成${count}张图片\n${options.prompts.trim()}`;
            }
            if (options.refPrefix) {
                options.prompts = `${options.refPrefix}

参考以上图片，执行以下各场景绘图：

${options.prompts};
`;
            }
            const max_count = options.max_count ?? 15;
            const payload = {
                prompt: options.prompts,
                size: options.size ?? '1280x720',
                watermark: options.watermark ?? false,
                image: options.image,
                max_images: max_count,
            };
            // 启动心跳机制
            let heartbeatInterval = null;
            if (options.onProgress) {
                heartbeatInterval = setInterval(() => {
                    options.onProgress?.({});
                }, 60000); // 每60秒调用一次
            }
            try {
                const res = await this.session.sandbox.request('/ai/image/sequential_generate', {
                    method: 'POST',
                    headers: {
                        'Content-Type': 'application/json',
                    },
                    body: JSON.stringify(payload),
                });
                const data = await res.json();
                // 清除心跳定时器
                if (heartbeatInterval) {
                    clearInterval(heartbeatInterval);
                }
                if (data.data) {
                    const urls = data.data
                        .map((item) => item.url)
                        .filter((url) => url?.trim());
                    for (const url of urls) {
                        this.session.track('Image Generated', { url });
                    }
                    return {
                        prompt: options.prompts,
                        data,
                        urls,
                    };
                }
                return data;
            }
            catch (error) {
                // 确保在出错时也清除心跳定时器
                if (heartbeatInterval) {
                    clearInterval(heartbeatInterval);
                }
                throw error;
            }
        }
        catch (error) {
            this.logger.error('generateImage error', error);
            return { error: `generateImage error: ${error}` };
        }
    }
    async generateImage(options) {
        try {
            const model = options.type === 'banana'
                ? 'Cloudsway-MaaS_Banana'
                : 'Doubao-Seedream-4.0';
            const res = await this.session.sandbox.request(`/ai/image/generate`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    model,
                    ...options,
                }),
            });
            const data = await res.json();
            if (data.data?.[0]?.url) {
                this.session.track('Image Generated', data);
                return {
                    url: data.data?.[0]?.url,
                };
            }
            else if (data.url) {
                // banana
                this.session.track('Image Generated', data);
                return data;
            }
            throw new Error(`generateImage error: ${JSON.stringify(data)}`);
        }
        catch (error) {
            this.logger.error('generateImage error', error);
            this.session.track('Image Generated', { error: error.message });
            throw error;
        }
    }
    async generateLineSketch(options) {
        try {
            const res = await this.session.sandbox.request(`/ai/image/line_sketch`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify(options),
            });
            const data = await res.json();
            if (data.url) {
                this.session.track('Line Sketch Generated', data);
                return data;
            }
            throw new Error(`generateLineSketch error: ${JSON.stringify(data)}`);
        }
        catch (error) {
            this.logger.error('generateLineSketch error', error);
            this.session.track('Line Sketch Generated', { error: error.message });
            throw error;
        }
    }
    async extendVideo(options) {
        try {
            const waitForFinish = options.waitForFinish ?? true;
            const res = await this.session.sandbox.request(`/ai/video/extend`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify(options),
            });
            const data = await res.json();
            if (data.taskUrl && waitForFinish) {
                return this.waitForVideoTaskComplete({
                    taskUrl: data.taskUrl,
                    onProgress: options.onProgress,
                    timeout: 300000,
                });
            }
            return data;
        }
        catch (error) {
            this.logger.error('Extend Video error', error);
            return { error: error.message };
        }
    }
    async referencesToVideo(options) {
        try {
            const waitForFinish = options.waitForFinish ?? true;
            const watermark = options.watermark ?? true;
            const resolution = '720p';
            const ratio = options.size === '1280x720' ? '16:9' : '9:16';
            const mute = options.mute ?? false;
            if (options.type === 'lite') {
                options.prompt = `${options.prompt} --rs ${resolution} --rt ${ratio} --camerafixed false --watermark ${watermark} --mute ${mute}`;
            }
            const res = await this.session.sandbox.request(`/ai/references/to/video/generate`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    ...options,
                    resolution,
                    mute,
                }),
            });
            const data = await res.json();
            if (data.taskUrl && waitForFinish) {
                return this.waitForVideoTaskComplete({
                    taskUrl: data.taskUrl,
                    onProgress: options.onProgress,
                    timeout: 300000,
                });
            }
            return data;
        }
        catch (error) {
            this.logger.error('Reference to Video error', error);
            return { error: error.message };
        }
    }
    // 用自己的模型生成 video
    async generateZeroCutVideo(options) {
        try {
            const waitForFinish = options.waitForFinish ?? true;
            const model = options.model ?? 'zero-1.0-fast';
            const parameters = {
                prompt: options.prompt,
                duration: options.duration,
                start_frame: options.startFrame,
                highlight_frame: options.highlightFrame,
                end_frame: options.endFrame,
                resolution: options.resolution ?? (model === 'zero-1.0' ? '1080p' : '720p'),
            };
            const workflowId = options.model === 'zero-1.0'
                ? '7570606875394572288'
                : '7571823801022414911';
            const result = await this.runCozeWorkflow(workflowId, parameters, options.onProgress, 1500, waitForFinish);
            return result;
        }
        catch (error) {
            this.logger.error('Generate ZeroCut Video error', error);
            return { error: error.message, content: null };
        }
    }
    async framesToVideo(options) {
        try {
            const watermark = options.watermark ?? true;
            const waitForFinish = options.waitForFinish ?? true;
            if (options.type === 'lite' || options.type === 'pro') {
                options.prompt = `${options.prompt} --ratio adaptive --camerafixed false --watermark ${watermark}`;
            }
            if (options.type === 'zero' || options.type === 'zero-fast') {
                const model = options.type === 'zero' ? 'zero-1.0' : 'zero-1.0-fast';
                const result = await this.generateZeroCutVideo({
                    model,
                    prompt: options.prompt,
                    duration: options.duration,
                    startFrame: options.start_frame,
                    highlightFrame: options.end_frame,
                    resolution: options.resolution,
                    onProgress: options.onProgress,
                    waitForFinish,
                });
                if (result.content) {
                    return {
                        url: result.content.video_url,
                        duration: options.duration,
                        resolution: result.content.resolution || '1080p',
                        ratio: 'auto',
                    };
                }
                return result;
            }
            const res = await this.session.sandbox.request(`/ai/frames/to/video/generate`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify(options),
            });
            const data = await res.json();
            if (data.taskUrl && waitForFinish) {
                return this.waitForVideoTaskComplete({
                    taskUrl: data.taskUrl,
                    onProgress: options.onProgress,
                    timeout: 300000,
                });
            }
            return data;
        }
        catch (error) {
            this.logger.error('framesToVideo error', error);
            return { error: error.message };
        }
    }
    async generateMusic(options) {
        const outputSchema = {
            name: 'Music',
            schema: {
                type: 'object',
                properties: {
                    type: {
                        type: 'string',
                        enum: ['bgm', 'song:doubao', 'song:minimax'],
                        description: 'The type of the music. bgm for background music, song for full song.',
                    },
                    duration: {
                        type: 'number',
                        description: 'The duration of the generated music in seconds.',
                    },
                    lyrics: {
                        type: 'string',
                        description: 'The lyrics of the music.',
                    },
                    genre: {
                        type: 'string',
                        enum: [
                            'Folk',
                            'Pop',
                            'Rock',
                            'Chinese Style',
                            'Hip Hop/Rap',
                            'R&B/Soul',
                            'Punk',
                            'Electronic',
                            'Jazz',
                            'Reggae',
                            'DJ',
                            'Pop Punk',
                            'Disco',
                            'Future Bass',
                            'Pop Rap',
                            'Trap Rap',
                            'R&B Rap',
                            'Chinoiserie Electronic',
                            'GuFeng Music',
                            'Pop Rock',
                            'Jazz Pop',
                            'Bossa Nova',
                            'Contemporary R&B',
                        ],
                        description: 'The genre of the music.',
                    },
                    mood: {
                        type: 'string',
                        enum: [
                            'Happy',
                            'Dynamic/Energetic',
                            'Sentimental/Melancholic/Lonely',
                            'Inspirational/Hopeful',
                            'Nostalgic/Memory',
                            'Excited',
                            'Sorrow/Sad',
                            'Chill',
                            'Relaxing',
                            'Romantic',
                            'Miss',
                            'Groovy/Funky',
                            'Dreamy/Ethereal',
                            'Calm/Relaxing',
                        ],
                        description: 'The mood of the music.',
                    },
                    gender: {
                        type: 'string',
                        enum: ['Female', 'Male'],
                        description: 'The gender of the singer.',
                    },
                    timbre: {
                        type: 'string',
                        enum: [
                            'Warm',
                            'Bright',
                            'Husky',
                            'Electrified voice',
                            'Sweet_AUDIO_TIMBRE',
                            'Cute_AUDIO_TIMBRE',
                            'Loud and sonorous',
                            'Powerful',
                            'Sexy/Lazy',
                        ],
                        description: 'The timbre of the music.',
                    },
                },
                required: ['type', 'duration'],
            },
        };
        const systemPrompt = `你是一位专业的音乐制作人，根据用户的描述创作音乐

音乐有两种类型，一种是带歌词的（Song)，另一种是纯音乐（BGM），根据用户具体要求创作

## 创作规则
- 分析用户描述，确定音乐的类型、时长、流派、情绪、性别、音色等

- 如果用户要求是生成 BGM，无歌词，否则应为用户生成歌词
- 如果用户提供了具体歌词，根据歌词创作，可少量改写，但主歌（verse）应严格遵守用户歌词内容
- 如果用户未提供具体歌词或者仅提供了歌词主题，你应该根据歌曲长度，帮助用户创作完整歌词

### 完整歌词通常包括以下桥段：
  - 前奏: intro，歌曲开始的音乐部分，主要用于引导歌曲的整体氛围
  - 主歌: verse，通常在前奏之后，歌曲中叙述歌曲故事或主题的部分
  - 副歌: chorus，一般在主歌之后，旋律有记忆点和感染力，是整首歌的高潮，进一步强化歌曲的主题和情感
  - 间奏: inst，歌曲中的纯音乐段落，用于连接不同的演唱部分
  - 尾奏: outro，歌曲结束后的音乐段落，用于营造歌曲结束的氛围
  - 桥段: bridge，通常出现在歌曲中段或接近结尾处，是一个过渡部分，用于连接不同的歌曲段落。

* 歌词示例

\`\`\`txt
[intro]
[verse]
记得那一天 那一天我们相恋
说好彼此都不说再见
遵守诺言 用心去相恋
我为你撑伞 你为我取暖
[inst]
[chorus]
当我把心交给你的那一天
你却消失在我的眼前
事到如今已经过了好多年
是否你还像从前
[outro]
\`\`\` 

## ‼️ 注意事项

- 歌词内容会被完整唱出，所以**不要**加入任何括号内的补充说明文字，包括[inst]桥段，[inst]桥段不需要任何文字
- genre、mood、timbre 是枚举字符串，只能选择其中一项，要严格遵守输出 schema 定义！

## 乐曲长度
- 如果用户要求生成的是 BGM，默认 30 秒，最少 30 秒，最多 120 秒
- 如果用户要求生成的是歌曲，默认 60 秒，最少 30 秒，最多 120 秒

## 输出
根据 schema 输出合法 JSON
`;
        const creationPayload = {
            model: 'Doubao-Seed-1.6-flash',
            messages: [
                {
                    role: 'system',
                    content: systemPrompt.trim(),
                },
                {
                    role: 'user',
                    content: options.prompt.trim(),
                },
            ],
            response_format: {
                type: 'json_schema',
                json_schema: outputSchema,
            },
        };
        const completion = await this.getCompletions(creationPayload);
        // console.log(completion);
        const creationResult = completion.choices[0]?.message?.content;
        if (!creationResult) {
            throw new Error('Failed to generate music');
        }
        try {
            const musicParams = JSON.parse(creationResult);
            console.log('generate music params ->', JSON.stringify(musicParams, null, 2));
            const { type, duration, lyrics, genre, mood, gender, timbre } = musicParams;
            if (!type || !duration) {
                throw new Error('Invalid music parameters');
            }
            if (type === 'bgm') {
                const finalPrompt = `${options.prompt}

---

${lyrics ? `lyrics: ${lyrics}` : ''}
${genre ? `genre: ${genre}` : ''}
${mood ? `mood: ${mood}` : ''}
${gender ? `gender: ${gender}` : ''}
${timbre ? `timbre: ${timbre}` : ''}`;
                return this.generateBGM({
                    duration,
                    prompt: finalPrompt,
                    onProgress: options.onProgress,
                });
            }
            else {
                const model = type.split(':')[1];
                return this.generateSong({
                    type: model,
                    lyrics,
                    duration,
                    genre,
                    mood,
                    gender,
                    timbre,
                    skipCopyCheck: options.skipCopyCheck,
                    onProgress: options.onProgress,
                });
            }
        }
        catch (ex) {
            throw new Error('Failed to parse music creation result');
        }
    }
    async generateSong(options) {
        try {
            const waitForFinish = options.waitForFinish ?? true;
            const payload = {
                Lyrics: options.lyrics,
                Duration: options.duration,
                Genre: options.genre ?? undefined,
                Mood: options.mood ?? undefined,
                Gender: options.gender ?? undefined,
                Timbre: options.timbre ?? undefined,
                SkipCopyCheck: options.skipCopyCheck,
            };
            console.log('generate-song payload ->', JSON.stringify(payload, null, 2));
            if (options.type === 'doubao') {
                const res = await this.session.sandbox.request(`/ai/music/song/generate`, {
                    method: 'POST',
                    headers: {
                        'Content-Type': 'application/json',
                    },
                    body: JSON.stringify(payload),
                });
                const data = await res.json();
                if (waitForFinish && data.taskUrl) {
                    const result = await this.waitForMusicTaskComplete({
                        taskUrl: data.taskUrl,
                        onProgress: options.onProgress,
                    });
                    if (result) {
                        return {
                            ...result,
                            lyrics: options.lyrics,
                        };
                    }
                }
                return data;
            }
            else {
                const prompt = `歌曲 genre：${options.genre || 'auto'}
歌曲 mood：${options.mood || 'auto'}
歌手 gender：${options.gender || 'auto'}
歌曲 timbre：${options.timbre || 'auto'}
歌曲时长约${options.duration}秒`;
                const res = await this.session.sandbox.request('/ai/music/song/generate/minimax', {
                    method: 'POST',
                    headers: {
                        'Content-Type': 'application/json',
                    },
                    body: JSON.stringify({
                        prompt,
                        lyrics: options.lyrics,
                    }),
                });
                const data = await res.json();
                const url = data.data.audio;
                if (url) {
                    this.session.track('Song Generated', {
                        id: data.data.id,
                        duration: data.data.duration,
                        prompt,
                    });
                    const captions = await this.voiceToCaptions({
                        url,
                    });
                    return {
                        url,
                        duration: data.extra_info.music_duration / 1000,
                        lyrics: options.lyrics,
                        captions,
                        prompt,
                    };
                }
                return data;
            }
        }
        catch (error) {
            this.logger.error('generateBGM error', error);
            return { error: error.message };
        }
    }
    async generateBGM(options) {
        try {
            const waitForFinish = options.waitForFinish ?? true;
            const res = await this.session.sandbox.request(`/ai/music/bgm/generate`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    Text: options.prompt,
                    Duration: options.duration,
                    Segments: options.segments,
                }),
            });
            const data = await res.json();
            if (waitForFinish && data.taskUrl) {
                return this.waitForMusicTaskComplete({
                    taskUrl: data.taskUrl,
                    onProgress: options.onProgress,
                });
            }
            return data;
        }
        catch (error) {
            this.logger.error('generateBGM error', error);
            return { error: error.message };
        }
    }
    async voiceToCaptions(options) {
        try {
            const res = await this.session.sandbox.request(`/ai/voice_to_caption`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    url: options.url,
                }),
            });
            const data = await res.json();
            this.session.track('Caption Generated', data);
            return data;
        }
        catch (error) {
            this.logger.error('voiceToCaption error', error);
            return { error: error.message };
        }
    }
    // 火山引擎版本
    async textToSpeechVolc(options) {
        const speech_rate = options.speed ?? 0; // -50 ~ 100
        const loudness_rate = options.volume ?? 0; // -50 ~ 100
        const emotion = options.emotion ?? 'storytelling';
        const explicit_language = options.explicit_language ?? 'zh';
        // 随机停顿 300 ～ 500 ms 模拟人对话
        const silence_duration = options.silence_duration ?? Math.floor(Math.random() * 200 + 300);
        try {
            const res = await this.session.sandbox.request(`/ai/text_to_speech/generate_volc`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    speaker: options.speaker,
                    text: options.text,
                    speech_rate,
                    loudness_rate,
                    emotion,
                    silence_duration,
                    explicit_language,
                    context_texts: options.context_texts,
                }),
            });
            const data = await res.json();
            this.session.track('TTS Generated', data);
            if (data.url && options.voice_to_caption) {
                try {
                    const res = await this.session.sandbox.request(`/ai/voice_to_caption`, {
                        method: 'POST',
                        headers: {
                            'Content-Type': 'application/json',
                        },
                        body: JSON.stringify({
                            url: data.url,
                        }),
                    });
                    data.captions = await res.json();
                    this.session.track('Caption Generated', data.captions);
                }
                catch (error) {
                    this.logger.error('voiceToCaption error', error);
                }
            }
            return data;
        }
        catch (error) {
            this.logger.error('textToSpeechVolc error', error);
            return { error: error.message };
        }
    }
    // minimax 版本的 tts
    async textToSpeech(options) {
        // TODO 支持分句生成
        try {
            const res = await this.session.sandbox.request(`/ai/text_to_speech/generate`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify(options),
            });
            const data = await res.json();
            this.session.track('TTS Generated', data);
            if (data.url && options.voice_to_caption) {
                try {
                    const res = await this.session.sandbox.request(`/ai/voice_to_caption`, {
                        method: 'POST',
                        headers: {
                            'Content-Type': 'application/json',
                        },
                        body: JSON.stringify({
                            url: data.url,
                        }),
                    });
                    data.captions = await res.json();
                    this.session.track('Caption Generated', data.captions);
                }
                catch (error) {
                    this.logger.error('voiceToCaption error', error);
                }
            }
            return data;
        }
        catch (error) {
            this.logger.error('textToSpeech error', error);
            return { error: error.message };
        }
    }
    async searchText(options) {
        try {
            options.count = options.count || 10;
            options.offset = options.offset || 0;
            options.type = options.type || 'smart';
            const res = await this.session.sandbox.request(`/ai/search/text?q=${encodeURIComponent(options.q)}&count=${options.count}&offset=${options.offset}&type=${options.type}`, {
                method: 'GET',
                headers: {
                    'Content-Type': 'application/json',
                },
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error('searchText error', error);
            return { error: error.message };
        }
    }
    async searchImage(options) {
        try {
            options.count = options.count || 10;
            options.offset = options.offset || 0;
            const res = await this.session.sandbox.request(`/ai/search/image?q=${encodeURIComponent(options.q)}&count=${options.count}&offset=${options.offset}`, {
                method: 'GET',
                headers: {
                    'Content-Type': 'application/json',
                },
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error('searchImage error', error);
            return { error: error.message };
        }
    }
    async generateSoundEffect(options) {
        try {
            const res = await this.session.sandbox.request(`/ai/sound_effect/generate`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify(options),
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error('generateSoundEffect error', error);
            return { error: error.message };
        }
    }
    async getCompletions(options) {
        try {
            const res = await this.session.sandbox.request(`/ai/chat/completions`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify(options),
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error(`getCompositions error: ${error.message}`);
            return { error: error.message };
        }
    }
    async actionImitation(options) {
        try {
            const waitForFinish = options.waitForFinish ?? true;
            const res = await this.session.sandbox.request(`/ai/imitation/volcano`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    image_url: options.imageUrl,
                    video_url: options.videoUrl,
                }),
            });
            const data = await res.json();
            if (data.taskUrl && waitForFinish) {
                return this.waitForVideoTaskComplete({
                    taskUrl: data.taskUrl,
                    onProgress: options.onProgress,
                    timeout: 300000,
                });
            }
            return data;
        }
        catch (error) {
            this.logger.error(`actionImitation error: ${error.message}`);
            return { error: error.message };
        }
    }
    async voSync(options) {
        try {
            const { videoUrl, audioUrl, audioInMs = 0, audioFadeOutMs = 0 } = options;
            const res = await this.session.sandbox.request(`/ai/vosync`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    video_url: videoUrl,
                    audio_url: audioUrl,
                    audio_in_ms: audioInMs,
                    audio_fade_out_ms: audioFadeOutMs,
                }),
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error(`voSync error: ${error.message}`);
            return { error: error.message };
        }
    }
    async voiceDesign(options) {
        try {
            const res = await this.session.sandbox.request(`/ai/voice/design`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    prompt: options.prompt,
                    preview_text: options.previewText,
                }),
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error(`voiceDesign error: ${error.message}`);
            return { error: error.message };
        }
    }
    async lipSync(options) {
        try {
            const waitForFinish = options.waitForFinish ?? true;
            const type = options.type || 'pixv';
            if (options.ref_photo_url && type !== 'vidu') {
                throw new Error('ref_photo_url is only supported for vidu model');
            }
            const endpoint = type === 'vidu'
                ? '/ai/lipsync'
                : type === 'pixv'
                    ? '/ai/lipsync/pixverse'
                    : '/ai/lipsync/volcano';
            const req_key = type === 'basic'
                ? 'realman_change_lips_basic_chimera'
                : 'realman_change_lips';
            const res = await this.session.sandbox.request(endpoint, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    video_url: options.videoUrl,
                    audio_url: options.audioUrl,
                    audio_in_ms: options.audioInMs || 0,
                    pad_audio: options.pad_audio,
                    ref_photo_url: options.ref_photo_url, // 这个参数只对 vidu 生效，可以多人对口型
                    // 后三个参数只对 basic 和 lite 生效
                    req_key,
                    separate_vocal: false,
                    open_scenedet: false,
                }),
            });
            const data = await res.json();
            if (data.taskUrl && waitForFinish) {
                return this.waitForVideoTaskComplete({
                    taskUrl: data.taskUrl,
                    onProgress: options.onProgress,
                    timeout: 300000,
                });
            }
            return data;
        }
        catch (error) {
            this.logger.error('generateSoundEffect error', error);
            return { error: error.message };
        }
    }
    async concatMedia(options) {
        try {
            const res = await this.session.sandbox.request(`/ai/media/concat`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    urls: options.mediaUrls,
                }),
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error('getMediaMetadata error', error);
            return { error: error.message };
        }
    }
    async getMediaMetadata(url) {
        try {
            const res = await this.session.sandbox.request(`/ai/metadata`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    url,
                }),
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error('getMediaMetadata error', error);
            return { error: error.message };
        }
    }
    async mediaTrim(options) {
        try {
            const res = await this.session.sandbox.request(`/ai/media/trim`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    url: options.mediaUrl,
                    startTime: options.startTime,
                    endTime: options.endTime,
                }),
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error('mediaTrim error', error);
            return { error: error.message };
        }
    }
    async mixAudios(options) {
        try {
            const res = await this.session.sandbox.request(`/ai/audio/mix`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    voice: options.voiceUrl,
                    bgm: options.bgmUrl,
                    gain: options.gain || -15,
                    ducking: options.ducking || false,
                }),
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error('mixAudios error', error);
            return { error: error.message };
        }
    }
    async muteVideo(options) {
        try {
            const res = await this.session.sandbox.request(`/ai/media/mute`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    url: options.mediaUrl,
                }),
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error('muteVideo error', error);
            return { error: error.message };
        }
    }
    async padAudio(options) {
        try {
            const res = await this.session.sandbox.request(`/ai/audio/pad`, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({
                    audio: options.audioUrl,
                    inMs: options.inMs || 0,
                    fadeOutMs: options.fadeOutMs || 0,
                    duration: options.duration || 0,
                }),
            });
            const data = await res.json();
            return data;
        }
        catch (error) {
            this.logger.error('padAudio error', error);
            return { error: error.message };
        }
    }
    async transferToCoze(options) {
        try {
            if (!options.mediaUrl && !options.text) {
                throw new Error('mediaUrl or text is required');
            }
            let buffer;
            let filename;
            if (options.mediaUrl) {
                // eslint-disable-next-line custom/no-fetch-in-src
                const res = await fetch(options.mediaUrl);
                if (!res.ok) {
                    throw new Error(`Failed to fetch media: ${res.statusText}`);
                }
                buffer = Buffer.from(await res.arrayBuffer());
                const url = new URL(options.mediaUrl);
                filename = path_1.default.basename(url.pathname);
            }
            else {
                filename = `${(0, uuid_1.v4)()}.txt`;
                buffer = Buffer.from(options.text || '');
            }
            const data = await (0, coze_1.transferToCoze)(buffer, filename, this.session.sandbox.token);
            return data;
        }
        catch (error) {
            this.logger.error('transferToCoze error', error);
            return { error: error.message };
        }
    }
    async generateVideoByTemplate(options) {
        return this.runCozeWorkflow(options.templateId, options.parameters, undefined, 1500, options.waitForFinish ?? true);
    }
    async getCozeWorkflowInfo(workflowId) {
        return (0, coze_1.getWorkflowInfo)(workflowId, this.session.sandbox.token);
    }
    async runCozeWorkflow(workflowId, parameters, onProgress, interval = 1500, waitForFinish = true) {
        const apiKey = this.session.sandbox.token;
        const process = await (0, coze_1.runWorkflow)(workflowId, {
            api_key: this.session.sandbox.token,
            sandbox_id: this.session.sandbox.sandboxId,
            ...parameters,
        }, true, apiKey);
        console.log({ 'zerocut-process': process });
        const executeId = process.execute_id;
        const taskUrl = `https://api.coze.cn/v1/workflows/${workflowId}/run_histories/${executeId}`;
        this.session.sandbox.request('/ai/workflow/save', {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
            },
            body: JSON.stringify({
                workflowId,
                executeId,
                debugUrl: process.debug_url,
            }),
        });
        if (!waitForFinish) {
            return {
                taskUrl,
                ...process,
            };
        }
        return this.waitForWorkflowFinish({
            taskUrl,
            onProgress,
            interval,
        });
    }
    async waitForWorkflowFinish(options) {
        try {
            const { taskUrl, onProgress, interval = 1500, timeout = 900000, } = options;
            const result = await Promise.race([
                waitForWorkflowFinish(taskUrl, this.session.sandbox.token, this.logger, onProgress, interval),
                new Promise(resolve => {
                    setTimeout(() => {
                        resolve({
                            status: 'running',
                            reason: `wait timeout ${timeout}ms`,
                            taskUrl,
                        });
                    }, timeout);
                }),
            ]);
            if (result?.content) {
                this.session.track('Video Generated', {
                    taskUrl,
                });
                return {
                    url: result.content.video_url,
                    duration: result.duration,
                    resolution: result.resolution,
                    ratio: 'auto',
                };
            }
            this.session.track('Video Creation Failed', {
                result: JSON.stringify(result),
            });
            throw new Error(`Video Creation Failed: ${JSON.stringify(result)}`);
        }
        catch (error) {
            this.logger.error('waitForWorkflowFinish error', error);
            this.session.track('Video Creation Failed', {
                reason: error.message,
            });
            throw error;
        }
    }
    async waitForVideoTaskComplete(options) {
        try {
            const { taskUrl, onProgress, interval = 1500, traceWorkflow = true, timeout = 900000, } = options;
            if (taskUrl.startsWith('https://api.coze.cn/v1/workflows/')) {
                return this.waitForWorkflowFinish(options);
            }
            if (traceWorkflow) {
                // 启动一个coze工作流作为记录和监控
                this.runCozeWorkflow('7574320984539791400', {
                    taskUrl,
                }, undefined, undefined, false).catch(error => {
                    this.logger.error('runCozeWorkflow error', error);
                });
            }
            const result = await Promise.race([
                waitForVideoTaskComplete(this.session.sandbox, taskUrl, this.logger, onProgress, interval),
                new Promise(resolve => {
                    setTimeout(() => {
                        resolve({
                            status: 'running',
                            reason: `wait timeout ${timeout}ms`,
                            taskUrl,
                        });
                    }, timeout);
                }),
            ]);
            if (result && result.content?.video_url) {
                this.session.track('Video Generated', {
                    id: result.id,
                    model: result.model,
                    fps: result.framespersecond,
                    duration: result.duration,
                    resolution: result.resolution,
                    ratio: result.ratio,
                    cost: result.usage?.total_tokens,
                });
                return {
                    url: result.content.video_url,
                    last_frame_url: result.content?.last_frame_url,
                    duration: result.duration,
                    resolution: result.resolution,
                    ratio: result.ratio,
                };
            }
            this.session.track('Video Creation Failed', {
                result: JSON.stringify(result),
            });
            throw new Error(`Video Creation Failed: ${JSON.stringify(result)}`);
        }
        catch (error) {
            this.logger.error('waitForVideoTaskComplete error', error);
            this.session.track('Video Creation Failed', {
                reason: error.message,
            });
            throw error;
        }
    }
    async waitForMusicTaskComplete(options) {
        try {
            const { taskUrl, onProgress, interval = 1500 } = options;
            const result = await waitForMusicTaskComplete(this.session.sandbox, taskUrl, this.logger, onProgress, interval);
            if (result) {
                this.session.track('Music Generated', {
                    id: result.TaskID,
                    duration: result.Duration,
                    prompt: result.Prompt,
                    captions: result.Captions,
                });
                return {
                    url: result.AudioUrl,
                    duration: result.Duration,
                    prompt: result.Prompt,
                    captions: result.Captions,
                };
            }
            this.session.track('Music Creation Failed', {
                result: '',
            });
        }
        catch (error) {
            this.logger.error('waitForMusicTaskComplete error', error);
            this.session.track('Music Creation Failed', {
                reason: error.message,
            });
            throw error;
        }
    }
};
exports.AI = AI;
exports.AI = AI = __decorate([
    (0, base_1.Logger)({ VERSION: constants_1.VERSION }),
    __metadata("design:paramtypes", [session_1.Session])
], AI);
//# sourceMappingURL=ai.js.map