contaigents/dist/services/tools/FFmpegTool.js

Modular AI Content Ecosystem with Audio Generation

import { BaseTool } from './BaseTool.js';
import { exec } from 'child_process';
import { promisify } from 'util';
import path from 'path';
import fs from 'fs/promises';
const execAsync = promisify(exec);
export class FFmpegTool extends BaseTool {
    constructor(baseDir) {
        super();
        this.maxTimeout = 300000; // 5 minutes for video processing
        this.baseDir = baseDir || process.cwd();
    }
    getName() {
        return 'ffmpeg';
    }
    getDescription() {
        return 'Comprehensive FFmpeg tool for audio and video processing. Supports format conversion, trimming, merging, filtering, and more.';
    }
    getParameters() {
        return [
            {
                name: 'operation',
                type: 'string',
                description: 'The FFmpeg operation to perform (extract_audio, convert_format, trim, merge, resize, overlay, social_media_convert, detect_shots, etc.)',
                required: true
            },
            {
                name: 'input_file',
                type: 'string',
                description: 'Path to the input file (relative to base directory)',
                required: true
            },
            {
                name: 'output_file',
                type: 'string',
                description: 'Path to the output file (relative to base directory)',
                required: true
            },
            {
                name: 'start_time',
                type: 'string',
                description: 'Start time for trimming (format: HH:MM:SS or seconds)',
                required: false
            },
            {
                name: 'duration',
                type: 'string',
                description: 'Duration for trimming (format: HH:MM:SS or seconds)',
                required: false
            },
            {
                name: 'end_time',
                type: 'string',
                description: 'End time for trimming (format: HH:MM:SS or seconds)',
                required: false
            },
            {
                name: 'format',
                type: 'string',
                description: 'Target format (mp3, wav, aac, flac, mp4, mkv, webm, mov, gif)',
                required: false
            },
            {
                name: 'quality',
                type: 'string',
                description: 'Quality setting (high, medium, low, or specific bitrate like 320k)',
                required: false,
                default: 'medium'
            },
            {
                name: 'resolution',
                type: 'string',
                description: 'Video resolution (1920x1080, 1280x720, 854x480, etc.)',
                required: false
            },
            {
                name: 'sample_rate',
                type: 'string',
                description: 'Audio sample rate (16000, 22050, 44100, 48000)',
                required: false
            },
            {
                name: 'channels',
                type: 'string',
                description: 'Audio channels (mono, stereo, or number like 1, 2)',
                required: false
            },
            {
                name: 'volume',
                type: 'string',
                description: 'Volume adjustment (0.5 for half, 2.0 for double, or dB like +3dB)',
                required: false
            },
            {
                name: 'speed',
                type: 'string',
                description: 'Playback speed multiplier (0.5 for half speed, 2.0 for double speed)',
                required: false
            },
            {
                name: 'overlay_file',
                type: 'string',
                description: 'Path to overlay image/video file for overlay operations',
                required: false
            },
            {
                name: 'overlay_position',
                type: 'string',
                description: 'Overlay position (top-left, top-right, bottom-left, bottom-right, center, or x:y coordinates)',
                required: false,
                default: 'top-right'
            },
            {
                name: 'text',
                type: 'string',
                description: 'Text to overlay on video',
                required: false
            },
            {
                name: 'font_size',
                type: 'string',
                description: 'Font size for text overlay',
                required: false,
                default: '24'
            },
            {
                name: 'additional_files',
                type: 'array',
                description: 'Additional input files for merge operations',
                required: false
            },
            {
                name: 'platform',
                type: 'string',
                description: 'Target social media platform (instagram_reel, tiktok, youtube_shorts, instagram_post, youtube)',
                required: false
            },
            {
                name: 'crop_strategy',
                type: 'string',
                description: 'Cropping strategy (center, top, bottom, left, right)',
                required: false,
                default: 'center'
            },
            {
                name: 'custom_options',
                type: 'string',
                description: 'Custom FFmpeg options to append to the command',
                required: false
            },
            {
                name: 'scene_threshold',
                type: 'string',
                description: 'Scene detection threshold (0.0-1.0, default: 0.3). Lower values detect more shots.',
                required: false,
                default: '0.3'
            },
            {
                name: 'output_format',
                type: 'string',
                description: 'Output format for shot detection: "timestamps" (default) or "metadata"',
                required: false,
                default: 'timestamps'
            }
        ];
    }
    getAgentGuidance() {
        return `
🔊 AUDIO OPERATIONS:
- extract_audio: Extract audio from video (-vn flag)
- convert_format: Convert between mp3, wav, aac, flac formats
- resample: Change sample rate (16kHz, 44.1kHz, 48kHz)
- normalize: Adjust/normalize volume levels
- trim: Clip audio from start_time to end_time or for duration
- merge: Concatenate or mix multiple audio tracks
- channels: Convert between mono/stereo
- filters: Apply noise reduction, silence removal, speed/pitch changes

🎥 VIDEO OPERATIONS:
- extract_frames: Extract images at intervals
- trim: Cut video segments
- resize: Scale video resolution
- convert_format: Change codec/container (mp4, mkv, webm, mov)
- social_media_convert: PREFERRED for social media platforms (Instagram, TikTok, YouTube)
- overlay: Add watermarks, images, or text
- extract_gif: Create GIF from video segment
- merge: Concatenate multiple videos
- transcode: Optimize for web (H.264/AAC in MP4)
- speed: Change playback speed
- detect_shots: Analyze video for shot boundaries/scene changes (outputs timestamps or metadata)

🎬 AUDIO + VIDEO SYNC:
- replace_audio: Replace video's audio track (dub/voiceover)
- combine: Merge separate audio and video files

🎯 VIDEO ANALYSIS & SPLITTING WORKFLOW:
- detect_shots: Find shot boundaries → timestamps.txt
- Then use trim operations to split: trim each segment using the detected timestamps
- Example: detect_shots → [0, 2.52, 8.14, 15.3] → trim 0-2.52, trim 2.52-8.14, trim 8.14-15.3

⚠️ CRITICAL ASPECT RATIO AWARENESS:
When converting videos for social media platforms:
1. ALWAYS check source video dimensions first using execute_command with ffprobe
2. For Instagram Reels (9:16 ratio): If source is landscape, use CENTER CROPPING not stretching
3. For YouTube Shorts (9:16): Same as Instagram Reels
4. For TikTok (9:16): Same as Instagram Reels
5. If user says video looks "squeezed" or "stretched", the aspect ratio was likely forced - recrop from original

🎯 SMART CROPPING EXAMPLES:
- Instagram Reel from landscape: Check source dimensions, calculate center crop to 9:16 ratio, then scale
- Square to vertical: Crop center portion that fits 9:16, then scale to 1080x1920
- Portrait to landscape: Add letterboxing or crop top/bottom

EXAMPLES:
- Extract audio: operation="extract_audio", input_file="video.mp4", output_file="audio.mp3"
- Trim video: operation="trim", input_file="video.mp4", output_file="clip.mp4", start_time="00:01:23", end_time="00:02:15"
- Convert format: operation="convert_format", input_file="audio.wav", output_file="audio.mp3", format="mp3"
- Resize video: operation="resize", input_file="video.mp4", output_file="small.mp4", resolution="1280x720"
- Add watermark: operation="overlay", input_file="video.mp4", output_file="watermarked.mp4", overlay_file="logo.png", overlay_position="top-right"
- Instagram Reel: operation="social_media_convert", input_file="video.mp4", output_file="reel.mp4", platform="instagram_reel"
- TikTok: operation="social_media_convert", input_file="video.mp4", output_file="tiktok.mp4", platform="tiktok"
- YouTube: operation="social_media_convert", input_file="video.mp4", output_file="youtube.mp4", platform="youtube"
- Detect shots: operation="detect_shots", input_file="video.mp4", output_file="shots.txt", scene_threshold="0.3"
- Split by shots: First detect shots, then use trim operation with timestamps from shots.txt
`;
    }
    async execute(parameters) {
        // Validate parameters
        const validationError = this.validateParameters(parameters);
        if (validationError) {
            return validationError;
        }
        const { operation, input_file, output_file } = parameters;
        // Smart aspect ratio validation for social media operations
        if (operation === 'social_media_convert' || operation === 'resize' ||
            (operation === 'convert_format' && parameters.platform)) {
            const aspectRatioWarning = await this.validateAspectRatio(input_file, parameters);
            if (aspectRatioWarning) {
                // Add warning to the operation but continue
                console.warn(`⚠️ Aspect Ratio Warning: ${aspectRatioWarning}`);
            }
        }
        try {
            // Check if FFmpeg is available
            await this.checkFFmpegAvailability();
            // Resolve file paths
            const inputPath = path.resolve(this.baseDir, input_file);
            let outputPath = path.resolve(this.baseDir, output_file);
            // Security check - ensure paths are within base directory
            if (!inputPath.startsWith(this.baseDir) || !outputPath.startsWith(this.baseDir)) {
                return {
                    success: false,
                    error: 'File paths must be within the base directory'
                };
            }
            // Check if input file exists
            try {
                await fs.access(inputPath);
            }
            catch {
                return {
                    success: false,
                    error: `Input file not found: ${input_file}`
                };
            }
            // Build FFmpeg command based on operation
            const command = await this.buildFFmpegCommand(operation, inputPath, outputPath, parameters);
            if (!command) {
                return {
                    success: false,
                    error: `Unsupported operation: ${operation}`
                };
            }
            // Execute FFmpeg command
            const startTime = Date.now();
            const result = await execAsync(command, {
                cwd: this.baseDir,
                timeout: this.maxTimeout,
                maxBuffer: 1024 * 1024 * 50 // 50MB buffer for large outputs
            });
            const executionTime = Date.now() - startTime;
            // Check if output file was created
            let outputStats;
            try {
                outputStats = await fs.stat(outputPath);
            }
            catch {
                // For extract_frames, check if any frame files were created
                if (operation === 'extract_frames' && outputPath.includes('%')) {
                    const dir = path.dirname(outputPath);
                    try {
                        const files = await fs.readdir(dir);
                        const frameFiles = files.filter(f => f.includes('frame') && f.endsWith('.png'));
                        if (frameFiles.length > 0) {
                            const firstFramePath = path.join(dir, frameFiles[0]);
                            outputStats = await fs.stat(firstFramePath);
                            // Update outputPath to the first frame for response
                            outputPath = firstFramePath;
                        }
                        else {
                            return {
                                success: false,
                                error: 'No frame files were created successfully'
                            };
                        }
                    }
                    catch {
                        return {
                            success: false,
                            error: 'Output directory not accessible or no frames created'
                        };
                    }
                }
                else {
                    return {
                        success: false,
                        error: 'Output file was not created successfully'
                    };
                }
            }
            return {
                success: true,
                data: {
                    operation,
                    input_file: path.relative(this.baseDir, inputPath),
                    output_file: path.relative(this.baseDir, outputPath),
                    file_size: outputStats.size,
                    execution_time: executionTime,
                    command: command.replace(this.baseDir, '.'), // Hide full paths in output
                    stdout: result.stdout,
                    stderr: result.stderr
                },
                message: `FFmpeg ${operation} completed successfully. Output: ${path.relative(this.baseDir, outputPath)} (${this.formatFileSize(outputStats.size)})`
            };
        }
        catch (error) {
            return {
                success: false,
                error: `FFmpeg operation failed: ${error.message}`,
                data: {
                    operation,
                    stdout: error.stdout || '',
                    stderr: error.stderr || '',
                    command: error.cmd || ''
                }
            };
        }
    }
    async checkFFmpegAvailability() {
        try {
            await execAsync('ffmpeg -version');
        }
        catch (error) {
            throw new Error('FFmpeg is not installed or not available in PATH. Please install FFmpeg first.');
        }
    }
    formatFileSize(bytes) {
        const units = ['B', 'KB', 'MB', 'GB'];
        let size = bytes;
        let unitIndex = 0;
        while (size >= 1024 && unitIndex < units.length - 1) {
            size /= 1024;
            unitIndex++;
        }
        return `${size.toFixed(1)}${units[unitIndex]}`;
    }
    async buildFFmpegCommand(operation, inputPath, outputPath, params) {
        const { start_time, duration, end_time, format, quality = 'medium', resolution, sample_rate, channels, volume, speed, overlay_file, overlay_position = 'top-right', text, font_size = '24', additional_files = [], custom_options = '', scene_threshold = '0.3', output_format = 'timestamps' } = params;
        let command = `ffmpeg -i "${inputPath}"`;
        switch (operation) {
            case 'extract_audio':
                command += ' -vn'; // No video
                if (format)
                    command += ` -acodec ${this.getAudioCodec(format)}`;
                if (sample_rate)
                    command += ` -ar ${sample_rate}`;
                if (channels)
                    command += ` -ac ${this.getChannelCount(channels)}`;
                break;
            case 'convert_format':
                if (format) {
                    const codec = this.getCodecForFormat(format);
                    if (codec.video)
                        command += ` -vcodec ${codec.video}`;
                    if (codec.audio)
                        command += ` -acodec ${codec.audio}`;
                }
                if (quality)
                    command += ` ${this.getQualityOptions(quality, format)}`;
                break;
            case 'trim':
                if (start_time)
                    command += ` -ss ${start_time}`;
                if (duration)
                    command += ` -t ${duration}`;
                else if (end_time)
                    command += ` -to ${end_time}`;
                break;
            case 'resize':
                if (resolution)
                    command += ` -vf scale=${resolution}`;
                break;
            case 'resample':
                if (sample_rate)
                    command += ` -ar ${sample_rate}`;
                if (channels)
                    command += ` -ac ${this.getChannelCount(channels)}`;
                break;
            case 'normalize':
                command += ' -af loudnorm';
                if (volume)
                    command += `,volume=${volume}`;
                break;
            case 'speed':
                if (speed) {
                    const speedVal = parseFloat(speed);
                    command += ` -filter:v "setpts=${1 / speedVal}*PTS" -filter:a "atempo=${speedVal}"`;
                }
                break;
            case 'overlay':
                if (overlay_file) {
                    const overlayPath = path.resolve(this.baseDir, overlay_file);
                    command = `ffmpeg -i "${inputPath}" -i "${overlayPath}"`;
                    const position = this.getOverlayPosition(overlay_position);
                    command += ` -filter_complex "[0:v][1:v]overlay=${position}"`;
                }
                else if (text) {
                    command += ` -vf "drawtext=text='${text}':fontsize=${font_size}:fontcolor=white:x=10:y=10"`;
                }
                break;
            case 'extract_frames':
                const fps = params.fps || '1';
                command += ` -vf fps=${fps}`;
                // For extract_frames, we need to handle the output path differently
                // If it contains %d pattern, use as-is, otherwise modify it
                if (!outputPath.includes('%')) {
                    const dir = path.dirname(outputPath);
                    const name = path.basename(outputPath, path.extname(outputPath));
                    const ext = path.extname(outputPath) || '.png';
                    outputPath = path.join(dir, `${name}_%04d${ext}`);
                }
                break;
            case 'extract_gif':
                if (start_time)
                    command += ` -ss ${start_time}`;
                if (duration)
                    command += ` -t ${duration}`;
                command += ' -vf "fps=10,scale=320:-1:flags=lanczos"';
                break;
            case 'merge':
                if (additional_files && additional_files.length > 0) {
                    // Create concat file list
                    const fileList = [inputPath, ...additional_files.map((f) => path.resolve(this.baseDir, f))];
                    command = 'ffmpeg';
                    fileList.forEach(file => {
                        command += ` -i "${file}"`;
                    });
                    command += ` -filter_complex "concat=n=${fileList.length}:v=1:a=1"`;
                }
                break;
            case 'replace_audio':
                if (additional_files && additional_files.length > 0) {
                    const audioPath = path.resolve(this.baseDir, additional_files[0]);
                    command = `ffmpeg -i "${inputPath}" -i "${audioPath}" -c:v copy -c:a aac -map 0:v:0 -map 1:a:0`;
                }
                break;
            case 'combine':
                if (additional_files && additional_files.length > 0) {
                    const audioPath = path.resolve(this.baseDir, additional_files[0]);
                    command = `ffmpeg -i "${inputPath}" -i "${audioPath}" -c:v copy -c:a aac`;
                }
                break;
            case 'transcode':
                command += ' -c:v libx264 -c:a aac -preset medium -crf 23';
                break;
            case 'social_media_convert':
                // Smart social media conversion with aspect ratio detection
                const platform = params.platform || 'instagram_reel';
                const cropStrategy = params.crop_strategy || 'center';
                switch (platform.toLowerCase()) {
                    case 'instagram_reel':
                    case 'tiktok':
                    case 'youtube_shorts':
                        // Target: 9:16 aspect ratio (1080x1920)
                        // First scale to ensure we have enough pixels, then crop to exact ratio
                        command += ' -vf "scale=1080:1920:force_original_aspect_ratio=increase,crop=1080:1920"';
                        break;
                    case 'instagram_post':
                        // Target: 1:1 aspect ratio (1080x1080)
                        command += ' -vf "scale=1080:1080:force_original_aspect_ratio=increase,crop=1080:1080"';
                        break;
                    case 'youtube':
                        // Target: 16:9 aspect ratio (1920x1080)
                        command += ' -vf "scale=1920:1080:force_original_aspect_ratio=increase,crop=1920:1080"';
                        break;
                }
                command += ' -c:v libx264 -c:a aac -preset medium -crf 23';
                break;
            case 'detect_shots':
                return this.buildShotDetectionCommand(inputPath, outputPath, params);
            default:
                return null;
        }
        // Add custom options if provided
        if (custom_options) {
            command += ` ${custom_options}`;
        }
        // Add output file and overwrite flag
        command += ` "${outputPath}" -y`;
        return command;
    }
    getAudioCodec(format) {
        const codecs = {
            'mp3': 'libmp3lame',
            'aac': 'aac',
            'wav': 'pcm_s16le',
            'flac': 'flac',
            'ogg': 'libvorbis'
        };
        return codecs[format] || 'copy';
    }
    getCodecForFormat(format) {
        const codecs = {
            'mp4': { video: 'libx264', audio: 'aac' },
            'mkv': { video: 'libx264', audio: 'aac' },
            'webm': { video: 'libvpx-vp9', audio: 'libopus' },
            'mov': { video: 'libx264', audio: 'aac' },
            'avi': { video: 'libx264', audio: 'mp3' },
            'mp3': { audio: 'libmp3lame' },
            'wav': { audio: 'pcm_s16le' },
            'flac': { audio: 'flac' }
        };
        return codecs[format] || { video: 'copy', audio: 'copy' };
    }
    getQualityOptions(quality, format) {
        if (quality.includes('k') || quality.includes('M')) {
            return `-b:a ${quality}`;
        }
        const isVideo = format && ['mp4', 'mkv', 'webm', 'mov', 'avi'].includes(format);
        switch (quality) {
            case 'high':
                return isVideo ? '-crf 18 -b:a 320k' : '-b:a 320k';
            case 'medium':
                return isVideo ? '-crf 23 -b:a 192k' : '-b:a 192k';
            case 'low':
                return isVideo ? '-crf 28 -b:a 128k' : '-b:a 128k';
            default:
                return '';
        }
    }
    getChannelCount(channels) {
        switch (channels.toLowerCase()) {
            case 'mono': return '1';
            case 'stereo': return '2';
            default: return channels;
        }
    }
    getOverlayPosition(position) {
        switch (position.toLowerCase()) {
            case 'top-left': return '10:10';
            case 'top-right': return 'W-w-10:10';
            case 'bottom-left': return '10:H-h-10';
            case 'bottom-right': return 'W-w-10:H-h-10';
            case 'center': return '(W-w)/2:(H-h)/2';
            default: return position; // Assume it's coordinates like "100:50"
        }
    }
    async validateAspectRatio(inputFile, params) {
        try {
            const inputPath = path.resolve(this.baseDir, inputFile);
            // Get video dimensions using ffprobe
            const result = await execAsync(`ffprobe -v error -select_streams v:0 -show_entries stream=width,height -of csv=s=x:p=0 "${inputPath}"`, { cwd: this.baseDir, timeout: 10000 });
            const dimensions = result.stdout.trim();
            if (!dimensions)
                return null;
            const [width, height] = dimensions.split('x').map(Number);
            if (!width || !height)
                return null;
            const sourceRatio = width / height;
            const platform = params.platform?.toLowerCase();
            // Define target ratios for different platforms
            const targetRatios = {
                'instagram_reel': { ratio: 9 / 16, name: '9:16 (vertical)' },
                'tiktok': { ratio: 9 / 16, name: '9:16 (vertical)' },
                'youtube_shorts': { ratio: 9 / 16, name: '9:16 (vertical)' },
                'instagram_post': { ratio: 1, name: '1:1 (square)' },
                'youtube': { ratio: 16 / 9, name: '16:9 (landscape)' }
            };
            if (platform && targetRatios[platform]) {
                const target = targetRatios[platform];
                const ratioDiff = Math.abs(sourceRatio - target.ratio);
                // If aspect ratios are significantly different, warn about potential cropping
                if (ratioDiff > 0.1) {
                    const sourceDesc = sourceRatio > 1 ? 'landscape' : sourceRatio < 1 ? 'portrait' : 'square';
                    return `Source video is ${sourceDesc} (${width}x${height}, ratio ${sourceRatio.toFixed(2)}) but target is ${target.name} (ratio ${target.ratio.toFixed(2)}). Content will be cropped to fit.`;
                }
            }
            // Check for common "squeezed" video indicators
            if (params.operation === 'resize' && params.resolution) {
                const [targetW, targetH] = params.resolution.split('x').map(Number);
                if (targetW && targetH) {
                    const targetRatio = targetW / targetH;
                    const ratioDiff = Math.abs(sourceRatio - targetRatio);
                    if (ratioDiff > 0.1) {
                        return `Resizing from ${width}x${height} (ratio ${sourceRatio.toFixed(2)}) to ${targetW}x${targetH} (ratio ${targetRatio.toFixed(2)}) may cause stretching. Consider cropping first.`;
                    }
                }
            }
            return null;
        }
        catch (error) {
            // Don't fail the operation, just skip validation
            return null;
        }
    }
    /**
     * Build FFmpeg command for shot detection
     *
     * WORKFLOW FOR SPLITTING VIDEOS BY SHOTS:
     * 1. First run detect_shots to get timestamps: operation="detect_shots", output_file="shots.txt"
     * 2. Read the timestamps from shots.txt
     * 3. Use trim operation for each segment: operation="trim", start_time="0", end_time="2.52", output_file="shot_001.mp4"
     * 4. Continue with next segment: operation="trim", start_time="2.52", end_time="8.14", output_file="shot_002.mp4"
     *
     * This approach gives you full control over naming, format, and processing of each shot.
     */
    buildShotDetectionCommand(inputPath, outputPath, params) {
        const { scene_threshold = '0.3', output_format = 'timestamps' } = params;
        switch (output_format.toLowerCase()) {
            case 'timestamps':
                // Output timestamps to a text file using scene detection
                return `ffmpeg -i "${inputPath}" -vf "select='gt(scene,${scene_threshold})',showinfo" -vsync vfr -f null - 2>&1 | grep -o "pts_time:[0-9.]*" | sed 's/pts_time://' > "${outputPath}"`;
            case 'metadata':
                // Output detailed scene detection information
                return `ffmpeg -i "${inputPath}" -vf "select='gt(scene,${scene_threshold})',metadata=print:file=${outputPath}" -f null - 2>&1`;
            default:
                // Default to timestamps
                return `ffmpeg -i "${inputPath}" -vf "select='gt(scene,${scene_threshold})',showinfo" -vsync vfr -f null - 2>&1 | grep -o "pts_time:[0-9.]*" | sed 's/pts_time://' > "${outputPath}"`;
        }
    }
}