contaigents
Version:
Modular AI Content Ecosystem with Audio Generation
527 lines (518 loc) • 19.6 kB
JavaScript
import { BaseTool } from './BaseTool.js';
import path from 'path';
import fs from 'fs/promises';
export class AudioProjectTool extends BaseTool {
constructor(baseDir) {
super();
this.baseDir = baseDir || process.cwd();
}
getName() {
return 'audio_project';
}
getDescription() {
return 'Create and manage simple audio projects with multiple clips. Supports creating JSON project files, adding audio clips with timing, and mixing/exporting to WAV format.';
}
getAgentGuidance() {
return `
## 🎵 AUDIO PROJECT BEST PRACTICES
When using the audio_project tool, follow these guidelines:
### 📁 PROJECT ORGANIZATION:
- **Use descriptive project names** like "podcast_episode_1.json"
- **Organize audio files** in an "audio/" subdirectory
- **Use relative paths** for portability
### ⏱️ TIMING CONSIDERATIONS:
- **start_time** is in seconds (e.g., 30.5 for 30.5 seconds)
- **Plan clip timing** to avoid unwanted overlaps
- **Leave gaps** between clips for natural pacing
### 🔊 VOLUME MANAGEMENT:
- **Default volume is 1.0** (full volume)
- **Use 0.0 to 1.0 range** for volume levels
- **Consider background music at 0.3-0.5** volume
- **Main narration typically at 0.8-1.0** volume
### 📝 XML FORMATTING FOR CLIPS ARRAY:
**IMPORTANT**: The clips parameter must be formatted as a JSON array in XML:
<clips>[{"file_path": "audio/intro.wav", "start_time": 0.0, "volume": 1.0}, {"file_path": "audio/main.wav", "start_time": 30.0, "volume": 0.9}]</clips>
### 🎯 WORKFLOW TIPS:
1. Generate audio files first with audio_generation tool
2. Create project with initial clips using proper JSON array format
3. Add more clips as needed
4. Mix and export when ready
### 📋 EXAMPLE USAGE:
<tool_call name="audio_project" id="1">
<operation>create</operation>
<project_path>projects/my_podcast.json</project_path>
<title>My Podcast Episode</title>
<clips>[{"file_path": "audio/intro.wav", "start_time": 0.0}, {"file_path": "audio/main.wav", "start_time": 15.0}]</clips>
</tool_call>
`;
}
getParameters() {
return [
{
name: 'operation',
type: 'string',
description: 'Operation to perform: create, add_clip, mix_export',
required: true
},
{
name: 'project_path',
type: 'string',
description: 'Path to the project JSON file',
required: true
},
{
name: 'clips',
type: 'array',
description: 'Array of clip objects with file_path, start_time, and optional volume (for create/add_clip operations)',
required: false
},
{
name: 'output_path',
type: 'string',
description: 'Output file path for mixed audio (for mix_export operation)',
required: false
},
{
name: 'title',
type: 'string',
description: 'Project title (for create operation)',
required: false
}
];
}
async execute(parameters) {
// Validate parameters
const validationError = this.validateParameters(parameters);
if (validationError) {
return validationError;
}
const { operation, project_path } = parameters;
try {
switch (operation) {
case 'create':
return await this.createProject(parameters);
case 'add_clip':
return await this.addClip(parameters);
case 'mix_export':
return await this.mixExport(parameters);
default:
return {
success: false,
error: `Unknown operation: ${operation}. Supported operations: create, add_clip, mix_export`
};
}
}
catch (error) {
return {
success: false,
error: `AudioProjectTool error: ${error.message}`
};
}
}
/**
* Create a new audio project
*/
async createProject(parameters) {
const { project_path, clips = [], title } = parameters;
// Resolve project path
const resolvedPath = path.resolve(this.baseDir, project_path);
// Ensure directory exists
const projectDir = path.dirname(resolvedPath);
await fs.mkdir(projectDir, { recursive: true });
// Check if project already exists
try {
await fs.access(resolvedPath);
return {
success: false,
error: `Project file already exists: ${project_path}`
};
}
catch {
// File doesn't exist, which is what we want
}
// Validate clips
const validationResult = await this.validateClips(clips);
if (!validationResult.success) {
return validationResult;
}
// Create project structure
const project = {
title: title || path.basename(project_path, '.json'),
clips: clips.map((clip) => ({
file_path: clip.file_path,
start_time: clip.start_time,
volume: clip.volume || 1.0
})),
created_at: new Date().toISOString(),
updated_at: new Date().toISOString()
};
// Save project file
await fs.writeFile(resolvedPath, JSON.stringify(project, null, 2));
return {
success: true,
data: {
project_path: project_path,
clips_count: project.clips.length,
title: project.title
},
message: `Audio project created: ${project_path} with ${project.clips.length} clips`
};
}
/**
* Add clips to an existing project
*/
async addClip(parameters) {
const { project_path, clips = [] } = parameters;
// Load existing project
const loadResult = await this.loadProject(project_path);
if (!loadResult.success) {
return loadResult;
}
const project = loadResult.data;
// Validate new clips
const validationResult = await this.validateClips(clips);
if (!validationResult.success) {
return validationResult;
}
// Add new clips
const newClips = clips.map((clip) => ({
file_path: clip.file_path,
start_time: clip.start_time,
volume: clip.volume || 1.0
}));
project.clips.push(...newClips);
project.updated_at = new Date().toISOString();
// Save updated project
const resolvedPath = path.resolve(this.baseDir, project_path);
await fs.writeFile(resolvedPath, JSON.stringify(project, null, 2));
return {
success: true,
data: {
project_path: project_path,
clips_added: newClips.length,
total_clips: project.clips.length
},
message: `Added ${newClips.length} clips to project. Total clips: ${project.clips.length}`
};
}
/**
* Mix project and export to WAV
*/
async mixExport(parameters) {
const { project_path, output_path } = parameters;
if (!output_path) {
return {
success: false,
error: 'output_path is required for mix_export operation'
};
}
// Load project
const loadResult = await this.loadProject(project_path);
if (!loadResult.success) {
return loadResult;
}
const project = loadResult.data;
if (project.clips.length === 0) {
return {
success: false,
error: 'Project has no clips to mix'
};
}
// Mix audio clips
const mixResult = await this.mixAudioClips(project.clips);
if (!mixResult.success) {
return mixResult;
}
// Save mixed audio
const resolvedOutputPath = path.resolve(this.baseDir, output_path);
const outputDir = path.dirname(resolvedOutputPath);
await fs.mkdir(outputDir, { recursive: true });
await fs.writeFile(resolvedOutputPath, mixResult.data.audioBuffer);
// Get file stats
const fileStats = await fs.stat(resolvedOutputPath);
return {
success: true,
data: {
output_path: output_path,
file_size: fileStats.size,
duration_seconds: mixResult.data.durationSeconds,
clips_mixed: project.clips.length
},
message: `Mixed ${project.clips.length} clips and exported to: ${output_path} (${this.formatFileSize(fileStats.size)})`
};
}
/**
* Load project from file
*/
async loadProject(projectPath) {
try {
const resolvedPath = path.resolve(this.baseDir, projectPath);
const projectData = await fs.readFile(resolvedPath, 'utf-8');
const project = JSON.parse(projectData);
return {
success: true,
data: project
};
}
catch (error) {
return {
success: false,
error: `Failed to load project: ${error.message}`
};
}
}
/**
* Validate clips array
*/
async validateClips(clips) {
if (!Array.isArray(clips)) {
return {
success: false,
error: 'clips must be an array'
};
}
for (let i = 0; i < clips.length; i++) {
const clip = clips[i];
if (!clip.file_path || typeof clip.file_path !== 'string') {
return {
success: false,
error: `Clip ${i}: file_path is required and must be a string`
};
}
if (typeof clip.start_time !== 'number' || clip.start_time < 0) {
return {
success: false,
error: `Clip ${i}: start_time must be a non-negative number`
};
}
if (clip.volume !== undefined && (typeof clip.volume !== 'number' || clip.volume < 0 || clip.volume > 1)) {
return {
success: false,
error: `Clip ${i}: volume must be a number between 0 and 1`
};
}
// Check if audio file exists
const audioPath = path.resolve(this.baseDir, clip.file_path);
try {
await fs.access(audioPath);
}
catch {
return {
success: false,
error: `Clip ${i}: Audio file not found: ${clip.file_path}`
};
}
}
return { success: true };
}
/**
* Format file size for display
*/
formatFileSize(bytes) {
if (bytes < 1024)
return `${bytes} B`;
if (bytes < 1024 * 1024)
return `${(bytes / 1024).toFixed(1)} KB`;
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
/**
* Mix audio clips into a single WAV file
*/
async mixAudioClips(clips) {
try {
// Load and parse all audio files
const audioData = [];
let maxSampleRate = 0;
let totalDurationSamples = 0;
for (const clip of clips) {
const audioPath = path.resolve(this.baseDir, clip.file_path);
const wavData = await this.loadWavFile(audioPath);
if (!wavData.success) {
return wavData;
}
const { samples, header } = wavData.data;
const startSample = Math.floor(clip.start_time * header.sampleRate);
const endSample = startSample + samples.length;
maxSampleRate = Math.max(maxSampleRate, header.sampleRate);
totalDurationSamples = Math.max(totalDurationSamples, endSample);
audioData.push({
samples,
startSample,
volume: clip.volume || 1.0,
sampleRate: header.sampleRate
});
}
// Create output buffer
const outputSamples = new Float32Array(totalDurationSamples);
// Mix all clips (simple additive mixing)
for (const audio of audioData) {
// Simple resampling if needed (basic nearest neighbor)
const resampleRatio = audio.sampleRate / maxSampleRate;
for (let i = 0; i < audio.samples.length; i++) {
const outputIndex = audio.startSample + Math.floor(i / resampleRatio);
if (outputIndex < outputSamples.length) {
outputSamples[outputIndex] += audio.samples[i] * audio.volume;
}
}
}
// Normalize to prevent clipping
let maxAmplitude = 0;
for (let i = 0; i < outputSamples.length; i++) {
maxAmplitude = Math.max(maxAmplitude, Math.abs(outputSamples[i]));
}
if (maxAmplitude > 1.0) {
const normalizationFactor = 0.95 / maxAmplitude; // Leave some headroom
for (let i = 0; i < outputSamples.length; i++) {
outputSamples[i] *= normalizationFactor;
}
}
// Convert to 16-bit PCM
const pcmData = new Int16Array(outputSamples.length);
for (let i = 0; i < outputSamples.length; i++) {
pcmData[i] = Math.max(-32768, Math.min(32767, Math.floor(outputSamples[i] * 32767)));
}
// Create WAV file with header
const wavBuffer = this.createWavFile(pcmData, maxSampleRate);
const durationSeconds = outputSamples.length / maxSampleRate;
return {
success: true,
data: {
audioBuffer: wavBuffer,
durationSeconds: durationSeconds
}
};
}
catch (error) {
return {
success: false,
error: `Failed to mix audio clips: ${error.message}`
};
}
}
/**
* Load and parse a WAV file
*/
async loadWavFile(filePath) {
try {
const buffer = await fs.readFile(filePath);
// Parse WAV header
const header = this.parseWavHeader(buffer);
if (!header) {
return {
success: false,
error: `Invalid WAV file: ${filePath}`
};
}
// Extract PCM data
const pcmData = buffer.subarray(header.dataOffset, header.dataOffset + header.dataLength);
// Convert to Float32Array based on bit depth
let samples;
if (header.bitsPerSample === 16) {
const int16Data = new Int16Array(pcmData.buffer, pcmData.byteOffset, pcmData.byteLength / 2);
samples = new Float32Array(int16Data.length);
for (let i = 0; i < int16Data.length; i++) {
samples[i] = int16Data[i] / 32768.0;
}
}
else {
return {
success: false,
error: `Unsupported bit depth: ${header.bitsPerSample}. Only 16-bit WAV files are supported.`
};
}
return {
success: true,
data: {
samples,
header
}
};
}
catch (error) {
return {
success: false,
error: `Failed to load WAV file ${filePath}: ${error.message}`
};
}
}
/**
* Parse WAV file header
*/
parseWavHeader(buffer) {
try {
// Check RIFF header
if (buffer.toString('ascii', 0, 4) !== 'RIFF')
return null;
if (buffer.toString('ascii', 8, 12) !== 'WAVE')
return null;
// Find fmt chunk
let offset = 12;
while (offset < buffer.length - 8) {
const chunkId = buffer.toString('ascii', offset, offset + 4);
const chunkSize = buffer.readUInt32LE(offset + 4);
if (chunkId === 'fmt ') {
const audioFormat = buffer.readUInt16LE(offset + 8);
if (audioFormat !== 1)
return null; // Only PCM supported
const numChannels = buffer.readUInt16LE(offset + 10);
const sampleRate = buffer.readUInt32LE(offset + 12);
const bitsPerSample = buffer.readUInt16LE(offset + 22);
// Find data chunk
let dataOffset = offset + 8 + chunkSize;
while (dataOffset < buffer.length - 8) {
const dataChunkId = buffer.toString('ascii', dataOffset, dataOffset + 4);
const dataChunkSize = buffer.readUInt32LE(dataOffset + 4);
if (dataChunkId === 'data') {
return {
sampleRate,
numChannels,
bitsPerSample,
dataOffset: dataOffset + 8,
dataLength: dataChunkSize
};
}
dataOffset += 8 + dataChunkSize;
}
}
offset += 8 + chunkSize;
}
return null;
}
catch {
return null;
}
}
/**
* Create WAV file buffer from PCM data
*/
createWavFile(pcmData, sampleRate) {
const numChannels = 1; // Mono
const bitsPerSample = 16;
const byteRate = sampleRate * numChannels * bitsPerSample / 8;
const blockAlign = numChannels * bitsPerSample / 8;
const dataLength = pcmData.length * 2; // 2 bytes per sample
const header = Buffer.alloc(44);
// RIFF header
header.write('RIFF', 0);
header.writeUInt32LE(36 + dataLength, 4);
header.write('WAVE', 8);
// Format chunk
header.write('fmt ', 12);
header.writeUInt32LE(16, 16);
header.writeUInt16LE(1, 20); // PCM format
header.writeUInt16LE(numChannels, 22);
header.writeUInt32LE(sampleRate, 24);
header.writeUInt32LE(byteRate, 28);
header.writeUInt16LE(blockAlign, 32);
header.writeUInt16LE(bitsPerSample, 34);
// Data chunk
header.write('data', 36);
header.writeUInt32LE(dataLength, 40);
// Convert PCM data to buffer
const pcmBuffer = Buffer.alloc(dataLength);
for (let i = 0; i < pcmData.length; i++) {
pcmBuffer.writeInt16LE(pcmData[i], i * 2);
}
return Buffer.concat([header, pcmBuffer]);
}
}