contaigents
Version:
Modular AI Content Ecosystem with Audio Generation
687 lines (677 loc) • 29.8 kB
JavaScript
import { BaseTool } from './BaseTool.js';
import { AudioGenerator } from '../audioGenerator.js';
import { LLMFactory } from '../llm/LLMFactory.js';
import path from 'path';
import fs from 'fs/promises';
export class AudioGenerationTool extends BaseTool {
constructor(baseDir) {
super();
this.baseDir = baseDir || process.cwd();
}
getName() {
return 'audio_generation';
}
getDescription() {
return 'Generate audio files from text using AI providers. Supports speech generation (Gemini TTS, Echogarden local TTS) and music generation (Lyria 2 + local MusicGen fallback). Automatically saves files to specified paths.';
}
getAgentGuidance() {
return `
## 🎵 AUDIO GENERATION BEST PRACTICES
When using the audio_generation tool, consider these guidelines for optimal results:
### 🎤 SPEECH GENERATION (content_type: "speech"):
- **Use clear, natural language** that flows well when spoken
- **Add punctuation** for proper pauses and intonation
- **Break up long sentences** for better pacing
- **Consider pronunciation** of technical terms or names
### 🎭 VOICE SELECTION (for speech):
**Gemini TTS (cloud-based):**
- **Achernar** (Female): Bright, energetic voice
- **Achird** (Male): Deep, authoritative voice
- **Algenib** (Male): Smooth, professional voice
- **Algieba** (Male): Warm, friendly voice
- **Alnilam** (Male): Clear, articulate voice
- **Aoede** (Female): Melodic, expressive voice
- **Autonoe** (Female): Gentle, soothing voice
- **Callirrhoe** (Female): Elegant, refined voice
- **Charon** (Male): Strong, commanding voice
- **Despina** (Female): Professional, clear voice
- **Enceladus** (Male): Calm, measured voice
- **Erinome** (Female): Soft, nurturing voice
- **Fenrir** (Male): Bold, dynamic voice
- **Gacrux** (Female): Rich, mature voice
- **Iapetus** (Male): Steady, reliable voice
- **Kore** (Female): Youthful, vibrant voice
- **Laomedeia** (Female): Graceful, flowing voice
- **Leda** (Female): Light, cheerful voice
- **Orus** (Male): Resonant, powerful voice
- **Pulcherrima** (Female): Beautiful, captivating voice
- **Puck** (Male): Playful, animated voice
- **Rasalgethi** (Male): Distinguished, noble voice
- **Sadachbia** (Male): Thoughtful, contemplative voice
- **Sadaltager** (Male): Confident, assured voice
- **Schedar** (Male): Wise, experienced voice
- **Sulafat** (Female): Bright, clear voice
- **Umbriel** (Male): Dark, mysterious voice
- **Vindemiatrix** (Female): Crisp, precise voice
- **Zephyr** (Female): Warm, friendly voice (default)
- **Zubenelgenubi** (Male): Balanced, harmonious voice
- **Choose based on content type and desired tone** (casual vs formal, energetic vs calm)
**Echogarden (local/offline):**
- **Bella**: Natural female voice (default for echogarden)
- **Nicole**: Professional female voice
- **Sarah**: Warm female voice
- **Nova**: Modern female voice
- **Sky**: Clear female voice
- **Jessica**: Friendly female voice
- **River**: Calm female voice
- **Emma**: Expressive female voice
- **No API key required** - runs completely offline with high-quality Kokoro neural voices
### 🎵 MUSIC GENERATION (content_type: "music"):
- **Describe the mood and style** clearly (e.g., "upbeat electronic", "calm ambient")
- **Specify genre** when relevant (jazz, classical, rock, electronic)
- **Include tempo** if important (fast, slow, moderate)
- **Mention instruments** for specific sounds (piano, guitar, strings)
- **Use negative_prompt** to avoid unwanted elements
- **Set prefer_local: true** for offline generation or high-volume usage
### 🎯 MUSIC STYLE EXAMPLES:
- "Upbeat electronic background music for technology podcast"
- "Calm ambient piano music for meditation"
- "Energetic rock guitar riff for intro"
- "Smooth jazz saxophone for background ambience"
### 🔄 PROVIDER FALLBACK:
- **Cloud-first**: System tries Lyria 2 first (premium quality, $0.06/30sec)
- **Replicate fallback**: Falls back to Replicate MusicGen (~$0.084/30sec)
- **HuggingFace fallback**: Falls back to HuggingFace Spaces (free!)
- **Instant setup**: No Docker or local installation required
- **Reliable**: All providers are production-ready cloud services
### 💡 CONTENT TIPS:
- Scripts and narrations work best with clear structure
- Add natural pauses with commas and periods
- Consider the listening context (podcast, presentation, etc.)
- Music clips are 30 seconds long - plan accordingly for longer needs
`;
}
getParameters() {
return [
{
name: 'text',
type: 'string',
description: 'Text content for speech generation OR music description for music generation',
required: true
},
{
name: 'output_path',
type: 'string',
description: 'Output file path (relative to project root)',
required: true
},
{
name: 'content_type',
type: 'string',
description: 'Type of audio content: "speech" (default), "music", or "sound_effect"',
required: false,
default: 'speech'
},
{
name: 'provider',
type: 'string',
description: 'AI provider for audio generation: "gemini" (cloud), "echogarden" (local/offline), or auto-selected based on content_type if not specified',
required: false
},
{
name: 'voice',
type: 'string',
description: 'Voice to use for speech generation. Gemini: "Achernar" (F), "Achird" (M), "Algenib" (M), "Algieba" (M), "Alnilam" (M), "Aoede" (F), "Autonoe" (F), "Callirrhoe" (F), "Charon" (M), "Despina" (F), "Enceladus" (M), "Erinome" (F), "Fenrir" (M), "Gacrux" (F), "Iapetus" (M), "Kore" (F), "Laomedeia" (F), "Leda" (F), "Orus" (M), "Pulcherrima" (F), "Puck" (M), "Rasalgethi" (M), "Sadachbia" (M), "Sadaltager" (M), "Schedar" (M), "Sulafat" (F), "Umbriel" (M), "Vindemiatrix" (F), "Zephyr" (F, default), "Zubenelgenubi" (M). Echogarden: "Bella" (default), "Nicole", "Sarah", "Nova", "Sky", "Jessica", "River", "Emma"',
required: false,
default: 'Zephyr'
},
{
name: 'style',
type: 'string',
description: 'Music style/genre for music generation (e.g., "electronic", "ambient", "jazz")',
required: false
},
{
name: 'negative_prompt',
type: 'string',
description: 'Elements to avoid in music generation (for music content_type)',
required: false
},
{
name: 'seed',
type: 'number',
description: 'Seed for reproducible music generation (for music content_type)',
required: false
},
{
name: 'duration',
type: 'number',
description: 'Duration in seconds for music generation (default: 30, max: 30 for most providers)',
required: false,
default: 30
},
{
name: 'model',
type: 'string',
description: 'Specific model to use (provider-dependent)',
required: false
}
];
}
async execute(parameters) {
// Validate parameters
const validationError = this.validateParameters(parameters);
if (validationError) {
return validationError;
}
const { text, output_path, content_type = 'speech', provider, voice = 'zephyr', style, negative_prompt, seed, duration = 30, model } = parameters;
try {
// Validate text content
if (!text || typeof text !== 'string' || !text.trim()) {
return {
success: false,
error: 'Text content cannot be empty'
};
}
// Validate duration for music generation
if (content_type === 'music' && duration !== undefined) {
if (typeof duration !== 'number' || duration <= 0 || duration > 30) {
return {
success: false,
error: 'Duration must be a positive number between 1 and 30 seconds'
};
}
}
// Validate and resolve output path
const pathValidation = await this.validateAndResolveOutputPath(output_path);
if (!pathValidation.success) {
return {
success: false,
error: pathValidation.error
};
}
// Route to appropriate generation method based on content type
let result;
if (content_type === 'music') {
result = await this.generateMusic(text, {
output_path: pathValidation.resolvedPath,
style,
negative_prompt,
seed,
duration,
provider
});
}
else if (content_type === 'speech') {
result = await this.generateSpeech(text, {
output_path: pathValidation.resolvedPath,
provider: provider || 'gemini',
voice,
model
});
}
else {
return {
success: false,
error: `Unsupported content_type: ${content_type}. Supported types: speech, music`
};
}
return result;
}
catch (error) {
return {
success: false,
error: `Failed to generate audio: ${error.message}`
};
}
}
/**
* Generate speech using TTS (Gemini, Echogarden)
*/
async generateSpeech(text, options) {
const { output_path, provider, voice, model } = options;
try {
let audioGenerator;
if (provider === 'echogarden') {
// Echogarden doesn't require API keys - runs locally
audioGenerator = new AudioGenerator({
provider,
voice,
model: model || 'kokoro' // Default to kokoro engine
});
}
else {
// Get configured LLM provider for API key (for cloud providers like Gemini)
const llmProvider = await this.getLLMProviderForAudio(provider);
if (!llmProvider) {
return {
success: false,
error: `No configured ${provider} provider found. Please configure the ${provider} provider using 'contaigents configure' command.`
};
}
// Initialize AudioGenerator with API key from LLM config
audioGenerator = new AudioGenerator({
provider,
apiKey: llmProvider.apiKey,
voice,
model
});
}
// Validate voice if available
if (!audioGenerator.isVoiceAvailable(voice)) {
const availableVoices = audioGenerator.getAvailableVoices();
return {
success: false,
error: `Voice "${voice}" is not available for provider "${provider}". Available voices: ${availableVoices.join(', ')}`
};
}
// Generate audio
console.log(`🎙️ Generating speech with ${provider} using voice: ${voice}`);
const audioBuffer = await audioGenerator.generateAudio(text);
// Save audio file
const savedPath = await audioGenerator.saveAudioToFile(output_path, audioBuffer);
// Get file stats
const fileStats = await fs.stat(savedPath);
const fileSize = fileStats.size;
// Estimate duration (rough estimate: ~150 words per minute, ~5 characters per word)
const estimatedWords = text.length / 5;
const estimatedDurationSeconds = (estimatedWords / 150) * 60;
return {
success: true,
data: {
output_file_path: path.relative(this.baseDir, savedPath),
absolute_path: savedPath,
file_size: fileSize,
duration_estimate: `${estimatedDurationSeconds.toFixed(1)}s`,
provider,
voice,
content_type: 'speech',
text_length: text.length,
model: model || audioGenerator.getAvailableVoices()[0]
},
message: `Speech generated successfully. Saved to: ${path.relative(this.baseDir, savedPath)} (${this.formatFileSize(fileSize)}, ~${estimatedDurationSeconds.toFixed(1)}s)`
};
}
catch (error) {
return {
success: false,
error: `Failed to generate speech: ${error.message}`
};
}
}
/**
* Generate music using cloud-first approach (Lyria 2 + Replicate fallback)
*/
async generateMusic(prompt, options) {
const { output_path, style, negative_prompt, seed, duration, provider } = options;
let lyria2Result = null;
// Smart provider selection logic - Cloud-First Architecture
if (!provider) {
// Try Lyria 2 first (premium quality)
console.log('🌐 Attempting music generation with Lyria 2...');
lyria2Result = await this.generateMusicWithLyria2(prompt, {
output_path,
style,
negative_prompt,
seed,
duration
});
if (lyria2Result.success) {
return lyria2Result;
}
console.log(`⚠️ Lyria 2 failed: ${lyria2Result.error}`);
console.log('🔄 Falling back to Replicate MusicGen...');
}
// Try Replicate MusicGen (reliable cloud fallback)
const replicateResult = await this.generateMusicWithReplicate(prompt, {
output_path,
style,
duration
});
if (replicateResult.success) {
return replicateResult;
}
console.log(`⚠️ Replicate failed: ${replicateResult.error}`);
console.log('🔄 Falling back to HuggingFace MusicGen...');
// Try HuggingFace MusicGen (free fallback)
const huggingFaceResult = await this.generateMusicWithHuggingFace(prompt, {
output_path,
style,
duration
});
if (huggingFaceResult.success) {
return huggingFaceResult;
}
// All cloud providers failed
return {
success: false,
error: `Music generation failed with all cloud providers. Lyria 2: ${lyria2Result?.error || 'not attempted'}. Replicate: ${replicateResult.error}. HuggingFace: ${huggingFaceResult.error}`
};
}
/**
* Generate music using Lyria 2 (Google Vertex AI)
*/
async generateMusicWithLyria2(prompt, options) {
const { output_path, style, negative_prompt, seed } = options;
try {
// Get configured Gemini provider for Vertex AI access
const llmProvider = await this.getLLMProviderForAudio('gemini');
if (!llmProvider) {
return {
success: false,
error: 'No configured Gemini provider found. Lyria 2 requires Gemini API configuration.'
};
}
// Enhance prompt with style if provided
const enhancedPrompt = style ? `${style}: ${prompt}` : prompt;
// Prepare request payload for Lyria 2
const requestBody = {
instances: [{
prompt: enhancedPrompt,
...(negative_prompt && { negative_prompt }),
...(seed && { seed })
}],
parameters: {}
};
// Check if we have the required environment variables
if (!process.env.GOOGLE_CLOUD_PROJECT) {
throw new Error('GOOGLE_CLOUD_PROJECT environment variable is required for Lyria 2');
}
// Make request to Lyria 2 via Vertex AI
// Note: This is a simplified version - in reality we'd need to use the Google Cloud client
const response = await fetch(`https://us-central1-aiplatform.googleapis.com/v1/projects/${process.env.GOOGLE_CLOUD_PROJECT}/locations/us-central1/publishers/google/models/lyria-002:predict`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${llmProvider.apiKey}`,
'Content-Type': 'application/json'
},
body: JSON.stringify(requestBody)
});
if (!response.ok) {
throw new Error(`Lyria 2 API error: ${response.status} ${response.statusText}`);
}
const result = await response.json();
if (!result.predictions || !result.predictions[0] || !result.predictions[0].audioContent) {
throw new Error('Invalid response from Lyria 2 API');
}
// Decode base64 audio content
const audioContent = result.predictions[0].audioContent;
const audioBuffer = Buffer.from(audioContent, 'base64');
// Save audio file
await fs.writeFile(output_path, audioBuffer);
// Get file stats
const fileStats = await fs.stat(output_path);
const fileSize = fileStats.size;
return {
success: true,
data: {
output_file_path: path.relative(this.baseDir, output_path),
absolute_path: output_path,
file_size: fileSize,
duration_estimate: '30.0s', // Lyria 2 generates 30-second clips
provider: 'lyria-2',
content_type: 'music',
prompt_length: prompt.length,
style,
negative_prompt,
seed
},
message: `Music generated successfully with Lyria 2. Saved to: ${path.relative(this.baseDir, output_path)} (${this.formatFileSize(fileSize)}, 30s)`
};
}
catch (error) {
return {
success: false,
error: `Lyria 2 music generation failed: ${error.message}`
};
}
}
/**
* Generate music using Replicate MusicGen
*/
async generateMusicWithReplicate(prompt, options) {
const { output_path, style, duration = 30 } = options;
try {
// Get configured Replicate provider
const replicateProvider = await this.getLLMProviderForAudio('replicate');
if (!replicateProvider) {
return {
success: false,
error: 'No configured Replicate provider found. Please configure Replicate using "contaigents configure" command.'
};
}
// Enhance prompt with style if provided
const enhancedPrompt = style ? `${style}: ${prompt}` : prompt;
console.log(`🎵 Generating music with Replicate MusicGen...`);
// Call Replicate MusicGen API using the correct endpoint format
const response = await fetch('https://api.replicate.com/v1/predictions', {
method: 'POST',
headers: {
'Authorization': `Bearer ${replicateProvider.apiKey}`,
'Content-Type': 'application/json',
'Prefer': 'wait'
},
body: JSON.stringify({
version: '671ac645ce5e552cc63a54a2bbff63fcf798043055d2dac5fc9e36a837eedcfb', // Latest version from API
input: {
prompt: enhancedPrompt,
model_version: 'melody-large',
output_format: 'wav',
duration: duration,
top_k: 250,
top_p: 0.0,
temperature: 1.0,
classifier_free_guidance: 3.0
}
})
});
if (!response.ok) {
const error = await response.json();
throw new Error(`Replicate API error: ${error.detail || response.statusText}`);
}
const result = await response.json();
if (result.error) {
throw new Error(result.error);
}
if (!result.output || !result.output[0]) {
throw new Error('No audio output received from Replicate MusicGen');
}
// Download the generated audio file
const audioUrl = result.output[0];
const audioResponse = await fetch(audioUrl);
if (!audioResponse.ok) {
throw new Error(`Failed to download audio: ${audioResponse.statusText}`);
}
const audioBuffer = Buffer.from(await audioResponse.arrayBuffer());
// Save audio file
await fs.writeFile(output_path, audioBuffer);
// Get file stats
const fileStats = await fs.stat(output_path);
const fileSize = fileStats.size;
return {
success: true,
data: {
output_file_path: path.relative(this.baseDir, output_path),
absolute_path: output_path,
file_size: fileSize,
duration_estimate: '30.0s',
provider: 'replicate-musicgen',
content_type: 'music',
prompt_length: prompt.length,
style,
model_version: 'melody',
cost_estimate: '$0.084'
},
message: `Music generated successfully with Replicate MusicGen. Saved to: ${path.relative(this.baseDir, output_path)} (${this.formatFileSize(fileSize)}, 30s, ~$0.084)`
};
}
catch (error) {
return {
success: false,
error: `Replicate MusicGen generation failed: ${error.message}`
};
}
}
/**
* Generate music using HuggingFace Spaces (free)
*/
async generateMusicWithHuggingFace(prompt, options) {
const { output_path, style } = options;
try {
// Enhance prompt with style if provided
const enhancedPrompt = style ? `${style}: ${prompt}` : prompt;
console.log(`🤗 Generating music with HuggingFace MusicGen...`);
// Import Gradio client dynamically
const { client } = await import('@gradio/client');
// Connect to the MusicGen space
const app = await client("https://facebook-musicgen.hf.space/");
// Call the MusicGen API using the correct endpoint (ID 0)
// Based on the space config: inputs are [textbox (id 5), audio (id 8)]
const result = await app.predict(0, [
enhancedPrompt, // textbox input (id 5) - prompt
null // audio input (id 8) - optional melody file
]);
console.log('HuggingFace result:', JSON.stringify(result, null, 2));
if (!result?.data || !Array.isArray(result.data)) {
throw new Error('No audio data received from HuggingFace MusicGen');
}
// The result data contains [video_output, audio_output]
// We want the audio output (second item)
let audioFile = null;
// Look for the audio file in the response
if (result.data.length >= 2 && result.data[1]) {
audioFile = result.data[1];
}
if (!audioFile || !audioFile.name) {
throw new Error(`No audio file found in HuggingFace response. Result: ${JSON.stringify(result.data)}`);
}
// Construct the download URL for the Gradio file
const audioUrl = `https://facebook-musicgen.hf.space/file=${audioFile.name}`;
// Download the generated audio file
const audioResponse = await fetch(audioUrl);
if (!audioResponse.ok) {
throw new Error(`Failed to download audio: ${audioResponse.statusText}`);
}
const audioBuffer = Buffer.from(await audioResponse.arrayBuffer());
// Save audio file
await fs.writeFile(output_path, audioBuffer);
// Get file stats
const fileStats = await fs.stat(output_path);
const fileSize = fileStats.size;
return {
success: true,
data: {
output_file_path: path.relative(this.baseDir, output_path),
absolute_path: output_path,
file_size: fileSize,
duration_estimate: '30.0s',
provider: 'huggingface-musicgen',
content_type: 'music',
prompt_length: prompt.length,
style,
model: 'facebook/musicgen-melody',
cost_estimate: 'Free'
},
message: `Music generated successfully with HuggingFace MusicGen. Saved to: ${path.relative(this.baseDir, output_path)} (${this.formatFileSize(fileSize)}, 30s, Free!)`
};
}
catch (error) {
return {
success: false,
error: `HuggingFace MusicGen generation failed: ${error.message}`
};
}
}
/**
* Get configured LLM provider for audio generation
*/
async getLLMProviderForAudio(provider) {
try {
// Map audio provider names to LLM provider names
const llmProviderName = this.mapAudioProviderToLLM(provider);
// Get the configured LLM provider
const llmProvider = LLMFactory.getProvider(llmProviderName);
// Check if it's configured
if (!(await llmProvider.isConfigured())) {
return null;
}
// Get the configuration
const config = llmProvider.getConfig();
// Return the API key
return {
apiKey: config.apiKey
};
}
catch (error) {
console.error(`Failed to get LLM provider for audio: ${error}`);
return null;
}
}
/**
* Map audio provider names to LLM provider names
*/
mapAudioProviderToLLM(audioProvider) {
switch (audioProvider.toLowerCase()) {
case 'gemini':
case 'google':
return 'Google';
case 'openai':
return 'OpenAI';
case 'replicate':
return 'Replicate';
default:
// Try to use the provider name as-is (capitalize first letter)
return audioProvider.charAt(0).toUpperCase() + audioProvider.slice(1).toLowerCase();
}
}
/**
* Validate and resolve output path with security checks
*/
async validateAndResolveOutputPath(outputPath) {
try {
// Normalize the path to prevent directory traversal
const normalizedPath = path.normalize(outputPath);
// Check for directory traversal attempts
if (normalizedPath.includes('..') || path.isAbsolute(normalizedPath)) {
return {
success: false,
error: 'Output path must be relative and cannot contain ".." or be absolute for security reasons'
};
}
// Resolve full path
const fullPath = path.join(this.baseDir, normalizedPath);
// Ensure the resolved path is still within the base directory
const relativePath = path.relative(this.baseDir, fullPath);
if (relativePath.startsWith('..')) {
return {
success: false,
error: 'Output path resolves outside of project directory'
};
}
// Create parent directories if they don't exist
const parentDir = path.dirname(fullPath);
await fs.mkdir(parentDir, { recursive: true });
return {
success: true,
resolvedPath: fullPath
};
}
catch (error) {
return {
success: false,
error: `Failed to resolve output path: ${error.message}`
};
}
}
/**
* Format file size in human-readable format
*/
formatFileSize(bytes) {
if (bytes === 0)
return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i];
}
}