contaigents
Version:
Modular AI Content Ecosystem with Audio Generation
420 lines (419 loc) • 18.6 kB
JavaScript
/**
* Audio Generator Utility
* Provides a unified interface for generating audio from text using different AI providers
*/
import { GoogleGenerativeAI } from '@google/generative-ai';
import * as Echogarden from 'echogarden';
import { writeFile } from 'fs';
import { promisify } from 'util';
import path from 'path';
// Convert callback-based writeFile to Promise-based
const writeFileAsync = promisify(writeFile);
/**
* AudioGenerator class for text-to-speech conversion using various AI providers
*/
export class AudioGenerator {
/**
* Create a new AudioGenerator instance
* @param {Object} config - Configuration options
* @param {string} config.provider - AI provider ('gemini', 'openai', 'echogarden', etc.)
* @param {string} config.apiKey - API key for the provider (not required for echogarden)
* @param {string} config.voice - Voice to use for audio generation
* @param {string} config.model - Model to use for audio generation
*/
constructor(config) {
this.geminiModel = '';
this.availableVoices = [];
this.lastMimeType = '';
this.provider = config.provider || 'gemini';
this.apiKey = config.apiKey || '';
this.voice = config.voice || '';
this.model = config.model || '';
// Initialize the appropriate client based on the provider
this._initializeClient();
}
/**
* Initialize the appropriate AI client based on the provider
* @private
*/
_initializeClient() {
switch (this.provider) {
case 'gemini':
if (!this.apiKey) {
throw new Error('Gemini API key is required');
}
this.client = new GoogleGenerativeAI(this.apiKey);
this.geminiModel = this.model || 'gemini-2.5-flash-preview-tts';
this.voice = this.voice || 'Zephyr';
this.availableVoices = ['Achernar', 'Achird', 'Algenib', 'Algieba', 'Alnilam', 'Aoede', 'Autonoe', 'Callirrhoe', 'Charon', 'Despina', 'Enceladus', 'Erinome', 'Fenrir', 'Gacrux', 'Iapetus', 'Kore', 'Laomedeia', 'Leda', 'Orus', 'Puck', 'Pulcherrima', 'Rasalgethi', 'Sadachbia', 'Sadaltager', 'Schedar', 'Sulafat', 'Umbriel', 'Vindemiatrix', 'Zephyr', 'Zubenelgenubi'];
break;
case 'echogarden':
// Echogarden doesn't require API keys - it runs locally
this.client = null; // We'll use Echogarden directly
this.voice = this.voice || 'Bella'; // Default voice for echogarden (Kokoro)
this.model = this.model || 'kokoro'; // Default engine for echogarden
// Available voices depend on the engine, we'll populate this dynamically
this.availableVoices = [];
break;
// Add more providers here as needed
case 'openai':
throw new Error('OpenAI provider not yet implemented');
default:
throw new Error(`Unsupported provider: ${this.provider}`);
}
}
/**
* Split text into chunks that fit within the AI provider's character limit
* @param {string} text - The text to split
* @param {number} maxChunkSize - Maximum characters per chunk
* @returns {string[]} - Array of text chunks
* @private
*/
_splitTextIntoChunks(text, maxChunkSize = 4000) {
// If text is already within the limit, return it as a single chunk
if (text.length <= maxChunkSize) {
return [text];
}
const chunks = [];
let currentChunk = '';
// Split text by sentences to avoid cutting in the middle of a sentence
const sentences = text.split(/(?<=[.!?])\s+/);
for (const sentence of sentences) {
// If adding this sentence would exceed the limit, save current chunk and start a new one
if (currentChunk.length + sentence.length > maxChunkSize) {
if (currentChunk.length > 0) {
chunks.push(currentChunk);
currentChunk = '';
}
// If a single sentence is longer than the limit, split it by words
if (sentence.length > maxChunkSize) {
const words = sentence.split(/\s+/);
let wordChunk = '';
for (const word of words) {
if (wordChunk.length + word.length + 1 > maxChunkSize) {
chunks.push(wordChunk);
wordChunk = word;
}
else {
wordChunk += (wordChunk ? ' ' : '') + word;
}
}
if (wordChunk) {
currentChunk = wordChunk;
}
}
else {
currentChunk = sentence;
}
}
else {
currentChunk += (currentChunk ? ' ' : '') + sentence;
}
}
// Add the last chunk if there's anything left
if (currentChunk) {
chunks.push(currentChunk);
}
return chunks;
}
/**
* Generate audio from text using Gemini
* @param {string} text - The text to convert to speech
* @returns {Promise<Buffer>} - The audio data as a Buffer
* @private
*/
async _generateAudioWithGemini(text) {
// Split text into chunks to handle Gemini's character limit
const textChunks = this._splitTextIntoChunks(text, 4000);
console.log(`📊 Text split into ${textChunks.length} chunks for processing`);
// Process each chunk and collect the audio buffers
const audioBuffers = [];
for (let i = 0; i < textChunks.length; i++) {
console.log(`🔄 Processing chunk ${i + 1} of ${textChunks.length} (${textChunks[i].length} characters)`);
// Configure Gemini TTS settings
const config = {
temperature: 1,
responseModalities: ['audio'],
speechConfig: {
voiceConfig: {
prebuiltVoiceConfig: {
voiceName: this.voice,
}
}
},
};
const contents = [
{
role: 'user',
parts: [
{
text: textChunks[i],
},
],
},
];
try {
// Get the Gemini model
const model = this.client.getGenerativeModel({
model: this.geminiModel
});
// Generate audio using Gemini
const response = await model.generateContentStream({
contents,
generationConfig: config
});
// Process the streaming response
let audioData = null;
for await (const chunk of response.stream) {
if (!chunk.candidates || !chunk.candidates[0].content || !chunk.candidates[0].content.parts) {
continue;
}
if (chunk.candidates[0].content.parts[0].inlineData) {
const inlineData = chunk.candidates[0].content.parts[0].inlineData;
const mimeType = inlineData.mimeType || '';
const data = inlineData.data || '';
console.log(`Received audio data with MIME type: ${mimeType}`);
// Get the audio data as a buffer
audioData = Buffer.from(data, 'base64');
// Store the MIME type for later use
this.lastMimeType = mimeType;
break;
}
}
if (audioData) {
audioBuffers.push(audioData);
}
else {
console.warn(`⚠️ No audio data received for chunk ${i + 1}`);
}
}
catch (chunkError) {
console.error(`❌ Error processing chunk ${i + 1}:`, chunkError.message);
// Continue with next chunk instead of failing the entire process
}
}
if (audioBuffers.length === 0) {
throw new Error('Failed to generate any audio data');
}
// Combine all audio buffers into a single buffer
return Buffer.concat(audioBuffers);
}
/**
* Generate audio from text using Echogarden (local TTS)
* @param {string} text - The text to convert to speech
* @returns {Promise<Buffer>} - The audio data as a Buffer
* @private
*/
async _generateAudioWithEchogarden(text) {
console.log(`🎙️ Generating audio with Echogarden using engine: ${this.model}, voice: ${this.voice}`);
try {
// Configure Echogarden synthesis options
const options = {
engine: this.model, // Cast to avoid TypeScript issues with string vs enum
voice: this.voice,
language: 'en', // Default to English, could be made configurable
outputAudioFormat: {
codec: 'wav'
}
};
console.log(`📊 Synthesizing text with Echogarden (${text.length} characters)`);
// Use Echogarden to synthesize the text
const result = await Echogarden.synthesize(text, options);
// Convert the audio result to Buffer
let audioBuffer;
if (result.audio instanceof Uint8Array) {
// If it's encoded audio (Uint8Array)
audioBuffer = Buffer.from(result.audio);
}
else if (result.audio && 'channels' in result.audio && 'sampleRate' in result.audio) {
// If it's RawAudio format, convert to WAV buffer
const rawAudio = result.audio;
// Convert Float32Array to 16-bit PCM
const samples = rawAudio.channels[0]; // Use first channel
const pcmData = new Int16Array(samples.length);
for (let i = 0; i < samples.length; i++) {
// Convert from [-1, 1] float to 16-bit signed integer
pcmData[i] = Math.max(-32768, Math.min(32767, Math.floor(samples[i] * 32767)));
}
// Create WAV header
const wavHeader = this._createWavHeader(Buffer.from(pcmData.buffer), {
numChannels: 1,
sampleRate: rawAudio.sampleRate,
bitsPerSample: 16
});
audioBuffer = Buffer.concat([wavHeader, Buffer.from(pcmData.buffer)]);
}
else {
console.log('Audio result type:', typeof result.audio);
console.log('Audio result keys:', result.audio ? Object.keys(result.audio) : 'null');
throw new Error('Unexpected audio format from Echogarden');
}
console.log(`✅ Generated ${audioBuffer.length} bytes of audio data with Echogarden`);
return audioBuffer;
}
catch (error) {
console.error(`❌ Error generating audio with Echogarden:`, error.message);
throw error;
}
}
/**
* Generate audio from text
* @param {string} text - The text to convert to speech
* @returns {Promise<Buffer>} - The audio data as a Buffer
*/
async generateAudio(text) {
console.log(`🎙️ Generating audio with ${this.provider} using voice: ${this.voice}`);
try {
switch (this.provider) {
case 'gemini':
return await this._generateAudioWithGemini(text);
case 'echogarden':
return await this._generateAudioWithEchogarden(text);
// Add more providers here as needed
default:
throw new Error(`Unsupported provider: ${this.provider}`);
}
}
catch (error) {
console.error(`❌ Error generating audio with ${this.provider}:`, error.message);
throw error;
}
}
/**
* Get the appropriate file extension based on MIME type
* @param {string} mimeType - The MIME type of the audio data
* @returns {string} - The file extension (without the dot)
* @private
*/
_getFileExtensionFromMimeType(mimeType) {
if (!mimeType)
return 'wav'; // Default to wav if no MIME type
const mimeToExt = {
'audio/wav': 'wav',
'audio/wave': 'wav',
'audio/x-wav': 'wav',
'audio/mp3': 'mp3',
'audio/mpeg': 'mp3',
'audio/mpeg3': 'mp3',
'audio/ogg': 'ogg',
'audio/webm': 'webm',
'audio/aac': 'aac',
'audio/flac': 'flac'
};
// Extract the base MIME type (before any parameters)
const baseMimeType = mimeType.split(';')[0].trim();
return mimeToExt[baseMimeType] || 'wav';
}
/**
* Create a WAV header for raw PCM audio data
* @param {Buffer} audioData - The raw audio data
* @param {Object} options - Audio format options
* @param {number} options.numChannels - Number of audio channels (default: 1)
* @param {number} options.sampleRate - Sample rate in Hz (default: 24000)
* @param {number} options.bitsPerSample - Bits per sample (default: 16)
* @returns {Buffer} - WAV header buffer
* @private
*/
_createWavHeader(audioData, options = {}) {
// Audio format parameters with defaults
const numChannels = options.numChannels || 1;
const sampleRate = options.sampleRate || 24000;
const bitsPerSample = options.bitsPerSample || 16;
console.log(`Creating WAV header with: channels=${numChannels}, rate=${sampleRate}Hz, bits=${bitsPerSample}`);
const byteRate = sampleRate * numChannels * bitsPerSample / 8;
const blockAlign = numChannels * bitsPerSample / 8;
const dataLength = audioData.length;
const header = Buffer.alloc(44);
// RIFF header
header.write('RIFF', 0);
header.writeUInt32LE(36 + dataLength, 4);
header.write('WAVE', 8);
// Format chunk
header.write('fmt ', 12);
header.writeUInt32LE(16, 16);
header.writeUInt16LE(1, 20); // PCM format
header.writeUInt16LE(numChannels, 22);
header.writeUInt32LE(sampleRate, 24);
header.writeUInt32LE(byteRate, 28);
header.writeUInt16LE(blockAlign, 32);
header.writeUInt16LE(bitsPerSample, 34);
// Data chunk
header.write('data', 36);
header.writeUInt32LE(dataLength, 40);
return header;
}
/**
* Save audio data to a file
* @param {string} filePath - Path to save the audio file
* @param {Buffer} audioData - The audio data to save
* @returns {Promise<string>} - The path to the saved file (may be different from input if extension changed)
*/
async saveAudioToFile(filePath, audioData) {
try {
// Get the file extension from the MIME type
const fileExt = this._getFileExtensionFromMimeType(this.lastMimeType);
// Make sure the file path has the correct extension
const pathInfo = path.parse(filePath);
const newFilePath = path.join(pathInfo.dir, `${pathInfo.name}.${fileExt}`);
// If the data doesn't have proper headers and is raw PCM, add WAV headers
let finalAudioData = audioData;
if (this.lastMimeType && this.lastMimeType.toLowerCase().includes('audio/l16')) {
console.log('Converting raw PCM audio to WAV format...');
// Parse sample rate from MIME type if available
let sampleRate = 24000; // Default
if (this.lastMimeType.includes('rate=')) {
const rateMatch = this.lastMimeType.match(/rate=(\d+)/);
if (rateMatch && rateMatch[1]) {
sampleRate = parseInt(rateMatch[1], 10);
}
}
console.log(`Using sample rate: ${sampleRate} Hz`);
// Create WAV header with the correct sample rate
const wavHeader = this._createWavHeader(audioData, {
numChannels: 1,
sampleRate: sampleRate,
bitsPerSample: 16
});
finalAudioData = Buffer.concat([wavHeader, audioData]);
}
// Save the file
await writeFileAsync(newFilePath, finalAudioData);
console.log(`💾 Saved audio to: ${newFilePath}`);
return newFilePath;
}
catch (error) {
console.error(`❌ Error saving audio file:`, error.message);
throw error;
}
}
/**
* Get available voices for the current provider
* @returns {string[]} - Array of available voice names
*/
getAvailableVoices() {
if (this.provider === 'echogarden') {
// Return common echogarden voices based on engine
switch (this.model) {
case 'kokoro':
return ['Bella', 'Nicole', 'Sarah', 'Nova', 'Sky', 'Jessica', 'River', 'Emma', 'Isabella', 'Alice', 'Lily'];
case 'vits':
return ['en-us-lessac-low', 'en-us-lessac-medium', 'en-us-lessac-high'];
case 'espeak':
return ['en', 'en-us', 'en-gb', 'en-au', 'en-ca'];
default:
return ['Bella', 'Nicole', 'Sarah', 'Nova', 'Sky', 'Jessica', 'River', 'Emma', 'Isabella', 'Alice', 'Lily'];
}
}
return this.availableVoices || [];
}
/**
* Check if a voice is available for the current provider
* @param {string} voice - Voice name to check
* @returns {boolean} - True if the voice is available
*/
isVoiceAvailable(voice) {
// Case-insensitive comparison
const lowerVoice = voice.toLowerCase();
return this.availableVoices?.some(v => v.toLowerCase() === lowerVoice) || false;
}
}