contaigents
Version:
Modular AI Content Ecosystem with Audio Generation
336 lines (335 loc) • 14.2 kB
JavaScript
/**
* Audio Generator Utility
* Provides a unified interface for generating audio from text using different AI providers
*/
import { GoogleGenerativeAI } from '@google/generative-ai';
import { writeFile } from 'fs';
import { promisify } from 'util';
import path from 'path';
// Convert callback-based writeFile to Promise-based
const writeFileAsync = promisify(writeFile);
/**
* AudioGenerator class for text-to-speech conversion using various AI providers
*/
export class AudioGenerator {
/**
* Create a new AudioGenerator instance
* @param {Object} config - Configuration options
* @param {string} config.provider - AI provider ('gemini', 'openai', etc.)
* @param {string} config.apiKey - API key for the provider
* @param {string} config.voice - Voice to use for audio generation
* @param {string} config.model - Model to use for audio generation
*/
constructor(config) {
this.geminiModel = '';
this.availableVoices = [];
this.lastMimeType = '';
this.provider = config.provider || 'gemini';
this.apiKey = config.apiKey;
this.voice = config.voice || '';
this.model = config.model || '';
// Initialize the appropriate client based on the provider
this._initializeClient();
}
/**
* Initialize the appropriate AI client based on the provider
* @private
*/
_initializeClient() {
switch (this.provider) {
case 'gemini':
if (!this.apiKey) {
throw new Error('Gemini API key is required');
}
this.client = new GoogleGenerativeAI(this.apiKey);
this.geminiModel = this.model || 'gemini-2.5-flash-preview-tts';
this.voice = this.voice || 'zephyr';
this.availableVoices = ['achernar', 'achird', 'algenib', 'algieba', 'alnilam', 'aoede', 'autonoe', 'callirrhoe', 'charon', 'despina', 'enceladus', 'erinome', 'fenrir', 'gacrux', 'iapetus', 'kore', 'laomedeia', 'leda', 'orus', 'puck', 'pulcherrima', 'rasalgethi', 'sadachbia', 'sadaltager', 'schedar', 'sulafat', 'umbriel', 'vindemiatrix', 'zephyr', 'zubenelgenubi'];
break;
// Add more providers here as needed
case 'openai':
throw new Error('OpenAI provider not yet implemented');
default:
throw new Error(`Unsupported provider: ${this.provider}`);
}
}
/**
* Split text into chunks that fit within the AI provider's character limit
* @param {string} text - The text to split
* @param {number} maxChunkSize - Maximum characters per chunk
* @returns {string[]} - Array of text chunks
* @private
*/
_splitTextIntoChunks(text, maxChunkSize = 4000) {
// If text is already within the limit, return it as a single chunk
if (text.length <= maxChunkSize) {
return [text];
}
const chunks = [];
let currentChunk = '';
// Split text by sentences to avoid cutting in the middle of a sentence
const sentences = text.split(/(?<=[.!?])\s+/);
for (const sentence of sentences) {
// If adding this sentence would exceed the limit, save current chunk and start a new one
if (currentChunk.length + sentence.length > maxChunkSize) {
if (currentChunk.length > 0) {
chunks.push(currentChunk);
currentChunk = '';
}
// If a single sentence is longer than the limit, split it by words
if (sentence.length > maxChunkSize) {
const words = sentence.split(/\s+/);
let wordChunk = '';
for (const word of words) {
if (wordChunk.length + word.length + 1 > maxChunkSize) {
chunks.push(wordChunk);
wordChunk = word;
}
else {
wordChunk += (wordChunk ? ' ' : '') + word;
}
}
if (wordChunk) {
currentChunk = wordChunk;
}
}
else {
currentChunk = sentence;
}
}
else {
currentChunk += (currentChunk ? ' ' : '') + sentence;
}
}
// Add the last chunk if there's anything left
if (currentChunk) {
chunks.push(currentChunk);
}
return chunks;
}
/**
* Generate audio from text using Gemini
* @param {string} text - The text to convert to speech
* @returns {Promise<Buffer>} - The audio data as a Buffer
* @private
*/
async _generateAudioWithGemini(text) {
// Split text into chunks to handle Gemini's character limit
const textChunks = this._splitTextIntoChunks(text, 4000);
console.log(`📊 Text split into ${textChunks.length} chunks for processing`);
// Process each chunk and collect the audio buffers
const audioBuffers = [];
for (let i = 0; i < textChunks.length; i++) {
console.log(`🔄 Processing chunk ${i + 1} of ${textChunks.length} (${textChunks[i].length} characters)`);
// Configure Gemini TTS settings
const config = {
temperature: 1,
responseModalities: ['audio'],
speechConfig: {
voiceConfig: {
prebuiltVoiceConfig: {
voiceName: this.voice,
}
}
},
};
const contents = [
{
role: 'user',
parts: [
{
text: textChunks[i],
},
],
},
];
try {
// Get the Gemini model
const model = this.client.getGenerativeModel({
model: this.geminiModel
});
// Generate audio using Gemini
const response = await model.generateContentStream({
contents,
generationConfig: config
});
// Process the streaming response
let audioData = null;
for await (const chunk of response.stream) {
if (!chunk.candidates || !chunk.candidates[0].content || !chunk.candidates[0].content.parts) {
continue;
}
if (chunk.candidates[0].content.parts[0].inlineData) {
const inlineData = chunk.candidates[0].content.parts[0].inlineData;
const mimeType = inlineData.mimeType || '';
const data = inlineData.data || '';
console.log(`Received audio data with MIME type: ${mimeType}`);
// Get the audio data as a buffer
audioData = Buffer.from(data, 'base64');
// Store the MIME type for later use
this.lastMimeType = mimeType;
break;
}
}
if (audioData) {
audioBuffers.push(audioData);
}
else {
console.warn(`⚠️ No audio data received for chunk ${i + 1}`);
}
}
catch (chunkError) {
console.error(`❌ Error processing chunk ${i + 1}:`, chunkError.message);
// Continue with next chunk instead of failing the entire process
}
}
if (audioBuffers.length === 0) {
throw new Error('Failed to generate any audio data');
}
// Combine all audio buffers into a single buffer
return Buffer.concat(audioBuffers);
}
/**
* Generate audio from text
* @param {string} text - The text to convert to speech
* @returns {Promise<Buffer>} - The audio data as a Buffer
*/
async generateAudio(text) {
console.log(`🎙️ Generating audio with ${this.provider} using voice: ${this.voice}`);
try {
switch (this.provider) {
case 'gemini':
return await this._generateAudioWithGemini(text);
// Add more providers here as needed
default:
throw new Error(`Unsupported provider: ${this.provider}`);
}
}
catch (error) {
console.error(`❌ Error generating audio with ${this.provider}:`, error.message);
throw error;
}
}
/**
* Get the appropriate file extension based on MIME type
* @param {string} mimeType - The MIME type of the audio data
* @returns {string} - The file extension (without the dot)
* @private
*/
_getFileExtensionFromMimeType(mimeType) {
if (!mimeType)
return 'wav'; // Default to wav if no MIME type
const mimeToExt = {
'audio/wav': 'wav',
'audio/wave': 'wav',
'audio/x-wav': 'wav',
'audio/mp3': 'mp3',
'audio/mpeg': 'mp3',
'audio/mpeg3': 'mp3',
'audio/ogg': 'ogg',
'audio/webm': 'webm',
'audio/aac': 'aac',
'audio/flac': 'flac'
};
// Extract the base MIME type (before any parameters)
const baseMimeType = mimeType.split(';')[0].trim();
return mimeToExt[baseMimeType] || 'wav';
}
/**
* Create a WAV header for raw PCM audio data
* @param {Buffer} audioData - The raw audio data
* @param {Object} options - Audio format options
* @param {number} options.numChannels - Number of audio channels (default: 1)
* @param {number} options.sampleRate - Sample rate in Hz (default: 24000)
* @param {number} options.bitsPerSample - Bits per sample (default: 16)
* @returns {Buffer} - WAV header buffer
* @private
*/
_createWavHeader(audioData, options = {}) {
// Audio format parameters with defaults
const numChannels = options.numChannels || 1;
const sampleRate = options.sampleRate || 24000;
const bitsPerSample = options.bitsPerSample || 16;
console.log(`Creating WAV header with: channels=${numChannels}, rate=${sampleRate}Hz, bits=${bitsPerSample}`);
const byteRate = sampleRate * numChannels * bitsPerSample / 8;
const blockAlign = numChannels * bitsPerSample / 8;
const dataLength = audioData.length;
const header = Buffer.alloc(44);
// RIFF header
header.write('RIFF', 0);
header.writeUInt32LE(36 + dataLength, 4);
header.write('WAVE', 8);
// Format chunk
header.write('fmt ', 12);
header.writeUInt32LE(16, 16);
header.writeUInt16LE(1, 20); // PCM format
header.writeUInt16LE(numChannels, 22);
header.writeUInt32LE(sampleRate, 24);
header.writeUInt32LE(byteRate, 28);
header.writeUInt16LE(blockAlign, 32);
header.writeUInt16LE(bitsPerSample, 34);
// Data chunk
header.write('data', 36);
header.writeUInt32LE(dataLength, 40);
return header;
}
/**
* Save audio data to a file
* @param {string} filePath - Path to save the audio file
* @param {Buffer} audioData - The audio data to save
* @returns {Promise<string>} - The path to the saved file (may be different from input if extension changed)
*/
async saveAudioToFile(filePath, audioData) {
try {
// Get the file extension from the MIME type
const fileExt = this._getFileExtensionFromMimeType(this.lastMimeType);
// Make sure the file path has the correct extension
const pathInfo = path.parse(filePath);
const newFilePath = path.join(pathInfo.dir, `${pathInfo.name}.${fileExt}`);
// If the data doesn't have proper headers and is raw PCM, add WAV headers
let finalAudioData = audioData;
if (this.lastMimeType && this.lastMimeType.toLowerCase().includes('audio/l16')) {
console.log('Converting raw PCM audio to WAV format...');
// Parse sample rate from MIME type if available
let sampleRate = 24000; // Default
if (this.lastMimeType.includes('rate=')) {
const rateMatch = this.lastMimeType.match(/rate=(\d+)/);
if (rateMatch && rateMatch[1]) {
sampleRate = parseInt(rateMatch[1], 10);
}
}
console.log(`Using sample rate: ${sampleRate} Hz`);
// Create WAV header with the correct sample rate
const wavHeader = this._createWavHeader(audioData, {
numChannels: 1,
sampleRate: sampleRate,
bitsPerSample: 16
});
finalAudioData = Buffer.concat([wavHeader, audioData]);
}
// Save the file
await writeFileAsync(newFilePath, finalAudioData);
console.log(`💾 Saved audio to: ${newFilePath}`);
return newFilePath;
}
catch (error) {
console.error(`❌ Error saving audio file:`, error.message);
throw error;
}
}
/**
* Get available voices for the current provider
* @returns {string[]} - Array of available voice names
*/
getAvailableVoices() {
return this.availableVoices || [];
}
/**
* Check if a voice is available for the current provider
* @param {string} voice - Voice name to check
* @returns {boolean} - True if the voice is available
*/
isVoiceAvailable(voice) {
return this.availableVoices?.includes(voice) || false;
}
}