contaigents
Version:
Modular AI Content Ecosystem with Audio Generation
631 lines (621 loc) ā¢ 26.3 kB
JavaScript
import { BaseTool } from './BaseTool.js';
import { exec } from 'child_process';
import { promisify } from 'util';
import path from 'path';
import fs from 'fs/promises';
import FormData from 'form-data';
import fetch from 'node-fetch';
const execAsync = promisify(exec);
// Dynamic import for @xenova/transformers to avoid loading issues
let WhisperPipeline = null;
let whisperInitialized = false;
export class SpeechToTextTool extends BaseTool {
constructor(baseDir) {
super();
this.maxTimeout = 300000; // 5 minutes for audio processing
this.baseDir = baseDir || process.cwd();
}
getName() {
return 'speech_to_text';
}
getDescription() {
return 'Convert speech/audio to text using Whisper models (local/cloud) or cloud providers (OpenAI, Gemini). Supports multiple audio formats and languages.';
}
getParameters() {
return [
{
name: 'input_file',
type: 'string',
description: 'Path to the audio/video file to transcribe',
required: true
},
{
name: 'output_file',
type: 'string',
description: 'Path to save the transcription text file (optional)',
required: false
},
{
name: 'provider',
type: 'string',
description: 'ASR provider (whisper_local, openai_whisper, gemini, auto)',
required: false,
default: 'auto'
},
{
name: 'model',
type: 'string',
description: 'Model to use (tiny, base, small, medium, large for Whisper; whisper-1 for OpenAI)',
required: false,
default: 'base'
},
{
name: 'language',
type: 'string',
description: 'Language code (en, es, fr, de, etc.) or auto-detect',
required: false,
default: 'auto'
},
{
name: 'format',
type: 'string',
description: 'Output format (text, srt, vtt, json)',
required: false,
default: 'text'
},
{
name: 'timestamps',
type: 'boolean',
description: 'Include timestamps in output',
required: false,
default: false
},
{
name: 'translate',
type: 'boolean',
description: 'Translate to English (Whisper only)',
required: false,
default: false
},
{
name: 'api_key',
type: 'string',
description: 'API key for cloud providers (optional if set in environment)',
required: false
}
];
}
getAgentGuidance() {
return `
š¤ SPEECH-TO-TEXT OPERATIONS:
š± PROVIDER OPTIONS:
- whisper_local: Use local Whisper installation (privacy-first, no API costs)
- openai_whisper: OpenAI Whisper API (high quality, requires API key)
- gemini: Google Gemini API (fast, supports many languages)
- auto: Smart provider selection (tries local first, falls back to cloud)
šÆ MODEL SELECTION:
Local Whisper Models (by accuracy/speed):
- tiny: Fastest, least accurate (~39 MB)
- base: Good balance (~74 MB) - DEFAULT
- small: Better accuracy (~244 MB)
- medium: High accuracy (~769 MB)
- large: Best accuracy (~1550 MB)
Cloud Models:
- OpenAI: whisper-1 (high quality)
- Gemini: Built-in speech recognition
š OUTPUT FORMATS:
- text: Plain text transcription
- srt: Subtitle format with timestamps
- vtt: WebVTT format for web players
- json: Detailed JSON with confidence scores
š RESPONSE SIZE CONTROL:
- max_chars: Maximum characters to return in response (default: 2000)
- Full transcription always saved to file
- Response includes is_trimmed flag if content was truncated
- Agents can read full content from saved file when needed
š LANGUAGE SUPPORT:
- auto: Auto-detect language
- en: English
- es: Spanish
- fr: French
- de: German
- ja: Japanese
- zh: Chinese
- And 90+ more languages
ā” SMART PROVIDER SELECTION:
1. Local Whisper (Node.js) - Privacy + No costs + No Python dependencies
2. OpenAI Whisper API - High quality fallback
3. Gemini API - Fast multilingual fallback
š§ LOCAL INSTALLATION:
- Node.js Whisper: npm install @xenova/transformers (pure JavaScript)
- No Python or pip required - everything runs in Node.js!
EXAMPLES:
- Basic transcription: input_file="audio.mp3", provider="auto"
- High quality: input_file="interview.wav", provider="openai_whisper", model="whisper-1"
- With subtitles: input_file="video.mp4", format="srt", timestamps=true
- Translate to English: input_file="spanish.mp3", translate=true, language="es"
- Local privacy: input_file="private.wav", provider="whisper_local", model="medium"
- Large file with preview: input_file="long_meeting.wav", max_chars=500
`;
}
async execute(parameters) {
// Validate parameters
const validationError = this.validateParameters(parameters);
if (validationError) {
return validationError;
}
const { input_file, output_file, provider = 'auto', model = 'base', language = 'auto', format = 'text', timestamps = false, translate = false, api_key, max_chars = 2000 } = parameters;
try {
// Resolve file paths
const inputPath = path.resolve(this.baseDir, input_file);
// Security check - ensure paths are within base directory
if (!inputPath.startsWith(this.baseDir)) {
return {
success: false,
error: 'File paths must be within the base directory'
};
}
// Check if input file exists
try {
await fs.access(inputPath);
}
catch {
return {
success: false,
error: `Input file not found: ${input_file}`
};
}
// Smart provider selection
const selectedProvider = await this.selectProvider(provider, api_key);
// Perform transcription based on provider
let transcriptionResult;
switch (selectedProvider) {
case 'whisper_local':
transcriptionResult = await this.transcribeWithLocalWhisper(inputPath, {
model, language, format, timestamps, translate
});
break;
case 'openai_whisper':
transcriptionResult = await this.transcribeWithOpenAI(inputPath, {
model: model === 'base' ? 'whisper-1' : model,
language, format, timestamps, translate, api_key
});
break;
case 'gemini':
transcriptionResult = await this.transcribeWithGemini(inputPath, {
language, format, timestamps, api_key
});
break;
default:
return {
success: false,
error: `Unsupported provider: ${selectedProvider}`
};
}
if (!transcriptionResult.success) {
return transcriptionResult;
}
// Save to output file if specified
let outputPath;
if (output_file) {
outputPath = path.resolve(this.baseDir, output_file);
if (!outputPath.startsWith(this.baseDir)) {
return {
success: false,
error: 'Output file path must be within the base directory'
};
}
await fs.writeFile(outputPath, transcriptionResult.text, 'utf8');
}
// Use the output file from the transcription result if available (for local Whisper)
const finalOutputFile = transcriptionResult.outputFile || outputPath;
const finalOutputSize = transcriptionResult.outputSize || (transcriptionResult.text ? Buffer.byteLength(transcriptionResult.text, 'utf8') : 0);
// Handle content trimming based on max_chars
const fullTranscription = transcriptionResult.text;
const fullLength = fullTranscription.length;
const isTrimmed = fullLength > max_chars;
const responseTranscription = isTrimmed ? fullTranscription.substring(0, max_chars) + '...' : fullTranscription;
return {
success: true,
data: {
transcription: responseTranscription,
is_trimmed: isTrimmed,
full_length: fullLength,
provider: selectedProvider,
model: model,
language: transcriptionResult.detectedLanguage || language,
format: format,
input_file: path.relative(this.baseDir, inputPath),
output_file: finalOutputFile ? path.relative(this.baseDir, finalOutputFile) : null,
output_size: finalOutputSize,
characters_transcribed: transcriptionResult.charactersTranscribed || fullLength,
confidence: transcriptionResult.confidence,
duration: transcriptionResult.duration,
word_count: fullTranscription.split(/\s+/).filter(word => word.length > 0).length,
timestamp: new Date().toISOString()
},
message: `Speech-to-text completed successfully using ${selectedProvider}. ${fullLength} characters transcribed.${isTrimmed ? ` Response trimmed to ${max_chars} chars - full content in file.` : ''}${finalOutputFile ? ` Output saved to: ${path.relative(this.baseDir, finalOutputFile)} (${finalOutputSize} bytes)` : ''}`
};
}
catch (error) {
return {
success: false,
error: `Speech-to-text failed: ${error.message}`,
data: {
provider: provider,
input_file: input_file
}
};
}
}
async selectProvider(requestedProvider, apiKey) {
if (requestedProvider !== 'auto') {
return requestedProvider;
}
// Try local Node.js Whisper first
try {
await this.initializeWhisper();
return 'whisper_local';
}
catch {
// Local Whisper not available
}
// Check for API keys and fall back to cloud providers
const openaiKey = apiKey || process.env.OPENAI_API_KEY;
if (openaiKey) {
return 'openai_whisper';
}
const geminiKey = apiKey || process.env.GEMINI_API_KEY;
if (geminiKey) {
return 'gemini';
}
throw new Error('No speech-to-text provider available. Install @xenova/transformers or provide API keys.');
}
async initializeWhisper() {
if (whisperInitialized && WhisperPipeline) {
return;
}
try {
const { pipeline } = await import('@xenova/transformers');
WhisperPipeline = await pipeline('automatic-speech-recognition', 'Xenova/whisper-base');
whisperInitialized = true;
}
catch (error) {
throw new Error('Failed to initialize Whisper: @xenova/transformers not available');
}
}
async transcribeWithLocalWhisper(inputPath, options) {
try {
// Initialize Whisper if not already done
await this.initializeWhisper();
// Configure the model based on options
let modelName = 'Xenova/whisper-base';
switch (options.model) {
case 'tiny':
modelName = 'Xenova/whisper-tiny';
break;
case 'small':
modelName = 'Xenova/whisper-small';
break;
case 'medium':
modelName = 'Xenova/whisper-medium';
break;
case 'large':
modelName = 'Xenova/whisper-large-v3';
break;
default:
modelName = 'Xenova/whisper-base';
}
// Create a new pipeline with the specified model
const { pipeline } = await import('@xenova/transformers');
const transcriber = await pipeline('automatic-speech-recognition', modelName);
// Prepare transcription options
const transcribeOptions = {
// Handle long audio files by chunking
chunk_length_s: 30,
stride_length_s: 5
};
if (options.language !== 'auto') {
transcribeOptions.language = options.language;
}
if (options.translate) {
transcribeOptions.task = 'translate';
}
if (options.timestamps) {
transcribeOptions.return_timestamps = true;
}
// Preprocess audio for Node.js environment
const audioData = await this.preprocessAudioForNode(inputPath);
// Perform transcription using preprocessed audio data with chunking support
const result = await transcriber(audioData, transcribeOptions);
let transcriptionText = '';
let detectedLanguage = options.language !== 'auto' ? options.language : undefined;
if (options.format === 'json') {
// Return detailed JSON format
transcriptionText = JSON.stringify(result, null, 2);
}
else if (options.format === 'srt' && result.chunks) {
// Convert to SRT format
transcriptionText = this.convertToSRT(result.chunks);
}
else if (options.format === 'vtt' && result.chunks) {
// Convert to VTT format
transcriptionText = this.convertToVTT(result.chunks);
}
else {
// Plain text format
transcriptionText = result.text || '';
}
// Always save result to file
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const outputFileName = options.output_file || `transcription_whisper_local_${timestamp}.${options.format === 'json' ? 'json' : 'txt'}`;
const outputPath = path.resolve(this.baseDir, outputFileName);
// Save to file
await fs.writeFile(outputPath, transcriptionText, 'utf8');
const resultSize = Buffer.byteLength(transcriptionText, 'utf8');
return {
success: true,
text: transcriptionText.trim(),
detectedLanguage,
confidence: result.confidence || undefined,
duration: result.duration || undefined,
outputFile: outputPath,
outputSize: resultSize,
charactersTranscribed: transcriptionText.length,
provider: 'whisper_local',
model: options.model || 'base'
};
}
catch (error) {
return {
success: false,
text: '',
error: `Local Whisper (Node.js) failed: ${error.message}`
};
}
}
async transcribeWithOpenAI(inputPath, options) {
try {
const apiKey = options.api_key || process.env.OPENAI_API_KEY;
if (!apiKey) {
throw new Error('OpenAI API key not found');
}
// Read the audio file
const audioBuffer = await fs.readFile(inputPath);
// Create form data
const formData = new FormData();
formData.append('file', audioBuffer, {
filename: path.basename(inputPath),
contentType: this.getAudioMimeType(inputPath)
});
formData.append('model', options.model || 'whisper-1');
if (options.language !== 'auto') {
formData.append('language', options.language);
}
if (options.format === 'srt') {
formData.append('response_format', 'srt');
}
else if (options.format === 'vtt') {
formData.append('response_format', 'vtt');
}
else if (options.format === 'json') {
formData.append('response_format', 'verbose_json');
}
else {
formData.append('response_format', 'text');
}
if (options.timestamps && options.format === 'json') {
formData.append('timestamp_granularities[]', 'word');
}
const endpoint = options.translate ?
'https://api.openai.com/v1/audio/translations' :
'https://api.openai.com/v1/audio/transcriptions';
const response = await fetch(endpoint, {
method: 'POST',
headers: {
'Authorization': `Bearer ${apiKey}`,
...formData.getHeaders()
},
body: formData
});
if (!response.ok) {
throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`);
}
const result = await response.text();
let transcription = result;
let detectedLanguage;
let confidence;
let duration;
if (options.format === 'json') {
const jsonResult = JSON.parse(result);
transcription = jsonResult.text;
detectedLanguage = jsonResult.language;
duration = jsonResult.duration;
// Calculate average confidence if available
if (jsonResult.words) {
const confidences = jsonResult.words.map((w) => w.confidence || 1);
confidence = confidences.reduce((a, b) => a + b, 0) / confidences.length;
}
}
return {
success: true,
text: transcription.trim(),
detectedLanguage,
confidence,
duration
};
}
catch (error) {
return {
success: false,
text: '',
error: `OpenAI Whisper failed: ${error.message}`
};
}
}
async transcribeWithGemini(inputPath, options) {
try {
const apiKey = options.api_key || process.env.GEMINI_API_KEY;
if (!apiKey) {
throw new Error('Gemini API key not found');
}
// Read and encode audio file
const audioBuffer = await fs.readFile(inputPath);
const audioBase64 = audioBuffer.toString('base64');
const mimeType = this.getAudioMimeType(inputPath);
const requestBody = {
contents: [{
parts: [{
inline_data: {
mime_type: mimeType,
data: audioBase64
}
}, {
text: options.language !== 'auto' ?
`Transcribe this audio in ${options.language}. Return only the transcription text.` :
'Transcribe this audio. Return only the transcription text.'
}]
}]
};
const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash:generateContent?key=${apiKey}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(requestBody)
});
if (!response.ok) {
throw new Error(`Gemini API error: ${response.status} ${response.statusText}`);
}
const result = await response.json();
if (!result.candidates || !result.candidates[0] || !result.candidates[0].content) {
throw new Error('No transcription returned from Gemini');
}
const transcription = result.candidates[0].content.parts[0].text;
return {
success: true,
text: transcription.trim(),
detectedLanguage: options.language !== 'auto' ? options.language : undefined
};
}
catch (error) {
return {
success: false,
text: '',
error: `Gemini failed: ${error.message}`
};
}
}
getAudioMimeType(filePath) {
const ext = path.extname(filePath).toLowerCase();
const mimeTypes = {
'.mp3': 'audio/mpeg',
'.wav': 'audio/wav',
'.m4a': 'audio/mp4',
'.aac': 'audio/aac',
'.ogg': 'audio/ogg',
'.flac': 'audio/flac',
'.mp4': 'video/mp4',
'.mov': 'video/quicktime',
'.avi': 'video/x-msvideo',
'.mkv': 'video/x-matroska'
};
return mimeTypes[ext] || 'audio/mpeg';
}
convertToSRT(chunks) {
let srtContent = '';
chunks.forEach((chunk, index) => {
const startTime = this.formatSRTTime(chunk.timestamp[0] || 0);
const endTime = this.formatSRTTime(chunk.timestamp[1] || chunk.timestamp[0] + 1);
srtContent += `${index + 1}\n${startTime} --> ${endTime}\n${chunk.text}\n\n`;
});
return srtContent;
}
convertToVTT(chunks) {
let vttContent = 'WEBVTT\n\n';
chunks.forEach((chunk) => {
const startTime = this.formatVTTTime(chunk.timestamp[0] || 0);
const endTime = this.formatVTTTime(chunk.timestamp[1] || chunk.timestamp[0] + 1);
vttContent += `${startTime} --> ${endTime}\n${chunk.text}\n\n`;
});
return vttContent;
}
formatSRTTime(seconds) {
const hours = Math.floor(seconds / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const secs = Math.floor(seconds % 60);
const ms = Math.floor((seconds % 1) * 1000);
return `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')},${ms.toString().padStart(3, '0')}`;
}
formatVTTTime(seconds) {
const hours = Math.floor(seconds / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const secs = Math.floor(seconds % 60);
const ms = Math.floor((seconds % 1) * 1000);
return `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}.${ms.toString().padStart(3, '0')}`;
}
async preprocessAudioForNode(inputPath) {
try {
// First, try to use FFmpeg to convert to WAV format if needed
const ext = path.extname(inputPath).toLowerCase();
let wavPath = inputPath;
if (ext !== '.wav') {
// Convert to WAV using FFmpeg
wavPath = inputPath.replace(path.extname(inputPath), '_temp.wav');
await execAsync(`ffmpeg -i "${inputPath}" -ar 16000 -ac 1 -f wav "${wavPath}" -y`);
}
// Read the WAV file
const audioBuffer = await fs.readFile(wavPath);
// Try to use wav-decoder for proper audio processing
try {
const { decode } = await import('wav-decoder');
const audioData = await decode(audioBuffer);
// Convert to Float32Array (mono, 16kHz)
let samples;
if (audioData.channelData.length > 1) {
// Convert stereo to mono by averaging channels
const left = audioData.channelData[0];
const right = audioData.channelData[1];
samples = new Float32Array(left.length);
for (let i = 0; i < left.length; i++) {
samples[i] = (left[i] + right[i]) / 2;
}
}
else {
samples = audioData.channelData[0];
}
// Clean up temporary file if created
if (wavPath !== inputPath) {
try {
await fs.unlink(wavPath);
}
catch (error) {
// Ignore cleanup errors
}
}
return samples;
}
catch (wavError) {
// Fallback: Simple WAV parsing for basic WAV files
return this.parseSimpleWav(audioBuffer);
}
}
catch (error) {
throw new Error(`Audio preprocessing failed: ${error.message}`);
}
}
parseSimpleWav(buffer) {
// Simple WAV parser for basic 16-bit PCM WAV files
// This is a fallback when wav-decoder is not available
// Skip WAV header (44 bytes for standard WAV)
const dataStart = 44;
const dataLength = buffer.length - dataStart;
const samples = new Float32Array(dataLength / 2);
// Convert 16-bit PCM to Float32Array
for (let i = 0; i < samples.length; i++) {
const sample = buffer.readInt16LE(dataStart + i * 2);
samples[i] = sample / 32768.0; // Normalize to [-1, 1]
}
return samples;
}
}