@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
344 lines (343 loc) • 14.3 kB
TypeScript
/**
* Audio File Processor
*
* Handles downloading, validating, and processing audio files to extract metadata
* and build text content suitable for LLM consumption. Audio files cannot be sent
* raw to most LLMs, so this processor extracts structured metadata (duration, codec,
* bitrate, tags) and formats it as text.
*
* Uses the `music-metadata` library (pure JavaScript, no native dependencies) for
* metadata extraction. Supports all major audio formats: MP3, WAV, OGG, FLAC, M4A,
* AAC, WMA, WebM, AIFF, AMR, APE, WavPack, and more.
*
* Key features:
* - Metadata extraction: duration, codec, bitrate, sample rate, channels
* - Tag extraction: title, artist, album, year, genre, track number, composer
* - Embedded cover art extraction
* - Graceful degradation for corrupt or partially readable files
* - LLM-friendly text content generation
*
* @module processors/media/AudioProcessor
*
* @example
* ```typescript
* import { audioProcessor, processAudio, isAudioFile } from "./AudioProcessor.js";
*
* // Check if a file is an audio file
* if (isAudioFile(fileInfo.mimetype, fileInfo.name)) {
* const result = await processAudio(fileInfo);
*
* if (result.success) {
* console.log(`Duration: ${result.data.metadata.durationFormatted}`);
* console.log(`Codec: ${result.data.metadata.codec}`);
* console.log(`Artist: ${result.data.tags.artist}`);
* console.log(`Text for LLM: ${result.data.textContent}`);
* }
* }
* ```
*/
import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
import type { AudioProcessorOptions, FileInfo, ProcessedAudio, ProcessorFileProcessingResult, ProcessOptions } from "../../types/index.js";
/**
* Audio Processor - extracts metadata and tags from audio files for LLM consumption.
*
* Audio files cannot be directly sent to most language models. This processor
* parses audio file headers to extract structured metadata (duration, codec,
* bitrate, sample rate, channels) and embedded tags (title, artist, album, etc.),
* then builds a human-readable text summary for the AI to reason about.
*
* Uses the `music-metadata` library which is a pure JavaScript implementation
* with no native dependencies, making it safe for all deployment environments.
*
* @example
* ```typescript
* const processor = new AudioProcessor();
*
* const result = await processor.processFile({
* id: 'audio-123',
* name: 'song.mp3',
* mimetype: 'audio/mpeg',
* size: 5242880,
* buffer: audioBuffer,
* });
*
* if (result.success) {
* console.log(result.data.textContent);
* // "[Audio File: song.mp3]
* // Duration: 3:45 | Codec: MPEG 1 Layer 3 | Bitrate: 320 kbps | ..."
* }
* ```
*/
export declare class AudioProcessor extends BaseFileProcessor<ProcessedAudio> {
constructor();
/**
* Override processFile for async audio metadata parsing with music-metadata.
*
* Processing pipeline:
* 1. Validate file type and size (base class)
* 2. Get file buffer (from direct buffer or download)
* 3. Parse audio metadata using music-metadata's parseBuffer()
* 4. Extract tags (title, artist, album, etc.)
* 5. Extract embedded cover art if present
* 6. Build LLM-friendly text content
*
* @param fileInfo - File information (can include URL or buffer)
* @param options - Optional processing options (auth headers, timeout, etc.)
* widened with the transcription knobs (#413/#440) so a caller-chosen
* backend, language or prompt reaches the transcriber instead of being
* discarded at the detector boundary.
* @returns Processing result with audio metadata or error
*/
processFile(fileInfo: FileInfo, options?: ProcessOptions & AudioProcessorOptions): Promise<ProcessorFileProcessingResult<ProcessedAudio>>;
/**
* Whether a transcription backend has the credentials it needs (#413).
*
* Availability is decided from the environment only — no network call — so
* selection stays cheap and cannot itself fail. A key that is present but
* rejected upstream surfaces later, as a transcription failure with the
* provider's own message, not as "unavailable".
*
* @param provider - Backend to check
* @returns True when at least one of the backend's credential vars is set
*/
private isProviderAvailable;
/**
* Choose the transcription backend for this file (#413).
*
* With no `requested` backend, the first configured entry of
* {@link TRANSCRIPTION_PROVIDER_ORDER} wins (OpenAI, then Google, then
* Azure). With one, it is normalised through the alias table and validated
* for availability — a backend the caller explicitly asked for is never
* silently swapped for a different one, because a caller who pinned Azure
* for a data-residency reason would not want OpenAI chosen behind their
* back. The mismatch is reported instead.
*
* @param requested - Caller's `AudioProcessorOptions.provider`, if any
* @returns The chosen backend, or the reason no backend could be chosen
*/
private selectProvider;
/**
* Attempt to transcribe audio with whichever backend is configured (#413).
*
* Transcription is attempted when:
* 1. A backend's credentials are present (see {@link selectProvider})
* 2. File size is within the 25MB ceiling
* 3. The file format is one the backend accepts
*
* Gracefully degrades: if transcription fails for any reason, metadata-only
* output is returned (transcription is additive, never blocks processing).
*
* @param buffer - Audio file content
* @param filename - Original filename (used for format detection)
* @param mimetype - MIME type of the audio file
* @param options - Caller's transcription knobs (provider, language, prompt)
* @returns Transcription result with transcript text, or the reason there is none
*/
private attemptTranscription;
/**
* Transcribe via OpenAI Whisper (#416).
*
* A native multipart POST to OpenAI's transcription endpoint. This used to
* go through @ai-sdk/openai's createOpenAI().transcription() plus the ai
* package's experimental_transcribe; both were dropped, and this is the
* only wire behaviour of theirs the processor ever depended on. The same
* request is already made natively by voice/providers/OpenAISTT.ts.
*
* `verbose_json` is requested for `language` and `duration` alongside the
* text — the response was previously parsed for `text` alone, so both were
* received and discarded, and `FileProcessingResult.metadata.language` had
* nothing to report (#409).
*
* @param buffer - Audio file content
* @param filename - Original filename, sent as the multipart part name
* @param mimetype - MIME type used for the upload blob
* @param options - Caller's language / model / prompt overrides
* @param skipped - Builds the "no transcript, and here is why" outcome
* @returns Transcription result, or the reason there is none
*/
private transcribeWithOpenAI;
/**
* Transcribe via one of the non-OpenAI STT handlers (#413).
*
* Google and Azure already have working, tested implementations under
* `src/lib/voice/providers/`, reached here through a dynamic import so a
* file-processing run that never transcribes does not pay to load them —
* the same reason `music-metadata` is loaded lazily above.
*
* @param chosen - Backend and display label from {@link selectProvider}
* @param buffer - Audio file content
* @param filename - Original filename; its extension picks the wire format
* @param mimetype - MIME type, used as the fallback format signal
* @param options - Caller's language / model overrides
* @param skipped - Builds the "no transcript, and here is why" outcome
* @returns Transcription result, or the reason there is none
*/
private transcribeWithHandler;
/**
* Stub implementation required by BaseFileProcessor.
* Not used because processFile is fully overridden.
*
* @param buffer - File buffer
* @param fileInfo - File information
* @returns Empty ProcessedAudio structure
*/
protected buildProcessedResult(buffer: Buffer, fileInfo: FileInfo): ProcessedAudio;
/**
* Parse audio metadata from a buffer using music-metadata.
*
* @param buffer - Audio file content
* @param fileInfo - File information (used for MIME type hint)
* @returns Parsed audio metadata from music-metadata
* @throws Error if the buffer cannot be parsed (corrupt file, unsupported format)
*/
private parseAudioMetadata;
/**
* Extract structured metadata from the parsed audio format information.
*
* @param audioMetadata - Parsed audio metadata from music-metadata
* @param fileSize - File size in bytes
* @returns Structured metadata object
*/
private extractMetadata;
/**
* Extract common tags from the parsed audio metadata.
*
* Maps music-metadata's common tag format to our simplified tag structure.
* Handles array-to-scalar conversions (e.g., comment[] -> first comment string).
*
* @param audioMetadata - Parsed audio metadata from music-metadata
* @returns Simplified tag object
*/
private extractTags;
/**
* Extract embedded cover art from the audio file.
*
* Uses music-metadata's selectCover() to pick the most appropriate
* cover image when multiple are embedded (e.g., front cover vs. back cover).
*
* @param audioMetadata - Parsed audio metadata from music-metadata
* @returns Cover art as Buffer, or null if no cover art is embedded
*/
private extractCoverArt;
/**
* Build an LLM-friendly text representation of the audio file.
*
* Produces a structured text block that gives the AI context about the
* audio file without requiring the actual audio stream. The format is
* designed to be scannable and information-dense.
*
* @param filename - Original filename
* @param metadata - Extracted audio metadata
* @param tags - Extracted audio tags
* @param transcript - Optional transcribed text from Whisper
* @param skippedReason - When `transcript` is absent, why transcription was
* skipped (no backend configured, unavailable pinned backend, format/size
* limit, or a backend call that failed) — inlined so the model is told the
* reason instead of silently receiving metadata with no transcript, which
* `AUDIO_TRANSCRIPTION_INSTRUCTIONS` (messageBuilder.ts) tells it to expect.
* @returns Formatted text content string
*
* @example Output:
* ```
* [Audio File: song.mp3]
* Duration: 3:45 | Codec: MPEG 1 Layer 3 | Bitrate: 320 kbps | Sample Rate: 44100 Hz | Channels: 2 (Stereo) | Lossless: No
* File Size: 5.00 MB
* Title: Yesterday | Artist: The Beatles | Album: Help! | Year: 1965 | Genre: Rock, Pop
* Track: 1/14 | Composer: Lennon-McCartney
*
* --- Transcript ---
* [full transcribed text here]
* ```
*/
private buildTextContent;
/**
* Format a duration in seconds to a human-readable string.
*
* Delegates to the shared formatter so audio and video describe the same
* file the same way — this used to render "0:02" where VideoProcessor
* rendered "2s".
*
* @param seconds - Duration in seconds
* @returns Formatted string: "45s", "3m 45s", "1h 2m 30s"
*/
private formatDuration;
/**
* Format bitrate to a human-readable string.
*
* @param bitrate - Bitrate in bits per second
* @returns Formatted string (e.g., "320 kbps", "1411 kbps")
*/
private formatBitrate;
/**
* Get a human-readable label for the number of audio channels.
*
* @param channels - Number of audio channels
* @returns Channel label (e.g., "Mono", "Stereo", "5.1 Surround")
*/
private getChannelLabel;
}
/**
* Singleton Audio processor instance.
* Use this for standard audio processing operations.
*
* @example
* ```typescript
* import { audioProcessor } from "./AudioProcessor.js";
*
* const result = await audioProcessor.processFile(fileInfo);
* ```
*/
export declare const audioProcessor: AudioProcessor;
/**
* Check if a file is an audio file.
* Matches by MIME type or file extension.
*
* @param mimetype - MIME type of the file
* @param filename - Filename (for extension-based detection)
* @returns true if the file is an audio file
*
* @example
* ```typescript
* if (isAudioFile('audio/mpeg', 'song.mp3')) {
* // Process as audio
* }
*
* if (isAudioFile('', 'recording.flac')) {
* // Also matches by extension
* }
* ```
*/
export declare function isAudioFile(mimetype: string, filename: string): boolean;
/**
* Process a single audio file.
* Convenience function that uses the singleton processor.
*
* @param fileInfo - File information (can include URL or buffer)
* @param options - Optional processing options (auth headers, timeout, etc.),
* widened with the transcription knobs (#413/#440, matching
* {@link AudioProcessor.processFile}) so a caller can pin a provider,
* language, model or prompt through this convenience export too.
* @returns Processing result with audio metadata or error
*
* @example
* ```typescript
* import { processAudio } from "./AudioProcessor.js";
*
* const result = await processAudio({
* id: 'audio-1',
* name: 'podcast.mp3',
* mimetype: 'audio/mpeg',
* size: 15728640,
* buffer: mp3Buffer,
* });
*
* if (result.success) {
* const { metadata, tags, textContent } = result.data;
* console.log(`${tags.title} by ${tags.artist} (${metadata.durationFormatted})`);
* // Send textContent to LLM for analysis
* } else {
* console.error(`Processing failed: ${result.error?.userMessage}`);
* }
* ```
*/
export declare function processAudio(fileInfo: FileInfo, options?: ProcessOptions & AudioProcessorOptions): Promise<ProcessorFileProcessingResult<ProcessedAudio>>;