@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
870 lines (869 loc) • 38.2 kB
JavaScript
/**
* Audio File Processor
*
* Handles downloading, validating, and processing audio files to extract metadata
* and build text content suitable for LLM consumption. Audio files cannot be sent
* raw to most LLMs, so this processor extracts structured metadata (duration, codec,
* bitrate, tags) and formats it as text.
*
* Uses the `music-metadata` library (pure JavaScript, no native dependencies) for
* metadata extraction. Supports all major audio formats: MP3, WAV, OGG, FLAC, M4A,
* AAC, WMA, WebM, AIFF, AMR, APE, WavPack, and more.
*
* Key features:
* - Metadata extraction: duration, codec, bitrate, sample rate, channels
* - Tag extraction: title, artist, album, year, genre, track number, composer
* - Embedded cover art extraction
* - Graceful degradation for corrupt or partially readable files
* - LLM-friendly text content generation
*
* @module processors/media/AudioProcessor
*
* @example
* ```typescript
* import { audioProcessor, processAudio, isAudioFile } from "./AudioProcessor.js";
*
* // Check if a file is an audio file
* if (isAudioFile(fileInfo.mimetype, fileInfo.name)) {
* const result = await processAudio(fileInfo);
*
* if (result.success) {
* console.log(`Duration: ${result.data.metadata.durationFormatted}`);
* console.log(`Codec: ${result.data.metadata.codec}`);
* console.log(`Artist: ${result.data.tags.artist}`);
* console.log(`Text for LLM: ${result.data.textContent}`);
* }
* }
* ```
*/
import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
import { getSTTProviderDescriptor, isSTTProviderConfigured, resolveDefaultSTTProvider, resolveSTTProviderName, STT_PROVIDER_DESCRIPTORS, } from "../../factories/sttDescriptors.js";
import { STTProcessor } from "../../utils/sttProcessor.js";
import { freshEnvSTTCredentials } from "../../voice/sttEnvCredentials.js";
import { SIZE_LIMITS_MB } from "../config/index.js";
import { extensionsForModality, mimeTypesForModality, } from "../config/fileTypeRegistry.js";
import { FileErrorCode } from "../errors/index.js";
import { withTimeout } from "../../utils/timeout.js";
import { formatMediaDuration } from "../../utils/mediaDuration.js";
import { logger } from "../../utils/logger.js";
import { tryImport } from "../../utils/tryImport.js";
let _musicMetadata = null;
async function loadMusicMetadata() {
if (_musicMetadata) {
return _musicMetadata;
}
_musicMetadata = await tryImport("music-metadata", "Audio processing");
return _musicMetadata;
}
// =============================================================================
// TYPES
// =============================================================================
// =============================================================================
// CONSTANTS
// =============================================================================
/**
* Audio processor configuration constants.
*/
const AUDIO_CONFIG = {
/** Maximum audio file size in MB (uses centralized constant from sizeLimits) */
MAX_SIZE_MB: SIZE_LIMITS_MB.AUDIO_MAX_MB,
/** Processing timeout in milliseconds (audio metadata parsing is fast) */
TIMEOUT_MS: 30000,
/** Maximum file size for Whisper API transcription (25MB) */
WHISPER_MAX_SIZE_MB: 25,
/** Transcription timeout in milliseconds (120 seconds for large files) */
TRANSCRIPTION_TIMEOUT_MS: 120_000,
};
/**
* Supported MIME types for audio files.
*
* Derived from the canonical registry so this processor cannot claim a format
* the detector standing in front of it does not recognise. It previously did:
* .aiff, .amr, .ape, .wv and .oga were all declared here and all resolved to
* "unknown" during detection, so a file in one of those formats never reached
* this processor at all.
*
* `audio/webm` is appended because WebM is registered as a video container and
* an audio-only .webm is legitimate input. Only the MIME type is accepted, not
* the extension: `isFileSupported` matches on extension OR mimetype, so
* claiming `.webm` here made `isAudioFile("video/webm", "clip.webm")` accept a
* video-only WebM as audio on its filename alone.
*/
const SUPPORTED_AUDIO_MIME_TYPES = [
...mimeTypesForModality("audio"),
"audio/webm",
];
/**
* Supported file extensions for audio files.
* Derived from the canonical registry — see the note above.
*/
const SUPPORTED_AUDIO_EXTENSIONS = extensionsForModality("audio");
/**
* Backend names this processor has always accepted in
* `AudioProcessorOptions.provider`, mapped onto STT provider names. Anything
* else is resolved through the STT descriptors (`whisper`, `deepgram`,
* `scribe`, `local`, …), so every shipped STT provider can be pinned here.
*/
const LEGACY_TRANSCRIPTION_ALIASES = {
openai: "whisper",
"openai-whisper": "whisper",
google: "google-stt",
"google-speech": "google-stt",
azure: "azure-stt",
"azure-speech": "azure-stt",
};
/**
* The label reported in log lines and in `ProcessedAudio.transcriptionProvider`.
* Whisper keeps the label it has always had; every other provider reports its
* STT provider name.
*/
function transcriptionLabel(provider) {
return provider === "whisper" ? "openai-whisper" : provider;
}
/** The STT provider a caller's backend name means, if it means one. */
function resolveTranscriptionProvider(requested) {
const normalized = requested.trim().toLowerCase();
return (LEGACY_TRANSCRIPTION_ALIASES[normalized] ??
resolveSTTProviderName(normalized));
}
/**
* Map an audio mimetype/extension onto the format vocabulary the STT handlers
* expect. Returns undefined when nothing matches, which leaves the handler on
* its own "wav" default rather than asserting a format that is wrong.
*/
function toSTTAudioFormat(extension, mimetype) {
const candidates = [
"mp3",
"wav",
"ogg",
"opus",
"m4a",
"flac",
"webm",
"mp4",
"mpeg",
"mpga",
];
if (extension) {
const match = candidates.find((format) => format === extension);
if (match) {
return match;
}
}
const subtype = mimetype?.split(";")[0].trim().toLowerCase().split("/")[1];
if (!subtype) {
return undefined;
}
// "audio/x-m4a" and "audio/x-wav" are both in the wild.
const normalized = subtype.startsWith("x-") ? subtype.slice(2) : subtype;
return candidates.find((format) => format === normalized);
}
// =============================================================================
// AUDIO PROCESSOR CLASS
// =============================================================================
/**
* Audio Processor - extracts metadata and tags from audio files for LLM consumption.
*
* Audio files cannot be directly sent to most language models. This processor
* parses audio file headers to extract structured metadata (duration, codec,
* bitrate, sample rate, channels) and embedded tags (title, artist, album, etc.),
* then builds a human-readable text summary for the AI to reason about.
*
* Uses the `music-metadata` library which is a pure JavaScript implementation
* with no native dependencies, making it safe for all deployment environments.
*
* @example
* ```typescript
* const processor = new AudioProcessor();
*
* const result = await processor.processFile({
* id: 'audio-123',
* name: 'song.mp3',
* mimetype: 'audio/mpeg',
* size: 5242880,
* buffer: audioBuffer,
* });
*
* if (result.success) {
* console.log(result.data.textContent);
* // "[Audio File: song.mp3]
* // Duration: 3:45 | Codec: MPEG 1 Layer 3 | Bitrate: 320 kbps | ..."
* }
* ```
*/
export class AudioProcessor extends BaseFileProcessor {
constructor() {
super({
maxSizeMB: AUDIO_CONFIG.MAX_SIZE_MB,
timeoutMs: AUDIO_CONFIG.TIMEOUT_MS,
supportedMimeTypes: [...SUPPORTED_AUDIO_MIME_TYPES],
supportedExtensions: [...SUPPORTED_AUDIO_EXTENSIONS],
fileTypeName: "audio",
defaultFilename: "audio.mp3",
});
}
// ===========================================================================
// PROCESSING OVERRIDE
// ===========================================================================
/**
* Override processFile for async audio metadata parsing with music-metadata.
*
* Processing pipeline:
* 1. Validate file type and size (base class)
* 2. Get file buffer (from direct buffer or download)
* 3. Parse audio metadata using music-metadata's parseBuffer()
* 4. Extract tags (title, artist, album, etc.)
* 5. Extract embedded cover art if present
* 6. Build LLM-friendly text content
*
* @param fileInfo - File information (can include URL or buffer)
* @param options - Optional processing options (auth headers, timeout, etc.)
* widened with the transcription knobs (#413/#440) so a caller-chosen
* backend, language or prompt reaches the transcriber instead of being
* discarded at the detector boundary.
* @returns Processing result with audio metadata or error
*/
async processFile(fileInfo, options) {
try {
// Step 1: Validate file type and size
const validationResult = this.validateFileWithResult(fileInfo);
if (!validationResult.success) {
return {
success: false,
error: validationResult.error,
};
}
// Step 2: Get file buffer (from direct buffer or download from URL)
let buffer;
if (fileInfo.buffer) {
buffer = fileInfo.buffer;
}
else if (fileInfo.url) {
const downloadResult = await this.downloadFileWithRetry(fileInfo, options);
if (!downloadResult.success) {
return {
success: false,
error: downloadResult.error,
};
}
if (!downloadResult.data) {
return {
success: false,
error: this.createError(FileErrorCode.DOWNLOAD_FAILED, {
reason: "Download succeeded but returned no data",
}),
};
}
buffer = downloadResult.data;
// Validate actual downloaded size against limit
if (!this.validateFileSize(buffer.length)) {
return {
success: false,
error: this.createError(FileErrorCode.FILE_TOO_LARGE, {
sizeMB: (buffer.length / (1024 * 1024)).toFixed(2),
maxMB: this.config.maxSizeMB,
type: this.config.fileTypeName,
}),
};
}
}
else {
return {
success: false,
error: this.createError(FileErrorCode.DOWNLOAD_FAILED, {
reason: "No buffer or URL provided for file",
}),
};
}
// Step 3: Parse audio metadata using music-metadata
const audioMetadata = await this.parseAudioMetadata(buffer, fileInfo);
// Step 4: Extract structured metadata from parsed result
const metadata = this.extractMetadata(audioMetadata, buffer.length);
// Step 5: Extract tags from common metadata
const tags = this.extractTags(audioMetadata);
// Step 6: Extract embedded cover art if present
const coverArt = await this.extractCoverArt(audioMetadata);
// Step 7: Attempt transcription if a backend is configured
const filename = this.getFilename(fileInfo);
const transcriptionResult = await this.attemptTranscription(buffer, filename, fileInfo.mimetype, options);
// Step 8: Build LLM-friendly text content (includes transcript if
// available, or the skip reason when it is not — #471)
const textContent = this.buildTextContent(filename, metadata, tags, transcriptionResult.transcript, transcriptionResult.transcriptionSkippedReason);
return {
success: true,
data: {
textContent,
metadata,
tags,
transcript: transcriptionResult.transcript,
hasTranscript: transcriptionResult.hasTranscript,
transcriptionProvider: transcriptionResult.transcriptionProvider,
transcriptionLanguage: transcriptionResult.transcriptionLanguage,
transcriptionDuration: transcriptionResult.transcriptionDuration,
...(transcriptionResult.transcriptionSkippedReason
? {
transcriptionSkippedReason: transcriptionResult.transcriptionSkippedReason,
}
: {}),
coverArt: coverArt ?? undefined,
buffer,
mimetype: fileInfo.mimetype || "audio/mpeg",
size: fileInfo.size,
filename,
},
};
}
catch (error) {
// Classify music-metadata parse errors as INVALID_FORMAT
// (corrupt/truncated files, unsupported codec variants, etc.)
const isParseError = error instanceof Error &&
(error.message.includes("parse") ||
error.message.includes("codec") ||
error.message.includes("header") ||
error.message.includes("format") ||
error.message.includes("unexpected end") ||
error.name === "CouldNotDetermineFileTypeError" ||
error.name === "UnsupportedFileTypeError");
const errorCode = isParseError
? FileErrorCode.INVALID_FORMAT
: FileErrorCode.PROCESSING_FAILED;
return {
success: false,
error: this.createError(errorCode, {
fileType: "audio",
error: error instanceof Error ? error.message : String(error),
}, error instanceof Error ? error : undefined),
};
}
}
// ===========================================================================
// PRIVATE: AUDIO TRANSCRIPTION
// ===========================================================================
/**
* Choose the transcription backend for this file (#413).
*
* With no `requested` backend, the STT default applies: the provider named
* by `NEUROLINK_STT_PROVIDER`, else the first configured STT provider in
* descriptor order, else the built-in local engine when it can run here.
* With one, it is resolved through the legacy names and the STT aliases and
* validated for availability — a backend the caller explicitly asked for is
* never silently swapped for a different one, because a caller who pinned
* Azure for a data-residency reason would not want OpenAI chosen behind
* their back. The mismatch is reported instead.
*
* Availability is decided from the environment only — no network call — so
* selection stays cheap and cannot itself fail. A key that is present but
* rejected upstream surfaces later, as a transcription failure with the
* provider's own message, not as "unavailable".
*
* @param requested - Caller's `AudioProcessorOptions.provider`, if any
* @returns The chosen backend, or the reason no backend could be chosen
*/
async selectProvider(requested) {
const normalized = requested?.trim();
if (normalized) {
const resolved = resolveTranscriptionProvider(normalized);
if (!resolved) {
return {
reason: `transcription provider "${requested}" is not one this processor can drive — ` +
`supported: ${STT_PROVIDER_DESCRIPTORS.map((d) => d.name).join(", ")}`,
};
}
if (!isSTTProviderConfigured(resolved)) {
const envVars = getSTTProviderDescriptor(resolved)?.envVars ?? [];
return {
reason: `transcription provider "${normalized}" was requested but is not configured — ` +
`set one of: ${envVars.join(", ")}`,
};
}
const label = transcriptionLabel(resolved);
logger.debug(`[AudioProcessor] Using caller-selected transcription provider: ${label}`);
return { provider: resolved, label };
}
const provider = resolveDefaultSTTProvider();
const descriptor = getSTTProviderDescriptor(provider);
// The local engine is the default of last resort, but only when it can
// actually run on this machine; otherwise "nothing is configured" is the
// honest reason, and the hosted options are what fixes it.
if (descriptor?.capabilities.local) {
const handler = await STTProcessor.resolveHandler(provider, freshEnvSTTCredentials(descriptor.credentialsKey));
if (!handler?.isConfigured()) {
return {
reason: `no transcription backend is configured — set one of: ${STT_PROVIDER_DESCRIPTORS.flatMap((d) => d.envVars).join(", ")}`,
};
}
}
const label = transcriptionLabel(provider);
logger.debug(`[AudioProcessor] Auto-selected transcription provider: ${label}`);
return { provider, label };
}
/**
* Attempt to transcribe audio with whichever backend is configured (#413).
*
* Transcription is attempted when:
* 1. A backend is available (see {@link selectProvider})
* 2. File size is within the 25MB ceiling
* 3. The file format is one the backend accepts
*
* The call itself goes through `STTProcessor`, the same path as
* `transcribe()` and `generate({ stt })`, so `OPENAI_STT_*`, the default
* provider rule and every STT handler apply here too.
*
* Gracefully degrades: if transcription fails for any reason, metadata-only
* output is returned (transcription is additive, never blocks processing).
*
* @param buffer - Audio file content
* @param filename - Original filename (used for format detection)
* @param mimetype - MIME type of the audio file
* @param options - Caller's transcription knobs (provider, language, prompt)
* @returns Transcription result with transcript text, or the reason there is none
*/
async attemptTranscription(buffer, filename, mimetype, options) {
/**
* Every exit below used to return an indistinguishable empty result, so
* "no API key", "file too large", "format the backend can't read" and "the
* call failed" were impossible to tell apart — from the outside it just
* looked like the audio had no speech (#416).
*
* `transcriptionProvider` is left undefined by default (no backend ran),
* but a caller that reached a backend and got a genuine empty transcript
* back passes its label through. `FileDetector` only writes
* `transcriptionLength` when `transcriptionProvider` is set, precisely so
* "a provider ran and reported no speech" (`transcriptionLength: 0`)
* stays distinguishable from "no provider ever ran" (field omitted).
*/
const skipped = (reason, transcriptionProvider) => {
logger.warn(`[AudioProcessor] No transcript for ${filename}: ${reason}. ` +
`The model will receive metadata only.`);
return {
transcript: undefined,
hasTranscript: false,
transcriptionProvider,
transcriptionLanguage: undefined,
transcriptionDuration: undefined,
transcriptionSkippedReason: reason,
};
};
try {
const selection = await this.selectProvider(options?.provider);
if (!("provider" in selection)) {
return skipped(selection.reason);
}
const { provider, label } = selection;
// Size ceiling. Whisper documents 25MB; the other backends' synchronous
// endpoints are lower still, so this is a floor on what is worth sending
// rather than a per-backend limit.
const fileSizeMB = buffer.length / (1024 * 1024);
if (fileSizeMB > AUDIO_CONFIG.WHISPER_MAX_SIZE_MB) {
return skipped(`file is ${fileSizeMB.toFixed(1)}MB, over the ${AUDIO_CONFIG.WHISPER_MAX_SIZE_MB}MB transcription limit — split or compress it`);
}
const descriptor = getSTTProviderDescriptor(provider);
const credentials = descriptor
? freshEnvSTTCredentials(descriptor.credentialsKey)
: undefined;
const handler = await STTProcessor.resolveHandler(provider, credentials);
// Format gate: the handler's own accepted list, checked before upload
// so an unreadable file is reported as such rather than as a failure.
const ext = filename.split(".").pop()?.toLowerCase();
const format = toSTTAudioFormat(ext, mimetype);
const supported = handler?.getSupportedFormats() ?? [];
if (!format || (supported.length > 0 && !supported.includes(format))) {
const displayName = provider === "whisper" ? "Whisper" : label;
return skipped(`format is not one ${displayName} accepts (extension "${ext ?? "none"}", mimetype "${mimetype ?? "none"}"); supported: ${supported.join(", ")}`);
}
const sttOptions = {
format,
timeoutMs: AUDIO_CONFIG.TRANSCRIPTION_TIMEOUT_MS,
maxAudioBytes: AUDIO_CONFIG.WHISPER_MAX_SIZE_MB * 1024 * 1024,
...(options?.language ? { language: options.language } : {}),
...(options?.transcriptionModel
? { model: options.transcriptionModel }
: {}),
...(options?.prompt ? { prompt: options.prompt } : {}),
};
// `withTimeout` bounds handlers that keep no timer of their own; the
// ones that do abort their request at the same deadline.
const result = await withTimeout(STTProcessor.transcribe(buffer, provider, sttOptions, credentials), AUDIO_CONFIG.TRANSCRIPTION_TIMEOUT_MS, label, "generate");
const text = result.text.trim();
if (text.length === 0) {
// A successful call that returned nothing is a legitimate outcome
// (silence, music, no speech) — distinct from a failure.
const displayName = provider === "whisper" ? "Whisper" : label;
return skipped(`${displayName} returned an empty transcript for this audio`, label);
}
logger.debug(`[AudioProcessor] Transcribed ${filename} via ${label} (${text.length} chars)`);
return {
transcript: text,
hasTranscript: true,
transcriptionProvider: label,
transcriptionLanguage: result.language && result.language.length > 0
? result.language
: options?.language,
transcriptionDuration: result.duration,
transcriptionSkippedReason: undefined,
};
}
catch (error) {
// Transcription stays best-effort — a failure must never kill the whole
// processing pipeline. But discarding the error outright made a bad API
// key, a rate limit and a network blip all look identical to "this
// file has no speech in it".
const message = error instanceof Error ? error.message : String(error);
return skipped(`transcription request failed — ${message}`);
}
}
// ===========================================================================
// STUB: buildProcessedResult (required by base class, unused due to override)
// ===========================================================================
/**
* Stub implementation required by BaseFileProcessor.
* Not used because processFile is fully overridden.
*
* @param buffer - File buffer
* @param fileInfo - File information
* @returns Empty ProcessedAudio structure
*/
buildProcessedResult(buffer, fileInfo) {
return {
textContent: "",
metadata: {
duration: 0,
durationFormatted: formatMediaDuration(0),
codec: "unknown",
lossless: false,
fileSize: buffer.length,
},
tags: {},
hasTranscript: false,
buffer,
mimetype: fileInfo.mimetype || "audio/mpeg",
size: fileInfo.size,
filename: this.getFilename(fileInfo),
};
}
// ===========================================================================
// PRIVATE: METADATA PARSING
// ===========================================================================
/**
* Parse audio metadata from a buffer using music-metadata.
*
* @param buffer - Audio file content
* @param fileInfo - File information (used for MIME type hint)
* @returns Parsed audio metadata from music-metadata
* @throws Error if the buffer cannot be parsed (corrupt file, unsupported format)
*/
async parseAudioMetadata(buffer, fileInfo) {
// Provide MIME type as a string hint to music-metadata for more accurate parsing.
// parseBuffer accepts (Uint8Array, fileInfo?: IFileInfo | string, options?)
// where string is interpreted as MIME type.
const mimeType = fileInfo.mimetype || undefined;
const { parseBuffer } = await loadMusicMetadata();
return parseBuffer(buffer, mimeType);
}
/**
* Extract structured metadata from the parsed audio format information.
*
* @param audioMetadata - Parsed audio metadata from music-metadata
* @param fileSize - File size in bytes
* @returns Structured metadata object
*/
extractMetadata(audioMetadata, fileSize) {
const format = audioMetadata.format;
const duration = format.duration ?? 0;
const durationFormatted = this.formatDuration(duration);
return {
duration,
durationFormatted,
codec: format.codec ?? format.container ?? "unknown",
codecProfile: format.codecProfile ?? undefined,
bitrate: format.bitrate ?? undefined,
sampleRate: format.sampleRate ?? undefined,
channels: format.numberOfChannels ?? undefined,
bitsPerSample: format.bitsPerSample ?? undefined,
lossless: format.lossless ?? false,
fileSize,
};
}
/**
* Extract common tags from the parsed audio metadata.
*
* Maps music-metadata's common tag format to our simplified tag structure.
* Handles array-to-scalar conversions (e.g., comment[] -> first comment string).
*
* @param audioMetadata - Parsed audio metadata from music-metadata
* @returns Simplified tag object
*/
extractTags(audioMetadata) {
const common = audioMetadata.common;
return {
title: common.title ?? undefined,
artist: common.artist ?? undefined,
album: common.album ?? undefined,
year: common.year ?? undefined,
genre: common.genre && common.genre.length > 0 ? common.genre : undefined,
track: common.track.no !== null || common.track.of !== null
? { no: common.track.no, of: common.track.of }
: undefined,
comment: common.comment && common.comment.length > 0
? (common.comment[0]?.text ?? undefined)
: undefined,
composer: common.composer && common.composer.length > 0
? common.composer[0]
: undefined,
};
}
/**
* Extract embedded cover art from the audio file.
*
* Uses music-metadata's selectCover() to pick the most appropriate
* cover image when multiple are embedded (e.g., front cover vs. back cover).
*
* @param audioMetadata - Parsed audio metadata from music-metadata
* @returns Cover art as Buffer, or null if no cover art is embedded
*/
async extractCoverArt(audioMetadata) {
const pictures = audioMetadata.common.picture;
if (!pictures || pictures.length === 0) {
return null;
}
const { selectCover } = await loadMusicMetadata();
const cover = selectCover(pictures);
if (!cover) {
return null;
}
return Buffer.from(cover.data);
}
// ===========================================================================
// PRIVATE: TEXT CONTENT BUILDING
// ===========================================================================
/**
* Build an LLM-friendly text representation of the audio file.
*
* Produces a structured text block that gives the AI context about the
* audio file without requiring the actual audio stream. The format is
* designed to be scannable and information-dense.
*
* @param filename - Original filename
* @param metadata - Extracted audio metadata
* @param tags - Extracted audio tags
* @param transcript - Optional transcribed text from Whisper
* @param skippedReason - When `transcript` is absent, why transcription was
* skipped (no backend configured, unavailable pinned backend, format/size
* limit, or a backend call that failed) — inlined so the model is told the
* reason instead of silently receiving metadata with no transcript, which
* `AUDIO_TRANSCRIPTION_INSTRUCTIONS` (messageBuilder.ts) tells it to expect.
* @returns Formatted text content string
*
* @example Output:
* ```
* [Audio File: song.mp3]
* Duration: 3:45 | Codec: MPEG 1 Layer 3 | Bitrate: 320 kbps | Sample Rate: 44100 Hz | Channels: 2 (Stereo) | Lossless: No
* File Size: 5.00 MB
* Title: Yesterday | Artist: The Beatles | Album: Help! | Year: 1965 | Genre: Rock, Pop
* Track: 1/14 | Composer: Lennon-McCartney
*
* --- Transcript ---
* [full transcribed text here]
* ```
*/
buildTextContent(filename, metadata, tags, transcript, skippedReason) {
const lines = [];
// Header line
lines.push(`[Audio File: ${filename}]`);
// Technical metadata line
const techParts = [];
techParts.push(`Duration: ${metadata.durationFormatted}`);
techParts.push(`Codec: ${metadata.codec}`);
if (metadata.codecProfile) {
techParts.push(`Profile: ${metadata.codecProfile}`);
}
if (metadata.bitrate) {
techParts.push(`Bitrate: ${this.formatBitrate(metadata.bitrate)}`);
}
if (metadata.sampleRate) {
techParts.push(`Sample Rate: ${metadata.sampleRate} Hz`);
}
if (metadata.channels) {
techParts.push(`Channels: ${metadata.channels} (${this.getChannelLabel(metadata.channels)})`);
}
if (metadata.bitsPerSample) {
techParts.push(`Bit Depth: ${metadata.bitsPerSample}-bit`);
}
techParts.push(`Lossless: ${metadata.lossless ? "Yes" : "No"}`);
lines.push(techParts.join(" | "));
// File size line
lines.push(`File Size: ${(metadata.fileSize / (1024 * 1024)).toFixed(2)} MB`);
// Tags line (only if any tags are present)
const tagParts = [];
if (tags.title) {
tagParts.push(`Title: ${tags.title}`);
}
if (tags.artist) {
tagParts.push(`Artist: ${tags.artist}`);
}
if (tags.album) {
tagParts.push(`Album: ${tags.album}`);
}
if (tags.year) {
tagParts.push(`Year: ${tags.year}`);
}
if (tags.genre && tags.genre.length > 0) {
tagParts.push(`Genre: ${tags.genre.join(", ")}`);
}
if (tagParts.length > 0) {
lines.push(tagParts.join(" | "));
}
// Secondary tags line (track, composer, comment)
const secondaryParts = [];
if (tags.track) {
const trackStr = tags.track.of !== null
? `${tags.track.no ?? "?"}/${tags.track.of}`
: `${tags.track.no ?? "?"}`;
secondaryParts.push(`Track: ${trackStr}`);
}
if (tags.composer) {
secondaryParts.push(`Composer: ${tags.composer}`);
}
if (tags.comment) {
secondaryParts.push(`Comment: ${tags.comment}`);
}
if (secondaryParts.length > 0) {
lines.push(secondaryParts.join(" | "));
}
// Transcript section (if transcription was performed), or — when it was
// not — the reason, so the model is told rather than left to guess or
// invent content for a recording it was never given a transcript of.
if (transcript) {
lines.push("");
lines.push("--- Transcript ---");
lines.push(transcript);
}
else if (skippedReason) {
lines.push("");
lines.push("--- Transcription Skipped ---");
lines.push(skippedReason);
}
return lines.join("\n");
}
// ===========================================================================
// PRIVATE: FORMATTING UTILITIES
// ===========================================================================
/**
* Format a duration in seconds to a human-readable string.
*
* Delegates to the shared formatter so audio and video describe the same
* file the same way — this used to render "0:02" where VideoProcessor
* rendered "2s".
*
* @param seconds - Duration in seconds
* @returns Formatted string: "45s", "3m 45s", "1h 2m 30s"
*/
formatDuration(seconds) {
return formatMediaDuration(seconds);
}
/**
* Format bitrate to a human-readable string.
*
* @param bitrate - Bitrate in bits per second
* @returns Formatted string (e.g., "320 kbps", "1411 kbps")
*/
formatBitrate(bitrate) {
const kbps = Math.round(bitrate / 1000);
return `${kbps} kbps`;
}
/**
* Get a human-readable label for the number of audio channels.
*
* @param channels - Number of audio channels
* @returns Channel label (e.g., "Mono", "Stereo", "5.1 Surround")
*/
getChannelLabel(channels) {
switch (channels) {
case 1:
return "Mono";
case 2:
return "Stereo";
case 6:
return "5.1 Surround";
case 8:
return "7.1 Surround";
default:
return `${channels}ch`;
}
}
}
// =============================================================================
// SINGLETON INSTANCE
// =============================================================================
/**
* Singleton Audio processor instance.
* Use this for standard audio processing operations.
*
* @example
* ```typescript
* import { audioProcessor } from "./AudioProcessor.js";
*
* const result = await audioProcessor.processFile(fileInfo);
* ```
*/
export const audioProcessor = new AudioProcessor();
// =============================================================================
// HELPER FUNCTIONS
// =============================================================================
/**
* Check if a file is an audio file.
* Matches by MIME type or file extension.
*
* @param mimetype - MIME type of the file
* @param filename - Filename (for extension-based detection)
* @returns true if the file is an audio file
*
* @example
* ```typescript
* if (isAudioFile('audio/mpeg', 'song.mp3')) {
* // Process as audio
* }
*
* if (isAudioFile('', 'recording.flac')) {
* // Also matches by extension
* }
* ```
*/
export function isAudioFile(mimetype, filename) {
return audioProcessor.isFileSupported(mimetype, filename);
}
/**
* Process a single audio file.
* Convenience function that uses the singleton processor.
*
* @param fileInfo - File information (can include URL or buffer)
* @param options - Optional processing options (auth headers, timeout, etc.),
* widened with the transcription knobs (#413/#440, matching
* {@link AudioProcessor.processFile}) so a caller can pin a provider,
* language, model or prompt through this convenience export too.
* @returns Processing result with audio metadata or error
*
* @example
* ```typescript
* import { processAudio } from "./AudioProcessor.js";
*
* const result = await processAudio({
* id: 'audio-1',
* name: 'podcast.mp3',
* mimetype: 'audio/mpeg',
* size: 15728640,
* buffer: mp3Buffer,
* });
*
* if (result.success) {
* const { metadata, tags, textContent } = result.data;
* console.log(`${tags.title} by ${tags.artist} (${metadata.durationFormatted})`);
* // Send textContent to LLM for analysis
* } else {
* console.error(`Processing failed: ${result.error?.userMessage}`);
* }
* ```
*/
export async function processAudio(fileInfo, options) {
return audioProcessor.processFile(fileInfo, options);
}