@juspay/neurolink/dist/adapters/audioFormatSupport.js

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

/**
 * Native audio delivery to providers that can listen.
 *
 * ## The gap this closes
 *
 * Until this module existed, attaching an audio file produced a message
 * containing only a metadata block:
 *
 *   ## Audio File: "recording.mp3"
 *   Duration: 19s | Codec: MPEG 2 Layer 3 | Bitrate: 32 kbps |
 *   Sample Rate: 22050 Hz | Channels: 1 (Mono)
 *
 * No audio bytes were ever handed to the provider. Every question about what
 * the recording *says* — transcribe this, who is speaking, what was agreed —
 * was answered from a description of the file, and Gemini has accepted inline
 * audio the whole time.
 *
 * The failure was invisible for an instructive reason: that metadata block
 * answers precisely the questions a test is most tempted to ask. "How long is
 * this audio?" and "what sample rate is it?" both succeed with no audio
 * attached, so a suite built on them reports working audio support. It took an
 * end-to-end test asking for a spoken word to expose it.
 *
 * ## Provider scope
 *
 * Deliberately a capability map rather than "send audio to everyone". A
 * provider that cannot accept an audio part responds with an opaque HTTP 400,
 * which is worse than the metadata summary it would otherwise have received —
 * so an unlisted provider keeps the existing text-only behaviour and loses
 * nothing.
 *
 * @module adapters/audioFormatSupport
 */
import { withTimeout } from "../utils/errorHandling.js";
import { logger } from "../utils/logger.js";
import { getFfmpegPath, runFfmpeg } from "./video/ffmpegAdapter.js";
/**
 * Ceiling for one audio conversion.
 *
 * Longer than the image equivalent because a lossless hour-long WAV is a
 * legitimate input and re-encoding it is not instant, but still bounded so a
 * wedged decoder cannot hold a generation request open indefinitely.
 */
const AUDIO_TRANSCODE_TIMEOUT_MS = 120_000;
/**
 * Providers that accept inline audio parts.
 *
 * Google's Gemini models (both Vertex and AI Studio) take audio as `inlineData`
 * alongside text. Other providers are omitted rather than assumed: OpenAI's
 * audio models use a different request shape than the chat-completions path
 * NeuroLink builds here, and sending an audio part to a provider that does not
 * expect one converts a working (if limited) response into a hard failure.
 */
const NATIVE_AUDIO_PROVIDERS = new Set([
    "vertex",
    "google-vertex",
    "googlevertex",
    "google-ai-studio",
    "googleaistudio",
    "google-ai",
    "googleai",
    "gemini",
]);
/**
 * The Vertex-family aliases within {@link NATIVE_AUDIO_PROVIDERS}.
 *
 * Vertex is a front end for two unrelated model families: Gemini, which
 * accepts inline audio, and Claude (routed to `@anthropic-ai/vertex-sdk`),
 * which does not. The provider name alone cannot tell them apart, so these
 * aliases get an extra model check in {@link supportsNativeAudio}; the
 * AI-Studio/Gemini aliases never serve Claude and skip it.
 */
const VERTEX_PROVIDER_NAMES = new Set([
    "vertex",
    "google-vertex",
    "googlevertex",
]);
/**
 * Audio MIME types the native providers accept as-is.
 *
 * Gemini's documented set. Anything outside it is transcoded rather than
 * rejected, because the container a user happens to have — a voice memo in
 * CAF, a Windows recording in WMA — says nothing about whether the audio
 * inside is useful.
 */
const NATIVE_AUDIO_MIME_TYPES = new Set([
    "audio/wav",
    "audio/x-wav",
    "audio/mpeg",
    "audio/mp3",
    "audio/aiff",
    "audio/x-aiff",
    "audio/aac",
    "audio/ogg",
    "audio/flac",
    "audio/x-flac",
]);
/** MIME type every transcode targets. Universally accepted and compact. */
const TRANSCODE_TARGET_MIME = "audio/mpeg";
/**
 * Whether `provider` can be handed raw audio bytes.
 *
 * `model` is optional and only consulted for the Vertex aliases, which serve
 * both Gemini (native audio) and Claude (no native audio — routed to
 * `@anthropic-ai/vertex-sdk`, whose message builder never reads an audio
 * part). The Claude test is the one `GoogleVertexProvider` routes by
 * (`isAnthropicModel` in providers/googleVertex/client.ts: case-insensitive,
 * anywhere in the id). A stricter test here would call a model Gemini that
 * Vertex actually sends to Claude. A caller that omits `model` on Vertex keeps
 * today's behaviour. AI Studio / bare "gemini" aliases never serve Claude.
 */
export function supportsNativeAudio(provider, model) {
    const normalizedProvider = provider.toLowerCase().trim();
    if (!NATIVE_AUDIO_PROVIDERS.has(normalizedProvider)) {
        return false;
    }
    if (VERTEX_PROVIDER_NAMES.has(normalizedProvider)) {
        return !(model ?? "").toLowerCase().includes("claude");
    }
    return true;
}
/** Whether `mimeType` must be re-encoded before a native provider will read it. */
export function needsAudioTranscode(mimeType) {
    return !NATIVE_AUDIO_MIME_TYPES.has(normalizeAudioMime(mimeType));
}
function normalizeAudioMime(mimeType) {
    return mimeType.split(";")[0].trim().toLowerCase();
}
/**
 * Re-encode audio to MP3 with ffmpeg.
 *
 * Temp files rather than stdin: several of the containers that need converting
 * (CAF, WavPack, AU) carry their metadata in a trailer or require seeking, and
 * a piped stream leaves ffmpeg unable to find it. The directory is removed in
 * `finally` whether or not the conversion succeeded.
 *
 * Node builtins are imported dynamically because the browser bundle stubs
 * `node:fs/promises` without `mkdtemp`; nothing in a browser spawns ffmpeg, so
 * the import belongs at the point of use.
 */
async function transcodeToMp3(buffer, extension) {
    const [{ randomUUID }, { mkdtemp, readFile, rm, writeFile }, { tmpdir }, { join },] = await Promise.all([
        import("node:crypto"),
        import("node:fs/promises"),
        import("node:os"),
        import("node:path"),
    ]);
    const workDir = await mkdtemp(join(tmpdir(), "neurolink-audio-"));
    const inputPath = join(workDir, `${randomUUID()}${extension}`);
    const outputPath = join(workDir, `${randomUUID()}.mp3`);
    try {
        await writeFile(inputPath, buffer);
        await runFfmpeg([
            "-y",
            "-v",
            "error",
            "-i",
            inputPath,
            // Downmix and cap the rate: speech is the point, and a 48 kHz stereo
            // re-encode of a mono voice memo triples the payload for nothing.
            "-ac",
            "1",
            "-ar",
            "16000",
            "-c:a",
            "libmp3lame",
            "-q:a",
            "4",
            outputPath,
        ], 
        // Without this the call inherits runFfmpeg's frame-extraction default of
        // 30s, which is sized for pulling a single video frame — so the 120s
        // ceiling above, chosen precisely because re-encoding a lossless
        // hour-long WAV is not instant, could never be reached. ffmpeg killed the
        // transcode at 30s and the outer race never got to run.
        { timeoutMs: AUDIO_TRANSCODE_TIMEOUT_MS });
        return await readFile(outputPath);
    }
    finally {
        await rm(workDir, { recursive: true, force: true }).catch(() => undefined);
    }
}
/**
 * Return audio bytes a native provider can read, transcoding when the source
 * container is one it does not accept.
 *
 * Never throws for audio reasons. When conversion is impossible — no ffmpeg, an
 * unreadable stream — the original bytes and MIME type come back with
 * `converted: false`, and the caller falls back to the metadata summary. That
 * keeps this from turning a previously-working (if limited) request into a
 * failure.
 *
 * @param buffer - Raw audio bytes.
 * @param mimeType - Detected MIME type of `buffer`.
 * @param extension - Source extension, used so ffmpeg picks the right demuxer.
 */
export async function toProviderCompatibleAudio(buffer, mimeType, extension) {
    const normalized = normalizeAudioMime(mimeType);
    if (!needsAudioTranscode(normalized)) {
        return { buffer, mimeType: normalized, converted: false };
    }
    // Resolving the binary first turns "ffmpeg is not installed" into one clear
    // warning rather than a spawn error surfacing from inside the conversion.
    const ffmpegAvailable = await getFfmpegPath()
        .then(() => true)
        .catch(() => false);
    if (!ffmpegAvailable) {
        logger.warn(`[audioFormatSupport] ${normalized} needs conversion before a provider can ` +
            `read it, but ffmpeg is unavailable — falling back to a metadata-only ` +
            `summary. Install ffmpeg (or set FFMPEG_PATH) to enable this format.`);
        return { buffer, mimeType: normalized, converted: false };
    }
    try {
        const converted = await withTimeout(transcodeToMp3(buffer, extension), AUDIO_TRANSCODE_TIMEOUT_MS, new Error(`audio transcode exceeded ${AUDIO_TRANSCODE_TIMEOUT_MS}ms`));
        if (converted.length === 0) {
            throw new Error("produced an empty audio stream");
        }
        logger.debug(`[audioFormatSupport] Transcoded ${normalized} → ${TRANSCODE_TARGET_MIME} ` +
            `(${buffer.length} → ${converted.length} bytes) for native delivery`);
        return {
            buffer: converted,
            mimeType: TRANSCODE_TARGET_MIME,
            converted: true,
        };
    }
    catch (error) {
        logger.warn(`[audioFormatSupport] Could not convert ${normalized} for native delivery ` +
            `— falling back to a metadata-only summary: ` +
            `${error instanceof Error ? error.message.split("\n")[0] : String(error)}`);
        return { buffer, mimeType: normalized, converted: false };
    }
}