naisys/dist/features/media/listen.js

NAISYS - Autonomous AI agent runner with built-in context management and cost tracking

import { mimeFromFilename } from "@naisys/common";
import fs from "fs";
import path from "path";
import { listenCmd } from "../../command/commandDefs.js";
const SUPPORTED_EXTENSIONS = new Set([
    ".wav",
    ".mp3",
    ".m4a",
    ".flac",
    ".ogg",
    ".webm",
]);
export function createListenService({ agentConfig }, modelService, contextManager, llmService, shellWrapper) {
    async function handleCommand(args) {
        const trimmed = args.trim();
        // Parse --transcribe flag
        let transcribe = false;
        let filepath;
        if (trimmed.startsWith("--transcribe ")) {
            transcribe = true;
            filepath = trimmed.slice("--transcribe ".length).trim();
        }
        else {
            filepath = trimmed;
        }
        if (!filepath) {
            return `Usage: ${listenCmd.name} ${listenCmd.usage}`;
        }
        // Resolve relative paths against the shell's current working directory
        if (!path.isAbsolute(filepath)) {
            const cwd = await shellWrapper.getCurrentPath();
            if (cwd) {
                filepath = path.resolve(cwd, filepath);
            }
        }
        // Validate the agent's shellModel supports hearing
        const shellModel = agentConfig().shellModel;
        const model = modelService.getLlmModel(shellModel);
        if (!model.supportsHearing) {
            return `Error: Model '${shellModel}' does not support audio input. ns-listen requires an audio-capable model.`;
        }
        // Validate file exists
        if (!fs.existsSync(filepath)) {
            return `Error: File not found: ${filepath}`;
        }
        // Validate extension
        const ext = path.extname(filepath).toLowerCase();
        if (!SUPPORTED_EXTENSIONS.has(ext)) {
            return `Error: Unsupported audio type '${ext}'. Supported: ${[...SUPPORTED_EXTENSIONS].join(", ")}`;
        }
        // Read file and encode to base64
        const fileBuffer = fs.readFileSync(filepath);
        const base64 = fileBuffer.toString("base64");
        const mimeType = mimeFromFilename(filepath);
        if (transcribe) {
            // One-shot: send audio to LLM for a text transcription
            const transcribeSystemPrompt = "You are an audio transcription assistant. Transcribe the given audio accurately. Include speaker labels if multiple speakers are detected. Be precise and faithful to the original audio.";
            const result = await llmService.query(shellModel, transcribeSystemPrompt, [
                {
                    role: "user",
                    content: [
                        {
                            type: "text",
                            text: `Transcribe this audio (${filepath}):`,
                        },
                        {
                            type: "audio",
                            base64,
                            mimeType,
                        },
                    ],
                },
            ], "listen");
            return `[${filepath}]\n${result.responses.join("\n")}`;
        }
        // Default: pin audio into context. appendAudio returns a block reason
        // when the model can't ingest audio (e.g. shellModel changed) — surface
        // it to the agent like look.ts does for appendImage. Mirrors the upfront
        // supportsHearing check above as defense in depth.
        return contextManager.appendAudio(base64, mimeType, filepath);
    }
    const registrableCommand = {
        command: listenCmd,
        handleCommand,
    };
    return {
        ...registrableCommand,
    };
}
//# sourceMappingURL=listen.js.map