UNPKG

@juspay/neurolink

Version:

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

796 lines (795 loc) • 25.1 kB
/** * Speech-to-Text (STT) Type Definitions for NeuroLink * * All STT-specific types: options, results, handlers, * provider-specific options, error codes, defaults, and type guards. * * @module types/stt */ import type { TTSAudioFormat } from "./tts.js"; /** * STT configuration options */ export type STTOptions = { /** Enable STT processing */ enabled?: boolean; /** Override STT provider */ provider?: string; /** Language code for transcription (e.g., "en-US") */ language?: string; /** Audio format of input */ format?: TTSAudioFormat; /** Sample rate in Hz */ sampleRate?: number; /** Enable punctuation in transcription */ punctuation?: boolean; /** Enable punctuation (alias) */ punctuate?: boolean; /** Enable profanity filter */ profanityFilter?: boolean; /** Enable speaker diarization */ speakerDiarization?: boolean; /** Enable speaker diarization (alias) */ diarization?: boolean; /** Number of speakers (for diarization) */ speakerCount?: number; /** Enable word-level timestamps */ wordTimestamps?: boolean; /** Model variant to use */ model?: string; /** Custom vocabulary/phrases */ vocabulary?: string[]; /** * Context prompt for engines that bias on one (Whisper and other * OpenAI-compatible servers). Engines without a prompt ignore it; * `vocabulary` terms are appended by the handlers that accept a prompt. */ prompt?: string; /** Per-request timeout in milliseconds, for handlers that honour one. */ timeoutMs?: number; /** Minimum confidence threshold */ confidenceThreshold?: number; /** * Maximum audio buffer size in bytes. STTProcessor rejects buffers over * this limit before any provider call, preventing OOM on multi-GB inputs. * Default: 25_000_000 (matches Whisper's documented 25MB ceiling). */ maxAudioBytes?: number; }; /** * STT result from transcription */ /** * Where a provider's `STTResult.confidence` came from, when it has to say — * Scribe reports a per-word `logprob` on some responses and only a * language-detection probability on others, and a caller reading the number * should know which it is getting. `"none"` means the provider reported no * transcript-level signal at all and the field is 0. */ export type STTConfidenceSource = "word_logprobs" | "language_probability" | "none"; export type STTResult = { /** Full transcribed text */ text: string; /** * Transcript confidence (0-1) — Deepgram's alternative confidence, * Whisper's `exp(avg_logprob)`, Scribe's mean per-word `exp(logprob)`. * Always present: a strict-TS consumer reads it without a guard. A provider * with no transcript-level signal reports 0 and, where it can, says so in * `metadata.confidenceSource` rather than passing off a different number * (a language-detection probability, say) as this one silently. */ confidence: number; /** Detected language code */ language?: string; /** Audio duration in seconds */ duration?: number; /** Word-level timings */ words?: WordTiming[]; /** Transcription segments */ segments?: TranscriptionSegment[]; /** Speaker labels (for diarization) */ speakers?: string[]; /** Performance metadata */ metadata?: { /** Processing latency in milliseconds */ latency: number; /** Provider name */ provider?: string; /** Model used */ model?: string; /** What `confidence` was derived from, for providers that must choose. */ confidenceSource?: STTConfidenceSource; /** Additional provider-specific metadata */ [key: string]: unknown; }; }; /** * STT language information */ export type STTLanguage = { /** Language code (e.g., "en-US") */ code: string; /** Language name */ name: string; /** Whether the language supports speaker diarization */ supportsDiarization?: boolean; /** Whether the language supports punctuation */ supportsPunctuation?: boolean; }; /** * Word-level timing information */ export type WordTiming = { /** The word */ word: string; /** Start time in seconds */ startTime?: number; /** Start time alias */ start?: number; /** End time in seconds */ endTime?: number; /** End time alias */ end?: number; /** Confidence score (0-1) */ confidence?: number; /** Speaker label (for diarization) */ speaker?: string; }; /** * Transcription segment for streaming STT */ export type TranscriptionSegment = { /** Segment index */ index?: number; /** Transcribed text */ text: string; /** Whether this is a final result */ isFinal: boolean; /** Confidence score (0-1) */ confidence?: number; /** Start time in audio (seconds) */ startTime?: number; /** Start time (alias for startTime) */ start?: number; /** End time in audio (seconds) */ endTime?: number; /** End time (alias for endTime) */ end?: number; /** Word-level timings */ words?: WordTiming[]; /** Speaker label */ speaker?: string; /** Detected language */ language?: string; }; export type STTHandler = { transcribe(audio: Buffer | ArrayBuffer, options: STTOptions): Promise<STTResult>; transcribeStream?(audioStream: AsyncIterable<Buffer>, options: STTOptions): AsyncIterable<TranscriptionSegment>; getSupportedLanguages?(): Promise<STTLanguage[]>; getSupportedFormats(): TTSAudioFormat[]; isConfigured(): boolean; /** * What would make `isConfigured()` true, in the handler's own words — the * env var or key for a hosted engine, the files and their download URLs for * a local one. Appended to the "not configured" error so the caller is told * what to do rather than "set the required API keys". */ describeConfiguration?(): string; maxAudioDuration?: number; supportsStreaming?: boolean; }; /** * STT error codes */ export declare const STT_ERROR_CODES: { readonly AUDIO_EMPTY: "STT_AUDIO_EMPTY"; readonly AUDIO_TOO_LONG: "STT_AUDIO_TOO_LONG"; readonly INVALID_AUDIO_FORMAT: "STT_INVALID_AUDIO_FORMAT"; readonly LANGUAGE_NOT_SUPPORTED: "STT_LANGUAGE_NOT_SUPPORTED"; readonly TRANSCRIPTION_FAILED: "STT_TRANSCRIPTION_FAILED"; readonly PROVIDER_NOT_CONFIGURED: "STT_PROVIDER_NOT_CONFIGURED"; readonly PROVIDER_NOT_SUPPORTED: "STT_PROVIDER_NOT_SUPPORTED"; readonly STREAM_ERROR: "STT_STREAM_ERROR"; readonly STREAMING_NOT_SUPPORTED: "STT_STREAMING_NOT_SUPPORTED"; }; /** * Default STT options */ export declare const DEFAULT_STT_OPTIONS: Required<Pick<STTOptions, "language" | "punctuation" | "profanityFilter" | "sampleRate">>; /** * Type guard for STTResult */ export declare function isSTTResult(value: unknown): value is STTResult; /** * Type guard for valid STTOptions */ export declare function isValidSTTOptions(options: unknown): options is STTOptions; /** * Type guard for TranscriptionSegment */ export declare function isTranscriptionSegment(value: unknown): value is TranscriptionSegment; export type AzureRecognitionMode = "interactive" | "conversation" | "dictation"; export type AzureOutputFormat = "simple" | "detailed"; export type AzureSTTOptions = STTOptions & { recognitionMode?: AzureRecognitionMode; outputFormat?: AzureOutputFormat; interimResults?: boolean; endpointId?: string; /** Custom endpoint ID (alias for endpointId) */ customEndpointId?: string; connectionTimeout?: number; silenceTimeout?: number; profanityOption?: "masked" | "removed" | "raw"; /** Profanity mode (alias for profanityOption) */ profanityMode?: "masked" | "removed" | "raw"; initialSilenceTimeout?: number; enableLogging?: boolean; phraseList?: string[]; /** Whether to request detailed output format */ detailed?: boolean; wordLevelConfidence?: boolean; initialSilenceTimeoutMs?: number; endSilenceTimeoutMs?: number; }; export type DeepgramModel = "nova-2" | "nova-2-general" | "nova-2-meeting" | "nova-2-phonecall" | "nova-2-voicemail" | "nova-2-finance" | "nova-2-medical" | "nova" | "enhanced" | "base"; export type DeepgramSTTOptions = STTOptions & { model?: DeepgramModel | "nova-3"; smartFormat?: boolean; search?: string[]; replace?: Array<{ find: string; replace: string; }>; utterances?: boolean; utterSplit?: number; /** Alias for utterSplit (legacy field name) */ uttSplit?: number; paragraphs?: boolean; keywords?: string[]; keywordBoost?: "legacy" | "medium" | "high"; fillerWords?: boolean; detectTopics?: boolean; detectEntities?: boolean; summarize?: boolean; redact?: ("pci" | "numbers" | "ssn")[]; }; export type GoogleSTTModel = "latest_short" | "latest_long" | "telephony" | "medical_conversation" | "medical_dictation" | "command_and_search" | "phone_call" | "video" | "default"; export type GoogleSTTAudioEncoding = "ENCODING_UNSPECIFIED" | "LINEAR16" | "FLAC" | "MULAW" | "AMR" | "AMR_WB" | "OGG_OPUS" | "SPEEX_WITH_HEADER_BYTE" | "MP3" | "WEBM_OPUS"; export type GoogleSTTOptions = STTOptions & { model?: GoogleSTTModel; encoding?: GoogleSTTAudioEncoding; sampleRateHertz?: number; audioChannelCount?: number; enableSeparateRecognitionPerChannel?: boolean; alternativeLanguageCodes?: string[]; maxAlternatives?: number; enableAutomaticPunctuation?: boolean; enableSpokenPunctuation?: boolean; enableSpokenEmojis?: boolean; speechContexts?: Array<{ phrases: string[]; boost?: number; }>; adaptation?: { phraseSets?: string[]; customClasses?: string[]; }; useEnhanced?: boolean; keywords?: string[]; }; /** * `whisper-1` is OpenAI's own model; any other string is passed through as-is, * which is what a self-hosted OpenAI-compatible transcription server (vLLM, * LiteLLM, a diarization sidecar) expects — e.g. `"qwen3-asr"`. */ export type WhisperModel = "whisper-1" | (string & NonNullable<unknown>); export type WhisperSTTOptions = STTOptions & { model?: WhisperModel; responseFormat?: "json" | "text" | "srt" | "verbose_json" | "vtt"; temperature?: number; prompt?: string; /** Translate audio to English instead of transcribing in original language */ translate?: boolean; /** * Request timeout in milliseconds. Default 30_000, which suits OpenAI's * hosted endpoint; a self-hosted server transcribing a whole meeting needs * minutes, so raise it (or set `OPENAI_STT_TIMEOUT_MS`). */ timeoutMs?: number; }; export type AzureWord = { Word: string; Offset: number; Duration: number; Confidence?: number; }; export type AzureNBest = { Confidence: number; Lexical: string; ITN: string; MaskedITN: string; Display: string; Words?: AzureWord[]; }; export type AzureRecognitionResult = { RecognitionStatus: "Success" | "NoMatch" | "InitialSilenceTimeout" | "BabbleTimeout" | "Error" | string; Offset?: number; Duration?: number; DisplayText?: string; NBest?: AzureNBest[]; }; export type AzureSpeakerRecognitionResult = AzureRecognitionResult & { SpeakerId?: string; }; export type DeepgramWord = { word: string; start: number; end: number; confidence: number; speaker?: number; punctuated_word?: string; }; export type DeepgramAlternative = { transcript: string; confidence: number; words: DeepgramWord[]; paragraphs?: { transcript: string; paragraphs: Array<{ sentences: Array<{ text: string; start: number; end: number; }>; }>; }; }; export type DeepgramChannel = { alternatives: DeepgramAlternative[]; /** Present when the request asked for `detect_language`. */ detected_language?: string; language_confidence?: number; }; export type DeepgramUtterance = { start: number; end: number; confidence: number; channel: number; transcript: string; words: DeepgramWord[]; speaker?: number; id?: string; }; export type DeepgramResult = { channels: DeepgramChannel[]; utterances?: DeepgramUtterance[]; }; export type DeepgramResponse = { metadata: { request_id: string; transaction_key?: string; sha256?: string; created: string; duration: number; channels: number; models: string[]; model_info?: Record<string, { name: string; version: string; }>; }; results: DeepgramResult; }; export type GoogleWordInfo = { startTime: string; endTime: string; word: string; confidence?: number; speakerTag?: number; }; export type GoogleSpeechRecognitionAlternative = { transcript: string; confidence: number; words?: GoogleWordInfo[]; }; export type GoogleSpeechRecognitionResult = { alternatives: GoogleSpeechRecognitionAlternative[]; channelTag?: number; languageCode?: string; resultEndTime?: string; }; export type GoogleLongRunningRecognizeResponse = { results: GoogleSpeechRecognitionResult[]; totalBilledTime?: string; }; export type GoogleRecognizeResponse = { results?: GoogleSpeechRecognitionResult[]; totalBilledTime?: string; }; export type GoogleOperationResponse = { name: string; done: boolean; metadata?: { progressPercent?: number; startTime?: string; lastUpdateTime?: string; }; response?: GoogleLongRunningRecognizeResponse; error?: { code: number; message: string; }; }; export type GoogleRecognitionConfig = { encoding: string; sampleRateHertz?: number; languageCode: string; enableAutomaticPunctuation?: boolean; enableWordTimeOffsets?: boolean; enableWordConfidence?: boolean; model?: string; useEnhanced?: boolean; maxAlternatives?: number; profanityFilter?: boolean; enableSpeakerDiarization?: boolean; diarizationSpeakerCount?: number; }; export type GoogleRecognitionAudio = { content: string; }; export type WhisperTranscriptionWord = { word: string; start: number; end: number; }; export type WhisperTranscriptionSegment = { id: number; /** OpenAI always sends these decoder stats; OpenAI-compatible servers may omit them. */ seek?: number; start: number; end: number; text: string; tokens?: number[]; temperature?: number; avg_logprob?: number; compression_ratio?: number; no_speech_prob?: number; /** * Not part of OpenAI's schema. Diarizing OpenAI-compatible servers (the * NeuroLink diarization sidecar, Deepgram's compat endpoint, …) label each * segment with a speaker; the handler surfaces it as `TranscriptionSegment.speaker`. */ speaker?: string; }; export type WhisperVerboseResponse = { task: string; language: string; duration: number; text: string; segments?: WhisperTranscriptionSegment[]; words?: WhisperTranscriptionWord[]; /** Extension: distinct speaker labels, in order of first appearance. */ speakers?: string[]; /** * Extension: a self-hosted server that scores language identification may * say whether it trusts its own detection. Read defensively; never required. */ language_detected?: boolean; /** Extension: candidate languages and the server's score for each. */ language_scores?: Array<{ language: string; score: number; }>; /** Extension: the server's confidence in `language`. */ language_confidence?: number; }; export type WhisperSimpleResponse = { text: string; }; export type ElevenLabsVoice = { voice_id: string; name: string; category: string; labels?: { accent?: string; description?: string; age?: string; gender?: string; use_case?: string; }; preview_url?: string; }; export type ElevenLabsVoicesResponse = { voices: ElevenLabsVoice[]; }; /** * Scribe model ids the batch `speech-to-text` endpoint accepts. `scribe_v2` * is the current model and the default; `scribe_v2_medical` is its clinical * fine-tune; `scribe_v1` (and the experimental variant) are deprecated. The * `(string & {})` member keeps the union open so a newly released id can be * used without a cast. */ export type ElevenLabsSTTModel = "scribe_v2" | "scribe_v2_medical" | "scribe_v1" | "scribe_v1_experimental" | (string & {}); export type ElevenLabsSTTOptions = STTOptions & { /** Scribe model id. Default `scribe_v2`. */ model?: ElevenLabsSTTModel; /** * Annotate non-speech audio events ("(laughter)", "(music)") in the * transcript. ElevenLabs defaults this to true; NeuroLink sends `false` so * the transcript reads as plain speech unless asked otherwise. * @default false */ tagAudioEvents?: boolean; /** Per-request timeout in milliseconds. Default 60_000. */ timeoutMs?: number; /** * API base URL including the `/v1` prefix. Default * `ELEVENLABS_BASE_URL` or `https://api.elevenlabs.io/v1`. */ baseUrl?: string; }; export type ElevenLabsSTTWord = { text: string; start?: number; end?: number; type?: "word" | "spacing" | "audio_event" | string; speaker_id?: string; logprob?: number; }; export type ElevenLabsSTTResponse = { language_code?: string; language_probability?: number; text: string; words?: ElevenLabsSTTWord[]; }; export type AzureVoiceInfo = { Name: string; DisplayName: string; LocalName: string; ShortName: string; Gender: string; Locale: string; LocaleName: string; VoiceType: string; Status: string; WordsPerMinute?: string; }; export type GoogleAudioConfig = { audioEncoding: string; speakingRate?: number; pitch?: number; volumeGainDb?: number; sampleRateHertz?: number; effectsProfileId?: string[]; }; export type GoogleVoiceSelectionParams = { languageCode: string; name?: string; ssmlGender?: string; }; export type GoogleSynthesisInput = { text?: string; ssml?: string; }; export type GoogleSynthesizeRequest = { input: GoogleSynthesisInput; voice: GoogleVoiceSelectionParams; audioConfig: GoogleAudioConfig; }; export type GoogleVoiceInfo = { languageCodes: string[]; name: string; ssmlGender: string; naturalSampleRateHertz: number; }; export type GoogleListVoicesResponse = { voices: GoogleVoiceInfo[]; }; export type GoogleSynthesizeResponse = { audioContent: string; }; export type OpenAIRealtimeEvent = { type: string; event_id?: string; [key: string]: unknown; }; export type OpenAISessionCreated = OpenAIRealtimeEvent & { type: "session.created"; session: { id: string; object: string; model: string; modalities: string[]; voice: string; input_audio_format: string; output_audio_format: string; turn_detection: { type: string; threshold?: number; prefix_padding_ms?: number; silence_duration_ms?: number; }; tools: unknown[]; tool_choice: string; temperature: number; max_response_output_tokens: string | number; }; }; export type OpenAIAudioDelta = OpenAIRealtimeEvent & { type: "response.audio.delta"; response_id: string; item_id: string; output_index: number; content_index: number; delta: string; }; export type OpenAITranscriptDelta = OpenAIRealtimeEvent & { type: "response.audio_transcript.delta" | "conversation.item.input_audio_transcription.completed"; delta?: string; transcript?: string; }; export type GeminiMessage = { setup?: { model: string; generationConfig?: { responseModalities?: string[]; speechConfig?: { voiceConfig?: { prebuiltVoiceConfig?: { voiceName?: string; }; }; }; }; systemInstruction?: { parts: Array<{ text: string; }>; }; tools?: unknown[]; }; realtimeInput?: { mediaChunks: Array<{ mimeType: string; data: string; }>; }; clientContent?: { turns: Array<{ role: string; parts: Array<{ text: string; }>; }>; turnComplete: boolean; }; }; export type GeminiResponse = { setupComplete?: Record<string, unknown>; serverContent?: { modelTurn?: { parts: Array<{ text?: string; inlineData?: { mimeType: string; data: string; }; }>; }; turnComplete?: boolean; interrupted?: boolean; }; toolCall?: { functionCalls: Array<{ id: string; name: string; args: Record<string, unknown>; }>; }; toolCallCancellation?: { ids: string[]; }; }; /** The three files the built-in Whistle engine needs on disk. */ export type WhistleAssetName = "needle.js" | "needle.wasm" | "whistle.cact"; /** One pinned Whistle asset: where it comes from and what it must hash to. */ export type WhistleAsset = { name: WhistleAssetName; url: string; sha256: string; /** Exact size in bytes, used for the download notice and a cheap pre-check. */ bytes: number; }; /** Absolute paths of the three Whistle files once they are present and verified. */ export type WhistleAssetPaths = Record<WhistleAssetName, string>; /** One word as the Whistle engine reports it (seconds from the start of the audio). */ export type WhistleWord = { word: string; start: number; end: number; probability: number; }; /** A finished batch transcription from the Whistle worker, all chunks joined. */ export type WhistleEngineOutput = { text: string; language: string | null; words: WhistleWord[]; /** Time to first token of the first chunk. */ ttftMs: number | null; /** Mean decode speed over the chunks, tokens per second. */ decodeTps: number | null; /** How many ≤30 s pieces the audio was cut into. */ chunks: number; }; /** One step of the engine's native stream: what this chunk committed, and the unconfirmed tail. */ export type WhistleStreamStep = { text: string; words: WhistleWord[]; pending: string; language: string | null; /** Seconds of audio received so far. */ received: number; passMs: number; }; /** Messages the handler sends to the Whistle worker thread. */ export type WhistleWorkerRequest = { id: number; op: "transcribe"; pcm: Float32Array; language: string | null; keywords: string | null; timestamps: boolean; } | { id: number; op: "streamProcess"; pcm: Float32Array; language: string | null; keywords: string | null; } | { id: number; op: "streamStop"; }; /** Messages the Whistle worker thread sends back. */ export type WhistleWorkerResponse = { kind: "ready"; loadMs: number; } | { kind: "bootError"; error: string; } | { kind: "transcribed"; id: number; result: WhistleEngineOutput; } | { kind: "streamStep"; id: number; result: WhistleStreamStep; } | { kind: "failed"; id: number; error: string; }; /** `workerData` for the Whistle worker thread. */ export type WhistleWorkerData = { paths: WhistleAssetPaths; }; /** The subset of the Emscripten "needle" module the Whistle worker calls. */ export type WhistleNeedleModule = { HEAPU8: Uint8Array; _malloc(size: number): number; _free(ptr: number): void; _needle_load(ptr: number, length: bigint): number; _needle_last_error(): number; _needle_transcribe(pcm: number, samples: number, language: number, keywords: number, timestamps: number, out: number, outCapacity: number): number; _needle_stream_transcribe_process(pcm: number, samples: number, language: number, keywords: number, out: number, outCapacity: number): number; _needle_stream_transcribe_stop(out: number, outCapacity: number): number; UTF8ToString(ptr: number): string; }; /** `createNeedle` — the factory `needle.js` exports. */ export type WhistleNeedleFactory = (moduleArg: { wasmBinary?: Uint8Array; print?: (text: string) => void; printErr?: (text: string) => void; }) => Promise<WhistleNeedleModule>; /** A worker request before the engine stamps its id (the `Omit` distributes over the union). */ export type WhistleWorkerOutgoing<T = WhistleWorkerRequest> = T extends unknown ? Omit<T, "id"> : never;