@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
369 lines (368 loc) • 17.3 kB
JavaScript
/**
* Speech-to-Text (STT) Processing Utility
*
* Central orchestrator for all STT operations across providers.
* Manages provider-specific STT handlers and audio transcription.
*
* @module utils/sttProcessor
*/
import { logger } from "./logger.js";
import { buildSpeakerTurns, listSpeakers } from "./speakerTurns.js";
import { getSTTProviderDescriptor, resolveCallSTTEndpoints, } from "../factories/sttDescriptors.js";
import { STT_ERROR_CODES } from "../types/index.js";
import { ErrorCategory, ErrorSeverity } from "../constants/enums.js";
import { STTError } from "../voice/errors.js";
import { HandlerRegistry } from "../core/handlerRegistry.js";
import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../observability/index.js";
/**
* STT processor class for orchestrating speech-to-text operations
*
* Follows the same pattern as TTSProcessor, CSVProcessor, ImageProcessor, and PDFProcessor.
* Provides a unified interface for STT transcription across multiple providers.
*
* @example
* ```typescript
* // Register a handler
* STTProcessor.registerHandler('whisper', whisperHandler);
*
* // Check if provider is supported
* if (STTProcessor.supports('whisper')) {
* // Provider is registered
* }
* ```
*/
export class STTProcessor {
/**
* Handler registry mapping provider names to STT handlers
*
* @private
*/
static registry = new HandlerRegistry("STTProcessor");
/**
* Default maximum audio duration for STT transcription (in seconds)
*
* Providers can override this value by specifying the `maxAudioDuration` property
* in their respective `STTHandler` implementation. If not specified, this default
* value will be used (5 minutes).
*
* @private
*/
static DEFAULT_MAX_AUDIO_DURATION = 300;
/**
* Register an STT handler for a specific provider
*
* Allows providers to register their STT implementation at runtime.
*
* @param providerName - Provider identifier (e.g., 'whisper', 'deepgram')
* @param handler - STT handler implementation
*
* @example
* ```typescript
* const whisperHandler: STTHandler = {
* transcribe: async (audio, options) => { ... },
* getSupportedFormats: () => ["mp3", "wav"],
* isConfigured: () => true
* };
*
* STTProcessor.registerHandler('whisper', whisperHandler);
* ```
*/
static registerHandler(providerName, handler) {
const normalizedName = providerName
? providerName.toLowerCase()
: providerName;
this.registry.register(providerName, handler);
logger.debug(`[STTProcessor] Registered STT handler for provider: ${normalizedName}`);
}
/**
* Get a registered STT handler by provider name.
*
* Exposed publicly so module-level auto-registration code can reuse an
* already-registered primary handler when backfilling its aliases.
*
* @param providerName - Provider identifier
* @returns Handler instance or undefined if not registered
*/
static getHandler(providerName) {
return this.registry.get(providerName);
}
/**
* The handler a call should use. A credentials slice for a shipped
* provider builds a fresh handler for this call; otherwise the registered
* one is used, and a shipped provider that was never registered (its key
* arrived after registration, or it is the local engine on a registry that
* was cleared) is built from the environment rather than reported as
* unsupported. Names that are neither registered nor shipped resolve to
* `undefined`.
*
* The handler classes are loaded lazily: `voice/index.ts` imports this
* module, so a static import back would be a cycle.
*/
static async resolveHandler(provider, credentials) {
const endpoint = resolveCallSTTEndpoints(credentials)[provider.trim().toLowerCase()];
if (endpoint) {
// A per-call named endpoint: a fresh handler, never the registry's.
const { createSTTHandler } = await import("../voice/index.js");
return createSTTHandler(provider, credentials);
}
const descriptor = getSTTProviderDescriptor(provider);
const slice = descriptor
? credentials?.[descriptor.credentialsKey]
: undefined;
if (descriptor && slice !== undefined) {
const { createSTTHandler } = await import("../voice/index.js");
return createSTTHandler(descriptor.name, credentials);
}
const registered = this.getHandler(provider);
if (registered || !descriptor) {
return registered;
}
const { createSTTHandler } = await import("../voice/index.js");
return createSTTHandler(descriptor.name, credentials);
}
/**
* List the names of all registered providers.
*/
static listProviders() {
return this.registry.list();
}
/**
* Removes every registered STT handler. Primarily for test isolation —
* production code should not need to call this.
*/
static clearHandlers() {
this.registry.clear();
}
/**
* Check if a provider is supported (has a registered STT handler)
*
* @param providerName - Provider identifier
* @returns True if handler is registered
*
* @example
* ```typescript
* if (STTProcessor.supports('whisper')) {
* console.log('Whisper STT is supported');
* }
* ```
*/
static supports(providerName) {
if (!providerName) {
logger.error("[STTProcessor] Provider name is required for supports check");
return false;
}
const isSupported = this.registry.supports(providerName);
if (!isSupported) {
logger.debug(`[STTProcessor] Provider ${providerName} is not supported`);
}
return isSupported;
}
/**
* Transcribe audio to text using a registered STT provider
*
* Orchestrates the speech-to-text transcription process:
* 1. Validates audio input (non-empty)
* 2. Looks up the provider handler
* 3. Verifies provider configuration
* 4. Delegates transcription to the provider
* 5. Enriches result with provider metadata
*
* @param audio - Audio data as Buffer or ArrayBuffer
* @param provider - Provider identifier
* @param options - STT configuration options
* @param credentials - Per-call credentials (`credentials.stt`). When it
* carries a slice for this provider — even an empty one — a fresh handler
* is built from that slice (environment as fallback) for this call alone,
* instead of the registry's shared instance: per-call beats instance
* beats environment, exactly as for text providers.
* @returns Transcription result with text and metadata
* @throws STTError if validation fails or provider not supported/configured
*
* @example
* ```typescript
* const result = await STTProcessor.transcribe(audioBuffer, "whisper", {
* language: "en-US",
* punctuation: true,
* });
*
* console.log(`Transcription: ${result.text}`);
* console.log(`Confidence: ${result.confidence}`);
* ```
*/
static async transcribe(audio, provider, options, credentials) {
// Create span early so preflight failures are captured
const span = SpanSerializer.createSpan(SpanType.STT, "stt.transcribe", {
"stt.operation": "transcribe",
"stt.provider": provider,
"stt.language": options.language,
"stt.format": options.format,
});
try {
// 1. Audio validation: reject empty + oversized audio
const byteLength = audio instanceof ArrayBuffer ? audio.byteLength : audio.length;
if (!byteLength || byteLength === 0) {
logger.error("[STTProcessor] Audio data is required for transcription");
throw new STTError({
code: STT_ERROR_CODES.AUDIO_EMPTY,
message: "Audio data is required for STT transcription",
severity: ErrorSeverity.LOW,
retriable: false,
context: { provider },
});
}
// NEW13: enforce a size upper bound so a multi-GB Buffer can't OOM the
// process. Default 25 MB matches Whisper's documented limit; callers
// can override via `options.maxAudioBytes`. Permanent errors at the
// provider level (e.g. Whisper rejecting >25MB) become this clean
// STTError instead of a memory crash or vendor 413.
const maxAudioBytes = options.maxAudioBytes ?? 25_000_000;
if (byteLength > maxAudioBytes) {
logger.error(`[STTProcessor] Audio buffer ${byteLength} bytes exceeds limit ${maxAudioBytes}`);
throw new STTError({
code: STT_ERROR_CODES.AUDIO_TOO_LONG,
message: `Audio buffer ${byteLength} bytes exceeds maximum ${maxAudioBytes} bytes for STT transcription. Increase maxAudioBytes in options or chunk the audio.`,
severity: ErrorSeverity.HIGH,
retriable: false,
context: { provider, byteLength, maxAudioBytes },
});
}
// 2. Handler lookup and error if provider not supported
const handler = await this.resolveHandler(provider, credentials);
if (!handler) {
logger.error(`[STTProcessor] Provider "${provider}" is not registered`);
throw new STTError({
code: STT_ERROR_CODES.PROVIDER_NOT_SUPPORTED,
message: `STT provider "${provider}" is not supported. Use STTProcessor.registerHandler() to register it.`,
severity: ErrorSeverity.HIGH,
retriable: false,
context: {
provider,
availableProviders: this.registry.list(),
},
});
}
// 3. Format compatibility check — fail fast when the caller passes
// an audio format the provider explicitly does not decode (e.g. MP3 to
// azure-stt). Without this, providers like Azure return a Success
// response with empty text, which then cascades into a confusing
// "prompt must be at least 1 character long" failure on the downstream
// LLM call. We only validate when both `options.format` and
// `handler.getSupportedFormats()` are present so we never block providers
// that prefer to do their own detection.
if (options.format && typeof handler.getSupportedFormats === "function") {
const supported = handler.getSupportedFormats();
if (Array.isArray(supported) &&
supported.length > 0 &&
!supported.includes(options.format)) {
logger.error(`[STTProcessor] Provider "${provider}" does not support audio format "${options.format}"`);
throw new STTError({
code: STT_ERROR_CODES.INVALID_AUDIO_FORMAT,
message: `STT provider "${provider}" does not support audio format "${options.format}". Supported formats: ${supported.join(", ")}.`,
severity: ErrorSeverity.HIGH,
retriable: false,
context: {
provider,
requestedFormat: options.format,
supportedFormats: supported,
},
});
}
}
// 4. Configuration check
if (!handler.isConfigured()) {
logger.warn(`[STTProcessor] Provider "${provider}" is not properly configured`);
const hint = handler.describeConfiguration?.();
throw new STTError({
code: STT_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
message: hint
? `STT provider "${provider}" is not configured. ${hint}`
: `STT provider "${provider}" is not configured. Please set the required API keys.`,
category: ErrorCategory.CONFIGURATION,
severity: ErrorSeverity.HIGH,
retriable: false,
context: { provider },
});
}
logger.debug(`[STTProcessor] Starting transcription with provider: ${provider}`);
// `diarization` is documented as an alias of `speakerDiarization`, but
// every handler reads only the latter; fold it in here so the alias
// works for all of them.
const handlerOptions = options.diarization && options.speakerDiarization === undefined
? { ...options, speakerDiarization: true }
: options;
// 5. Call handler.transcribe() - providers handle their own timeouts
const result = await handler.transcribe(audio, handlerOptions);
// 6. Post-processing: enrich result with provider metadata, and give
// every diarizing provider the same shape. Providers label speakers in
// different places (per word, per segment); when a handler returned
// speaker-labelled words but no speaker-labelled segments, derive the
// turns here so callers can read `segments[].speaker` regardless of
// provider.
const enrichedResult = {
...result,
metadata: {
...result.metadata,
provider,
latency: result.metadata?.latency ?? 0,
},
};
const hasSpeakerSegments = enrichedResult.segments?.some((segment) => segment.speaker !== undefined);
if (!hasSpeakerSegments && enrichedResult.words) {
const turns = buildSpeakerTurns(enrichedResult.words);
if (turns.length > 0) {
enrichedResult.segments = turns;
}
}
if (!enrichedResult.speakers?.length && enrichedResult.segments) {
const speakers = listSpeakers(enrichedResult.segments);
if (speakers.length > 0) {
enrichedResult.speakers = speakers;
}
}
// Don't log transcript content at INFO — voice transcriptions can carry
// PII / health / financial data, and INFO is typically persisted in
// production log aggregation (CloudWatch, Datadog, etc.). GDPR / CCPA
// concern. Length and provider are safe to record.
logger.debug(`[STTProcessor] Transcription completed for provider "${provider}" (${result.text.length} chars)`);
// 7. Record successful span
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
getMetricsAggregator().recordSpan(endedSpan);
// 8. Return STTResult with text, confidence, metadata
return enrichedResult;
}
catch (err) {
// Record error span
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR, err instanceof Error ? err.message : String(err));
getMetricsAggregator().recordSpan(endedSpan);
// Re-throw STTError as-is
if (err instanceof STTError) {
throw err;
}
// Wrap other errors in STTError
const errorMessage = err instanceof Error ? err.message : String(err || "Unknown error");
logger.error(`[STTProcessor] Transcription failed for provider "${provider}": ${errorMessage}`);
throw new STTError({
code: STT_ERROR_CODES.TRANSCRIPTION_FAILED,
message: `STT transcription failed for provider "${provider}": ${errorMessage}`,
category: ErrorCategory.EXECUTION,
severity: ErrorSeverity.HIGH,
retriable: true,
context: {
provider,
audioByteLength: audio instanceof ArrayBuffer ? audio.byteLength : audio.length,
// Sanitize: strip free-text user-supplied fields (e.g. WhisperSTTOptions.prompt)
// from the error context so error-monitoring pipelines (Sentry, Datadog APM)
// don't ingest user audio prompt text.
options: {
format: options.format,
language: options.language,
wordTimestamps: options.wordTimestamps,
maxAudioBytes: options.maxAudioBytes,
speakerDiarization: options.speakerDiarization,
},
},
originalError: err instanceof Error ? err : undefined,
});
}
}
}