UNPKG

@siteed/expo-audio-studio

Version:

Comprehensive audio processing library for React Native and Expo with recording, analysis, visualization, and streaming capabilities across iOS, Android, and web

906 lines (804 loc) 33.2 kB
// src/ExpoAudioStreamModule.web.ts import { LegacyEventEmitter } from 'expo-modules-core' import { AudioAnalysis } from './AudioAnalysis/AudioAnalysis.types' import { AudioRecording, AudioStreamStatus, BitDepth, ConsoleLike, RecordingConfig, RecordingInterruptionReason, StartRecordingResult, } from './ExpoAudioStream.types' import { WebRecorder } from './WebRecorder.web' import { AudioEventPayload } from './events' import { encodingToBitDepth } from './utils/encodingToBitDepth' export interface AudioStreamEvent { type: string device?: string timestamp: Date } export interface ExpoAudioStreamOptions { logger?: ConsoleLike eventCallback?: (event: AudioStreamEvent) => void } export interface EmitAudioEventProps { data: Float32Array position: number compression?: { data: Blob size: number totalSize: number mimeType: string format: string bitrate: number } } export type EmitAudioEventFunction = (_: EmitAudioEventProps) => void export type EmitAudioAnalysisFunction = (_: AudioAnalysis) => void export interface ExpoAudioStreamWebProps { logger?: ConsoleLike audioWorkletUrl: string featuresExtratorUrl: string maxBufferSize?: number // Maximum number of chunks to keep in memory } export class ExpoAudioStreamWeb extends LegacyEventEmitter { customRecorder: WebRecorder | null audioChunks: Float32Array[] isRecording: boolean isPaused: boolean recordingStartTime: number pausedTime: number currentDurationMs: number currentSize: number currentInterval: number currentIntervalAnalysis: number lastEmittedSize: number lastEmittedTime: number lastEmittedCompressionSize: number lastEmittedAnalysisTime: number streamUuid: string | null extension: 'webm' | 'wav' = 'wav' // Default extension is 'wav' recordingConfig?: RecordingConfig bitDepth: BitDepth // Bit depth of the audio audioWorkletUrl: string featuresExtratorUrl: string logger?: ConsoleLike latestPosition: number = 0 totalCompressedSize: number = 0 private readonly maxBufferSize: number private eventCallback?: (event: AudioStreamEvent) => void constructor({ audioWorkletUrl, featuresExtratorUrl, logger, maxBufferSize = 100, // Default to storing last 100 chunks (1 chunk = 0.5 seconds) }: ExpoAudioStreamWebProps) { const mockNativeModule = { addListener: () => {}, removeListeners: () => {}, } super(mockNativeModule) // Pass the mock native module to the parent class this.logger = logger this.customRecorder = null this.audioChunks = [] this.isRecording = false this.isPaused = false this.recordingStartTime = 0 this.pausedTime = 0 this.currentDurationMs = 0 this.currentSize = 0 this.bitDepth = 32 // Default this.currentInterval = 1000 // Default interval in ms this.currentIntervalAnalysis = 500 // Default analysis interval in ms this.lastEmittedSize = 0 this.lastEmittedTime = 0 this.latestPosition = 0 this.lastEmittedCompressionSize = 0 this.lastEmittedAnalysisTime = 0 this.streamUuid = null // Initialize UUID on first recording start this.audioWorkletUrl = audioWorkletUrl this.featuresExtratorUrl = featuresExtratorUrl this.maxBufferSize = maxBufferSize } // Utility to handle user media stream async getMediaStream() { try { this.logger?.debug('Requesting user media (microphone)...') // First check if the browser supports the necessary audio APIs if (!navigator?.mediaDevices?.getUserMedia) { this.logger?.error( 'Browser does not support mediaDevices.getUserMedia' ) throw new Error('Browser does not support audio recording') } // Get media with detailed audio constraints for better diagnostics const constraints = { audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true, // Add deviceId constraint if specified ...(this.recordingConfig?.deviceId ? { deviceId: { exact: this.recordingConfig.deviceId, }, } : {}), }, } this.logger?.debug('Media constraints:', constraints) const stream = await navigator.mediaDevices.getUserMedia(constraints) // Get detailed info about the audio track for debugging const audioTracks = stream.getAudioTracks() if (audioTracks.length > 0) { const track = audioTracks[0] const settings = track.getSettings() this.logger?.debug('Audio track obtained:', { label: track.label, id: track.id, enabled: track.enabled, muted: track.muted, readyState: track.readyState, settings, }) } else { this.logger?.warn('Stream has no audio tracks!') } return stream } catch (error) { this.logger?.error('Failed to get media stream:', error) throw error } } // Prepare recording with options async prepareRecording( recordingConfig: RecordingConfig = {} ): Promise<boolean> { if (this.isRecording) { this.logger?.warn( 'Cannot prepare: Recording is already in progress' ) return false } try { // Check permissions and initialize basic settings await this.getMediaStream().then((stream) => { // Just verify we can access the microphone by getting a stream, then release it stream.getTracks().forEach((track) => track.stop()) }) this.bitDepth = encodingToBitDepth({ encoding: recordingConfig.encoding ?? 'pcm_32bit', }) // Store recording configuration for later use this.recordingConfig = recordingConfig // Use custom filename if provided, otherwise fallback to timestamp if (recordingConfig.filename) { // Remove any existing extension from the filename this.streamUuid = recordingConfig.filename.replace( /\.[^/.]+$/, '' ) } else { this.streamUuid = Date.now().toString() } this.logger?.debug('Recording preparation completed successfully') return true } catch (error) { this.logger?.error('Error preparing recording:', error) return false } } // Start recording with options async startRecording( recordingConfig: RecordingConfig = {} ): Promise<StartRecordingResult> { if (this.isRecording) { throw new Error('Recording is already in progress') } // If we haven't prepared or have different settings, prepare now if ( !this.recordingConfig || this.recordingConfig.sampleRate !== recordingConfig.sampleRate || this.recordingConfig.channels !== recordingConfig.channels || this.recordingConfig.encoding !== recordingConfig.encoding ) { await this.prepareRecording(recordingConfig) } else { this.logger?.debug( 'Using previously prepared recording configuration' ) } // Save recording config for reference this.recordingConfig = recordingConfig const audioContext = new (window.AudioContext || // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - Allow webkitAudioContext for Safari window.webkitAudioContext)() const stream = await this.getMediaStream() const source = audioContext.createMediaStreamSource(stream) this.customRecorder = new WebRecorder({ logger: this.logger, audioContext, source, recordingConfig, emitAudioEventCallback: this.customRecorderEventCallback.bind(this), emitAudioAnalysisCallback: this.customRecorderAnalysisCallback.bind(this), onInterruption: this.handleRecordingInterruption.bind(this), }) await this.customRecorder.init() this.customRecorder.start() this.isRecording = true this.recordingStartTime = Date.now() this.pausedTime = 0 this.isPaused = false this.lastEmittedSize = 0 this.lastEmittedTime = 0 this.lastEmittedCompressionSize = 0 this.currentInterval = recordingConfig.interval ?? 1000 this.currentIntervalAnalysis = recordingConfig.intervalAnalysis ?? 500 this.lastEmittedAnalysisTime = Date.now() // Use custom filename if provided, otherwise fallback to timestamp if (recordingConfig.filename) { // Remove any existing extension from the filename this.streamUuid = recordingConfig.filename.replace(/\.[^/.]+$/, '') } else { this.streamUuid = Date.now().toString() } const fileUri = `${this.streamUuid}.${this.extension}` const streamConfig: StartRecordingResult = { fileUri, mimeType: `audio/${this.extension}`, bitDepth: this.bitDepth, channels: recordingConfig.channels ?? 1, sampleRate: recordingConfig.sampleRate ?? 44100, compression: recordingConfig.output?.compressed?.enabled ? { ...recordingConfig.output.compressed, bitrate: recordingConfig.output.compressed.bitrate ?? 128000, size: 0, mimeType: 'audio/webm', format: recordingConfig.output.compressed.format ?? 'opus', compressedFileUri: '', } : undefined, } return streamConfig } /** * Centralized handler for recording interruptions */ private handleRecordingInterruption(event: { reason: RecordingInterruptionReason | string isPaused: boolean timestamp: number message?: string }): void { this.logger?.debug(`Received recording interruption: ${event.reason}`) // Update local state if the interruption should pause recording if (event.isPaused) { this.isPaused = true // If this is a device disconnection, handle according to behavior setting if (event.reason === 'deviceDisconnected') { this.pausedTime = Date.now() // Check if we should try fallback to another device if ( this.recordingConfig?.deviceDisconnectionBehavior === 'fallback' ) { this.logger?.debug( 'Device disconnected with fallback behavior - attempting to switch to default device' ) // Try to restart with default device this.handleDeviceFallback().catch((error) => { // If fallback fails, emit warning this.logger?.error('Device fallback failed:', error) this.emit('onRecordingInterrupted', { reason: 'deviceSwitchFailed', isPaused: true, timestamp: Date.now(), message: 'Failed to switch to fallback device. Recording paused.', }) }) } else { // Just warn about disconnection if fallback not enabled this.logger?.warn( 'Device disconnected - recording paused automatically' ) this.emit('onRecordingInterrupted', event) } } else { // For other interruption types, just emit the event this.emit('onRecordingInterrupted', event) } } else { // If not causing a pause, just forward the event this.emit('onRecordingInterrupted', event) } } /** * Handler for audio events from the WebRecorder */ private customRecorderEventCallback({ data, position, compression, }: EmitAudioEventProps): void { // Keep only the latest chunks based on maxBufferSize this.audioChunks.push(new Float32Array(data)) if (this.audioChunks.length > this.maxBufferSize) { this.audioChunks.shift() // Remove oldest chunk } this.currentSize += data.byteLength this.emitAudioEvent({ data, position, compression }) this.lastEmittedTime = Date.now() this.lastEmittedSize = this.currentSize this.lastEmittedCompressionSize = compression?.size ?? 0 } /** * Handler for audio analysis events from the WebRecorder */ private customRecorderAnalysisCallback( audioAnalysisData: AudioAnalysis ): void { this.emit('AudioAnalysis', audioAnalysisData) } // Get recording duration private getRecordingDuration(): number { if (!this.isRecording) { return 0 } return this.currentDurationMs } emitAudioEvent({ data, position, compression }: EmitAudioEventProps) { const fileUri = `${this.streamUuid}.${this.extension}` if (compression?.size) { this.lastEmittedCompressionSize = compression.size this.totalCompressedSize = compression.totalSize } // Update latest position for tracking this.latestPosition = position // Calculate duration of this chunk in ms const sampleRate = this.recordingConfig?.sampleRate || 44100 const chunkDurationMs = (data.length / sampleRate) * 1000 // Handle duration calculation if (this.customRecorder?.isFirstChunkAfterSwitch) { this.logger?.debug( `Processing first chunk after device switch, duration preserved at ${this.currentDurationMs}ms` ) this.customRecorder.isFirstChunkAfterSwitch = false } else { this.currentDurationMs += chunkDurationMs } const audioEventPayload: AudioEventPayload = { fileUri, mimeType: `audio/${this.extension}`, lastEmittedSize: this.lastEmittedSize, deltaSize: data.byteLength, position, totalSize: this.currentSize, buffer: data, streamUuid: this.streamUuid ?? '', compression: compression ? { data: compression?.data, totalSize: this.totalCompressedSize, eventDataSize: compression?.size ?? 0, position, } : undefined, } this.emit('AudioData', audioEventPayload) } // Stop recording async stopRecording(): Promise<AudioRecording> { if (!this.customRecorder) { throw new Error('Recorder is not initialized') } this.logger?.debug('Starting stop process') try { const { compressedBlob, uncompressedBlob } = await this.customRecorder.stop() this.isRecording = false this.isPaused = false let compression: AudioRecording['compression'] let fileUri = `${this.streamUuid}.${this.extension}` let mimeType = `audio/${this.extension}` // Handle both compressed and uncompressed blobs according to new output configuration const primaryEnabled = this.recordingConfig?.output?.primary?.enabled ?? true const compressedEnabled = this.recordingConfig?.output?.compressed?.enabled ?? false // Process compressed blob if available and enabled if (compressedBlob && compressedEnabled) { const compressedUri = URL.createObjectURL(compressedBlob) const compressedInfo = { compressedFileUri: compressedUri, size: compressedBlob.size, mimeType: 'audio/webm', format: this.recordingConfig?.output?.compressed?.format ?? 'opus', bitrate: this.recordingConfig?.output?.compressed?.bitrate ?? 128000, } // Store compression info compression = compressedInfo // If primary is disabled, use compressed as main file if (!primaryEnabled) { this.logger?.debug( 'Using compressed audio as primary output (primary disabled)' ) fileUri = compressedUri mimeType = 'audio/webm' } } // Process uncompressed WAV if available and primary is enabled if (uncompressedBlob && primaryEnabled) { const wavUri = URL.createObjectURL(uncompressedBlob) fileUri = wavUri mimeType = 'audio/wav' } else if (!primaryEnabled && !compressedEnabled) { // No outputs enabled - streaming only mode this.logger?.debug('No outputs enabled - streaming only mode') fileUri = '' mimeType = 'audio/wav' } // Use the stored streamUuid for the final filename const filename = fileUri ? `${this.streamUuid}.${this.extension}` : 'stream-only' const result: AudioRecording = { fileUri, filename, bitDepth: this.bitDepth, createdAt: this.recordingStartTime, channels: this.recordingConfig?.channels ?? 1, sampleRate: this.recordingConfig?.sampleRate ?? 44100, durationMs: this.currentDurationMs, size: primaryEnabled ? this.currentSize : 0, mimeType, compression, } // Reset after creating the result this.streamUuid = null // Reset recording state variables to prepare for next recording this.currentDurationMs = 0 this.currentSize = 0 this.lastEmittedSize = 0 this.totalCompressedSize = 0 this.lastEmittedCompressionSize = 0 this.audioChunks = [] return result } catch (error) { this.logger?.error('Error stopping recording:', error) throw error } } // Pause recording async pauseRecording() { if (!this.isRecording) { throw new Error('Recording is not active') } if (this.isPaused) { this.logger?.debug('Recording already paused, skipping') return } try { if (this.customRecorder) { this.customRecorder.pause() } this.isPaused = true this.pausedTime = Date.now() } catch (error) { this.logger?.error('Error in pauseRecording', error) // Even if the pause operation failed, make sure our state is consistent this.isPaused = true this.pausedTime = Date.now() } } // Resume recording async resumeRecording() { if (!this.isPaused) { throw new Error('Recording is not paused') } this.logger?.debug('Resuming recording', { deviceDisconnectionBehavior: this.recordingConfig?.deviceDisconnectionBehavior, isDeviceDisconnected: this.customRecorder?.isDeviceDisconnected, }) try { // If we have no recorder, or if the device is disconnected, always attempt fallback if ( !this.customRecorder || this.customRecorder.isDeviceDisconnected ) { this.logger?.debug( 'No recorder exists or device disconnected - attempting fallback on resume' ) await this.handleDeviceFallback() // handleDeviceFallback will manage resuming if successful, or emit error if failed. return } // Normal resume path - device is still connected this.customRecorder.resume() this.isPaused = false // Adjust the recording start time to account for the pause duration const pauseDuration = Date.now() - this.pausedTime this.recordingStartTime += pauseDuration this.pausedTime = 0 this.emit('onRecordingInterrupted', { reason: 'userResumed', isPaused: false, timestamp: Date.now(), }) } catch (error) { this.logger?.error('Resume failed:', error) // Fallback to emitting a general failure if resume fails unexpectedly this.emit('onRecordingInterrupted', { reason: 'resumeFailed', // Use a more specific reason isPaused: true, // Remain paused if resume fails timestamp: Date.now(), message: 'Failed to resume recording. Please stop and start again.', }) } } // Get current status status() { const durationMs = this.getRecordingDuration() const status: AudioStreamStatus = { isRecording: this.isRecording, isPaused: this.isPaused, durationMs, size: this.currentSize, interval: this.currentInterval, intervalAnalysis: this.currentIntervalAnalysis, mimeType: `audio/${this.extension}`, compression: this.recordingConfig?.output?.compressed?.enabled ? { size: this.totalCompressedSize, mimeType: 'audio/webm', format: this.recordingConfig.output.compressed.format ?? 'opus', bitrate: this.recordingConfig.output.compressed.bitrate ?? 128000, compressedFileUri: `${this.streamUuid}.webm`, } : undefined, } return status } /** * Handles device fallback when the current device is disconnected */ private async handleDeviceFallback(): Promise<boolean> { this.logger?.debug('Starting device fallback procedure') if (!this.isRecording) { return false } try { // Save important state before switching const currentPosition = this.latestPosition const existingAudioChunks = [...this.audioChunks] // Save compressed chunks if available let compressedChunks: Blob[] = [] if (this.customRecorder) { try { compressedChunks = this.customRecorder.getCompressedChunks() } catch (err) { this.logger?.warn('Failed to get compressed chunks:', err) } } // Save the current counter value for continuity let currentDataPointCounter = 0 if (this.customRecorder) { currentDataPointCounter = this.customRecorder.getDataPointCounter() } // Clean up existing recorder if (this.customRecorder) { try { this.customRecorder.cleanup() } catch (cleanupError) { this.logger?.warn('Error during cleanup:', cleanupError) } } // Keep recording state true but mark as paused this.isPaused = true this.pausedTime = Date.now() // Store current size and other stats const previousTotalSize = this.currentSize const previousLastEmittedSize = this.lastEmittedSize const previousCompressedSize = this.totalCompressedSize // Try to get a fallback device const fallbackDeviceInfo = await this.getFallbackDevice() if (!fallbackDeviceInfo) { this.emit('onRecordingInterrupted', { reason: 'deviceSwitchFailed', isPaused: true, timestamp: Date.now(), message: 'Failed to switch to fallback device. Recording paused.', }) return false } // Start recording with the new device try { const stream = await this.requestPermissionsAndGetUserMedia( fallbackDeviceInfo.deviceId ) const audioContext = new (window.AudioContext || // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - Allow webkitAudioContext for Safari window.webkitAudioContext)() const source = audioContext.createMediaStreamSource(stream) // Create a new recorder with the fallback device this.customRecorder = new WebRecorder({ logger: this.logger, audioContext, source, recordingConfig: this.recordingConfig || {}, emitAudioEventCallback: this.customRecorderEventCallback.bind(this), emitAudioAnalysisCallback: this.customRecorderAnalysisCallback.bind(this), onInterruption: this.handleRecordingInterruption.bind(this), }) await this.customRecorder.init() // Set the initial position to continue from the previous device this.customRecorder.setPosition(currentPosition) // Reset the data point counter to continue from where the previous device left off if (currentDataPointCounter > 0) { this.customRecorder.resetDataPointCounter( currentDataPointCounter ) } // Prepare the recorder to handle the device switch properly this.customRecorder.prepareForDeviceSwitch() // Restore the existing audio chunks if (existingAudioChunks.length > 0) { this.audioChunks = existingAudioChunks } // Restore compressed chunks if available if (compressedChunks.length > 0) { this.customRecorder.setCompressedChunks(compressedChunks) } // Start the new recorder while preserving counters this.customRecorder.start(true) // Update recording state this.isPaused = false this.recordingStartTime = Date.now() // Restore size counters to maintain continuity this.currentSize = previousTotalSize this.lastEmittedSize = previousLastEmittedSize this.totalCompressedSize = previousCompressedSize // Notify that we switched to a fallback device if (this.eventCallback) { this.eventCallback({ type: 'deviceFallback', device: fallbackDeviceInfo.deviceId, timestamp: new Date(), }) } return true } catch (error) { this.logger?.error( 'Failed to start recording with fallback device', error ) this.isPaused = true this.emit('onRecordingInterrupted', { reason: 'deviceSwitchFailed', isPaused: true, timestamp: Date.now(), message: 'Failed to switch to fallback device. Recording paused.', }) return false } } catch (error) { this.logger?.error('Failed to use fallback device', error) this.isPaused = true this.emit('onRecordingInterrupted', { reason: 'deviceSwitchFailed', isPaused: true, timestamp: Date.now(), message: 'Failed to switch to fallback device. Recording paused.', }) return false } } /** * Attempts to get a fallback audio device */ private async getFallbackDevice(): Promise<MediaDeviceInfo | null> { try { // Get list of available audio input devices const devices = await navigator.mediaDevices.enumerateDevices() const audioInputDevices = devices.filter( (device) => device.kind === 'audioinput' ) if (audioInputDevices.length === 0) { return null } // Try to find a device that's not the current one if (this.customRecorder) { try { // Use mediaDevices.enumerateDevices to find the current active device const tracks = navigator.mediaDevices .getUserMedia({ audio: true }) .then((stream) => { const track = stream.getAudioTracks()[0] return track ? track.label : '' }) .catch(() => '') const currentTrackLabel = await tracks if (currentTrackLabel) { // Find a device with a different label const differentDevice = audioInputDevices.find( (device) => device.label && device.label !== currentTrackLabel ) if (differentDevice) { return differentDevice } } } catch (err) { this.logger?.warn( 'Error determining current device, using default', err ) } } // Return the first available device (default device) return audioInputDevices[0] } catch (error) { this.logger?.error('Error finding fallback device:', error) return null } } /** * Gets user media with specific device ID */ private async requestPermissionsAndGetUserMedia( deviceId: string ): Promise<MediaStream> { try { // Request the specific device return await navigator.mediaDevices.getUserMedia({ audio: { deviceId: { exact: deviceId }, }, }) } catch (error) { this.logger?.error( `Failed to get media for device ${deviceId}`, error ) // Try with default constraints as fallback return await navigator.mediaDevices.getUserMedia({ audio: true }) } } init(options?: ExpoAudioStreamOptions): Promise<void> { try { this.logger = options?.logger this.eventCallback = options?.eventCallback return Promise.resolve() } catch (error) { this.logger?.error('Error initializing ExpoAudioStream', error) return Promise.reject(error) } } }