@4players/odin
Version:
A cross-platform SDK enabling developers to integrate real-time VoIP chat technology into their projects
454 lines (453 loc) • 17.8 kB
JavaScript
import { __awaiter } from "tslib";
import { OdinEvent, } from './types';
import { isBlinkBrowser } from './utils';
import { workletScript } from './worker';
/**
* Class responsible for handling audio encoding/decoding operations and emitting events related
* to voice activity status.
*/
export class OdinAudioService {
/**
* Initializes and returns a singleton instance of the audio service. If the instance already exists, returns the existing instance.
*
* @param _worker The worker instance responsible for audio encoding/decoding
* @param _audioDataChannel The RTC data channel used for transmitting audio data
* @param _audioContexts The web audio context used for input/output audio processing
*/
constructor(_worker, _audioDataChannel, _audioContexts) {
this._worker = _worker;
this._audioDataChannel = _audioDataChannel;
this._audioContexts = _audioContexts;
/**
* Array to hold OdinMedia instances.
*/
this._medias = [];
/**
* Percent of outgoing media packets to artificially drop.
*/
this._artificialPacketLoss = 0;
/**
* Whether or not RNN VAD statistics are enabled.
*/
this._voiceProcessingStatsEnabled = false;
/**
* Default settings for audio processing.
*/
this._audioSettings = {
voiceActivityDetection: true,
voiceActivityDetectionAttackProbability: 0.9,
voiceActivityDetectionReleaseProbability: 0.8,
volumeGate: true,
volumeGateAttackLoudness: -30,
volumeGateReleaseLoudness: -40,
};
this._worker.onmessage = (event) => {
switch (event.data.type) {
case 'packet':
if (this._audioDataChannel.readyState === 'open') {
if (this._artificialPacketLoss > 0 && Math.random() * 100 < this._artificialPacketLoss) {
console.warn(`Dropping packet due to artificial packet loss percentage: ${this._artificialPacketLoss}%`);
}
else {
this._audioDataChannel.send(event.data.bytes);
}
}
break;
case 'stats':
this._room.eventTarget.dispatchEvent(new OdinEvent('AudioStats', { room: this._room, stats: event.data }));
break;
case 'vad_status':
this._room.eventTarget.dispatchEvent(new OdinEvent('VoiceProcessingStats', { room: this._room, stats: event.data }));
break;
case 'talk_status':
this.updateMediaActivity(event.data.media_id, event.data.is_talking);
break;
}
};
}
/**
* Retrieves the `OdinPeer` instance that matches a specified media ID.
*
* @param id The identifier of the peer's associated media
* @return The peer object matching the specified media ID or `undefined` if none found
*/
getPeerByMediaId(id) {
if (this._room.ownPeer.medias.get(id)) {
return this._room.ownPeer;
}
for (const [, peer] of this._room.remotePeers) {
if (peer.medias.get(id)) {
return peer;
}
}
return undefined;
}
/**
* Initializes and returns a singleton instance of the audio service.
*
* @param worker The worker instance handing all the encoding/decoding
* @param audioChannel The RTC data channel used to transfer audio data
* @param audioContexts The web audio contexts for processing capture/playback data
*/
static setInstance(worker, audioChannel, audioContexts) {
this._instance = new OdinAudioService(worker, audioChannel, audioContexts);
return this._instance;
}
/**
* Returns a singleton instance of the audio service.
*/
static getInstance() {
return this._instance;
}
/**
* Returns the underlying audio worker instance.
*/
get audioWorker() {
return this._worker;
}
/**
* Returns the underlying room instance.
*/
get room() {
return this._room;
}
/**
* Sets the room this audio service is dealing with.
*
* @param room A room instance
*/
set room(room) {
this._room = room;
}
/**
* Returns the artificial packet loss for outgoing media packets.
*/
get artificialPacketLossPercentage() {
return this._artificialPacketLoss;
}
/**
* Sets the artificial packet loss for outgoing media packets.
*
* @param room Packet loss in percent (0-100)
*/
set artificialPacketLossPercentage(value) {
if (value >= 0 && value <= 100) {
this._artificialPacketLoss = value;
}
}
/**
* Returns the sample rate of the input audio context.
*/
get inputSampleRate() {
return this._audioContexts.input.sampleRate;
}
/**
* Returns the sample rate of the output audio context.
*/
get outputSampleRate() {
return this._audioContexts.output.sampleRate;
}
/**
* Returns the `OdinMedia` instance matching the specified ID if known.
*
* @param id The media ID to search for
*/
getMedia(id) {
return this._medias.find((media) => media.id === id);
}
/**
* Returns true if the audio service knows a media with the specified ID.
*
* @param id The media ID to search for
*/
hasMedia(id) {
return !!this.getMedia(id);
}
/**
* Updates the activity status of a specific media. This method is responsible for setting the activity state of a media and
* triggering the necessary events when the media activity status changes.
*
* @param id The ID of the media whose activity status needs updating
* @param isActive The updated activity status of the media
*/
updateMediaActivity(id, isActive) {
const media = this._medias.find((media) => media.id === id);
const peer = this.getPeerByMediaId(id);
if (!media) {
return;
}
media.active = isActive;
if (peer) {
media.eventTarget.dispatchEvent(new OdinEvent('Activity', { room: this._room, peer, media }));
peer.eventTarget.dispatchEvent(new OdinEvent('MediaActivity', { room: this._room, peer, media }));
this._room.eventTarget.dispatchEvent(new OdinEvent('MediaActivity', { room: this._room, peer, media }));
}
}
/**
* Registers a media to handle its talk status.
*
* @param media The media that gets registered
*/
registerMedia(media) {
this._medias.push(media);
}
/**
* Unregister a media to stop the talk status handling.
*
* @param media The media that gets unregistered
*/
unregisterMedia(media) {
this._medias = this._medias.filter((mediaItem) => {
return mediaItem.id !== media.id;
});
}
/**
* Starts an internal encoder for the specified media.
*
* @param media The media to start an encoder for
*/
startEncoder(media) {
this._worker.postMessage({
type: 'start_encoder',
media_id: media.id,
properties: {
sample_rate: this.inputSampleRate,
fec: true,
voip: true,
},
});
if (this._voiceProcessingStatsEnabled) {
this._worker.postMessage({
type: 'start_vad_meter',
});
}
}
/**
* Stops an internal encoder for the specified media.
*
* @param media The media to stop an encoder for
*/
stopEncoder(media) {
this._worker.postMessage({
type: 'stop_encoder',
media_id: media.id,
});
if (this._voiceProcessingStatsEnabled) {
this._worker.postMessage({
type: 'stop_vad_meter',
});
}
}
/**
* Starts an internal decoder for the specified media.
*
* @param media The media to start a decoder for
*/
startDecoder(media) {
this._worker.postMessage({
type: 'start_decoder',
media_id: media.id,
properties: {
sample_rate: this.outputSampleRate,
},
});
}
/**
* Stops an internal decoder for the specified media.
*
* @param media The media to stop a decoder for
*/
stopDecoder(media) {
this._worker.postMessage({
type: 'stop_decoder',
media_id: media.id,
});
}
/**
* Set up the audio device, encoder and decoder.
*/
setupAudio() {
return __awaiter(this, void 0, void 0, function* () {
yield this._audioContexts.output.audioWorklet.addModule(workletScript);
this._decoderNode = new AudioWorkletNode(this._audioContexts.output, 'eyvor-decoder', {
numberOfInputs: 0,
numberOfOutputs: 1,
});
yield this._audioContexts.input.audioWorklet.addModule(workletScript);
this._encoderNode = new AudioWorkletNode(this._audioContexts.input, 'eyvor-encoder', {
numberOfInputs: 1,
numberOfOutputs: 0,
});
const encoderPipe = new MessageChannel();
const decoderPipe = new MessageChannel();
this._worker.postMessage({
type: 'initialize',
encoder: { worklet: encoderPipe.port1 },
decoder: { worklet: decoderPipe.port1 },
stats_interval: 1000,
}, [encoderPipe.port1, decoderPipe.port1]);
this._encoderNode.port.postMessage({ type: 'initialize', worker: encoderPipe.port2 }, [encoderPipe.port2]);
this._decoderNode.port.postMessage({ type: 'initialize', worker: decoderPipe.port2 }, [decoderPipe.port2]);
this._decoderNode.connect(this._audioContexts.output.destination);
});
}
/**
* Initiates the audio input recording. In case of using a Blink-based browser, it handles echo cancellation as well.
*
* @param mediaStream The media stream object that contains the audio track to be processed
* @param audioSettings Settings related to audio processing
*/
updateInputStream(mediaStream) {
var _a, _b, _c, _d, _e;
return __awaiter(this, void 0, void 0, function* () {
if (!this._encoderNode || !this._decoderNode) {
return;
}
const audioTrack = mediaStream.getAudioTracks()[0];
// Apply ugly workaround to apply echo cancellation in Chromium based browsers
if (isBlinkBrowser() && ((_a = audioTrack === null || audioTrack === void 0 ? void 0 : audioTrack.getConstraints()) === null || _a === void 0 ? void 0 : _a.echoCancellation)) {
const outputDestination = this._audioContexts.output.createMediaStreamDestination();
this._decoderNode.disconnect(this._audioContexts.output.destination);
this._decoderNode.connect(outputDestination);
const webrtc = yield this.webrtcLoopback(audioTrack, outputDestination.stream.getAudioTracks()[0]);
const inputStream = new MediaStream([webrtc.input]);
const outputStream = new MediaStream([webrtc.output]);
this.webrtcDummy(inputStream);
this.webrtcDummy(outputStream);
(_b = this._audioSource) === null || _b === void 0 ? void 0 : _b.disconnect();
this._audioSource = this._audioContexts.input.createMediaStreamSource(inputStream);
this._audioSource.connect(this._encoderNode);
(_c = this._audioElement) === null || _c === void 0 ? void 0 : _c.remove();
this._audioElement = new Audio();
this._audioElement.srcObject = outputStream;
this._audioElement.volume = 1;
yield this._audioElement.play();
}
else {
(_d = this._audioSource) === null || _d === void 0 ? void 0 : _d.disconnect();
this._audioSource = this._audioContexts.input.createMediaStreamSource(mediaStream);
(_e = this._audioSource) === null || _e === void 0 ? void 0 : _e.connect(this._encoderNode);
}
});
}
/**
* Updates settings for voice activity detection and volume gate. These settings control how voice activity is detected and how
* the volume gate is operated.
*
* @param settings The new configurations and thresholds for voice activity detection and volume gate
*/
setVoiceProcessingConfig(settings) {
this._audioSettings = settings;
this._worker.postMessage({
type: 'set_speech_detection_config',
enabled: settings.voiceActivityDetection,
going_active_threshold: settings.voiceActivityDetectionAttackProbability,
going_inactive_threshold: settings.voiceActivityDetectionReleaseProbability,
});
this._worker.postMessage({
type: 'set_volume_gate_config',
enabled: settings.volumeGate,
going_active_threshold: settings.volumeGateAttackLoudness,
going_inactive_threshold: settings.volumeGateReleaseLoudness,
});
}
/**
* Returns settings for voice activity detection and volume gate.
*/
getVoiceProcessingConfig() {
return this._audioSettings;
}
/**
* Enables/disables emitting of voice activity detection stats.
*/
setVoiceProcessingStatsEnabled(enabled) {
this._worker.postMessage({
type: enabled ? 'start_vad_meter' : 'stop_vad_meter',
});
this._voiceProcessingStatsEnabled = enabled;
}
/**
* Stops all audio encoding/decoding and closes the related connections. This includes disconnecting the encoder and decoder nodes,
* closing the audio data channel, and stopping any audio currently playing.
*/
stopAllAudio() {
return __awaiter(this, void 0, void 0, function* () {
if (this._encoderNode) {
this._encoderNode.disconnect();
}
if (this._decoderNode) {
this._decoderNode.disconnect();
}
if (this._audioSource) {
this._audioSource.disconnect();
this._audioSource = undefined;
}
if (this._audioContexts.input && this._audioContexts.input.state !== 'closed') {
yield this._audioContexts.input.close();
}
if (this._audioContexts.output &&
this._audioContexts.input !== this._audioContexts.output &&
this._audioContexts.output.state !== 'closed') {
yield this._audioContexts.output.close();
}
if (this._audioDataChannel) {
yield this._audioDataChannel.close();
}
if (this._audioElement) {
this._audioElement.pause();
this._audioElement = undefined;
}
});
}
/**
* Helper functions to allow echo cancellation on Chromium based browsers by piping all incoming audio through WebRTC. This is a
* workaround to a known issue in Chromium.
*
* See https://bugs.chromium.org/p/chromium/issues/detail?id=687574
*/
webrtcDummy(stream) {
let audio = new Audio();
audio.muted = true;
audio.srcObject = stream;
audio.addEventListener('canplaythrough', () => {
audio = null;
});
}
webrtcLoopback(input, output) {
return __awaiter(this, void 0, void 0, function* () {
// setup peer connections
const connection_one = new RTCPeerConnection();
const connection_two = new RTCPeerConnection();
connection_one.onicecandidate = (event) => __awaiter(this, void 0, void 0, function* () {
if (event.candidate) {
yield connection_two.addIceCandidate(event.candidate);
}
});
connection_two.onicecandidate = (event) => __awaiter(this, void 0, void 0, function* () {
if (event.candidate) {
yield connection_one.addIceCandidate(event.candidate);
}
});
// add tracks and catch loopbacks
connection_one.addTrack(input);
connection_two.addTrack(output);
const loopback_input = new Promise((resolve) => {
connection_two.ontrack = (event) => resolve(event.track);
});
const loopback_output = new Promise((resolve) => {
connection_one.ontrack = (event) => resolve(event.track);
});
// open connection
const offer = yield connection_one.createOffer();
yield connection_one.setLocalDescription(offer);
yield connection_two.setRemoteDescription(offer);
const answer = yield connection_two.createAnswer();
yield connection_two.setLocalDescription(answer);
yield connection_one.setRemoteDescription(answer);
return {
input: yield loopback_input,
output: yield loopback_output,
};
});
}
}