infobip-rtc
Version:
Infobip RTC JavaScript SDK - Infobip WebRTC API Implementation
1,087 lines • 88.1 kB
JavaScript
import { CameraOrientation } from "../options/CameraOrientation";
import { Audio, Media, Participant, State, Video } from "../../util/Participant";
import { CallStatus } from "../CallStatus";
import MonitoredPeerConnection from "../../log/monitor/MonitoredPeerConnection";
import { MediaUpdateStatus } from "./MediaUpdateStatus";
import { ErrorLog } from "../../log/Log";
import HangupReasonFactory from "./util/HangupReasonFactory";
import HangupStatusFactory from "../util/HangupStatusFactory";
import { MediaType } from "../../log/util/MediaType";
import SimulcastEncodings from "../../util/SimulcastEncodings";
import VideoType from "../../util/VideoType";
import { ApplicationErrorCode } from "../ApplicationErrorCode";
import { WsEvent } from "../ws/WsEvent";
import { ApiEventEmitter } from "../../util/ApiEventEmitter";
import CallProperties from "./CallProperties";
import { EndpointType, PhoneEndpoint, SipEndpoint, ViberEndpoint, WebrtcEndpoint, WebsocketEndpoint, WhatsAppEndpoint } from "../../util/Endpoint";
import { WsAction } from "../ws/WsAction";
import { EmptyAudioStream } from "./EmptyAudioStream";
import { PeerConnectionTag } from "../../log/monitor/media/PeerConnectionTag";
import { DefaultVideoFilterManager } from "../options/filters/video/DefaultVideoFilterManager";
import { NetworkQualityMonitor } from "./network/NetworkQualityMonitor";
import { CallsApiEvent } from "../event/CallsApiEvents";
import VideoRemovalReason from "../../util/VideoRemovalReason";
import { DefaultLocalCapturer } from "./DefaultLocalCapturer";
import { DefaultServerCapturer } from "./DefaultServerCapturer";
import { DefaultDataChannel } from "./DefaultDataChannel";
import { AudioQualityMode } from "../options/AudioQualityMode";
import { RTCMediaDevice } from "../../RTCMediaDevice";
import { BitrateConfig, configureForSending } from "./util/BitrateUtil";
import { ManagedEventEmitter } from "../../util/ManagedEventEmitter";
import { DefaultAudioFilterManager } from "../options/filters/audio/DefaultAudioFilterManager";
import Browser from "../../util/Browser";
import { AdvisorRole, AuditorRole, DefaultRole, ListenerRole, MonitorRole, RoleType } from "../../util/Role";
import { AsyncLock } from "./util/AsyncLock";
import { DefaultAudioStreamEnergyMonitor } from "./AudioStreamEnergyMonitor";
const LOW_DATA_AUDIO_BITRATE = 10000;
var ApplicationCallState;
(function (ApplicationCallState) {
ApplicationCallState[ApplicationCallState["STANDALONE"] = 0] = "STANDALONE";
ApplicationCallState[ApplicationCallState["IN_DIALOG"] = 1] = "IN_DIALOG";
ApplicationCallState[ApplicationCallState["IN_CONFERENCE"] = 2] = "IN_CONFERENCE";
})(ApplicationCallState || (ApplicationCallState = {}));
export class DefaultApplicationCall {
constructor(eventEmitter, gateway, logger, rtcConfig, device, _callsConfigurationId, applicationCallOptions, currentUserIdentity, token, apiUrl, callId = null) {
this.gateway = gateway;
this.logger = logger;
this.rtcConfig = rtcConfig;
this.device = device;
this._callsConfigurationId = _callsConfigurationId;
this.applicationCallOptions = applicationCallOptions;
this.currentUserIdentity = currentUserIdentity;
this.token = token;
this.apiUrl = apiUrl;
this.hasRemoteDescription = false;
this.remoteCandidates = [];
this.isEarlyMedia = false;
this.joinedConference = false;
this.joinedDialog = false;
this.participants = {};
this.remoteVideos = {};
this.videoFilterManager = null;
this.audioFilterManager = null;
this.reconnecting = false;
this._recordingState = {};
this.cameraVideoEncodings = new Browser().isMobile() ? SimulcastEncodings.cameraEncodingsMobile : SimulcastEncodings.cameraEncodings;
this.updateCameraStreamLock = new AsyncLock();
this.eventEmitter = new ManagedEventEmitter(eventEmitter);
this.callStartTime = new Date();
this.callStatus = CallStatus.INITIALIZED;
this.mediaUpdateStatus = MediaUpdateStatus.IDLE;
this.apiEventEmitter = new ApiEventEmitter(this.logger);
this.callId = callId || crypto.randomUUID();
this.localScreenShare = { active: false };
this.networkQualityMonitor = new NetworkQualityMonitor();
let videoFilter = this.applicationCallOptions?.videoOptions?.videoFilter;
if (videoFilter) {
this.videoFilterManager = new DefaultVideoFilterManager(videoFilter, this.logger, this.callId);
}
let audioFilter = this.applicationCallOptions?.audioOptions?.audioFilter;
if (audioFilter) {
this.audioFilterManager = new DefaultAudioFilterManager(audioFilter);
}
this._audioQualityMode = this.applicationCallOptions?.audioOptions?.audioQualityMode || AudioQualityMode.AUTO;
if (this.applicationCallOptions?.dataChannel) {
this.createDataChannel();
}
if (this.applicationCallOptions?.audioOptions?.detectTalkingWhileMuted) {
this.createAudioStreamEnergyMonitor();
}
this.initEventHandlers();
this.handleDeviceChange();
}
on(name, handler) {
if (!Object.values(CallsApiEvent)
.find(apiEvent => apiEvent === name)) {
throw new Error(`Unknown event: ${name}!`);
}
this.apiEventEmitter.on(name, handler);
}
id() {
return this.callId;
}
options() {
return this.applicationCallOptions;
}
customData() {
return this.applicationCallOptions?.customData;
}
callsConfigurationId() {
return this._callsConfigurationId;
}
duration() {
if (!this.callEstablishTime) {
return 0;
}
return !this.callEndTime ? this.getDurationInSeconds(new Date()) : this.getDurationInSeconds(this.callEndTime);
}
endTime() {
return this.callEndTime;
}
establishTime() {
return this.callEstablishTime;
}
hangup() {
if (this.callStatus !== CallStatus.FINISHED) {
this.callStatus = CallStatus.FINISHED;
this.dataChannelCleanup();
this.scheduleHangup({ callId: this.callId, status: { id: '0', name: 'NO_ERROR', description: 'No Error.' } });
this.gateway.send({ action: WsAction.HANGUP, callId: this.callId, reason: 'Normal call clearing' });
}
}
startTime() {
return this.callStartTime;
}
status() {
return this.callStatus;
}
async mute(shouldMute) {
this.validateRoleMediaAllowed();
this.validateMediaActionAllowed();
return await this.muteInternal(shouldMute);
}
muted() {
return !this.localAudio.active;
}
async sendDTMF(dtmf) {
if (!dtmf || dtmf.toString().match(/[^0-9*#A-D]/)) {
throw ApplicationErrorCode.MEDIA_ERROR;
}
if (this.callStatus !== CallStatus.ESTABLISHED) {
throw {
id: '10103',
name: 'MEDIA_ERROR',
description: 'Call not established, action not allowed!'
};
}
this.validateRoleMediaAllowed();
this.validateMediaActionAllowed();
this.sendDTMFInfo(dtmf, CallProperties.DTMF_TONE_DURATION);
}
pauseIncomingVideo() {
this.validateMediaActionAllowed();
if (!this.joinedConference && !this.joinedDialog) {
return;
}
this.gateway.send({
action: WsAction.PAUSE_INCOMING_VIDEO
});
}
resumeIncomingVideo() {
this.validateMediaActionAllowed();
if (!this.joinedConference && !this.joinedDialog) {
return;
}
this.gateway.send({
action: WsAction.RESUME_INCOMING_VIDEO
});
}
async setAudioInputDevice(deviceId) {
this.device.setAudioInputDevice(deviceId);
if (this.callStatus !== CallStatus.ESTABLISHED) {
return;
}
try {
await this.audioFilterManager?.stop();
await this.replaceAudioStream();
}
catch (error) {
return this.throwMediaError(error);
}
}
audioFilter() {
return this.audioFilterManager?.getAudioFilter();
}
async setAudioFilter(audioFilter) {
if (this.audioFilterManager?.getAudioFilter() === audioFilter) {
return;
}
try {
await this.audioFilterManager?.stop();
this.audioFilterManager = audioFilter ? new DefaultAudioFilterManager(audioFilter) : null;
await this.replaceAudioStream();
}
catch (error) {
return this.throwMediaError(error);
}
}
clearAudioFilter() {
return this.setAudioFilter(null);
}
videoFilter() {
return this.videoFilterManager?.getVideoFilter();
}
async setVideoFilter(videoFilter) {
if (this.videoFilterManager && this.videoFilterManager.getVideoFilter() === videoFilter) {
return;
}
let previousFilter = this.videoFilterManager;
this.videoFilterManager = videoFilter ? new DefaultVideoFilterManager(videoFilter, this.logger, this.callId) : null;
await previousFilter?.stop();
if (this.hasCameraVideo()) {
await this.videoFilterManager?.start(this.localCameraVideoStream, 0, this.apiEventEmitter);
await this.updateTransceiver(this.localCameraVideo, this.localCameraVideoStream, this.cameraVideoEncodings);
this.apiEventEmitter.emit(CallsApiEvent.CAMERA_VIDEO_UPDATED, { stream: this.localCameraVideoStream });
}
}
clearVideoFilter() {
return this.setVideoFilter(null);
}
async setVideoInputDevice(deviceId) {
this.device.setVideoInputDevice(deviceId);
if (this.callStatus !== CallStatus.ESTABLISHED) {
return;
}
await this.updateCameraStream(this.cameraOrientation());
}
cameraOrientation() {
return this.device.getCameraOrientation();
}
setCameraOrientation(cameraOrientation) {
return this.updateCameraStream(cameraOrientation, false);
}
localCapturer() {
return this.defaultLocalCaputuer ??= new DefaultLocalCapturer(() => this.localCameraVideoStream, () => this.localScreenShareStream, () => this.videoSubscriberPC?.peerConnection, (identity, videoType) => this.getMidByIdentityAndVideoType(identity, videoType), this.currentUserIdentity);
}
serverCapturer() {
return this.defaultServerCapturer ??= new DefaultServerCapturer(() => this.localCameraVideoStream, () => this.localScreenShareStream, () => this.videoSubscriberPC?.peerConnection, (identity, videoType) => this.getMidByIdentityAndVideoType(identity, videoType), this.currentUserIdentity, this.token, this.apiUrl, this.logger);
}
dataChannel() {
return this._dataChannel;
}
setAudioQualityMode(audioQualityMode) {
this._audioQualityMode = audioQualityMode;
const senders = this.audioPC.peerConnection.getSenders();
senders.forEach((sender) => {
if (sender.track.kind !== 'audio') {
return;
}
const parameters = sender.getParameters();
parameters.encodings ??= [{}];
if (parameters.encodings[0]) {
parameters.encodings[0].priority = 'high';
parameters.encodings[0].networkPriority = 'high';
}
parameters.encodings.forEach(encoding => {
switch (audioQualityMode) {
case AudioQualityMode.LOW_DATA:
encoding.maxBitrate = LOW_DATA_AUDIO_BITRATE;
break;
case AudioQualityMode.AUTO:
case AudioQualityMode.HIGH_QUALITY:
encoding.maxBitrate = undefined;
break;
}
});
sender.setParameters(parameters)
.catch(err => this.logger.error(`Changing data consumption mode failed: ${err?.message}`));
});
}
audioQualityMode() {
return this._audioQualityMode;
}
hasCameraVideo() {
return this.localCameraVideo && this.localCameraVideo.active;
}
hasScreenShare() {
return this.localScreenShare && this.localScreenShare.active;
}
async stopVideo() {
this.validateMediaActionAllowed();
this.validateMediaUpdateStatus();
return this.stopVideoInternal(VideoRemovalReason.USER_REQUEST);
}
async cameraVideo(localVideo) {
this.validateRoleMediaAllowed();
this.validateMediaActionAllowed();
this.validateMediaUpdateStatus();
if (localVideo !== this.localCameraVideo.active) {
this.localCameraVideo.active = localVideo;
return localVideo ? this.addCameraVideo() : this.removeCameraVideo(VideoRemovalReason.USER_REQUEST);
}
}
async screenShare(screenShare) {
this.validateRoleMediaAllowed();
this.validateMediaActionAllowed();
this.validateMediaUpdateStatus();
if (screenShare !== this.localScreenShare.active) {
this.localScreenShare.active = screenShare;
return screenShare ? this.addScreenShareVideo() : this.removeScreenShareVideo(VideoRemovalReason.USER_REQUEST);
}
}
async startScreenShare(displayOptions) {
this.validateRoleMediaAllowed();
this.validateMediaActionAllowed();
this.validateMediaUpdateStatus();
this.localScreenShare.active = true;
return this.addScreenShareVideo(displayOptions);
}
async stopScreenShare() {
this.validateMediaActionAllowed();
this.validateMediaUpdateStatus();
this.localScreenShare.active = false;
return this.removeScreenShareVideo(VideoRemovalReason.USER_REQUEST);
}
recordingState() {
return this._recordingState;
}
createDataChannel() {
this._dataChannel = new DefaultDataChannel(this.gateway, this.logger, this.callId, this.currentUserIdentity, identity => this.participants[identity], ice => this.onIceCandidate(WsAction.ICE_CANDIDATE_DATA_CHANNEL, ice), () => this.joinedDialog || this.joinedConference, this.apiEventEmitter, this.conferenceId);
}
createAudioPeerConnection() {
this.audioPC = MonitoredPeerConnection.create(this.rtcConfig, this.callId, PeerConnectionTag.Audio, this.conferenceId || this.dialogId, MediaType.AUDIO, this.logger, this.gateway);
this.audioPC.peerConnection.onicecandidate = ice => this.onIceCandidate(WsAction.ICE_CANDIDATE, ice);
this.audioPC.peerConnection.ontrack = event => this.onAudioPcTrack(event);
this.audioPC.peerConnection.onconnectionstatechange = () => this.onAudioConnectionStateChanged();
this.audioPC.monitor.onNetworkQualityStatistics((networkQualityStatistics, currentMediaStats) => this.onNetworkQualityStatisticsChanged(networkQualityStatistics, currentMediaStats));
}
setLocalAudioStream(audioStream) {
this.localAudioStream = audioStream;
let track = this.localAudioStream.getAudioTracks()[0];
if (!track) {
this.emptyAudioStream = new EmptyAudioStream();
return;
}
track.enabled = this.localAudio.active;
if (!this.localAudio.active) {
this.logger.info("Starting audio stream energy monitor for a call that started muted!", this.callId);
this.audioStreamEnergyMonitor?.start(this.localAudioStream);
}
}
async setLocalVideoStream(videoStream) {
this.localCameraVideoStream = videoStream;
await this.videoFilterManager?.start(this.localCameraVideoStream, 0, this.apiEventEmitter);
this.mediaUpdateStatus = this.reconnecting ? MediaUpdateStatus.RECONNECTING : MediaUpdateStatus.ADDING_CAMERA_VIDEO;
}
createAudioTransceiver() {
let stream = this.emptyAudioStream ? this.emptyAudioStream.stream() : this.localAudioStream;
let track = stream.getAudioTracks()[0];
this.localAudio.transceiver = this.audioPC.peerConnection.addTransceiver(track, {
streams: [stream],
direction: 'sendrecv'
});
}
async setLocalDescription(pc, localDescription) {
await pc.setLocalDescription(localDescription);
return localDescription;
}
setRemoteCandidates() {
this.hasRemoteDescription = true;
if (this.remoteCandidates.length > 0) {
this.remoteCandidates.forEach(candidate => this.addIceCandidate(candidate));
this.remoteCandidates = [];
}
}
async addCameraVideo() {
this.mediaUpdateStatus = MediaUpdateStatus.ADDING_CAMERA_VIDEO;
try {
this.localCameraVideoStream = await this.getCameraVideoStream(this.cameraOrientation());
await this.videoFilterManager?.start(this.localCameraVideoStream, 0, this.apiEventEmitter);
}
catch (error) {
this.localCameraVideo.active = false;
this.handleGetUserMediaError(error, "camera");
return;
}
await this.publishVideo(this.joinedConference || this.joinedDialog);
}
async addScreenShareVideo(displayOptions) {
this.mediaUpdateStatus = MediaUpdateStatus.ADDING_SCREEN_SHARE;
let isConference = this.joinedConference || this.joinedDialog;
if (!this.videoPublisherPC) {
this.createVideoPublisherPC(isConference);
}
try {
this.localScreenShareStream = await this.getScreenShareVideoStream(displayOptions);
}
catch (error) {
this.handleGetUserMediaError(error, "screen-share");
return;
}
try {
await this.updateTransceiver(this.localScreenShare, this.localScreenShareStream, SimulcastEncodings.screenShareEncodings);
await this.negotiateVideoPublisher(isConference);
}
catch (error) {
this.handlePublishVideoFlowError(error);
}
}
handleCallFlowError(error, sendHangup = false) {
this.logger.log(new ErrorLog(this.callId, error));
if (sendHangup) {
let reason = HangupReasonFactory.getHangupReason(error);
this.gateway.send({ action: WsAction.HANGUP, callId: this.callId, reason: reason });
}
let hangupStatus = HangupStatusFactory.getApplicationHangupStatus(error);
this.eventEmitter.emit('call-error', { status: hangupStatus });
}
validateMediaUpdateStatus() {
if (this.mediaUpdateStatus !== MediaUpdateStatus.IDLE && this.mediaUpdateStatus !== MediaUpdateStatus.MEDIA_PENDING) {
throw new Error("User media already updating.");
}
}
async getLocalMediaStream(audio, video, cameraOrientation, cameraVideoFrameRate) {
if (!audio && !video) {
return new MediaStream();
}
return await this.device.getLocalStream(audio, video, cameraOrientation, true, true, cameraVideoFrameRate);
}
isFinished() {
return this.callStatus === CallStatus.FINISHED;
}
stopAllTracks(mediaStream) {
mediaStream.getTracks().forEach((track) => track.stop());
}
createAudioStreamEnergyMonitor() {
this.audioStreamEnergyMonitor = DefaultAudioStreamEnergyMonitor.create(() => this.emitTalkingWhileMutedEvent(), this.logger, this.callId);
}
audioStreamEnergyMonitorCleanup() {
if (this.audioStreamEnergyMonitor) {
this.audioStreamEnergyMonitor.destroy();
this.audioStreamEnergyMonitor = null;
}
}
dataChannelCleanup() {
this._dataChannel?.destroy();
}
audioCleanup() {
let totalMediaStats;
if (this.audioPC) {
totalMediaStats = this.audioPC.close();
this.audioPC = null;
this.hasRemoteDescription = false;
}
if (this.emptyAudioStream) {
this.emptyAudioStream.close();
}
if (this.localAudioStream) {
this.localAudioStream.getAudioTracks().forEach((track) => track.stop());
}
return totalMediaStats;
}
emitErrorEvent(errorCode) {
this.apiEventEmitter.emit(CallsApiEvent.ERROR, { errorCode });
}
getMidByIdentityAndVideoType(identity, type) {
return Object.values(this.remoteVideos).find(value => value.participant.endpoint.identifier === identity && value.type === type)?.mid;
}
validateMediaActionAllowed() {
if (this.reconnecting) {
throw {
id: '10103',
name: 'MEDIA_ERROR',
description: 'Call is reconnecting, action not allowed!'
};
}
}
validateRoleMediaAllowed() {
if (this.joinedConference && !this.isRoleMediaAllowed()) {
throw {
id: '10103',
name: 'MEDIA_ERROR',
description: 'Action not allowed for current role!'
};
}
}
async muteInternal(shouldMute) {
if (!this.localAudioStream) {
throw ApplicationErrorCode.MEDIA_ERROR;
}
if (this.emptyAudioStream && !shouldMute) {
this.emptyAudioStream.close();
delete this.emptyAudioStream;
try {
await this.audioFilterManager?.stop();
this.localAudioStream = await this.getAudioStream();
await this.audioFilterManager?.start(this.localAudioStream);
const track = this.localAudioStream.getAudioTracks()[0];
await this.replaceTrack(this.localAudio, track);
}
catch (error) {
return this.throwMediaError(error);
}
}
let audioTracks = this.localAudioStream.getAudioTracks();
if (audioTracks.length === 0) {
throw ApplicationErrorCode.MEDIA_ERROR;
}
this.localAudio.active = !shouldMute;
let audioTrack = audioTracks[0];
audioTrack.enabled = this.localAudio.active;
this.sendMute(shouldMute);
if (shouldMute) {
this.logger.info("Starting audio stream energy monitor because call was muted!", this.callId);
this.audioStreamEnergyMonitor?.start(this.localAudioStream);
}
else {
this.logger.info("Stopping audio stream energy monitor because call was unmuted!", this.callId);
this.audioStreamEnergyMonitor?.stop();
}
}
async stopVideoInternal(videoRemovalReason) {
if (this.hasScreenShare() && this.hasCameraVideo()) {
return await this.removeCameraAndScreenShare(videoRemovalReason);
}
if (this.hasScreenShare()) {
this.localScreenShare.active = false;
return await this.removeScreenShareVideo(videoRemovalReason);
}
if (this.hasCameraVideo()) {
this.localCameraVideo.active = false;
return await this.removeCameraVideo(videoRemovalReason);
}
}
async replaceAudioStream() {
if (this.emptyAudioStream) {
return;
}
this.localAudioStream?.getAudioTracks().forEach((track) => track.stop());
this.localAudioStream = await this.getAudioStream();
await this.audioFilterManager?.start(this.localAudioStream);
const track = this.localAudioStream.getAudioTracks()[0];
track.enabled = this.localAudio.active;
if (!track.enabled) {
this.logger.debug("Starting energy monitor after stream got replaced.", this.callId);
this.audioStreamEnergyMonitor?.start(this.localAudioStream);
}
await this.replaceTrack(this.localAudio, track);
}
async getAudioStream() {
const mediaStream = await this.getLocalMediaStream(true, false);
return new MediaStream([mediaStream.getAudioTracks()[0]]);
}
sendMute(shouldMute) {
this.gateway.send({
action: WsAction.MUTE,
muted: shouldMute,
callId: this.callId
});
}
async removeScreenShareVideoWithReason(reason) {
if (this.localScreenShare.active) {
this.validateMediaUpdateStatus();
this.localScreenShare.active = false;
return this.removeScreenShareVideo(reason);
}
}
initEventHandlers() {
this.eventEmitter.on(WsEvent.TRICKLE_ICE, event => this.handleTrickleIce(event));
this.eventEmitter.once(WsEvent.RINGING, event => this.ringingHandler(event));
this.eventEmitter.on(WsEvent.CALL_RESPONSE, event => this.responseHandler(event));
this.eventEmitter.on(WsEvent.CALL_ACCEPTED, event => this.acceptedHandler());
this.eventEmitter.once(WsEvent.HANGUP, event => this.hangupHandler(event));
this.eventEmitter.once(WsEvent.CALL_ERROR, event => this.errorHandler(event));
this.eventEmitter.on(WsEvent.JOINED_VIDEO_CALL, () => this.videoCallJoinedHandler());
this.eventEmitter.on(WsEvent.PUBLISHED_VIDEO_CALL, event => this.videoCallPublishedHandler(event));
this.eventEmitter.on(WsEvent.UNPUBLISHED_VIDEO_CALL, () => this.videoCallUnpublishedHandler());
this.eventEmitter.on(WsEvent.JOIN_VIDEO_CALL_ERROR, event => this.handleJoinVideoCallError(event));
this.eventEmitter.on(WsEvent.PUBLISH_VIDEO_CALL_ERROR, event => this.publishVideoCallErrorHandler(event));
this.eventEmitter.on(WsEvent.CALL_RECORDING_STARTED, event => this.handleCallRecordingStarted(event));
this.eventEmitter.on(WsEvent.CALL_RECORDING_STOPPED, event => this.handleCallRecordingStopped(event));
this.eventEmitter.on(WsEvent.JOINED_APPLICATION_CONFERENCE, event => this.handleJoinedApplicationConference(event));
this.eventEmitter.on(WsEvent.PARTICIPANT_JOINING, event => this.handleParticipantJoining(event));
this.eventEmitter.on(WsEvent.PARTICIPANT_JOINED, event => this.handleParticipantJoined(event));
this.eventEmitter.on(WsEvent.PARTICIPANT_MEDIA_CHANGED, event => this.handleParticipantMediaChanged(event));
this.eventEmitter.on(WsEvent.ROLE_CHANGED, event => this.handleRoleChanged(event));
this.eventEmitter.on(WsEvent.PARTICIPANT_ROLE_CHANGED, event => this.handleParticipantRoleChanged(event));
this.eventEmitter.on(WsEvent.STARTED_TALKING, event => this.handleStartedTalking());
this.eventEmitter.on(WsEvent.STOPPED_TALKING, event => this.handleStoppedTalking());
this.eventEmitter.on(WsEvent.PARTICIPANT_STARTED_TALKING, event => this.handleParticipantStartedTalking(event));
this.eventEmitter.on(WsEvent.PARTICIPANT_STOPPED_TALKING, event => this.handleParticipantStoppedTalking(event));
this.eventEmitter.on(WsEvent.PARTICIPANT_LEFT, event => this.handleParticipantLeft(event));
this.eventEmitter.on(WsEvent.LEFT_APPLICATION_CONFERENCE, event => this.handleLeftApplicationConference(event));
this.eventEmitter.on(WsEvent.JOINED_VIDEO_CONFERENCE, () => this.handleJoinedVideoConference());
this.eventEmitter.on(WsEvent.PUBLISHED_VIDEO_CONFERENCE, event => this.videoConferencePublishedHandler(event));
this.eventEmitter.on(WsEvent.UNPUBLISHED_VIDEO_CONFERENCE, () => this.videoConferenceUnpublishedHandler());
this.eventEmitter.on(WsEvent.JOIN_VIDEO_CONFERENCE_ERROR, event => this.handleJoinVideoConferenceError(event));
this.eventEmitter.on(WsEvent.PUBLISH_VIDEO_CONFERENCE_ERROR, event => this.publishVideoConferenceErrorHandler(event));
this.eventEmitter.on(WsEvent.SUBSCRIBED_VIDEO, event => this.subscribedVideoHandler(event));
this.eventEmitter.on(WsEvent.SUBSCRIBE_VIDEO_CONFERENCE_ERROR, event => this.subscribeVideoConferenceError(event));
this.eventEmitter.on(WsEvent.UPDATED_VIDEO, event => this.updatedVideoHandler(event));
this.eventEmitter.on(WsEvent.CONFERENCE_RECORDING_STARTED, event => this.handleConferenceRecordingStarted(event));
this.eventEmitter.on(WsEvent.CONFERENCE_RECORDING_STOPPED, event => this.handleConferenceRecordingStopped(event));
this.eventEmitter.on(WsEvent.DIALOG_CREATED, event => this.handleDialogCreated(event));
this.eventEmitter.on(WsEvent.DIALOG_ESTABLISHED, event => this.handleDialogEstablished(event));
this.eventEmitter.on(WsEvent.DIALOG_FINISHED, event => this.handleDialogFinished(event));
this.eventEmitter.on(WsEvent.DIALOG_FAILED, event => this.handleDialogFailed(event));
this.eventEmitter.on(WsEvent.DIALOG_RECORDING_STARTED, event => this.handleDialogRecordingStarted(event));
this.eventEmitter.on(WsEvent.DIALOG_RECORDING_STOPPED, event => this.handleDialogRecordingStopped(event));
this.eventEmitter.on(WsEvent.SETUP_DATA_CHANNEL, event => this.handleSetupDataChannel(event));
this.eventEmitter.on(WsEvent.SETUP_DATA_CHANNEL_ERROR, event => this.setupDataChannelError(event));
this.eventEmitter.on(WsEvent.PARTICIPANT_NETWORK_QUALITY, event => this.handleParticipantNetworkQuality(event));
this.eventEmitter.on("receiving_video_media", event => this.videoMediaReceivingHandler());
this.eventEmitter.on("reconnecting", () => this.handleReconnecting());
this.eventEmitter.on("reconnected", () => this.handleReconnected());
this.eventEmitter.on(WsEvent.CALL_RECONNECTED, event => this.handleCallReconnected(event));
this.eventEmitter.on(WsEvent.PARTICIPANT_DISCONNECTED, event => this.handleParticipantDisconnected(event));
this.eventEmitter.on(WsEvent.PARTICIPANT_RECONNECTED, event => this.handleParticipantReconnected(event));
this.eventEmitter.on(WsEvent.MESSAGE_RECEIVED, event => this.handleMessageReceived(event));
this.eventEmitter.on(WsEvent.MACHINE_DETECTION_FINISHED, event => this.handleMachineDetectionFinished(event));
this.eventEmitter.on(WsEvent.MACHINE_DETECTION_FAILED, event => this.handleMachineDetectionFailed(event));
}
handleDeviceChange() {
navigator.mediaDevices.ondevicechange = async () => {
let currentlyUsedDeviceId = this.device.getAudioInputDevice() || "default";
if (currentlyUsedDeviceId === "default") {
this.switchToDefaultDevice();
}
else {
let shouldChange = await this.device.audioInputDeviceShouldChange();
if (shouldChange) {
this.switchToDefaultDevice();
}
}
};
}
switchToDefaultDevice() {
this.setAudioInputDevice("default")
.catch(err => {
this.logger.error(`Switching audio input device failed (${err?.message})`, this.callId);
});
}
updateVideoBitrate() {
if (this.videoPublisherPC.peerConnection.connectionState !== 'connected') {
return;
}
let desiredBitrate = [
this.hasCameraVideo() ? BitrateConfig.CAMERA : 0,
this.hasScreenShare() ? BitrateConfig.SCREENSHARE : 0
].reduce((prev, curr) => prev + curr, 0);
configureForSending(this.videoPublisherPC.peerConnection, "video", desiredBitrate, this.logger);
}
onVideoPublisherConnectionStateChanged() {
this.updateVideoBitrate();
}
onAudioConnectionStateChanged() {
if (this.audioPC.peerConnection.connectionState === 'connected') {
this.setAudioQualityMode(this._audioQualityMode);
}
}
ringingHandler(event) {
this.callStatus = CallStatus.RINGING;
this.updateCustomData(event);
this.apiEventEmitter.emit(CallsApiEvent.RINGING, {});
}
async responseHandler(event) {
this.isEarlyMedia = event.isEarlyMedia;
try {
await this.audioPC.peerConnection.setRemoteDescription(event.description);
this.setRemoteCandidates();
}
catch (error) {
this.handleCallFlowError(error, true);
}
}
acceptedHandler() {
if (this.callStatus === CallStatus.ESTABLISHED) {
return;
}
this.callStatus = CallStatus.ESTABLISHED;
this.callEstablishTime = new Date();
this.apiEventEmitter.emit(CallsApiEvent.ESTABLISHED, {
stream: this.remoteAudioStream
});
}
videoCallJoinedHandler() {
if (this.joinedConference || this.joinedDialog) {
return;
}
if (this.hasCameraVideo() || this.hasScreenShare()) {
this.publishVideo(false);
}
}
changeMonitorConferenceId(conferenceId) {
if (this.audioPC) {
this.audioPC.monitor.conferenceId = conferenceId;
}
if (this.videoPublisherPC) {
this.videoPublisherPC.monitor.conferenceId = conferenceId;
}
if (this.videoSubscriberPC) {
this.videoSubscriberPC.monitor.conferenceId = conferenceId;
}
}
async handleSetupDataChannel(event) {
this._dataChannel?.initialize(event, this.rtcConfig);
}
handleJoinedApplicationConference(event) {
this.joinedConference = true;
this.conferenceId = event.id;
this.changeMonitorConferenceId(event.id);
this.participants = this.createParticipantMap(event.participants);
this._recordingState.conferenceRecording = event.recordingType;
if (!this.isRoleMediaAllowed()) {
this.disableMedia();
}
this.apiEventEmitter.emit(CallsApiEvent.CONFERENCE_JOINED, {
id: event.id,
name: event.name,
participants: Object.values(this.participants),
recordingType: event.recordingType
});
}
handleDialogCreated(event) {
this.joinedDialog = true;
this.dialogId = event.id;
this.changeMonitorConferenceId(event.id);
this.participants = this.createParticipantMap(event.participants);
const participant = Object.values(this.participants)
.find(participant => participant.endpoint.identifier !== this.currentUserIdentity);
this._recordingState.dialogRecording = event.recordingType;
this.apiEventEmitter.emit(CallsApiEvent.DIALOG_JOINED, {
id: event.id,
remote: participant,
recordingType: event.recordingType
});
}
createParticipantMap(participants) {
return participants.reduce((map, p) => {
let participant = this.loadParticipant(p);
let identifier = participant.endpoint.identifier;
map[identifier] = participant;
return map;
}, {});
}
handleDialogEstablished(event) {
this.participants = this.createParticipantMap(event.participants);
const participant = Object.values(this.participants)
.find(participant => participant.endpoint.identifier !== this.currentUserIdentity);
this._recordingState.dialogRecording = event.recordingType;
if (!this.joinedDialog) {
this.joinedDialog = true;
this.dialogId = event.id;
this.changeMonitorConferenceId(event.id);
this.apiEventEmitter.emit(CallsApiEvent.DIALOG_JOINED, {
id: event.id,
remote: participant,
recordingType: event.recordingType
});
}
if (participant.media.audio.muted) {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_MUTED, { participant: participant });
}
}
handleParticipantJoining(event) {
if (this.joinedConference) {
const participant = this.loadParticipant(event.participant);
const identifier = participant.endpoint.identifier;
if (!this.participants[identifier]) {
this.participants[identifier] = participant;
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_JOINING, { participant: participant });
}
}
}
handleParticipantJoined(event) {
if (this.joinedConference) {
const participant = this.loadParticipant(event.participant);
const identifier = participant.endpoint.identifier;
if (!this.participants[identifier] || this.participants[identifier].state === State.JOINING) {
this.participants[identifier] = participant;
this._dataChannel?.addParticipant(identifier);
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_JOINED, { participant: participant });
}
}
}
handleParticipantMediaChanged(event) {
if (this.joinedConference || this.joinedDialog) {
const endpoint = this.getEndpoint(event.endpoint);
const identifier = endpoint.identifier;
if (this.participants[identifier]) {
let participant = this.participants[identifier];
if (event.media.audio?.muted !== undefined) {
participant.media.audio.muted = event.media.audio.muted;
this.emitMutedEvent(participant);
}
if (event.media.audio?.deaf !== undefined && !this.joinedDialog) {
participant.media.audio.deaf = event.media.audio.deaf;
this.emitDeafEvent(participant);
}
if (event.media.video?.blind !== undefined && !this.joinedDialog) {
participant.media.video.blind = event.media.video.blind;
this.emitBlindEvent(participant);
}
}
}
}
handleRoleChanged(event) {
const role = this.getRole(event.role);
const localParticipant = this.participants[this.currentUserIdentity];
if (localParticipant) {
localParticipant.role = role;
}
if (!this.isRoleMediaAllowed()) {
this.disableMedia();
}
this.apiEventEmitter.emit(CallsApiEvent.ROLE_CHANGED, { role });
}
isRoleMediaAllowed() {
const localRoleType = this.participants[this.currentUserIdentity]?.role?.type;
return !localRoleType || [RoleType.DEFAULT, RoleType.ADVISOR].includes(localRoleType);
}
async disableMedia() {
let actions = [];
if (!this.muted()) {
actions.push(this.muteInternal(true));
}
if (this.hasCameraVideo() || this.hasScreenShare()) {
actions.push(this.stopVideoInternal(VideoRemovalReason.ROLE_CHANGED));
}
await Promise.all(actions);
}
handleParticipantRoleChanged(event) {
if (!this.joinedConference) {
return;
}
const participant = this.loadParticipant(event.participant);
const identifier = participant.endpoint.identifier;
this.participants[identifier] = participant;
this.emitParticipantRoleChanged(participant);
}
emitDeafEvent(participant) {
if (participant.media.audio.deaf) {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_DEAF, { participant: participant });
}
else {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_UNDEAF, { participant: participant });
}
}
emitMutedEvent(participant) {
if (participant.media.audio.muted) {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_MUTED, { participant: participant });
}
else {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_UNMUTED, { participant: participant });
}
}
emitBlindEvent(participant) {
if (participant.media.video.blind) {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_BLINDED, { participant: participant });
}
else {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_UNBLINDED, { participant: participant });
}
}
emitDisconnectedEvent(participant) {
if (participant.disconnected) {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_DISCONNECTED, { participant: participant });
}
else {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_RECONNECTED, { participant: participant });
}
}
emitParticipantRoleChanged(participant) {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_ROLE_CHANGED, { participant: participant });
}
handleStartedTalking() {
if (this.joinedConference) {
const localParticipant = this.participants[this.currentUserIdentity];
if (localParticipant) {
localParticipant.media.audio.talking = true;
}
this.emitTalkingEvent(true);
}
}
handleStoppedTalking() {
if (this.joinedConference) {
const localParticipant = this.participants[this.currentUserIdentity];
if (localParticipant) {
localParticipant.media.audio.talking = false;
}
this.emitTalkingEvent(false);
}
}
handleParticipantStartedTalking(event) {
if (this.joinedConference) {
const endpoint = this.getEndpoint(event.endpoint);
const identifier = endpoint.identifier;
if (this.participants[identifier]) {
let participant = this.participants[identifier];
participant.media.audio.talking = true;
this.emitParticipantTalkingEvent(participant, true);
}
}
}
handleParticipantStoppedTalking(event) {
if (this.joinedConference) {
const endpoint = this.getEndpoint(event.endpoint);
const identifier = endpoint.identifier;
if (this.participants[identifier]) {
let participant = this.participants[identifier];
participant.media.audio.talking = false;
this.emitParticipantTalkingEvent(participant, false);
}
}
}
emitTalkingEvent(isTalking) {
if (isTalking) {
this.apiEventEmitter.emit(CallsApiEvent.STARTED_TALKING, {});
}
else {
this.apiEventEmitter.emit(CallsApiEvent.STOPPED_TALKING, {});
}
}
emitParticipantTalkingEvent(participant, isTalking) {
if (isTalking) {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_STARTED_TALKING, { participant: participant });
}
else {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_STOPPED_TALKING, { participant: participant });
}
}
handleParticipantLeft(event) {
if (this.joinedConference) {
const participant = this.loadParticipant(event.participant);
const identifier = participant.endpoint.identifier;
delete this.participants[identifier];
this._dataChannel?.removeParticipant(identifier);
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_LEFT, { participant: participant });
}
}
async handleJoinedVideoConference() {
this.videoPublisherCleanup(true);
if (this.hasCameraVideo() || this.hasScreenShare()) {
if (this.mediaUpdateStatus === MediaUpdateStatus.IDLE) {
this.mediaUpdateStatus = MediaUpdateStatus.MIGRATING;
}
await this.migrateLocalVideos();
try {
await this.negotiateVideoPublisher(true);
}
catch (error) {
this.handlePublishVideoFlowError(error);
}
}
}
subscribedVideoHandler(event) {
this.remoteVideos = this.mapStreamEvent(event.streams);
this.createVideoSubscriberPC();
this.videoSubscriberPC.peerConnection.setRemoteDescription(event.description)
.then(() => this.videoSubscriberPC.peerConnection.createAnswer())
.then(answer => this.setLocalDescription(this.videoSubscriberPC.peerConnection, answer))
.then(answer => {
this.gateway.send({
action: WsAction.START_VIDEO_CONFERENCE,
description: answer
});
})
.catch(error => this.handleSubscribeVideoFlowError(error));
}
handleSubscribeVideoFlowError(error) {
this.logger.error(`Subscribe video flow error occurred: ${error}`, this.callId);
this.videoSubscriberCleanup();
let errorCode = HangupStatusFactory.getApplicationHangupStatus(error);
this.apiEventEmitter.emit(CallsApiEvent.ERROR, { errorCode });
}
updatedVideoHandler(event) {
if (event.streams) {
Object.keys(this.remoteVideos)
.filter(mid => !event.streams[mid])
.forEach(mid => {
let remoteVideo = this.remoteVideos[mid];
delete this.remoteVideos[mid];
this.emitParticipantVideoRemovedEvent(remoteVideo.type, remoteVideo.participant);
});
this.remoteVideos = this.mapStreamEvent(event.streams);
}
else {
this.videoSubscriberPC.peerConnection.restartIce();
}
this.videoSubscriberPC.peerConnection.setRemoteDescription(event.description)
.then(() => this.videoSubscriberPC.peerConnection.createAnswer())
.then(answer => this.setLocalDescription(this.videoSubscriberPC.peerConnection, answer))
.then(answer => {
this.gateway.send({
action: WsAction.START_VIDEO_CONFERENCE,
description: answer
});
})
.catch(error => this.handleSubscribeVideoFlowError(error));
}
emitParticipantVideoRemovedEvent(type, participant) {
if (type === VideoType.CAMERA) {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_CAMERA_VIDEO_REMOVED, { participant: participant });
}
else if (type === VideoType.SCREENSHARE) {
this.apiEventEmitter.emit(CallsApiEvent.PARTICIPANT_SCREEN_SHARE_REMOVED, { participant: participant });
}
}
async updateCameraStream(cameraOrientation = CameraOrientation.FRONT, useExactDevice = true) {
return this.updateCameraStreamLock.withLock(async () => {
if (!this.hasCameraVideo()) {
throw new Error("Camera video is not enabled.");
}
this.localCameraVideoStream?.getVideoTracks().forEach(track => track.stop());
try {
await this.videoFilterManager?.stop();
const stream = await this.getCameraVideoStream(cameraOrientation, useExactDevice);
this.localCameraVideoStream?.getVideoTracks().forEach(track => track.stop());
this.localCameraVideoStream = stream;
this.apiEventEmitter.emit(CallsApiEvent.CAMERA_VIDEO_UPDATED, { stream: stream });
await this.videoFilterManager?.start(stream, 0, this.apiEventEmitter);
return this.replaceTrack(this.localCameraVideo, stream.getVideoTracks()[0]);
}
catch (error) {
this.throwMediaError(error);
}
});
}
async handleLeftApplicationConference(event) {
this.joinedConference = false;
this.conferenceId = null;
this.changeMonitorConferenceId(null);
this.participants = {};
this.remoteVideos = {};
this.videoPublisherCleanup(true);
this.videoSubscriberCleanup();
this.dataChannelCleanup();
this._recordingState.conferenceRecording = "UNDEFINED";
if (this.hasCameraVideo() || this.hasScreenShare()) {
this.mediaUpdateStatus = MediaUpdateStatus.MIGRATING;
await this.migrateLocalVideos();
}
this.apiEventEmitter.emit(CallsApiEvent.CONFERENCE_LEFT, { errorCode: event.status });
}
async handleDialogFinished(event) {
this.joinedDialog = false;
this.dialogId = null;
this.changeMonitorConferenceId(null);
this.participants = {};
this.remoteVideos = {};
this.videoPublisherCleanup(true);
this.videoSubscriberCleanup();
this.dataChannelCleanup();
this._recordingState.dialogRecording = "UNDEFINED";
if (this.hasCameraVideo() || this.hasScreenShare()) {
this.mediaUpdateStatus = MediaUpdateStatus.MIGRATING;
await this.migrateLocalVideos();
}
this.apiEventEmitter.emit(CallsApiEvent.DIALOG_LEFT, { errorCode: event.status });
}
async handleReconnecting() {
if (this.callStatus !== CallStatus.ESTABLISHED) {
this.logger.warn("Reconnect failed: call is not established");
this.hangupHandler({ callId: this.callId, status: ApplicationErrorCode.NETWORK_ERROR });
return;
}
if (!this.applicationCallOptions?.autoReconnect) {
this.logger.debug("Websocket is reconnecting, but the active call doesn't support reconnect. Hanging up...");
this.hangupHandler({ callId: this.callId, status: ApplicationErrorCode.NETWORK_ERROR });
return;
}
this.reconnecting = true;
this.apiEventEmitter.emit(CallsApiEvent.RECONNECTING, {});
}
async handleReconnected() {
this.logger.info(`Reconnecting call ${this.callId}, restarting ICE...`);
const audioOffer = await this.audioPC?.restartIce();
this.gateway.send({
action: WsAction.RECONNECT_APPLICATION_CALL,
callId: this.callId,
audioOffer,
});
}
async handleCallReconnected(event) {
this.callStatus = CallStatus.ESTABLISHED;
await this.audioPC.peerConnection.setRemoteDescription(event.description);
await this.handleApplicationCallChanges(event);
let state