@100mslive/hms-video-store
Version:
@100mslive Core SDK which abstracts the complexities of webRTC while providing a reactive store for data management with a unidirectional data flow
1,328 lines (1,204 loc) • 52 kB
text/typescript
import { JoinParameters } from './models/JoinParameters';
import { TransportFailureCategory } from './models/TransportFailureCategory';
import { TransportState } from './models/TransportState';
import ITransportObserver from './ITransportObserver';
import { RetryScheduler } from './RetryScheduler';
import { AdditionalAnalyticsProperties } from '../analytics/AdditionalAnalyticsProperties';
import AnalyticsEvent from '../analytics/AnalyticsEvent';
import AnalyticsEventFactory from '../analytics/AnalyticsEventFactory';
import { AnalyticsEventsService } from '../analytics/AnalyticsEventsService';
import { AnalyticsTimer, TimedEvent } from '../analytics/AnalyticsTimer';
import { HTTPAnalyticsTransport } from '../analytics/HTTPAnalyticsTransport';
import { SignalAnalyticsTransport } from '../analytics/signal-transport/SignalAnalyticsTransport';
import { PublishStatsAnalytics, SubscribeStatsAnalytics } from '../analytics/stats';
import { PluginUsageTracker } from '../common/PluginUsageTracker';
import { HMSConnectionRole, HMSTrickle } from '../connection/model';
import { IPublishConnectionObserver } from '../connection/publish/IPublishConnectionObserver';
import HMSPublishConnection from '../connection/publish/publishConnection';
import ISubscribeConnectionObserver from '../connection/subscribe/ISubscribeConnectionObserver';
import HMSSubscribeConnection from '../connection/subscribe/subscribeConnection';
import { DeviceManager } from '../device-manager';
import { HMSDiagnosticsConnectivityListener } from '../diagnostics/interfaces';
import { ErrorCodes } from '../error/ErrorCodes';
import { ErrorFactory } from '../error/ErrorFactory';
import { HMSAction } from '../error/HMSAction';
import { HMSException } from '../error/HMSException';
import { EventBus } from '../events/EventBus';
import { HMSICEServer, HMSRole, HMSTrackUpdate, HMSUpdateListener } from '../interfaces';
import { HMSLocalStream } from '../media/streams/HMSLocalStream';
import { HMSLocalTrack, HMSLocalVideoTrack, HMSTrack } from '../media/tracks';
import { TrackState } from '../notification-manager';
import { HMSWebrtcInternals } from '../rtc-stats/HMSWebrtcInternals';
import { Store } from '../sdk/store';
import InitService from '../signal/init';
import { InitConfig, InitFlags } from '../signal/init/models';
import { ISignalEventsObserver } from '../signal/ISignalEventsObserver';
import JsonRpcSignal from '../signal/jsonrpc';
import {
ICE_DISCONNECTION_TIMEOUT,
LEAVE_REASON,
PROTOCOL_SPEC,
PROTOCOL_VERSION,
PUBLISH_STATS_PUSH_INTERVAL,
PUBLISH_STATS_SAMPLE_WINDOW,
RENEGOTIATION_CALLBACK_ID,
SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID,
SUBSCRIBE_STATS_PUSH_INTERVAL,
SUBSCRIBE_STATS_SAMPLE_WINDOW,
SUBSCRIBE_TIMEOUT,
} from '../utils/constants';
import HMSLogger from '../utils/logger';
import { getNetworkInfo } from '../utils/network-info';
import { PromiseCallbacks } from '../utils/promise';
const TAG = '[HMSTransport]:';
// @DISCUSS: action and extra are not used at all.
interface CallbackTriple {
promise: PromiseCallbacks<boolean>;
action: HMSAction;
extra: any;
}
interface NegotiateJoinParams {
name: string;
data: string;
autoSubscribeVideo: boolean;
}
export default class HMSTransport {
private state: TransportState = TransportState.Disconnected;
private trackStates: Map<string, TrackState> = new Map();
private publishConnection: HMSPublishConnection | null = null;
private subscribeConnection: HMSSubscribeConnection | null = null;
private initConfig?: InitConfig;
private endpoint!: string;
private joinParameters?: JoinParameters;
private retryScheduler: RetryScheduler;
private webrtcInternals?: HMSWebrtcInternals;
private publishStatsAnalytics?: PublishStatsAnalytics;
private subscribeStatsAnalytics?: SubscribeStatsAnalytics;
private maxSubscribeBitrate = 0;
private connectivityListener?: HMSDiagnosticsConnectivityListener;
private sfuNodeId?: string;
joinRetryCount = 0;
private publishDisconnectTimer = 0;
private listener?: HMSUpdateListener;
private onScreenshareStop = () => {};
private screenStream = new Set<MediaStream>();
constructor(
private observer: ITransportObserver,
private deviceManager: DeviceManager,
private store: Store,
private eventBus: EventBus,
private analyticsEventsService: AnalyticsEventsService,
private analyticsTimer: AnalyticsTimer,
private pluginUsageTracker: PluginUsageTracker,
) {
this.webrtcInternals = new HMSWebrtcInternals(this.store, this.eventBus);
const onStateChange = async (state: TransportState, error?: HMSException) => {
if (state !== this.state) {
this.state = state;
await this.observer.onStateChange(this.state, error);
}
};
this.retryScheduler = new RetryScheduler(onStateChange, this.sendErrorAnalyticsEvent.bind(this));
this.eventBus.statsUpdate.subscribe(stats => {
const currentSubscribeBitrate = stats.getLocalPeerStats()?.subscribe?.bitrate || 0;
this.maxSubscribeBitrate = Math.max(this.maxSubscribeBitrate, currentSubscribeBitrate);
});
this.eventBus.localAudioEnabled.subscribe(({ track, enabled }) => this.trackUpdate(track, enabled));
this.eventBus.localVideoEnabled.subscribe(({ track, enabled }) => this.trackUpdate(track, enabled));
}
/**
* Map of callbacks used to wait for an event to fire.
* Used here for:
* 1. publish/unpublish waits for [IPublishConnectionObserver.onRenegotiationNeeded] to complete
*/
private readonly callbacks = new Map<string, CallbackTriple>();
setListener = (listener: HMSUpdateListener) => {
this.listener = listener;
};
setOnScreenshareStop = (onStop: () => void) => {
this.onScreenshareStop = onStop;
};
getWebsocketEndpoint(): string | undefined {
if (!this.initConfig) {
return;
}
return this.initConfig.endpoint;
}
private signalObserver: ISignalEventsObserver = {
// eslint-disable-next-line complexity
onOffer: async (jsep: RTCSessionDescriptionInit & { sfu_node_id?: string }) => {
try {
if (!this.subscribeConnection) {
return;
}
if (
jsep.sfu_node_id &&
this.subscribeConnection.sfuNodeId &&
this.subscribeConnection.sfuNodeId !== jsep.sfu_node_id
) {
HMSLogger.d(TAG, 'ignoring old offer');
return;
}
await this.subscribeConnection.setRemoteDescription(jsep);
HMSLogger.d(
TAG,
`[SUBSCRIBE] Adding ${this.subscribeConnection.candidates.length} ice-candidates`,
this.subscribeConnection.candidates,
);
for (const candidate of this.subscribeConnection.candidates) {
await this.subscribeConnection.addIceCandidate(candidate);
}
this.subscribeConnection.candidates.length = 0;
const answer = await this.subscribeConnection.createAnswer();
await this.subscribeConnection.setLocalDescription(answer);
this.signal.answer(answer);
HMSLogger.d(TAG, '[role=SUBSCRIBE] onOffer renegotiation DONE ✅');
} catch (err) {
HMSLogger.d(TAG, '[role=SUBSCRIBE] onOffer renegotiation FAILED ❌', err);
this.state = TransportState.Failed;
let ex: HMSException;
if (err instanceof HMSException) {
ex = err;
} else {
ex = ErrorFactory.GenericErrors.Unknown(HMSAction.SUBSCRIBE, (err as Error).message);
}
this.observer.onFailure(ex);
this.eventBus.analytics.publish(AnalyticsEventFactory.subscribeFail(ex));
}
},
onTrickle: async (trickle: HMSTrickle) => {
const connection =
trickle.target === HMSConnectionRole.Publish ? this.publishConnection : this.subscribeConnection;
if (!connection?.remoteDescription) {
// ICE candidates can't be added without any remote session description
connection?.candidates.push(trickle.candidate);
} else {
await connection.addIceCandidate(trickle.candidate);
}
},
onNotification: (message: any) => this.observer.onNotification(message),
onServerError: async (error: HMSException) => {
await this.observer.onStateChange(TransportState.Failed, error);
},
onFailure: (error: HMSException) => {
// @DISCUSS: Should we remove this? Pong failure would have already scheduled signal retry.
if (this.joinParameters) {
this.retryScheduler.schedule({
category: TransportFailureCategory.SignalDisconnect,
error,
task: this.retrySignalDisconnectTask,
originalState: this.state,
});
}
},
onOffline: async (reason: string) => {
HMSLogger.d(TAG, 'socket offline', TransportState[this.state]);
try {
if (this.state !== TransportState.Leaving && this.joinParameters) {
this.retryScheduler.schedule({
category: TransportFailureCategory.SignalDisconnect,
error: ErrorFactory.WebSocketConnectionErrors.WebSocketConnectionLost(HMSAction.RECONNECT_SIGNAL, reason),
task: this.retrySignalDisconnectTask,
originalState: this.state,
});
}
} catch (e) {
console.error(e);
}
},
// this is called when socket connection is successful
onOnline: () => {
HMSLogger.d(TAG, 'socket online', TransportState[this.state]);
this.analyticsSignalTransport.flushFailedEvents(this.store.getLocalPeer()?.peerId);
},
// this is called when window.online event is triggered
onNetworkOnline: () => {
this.analyticsEventsService.flushFailedClientEvents();
},
};
public readonly signal = new JsonRpcSignal(this.signalObserver);
private analyticsSignalTransport = new SignalAnalyticsTransport(this.signal);
private publishDtlsStateTimer = 0;
private lastPublishDtlsState: RTCDtlsTransportState = 'new';
getWebrtcInternals() {
return this.webrtcInternals;
}
isFlagEnabled(flag: InitFlags) {
const config = this.initConfig?.config;
const flags = config?.enabledFlags || [];
return flags.includes(flag);
}
setConnectivityListener(listener: HMSDiagnosticsConnectivityListener) {
this.connectivityListener = listener;
}
async preview(
token: string,
endpoint: string,
peerId: string,
customData: { name: string; metaData: string },
autoSubscribeVideo = false,
iceServers?: HMSICEServer[],
): Promise<InitConfig | void> {
const initConfig = await this.connect(token, endpoint, peerId, customData, autoSubscribeVideo, iceServers);
this.state = TransportState.Preview;
this.observer.onStateChange(this.state);
return initConfig;
}
async join(
authToken: string,
peerId: string,
customData: { name: string; metaData: string },
initEndpoint: string,
autoSubscribeVideo = false,
iceServers?: HMSICEServer[],
): Promise<void> {
HMSLogger.d(TAG, 'join: started ⏰');
try {
if (!this.signal.isConnected || !this.initConfig) {
await this.connect(authToken, initEndpoint, peerId, customData, autoSubscribeVideo, iceServers);
}
this.validateNotDisconnected('connect');
if (this.initConfig) {
await this.waitForLocalRoleAvailability();
await this.createConnectionsAndNegotiateJoin(customData, autoSubscribeVideo);
this.initStatsAnalytics();
HMSLogger.d(TAG, '✅ join: Negotiated over PUBLISH connection');
}
} catch (error) {
HMSLogger.e(TAG, `join: failed ❌ [token=${authToken}]`, error);
this.state = TransportState.Failed;
const ex = error as HMSException;
// set isTerminal to true if not already when error code is 500(internal biz server error)
ex.isTerminal = ex.isTerminal || ex.code === 500;
await this.observer.onStateChange(this.state, ex);
throw ex;
}
HMSLogger.d(TAG, '✅ join: successful');
this.state = TransportState.Joined;
this.observer.onStateChange(this.state);
}
// eslint-disable-next-line complexity
async connect(
token: string,
endpoint: string,
peerId: string,
customData: { name: string; metaData: string },
autoSubscribeVideo = false,
iceServers?: HMSICEServer[],
): Promise<InitConfig | void> {
this.setTransportStateForConnect();
this.joinParameters = new JoinParameters(
token,
peerId,
customData.name,
customData.metaData,
endpoint,
autoSubscribeVideo,
iceServers,
);
try {
const response = await this.internalConnect(token, endpoint, peerId, iceServers);
return response;
} catch (error) {
if (!(error instanceof HMSException)) {
throw error;
}
const shouldRetry =
[
ErrorCodes.WebSocketConnectionErrors.WEBSOCKET_CONNECTION_LOST,
ErrorCodes.WebSocketConnectionErrors.FAILED_TO_CONNECT,
ErrorCodes.WebSocketConnectionErrors.ABNORMAL_CLOSE,
].includes(error.code) ||
error.code.toString().startsWith('5') ||
error.code.toString().startsWith('429') ||
(error.code === ErrorCodes.APIErrors.ENDPOINT_UNREACHABLE && !navigator.onLine);
if (shouldRetry) {
const task = async () => {
await this.internalConnect(token, endpoint, peerId, iceServers);
return Boolean(this.initConfig && this.initConfig.endpoint);
};
await this.retryScheduler.schedule({
category: TransportFailureCategory.ConnectFailed,
error,
task,
originalState: this.state,
changeState: false,
});
} else {
throw error;
}
}
}
async leave(notifyServer: boolean, reason = LEAVE_REASON.USER_REQUEST): Promise<void> {
this.retryScheduler.reset();
this.joinParameters = undefined;
HMSLogger.d(TAG, 'leaving in transport');
try {
const usage = this.pluginUsageTracker.getPluginUsage('HMSKrispPlugin');
if (usage) {
this.eventBus.analytics.publish(AnalyticsEventFactory.getKrispUsage(usage));
}
this.state = TransportState.Leaving;
this.publishStatsAnalytics?.stop();
this.subscribeStatsAnalytics?.stop();
this.webrtcInternals?.cleanup();
this.clearPeerConnections();
if (notifyServer) {
try {
this.signal.leave(reason);
HMSLogger.d(TAG, 'signal leave done');
} catch (err) {
HMSLogger.w(TAG, 'failed to send leave on websocket to server', err);
}
}
this.analyticsEventsService.flushFailedClientEvents();
this.analyticsEventsService.reset();
await this.signal.close();
} catch (err) {
this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(err as Error));
HMSLogger.e(TAG, 'leave: FAILED ❌', err);
} finally {
this.state = TransportState.Disconnected;
this.observer.onStateChange(this.state);
}
}
handleLocalRoleUpdate = async ({ oldRole, newRole }: { oldRole: HMSRole; newRole: HMSRole }) => {
const changedFromNonWebRTCToWebRTC = !this.doesRoleNeedWebRTC(oldRole) && this.doesRoleNeedWebRTC(newRole);
if (!changedFromNonWebRTCToWebRTC) {
return;
}
HMSLogger.d(
TAG,
'Local peer role updated to webrtc role, creating PeerConnections and performing inital publish negotiation ⏳',
);
this.createPeerConnections();
await this.negotiateOnFirstPublish();
};
async publish(tracks: Array<HMSLocalTrack>): Promise<void> {
for (const track of tracks) {
try {
await this.publishTrack(track);
this.connectivityListener?.onMediaPublished(track);
} catch (error) {
this.eventBus.analytics.publish(
AnalyticsEventFactory.publish({
devices: this.deviceManager.getDevices(),
error: error as Error,
}),
);
}
}
}
async unpublish(tracks: Array<HMSLocalTrack>): Promise<void> {
for (const track of tracks) {
await this.unpublishTrack(track);
}
}
setSFUNodeId(id?: string) {
this.signal.setSfuNodeId(id);
if (!this.sfuNodeId) {
this.sfuNodeId = id;
this.publishConnection?.setSfuNodeId(id);
this.subscribeConnection?.setSfuNodeId(id);
} else if (id && this.sfuNodeId !== id) {
this.sfuNodeId = id;
this.handleSFUMigration();
}
}
// eslint-disable-next-line complexity
async handleSFUMigration() {
HMSLogger.time('sfu migration');
this.clearPeerConnections();
const peers = this.store.getPeerMap();
this.store.removeRemoteTracks();
for (const peerId in peers) {
const peer = peers[peerId];
if (peer.isLocal) {
continue;
}
peer.audioTrack = undefined;
peer.videoTrack = undefined;
peer.auxiliaryTracks = [];
}
const localPeer = this.store.getLocalPeer();
if (!localPeer) {
return;
}
this.createPeerConnections();
this.trackStates.clear();
await this.negotiateOnFirstPublish();
const streamMap = new Map<string, HMSLocalStream>();
if (localPeer.audioTrack) {
const stream = localPeer.audioTrack.stream as HMSLocalStream;
if (!streamMap.get(stream.id)) {
streamMap.set(stream.id, new HMSLocalStream(new MediaStream()));
}
const newTrack = localPeer.audioTrack.clone(streamMap.get(stream.id)!);
this.store.removeTrack(localPeer.audioTrack);
localPeer.audioTrack.cleanup();
await this.publishTrack(newTrack);
localPeer.audioTrack = newTrack;
}
if (localPeer.videoTrack) {
const stream = localPeer.videoTrack.stream as HMSLocalStream;
if (!streamMap.get(stream.id)) {
streamMap.set(stream.id, new HMSLocalStream(new MediaStream()));
}
this.store.removeTrack(localPeer.videoTrack);
const newTrack = localPeer.videoTrack.clone(streamMap.get(stream.id)!);
localPeer.videoTrack.cleanup();
await this.publishTrack(newTrack);
localPeer.videoTrack = newTrack;
}
const auxTracks = [];
while (localPeer.auxiliaryTracks.length > 0) {
const track = localPeer.auxiliaryTracks.shift();
if (track) {
const stream = track.stream as HMSLocalStream;
if (!streamMap.get(stream.id)) {
/**
* For screenshare, you need to clone the current stream only, cloning the track will not work otherwise, it will have all
* correct states but bytes sent and all other stats would be 0
**/
streamMap.set(
stream.id,
new HMSLocalStream(track.source === 'screen' ? stream.nativeStream.clone() : new MediaStream()),
);
}
this.store.removeTrack(track);
const newTrack = track.clone(streamMap.get(stream.id)!);
if (newTrack.type === 'video' && newTrack.source === 'screen') {
/**
* Store all the stream so they can be stopped when screenshare stopped. Stopping before is not helping
*/
this.screenStream.add(stream.nativeStream);
this.screenStream.add(newTrack.stream.nativeStream);
newTrack.nativeTrack.addEventListener('ended', this.onScreenshareStop);
}
track.cleanup();
await this.publishTrack(newTrack);
auxTracks.push(newTrack);
}
}
localPeer.auxiliaryTracks = auxTracks;
streamMap.clear();
this.listener?.onSFUMigration?.();
HMSLogger.timeEnd('sfu migration');
}
/**
* TODO: check if track.publishedTrackId be used instead of the hack to match with track with same type and
* source. The hack won't work if there are multiple tracks with same source and type.
*/
trackUpdate(track: HMSLocalTrack, enabled: boolean) {
const currentTrackStates = Array.from(this.trackStates.values());
const originalTrackState = currentTrackStates.find(
trackState => track.type === trackState.type && track.source === trackState.source,
);
/**
* on call interruption, we just send disabled track update to biz to send to remote peers WITHOUT sending to the local peer
* in this case, track.enabled would still be true which is why we are using the value from the localVideoEnabled event
* */
if (originalTrackState) {
const newTrackState = new TrackState({
...originalTrackState,
mute: !enabled,
});
this.trackStates.set(originalTrackState.track_id, newTrackState);
HMSLogger.d(TAG, 'Track Update', this.trackStates, track);
this.signal.trackUpdate(new Map([[originalTrackState.track_id, newTrackState]]));
const peer = this.store.getLocalPeer();
// don't send update in case of call interruption
if (peer && enabled === track.enabled) {
this.listener?.onTrackUpdate(enabled ? HMSTrackUpdate.TRACK_UNMUTED : HMSTrackUpdate.TRACK_MUTED, track, peer);
}
}
}
private async publishTrack(track: HMSLocalTrack): Promise<void> {
track.publishedTrackId = track.getTrackIDBeingSent();
HMSLogger.d(
TAG,
`⏳ publishTrack: trackId=${track.trackId}, toPublishTrackId=${track.publishedTrackId}`,
`${track}`,
);
this.trackStates.set(track.publishedTrackId, new TrackState(track));
const p = new Promise<boolean>((resolve, reject) => {
this.callbacks.set(RENEGOTIATION_CALLBACK_ID, {
promise: { resolve, reject },
action: HMSAction.PUBLISH,
extra: {},
});
});
const stream = track.stream as HMSLocalStream;
stream.setConnection(this.publishConnection!);
const simulcastLayers = this.store.getSimulcastLayers(track.source!);
stream.addTransceiver(track, simulcastLayers);
HMSLogger.time(`publish-${track.trackId}-${track.type}`);
await p;
HMSLogger.timeEnd(`publish-${track.trackId}-${track.type}`);
// add track to store after publish
this.store.addTrack(track);
await stream
.setMaxBitrateAndFramerate(track)
.then(() => {
HMSLogger.d(
TAG,
`Setting maxBitrate=${track.settings.maxBitrate} kpbs${
track instanceof HMSLocalVideoTrack ? ` and maxFramerate=${track.settings.maxFramerate}` : ''
} for ${track.source} ${track.type} ${track.trackId}`,
);
})
.catch(error => HMSLogger.w(TAG, 'Failed setting maxBitrate and maxFramerate', error));
track.isPublished = true;
HMSLogger.d(TAG, `✅ publishTrack: trackId=${track.trackId}`, `${track}`, this.callbacks);
}
private async unpublishTrack(track: HMSLocalTrack): Promise<void> {
HMSLogger.d(TAG, `⏳ unpublishTrack: trackId=${track.trackId}`, `${track}`);
if (track.publishedTrackId && this.trackStates.has(track.publishedTrackId)) {
this.trackStates.delete(track.publishedTrackId);
} else {
// TODO: hotfix to unpublish replaced video track id, solve it properly
// it won't work when there are multiple regular video tracks, hmslocalvideotrack can store
// the original initial track id for a proper fix
const currentTrackStates = Array.from(this.trackStates.values());
const originalTrackState = currentTrackStates.find(
trackState => track.type === trackState.type && track.source === trackState.source,
);
if (originalTrackState) {
this.trackStates.delete(originalTrackState.track_id);
}
}
const p = new Promise<boolean>((resolve, reject) => {
this.callbacks.set(RENEGOTIATION_CALLBACK_ID, {
promise: { resolve, reject },
action: HMSAction.UNPUBLISH,
extra: {},
});
});
const stream = track.stream as HMSLocalStream;
stream.removeSender(track);
await p;
await track.cleanup();
if (track.source === 'screen' && this.screenStream) {
// stop older screenshare tracks to remove the screenshare banner
this.screenStream.forEach(stream => {
stream.getTracks().forEach(_track => {
_track.stop();
});
this.screenStream.delete(stream);
});
}
// remove track from store on unpublish
this.store.removeTrack(track);
HMSLogger.d(TAG, `✅ unpublishTrack: trackId=${track.trackId}`, this.callbacks);
}
private async clearPeerConnections() {
clearTimeout(this.publishDtlsStateTimer);
this.publishDtlsStateTimer = 0;
clearTimeout(this.publishDisconnectTimer);
this.publishDisconnectTimer = 0;
this.lastPublishDtlsState = 'new';
this.publishConnection?.close();
this.subscribeConnection?.close();
this.publishConnection = null;
this.subscribeConnection = null;
}
private waitForLocalRoleAvailability() {
if (this.store.hasRoleDetailsArrived()) {
return;
} else {
return new Promise<void>(resolve => {
this.eventBus.policyChange.subscribeOnce(() => resolve());
});
}
}
private async createConnectionsAndNegotiateJoin(
customData: { name: string; metaData: string },
autoSubscribeVideo = false,
) {
const isWebRTC = this.doesLocalPeerNeedWebRTC();
if (isWebRTC) {
this.createPeerConnections();
}
this.analyticsTimer.start(TimedEvent.JOIN_RESPONSE);
await this.negotiateJoinWithRetry({
name: customData.name,
data: customData.metaData,
autoSubscribeVideo,
isWebRTC,
});
this.analyticsTimer.end(TimedEvent.JOIN_RESPONSE);
}
private createPeerConnections() {
const logConnectionState = (
role: HMSConnectionRole,
newState: RTCIceConnectionState | RTCPeerConnectionState,
ice = false,
) => {
const log = ['disconnected', 'failed'].includes(newState)
? HMSLogger.w.bind(HMSLogger)
: HMSLogger.d.bind(HMSLogger);
log(TAG, `${HMSConnectionRole[role]} ${ice ? 'ice' : ''} connection state change: ${newState}`);
};
if (this.initConfig) {
const publishConnectionObserver: IPublishConnectionObserver = {
onRenegotiationNeeded: async () => {
await this.performPublishRenegotiation();
},
// eslint-disable-next-line complexity
onDTLSTransportStateChange: (state?: RTCDtlsTransportState) => {
const log = state === 'failed' ? HMSLogger.w.bind(HMSLogger) : HMSLogger.d.bind(HMSLogger);
log(TAG, `Publisher on dtls transport state change: ${state}`);
if (!state || this.lastPublishDtlsState === state) {
return;
}
this.lastPublishDtlsState = state;
if (this.publishDtlsStateTimer !== 0) {
clearTimeout(this.publishDtlsStateTimer);
this.publishDtlsStateTimer = 0;
}
if (state !== 'connecting' && state !== 'failed') {
return;
}
const timeout = this.initConfig?.config?.dtlsStateTimeouts?.[state];
if (!timeout || timeout <= 0) {
return;
}
// if we're in connecting check again after timeout
// hotfix: mitigate https://100ms.atlassian.net/browse/LIVE-1924
this.publishDtlsStateTimer = window.setTimeout(() => {
const newState = this.publishConnection?.nativeConnection.connectionState;
if (newState && state && newState === state) {
// stuck in either `connecting` or `failed` state for long time
const err = ErrorFactory.WebrtcErrors.ICEFailure(
HMSAction.PUBLISH,
`DTLS transport state ${state} timeout:${timeout}ms`,
true,
);
this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(err));
this.observer.onFailure(err);
}
}, timeout);
},
onDTLSTransportError: (error: Error) => {
HMSLogger.e(TAG, `onDTLSTransportError ${error.name} ${error.message}`, error);
this.eventBus.analytics.publish(AnalyticsEventFactory.disconnect(error));
},
onIceConnectionChange: async (newState: RTCIceConnectionState) => {
logConnectionState(HMSConnectionRole.Publish, newState, true);
},
onConnectionStateChange: async (newState: RTCPeerConnectionState) => {
logConnectionState(HMSConnectionRole.Publish, newState, false);
if (newState === 'new') {
return;
}
if (newState === 'connected') {
this.connectivityListener?.onICESuccess(true);
this.publishConnection?.handleSelectedIceCandidatePairs();
} else if (newState === 'failed') {
await this.handleIceConnectionFailure(
HMSConnectionRole.Publish,
ErrorFactory.WebrtcErrors.ICEFailure(
HMSAction.PUBLISH,
`local candidate - ${this.publishConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.publishConnection?.selectedCandidatePair?.remote?.candidate}`,
),
);
} else {
this.publishDisconnectTimer = window.setTimeout(() => {
if (this.publishConnection?.connectionState !== 'connected') {
this.handleIceConnectionFailure(
HMSConnectionRole.Publish,
ErrorFactory.WebrtcErrors.ICEDisconnected(
HMSAction.PUBLISH,
`local candidate - ${this.publishConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.publishConnection?.selectedCandidatePair?.remote?.candidate}`,
),
);
}
}, ICE_DISCONNECTION_TIMEOUT);
}
},
onIceCandidate: candidate => {
this.connectivityListener?.onICECandidate(candidate, true);
},
onSelectedCandidatePairChange: candidatePair => {
this.connectivityListener?.onSelectedICECandidatePairChange(candidatePair, true);
},
};
const subscribeConnectionObserver: ISubscribeConnectionObserver = {
onApiChannelMessage: (message: string) => {
this.observer.onNotification(JSON.parse(message));
},
onTrackAdd: (track: HMSTrack) => {
HMSLogger.d(TAG, '[Subscribe] onTrackAdd', `${track}`);
this.observer.onTrackAdd(track);
},
onTrackRemove: (track: HMSTrack) => {
HMSLogger.d(TAG, '[Subscribe] onTrackRemove', `${track}`);
this.observer.onTrackRemove(track);
},
onIceConnectionChange: async (newState: RTCIceConnectionState) => {
logConnectionState(HMSConnectionRole.Subscribe, newState, true);
if (newState === 'connected') {
const callback = this.callbacks.get(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
this.callbacks.delete(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
this.connectivityListener?.onICESuccess(false);
if (callback) {
callback.promise.resolve(true);
}
}
},
onConnectionStateChange: async (newState: RTCPeerConnectionState) => {
logConnectionState(HMSConnectionRole.Subscribe, newState, false);
if (newState === 'failed') {
await this.handleIceConnectionFailure(
HMSConnectionRole.Subscribe,
ErrorFactory.WebrtcErrors.ICEFailure(
HMSAction.SUBSCRIBE,
`local candidate - ${this.subscribeConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.subscribeConnection?.selectedCandidatePair?.remote?.candidate}`,
),
);
} else if (newState === 'disconnected') {
setTimeout(() => {
if (this.subscribeConnection?.connectionState === 'disconnected') {
this.handleIceConnectionFailure(
HMSConnectionRole.Subscribe,
ErrorFactory.WebrtcErrors.ICEDisconnected(
HMSAction.SUBSCRIBE,
`local candidate - ${this.subscribeConnection?.selectedCandidatePair?.local?.candidate}; remote candidate - ${this.subscribeConnection?.selectedCandidatePair?.remote?.candidate}`,
),
);
}
}, ICE_DISCONNECTION_TIMEOUT);
} else if (newState === 'connected') {
this.subscribeConnection?.handleSelectedIceCandidatePairs();
const callback = this.callbacks.get(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
this.callbacks.delete(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID);
if (callback) {
callback.promise.resolve(true);
}
}
},
onIceCandidate: candidate => {
this.connectivityListener?.onICECandidate(candidate, false);
},
onSelectedCandidatePairChange: candidatePair => {
this.connectivityListener?.onSelectedICECandidatePairChange(candidatePair, false);
},
};
if (!this.publishConnection) {
this.publishConnection = new HMSPublishConnection(
this.signal,
this.initConfig.rtcConfiguration,
publishConnectionObserver,
);
}
if (!this.subscribeConnection) {
this.subscribeConnection = new HMSSubscribeConnection(
this.signal,
this.initConfig.rtcConfiguration,
this.isFlagEnabled.bind(this),
subscribeConnectionObserver,
);
}
}
this.webrtcInternals?.setPeerConnections({
publish: this.publishConnection?.nativeConnection,
subscribe: this.subscribeConnection?.nativeConnection,
});
}
private async negotiateJoinWithRetry({
name,
data,
autoSubscribeVideo,
isWebRTC = true,
}: NegotiateJoinParams & { isWebRTC: boolean }) {
try {
await this.negotiateJoin({ name, data, autoSubscribeVideo, isWebRTC });
} catch (error) {
HMSLogger.e(TAG, 'Join negotiation failed ❌', error);
const hmsError =
error instanceof HMSException
? error
: ErrorFactory.WebsocketMethodErrors.ServerErrors(
500,
HMSAction.JOIN,
`Websocket join error - ${(error as Error).message}`,
);
const shouldRetry =
parseInt(`${hmsError.code / 100}`) === 5 ||
[ErrorCodes.WebSocketConnectionErrors.WEBSOCKET_CONNECTION_LOST, 429].includes(hmsError.code);
if (hmsError.code === 410) {
hmsError.isTerminal = true;
}
if (shouldRetry) {
this.joinRetryCount = 0;
hmsError.isTerminal = false;
const task = async () => {
this.joinRetryCount++;
return await this.negotiateJoin({ name, data, autoSubscribeVideo, isWebRTC });
};
await this.retryScheduler.schedule({
category: TransportFailureCategory.JoinWSMessageFailed,
error: hmsError,
task,
originalState: TransportState.Joined,
changeState: false,
});
} else {
throw error;
}
}
}
private async negotiateJoin({
name,
data,
autoSubscribeVideo,
isWebRTC = true,
}: NegotiateJoinParams & { isWebRTC: boolean }): Promise<boolean> {
if (isWebRTC) {
return await this.negotiateJoinWebRTC({ name, data, autoSubscribeVideo });
} else {
return await this.negotiateJoinNonWebRTC({ name, data, autoSubscribeVideo });
}
}
private async negotiateJoinWebRTC({ name, data, autoSubscribeVideo }: NegotiateJoinParams): Promise<boolean> {
HMSLogger.d(TAG, '⏳ join: Negotiating over PUBLISH connection');
if (!this.publishConnection) {
HMSLogger.e(TAG, 'Publish peer connection not found, cannot negotiate');
return false;
}
const offer = await this.publishConnection.createOffer();
await this.publishConnection.setLocalDescription(offer);
const serverSubDegrade = this.isFlagEnabled(InitFlags.FLAG_SERVER_SUB_DEGRADATION);
const simulcast = this.isFlagEnabled(InitFlags.FLAG_SERVER_SIMULCAST);
const onDemandTracks = this.isFlagEnabled(InitFlags.FLAG_ON_DEMAND_TRACKS);
const answer = await this.signal.join(
name,
data,
!autoSubscribeVideo,
serverSubDegrade,
simulcast,
onDemandTracks,
offer,
);
this.setSFUNodeId(answer?.sfu_node_id);
await this.publishConnection.setRemoteDescription(answer);
for (const candidate of this.publishConnection.candidates) {
await this.publishConnection.addIceCandidate(candidate);
}
this.publishConnection.initAfterJoin();
return !!answer;
}
private async negotiateJoinNonWebRTC({ name, data, autoSubscribeVideo }: NegotiateJoinParams): Promise<boolean> {
HMSLogger.d(TAG, '⏳ join: Negotiating Non-WebRTC');
const serverSubDegrade = this.isFlagEnabled(InitFlags.FLAG_SERVER_SUB_DEGRADATION);
const simulcast = this.isFlagEnabled(InitFlags.FLAG_SERVER_SIMULCAST);
const onDemandTracks = this.isFlagEnabled(InitFlags.FLAG_ON_DEMAND_TRACKS);
const response = await this.signal.join(
name,
data,
!autoSubscribeVideo,
serverSubDegrade,
simulcast,
onDemandTracks,
);
this.setSFUNodeId(response?.sfu_node_id);
return !!response;
}
/**
* Negotiate on first publish after changing role from non-webrtc peer to webrtc peer by sending offer
*/
private async negotiateOnFirstPublish() {
HMSLogger.d(TAG, '⏳ Negotiating offer over PUBLISH connection');
if (!this.publishConnection) {
HMSLogger.e(TAG, 'Publish peer connection not found, cannot negotiate');
return false;
}
try {
const offer = await this.publishConnection.createOffer(this.trackStates);
await this.publishConnection.setLocalDescription(offer);
const answer = await this.signal.offer(offer, this.trackStates);
await this.publishConnection.setRemoteDescription(answer);
for (const candidate of this.publishConnection.candidates) {
await this.publishConnection.addIceCandidate(candidate);
}
this.publishConnection.initAfterJoin();
return !!answer;
} catch (ex) {
// resolve for now as this might happen during migration
if (ex instanceof HMSException && ex.code === 421) {
return true;
}
throw ex;
}
}
private async performPublishRenegotiation(constraints?: RTCOfferOptions) {
HMSLogger.d(TAG, `⏳ [role=PUBLISH] onRenegotiationNeeded START`, this.trackStates);
const callback = this.callbacks.get(RENEGOTIATION_CALLBACK_ID);
if (!callback) {
HMSLogger.w(TAG, 'no callback found for renegotiation');
return;
}
if (!this.publishConnection) {
HMSLogger.e(TAG, 'Publish peer connection not found, cannot renegotiate');
return;
}
try {
const offer = await this.publishConnection.createOffer(this.trackStates, constraints);
await this.publishConnection.setLocalDescription(offer);
HMSLogger.time(`renegotiation-offer-exchange`);
const answer = await this.signal.offer(offer, this.trackStates);
this.callbacks.delete(RENEGOTIATION_CALLBACK_ID);
HMSLogger.timeEnd(`renegotiation-offer-exchange`);
await this.publishConnection.setRemoteDescription(answer);
callback.promise.resolve(true);
HMSLogger.d(TAG, `[role=PUBLISH] onRenegotiationNeeded DONE ✅`);
} catch (err) {
let ex: HMSException;
if (err instanceof HMSException) {
ex = err;
} else {
ex = ErrorFactory.GenericErrors.Unknown(HMSAction.PUBLISH, (err as Error).message);
}
// resolve for now as this might happen during migration
if (ex.code === 421) {
callback.promise.resolve(true);
} else {
callback.promise.reject(ex);
}
HMSLogger.d(TAG, `[role=PUBLISH] onRenegotiationNeeded FAILED ❌`);
}
}
private async handleIceConnectionFailure(role: HMSConnectionRole, error: HMSException) {
// ice retry is already in progress(from disconnect state)
if (
this.retryScheduler.isTaskInProgress(
HMSConnectionRole.Publish
? TransportFailureCategory.PublishIceConnectionFailed
: TransportFailureCategory.SubscribeIceConnectionFailed,
)
) {
return;
}
if (role === HMSConnectionRole.Publish) {
this.retryScheduler.schedule({
category: TransportFailureCategory.PublishIceConnectionFailed,
error,
task: this.retryPublishIceFailedTask,
originalState: TransportState.Joined,
});
} else {
this.retryScheduler.schedule({
category: TransportFailureCategory.SubscribeIceConnectionFailed,
error,
task: this.retrySubscribeIceFailedTask,
originalState: TransportState.Joined,
});
}
}
private async internalConnect(token: string, initEndpoint: string, peerId: string, iceServers?: HMSICEServer[]) {
HMSLogger.d(TAG, 'connect: started ⏰');
const connectRequestedAt = new Date();
try {
this.analyticsTimer.start(TimedEvent.INIT);
this.initConfig = await InitService.fetchInitConfig({
token,
peerId,
userAgent: this.store.getUserAgent(),
initEndpoint,
iceServers,
});
this.connectivityListener?.onInitSuccess(this.initConfig.endpoint);
const room = this.store.getRoom();
if (room) {
room.effectsKey = this.initConfig.config.vb?.effectsKey;
room.isEffectsEnabled = this.isFlagEnabled(InitFlags.FLAG_EFFECTS_SDK_ENABLED);
room.disableNoneLayerRequest = this.isFlagEnabled(InitFlags.FLAG_DISABLE_NONE_LAYER_REQUEST);
room.isVBEnabled = this.isFlagEnabled(InitFlags.FLAG_VB_ENABLED);
room.isHipaaEnabled = this.isFlagEnabled(InitFlags.FLAG_HIPAA_ENABLED);
room.isNoiseCancellationEnabled = this.isFlagEnabled(InitFlags.FLAG_NOISE_CANCELLATION);
}
this.analyticsTimer.end(TimedEvent.INIT);
HTTPAnalyticsTransport.setWebsocketEndpoint(this.initConfig.endpoint);
// if leave was called while init was going on, don't open websocket
this.validateNotDisconnected('post init');
await this.openSignal(token, peerId);
this.observer.onConnected();
this.connectivityListener?.onSignallingSuccess();
this.store.setSimulcastEnabled(this.isFlagEnabled(InitFlags.FLAG_SERVER_SIMULCAST));
HMSLogger.d(TAG, 'Adding Analytics Transport: JsonRpcSignal');
this.analyticsEventsService.setTransport(this.analyticsSignalTransport);
this.analyticsEventsService.flush();
return this.initConfig;
} catch (error) {
if (this.state !== TransportState.Reconnecting) {
this.eventBus.analytics.publish(
AnalyticsEventFactory.connect(
error as Error,
this.getAdditionalAnalyticsProperties(),
connectRequestedAt,
new Date(),
initEndpoint,
),
);
}
HMSLogger.e(TAG, '❌ internal connect: failed', error);
throw error;
}
}
// leave could be called between any two async tasks, which would make
// the state disconnected instead of connecting, throw error for those cases.
private validateNotDisconnected(stage: string) {
if (this.state === TransportState.Disconnected) {
HMSLogger.w(TAG, 'aborting join as transport state is disconnected');
throw ErrorFactory.GenericErrors.ValidationFailed(`leave called before join could complete - stage=${stage}`);
}
}
private async openSignal(token: string, peerId: string) {
if (!this.initConfig) {
throw ErrorFactory.APIErrors.InitConfigNotAvailable(HMSAction.INIT, 'Init Config not found');
}
HMSLogger.d(TAG, '⏳ internal connect: connecting to ws endpoint', this.initConfig.endpoint);
const url = new URL(this.initConfig.endpoint);
url.searchParams.set('peer', peerId);
url.searchParams.set('token', token);
url.searchParams.set('user_agent_v2', this.store.getUserAgent());
url.searchParams.set('protocol_version', PROTOCOL_VERSION);
url.searchParams.set('protocol_spec', PROTOCOL_SPEC);
this.endpoint = url.toString();
this.analyticsTimer.start(TimedEvent.WEBSOCKET_CONNECT);
await this.signal.open(this.endpoint);
this.analyticsTimer.end(TimedEvent.WEBSOCKET_CONNECT);
this.analyticsTimer.start(TimedEvent.ON_POLICY_CHANGE);
this.analyticsTimer.start(TimedEvent.ROOM_STATE);
HMSLogger.d(TAG, '✅ internal connect: connected to ws endpoint');
}
private initStatsAnalytics() {
if (this.isFlagEnabled(InitFlags.FLAG_PUBLISH_STATS)) {
this.publishStatsAnalytics = new PublishStatsAnalytics(
this.store,
this.eventBus,
this.getValueFromInitConfig('publishStats', 'maxSampleWindowSize', PUBLISH_STATS_SAMPLE_WINDOW),
this.getValueFromInitConfig('publishStats', 'maxSamplePushInterval', PUBLISH_STATS_PUSH_INTERVAL),
);
this.getWebrtcInternals()?.start();
}
if (this.isFlagEnabled(InitFlags.FLAG_SUBSCRIBE_STATS)) {
this.subscribeStatsAnalytics = new SubscribeStatsAnalytics(
this.store,
this.eventBus,
this.getValueFromInitConfig('subscribeStats', 'maxSampleWindowSize', SUBSCRIBE_STATS_SAMPLE_WINDOW),
this.getValueFromInitConfig('subscribeStats', 'maxSamplePushInterval', SUBSCRIBE_STATS_PUSH_INTERVAL),
);
this.getWebrtcInternals()?.start();
}
}
private getValueFromInitConfig(
baseKey: 'publishStats' | 'subscribeStats',
subKey: 'maxSampleWindowSize' | 'maxSamplePushInterval',
defaultValue: number,
) {
return this.initConfig?.config[baseKey]?.[subKey] || defaultValue;
}
/**
* Role does not need WebRTC(peer connections to communicate to SFU) if it cannot publish or subscribe to anything
* @returns boolean denoting if a peer cannot publish(video, audio or screen) and cannot subscribe to any role
*/
private doesRoleNeedWebRTC(role: HMSRole) {
if (!this.isFlagEnabled(InitFlags.FLAG_NON_WEBRTC_DISABLE_OFFER)) {
return true;
}
const isPublishing = Boolean(role.publishParams.allowed && role.publishParams.allowed?.length > 0);
const isSubscribing = Boolean(
role.subscribeParams.subscribeToRoles && role.subscribeParams.subscribeToRoles?.length > 0,
);
return isPublishing || isSubscribing;
}
private doesLocalPeerNeedWebRTC() {
const localRole = this.store.getLocalPeer()?.role;
if (!localRole) {
return true;
}
return this.doesRoleNeedWebRTC(localRole);
}
private retryPublishIceFailedTask = async () => {
/**
* Proceed with the retry even if the connection state is connected as the offer could have failed
* which will cause missing tiles if it is not sent again.
* Do iceRestart only if not connected
*/
if (this.publishConnection) {
const p = new Promise<boolean>((resolve, reject) => {
this.callbacks.set(RENEGOTIATION_CALLBACK_ID, {
promise: { resolve, reject },
action: HMSAction.RESTART_ICE,
extra: {},
});
});
await this.performPublishRenegotiation({ iceRestart: this.publishConnection.connectionState !== 'connected' });
await p;
}
return true;
};
private retrySubscribeIceFailedTask = async () => {
if (this.subscribeConnection && this.subscribeConnection.connectionState !== 'connected') {
const p = new Promise<boolean>((resolve, reject) => {
// Use subscribe constant string
this.callbacks.set(SUBSCRIBE_ICE_CONNECTION_CALLBACK_ID, {
promise: { resolve, reject },
action: HMSAction.RESTART_ICE,
extra: {},
});
});
const timeout = new Promise(resolve => {
setTimeout(resolve, SUBSCRIBE_TIMEOUT, false);
});
return Promise.race([p, timeout]) as Promise<boolean>;
}
return true;
};
private retrySignalDisconnectTask = async () => {
HMSLogger.d(TAG, 'retrySignalDisconnectTask', { signalConnected: this.signal.isConnected });
// Check if ws is disconnected - otherwise if only publishIce fails
// and ws connect is success then we don't need to reconnect to WebSocket
if (!this.signal.isConnected) {
await this.internalConnect(
this.joinParameters!.authToken,
this.joinParameters!.endpoint,
this.joinParameters!.peerId,
this.joinParameters!.iceServers,
);
}
// Only retry publish failed task after joining the call - not needed in preview signal reconnect
const ok = this.store.getRoom()?.joinedAt
? this.signal.isConnected && (await this.retryPublishIceFailedTask())
: this.signal.isConnected;
// Send track update to sync local track state changes during reconnection
this.signal.trackUpdate(this.trackStates);
return ok;
};
private setTransportStateForConnect() {
if (this.state === TransportState.Failed) {
this.state = TransportState.Disconnected;
}
if (this.state !== TransportState.Disconnected && this.state !== TransportState.Reconnecting) {
throw ErrorFactory.WebsocketMethodErrors.AlreadyJoined(
HMSAction.JOIN,
`Cannot join a meeting in ${this.state} state`,
);
}
if (this.state === TransportState.Disconnected) {
this.state = TransportState.Connecting;
this.observer.onStateChange(this.state);
}
}
private sendErrorAnalyticsEvent(error: HMSException, category: TransportFailureCategory) {
const additionalProps = this.getAdditionalAnalyticsProperties();
let event: AnalyticsEvent;
switch (category) {
case TransportFailureCategory.ConnectFailed:
event = AnalyticsEventFactory.connect(error, additionalProps);
break;
case TransportFailureCategory.SignalDisconnect:
event = AnalyticsEventFactory.disconnect(error, additionalProps);
break;
case TransportFailureCategory.JoinWS