livekit-client
Version:
JavaScript/TypeScript client SDK for LiveKit
1,527 lines (1,336 loc) • 65.8 kB
text/typescript
import { Mutex } from '@livekit/mutex';
import {
type AddTrackRequest,
ClientConfigSetting,
ClientConfiguration,
type ConnectionQualityUpdate,
DataChannelInfo,
DataChannelReceiveState,
DataPacket,
DataPacket_Kind,
DataTrackSubscriberHandles,
DisconnectReason,
EncryptedPacket,
EncryptedPacketPayload,
Encryption_Type,
type JoinResponse,
type LeaveRequest,
LeaveRequest_Action,
MediaSectionsRequirement,
ParticipantInfo,
PublishDataTrackResponse,
ReconnectReason,
type ReconnectResponse,
RequestResponse,
Room as RoomModel,
RoomMovedResponse,
RpcAck,
RpcResponse,
SessionDescription,
SignalTarget,
SpeakerInfo,
type StreamStateUpdate,
SubscribedQualityUpdate,
type SubscriptionPermissionUpdate,
type SubscriptionResponse,
SyncState,
TrackInfo,
type TrackPublishedResponse,
TrackUnpublishedResponse,
Transcription,
UnpublishDataTrackResponse,
UpdateSubscription,
type UserPacket,
} from '@livekit/protocol';
import { EventEmitter } from 'events';
import type { MediaAttributes } from 'sdp-transform';
import type TypedEventEmitter from 'typed-emitter';
import type { SignalOptions } from '../api/SignalClient';
import {
SignalClient,
SignalConnectionState,
toProtoSessionDescription,
} from '../api/SignalClient';
import type { BaseE2EEManager } from '../e2ee/E2eeManager';
import { asEncryptablePacket } from '../e2ee/utils';
import log, { LoggerNames, getLogger } from '../logger';
import type { InternalRoomOptions } from '../options';
import TypedPromise from '../utils/TypedPromise';
import { DataPacketBuffer } from '../utils/dataPacketBuffer';
import { TTLMap } from '../utils/ttlmap';
import PCTransport, { PCEvents } from './PCTransport';
import { PCTransportManager, PCTransportState } from './PCTransportManager';
import type { ReconnectContext, ReconnectPolicy } from './ReconnectPolicy';
import { DEFAULT_MAX_AGE_MS, type RegionUrlProvider } from './RegionUrlProvider';
import { DataTrackInfo } from './data-track/types';
import { roomConnectOptionDefaults } from './defaults';
import {
ConnectionError,
ConnectionErrorReason,
NegotiationError,
SignalReconnectError,
TrackInvalidError,
UnexpectedConnectionState,
} from './errors';
import { EngineEvent } from './events';
import { RpcError } from './rpc';
import CriticalTimers from './timers';
import type LocalTrack from './track/LocalTrack';
import type LocalTrackPublication from './track/LocalTrackPublication';
import LocalVideoTrack from './track/LocalVideoTrack';
import type { SimulcastTrackInfo } from './track/LocalVideoTrack';
import type RemoteTrackPublication from './track/RemoteTrackPublication';
import type { Track } from './track/Track';
import type { TrackPublishOptions, VideoCodec } from './track/options';
import { getTrackPublicationInfo } from './track/utils';
import type { LoggerOptions } from './types';
import {
isCompressionStreamSupported,
isVideoCodec,
isVideoTrack,
isWeb,
sleep,
supportsAddTrack,
supportsTransceiver,
toHttpUrl,
} from './utils';
const lossyDataChannel = '_lossy';
const reliableDataChannel = '_reliable';
const dataTrackDataChannel = '_data_track';
const minReconnectWait = 2 * 1000;
const leaveReconnect = 'leave-reconnect';
const reliabeReceiveStateTTL = 30_000;
const lossyDataChannelBufferThresholdMin = 8 * 1024;
const lossyDataChannelBufferThresholdMax = 256 * 1024;
const initialMediaSectionsAudio = 3;
const initialMediaSectionsVideo = 3;
enum PCState {
New,
Connected,
Disconnected,
Reconnecting,
Closed,
}
export enum DataChannelKind {
RELIABLE = DataPacket_Kind.RELIABLE,
LOSSY = DataPacket_Kind.LOSSY,
DATA_TRACK_LOSSY,
}
/** @internal */
export default class RTCEngine extends (EventEmitter as new () => TypedEventEmitter<EngineEventCallbacks>) {
client: SignalClient;
rtcConfig: RTCConfiguration = {};
peerConnectionTimeout: number = roomConnectOptionDefaults.peerConnectionTimeout;
fullReconnectOnNext: boolean = false;
pcManager?: PCTransportManager;
/**
* @internal
*/
latestJoinResponse?: JoinResponse;
/**
* @internal
*/
latestRemoteOfferId: number = 0;
/** @internal */
e2eeManager: BaseE2EEManager | undefined;
get isClosed() {
return this._isClosed;
}
get isNewlyCreated() {
return this._isNewlyCreated;
}
get pendingReconnect() {
return !!this.reconnectTimeout;
}
private lossyDC?: RTCDataChannel;
// @ts-ignore noUnusedLocals
private lossyDCSub?: RTCDataChannel;
private reliableDC?: RTCDataChannel;
// @ts-ignore noUnusedLocals
private reliableDCSub?: RTCDataChannel;
private dataTrackDC?: RTCDataChannel;
// @ts-ignore noUnusedLocals
private dataTrackDCSub?: RTCDataChannel;
private dcBufferStatus: Map<DataChannelKind, boolean>;
private subscriberPrimary: boolean = false;
private pcState: PCState = PCState.New;
private _isClosed: boolean = true;
private _isNewlyCreated: boolean = true;
private pendingTrackResolvers: {
[key: string]: { resolve: (info: TrackInfo) => void; reject: () => void };
} = {};
// keep join info around for reconnect, this could be a region url
private url?: string;
private token?: string;
private signalOpts?: SignalOptions;
private reconnectAttempts: number = 0;
private reconnectStart: number = 0;
private clientConfiguration?: ClientConfiguration;
private attemptingReconnect: boolean = false;
private reconnectPolicy: ReconnectPolicy;
private reconnectTimeout?: ReturnType<typeof setTimeout>;
private participantSid?: string;
/** keeps track of how often an initial join connection has been tried */
private joinAttempts: number = 0;
/** specifies how often an initial join connection is allowed to retry */
private maxJoinAttempts: number = 1;
private closingLock: Mutex;
private dataProcessLock: Mutex;
private shouldFailNext: boolean = false;
private regionUrlProvider?: RegionUrlProvider;
private log = log;
private loggerOptions: LoggerOptions;
private publisherConnectionPromise: Promise<void> | undefined;
private reliableDataSequence: number = 1;
private reliableMessageBuffer = new DataPacketBuffer();
private reliableReceivedState: TTLMap<string, number> = new TTLMap(reliabeReceiveStateTTL);
private lossyDataStatCurrentBytes: number = 0;
private lossyDataStatByterate: number = 0;
private lossyDataStatInterval: ReturnType<typeof setInterval> | undefined;
private lossyDataDropCount: number = 0;
private midToTrackId: { [key: string]: string } = {};
/** used to indicate whether the browser is currently waiting to reconnect */
private isWaitingForNetworkReconnect: boolean = false;
constructor(private options: InternalRoomOptions) {
super();
this.log = getLogger(options.loggerName ?? LoggerNames.Engine);
this.loggerOptions = {
loggerName: options.loggerName,
loggerContextCb: () => this.logContext,
};
this.client = new SignalClient(undefined, this.loggerOptions);
this.client.signalLatency = this.options.expSignalLatency;
this.reconnectPolicy = this.options.reconnectPolicy;
this.closingLock = new Mutex();
this.dataProcessLock = new Mutex();
this.dcBufferStatus = new Map([
[DataChannelKind.RELIABLE, true],
[DataChannelKind.LOSSY, true],
[DataChannelKind.DATA_TRACK_LOSSY, true],
]);
this.client.onParticipantUpdate = (updates) =>
this.emit(EngineEvent.ParticipantUpdate, updates);
this.client.onConnectionQuality = (update) =>
this.emit(EngineEvent.ConnectionQualityUpdate, update);
this.client.onRoomUpdate = (update) => this.emit(EngineEvent.RoomUpdate, update);
this.client.onSubscriptionError = (resp) => this.emit(EngineEvent.SubscriptionError, resp);
this.client.onSubscriptionPermissionUpdate = (update) =>
this.emit(EngineEvent.SubscriptionPermissionUpdate, update);
this.client.onSpeakersChanged = (update) => this.emit(EngineEvent.SpeakersChanged, update);
this.client.onStreamStateUpdate = (update) => this.emit(EngineEvent.StreamStateChanged, update);
this.client.onRequestResponse = (response) =>
this.emit(EngineEvent.SignalRequestResponse, response);
this.client.onParticipantUpdate = (updates) =>
this.emit(EngineEvent.ParticipantUpdate, updates);
this.client.onJoined = (joinResponse) => this.emit(EngineEvent.Joined, joinResponse);
}
/** @internal */
get logContext() {
return {
room: this.latestJoinResponse?.room?.name,
roomID: this.latestJoinResponse?.room?.sid,
participant: this.latestJoinResponse?.participant?.identity,
participantID: this.participantSid,
};
}
async join(
url: string,
token: string,
opts: SignalOptions,
abortSignal?: AbortSignal,
/** setting this to true results in dual peer connection mode being used */
useV0Path: boolean = false,
): Promise<JoinResponse> {
this._isNewlyCreated = false;
this.url = url;
this.token = token;
this.signalOpts = opts;
this.maxJoinAttempts = opts.maxRetries;
try {
this.joinAttempts += 1;
this.setupSignalClientCallbacks();
let offerProto: SessionDescription | undefined;
if (!useV0Path && isCompressionStreamSupported()) {
if (!this.pcManager) {
await this.configure();
this.createDataChannels();
this.addMediaSections(initialMediaSectionsAudio, initialMediaSectionsVideo);
}
const offer = await this.pcManager?.publisher.createInitialOffer();
if (offer) {
offerProto = toProtoSessionDescription(offer.offer, offer.offerId);
}
}
if (abortSignal?.aborted) {
throw ConnectionError.cancelled('Connection aborted');
}
const joinResponse = await this.client.join(
url,
token,
opts,
abortSignal,
useV0Path,
offerProto,
);
this._isClosed = false;
this.latestJoinResponse = joinResponse;
this.participantSid = joinResponse.participant?.sid;
this.subscriberPrimary = joinResponse.subscriberPrimary;
if (!useV0Path && isCompressionStreamSupported()) {
this.pcManager?.updateConfiguration(this.makeRTCConfiguration(joinResponse));
} else {
if (!this.pcManager) {
await this.configure(joinResponse, !useV0Path);
}
// create offer
if (!this.subscriberPrimary || joinResponse.fastPublish) {
this.negotiate().catch((err) => {
log.error(err, this.logContext);
});
}
}
this.registerOnLineListener();
this.clientConfiguration = joinResponse.clientConfiguration;
this.emit(EngineEvent.SignalConnected, joinResponse);
return joinResponse;
} catch (e) {
if (e instanceof ConnectionError) {
if (e.reason === ConnectionErrorReason.ServerUnreachable) {
this.log.warn(
`Couldn't connect to server, attempt ${this.joinAttempts} of ${this.maxJoinAttempts}`,
this.logContext,
);
if (this.joinAttempts < this.maxJoinAttempts) {
return this.join(url, token, opts, abortSignal, useV0Path);
}
} else if (e.reason === ConnectionErrorReason.ServiceNotFound) {
this.log.warn(`Initial connection failed: ${e.message} – Retrying`);
return this.join(url, token, opts, abortSignal, true);
}
}
throw e;
}
}
async close() {
const unlock = await this.closingLock.lock();
if (this.isClosed) {
unlock();
return;
}
try {
this._isClosed = true;
this.joinAttempts = 0;
this.emit(EngineEvent.Closing);
this.removeAllListeners();
this.deregisterOnLineListener();
this.clearPendingReconnect();
this.cleanupLossyDataStats();
await this.cleanupPeerConnections();
await this.cleanupClient();
} finally {
unlock();
}
}
async cleanupPeerConnections() {
await this.pcManager?.close();
this.pcManager = undefined;
const dcCleanup = (dc: RTCDataChannel | undefined) => {
if (!dc) return;
dc.close();
dc.onbufferedamountlow = null;
dc.onclose = null;
dc.onclosing = null;
dc.onerror = null;
dc.onmessage = null;
dc.onopen = null;
};
dcCleanup(this.lossyDC);
dcCleanup(this.lossyDCSub);
dcCleanup(this.reliableDC);
dcCleanup(this.reliableDCSub);
dcCleanup(this.dataTrackDC);
dcCleanup(this.dataTrackDCSub);
this.lossyDC = undefined;
this.lossyDCSub = undefined;
this.reliableDC = undefined;
this.reliableDCSub = undefined;
this.dataTrackDC = undefined;
this.dataTrackDCSub = undefined;
this.reliableMessageBuffer = new DataPacketBuffer();
this.reliableDataSequence = 1;
this.reliableReceivedState.clear();
}
cleanupLossyDataStats() {
this.lossyDataStatByterate = 0;
this.lossyDataStatCurrentBytes = 0;
if (this.lossyDataStatInterval) {
clearInterval(this.lossyDataStatInterval);
this.lossyDataStatInterval = undefined;
}
this.lossyDataDropCount = 0;
}
async cleanupClient() {
await this.client.close();
this.client.resetCallbacks();
}
addTrack(req: AddTrackRequest): Promise<TrackInfo> {
if (this.pendingTrackResolvers[req.cid]) {
throw new TrackInvalidError('a track with the same ID has already been published');
}
return new Promise<TrackInfo>((resolve, reject) => {
const publicationTimeout = setTimeout(() => {
delete this.pendingTrackResolvers[req.cid];
reject(
ConnectionError.timeout('publication of local track timed out, no response from server'),
);
}, 10_000);
this.pendingTrackResolvers[req.cid] = {
resolve: (info: TrackInfo) => {
clearTimeout(publicationTimeout);
resolve(info);
},
reject: () => {
clearTimeout(publicationTimeout);
reject(new Error('Cancelled publication by calling unpublish'));
},
};
this.client.sendAddTrack(req);
});
}
/**
* Removes sender from PeerConnection, returning true if it was removed successfully
* and a negotiation is necessary
* @param sender
* @returns
*/
removeTrack(sender: RTCRtpSender): boolean {
if (sender.track && this.pendingTrackResolvers[sender.track.id]) {
const { reject } = this.pendingTrackResolvers[sender.track.id];
if (reject) {
reject();
}
delete this.pendingTrackResolvers[sender.track.id];
}
try {
this.pcManager!.removeTrack(sender);
return true;
} catch (e: unknown) {
this.log.warn('failed to remove track', { ...this.logContext, error: e });
}
return false;
}
updateMuteStatus(trackSid: string, muted: boolean) {
this.client.sendMuteTrack(trackSid, muted);
}
get dataSubscriberReadyState(): string | undefined {
return this.reliableDCSub?.readyState;
}
async getConnectedServerAddress(): Promise<string | undefined> {
return this.pcManager?.getConnectedAddress();
}
/* @internal */
setRegionUrlProvider(provider: RegionUrlProvider) {
this.regionUrlProvider = provider;
}
private async configure(joinResponse?: JoinResponse, useSinglePeerConnection?: boolean) {
// already configured
if (this.pcManager && this.pcManager.currentState !== PCTransportState.NEW) {
return;
}
if (!joinResponse) {
const rtcConfig = this.makeRTCConfiguration();
this.pcManager = new PCTransportManager('publisher-only', this.loggerOptions, rtcConfig);
} else {
this.participantSid = joinResponse.participant?.sid;
const rtcConfig = this.makeRTCConfiguration(joinResponse);
this.pcManager = new PCTransportManager(
useSinglePeerConnection
? 'publisher-only'
: joinResponse.subscriberPrimary
? 'subscriber-primary'
: 'publisher-primary',
this.loggerOptions,
rtcConfig,
);
}
this.emit(EngineEvent.TransportsCreated, this.pcManager.publisher, this.pcManager.subscriber);
this.pcManager.onIceCandidate = (candidate, target) => {
this.client.sendIceCandidate(candidate, target);
};
this.pcManager.onPublisherOffer = (offer, offerId) => {
this.client.sendOffer(offer, offerId);
};
this.pcManager.onDataChannel = this.handleDataChannel;
this.pcManager.onStateChange = async (connectionState, publisherState, subscriberState) => {
this.log.debug(`primary PC state changed ${connectionState}`, this.logContext);
if (['closed', 'disconnected', 'failed'].includes(publisherState)) {
// reset publisher connection promise
this.publisherConnectionPromise = undefined;
}
if (connectionState === PCTransportState.CONNECTED) {
const shouldEmit = this.pcState === PCState.New;
this.pcState = PCState.Connected;
if (shouldEmit) {
this.emit(EngineEvent.Connected, this.latestJoinResponse!);
}
} else if (connectionState === PCTransportState.FAILED) {
// on Safari, PeerConnection will switch to 'disconnected' during renegotiation
if (this.pcState === PCState.Connected || this.pcState === PCState.Reconnecting) {
this.pcState = PCState.Disconnected;
this.handleDisconnect(
'peerconnection failed',
subscriberState === 'failed'
? ReconnectReason.RR_SUBSCRIBER_FAILED
: ReconnectReason.RR_PUBLISHER_FAILED,
);
}
}
// detect cases where both signal client and peer connection are severed and assume that user has lost network connection
const isSignalSevered =
this.client.isDisconnected ||
this.client.currentState === SignalConnectionState.RECONNECTING;
const isPCSevered = [
PCTransportState.FAILED,
PCTransportState.CLOSING,
PCTransportState.CLOSED,
].includes(connectionState);
if (isSignalSevered && isPCSevered && !this._isClosed) {
this.emit(EngineEvent.Offline);
}
};
this.pcManager.onTrack = (ev: RTCTrackEvent) => {
// this fires after the underlying transceiver is stopped and potentially
// peer connection closed, so do not bubble up if there are no streams
if (ev.streams.length === 0) return;
this.emit(EngineEvent.MediaTrackAdded, ev.track, ev.streams[0], ev.receiver);
};
}
private setupSignalClientCallbacks() {
// configure signaling client
this.client.onAnswer = async (sd, offerId, midToTrackId) => {
if (!this.pcManager) {
return;
}
this.log.debug('received server answer', {
...this.logContext,
RTCSdpType: sd.type,
sdp: sd.sdp,
midToTrackId,
});
this.midToTrackId = midToTrackId;
await this.pcManager.setPublisherAnswer(sd, offerId);
};
// add candidate on trickle
this.client.onTrickle = (candidate, target) => {
if (!this.pcManager) {
return;
}
this.log.debug('got ICE candidate from peer', { ...this.logContext, candidate, target });
this.pcManager.addIceCandidate(candidate, target);
};
// when server creates an offer for the client
this.client.onOffer = async (sd, offerId, midToTrackId) => {
this.latestRemoteOfferId = offerId;
if (!this.pcManager) {
return;
}
this.midToTrackId = midToTrackId;
const answer = await this.pcManager.createSubscriberAnswerFromOffer(sd, offerId);
if (answer) {
this.client.sendAnswer(answer, offerId);
}
};
this.client.onLocalTrackPublished = (res: TrackPublishedResponse) => {
this.log.debug('received trackPublishedResponse', {
...this.logContext,
cid: res.cid,
track: res.track?.sid,
});
if (!this.pendingTrackResolvers[res.cid]) {
this.log.error(`missing track resolver for ${res.cid}`, {
...this.logContext,
cid: res.cid,
});
return;
}
const { resolve } = this.pendingTrackResolvers[res.cid];
delete this.pendingTrackResolvers[res.cid];
resolve(res.track!);
};
this.client.onLocalTrackUnpublished = (response: TrackUnpublishedResponse) => {
this.emit(EngineEvent.LocalTrackUnpublished, response);
};
this.client.onLocalTrackSubscribed = (trackSid: string) => {
this.emit(EngineEvent.LocalTrackSubscribed, trackSid);
};
this.client.onTokenRefresh = (token: string) => {
this.token = token;
this.regionUrlProvider?.updateToken(token);
};
this.client.onRemoteMuteChanged = (trackSid: string, muted: boolean) => {
this.emit(EngineEvent.RemoteMute, trackSid, muted);
};
this.client.onSubscribedQualityUpdate = (update: SubscribedQualityUpdate) => {
this.emit(EngineEvent.SubscribedQualityUpdate, update);
};
this.client.onRoomMoved = (res: RoomMovedResponse) => {
this.participantSid = res.participant?.sid;
if (this.latestJoinResponse) {
this.latestJoinResponse.room = res.room;
}
this.emit(EngineEvent.RoomMoved, res);
};
this.client.onMediaSectionsRequirement = (requirement: MediaSectionsRequirement) => {
this.addMediaSections(requirement.numAudios, requirement.numVideos);
this.negotiate();
};
this.client.onPublishDataTrackResponse = (event: PublishDataTrackResponse) => {
this.emit(EngineEvent.PublishDataTrackResponse, event);
};
this.client.onUnPublishDataTrackResponse = (event: UnpublishDataTrackResponse) => {
this.emit(EngineEvent.UnPublishDataTrackResponse, event);
};
this.client.onDataTrackSubscriberHandles = (event: DataTrackSubscriberHandles) => {
this.emit(EngineEvent.DataTrackSubscriberHandles, event);
};
this.client.onClose = () => {
this.handleDisconnect('signal', ReconnectReason.RR_SIGNAL_DISCONNECTED);
};
this.client.onLeave = (leave: LeaveRequest) => {
this.log.debug('client leave request', { ...this.logContext, reason: leave?.reason });
if (leave.regions && this.regionUrlProvider) {
this.log.debug('updating regions', this.logContext);
this.regionUrlProvider.setServerReportedRegions({
updatedAtInMs: Date.now(),
maxAgeInMs: DEFAULT_MAX_AGE_MS,
regionSettings: leave.regions,
});
}
switch (leave.action) {
case LeaveRequest_Action.DISCONNECT:
this.emit(EngineEvent.Disconnected, leave?.reason);
this.close();
break;
case LeaveRequest_Action.RECONNECT:
this.fullReconnectOnNext = true;
// reconnect immediately instead of waiting for next attempt
this.handleDisconnect(leaveReconnect);
break;
case LeaveRequest_Action.RESUME:
// reconnect immediately instead of waiting for next attempt
this.handleDisconnect(leaveReconnect);
default:
break;
}
};
}
private makeRTCConfiguration(
serverResponse?: JoinResponse | ReconnectResponse,
): RTCConfiguration {
const rtcConfig = { ...this.rtcConfig };
if (this.signalOpts?.e2eeEnabled) {
this.log.debug('E2EE - setting up transports with insertable streams', this.logContext);
// this makes sure that no data is sent before the transforms are ready
// @ts-ignore
rtcConfig.encodedInsertableStreams = true;
}
// @ts-ignore
rtcConfig.sdpSemantics = 'unified-plan';
// @ts-ignore
rtcConfig.continualGatheringPolicy = 'gather_continually';
if (!serverResponse) {
return rtcConfig;
}
// update ICE servers before creating PeerConnection
if (serverResponse.iceServers && !rtcConfig.iceServers) {
const rtcIceServers: RTCIceServer[] = [];
serverResponse.iceServers.forEach((iceServer) => {
const rtcIceServer: RTCIceServer = {
urls: iceServer.urls,
};
if (iceServer.username) rtcIceServer.username = iceServer.username;
if (iceServer.credential) {
rtcIceServer.credential = iceServer.credential;
}
rtcIceServers.push(rtcIceServer);
});
rtcConfig.iceServers = rtcIceServers;
}
if (
serverResponse.clientConfiguration &&
serverResponse.clientConfiguration.forceRelay === ClientConfigSetting.ENABLED
) {
rtcConfig.iceTransportPolicy = 'relay';
}
return rtcConfig;
}
private addMediaSections(numAudios: number, numVideos: number) {
const transceiverInit: RTCRtpTransceiverInit = { direction: 'recvonly' };
for (let i: number = 0; i < numAudios; i++) {
this.pcManager?.addPublisherTransceiverOfKind('audio', transceiverInit);
}
for (let i: number = 0; i < numVideos; i++) {
this.pcManager?.addPublisherTransceiverOfKind('video', transceiverInit);
}
}
private createDataChannels() {
if (!this.pcManager) {
return;
}
// clear old data channel callbacks if recreate
if (this.lossyDC) {
this.lossyDC.onmessage = null;
this.lossyDC.onerror = null;
}
if (this.reliableDC) {
this.reliableDC.onmessage = null;
this.reliableDC.onerror = null;
}
if (this.dataTrackDC) {
this.dataTrackDC.onmessage = null;
this.dataTrackDC.onerror = null;
}
// create data channels
this.lossyDC = this.pcManager.createPublisherDataChannel(lossyDataChannel, {
ordered: false,
maxRetransmits: 0,
});
this.reliableDC = this.pcManager.createPublisherDataChannel(reliableDataChannel, {
ordered: true,
});
this.dataTrackDC = this.pcManager.createPublisherDataChannel(dataTrackDataChannel, {
ordered: false,
maxRetransmits: 0,
});
// also handle messages over the pub channel, for backwards compatibility
this.lossyDC.onmessage = this.handleDataMessage;
this.reliableDC.onmessage = this.handleDataMessage;
this.dataTrackDC.onmessage = this.handleDataTrackMessage;
// handle datachannel errors
this.lossyDC.onerror = this.handleDataError;
this.reliableDC.onerror = this.handleDataError;
this.dataTrackDC.onerror = this.handleDataError;
// set up dc buffer threshold, set to 64kB (otherwise 0 by default)
this.lossyDC.bufferedAmountLowThreshold = 65535;
this.reliableDC.bufferedAmountLowThreshold = 65535;
this.dataTrackDC.bufferedAmountLowThreshold = 65535;
// handle buffer amount low events
this.lossyDC.onbufferedamountlow = () => this.handleBufferedAmountLow(DataChannelKind.LOSSY);
this.reliableDC.onbufferedamountlow = () =>
this.handleBufferedAmountLow(DataChannelKind.RELIABLE);
this.dataTrackDC.onbufferedamountlow = () =>
this.handleBufferedAmountLow(DataChannelKind.DATA_TRACK_LOSSY);
this.cleanupLossyDataStats();
this.lossyDataStatInterval = setInterval(() => {
this.lossyDataStatByterate = this.lossyDataStatCurrentBytes;
this.lossyDataStatCurrentBytes = 0;
const dc = this.dataChannelForKind(DataChannelKind.LOSSY);
if (dc) {
// control buffered latency to ~100ms
const threshold = this.lossyDataStatByterate / 10;
dc.bufferedAmountLowThreshold = Math.min(
Math.max(threshold, lossyDataChannelBufferThresholdMin),
lossyDataChannelBufferThresholdMax,
);
}
}, 1000);
}
private handleDataChannel = async ({ channel }: RTCDataChannelEvent) => {
if (!channel) {
return;
}
let handler;
if (channel.label === reliableDataChannel) {
this.reliableDCSub = channel;
handler = this.handleDataMessage;
} else if (channel.label === lossyDataChannel) {
this.lossyDCSub = channel;
handler = this.handleDataMessage;
} else if (channel.label === dataTrackDataChannel) {
this.dataTrackDCSub = channel;
handler = this.handleDataTrackMessage;
} else {
return;
}
this.log.debug(`on data channel ${channel.id}, ${channel.label}`, this.logContext);
channel.onmessage = handler;
};
private handleDataMessage = async (message: MessageEvent) => {
// make sure to respect incoming data message order by processing message events one after the other
const unlock = await this.dataProcessLock.lock();
try {
// decode
let buffer: ArrayBuffer | undefined;
if (message.data instanceof ArrayBuffer) {
buffer = message.data;
} else if (message.data instanceof Blob) {
buffer = await message.data.arrayBuffer();
} else {
this.log.error('unsupported data type', { ...this.logContext, data: message.data });
return;
}
const dp = DataPacket.fromBinary(new Uint8Array(buffer));
if (dp.sequence > 0 && dp.participantSid !== '') {
const lastSeq = this.reliableReceivedState.get(dp.participantSid);
if (lastSeq && dp.sequence <= lastSeq) {
// ignore duplicate or out-of-order packets in reliable channel
return;
}
this.reliableReceivedState.set(dp.participantSid, dp.sequence);
}
if (dp.value?.case === 'speaker') {
// dispatch speaker updates
this.emit(EngineEvent.ActiveSpeakersUpdate, dp.value.value.speakers);
} else if (dp.value?.case === 'encryptedPacket') {
if (!this.e2eeManager) {
this.log.error('Received encrypted packet but E2EE not set up', this.logContext);
return;
}
const decryptedData = await this.e2eeManager?.handleEncryptedData(
dp.value.value.encryptedValue,
dp.value.value.iv,
dp.participantIdentity,
dp.value.value.keyIndex,
);
const decryptedPacket = EncryptedPacketPayload.fromBinary(decryptedData.payload);
const newDp = new DataPacket({
value: decryptedPacket.value,
participantIdentity: dp.participantIdentity,
participantSid: dp.participantSid,
});
if (newDp.value?.case === 'user') {
// compatibility
applyUserDataCompat(newDp, newDp.value.value);
}
this.emit(EngineEvent.DataPacketReceived, newDp, dp.value.value.encryptionType);
} else {
if (dp.value?.case === 'user') {
// compatibility
applyUserDataCompat(dp, dp.value.value);
}
this.emit(EngineEvent.DataPacketReceived, dp, Encryption_Type.NONE);
}
} finally {
unlock();
}
};
private handleDataTrackMessage = async (message: MessageEvent) => {
// Decode / normalize into a common format
let buffer: ArrayBuffer | undefined;
if (message.data instanceof ArrayBuffer) {
buffer = message.data;
} else if (message.data instanceof Blob) {
buffer = await message.data.arrayBuffer();
} else {
this.log.error('unsupported data type', { ...this.logContext, data: message.data });
return;
}
this.emit('dataTrackPacketReceived', new Uint8Array(buffer));
};
private handleDataError = (event: Event) => {
const channel = event.currentTarget as RTCDataChannel;
const channelKind = channel.maxRetransmits === 0 ? 'lossy' : 'reliable';
if (event instanceof ErrorEvent && event.error) {
const { error } = event.error;
this.log.error(`DataChannel error on ${channelKind}: ${event.message}`, {
...this.logContext,
error,
});
} else {
this.log.error(`Unknown DataChannel error on ${channelKind}`, { ...this.logContext, event });
}
};
private handleBufferedAmountLow = (channelKind: DataChannelKind) => {
this.updateAndEmitDCBufferStatus(channelKind);
};
async createSender(
track: LocalTrack,
opts: TrackPublishOptions,
encodings?: RTCRtpEncodingParameters[],
) {
if (supportsTransceiver()) {
const sender = await this.createTransceiverRTCRtpSender(track, opts, encodings);
return sender;
}
if (supportsAddTrack()) {
this.log.warn('using add-track fallback', this.logContext);
const sender = await this.createRTCRtpSender(track.mediaStreamTrack);
return sender;
}
throw new UnexpectedConnectionState('Required webRTC APIs not supported on this device');
}
async createSimulcastSender(
track: LocalVideoTrack,
simulcastTrack: SimulcastTrackInfo,
opts: TrackPublishOptions,
encodings?: RTCRtpEncodingParameters[],
) {
// store RTCRtpSender
if (supportsTransceiver()) {
return this.createSimulcastTransceiverSender(track, simulcastTrack, opts, encodings);
}
if (supportsAddTrack()) {
this.log.debug('using add-track fallback', this.logContext);
return this.createRTCRtpSender(track.mediaStreamTrack);
}
throw new UnexpectedConnectionState('Cannot stream on this device');
}
private async createTransceiverRTCRtpSender(
track: LocalTrack,
opts: TrackPublishOptions,
encodings?: RTCRtpEncodingParameters[],
) {
if (!this.pcManager) {
throw new UnexpectedConnectionState('publisher is closed');
}
const streams: MediaStream[] = [];
if (track.mediaStream) {
streams.push(track.mediaStream);
}
if (isVideoTrack(track)) {
track.codec = opts.videoCodec;
}
const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly', streams };
if (encodings) {
transceiverInit.sendEncodings = encodings;
}
// addTransceiver for react-native is async. web is synchronous, but await won't effect it.
const transceiver = await this.pcManager.addPublisherTransceiver(
track.mediaStreamTrack,
transceiverInit,
);
return transceiver.sender;
}
private async createSimulcastTransceiverSender(
track: LocalVideoTrack,
simulcastTrack: SimulcastTrackInfo,
opts: TrackPublishOptions,
encodings?: RTCRtpEncodingParameters[],
) {
if (!this.pcManager) {
throw new UnexpectedConnectionState('publisher is closed');
}
const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
if (encodings) {
transceiverInit.sendEncodings = encodings;
}
// addTransceiver for react-native is async. web is synchronous, but await won't effect it.
const transceiver = await this.pcManager.addPublisherTransceiver(
simulcastTrack.mediaStreamTrack,
transceiverInit,
);
if (!opts.videoCodec) {
return;
}
track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
return transceiver.sender;
}
private async createRTCRtpSender(track: MediaStreamTrack) {
if (!this.pcManager) {
throw new UnexpectedConnectionState('publisher is closed');
}
return this.pcManager.addPublisherTrack(track);
}
// websocket reconnect behavior. if websocket is interrupted, and the PeerConnection
// continues to work, we can reconnect to websocket to continue the session
// after a number of retries, we'll close and give up permanently
private handleDisconnect = (connection: string, disconnectReason?: ReconnectReason) => {
if (this._isClosed) {
return;
}
this.log.warn(`${connection} disconnected`, this.logContext);
if (this.reconnectAttempts === 0) {
// only reset start time on the first try
this.reconnectStart = Date.now();
}
const disconnect = (duration: number) => {
this.log.warn(
`could not recover connection after ${this.reconnectAttempts} attempts, ${duration}ms. giving up`,
this.logContext,
);
this.emit(EngineEvent.Disconnected);
this.close();
};
const duration = Date.now() - this.reconnectStart;
let delay = this.getNextRetryDelay({
elapsedMs: duration,
retryCount: this.reconnectAttempts,
});
if (delay === null) {
disconnect(duration);
return;
}
if (connection === leaveReconnect) {
delay = 0;
}
this.log.debug(`reconnecting in ${delay}ms`, this.logContext);
this.clearReconnectTimeout();
if (this.token && this.regionUrlProvider) {
// token may have been refreshed, we do not want to recreate the regionUrlProvider
// since the current engine may have inherited a regional url
this.regionUrlProvider.updateToken(this.token);
}
this.reconnectTimeout = CriticalTimers.setTimeout(
() =>
this.attemptReconnect(disconnectReason).finally(() => (this.reconnectTimeout = undefined)),
delay,
);
};
private async attemptReconnect(reason?: ReconnectReason) {
if (this._isClosed) {
return;
}
// guard for attempting reconnection multiple times while one attempt is still not finished
if (this.attemptingReconnect) {
log.warn('already attempting reconnect, returning early', this.logContext);
return;
}
if (
this.clientConfiguration?.resumeConnection === ClientConfigSetting.DISABLED ||
// signaling state could change to closed due to hardware sleep
// those connections cannot be resumed
(this.pcManager?.currentState ?? PCTransportState.NEW) === PCTransportState.NEW
) {
this.fullReconnectOnNext = true;
}
try {
this.attemptingReconnect = true;
if (this.fullReconnectOnNext) {
await this.restartConnection();
} else {
await this.resumeConnection(reason);
}
this.clearPendingReconnect();
this.fullReconnectOnNext = false;
} catch (e) {
this.reconnectAttempts += 1;
let recoverable = true;
if (e instanceof UnexpectedConnectionState) {
this.log.debug('received unrecoverable error', { ...this.logContext, error: e });
// unrecoverable
recoverable = false;
} else if (!(e instanceof SignalReconnectError)) {
// cannot resume
this.fullReconnectOnNext = true;
}
if (recoverable) {
this.handleDisconnect('reconnect', ReconnectReason.RR_UNKNOWN);
} else {
this.log.info(
`could not recover connection after ${this.reconnectAttempts} attempts, ${
Date.now() - this.reconnectStart
}ms. giving up`,
this.logContext,
);
this.emit(EngineEvent.Disconnected);
await this.close();
}
} finally {
this.attemptingReconnect = false;
}
}
private getNextRetryDelay(context: ReconnectContext) {
try {
return this.reconnectPolicy.nextRetryDelayInMs(context);
} catch (e) {
this.log.warn('encountered error in reconnect policy', { ...this.logContext, error: e });
}
// error in user code with provided reconnect policy, stop reconnecting
return null;
}
private async restartConnection(regionUrl?: string) {
try {
if (!this.url || !this.token) {
// permanent failure, don't attempt reconnection
throw new UnexpectedConnectionState('could not reconnect, url or token not saved');
}
this.log.info(`reconnecting, attempt: ${this.reconnectAttempts}`, this.logContext);
this.emit(EngineEvent.Restarting);
if (!this.client.isDisconnected) {
await this.client.sendLeave();
}
await this.cleanupPeerConnections();
await this.cleanupClient();
let joinResponse: JoinResponse;
try {
if (!this.signalOpts) {
this.log.warn(
'attempted connection restart, without signal options present',
this.logContext,
);
throw new SignalReconnectError();
}
// in case a regionUrl is passed, the region URL takes precedence
joinResponse = await this.join(
regionUrl ?? this.url,
this.token,
this.signalOpts,
undefined,
!this.options.singlePeerConnection,
);
} catch (e) {
if (e instanceof ConnectionError && e.reason === ConnectionErrorReason.NotAllowed) {
throw new UnexpectedConnectionState('could not reconnect, token might be expired');
}
throw new SignalReconnectError();
}
if (this.shouldFailNext) {
this.shouldFailNext = false;
throw new Error('simulated failure');
}
this.client.setReconnected();
this.emit(EngineEvent.SignalRestarted, joinResponse);
await this.waitForPCReconnected();
// re-check signal connection state before setting engine as resumed
if (this.client.currentState !== SignalConnectionState.CONNECTED) {
throw new SignalReconnectError('Signal connection got severed during reconnect');
}
this.regionUrlProvider?.resetAttempts();
// reconnect success
this.emit(EngineEvent.Restarted);
} catch (error) {
const nextRegionUrl = await this.regionUrlProvider?.getNextBestRegionUrl();
if (nextRegionUrl) {
await this.restartConnection(nextRegionUrl);
return;
} else {
// no more regions to try (or we're not on cloud)
this.regionUrlProvider?.resetAttempts();
throw error;
}
}
}
private async resumeConnection(reason?: ReconnectReason): Promise<void> {
if (!this.url || !this.token) {
// permanent failure, don't attempt reconnection
throw new UnexpectedConnectionState('could not reconnect, url or token not saved');
}
// trigger publisher reconnect
if (!this.pcManager) {
throw new UnexpectedConnectionState('publisher and subscriber connections unset');
}
this.log.info(`resuming signal connection, attempt ${this.reconnectAttempts}`, this.logContext);
this.emit(EngineEvent.Resuming);
let res: ReconnectResponse | undefined;
try {
this.setupSignalClientCallbacks();
res = await this.client.reconnect(this.url, this.token, this.participantSid, reason);
} catch (error) {
let message = '';
if (error instanceof Error) {
message = error.message;
this.log.error(error.message, { ...this.logContext, error });
}
if (error instanceof ConnectionError && error.reason === ConnectionErrorReason.NotAllowed) {
throw new UnexpectedConnectionState('could not reconnect, token might be expired');
}
if (error instanceof ConnectionError && error.reason === ConnectionErrorReason.LeaveRequest) {
throw error;
}
throw new SignalReconnectError(message);
}
this.emit(EngineEvent.SignalResumed);
if (res) {
const rtcConfig = this.makeRTCConfiguration(res);
this.pcManager.updateConfiguration(rtcConfig);
if (this.latestJoinResponse) {
this.latestJoinResponse.serverInfo = res.serverInfo;
}
} else {
this.log.warn('Did not receive reconnect response', this.logContext);
}
if (this.shouldFailNext) {
this.shouldFailNext = false;
throw new Error('simulated failure');
}
await this.pcManager.triggerIceRestart();
await this.waitForPCReconnected();
// re-check signal connection state before setting engine as resumed
if (this.client.currentState !== SignalConnectionState.CONNECTED) {
throw new SignalReconnectError('Signal connection got severed during reconnect');
}
this.client.setReconnected();
// recreate publish datachannel if it's id is null
// (for safari https://bugs.webkit.org/show_bug.cgi?id=184688)
if (this.reliableDC?.readyState === 'open' && this.reliableDC.id === null) {
this.createDataChannels();
}
if (res?.lastMessageSeq) {
this.resendReliableMessagesForResume(res.lastMessageSeq);
}
// resume success
this.emit(EngineEvent.Resumed);
}
async waitForPCInitialConnection(timeout?: number, abortController?: AbortController) {
if (!this.pcManager) {
throw new UnexpectedConnectionState('PC manager is closed');
}
await this.pcManager.ensurePCTransportConnection(abortController, timeout);
}
private async waitForPCReconnected() {
this.pcState = PCState.Reconnecting;
this.log.debug('waiting for peer connection to reconnect', this.logContext);
try {
await sleep(minReconnectWait); // FIXME setTimeout again not ideal for a connection critical path
if (!this.pcManager) {
throw new UnexpectedConnectionState('PC manager is closed');
}
await this.pcManager.ensurePCTransportConnection(undefined, this.peerConnectionTimeout);
this.pcState = PCState.Connected;
} catch (e: any) {
// TODO do we need a `failed` state here for the PC?
this.pcState = PCState.Disconnected;
throw ConnectionError.internal(`could not establish PC connection, ${e.message}`);
}
}
waitForRestarted = () => {
return new Promise<void>((resolve, reject) => {
if (this.pcState === PCState.Connected) {
resolve();
}
const onRestarted = () => {
this.off(EngineEvent.Disconnected, onDisconnected);
resolve();
};
const onDisconnected = () => {
this.off(EngineEvent.Restarted, onRestarted);
reject();
};
this.once(EngineEvent.Restarted, onRestarted);
this.once(EngineEvent.Disconnected, onDisconnected);
});
};
/** @internal */
async publishRpcResponse(
destinationIdentity: string,
requestId: string,
payload: string | null,
error: RpcError | null,
) {
const packet = new DataPacket({
destinationIdentities: [destinationIdentity],
kind: DataPacket_Kind.RELIABLE,
value: {
case: 'rpcResponse',
value: new RpcResponse({
requestId,
value: error
? { case: 'error', value: error.toProto() }
: { case: 'payload', value: payload ?? '' },
}),
},
});
await this.sendDataPacket(packet, DataChannelKind.RELIABLE);
}
/** @internal */
async publishRpcAck(destinationIdentity: string, requestId: string) {
const packet = new DataPacket({
destinationIdentities: [destinationIdentity],
kind: DataPacket_Kind.RELIABLE,
value: {
case: 'rpcAck',
value: new RpcAck({
requestId,
}),
},
});
await this.sendDataPacket(packet, DataChannelKind.RELIABLE);
}
/* @internal */
async sendDataPacket(packet: DataPacket, kind: DataChannelKind) {
// make sure we do have a data connection
await this.ensurePublisherConnected(kind);
if (this.e2eeManager && this.e2eeManager.isDataChannelEncryptionEnabled) {
const encryptablePacket = asEncryptablePacket(packet);
if (encryptablePacket) {
const encryptedData = await this.e2eeManager.encryptData(encryptablePacket.toBinary());
packet.value = {
case: 'encryptedPacket',
value: new EncryptedPacket({
encryptedValue: encryptedData.payload,
iv: encryptedData.iv,
keyIndex: encryptedData.keyIndex,
}),
};
}
}
if (kind === DataChannelKind.RELIABLE) {
packet.sequence = this.reliableDataSequence;
this.reliableDataSequence += 1;
}
const msg = packet.toBinary();
switch (kind) {
case DataChannelKind.LOSSY:
case DataChannelKind.DATA_TRACK_LOSSY:
return this.sendLossyBytes(msg, kind);
case DataChannelKind.RELIABLE:
const dc = this.dataChannelForKind(kind);
if (dc) {
await this.waitForBufferStatusLow(kind);
this.reliableMessageBuffer.push({ data: msg, sequence: packet.sequence });
if (this.attemptingReconnect) {
return;
}
dc.send(msg);
}
this.updateAndEmitDCBufferStatus(kind);
break;
}
}
/* @internal */
async sendLossyBytes(
bytes: Uint8Array,
kind: Exclude<DataChannelKind, DataChannelKind.RELIABLE>,
bufferStatusLowBehavior: 'drop' | 'wait' = 'drop',
) {
// make sure we do have a data connection
await this.ensurePublisherConnected(kind);
const dc = this.dataChannelForKind(kind);
if (dc) {
if (!this.isBufferStatusLow(kind)) {
// Depending on the exact circumstance that data is being sent, either drop or wait for the
// buffer status to not be low before continuing.
switch (bufferStatusLowBehavior) {
case 'wait':
await this.waitForBufferStatusLow(kind);
break;
case 'drop':
// this.log.warn(`dropping lossy data channel message`, this.logContext);
// Drop messages to reduce latency
this.lossyDataDropCount += 1;
if (this.lossyDataDropCount % 100 === 0) {
this.log.warn(
`dropping lossy data channel messages, total dropped: ${this.lossyDataDropCount}`,
this.logContext,
);
}
return;
}
}
this.lossyDataStatCurrentBytes += bytes.byteLength;
if (this.attemptingReconnect) {
return;
}
dc.send(bytes);
}
this.updateAndEmitDCBufferStatus(kind);
}
private async resendReliableMessagesForResume(lastMessageSeq: number) {
await this.ensurePublisherConnected(DataChannelKind.RELIABLE);
const dc = this.dataChannelForKind(DataChannelKind.RELIABLE);
if (dc) {
this.reliableMessageBuffer.popToSequence(lastMessageSeq);
this.reliableMessageBuffer.getAll().forEach((msg) => {
dc.send(msg.data);
});
}
this.updateAndEmitDCBufferStatus(DataChannelKind.RELIABLE);
}
private updateAndEmitDCBufferStatus = (kind: DataChannelKind) => {
if (kind === DataChannelKind.RELIABLE) {
const dc = this.dataChannelForKind(kind);
if (dc) {
this.reliableMessageBuffer.alignBufferedAmount(dc.bufferedAmount);
}
}
const status = this.isBuf