@ceeblue/webrtc-client
Version:
Ceeblue WebRTC Client
1,236 lines (1,231 loc) • 123 kB
JavaScript
import*as utils from'@ceeblue/web-utils';import {EventEmitter,Util,WebSocketReliable,Connect,NetAddress,Loggable,Numbers}from'@ceeblue/web-utils';export{utils };import*as sdpTransform from'sdp-transform';/******************************************************************************
Copyright (c) Microsoft Corporation.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
***************************************************************************** */
/* global Reflect, Promise, SuppressedError, Symbol */
function __awaiter(thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
}
typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
var e = new Error(message);
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
};/**
* Copyright 2023 Ceeblue B.V.
* This file is part of https://github.com/CeeblueTV/webrtc-client which is released under GNU Affero General Public License.
* See file LICENSE or go to https://spdx.org/licenses/AGPL-3.0-or-later.html for full license details.
*/
/**
* List the media type in their string version as received from server
*/
var MType;
(function (MType) {
MType["AUDIO"] = "audio";
MType["VIDEO"] = "video";
MType["DATA"] = "data";
})(MType || (MType = {}));
function filter(tracks, medias, codecs) {
for (let i = 0; i < medias.length; ++i) {
const media = medias[i];
if (!codecs.has(media.codec.toLowerCase())) {
tracks.delete(media.idx);
medias.splice(i--, 1);
}
}
}
/**
* Metadata representation
*/
class Metadata {
constructor() {
this.type = '';
/**
* Width resolution size
*/
this.width = 0;
/**
* Height resolution size
*/
this.height = 0;
/**
* Sources available to play the stream
*/
this.sources = new Map(); // url <=> source
/**
* Tracks sorted by descending BPS
*/
this.tracks = new Map();
/**
* Audio tracks sorted by descending BPS
*/
this.audios = [];
/**
* Video tracks sorted by descending BPS
*/
this.videos = [];
/**
* Data track
*/
this.datas = [];
}
/**
* Return a new metadata subset with only track with a codec supported
* @param codecs codecs supported
* @returns the subset of metadata
*/
subset(codecs) {
const metadata = Object.assign({}, this);
if (codecs) {
filter(metadata.tracks, metadata.audios, codecs);
filter(metadata.tracks, metadata.videos, codecs);
// Fix UP/DOWN
for (const [, track] of metadata.tracks) {
while (track.up && !metadata.tracks.has(track.up.idx)) {
track.up = track.up.up;
}
while (track.down && !metadata.tracks.has(track.down.idx)) {
track.down = track.down.down;
}
}
}
return metadata;
}
}/**
* Copyright 2023 Ceeblue B.V.
* This file is part of https://github.com/CeeblueTV/webrtc-client which is released under GNU Affero General Public License.
* See file LICENSE or go to https://spdx.org/licenses/AGPL-3.0-or-later.html for full license details.
*/
const sortByMAXBPS = (track1, track2) => track2.maxbps - track1.maxbps;
var StreamState;
(function (StreamState) {
StreamState["UNKNOWN"] = "";
StreamState["ONLINE"] = "Stream is online";
StreamState["OFFLINE"] = "Stream is offline";
StreamState["INITIALIZING"] = "Stream is initializing";
StreamState["BOOTING"] = "Stream is booting";
StreamState["WAITING"] = "Stream is waiting for data";
})(StreamState || (StreamState = {}));
/**
* Use StreamMetadata to get real-time information on a server stream, including:
* - the list of tracks and their properties,
* - the list of availables sources and their properties,
* @example
* const streamMetadata = new StreamMetadata(Connect.buildURL(endPoint, streamName));
* streamMetadata.onMetadata = metadata => {
* console.log(metadata);
* }
*
*/
class StreamMetadata extends EventEmitter {
/**
* Event fired when stream state is changing
* @param state
*/
onState(state) {
this.log('onState', state).info();
}
/**
* Event fired when the stream is closed
* @param error error description on an improper closure
* @event
*/
onClose(error) {
this.log('onClose').info();
}
/**
* Event fired when metadata is present in the stream
* @param metadata
* @event
*/
onMetadata(metadata) {
this.log(Util.stringify(metadata)).info();
}
/**
* URL of the connection
*/
get url() {
return this._ws.url;
}
/**
* State of the stream as indicated by the server
*/
get streamState() {
return this._streamState;
}
/**
* Returns the {@link Connect.Params} object containing the connection parameters
*/
get connectParams() {
return this._connectParams;
}
/**
* Returns the {@link Metadata} object description
*/
get metadata() {
return this._metadata;
}
/**
* Returns true if the connection is closed
*/
get closed() {
return this._ws.closed;
}
/**
* Create a new StreamMetadata instance, connects to the server using WebSocket and
* listen to metadata events.
*/
constructor(connectParams) {
super();
const states = new Map();
for (const state of Object.values(StreamState)) {
states.set(state, state);
}
// Server can server the following text to indicate a unknown status
states.set('Stream status is unknown?!', StreamState.UNKNOWN);
this._connectParams = connectParams;
this._streamState = StreamState.UNKNOWN;
this._ws = new WebSocketReliable(Connect.buildURL(Connect.Type.META, connectParams));
this._ws.onClose = (error) => this.close(error);
this._ws.onMessage = (message) => {
var _a, _b;
try {
const data = JSON.parse(message);
if (data.error) {
const state = states.get(data.error);
if (state) {
this.onState((this._streamState = state));
}
else {
// Unrecoverable issue!
this.close({
type: 'StreamMetadataError',
name: data.error,
stream: (_a = connectParams.streamName) !== null && _a !== void 0 ? _a : ''
});
}
return;
}
// Metadata
this._metadata = new Metadata();
this._metadata.type = data.type;
this._metadata.width = data.width;
this._metadata.height = data.height;
this._metadata.sources.clear();
for (const source of data.source || []) {
this._metadata.sources.set(source.hrn, source);
}
const tracks = [];
this._metadata.tracks.clear();
if ((_b = data.meta) === null || _b === void 0 ? void 0 : _b.tracks) {
for (const [name, track] of Util.iterableEntries(data.meta.tracks)) {
track.name = name;
track.type = track.type.toLowerCase();
switch (track.type) {
case 'audio':
this._metadata.audios.push(track);
continue;
case 'video':
this._metadata.videos.push(track);
continue;
case 'meta':
track.type = MType.DATA; // Fix meta string to explicit DATA type
this._metadata.datas.push(track);
break;
default:
this.log(`Unknown track type ${track.type}`).warn();
}
tracks.push(track);
}
}
// Sorts audios/videos by descending BPS
this._addSortedTrack(this._metadata.audios, this._metadata.tracks);
this._addSortedTrack(this._metadata.videos, this._metadata.tracks);
for (const track of tracks) {
this._metadata.tracks.set(track.idx, track);
}
// SUCCESS
this.onState((this._streamState = StreamState.ONLINE));
}
catch (e) {
this.log(Util.stringify(e)).error();
return;
}
// Announce metadata change
this.onMetadata(this._metadata);
};
}
/**
* Close the stream metadata channel
* @param error error description on an improper closure
*/
close(error) {
if (this._ws.onClose === Util.EMPTY_FUNCTION) {
return;
}
this._ws.onClose = Util.EMPTY_FUNCTION;
this._ws.close();
this._metadata = new Metadata(); // reset metadata
this.onClose(error);
}
_addSortedTrack(medias, tracks) {
medias.sort(sortByMAXBPS);
for (let i = 0; i < medias.length; ++i) {
const media = medias[i];
tracks.set(media.idx, media); // sorted by descending BPS!
if (i) {
media.up = medias[i - 1];
medias[i - 1].down = media;
}
}
}
}/**
* Copyright 2023 Ceeblue B.V.
* This file is part of https://github.com/CeeblueTV/webrtc-client which is released under GNU Affero General Public License.
* See file LICENSE or go to https://spdx.org/licenses/AGPL-3.0-or-later.html for full license details.
*/
class PlayerStats extends utils.PlayerStats {
constructor() {
super();
// States used for incremental stats computation
this._prevTime = 0;
this._prevAudioBytes = 0;
this._prevVideoBytes = 0;
this._prevVideoEmittedCount = 0;
this._prevAudioEmittedCount = 0;
this._prevVideoJitterDelay = 0;
this._prevAudioJitterDelay = 0;
this._prevSkippedAudio = 0;
this._prevAudioConcealedSamples = 0;
this._prevSkippedVideo = 0;
this._prevVideoDroppedFrames = 0;
this._prevVideoTime = 0;
this._prevRealTime = 0;
this.protocol = 'WebRTC';
}
/**
* @override{@inheritDoc IStats.onRelease}
* @event
*/
onRelease() { }
/**
* @returns a JSON representation of the player stats, which is the object itself in this case
*/
serialize() {
return __awaiter(this, void 0, void 0, function* () {
return this;
});
}
/**
* Computes and updates all player statistics based on the current connection infos, metadata, and playback state.
* Updates the internal properties of this class including those inherited from {@link utils.PlayerStats}.
* @param infos ConnectionInfos: WebRTC connection and input stats.
* @param metadata Metadata: Stream metadata and track info.
* @param currentTime number: Current playback time (media time) in seconds.
* @param audioTrackId number (optional): Selected audio track ID.
* @param videoTrackId number (optional): Selected video track ID.
*/
compute(infos, metadata, currentTime, audioTrackId, videoTrackId) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l;
const audioIn = (_a = infos.inputs) === null || _a === void 0 ? void 0 : _a.audio;
const videoIn = (_b = infos.inputs) === null || _b === void 0 ? void 0 : _b.video;
// videoTrackId
this.videoTrackId = videoTrackId;
const videoTrack = videoTrackId != null ? metadata.tracks.get(videoTrackId) : undefined;
// audioTrackId
this.audioTrackId = audioTrackId;
const audioTrack = audioTrackId != null ? metadata.tracks.get(audioTrackId) : undefined;
// recvByteRate
let now = performance.now();
const deltaTime = Math.max(1, now - this._prevTime);
this._prevTime = now;
// audioByteRate
const audioBytes = audioIn === null || audioIn === void 0 ? void 0 : audioIn.bytesReceived;
if (audioBytes != null) {
this.audioByteRate = Math.max(0, audioBytes - this._prevAudioBytes) / deltaTime;
this._prevAudioBytes = audioBytes;
}
else {
this.audioByteRate = undefined;
}
// videoByteRate
const videoBytes = videoIn === null || videoIn === void 0 ? void 0 : videoIn.bytesReceived;
if (videoBytes != null) {
this.videoByteRate = Math.max(0, videoBytes - this._prevVideoBytes) / deltaTime;
this._prevVideoBytes = videoBytes;
}
else {
this.videoByteRate = undefined;
}
// bufferAmount, computed as the max of audio/video jitter
const videoJitterDelay = videoIn === null || videoIn === void 0 ? void 0 : videoIn.jitterBufferDelay;
const videoEmittedCount = videoIn === null || videoIn === void 0 ? void 0 : videoIn.jitterBufferEmittedCount;
let videoBuffering;
if (videoJitterDelay != null && videoEmittedCount != null) {
if (videoEmittedCount > this._prevVideoEmittedCount) {
videoBuffering =
(1000 * Math.max(0, videoJitterDelay - this._prevVideoJitterDelay)) /
(videoEmittedCount - this._prevVideoEmittedCount);
}
this._prevVideoEmittedCount = videoEmittedCount;
this._prevVideoJitterDelay = videoJitterDelay;
}
const audioJitterDelay = audioIn === null || audioIn === void 0 ? void 0 : audioIn.jitterBufferDelay;
const audioEmittedCount = audioIn === null || audioIn === void 0 ? void 0 : audioIn.jitterBufferEmittedCount;
let audioBuffering;
if (audioJitterDelay != null && audioEmittedCount != null) {
if (audioEmittedCount > this._prevAudioEmittedCount) {
audioBuffering =
(1000 * Math.max(0, audioJitterDelay - this._prevAudioJitterDelay)) /
(audioEmittedCount - this._prevAudioEmittedCount);
}
this._prevAudioEmittedCount = audioEmittedCount;
this._prevAudioJitterDelay = audioJitterDelay;
}
if (videoBuffering != null || audioBuffering != null) {
this.bufferAmount = Math.max(videoBuffering !== null && videoBuffering !== void 0 ? videoBuffering : 0, audioBuffering !== null && audioBuffering !== void 0 ? audioBuffering : 0);
}
else {
this.bufferAmount = undefined;
}
// videoPerSecond
this.videoPerSecond = videoIn === null || videoIn === void 0 ? void 0 : videoIn.framesPerSecond;
// skippedAudio
const audioConcealedSamples = audioIn === null || audioIn === void 0 ? void 0 : audioIn.concealedSamples;
if (audioConcealedSamples != null && audioTrack && audioTrack.rate) {
const deltaConcealedSamples = Math.max(audioConcealedSamples - this._prevAudioConcealedSamples, 0);
this._prevAudioConcealedSamples = audioConcealedSamples; // in samples
this.skippedAudio = this._prevSkippedAudio + (deltaConcealedSamples / audioTrack.rate) * 1000;
this._prevSkippedAudio = this.skippedAudio; // in ms
}
else {
this.skippedAudio = undefined;
}
// skippedVideo
const videoDroppedFrames = videoIn === null || videoIn === void 0 ? void 0 : videoIn.framesDropped;
if (videoDroppedFrames != null && this.videoPerSecond) {
const deltaDroppedFrames = Math.max(videoDroppedFrames - this._prevVideoDroppedFrames, 0);
this._prevVideoDroppedFrames = videoDroppedFrames; // in frames
this.skippedVideo = this._prevSkippedVideo + (deltaDroppedFrames / this.videoPerSecond) * 1000;
this._prevSkippedVideo = this.skippedVideo; // in ms
}
else {
this.skippedVideo = undefined;
}
// stallCount
this.stallCount = videoIn === null || videoIn === void 0 ? void 0 : videoIn.freezeCount;
// audioTrackBandwidth
this.audioTrackBandwidth = (_c = audioTrack === null || audioTrack === void 0 ? void 0 : audioTrack.ebps) !== null && _c !== void 0 ? _c : audioTrack === null || audioTrack === void 0 ? void 0 : audioTrack.bps;
// videoTrackBandwidth
this.videoTrackBandwidth = (_d = videoTrack === null || videoTrack === void 0 ? void 0 : videoTrack.ebps) !== null && _d !== void 0 ? _d : videoTrack === null || videoTrack === void 0 ? void 0 : videoTrack.bps;
// playbackSpeed
now = performance.now();
const videoTime = currentTime;
let measuredSpeed = 0;
if (this._prevVideoTime !== undefined && this._prevRealTime !== undefined) {
const deltaVideoTime = videoTime - this._prevVideoTime;
const deltaRealTime = (now - this._prevRealTime) / 1000;
if (deltaVideoTime >= 0 && deltaRealTime > 0.05) {
measuredSpeed = deltaVideoTime / deltaRealTime;
}
}
this._prevVideoTime = videoTime;
this._prevRealTime = now;
this.playbackSpeed = measuredSpeed;
// rtt
this.rtt = (_e = infos === null || infos === void 0 ? void 0 : infos.candidate) === null || _e === void 0 ? void 0 : _e.currentRoundTripTime;
// jitter
if ((videoIn === null || videoIn === void 0 ? void 0 : videoIn.jitter) != null || (audioIn === null || audioIn === void 0 ? void 0 : audioIn.jitter) != null) {
this.jitter = Math.max((_f = videoIn === null || videoIn === void 0 ? void 0 : videoIn.jitter) !== null && _f !== void 0 ? _f : 0, (_g = audioIn === null || audioIn === void 0 ? void 0 : audioIn.jitter) !== null && _g !== void 0 ? _g : 0);
}
else {
this.jitter = undefined;
}
// lostPacketCount, can go down because of how packet loss is computed in WebRTC : received count - expected count, without taking duplicate into account, which can lead to negative values
if ((videoIn === null || videoIn === void 0 ? void 0 : videoIn.packetsLost) != null || (audioIn === null || audioIn === void 0 ? void 0 : audioIn.packetsLost) != null) {
this.lostPacketCount = ((_h = videoIn === null || videoIn === void 0 ? void 0 : videoIn.packetsLost) !== null && _h !== void 0 ? _h : 0) + ((_j = audioIn === null || audioIn === void 0 ? void 0 : audioIn.packetsLost) !== null && _j !== void 0 ? _j : 0);
}
else {
this.lostPacketCount = undefined;
}
// nackCount
if ((videoIn === null || videoIn === void 0 ? void 0 : videoIn.nackCount) != null || (audioIn === null || audioIn === void 0 ? void 0 : audioIn.nackCount) != null) {
this.nackCount = ((_k = videoIn === null || videoIn === void 0 ? void 0 : videoIn.nackCount) !== null && _k !== void 0 ? _k : 0) + ((_l = audioIn === null || audioIn === void 0 ? void 0 : audioIn.nackCount) !== null && _l !== void 0 ? _l : 0);
}
else {
this.nackCount = undefined;
}
}
}/**
* Copyright 2023 Ceeblue B.V.
* This file is part of https://github.com/CeeblueTV/webrtc-client which is released under GNU Affero General Public License.
* See file LICENSE or go to https://spdx.org/licenses/AGPL-3.0-or-later.html for full license details.
*/
/**
* Check if the connector is a controller
* @returns true if the connector is a controller
*/
function IsController(connector) {
return 'send' in connector;
}/**
* Copyright 2023 Ceeblue B.V.
* This file is part of https://github.com/CeeblueTV/webrtc-client which is released under GNU Affero General Public License.
* See file LICENSE or go to https://spdx.org/licenses/AGPL-3.0-or-later.html for full license details.
*/
const PEER_CONNECTION_IDLE_TIMEOUT = 15000;
const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
/**
* Set stereo=1 for the opus codecs in the sdp
* This is necessary for the opus codec to work in stereo mode
* on browsers like Safari and Chrome
*
* @param sdp the original sdp
* @returns the sdp with stereo=1 for the opus codecs
*/
function setStereoForOpus(sdp) {
const sdpObj = sdpTransform.parse(sdp);
for (const media of sdpObj.media) {
if (media.type === 'audio') {
const opusPayloads = [];
// Search for the opus codec payload ID in rtpmap
for (const rtp of media.rtp) {
if (rtp.codec === 'opus') {
opusPayloads.push(rtp.payload);
}
}
if (!opusPayloads.length) {
continue;
}
// Set stereo=1 for the opus codecs fmtp
for (const fmtp of media.fmtp) {
if (opusPayloads.includes(fmtp.payload)) {
const newConfig = sdpTransform.parseParams(fmtp.config);
newConfig.stereo = 1;
fmtp.config = '';
for (const key in newConfig) {
fmtp.config += (fmtp.config ? ';' : '') + key + '=' + newConfig[key];
}
}
}
}
}
return sdpTransform.write(sdpObj);
}
/**
* SIPConnector is a common abstract class for negotiating a new RTCPeerConnection connection
* with the server.
*
* The child class must implement the _sip method to send the offer to the server and get the answer.
*
*/
class SIPConnector extends EventEmitter {
/**
* @override{@inheritDoc IConnector.onOpen}
* @event
*/
onOpen(stream) {
this.log('onOpen').info();
}
/**
* @override{@inheritDoc IConnector.onClose}
* @event
*/
onClose(error) {
if (error) {
this.log('onClose', error).error();
}
else {
this.log('onClose').info();
}
}
/**
* @override{@inheritDoc IConnector.opened}
*/
get opened() {
return this._peerConnection && this._peerConnection.ontrack === Util.EMPTY_FUNCTION ? true : false;
}
/**
* @override{@inheritDoc IConnector.closed}
*/
get closed() {
return this._closed;
}
/**
* @override{@inheritDoc IConnector.stream}
*/
get stream() {
return this._stream;
}
/**
* @override{@inheritDoc IConnector.streamName}
*/
get streamName() {
return this._streamName;
}
/**
* @override{@inheritDoc IConnector.codecs}
*/
get codecs() {
return this._codecs;
}
/**
* Create a new SIPConnector instance. The RTCPeerConnection is created only when calling _open().
*
* By default, a listener channel is negotiated.
* To create a streamer channel, pass a stream parameter.
*/
constructor(connectParams, stream) {
var _a;
super();
this._closed = false;
this._streamName = (_a = connectParams.streamName) !== null && _a !== void 0 ? _a : '';
this._endPoint = connectParams.endPoint;
this._stream = stream;
this._connectionInfosTime = 0;
this._codecs = new Set();
}
/**
* Returns connection info, such as round trip time, requests sent and received,
* bytes sent and received, and bitrates
* NOTE: This call is resource-intensive for the CPU.
* @returns {Promise<ConnectionInfos>} A promise that resolves to an RTCStatsReport object
*/
connectionInfos() {
return __awaiter(this, arguments, void 0, function* (cacheDuration = 1000) {
if (!this._peerConnection) {
return Promise.reject('Not connected');
}
// update only evey seconds!
if (!this._connectionInfos || Util.time() - cacheDuration > this._connectionInfosTime) {
const infos = yield this._peerConnection.getStats(null);
this._connectionInfos = {
inputs: {},
outputs: {}
};
const tracks = new Map();
const rtps = new Map();
for (const info of infos.values()) {
// info.kind|mediaType
// => in a previous implementation 'kind' was named 'mediaType'
// see https://developer.mozilla.org/en-US/docs/Web/API/RTCInboundRtpStreamStats#standard_fields_included_for_all_media_types
switch (info.type) {
case 'track':
tracks.set(info.id, info);
break;
case 'outbound-rtp':
rtps.set(info.trackId, info);
this._connectionInfos.outputs[(info.kind || info.mediaType) === 'audio' ? 'audio' : 'video'] =
info;
break;
case 'inbound-rtp':
rtps.set(info.trackId, info);
this._connectionInfos.inputs[(info.kind || info.mediaType) === 'audio' ? 'audio' : 'video'] =
info;
break;
case 'candidate-pair':
if (info.selected != null) {
if (!info.selected) {
continue;
}
}
else if (info.nominated != null) {
if (!info.nominated) {
continue;
}
}
this._connectionInfos.candidate = info;
break;
}
}
// Report track infos with inbound/outbound-rtp (safari requirement!)
for (const [id, track] of tracks) {
const rtp = rtps.get(id);
if (rtp) {
Object.assign(rtp, Object.assign(Object.assign({}, track), rtp));
}
}
this._connectionInfosTime = Util.time();
}
return this._connectionInfos;
});
}
/**
* @override{@inheritDoc IConnector.close}
*/
close(error) {
if (this._closed) {
return;
} // Already closed!
this._closed = true;
this._clearPeerConnectionIdleTimeout();
const peerConnection = this._peerConnection;
if (peerConnection) {
this._peerConnection = undefined;
// Stop all tracks
peerConnection.getReceivers().forEach(receiver => receiver.track && receiver.track.stop());
peerConnection.getSenders().forEach(sender => sender.track && sender.track.stop());
// Close
peerConnection.close();
}
if (this._stream) {
this._stream.getTracks().forEach(track => track.stop());
}
this.onClose(error);
}
/**
* Main function which creates the RTCPeerConnection, creates the offer,
* calls the _sip method, then set the answer and calls onOpen
*/
_open(iceServer) {
// If iceServer is not provided, use the default one
if (!iceServer) {
const domain = new NetAddress(this._endPoint, 443).domain;
iceServer = {
urls: ['turn:' + domain + ':3478?transport=tcp', 'turn:' + domain + ':3478'],
username: 'ceeblue',
credential: 'ceeblue'
};
}
// Start the RTCPeerConnection and create an offer
try {
this._peerConnection = new RTCPeerConnection({ iceServers: [iceServer] });
}
catch (e) {
this.close({ type: 'ConnectorError', name: 'RTCPeerConnection failed', detail: Util.stringify(e) });
return;
}
this._peerConnection.onconnectionstatechange = (ev) => {
const target = ev.target;
if (target) {
const connectionState = target === null || target === void 0 ? void 0 : target['connectionState'];
this.log(`Peer connection state: ${connectionState}`).info();
switch (connectionState) {
case 'connected':
case 'connecting':
this._clearPeerConnectionIdleTimeout();
break;
case 'disconnected':
case 'failed':
this.log(`Peer connection state: ${connectionState}`).warn();
this._startPeerConnectionIdleTimeout();
break;
case 'closed':
this.log(`Peer connection state: ${connectionState}`).warn();
this.close();
break;
}
}
};
if (this._stream) {
// streamer
for (const track of this._stream.getTracks()) {
this._peerConnection.addTrack(track);
}
}
else {
// listener
this._peerConnection.ontrack = ev => {
this._stream = ev.streams[0];
this._tryToOpen();
};
}
// Add transceivers for Safari
// This is necessary for Safari to handle the audio and video tracks correctly
if (isSafari) {
if (!this._stream) {
this._peerConnection.addTransceiver('audio', { direction: 'recvonly' });
this._peerConnection.addTransceiver('video', { direction: 'recvonly' });
}
else {
const transceivers = this._peerConnection.getTransceivers();
for (const transceiver of transceivers) {
if (transceiver.receiver.track.kind === 'audio' || transceiver.receiver.track.kind === 'video') {
transceiver.direction = 'sendonly';
}
}
}
}
let sdp;
this._peerConnection
.createOffer({ offerToReceiveAudio: !this._stream, offerToReceiveVideo: !this._stream })
.then(offer => {
if (!this._peerConnection) {
return;
}
offer.sdp = sdp = offer.sdp ? setStereoForOpus(offer.sdp) : '';
this.log(`Offer\r\n${sdp}`).debug();
return this._peerConnection.setLocalDescription(offer);
})
.then(_ => {
if (!this._peerConnection) {
return;
} // has been closed!
if (!sdp) {
return Promise.reject('invalid empty sdp offer');
}
return this._sip(sdp); // Send the offer to the backend and get answer!
})
.then(answer => {
if (!answer || !this._peerConnection) {
return;
} // has been closed!
this.log(`Answer\r\n${answer}`).debug();
this.updateCodecs(answer);
return this._peerConnection.setRemoteDescription(new RTCSessionDescription({ type: 'answer', sdp: answer }));
})
.then(() => this._tryToOpen())
.catch(e => this.close({ type: 'ConnectorError', name: 'SIP failed', detail: Util.stringify(e) }));
}
/**
* Hot-swap a track on the existing RTCPeerConnection without renegotiation.
* @param kind 'audio' | 'video'
* @param track A MediaStreamTrack to send, or null to stop sending that kind.
*/
replaceTrack(kind, track) {
return __awaiter(this, void 0, void 0, function* () {
if (this._closed || !this._peerConnection) {
throw Error('Connector is closed');
}
if (!this._stream) {
throw Error('No local stream to update');
}
// Prefer transceivers: they keep the media kind visible via the receiver even when sender.track is null
let tx;
if (typeof this._peerConnection.getTransceivers === 'function') {
tx = this._peerConnection
.getTransceivers()
.find(t => t.sender &&
((t.receiver && t.receiver.track && t.receiver.track.kind === kind) ||
(t.sender.track && t.sender.track.kind === kind)));
}
let sender = tx === null || tx === void 0 ? void 0 : tx.sender;
// Fallback: sender scan (works when sender.track exists)
if (!sender) {
sender = this._peerConnection.getSenders().find(s => s.track && s.track.kind === kind);
}
if (!sender) {
// We could call addTrack() here, but that would require renegotiation.
this.close({
type: 'ConnectorError',
name: 'Replace track failed',
detail: `No existing ${kind} sender to replace; restart is required for this direction.`
});
return;
}
// Perform hot swap (no renegotiation)
yield sender.replaceTrack(track);
// Keep our public MediaStream in sync for consumers of stream
const current = this._stream;
const toRemove = kind === 'video' ? current.getVideoTracks() : current.getAudioTracks();
toRemove.forEach(t => current.removeTrack(t));
if (track) {
current.addTrack(track);
}
});
}
/**
* Fill the codecs set with the codecs found in the sdp
*
* @param sdp the sdp to parse
*/
updateCodecs(sdp) {
const sdpObj = sdpTransform.parse(sdp);
for (const media of sdpObj.media) {
for (const rtp of media.rtp) {
this._codecs.add(rtp.codec.toLowerCase());
}
}
}
/**
* To call onOpen just once
*/
_tryToOpen() {
if (!this._stream || !this._peerConnection || this._peerConnection.ontrack === Util.EMPTY_FUNCTION) {
return;
}
this._peerConnection.ontrack = Util.EMPTY_FUNCTION; // Just once!
this.onOpen(this._stream);
}
_startPeerConnectionIdleTimeout() {
if (!this._peerConnectionIdleTimeout) {
this._peerConnectionIdleTimeout = setTimeout(() => {
this.log('Peer connection idle timeout!').error();
this.close({ type: 'ConnectorError', name: 'Connection idle' });
}, PEER_CONNECTION_IDLE_TIMEOUT);
}
}
_clearPeerConnectionIdleTimeout() {
if (this._peerConnectionIdleTimeout) {
clearTimeout(this._peerConnectionIdleTimeout);
this._peerConnectionIdleTimeout = undefined;
}
}
}/**
* Copyright 2023 Ceeblue B.V.
* This file is part of https://github.com/CeeblueTV/webrtc-client which is released under GNU Affero General Public License.
* See file LICENSE or go to https://spdx.org/licenses/AGPL-3.0-or-later.html for full license details.
*/
const REPORT_WATCHDOG_TIMEOUT = 30000;
/**
* Use WSController to negotiate a new RTCPeerConnection connection with the server
* using WebSocket custom signaling and keep that connection open for communication.
* @example
* // Listener channel (no 'stream' parameter), listen to a stream without sending data
* const connection = new WSController({endPoint, streamName});
* connection.onOpen = stream => videoElement.srcObject = stream;
*
* // Streamer channel (with 'stream' parameter), sends and receives media streams
* const connection = new WSController({endPoint, streamName}, {stream: await navigator.mediaDevices.getUserMedia()});
* connection.onOpen = stream => {}
*/
class WSController extends SIPConnector {
/**
* @override{@inheritDoc IController.onRTPProps}
* @event
*/
onRTPProps(rtpProps) { }
/**
* @override{@inheritDoc IController.onMediaReport}
* @event
*/
onMediaReport(mediaReport) { }
/**
* @override{@inheritDoc IController.onVideoBitrate}
* @event
*/
onVideoBitrate(videoBitrate, videoBitrateConstraint) {
this.log(`onVideoBitrate ${Util.stringify({ video_bitrate: videoBitrate, video_bitrate_constraint: videoBitrateConstraint })}`).info();
}
/**
* @override{@inheritDoc IController.onPlaying}
* @event
*/
onPlaying(playing) {
this.log(`onPlaying ${Util.stringify(playing)}`).debug();
}
/**
* Instantiate the WSController, connect to the WebSocket endpoint
* and call _open() to create the RTCPeerConnection.
*
* By default, a listener channel is negotiated.
* To create a streamer channel, pass a stream parameter.
*/
constructor(connectParams, stream) {
super(connectParams, stream);
this._reportReceivedTimestamp = Util.time();
this._ws = new WebSocketReliable(Connect.buildURL(Connect.Type.WEBRTC, connectParams, 'wss'));
this._ws.onClose = (error) => this.close(error);
this._ws.onOpen = () => {
this._startReportWatchdog();
// [ENG-142] Add a way to get the server's configuration for 'iceServers'
this._open(connectParams.iceServer);
};
this._ws.onMessage = (message) => {
try {
this._eventHandler(JSON.parse(message));
}
catch (e) {
this.log(`Invalid signaling message, ${Util.stringify(e)}`).warn();
}
};
}
/**
* @override{@inheritDoc IController.setRTPProps}
*/
setRTPProps(nack, drop) {
this.send('rtp_props', { nack, drop });
}
/**
* @override{@inheritDoc IController.setVideoBitrate}
*/
setVideoBitrate(value) {
this.send('video_bitrate', { video_bitrate: value });
}
/**
* @override{@inheritDoc IController.setTracks}
*/
setTracks(tracks) {
this.send('tracks', tracks);
}
/**
* @override{@inheritDoc IController.send}
*/
send(type, params) {
try {
// Send immediatly when SIPConnector is opened (offer+answed)
// OR delay command sending (will be flushed one time onOpen)
this.log(`Command ${type} ${Util.stringify(params)}`).info();
this._ws.send(JSON.stringify(Object.assign({ type }, params)));
}
catch (ex) {
this.log(Util.stringify(ex)).error();
}
}
/**
* @override{@inheritDoc SIPConnector.close}
*/
close(error) {
if (this._ws.onClose === Util.EMPTY_FUNCTION) {
return;
}
this._ws.onClose = Util.EMPTY_FUNCTION;
this._ws.close();
this._clearReportWatchdog();
if (this._promise) {
this._promise(Error('closing'));
this._promise = undefined;
}
super.close(error);
}
/**
* @override{@inheritDoc SIPConnector._sip}
*/
_sip(offer) {
return new Promise((onSuccess, onFail) => {
this._promise = (result) => {
if (result instanceof Error) {
onFail(result);
}
else {
onSuccess(result);
}
};
this._ws.send(JSON.stringify({ type: 'offer_sdp', offer_sdp: offer }));
});
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
_eventHandler(ev) {
var _a, _b;
this.log(`EventHandler ${Util.stringify(ev, { recursion: 2 })}`).debug();
switch (ev.type) {
case 'on_answer_sdp': {
if (ev.result !== true) {
this.close({ type: 'ConnectorError', name: 'Access denied' });
return;
}
if (this._promise) {
this._promise(ev.answer_sdp);
}
break;
}
case 'on_error': {
if (!this.opened) {
// error on start or offer/answer => irrecoverable error!
this.close({ type: 'ConnectorError', name: 'Connection failed', detail: Util.stringify(ev) });
return;
}
this.log(Util.stringify(ev)).warn();
break;
}
case 'on_video_bitrate': {
this.onVideoBitrate(ev.video_bitrate, ev.video_bitrate_constraint);
break;
}
case 'on_stop': {
this.log('on_stop').info();
// Close the signaling channel when live stream on_stop
this.close();
return;
}
case 'on_track_drop': {
const mediatype = (_a = ev.mediatype) !== null && _a !== void 0 ? _a : '?';
const trackId = (_b = ev.track) !== null && _b !== void 0 ? _b : '?';
this.log(`${mediatype} track #${trackId} dropped`).warn();
break;
}
case 'on_rtp_props': {
this.onRTPProps(ev);
break;
}
case 'on_media_receive': {
this._reportReceivedTimestamp = Util.time();
if (ev.stats.loss_perc && !ev.stats.loss_num) {
// Fix an abnormal loss_perc=100% whereas loss_num=0
// happen general on start
// WIP => would be fixed on server side
ev.stats.loss_perc = 0;
}
this.onMediaReport(ev);
break;
}
case 'set_speed': // ignore on_speed report
case 'on_seek': // ignore on_seek report
break;
case 'on_time': {
this._reportReceivedTimestamp = Util.time();
this.onPlaying(ev);
break;
}
default: {
this.log(`Unhandled event: ${ev.type}`).warn();
break;
}
}
}
_startReportWatchdog() {
this._reportReceivedTimestamp = Util.time();
this._reportWatchdogInterval = setInterval(() => {
const timeout = Util.time() - this._reportReceivedTimestamp;
if (timeout >= REPORT_WATCHDOG_TIMEOUT / 3) {
this.log(`No updates received for the last ${(timeout / 1000).toFixed(1)}s`).warn();
}
if (timeout >= REPORT_WATCHDOG_TIMEOUT) {
this.close({ type: 'ConnectorError', name: 'Connection idle' });
}
}, REPORT_WATCHDOG_TIMEOUT / 6);
}
_clearReportWatchdog() {
if (this._reportWatchdogInterval) {
clearInterval(this._reportWatchdogInterval);
this._reportWatchdogInterval = undefined;
}
}
}/**
* Copyright 2023 Ceeblue B.V.
* This file is part of https://github.com/CeeblueTV/webrtc-client which is released under GNU Affero General Public License.
* See file LICENSE or go to https://spdx.org/licenses/AGPL-3.0-or-later.html for full license details.
*/
/**
* Use HTTPConnector to negotiate a new RTCPeerConnection connection with the server
* using WHIP (WebRTC HTTP Ingest Protocol) or WHEP (WebRTC HTTP Egress Protocol).
* @example
* // Listener channel (no initial 'stream' parameter), listen to a stream without sending data
* const connection = new HTTPConnector({endPoint, streamName});
* // we get the media stream from server
* connection.onOpen = stream => videoElement.srcObject = stream;
*
* // Streamer channel (with initial 'stream' parameter), sends and receives media streams
* const connection = new HTTPConnector({endPoint, streamName}, {stream: await navigator.mediaDevices.getUserMedia()});
* // the media stream here is our local camera as passed in the above constructor
* connection.onOpen = stream => {}
*/
class HTTPConnector extends SIPConnector {
/**
* Start the HTTPConnector.
*
* By default, a listener channel is negotiated.
* To create a streamer channel, give a media stream parameter
*/
constructor(connectParams, stream) {
super(connectParams, stream);
this._url = Connect.buildURL(Connect.Type.WEBRTC, connectParams, 'https');
this._fetch = new AbortController();
// [ENG-142] Add a way to get the server's configuration for 'iceServers'
setTimeout(() => {
// We wait for the next event loop to let the user set the event handlers because it can be closed immediately
this._open(connectParams.iceServer);
}, 0);
}
/**
* @override{@inheritDoc SIPConnector.close}
*/
close(error) {
this._fetch.abort();
super.close(error);
}
/**
* @override{@inheritDoc SIPConnector._sip}
*/
_sip(offer) {
return __awaiter(this, void 0, void 0, function* () {
const response = yield fetch(this._url, {
method: 'POST',
body: offer,
headers: {
'Content-Type': 'application/sdp'
},
signal: this._fetch.signal
});
if (response.ok) {
return response.text();
}
return Promise.reject(`HTTP ${response.status} ${response.statusText} status`);
});
}
}/**
* Copyright 2023 Ceeblue B.V.
* This file is part of https://github.com/CeeblueTV/webrtc-client which is released under GNU Affero General Public License.
* See file LICENSE or go to https://spdx.org/licenses/AGPL-3.0-or-later.html for full license details.
*/
/**
* WSStreamData is the WebSocket implementation of IStreamData
* @example
* const streamData = new WSStreamData({endPoint, streamName});
* streamData.tracks = [0, 1]; // subscribe to data tracks 0 and 1
* streamData.onData = track, time, data => {
* console.log(`Data received on track ${track} at ${time} : ${Util.stringify(data)}`);
* }
*/
class WSStreamData extends EventEmitter {
/**
* @override{@inheritDoc IStreamData.onClose}
* @event
*/
onClose(error) {
this.log('onClose').info();
}
/**
* @override{@inheritDoc IStreamData.onData}
* @event
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
onData(track, time, data) {
this.log(`Data received on track ${track} at ${time}: ${Util.stringify(data)}`).info();
}
/**
* @override{@inheritDoc IStreamData.url}
*/
get url() {
return this._url;
}
/**
* @override{@inheritDoc IStreamData.tracks}
*/
get tracks() {
return [...this._tracks];
}
/**
* @override{@inheritDoc IStreamData.tracks}
*/
set tracks(tracks) {
this._tracks = [...tracks];
this._sendTracks();
}
/**
* @override{@inheritDoc IStreamData.closed}
*/
get closed() {
return !this._ws || this._ws.closed;
}
/**
* Build the stream data instance, it only connects to the server when tracks are set.
*/
constructor(connectParams) {
super();
this._url = Connect.buildURL(Connect.Type.DATA, connectParams).toString();
this._tracks = Array();
this._ws = new WebSocketReliable();
this._ws.onOpen = () => this._sendTracks(); // On open sends tracks subscription!
this._ws.onClose = (error) => this.onClose(error);
this._ws.onMessage = (message) => {
let json;
try {
json = JSON.parse(message);
if (json.error) {
throw Error(json.error);
}
}
catch (e) {
this.log(U