@simplito/privmx-webendpoint/dist/webStreams/WebRtcClient.js

PrivMX Web Endpoint library

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.WebRtcClient = void 0;
const WebWorkerHelper_1 = require("./WebWorkerHelper");
const WebRtcConfig_1 = require("./WebRtcConfig");
const Types_1 = require("../Types");
const KeyStore_1 = require("./KeyStore");
const PeerConnectionsManager_1 = require("./PeerConnectionsManager");
const Logger_1 = require("./Logger");
const Queue_1 = require("./Queue");
const LocalAudioLevelMeter_1 = require("./audio/LocalAudioLevelMeter");
const ActiveSpeakerDetector_1 = require("./audio/ActiveSpeakerDetector");
const EventDispatcher_1 = require("../service/EventDispatcher");
const DataChannelCryptor_1 = require("./DataChannelCryptor");
class WebRtcClient {
    assetsDir;
    uniqId;
    e2eeWorker;
    webWorkerApi;
    configuration;
    keyStore = new KeyStore_1.KeyStore();
    publishStreamHandle;
    // to moze byc uzyte kiedy wymagany jest update credentials (jak straca waznosc)
    peerCredentials;
    remoteStreamsListeners = new Map();
    sequenceNumberByRemoteStreamId = new Map();
    dataChannelByRemoteStreamId = new Map();
    dataChannelCryptor;
    sequenceNumberOfSender;
    peerConnectionsManager;
    streamsApiInterface;
    activeSpeakerDetector;
    audioLevelCallback;
    // private mediaServerAvailPublishers: {[publisherId: number]: Publisher} = {};
    encByReceiver = new WeakMap();
    logger = new Logger_1.Logger();
    peerConnectionReconfigureQueue;
    lastProcessedAnswer = {};
    lastMeasuredLocalRMS = LocalAudioLevelMeter_1.LocalAudioLevelMeter.RMS_VALUE_OF_SILENCE;
    eventsDispatcher = new EventDispatcher_1.StateChangeDispatcher();
    localAudioLevelMeters = new Map();
    bootstrapDataChannel;
    constructor(assetsDir) {
        this.assetsDir = assetsDir;
        this.uniqId = "" + Math.random() + "-" + Math.random();
        this.sequenceNumberOfSender = 1;
        this.peerConnectionsManager = new PeerConnectionsManager_1.PeerConnectionManager((roomId) => {
            return this.createPeerConnectionMultiForRoom(roomId, this.getPeerConnectionConfiguration());
        }, (sessionId, candidate) => {
            return this.streamsApiInterface.trickle(sessionId, candidate);
        });
        this.peerConnectionReconfigureQueue = new Queue_1.Queue();
        this.peerConnectionReconfigureQueue.assignProcessorFunc(async (_item) => {
            if (_item.jsep && _item.jsep.type === "offer") {
                await this.reconfigureSingle(_item._room, _item.jsep);
            }
            else {
                await this.reconfigureSingleCreateOffer(_item._room);
            }
        });
        this.activeSpeakerDetector = new ActiveSpeakerDetector_1.ActiveSpeakerDetector(ActiveSpeakerDetector_1.DEFAULTS);
        this.dataChannelCryptor = new DataChannelCryptor_1.DataChannelCryptor(this.getKeyStore());
    }
    async ensureLocalAudioLevelMeter(track) {
        if (this.localAudioLevelMeters.has(track.id)) {
            return;
        }
        const worker = await this.getWorker();
        const meter = new LocalAudioLevelMeter_1.LocalAudioLevelMeter(track, (onRms) => {
            const rmsToReport = track.enabled ? onRms : LocalAudioLevelMeter_1.LocalAudioLevelMeter.RMS_VALUE_OF_SILENCE;
            worker.postMessage({ operation: "rms", rms: rmsToReport });
            this.lastMeasuredLocalRMS = onRms;
        });
        this.localAudioLevelMeters.set(track.id, meter);
        try {
            await meter.init(this.assetsDir + "/rms-processor.js");
        }
        catch (e) {
            this.localAudioLevelMeters.delete(track.id);
            meter.stop();
            throw e;
        }
    }
    stopLocalAudioLevelMeter(track) {
        const meter = this.localAudioLevelMeters.get(track.id);
        if (!meter) {
            return;
        }
        this.localAudioLevelMeters.delete(track.id);
        meter.stop();
    }
    setAudioLevelCallback(func) {
        this.audioLevelCallback = func;
    }
    bindApiInterface(streamsApiInterface) {
        this.streamsApiInterface = streamsApiInterface;
    }
    addRemoteStreamListener(listener) {
        let listeners = this.remoteStreamsListeners.get(listener.streamRoomId) || [];
        const exists = listeners.find((x) => x.streamId === listener.streamId);
        if (exists) {
            throw new Error("RemoteStreamListener with given params already exists.");
        }
        listeners.push(listener);
        this.remoteStreamsListeners.set(listener.streamRoomId, listeners);
    }
    getStreamStateChangeDispatcher() {
        return this.eventsDispatcher;
    }
    getConnectionManager() {
        if (!this.peerConnectionsManager) {
            throw new Error("No peerConnectionManager initialized.");
        }
        return this.peerConnectionsManager;
    }
    getWebRtcEventDispatcher() {
        return this.eventsDispatcher;
    }
    async getWorker() {
        if (!this.e2eeWorker) {
            const workerApi = await this.getWorkerApi();
            this.e2eeWorker = workerApi.getWorker();
        }
        if (!this.e2eeWorker) {
            throw new Error("Worker not initialized.");
        }
        return this.e2eeWorker;
    }
    async initPipeline(receiverTrackId, publisherId) {
        const worker = await this.getWorker();
        const waitPromise = new Promise((resolve) => {
            const listener = (ev) => {
                if (ev.data.operation === "init-pipeline" && ev.data.id === receiverTrackId) {
                    worker.removeEventListener("message", listener);
                    resolve();
                }
            };
            worker.addEventListener("message", listener);
            worker.postMessage({
                operation: "init-pipeline",
                id: receiverTrackId,
                publisherId: publisherId,
            });
        });
        return waitPromise;
    }
    async getWorkerApi() {
        if (!this.webWorkerApi) {
            this.webWorkerApi = new WebWorkerHelper_1.WebWorker(this.assetsDir, (frameInfo) => {
                if (this.audioLevelCallback && typeof this.audioLevelCallback === "function") {
                    // report local rms to activeSpeakerDetector to have notifications for local streams
                    this.activeSpeakerDetector.onFrame({
                        id: 0,
                        rms: this.lastMeasuredLocalRMS,
                        timestamp: Date.now(),
                    });
                    const speakers = this.activeSpeakerDetector.onFrame({
                        id: frameInfo.publisherId,
                        rms: frameInfo.rms,
                        timestamp: Date.now(),
                    });
                    // if (laudestParticipant === frameInfo.publisherId) {
                    this.audioLevelCallback({ levels: speakers });
                    // }
                }
            });
            await this.webWorkerApi.init_e2ee();
        }
        return this.webWorkerApi;
    }
    getPeerConnectionConfiguration() {
        if (!this.configuration) {
            // throw new Error("No peerConnectionConfiguration created");
            this.configuration = WebRtcConfig_1.WebRtcConfig.generateTurnConfiguration(this.peerCredentials);
        }
        return this.configuration;
    }
    async setTurnCredentials(turnCredentials) {
        this.peerCredentials = turnCredentials;
    }
    async createPeerConnectionWithLocalStream(streamHandle, streamRoomId, stream, dataTracks) {
        this.publishStreamHandle = streamHandle;
        this.configuration = WebRtcConfig_1.WebRtcConfig.generateTurnConfiguration(this.peerCredentials);
        const peerConnManager = this.getConnectionManager();
        peerConnManager.initialize(streamRoomId, "publisher");
        const pc = this.getConnectionManager().getConnectionWithSession(streamRoomId, "publisher").pc;
        if (stream.getTracks().length > 0) {
            const tracks = stream.getTracks();
            this.e2eeWorker = await this.getWorker();
            for (const track of tracks) {
                if (track.kind === "audio") {
                    // add RMSProcessor
                    await this.ensureLocalAudioLevelMeter(track);
                }
                const streamSender = pc.addTrack(track, stream);
                this.setupSenderTransform(streamSender);
            }
        }
        if (dataTracks) {
            for (const dataTrack of dataTracks) {
                const dataChannel = pc.createDataChannel("JanusDataChannel", {
                    ordered: true,
                    negotiated: false,
                });
                dataTrack.dataChannelMeta.dataChannel = dataChannel;
            }
        }
        return pc;
    }
    removeSenderPeerConnectionOnUnpublish(streamRoomId, stream) {
        const peerConnManager = this.getConnectionManager();
        const session = peerConnManager.getConnectionWithSession(streamRoomId, "publisher");
        for (const track of stream.getAudioTracks()) {
            this.stopLocalAudioLevelMeter(track);
        }
        session.pc.close();
        session.pc = undefined;
    }
    async updatePeerConnectionWithLocalStream(streamRoomId, localStream, tracksToAdd, tracksToRemove) {
        this.configuration = WebRtcConfig_1.WebRtcConfig.generateTurnConfiguration(this.peerCredentials);
        const peerConnManager = this.getConnectionManager();
        peerConnManager.initialize(streamRoomId, "publisher");
        const pc = this.getConnectionManager().getConnectionWithSession(streamRoomId, "publisher").pc;
        if (tracksToAdd.length > 0) {
            this.e2eeWorker = await this.getWorker();
            for (const track of tracksToAdd) {
                if (track.kind === "audio") {
                    await this.ensureLocalAudioLevelMeter(track);
                }
                const videoSender = pc.addTrack(track, localStream);
                if (window.RTCRtpScriptTransform) {
                    const options = {
                        operation: "encode",
                    };
                    videoSender.transform = new RTCRtpScriptTransform(this.e2eeWorker, options);
                }
                else {
                    const senderStreams = videoSender.createEncodedStreams();
                    this.e2eeWorker.postMessage({
                        operation: "encode",
                        readableStream: senderStreams.readable,
                        writableStream: senderStreams.writable,
                    }, [senderStreams.readable, senderStreams.writable]);
                }
            }
        }
        if (tracksToRemove.length > 0) {
            const senders = pc.getSenders();
            for (const oldTrack of tracksToRemove) {
                if (oldTrack.kind === "audio") {
                    this.stopLocalAudioLevelMeter(oldTrack);
                }
                const sender = senders.find((s) => s.track === oldTrack);
                if (sender) {
                    pc.removeTrack(sender);
                }
            }
        }
        return pc;
    }
    async encryptDataChannelData(data) {
        const nextSequenceNumber = ++this.sequenceNumberOfSender;
        return this.dataChannelCryptor.encryptToWireFormat({
            plaintext: data,
            sequenceNumber: nextSequenceNumber,
        });
    }
    createPeerConnectionMultiForRoom(roomId, configuration, _handle, _session) {
        const extConf = configuration;
        extConf.encodedInsertableStreams = true;
        const connection = new RTCPeerConnection(extConf);
        // gethering state change
        connection.addEventListener("icegatheringstatechange", (event) => {
            this.logger.debug("on ice state change: ", event);
        });
        // ice candidate error
        connection.addEventListener("icecandidateerror", (event) => {
            this.logger.debug("on ice error: ", event);
        });
        connection.addEventListener("connectionstatechange", (event) => {
            this.logger.debug("connectionstatechange: ", event);
            if (connection.connectionState === "connected") {
                this.logger.debug("Peers connected!");
            }
            else {
                this.logger.debug("connection state: ", connection.connectionState);
            }
            this.eventsDispatcher.emit({
                streamHandle: this.publishStreamHandle,
                state: connection.connectionState,
            });
        });
        connection.addEventListener("datachannel", (event) => {
            this.logger.debug("================ RECV datachannel: ", event.channel.id, event.channel.label);
            const dc = event.channel;
            dc.binaryType = "arraybuffer";
            dc.onmessage = async (dataEvent) => {
                this.logger.debug("================ ON MESSAGE....");
                const remoteStreamId = Number(event.channel.label);
                const frame = dataEvent.data instanceof Uint8Array
                    ? dataEvent.data
                    : dataEvent.data instanceof ArrayBuffer
                        ? new Uint8Array(dataEvent.data)
                        : new Uint8Array(dataEvent.data.buffer);
                try {
                    const lastSeq = this.sequenceNumberByRemoteStreamId.get(remoteStreamId) || 0;
                    const decrypted = await this.dataChannelCryptor.decryptFromWireFormat({
                        frame,
                        lastSequenceNumber: lastSeq,
                    });
                    this.sequenceNumberByRemoteStreamId.set(remoteStreamId, decrypted.seq);
                    this.logger.debug("Calling listener for dataChannel with values: ", roomId, remoteStreamId, decrypted.data, Types_1.DataChannelCryptorDecryptStatus.OK);
                    this.callRegisteredListenersForDataChannel(roomId, remoteStreamId, decrypted.data, Types_1.DataChannelCryptorDecryptStatus.OK);
                }
                catch (e) {
                    if (e instanceof DataChannelCryptor_1.DataChannelCryptorError) {
                        this.callRegisteredListenersForDataChannel(roomId, remoteStreamId, new Uint8Array(), e.code);
                    }
                    else {
                        throw e;
                    }
                }
            };
            this.dataChannelByRemoteStreamId.set(Number(dc.label), dc);
        });
        connection.addEventListener("iceconnectionstatechange", (event) => {
            this.logger.debug("iceconnectionstatechange: ", event);
        });
        connection.addEventListener("negotiationneeded", async (_event) => {
            this.logger.debug("negotiationneeded: ", _event);
            // await this.startNegotiationMulti(roomId, (_event as any).target);
        });
        connection.addEventListener("signalingstatechange", (event) => {
            this.logger.debug("signalingstatechange: ", event);
        });
        connection.addEventListener("track", async (event) => {
            await this.addRemoteTrack(roomId, event);
        });
        return connection;
    }
    async startNegotiationMulti(roomId, _rtcPeerConnection, _withIceRestart) {
        try {
            if (!this.peerConnectionReconfigureQueue) {
                throw new Error("ReconfigureQueue does not exist.");
            }
            this.peerConnectionReconfigureQueue.enqueue({
                taskId: Math.floor(1 + Math.random() * 10000),
                _room: roomId,
            });
            try {
                await this.peerConnectionReconfigureQueue.processAll();
            }
            catch (e) {
                console.error("Error on onSubscriberAttached", e);
            }
        }
        catch (e) {
            console.error("Error on startNegotiationMulti", e);
        }
    }
    async updateKeys(_streamRoomId, keys) {
        this.logger.debug("=======> UPDATE KEYS", _streamRoomId, keys.length);
        this.keyStore.setKeys(keys);
        (await this.getWorkerApi()).setKeys(keys);
    }
    getKeyStore() {
        return this.keyStore;
    }
    setupSenderTransform(videoSender) {
        if (window.RTCRtpScriptTransform) {
            const options = {
                operation: "encode",
            };
            videoSender.transform = new RTCRtpScriptTransform(this.e2eeWorker, options);
        }
        else {
            this.logger.debug("Worker - encoding frames using EncodedStreams");
            const senderStreams = videoSender.createEncodedStreams();
            this.e2eeWorker.postMessage({
                operation: "encode",
                readableStream: senderStreams.readable,
                writableStream: senderStreams.writable,
            }, [senderStreams.readable, senderStreams.writable]);
        }
    }
    async setupReceiverTransform(receiver, publisherId, worker) {
        if ("RTCRtpScriptTransform" in window && !receiver.transform) {
            this.logger.debug("-> using RtpScriptTransform");
            const id = receiver.track.id;
            receiver.transform = new window.RTCRtpScriptTransform(worker, {
                operation: "decode",
                id,
                publisherId,
            });
            return;
        }
        this.logger.debug("-> using EncodedStreams");
        // Fallback: Encoded Streams
        if (!this.encByReceiver.has(receiver) &&
            "createEncodedStreams" in receiver &&
            typeof receiver.createEncodedStreams === "function") {
            this.logger.debug("-> call for createEncodedStreams()");
            const { readable, writable } = await receiver.createEncodedStreams();
            const enc = {
                readable,
                writable,
                id: receiver.track.id,
                publisherId: publisherId,
                posted: false,
            };
            this.encByReceiver.set(receiver, enc);
            this.logger.debug("-> posting EncodedStreams to worker (should happen only once)");
            await this.initPipeline(enc.id, enc.publisherId);
            worker.postMessage({
                operation: "decode",
                id: enc.id,
                publisherId: enc.publisherId,
                readableStream: enc.readable,
                writableStream: enc.writable,
            }, [enc.readable, enc.writable]);
        }
        else {
            this.logger.debug("-> EncodedStreams posted to worker already.");
        }
    }
    async waitUntilConnected(pc) {
        if (pc.iceConnectionState === "connected" || pc.iceConnectionState === "completed")
            return Promise.resolve();
        return new Promise((resolve, reject) => {
            const onChange = () => {
                if (pc.iceConnectionState === "connected" ||
                    pc.iceConnectionState === "completed") {
                    // pc.removeEventListener('iceconnectionstatechange', onChange);
                    resolve();
                }
                else if (pc.iceConnectionState === "failed" ||
                    pc.connectionState === "failed" ||
                    pc.connectionState === "closed") {
                    pc.removeEventListener("iceconnectionstatechange", onChange);
                    reject(new Error("ICE/DTLS not connected"));
                }
            };
            pc.addEventListener("iceconnectionstatechange", onChange);
        });
    }
    async teardownReceiver(receiver, worker) {
        const enc = this.encByReceiver.get(receiver);
        if (enc) {
            worker.postMessage({ operation: "stop", id: enc.id });
            this.encByReceiver.delete(receiver);
        }
    }
    async addRemoteTrack(roomId, event) {
        const worker = await this.getWorker();
        const track = event.track;
        const receiver = event.receiver;
        const publisherId = Number(event.streams[0].id);
        const peerConnection = this.getConnectionManager().getConnectionWithSession(roomId, "subscriber").pc;
        this.logger.debug("waitUntilConnected...");
        await this.waitUntilConnected(peerConnection);
        this.logger.debug("setupReceiverTransform...");
        await this.setupReceiverTransform(receiver, publisherId, worker);
        track.addEventListener("ended", async () => await this.teardownReceiver(receiver, worker));
        this.callRegisteredListeners(roomId, event);
    }
    callRegisteredListeners(roomId, event) {
        const remoteStreamId = Number(event.streams[0].id);
        const listeners = this.remoteStreamsListeners.get(roomId);
        if (!listeners) {
            return;
        }
        const filteredListeners = listeners.filter((x) => x.streamId === remoteStreamId || x.streamId === undefined);
        for (const listener of filteredListeners) {
            if (listener.onRemoteStreamTrack &&
                typeof listener.onRemoteStreamTrack === "function") {
                listener.onRemoteStreamTrack(event);
            }
        }
    }
    callRegisteredListenersForDataChannel(roomId, remoteStreamId, data, statusCode) {
        const listeners = this.remoteStreamsListeners.get(roomId);
        if (!listeners) {
            return;
        }
        const filteredListeners = listeners.filter((x) => x.streamId === remoteStreamId || x.streamId === undefined);
        for (const listener of filteredListeners) {
            if (listener.onRemoteData && typeof listener.onRemoteData === "function") {
                listener.onRemoteData(data, statusCode);
            }
        }
    }
    async onSubscriptionUpdated(_room, offer) {
        if (!this.peerConnectionReconfigureQueue) {
            throw new Error("ReconfigureQueue does not exist.");
        }
        this.peerConnectionReconfigureQueue.enqueue({
            taskId: Math.floor(1 + Math.random() * 10000),
            _room,
            jsep: offer,
        });
        try {
            await this.peerConnectionReconfigureQueue.processAll();
        }
        catch (e) {
            console.error("Error on onSubscriberAttached", e);
        }
    }
    async onSubscriptionUpdatedSingle(_room, offer) {
        return this.reconfigureSingle(_room, offer);
    }
    async reconfigureSingle(room, offer) {
        if (!this.configuration) {
            throw new Error("Configuration missing.");
        }
        const janusConnection = this.getConnectionManager().getConnectionWithSession(room, "subscriber");
        const peerConnection = janusConnection.pc;
        this.logger.debug("SUBSCRIBER RECV OFFER FROM PUBLISHER: ", offer.sdp);
        this.logger.debug("1. Setting up remoteDescription...");
        if (!this.bootstrapDataChannel) {
            const bootstrap = peerConnection.createDataChannel("JanusDataChannel");
            bootstrap.onerror = (e) => {
                console.error(e);
                throw new Error("Cannot initialize Bootrstrap dataChannel");
            };
        }
        await peerConnection.setRemoteDescription(new RTCSessionDescription({ type: offer.type, sdp: offer.sdp }));
        this.logger.debug("offer from Janus: ", JSON.stringify(offer, null, 2));
        this.logger.debug("2. Creating an answer...", "peerConnection state", peerConnection.connectionState);
        const answer = await peerConnection.createAnswer();
        this.logger.debug("3. Setting up localDescription...");
        await peerConnection.setLocalDescription(new RTCSessionDescription(answer));
        // this.subscriberAttachedProcessing = false
        this.lastProcessedAnswer[room] = answer;
        return answer;
    }
    async reconfigureSingleCreateOffer(room) {
        if (!this.configuration) {
            throw new Error("Configuration missing.");
        }
        const janusConnection = this.getConnectionManager().getConnectionWithSession(room, "publisher");
        const peerConnection = janusConnection.pc;
        const offer = await peerConnection.createOffer();
        await peerConnection.setLocalDescription(new RTCSessionDescription({ type: "offer", sdp: offer.sdp }));
        return offer;
    }
}
exports.WebRtcClient = WebRtcClient;