@casual-simulation/aux-vm-browser/managers/LivekitManager.js

A set of utilities required to securely run an AUX in a web browser.

import { asyncResult, hasValue, ON_ROOM_JOINED, ON_ROOM_LEAVE, ON_ROOM_OPTIONS_CHANGED, ON_ROOM_REMOTE_JOINED, ON_ROOM_REMOTE_LEAVE, ON_ROOM_SPEAKERS_CHANGED, ON_ROOM_STREAMING, ON_ROOM_STREAM_LOST, ON_ROOM_TRACK_SUBSCRIBED, ON_ROOM_TRACK_UNSUBSCRIBED, } from '@casual-simulation/aux-common';
import { Subject } from 'rxjs';
/**
 * Defines a class that is able to manage Livekit rooms and make streams available to scripts.
 */
export class LivekitManager {
    /**
     * Gets an observable that resolves whenever a track needs to be attached to the document.
     */
    get onTrackNeedsAttachment() {
        return this._onTrackNeedsAttachment;
    }
    /**
     * Gets an observable that resolves whenever a track needs to be detached from the document.
     */
    get onTrackNeedsDetachment() {
        return this._onTrackNeedsDetachment;
    }
    get closed() {
        return this._closed;
    }
    constructor(helper) {
        this._rooms = [];
        this._addressToTrack = new Map();
        this._addressToPublication = new Map();
        this._addressToParticipant = new Map();
        this._addressToVideo = new Map();
        this._trackToAddress = new Map();
        this._closed = false;
        this._onTrackNeedsAttachment = new Subject();
        this._onTrackNeedsDetachment = new Subject();
        this._helper = helper;
    }
    unsubscribe() {
        if (this._closed) {
            return;
        }
        this._closed = true;
        for (let room of this._rooms) {
            room.disconnect(true);
        }
        this._rooms = [];
        this._addressToTrack = null;
        this._trackToAddress = null;
    }
    async _requestMediaPermissions() {
        // Request audio and video permission (required)
        try {
            await navigator.mediaDevices.getUserMedia({
                video: true,
                audio: true,
            });
            console.log('[LivekitManager] Media permissions granted.');
        }
        catch (error) {
            console.error('[LivekitManager] Media permissions denied:', error);
            throw new Error('Media permissions are required to join the room.');
        }
    }
    async joinRoom(join) {
        try {
            await this._requestMediaPermissions();
            this._livekit = await import('livekit-client');
            const room = new this._livekit.Room({
                adaptiveStream: false,
                dynacast: true,
                ...join.options,
            });
            room.on(this._livekit.RoomEvent.TrackSubscribed, this._onTrackSubscribed(room))
                .on(this._livekit.RoomEvent.TrackUnsubscribed, this._onTrackUnsubscribed(room))
                .on(this._livekit.RoomEvent.Disconnected, this._onDisconnected(room))
                .on(this._livekit.RoomEvent.Reconnected, this._onReconnected(room))
                .on(this._livekit.RoomEvent.LocalTrackPublished, this._onLocalTrackPublished(room))
                .on(this._livekit.RoomEvent.LocalTrackUnpublished, this._onLocalTrackUnpublished(room))
                .on(this._livekit.RoomEvent.TrackMuted, this._onTrackMuted(room))
                .on(this._livekit.RoomEvent.TrackUnmuted, this._onTrackUnmuted(room))
                .on(this._livekit.RoomEvent.ActiveSpeakersChanged, this._onActiveSpeakersChanged(room))
                .on(this._livekit.RoomEvent.ParticipantConnected, this._onParticipantConnected(room))
                .on(this._livekit.RoomEvent.ParticipantDisconnected, this._onParticipantDisconnected(room));
            await room.connect(join.url, join.token, {});
            try {
                await this._setRoomOptions(room, {
                    video: true,
                    audio: true,
                    ...join.options,
                });
            }
            catch (err) {
                console.warn('[LivekitManager] Unable to set room options:', err);
            }
            this._rooms.push(room);
            const options = this._getRoomOptions(room);
            join.resolve(options);
            let actions = [
                {
                    eventName: ON_ROOM_JOINED,
                    bots: null,
                    arg: { roomName: room.name, options },
                },
                {
                    eventName: ON_ROOM_STREAMING,
                    bots: null,
                    arg: { roomName: room.name, options },
                },
            ];
            // Send initial ON_ROOM_REMOTE_JOINED events
            for (let participant of room.remoteParticipants.values()) {
                actions.push({
                    eventName: ON_ROOM_REMOTE_JOINED,
                    bots: null,
                    arg: {
                        roomName: room.name,
                        remoteId: participant.identity,
                    },
                });
            }
            this._helper.transaction(...this._helper.actions(actions));
        }
        catch (err) {
            join.reject('server_error', err.toString());
        }
    }
    async leaveRoom(leave) {
        try {
            const index = this._rooms.findIndex((r) => r.name === leave.roomName);
            let room;
            let actions = [];
            if (index >= 0) {
                room = this._rooms[index];
                this._rooms.splice(index, 1);
                if (room) {
                    // Send ON_ROOM_REMOTE_LEAVE events
                    for (let participant of room.remoteParticipants.values()) {
                        actions.push({
                            eventName: ON_ROOM_REMOTE_LEAVE,
                            bots: null,
                            arg: {
                                roomName: room.name,
                                remoteId: participant.identity,
                            },
                        });
                    }
                    room.disconnect(true);
                }
            }
            leave.resolve();
            if (room) {
                actions.push({
                    eventName: ON_ROOM_STREAM_LOST,
                    bots: null,
                    arg: { roomName: leave.roomName },
                }, {
                    eventName: ON_ROOM_LEAVE,
                    bots: null,
                    arg: { roomName: leave.roomName },
                });
                this._helper.transaction(...this._helper.actions(actions));
            }
        }
        catch (err) {
            leave.reject('error', err.toString());
        }
    }
    async setRoomOptions(setRoomOptions) {
        try {
            const room = this._rooms.find((r) => r.name === setRoomOptions.roomName);
            if (!room || !this._livekit) {
                setRoomOptions.reject('room_not_found', 'The specified room was not found.');
                return;
            }
            const changed = await this._setRoomOptions(room, setRoomOptions.options);
            const options = this._getRoomOptions(room);
            let rejected = false;
            for (let key in setRoomOptions.options) {
                const targetValue = setRoomOptions.options[key];
                const currentValue = options[key];
                if (targetValue !== currentValue) {
                    setRoomOptions.reject('error', `Unable to set "${key}" to ${targetValue}`);
                    rejected = true;
                }
            }
            if (!rejected) {
                setRoomOptions.resolve(options);
            }
            if (changed) {
                this._helper.action(ON_ROOM_OPTIONS_CHANGED, null, {
                    roomName: room.name,
                    options,
                });
            }
        }
        catch (err) {
            setRoomOptions.reject('error', err.toString());
        }
    }
    async _setRoomOptions(room, options) {
        let promises = [];
        if ('video' in options) {
            promises.push(room.localParticipant.setCameraEnabled(!!options.video));
        }
        if ('audio' in options) {
            promises.push(room.localParticipant.setMicrophoneEnabled(!!options.audio));
        }
        if ('screen' in options) {
            promises.push(room.localParticipant.setScreenShareEnabled(!!options.screen));
        }
        const results = await Promise.allSettled(promises);
        return results.some((r) => r.status === 'fulfilled');
    }
    async getRoomOptions(getRoomOptions) {
        try {
            const room = this._rooms.find((r) => r.name === getRoomOptions.roomName);
            if (!room) {
                getRoomOptions.reject('room_not_found', 'The specified room was not found.');
                return;
            }
            const options = this._getRoomOptions(room);
            getRoomOptions.resolve(options);
        }
        catch (err) {
            getRoomOptions.reject('error', err.toString());
        }
    }
    _getRoomOptions(room) {
        return this._getParticipantOptions(room.localParticipant);
    }
    _getParticipantOptions(participant) {
        let options = {
            video: false,
            audio: false,
            screen: false,
        };
        if (this._livekit) {
            for (let [id, pub] of participant.trackPublications) {
                if (!pub.isMuted && pub.isEnabled) {
                    if (pub.kind === this._livekit.Track.Kind.Audio &&
                        pub.source === this._livekit.Track.Source.Microphone) {
                        options.audio = true;
                    }
                    else if (pub.kind === this._livekit.Track.Kind.Video &&
                        pub.source === this._livekit.Track.Source.Camera) {
                        options.video = true;
                    }
                    else if (pub.source === this._livekit.Track.Source.ScreenShare) {
                        options.screen = true;
                    }
                }
            }
        }
        return options;
    }
    handleEvents(events) {
        for (let event of events) {
            if (event.type === 'get_room_track_options') {
                this._getRoomTrackOptions(event);
            }
            else if (event.type === 'set_room_track_options') {
                this._setRoomTrackOptions(event);
            }
            else if (event.type === 'get_room_remote_options') {
                this._getRoomRemoteOptions(event);
            }
        }
    }
    async _getRoomTrackOptions(event) {
        try {
            const room = this._rooms.find((r) => r.name === event.roomName);
            if (!room || !this._livekit) {
                this._helper.transaction(asyncResult(event.taskId, {
                    success: false,
                    errorCode: 'room_not_found',
                    errorMessage: 'The specified room was not found.',
                    roomName: event.roomName,
                }));
                return;
            }
            const pub = this._addressToPublication.get(event.address);
            const participant = this._addressToParticipant.get(event.address);
            if (!pub || !participant) {
                this._helper.transaction(asyncResult(event.taskId, {
                    success: false,
                    errorCode: 'track_not_found',
                    errorMessage: 'The specified track was not found.',
                    roomName: event.roomName,
                    address: event.address,
                }));
                return;
            }
            const options = this._getTrackOptions(pub, participant);
            this._helper.transaction(asyncResult(event.taskId, {
                success: true,
                roomName: room.name,
                address: event.address,
                options,
            }));
        }
        catch (err) {
            if (hasValue(event.taskId)) {
                this._helper.transaction(asyncResult(event.taskId, {
                    success: false,
                    errorCode: 'error',
                    errorMessage: err.toString(),
                    roomName: event.roomName,
                    address: event.address,
                }));
            }
        }
    }
    async _setRoomTrackOptions(event) {
        try {
            const room = this._rooms.find((r) => r.name === event.roomName);
            if (!room || !this._livekit) {
                this._helper.transaction(asyncResult(event.taskId, {
                    success: false,
                    errorCode: 'room_not_found',
                    errorMessage: 'The specified room was not found.',
                    roomName: event.roomName,
                }));
                return;
            }
            const pub = this._addressToPublication.get(event.address);
            const participant = this._addressToParticipant.get(event.address);
            if (!pub || !participant) {
                this._helper.transaction(asyncResult(event.taskId, {
                    success: false,
                    errorCode: 'track_not_found',
                    errorMessage: 'The specified track was not found.',
                    roomName: event.roomName,
                    address: event.address,
                }));
                return;
            }
            let promises = [];
            if ('muted' in event.options) {
                if (pub instanceof this._livekit.LocalTrackPublication) {
                    if (event.options.muted) {
                        promises.push(pub.mute().then(() => { }));
                    }
                    else {
                        promises.push(pub.unmute().then(() => { }));
                    }
                }
                else if (pub instanceof this._livekit.RemoteTrackPublication) {
                    pub.setEnabled(!event.options.muted);
                }
            }
            if ('videoQuality' in event.options) {
                if (pub instanceof this._livekit.RemoteTrackPublication) {
                    let quality = this._livekit.VideoQuality.HIGH;
                    if (event.options.videoQuality === 'medium') {
                        quality = this._livekit.VideoQuality.MEDIUM;
                    }
                    else if (event.options.videoQuality === 'low') {
                        quality = this._livekit.VideoQuality.LOW;
                    }
                    else if (event.options.videoQuality === 'high') {
                        quality = this._livekit.VideoQuality.HIGH;
                    }
                    else if (event.options.videoQuality === 'off') {
                        pub.setEnabled(false);
                    }
                    pub.setVideoQuality(quality);
                }
            }
            await Promise.allSettled(promises);
            const options = this._getTrackOptions(pub, participant);
            this._helper.transaction(asyncResult(event.taskId, {
                success: true,
                roomName: room.name,
                address: event.address,
                options,
            }));
        }
        catch (err) {
            if (hasValue(event.taskId)) {
                this._helper.transaction(asyncResult(event.taskId, {
                    success: false,
                    errorCode: 'error',
                    errorMessage: err.toString(),
                    roomName: event.roomName,
                    address: event.address,
                }));
            }
        }
    }
    _findRemoteParticipant(room, identity) {
        for (let p of room.remoteParticipants.values()) {
            if (p.identity === identity) {
                return p;
            }
        }
        return null;
    }
    async _getRoomRemoteOptions(event) {
        try {
            const room = this._rooms.find((r) => r.name === event.roomName);
            if (!room || !this._livekit) {
                this._helper.transaction(asyncResult(event.taskId, {
                    success: false,
                    errorCode: 'room_not_found',
                    errorMessage: 'The specified room was not found.',
                    roomName: event.roomName,
                }));
                return;
            }
            const participant = this._findRemoteParticipant(room, event.remoteId);
            if (!participant) {
                this._helper.transaction(asyncResult(event.taskId, {
                    success: false,
                    errorCode: 'remote_not_found',
                    errorMessage: 'The specified remote was not found.',
                    roomName: event.roomName,
                    remoteId: event.remoteId,
                }));
                return;
            }
            const basicOptions = this._getParticipantOptions(participant);
            const options = {
                ...basicOptions,
                audioLevel: participant.audioLevel,
                connectionQuality: participant.connectionQuality,
            };
            this._helper.transaction(asyncResult(event.taskId, {
                success: true,
                roomName: room.name,
                remoteId: event.remoteId,
                options,
            }));
        }
        catch (err) {
            if (hasValue(event.taskId)) {
                this._helper.transaction(asyncResult(event.taskId, {
                    success: false,
                    errorCode: 'error',
                    errorMessage: err.toString(),
                    roomName: event.roomName,
                    remoteId: event.remoteId,
                }));
            }
        }
    }
    /**
     * Gets the media stream that is referenced by the given address.
     * @param address The address that should be used.
     */
    getMediaByAddress(address) {
        const track = this._addressToTrack.get(address);
        if (track) {
            return track.mediaStream;
        }
        return null;
    }
    /**
     * Gets the video that is referenced by the given address.
     * @param address The address that should be used.
     */
    getVideoByAddress(address) {
        const track = this._addressToVideo.get(address);
        if (track) {
            return track;
        }
        return null;
    }
    _onTrackSubscribed(room) {
        return (track, pub, participant) => {
            console.log('[LivekitManager] Track subscribed!', track);
            const address = this._getTrackAddress(pub, participant);
            this._saveTrack(address, track, pub, participant);
            this._helper.action(ON_ROOM_TRACK_SUBSCRIBED, null, this._trackArg(room.name, pub, participant, address, track));
            if (track.kind === this._livekit.Track.Kind.Audio ||
                track.kind === this._livekit.Track.Kind.Video) {
                this._onTrackNeedsAttachment.next(track);
            }
        };
    }
    _onTrackUnsubscribed(room) {
        return (track, pub, participant) => {
            console.log('[LivekitManager] Track unsubscribed!', track);
            const address = this._deleteTrack(track);
            if (address) {
                this._helper.action(ON_ROOM_TRACK_UNSUBSCRIBED, null, this._trackArg(room.name, pub, participant, address, track));
            }
            if (track.kind === this._livekit.Track.Kind.Audio ||
                track.kind === this._livekit.Track.Kind.Video) {
                this._onTrackNeedsDetachment.next(track);
            }
        };
    }
    _onLocalTrackPublished(room) {
        return (pub, participant) => {
            const track = pub.track;
            console.log('[LivekitManager] Track subscribed!', track);
            const address = this._getTrackAddress(pub, participant);
            this._saveTrack(address, track, pub, participant);
            this._helper.action(ON_ROOM_TRACK_SUBSCRIBED, null, this._trackArg(room.name, pub, participant, address, track));
            if (track.kind === this._livekit.Track.Kind.Video) {
                this._onTrackNeedsAttachment.next(track);
            }
        };
    }
    _onLocalTrackUnpublished(room) {
        return (pub, participant) => {
            const track = pub.track;
            console.log('[LivekitManager] Track unsubscribed!', track);
            const address = this._deleteTrack(track);
            if (address) {
                this._helper.action(ON_ROOM_TRACK_UNSUBSCRIBED, null, this._trackArg(room.name, pub, participant, address, track));
            }
            if (track.kind === this._livekit.Track.Kind.Video) {
                this._onTrackNeedsDetachment.next(track);
            }
        };
    }
    _trackArg(roomName, pub, participant, address, track) {
        return {
            roomName: roomName,
            address: address,
            ...this._getTrackOptions(pub, participant, track),
        };
    }
    _getTrackOptions(pub, participant, t) {
        const track = t ?? pub.track;
        const isRemote = pub instanceof this._livekit.RemoteTrackPublication;
        const common = {
            isRemote: isRemote,
            remoteId: participant.identity,
            muted: pub.isMuted || !pub.isEnabled,
            kind: this._getTrackKind(track),
            source: track.source,
        };
        if (pub.kind === this._livekit.Track.Kind.Video) {
            return {
                ...common,
                dimensions: pub.dimensions,
                aspectRatio: pub.dimensions.width / pub.dimensions.height,
                videoQuality: this._getTrackQuality(pub),
            };
        }
        else {
            return {
                ...common,
            };
        }
    }
    _onDisconnected(room) {
        return () => {
            console.log('[LivekitManager] Disconnected!');
            this._helper.action(ON_ROOM_STREAM_LOST, null, {
                roomName: room.name,
            });
        };
    }
    _onReconnected(room) {
        return () => {
            console.log('[LivekitManager] Reconnected!');
            this._helper.action(ON_ROOM_STREAMING, null, {
                roomName: room.name,
            });
        };
    }
    _onTrackMuted(room) {
        return (pub, participant) => {
            console.log('[LivekitManager] Track muted!', pub, participant);
            const address = this._trackToAddress.get(pub.track);
            this._helper.action(ON_ROOM_TRACK_UNSUBSCRIBED, null, this._trackArg(room.name, pub, participant, address, null));
        };
    }
    _onTrackUnmuted(room) {
        return (pub, participant) => {
            console.log('[LivekitManager] Track unmuted!', pub, participant);
            const address = this._trackToAddress.get(pub.track);
            this._helper.action(ON_ROOM_TRACK_SUBSCRIBED, null, this._trackArg(room.name, pub, participant, address, null));
        };
    }
    _onActiveSpeakersChanged(room) {
        return (speakers) => {
            this._helper.action(ON_ROOM_SPEAKERS_CHANGED, null, {
                roomName: room.name,
                speakerIds: speakers.map((s) => s.identity),
            });
        };
    }
    _onParticipantConnected(room) {
        return (participant) => {
            this._helper.action(ON_ROOM_REMOTE_JOINED, null, {
                roomName: room.name,
                remoteId: participant.identity,
            });
        };
    }
    _onParticipantDisconnected(room) {
        return (participant) => {
            this._helper.action(ON_ROOM_REMOTE_LEAVE, null, {
                roomName: room.name,
                remoteId: participant.identity,
            });
        };
    }
    _saveTrack(address, track, pub, participant) {
        this._addressToTrack.set(address, track);
        this._addressToPublication.set(address, pub);
        this._addressToParticipant.set(address, participant);
        if (track.kind === this._livekit.Track.Kind.Video) {
            this._addressToVideo.set(address, track.attach());
        }
        this._trackToAddress.set(track, address);
    }
    _deleteTrack(track) {
        const address = this._trackToAddress.get(track);
        if (address) {
            this._addressToTrack.delete(address);
            this._addressToVideo.delete(address);
            this._addressToPublication.delete(address);
            this._addressToParticipant.delete(address);
            this._trackToAddress.delete(track);
            return address;
        }
        return null;
    }
    _getTrackKind(track) {
        return track.kind === this._livekit.Track.Kind.Video
            ? 'video'
            : 'audio';
    }
    _getTrackQuality(publication) {
        if (publication instanceof this._livekit.RemoteTrackPublication) {
            if (publication.isMuted || !publication.isEnabled) {
                return 'off';
            }
            const quality = publication.videoQuality;
            if (quality === this._livekit.VideoQuality.HIGH) {
                return 'high';
            }
            else if (quality === this._livekit.VideoQuality.MEDIUM) {
                return 'medium';
            }
            else if (quality === this._livekit.VideoQuality.LOW) {
                return 'low';
            }
            else {
                return 'high';
            }
        }
        else if (publication.videoTrack) {
            return publication.isMuted || !publication.isEnabled
                ? 'off'
                : 'high';
        }
        return null;
    }
    _getTrackAddress(publication, participant) {
        if (publication.kind === this._livekit.Track.Kind.Video) {
            return `casualos://video-element/${participant.identity}-${publication.trackSid}`;
        }
        else {
            return `casualos://audio-element/${participant.identity}-${publication.trackSid}`;
        }
    }
}
//# sourceMappingURL=LivekitManager.js.map