@epicgames-ps/lib-pixelstreamingfrontend-ue5.4/src/WebRtcPlayer/WebRtcPlayerController.ts

Frontend library for Unreal Engine 5.4 Pixel Streaming

// Copyright Epic Games, Inc. All Rights Reserved.

import { WebSocketController } from '../WebSockets/WebSocketController';
import { ExtraOfferParameters, ExtraAnswerParameters } from '../WebSockets/MessageSend';
import { StreamController } from '../VideoPlayer/StreamController';
import {
    MessageAnswer,
    MessageOffer,
    MessageConfig,
    MessageStreamerList,
    MessageStreamerIDChanged
} from '../WebSockets/MessageReceive';
import { FreezeFrameController } from '../FreezeFrame/FreezeFrameController';
import { AFKController } from '../AFK/AFKController';
import { DataChannelController } from '../DataChannel/DataChannelController';
import { PeerConnectionController } from '../PeerConnectionController/PeerConnectionController';
import { KeyboardController } from '../Inputs/KeyboardController';
import { AggregatedStats } from '../PeerConnectionController/AggregatedStats';
import {
    Config,
    Flags,
    ControlSchemeType,
    TextParameters,
    OptionParameters,
    NumericParameters
} from '../Config/Config';
import {
    EncoderSettings,
    InitialSettings,
    WebRTCSettings
} from '../DataChannel/InitialSettings';
import { LatencyTestResults } from '../DataChannel/LatencyTestResults';
import { Logger } from '../Logger/Logger';
import { FileTemplate, FileUtil } from '../Util/FileUtil';
import { InputClassesFactory } from '../Inputs/InputClassesFactory';
import { VideoPlayer } from '../VideoPlayer/VideoPlayer';
import {
    StreamMessageController,
    MessageDirection
} from '../UeInstanceMessage/StreamMessageController';
import { ResponseController } from '../UeInstanceMessage/ResponseController';
import * as MessageReceive from '../WebSockets/MessageReceive';
import { MessageOnScreenKeyboard } from '../WebSockets/MessageReceive';
import { SendMessageController } from '../UeInstanceMessage/SendMessageController';
import { ToStreamerMessagesController } from '../UeInstanceMessage/ToStreamerMessagesController';
import { MouseController } from '../Inputs/MouseController';
import { GamePadController } from '../Inputs/GamepadController';
import { DataChannelSender } from '../DataChannel/DataChannelSender';
import {
    CoordinateConverter,
    UnquantizedDenormalizedUnsignedCoord
} from '../Util/CoordinateConverter';
import { PixelStreaming } from '../PixelStreaming/PixelStreaming';
import { ITouchController } from '../Inputs/ITouchController';
import {
    DataChannelCloseEvent,
    DataChannelErrorEvent,
    DataChannelOpenEvent,
    HideFreezeFrameEvent,
    LoadFreezeFrameEvent,
    PlayStreamErrorEvent,
    PlayStreamEvent,
    PlayStreamRejectedEvent,
    StreamerListMessageEvent,
    StreamerIDChangedMessageEvent
} from '../Util/EventEmitter';
import {
    DataChannelLatencyTestRequest,
    DataChannelLatencyTestResponse
} from "../DataChannel/DataChannelLatencyTestResults";
/**
 * Entry point for the WebRTC Player
 */
export class WebRtcPlayerController {
    config: Config;
    responseController: ResponseController;
    sdpConstraints: RTCOfferOptions;
    webSocketController: WebSocketController;
    // The primary data channel. This is bidirectional when p2p and send only when using an SFU
    sendrecvDataChannelController: DataChannelController;
    // A recv only data channel required when using an SFU
    recvDataChannelController: DataChannelController;
    dataChannelSender: DataChannelSender;
    datachannelOptions: RTCDataChannelInit;
    videoPlayer: VideoPlayer;
    streamController: StreamController;
    peerConnectionController: PeerConnectionController;
    inputClassesFactory: InputClassesFactory;
    freezeFrameController: FreezeFrameController;
    shouldShowPlayOverlay = true;
    afkController: AFKController;
    videoElementParentClientRect: DOMRect;
    latencyStartTime: number;
    pixelStreaming: PixelStreaming;
    streamMessageController: StreamMessageController;
    sendMessageController: SendMessageController;
    toStreamerMessagesController: ToStreamerMessagesController;
    keyboardController: KeyboardController;
    mouseController: MouseController;
    touchController: ITouchController;
    gamePadController: GamePadController;
    coordinateConverter: CoordinateConverter;
    isUsingSFU: boolean;
    isQualityController: boolean;
    statsTimerHandle: number;
    file: FileTemplate;
    preferredCodec: string;
    peerConfig: RTCConfiguration;
    videoAvgQp: number;
    locallyClosed: boolean;
    shouldReconnect: boolean;
    isReconnecting: boolean;
    reconnectAttempt: number;
    disconnectMessage: string;
    subscribedStream: string;
    signallingUrlBuilder: () => string;
    autoJoinTimer: ReturnType<typeof setTimeout> = undefined;

    /**
     *
     * @param config - the frontend config object
     * @param pixelStreaming - the PixelStreaming object
     */
    constructor(config: Config, pixelStreaming: PixelStreaming) {
        this.config = config;
        this.pixelStreaming = pixelStreaming;
        this.responseController = new ResponseController();
        this.file = new FileTemplate();

        this.sdpConstraints = {
            offerToReceiveAudio: true,
            offerToReceiveVideo: true
        };

        // set up the afk logic class and connect up its method for closing the signaling server
        this.afkController = new AFKController(
            this.config,
            this.pixelStreaming,
            this.onAfkTriggered.bind(this)
        );
        this.afkController.onAFKTimedOutCallback = () => {
            this.closeSignalingServer('You have been disconnected due to inactivity');
        };

        this.freezeFrameController = new FreezeFrameController(
            this.pixelStreaming.videoElementParent
        );

        this.videoPlayer = new VideoPlayer(
            this.pixelStreaming.videoElementParent,
            this.config
        );
        this.videoPlayer.onVideoInitialized = () =>
            this.handleVideoInitialized();

        // When in match viewport resolution mode, when the browser viewport is resized we send a resize command back to UE.
        this.videoPlayer.onMatchViewportResolutionCallback = (
            width: number,
            height: number
        ) => {
            const descriptor = {
                'Resolution.Width': width,
                'Resolution.Height': height
            };

            this.streamMessageController.toStreamerHandlers.get(
                'Command'
            )([JSON.stringify(descriptor)]);
        };

        // Every time video player is resized in browser we need to reinitialize the mouse coordinate conversion and freeze frame sizing logic.
        this.videoPlayer.onResizePlayerCallback = () => {
            this.setUpMouseAndFreezeFrame();
        };

        this.streamController = new StreamController(this.videoPlayer);

        this.coordinateConverter = new CoordinateConverter(this.videoPlayer);

        this.sendrecvDataChannelController = new DataChannelController();
        this.recvDataChannelController = new DataChannelController();
        this.registerDataChannelEventEmitters(
            this.sendrecvDataChannelController
        );
        this.registerDataChannelEventEmitters(this.recvDataChannelController);
        this.dataChannelSender = new DataChannelSender(
            this.sendrecvDataChannelController
        );
        this.dataChannelSender.resetAfkWarningTimerOnDataSend = () =>
            this.afkController.resetAfkWarningTimer();

        this.streamMessageController = new StreamMessageController();

        // set up websocket methods
        this.webSocketController = new WebSocketController();
        this.webSocketController.onConfig = (
            messageConfig: MessageReceive.MessageConfig
        ) => this.handleOnConfigMessage(messageConfig);
        this.webSocketController.onStreamerList = (
            messageList: MessageReceive.MessageStreamerList
        ) => this.handleStreamerListMessage(messageList);
        this.webSocketController.onStreamerIDChanged = (
            message: MessageReceive.MessageStreamerIDChanged
        ) => this.handleStreamerIDChangedMessage(message);
        this.webSocketController.onPlayerCount = (playerCount: MessageReceive.MessagePlayerCount) => { 
            this.pixelStreaming._onPlayerCount(playerCount.count); 
        };
        this.webSocketController.onOpen.addEventListener('open', () => {
            const BrowserSendsOffer = this.config.isFlagEnabled(
                Flags.BrowserSendOffer
            );
            if(!BrowserSendsOffer)
            {
                this.webSocketController.requestStreamerList();
            }
        });
        this.webSocketController.onClose.addEventListener('close', (event : CustomEvent) => {
            // when we refresh the page during a stream we get the going away code.
            // in that case we don't want to reconnect since we're navigating away.
            // https://developer.mozilla.org/en-US/docs/Web/API/CloseEvent/code
            // lists all the codes. 
            const CODE_GOING_AWAY = 1001;

            const willTryReconnect = this.shouldReconnect
               && event.detail.code != CODE_GOING_AWAY
               && this.config.getNumericSettingValue(NumericParameters.MaxReconnectAttempts) > 0

            const disconnectMessage = this.disconnectMessage ? this.disconnectMessage : event.detail.reason;
            this.pixelStreaming._onDisconnect(disconnectMessage, !willTryReconnect && !this.isReconnecting);

            this.afkController.stopAfkWarningTimer();

            // stop sending stats on interval if we have closed our connection
            if (this.statsTimerHandle && this.statsTimerHandle !== undefined) {
                window.clearInterval(this.statsTimerHandle);
            }

            // reset the stream quality icon.
            this.setVideoEncoderAvgQP(0);

            // unregister all input device event handlers on disconnect
            this.setTouchInputEnabled(false);
            this.setMouseInputEnabled(false);
            this.setKeyboardInputEnabled(false);
            this.setGamePadInputEnabled(false);

            if (willTryReconnect) {
                // need a small delay here to prevent reconnect spamming
                setTimeout(() => {
                    this.isReconnecting = true;
                    this.reconnectAttempt++;
                    this.tryReconnect(event.detail.reason);
                }, 2000);
            }
        });

        // set up the final webRtc player controller methods from within our application so a connection can be activated
        this.sendMessageController = new SendMessageController(
            this.dataChannelSender,
            this.streamMessageController
        );
        this.toStreamerMessagesController = new ToStreamerMessagesController(
            this.sendMessageController
        );
        this.registerMessageHandlers();
        this.streamMessageController.populateDefaultProtocol();

        this.inputClassesFactory = new InputClassesFactory(
            this.streamMessageController,
            this.videoPlayer,
            this.coordinateConverter
        );

        this.isUsingSFU = false;
        this.isQualityController = false;
        this.preferredCodec = '';
        this.shouldReconnect = true;
        this.isReconnecting = false;
        this.reconnectAttempt = 0;

        this.config._addOnOptionSettingChangedListener(
            OptionParameters.StreamerId,
            (streamerid) => {
                if(streamerid === "") {
                    return;
                }

                // close the current peer connection and create a new one
                this.peerConnectionController.peerConnection.close();
                this.peerConnectionController.createPeerConnection(
                    this.peerConfig,
                    this.preferredCodec
                );
                this.subscribedStream = streamerid;
                this.webSocketController.sendSubscribe(streamerid);
            }
        );

        this.setVideoEncoderAvgQP(-1);

        this.signallingUrlBuilder =  () => {
            let signallingServerUrl = this.config.getTextSettingValue(
                TextParameters.SignallingServerUrl
            );
    
            // If we are connecting to the SFU add a special url parameter to the url
            if (this.config.isFlagEnabled(Flags.BrowserSendOffer)) {
                signallingServerUrl += '?' + Flags.BrowserSendOffer + '=true';
            }
    
            // This code is no longer needed, but is a good example for how subsequent config flags can be appended
            // if (this.config.isFlagEnabled(Flags.BrowserSendOffer)) {
            //     signallingServerUrl += (signallingServerUrl.includes('?') ? '&' : '?') + Flags.BrowserSendOffer + '=true';
            // }
    
            return signallingServerUrl;
        }
    }

    /**
     * Make a request to UnquantizedAndDenormalizeUnsigned coordinates
     * @param x x axis coordinate
     * @param y y axis coordinate
     */
    requestUnquantizedAndDenormalizeUnsigned(
        x: number,
        y: number
    ): UnquantizedDenormalizedUnsignedCoord {
        return this.coordinateConverter.unquantizeAndDenormalizeUnsigned(x, y);
    }

    /**
     * Handles when a message is received
     * @param event - Message Event
     */
    handleOnMessage(event: MessageEvent) {
        const message = new Uint8Array(event.data);
        Logger.Log(Logger.GetStackTrace(), 'Message incoming:' + message, 6);

        //try {
        const messageType =
            this.streamMessageController.fromStreamerMessages.get(
                message[0]
            );
        this.streamMessageController.fromStreamerHandlers.get(messageType)(
            event.data
        );
        //} catch (e) {
        //Logger.Error(Logger.GetStackTrace(), `Custom data channel message with message type that is unknown to the Pixel Streaming protocol. Does your PixelStreamingProtocol need updating? The message type was: ${message[0]}`);
        //}
    }

    /**
     * Register message all handlers
     */
    registerMessageHandlers() {
        // From Streamer
        // Message events from the streamer have a data type of ArrayBuffer as we force this type in the DatachannelController
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'QualityControlOwnership',
            (data: ArrayBuffer) => this.onQualityControlOwnership(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'Response',
            (data: ArrayBuffer) => this.responseController.onResponse(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'Command',
            (data: ArrayBuffer) => {
                this.onCommand(data);
            }
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'FreezeFrame',
            (data: ArrayBuffer) => this.onFreezeFrameMessage(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'UnfreezeFrame',
            () => this.invalidateFreezeFrameAndEnableVideo()
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'VideoEncoderAvgQP',
            (data: ArrayBuffer) => this.handleVideoEncoderAvgQP(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'LatencyTest',
            (data: ArrayBuffer) => this.handleLatencyTestResult(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'DataChannelLatencyTest',
            (data: ArrayBuffer) => this.handleDataChannelLatencyTestResponse(data)
        )
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'InitialSettings',
            (data: ArrayBuffer) => this.handleInitialSettings(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'FileExtension',
            (data: ArrayBuffer) => this.onFileExtension(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'FileMimeType',
            (data: ArrayBuffer) => this.onFileMimeType(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'FileContents',
            (data: ArrayBuffer) => this.onFileContents(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'TestEcho',
            () => {
                /* Do nothing */
            }
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'InputControlOwnership',
            (data: ArrayBuffer) => this.onInputControlOwnership(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'GamepadResponse',
            (data: ArrayBuffer) => this.onGamepadResponse(data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.FromStreamer,
            'Protocol',
            (data: ArrayBuffer) => this.onProtocolMessage(data)
        );

        // To Streamer
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'IFrameRequest',
            () =>
                this.sendMessageController.sendMessageToStreamer(
                    'IFrameRequest'
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'RequestQualityControl',
            () =>
                this.sendMessageController.sendMessageToStreamer(
                    'RequestQualityControl'
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'FpsRequest',
            () => this.sendMessageController.sendMessageToStreamer('FpsRequest')
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'AverageBitrateRequest',
            () =>
                this.sendMessageController.sendMessageToStreamer(
                    'AverageBitrateRequest'
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'StartStreaming',
            () =>
                this.sendMessageController.sendMessageToStreamer(
                    'StartStreaming'
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'StopStreaming',
            () =>
                this.sendMessageController.sendMessageToStreamer(
                    'StopStreaming'
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'LatencyTest',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'LatencyTest', data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'RequestInitialSettings',
            () =>
                this.sendMessageController.sendMessageToStreamer(
                    'RequestInitialSettings'
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'TestEcho',
            () => {
                /* Do nothing */
            }
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'UIInteraction',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'UIInteraction', data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'Command',
            (data: Array<number | string>) => 
                this.sendMessageController.sendMessageToStreamer(
                    'Command', data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'TextboxEntry',
            (data: Array<number | string>) => 
                this.sendMessageController.sendMessageToStreamer(
                    'TextboxEntry', data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'KeyDown',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'KeyDown',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'KeyUp',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer('KeyUp', data)
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'KeyPress',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'KeyPress',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'MouseEnter',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'MouseEnter',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'MouseLeave',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'MouseLeave',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'MouseDown',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'MouseDown',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'MouseUp',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'MouseUp',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'MouseMove',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'MouseMove',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'MouseWheel',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'MouseWheel',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'MouseDouble',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'MouseDouble',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'TouchStart',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'TouchStart',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'TouchEnd',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'TouchEnd',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'TouchMove',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'TouchMove',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'GamepadConnected',
            () =>
                this.sendMessageController.sendMessageToStreamer(
                    'GamepadConnected'
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'GamepadButtonPressed',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'GamepadButtonPressed',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'GamepadButtonReleased',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'GamepadButtonReleased',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'GamepadAnalog',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'GamepadAnalog',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'GamepadDisconnected',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'GamepadDisconnected',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'XREyeViews',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'XREyeViews',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'XRHMDTransform',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'XRHMDTransform',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'XRControllerTransform',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'XRControllerTransform',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'XRSystem',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'XRSystem',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'XRButtonTouched',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'XRButtonTouched',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'XRButtonTouchReleased',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'XRButtonTouchReleased',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'XRButtonPressed',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'XRButtonPressed',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'XRButtonReleased',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'XRButtonReleased',
                    data
                )
        );
        this.streamMessageController.registerMessageHandler(
            MessageDirection.ToStreamer,
            'XRAnalog',
            (data: Array<number | string>) =>
                this.sendMessageController.sendMessageToStreamer(
                    'XRAnalog',
                    data
                )
        );
    }

    /**
     * Activate the logic associated with a command from UE
     * @param message
     */
    onCommand(message: ArrayBuffer) {
        Logger.Log(
            Logger.GetStackTrace(),
            'DataChannelReceiveMessageType.Command',
            6
        );
        const commandAsString = new TextDecoder('utf-16').decode(
            message.slice(1)
        );

        Logger.Log(
            Logger.GetStackTrace(),
            'Data Channel Command: ' + commandAsString,
            6
        );
        const command: MessageOnScreenKeyboard = JSON.parse(commandAsString);
        if (command.command === 'onScreenKeyboard') {
            this.pixelStreaming._activateOnScreenKeyboard(command);
        }
    }

    /**
     * Handles a protocol message received from the streamer
     * @param message the message data from the streamer
     */
    onProtocolMessage(message: ArrayBuffer) {
        try {
            const protocolString = new TextDecoder('utf-16').decode(
                message.slice(1)
            );
            const protocolJSON = JSON.parse(protocolString);
            if (
                !Object.prototype.hasOwnProperty.call(protocolJSON, 'Direction')
            ) {
                Logger.Error(
                    Logger.GetStackTrace(),
                    'Malformed protocol received. Ensure the protocol message contains a direction'
                );
            }
            const direction = protocolJSON.Direction;
            delete protocolJSON.Direction;
            Logger.Log(
                Logger.GetStackTrace(),
                `Received new ${
                    direction == MessageDirection.FromStreamer
                        ? 'FromStreamer'
                        : 'ToStreamer'
                } protocol. Updating existing protocol...`
            );
            Object.keys(protocolJSON).forEach((messageType) => {
                const message = protocolJSON[messageType];
                switch (direction) {
                    case MessageDirection.ToStreamer:
                        // Check that the message contains all the relevant params
                        if (
                            !Object.prototype.hasOwnProperty.call(
                                message,
                                'id'
                            )
                        ) {
                            Logger.Error(
                                Logger.GetStackTrace(),
                                `ToStreamer->${messageType} protocol definition was malformed as it didn't contain at least an id\n
                                           Definition was: ${JSON.stringify(
                                               message,
                                               null,
                                               2
                                           )}`
                            );
                            // return in a forEach is equivalent to a continue in a normal for loop
                            return;
                        }

                        // UE5.1 and UE5.2 don't send a structure for these message types, but they actually do have a structure so ignore updating them
                        if((messageType === "UIInteraction" || messageType === "Command" || messageType === "LatencyTest")) {
                            return;
                        }

                        if (
                            this.streamMessageController.toStreamerHandlers.get(
                                messageType
                            )
                        ) {
                            // If we've registered a handler for this message type we can add it to our supported messages. ie registerMessageHandler(...)
                            this.streamMessageController.toStreamerMessages.set(
                                messageType,
                                message
                            );
                        } else {
                            Logger.Error(
                                Logger.GetStackTrace(),
                                `There was no registered handler for "${messageType}" - try adding one using registerMessageHandler(MessageDirection.ToStreamer, "${messageType}", myHandler)`
                            );
                        }
                        break;
                    case MessageDirection.FromStreamer:
                        // Check that the message contains all the relevant params
                        if (
                            !Object.prototype.hasOwnProperty.call(message, 'id')
                        ) {
                            Logger.Error(
                                Logger.GetStackTrace(),
                                `FromStreamer->${messageType} protocol definition was malformed as it didn't contain at least an id\n
                            Definition was: ${JSON.stringify(message, null, 2)}`
                            );
                            // return in a forEach is equivalent to a continue in a normal for loop
                            return;
                        }
                        if (
                            this.streamMessageController.fromStreamerHandlers.get(
                                messageType
                            )
                        ) {
                            // If we've registered a handler for this message type. ie registerMessageHandler(...)
                            this.streamMessageController.fromStreamerMessages.set(
                                message.id,
                                messageType
                            );
                        } else {
                            Logger.Error(
                                Logger.GetStackTrace(),
                                `There was no registered handler for "${message}" - try adding one using registerMessageHandler(MessageDirection.FromStreamer, "${messageType}", myHandler)`
                            );
                        }
                        break;
                    default:
                        Logger.Error(
                            Logger.GetStackTrace(),
                            `Unknown direction: ${direction}`
                        );
                }
            });

            // Once the protocol has been received, we can send our control messages
            this.toStreamerMessagesController.SendRequestInitialSettings();
            this.toStreamerMessagesController.SendRequestQualityControl();
        } catch (e) {
            Logger.Log(Logger.GetStackTrace(), e);
        }
    }

    /**
     * Handles an input control message when it is received from the streamer
     * @param message The input control message
     */
    onInputControlOwnership(message: ArrayBuffer) {
        const view = new Uint8Array(message);
        Logger.Log(
            Logger.GetStackTrace(),
            'DataChannelReceiveMessageType.InputControlOwnership',
            6
        );
        const inputControlOwnership = new Boolean(view[1]).valueOf();
        Logger.Log(
            Logger.GetStackTrace(),
            `Received input controller message - will your input control the stream: ${inputControlOwnership}`
        );
        this.pixelStreaming._onInputControlOwnership(inputControlOwnership);
    }

    /**
     * 
     * @param message 
     */
    onGamepadResponse(message: ArrayBuffer) {
        const responseString = new TextDecoder('utf-16').decode(message.slice(1));
        const responseJSON = JSON.parse(responseString);
        this.gamePadController.onGamepadResponseReceived(responseJSON.controllerId);
    }

    onAfkTriggered(): void {
        this.afkController.onAfkClick();

        // if the stream is paused play it, if we can
        if (this.videoPlayer.isPaused() && this.videoPlayer.hasVideoSource()) {
            this.playStream();
        }
    }

    /**
     * Set whether we should timeout when afk.
     * @param afkEnabled If true we timeout when idle for some given amount of time.
     */
    setAfkEnabled(afkEnabled: boolean): void {
        if (afkEnabled) {
            this.onAfkTriggered();
        } else {
            this.afkController.stopAfkWarningTimer();
        }
    }

    /**
     * Attempt a reconnection to the signalling server
     */
    tryReconnect(message: string) {
        // if there is no webSocketController return immediately or this will not work
        if (!this.webSocketController) {
            Logger.Log(
                Logger.GetStackTrace(),
                'The Web Socket Controller does not exist so this will not work right now.'
            );
            return;
        }

        // if the connection is open, first close it. wait some time and try again.
        this.isReconnecting = true;
        if (this.webSocketController.webSocket && this.webSocketController.webSocket.readyState != WebSocket.CLOSED) {
            this.closeSignalingServer(`${message} Restarting stream...`);
            setTimeout(() => {
                this.tryReconnect(message);
            }, 3000);
        } else {
            this.pixelStreaming._onWebRtcAutoConnect();
            this.connectToSignallingServer();
        }
    }

    /**
     * Loads a freeze frame if it is required otherwise shows the play overlay
     */
    loadFreezeFrameOrShowPlayOverlay() {
        this.pixelStreaming.dispatchEvent(
            new LoadFreezeFrameEvent({
                shouldShowPlayOverlay: this.shouldShowPlayOverlay,
                isValid: this.freezeFrameController.valid,
                jpegData: this.freezeFrameController.jpeg
            })
        );
        if (this.shouldShowPlayOverlay === true) {
            Logger.Log(Logger.GetStackTrace(), 'showing play overlay');
            this.resizePlayerStyle();
        } else {
            Logger.Log(Logger.GetStackTrace(), 'showing freeze frame');
            this.freezeFrameController.showFreezeFrame();
        }
        setTimeout(() => {
            this.videoPlayer.setVideoEnabled(false);
        }, this.freezeFrameController.freezeFrameDelay);
    }

    /**
     * Process the freeze frame and load it
     * @param message The freeze frame data in bytes
     */
    onFreezeFrameMessage(message: ArrayBuffer) {
        Logger.Log(
            Logger.GetStackTrace(),
            'DataChannelReceiveMessageType.FreezeFrame',
            6
        );
        const view = new Uint8Array(message);
        this.freezeFrameController.processFreezeFrameMessage(view, () =>
            this.loadFreezeFrameOrShowPlayOverlay()
        );
    }

    /**
     * Enable the video after hiding a freeze frame
     */
    invalidateFreezeFrameAndEnableVideo() {
        Logger.Log(
            Logger.GetStackTrace(),
            'DataChannelReceiveMessageType.FreezeFrame',
            6
        );
        setTimeout(() => {
            this.pixelStreaming.dispatchEvent(
                new HideFreezeFrameEvent()
            );
            this.freezeFrameController.hideFreezeFrame();
        }, this.freezeFrameController.freezeFrameDelay);
        if (this.videoPlayer.getVideoElement()) {
            this.videoPlayer.setVideoEnabled(true);
        }
    }

    /**
     * Prep datachannel data for processing file extension
     * @param data the file extension data
     */
    onFileExtension(data: ArrayBuffer) {
        const view = new Uint8Array(data);
        FileUtil.setExtensionFromBytes(view, this.file);
    }

    /**
     * Prep datachannel data for processing the file mime type
     * @param data the file mime type data
     */
    onFileMimeType(data: ArrayBuffer) {
        const view = new Uint8Array(data);
        FileUtil.setMimeTypeFromBytes(view, this.file);
    }

    /**
     * Prep datachannel data for processing the file contents
     * @param data the file contents data
     */
    onFileContents(data: ArrayBuffer) {
        const view = new Uint8Array(data);
        FileUtil.setContentsFromBytes(view, this.file);
    }

    /**
     * Plays the stream audio and video source and sets up other pieces while the stream starts
     */
    playStream() {
        if (!this.videoPlayer.getVideoElement()) {
            const message =
                'Could not play video stream because the video player was not initialized correctly.';
            this.pixelStreaming.dispatchEvent(
                new PlayStreamErrorEvent({ message })
            );
            Logger.Error(Logger.GetStackTrace(), message);

            // close the connection
            this.closeSignalingServer('Stream not initialized correctly');
            return;
        }

        if (!this.videoPlayer.hasVideoSource()) {
            Logger.Warning(
                Logger.GetStackTrace(),
                'Cannot play stream, the video element has no srcObject to play.'
            );
            return;
        }

        this.setTouchInputEnabled(this.config.isFlagEnabled(Flags.TouchInput));
        this.pixelStreaming.dispatchEvent(new PlayStreamEvent());

        if (this.streamController.audioElement.srcObject) {
            const startMuted = this.config.isFlagEnabled(Flags.StartVideoMuted)
            this.streamController.audioElement.muted = startMuted;

            if (startMuted) {
              this.playVideo();
            } else {
                this.streamController.audioElement
                    .play()
                    .then(() => {
                        this.playVideo();
                    })
                    .catch((onRejectedReason) => {
                        Logger.Log(Logger.GetStackTrace(), onRejectedReason);
                        Logger.Log(
                            Logger.GetStackTrace(),
                            'Browser does not support autoplaying video without interaction - to resolve this we are going to show the play button overlay.'
                        );
                        this.pixelStreaming.dispatchEvent(
                            new PlayStreamRejectedEvent({
                                reason: onRejectedReason
                            })
                        );
                    });
            }
        } else {
            this.playVideo();
        }

        this.shouldShowPlayOverlay = false;
        this.freezeFrameController.showFreezeFrame();
    }

    /**
     * Plays the video stream
     */
    private playVideo() {
        // handle play() with promise as it is an asynchronous call
        this.videoPlayer.play().catch((onRejectedReason: string) => {
            if (this.streamController.audioElement.srcObject) {
                this.streamController.audioElement.pause();
            }
            Logger.Log(Logger.GetStackTrace(), onRejectedReason);
            Logger.Log(
                Logger.GetStackTrace(),
                'Browser does not support autoplaying video without interaction - to resolve this we are going to show the play button overlay.'
            );
            this.pixelStreaming.dispatchEvent(
                new PlayStreamRejectedEvent({ reason: onRejectedReason })
            );
        });
    }

    /**
     * Enable the video to play automatically if enableAutoplay is true
     */
    autoPlayVideoOrSetUpPlayOverlay() {
        if (this.config.isFlagEnabled(Flags.AutoPlayVideo)) {
            // attempt to play the video
            this.playStream();
        }
        this.resizePlayerStyle();
    }

    /**
     * Connect to the Signaling server
     */
    connectToSignallingServer() {
        this.locallyClosed = false;
        this.shouldReconnect = true;
        this.disconnectMessage = null;
        const signallingUrl = this.signallingUrlBuilder();
        this.webSocketController.connect(signallingUrl);
    }

    /**
     * This will start the handshake to the signalling server
     * @param peerConfig  - RTC Configuration Options from the Signaling server
     * @remark RTC Peer Connection on Ice Candidate event have it handled by handle Send Ice Candidate
     */
    startSession(peerConfig: RTCConfiguration) {
        this.peerConfig = peerConfig;
        // check for forcing turn
        if (this.config.isFlagEnabled(Flags.ForceTURN)) {
            // check for a turn server
            const hasTurnServer = this.checkTurnServerAvailability(peerConfig);

            // close and error if turn is forced and there is no turn server
            if (!hasTurnServer) {
                Logger.Info(
                    Logger.GetStackTrace(),
                    'No turn server was found in the Peer Connection Options. TURN cannot be forced, closing connection. Please use STUN instead'
                );
                this.closeSignalingServer('TURN cannot be forced, closing connection. Please use STUN instead.');
                return;
            }
        }

        // set up the peer connection controller
        this.peerConnectionController = new PeerConnectionController(
            this.peerConfig,
            this.config,
            this.preferredCodec
        );

        // set up peer connection controller video stats
        this.peerConnectionController.onVideoStats = (event: AggregatedStats) =>
            this.handleVideoStats(event);

        /* When the Peer Connection wants to send an offer have it handled */
        this.peerConnectionController.onSendWebRTCOffer = (
            offer: RTCSessionDescriptionInit
        ) => this.handleSendWebRTCOffer(offer);

        /* When the Peer Connection wants to send an answer have it handled */
        this.peerConnectionController.onSendWebRTCAnswer = (
            offer: RTCSessionDescriptionInit
        ) => this.handleSendWebRTCAnswer(offer);

        /* When the Peer Connection ice candidate is added have it handled */
        this.peerConnectionController.onPeerIceCandidate = (
            peerConnectionIceEvent: RTCPeerConnectionIceEvent
        ) => this.handleSendIceCandidate(peerConnectionIceEvent);

        /* When the Peer Connection has a data channel created for it by the browser, handle it */
        this.peerConnectionController.onDataChannel = (
            datachannelEvent: RTCDataChannelEvent
        ) => this.handleDataChannel(datachannelEvent);

        // set up webRtc text overlays
        this.peerConnectionController.showTextOverlayConnecting = () =>
            this.pixelStreaming._onWebRtcConnecting();
        this.peerConnectionController.showTextOverlaySetupFailure = () =>
            this.pixelStreaming._onWebRtcFailed();
        let webRtcConnectedSent = false;
        this.peerConnectionController.onIceConnectionStateChange = () => {
            // Browsers emit "connected" when getting first connection and "completed" when finishing
            // candidate checking. However, sometimes browsers can skip "connected" and only emit "completed".
            // Therefore need to check both cases and emit onWebRtcConnected only once on the first hit.
            if (!webRtcConnectedSent && 
                ["connected", "completed"].includes(this.peerConnectionController.peerConnection.iceConnectionState)) {
                this.pixelStreaming._onWebRtcConnected();
                webRtcConnectedSent = true;
            }
        };

        /* RTC Peer Connection on Track event -> handle on track */
        this.peerConnectionController.onTrack = (trackEvent: RTCTrackEvent) =>
            this.streamController.handleOnTrack(trackEvent);

        /* Start the Hand shake process by creating an Offer */
        const BrowserSendsOffer = this.config.isFlagEnabled(
            Flags.BrowserSendOffer
        );
        if (BrowserSendsOffer) {
            // If browser is sending the offer, create an offer and send it to the streamer
            this.sendrecvDataChannelController.createDataChannel(
                this.peerCon