UNPKG

@3dsource/angular-unreal-module

Version:

Angular Unreal module for connect with unreal engine stream

1,276 lines (1,246 loc) 290 kB
import * as i0 from '@angular/core'; import { InjectionToken, inject, Injectable, ChangeDetectionStrategy, Component, Pipe, DestroyRef, signal, ElementRef, input, HostListener, Input, ViewChild, computed, output } from '@angular/core'; import { filter, withLatestFrom, distinctUntilChanged, switchMap, first, catchError, map as map$1, tap, delay, takeUntil, debounceTime, exhaustMap, takeWhile, skip as skip$1 } from 'rxjs/operators'; import { createAction, props, createReducer, on, createFeature, Store, createSelector } from '@ngrx/store'; import { Actions, createEffect, ofType } from '@ngrx/effects'; import { skip, share, merge, Subject, interval, map, from, take, fromEvent, timer, of, combineLatest, switchMap as switchMap$1, timeout, retryWhen, tap as tap$1, startWith, combineLatestWith, takeUntil as takeUntil$1, auditTime, EMPTY, debounceTime as debounceTime$1, scan, concatMap, animationFrameScheduler, Observable, BehaviorSubject, first as first$1, distinctUntilChanged as distinctUntilChanged$1, concat } from 'rxjs'; import { concatLatestFrom, mapResponse } from '@ngrx/operators'; import { Falsy, Truthy, Logger, calculateMedian, clampf, Signal, tapLog, generateUuid, COLOR_CODES, where, KeyboardNumericCode, InvertedKeyMap, Semaphore, isEmpty, lerp, getCanvasCached, getSnapshot, whereNot, HEXtoRGB, RGBtoHSV, inverseLerp, HSVtoRGB, RGBtoHEX, fpIsASameAsB, fitIntoRectangle } from '@3dsource/utils'; import { toSignal, takeUntilDestroyed, toObservable } from '@angular/core/rxjs-interop'; import { HttpClient } from '@angular/common/http'; import { DialogRef, DIALOG_DATA, Dialog } from '@angular/cdk/dialog'; import { ScrollStrategyOptions } from '@angular/cdk/overlay'; import { SourceButtonComponent, SourceIconButtonComponent, SourceLoadingComponent } from '@3dsource/source-ui-native'; import { MetaBoxCommand } from '@3dsource/types-unreal'; import { NgClass, NgOptimizedImage, AsyncPipe, JsonPipe } from '@angular/common'; import { DomSanitizer } from '@angular/platform-browser'; import * as i1 from '@angular/forms'; import { FormsModule } from '@angular/forms'; const WSCloseCode_NORMAL_CLOSURE = 3000; const WSCloseCode_NORMAL_AFK_TIMEOUT = 3001; const WSCloseCode_NORMAL_MANUAL_DISCONNECT = 3002; const WSCloseCode_NORMAL_CIRRUS_CLOSED = 3003; const WSCloseCodes = [ WSCloseCode_NORMAL_CLOSURE, WSCloseCode_NORMAL_AFK_TIMEOUT, WSCloseCode_NORMAL_MANUAL_DISCONNECT, WSCloseCode_NORMAL_CIRRUS_CLOSED, ]; const DisconnectReason = { afk: 'afk', none: 'none', reset: 'reset', wsOnError: 'wsOnError', dataChannelClosed: 'dataChannelClosed', dataChannelTimeout: 'dataChannelTimeout', }; /* export interface RequestReservation extends InstanceMessageBase { type: 'requestReservation'; } */ const OrchestrationMessageTypes = { streamerList: 'streamerList', listStreamers: 'listStreamers', requestReservation: 'requestReservation', instanceReady: 'instanceReady', instanceReserved: 'instanceReserved', ssInfo: 'ssInfo', playerCount: 'playerCount', answer: 'answer', iceCandidate: 'iceCandidate', ping: 'ping', config: 'config', offer: 'offer', }; const UnrealInternalSignalEvents = { OnVideoInitialized: 'OnVideoInitialized', RestAfkTimer: 'RestAfkTimer', FreezeFrameStart: 'FreezeFrameStart', UnrealCallback: 'UnrealCallback', ClickableOverlay: 'ClickableOverlay', VideoPlayClick: 'VideoPlayClick', ShowPlayButton: 'ShowPlayButton', ForceResizeViewport: 'ForceResizeViewport', VideoAdaptedToContainer: 'VideoAdaptedToContainer', TelemetryStart: 'TelemetryStart', TelemetryStop: 'TelemetryStop', TelemetryReset: 'TelemetryReset', }; function AnswerHandler(msg) { this.onWebRtcAnswer$.next(msg); } function ConfigHandler(msg) { this.onConfig$.next(msg); } function IceCandidateHandler(msg) { this.onWebRtcIce$.next(msg.candidate); } function InstanceReadyHandler(_) { } const scoped = (templateString) => `[UNREAL] ${templateString[0]}`; const trackMixpanelEvent = createAction(scoped `track mixpanel event`, props()); const changeLowBandwidth = createAction(scoped `change low bandwidth`, props()); const setMaxFps = createAction(scoped `change fps`, props()); const destroyRemoteConnections = createAction(scoped `destroyRemoteConnections`, props()); const destroyConnectionsAndResetState = createAction(scoped `destroyConnections and reset`); const setCirrusConnected = createAction(scoped `cirrusConnected`); const setCirrusDisconnected = createAction(scoped `cirrusDisconnected`); const changeStatusMainVideoOnScene = createAction(scoped `change status main video on scene`, props()); const setAwsInstance = createAction(scoped `set aws instance`, props()); const setStatusMessage = createAction(scoped `set status message`, props()); const setStatusPercentSignallingServer = createAction(scoped `change status percent signalling server`, props()); const setFreezeFrame = createAction(scoped `set freeze frame`, props()); const setMatchUrls = createAction(scoped `set match urls`, props()); const setStreamClientCompanyId = createAction(scoped `set stream client company id`, props()); const setStreamViewId = createAction(scoped `set stream view id`, props()); const setIntroImageSrc = createAction(scoped `set intro image src`, props()); const setLoadingImageSrc = createAction(scoped `set loading image src`, props()); const setIntroVideoSrc = createAction(scoped `set intro video src`, props()); const resetIntroSrc = createAction(scoped `reset intro src`); const setFreezeFrameFromVideo = createAction(scoped `set freeze frame from video`, props()); const setEstablishingConnection = createAction(scoped `set establishing connection`, props()); const setDataChannelConnected = createAction(scoped `set data channel connected`, props()); const setConfig = createAction(scoped `set config`, props()); const setErrorMessage = createAction(scoped `set error message`, props()); const setViewportReady = createAction(scoped `set viewport ready`, props()); const changeStreamResolutionAction = createAction(scoped `change stream resolution`, props()); const changeStreamResolutionSuccessAction = createAction(scoped `change stream resolution success action`, props()); const setSignalingName = createAction(scoped `set aws instanceName`, props()); const updateCirrusInfo = createAction(scoped `set unrealInfo`, props()); const commandStarted = createAction(scoped `command started`, props()); const commandCompleted = createAction(scoped `command completed`, props()); const setLoopBackCommandIsCompleted = createAction(scoped `set loopBack command is completed`); const showUnrealErrorMessage = createAction(scoped `show unreal error message`, props()); const initSignalling = createAction(scoped `init signalling`); const resetConfig = createAction(scoped `reset config`); const resetAfkAction = createAction(scoped `reset afk action`); const resetWarnTimeout = createAction(scoped `reset config warn timeout`); const resetUnrealStateAction = createAction(scoped `reset state`); const resetUnrealState = createAction(scoped `reset unreal state`); const SpecialKeyCodes = { BackSpace: 8, Shift: 16, Control: 17, Alt: 18, RightShift: 253, RightControl: 254, RightAlt: 255, }; // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/button const MouseButton = { MainButton: 0, // Left button. AuxiliaryButton: 1, // Wheel button. SecondaryButton: 2, // Right button. FourthButton: 3, // Browser Back button. FifthButton: 4, // Browser Forward button. }; // https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/buttons const MouseButtonsMask = { PrimaryButton: 1, // Left button. SecondaryButton: 2, // Right button. AuxiliaryButton: 4, // Wheel button. FourthButton: 8, // Browser Back button. FifthButton: 16, // Browser Forward button. }; const EControlSchemeType = { // A mouse can lock inside the WebRTC player so the user can simply move the // mouse to control the orientation of the camera. The user presses the // Escape key to unlock the mouse. LockedMouse: 0, // A mouse can hover over the WebRTC player so the user needs to click and // drag to control the orientation of the camera. HoveringMouse: 1, Default: 2, }; // Must be kept in sync with PixelStreamingProtocol::EToUE4Msg C++ descriptors. const EMessageType = { /* * Control Messages. Range = 0..49. */ IFrameRequest: 0, RequestQualityControl: 1, MaxFpsRequest: 2, AverageBitrateRequest: 3, StartStreaming: 4, StopStreaming: 5, LatencyTest: 6, RequestInitialSettings: 7, /* * Input Messages. Range = 50..89. */ // Generic Input Messages. Range = 50..59. UIInteraction: 50, Command: 51, // Keyboard Input Message. Range = 60..69. KeyDown: 60, KeyUp: 61, KeyPress: 62, // Mouse Input Messages. Range = 70..79. MouseEnter: 70, MouseLeave: 71, MouseDown: 72, MouseUp: 73, MouseMove: 74, MouseWheel: 75, // Touch Input Messages. Range = 80..89. TouchStart: 80, TouchEnd: 81, TouchMove: 82, }; const EToClientMessageType = { QualityControlOwnership: 0, Response: 1, Command: 2, FreezeFrame: 3, UnfreezeFrame: 4, VideoEncoderAvgQP: 5, LatencyTest: 6, InitialSettings: 7, InputControlOwnership: 12, Protocol: 255, }; const UNREAL_CONFIG = new InjectionToken('Unreal Initial Configuration'); const InputOptions = { controlScheme: EControlSchemeType.Default, suppressBrowserKeys: true, fakeMouseWithTouches: false, }; const UnrealStatusMessage = { CONNECTING_TO_SESSION: 'Connecting to session.', STARTING_YOUR_SESSION: 'Starting your session', }; const DEBOUNCE_TO_MANY_RESIZE_CALLS = 100; const SAME_SIZE_THRESHOLD = 1.01; const MINIMAL_FPS = 6; const STREAMING_VIDEO_ID = 'streamingVideo'; const CONSOLE_COMMAND_ENABLE_MESSAGES = 'EnableAllScreenMessages'; const CONSOLE_COMMAND_DISABLE_MESSAGES = 'DisableAllScreenMessages'; const CONSOLE_COMMAND_PIXEL_QUALITY = 'PixelStreaming.FreezeFrameQuality 95'; const FULL_HD_WIDTH = 1920; const FULL_HD_HEIGHT = 1080; const WS_TIMEOUT = 2000; const WS_OPEN_STATE = 1; const DEFAULT_TIMEOUT_PERIOD = 15; const DEFAULT_WARN_TIMEOUT = 120; const DATA_CHANNEL_CONNECTION_TIMEOUT = 8000; // 8000 ms; const SIGNALLING_PERCENT_VALUE = 56; const SCREEN_LOCKER_CONTAINER_ID = '3dsource_start_screen'; const initialState = { lowBandwidthStats: undefined, wasInitialized: false, isFirstSuccessLoad: false, lowBandwidth: false, cirrusConnected: false, establishingConnection: false, viewportReady: false, dataChannelConnected: false, isVideoPlaying: false, statusPercentSignallingServer: null, statusMessage: null, errorMessage: null, ssInfo: null, ssData: null, streamResolution: { width: null, height: null }, freezeFrameFromVideo: { dataUrl: null, progress: null }, freezeFrame: { dataUrl: null, progress: null }, disconnectReason: DisconnectReason.none, awsInstance: { wsUrl: null, instanceName: null, pollingUrl: null, }, streamConfig: { autoStart: true, warnTimeout: DEFAULT_WARN_TIMEOUT, matchMakerUrls: [], }, loaderCommands: { commandsInProgress: [], totalCommandsStarted: 0, totalCommandsCompleted: 0, }, matchUrls: [], streamClientCompanyId: '', streamViewId: 'default', videoIntroSrc: null, imageIntroSrc: null, imageLoadingSrc: '', }; const unrealReducer = createReducer(initialState, on(changeLowBandwidth, (state, { lowBandwidth, stats }) => { return { ...state, lowBandwidth: lowBandwidth, lowBandwidthStats: lowBandwidth ? stats : undefined, }; }), on(changeStatusMainVideoOnScene, (state, { isVideoPlaying }) => { return { ...state, isVideoPlaying: isVideoPlaying, }; }), on(setAwsInstance, (state, { instanceName, wsUrl, pollingUrl }) => { return { ...state, awsInstance: { instanceName, wsUrl, pollingUrl, }, }; }), on(setViewportReady, (state, { value }) => { return { ...state, viewportReady: value, statusMessage: value ? null : state.statusMessage, errorMessage: value ? null : state.errorMessage, }; }), on(updateCirrusInfo, (state, { ssInfo, ssData }) => { return { ...state, ssInfo: ssInfo, // For back compatibility ssData: ssData, // Contains all the data from the ssInfo as object }; }), on(changeStreamResolutionSuccessAction, (state, { width, height }) => { return { ...state, streamResolution: { width: width, height: height, }, }; }), on(setFreezeFrame, (state, freezeFrame) => { return { ...state, freezeFrame: { dataUrl: freezeFrame.progress === 0 || freezeFrame.progress === 1 || freezeFrame.progress === null ? freezeFrame.dataUrl : state.freezeFrame.dataUrl, progress: freezeFrame.progress || null, }, }; }), on(setErrorMessage, (state, errorMessage) => { if (state.dataChannelConnected) { return state; } return { ...state, errorMessage: errorMessage, statusMessage: null, }; }), on(setFreezeFrameFromVideo, (state, freezeFrameFromVideo) => { return { ...state, freezeFrameFromVideo: { dataUrl: freezeFrameFromVideo.dataUrl, progress: freezeFrameFromVideo.progress || null, }, }; }), on(setStatusMessage, (state, { message }) => { return { ...state, statusMessage: message, }; }), on(setEstablishingConnection, (state, { value }) => { return { ...state, establishingConnection: value, }; }), on(setStatusPercentSignallingServer, (state, { percent }) => { return { ...state, statusPercentSignallingServer: percent, }; }), on(setDataChannelConnected, (state, { value }) => { return { ...state, dataChannelConnected: value, wasInitialized: value ? true : state.wasInitialized, }; }), on(setConfig, (state, { config }) => { return { ...state, streamConfig: { ...state.streamConfig, ...config }, }; }), on(resetConfig, (state) => { return { ...state, streamConfig: initialState.streamConfig, }; }), on(resetWarnTimeout, (state) => { return { ...state, streamConfig: { ...state.streamConfig, warnTimeout: DEFAULT_WARN_TIMEOUT, }, }; }), on(setCirrusConnected, (state) => { return { ...state, cirrusConnected: true, }; }), on(setCirrusDisconnected, (state) => { return { ...state, cirrusConnected: false, }; }), on(initSignalling, (state) => { return { ...state, disconnectReason: DisconnectReason.none, }; }), on(setSignalingName, (state, { instanceName }) => { return { ...state, awsInstance: { ...state.awsInstance, instanceName }, }; }), on(setLoopBackCommandIsCompleted, (state) => { return { ...state, isFirstSuccessLoad: true, }; }), on(setMatchUrls, (state, { urls }) => { return { ...state, matchUrls: urls, }; }), on(setStreamClientCompanyId, (state, { id }) => { return { ...state, streamClientCompanyId: id, }; }), on(setStreamViewId, (state, { id }) => { return { ...state, streamViewId: id, }; }), on(setIntroImageSrc, (state, { src }) => { return { ...state, imageIntroSrc: src, }; }), on(setLoadingImageSrc, (state, { src }) => { return { ...state, imageLoadingSrc: src, }; }), on(setIntroVideoSrc, (state, { src }) => { return { ...state, videoIntroSrc: src, }; }), on(resetIntroSrc, (state) => { return { ...state, imageIntroSrc: '', videoIntroSrc: '', }; }), on(commandStarted, (state, { id, command }) => { return { ...state, loaderCommands: { ...state.loaderCommands, totalCommandsStarted: state.loaderCommands.totalCommandsStarted + 1, commandsInProgress: [ ...state.loaderCommands.commandsInProgress, { id, command, timeStamp: new Date().getTime() }, ], }, }; }), on(commandCompleted, (state, { id }) => { return { ...state, loaderCommands: removeExileCommands(state.loaderCommands, id), }; }), on(resetUnrealState, (state) => { return { ...initialState, wasInitialized: state.wasInitialized, isFirstSuccessLoad: state.isFirstSuccessLoad, matchUrls: state.matchUrls, streamClientCompanyId: state.streamClientCompanyId, streamViewId: state.streamViewId, imageIntroSrc: state.imageIntroSrc, videoIntroSrc: state.videoIntroSrc, imageLoadingSrc: state.imageLoadingSrc, }; }), on(resetUnrealStateAction, () => { return initialState; })); const unrealFeature = createFeature({ name: 'unrealFeature', reducer: unrealReducer, }); class SubService { constructor() { this.store = inject(Store); this.disconnect$ = this.store .select(unrealFeature.selectCirrusConnected) .pipe(skip(1), filter(Falsy), share()); } } class AFKService extends SubService { constructor() { super(); // Optionally detect if the user is not interacting (AFK) and disconnect them. this.enabled = true; // Set to true to enable the AFK system. this.closeTimeout = DEFAULT_TIMEOUT_PERIOD; // The time after the warning when we disconnect the user. this.active = false; // Whether the AFK system is currently looking for inactivity. this.countdown = 0; // The inactivity warning overlay has a countdown to show time until disconnect. this.selectWarnTimeout = toSignal(this.store.select(selectWarnTimeout)); this.initAfk(); } initAfk() { merge(this.store.select(selectWarnTimeout), fromSignal(UnrealInternalSignalEvents.RestAfkTimer)) .pipe(withLatestFrom(this.store.select(unrealFeature.selectViewportReady)), filter(([, viewportReady]) => viewportReady)) .subscribe(() => this.resetAfkWarningTimer()); this.store .select(unrealFeature.selectViewportReady) .pipe(distinctUntilChanged(), filter(Truthy)) .subscribe(() => this.startAfkWarningTimer()); this.disconnect$.subscribe(() => this.stop()); } init() { // do nothing, just to not forget to initialize Service } hideOverlay() { sendSignal(UnrealInternalSignalEvents.ClickableOverlay); } /** * Start a timer which when elapsed will warn the user they are inactive. */ startAfkWarningTimer() { this.active = this.enabled; this.resetAfkWarningTimer(); } /** * If the user interacts, then reset the warning timer. */ resetAfkWarningTimer() { if (this.active) { this.stop(); this.warnTimer = setTimeout(() => { this.showAfkOverlay(); }, this.selectWarnTimeout() * 1000); } } /** * Update the count-down spans number for the overlay * @param countdown the count down number to be inserted into the span for updating */ updateCountDown(countdown) { this.dispatchMessage(String(countdown)); } /** * Update the text overlays inner text * @param message the update text to be inserted into the overlay */ dispatchMessage(message) { sendSignal(UnrealInternalSignalEvents.ClickableOverlay, { message, isActivityDetected: true, className: 'clickableState', onOverlayClick: () => this.reset(), }); } stop() { clearInterval(this.countdownTimer); clearTimeout(this.warnTimer); this.hideOverlay(); } reset() { this.hideOverlay(); clearInterval(this.countdownTimer); this.startAfkWarningTimer(); } showAfkOverlay() { // Pause the timer while the user is looking at the inactivity warning overlay. this.active = false; this.countdown = this.closeTimeout; this.updateCountDown(this.countdown); if (InputOptions.controlScheme == EControlSchemeType.LockedMouse) { document.exitPointerLock(); } this.countdownTimer = setInterval(() => { this.countdown--; if (this.countdown === 0) { // The user failed to click so disconnect them this.hideOverlay(); // TODO possible way (Blinking), because postponed close event: destroyRemoteConnections({ disconnectReason: DisconnectReason.afk }), this.store.dispatch(destroyConnectionsAndResetState()); clearInterval(this.countdownTimer); } else { this.updateCountDown(this.countdown); } }, 1000); } static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: AFKService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); } static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: AFKService }); } } i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: AFKService, decorators: [{ type: Injectable }], ctorParameters: () => [] }); class FreezeFrameService extends SubService { constructor() { super(...arguments); this.receiving = false; this.size = 0; this.freezeFrameOverlay = new Image(); } init() { this.store .select(unrealFeature.selectViewportReady) .pipe(filter(Truthy)) .subscribe(() => this.invalidate()); } setData(view) { view = view.slice(1 + 4); const jpeg = new Uint8Array(this.jpeg.length + view.length); jpeg.set(this.jpeg, 0); jpeg.set(view, this.jpeg.length); this.jpeg = jpeg; if (this.jpeg.length === this.size) { this.receiving = false; Logger.info(`FRAME: Received complete freeze frame ${this.size}`); this.showFreezeFrame(); } else if (this.jpeg.length > this.size) { Logger.error(`FRAME: Received bigger freeze frame than advertised: ${this.jpeg.length}/${this.size}`); this.store.dispatch(setFreezeFrame({ dataUrl: null, progress: 0 })); this.jpeg = undefined; this.receiving = false; } else { this.dispatchInProgress(); /* Logger.warn( `received next chunk (${view.length} bytes) of freeze frame: ${this.jpeg.length}/${this.size}`, );*/ } } dispatchInProgress() { this.store.dispatch(setFreezeFrame({ dataUrl: null, progress: (this.jpeg?.length || 0) / this.size, })); } start(view) { this.size = new DataView(view.slice(1, 5).buffer).getInt32(0, true); this.jpeg = view.slice(1 + 4); if (this.jpeg.length < this.size) { this.dispatchInProgress(); sendSignal(UnrealInternalSignalEvents.FreezeFrameStart); Logger.info(`FRAME: Received first chunk of freeze frame: ${this.jpeg.length}/${this.size}`); this.receiving = true; } else { Logger.info(`FRAME: Received complete freeze frame: ${this.jpeg.length}/${this.size}`); this.receiving = false; this.showFreezeFrame(); } } invalidate() { this.store.dispatch(setFreezeFrame({ dataUrl: null, progress: null })); this.jpeg = undefined; this.receiving = false; } showFreezeFrame() { if (!this.jpeg) { return; } const base64 = btoa(this.jpeg?.reduce((data, byte) => data + String.fromCharCode(byte), '')); this.freezeFrameOverlay.src = 'data:image/jpeg;base64,' + base64; this.store.dispatch(setFreezeFrame({ dataUrl: this.freezeFrameOverlay.src, progress: 1, })); } static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: FreezeFrameService, deps: null, target: i0.ɵɵFactoryTarget.Injectable }); } static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: FreezeFrameService }); } } i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: FreezeFrameService, decorators: [{ type: Injectable }] }); class DataFlowMonitor { /** * Initializes the DataFlowMonitor monitor. * @param yellowFlagThresholdPercentage - The percentage drop to trigger a YELLOW warning (default: 15%). * @param redFlagThresholdPercentage - The percentage drop to trigger a RED warning (default: 30%). * @param historyBufferLength - buffer length (default: 100). * @param splitPoint - The point at which to split the history buffer into two halves (default: 0.5). */ constructor(yellowFlagThresholdPercentage = 15, redFlagThresholdPercentage = 30, historyBufferLength = 100, splitPoint = 0.5) { this.yellowFlagThresholdPercentage = yellowFlagThresholdPercentage; this.redFlagThresholdPercentage = redFlagThresholdPercentage; this.historyBufferLength = historyBufferLength; this.splitPoint = splitPoint; this.dataHistory = []; } reset() { this.dataHistory.length = 0; } config(data) { this.reset(); this.yellowFlagThresholdPercentage = data.yellowFlag; this.redFlagThresholdPercentage = data.redFlag; } /** * Adds a new bitrate measurement and checks for significant drops. * @param currentValue - The current bitrate in kbps. * @returns BitrateCheckResult indicating if a drop was detected. */ addValue(currentValue) { if (isNaN(currentValue) || currentValue < 1) { return {}; } const historyLength = this.dataHistory.length; if (historyLength > this.historyBufferLength) { this.dataHistory.shift(); } this.dataHistory.push(currentValue); if (historyLength < this.historyBufferLength) { this.dataHistory.length = this.historyBufferLength; this.dataHistory.fill(currentValue, 0, this.historyBufferLength); } const splitIndex = Math.floor(historyLength * this.splitPoint); const firstHalf = this.dataHistory.slice(0, splitIndex); const secondHalf = this.dataHistory.slice(splitIndex); const firstHalfMedian = calculateMedian(firstHalf); const secondHalfMedian = calculateMedian(secondHalf); const dropPercentage = clampf(0, 100, ((firstHalfMedian - secondHalfMedian) / firstHalfMedian) * 100); const isRedFlag = dropPercentage >= this.redFlagThresholdPercentage; if (dropPercentage >= this.yellowFlagThresholdPercentage) { return { config: { yellowFlagThresholdPercentage: this.yellowFlagThresholdPercentage, redFlagThresholdPercentage: this.redFlagThresholdPercentage, historyBufferLength: this.historyBufferLength, splitPoint: this.splitPoint, }, isDropDetected: true, dropPercentage, dataHistory: [...this.dataHistory], activeMedian: secondHalfMedian, quality: isRedFlag ? 'red' : 'orange', message: `Significant flow drop detected: ${dropPercentage.toFixed(2)}% (from ${firstHalfMedian} to ${secondHalfMedian})`, }; } return { config: { yellowFlagThresholdPercentage: this.yellowFlagThresholdPercentage, redFlagThresholdPercentage: this.redFlagThresholdPercentage, historyBufferLength: this.historyBufferLength, splitPoint: this.splitPoint, }, isDropDetected: false, dropPercentage, dataHistory: [...this.dataHistory], activeMedian: secondHalfMedian, quality: 'lime', message: 'Stable flow', }; } } /** * Default LBM Filter Parameters */ const DefaultFilterModel = { minimumBitrate: 1600, yellowFlag: 30, redFlag: 50, minimumFps: 5, monitoringDelayTime: 5000, initialBitrateEstimate: 1600, initialErrorCovariance: 1, processNoise: 3, measurementNoise: 500, panelOpen: false, }; /** * Global LBM Filter Parameters */ const FilterModel = { ...DefaultFilterModel }; /** * Bitrate Monitor Static Class */ const BITRATE_MONITOR = new DataFlowMonitor(FilterModel.yellowFlag, FilterModel.redFlag, 20); class VideoService extends SubService { constructor() { super(); this.latencyTestTimings = new LatencyTimings(); this.videoTrack$ = new Subject(); this.VideoEncoderQP = 0; this.aggregatedStats = {}; this.kalmanFilter1D = new KalmanFilter1D(FilterModel.initialBitrateEstimate, FilterModel.initialErrorCovariance, FilterModel.processNoise, FilterModel.measurementNoise); /** * Aggregate video stats and emit it as videoStats$ */ this.videoStats$ = this.videoTrack$.pipe( // IMPORTANT! DO NOT CHANGE THOSE NUMBERS, LBM Stats are based on those values switchMap(({ pcClient }) => interval(250).pipe(map(() => pcClient))), switchMap((pcClient) => from(this.getStats(pcClient))), filter(Truthy), share()); Signal.on('setKalmanParams').subscribe((data) => { this.kalmanFilter1D.config(data); BITRATE_MONITOR.config(data); }); } setContainer(container) { this.container = container; } setLatencyTimings(latencyTimings) { this.latencyTestTimings.SetUETimings(latencyTimings); } setEncoder(data) { this.VideoEncoderQP = Number(new TextDecoder('utf-16').decode(data)); } create() { this.disconnect$.pipe(take(1)).subscribe(() => this.destroy()); this.destroy(); this.createWebRtcVideo(); this.createWebRtcAudio(); } attachVideoStream(stream, pcClient) { if (this.video) { this.video.srcObject = stream; this.videoTrack$.next({ stream, pcClient }); } else { console.error('Video element is not defined.'); } } attachAudioStream(stream) { // do nothing the video has the same media stream as the audio track we have here (they are linked) if (this.video?.srcObject === stream) { return; } // video element has some other media stream that is not associated with this audio track else if (this.audio && this.video?.srcObject && this.video?.srcObject !== stream) { this.audio.srcObject = stream; this.playAudio(this.audio); } } play() { void this.video?.play(); } async getStats(pcClient) { if (!pcClient) { return null; } const stats = await pcClient.getStats(null); return this.generateAggregatedStatsFunction(stats); } generateAggregatedStatsFunction(stats) { const newStat = {}; // store each type of codec we can get stats on newStat.codecs = {}; newStat.currentRoundTripTime = -1; stats.forEach((stat) => { // Get the inbound-rtp for video if (stat.type === 'inbound-rtp' && stat.kind === 'video') { Object.assign(newStat, stat); newStat.bytesReceivedStart = this.aggregatedStats && this.aggregatedStats.bytesReceivedStart ? this.aggregatedStats.bytesReceivedStart : stat.bytesReceived; newStat.framesDecodedStart = this.aggregatedStats && this.aggregatedStats.framesDecodedStart ? this.aggregatedStats.framesDecodedStart : stat.framesDecoded; newStat.timestampStart = this.aggregatedStats && this.aggregatedStats.timestampStart ? this.aggregatedStats.timestampStart : stat.timestamp; if (this.aggregatedStats && this.aggregatedStats.timestamp) { // Get the mimetype of the video codec being used if (stat.codecId && this.aggregatedStats.codecs && Object.hasOwn(this.aggregatedStats.codecs, stat.codecId)) { newStat.videoCodec = this.aggregatedStats.codecs[stat.codecId]; } if (this.aggregatedStats.bytesReceived) { // bitrate = bits received since last time / number of ms since last time // This is automatically in kbits (where k=1000) since time is in ms and stat we want is in seconds (so a '* 1000' then a '/ 1000' would negate each other) newStat.bitrate = Math.floor((8 * (newStat.bytesReceived - this.aggregatedStats.bytesReceived)) / (newStat.timestamp - this.aggregatedStats.timestamp)); } if (this.aggregatedStats.bytesReceivedStart) { newStat.avgBitrate = Math.floor((8 * (newStat?.bytesReceived - this.aggregatedStats.bytesReceivedStart)) / (newStat.timestamp - this.aggregatedStats.timestampStart)); } if (this.aggregatedStats.framesDecodedStart) { newStat.avgFrameRate = Math.floor((newStat.framesDecoded - this.aggregatedStats.framesDecodedStart) / ((newStat.timestamp - this.aggregatedStats.timestampStart) / 1000)); } } } if (stat.type === 'candidate-pair' && Object.hasOwn(stat, 'currentRoundTripTime')) { newStat.currentRoundTripTime = stat.currentRoundTripTime ?? 0; } // Store mimetype of each codec if (Object.hasOwn(newStat, 'codecs') && stat.type === 'codec' && stat.mimeType && stat.id) { const codecId = stat.id; newStat.codecs[codecId] = stat.mimeType .replace('video/', '') .replace('audio/', ''); } }); newStat.pixelRatio = +window.devicePixelRatio.toFixed(2); if (this.aggregatedStats.receiveToCompositeMs) { newStat.receiveToCompositeMs = this.aggregatedStats.receiveToCompositeMs; this.latencyTestTimings.SetFrameDisplayDeltaTime(this.aggregatedStats.receiveToCompositeMs); } // Calculate duration of run newStat.runTime = Math.floor((newStat.timestamp - newStat.timestampStart) / 1000); newStat.kalmanBitrate = Math.floor(this.kalmanFilter1D.update(newStat.bitrate || FilterModel.initialBitrateEstimate)); newStat.dataFlowCheckResult = BITRATE_MONITOR.addValue(newStat.kalmanBitrate); newStat.bitrateDrop = Math.floor(newStat.dataFlowCheckResult.dropPercentage); this.aggregatedStats = newStat; return this.onAggregatedStats(newStat); } onAggregatedStats(aggregatedStats) { return { quality: mapQpToQuality(this.VideoEncoderQP), aggregatedStats: { ...aggregatedStats, VideoEncoderQP: this.VideoEncoderQP, }, }; } destroy() { this.video = null; this.audio = null; while (this.container?.firstChild) { this.container.removeChild(this.container.firstChild); } this.store.dispatch(changeStatusMainVideoOnScene({ isVideoPlaying: false })); } createWebRtcVideo() { dispatchResize(); const video = document.createElement('video'); video.id = STREAMING_VIDEO_ID; video.playsInline = true; video.muted = true; video.autoplay = true; video.preload = 'auto'; video.disablePictureInPicture = true; this.video = video; this.container?.appendChild(this.video); this.video.addEventListener('play', () => { this.store.dispatch(changeStatusMainVideoOnScene({ isVideoPlaying: true })); }); fromEvent(this.video, 'loadedmetadata') .pipe(first(), filter((data) => this.video === data.target), tapLog('VideoService loadedmetadata:')) .subscribe(() => sendSignal(UnrealInternalSignalEvents.OnVideoInitialized, this.video)); } createWebRtcAudio() { const audio = new Audio(); audio.id = 'streamingAudio'; this.audio = audio; } playAudio(audio) { from(audio.play()) .pipe(catchError(() => fromEvent(document, 'click')), take(1)) .subscribe({ next: () => audio.play(), error: () => ({}), }); } static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: VideoService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); } static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: VideoService }); } } i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: VideoService, decorators: [{ type: Injectable }], ctorParameters: () => [] }); class CommandTelemetryService { constructor() { this.appId = 'metabox-telemetry'; this.commandsSent = {}; this.exileTimout = 60000; this.pollingTime = 5000; this.commandSequenceNumber = 0; this.payloads = []; this.httpClient = inject(HttpClient); this.unrealInitialConfig = inject(UNREAL_CONFIG, { optional: true, }); if (!this.unrealInitialConfig?.commandTelemetryReceiver) { return; } this.reset(); this.sessionId = sessionStorage.getItem(this.appId) || this.uuid; this.userId = localStorage.getItem(this.appId) || this.uuid; sessionStorage.setItem(this.appId, this.sessionId); localStorage.setItem(this.appId, this.userId); this.start(); } get timeNow() { return Date.now(); } get sessionTime() { return this.timeNow - this.startTime; } get canSkipSending() { return !(this.timeNow - this.lastTime > this.pollingTime * 2); } decorate(funcToDecorate) { this.listenCallbacks(); return this.wrapper.bind(this, funcToDecorate); } start() { fromSignal(UnrealInternalSignalEvents.TelemetryStart).subscribe(({ externalId }) => { this.trackStartCommand(`EXT-${externalId}`); }); fromSignal(UnrealInternalSignalEvents.TelemetryStop).subscribe(({ externalId, payload }) => { const trackingId = `EXT-${externalId}`; this.trackStopCommand(trackingId, undefined, payload?.multi, payload); }); fromSignal(UnrealInternalSignalEvents.TelemetryReset).subscribe(() => { this.reset(); }); timer(0, this.pollingTime).subscribe(() => { this.send(this.payloads); }); } reset() { this.uuid = generateUuid(); this.commandSequenceNumber = 0; this.viewId = this.uuid; this.startTime = this.timeNow; this.payloads.length = 0; this.commandsSent = {}; } trackStartCommand(trackingId) { const out = { command: trackingId, trackingId, }; const timers = { sessionElapsedTime: this.sessionTime, }; this.pushStatToBuffer('commandSent', out, timers); this.trackTime(out); } trackStopCommand(trackingId, timeStampsRaw, multi = false, response = {}) { const commandResponse = this.commandsSent[trackingId]; if (!commandResponse) { return; } const totalRoundTripTime = this.timeNow - commandResponse.timeStampJs; /** * Time between telemetry was reset and command was sent. */ const commandSentJsTime = commandResponse.timeStampJs - this.startTime; let unrealTimers = {}; const t = timeStampsRaw?.timeStampCommand; if (t) { unrealTimers = { unrealRoundTripTime: t.onCompleteCommand.timestamp - t.onRequestCommand.timestamp, unrealExecutionDuration: t.onCompleteCommand.timestamp - t.onStartCommand.timestamp, timeDiffRequest: t.onRequestCommand.timestamp - commandResponse.timeStampJs, timeDiffCallBack: this.timeNow - t.onCompleteCommand.timestamp, }; } const timers = { sessionElapsedTime: this.sessionTime, commandSentJsTime, totalRoundTripTime, ...unrealTimers, }; this.pushStatToBuffer('commandStop', { ...commandResponse.command, response }, timers, timeStampsRaw); if (!multi) { delete this.commandsSent[trackingId]; } } pushStatToBuffer(type, commandContent, computedTimers, timeStampsRaw) { this.pushData({ type, commandSequenceNumber: this.commandSequenceNumber++, commandContent, timeStampsRaw, computedTimers, }); } trackTime(command) { this.commandsSent[command.trackingId] = { timeStampJs: this.timeNow, command, }; } send(payload) { if (payload.length === 0 || (this.canSkipSending && payload.length < 10)) { return; } this.lastTime = this.timeNow; const out = { appId: this.appId, sessionId: this.sessionId, viewId: this.viewId, userId: this.userId, ua: navigator.userAgent, timeStamp: new Date(this.timeNow).toString(), href: location.href, resolution: { width: window.screen.width, height: window.screen.height, pixelRatio: window.devicePixelRatio, }, payload, }; const lastIndex = payload[payload.length - 1].commandSequenceNumber; this.httpClient .post(this.unrealInitialConfig?.commandTelemetryReceiver || '', out) .pipe(first(), catchError(() => of(null))) .subscribe(() => { this.payloads = this.payloads.filter((p) => p.commandSequenceNumber > lastIndex); }); } /** * Listens for Unreal Engine callbacks. * Subscribes to the Unreal Engine event loop back and filters out events that do not have a tracking ID. * When a callback with a tracking ID is received, * it stops tracking the time for that command and filters out commands * that have exceeded the timeout limit. */ listenCallbacks() { fromSignal(UnrealInternalSignalEvents.UnrealCallback) .pipe(filter(({ json }) => { const callback = 'commandCallback' in json ? json.commandCallback : json; return Truthy(callback?.trackingId); })) .subscribe(({ json }) => { const callback = json.commandCallback; const timeStampsRaw = json.timeStamp; const trackingId = callback?.trackingId || ''; this.trackStopCommand(trackingId, timeStampsRaw); this.removeExileCommands(); }); } pushData(payload) { this.payloads.push(payload); } /** * Removes commands that have exceeded the timeout limit. * Iterates over all the commands sent and checks if the difference between the current time * and the time the command was sent is greater than the timeout limit. * If it is, the command is marked for deletion. * After checking all commands, those marked for deletion are removed from the commands sent. * This method is used to ensure that commands that are not responded to within a certain time frame * do not remain in the commandsSent object indefinitely, which could lead to memory leaks over time. */ removeExileCommands() { const time = new Date().getTime(); const markForDelete = []; Object.entries(this.commandsSent).forEach(([trackingId, { timeStampJs }]) => { if (time - timeStampJs > this.exileTimout) { markForDelete.push(trackingId); } }); markForDelete.forEach((trackingId) => { delete this.commandsSent[trackingId]; }); } /** * Wraps the provided function with telemetry tracking. * Generates a unique tracking ID and adds it to the data object. * Pushes the command sent to the payloads with the tracking ID. * Starts tracking the time for the command. * Finally, calls the provided function with the modified data object. * * @param {IToBeDecorated} funcToDecorate - The function to be decorated with telemetry tracking. * @param {any} data - The data object to be passed to the function. It will be augmented with a unique tracking ID. */ wrapper(funcToDecorate, data) { const out = { ...data, trackingId: generateUuid() }; const timers = { sessionElapsedTime: this.sessionTime, }; this.pushStatToBuffer('commandSent', { ...out, trackingId: `${out.trackingId}`, }, timers); this.trackTime(out); funcToDecorate(out); } static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: CommandTelemetryService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); } static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: CommandTelemetryService }); } } i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.1.3", ngImport: i0, type: CommandTelemetryService, decorators: [{ type: Injectable }], ctorParameters: () => [] }); function TelemetryStart(externalId) { sendSignal(UnrealInternalSignalEvents.TelemetryStart, { externalId }); } function TelemetryStop(externalId, payload) { sendSignal(UnrealInternalSignalEvents.TelemetryStop, { externalId, payload }); } function ResetTelemetry() { sendSignal(UnrealInternalSignalEvents.TelemetryReset); } function StreamerListHandler(msg) { this.send(JSON.stringify({ type: 'subscribe', streamerId: msg.ids[0], })); } function OfferHandler(msg) { this.onOffer$.next(msg); } class SignallingService extends SubService { constructor() { super(); this.httpClient = inject(HttpClient); this.regionsPingService = inject(RegionsPingService); this.onOffer$ = new Subject(); this.onConfig$ = new Subject(); this.onWebRtcIce$ = new Subject(); this.onWebRtcAnswer$ = new Subject(); this.wsMsgHandlers = {}; this.establishingConnection = toSignal(this.store.select(unrealFeature.selectEstablishingConnection)); this.isCirrusConnected = toSignal(this.store.select(unrealFeature.selectCirrusConnected)); this.establishingConnectionDrop$ = this.store .select(unrealFeature.selectEstablishingConnection) .pipe(skip(1), filter(Falsy), tapLog('selectEstablishingConnection'), share()); this.setHandlersFromStream(); this.store .select(unrealFeature.selectDataChannelConnected) .pipe(filter(Truthy)) .subscribe(() => { this.send(JSON.stringify({ type: 'p2pEstablished', source: 'front', sessionId: generateUuid(), })); }); combineLatest([ this.store .select(selectMatchUrls) .pipe(tapLog('MatchMakerUrls changed:')), this.store.select(selectIsAutostart).pipe(tapLog('Autostart is:')), ]) .pipe(filter(([, autoStart]) => autoStart), map$1(([matchMakerUrls]) => [...matchMakerUrls].filter(Truthy)), filter((urls) => urls.length > 0)) .subscribe((urls) => this.connectToSignaling(urls)); this.store .select(selectWsUrl) .pipe(filter(Truthy)) .subscribe((url) => { this.initWebSocket(url); }); } initEstablishingConnection() { this.showStatusMessage(UnrealStatusMessage.STARTING_YOUR_SESSION); this.store.dispatch(setEstablishingConnection({ value: true })); } stopEstablishingConnection() { this.store.dispatch(setEstablishingConnection({ value: false })); }