@3dsource/angular-unreal-module
Version:
Angular Unreal module for connect with unreal engine stream
1,211 lines (1,180 loc) • 319 kB
JavaScript
import * as i0 from '@angular/core';
import { InjectionToken, inject, Injectable, signal, makeEnvironmentProviders, provideEnvironmentInitializer, ChangeDetectionStrategy, Component, Pipe, ElementRef, input, HostListener, Input, ViewChild, DestroyRef, computed, viewChild, effect, untracked, output, afterNextRender } from '@angular/core';
import { filter, withLatestFrom, distinctUntilChanged, switchMap, catchError, timeout, tap, map as map$1, takeUntil, exhaustMap, debounceTime, takeWhile, delay, skip as skip$1 } from 'rxjs/operators';
import { createAction, props, Store, provideState, createReducer, on, createFeature, createSelector } from '@ngrx/store';
import { Actions, ofType, provideEffects, createEffect } from '@ngrx/effects';
import { skip, share, merge, Subject, interval, of, map, from, take, fromEvent, timer, throwError, defer, Observable, switchMap as switchMap$1, retry, timeout as timeout$1, tap as tap$1, startWith, takeUntil as takeUntil$1, auditTime, EMPTY, debounceTime as debounceTime$1, mergeMap, scan, concatMap, animationFrameScheduler, BehaviorSubject, combineLatestWith, distinctUntilChanged as distinctUntilChanged$1, concat } from 'rxjs';
import { Falsy, Truthy, Logger, calculateMedian, clampf, Signal, tapLog, generateUuid, COLOR_CODES, where, KeyboardNumericCode, InvertedKeyMap, Semaphore, lerp, getCanvasCached, getSnapshot, whereNot, HEXtoRGB, RGBtoHSV, inverseLerp, HSVtoRGB, RGBtoHEX, fpIsASameAsB, fitIntoRectangle } from '@3dsource/utils';
import { HttpClient } from '@angular/common/http';
import { DialogRef, DIALOG_DATA, Dialog } from '@angular/cdk/dialog';
import { ScrollStrategyOptions } from '@angular/cdk/overlay';
import { MetaBoxCommand } from '@3dsource/types-unreal';
import { SourceButtonComponent, SourceLoadingComponent } from '@3dsource/source-ui-native';
import * as i1 from '@angular/common';
import { CommonModule, NgOptimizedImage, AsyncPipe, JsonPipe } from '@angular/common';
import { toSignal, takeUntilDestroyed, toObservable } from '@angular/core/rxjs-interop';
import { DomSanitizer } from '@angular/platform-browser';
import * as i1$1 from '@angular/forms';
import { FormsModule } from '@angular/forms';
const WSCloseCode_NORMAL_CLOSURE = 3000;
const WSCloseCode_NORMAL_AFK_TIMEOUT = 3001;
const WSCloseCode_NORMAL_MANUAL_DISCONNECT = 3002;
const WSCloseCode_FORCE_CIRRUS_CLOSE = 3003;
const WSCloseCode_UNKNOWN = 1000;
const WSCloseCode_CIRRUS_PLAYER_DISCONNECTED = 1001;
const WSCloseCode_CIRRUS_ABNORMAL_CLOSURE = 1006;
const WSCloseCode_CIRRUS_MAX_PLAYERS_ERROR = 1013;
const WSCloseCode_CIRRUS_STREAMER_KIKED_PLAYER = 1011;
const WSCloseCodes = [
WSCloseCode_NORMAL_CLOSURE,
WSCloseCode_NORMAL_AFK_TIMEOUT,
WSCloseCode_NORMAL_MANUAL_DISCONNECT,
WSCloseCode_FORCE_CIRRUS_CLOSE,
WSCloseCode_UNKNOWN,
WSCloseCode_CIRRUS_PLAYER_DISCONNECTED,
WSCloseCode_CIRRUS_ABNORMAL_CLOSURE,
WSCloseCode_CIRRUS_MAX_PLAYERS_ERROR,
WSCloseCode_CIRRUS_STREAMER_KIKED_PLAYER,
];
const DisconnectReason = {
Afk: 'Afk',
None: 'None',
Destroy: 'Destroy',
DataChannelClosed: 'DataChannelClosed',
DataChannelTimeout: 'DataChannelTimeout',
WebRTCError: 'WebRTCError',
WebSocketError: 'WebSocketError',
WebSocketClose: 'WebSocketClose',
DropConnection: 'DropConnection',
OrchestrationPlayerDisconnected: 'OrchestrationPlayerDisconnected',
OrchestrationStreamerDisconnected: 'OrchestrationStreamerDisconnected',
};
/*
export interface RequestReservation extends InstanceMessageBase {
type: 'requestReservation';
}
*/
const OrchestrationMessageTypes = {
playerDisconnected: 'playerDisconnected',
streamerDisconnected: 'streamerDisconnected',
interruptClientStream: 'interruptClientStream',
error: 'error',
requestStream: 'requestStream',
streamerList: 'streamerList',
listStreamers: 'listStreamers',
instanceReady: 'instanceReady',
instanceReserved: 'instanceReserved',
ssInfo: 'ssInfo',
playerCount: 'playerCount',
answer: 'answer',
iceCandidate: 'iceCandidate',
p2pEstablished: 'p2pEstablished',
ping: 'ping',
config: 'config',
offer: 'offer',
};
const UnrealInternalSignalEvents = {
OnVideoInitialized: 'OnVideoInitialized',
RestAfkTimer: 'RestAfkTimer',
FreezeFrameStart: 'FreezeFrameStart',
UnrealCallback: 'UnrealCallback',
ClickableOverlay: 'ClickableOverlay',
VideoPlayClick: 'VideoPlayClick',
ShowPlayButton: 'ShowPlayButton',
ForceResizeViewport: 'ForceResizeViewport',
VideoAdaptedToContainer: 'VideoAdaptedToContainer',
TelemetryStart: 'TelemetryStart',
TelemetryStop: 'TelemetryStop',
TelemetryReset: 'TelemetryReset',
};
function AnswerHandler(msg) {
this.onWebRtcAnswer$.next(msg);
}
function ConfigHandler(msg) {
this.onConfig$.next(msg);
}
function IceCandidateHandler(msg) {
this.onWebRtcIce$.next(msg.candidate);
}
function InstanceReadyHandler() {
this.send({
type: OrchestrationMessageTypes.listStreamers,
correlationId: this.correlationId,
});
}
const scoped = (templateString) => `[UNREAL] ${templateString[0]}`;
const trackMixpanelEvent = createAction(scoped `track mixpanel event`, props());
const changeLowBandwidth = createAction(scoped `change low bandwidth`, props());
const setMaxFps = createAction(scoped `change fps`, props());
const destroyRemoteConnections = createAction(scoped `destroyRemoteConnections`, props());
const destroyUnrealScene = createAction(scoped `destroyUnrealScene`);
const setCirrusConnected = createAction(scoped `cirrusConnected`);
const setCirrusDisconnected = createAction(scoped `cirrusDisconnected`);
const changeStatusMainVideoOnScene = createAction(scoped `change status main video on scene`, props());
const setAwsInstance = createAction(scoped `set aws instance`, props());
const setStatusMessage = createAction(scoped `set status message`, props());
const setStatusPercentSignallingServer = createAction(scoped `change status percent signalling server`, props());
const setFreezeFrame = createAction(scoped `set freeze frame`, props());
const setStreamClientCompanyId = createAction(scoped `set stream client company id`, props());
const setStreamViewId = createAction(scoped `set stream view id`, props());
const setIntroImageSrc = createAction(scoped `set intro image src`, props());
const setLoadingImageSrc = createAction(scoped `set loading image src`, props());
const setIntroVideoSrc = createAction(scoped `set intro video src`, props());
const resetIntroSrc = createAction(scoped `reset intro src`);
const setFreezeFrameFromVideo = createAction(scoped `set freeze frame from video`, props());
const setEstablishingConnection = createAction(scoped `set establishing connection`, props());
const setDataChannelConnected = createAction(scoped `set data channel connected`, props());
const setConfig = createAction(scoped `set config`, props());
const disconnectStream = createAction(scoped `disconnectStream`, props());
const dropConnection = createAction(scoped `dropConnection`);
const setViewportReady = createAction(scoped `set viewport ready`);
const setViewportNotReady = createAction(scoped `set viewport not ready`);
const changeStreamResolutionAction = createAction(scoped `change stream resolution`, props());
const changeStreamResolutionSuccessAction = createAction(scoped `change stream resolution success action`, props());
const setSignalingName = createAction(scoped `set aws instanceName`, props());
const setOrchestrationParameters = createAction(scoped `set orchestration parameters`, props());
const setOrchestrationMessage = createAction(scoped `set orchestration message`, props());
const setOrchestrationProgress = createAction(scoped `set orchestration progress`, props());
const setOrchestrationContext = createAction(scoped `set orchestration params`, props());
const updateCirrusInfo = createAction(scoped `set unrealInfo`, props());
const commandStarted = createAction(scoped `command started`, props());
const commandCompleted = createAction(scoped `command completed`, props());
const setLoopBackCommandIsCompleted = createAction(scoped `set loopBack command is completed`);
const setAfkTimerVisible = createAction(scoped `set afk timer visible`);
const setAfkTimerHide = createAction(scoped `set afk timer hide`);
const showUnrealErrorMessage = createAction(scoped `show unreal error message`, props());
const initSignalling = createAction(scoped `init signalling`, (data = {
resetDisconnectionReason: true,
}) => ({
resetDisconnectionReason: data.resetDisconnectionReason,
}));
const startStream = createAction(scoped `startStream`, props());
const resetConfig = createAction(scoped `reset config`);
const resetAfkAction = createAction(scoped `reset afk action`);
const resetWarnTimeout = createAction(scoped `reset config warn timeout`);
const abortEstablishingConnection = createAction(scoped `abortEstablishingConnection`);
const setUnrealPlaywrightConfig = createAction(scoped `set unreal playwright config`);
const SpecialKeyCodes = {
BackSpace: 8,
Shift: 16,
Control: 17,
Alt: 18,
RightShift: 253,
RightControl: 254,
RightAlt: 255,
};
// https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/button
const MouseButton = {
MainButton: 0, // Left button.
AuxiliaryButton: 1, // Wheel button.
SecondaryButton: 2, // Right button.
FourthButton: 3, // Browser Back button.
FifthButton: 4, // Browser Forward button.
};
// https://developer.mozilla.org/en-US/docs/Web/API/MouseEvent/buttons
const MouseButtonsMask = {
PrimaryButton: 1, // Left button.
SecondaryButton: 2, // Right button.
AuxiliaryButton: 4, // Wheel button.
FourthButton: 8, // Browser Back button.
FifthButton: 16, // Browser Forward button.
};
const EControlSchemeType = {
// A mouse can lock inside the WebRTC player so the user can simply move the
// mouse to control the orientation of the camera. The user presses the
// Escape key to unlock the mouse.
LockedMouse: 0,
// A mouse can hover over the WebRTC player so the user needs to click and
// drag to control the orientation of the camera.
HoveringMouse: 1,
Default: 2,
};
// Must be kept in sync with PixelStreamingProtocol::EToUE4Msg C++ descriptors.
const EMessageType = {
/*
* Control Messages. Range = 0..49.
*/
IFrameRequest: 0,
RequestQualityControl: 1,
MaxFpsRequest: 2,
AverageBitrateRequest: 3,
StartStreaming: 4,
StopStreaming: 5,
LatencyTest: 6,
RequestInitialSettings: 7,
/*
* Input Messages. Range = 50..89.
*/
// Generic Input Messages. Range = 50..59.
UIInteraction: 50,
Command: 51,
// Keyboard Input Message. Range = 60..69.
KeyDown: 60,
KeyUp: 61,
KeyPress: 62,
// Mouse Input Messages. Range = 70..79.
MouseEnter: 70,
MouseLeave: 71,
MouseDown: 72,
MouseUp: 73,
MouseMove: 74,
MouseWheel: 75,
// Touch Input Messages. Range = 80..89.
TouchStart: 80,
TouchEnd: 81,
TouchMove: 82,
};
const EToClientMessageType = {
QualityControlOwnership: 0,
Response: 1,
Command: 2,
FreezeFrame: 3,
UnfreezeFrame: 4,
VideoEncoderAvgQP: 5,
LatencyTest: 6,
InitialSettings: 7,
FileExtension: 8,
FileMimeType: 9,
FileContents: 10,
TestEcho: 11,
InputControlOwnership: 12,
GamepadResponse: 13,
Protocol: 255,
};
const UNREAL_CONFIG = new InjectionToken('Unreal Initial Configuration');
const InputOptions = {
controlScheme: EControlSchemeType.Default,
suppressBrowserKeys: true,
fakeMouseWithTouches: false,
};
const UnrealStatusMessage = {
CONNECTING_TO_SESSION: 'Connecting to session.',
STARTING_YOUR_SESSION: 'Starting your session',
};
const DEBOUNCE_TO_MANY_RESIZE_CALLS = 100;
const SAME_SIZE_THRESHOLD = 1.01;
const MINIMAL_FPS = 6;
const STREAMING_VIDEO_ID = 'streamingVideo';
const CONSOLE_COMMAND_ENABLE_MESSAGES = 'EnableAllScreenMessages';
const CONSOLE_COMMAND_DISABLE_MESSAGES = 'DisableAllScreenMessages';
const CONSOLE_COMMAND_PIXEL_QUALITY = 'PixelStreaming.FreezeFrameQuality 95';
const FULL_HD_WIDTH = 1920;
const FULL_HD_HEIGHT = 1080;
const WS_TIMEOUT = 2000;
const POLLING_TIME = 4000;
const WS_OPEN_STATE = 1;
const DEFAULT_AFK_TIMEOUT_PERIOD = 15;
const DEFAULT_AFK_TIMEOUT = 120;
const DATA_CHANNEL_CONNECTION_TIMEOUT = 8000; //ms;
const SIGNALLING_PERCENT_VALUE = 56;
const SCREEN_LOCKER_CONTAINER_ID = '3dsource_start_screen';
class SubService {
constructor() {
this.store = inject(Store);
this.disconnect$ = this.store
.select(unrealFeature.selectCirrusConnected)
.pipe(skip(1), filter(Falsy), share());
}
}
class AFKService extends SubService {
constructor() {
super();
// Optionally detect if the user is not interacting (AFK) and disconnect them.
this.enabled = true; // Set to true to enable the AFK system.
this.closeTimeout = DEFAULT_AFK_TIMEOUT_PERIOD; // The time after the warning when we disconnect the user.
this.active = false; // Whether the AFK system is currently looking for inactivity.
this.countdown = 0; // The inactivity warning overlay has a countdown to show time until disconnect.
this.selectWarnTimeout = this.store.selectSignal(selectWarnTimeout);
this.isViewportReady = this.store.selectSignal(unrealFeature.selectViewportReady);
this.init();
}
init() {
merge(this.store.select(selectWarnTimeout), fromSignal(UnrealInternalSignalEvents.RestAfkTimer))
.pipe(withLatestFrom(this.store.select(unrealFeature.selectViewportReady)), filter(([, viewportReady]) => viewportReady))
.subscribe(() => this.resetAfkWarningTimer());
this.store
.select(unrealFeature.selectViewportReady)
.pipe(distinctUntilChanged(), filter(Truthy))
.subscribe(() => this.startAfkWarningTimer());
this.disconnect$.subscribe(() => this.stop());
}
hideOverlay() {
sendSignal(UnrealInternalSignalEvents.ClickableOverlay);
this.store.dispatch(setAfkTimerHide());
}
/**
* Start a timer which when elapsed will warn the user they are inactive.
*/
startAfkWarningTimer() {
this.active = this.enabled;
this.resetAfkWarningTimer();
}
/**
* If the user interacts, then reset the warning timer.
*/
resetAfkWarningTimer() {
if (this.active) {
this.clearTimers();
this.warnTimer = setTimeout(() => {
this.showAfkOverlay();
}, this.selectWarnTimeout() * 1000);
}
}
/**
* Update the count-down spans number for the overlay
* @param countdown the count down number to be inserted into the span for updating
*/
updateCountDown(countdown) {
this.dispatchMessage(String(countdown));
}
/**
* Update the text overlays inner text
* @param message the update text to be inserted into the overlay
*/
dispatchMessage(message) {
sendSignal(UnrealInternalSignalEvents.ClickableOverlay, {
message,
isActivityDetected: true,
className: 'clickableState',
onOverlayClick: () => this.reset(),
});
}
clearTimers() {
clearInterval(this.countdownTimer);
clearTimeout(this.warnTimer);
}
stop() {
this.clearTimers();
this.hideOverlay();
}
reset() {
this.hideOverlay();
clearInterval(this.countdownTimer);
this.startAfkWarningTimer();
}
showAfkOverlay() {
if (!this.isViewportReady()) {
return;
}
// Pause the timer while the user is looking at the inactivity warning overlay.
this.active = false;
this.countdown = this.closeTimeout;
this.store.dispatch(setAfkTimerVisible());
this.updateCountDown(this.countdown);
if (InputOptions.controlScheme == EControlSchemeType.LockedMouse) {
document.exitPointerLock();
}
this.countdownTimer = setInterval(() => {
this.countdown--;
if (this.countdown === 0) {
// The user failed to click so disconnect them
this.hideOverlay();
this.store.dispatch(disconnectStream({
reason: DisconnectReason.Afk,
message: `AFK timeout:${this.selectWarnTimeout()} seconds, popup timeout:${this.closeTimeout} seconds`,
}));
clearInterval(this.countdownTimer);
}
else {
this.updateCountDown(this.countdown);
}
}, 1000);
}
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: AFKService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); }
static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: AFKService }); }
}
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: AFKService, decorators: [{
type: Injectable
}], ctorParameters: () => [] });
class FreezeFrameService extends SubService {
constructor() {
super();
this.receiving = false;
this.size = 0;
this.freezeFrameOverlay = new Image();
this.init();
}
init() {
this.store
.select(unrealFeature.selectViewportReady)
.pipe(filter(Truthy))
.subscribe(() => this.invalidate());
}
setData(view) {
view = view.slice(1 + 4);
const jpeg = new Uint8Array(this.jpeg.length + view.length);
jpeg.set(this.jpeg, 0);
jpeg.set(view, this.jpeg.length);
this.jpeg = jpeg;
if (this.jpeg.length === this.size) {
this.receiving = false;
Logger.info(`FRAME: Received complete freeze frame ${this.size}`);
this.showFreezeFrame();
}
else if (this.jpeg.length > this.size) {
Logger.error(`FRAME: Received bigger freeze frame than advertised: ${this.jpeg.length}/${this.size}`);
this.store.dispatch(setFreezeFrame({ dataUrl: null, progress: 0 }));
this.jpeg = undefined;
this.receiving = false;
}
else {
this.dispatchInProgress();
}
}
dispatchInProgress() {
this.store.dispatch(setFreezeFrame({
dataUrl: null,
progress: (this.jpeg?.length || 0) / this.size,
}));
}
start(view) {
this.size = new DataView(view.slice(1, 5).buffer).getInt32(0, true);
this.jpeg = view.slice(1 + 4);
if (this.jpeg.length < this.size) {
this.dispatchInProgress();
sendSignal(UnrealInternalSignalEvents.FreezeFrameStart);
Logger.info(`FRAME: Received first chunk of freeze frame: ${this.jpeg.length}/${this.size}`);
this.receiving = true;
}
else {
Logger.info(`FRAME: Received complete freeze frame: ${this.jpeg.length}/${this.size}`);
this.receiving = false;
this.showFreezeFrame();
}
}
invalidate() {
this.store.dispatch(setFreezeFrame({ dataUrl: null, progress: null }));
this.jpeg = undefined;
this.receiving = false;
}
showFreezeFrame() {
if (!this.jpeg) {
return;
}
const base64 = btoa(this.jpeg?.reduce((data, byte) => data + String.fromCharCode(byte), ''));
this.freezeFrameOverlay.src = 'data:image/jpeg;base64,' + base64;
this.store.dispatch(setFreezeFrame({
dataUrl: this.freezeFrameOverlay.src,
progress: 1,
}));
}
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: FreezeFrameService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); }
static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: FreezeFrameService }); }
}
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: FreezeFrameService, decorators: [{
type: Injectable
}], ctorParameters: () => [] });
class DataFlowMonitor {
/**
* Initializes the DataFlowMonitor monitor.
* @param yellowFlagThresholdPercentage - The percentage drop to trigger a YELLOW warning (default: 15%).
* @param redFlagThresholdPercentage - The percentage drop to trigger a RED warning (default: 30%).
* @param historyBufferLength - buffer length (default: 100).
* @param splitPoint - The point at which to split the history buffer into two halves (default: 0.5).
*/
constructor(yellowFlagThresholdPercentage = 15, redFlagThresholdPercentage = 30, historyBufferLength = 100, splitPoint = 0.5) {
this.yellowFlagThresholdPercentage = yellowFlagThresholdPercentage;
this.redFlagThresholdPercentage = redFlagThresholdPercentage;
this.historyBufferLength = historyBufferLength;
this.splitPoint = splitPoint;
this.dataHistory = [];
}
reset() {
this.dataHistory.length = 0;
}
config(data) {
this.reset();
this.yellowFlagThresholdPercentage = data.yellowFlag;
this.redFlagThresholdPercentage = data.redFlag;
}
/**
* Adds a new bitrate measurement and checks for significant drops.
* @param currentValue - The current bitrate in kbps.
* @returns BitrateCheckResult indicating if a drop was detected.
*/
addValue(currentValue) {
if (isNaN(currentValue) || currentValue < 1) {
return {};
}
const historyLength = this.dataHistory.length;
if (historyLength > this.historyBufferLength) {
this.dataHistory.shift();
}
this.dataHistory.push(currentValue);
if (historyLength < this.historyBufferLength) {
this.dataHistory.length = this.historyBufferLength;
this.dataHistory.fill(currentValue, 0, this.historyBufferLength);
}
const splitIndex = Math.floor(historyLength * this.splitPoint);
const firstHalf = this.dataHistory.slice(0, splitIndex);
const secondHalf = this.dataHistory.slice(splitIndex);
const firstHalfMedian = calculateMedian(firstHalf);
const secondHalfMedian = calculateMedian(secondHalf);
const dropPercentage = clampf(0, 100, ((firstHalfMedian - secondHalfMedian) / firstHalfMedian) * 100);
const isRedFlag = dropPercentage >= this.redFlagThresholdPercentage;
if (dropPercentage >= this.yellowFlagThresholdPercentage) {
return {
config: {
yellowFlagThresholdPercentage: this.yellowFlagThresholdPercentage,
redFlagThresholdPercentage: this.redFlagThresholdPercentage,
historyBufferLength: this.historyBufferLength,
splitPoint: this.splitPoint,
},
isDropDetected: true,
dropPercentage,
dataHistory: [...this.dataHistory],
activeMedian: secondHalfMedian,
quality: isRedFlag ? 'red' : 'orange',
message: `Significant flow drop detected: ${dropPercentage.toFixed(2)}% (from ${firstHalfMedian} to ${secondHalfMedian})`,
};
}
return {
config: {
yellowFlagThresholdPercentage: this.yellowFlagThresholdPercentage,
redFlagThresholdPercentage: this.redFlagThresholdPercentage,
historyBufferLength: this.historyBufferLength,
splitPoint: this.splitPoint,
},
isDropDetected: false,
dropPercentage,
dataHistory: [...this.dataHistory],
activeMedian: secondHalfMedian,
quality: 'lime',
message: 'Stable flow',
};
}
}
/**
* Default LBM Filter Parameters
*/
const DefaultFilterModel = {
minimumBitrate: 1600,
yellowFlag: 30,
redFlag: 50,
minimumFps: 5,
monitoringDelayTime: 5000,
initialBitrateEstimate: 1600,
initialErrorCovariance: 1,
processNoise: 3,
measurementNoise: 500,
panelOpen: false,
};
/**
* Global LBM Filter Parameters
*/
const FilterModel = { ...DefaultFilterModel };
/**
* Bitrate Monitor Static Class
*/
const BITRATE_MONITOR = new DataFlowMonitor(FilterModel.yellowFlag, FilterModel.redFlag, 20);
class VideoService extends SubService {
constructor() {
super();
this.video = null;
this.audio = null;
this.container = null;
this.latencyTestTimings = new LatencyTimings();
this.videoTrack$ = new Subject();
this.VideoEncoderQP = 0;
this.aggregatedStats = {};
this.kalmanFilter1D = new KalmanFilter1D(FilterModel.initialBitrateEstimate, FilterModel.initialErrorCovariance, FilterModel.processNoise, FilterModel.measurementNoise);
/**
* Aggregate video stats and emit it as videoStats$
*/
this.videoStats$ = this.videoTrack$.pipe(
// If videoTrack$ emits null, switch to EMPTY → the current interval is canceled.
switchMap((track) => track
? // IMPORTANT! DO NOT CHANGE THOSE NUMBERS, LBM Stats are based on those values
interval(250).pipe(map(() => track.pcClient), switchMap((pcClient) => from(this.getStats(pcClient))), filter(Truthy))
: of(null)), filter(Truthy), share());
this.init();
}
init() {
Signal.on('setKalmanParams').subscribe((data) => {
this.kalmanFilter1D.config(data);
BITRATE_MONITOR.config(data);
});
}
setContainer(container = null) {
this.container = container;
}
setLatencyTimings(latencyTimings) {
this.latencyTestTimings.SetUETimings(latencyTimings);
}
setEncoder(data) {
this.VideoEncoderQP = Number(new TextDecoder('utf-16').decode(data));
}
create() {
this.disconnect$.pipe(take(1)).subscribe(() => this.destroy());
this.destroy();
this.createWebRtcVideo();
this.createWebRtcAudio();
}
attachVideoStream(stream, pcClient) {
if (this.video) {
this.video.srcObject = stream;
this.videoTrack$.next({ stream, pcClient });
}
else {
console.error('Video element is not defined.');
}
}
attachAudioStream(stream) {
// do nothing the video has the same media stream as the audio track we have here (they are linked)
if (this.video?.srcObject === stream) {
return;
}
// the video element has some other media stream not associated with this audio track
else if (this.audio &&
this.video?.srcObject &&
this.video?.srcObject !== stream) {
this.audio.srcObject = stream;
this.playAudio(this.audio);
}
}
play() {
void this.safePlay(this.video);
}
async safePlay(video) {
if (!video) {
return;
}
try {
await video.play();
return;
}
catch (e) {
if (e?.name !== 'NotAllowedError')
throw e;
// Try silent inline autoplay
video.muted = true;
video.setAttribute('playsinline', '');
try {
await video.play();
return;
}
catch {
/* fall through */
}
// Last resort: wait for the next real user gesture and play with/without sound
const resume = async () => {
try {
// choose whether to unmute here
// video.muted = false;
await video.play();
}
catch {
/* ignore, maybe another gesture needed */
}
finally {
window.removeEventListener('pointerdown', resume, true);
window.removeEventListener('keydown', resume, true);
}
};
window.addEventListener('pointerdown', resume, {
once: true,
capture: true,
});
window.addEventListener('keydown', resume, { once: true, capture: true });
}
}
async getStats(pcClient) {
if (!pcClient || pcClient.connectionState === 'closed') {
return null;
}
try {
const stats = await pcClient.getStats(null);
return this.generateAggregatedStatsFunction(stats);
}
catch (err) {
// Handle InvalidStateError or any transient WebRTC errors gracefully
if (err instanceof DOMException && err.name === 'InvalidStateError') {
console.warn('Peer connection is closed or unusable, skipping stats.');
return null;
}
throw err;
}
}
generateAggregatedStatsFunction(stats) {
const newStat = {};
// store each type of codec we can get stats on
newStat.codecs = {};
newStat.currentRoundTripTime = -1;
stats.forEach((report) => {
// Get the inbound-rtp for video
if (report.type === 'inbound-rtp' && report.kind === 'video') {
Object.assign(newStat, report);
newStat.bytesReceivedStart =
this.aggregatedStats && this.aggregatedStats.bytesReceivedStart
? this.aggregatedStats.bytesReceivedStart
: report.bytesReceived;
newStat.framesDecodedStart =
this.aggregatedStats && this.aggregatedStats.framesDecodedStart
? this.aggregatedStats.framesDecodedStart
: report.framesDecoded;
newStat.timestampStart =
this.aggregatedStats && this.aggregatedStats.timestampStart
? this.aggregatedStats.timestampStart
: report.timestamp;
if (this.aggregatedStats && this.aggregatedStats.timestamp) {
// Get the mimetype of the video codec being used
if (report.codecId &&
this.aggregatedStats.codecs &&
Object.hasOwn(this.aggregatedStats.codecs, report.codecId)) {
newStat.videoCodec = this.aggregatedStats.codecs[report.codecId];
}
if (this.aggregatedStats.bytesReceived) {
// bitrate = bits received since last time / number of ms since last time
// This is automatically in kbits (where k=1000) since time is in ms and stat we want is in seconds (so a '* 1000' then a '/ 1000' would negate each other)
newStat.bitrate = Math.floor((8 *
(newStat.bytesReceived - this.aggregatedStats.bytesReceived)) /
(newStat.timestamp - this.aggregatedStats.timestamp));
}
if (this.aggregatedStats.bytesReceivedStart) {
newStat.avgBitrate = Math.floor((8 *
(newStat?.bytesReceived -
this.aggregatedStats.bytesReceivedStart)) /
(newStat.timestamp - this.aggregatedStats.timestampStart));
}
if (this.aggregatedStats.framesDecodedStart) {
newStat.avgFrameRate = Math.floor((newStat.framesDecoded -
this.aggregatedStats.framesDecodedStart) /
((newStat.timestamp - this.aggregatedStats.timestampStart) /
1000));
}
}
}
if (report.type === 'candidate-pair' &&
Object.hasOwn(report, 'currentRoundTripTime')) {
newStat.currentRoundTripTime = report.currentRoundTripTime ?? 0;
}
if (report.type === 'transport' &&
Object.hasOwn(report, 'selectedCandidatePairId')) {
const selectedPair = stats.get(report.selectedCandidatePairId);
const local = stats.get(selectedPair.localCandidateId);
const remote = stats.get(selectedPair.remoteCandidateId);
newStat.selectedPair = {
local: { type: local?.candidateType, url: local?.url },
remote: { type: remote?.candidateType, url: remote?.url },
};
}
// Store mimetype of each codec
if (Object.hasOwn(newStat, 'codecs') &&
report.type === 'codec' &&
report.mimeType &&
report.id) {
const codecId = report.id;
newStat.codecs[codecId] = report.mimeType
.replace('video/', '')
.replace('audio/', '');
}
});
newStat.pixelRatio = +window.devicePixelRatio.toFixed(2);
if (this.aggregatedStats.receiveToCompositeMs) {
newStat.receiveToCompositeMs = this.aggregatedStats.receiveToCompositeMs;
this.latencyTestTimings.SetFrameDisplayDeltaTime(this.aggregatedStats.receiveToCompositeMs);
}
// Calculate duration of run
newStat.runTime = Math.floor((newStat.timestamp - newStat.timestampStart) / 1000);
newStat.kalmanBitrate = Math.floor(this.kalmanFilter1D.update(newStat.bitrate || FilterModel.initialBitrateEstimate));
newStat.dataFlowCheckResult = BITRATE_MONITOR.addValue(newStat.kalmanBitrate);
newStat.bitrateDrop = Math.floor(newStat.dataFlowCheckResult.dropPercentage);
this.aggregatedStats = newStat;
return this.onAggregatedStats(newStat);
}
onAggregatedStats(aggregatedStats) {
return {
quality: mapQpToQuality(this.VideoEncoderQP),
aggregatedStats: {
...aggregatedStats,
VideoEncoderQP: this.VideoEncoderQP,
},
};
}
destroy() {
this.video = null;
this.audio = null;
this.videoTrack$.next(null);
while (this.container?.firstChild) {
this.container.removeChild(this.container.firstChild);
}
this.store.dispatch(changeStatusMainVideoOnScene({ isVideoPlaying: false }));
}
createWebRtcVideo() {
dispatchResize();
const video = document.createElement('video');
video.id = STREAMING_VIDEO_ID;
video.playsInline = true;
video.muted = true;
video.autoplay = true;
video.preload = 'auto';
video.disablePictureInPicture = true;
this.video = video;
this.container?.appendChild(this.video);
this.video.addEventListener('play', () => this.store.dispatch(changeStatusMainVideoOnScene({ isVideoPlaying: true })));
fromEvent(this.video, 'loadedmetadata')
.pipe(take(1), filter((data) => this.video === data.target), tapLog('VideoService loadedmetadata:'))
.subscribe(() => {
if (this.video) {
sendSignal(UnrealInternalSignalEvents.OnVideoInitialized, this.video);
}
});
}
createWebRtcAudio() {
const audio = new Audio();
audio.id = 'streamingAudio';
this.audio = audio;
}
playAudio(audio) {
from(audio.play())
.pipe(catchError(() => fromEvent(document, 'click')), take(1))
.subscribe({
next: () => audio.play(),
error: () => ({}),
});
}
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: VideoService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); }
static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: VideoService }); }
}
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: VideoService, decorators: [{
type: Injectable
}], ctorParameters: () => [] });
class CommandTelemetryService {
constructor() {
this.appId = 'metabox-telemetry';
this.commandsSent = {};
this.exileTimeout = 60000;
this.pollingTime = 5000;
this.commandSequenceNumber = 0;
this.payloads = [];
this.httpClient = inject(HttpClient);
this.unrealInitialConfig = inject(UNREAL_CONFIG, {
optional: true,
});
this.init();
}
init() {
if (!this.unrealInitialConfig?.commandTelemetryReceiver) {
return;
}
this.reset();
this.sessionId = sessionStorage.getItem(this.appId) || this.uuid;
this.userId = localStorage.getItem(this.appId) || this.uuid;
sessionStorage.setItem(this.appId, this.sessionId);
localStorage.setItem(this.appId, this.userId);
this.start();
}
get timeNow() {
return Date.now();
}
get sessionTime() {
return this.timeNow - this.startTime;
}
get canSkipSending() {
return !(this.timeNow - this.lastTime > this.pollingTime * 2);
}
decorate(funcToDecorate) {
this.listenCallbacks();
return this.wrapper.bind(this, funcToDecorate);
}
start() {
fromSignal(UnrealInternalSignalEvents.TelemetryStart).subscribe(({ externalId }) => {
this.trackStartCommand(`EXT-${externalId}`);
});
fromSignal(UnrealInternalSignalEvents.TelemetryStop).subscribe(({ externalId, payload }) => {
const trackingId = `EXT-${externalId}`;
this.trackStopCommand(trackingId, undefined, payload?.multi, payload);
});
fromSignal(UnrealInternalSignalEvents.TelemetryReset).subscribe(() => {
this.reset();
});
timer(0, this.pollingTime).subscribe(() => {
this.send(this.payloads);
});
}
reset() {
this.uuid = generateUuid();
this.commandSequenceNumber = 0;
this.viewId = this.uuid;
this.startTime = this.timeNow;
this.payloads.length = 0;
this.commandsSent = {};
}
trackStartCommand(trackingId) {
const out = {
command: trackingId,
trackingId,
};
const timers = {
sessionElapsedTime: this.sessionTime,
};
this.pushStatToBuffer('commandSent', out, timers);
this.trackTime(out);
}
trackStopCommand(trackingId, timeStampsRaw, multi = false, response = {}) {
const commandResponse = this.commandsSent[trackingId];
if (!commandResponse) {
return;
}
const totalRoundTripTime = this.timeNow - commandResponse.timeStampJs;
/**
* Time between telemetry was reset and command was sent.
*/
const commandSentJsTime = commandResponse.timeStampJs - this.startTime;
let unrealTimers = {};
const t = timeStampsRaw?.timeStampCommand;
if (t) {
unrealTimers = {
unrealRoundTripTime: t.onCompleteCommand.timestamp - t.onRequestCommand.timestamp,
unrealExecutionDuration: t.onCompleteCommand.timestamp - t.onStartCommand.timestamp,
timeDiffRequest: t.onRequestCommand.timestamp - commandResponse.timeStampJs,
timeDiffCallBack: this.timeNow - t.onCompleteCommand.timestamp,
};
}
const timers = {
sessionElapsedTime: this.sessionTime,
commandSentJsTime,
totalRoundTripTime,
...unrealTimers,
};
this.pushStatToBuffer('commandStop', { ...commandResponse.command, response }, timers, timeStampsRaw);
if (!multi) {
delete this.commandsSent[trackingId];
}
}
pushStatToBuffer(type, commandContent, computedTimers, timeStampsRaw) {
this.pushData({
type,
commandSequenceNumber: this.commandSequenceNumber++,
commandContent,
timeStampsRaw,
computedTimers,
});
}
trackTime(command) {
this.commandsSent[command.trackingId] = {
timeStampJs: this.timeNow,
command,
};
}
send(payload) {
if (payload.length === 0 || (this.canSkipSending && payload.length < 10)) {
return;
}
this.lastTime = this.timeNow;
const out = {
appId: this.appId,
sessionId: this.sessionId,
viewId: this.viewId,
userId: this.userId,
ua: navigator.userAgent,
timeStamp: new Date(this.timeNow).toString(),
href: location.href,
resolution: {
width: window.screen.width,
height: window.screen.height,
pixelRatio: window.devicePixelRatio,
},
payload,
};
const lastIndex = payload[payload.length - 1].commandSequenceNumber;
this.httpClient
.post(this.unrealInitialConfig?.commandTelemetryReceiver || '', out)
.pipe(catchError(() => of(null)), take(1))
.subscribe(() => {
this.payloads = this.payloads.filter((p) => p.commandSequenceNumber > lastIndex);
});
}
/**
* Listens for Unreal Engine callbacks.
* Subscribes to the Unreal Engine event loop back and filters out events that do not have a tracking ID.
* When a callback with a tracking ID is received,
* it stops tracking the time for that command and filters out commands
* that have exceeded the timeout limit.
*/
listenCallbacks() {
fromSignal(UnrealInternalSignalEvents.UnrealCallback)
.pipe(filter(({ json }) => {
const callback = 'commandCallback' in json ? json.commandCallback : json;
return Truthy(callback?.trackingId);
}))
.subscribe(({ json }) => {
const callback = json.commandCallback;
const timeStampsRaw = json.timeStamp;
const trackingId = callback?.trackingId || '';
this.trackStopCommand(trackingId, timeStampsRaw);
this.removeExileCommands();
});
}
pushData(payload) {
this.payloads.push(payload);
}
/**
* Removes commands that have exceeded the timeout limit.
* Iterates over all the commands sent and checks if the difference between the current time
* and the time the command was sent is greater than the timeout limit.
* If it is, the command is marked for deletion.
* After checking all commands, those marked for deletion are removed from the commands sent.
* This method is used to ensure that commands that are not responded to within a certain time frame
* do not remain in the commandsSent object indefinitely, which could lead to memory leaks over time.
*/
removeExileCommands() {
const time = new Date().getTime();
const markForDelete = [];
Object.entries(this.commandsSent).forEach(([trackingId, { timeStampJs }]) => {
if (time - timeStampJs > this.exileTimeout) {
markForDelete.push(trackingId);
}
});
markForDelete.forEach((trackingId) => {
delete this.commandsSent[trackingId];
});
}
/**
* Wraps the provided function with telemetry tracking.
* Generates a unique tracking ID and adds it to the data object.
* Pushes the command sent to the payloads with the tracking ID.
* Starts tracking the time for the command.
* Finally, calls the provided function with the modified data object.
*
* @param {IToBeDecorated} funcToDecorate - The function to be decorated with telemetry tracking.
* @param {any} data - The data object to be passed to the function. It will be augmented with a unique tracking ID.
*/
wrapper(funcToDecorate, data) {
const out = { ...data, trackingId: generateUuid() };
const timers = {
sessionElapsedTime: this.sessionTime,
};
this.pushStatToBuffer('commandSent', {
...out,
trackingId: `${out.trackingId}`,
}, timers);
this.trackTime(out);
funcToDecorate(out);
}
static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: CommandTelemetryService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); }
static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: CommandTelemetryService }); }
}
i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "20.3.15", ngImport: i0, type: CommandTelemetryService, decorators: [{
type: Injectable
}], ctorParameters: () => [] });
function TelemetryStart(externalId) {
sendSignal(UnrealInternalSignalEvents.TelemetryStart, { externalId });
}
function TelemetryStop(externalId, payload) {
sendSignal(UnrealInternalSignalEvents.TelemetryStop, { externalId, payload });
}
function ResetTelemetry() {
sendSignal(UnrealInternalSignalEvents.TelemetryReset);
}
function StreamerListHandler(msg) {
this.send(JSON.stringify({
type: 'subscribe',
streamerId: msg.ids[0],
}));
}
function OfferHandler(msg) {
this.onOffer$.next(msg);
}
function httpUrlToWs(url) {
return url.replace('http://', 'ws://').replace('https://', 'wss://');
}
function createWebSocket(wsUrl) {
if (typeof window === 'undefined' || !('WebSocket' in window)) {
return throwError(() => new Error("Your browser doesn't support WebSocket"));
}
Logger.info('Creating socket', wsUrl);
return defer(() => new Observable((observer) => {
const ws = new WebSocket(wsUrl);
const onOpen = () => {
observer.next(ws); // connection established
};
const onError = (ev) => {
const anyEv = ev;
const err = anyEv?.error instanceof Error
? anyEv.error
: new Error('WebSocket failed to connect');
observer.error(err);
};
const onClose = (ev) => {
if (ws.readyState !== WebSocket.OPEN) {
observer.error(new Error(`WebSocket closed before open (code ${ev.code})`));
}
};
ws.addEventListener('open', onOpen);
ws.addEventListener('error', onError);
ws.addEventListener('close', onClose);
return () => {
ws.removeEventListener('open', onOpen);
ws.removeEventListener('error', onError);
ws.removeEventListener('close', onClose);
};
})).pipe(timeout({
first: WS_TIMEOUT, // fail if no 'open' event in this window
with: () => throwError(() => new Error(`WebSocket connection timed out after ${WS_TIMEOUT}ms`)),
}));
}
/*
* Copyright (c) 2024.
* Sergii Karanda - All Rights Reserved
*/
function PlayerDisconnectHandler() {
this.store.dispatch(disconnectStream({
reason: DisconnectReason.OrchestrationPlayerDisconnected,
message: 'Orchestration: PlayerDisconnectHandler',
}));
}
function StreamerDisconnectHandler() {
this.store.dispatch(disconnectStream({
reason: DisconnectReason.OrchestrationStreamerDisconnected,
message: 'Orchestration: StreamerDisconnectHandler',
}));
}
// Structured constants holding code + description
const ORCHESTRATION_ERROR_CODES = {
NO_AVAILABLE_INSTANCES: {
code: 4001,
description: 'There are no instances to run for the requested environment.',
},
LIMIT_REACHED: {
code: 4002,
description: 'The limit of instances for the requested environment is reached.',
},
};
// Set for codes that should trigger a retry
const RETRY_CODES = new Set([
ORCHESTRATION_ERROR_CODES.NO_AVAILABLE_INSTANCES.code,
ORCHESTRATION_ERROR_CODES.LIMIT_REACHED.code,
]);
function OrchestrationErrorHandler(msg) {
if (RETRY_CODES.has(msg.payload.code)) {
const errorInfo = Object.values(ORCHESTRATION_ERROR_CODES).find(({ code }) => code === msg.payload.code);
this.store.dispatch(setOrchestrationMessage({ message: errorInfo?.description }));
setTimeout(() => this.sendRequestStream(), POLLING_TIME);
}
}
class SignallingService extends SubService {
constructor() {
super();
this.action$ = inject(Actions);
this.httpClient = inject(HttpClient);
this.regionsPingService = inject(RegionsPingService);
this.region = signal('', ...(ngDevMode ? [{ debugName: "region" }] : []));
this.selectClientAndViewIds = this.store.selectSignal(selectClientAndViewIds);
this.environmentId = this.store.selectSignal(unrealFeature.selectEnvironmentId);
this.streamRequestContext = this.store.selectSignal(unrealFeature.selectStreamRequestContext);
this.onOffer$ = new Subject();
this.onConfig$ = new Subject();
this.onWebRtcIce$ = new Subject();
this.onWebRtcAnswer$ = new Subject();
this.abort$ = this.action$.pipe(ofType(abortEstablishingConnection));
this.wsMsgHandlers = {};
this.init();
}
init() {
this.setHandlersFromStream();
this.store
.select(unrealFeature.selectDataChannelConnected)
.pipe(filter(Truthy))
.subscribe(() => {
this.send