@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
1,040 lines • 69.9 kB
JavaScript
import { Object3D, PerspectiveCamera, Quaternion, Vector3 } from "three";
import { enableSpatialConsole, isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../debug/index.js";
import { Application, internalOnUserInputRegistered } from "../engine_application.js";
import { Context, FrameEvent } from "../engine_context.js";
import { ContextEvent, ContextRegistry } from "../engine_context_registry.js";
import { isDestroyed } from "../engine_gameobject.js";
import { Gizmos } from "../engine_gizmos.js";
import { registerFrameEventCallback, unregisterFrameEventCallback } from "../engine_lifecycle_functions_internal.js";
import { getBoundingBox, getTempQuaternion, getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../engine_three_utils.js";
import { delay, DeviceUtilities, getParam } from "../engine_utils.js";
import { invokeXRSessionEnd, invokeXRSessionStart } from "./events.js";
import { flipForwardMatrix, flipForwardQuaternion, ImplictXRRig } from "./internal.js";
import { NeedleXRController } from "./NeedleXRController.js";
import { NeedleXRSync } from "./NeedleXRSync.js";
import { SceneTransition } from "./SceneTransition.js";
import { TemporaryXRContext } from "./TempXRContext.js";
import { InternalUSDZRegistry } from "./usdz.js";
const measure_SessionStartedMarker = "NeedleXRSession onStart";
const measure_SessionEndedMarker = "NeedleXRSession onEnd";
const debug = getParam("debugwebxr");
const debugFPS = getParam("stats");
let debugFPSFramesSinceLastUpdate = 0;
function getDOMOverlayElement(domElement) {
let arOverlayElement = null;
// for react cases we dont have an Engine Element
const element = domElement;
if (element.getAROverlayContainer)
arOverlayElement = element.getAROverlayContainer();
else
arOverlayElement = domElement;
return arOverlayElement;
}
handleSessionGranted();
async function handleSessionGranted() {
// TODO: asap session granted doesnt handle the pre-room yet
if (getParam("debugasap")) {
let asapSession = globalThis["needle:XRSession"];
// @ts-ignore // getting TS2848 here, not sure why
if (asapSession instanceof (Promise)) {
delete globalThis["needle:XRSession"];
ContextRegistry.addContextCreatedCallback(async (cb) => {
if (!asapSession)
return;
// TODO: add support to pass this to the temporary room
enableSpatialConsole(true);
const session = await asapSession;
if (session) {
const sessionInit = NeedleXRSession.getDefaultSessionInit("immersive-vr");
NeedleXRSession.setSession("immersive-vr", session, sessionInit, cb.context);
}
else {
console.error("NeedleXRSession: ASAP session was rejected");
}
asapSession = undefined;
});
return;
}
}
if ('xr' in navigator) {
// WebXRViewer (based on Firefox) has a bug where addEventListener
// throws a silent exception and aborts execution entirely.
if (/WebXRViewer\//i.test(navigator.userAgent)) {
console.warn('WebXRViewer does not support addEventListener');
return;
}
navigator.xr?.addEventListener('sessiongranted', async () => {
enableSpatialConsole(true);
console.log("Received Session Granted...");
await delay(100);
const lastSessionMode = sessionStorage.getItem("needle_xr_session_mode");
const lastSessionInit = sessionStorage.getItem("needle_xr_session_init") ?? null;
const init = lastSessionInit ? JSON.parse(lastSessionInit) : null;
let info = null;
if (contextIsLoading()) {
await TemporaryXRContext.start(lastSessionMode || "immersive-vr", init || NeedleXRSession.getDefaultSessionInit("immersive-vr"));
await waitForContextLoadingFinished();
info = await TemporaryXRContext.handoff();
}
if (info) {
NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current);
}
else if (lastSessionMode && lastSessionInit) {
console.log("Session Granted: Restore last session");
const init = JSON.parse(lastSessionInit);
NeedleXRSession.start(lastSessionMode, init).catch(e => console.warn(e));
}
else {
// if no session was found we start VR by default
NeedleXRSession.start("immersive-vr").catch(e => console.warn("Session Granted failed:", e));
}
// make sure we only subscribe to the event once
}, { once: true });
}
}
function saveSessionInfo(mode, init) {
sessionStorage.setItem("needle_xr_session_mode", mode);
sessionStorage.setItem("needle_xr_session_init", JSON.stringify(init));
}
function deleteSessionInfo() {
sessionStorage.removeItem("needle_xr_session_mode");
sessionStorage.removeItem("needle_xr_session_init");
}
const contexts_loading = new Set();
ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, async (cb) => {
contexts_loading.add(cb.context);
});
ContextRegistry.registerCallback(ContextEvent.ContextCreated, async (cb) => {
contexts_loading.delete(cb.context);
const autostart = cb.context?.domElement.getAttribute("autostart") || null;
handleAutoStart(autostart);
});
function contextIsLoading() { return contexts_loading.size > 0; }
function waitForContextLoadingFinished() {
return new Promise(res => {
const startTime = Date.now();
const interval = setInterval(() => {
if (!contextIsLoading() || Date.now() - startTime > 60000) {
clearInterval(interval);
res();
}
}, 100);
});
}
if (DeviceUtilities.isDesktop() && isDevEnvironment()) {
window.addEventListener("keydown", (evt) => {
if (evt.key === "x" || evt.key === "Escape") {
if (NeedleXRSession.active) {
NeedleXRSession.stop();
}
}
});
}
function handleAutoStart(value) {
if (!value)
return;
switch (value?.toLowerCase()) {
case "ar":
Application.registerWaitForInteraction(() => {
// TODO: move WebARSessionRoot to core
NeedleXRSession.start("ar");
});
break;
}
}
// if (getParam("simulatewebxrloading")) {
// ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, async _cb => {
// await delay(3000);
// setTimeout(async () => {
// const info = await TemporaryXRContext.handoff();
// if (info) NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current);
// else
// NeedleXRSession.start("immersive-vr")
// }, 6000)
// });
// let triggered = false;
// window.addEventListener("click", () => {
// if (triggered) return;
// triggered = true;
// TemporaryXRContext.start("immersive-vr", NeedleXRSession.getDefaultSessionInit("immersive-vr"));
// });
// }
/**
* This class manages an XRSession to provide helper methods and events. It provides easy access to the XRInputSources (controllers and hands)
* - Start a XRSession with `NeedleXRSession.start(...)`
* - Stop a XRSession with `NeedleXRSession.stop()`
* - Access a running XRSession with `NeedleXRSession.active`
*
* If a XRSession is active you can use all XR-related event methods on your components to receive XR events e.g. `onEnterXR`, `onUpdateXR`, `onLeaveXR`
* ```ts
* export class MyComponent extends Behaviour {
* // callback invoked whenever the XRSession is started or your component is added to a scene with an active XRSession
* onEnterXR(args: NeedleXREventArgs) {
* console.log("Entered XR");
* // access the NeedleXRSession via args.xr
* }
* // callback invoked whenever a controller is added (or you switch from controller to hand tracking)
* onControllerAdded(args: NeedleXRControllerEventArgs) { }
* }
* ```
*
* ### XRRig
* The XRRig can be accessed via the `rig` property
* Set a custom XRRig via `NeedleXRSession.addRig(...)` or `NeedleXRSession.removeRig(...)`
* By default the active XRRig with the highest priority in the scene is used
*/
export class NeedleXRSession {
static _sync = null;
static getXRSync(context) {
if (!this._sync)
this._sync = new NeedleXRSync(context);
return this._sync;
}
static get currentSessionRequest() { return this._currentSessionRequestMode; }
static _currentSessionRequestMode = null;
/**
* @returns the active @type {NeedleXRSession} (if any active) or null
*/
static get active() { return this._activeSession; }
/** The active xr session mode (if any xr session is active)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSessionMode
*/
static get activeMode() { return this._activeSession?.mode ?? null; }
/** XRSystem via navigator.xr access
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem
*/
static get xrSystem() {
return ('xr' in navigator) ? navigator.xr : undefined;
}
/**
* @returns true if the browser supports WebXR (`immersive-vr` or `immersive-ar`)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isXRSupported() { return Promise.all([this.isVRSupported(), this.isARSupported()]).then(res => res.some(e => e)).catch(() => false); }
/**
* @returns true if the browser supports immersive-vr (WebXR)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isVRSupported() { return this.isSessionSupported("immersive-vr"); }
/**
* @returns true if the browser supports immersive-ar (WebXR)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isARSupported() { return this.isSessionSupported("immersive-ar"); }
/**
* @param mode The XRSessionMode to check if it is supported
* @returns true if the browser supports the given XRSessionMode
*/
static isSessionSupported(mode) { return this.xrSystem?.isSessionSupported(mode).catch(err => { if (debug)
console.error(err); return false; }) ?? Promise.resolve(false); }
static _currentSessionRequest;
static _activeSession;
/** Register to listen to XRSession start events. Unsubscribe with `offXRSessionStart` */
static onSessionRequestStart(evt) {
this._sessionRequestStartListeners.push(evt);
}
/** Unsubscribe from request start evt. Register with `onSessionRequestStart` */
static offSessionRequestStart(evt) {
const index = this._sessionRequestStartListeners.indexOf(evt);
if (index >= 0)
this._sessionRequestStartListeners.splice(index, 1);
}
static _sessionRequestStartListeners = [];
/** Called after the session request has finished */
static onSessionRequestEnd(evt) {
this._sessionRequestEndListeners.push(evt);
}
/** Unsubscribe from request end evt */
static offSessionRequestEnd(evt) {
const index = this._sessionRequestEndListeners.indexOf(evt);
if (index >= 0)
this._sessionRequestEndListeners.splice(index, 1);
}
static _sessionRequestEndListeners = [];
/** Listen to XR session started. Unsubscribe with `offXRSessionStart` */
static onXRSessionStart(evt) {
this._xrStartListeners.push(evt);
}
;
/** Unsubscribe from XRSession started events */
static offXRSessionStart(evt) {
const index = this._xrStartListeners.indexOf(evt);
if (index >= 0)
this._xrStartListeners.splice(index, 1);
}
static _xrStartListeners = [];
/** Listen to XR session ended. Unsubscribe with `offXRSessionEnd` */
static onXRSessionEnd(evt) {
this._xrEndListeners.push(evt);
}
;
/** Unsubscribe from XRSession started events */
static offXRSessionEnd(evt) {
const index = this._xrEndListeners.indexOf(evt);
if (index >= 0)
this._xrEndListeners.splice(index, 1);
}
static _xrEndListeners = [];
/** Listen to controller added events.
* Events are cleared when starting a new session
**/
static onControllerAdded(evt) {
this._controllerAddedListeners.push(evt);
}
/** Unsubscribe from controller added evts */
static offControllerAdded(evt) {
const index = this._controllerAddedListeners.indexOf(evt);
if (index >= 0)
this._controllerAddedListeners.splice(index, 1);
}
static _controllerAddedListeners = [];
/** Listen to controller removed events
* Events are cleared when starting a new session
**/
static onControllerRemoved(evt) {
this._controllerRemovedListeners.push(evt);
}
/** Unsubscribe from controller removed events */
static offControllerRemoved(evt) {
const index = this._controllerRemovedListeners.indexOf(evt);
if (index >= 0)
this._controllerRemovedListeners.splice(index, 1);
}
static _controllerRemovedListeners = [];
/** If the browser supports offerSession - creating a VR or AR button in the browser navigation bar */
static offerSession(mode, init, context) {
if ('xr' in navigator && navigator.xr && 'offerSession' in navigator.xr) {
if (typeof navigator.xr.offerSession === "function") {
console.log("WebXR offerSession is available - requesting mode: " + mode);
if (init == "default") {
init = this.getDefaultSessionInit(mode);
}
navigator.xr.offerSession(mode, {
...init
}).then((session) => {
return NeedleXRSession.setSession(mode, session, init, context);
}).catch(_ => {
console.log("XRSession offer rejected (perhaps because another call to offerSession was made or a call to requestSession was made)");
});
}
return true;
}
return false;
}
/** @returns a new XRSession init object with defaults */
static getDefaultSessionInit(mode) {
switch (mode) {
case "immersive-ar":
const arFeatures = ['anchors', 'local-floor', 'layers', 'dom-overlay', 'hit-test', 'unbounded'];
// Don't request handtracking by default on VisionOS
if (!DeviceUtilities.isVisionOS())
arFeatures.push('hand-tracking');
return {
optionalFeatures: arFeatures,
};
case "immersive-vr":
const vrFeatures = ['local-floor', 'bounded-floor', 'high-fixed-foveation-level', 'layers'];
// Don't request handtracking by default on VisionOS
if (!DeviceUtilities.isVisionOS())
vrFeatures.push('hand-tracking');
return {
optionalFeatures: vrFeatures,
};
default:
console.warn("No default session init for mode", mode);
return {};
}
}
/** start a new webXR session (make sure to stop already running sessions before calling this method)
* @param mode The XRSessionMode to start (e.g. `immersive-vr` or `immersive-ar`) or `ar` to start `immersive-ar` on supported devices OR on iOS devices it will export an interactive USDZ and open in Quicklook.
* Get more information about WebXR modes: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionMode
* @param init The XRSessionInit to use (optional), docs: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionInit
* @param context The Needle Engine context to use
*/
static async start(mode, init, context) {
// handle iOS platform where "immersive-ar" is not supported
// TODO: should we add a separate mode (e.g. "AR")? https://linear.app/needle/issue/NE-5303
if (DeviceUtilities.isiOS()) {
if (mode === "ar") {
const arSupported = await this.isARSupported();
if (!arSupported) {
InternalUSDZRegistry.exportAndOpen();
return null;
}
else {
mode = "immersive-ar";
}
}
}
else if (mode == "ar") {
mode = "immersive-ar";
}
if (isDevEnvironment() && getParam("debugxrpreroom")) {
console.warn("Debug: Starting temporary XR session");
await TemporaryXRContext.start(mode, init || NeedleXRSession.getDefaultSessionInit(mode));
return null;
}
if (this._currentSessionRequest) {
console.warn("A XRSession is already being requested");
if (debug || isDevEnvironment())
showBalloonWarning("A XRSession is already being requested");
return this._currentSessionRequest.then(() => this._activeSession);
}
if (this._activeSession) {
console.error("A XRSession is already running");
return this._activeSession;
}
// Make sure we have a context
if (!context)
context = Context.Current;
if (!context)
context = ContextRegistry.All[0];
if (!context)
throw new Error("No Needle Engine Context found");
//performance.mark('NeedleXRSession start');
// setup session init args, make sure we have default values
if (!init)
init = {};
switch (mode) {
// Setup VR initialization parameters
case "immersive-ar":
{
const supported = await this.xrSystem?.isSessionSupported('immersive-ar');
if (supported !== true) {
console.error(mode + ' is not supported by this browser.');
return null;
}
const defaultInit = this.getDefaultSessionInit(mode);
const domOverlayElement = getDOMOverlayElement(context.domElement);
if (domOverlayElement && !DeviceUtilities.isQuest()) { // excluding quest since dom-overlay breaks sessiongranted
defaultInit.domOverlay = { root: domOverlayElement };
defaultInit.optionalFeatures.push('dom-overlay');
}
init = {
...defaultInit,
...init,
};
}
break;
// Setup AR initialization parameters
case "immersive-vr":
{
const supported = await this.xrSystem?.isSessionSupported('immersive-vr');
if (supported !== true) {
console.error(mode + ' is not supported by this browser.');
return null;
}
const defaultInit = this.getDefaultSessionInit(mode);
init = {
...defaultInit,
...init,
};
}
break;
default:
console.warn("No default session init for mode", mode);
break;
}
// Fix: define these here because VariantLauncher crashes otherwise. Spec: https://immersive-web.github.io/webxr/#feature-dependencies
// Issue: https://linear.app/needle/issue/NE-5136
init.optionalFeatures ??= [];
init.requiredFeatures ??= [];
// we stop a temporary session here (if any runs)
await TemporaryXRContext.stop();
const scripts = mode == "immersive-ar" ? context.scripts_immersive_ar : context.scripts_immersive_vr;
if (debug)
console.log("%c" + `Requesting ${mode} session`, "font-weight:bold;", init, scripts);
else
console.log("%c" + `Requesting ${mode} session`, "font-weight:bold;");
for (const script of scripts) {
if (script.onBeforeXR)
script.onBeforeXR(mode, init);
}
for (const listener of this._sessionRequestStartListeners) {
listener({ mode, init });
}
if (debug)
showBalloonMessage("Requesting " + mode + " session (" + Date.now() + ")");
this._currentSessionRequest = navigator.xr?.requestSession(mode, init);
this._currentSessionRequestMode = mode;
/**@type {XRSystem} */
const newSession = await (this._currentSessionRequest)?.catch(e => {
console.error(e, "Code: " + e.code);
if (e.code === 9)
showBalloonWarning("Make sure your device has the required permissions (e.g. camera access)");
console.log("If the specified XR configuration is not supported (e.g. entering AR doesnt work) - make sure you access the website on a secure connection (HTTPS) and your device has the required permissions (e.g. camera access)");
const notSecure = location.protocol === 'http:';
if (notSecure)
showBalloonWarning("XR requires a secure connection (HTTPS)");
});
this._currentSessionRequest = undefined;
this._currentSessionRequestMode = null;
for (const listener of this._sessionRequestEndListeners) {
listener({ mode, init, newSession: newSession || null });
}
if (!newSession) {
console.warn("XR Session request was rejected");
return null;
}
const session = this.setSession(mode, newSession, init, context);
//performance.mark('NeedleXRSession end');
//performance.measure('NeedleXRSession Startup', 'NeedleXRSession start', 'NeedleXRSession end');
return session;
}
static setSession(mode, session, init, context) {
if (this._activeSession) {
console.error("A XRSession is already running");
return this._activeSession;
}
const scripts = mode == "immersive-ar" ? context.scripts_immersive_ar : context.scripts_immersive_vr;
this._activeSession = new NeedleXRSession(mode, session, context, {
scripts: scripts,
controller_added: this._controllerAddedListeners,
controller_removed: this._controllerRemovedListeners,
init: init
});
session.addEventListener("end", this.onEnd);
if (debug)
console.log("%c" + `Started ${mode} session`, "font-weight:bold;", scripts);
else
console.log("%c" + `Started ${mode} session`, "font-weight:bold;");
return this._activeSession;
}
static $_stop_request = Symbol();
/** stops the active XR session */
static stop() {
const session = this._activeSession;
if (session) {
if (session[this.$_stop_request] === undefined) {
if (debug)
console.log("[NeedleXRSession] Stopping XR Session... (new)");
// We can not call session.end() immediately because we might be within a frame callback
// For example if a use calls `NeedleXRSession.stop()` within a `update()` callback
session[this.$_stop_request] = setTimeout(() => {
session.end();
});
}
else if (debug) {
console.warn("[NeedleXRSession] XR Session stop already requested");
}
}
}
static onEnd = () => {
if (debug)
console.log("XR Session ended");
this._activeSession = null;
};
/** The needle engine context this session was started from */
context;
get sync() {
return NeedleXRSession._sync;
}
/** Returns true if the xr session is still active */
get running() { return !this._ended && this.session != null; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession
*/
session;
/** XR Session Mode: AR or VR */
mode;
/**
* The XRSession interface's read-only interactionMode property describes the best space (according to the user agent) for the application to draw an interactive UI for the current session.
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/interactionMode
*/
get interactionMode() { return this.session["interactionMode"]; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/visibilityState
* @returns {XRVisibilityState} The visibility state of the XRSession
*/
get visibilityState() { return this.session.visibilityState; }
/**
* Check if the session is `visible-blurred` - this means e.g. the keyboard is shown
*/
get isVisibleBlurred() { return this.session.visibilityState === 'visible-blurred'; }
/**
* Check if the session has system keyboard support
*/
get isSystemKeyboardSupported() { return this.session.isSystemKeyboardSupported; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/environmentBlendMode
*/
get environmentBlendMode() { return this.session.environmentBlendMode; }
/**
* The current XR frame
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRFrame
*/
get frame() { return this.context.xrFrame; }
/** The currently active/connected controllers */
controllers = [];
/** shorthand to query the left controller. Use `controllers` to get access to all connected controllers */
get leftController() { return this.controllers.find(c => c.side === "left"); }
/** shorthand to query the right controller. Use `controllers` to get access to all connected controllers */
get rightController() { return this.controllers.find(c => c.side === "right"); }
/** @returns the given controller if it is connected */
getController(side) {
if (typeof side === "number")
return this.controllers[side] || null;
return this.controllers.find(c => c.side === side) || null;
}
/** Returns true if running in pass through mode in immersive AR (e.g. user is wearing a headset while in AR) */
get isPassThrough() {
if (this.environmentBlendMode !== 'opaque' && this.interactionMode === "world-space")
return true;
// since we can not rely on interactionMode check we check the controllers too
// https://linear.app/needle/issue/NE-4057
// the following is a workaround for the issue above
if (this.mode === "immersive-ar" && this.environmentBlendMode !== 'opaque') {
// if we have any tracked pointer controllers we're also in passthrough
if (this.controllers.some(c => c.inputSource.targetRayMode === "tracked-pointer"))
return true;
}
if (isDevEnvironment() && DeviceUtilities.isDesktop() && this.mode === "immersive-ar") {
return true;
}
return false;
}
get isAR() { return this.mode === 'immersive-ar'; }
get isVR() { return this.mode === 'immersive-vr'; }
/** If the AR mode is not immersive (meaning the user is e.g. holding a phone instead of wearing a AR passthrough headset) */
get isScreenBasedAR() { return this.isAR && !this.isPassThrough; }
get posePosition() { return this._transformPosition; }
get poseOrientation() { return this._transformOrientation; }
/** @returns the context.renderer.xr.getReferenceSpace() result */
get referenceSpace() { return this.context.renderer.xr.getReferenceSpace(); }
/** @returns the XRFrame `viewerpose` using the xr `referenceSpace` */
get viewerPose() { return this._viewerPose; }
/** @returns `true` if any image is currently being tracked */
/** returns true if images are currently being tracked */
get isTrackingImages() {
if (this.frame && "getImageTrackingResults" in this.frame && typeof this.frame.getImageTrackingResults === "function") {
try {
const trackingResult = this.frame.getImageTrackingResults();
for (const result of trackingResult) {
const state = result.trackingState;
if (state === "tracked")
return true;
}
}
catch {
// Looks like we get a NotSupportedException on Android since the method is known
// but the feature is not supported by the session
// TODO Can we check here if we even requested the image-tracking feature instead of catching?
return false;
}
}
return false;
}
/** The currently active XR rig */
get rig() {
const rig = this._rigs[0] ?? null;
if (rig?.gameObject && isDestroyed(rig.gameObject) || rig?.isActive === false) {
this.updateActiveXRRig();
return this._rigs[0] ?? null;
}
return rig;
}
_rigScale = 1;
_lastRigScaleUpdate = -1;
/** Get the XR Rig worldscale.
*
* **For AR**
* If you want to modify the scale in AR at runtime get the WebARSessionRoot component via `findObjectOfType(WebARSessionRoot)` and then set the `arScale` value.
*
*/
get rigScale() {
// const ar_scaleValue = this.context.domElement.getAttribute("ar-scale");
// if (ar_scaleValue) {
// const scale = parseFloat(ar_scaleValue);
// if (!isNaN(scale)) return scale;
// }
if (!this._rigs[0])
return 1;
if (this._lastRigScaleUpdate !== this.context.time.frame) {
this._lastRigScaleUpdate = this.context.time.frame;
this._rigScale = this._rigs[0].gameObject.worldScale.x;
}
return this._rigScale;
}
/** add a rig to the available XR rigs - if it's priority is higher than the currently active rig it will be enabled */
addRig(rig) {
const i = this._rigs.indexOf(rig);
if (i >= 0)
return;
if (rig.priority === undefined)
rig.priority = 0;
this._rigs.push(rig);
this.updateActiveXRRig();
}
/** Remove a rig from the available XR Rigs */
removeRig(rig) {
const i = this._rigs.indexOf(rig);
if (i === -1)
return;
this._rigs.splice(i, 1);
this.updateActiveXRRig();
}
/** Sets a XRRig to be active which will parent the camera to this rig */
setRigActive(rig) {
const i = this._rigs.indexOf(rig);
const currentlyActive = this._rigs[0];
this._rigs.splice(i, 1);
this._rigs.unshift(rig);
// if there's another rig currently active we need to make sure we have at least the same priority
rig.priority = currentlyActive?.priority ?? 0;
this.updateActiveXRRig();
}
/**
* @returns the user position in the rig space
*/
getUserOffsetInRig() {
const positionInRig = this.context.mainCamera?.position;
if (!positionInRig || !this.rig)
return getTempVector(0, 0, 0);
const vec = getTempVector(positionInRig);
vec.x *= -1;
vec.z *= -1;
vec.applyQuaternion(getTempQuaternion(this.rig.gameObject.quaternion));
return vec;
}
updateActiveXRRig() {
const previouslyActiveRig = this._rigs[0] ?? null;
// ensure that the default rig is in the scene
if (this._defaultRig.gameObject.parent !== this.context.scene)
this.context.scene.add(this._defaultRig.gameObject);
// ensure the fallback rig is always active!!!
this._defaultRig.gameObject.visible = true;
// ensure that the default rig is in the list of available rigs
if (!this._rigs.includes(this._defaultRig))
this._rigs.push(this._defaultRig);
// find the rig with the highest priority and make sure it's at the beginning of the array
let highestPriorityRig = this._rigs[0];
if (highestPriorityRig && highestPriorityRig.priority === undefined)
highestPriorityRig.priority = 0;
for (let i = 1; i < this._rigs.length; i++) {
const rig = this._rigs[i];
if (!rig.isActive)
continue;
if (isDestroyed(rig.gameObject)) {
this._rigs.splice(i, 1);
i--;
continue;
}
if (!highestPriorityRig || highestPriorityRig.isActive === false || (rig.priority !== undefined && rig.priority > highestPriorityRig.priority)) {
highestPriorityRig = rig;
}
}
// make sure the highest priority rig is at the beginning if it isnt already
if (previouslyActiveRig !== highestPriorityRig) {
const index = this._rigs.indexOf(highestPriorityRig);
if (index >= 0)
this._rigs.splice(index, 1);
this._rigs.unshift(highestPriorityRig);
}
if (debug) {
if (previouslyActiveRig === highestPriorityRig)
console.log("Updated Active XR Rig:", highestPriorityRig, "prev:", previouslyActiveRig);
else
console.log("Updated Active XRRig:", highestPriorityRig, " (the same as before)");
}
}
_rigs = [];
_viewerHitTestSource = null;
/** Returns a XR hit test result (if hit-testing is available) in rig space
* @param source If provided, the hit test will be performed for the given controller
*/
getHitTest(source) {
if (source) {
return this.getControllerHitTest(source);
}
if (!this._viewerHitTestSource)
return null;
const hitTestSource = this._viewerHitTestSource;
const hitTestResults = this.frame.getHitTestResults(hitTestSource);
if (hitTestResults.length > 0) {
const hit = hitTestResults[0];
return this.convertHitTestResult(hit);
}
return null;
}
getControllerHitTest(controller) {
const hitTestSource = controller.getHitTestSource();
if (!hitTestSource)
return null;
const res = this.frame.getHitTestResultsForTransientInput(hitTestSource);
for (const result of res) {
if (result.inputSource === controller.inputSource) {
for (const hit of result.results) {
return this.convertHitTestResult(hit);
}
}
}
return null;
}
convertHitTestResult(result) {
const referenceSpace = this.context.renderer.xr.getReferenceSpace();
const pose = referenceSpace && result.getPose(referenceSpace);
if (pose) {
const pos = getTempVector(pose.transform.position);
const rot = getTempQuaternion(pose.transform.orientation);
const camera = this.context.mainCamera;
if (camera?.parent !== this._cameraRenderParent) {
pos.applyMatrix4(flipForwardMatrix);
}
if (camera?.parent) {
pos.applyMatrix4(camera.parent.matrixWorld);
rot.multiply(flipForwardQuaternion);
// apply parent quaternion (if parent is moved/rotated)
const parentRotation = getWorldQuaternion(camera.parent);
// ensure that "up" (y+) is pointing away from the wall
parentRotation.premultiply(flipForwardQuaternion);
rot.premultiply(parentRotation);
}
return { hit: result, position: pos, quaternion: rot };
}
return null;
}
/** convert a XRRigidTransform from XR session space to threejs / Needle Engine XR space */
convertSpace(transform) {
const pos = getTempVector(transform.position);
pos.applyMatrix4(flipForwardMatrix);
const rot = getTempQuaternion(transform.orientation);
rot.premultiply(flipForwardQuaternion);
return { position: pos, quaternion: rot };
}
/** this is the implictly created XR rig */
_defaultRig;
/** all scripts that receive some sort of XR update event */
_xr_scripts;
/** scripts that have onUpdateXR event methods */
_xr_update_scripts = [];
/** scripts that are in the scene but inactive (e.g. disabled parent gameObject) */
_inactive_scripts = [];
_controllerAdded;
_controllerRemoved;
_originalCameraWorldPosition;
_originalCameraWorldRotation;
_originalCameraWorldScale;
_originalCameraParent;
/** we store the main camera reference here each frame to make sure we have a rendering camera
* this e.g. the case when the XR rig with the camera gets disabled (and thus this.context.mainCamera is unassigned)
*/
_mainCamera = null;
constructor(mode, session, context, extra) {
//performance.mark(measure_SessionStartedMarker);
saveSessionInfo(mode, extra.init);
this.session = session;
this.mode = mode;
this.context = context;
if (debug || getParam("console"))
enableSpatialConsole(true);
this._xr_scripts = [...extra.scripts];
this._xr_update_scripts = this._xr_scripts.filter(e => typeof e.onUpdateXR === "function");
this._controllerAdded = extra.controller_added;
this._controllerRemoved = extra.controller_removed;
registerFrameEventCallback(this.onBefore, FrameEvent.LateUpdate);
this.context.pre_render_callbacks.push(this.onBeforeRender);
this.context.post_render_callbacks.push(this.onAfterRender);
if (extra.init.optionalFeatures?.includes("hit-test") || extra.init.requiredFeatures?.includes("hit-test")) {
session.requestReferenceSpace('viewer').then((referenceSpace) => {
return session.requestHitTestSource?.call(session, { space: referenceSpace })?.then((source) => {
return this._viewerHitTestSource = source;
}).catch(e => console.error(e));
}).catch(e => console.error(e));
}
if (this.context.mainCamera) {
this._originalCameraWorldPosition = getWorldPosition(this.context.mainCamera, new Vector3());
this._originalCameraWorldRotation = getWorldQuaternion(this.context.mainCamera, new Quaternion());
this._originalCameraWorldScale = getWorldScale(this.context.mainCamera, new Vector3());
this._originalCameraParent = this.context.mainCamera.parent;
}
this._defaultRig = new ImplictXRRig();
this.context.scene.add(this._defaultRig.gameObject);
this.addRig(this._defaultRig);
// register already connected input sources
// this is for when the session is already running (via a temporary xr session)
// and the controllers are already connected
for (let i = 0; i < session.inputSources.length; i++) {
const inputSource = session.inputSources[i];
if (!inputSource.handedness) {
console.warn("Input source in xr session has no handedness - ignoring", i);
continue;
}
this.onInputSourceAdded(inputSource);
}
// handle controller and input source changes changes
this.session.addEventListener('end', this.onEnd);
// handle input sources change
this.session.addEventListener("inputsourceschange",
/* @ts-ignore (ignore CI XRInputSourceChangeEvent mismatch) */
(evt) => {
// handle removed controllers
for (const removedInputSource of evt.removed) {
this.disconnectInputSource(removedInputSource);
}
for (const newInputSource of evt.added) {
this.onInputSourceAdded(newInputSource);
}
});
// Unfortunately the code below doesnt work: the session never receives any input sources sometimes
// https://developer.mozilla.org/en-US/docs/Web/API/XRSession/visibilitychange_event
// this.session.addEventListener("visibilitychange", (evt: XRSessionEvent) => {
// // sometimes when entering an XR session the controllers are not added/not in the list and we don't receive an event
// // this is a workaround trying to add controllers when the scene visibility changes to "visible"
// // e.g. due to a user opening and closing the menu
// if (this.controllers.length === 0 && evt.session.visibilityState === "visible") {
// for (const controller of evt.session.inputSources) {
// this.onInputSourceAdded(controller);
// }
// }
// })
// we set the session on the webxr manager at the end because we want to receive inputsource events first
// e.g. in case there's a bug in the threejs codebase
this.context.xr = this;
this.context.renderer.xr.setSession(this.session).then(this.onRendererSessionSet);
// disable three.js renderer controller autoUpdate (added in ac67b31e3548386f8a93e23a4176554c92bbd0d9)
if ("controllerAutoUpdate" in this.context.renderer.xr) {
console.debug("Disabling three.js controllerAutoUpdate");
this.context.renderer.xr.controllerAutoUpdate = false;
}
else if (debug) {
console.warn("controllerAutoUpdate is not available in three.js - cannot disable it");
}
}
/** called when renderer.setSession is fulfilled */
onRendererSessionSet = () => {
if (!this.running)
return;
this.context.renderer.xr.enabled = true;
// calling update camera here once to prevent the xr camera left/right NaN in the first frame due to false near/far plane values, see NE-4126
this.context.renderer.xr.updateCamera(this.context.mainCamera);
this.context.mainCameraComponent?.applyClearFlags();
};
onInputSourceAdded = (newInputSource) => {
// do not create XR controllers for screen input sources
if (newInputSource.targetRayMode === "screen") {
return;
}
let index = 0;
for (let i = 0; i < this.session.inputSources.length; i++) {
if (this.session.inputSources[i] === newInputSource) {
index = i;
break;
}
}
// check if an xr controller for this input source already exists
// in case we have both an event from inputsourceschange and from the construtor initial input sources
if (this.controllers.find(c => c.inputSource === newInputSource)) {
console.debug("Controller already exists for input source", index);
return;
}
else if (this._newControllers.find(c => c.inputSource === newInputSource)) {
console.debug("Controller already registered for input source", index);
return;
}
// TODO: check if this is a transient input source AND we can figure out which existing controller it likely belongs to
// TODO: do not draw raycasts for controllers that don't have primary input actions / until we know that they have primary input actions
const newController = new NeedleXRController(this, newInputSource, index);
this._newControllers.push(newController);
};
/** Disconnects the controller, invokes events and notifies previou controller (if any) */
disconnectInputSource(inputSource) {
const handleRemove = (oldController, _array, i) => {
if (oldController.inputSource === inputSource) {
if (debug)
console.log("Disconnecting controller", oldController.index);
this.controllers.splice(i, 1);
this.invokeControllerEvent(oldController, this._controllerRemoved, "removed");
const args = {
xr: this,
controller: oldController,
change: "removed"
};
for (const script of this._xr_scripts) {
if (script.onXRControllerRemoved)
script.onXRControllerRemoved(args);
}
oldController.onDisconnected();
}
};
for (let i = this.controllers.length - 1; i >= 0; i--) {
const oldController = this.controllers[i];
handleRemove(oldController, this.controllers, i);
}
for (let i = this._newControllers.length - 1; i >= 0; i--) {
const oldController = this._newControllers[i];
handleRemove(oldController, this._newControllers, i);
}
}
/** End the XR Session */
end() {
// this can be called by external code to end the session
// the actual cleanup happens in onEnd which subscribes to the session end event
// so users can also just regularly call session.end() and the cleanup will happen automatically
if (this._ended)
return;
this.session.end().catch(e => console.warn(e));
}
_ended = false;
_newControllers = [];
onEnd = (_evt) => {
if (this._ended)
return;
this._ended = true;
console.debug("XR Session ended");
deleteSessionInfo();
this.onAfterRender();
this.revertCustomForward();
this._didStart = false;
this._previousCameraParent = null;
unregisterFrameEventCallback(this.onBefore, FrameEvent.LateUpdate);
const index = this.context.pre_render_callbacks.indexOf(this.onBeforeRender);
if (index >= 0)
this.context.pre_render_callbacks.splice(index, 1);
const index2 = this.context.post_render_callbacks.indexOf(this.onAfterRender);
if (index2 >= 0)
this.context.post_render_callbacks.splice(index2, 1);
this.context.xr = null;
this.context.renderer.xr.enabled = false;
// apply the clearflags at the beginning of the next frame
this.context.pre_update_oneshot_callbacks.push(() => {
this.context.mainCameraComponent?.applyClearFlags();
this.context.mainCameraComponent?.applyClippingPlane();
});
invokeXRSessionEnd({ session: this });
for (const listener of NeedleXRSession._xrEndListeners) {
listener({ xr: this });
}
// make sure we disconnect all controllers
// we copy the array because the disconnectInputSource method modifies the controllers array
const copy = [...this.controllers];
for (let i = 0; i < copy.length; i++) {
this.disconnectInputSource(copy[i].inputSource);
}
this._newControllers.length = 0;
this.controllers.length = 0;
// we want to call leave XR for *all* scripts that are still registered
// even if they might already be destroyed e.g. by the WebXR component (it destroys the default controller scripts)
// they should still receive this callback to be properly cleaned up
for (const listener of this._xr_scripts) {
listener?.onLeaveXR?.({ xr: this });
}
this.sync?.onExitXR(this);
if (this.context.mainCamera) {
// if we have a main camera we want to move it back to it's original parent
this._originalCameraParent?.add(this.context.mainCamera);
if (this._originalCameraWorldPosition) {
setWorldPosition(this.context.mainCamera, this._originalCameraWorldPosition);
}
if (this._originalCameraWorldRotation) {
setWorldQuaternion(this.context.mainCamera, this._originalCameraWorldRotation);
}
if (this._originalCameraWorldScale) {
setWorldScale(this.context.mainCamera, this._originalCameraWorldScale);
}
}
// mark for size change since DPI might have changed
this.context.requestSizeUpdate();
this._defaultRig.gameObject.removeFromParent();
enableSpatialConsole(false);
//performance.mark(measure_SessionEndedMarker);
//performance.measure('NeedleXRSession', measure_SessionStartedMarker, measure_SessionEnded