@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
1,041 lines (1,040 loc) • 77.3 kB
JavaScript
import { Object3D, PerspectiveCamera, Quaternion, Vector3 } from "three";
import { enableSpatialConsole, isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../debug/index.js";
import { Application, internalOnUserInputRegistered } from "../engine_application.js";
import { Context, FrameEvent } from "../engine_context.js";
import { ContextEvent, ContextRegistry } from "../engine_context_registry.js";
import { isDestroyed } from "../engine_gameobject.js";
import { Gizmos } from "../engine_gizmos.js";
import { Telemetry } from "../engine_license.js";
import { registerFrameEventCallback, unregisterFrameEventCallback } from "../engine_lifecycle_functions_internal.js";
import { getBoundingBox, getTempQuaternion, getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../engine_three_utils.js";
import { DeviceUtilities, getParam } from "../engine_utils.js";
import { invokeXRSessionEnd, invokeXRSessionStart } from "./events.js";
import { flipForwardMatrix, flipForwardQuaternion, ImplictXRRig } from "./internal.js";
import { NeedleXRController } from "./NeedleXRController.js";
import { NeedleXRSync } from "./NeedleXRSync.js";
import { SceneTransition } from "./SceneTransition.js";
import { TemporaryXRContext } from "./TempXRContext.js";
import { InternalUSDZRegistry } from "./usdz.js";
const measure_SessionStartedMarker = "NeedleXRSession onStart";
const measure_SessionEndedMarker = "NeedleXRSession onEnd";
const debug = getParam("debugwebxr");
const debugFPS = getParam("stats");
let debugFPSFramesSinceLastUpdate = 0;
function getDOMOverlayElement(domElement) {
let arOverlayElement = null;
// for react cases we dont have an Engine Element
const element = domElement;
if (element.getAROverlayContainer)
arOverlayElement = element.getAROverlayContainer();
else
arOverlayElement = domElement;
return arOverlayElement;
}
handleSessionGranted();
async function handleSessionGranted() {
// await delay(400);
let defaultMode = "immersive-vr";
try {
// In app clips we default to AR
if (DeviceUtilities.isNeedleAppClip()) {
defaultMode = "immersive-ar";
}
// Check if VR is even supported, otherwise try AR
else if (!(await navigator.xr?.isSessionSupported("immersive-vr"))) {
defaultMode = "immersive-ar";
}
// Check if AR is supported, otherwise we can't do anything
if (!(await navigator.xr?.isSessionSupported("immersive-ar")) && defaultMode === "immersive-ar") {
// console.warn("[NeedleXRSession:granted] Neither VR nor AR supported, aborting session start.");
// showBalloonMessage("NeidleXRSession: Neither VR nor AR supported, aborting session start.");
return;
}
}
catch (e) {
console.debug("[NeedleXRSession:granted] Error while checking XR support:", e);
// showBalloonWarning("NeedleXRSession: Error while checking XR support: " + (e as Error).message);
return;
}
// showBalloonMessage("sessiongranted: " + defaultMode);
// TODO: asap session granted doesnt handle the pre-room yet
if (getParam("debugasap")) {
let asapSession = globalThis["needle:XRSession"];
// @ts-ignore // getting TS2848 here, not sure why
if (asapSession instanceof (Promise)) {
delete globalThis["needle:XRSession"];
ContextRegistry.addContextCreatedCallback(async (cb) => {
if (!asapSession)
return;
// TODO: add support to pass this to the temporary room
enableSpatialConsole(true);
const session = await asapSession;
if (session) {
const sessionInit = NeedleXRSession.getDefaultSessionInit(defaultMode);
NeedleXRSession.setSession(defaultMode, session, sessionInit, cb.context);
}
else {
console.error("[NeedleXRSession:granted] ASAP session was rejected");
}
asapSession = undefined;
});
return;
}
}
// console.log("Attaching sessiongranted handler...", { haveXR: 'xr' in navigator });
// setTimeout(() => console.log("Session Granted handler attached.", { haveXR: 'xr' in navigator }), 1000);
if ('xr' in navigator) {
// WebXRViewer (based on Firefox) has a bug where addEventListener
// throws a silent exception and aborts execution entirely.
if (/WebXRViewer\//i.test(navigator.userAgent)) {
console.warn('WebXRViewer does not support addEventListener');
return;
}
navigator.xr?.addEventListener('sessiongranted', async () => {
// enableSpatialConsole(true);
const lastSessionMode = sessionStorage.getItem("needle_xr_session_mode");
const lastSessionInit = sessionStorage.getItem("needle_xr_session_init") ?? null;
const init = lastSessionInit ? JSON.parse(lastSessionInit) : null;
let info = null;
if (contextIsLoading()) {
await TemporaryXRContext.start(lastSessionMode || defaultMode, init || NeedleXRSession.getDefaultSessionInit(defaultMode))
.catch(e => console.warn("[NeedleXRSession:granted] TemporaryXRContext start failed:", e));
await waitForContextLoadingFinished();
info = await TemporaryXRContext.handoff();
}
if (info) {
NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current);
}
else if (lastSessionMode && lastSessionInit) {
console.log("[NeedleXRSession:granted] Restore last session");
const init = JSON.parse(lastSessionInit);
NeedleXRSession.start(lastSessionMode, init).catch(e => console.warn(e));
}
else {
// if no session was found we start VR by default
NeedleXRSession.start(defaultMode).catch(e => console.warn("[NeedleXRSession:granted] failed:", e));
}
// make sure we only subscribe to the event once
}, { once: true });
}
else {
// showBalloonWarning("NeedleXRSession: WebXR not available in this browser.");
}
}
function saveSessionInfo(mode, init) {
sessionStorage.setItem("needle_xr_session_mode", mode);
sessionStorage.setItem("needle_xr_session_init", JSON.stringify(init));
}
function deleteSessionInfo() {
sessionStorage.removeItem("needle_xr_session_mode");
sessionStorage.removeItem("needle_xr_session_init");
}
const contexts_loading = new Set();
ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, async (cb) => {
contexts_loading.add(cb.context);
});
ContextRegistry.registerCallback(ContextEvent.ContextCreated, async (cb) => {
contexts_loading.delete(cb.context);
const autostart = cb.context?.domElement.getAttribute("autostart") || null;
handleAutoStart(autostart);
});
function contextIsLoading() { return contexts_loading.size > 0; }
function waitForContextLoadingFinished() {
return new Promise(res => {
const startTime = Date.now();
const interval = setInterval(() => {
if (!contextIsLoading() || Date.now() - startTime > 60000) {
clearInterval(interval);
res();
}
}, 100);
});
}
if (DeviceUtilities.isDesktop() && isDevEnvironment()) {
window.addEventListener("keydown", (evt) => {
if (evt.key === "x" || evt.key === "Escape") {
if (NeedleXRSession.active) {
NeedleXRSession.stop();
}
}
});
}
function handleAutoStart(value) {
if (!value)
return;
switch (value?.toLowerCase()) {
case "ar":
Application.registerWaitForInteraction(() => {
// TODO: move WebARSessionRoot to core
NeedleXRSession.start("ar");
});
break;
}
}
// if (getParam("simulatewebxrloading")) {
// ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, async _cb => {
// await delay(3000);
// setTimeout(async () => {
// const info = await TemporaryXRContext.handoff();
// if (info) NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current);
// else
// NeedleXRSession.start("immersive-vr")
// }, 6000)
// });
// let triggered = false;
// window.addEventListener("click", () => {
// if (triggered) return;
// triggered = true;
// TemporaryXRContext.start("immersive-vr", NeedleXRSession.getDefaultSessionInit("immersive-vr"));
// });
// }
const $initialFov = Symbol("initial-fov");
const $initialNear = Symbol("initial-near");
/**
* This class manages an XRSession to provide helper methods and events. It provides easy access to the XRInputSources (controllers and hands)
* - Start a XRSession with `NeedleXRSession.start(...)`
* - Stop a XRSession with `NeedleXRSession.stop()`
* - Access a running XRSession with `NeedleXRSession.active`
*
* If a XRSession is active you can use all XR-related event methods on your components to receive XR events e.g. `onEnterXR`, `onUpdateXR`, `onLeaveXR`
* ```ts
* export class MyComponent extends Behaviour {
* // callback invoked whenever the XRSession is started or your component is added to a scene with an active XRSession
* onEnterXR(args: NeedleXREventArgs) {
* console.log("Entered XR");
* // access the NeedleXRSession via args.xr
* }
* // callback invoked whenever a controller is added (or you switch from controller to hand tracking)
* onControllerAdded(args: NeedleXRControllerEventArgs) { }
* }
* ```
*
* ### XRRig
* The XRRig can be accessed via the `rig` property
* Set a custom XRRig via `NeedleXRSession.addRig(...)` or `NeedleXRSession.removeRig(...)`
* By default the active XRRig with the highest priority in the scene is used
*
* ### Screenshots in XR
* Screenshots work automatically during XR sessions, including AR camera feed compositing. See {@link screenshot2} for more information.
*
* @category XR
* @see {@link screenshot2} for taking screenshots in XR sessions
*/
export class NeedleXRSession {
static _sync = null;
static getXRSync(context) {
if (!this._sync)
this._sync = new NeedleXRSync(context);
return this._sync;
}
static get currentSessionRequest() { return this._currentSessionRequestMode; }
static _currentSessionRequestMode = null;
/**
* @returns the active @type {NeedleXRSession} (if any active) or null
*/
static get active() { return this._activeSession; }
/** The active xr session mode (if any xr session is active)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSessionMode
*/
static get activeMode() { return this._activeSession?.mode ?? null; }
/** XRSystem via navigator.xr access
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem
*/
static get xrSystem() {
return ('xr' in navigator) ? navigator.xr : undefined;
}
/**
* @returns true if the browser supports WebXR (`immersive-vr` or `immersive-ar`)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isXRSupported() { return Promise.all([this.isVRSupported(), this.isARSupported()]).then(res => res.some(e => e)).catch(() => false); }
/**
* @returns true if the browser supports immersive-vr (WebXR)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isVRSupported() { return this.isSessionSupported("immersive-vr"); }
/**
* @returns true if the browser supports immersive-ar (WebXR)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isARSupported() { return this.isSessionSupported("immersive-ar"); }
/**
* @param mode The XRSessionMode to check if it is supported
* @returns true if the browser supports the given XRSessionMode
*/
static isSessionSupported(mode) { return this.xrSystem?.isSessionSupported(mode).catch(err => { if (debug)
console.error(err); return false; }) ?? Promise.resolve(false); }
static _currentSessionRequest;
static _activeSession;
/** Register to listen to XRSession start events. Unsubscribe with `offXRSessionStart` */
static onSessionRequestStart(evt) {
this._sessionRequestStartListeners.push(evt);
}
/** Unsubscribe from request start evt. Register with `onSessionRequestStart` */
static offSessionRequestStart(evt) {
const index = this._sessionRequestStartListeners.indexOf(evt);
if (index >= 0)
this._sessionRequestStartListeners.splice(index, 1);
}
static _sessionRequestStartListeners = [];
/** Called after the session request has finished */
static onSessionRequestEnd(evt) {
this._sessionRequestEndListeners.push(evt);
}
/** Unsubscribe from request end evt */
static offSessionRequestEnd(evt) {
const index = this._sessionRequestEndListeners.indexOf(evt);
if (index >= 0)
this._sessionRequestEndListeners.splice(index, 1);
}
static _sessionRequestEndListeners = [];
/** Listen to XR session started. Unsubscribe with `offXRSessionStart` */
static onXRSessionStart(evt) {
this._xrStartListeners.push(evt);
}
;
/** Unsubscribe from XRSession started events */
static offXRSessionStart(evt) {
const index = this._xrStartListeners.indexOf(evt);
if (index >= 0)
this._xrStartListeners.splice(index, 1);
}
static _xrStartListeners = [];
/** Listen to XR session ended. Unsubscribe with `offXRSessionEnd` */
static onXRSessionEnd(evt) {
this._xrEndListeners.push(evt);
}
;
/** Unsubscribe from XRSession started events */
static offXRSessionEnd(evt) {
const index = this._xrEndListeners.indexOf(evt);
if (index >= 0)
this._xrEndListeners.splice(index, 1);
}
static _xrEndListeners = [];
/** Listen to controller added events.
* Events are cleared when starting a new session
**/
static onControllerAdded(evt) {
this._controllerAddedListeners.push(evt);
}
/** Unsubscribe from controller added evts */
static offControllerAdded(evt) {
const index = this._controllerAddedListeners.indexOf(evt);
if (index >= 0)
this._controllerAddedListeners.splice(index, 1);
}
static _controllerAddedListeners = [];
/** Listen to controller removed events
* Events are cleared when starting a new session
**/
static onControllerRemoved(evt) {
this._controllerRemovedListeners.push(evt);
}
/** Unsubscribe from controller removed events */
static offControllerRemoved(evt) {
const index = this._controllerRemovedListeners.indexOf(evt);
if (index >= 0)
this._controllerRemovedListeners.splice(index, 1);
}
static _controllerRemovedListeners = [];
/** If the browser supports offerSession - creating a VR or AR button in the browser navigation bar */
static offerSession(mode, init, context) {
if ('xr' in navigator && navigator.xr && 'offerSession' in navigator.xr) {
if (typeof navigator.xr.offerSession === "function") {
console.log("WebXR offerSession is available - requesting mode: " + mode);
if (init == "default") {
init = this.getDefaultSessionInit(mode);
}
navigator.xr.offerSession(mode, {
...init
}).then((session) => {
return NeedleXRSession.setSession(mode, session, init, context);
}).catch(_ => {
console.log("XRSession offer rejected (perhaps because another call to offerSession was made or a call to requestSession was made)");
});
}
return true;
}
return false;
}
/** @returns a new XRSession init object with defaults */
static getDefaultSessionInit(mode) {
switch (mode) {
case "immersive-ar":
const arFeatures = ['anchors', 'local-floor', 'layers', 'dom-overlay', 'hit-test', 'unbounded'];
// Don't request handtracking by default on VisionOS
if (!DeviceUtilities.isVisionOS())
arFeatures.push('hand-tracking');
return {
optionalFeatures: arFeatures,
};
case "immersive-vr":
const vrFeatures = ['local-floor', 'bounded-floor', 'high-fixed-foveation-level', 'layers'];
// Don't request handtracking by default on VisionOS
if (!DeviceUtilities.isVisionOS())
vrFeatures.push('hand-tracking');
return {
optionalFeatures: vrFeatures,
};
default:
console.warn("No default session init for mode", mode);
return {};
}
}
/** start a new webXR session (make sure to stop already running sessions before calling this method)
* @param mode The XRSessionMode to start (e.g. `immersive-vr` or `immersive-ar`) or `ar` to start `immersive-ar` on supported devices OR on iOS devices it will export an interactive USDZ and open in Quicklook.
* Get more information about WebXR modes: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionMode
* @param init The XRSessionInit to use (optional), docs: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionInit
* @param context The Needle Engine context to use
*/
static async start(mode, init, context) {
// setup session init args, make sure we have default values
if (!init)
init = {};
// handle iOS platform where "immersive-ar" is special:
// - we either launch QuickLook
// - or forward to the Needle App Clip experience for WebXR AR
// TODO: should we add a separate mode (e.g. "AR")? https://linear.app/needle/issue/NE-5303
if (DeviceUtilities.isiOS()) {
const arSupported = await this.isARSupported().catch(() => false);
// On VisionOS, we use QuickLook for AR experiences; no AppClip support for now.
if (DeviceUtilities.isVisionOS() && !arSupported && (mode === "ar" || mode === "immersive-ar")) {
mode = "quicklook";
}
if (mode === "quicklook") {
Telemetry.sendEvent(Context.Current, "xr", {
action: "quicklook_export",
source: "NeedleXRSession.start",
});
InternalUSDZRegistry.exportAndOpen();
return null;
}
if (!arSupported && (mode === "immersive-ar" || mode === "ar")) {
this.invokeSessionRequestStart("immersive-ar", init);
// Forward to the AppClip experience (Using the apple.com url the appclip overlay shows immediately)
// const url =`https://appclip.needle.tools/ar?url=${(location.href)}`;
const url = new URL("https://appclip.apple.com/id?p=tools.needle.launch-app.Clip");
url.searchParams.set("url", location.href);
const urlStr = url.toString();
Telemetry.sendEvent(Context.Current, "xr", {
action: "app_clip_launch",
source: "NeedleXRSession.start",
url: urlStr,
});
// if we are in an iframe, we need to navigate the top window
const topWindow = window.top || window;
try {
console.debug("iOS device detected - opening Needle App Clip for AR experience", { mode, init, url });
// navigate to app clip url but keep the current url in history, open in same tab
// eslint-disable-next-line xss/no-location-href-assign
topWindow.location.href = urlStr;
}
catch (e) {
console.warn("Error navigating to AppClip " + urlStr + "\n", e);
// if top window navigation fails and we are in an iframe, we try to navigate the top window directly
const weAreInIframe = window !== window.top;
if (weAreInIframe) {
// we can try to open a new tab as a fallback
window.open(urlStr, "_blank");
}
// eslint-disable-next-line xss/no-location-href-assign
else
window.location.href = urlStr;
}
setTimeout(() => {
this.invokeSessionRequestEnd("immersive-ar", init || {}, null);
}, 3000);
return null;
}
}
if (mode === "quicklook") {
console.warn("QuickLook mode is only supported on iOS devices");
return null;
}
// Since we now know we are not on iOS, ar mode becomes "immersive-ar"
if (mode == "ar") {
mode = "immersive-ar";
}
if (isDevEnvironment() && getParam("debugxrpreroom")) {
console.warn("Debug: Starting temporary XR session");
await TemporaryXRContext.start(mode, init || NeedleXRSession.getDefaultSessionInit(mode));
return null;
}
if (this._currentSessionRequest) {
console.warn("A XRSession is already being requested");
if (debug || isDevEnvironment())
showBalloonWarning("A XRSession is already being requested");
return this._currentSessionRequest.then(() => this._activeSession);
}
if (this._activeSession) {
console.error("A XRSession is already running");
return this._activeSession;
}
// Make sure we have a context
if (!context)
context = Context.Current;
if (!context)
context = ContextRegistry.All[0];
if (!context)
throw new Error("No Needle Engine Context found");
//performance.mark('NeedleXRSession start');
switch (mode) {
// Setup VR initialization parameters
case "immersive-ar":
{
const supported = await this.xrSystem?.isSessionSupported('immersive-ar');
if (supported !== true) {
console.error(mode + ' is not supported by this browser.');
return null;
}
const defaultInit = this.getDefaultSessionInit(mode);
const domOverlayElement = getDOMOverlayElement(context.domElement);
if (domOverlayElement && !DeviceUtilities.isQuest()) { // excluding quest since dom-overlay breaks sessiongranted
defaultInit.domOverlay = { root: domOverlayElement };
defaultInit.optionalFeatures.push('dom-overlay');
}
init = {
...defaultInit,
...init,
};
}
break;
// Setup AR initialization parameters
case "immersive-vr":
{
const supported = await this.xrSystem?.isSessionSupported('immersive-vr');
if (supported !== true) {
console.error(mode + ' is not supported by this browser.');
return null;
}
const defaultInit = this.getDefaultSessionInit(mode);
init = {
...defaultInit,
...init,
};
}
break;
default:
console.warn("No default session init for mode", mode);
break;
}
// Fix: define these here because VariantLauncher crashes otherwise. Spec: https://immersive-web.github.io/webxr/#feature-dependencies
// Issue: https://linear.app/needle/issue/NE-5136
init.optionalFeatures ??= [];
init.requiredFeatures ??= [];
// we stop a temporary session here (if any runs)
await TemporaryXRContext.stop();
const scripts = mode == "immersive-ar" ? context.scripts_immersive_ar : context.scripts_immersive_vr;
if (debug)
console.log("%c" + `Requesting ${mode} session`, "font-weight:bold;", init, scripts);
else
console.log("%c" + `Requesting ${mode} session`, "font-weight:bold;");
for (const script of scripts) {
if (script.onBeforeXR && script.activeAndEnabled && !script.destroyed) {
script.onBeforeXR(mode, init);
}
}
this.invokeSessionRequestStart(mode, init);
if (debug)
showBalloonMessage("Requesting " + mode + " session (" + Date.now() + ")");
Telemetry.sendEvent(Context.Current, "xr", {
action: "session_request",
mode: mode,
features: ((init.requiredFeatures ?? []).concat(init.optionalFeatures ?? [])).join(","),
source: "NeedleXRSession.start",
});
this._currentSessionRequest = navigator?.xr?.requestSession(mode, init);
this._currentSessionRequestMode = mode;
/**@type {XRSystem} */
const newSession = await (this._currentSessionRequest)?.catch(e => {
console.error(e, "Code: " + e?.code);
if (e?.code === 9)
showBalloonWarning("Couldn't start XR session. Make sure you allow the required permissions.");
console.log("If the specified XR configuration is not supported (e.g. entering AR doesnt work) - make sure you access the website on a secure connection (HTTPS) and your device has the required permissions (e.g. camera access)");
const notSecure = location.protocol === 'http:';
if (notSecure)
showBalloonWarning("XR requires a secure connection (HTTPS)");
});
this._currentSessionRequest = undefined;
this._currentSessionRequestMode = null;
this.invokeSessionRequestEnd(mode, init, newSession);
if (!newSession) {
console.warn("XR Session request was rejected");
return null;
}
const session = this.setSession(mode, newSession, init, context);
//performance.mark('NeedleXRSession end');
//performance.measure('NeedleXRSession Startup', 'NeedleXRSession start', 'NeedleXRSession end');
return session;
}
static invokeSessionRequestStart(mode, init) {
for (const listener of this._sessionRequestStartListeners) {
listener({ mode, init });
}
}
static invokeSessionRequestEnd(mode, init, session) {
for (const listener of this._sessionRequestEndListeners) {
listener({ mode, init, newSession: session || null });
}
}
static setSession(mode, session, init, context) {
if (this._activeSession) {
console.error("A XRSession is already running");
return this._activeSession;
}
const scripts = mode == "immersive-ar" ? context.scripts_immersive_ar : context.scripts_immersive_vr;
this._activeSession = new NeedleXRSession(mode, session, context, {
scripts: scripts,
controller_added: this._controllerAddedListeners,
controller_removed: this._controllerRemovedListeners,
init: init
});
session.addEventListener("end", this.onEnd);
if (debug)
console.log("%c" + `Started ${mode} session`, "font-weight:bold;", scripts);
else
console.log("%c" + `Started ${mode} session`, "font-weight:bold;");
return this._activeSession;
}
static $_stop_request = Symbol();
/** stops the active XR session */
static stop() {
const session = this._activeSession;
if (session) {
if (session[this.$_stop_request] === undefined) {
if (debug)
console.log("[NeedleXRSession] Stopping XR Session... (new)");
// We can not call session.end() immediately because we might be within a frame callback
// For example if a use calls `NeedleXRSession.stop()` within a `update()` callback
session[this.$_stop_request] = setTimeout(() => {
session.end();
});
}
else if (debug) {
console.warn("[NeedleXRSession] XR Session stop already requested");
}
}
}
static onEnd = () => {
if (debug)
console.log("XR Session ended");
this._activeSession = null;
};
/** The needle engine context this session was started from */
context;
get sync() {
return NeedleXRSession._sync;
}
/** Returns true if the xr session is still active */
get running() { return !this._ended && this.session != null; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession
*/
session;
/** XR Session Mode: AR or VR */
mode;
/**
* The XRSession interface's read-only interactionMode property describes the best space (according to the user agent) for the application to draw an interactive UI for the current session.
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/interactionMode
*/
get interactionMode() { return this.session["interactionMode"]; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/visibilityState
* @returns {XRVisibilityState} The visibility state of the XRSession
*/
get visibilityState() { return this.session.visibilityState; }
/**
* Check if the session is `visible-blurred` - this means e.g. the keyboard is shown
*/
get isVisibleBlurred() { return this.session.visibilityState === 'visible-blurred'; }
/**
* Check if the session has system keyboard support
*/
get isSystemKeyboardSupported() { return this.session.isSystemKeyboardSupported; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/environmentBlendMode
*/
get environmentBlendMode() { return this.session.environmentBlendMode; }
/**
* The current XR frame
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRFrame
*/
get frame() { return this.context.xrFrame; }
/** The currently active/connected controllers */
controllers = [];
/** shorthand to query the left controller. Use `controllers` to get access to all connected controllers */
get leftController() { return this.controllers.find(c => c.side === "left"); }
/** shorthand to query the right controller. Use `controllers` to get access to all connected controllers */
get rightController() { return this.controllers.find(c => c.side === "right"); }
/** @returns the given controller if it is connected */
getController(side) {
if (typeof side === "number")
return this.controllers[side] || null;
return this.controllers.find(c => c.side === side) || null;
}
/** Returns true if running in pass through mode in immersive AR (e.g. user is wearing a headset while in AR) */
get isPassThrough() {
if (this.environmentBlendMode !== 'opaque' && this.interactionMode === "world-space")
return true;
// since we can not rely on interactionMode check we check the controllers too
// https://linear.app/needle/issue/NE-4057
// the following is a workaround for the issue above
if (this.mode === "immersive-ar" && this.environmentBlendMode !== 'opaque') {
// if we have any tracked pointer controllers we're also in passthrough
if (this.controllers.some(c => c.inputSource.targetRayMode === "tracked-pointer"))
return true;
}
if (isDevEnvironment() && DeviceUtilities.isDesktop() && this.mode === "immersive-ar") {
return true;
}
return false;
}
get isAR() { return this.mode === 'immersive-ar'; }
get isVR() { return this.mode === 'immersive-vr'; }
/** If the AR mode is not immersive (meaning the user is e.g. holding a phone instead of wearing a AR passthrough headset) */
get isScreenBasedAR() { return this.isAR && !this.isPassThrough; }
get posePosition() { return this._transformPosition; }
get poseOrientation() { return this._transformOrientation; }
/** @returns the context.renderer.xr.getReferenceSpace() result */
get referenceSpace() { return this.context.renderer.xr.getReferenceSpace(); }
/** @returns the XRFrame `viewerpose` using the xr `referenceSpace` */
get viewerPose() { return this._viewerPose; }
/** @returns `true` if any image is currently being tracked */
/** returns true if images are currently being tracked */
get isTrackingImages() {
if (this.frame && "getImageTrackingResults" in this.frame && typeof this.frame.getImageTrackingResults === "function") {
try {
const trackingResult = this.frame.getImageTrackingResults();
for (const result of trackingResult) {
const state = result.trackingState;
if (state === "tracked")
return true;
}
}
catch {
// Looks like we get a NotSupportedException on Android since the method is known
// but the feature is not supported by the session
// TODO Can we check here if we even requested the image-tracking feature instead of catching?
return false;
}
}
return false;
}
/** The currently active XR rig */
get rig() {
const rig = this._rigs[0] ?? null;
if (rig?.gameObject && isDestroyed(rig.gameObject) || rig?.isActive === false) {
this.updateActiveXRRig();
return this._rigs[0] ?? null;
}
return rig;
}
_rigScale = 1;
_lastRigScaleUpdate = -1;
/** Get the XR Rig worldscale.
*
* **For AR**
* If you want to modify the scale in AR at runtime get the WebARSessionRoot component via `findObjectOfType(WebARSessionRoot)` and then set the `arScale` value.
*
*/
get rigScale() {
// const ar_scaleValue = this.context.domElement.getAttribute("ar-scale");
// if (ar_scaleValue) {
// const scale = parseFloat(ar_scaleValue);
// if (!isNaN(scale)) return scale;
// }
if (!this._rigs[0])
return 1;
if (this._lastRigScaleUpdate !== this.context.time.frame) {
this._lastRigScaleUpdate = this.context.time.frame;
this._rigScale = this._rigs[0].gameObject.worldScale.x;
}
return this._rigScale;
}
/** add a rig to the available XR rigs - if it's priority is higher than the currently active rig it will be enabled */
addRig(rig) {
const i = this._rigs.indexOf(rig);
if (i >= 0)
return;
if (rig.priority === undefined)
rig.priority = 0;
this._rigs.push(rig);
this.updateActiveXRRig();
}
/** Remove a rig from the available XR Rigs */
removeRig(rig) {
const i = this._rigs.indexOf(rig);
if (i === -1)
return;
this._rigs.splice(i, 1);
this.updateActiveXRRig();
}
/** Sets a XRRig to be active which will parent the camera to this rig */
setRigActive(rig) {
const i = this._rigs.indexOf(rig);
const currentlyActive = this._rigs[0];
this._rigs.splice(i, 1);
this._rigs.unshift(rig);
// if there's another rig currently active we need to make sure we have at least the same priority
rig.priority = currentlyActive?.priority ?? 0;
this.updateActiveXRRig();
}
/**
* @returns the user position in the rig space
*/
getUserOffsetInRig() {
const positionInRig = this.context.mainCamera?.position;
if (!positionInRig || !this.rig)
return getTempVector(0, 0, 0);
const vec = getTempVector(positionInRig);
vec.x *= -1;
vec.z *= -1;
vec.applyQuaternion(getTempQuaternion(this.rig.gameObject.quaternion));
return vec;
}
updateActiveXRRig() {
const previouslyActiveRig = this._rigs[0] ?? null;
// ensure that the default rig is in the scene
if (this._defaultRig.gameObject.parent !== this.context.scene)
this.context.scene.add(this._defaultRig.gameObject);
// ensure the fallback rig is always active!!!
this._defaultRig.gameObject.visible = true;
// ensure that the default rig is in the list of available rigs
if (!this._rigs.includes(this._defaultRig))
this._rigs.push(this._defaultRig);
// find the rig with the highest priority and make sure it's at the beginning of the array
let highestPriorityRig = this._rigs[0];
if (highestPriorityRig && highestPriorityRig.priority === undefined)
highestPriorityRig.priority = 0;
for (let i = 1; i < this._rigs.length; i++) {
const rig = this._rigs[i];
if (!rig.isActive)
continue;
if (isDestroyed(rig.gameObject)) {
this._rigs.splice(i, 1);
i--;
continue;
}
if (!highestPriorityRig || highestPriorityRig.isActive === false || (rig.priority !== undefined && rig.priority > highestPriorityRig.priority)) {
highestPriorityRig = rig;
}
}
// make sure the highest priority rig is at the beginning if it isnt already
if (previouslyActiveRig !== highestPriorityRig) {
const index = this._rigs.indexOf(highestPriorityRig);
if (index >= 0)
this._rigs.splice(index, 1);
this._rigs.unshift(highestPriorityRig);
}
if (debug) {
if (previouslyActiveRig === highestPriorityRig)
console.log("Updated Active XR Rig:", highestPriorityRig, "prev:", previouslyActiveRig);
else
console.log("Updated Active XRRig:", highestPriorityRig, " (the same as before)");
}
}
_rigs = [];
_viewerHitTestSource = null;
/** Returns a XR hit test result (if hit-testing is available) in rig space
* @param source If provided, the hit test will be performed for the given controller
*/
getHitTest(source) {
if (source) {
return this.getControllerHitTest(source);
}
if (!this._viewerHitTestSource)
return null;
const hitTestSource = this._viewerHitTestSource;
const hitTestResults = this.frame.getHitTestResults(hitTestSource);
if (hitTestResults.length > 0) {
const hit = hitTestResults[0];
return this.convertHitTestResult(hit);
}
return null;
}
getControllerHitTest(controller) {
const hitTestSource = controller.getHitTestSource();
if (!hitTestSource)
return null;
const res = this.frame.getHitTestResultsForTransientInput(hitTestSource);
for (const result of res) {
if (result.inputSource === controller.inputSource) {
for (const hit of result.results) {
return this.convertHitTestResult(hit);
}
}
}
return null;
}
convertHitTestResult(result) {
const referenceSpace = this.context.renderer.xr.getReferenceSpace();
const pose = referenceSpace && result.getPose(referenceSpace);
if (pose) {
const pos = getTempVector(pose.transform.position);
const rot = getTempQuaternion(pose.transform.orientation);
const camera = this.context.mainCamera;
if (camera?.parent !== this._cameraRenderParent) {
pos.applyMatrix4(flipForwardMatrix);
}
if (camera?.parent) {
pos.applyMatrix4(camera.parent.matrixWorld);
rot.multiply(flipForwardQuaternion);
// apply parent quaternion (if parent is moved/rotated)
const parentRotation = getWorldQuaternion(camera.parent);
// ensure that "up" (y+) is pointing away from the wall
parentRotation.premultiply(flipForwardQuaternion);
rot.premultiply(parentRotation);
}
return { hit: result, position: pos, quaternion: rot };
}
return null;
}
/** convert a XRRigidTransform from XR session space to threejs / Needle Engine XR space */
convertSpace(transform) {
const pos = getTempVector(transform.position);
pos.applyMatrix4(flipForwardMatrix);
const rot = getTempQuaternion(transform.orientation);
rot.premultiply(flipForwardQuaternion);
return { position: pos, quaternion: rot };
}
/** this is the implictly created XR rig */
_defaultRig;
/** all scripts that receive some sort of XR update event */
_xr_scripts;
/** scripts that have onUpdateXR event methods */
_xr_update_scripts = [];
/** scripts that are in the scene but inactive (e.g. disabled parent gameObject) */
_inactive_scripts = [];
_controllerAdded;
_controllerRemoved;
_originalCameraWorldPosition;
_originalCameraWorldRotation;
_originalCameraWorldScale;
_originalCameraParent;
/** we store the main camera reference here each frame to make sure we have a rendering camera
* this e.g. the case when the XR rig with the camera gets disabled (and thus this.context.mainCamera is unassigned)
*/
_mainCamera = null;
constructor(mode, session, context, extra) {
//performance.mark(measure_SessionStartedMarker);
saveSessionInfo(mode, extra.init);
this.session = session;
this.mode = mode;
this.context = context;
if (debug || getParam("console"))
enableSpatialConsole(true);
this._xr_scripts = [...extra.scripts];
this._xr_update_scripts = this._xr_scripts.filter(e => typeof e.onUpdateXR === "function");
this._controllerAdded = extra.controller_added;
this._controllerRemoved = extra.controller_removed;
registerFrameEventCallback(this.onBefore, FrameEvent.LateUpdate);
this.context.pre_render_callbacks.push(this.onBeforeRender);
this.context.post_render_callbacks.push(this.onAfterRender);
if (extra.init.optionalFeatures?.includes("hit-test") || extra.init.requiredFeatures?.includes("hit-test")) {
session.requestReferenceSpace('viewer').then((referenceSpace) => {
return session.requestHitTestSource?.call(session, { space: referenceSpace })?.then((source) => {
return this._viewerHitTestSource = source;
}).catch(e => console.error(e));
}).catch(e => console.error(e));
}
if (this.context.mainCamera) {
this._originalCameraWorldPosition = getWorldPosition(this.context.mainCamera, new Vector3());
this._originalCameraWorldRotation = getWorldQuaternion(this.context.mainCamera, new Quaternion());
this._originalCameraWorldScale = getWorldScale(this.context.mainCamera, new Vector3());
this._originalCameraParent = this.context.mainCamera.parent;
if (this.context.mainCamera instanceof PerspectiveCamera) {
this.context.mainCamera[$initialFov] = this.context.mainCamera.fov;
// if (this.mode === "immersive-ar" && this.context.mainCamera.near > .1) {
// console.log("[WebXR] Setting near plane to 0.1 for better AR experience (was " + this.context.mainCamera.near + "). The initial near plane will be restored when the session ends.");
// this.context.mainCamera.near = 0.1;
// this.context.mainCamera[$initialNear] = this.context.mainCamera.near;
// }
}
}
this._defaultRig = new ImplictXRRig();
this.context.scene.add(this._defaultRig.gameObject);
this.addRig(this._defaultRig);
// register already connected input sources
// this is for when the session is already running (via a temporary xr session)
// and the controllers are already connected
for (let i = 0; i < session.inputSources.length; i++) {
const inputSource = session.inputSources[i];
if (!inputSource.handedness) {
console.warn("Input source in xr session has no handedness - ignoring", i);
continue;
}
this.onInputSourceAdded(inputSource);
}
// handle controller and input source changes changes
this.session.addEventListener('end', this.onEnd);
// handle input sources change
this.session.addEventListener("inputsourceschange",
/* @ts-ignore (ignore CI XRInputSourceChangeEvent mismatch) */
(evt) => {
// handle removed controllers
for (const removedInputSource of evt.removed) {
this.disconnectInputSource(removedInputSource);
}
for (const newInputSource of evt.added) {
this.onInputSourceAdded(newInputSource);
}
});
// Unfortunately the code below doesnt work: the session never receives any input sources sometimes
// https://developer.mozilla.org/en-US/docs/Web/API/XRSession/visibilitychange_event
// this.session.addEventListener("visibilitychange", (evt: XRSessionEvent) => {
// // sometimes when entering an XR session the controllers are not added/not in the list and we don't receive an event
// // this is a workaround trying to add controllers when the scene visibility changes to "visible"
// // e.g. due to a user opening and closing the menu
// if (this.controllers.length === 0 && evt.session.visibilityState === "visible") {
// for (const controller of evt.session.inputSources) {
// this.onInputSourceAdded(controller);
// }
// }
// })
// we set the session on the webxr manager at the end because we want to receive inputsource events first
// e.g. in case there's a bug in the threejs codebase
this.context.xr = this;
this.context.renderer.xr.setSession(this.session).then(this.onRendererSessionSet);
// disable three.js renderer controller autoUpdate (added in ac67b31e3548386f8a93e23a4176554c92bbd0d9)
if ("controllerAutoUpdate" in this.context.renderer.xr) {
console.debug("Disabling three.js controllerAutoUpdate");
this.context.renderer.xr.controllerAutoUpdate = false;
}
else if (debug) {
console.warn("controllerAutoUpdate is not available in three.js - cannot disable it");
}
// In the appclip we want to make sure the canvas resolution is correctly applied with the DPR
// this change has been added to Needle Go AppClip in daa9c186e49fe9d6f14d200e307eb0664e35ad72
// But for an immediate fix for Needle Engine projects we also add it here.
// We can most likely remove the folllowing block in march 2026
if (DeviceUtilities.isNeedleAppClip()) {
window.requestAnimationFrame(() => {
const canvas = this.context.renderer.domElement;
const dpr = window.devicePixelRatio || 1;
const currentWidth = canvas.width;
const currentHeight = canvas.height;
// Check if DPR is already applied (avoid double-scaling)
const expectedWidth = Math.floor(window.innerWidth * dpr);
const expectedHeight = Math.floor(window.innerHeight * dpr);
if (Math.abs(currentWidth - expectedWidth) > 2 || Math.abs(currentHeight - expectedHeight) > 2) {
canvas.width = expectedWidth;
canvas.height = expectedHeight;
console.debug("Applied DPR scaling for Needle AppClip XR session", dpr, canvas.width, canvas.height);
}
});
}
}
/** called when renderer.setSession is fulfilled */
onRendererSessionSet = () => {
if (!this.running)
return;
this.context.renderer.xr.enabled = true;
// calling update camera here once to prevent the xr camera left/right NaN in the first frame due to false near/far plane values, see NE-4126
this.context.renderer.xr.updateCamera(this.context.mainCamera);
this.context.mainCameraComponent?.applyClearFlags();
};
onInputSourceAdded = (newInputSource) => {
// do not create XR controllers for screen input sources
if (newInputSource.targetRayMode === "screen") {
return;