@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
1,031 lines • 82.1 kB
JavaScript
import { Object3D, PerspectiveCamera, Quaternion, Vector3 } from "three";
import { enableSpatialConsole, isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../debug/index.js";
import { Application, internalOnUserInputRegistered } from "../engine_application.js";
import { Context, FrameEvent } from "../engine_context.js";
import { ContextEvent, ContextRegistry } from "../engine_context_registry.js";
import { isDestroyed } from "../engine_gameobject.js";
import { Gizmos } from "../engine_gizmos.js";
import { Telemetry } from "../engine_license.js";
import { registerFrameEventCallback, unregisterFrameEventCallback } from "../engine_lifecycle_functions_internal.js";
import { getBoundingBox, getTempQuaternion, getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../engine_three_utils.js";
import { DeviceUtilities, getParam } from "../engine_utils.js";
import { invokeXRSessionEnd, invokeXRSessionStart } from "./events.js";
import { flipForwardMatrix, flipForwardQuaternion, ImplictXRRig } from "./internal.js";
import { NeedleXRController } from "./NeedleXRController.js";
import { NeedleXRSync } from "./NeedleXRSync.js";
import { SceneTransition } from "./SceneTransition.js";
import { TemporaryXRContext } from "./TempXRContext.js";
import { InternalUSDZRegistry } from "./usdz.js";
const measure_SessionStartedMarker = "NeedleXRSession onStart";
const measure_SessionEndedMarker = "NeedleXRSession onEnd";
const debug = getParam("debugwebxr");
const debugFPS = getParam("stats");
let debugFPSFramesSinceLastUpdate = 0;
function getDOMOverlayElement(domElement) {
let arOverlayElement = null;
// for react cases we dont have an Engine Element
const element = domElement;
if (element.getAROverlayContainer)
arOverlayElement = element.getAROverlayContainer();
else
arOverlayElement = domElement;
return arOverlayElement;
}
handleSessionGranted();
async function handleSessionGranted() {
let defaultMode = "immersive-vr";
try {
// In app clips we default to AR
if (DeviceUtilities.isNeedleAppClip()) {
defaultMode = "immersive-ar";
}
// Check if VR is even supported, otherwise try AR
else if (!(await navigator.xr?.isSessionSupported("immersive-vr"))) {
defaultMode = "immersive-ar";
}
// Check if AR is supported, otherwise we can't do anything
if (!(await navigator.xr?.isSessionSupported("immersive-ar")) && defaultMode === "immersive-ar") {
// console.warn("[NeedleXRSession:granted] Neither VR nor AR supported, aborting session start.");
// showBalloonMessage("NeidleXRSession: Neither VR nor AR supported, aborting session start.");
return;
}
}
catch (e) {
console.debug("[NeedleXRSession:granted] Error while checking XR support:", e);
// showBalloonWarning("NeedleXRSession: Error while checking XR support: " + (e as Error).message);
return;
}
// TODO: asap session granted doesnt handle the pre-room yet
if (getParam("debugasap")) {
let asapSession = globalThis["needle:XRSession"];
// @ts-ignore // getting TS2848 here, not sure why
if (asapSession instanceof (Promise)) {
delete globalThis["needle:XRSession"];
ContextRegistry.addContextCreatedCallback(async (cb) => {
if (!asapSession)
return;
// TODO: add support to pass this to the temporary room
enableSpatialConsole(true);
const session = await asapSession;
if (session) {
const sessionInit = NeedleXRSession.getDefaultSessionInit(defaultMode);
NeedleXRSession.setSession(defaultMode, session, sessionInit, cb.context);
}
else {
console.error("[NeedleXRSession:granted] ASAP session was rejected");
}
asapSession = undefined;
});
return;
}
}
// console.log("Attaching sessiongranted handler...", { haveXR: 'xr' in navigator });
// setTimeout(() => console.log("Session Granted handler attached.", { haveXR: 'xr' in navigator }), 1000);
if ('xr' in navigator) {
// WebXRViewer (based on Firefox) has a bug where addEventListener
// throws a silent exception and aborts execution entirely.
if (/WebXRViewer\//i.test(navigator.userAgent)) {
console.warn('WebXRViewer does not support addEventListener');
return;
}
navigator.xr?.addEventListener('sessiongranted', async () => {
// enableSpatialConsole(true);
const lastSessionMode = sessionStorage.getItem("needle_xr_session_mode");
const lastSessionInit = sessionStorage.getItem("needle_xr_session_init") ?? null;
const init = lastSessionInit ? JSON.parse(lastSessionInit) : null;
let info = null;
if (contextIsLoading()) {
await TemporaryXRContext.start(lastSessionMode || defaultMode, init || NeedleXRSession.getDefaultSessionInit(defaultMode))
.catch(e => console.warn("[NeedleXRSession:granted] TemporaryXRContext start failed:", e));
await waitForContextLoadingFinished();
info = await TemporaryXRContext.handoff();
}
if (info) {
NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current);
}
else if (lastSessionMode && lastSessionInit) {
console.log("[NeedleXRSession:granted] Restore last session");
const init = JSON.parse(lastSessionInit);
NeedleXRSession.start(lastSessionMode, init).catch(e => console.warn(e));
}
else {
// if no session was found we start VR by default
NeedleXRSession.start(defaultMode).catch(e => console.warn("[NeedleXRSession:granted] failed:", e));
}
// make sure we only subscribe to the event once
}, { once: true });
}
else {
// showBalloonWarning("NeedleXRSession: WebXR not available in this browser.");
}
}
function saveSessionInfo(mode, init) {
sessionStorage.setItem("needle_xr_session_mode", mode);
sessionStorage.setItem("needle_xr_session_init", JSON.stringify(init));
}
function deleteSessionInfo() {
sessionStorage.removeItem("needle_xr_session_mode");
sessionStorage.removeItem("needle_xr_session_init");
}
const contexts_loading = new Set();
ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, async (cb) => {
contexts_loading.add(cb.context);
});
ContextRegistry.registerCallback(ContextEvent.ContextCreated, async (cb) => {
contexts_loading.delete(cb.context);
const autostart = cb.context?.domElement.getAttribute("autostart") || null;
handleAutoStart(autostart);
});
function contextIsLoading() { return contexts_loading.size > 0; }
function waitForContextLoadingFinished() {
return new Promise(res => {
const startTime = Date.now();
const interval = setInterval(() => {
if (!contextIsLoading() || Date.now() - startTime > 60000) {
clearInterval(interval);
res();
}
}, 100);
});
}
if (typeof window !== "undefined" && DeviceUtilities.isDesktop() && isDevEnvironment()) {
window.addEventListener("keydown", (evt) => {
if (evt.key === "x" || evt.key === "Escape") {
if (NeedleXRSession.active) {
NeedleXRSession.stop();
}
}
});
}
function handleAutoStart(value) {
if (!value)
return;
switch (value?.toLowerCase()) {
case "ar":
Application.registerWaitForInteraction(() => {
// TODO: move WebARSessionRoot to core
NeedleXRSession.start("ar");
});
break;
}
}
// if (getParam("simulatewebxrloading")) {
// ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, async _cb => {
// await delay(3000);
// setTimeout(async () => {
// const info = await TemporaryXRContext.handoff();
// if (info) NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current);
// else
// NeedleXRSession.start("immersive-vr")
// }, 6000)
// });
// let triggered = false;
// window.addEventListener("click", () => {
// if (triggered) return;
// triggered = true;
// TemporaryXRContext.start("immersive-vr", NeedleXRSession.getDefaultSessionInit("immersive-vr"));
// });
// }
const $initialFov = Symbol("initial-fov");
const $initialNear = Symbol("initial-near");
/**
* This class manages an XRSession to provide helper methods and events. It provides easy access to the XRInputSources (controllers and hands)
* - Start a XRSession with `NeedleXRSession.start(...)`
* - Stop a XRSession with `NeedleXRSession.stop()`
* - Access a running XRSession with `NeedleXRSession.active`
*
* If a XRSession is active you can use all XR-related event methods on your components to receive XR events e.g. `onEnterXR`, `onUpdateXR`, `onLeaveXR`
* ```ts
* export class MyComponent extends Behaviour {
* // callback invoked whenever the XRSession is started or your component is added to a scene with an active XRSession
* onEnterXR(args: NeedleXREventArgs) {
* console.log("Entered XR");
* // access the NeedleXRSession via args.xr
* }
* // callback invoked whenever a controller is added (or you switch from controller to hand tracking)
* onControllerAdded(args: NeedleXRControllerEventArgs) { }
* }
* ```
*
* ### XRRig
* The XRRig can be accessed via the `rig` property
* Set a custom XRRig via `NeedleXRSession.addRig(...)` or `NeedleXRSession.removeRig(...)`
* By default the active XRRig with the highest priority in the scene is used
*
* ### Screenshots in XR
* Screenshots work automatically during XR sessions, including AR camera feed compositing. See {@link screenshot2} for more information.
*
* @category XR
* @see {@link screenshot2} for taking screenshots in XR sessions
*/
export class NeedleXRSession {
static _sync = null;
static getXRSync(context) {
if (!this._sync)
this._sync = new NeedleXRSync(context);
return this._sync;
}
static get currentSessionRequest() { return this._currentSessionRequestMode; }
static _currentSessionRequestMode = null;
/**
* @returns the active @type {NeedleXRSession} (if any active) or null
*/
static get active() { return this._activeSession; }
/** The active xr session mode (if any xr session is active)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSessionMode
*/
static get activeMode() { return this._activeSession?.mode ?? null; }
/** XRSystem via navigator.xr access
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem
*/
static get xrSystem() {
return ('xr' in navigator) ? navigator.xr : undefined;
}
/**
* @returns true if the browser supports WebXR (`immersive-vr` or `immersive-ar`)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isXRSupported() { return Promise.all([this.isVRSupported(), this.isARSupported()]).then(res => res.some(e => e)).catch(() => false); }
/**
* @returns true if the browser supports immersive-vr (WebXR)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isVRSupported() { return this.isSessionSupported("immersive-vr"); }
/**
* @returns true if the browser supports immersive-ar (WebXR)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isARSupported() { return this.isSessionSupported("immersive-ar"); }
/**
* @param mode The XRSessionMode to check if it is supported
* @returns true if the browser supports the given XRSessionMode
*/
static isSessionSupported(mode) {
// We cache the result of the session support check to avoid unnecessary calls to isSessionSupported. This is especially important for user input events that internally run session support checks during onclick where the event must be resolved as fast as possible.
if (this._sessionSupportedCache[mode] !== undefined) {
return Promise.resolve(this._sessionSupportedCache[mode]);
}
return this.xrSystem?.isSessionSupported(mode).then(supported => {
this._sessionSupportedCache[mode] = supported;
return supported;
}).catch(err => { if (debug)
console.error(err); return false; }) ?? Promise.resolve(false);
}
static _sessionSupportedCache = {};
static _currentSessionRequest;
static _activeSession;
/** Register to listen to XRSession start events. Unsubscribe with `offXRSessionStart` */
static onSessionRequestStart(evt) {
this._sessionRequestStartListeners.push(evt);
}
/** Unsubscribe from request start evt. Register with `onSessionRequestStart` */
static offSessionRequestStart(evt) {
const index = this._sessionRequestStartListeners.indexOf(evt);
if (index >= 0)
this._sessionRequestStartListeners.splice(index, 1);
}
static _sessionRequestStartListeners = [];
/** Called after the session request has finished */
static onSessionRequestEnd(evt) {
this._sessionRequestEndListeners.push(evt);
}
/** Unsubscribe from request end evt */
static offSessionRequestEnd(evt) {
const index = this._sessionRequestEndListeners.indexOf(evt);
if (index >= 0)
this._sessionRequestEndListeners.splice(index, 1);
}
static _sessionRequestEndListeners = [];
/** Listen to XR session started. Unsubscribe with `offXRSessionStart` */
static onXRSessionStart(evt) {
this._xrStartListeners.push(evt);
}
;
/** Unsubscribe from XRSession started events */
static offXRSessionStart(evt) {
const index = this._xrStartListeners.indexOf(evt);
if (index >= 0)
this._xrStartListeners.splice(index, 1);
}
static _xrStartListeners = [];
/** Listen to XR session ended. Unsubscribe with `offXRSessionEnd` */
static onXRSessionEnd(evt) {
this._xrEndListeners.push(evt);
}
;
/** Unsubscribe from XRSession started events */
static offXRSessionEnd(evt) {
const index = this._xrEndListeners.indexOf(evt);
if (index >= 0)
this._xrEndListeners.splice(index, 1);
}
static _xrEndListeners = [];
/** Listen to controller added events.
* Events are cleared when starting a new session
**/
static onControllerAdded(evt) {
this._controllerAddedListeners.push(evt);
}
/** Unsubscribe from controller added evts */
static offControllerAdded(evt) {
const index = this._controllerAddedListeners.indexOf(evt);
if (index >= 0)
this._controllerAddedListeners.splice(index, 1);
}
static _controllerAddedListeners = [];
/** Listen to controller removed events
* Events are cleared when starting a new session
**/
static onControllerRemoved(evt) {
this._controllerRemovedListeners.push(evt);
}
/** Unsubscribe from controller removed events */
static offControllerRemoved(evt) {
const index = this._controllerRemovedListeners.indexOf(evt);
if (index >= 0)
this._controllerRemovedListeners.splice(index, 1);
}
static _controllerRemovedListeners = [];
/** If the browser supports offerSession - creating a VR or AR button in the browser navigation bar */
static offerSession(mode, init, context) {
if ('xr' in navigator && navigator.xr && 'offerSession' in navigator.xr) {
if (typeof navigator.xr.offerSession === "function") {
console.log("WebXR offerSession is available - requesting mode: " + mode);
if (init == "default") {
init = this.getDefaultSessionInit(mode);
}
navigator.xr.offerSession(mode, {
...init
}).then((session) => {
return NeedleXRSession.setSession(mode, session, init, context);
}).catch(_ => {
console.log("XRSession offer rejected (perhaps because another call to offerSession was made or a call to requestSession was made)");
});
}
return true;
}
return false;
}
/** @returns a new XRSession init object with defaults */
static getDefaultSessionInit(mode) {
switch (mode) {
case "immersive-ar":
const arFeatures = ['anchors', 'local-floor', 'layers', 'dom-overlay', 'hit-test', 'unbounded'];
// Don't request handtracking by default on VisionOS
if (!DeviceUtilities.isVisionOS())
arFeatures.push('hand-tracking');
return {
optionalFeatures: arFeatures,
};
case "immersive-vr":
const vrFeatures = ['local-floor', 'bounded-floor', 'high-fixed-foveation-level', 'layers'];
// Don't request handtracking by default on VisionOS
if (!DeviceUtilities.isVisionOS())
vrFeatures.push('hand-tracking');
return {
optionalFeatures: vrFeatures,
};
default:
console.warn("No default session init for mode", mode);
return {};
}
}
/** start a new webXR session (make sure to stop already running sessions before calling this method)
* @param mode The XRSessionMode to start (e.g. `immersive-vr` or `immersive-ar`) or `ar` to start `immersive-ar` on supported devices OR on iOS devices it will export an interactive USDZ and open in Quicklook.
* Get more information about WebXR modes: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionMode
* @param init The XRSessionInit to use (optional), docs: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionInit
* @param context The Needle Engine context to use
*/
static async start(mode, init, context) {
// setup session init args, make sure we have default values
if (!init)
init = {};
// handle iOS platform where "immersive-ar" is special:
// - we either launch QuickLook
// - or forward to the Needle App Clip experience for WebXR AR
if (DeviceUtilities.isiOS()) {
// IMPORTANT: on iOS we should have prefetched immersive-ar support already on app start (this is done in xrInit()) - if not then clicking the Enter AR button for the very first time does not immediately open the AppClip overlay.
// Without an async await here the overlay *does* however open even when clicking for the very first time. (To test this you need to clear the local experience cache on device)
const arSupported = this._sessionSupportedCache["immersive-ar"] ?? await this.isARSupported();
// On VisionOS, we use QuickLook for AR experiences; no AppClip support for now.
if (DeviceUtilities.isVisionOS() && !arSupported && (mode === "ar" || mode === "immersive-ar")) {
mode = "quicklook";
}
if (mode === "quicklook") {
Telemetry.sendEvent(Context.Current, "xr", {
action: "quicklook_export",
source: "NeedleXRSession.start",
});
InternalUSDZRegistry.exportAndOpen();
return null;
}
if (!arSupported && (mode === "immersive-ar" || mode === "ar")) {
this.invokeSessionRequestStart("immersive-ar", init, context ?? Context.Current);
// if we are in an iframe, we need to navigate the top window
const topWindow = window.top || window;
const originalUrl = topWindow.location.href;
// Forward to the AppClip experience (Using the apple.com url the appclip overlay shows immediately)
// const url =`https://appclip.needle.tools/ar?url=${(location.href)}`;
const url = new URL("https://appclip.apple.com/id?p=tools.needle.launch-app.Clip");
url.searchParams.set("url", location.href);
const urlStr = url.toString();
Telemetry.sendEvent(Context.Current, "xr", {
action: "app_clip_launch",
source: "NeedleXRSession.start",
url: urlStr,
});
try {
console.debug("iOS device detected - opening Needle App Clip for AR experience", { mode, init, url });
// Prewarm the appclip by opening the URL
// BUT on safari triggering this with multiple URLs causes the
// Browser to show the dialogue AGAIN in the background
// SO: we don't do it here...
const key = "appclip-prewarm:" + new URL(url).origin;
const allowMultipleNavigations = sessionStorage.getItem(key) === null;
if (allowMultipleNavigations) {
const secondUrl = new URL(url);
secondUrl.searchParams.set("prewarm", "1");
// eslint-disable-next-line xss/no-location-href-assign
topWindow.location.href = secondUrl.toString();
// wait a bit to give the device time to fetch the app clip metadata - this seems to fix the double tap issue (500 ms seem to work)
await new Promise(res => setTimeout(res, 300));
}
// navigate to app clip url but keep the current url in history, open in same tab
// eslint-disable-next-line xss/no-location-href-assign
topWindow.location.href = urlStr;
if (allowMultipleNavigations && !DeviceUtilities.isSafari()) {
setTimeout(() => {
const url2 = new URL(url);
url2.searchParams.set("prewarm", "1");
// eslint-disable-next-line xss/no-location-href-assign
topWindow.location.href = url2.toString();
}, 500);
}
sessionStorage.setItem(key, "1");
}
catch (e) {
console.warn(`Error navigating to AppClip ${urlStr}\n`, e);
// if top window navigation fails and we are in an iframe, we try to navigate the top window directly
const weAreInIframe = window !== window.top;
if (weAreInIframe) {
// we can try to open a new tab as a fallback
window.open(urlStr, "_blank");
}
// eslint-disable-next-line xss/no-location-href-assign
else
window.location.href = urlStr;
}
setTimeout(() => {
this.invokeSessionRequestEnd("immersive-ar", init || {}, null);
}, 3000);
return null;
}
}
if (mode === "quicklook") {
console.error("QuickLook mode is only supported on iOS devices.");
return null;
}
// Since we now know we are not on iOS, ar mode becomes "immersive-ar"
if (mode == "ar") {
mode = "immersive-ar";
}
if (isDevEnvironment() && getParam("debugxrpreroom")) {
console.warn("Debug: Starting temporary XR session");
await TemporaryXRContext.start(mode, init || NeedleXRSession.getDefaultSessionInit(mode));
return null;
}
if (this._currentSessionRequest) {
console.warn("A XRSession is already being requested");
if (debug || isDevEnvironment())
showBalloonWarning("A XRSession is already being requested");
return this._currentSessionRequest.then(() => this._activeSession);
}
if (this._activeSession) {
console.error("A XRSession is already running");
return this._activeSession;
}
// Make sure we have a context
if (!context)
context = Context.Current;
if (!context)
context = ContextRegistry.All[0];
if (!context)
throw new Error("No Needle Engine Context found");
//performance.mark('NeedleXRSession start');
switch (mode) {
// Setup VR initialization parameters
case "immersive-ar":
{
const supported = await this.xrSystem?.isSessionSupported('immersive-ar');
if (supported !== true) {
console.error(mode + ' is not supported by this browser.');
return null;
}
const defaultInit = this.getDefaultSessionInit(mode);
const domOverlayElement = getDOMOverlayElement(context.domElement);
if (domOverlayElement && !DeviceUtilities.isQuest()) { // excluding quest since dom-overlay breaks sessiongranted
defaultInit.domOverlay = { root: domOverlayElement };
defaultInit.optionalFeatures.push('dom-overlay');
}
init = {
...defaultInit,
...init,
};
}
break;
// Setup AR initialization parameters
case "immersive-vr":
{
const supported = await this.xrSystem?.isSessionSupported('immersive-vr');
if (supported !== true) {
console.error(mode + ' is not supported by this browser.');
return null;
}
const defaultInit = this.getDefaultSessionInit(mode);
init = {
...defaultInit,
...init,
};
}
break;
default:
console.warn("No default session init for mode", mode);
break;
}
// Fix: define these here because VariantLauncher crashes otherwise. Spec: https://immersive-web.github.io/webxr/#feature-dependencies
// Issue: https://linear.app/needle/issue/NE-5136
init.optionalFeatures ??= [];
init.requiredFeatures ??= [];
// we stop a temporary session here (if any runs)
await TemporaryXRContext.stop();
const scripts = mode == "immersive-ar" ? context.scripts_immersive_ar : context.scripts_immersive_vr;
if (debug)
console.log("%c" + `Requesting ${mode} session`, "font-weight:bold;", init, scripts);
else
console.log("%c" + `Requesting ${mode} session`, "font-weight:bold;");
for (const script of scripts) {
if (script.onBeforeXR && script.activeAndEnabled && !script.destroyed) {
script.onBeforeXR(mode, init);
}
}
this.invokeSessionRequestStart(mode, init, context ?? Context.Current);
if (debug)
showBalloonMessage("Requesting " + mode + " session (" + Date.now() + ")");
Telemetry.sendEvent(Context.Current, "xr", {
action: "session_request",
mode: mode,
features: ((init.requiredFeatures ?? []).concat(init.optionalFeatures ?? [])).join(","),
source: "NeedleXRSession.start",
});
this._currentSessionRequest = navigator?.xr?.requestSession(mode, init);
this._currentSessionRequestMode = mode;
/**@type {XRSystem} */
const newSession = await (this._currentSessionRequest)?.catch(e => {
console.error(e, "Code: " + e?.code);
if (e?.code === 9)
showBalloonWarning("Couldn't start XR session. Make sure you allow the required permissions.");
console.log("If the specified XR configuration is not supported (e.g. entering AR doesnt work) - make sure you access the website on a secure connection (HTTPS) and your device has the required permissions (e.g. camera access)");
const notSecure = location.protocol === 'http:';
if (notSecure)
showBalloonWarning("XR requires a secure connection (HTTPS)");
});
this._currentSessionRequest = undefined;
this._currentSessionRequestMode = null;
this.invokeSessionRequestEnd(mode, init, newSession);
if (!newSession) {
console.warn("XR Session request was rejected");
return null;
}
const session = this.setSession(mode, newSession, init, context);
//performance.mark('NeedleXRSession end');
//performance.measure('NeedleXRSession Startup', 'NeedleXRSession start', 'NeedleXRSession end');
return session;
}
static invokeSessionRequestStart(mode, init, context) {
for (const listener of this._sessionRequestStartListeners) {
listener({ mode, init, context });
}
}
static invokeSessionRequestEnd(mode, init, session) {
for (const listener of this._sessionRequestEndListeners) {
listener({ mode, init, newSession: session || null });
}
}
static setSession(mode, session, init, context) {
if (this._activeSession) {
console.error("A XRSession is already running");
return this._activeSession;
}
const scripts = mode == "immersive-ar" ? context.scripts_immersive_ar : context.scripts_immersive_vr;
this._activeSession = new NeedleXRSession(mode, session, context, {
scripts: scripts,
controller_added: this._controllerAddedListeners,
controller_removed: this._controllerRemovedListeners,
init: init
});
session.addEventListener("end", this.onEnd);
if (debug)
console.log("%c" + `Started ${mode} session`, "font-weight:bold;", scripts);
else
console.log("%c" + `Started ${mode} session`, "font-weight:bold;");
return this._activeSession;
}
static $_stop_request = Symbol();
/** stops the active XR session */
static stop() {
const session = this._activeSession;
if (session) {
if (session[this.$_stop_request] === undefined) {
if (debug)
console.log("[NeedleXRSession] Stopping XR Session... (new)");
// We can not call session.end() immediately because we might be within a frame callback
// For example if a use calls `NeedleXRSession.stop()` within a `update()` callback
session[this.$_stop_request] = setTimeout(() => {
session.end();
});
}
else if (debug) {
console.warn("[NeedleXRSession] XR Session stop already requested");
}
}
}
static onEnd = () => {
if (debug)
console.log("XR Session ended");
this._activeSession = null;
};
/** The needle engine context this session was started from */
context;
get sync() {
return NeedleXRSession._sync;
}
/** Returns true if the xr session is still active */
get running() { return !this._ended && this.session != null; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession
*/
session;
/** XR Session Mode: AR or VR */
mode;
/**
* The XRSession interface's read-only interactionMode property describes the best space (according to the user agent) for the application to draw an interactive UI for the current session.
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/interactionMode
*/
get interactionMode() { return this.session["interactionMode"]; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/visibilityState
* @returns {XRVisibilityState} The visibility state of the XRSession
*/
get visibilityState() { return this.session.visibilityState; }
/**
* Check if the session is `visible-blurred` - this means e.g. the keyboard is shown
*/
get isVisibleBlurred() { return this.session.visibilityState === 'visible-blurred'; }
/**
* Check if the session has system keyboard support
*/
get isSystemKeyboardSupported() { return this.session.isSystemKeyboardSupported; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/environmentBlendMode
*/
get environmentBlendMode() { return this.session.environmentBlendMode; }
/**
* The current XR frame
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRFrame
*/
get frame() { return this.context.xrFrame; }
/** The currently active/connected controllers */
controllers = [];
/** shorthand to query the left controller. Use `controllers` to get access to all connected controllers */
get leftController() { return this.controllers.find(c => c.side === "left"); }
/** shorthand to query the right controller. Use `controllers` to get access to all connected controllers */
get rightController() { return this.controllers.find(c => c.side === "right"); }
/** @returns the given controller if it is connected */
getController(side) {
if (typeof side === "number")
return this.controllers[side] || null;
return this.controllers.find(c => c.side === side) || null;
}
/** Returns true if running in pass through mode in immersive AR (e.g. user is wearing a headset while in AR) */
get isPassThrough() {
if (this.environmentBlendMode !== 'opaque' && this.interactionMode === "world-space")
return true;
// since we can not rely on interactionMode check we check the controllers too
// https://linear.app/needle/issue/NE-4057
// the following is a workaround for the issue above
if (this.mode === "immersive-ar" && this.environmentBlendMode !== 'opaque') {
// if we have any tracked pointer controllers we're also in passthrough
if (this.controllers.some(c => c.inputSource.targetRayMode === "tracked-pointer"))
return true;
}
if (isDevEnvironment() && DeviceUtilities.isDesktop() && this.mode === "immersive-ar") {
return true;
}
return false;
}
get isAR() { return this.mode === 'immersive-ar'; }
get isVR() { return this.mode === 'immersive-vr'; }
/** If the AR mode is not immersive (meaning the user is e.g. holding a phone instead of wearing a AR passthrough headset) */
get isScreenBasedAR() { return this.isAR && !this.isPassThrough; }
get posePosition() { return this._transformPosition; }
get poseOrientation() { return this._transformOrientation; }
/** @returns the context.renderer.xr.getReferenceSpace() result */
get referenceSpace() { return this.context.renderer.xr.getReferenceSpace(); }
/** @returns the XRFrame `viewerpose` using the xr `referenceSpace` */
get viewerPose() { return this._viewerPose; }
/** @returns `true` if any image is currently being tracked or emulated */
get isTrackingImages() {
if (this.frame && "getImageTrackingResults" in this.frame && typeof this.frame.getImageTrackingResults === "function") {
try {
const trackingResult = this.frame.getImageTrackingResults();
for (const result of trackingResult) {
const state = result.trackingState;
if (state === "tracked" || state === "emulated") {
return true;
}
}
}
catch {
// Looks like we get a NotSupportedException on Android since the method is known
// but the feature is not supported by the session
// TODO Can we check here if we even requested the image-tracking feature instead of catching?
return false;
}
}
return false;
}
/** The currently active XR rig */
get rig() {
const rig = this._rigs[0] ?? null;
if (rig?.gameObject && isDestroyed(rig.gameObject) || rig?.isActive === false) {
this.updateActiveXRRig();
return this._rigs[0] ?? null;
}
return rig;
}
_rigScale = 1;
_lastRigScaleUpdate = -1;
/** Get the XR Rig worldscale.
*
* **For AR**
* If you want to modify the scale in AR at runtime get the WebARSessionRoot component via `findObjectOfType(WebARSessionRoot)` and then set the `arScale` value.
* @returns the scale of the XR rig
*
*/
get rigScale() {
// const ar_scaleValue = this.context.domElement.getAttribute("ar-scale");
// if (ar_scaleValue) {
// const scale = parseFloat(ar_scaleValue);
// if (!isNaN(scale)) return scale;
// }
if (!this._rigs[0])
return 1;
if (this._lastRigScaleUpdate !== this.context.time.frame) {
this._lastRigScaleUpdate = this.context.time.frame;
this._rigScale = this._rigs[0].gameObject.worldScale.x;
}
return this._rigScale;
}
/** add a rig to the available XR rigs - if it's priority is higher than the currently active rig it will be enabled */
addRig(rig) {
const i = this._rigs.indexOf(rig);
if (i >= 0)
return;
if (rig.priority === undefined)
rig.priority = 0;
this._rigs.push(rig);
this.updateActiveXRRig();
}
/** Remove a rig from the available XR Rigs */
removeRig(rig) {
const i = this._rigs.indexOf(rig);
if (i === -1)
return;
this._rigs.splice(i, 1);
this.updateActiveXRRig();
}
/** Sets a XRRig to be active which will parent the camera to this rig */
setRigActive(rig) {
const i = this._rigs.indexOf(rig);
const currentlyActive = this._rigs[0];
this._rigs.splice(i, 1);
this._rigs.unshift(rig);
// if there's another rig currently active we need to make sure we have at least the same priority
rig.priority = currentlyActive?.priority ?? 0;
this.updateActiveXRRig();
}
/**
* @returns the user position in the rig space
*/
getUserOffsetInRig() {
const positionInRig = this.context.mainCamera?.position;
if (!positionInRig || !this.rig)
return getTempVector(0, 0, 0);
const vec = getTempVector(positionInRig);
vec.x *= -1;
vec.z *= -1;
vec.applyQuaternion(getTempQuaternion(this.rig.gameObject.quaternion));
return vec;
}
updateActiveXRRig() {
const previouslyActiveRig = this._rigs[0] ?? null;
// ensure that the default rig is in the scene
if (this._defaultRig.gameObject.parent !== this.context.scene)
this.context.scene.add(this._defaultRig.gameObject);
// ensure the fallback rig is always active!!!
this._defaultRig.gameObject.visible = true;
// ensure that the default rig is in the list of available rigs
if (!this._rigs.includes(this._defaultRig))
this._rigs.push(this._defaultRig);
// find the rig with the highest priority and make sure it's at the beginning of the array
let highestPriorityRig = this._rigs[0];
if (highestPriorityRig && highestPriorityRig.priority === undefined)
highestPriorityRig.priority = 0;
for (let i = 1; i < this._rigs.length; i++) {
const rig = this._rigs[i];
if (!rig.isActive)
continue;
if (isDestroyed(rig.gameObject)) {
this._rigs.splice(i, 1);
i--;
continue;
}
if (!highestPriorityRig || highestPriorityRig.isActive === false || (rig.priority !== undefined && rig.priority > highestPriorityRig.priority)) {
highestPriorityRig = rig;
}
}
// make sure the highest priority rig is at the beginning if it isnt already
if (previouslyActiveRig !== highestPriorityRig) {
const index = this._rigs.indexOf(highestPriorityRig);
if (index >= 0)
this._rigs.splice(index, 1);
this._rigs.unshift(highestPriorityRig);
}
if (debug) {
if (previouslyActiveRig === highestPriorityRig)
console.log("Updated Active XR Rig:", highestPriorityRig, "prev:", previouslyActiveRig);
else
console.log("Updated Active XRRig:", highestPriorityRig, " (the same as before)");
}
}
_rigs = [];
_viewerHitTestSource = null;
/** Returns a XR hit test result (if hit-testing is available) in rig space
* @param source If provided, the hit test will be performed for the given controller
*/
getHitTest(source) {
if (source) {
return this.getControllerHitTest(source);
}
if (!this._viewerHitTestSource)
return null;
const hitTestSource = this._viewerHitTestSource;
const hitTestResults = this.frame.getHitTestResults(hitTestSource);
if (hitTestResults.length > 0) {
const hit = hitTestResults[0];
return this.convertHitTestResult(hit);
}
return null;
}
getControllerHitTest(controller) {
const hitTestSource = controller.getHitTestSource();
if (!hitTestSource)
return null;
const res = this.frame.getHitTestResultsForTransientInput(hitTestSource);
for (const result of res) {
if (result.inputSource === controller.inputSource) {
for (const hit of result.results) {
return this.convertHitTestResult(hit);
}
}
}
return null;
}
convertHitTestResult(result) {
const referenceSpace = this.context.renderer.xr.getReferenceSpace();
const pose = referenceSpace && result.getPose(referenceSpace);
if (pose) {
const pos = getTempVector(pose.transform.position);
const rot = getTempQuaternion(pose.transform.orientation);
const camera = this.context.mainCamera;
if (camera?.parent !== this._cameraRenderParent) {
pos.applyMatrix4(flipForwardMatrix);
}
if (camera?.parent) {
pos.applyMatrix4(camera.parent.matrixWorld);
rot.multiply(flipForwardQuaternion);
// apply parent quaternion (if parent is moved/rotated)
const parentRotation = getWorldQuaternion(camera.parent);
// ensure that "up" (y+) is pointing away from the wall
parentRotation.premultiply(flipForwardQuaternion);
rot.premultiply(parentRotation);
}
return { hit: result, position: pos, quaternion: rot };
}
return null;
}
/** convert a XRRigidTransform from XR session space to threejs / Needle Engine XR space */
convertSpace(transform) {
const pos = getTempVector(transform.position);
pos.applyMatrix4(flipForwardMatrix);
const rot = getTempQuaternion(transform.orientation);
rot.premultiply(flipForwardQuaternion);
return { position: pos, quaternion: rot };
}
/** this is the implictly created XR rig */
_defaultRig;
/** all scripts that receive some sort of XR update event */
_xr_scripts;
/** scripts that have onUpdateXR event methods */
_xr_update_scripts = [];
/** scripts that are in the scene but inactive (e.g. disabled parent gameObject) */
_inactive_scripts = [];
/** tracks scripts that have received onEnterXR — prevents spurious onLeaveXR calls */
_scripts_in_xr = new Set();
_controllerAdded;
_controllerRemoved;
_originalCameraWorldPosition;
_originalCameraWorldRotation;
_originalCameraWorldScale;
_originalCameraParent;
/** we store the main camera reference here each frame to make sure we have a rendering camera
* this e.g. the case when the XR rig with the camera gets disabled (and thus this.context.mainCamera is unassigned)
*/
_mainCamera = null;
constructor(mode, session, context, extra) {
//performance.mark(measure_SessionStartedMarker);
saveSessionInfo(mode, extra.init);
this.session = session;
this.mode = mode;
this.context = context;
if (debug || getParam("console"))
enableSpatialConsole(true);
this._xr_scripts = [...extra.scripts];
this._xr_update_scripts = this._xr_scripts.filter(e => typeof e.onUpdateXR === "function");
this._controllerAdded = extra.controller_added;
this._controllerRemoved = extra.controller_removed;
registerFrameEventCallback(this.onBefore, FrameEvent.LateUpdate);
this.context.pre_render_callbacks.push(this.onBeforeRender);
this.context.post_render_callbacks.push(this.onAfterRender);
if (extra.init.optionalFeatures?.includes("hit-test") || extra.init.requiredFeatures?.includes("hit-test")) {
session.requestReferenceSpace('viewer').then((referenceSpace) => {
return session.requestHitTestSource?.call(session, { space: referenceSpace })?.then((source) => {
return this._viewerHitTestSource = source;
}).catch(e => console.error(e));
}).catch(e => console.error(e));
}
if (this.context.mainCamera) {
this._originalCameraWorldPosition = getWorldPosition(this.context.mainCamera, new Vector3());
this._originalCameraWorldRotation = getWorldQuaternion(this.context.mainCamera, new Quaternion());
this._originalCameraWorldScale = getWorldScale(this.context.mainCamera, new Vector3());
this._originalCameraParent = this.context.mainCamera.parent;
if (this.context.mainCamera instanceof PerspectiveCamera) {
this.context.mainCamera[$initialFov] = this.context.mainCamera.fov;
// if (this.mode === "immersive-ar" && this.context.mainCamera.near > .1) {
// console.log("[WebXR] Setting near plane to 0.1 for better AR experience (was " + this.context.mainCamera.near + "). The initial near plane will be restored when the session ends.");
// this.context.mainCamera.near = 0.1;
// this.context.mainCamera[$initialNear] = this.context.mainCamera.near;
// }
}
}
this._defaultRig = new ImplictXRRig();
this.context.scene.add(this._defaultRig.gameObject);
this.addRig(this._defaultRig);
// register already connected input sources
// this is for when the session is already running (via a temporary xr session)
// and the controllers are already connected
for (let i = 0; i < session.inputSources.length; i++) {
const inputSource = session.inputSources[i];
if (!inputSource.handedness) {
console.warn("Input source in xr session has no handedness - ignoring", i);
continue;
}
this.onInputSourceAdded(inputSource);
}
// handle controller and input source changes changes
this.session.addEventListener('end', this.onEnd);
// handle input sources change
this.session.addEventListener("inputsourceschange",
/* @ts-ignore (ignore CI XRInputSourceChangeEvent mismatch) */
(evt) => {
// handle removed controllers
for (const removedInputSource of evt.removed) {
this.disconnectInputSource(removedInputSource);
}
for (const newInputSource of evt.added) {
this.onInputSourceAdded(newInputSource);
}
});
// Unfortunately the code below doesnt work: the session never receives any input sources sometimes
// https://developer.mozilla.org/en-US/docs/Web/API/XRSession/visibilitychange_event
// this.session.addEventListener("visibilitychange", (evt: XRSessionEvent) => {
// // sometimes when entering an XR session the controllers are not added/not in the list and we don't receive an event
// // this is a workaround trying to add controllers when the scene visibility changes to "visible"
// // e.g. due to a user opening and closing the menu
// if (this.controllers.length === 0 && evt.session.visibilityState === "visible") {
// for (const controller of evt.session.inputSources) {
// this.onInputSourceAdded(controller);
// }
// }
// })
//