@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
1,084 lines (957 loc) • 83 kB
text/typescript
import { Camera, Object3D, PerspectiveCamera, Quaternion, Vector3 } from "three";
import { enableSpatialConsole, isDevEnvironment, showBalloonMessage, showBalloonWarning } from "../debug/index.js";
import { Application, internalOnUserInputRegistered } from "../engine_application.js";
import { Context, FrameEvent } from "../engine_context.js";
import { ContextEvent, ContextRegistry } from "../engine_context_registry.js";
import { isDestroyed } from "../engine_gameobject.js";
import { Gizmos } from "../engine_gizmos.js";
import { Telemetry } from "../engine_license.js";
import { registerFrameEventCallback, unregisterFrameEventCallback } from "../engine_lifecycle_functions_internal.js";
import { getBoundingBox, getTempQuaternion, getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale, setWorldPosition, setWorldQuaternion, setWorldScale } from "../engine_three_utils.js";
import type { ICamera, IComponent, INeedleXRSession } from "../engine_types.js";
import { delay, DeviceUtilities, getParam } from "../engine_utils.js";
import { invokeXRSessionEnd, invokeXRSessionStart } from "./events.js"
import { flipForwardMatrix, flipForwardQuaternion, ImplictXRRig } from "./internal.js";
import { NeedleXRController } from "./NeedleXRController.js";
import { NeedleXRSync } from "./NeedleXRSync.js";
import { SceneTransition } from "./SceneTransition.js";
import { SessionInfo, TemporaryXRContext } from "./TempXRContext.js";
import { InternalUSDZRegistry } from "./usdz.js";
import type { IXRRig } from "./XRRig.js";
const measure_SessionStartedMarker = "NeedleXRSession onStart";
const measure_SessionEndedMarker = "NeedleXRSession onEnd";
/** @link https://developer.mozilla.org/en-US/docs/Web/API/XRFrame/fillPoses */
declare type FillPosesFunction = (spaces: IterableIterator<XRJointSpace>, referenceSpace: XRSpace, targetArray: Float32Array) => void;
declare type NeedleXRFrame = XRFrame & { fillPoses?: FillPosesFunction };
/** NeedleXRSession event argument.
* Use `args.xr` to access the NeedleXRSession */
export type NeedleXREventArgs = { readonly xr: NeedleXRSession }
export type SessionChangedEvt = (args: NeedleXREventArgs) => void;
export type SessionRequestedEvent = (args: { readonly mode: XRSessionMode, readonly init: XRSessionInit }) => void;
export type SessionRequestedEndEvent = (args: { readonly mode: XRSessionMode, readonly init: XRSessionInit, newSession: XRSession | null }) => void;
/** Result of a XR hit-test
* @property {XRHitTestResult} hit The original XRHitTestResult
* @property {Vector3} position The hit position in world space
* @property {Quaternion} quaternion The hit rotation in world space
*/
export type NeedleXRHitTestResult = { readonly hit: XRHitTestResult, readonly position: Vector3, readonly quaternion: Quaternion };
const debug = getParam("debugwebxr");
const debugFPS = getParam("stats");
let debugFPSFramesSinceLastUpdate = 0;
// TODO: move this into the IComponent interface!?
export interface INeedleXRSessionEventReceiver extends Pick<IComponent, "destroyed"> {
get activeAndEnabled(): boolean;
supportsXR?(mode: XRSessionMode): boolean;
/** Called before requesting a XR session */
onBeforeXR?(mode: XRSessionMode, args: XRSessionInit): void;
onEnterXR?(args: NeedleXREventArgs): void;
onUpdateXR?(args: NeedleXREventArgs): void;
onLeaveXR?(args: NeedleXREventArgs): void;
onXRControllerAdded?(args: NeedleXRControllerEventArgs): void;
onXRControllerRemoved?(args: NeedleXRControllerEventArgs): void;
}
/** Contains a reference to the currently active webxr session and the controller that has changed */
export type NeedleXRControllerEventArgs = NeedleXREventArgs & { controller: NeedleXRController, change: "added" | "removed" }
/** Event Arguments when a controller changed event is invoked (added or removed)
* Access the controller via `args.controller`, the `args.change` property indicates if the controller was added or removed
*/
export type ControllerChangedEvt = (args: NeedleXRControllerEventArgs) => void;
function getDOMOverlayElement(domElement: HTMLElement) {
let arOverlayElement: HTMLElement | null = null;
// for react cases we dont have an Engine Element
const element: any = domElement;
if (element.getAROverlayContainer)
arOverlayElement = element.getAROverlayContainer();
else arOverlayElement = domElement;
return arOverlayElement;
}
handleSessionGranted();
async function handleSessionGranted() {
// await delay(400);
let defaultMode: XRSessionMode = "immersive-vr";
try {
// In app clips we default to AR
if (DeviceUtilities.isNeedleAppClip()) {
defaultMode = "immersive-ar";
}
// Check if VR is even supported, otherwise try AR
else if (!(await navigator.xr?.isSessionSupported("immersive-vr"))) {
defaultMode = "immersive-ar";
}
// Check if AR is supported, otherwise we can't do anything
if (!(await navigator.xr?.isSessionSupported("immersive-ar")) && defaultMode === "immersive-ar") {
// console.warn("[NeedleXRSession:granted] Neither VR nor AR supported, aborting session start.");
// showBalloonMessage("NeidleXRSession: Neither VR nor AR supported, aborting session start.");
return;
}
} catch (e) {
console.debug("[NeedleXRSession:granted] Error while checking XR support:", e);
// showBalloonWarning("NeedleXRSession: Error while checking XR support: " + (e as Error).message);
return;
}
// showBalloonMessage("sessiongranted: " + defaultMode);
// TODO: asap session granted doesnt handle the pre-room yet
if (getParam("debugasap")) {
let asapSession = globalThis["needle:XRSession"] as XRSession | undefined | Promise<XRSession>;
// @ts-ignore // getting TS2848 here, not sure why
if (asapSession instanceof Promise<XRSession>) {
delete globalThis["needle:XRSession"];
ContextRegistry.addContextCreatedCallback(async cb => {
if (!asapSession) return;
// TODO: add support to pass this to the temporary room
enableSpatialConsole(true);
const session = await asapSession;
if (session) {
const sessionInit = NeedleXRSession.getDefaultSessionInit(defaultMode);
NeedleXRSession.setSession(defaultMode, session, sessionInit, cb.context);
}
else {
console.error("[NeedleXRSession:granted] ASAP session was rejected");
}
asapSession = undefined;
});
return;
}
}
// console.log("Attaching sessiongranted handler...", { haveXR: 'xr' in navigator });
// setTimeout(() => console.log("Session Granted handler attached.", { haveXR: 'xr' in navigator }), 1000);
if ('xr' in navigator) {
// WebXRViewer (based on Firefox) has a bug where addEventListener
// throws a silent exception and aborts execution entirely.
if (/WebXRViewer\//i.test(navigator.userAgent)) {
console.warn('WebXRViewer does not support addEventListener');
return;
}
navigator.xr?.addEventListener('sessiongranted', async () => {
// enableSpatialConsole(true);
const lastSessionMode = sessionStorage.getItem("needle_xr_session_mode") as XRSessionMode;
const lastSessionInit = sessionStorage.getItem("needle_xr_session_init") ?? null;
const init = lastSessionInit ? JSON.parse(lastSessionInit) : null;
let info: SessionInfo | null = null;
if (contextIsLoading()) {
await TemporaryXRContext.start(lastSessionMode || defaultMode, init || NeedleXRSession.getDefaultSessionInit(defaultMode))
.catch(e => console.warn("[NeedleXRSession:granted] TemporaryXRContext start failed:", e));
await waitForContextLoadingFinished();
info = await TemporaryXRContext.handoff();
}
if (info) {
NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current);
}
else if (lastSessionMode && lastSessionInit) {
console.log("[NeedleXRSession:granted] Restore last session")
const init = JSON.parse(lastSessionInit);
NeedleXRSession.start(lastSessionMode as XRSessionMode, init).catch(e => console.warn(e));
}
else {
// if no session was found we start VR by default
NeedleXRSession.start(defaultMode).catch(e => console.warn("[NeedleXRSession:granted] failed:", e));
}
// make sure we only subscribe to the event once
}, { once: true });
}
else {
// showBalloonWarning("NeedleXRSession: WebXR not available in this browser.");
}
}
function saveSessionInfo(mode: XRSessionMode, init: XRSessionInit) {
sessionStorage.setItem("needle_xr_session_mode", mode);
sessionStorage.setItem("needle_xr_session_init", JSON.stringify(init));
}
function deleteSessionInfo() {
sessionStorage.removeItem("needle_xr_session_mode");
sessionStorage.removeItem("needle_xr_session_init");
}
const contexts_loading: Set<Context> = new Set();
ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, async cb => {
contexts_loading.add(cb.context);
});
ContextRegistry.registerCallback(ContextEvent.ContextCreated, async cb => {
contexts_loading.delete(cb.context);
const autostart = cb.context?.domElement.getAttribute("autostart") || null;
handleAutoStart(autostart);
});
function contextIsLoading() { return contexts_loading.size > 0; }
function waitForContextLoadingFinished(): Promise<void> {
return new Promise(res => {
const startTime = Date.now();
const interval = setInterval(() => {
if (!contextIsLoading() || Date.now() - startTime > 60000) {
clearInterval(interval);
res();
}
}, 100);
});
}
if (DeviceUtilities.isDesktop() && isDevEnvironment()) {
window.addEventListener("keydown", (evt) => {
if (evt.key === "x" || evt.key === "Escape") {
if (NeedleXRSession.active) {
NeedleXRSession.stop();
}
}
});
}
function handleAutoStart(value: string | null) {
if (!value) return;
switch (value?.toLowerCase()) {
case "ar":
Application.registerWaitForInteraction(() => {
// TODO: move WebARSessionRoot to core
NeedleXRSession.start("ar");
})
break;
}
}
// if (getParam("simulatewebxrloading")) {
// ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, async _cb => {
// await delay(3000);
// setTimeout(async () => {
// const info = await TemporaryXRContext.handoff();
// if (info) NeedleXRSession.setSession(info.mode, info.session, info.init, Context.Current);
// else
// NeedleXRSession.start("immersive-vr")
// }, 6000)
// });
// let triggered = false;
// window.addEventListener("click", () => {
// if (triggered) return;
// triggered = true;
// TemporaryXRContext.start("immersive-vr", NeedleXRSession.getDefaultSessionInit("immersive-vr"));
// });
// }
const $initialFov = Symbol("initial-fov");
const $initialNear = Symbol("initial-near");
/**
* This class manages an XRSession to provide helper methods and events. It provides easy access to the XRInputSources (controllers and hands)
* - Start a XRSession with `NeedleXRSession.start(...)`
* - Stop a XRSession with `NeedleXRSession.stop()`
* - Access a running XRSession with `NeedleXRSession.active`
*
* If a XRSession is active you can use all XR-related event methods on your components to receive XR events e.g. `onEnterXR`, `onUpdateXR`, `onLeaveXR`
* ```ts
* export class MyComponent extends Behaviour {
* // callback invoked whenever the XRSession is started or your component is added to a scene with an active XRSession
* onEnterXR(args: NeedleXREventArgs) {
* console.log("Entered XR");
* // access the NeedleXRSession via args.xr
* }
* // callback invoked whenever a controller is added (or you switch from controller to hand tracking)
* onControllerAdded(args: NeedleXRControllerEventArgs) { }
* }
* ```
*
* ### XRRig
* The XRRig can be accessed via the `rig` property
* Set a custom XRRig via `NeedleXRSession.addRig(...)` or `NeedleXRSession.removeRig(...)`
* By default the active XRRig with the highest priority in the scene is used
*
* ### Screenshots in XR
* Screenshots work automatically during XR sessions, including AR camera feed compositing. See {@link screenshot2} for more information.
*
* @category XR
* @see {@link screenshot2} for taking screenshots in XR sessions
*/
export class NeedleXRSession implements INeedleXRSession {
private static _sync: NeedleXRSync | null = null;
static getXRSync(context: Context) {
if (!this._sync) this._sync = new NeedleXRSync(context);
return this._sync;
}
static get currentSessionRequest(): XRSessionMode | null { return this._currentSessionRequestMode; }
private static _currentSessionRequestMode: XRSessionMode | null = null;
/**
* @returns the active @type {NeedleXRSession} (if any active) or null
*/
static get active(): NeedleXRSession | null { return this._activeSession; }
/** The active xr session mode (if any xr session is active)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSessionMode
*/
static get activeMode() { return this._activeSession?.mode ?? null; }
/** XRSystem via navigator.xr access
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem
*/
static get xrSystem(): XRSystem | undefined {
return ('xr' in navigator) ? navigator.xr : undefined;
}
/**
* @returns true if the browser supports WebXR (`immersive-vr` or `immersive-ar`)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isXRSupported() { return Promise.all([this.isVRSupported(), this.isARSupported()]).then(res => res.some(e => e)).catch(() => false); }
/**
* @returns true if the browser supports immersive-vr (WebXR)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isVRSupported() { return this.isSessionSupported("immersive-vr"); }
/**
* @returns true if the browser supports immersive-ar (WebXR)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSystem/isSessionSupported
*/
static isARSupported() { return this.isSessionSupported("immersive-ar"); }
/**
* @param mode The XRSessionMode to check if it is supported
* @returns true if the browser supports the given XRSessionMode
*/
static isSessionSupported(mode: XRSessionMode) { return this.xrSystem?.isSessionSupported(mode).catch(err => { if (debug) console.error(err); return false }) ?? Promise.resolve(false); }
private static _currentSessionRequest?: Promise<XRSession>;
private static _activeSession: NeedleXRSession | null;
/** Register to listen to XRSession start events. Unsubscribe with `offXRSessionStart` */
static onSessionRequestStart(evt: SessionRequestedEvent) {
this._sessionRequestStartListeners.push(evt);
}
/** Unsubscribe from request start evt. Register with `onSessionRequestStart` */
static offSessionRequestStart(evt: SessionRequestedEvent) {
const index = this._sessionRequestStartListeners.indexOf(evt);
if (index >= 0) this._sessionRequestStartListeners.splice(index, 1);
}
private static readonly _sessionRequestStartListeners: SessionRequestedEvent[] = [];
/** Called after the session request has finished */
static onSessionRequestEnd(evt: SessionRequestedEndEvent) {
this._sessionRequestEndListeners.push(evt);
}
/** Unsubscribe from request end evt */
static offSessionRequestEnd(evt: SessionRequestedEndEvent) {
const index = this._sessionRequestEndListeners.indexOf(evt);
if (index >= 0) this._sessionRequestEndListeners.splice(index, 1);
}
private static readonly _sessionRequestEndListeners: SessionRequestedEndEvent[] = [];
/** Listen to XR session started. Unsubscribe with `offXRSessionStart` */
static onXRSessionStart(evt: SessionChangedEvt) {
this._xrStartListeners.push(evt);
};
/** Unsubscribe from XRSession started events */
static offXRSessionStart(evt: SessionChangedEvt) {
const index = this._xrStartListeners.indexOf(evt);
if (index >= 0) this._xrStartListeners.splice(index, 1);
}
private static readonly _xrStartListeners: SessionChangedEvt[] = [];
/** Listen to XR session ended. Unsubscribe with `offXRSessionEnd` */
static onXRSessionEnd(evt: SessionChangedEvt) {
this._xrEndListeners.push(evt);
};
/** Unsubscribe from XRSession started events */
static offXRSessionEnd(evt: SessionChangedEvt) {
const index = this._xrEndListeners.indexOf(evt);
if (index >= 0) this._xrEndListeners.splice(index, 1);
}
private static readonly _xrEndListeners: SessionChangedEvt[] = [];
/** Listen to controller added events.
* Events are cleared when starting a new session
**/
static onControllerAdded(evt: ControllerChangedEvt) {
this._controllerAddedListeners.push(evt);
}
/** Unsubscribe from controller added evts */
static offControllerAdded(evt: ControllerChangedEvt) {
const index = this._controllerAddedListeners.indexOf(evt);
if (index >= 0) this._controllerAddedListeners.splice(index, 1);
}
private static readonly _controllerAddedListeners: ControllerChangedEvt[] = [];
/** Listen to controller removed events
* Events are cleared when starting a new session
**/
static onControllerRemoved(evt: ControllerChangedEvt) {
this._controllerRemovedListeners.push(evt);
}
/** Unsubscribe from controller removed events */
static offControllerRemoved(evt: ControllerChangedEvt) {
const index = this._controllerRemovedListeners.indexOf(evt);
if (index >= 0) this._controllerRemovedListeners.splice(index, 1);
}
private static readonly _controllerRemovedListeners: ControllerChangedEvt[] = [];
/** If the browser supports offerSession - creating a VR or AR button in the browser navigation bar */
static offerSession(mode: XRSessionMode, init: XRSessionInit | "default", context: Context): boolean {
if ('xr' in navigator && navigator.xr && 'offerSession' in navigator.xr) {
if (typeof navigator.xr.offerSession === "function") {
console.log("WebXR offerSession is available - requesting mode: " + mode);
if (init == "default") {
init = this.getDefaultSessionInit(mode);
}
navigator.xr.offerSession(mode, {
...init
}).then((session) => {
return NeedleXRSession.setSession(mode, session, init as XRSessionInit, context);
}).catch(_ => {
console.log("XRSession offer rejected (perhaps because another call to offerSession was made or a call to requestSession was made)")
});
}
return true;
}
return false;
}
/** @returns a new XRSession init object with defaults */
static getDefaultSessionInit(mode: Omit<XRSessionMode, "inline">): XRSessionInit {
switch (mode) {
case "immersive-ar":
const arFeatures = ['anchors', 'local-floor', 'layers', 'dom-overlay', 'hit-test', 'unbounded'];
// Don't request handtracking by default on VisionOS
if (!DeviceUtilities.isVisionOS())
arFeatures.push('hand-tracking');
return {
optionalFeatures: arFeatures,
}
case "immersive-vr":
const vrFeatures = ['local-floor', 'bounded-floor', 'high-fixed-foveation-level', 'layers'];
// Don't request handtracking by default on VisionOS
if (!DeviceUtilities.isVisionOS())
vrFeatures.push('hand-tracking');
return {
optionalFeatures: vrFeatures,
}
default:
console.warn("No default session init for mode", mode);
return {};
}
}
/** start a new webXR session (make sure to stop already running sessions before calling this method)
* @param mode The XRSessionMode to start (e.g. `immersive-vr` or `immersive-ar`) or `ar` to start `immersive-ar` on supported devices OR on iOS devices it will export an interactive USDZ and open in Quicklook.
* Get more information about WebXR modes: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionMode
* @param init The XRSessionInit to use (optional), docs: https://developer.mozilla.org/en-US/docs/Web/API/XRSessionInit
* @param context The Needle Engine context to use
*/
static async start(mode: XRSessionMode | "ar" | "quicklook", init?: XRSessionInit, context?: Context): Promise<NeedleXRSession | null> {
// setup session init args, make sure we have default values
if (!init) init = {};
// handle iOS platform where "immersive-ar" is special:
// - we either launch QuickLook
// - or forward to the Needle App Clip experience for WebXR AR
// TODO: should we add a separate mode (e.g. "AR")? https://linear.app/needle/issue/NE-5303
if (DeviceUtilities.isiOS()) {
const arSupported = await this.isARSupported().catch(() => false);
// On VisionOS, we use QuickLook for AR experiences; no AppClip support for now.
if (DeviceUtilities.isVisionOS() && !arSupported && (mode === "ar" || mode === "immersive-ar")) {
mode = "quicklook";
}
if (mode === "quicklook") {
Telemetry.sendEvent(Context.Current, "xr", {
action: "quicklook_export",
source: "NeedleXRSession.start",
});
InternalUSDZRegistry.exportAndOpen();
return null;
}
if (!arSupported && (mode === "immersive-ar" || mode === "ar")) {
this.invokeSessionRequestStart("immersive-ar", init);
// Forward to the AppClip experience (Using the apple.com url the appclip overlay shows immediately)
// const url =`https://appclip.needle.tools/ar?url=${(location.href)}`;
const url = new URL("https://appclip.apple.com/id?p=tools.needle.launch-app.Clip");
url.searchParams.set("url", location.href);
const urlStr = url.toString();
Telemetry.sendEvent(Context.Current, "xr", {
action: "app_clip_launch",
source: "NeedleXRSession.start",
url: urlStr,
});
// if we are in an iframe, we need to navigate the top window
const topWindow = window.top || window;
try {
console.debug("iOS device detected - opening Needle App Clip for AR experience", { mode, init, url });
// navigate to app clip url but keep the current url in history, open in same tab
// eslint-disable-next-line xss/no-location-href-assign
topWindow.location.href = urlStr;
}
catch (e) {
console.warn("Error navigating to AppClip " + urlStr + "\n", e);
// if top window navigation fails and we are in an iframe, we try to navigate the top window directly
const weAreInIframe = window !== window.top;
if (weAreInIframe) {
// we can try to open a new tab as a fallback
window.open(urlStr, "_blank");
}
// eslint-disable-next-line xss/no-location-href-assign
else window.location.href = urlStr;
}
setTimeout(() => {
this.invokeSessionRequestEnd("immersive-ar", init || {}, null);
}, 3000);
return null;
}
}
if (mode === "quicklook") {
console.warn("QuickLook mode is only supported on iOS devices");
return null;
}
// Since we now know we are not on iOS, ar mode becomes "immersive-ar"
if (mode == "ar") {
mode = "immersive-ar";
}
if (isDevEnvironment() && getParam("debugxrpreroom")) {
console.warn("Debug: Starting temporary XR session");
await TemporaryXRContext.start(mode, init || NeedleXRSession.getDefaultSessionInit(mode));
return null;
}
if (this._currentSessionRequest) {
console.warn("A XRSession is already being requested");
if (debug || isDevEnvironment()) showBalloonWarning("A XRSession is already being requested");
return this._currentSessionRequest.then(() => this._activeSession!);
}
if (this._activeSession) {
console.error("A XRSession is already running");
return this._activeSession;
}
// Make sure we have a context
if (!context) context = Context.Current;
if (!context) context = ContextRegistry.All[0] as Context;
if (!context) throw new Error("No Needle Engine Context found");
//performance.mark('NeedleXRSession start');
switch (mode) {
// Setup VR initialization parameters
case "immersive-ar":
{
const supported = await this.xrSystem?.isSessionSupported('immersive-ar')
if (supported !== true) {
console.error(mode + ' is not supported by this browser.');
return null;
}
const defaultInit: XRSessionInit = this.getDefaultSessionInit(mode);
const domOverlayElement = getDOMOverlayElement(context.domElement);
if (domOverlayElement && !DeviceUtilities.isQuest()) { // excluding quest since dom-overlay breaks sessiongranted
defaultInit.domOverlay = { root: domOverlayElement };
defaultInit.optionalFeatures!.push('dom-overlay');
}
init = {
...defaultInit,
...init,
}
}
break;
// Setup AR initialization parameters
case "immersive-vr":
{
const supported = await this.xrSystem?.isSessionSupported('immersive-vr')
if (supported !== true) {
console.error(mode + ' is not supported by this browser.');
return null;
}
const defaultInit: XRSessionInit = this.getDefaultSessionInit(mode);
init = {
...defaultInit,
...init,
}
}
break;
default:
console.warn("No default session init for mode", mode);
break;
}
// Fix: define these here because VariantLauncher crashes otherwise. Spec: https://immersive-web.github.io/webxr/#feature-dependencies
// Issue: https://linear.app/needle/issue/NE-5136
init.optionalFeatures ??= [];
init.requiredFeatures ??= [];
// we stop a temporary session here (if any runs)
await TemporaryXRContext.stop();
const scripts = mode == "immersive-ar" ? context.scripts_immersive_ar : context.scripts_immersive_vr
if (debug)
console.log("%c" + `Requesting ${mode} session`, "font-weight:bold;", init, scripts);
else
console.log("%c" + `Requesting ${mode} session`, "font-weight:bold;");
for (const script of scripts) {
if (script.onBeforeXR && script.activeAndEnabled && !script.destroyed) {
script.onBeforeXR(mode, init);
}
}
this.invokeSessionRequestStart(mode, init);
if (debug) showBalloonMessage("Requesting " + mode + " session (" + Date.now() + ")");
Telemetry.sendEvent(Context.Current, "xr", {
action: "session_request",
mode: mode,
features: ((init.requiredFeatures ?? []).concat(init.optionalFeatures ?? [])).join(","),
source: "NeedleXRSession.start",
});
this._currentSessionRequest = navigator?.xr?.requestSession(mode, init);
this._currentSessionRequestMode = mode;
/**@type {XRSystem} */
const newSession = await (this._currentSessionRequest)?.catch(e => {
console.error(e, "Code: " + e?.code);
if (e?.code === 9) showBalloonWarning("Couldn't start XR session. Make sure you allow the required permissions.")
console.log("If the specified XR configuration is not supported (e.g. entering AR doesnt work) - make sure you access the website on a secure connection (HTTPS) and your device has the required permissions (e.g. camera access)");
const notSecure = location.protocol === 'http:';
if (notSecure) showBalloonWarning("XR requires a secure connection (HTTPS)");
});
this._currentSessionRequest = undefined;
this._currentSessionRequestMode = null;
this.invokeSessionRequestEnd(mode, init, newSession);
if (!newSession) {
console.warn("XR Session request was rejected");
return null;
}
const session = this.setSession(mode, newSession, init, context);
//performance.mark('NeedleXRSession end');
//performance.measure('NeedleXRSession Startup', 'NeedleXRSession start', 'NeedleXRSession end');
return session;
}
private static invokeSessionRequestStart(mode: XRSessionMode, init: XRSessionInit) {
for (const listener of this._sessionRequestStartListeners) {
listener({ mode, init });
}
}
private static invokeSessionRequestEnd(mode: XRSessionMode, init: XRSessionInit, session: XRSession | null | undefined | void) {
for (const listener of this._sessionRequestEndListeners) {
listener({ mode, init, newSession: session || null });
}
}
static setSession(mode: XRSessionMode, session: XRSession, init: XRSessionInit, context: Context) {
if (this._activeSession) {
console.error("A XRSession is already running");
return this._activeSession;
}
const scripts = mode == "immersive-ar" ? context.scripts_immersive_ar : context.scripts_immersive_vr;
this._activeSession = new NeedleXRSession(mode, session, context, {
scripts: scripts,
controller_added: this._controllerAddedListeners,
controller_removed: this._controllerRemovedListeners,
init: init
});
session.addEventListener("end", this.onEnd);
if (debug)
console.log("%c" + `Started ${mode} session`, "font-weight:bold;", scripts);
else
console.log("%c" + `Started ${mode} session`, "font-weight:bold;");
return this._activeSession;
}
private static $_stop_request = Symbol();
/** stops the active XR session */
static stop(): void {
const session = this._activeSession;
if (session) {
if (session[this.$_stop_request] === undefined) {
if (debug) console.log("[NeedleXRSession] Stopping XR Session... (new)");
// We can not call session.end() immediately because we might be within a frame callback
// For example if a use calls `NeedleXRSession.stop()` within a `update()` callback
session[this.$_stop_request] = setTimeout(() => {
session.end();
});
}
else if (debug) {
console.warn("[NeedleXRSession] XR Session stop already requested");
}
}
}
private static onEnd = () => {
if (debug) console.log("XR Session ended");
this._activeSession = null;
}
/** The needle engine context this session was started from */
readonly context: Context;
get sync(): NeedleXRSync | null {
return NeedleXRSession._sync;
}
/** Returns true if the xr session is still active */
get running(): boolean { return !this._ended && this.session != null; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession
*/
readonly session: XRSession;
/** XR Session Mode: AR or VR */
readonly mode: XRSessionMode;
/**
* The XRSession interface's read-only interactionMode property describes the best space (according to the user agent) for the application to draw an interactive UI for the current session.
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/interactionMode
*/
get interactionMode(): "screen-space" | "world-space" { return this.session["interactionMode"]!; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/visibilityState
* @returns {XRVisibilityState} The visibility state of the XRSession
*/
get visibilityState(): XRVisibilityState { return this.session.visibilityState; }
/**
* Check if the session is `visible-blurred` - this means e.g. the keyboard is shown
*/
get isVisibleBlurred(): boolean { return this.session.visibilityState === 'visible-blurred' }
/**
* Check if the session has system keyboard support
*/
get isSystemKeyboardSupported(): boolean { return this.session.isSystemKeyboardSupported; }
/**
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRSession/environmentBlendMode
*/
get environmentBlendMode(): XREnvironmentBlendMode { return this.session.environmentBlendMode; }
/**
* The current XR frame
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRFrame
*/
get frame(): NeedleXRFrame { return this.context.xrFrame! as NeedleXRFrame; }
/** The currently active/connected controllers */
readonly controllers: NeedleXRController[] = [];
/** shorthand to query the left controller. Use `controllers` to get access to all connected controllers */
get leftController() { return this.controllers.find(c => c.side === "left"); }
/** shorthand to query the right controller. Use `controllers` to get access to all connected controllers */
get rightController() { return this.controllers.find(c => c.side === "right"); }
/** @returns the given controller if it is connected */
getController(side: XRHandedness | number) {
if (typeof side === "number") return this.controllers[side] || null;
return this.controllers.find(c => c.side === side) || null;
}
/** Returns true if running in pass through mode in immersive AR (e.g. user is wearing a headset while in AR) */
get isPassThrough() {
if (this.environmentBlendMode !== 'opaque' && this.interactionMode === "world-space") return true;
// since we can not rely on interactionMode check we check the controllers too
// https://linear.app/needle/issue/NE-4057
// the following is a workaround for the issue above
if (this.mode === "immersive-ar" && this.environmentBlendMode !== 'opaque') {
// if we have any tracked pointer controllers we're also in passthrough
if (this.controllers.some(c => c.inputSource.targetRayMode === "tracked-pointer"))
return true;
}
if (isDevEnvironment() && DeviceUtilities.isDesktop() && this.mode === "immersive-ar") {
return true;
}
return false;
}
get isAR() { return this.mode === 'immersive-ar'; }
get isVR() { return this.mode === 'immersive-vr'; }
/** If the AR mode is not immersive (meaning the user is e.g. holding a phone instead of wearing a AR passthrough headset) */
get isScreenBasedAR() { return this.isAR && !this.isPassThrough; }
get posePosition() { return this._transformPosition; }
get poseOrientation() { return this._transformOrientation; }
/** @returns the context.renderer.xr.getReferenceSpace() result */
get referenceSpace(): XRSpace | null { return this.context.renderer.xr.getReferenceSpace(); }
/** @returns the XRFrame `viewerpose` using the xr `referenceSpace` */
get viewerPose(): XRViewerPose | undefined { return this._viewerPose; }
/** @returns `true` if any image is currently being tracked */
/** returns true if images are currently being tracked */
get isTrackingImages() {
if (this.frame && "getImageTrackingResults" in this.frame && typeof this.frame.getImageTrackingResults === "function") {
try {
const trackingResult = this.frame.getImageTrackingResults();
for (const result of trackingResult) {
const state = result.trackingState;
if (state === "tracked") return true;
}
}
catch {
// Looks like we get a NotSupportedException on Android since the method is known
// but the feature is not supported by the session
// TODO Can we check here if we even requested the image-tracking feature instead of catching?
return false;
}
}
return false;
}
/** The currently active XR rig */
get rig(): IXRRig | null {
const rig = this._rigs[0] ?? null;
if (rig?.gameObject && isDestroyed(rig.gameObject) || rig?.isActive === false) {
this.updateActiveXRRig();
return this._rigs[0] ?? null;
}
return rig;
}
private _rigScale: number = 1;
private _lastRigScaleUpdate: number = -1;
/** Get the XR Rig worldscale.
*
* **For AR**
* If you want to modify the scale in AR at runtime get the WebARSessionRoot component via `findObjectOfType(WebARSessionRoot)` and then set the `arScale` value.
*
*/
get rigScale() {
// const ar_scaleValue = this.context.domElement.getAttribute("ar-scale");
// if (ar_scaleValue) {
// const scale = parseFloat(ar_scaleValue);
// if (!isNaN(scale)) return scale;
// }
if (!this._rigs[0]) return 1;
if (this._lastRigScaleUpdate !== this.context.time.frame) {
this._lastRigScaleUpdate = this.context.time.frame;
this._rigScale = this._rigs[0].gameObject.worldScale.x;
}
return this._rigScale;
}
/** add a rig to the available XR rigs - if it's priority is higher than the currently active rig it will be enabled */
addRig(rig: IXRRig) {
const i = this._rigs.indexOf(rig);
if (i >= 0) return;
if (rig.priority === undefined) rig.priority = 0;
this._rigs.push(rig);
this.updateActiveXRRig();
}
/** Remove a rig from the available XR Rigs */
removeRig(rig: IXRRig) {
const i = this._rigs.indexOf(rig);
if (i === -1) return;
this._rigs.splice(i, 1);
this.updateActiveXRRig();
}
/** Sets a XRRig to be active which will parent the camera to this rig */
setRigActive(rig: IXRRig) {
const i = this._rigs.indexOf(rig);
const currentlyActive = this._rigs[0];
this._rigs.splice(i, 1);
this._rigs.unshift(rig);
// if there's another rig currently active we need to make sure we have at least the same priority
rig.priority = currentlyActive?.priority ?? 0;
this.updateActiveXRRig();
}
/**
* @returns the user position in the rig space
*/
getUserOffsetInRig() {
const positionInRig = this.context.mainCamera?.position;
if (!positionInRig || !this.rig) return getTempVector(0, 0, 0);
const vec = getTempVector(positionInRig);
vec.x *= -1;
vec.z *= -1;
vec.applyQuaternion(getTempQuaternion(this.rig.gameObject.quaternion));
return vec;
}
private updateActiveXRRig() {
const previouslyActiveRig = this._rigs[0] ?? null;
// ensure that the default rig is in the scene
if (this._defaultRig.gameObject.parent !== this.context.scene)
this.context.scene.add(this._defaultRig.gameObject);
// ensure the fallback rig is always active!!!
this._defaultRig.gameObject.visible = true;
// ensure that the default rig is in the list of available rigs
if (!this._rigs.includes(this._defaultRig))
this._rigs.push(this._defaultRig);
// find the rig with the highest priority and make sure it's at the beginning of the array
let highestPriorityRig: IXRRig = this._rigs[0];
if (highestPriorityRig && highestPriorityRig.priority === undefined) highestPriorityRig.priority = 0;
for (let i = 1; i < this._rigs.length; i++) {
const rig = this._rigs[i];
if (!rig.isActive) continue;
if (isDestroyed(rig.gameObject)) {
this._rigs.splice(i, 1);
i--;
continue;
}
if (!highestPriorityRig || highestPriorityRig.isActive === false || (rig.priority !== undefined && rig.priority > highestPriorityRig.priority!)) {
highestPriorityRig = rig;
}
}
// make sure the highest priority rig is at the beginning if it isnt already
if (previouslyActiveRig !== highestPriorityRig) {
const index = this._rigs.indexOf(highestPriorityRig);
if (index >= 0) this._rigs.splice(index, 1);
this._rigs.unshift(highestPriorityRig);
}
if (debug) {
if (previouslyActiveRig === highestPriorityRig)
console.log("Updated Active XR Rig:", highestPriorityRig, "prev:", previouslyActiveRig);
else console.log("Updated Active XRRig:", highestPriorityRig, " (the same as before)");
}
}
private _rigs: IXRRig[] = [];
private _viewerHitTestSource: XRHitTestSource | null = null;
/** Returns a XR hit test result (if hit-testing is available) in rig space
* @param source If provided, the hit test will be performed for the given controller
*/
getHitTest(source?: NeedleXRController): NeedleXRHitTestResult | null {
if (source) {
return this.getControllerHitTest(source);
}
if (!this._viewerHitTestSource) return null;
const hitTestSource = this._viewerHitTestSource;
const hitTestResults = this.frame.getHitTestResults(hitTestSource);
if (hitTestResults.length > 0) {
const hit = hitTestResults[0];
return this.convertHitTestResult(hit);
}
return null;
}
private getControllerHitTest(controller: NeedleXRController): NeedleXRHitTestResult | null {
const hitTestSource = controller.getHitTestSource();
if (!hitTestSource) return null;
const res = this.frame.getHitTestResultsForTransientInput(hitTestSource);
for (const result of res) {
if (result.inputSource === controller.inputSource) {
for (const hit of result.results) {
return this.convertHitTestResult(hit);
}
}
}
return null;
}
private convertHitTestResult(result: XRHitTestResult): NeedleXRHitTestResult | null {
const referenceSpace = this.context.renderer.xr.getReferenceSpace();
const pose = referenceSpace && result.getPose(referenceSpace);
if (pose) {
const pos = getTempVector(pose.transform.position);
const rot = getTempQuaternion(pose.transform.orientation);
const camera = this.context.mainCamera;
if (camera?.parent !== this._cameraRenderParent) {
pos.applyMatrix4(flipForwardMatrix);
}
if (camera?.parent) {
pos.applyMatrix4(camera.parent.matrixWorld);
rot.multiply(flipForwardQuaternion);
// apply parent quaternion (if parent is moved/rotated)
const parentRotation = getWorldQuaternion(camera.parent);
// ensure that "up" (y+) is pointing away from the wall
parentRotation.premultiply(flipForwardQuaternion);
rot.premultiply(parentRotation);
}
return { hit: result, position: pos, quaternion: rot };
}
return null;
}
/** convert a XRRigidTransform from XR session space to threejs / Needle Engine XR space */
convertSpace(transform: XRRigidTransform): { position: Vector3, quaternion: Quaternion } {
const pos = getTempVector(transform.position);
pos.applyMatrix4(flipForwardMatrix);
const rot = getTempQuaternion(transform.orientation);
rot.premultiply(flipForwardQuaternion);
return { position: pos, quaternion: rot };
}
/** this is the implictly created XR rig */
private readonly _defaultRig: IXRRig;
/** all scripts that receive some sort of XR update event */
private readonly _xr_scripts: INeedleXRSessionEventReceiver[];
/** scripts that have onUpdateXR event methods */
private readonly _xr_update_scripts: INeedleXRSessionEventReceiver[] = [];
/** scripts that are in the scene but inactive (e.g. disabled parent gameObject) */
private readonly _inactive_scripts: INeedleXRSessionEventReceiver[] = [];
private readonly _controllerAdded: ControllerChangedEvt[];
private readonly _controllerRemoved: ControllerChangedEvt[];
private readonly _originalCameraWorldPosition?: Vector3 | null;
private readonly _originalCameraWorldRotation?: Quaternion | null;
private readonly _originalCameraWorldScale?: Vector3 | null;
private readonly _originalCameraParent?: Object3D | null;
/** we store the main camera reference here each frame to make sure we have a rendering camera
* this e.g. the case when the XR rig with the camera gets disabled (and thus this.context.mainCamera is unassigned)
*/
private _mainCamera: ICamera | null = null;
private constructor(mode: XRSessionMode, session: XRSession, context: Context, extra: {
scripts: INeedleXRSessionEventReceiver[],
controller_added: ControllerChangedEvt[],
controller_removed: ControllerChangedEvt[],
/** the initialization arguments */
init: XRSessionInit,
}) {
//performance.mark(measure_SessionStartedMarker);
saveSessionInfo(mode, extra.init);
this.session = session;
this.mode = mode;
this.context = context;
if (debug || getParam("console")) enableSpatialConsole(true);
this._xr_scripts = [...extra.scripts];
this._xr_update_scripts = this._xr_scripts.filter(e => typeof e.onUpdateXR === "function");
this._controllerAdded = extra.controller_added;
this._controllerRemoved = extra.controller_removed;
registerFrameEventCallback(this.onBefore, FrameEvent.LateUpdate);
this.context.pre_render_callbacks.push(this.onBeforeRender);
this.context.post_render_callbacks.push(this.onAfterRender);
if (extra.init.optionalFeatures?.includes("hit-test") || extra.init.requiredFeatures?.includes("hit-test")) {
session.requestReferenceSpace('viewer').then((referenceSpace) => {
return session.requestHitTestSource?.call(session, { space: referenceSpace })?.then((source) => {
return this._viewerHitTestSource = source;
}).catch(e => console.error(e));
}).catch(e => console.error(e));
}
if (this.context.mainCamera) {
this._originalCameraWorldPosition = getWorldPosition(this.context.mainCamera, new Vector3());
this._originalCameraWorldRotation = getWorldQuaternion(this.context.mainCamera, new Quaternion());
this._originalCameraWorldScale = getWorldScale(this.context.mainCamera, new Vector3());
this._originalCameraParent = this.context.mainCamera.parent;
if (this.context.mainCamera instanceof PerspectiveCamera) {
this.context.mainCamera[$initialFov] = this.context.mainCamera.fov;
// if (this.mode === "immersive-ar" && this.context.mainCamera.near > .1) {
// console.log("[WebXR] Setting near plane to 0.1 for better AR experience (was " + this.context.mainCamera.near + "). The initial near plane will be restored when the session ends.");
// this.context.mainCamera.near = 0.1;
// this.context.mainCamera[$initialNear] = this.context.mainCamera.near;
// }
}
}
this._defaultRig = new ImplictXRRig();
this.context