@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
991 lines • 50.5 kB
JavaScript
import { fetchProfile, MotionController } from "@webxr-input-profiles/motion-controllers";
import { AxesHelper, Euler, MathUtils, Matrix4, Object3D, Quaternion, Ray, Vector3 } from "three";
import { Context } from "../engine_context.js";
import { Gizmos } from "../engine_gizmos.js";
import { InputEvents, NEPointerEvent } from "../engine_input.js";
import { getTempQuaternion, getTempVector, getWorldQuaternion } from "../engine_three_utils.js";
import { getParam } from "../engine_utils.js";
import { flipForwardMatrix, flipForwardQuaternion } from "./internal.js";
const debug = getParam("debugwebxr");
/** when enabled we will not use the browser select event but instead
* we will emit the input event based on our own pinch detection
* this is a workaround for visionOS not emitting the select events, see https://linear.app/needle/issue/NE-4212
*/
const debugCustomGesture = getParam("debugcustomgesture");
// https://github.com/immersive-web/webxr-input-profiles/blob/4484a05e30bcd43fe86bb4e06b7a707861a26796/packages/registry/profiles/meta/meta-quest-touch-plus.json
const DEFAULT_PROFILES_PATH = 'https://cdn.jsdelivr.net/npm/@webxr-input-profiles/assets@1.0/dist/profiles';
const DEFAULT_PROFILE = 'generic-trigger';
const metacarpalToGripQuaternion = new Quaternion().setFromEuler(new Euler(MathUtils.degToRad(0), MathUtils.degToRad(-90), MathUtils.degToRad(-90)));
const metacarpalToGripPosition = new Vector3(0.04, -0.04, 0.0);
/**
* A NeedleXRController wraps a connected XRInputDevice that is either a physical controller or a hand
* You can access specific buttons using `getButton` and `getStick`
* To get spatial data in rig space (position, rotation) use the `gripPosition`, `gripQuaternion`, `rayPosition` and `rayQuaternion` properties
* To get spatial data in world space use the `gripWorldPosition`, `gripWorldQuaternion`, `rayWorldPosition` and `rayWorldQuaternion` properties
* Inputs will also be emitted as pointer events on `this.context.input` - so you can receive controller inputs on objects using the appropriate input events on your components (e.g. `onPointerDown`, `onPointerUp` etc) - use the `pointerType` property to check if the event is from a controller or not
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRInputSource
*/
export class NeedleXRController {
/** the Needle XR Session */
xr;
get context() { return this.xr.context; }
/**
* https://developer.mozilla.org/en-US/docs/Web/API/XRInputSource
*/
inputSource;
/** the input source index */
index = 0;
/** When enabled the controller will create input events in the Needle Engine input system (e.g. when a button is pressed or the controller is moved)
* You can disable this if you don't want inputs to go through the input system but be aware that this will result in `onPointerDown` component callbacks to not be invoked anymore for this XRController
*/
emitEvents = true;
/** Is the controller still connected? */
get connected() {
return this._connected;
}
_connected = true;
get isTracking() { return this._isTracking; }
_isTracking = false;
/** the input source gamepad giving raw access to the gamepad values
* You should usually use the `getButton` and `getStick` methods instead to get access to named buttons and sticks
*/
get gamepad() { return this.__gamepad ??= this.inputSource.gamepad; }
__gamepad;
/** @returns true if this is a hand (otherwise this is a controller) */
get isHand() { return this.hand != undefined; }
/**
* If this is a hand then this is the hand info (XRHand)
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRHand
*/
get hand() { return this.__hand ??= this.inputSource.hand; }
__hand;
/** threejs XRHandSpace, shorthand for `context.renderer.xr.getHand(controllerIndex)`
* @link https://threejs.org/docs/#api/en/renderers/webxr/WebXRManager.getHand
*/
get handObject() { return this.context.renderer.xr.getHand(this.index); }
/** The input source profiles */
get profiles() { return this.inputSource.profiles; }
/** The device input layout */
get layout() { return this._layout; }
/** shorthand for `inputSource.targetRayMode` */
get targetRayMode() { return this.inputSource.targetRayMode; }
/** shorthand for `inputSource.targetRaySpace` */
get targetRaySpace() { return this.inputSource.targetRaySpace; }
/** shorthand for `inputSource.gripSpace` */
get gripSpace() { return this.inputSource.gripSpace; }
/**
* If the controller if held in the left or right hand (or if it's a left or right hand)
**/
get side() { return this.__side ??= this.inputSource.handedness; }
__side = undefined;
/** is right side. shorthand for `side === 'right'` */
get isRight() { return this.side === 'right'; }
/** is left side. shorthand for `side === 'left'` */
get isLeft() { return this.side === 'left'; }
/** is XR stylus, e.g. Logitech MX Ink */
get isStylus() { return this._isMxInk; }
/** The XRTransientInputHitTestSource can be used to perform hit tests with the controller ray against the real world.
* see https://developer.mozilla.org/en-US/docs/Web/API/XRSession/requestHitTestSourceForTransientInput for more information
* Requires the hit-test feature to be enabled in the XRSession
*
* NOTE: The hit test source should be cancelled once it's not needed anymore. Call `cancelHitTestSource` to do this
*/
getHitTestSource() {
if (!this._hitTestSource)
this._requestHitTestSource();
return this._hitTestSource;
}
get hasHitTestSource() {
return this._hitTestSource;
}
/** Make sure to cancel the hittest source once it's not needed anymore */
cancelHitTestSource() {
if (this._hitTestSource) {
this._hitTestSource.cancel();
this._hitTestSource = undefined;
}
}
_hitTestSource = undefined;
_hasSelectEvent = false;
get hasSelectEvent() { return this._hasSelectEvent; }
_isMxInk = false;
_isMetaQuestTouchController = false;
/** Perform a hit test against the XR planes or meshes. shorthand for `xr.getHitTest(controller)`
* @returns the hit test result (with position and rotation in worldspace) or null if no hit was found
*/
getHitTest() {
return this.xr.getHitTest(this);
}
/** This is cleared at the beginning of each frame */
_handJointPoses = new Map();
/** Get the hand joint pose from the current XRFrame. Results are cached for a frame to avoid calling getJointPose multiple times */
getHandJointPose(joint, frame) {
frame = frame || this.xr.frame;
if (!this.hand || !frame?.getJointPose || !this.xr.referenceSpace)
return null;
let pose = this._handJointPoses?.get(joint);
if (pose)
return pose;
pose = frame.getJointPose(joint, this.xr.referenceSpace);
if (pose)
this._handJointPoses.set(joint, pose);
return pose;
}
/** Grip matrix in grip space */
_gripMatrix = new Matrix4();
/** Grip position in grip space */
_gripPosition = new Vector3();
/** Grip rotation in grip space */
_gripQuaternion = new Quaternion();
_linearVelocity = new Vector3();
_rayPositionRaw = new Vector3();
_rayRotationRaw = new Quaternion();
/** ray matrix in grip space */
_rayMatrix = new Matrix4();
/** Ray position in rig space */
_rayPosition = new Vector3();
/** Ray rotation in rig space */
_rayQuaternion = new Quaternion();
/** Grip position in rig space */
get gripPosition() { return getTempVector(this._gripPosition); }
/** Grip rotation in rig space */
get gripQuaternion() { return getTempQuaternion(this._gripQuaternion); }
get gripMatrix() { return this._gripMatrix; }
/** Grip linear velocity in rig space
* @link https://developer.mozilla.org/en-US/docs/Web/API/XRPose/linearVelocity
*/
get gripLinearVelocity() {
return getTempVector(this._linearVelocity).applyQuaternion(flipForwardQuaternion);
}
/** Ray position in rig space */
get rayPosition() { return getTempVector(this._rayPosition); }
/** Ray rotation in rig space */
get rayQuaternion() { return getTempQuaternion(this._rayQuaternion); }
/** Controller grip position in worldspace */
get gripWorldPosition() { return getTempVector(this._gripWorldPosition); }
_gripWorldPosition = new Vector3();
/** Controller grip rotation in wordspace */
get gripWorldQuaternion() {
return getTempQuaternion(this._gripWorldQuaternion);
}
_gripWorldQuaternion = new Quaternion();
/** Controller ray position in worldspace (this value is calculated once per frame by default - call `updateRayWorldPosition` to force an update) */
get rayWorldPosition() {
return getTempVector(this._rayWorldPosition);
}
_rayWorldPosition = new Vector3();
/** Recalculates the ray world position */
updateRayWorldPosition() {
const parent = this.xr.context.mainCamera?.parent;
this._rayWorldPosition.copy(this._rayPositionRaw);
if (parent)
this._rayWorldPosition.applyMatrix4(parent.matrixWorld);
}
/** Controller ray rotation in wordspace (this value is calculated once per frame by default - call `updateRayWorldQuaternion` to force an update) */
get rayWorldQuaternion() {
return getTempQuaternion(this._rayWorldQuaternion);
}
_rayWorldQuaternion = new Quaternion();
get pinchPosition() {
return getTempVector(this._pinchPosition);
}
_pinchPosition = new Vector3();
/** Recalculates the ray world quaternion */
updateRayWorldQuaternion() {
const parent = this.xr.context.mainCamera?.parent;
const parentWorldQuaternion = parent ? getWorldQuaternion(parent) : undefined;
this._rayWorldQuaternion.copy(this._rayRotationRaw)
// flip forward because we want +Z to be forward
.multiply(flipForwardQuaternion);
if (parentWorldQuaternion)
this._rayWorldQuaternion.premultiply(parentWorldQuaternion);
}
/** The controller ray in worldspace */
get ray() {
this._ray.origin.copy(this.rayWorldPosition);
this._ray.direction.copy(getTempVector(0, 0, 1).applyQuaternion(this.rayWorldQuaternion));
return this._ray;
}
_ray;
/** Recalculated once per update */
_hand_wristDotUp = undefined;
/**
* The dot product of the hand palm with the up vector.
* This is a number between -1 and 1, where 1 means the palm is directly up and -1 means the palm is directly down (upside down).
* This value is undefined if there's no hand
*/
get handWristDotUp() {
if (this._hand_wristDotUp !== undefined)
return this._hand_wristDotUp;
const handPalm = this.handObject?.joints["wrist"];
if (handPalm) {
const up = getTempVector(0, 1, 0).applyQuaternion(handPalm.quaternion);
const dot = getTempVector(0, 1, 0).dot(up);
return this._hand_wristDotUp = dot;
}
return undefined;
}
/**
* @returns true if the hand is upside down
*/
get isHandUpsideDown() {
return this.handWristDotUp !== undefined ? this.handWristDotUp < -.7 : false;
}
/**
* @returns true if the hand is upside down and we got a pinch down event this frame.
*/
get isTeleportGesture() {
return this.isHandUpsideDown && this.getGesture("pinch")?.isDown;
}
/** The controller object space.
* You can use it to attach objects to the controller.
* Children will be automatically detached and put into the scene when the controller disconnects
*/
get object() { return this._object; }
_object;
_gripSpaceObject;
_raySpaceObject;
/** Assigned the model that you use for rendering. This can be used as a hint for other components */
model = null;
_debugAxesHelper = new AxesHelper(.15);
_debugGripAxesHelper = new AxesHelper(.07);
_debugRayAxesHelper = new AxesHelper(.07);
/** returns the URL of the default controller model */
async getModelUrl() {
return this.getMotionController?.then(res => res?.assetUrl || null);
}
constructor(session, device, index) {
this.xr = session;
this.inputSource = device;
this.index = index;
this._object = new Object3D();
this._object.name = `NeedleXRController_${index}`;
if (debug) {
this._object.add(this._debugAxesHelper);
this._gripSpaceObject = new Object3D();
this._raySpaceObject = new Object3D();
this._gripSpaceObject.name = `NeedleXRController_${index}_gripSpace`;
this._raySpaceObject.name = `NeedleXRController_${index}_raySpace`;
this._gripSpaceObject.add(this._debugGripAxesHelper);
this._raySpaceObject.add(this._debugRayAxesHelper);
this.xr.context.scene.add(this._gripSpaceObject);
this.xr.context.scene.add(this._raySpaceObject);
}
this.xr.context.scene.add(this._object);
this._ray = new Ray();
this.pointerInit = {
origin: this,
pointerType: this.hand ? "hand" : "controller",
deviceIndex: this.index,
pointerId: -1,
mode: this.inputSource.targetRayMode,
ray: this._ray,
device: this._object,
buttonName: "none",
};
this.initialize();
this.subscribeEvents();
}
_hitTestSourcePromise = null;
_requestHitTestSource() {
if (this._hitTestSourcePromise)
return this._hitTestSourcePromise;
// We only request a hit test source when we need it - meaning e.g. when we want to place the scene in AR
// Make sure to cancel the hittest source when we don't need it anymore for performance reasons
// // TODO: change this to check if we have hit-testing enabled instead of pass through.
if (this.xr.mode === "immersive-ar" && this.inputSource.targetRayMode === "tracked-pointer" && this.xr.session.requestHitTestSourceForTransientInput) {
// request hittest source
return this._hitTestSourcePromise = this.xr.session.requestHitTestSourceForTransientInput({
profile: this.inputSource.profiles[0],
offsetRay: new XRRay(),
})?.then(hitTestSource => {
this._hitTestSourcePromise = null;
if (!this.connected) {
hitTestSource.cancel();
return null;
}
return this._hitTestSource = hitTestSource;
}) ?? null;
}
return null;
}
onPointerHits = _evt => {
};
onUpdate(frame) {
this.onUpdateFrame(frame);
this.updateInputEvents();
this.onUpdateMove();
//performance.mark('NeedleXRController onUpdate end');
//performance.measure('NeedleXRController onUpdate', 'NeedleXRController onUpdate start', 'NeedleXRController onUpdate end');
}
onRenderDebug() {
Gizmos.DrawSphere(this.rayWorldPosition, .003);
Gizmos.DrawDirection(this.rayWorldPosition, getTempVector(0, 0, 10).applyQuaternion(this.rayWorldQuaternion));
const labelPosition = this.inputSource.gripSpace ? this.gripWorldPosition : this.object.worldPosition;
const debugLabelPosition = labelPosition.sub(this.object.worldForward.multiplyScalar(.1));
const profileStr = this.inputSource.profiles.join("\n");
let debugStr = `Controller[${this.index}] (${this.inputSource.targetRayMode}, ${this.side})
C:${this.connected ? "x" : "-"} T:${this.isTracking ? "x" : "-"} Hand:${this.inputSource.hand ? "x" : "-"} Pen: ${this._isMxInk ? "x" : "-"}`;
if (this.inputSource.hand)
debugStr += `\nPinch: ${this.getGesture("pinch")?.value.toFixed(3)}`;
debugStr += "\n" + profileStr;
debugStr += "\n" + (this.inputSource.targetRaySpace ? `Ray: x` : "Ray: -") +
(this.inputSource.gripSpace ? " Grip: x" : " Grip: -") +
(this.inputSource.gamepad ? ` Gamepad: ${this.inputSource.gamepad.mapping}` : " Gamepad: -");
if (this.inputSource.gamepad) {
const gp = this.inputSource.gamepad;
let gamepadStr = "[btns " + gp.buttons.length + "]: " + gp.buttons.map(b => b.value.toPrecision(1)).join(",");
gamepadStr += "\n[axes " + gp.axes.length + "]: " + gp.axes.map(a => a.toPrecision(1)).join(",");
debugStr += "\n" + gamepadStr;
}
Gizmos.DrawLabel(debugLabelPosition, debugStr, .006);
}
onUpdateFrame(frame) {
// make sure this is cleared every frame
this._handJointPoses.clear();
this._hand_wristDotUp = undefined;
if (!this.xr.referenceSpace) {
this._isTracking = false;
return;
}
const rayPose = frame.getPose(this.inputSource.targetRaySpace, this.xr.referenceSpace);
this._isTracking = rayPose != null;
let gripPositionRaw = null;
let gripQuaternionRaw = null;
let rayPositionRaw = null;
let rayQuaternionRaw = null;
if (rayPose) {
const t = rayPose.transform;
this._rayMatrix
.fromArray(t.matrix)
.premultiply(flipForwardMatrix);
this._rayMatrix.decompose(this._rayPosition, this._rayQuaternion, getTempVector(1, 1, 1));
rayPositionRaw = getTempVector(t.position);
rayQuaternionRaw = getTempQuaternion(t.orientation);
this._rayPositionRaw.copy(rayPositionRaw);
this._rayRotationRaw.copy(rayQuaternionRaw);
}
if (this.inputSource.gripSpace) {
const gripPose = frame.getPose(this.inputSource.gripSpace, this.xr.referenceSpace);
if (gripPose) {
const t = gripPose.transform;
gripPositionRaw = getTempVector(t.position);
gripQuaternionRaw = getTempQuaternion(t.orientation);
this._gripMatrix
.fromArray(t.matrix)
.premultiply(flipForwardMatrix);
this._gripMatrix.decompose(this._gripPosition, this._gripQuaternion, getTempVector(1, 1, 1));
if ("linearVelocity" in gripPose && gripPose.linearVelocity) {
const p = gripPose.linearVelocity;
this._linearVelocity.set(p.x, p.y, p.z);
}
}
}
// update controller object parent – needs to be parented to the rig, which
// implicitly is the same object as the camera parent.
if (this.xr.context.mainCamera?.parent) {
if (this._object.parent !== this.xr.context.mainCamera?.parent)
this.xr.context.mainCamera.parent.add(this._object);
if (this._gripSpaceObject !== undefined && this._gripSpaceObject?.parent !== this.xr.context.mainCamera?.parent)
this.xr.context.mainCamera.parent.add(this._gripSpaceObject);
if (this._raySpaceObject !== undefined && this._raySpaceObject?.parent !== this.xr.context.mainCamera?.parent)
this.xr.context.mainCamera.parent.add(this._raySpaceObject);
}
// for controllers, we set the position and rotation of the object to the ray position and rotation
// for hands, we take the wrist position and rotation
const hand = this.hand;
if (hand) {
// https://www.w3.org/TR/webxr-hand-input-1/#xrhand-interface
let gotWrist = false;
// TODO check why types are not correct here
const wrist = hand.get("wrist");
const wristPose = wrist && this.getHandJointPose(wrist, frame);
if (wristPose) {
gotWrist = true;
const p = wristPose.transform.position;
const q = wristPose.transform.orientation;
this._object.position.set(p.x, p.y, p.z);
this._object.quaternion.set(q.x, q.y, q.z, q.w).multiply(flipForwardQuaternion);
}
if (!gotWrist) {
this._object.position.copy(this._rayPosition);
this._object.quaternion.copy(this._rayQuaternion).multiply(flipForwardQuaternion);
}
//@ts-ignore
const middle = hand.get("middle-finger-metacarpal");
const middlePose = middle && this.getHandJointPose(middle, frame);
if (middlePose) {
// for some reason the grip rotation is different from the wrist rotation
// but we want to use the wrist rotation for the grip
this._gripMatrix
.fromArray(middlePose.transform.matrix)
.premultiply(flipForwardMatrix);
this._gripMatrix.decompose(this._gripPosition, this._gripQuaternion, getTempVector(1, 1, 1));
// If we don't have a grip space, we update the data from the metacarpal bone instead.
// this way, things looking for a grip pose will still find one (e.g. XRControllerFollow).
// For example, hands on VisionOS do not provide a gripSpace.
if (true || !this.inputSource.gripSpace) {
gripPositionRaw = getTempVector().copy(middlePose.transform.position);
gripQuaternionRaw = getTempQuaternion().copy(middlePose.transform.orientation);
gripQuaternionRaw.multiply(metacarpalToGripQuaternion);
gripPositionRaw.add(getTempVector(metacarpalToGripPosition).applyQuaternion(gripQuaternionRaw));
}
}
}
// on VisionOS we get a gripSpace that matches where the controller is for transient input sources
else if (this.inputSource.gripSpace && this.targetRayMode === "transient-pointer" && gripPositionRaw && gripQuaternionRaw) {
this._object.position.copy(gripPositionRaw);
this._object.quaternion.copy(gripQuaternionRaw).multiply(flipForwardQuaternion);
}
else if (rayPositionRaw && rayQuaternionRaw) {
this._object.position.copy(rayPositionRaw);
this._object.quaternion.copy(rayQuaternionRaw).multiply(flipForwardQuaternion);
}
if (debug) {
if (rayPositionRaw && rayQuaternionRaw) {
this._raySpaceObject?.position.copy(rayPositionRaw);
this._raySpaceObject?.quaternion.copy(rayQuaternionRaw).multiply(flipForwardQuaternion);
}
if (gripPositionRaw && gripQuaternionRaw) {
this._gripSpaceObject?.position.copy(gripPositionRaw);
this._gripSpaceObject?.quaternion.copy(gripQuaternionRaw).multiply(flipForwardQuaternion);
}
}
// UPDATE WORLD TRANSFORM DATA
const parent = this.xr.context.mainCamera?.parent;
const parentWorldQuaternion = parent ? getWorldQuaternion(parent) : undefined;
// GRIP
if (gripPositionRaw && gripQuaternionRaw) {
this._gripWorldPosition.copy(gripPositionRaw);
if (parent)
this._gripWorldPosition.applyMatrix4(parent.matrixWorld);
this._gripWorldQuaternion.copy(gripQuaternionRaw);
// flip forward because we want +Z to be forward
this._gripWorldQuaternion.multiply(flipForwardQuaternion);
if (parentWorldQuaternion)
this._gripWorldQuaternion.premultiply(parentWorldQuaternion);
}
// RAY
this.updateRayWorldPosition();
this.updateRayWorldQuaternion();
}
/** Called when the input source disconnects */
onDisconnected() {
this._connected = false;
if (debug)
console.warn("Controller disconnected", this.index);
// move all attached objects into the scene
for (const child of this._object.children) {
this.xr.context.scene.attach(child);
}
this._object?.removeFromParent();
this._debugAxesHelper?.removeFromParent();
this._debugGripAxesHelper?.removeFromParent();
this._debugRayAxesHelper?.removeFromParent();
this._gripSpaceObject?.removeFromParent();
this._raySpaceObject?.removeFromParent();
this.unsubscribeEvents();
if (this._hitTestSource) {
this._hitTestSource.cancel();
this._hitTestSource = undefined;
}
}
/**
* Get a gamepad button
* @link https://github.com/immersive-web/webxr-gamepads-module/blob/main/gamepads-module-explainer.md
* @param key the controller button name e.g. x-button
* @returns the gamepad button if it exists on the controller - otherwise undefined
*/
getButton(key) {
if (!this._layout)
return undefined;
switch (key) {
case "primary-button":
if (this.isLeft)
key = "x-button";
else if (this.isRight)
key = "a-button";
else
return undefined;
break;
case "primary":
if (this.hand) {
return this.getGesture("pinch");
}
return this.toNeedleGamepadButton(0, key);
}
if (this._buttonMap.has(key)) {
return this.toNeedleGamepadButton(this._buttonMap.get(key), key);
}
const componentModel = this._layout?.components[key];
if (componentModel?.gamepadIndices) {
switch (componentModel.type) {
case "button":
case "squeeze":
if (this.inputSource.gamepad) {
const index = componentModel.gamepadIndices.button;
this._buttonMap.set(key, index);
return this.toNeedleGamepadButton(index, key);
}
break;
default:
console.warn("Unsupported component type", componentModel.type);
break;
}
}
this._buttonMap.set(key, undefined);
return undefined;
}
/** Get a gesture state */
getGesture(key) {
const state = this.states[key];
if (!state)
return null;
this.states[key] = state;
const needleButton = this._needleGamepadButtons[key] || new NeedleGamepadButton(undefined, key);
needleButton.pressed = state.pressed;
needleButton.value = state.value;
needleButton.isDown = state.isDown;
needleButton.isUp = state.isUp;
this._needleGamepadButtons[key] = needleButton;
return needleButton;
}
getPointerId(button) {
if (button === "primary") {
button = 0;
}
else if (button === "pinch") {
button = 0;
}
if (typeof button !== "number") {
const needleButton = this._buttonMap.get(button);
if (needleButton === undefined) {
return undefined;
}
button = needleButton;
}
return this.index * 10 + button;
}
_needleGamepadButtons = {};
/** combine the InputState information + the GamepadButton information (since GamepadButtons can not be extended) */
toNeedleGamepadButton(index, name) {
if (!this.inputSource.gamepad?.buttons)
return undefined;
const button = this.inputSource.gamepad?.buttons[index];
const state = this.states[index];
const needleButton = this._needleGamepadButtons[index] || new NeedleGamepadButton(index, name);
if (button) {
needleButton.pressed = button.pressed;
needleButton.value = button.value;
needleButton.touched = button.touched;
}
if (state) {
needleButton.isDown = state.isDown;
needleButton.isUp = state.isUp;
}
this._needleGamepadButtons[index] = needleButton;
return needleButton;
}
/**
* Get the values of a controller joystick
* @link https://github.com/immersive-web/webxr-gamepads-module/blob/main/gamepads-module-explainer.md
* @returns the stick values where x is left/right, y is up/down and z is the button value
*/
getStick(key) {
if (!this._layout)
return { x: 0, y: 0, z: 0 };
if (key === "primary") {
const x = this.inputSource.gamepad?.axes[0] || 0;
const y = this.inputSource.gamepad?.axes[1] || 0;
// the primary thumbstick is button 3 (see gamepads module explainer)
const z = this.inputSource.gamepad?.buttons[3]?.value || 0;
return { x, y, z };
}
const componentModel = this._layout?.components[key];
if (componentModel?.gamepadIndices) {
switch (componentModel.type) {
case "thumbstick":
if (this.inputSource.gamepad) {
const xIndex = componentModel.gamepadIndices.xAxis;
const yIndex = componentModel.gamepadIndices.yAxis;
let x = this.inputSource.gamepad?.axes[xIndex];
let y = this.inputSource.gamepad?.axes[yIndex];
x *= -1;
y *= -1;
const buttonIndex = componentModel.gamepadIndices.button;
const z = this.inputSource.gamepad?.buttons[buttonIndex]?.value;
return { x, y, z };
}
}
}
return { x: 0, y: 0, z: 0 };
}
_buttonMap = new Map();
// the motion controller contains the controller scheme, we use this to simplify button access
_motioncontroller;
_layout;
getMotionController;
initialize() {
// WORKAROUND for hand controllers that don't have a select event
this._hasSelectEvent = this.profiles.includes("generic-hand-select") || this.profiles.some(p => p.startsWith("generic-trigger"));
// Used to determine special layout for Quest controllers, e.g. last button is menu button
this._isMetaQuestTouchController = this.profiles.includes("meta-quest-touch-plus") || this.profiles.includes("oculus-touch-v3");
// Proper profile starting with v69 and browser 35.1
this._isMxInk = this.profiles.includes("logitech-mx-ink");
if (!this._layout) {
// Ignore transient-pointer since we likely don't want to spawn a controller visual just for a temporary pointer.
// TODO we should check how this is actually handled on Quest Browser when the transient-pointer flag is on.
if (this.inputSource.targetRayMode === "transient-pointer")
return;
// TODO: we should fetch the profiles or better yet the profile list once and cache it
const fetchProfileCall = fetchProfile(this.inputSource, DEFAULT_PROFILES_PATH, DEFAULT_PROFILE);
/** @ts-ignore */
this.getMotionController = fetchProfileCall.then(res => {
if (!this.connected)
return null;
this._motioncontroller = new MotionController(this.inputSource, res.profile, res.assetPath || "");
const profile = res.profile;
const layout = profile.layouts[this.inputSource.handedness];
this._layout = layout;
if (this._layout) {
if (!this._layout.gamepad?.length) {
this._layout.gamepad = [];
for (const key in this._layout.components) {
const component = this._layout.components[key];
this._layout.gamepad[component.gamepadIndices.button] = key;
}
}
}
// if (debug) console.log(this._layout, this.inputSource);
// debugger;
// this.getButton("a-button")
return this._motioncontroller;
}).catch(err => {
if (this.inputSource)
console.warn("Couldn't initialize motion controller profile for ", this.inputSource, err);
return null;
});
}
}
/**
* When enabled the controller will automatically emit pointer down events to the Needle Engine Input System.
* @default true
*/
emitPointerDownEvent = true;
/**
* When enabled the controller will automatically emit pointer up events to the Needle Engine Input System.
* @default true
*/
emitPointerUpEvent = true;
/**
* When enabled the controller will automatically emit pointer move events to the Needle Engine Input System.
* @default true
*/
emitPointerMoveEvent = true;
/**
* The distance threshold for pointer move events. This value is in units in rig space
* @default 0.03
*/
pointerMoveDistanceThreshold = 0.03;
/**
* The angle threshold for pointer move events. This value is in radians.
* @default 0.05
*/
pointerMoveAngleThreshold = 0.05;
subscribeEvents() {
// https://developer.mozilla.org/en-US/docs/Web/API/XRSession/selectstart_event
this.xr.session.addEventListener("selectstart", this.onSelectStart);
this.xr.session.addEventListener("selectend", this.onSelectEnd);
// https://developer.mozilla.org/en-US/docs/Web/API/XRSession/squeeze_event
this.xr.session.addEventListener("squeezestart", this.onSequeezeStart);
this.xr.session.addEventListener("squeezeend", this.onSequeezeEnd);
}
unsubscribeEvents() {
this.xr.session.removeEventListener("selectstart", this.onSelectStart);
this.xr.session.removeEventListener("selectend", this.onSelectEnd);
this.xr.session.removeEventListener("squeezestart", this.onSequeezeStart);
this.xr.session.removeEventListener("squeezeend", this.onSequeezeEnd);
}
_selectButtonIndex = undefined;
_squeezeButtonIndex = undefined;
onSelectStart = (evt) => {
if (!this.emitPointerDownEvent)
return;
if (this.inputSource !== evt.inputSource)
return;
// if a selectstart event happens right after an input source is connected, we may even receive this event before
// requestAnimationFrame callback with the current session. So, we need to update the frame here.
this.onUpdateFrame(evt.frame);
// if we receive a select event we can be true that this device supports select events
this._hasSelectEvent = true;
const selectComponentId = this._layout?.selectComponentId;
const i = this._layout?.components[selectComponentId]?.gamepadIndices?.button;
if (i !== undefined)
this._selectButtonIndex = i;
if (debugCustomGesture)
return;
/*
if (!_didReceiveSelectStartEvent) {
_didReceiveSelectStartEvent = true;
// safeguard first pinch event - check if the pinch gesture is already down
const pinch = this.getGesture("pinch");
if (pinch?.pressed) {
console.warn("Select start event was received but the pinch gesture is already down. This might happen the first time you start pinching", this.index, this.side);
return;
}
}
*/
if (debug)
Gizmos.DrawDirection(this.rayWorldPosition, getTempVector(0, .01, 1).applyQuaternion(this.rayWorldQuaternion), 0xff0000, 10);
this.emitPointerEvent(InputEvents.PointerDown, this._selectButtonIndex || 0, "xr-standard-trigger", true, evt);
};
onSelectEnd = (evt) => {
if (!this.emitPointerUpEvent)
return;
if (debugCustomGesture)
return;
if (this.inputSource !== evt.inputSource)
return;
this.emitPointerEvent(InputEvents.PointerUp, this._selectButtonIndex || 0, "xr-standard-trigger", true, evt);
};
onSequeezeStart = (evt) => {
if (!this.emitPointerDownEvent)
return;
if (this.inputSource !== evt.inputSource)
return;
this._squeezeButtonIndex = this._layout?.components["xr-standard-squeeze"]?.gamepadIndices?.button;
if (this._squeezeButtonIndex !== undefined) {
if (debug)
Gizmos.DrawDirection(this.rayWorldPosition, getTempVector(0, .01, 1).applyQuaternion(this.rayWorldQuaternion), 0x0000ff, 10);
this.emitPointerEvent(InputEvents.PointerDown, this._squeezeButtonIndex || 0, "xr-standard-squeeze", true, evt);
}
};
onSequeezeEnd = (evt) => {
if (!this.emitPointerUpEvent)
return;
if (this.inputSource !== evt.inputSource)
return;
if (this._squeezeButtonIndex !== undefined)
this.emitPointerEvent(InputEvents.PointerUp, this._squeezeButtonIndex || 0, "xr-standard-squeeze", true, evt);
};
/** Index = button index */
states = {};
// If we want to invoke button events for ALL buttons we need to keep track of the previous state
// instead of using XR input select start events which is only raised for the primary button
// we should probably do both but then we need to ignore the primary index in the following function (to not raise an event for the same button twice)
// and start with index = 1
updateInputEvents() {
// https://immersive-web.github.io/webxr-gamepads-module/#xr-standard-heading
if (this.gamepad?.buttons) {
for (let index = 0; index < this.gamepad.buttons.length; index++) {
const button = this.gamepad.buttons[index];
const state = this.states[index] || new InputState();
let eventName = null;
// Special handling for MX Ink stylus on Quest OS v69+.
// We're never getting a "pressed" state here, so we determine pressed state based on the value.
if (this._isMxInk && (index === 4 || index === 5)) {
if (button.value > 0 && !state.pressed) {
eventName = "pointerdown";
state.isDown = true;
state.isUp = false;
}
else if (button.value === 0 && state.pressed) {
eventName = "pointerup";
state.isDown = false;
state.isUp = true;
}
else if (state.pressed) {
eventName = "pointermove";
state.isDown = false;
state.isUp = false;
}
state.pressed = button.value > 0;
state.value = button.value;
}
// Regular controller handling.
else {
// is down
if (button.pressed && !state.pressed) {
eventName = "pointerdown";
state.isDown = true;
state.isUp = false;
}
// is up
else if (!button.pressed && state.pressed) {
eventName = "pointerup";
state.isDown = false;
state.isUp = true;
}
else {
state.isDown = false;
state.isUp = false;
}
state.pressed = button.pressed;
state.value = button.value;
}
this.states[index] = state;
// the selection event is handled in the "selectstart" callback
const emitEvent = index !== this._selectButtonIndex && index !== this._squeezeButtonIndex;
if (eventName != null && emitEvent) {
let name = this._layout?.gamepad[index];
if (this._isMxInk && index === 4)
name = "stylus-touch";
if (this._isMxInk && index === 5)
name = "stylus-tip";
if (debug || debugCustomGesture)
console.log("Emitting pointer event", eventName, index, name, button.value, this.gamepad, this._layout);
this.emitPointerEvent(eventName, index, name ?? "none", false, null, button.value);
}
}
// For Quest controllers, the last button is the menu button
if (this._isMetaQuestTouchController) {
const menuButtonIndex = this.gamepad.buttons.length - 1;
const menuButtonState = this.states[menuButtonIndex];
if (menuButtonState) {
if (menuButtonState.isDown) {
const menu = this.context.menu;
if (menu.spatialMenuIsVisible)
menu.setSpatialMenuVisible(false);
else
this.context.menu.setSpatialMenuVisible(true);
}
}
}
}
// update hand gesture states
if (this.hand) {
const handObject = this.handObject;
if (handObject) {
// update pinch state
const indexTip = handObject.joints["index-finger-tip"];
const thumbTip = handObject.joints["thumb-tip"];
if (indexTip && thumbTip) {
const distance = indexTip.position.distanceTo(thumbTip.position);
// upddate position of the pinch point
this._pinchPosition.lerpVectors(indexTip.position, thumbTip.position, .5);
const parent = this.xr.context.mainCamera?.parent;
if (parent)
this._pinchPosition.applyMatrix4(parent.matrixWorld);
if (distance !== 0) { // ignore exactly 0 which happens when we switch from hands to controllers
const pinchThreshold = .02;
const pinchHysteresis = .01;
const state = this.states["pinch"] || new InputState();
const maxDistance = (pinchThreshold + pinchHysteresis) * 1.5;
state.value = 1 - ((distance - pinchThreshold) / maxDistance);
const isPressed = distance < (pinchThreshold - pinchHysteresis);
const isReleased = distance > (pinchThreshold + pinchHysteresis);
if (isPressed && !state.pressed) {
if (debugCustomGesture)
console.log("pinch start", distance);
state.isDown = true;
state.isUp = false;
state.pressed = true;
}
else if (isReleased && state.pressed) {
state.isDown = false;
state.isUp = true;
state.pressed = false;
}
else {
state.isDown = false;
state.isUp = false;
}
this.states["pinch"] = state;
}
/** Workaround for visionOS not emitting selectstart. See https://linear.app/needle/issue/NE-4212
* If a selectstart event was never received we do a manual check here if the user is pinching
* Update: VisionOS 1.1 now properly emits select events from transient input sources, based on gaze.
* We're keeping this code commented for now since there may be future changes before VisionOS WebXR ships.
*/
/*
if (!_didReceiveSelectStartEvent && (state.isDown || state.isUp)) {
const eventName = isPressed ? "pointerdown" : "pointerup";
const pressure = distance / pinchThreshold;
if (debugCustomGesture) {
const p = this.rayWorldPosition.add(this.object.worldForward.multiplyScalar(.2));
p.y += .05;
p.y += Math.random() * .02;
Gizmos.DrawLabel(p, "pinch:" + eventName + ", " + this.index + ", " + this.side + "\n" + handObject.uuid, 0.01, 5, 0x000000, new RGBAColor(1, 1, 1, .1));
}
this.emitPointerEvent(eventName, 0, "pinch", false, null, pressure);
}
*/
}
}
}
}
_didMoveLastFrame = false;
_lastPointerMovePosition = new Vector3();
_lastPointerMoveQuaternion = new Quaternion();
onUpdateMove() {
if (!this.emitPointerMoveEvent)
return;
let didMove = false;
const dist = this._lastPointerMovePosition.distanceTo(this.gripWorldPosition);
if (dist > this.pointerMoveDistanceThreshold * this.xr.rigScale)
didMove = true;
if (!didMove) {
const angle = this._lastPointerMoveQuaternion.angleTo(this.gripWorldQuaternion);
if (angle > this.pointerMoveAngleThreshold)
didMove = true;
}
if (didMove) {
this._didMoveLastFrame = true;
this._lastPointerMovePosition.copy(this.gripWorldPosition);
this._lastPointerMoveQuaternion.copy(this.gripWorldQuaternion);
if (debug)
Gizmos.DrawLabel(this.rayWorldPosition.add(this.object.worldForward.multiplyScalar(.1)), "move", .01);
let button = this.xr.context.input.getFirstPressedButtonForPointer(this.index);
if (button === undefined)
button = 0;
const pressure = this.gamepad?.buttons[button]?.value;
this.emitPointerEvent("pointermove", button, "none", false, null, pressure);
}
else {
this._didMoveLastFrame = false;
}
}
/** cached spatial pointer init object. We re-use it to not have */
pointerInit;
emitPointerEvent(type, button, buttonName, primary, source = null, pressure) {
if (!this.emitEvents) {
if (debug && type !== InputEvents.PointerMove)
console.warn("Pointer events are disabled for this controller", this.index, type, button);
return;
}
// Currently we do only want to emit pointer events for NON screen based events
// that means if the input device is spatial (AR touch on a screen should be handled via touchdown etc still)
// Not sure if *this* is enough to determine if the event is spatial or not
if (this.xr.mode === "immersive-vr" || this.xr.isPassThrough) {
this.pointerInit.origin = this;
this.pointerInit.pointerId = this.getPointerId(button);
this.pointerInit.pointerType = this.hand ? "hand" : "controller";
this.pointerInit.button = button;
this.pointerInit.buttonName = buttonName;
this.pointerInit.isPrimary = primary;
this.pointerInit.mode = this.inputSource.targetRayMode;
this.pointerInit.ray = this.ray;
this.pointerInit.device = this.object;
this.pointerInit.pressure = pressure;
this.pointerInit.clientX = this._rayPosition.x / this.xr.rigScale;
this.pointerInit.clientY = this._rayPosition.y / this.xr.rigScale;
this.pointerInit.clientZ = this._rayPosition.z / this.xr.rigScale;
const prevContext = Context.Current;
Context.Current = this.xr.context;
if (debug && type !== "pointermove")
console.warn("Pointer event", type, button, buttonName, { ...this.pointerInit });
this.xr.context.input.createInputEvent(new NEPointerEvent(type, source, this.pointerInit));
Context.Current = prevContext;
}
}
}
class InputState {
/** if the button was pressed the last update */
isDown = false;
/** if the button was released the last update */
isUp = false;
pressed = false;
value = 0;
}
;
/** Enhanced GamepadButton with `isDown` an