@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
519 lines • 21.8 kB
JavaScript
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
import { AdditiveBlending, DoubleSide, Line3, Mesh, MeshBasicMaterial, Plane, SphereGeometry, Vector3 } from "three";
import { Line2 } from "three/examples/jsm/lines/Line2.js";
import { LineGeometry } from "three/examples/jsm/lines/LineGeometry.js";
import { LineMaterial } from "three/examples/jsm/lines/LineMaterial.js";
import { GroundedSkybox } from "three/examples/jsm/objects/GroundedSkybox.js";
import { isDevEnvironment } from "../../../engine/debug/index.js";
import { Gizmos } from "../../../engine/engine_gizmos.js";
import { Mathf } from "../../../engine/engine_math.js";
import { serializable } from "../../../engine/engine_serialization.js";
import { getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale } from "../../../engine/engine_three_utils.js";
import { getParam } from "../../../engine/engine_utils.js";
import { Behaviour, GameObject } from "../../Component.js";
import { hasPointerEventComponent } from "../../ui/PointerEvents.js";
import { TeleportTarget } from "../TeleportTarget.js";
const debug = getParam("debugwebxr");
/**
* XRControllerMovement is a component that allows to move the XR rig using the XR controller input.
*
* It supports movement using the left controller's thumbstick and rotation using the right controller's thumbstick.
*
* Additionally it supports teleporting using the right controller's thumbstick or by pinching the index finger tip in front of the hand (if hand tracking is enabled).
* It also visualizes controller rays and hit points in the scene.
*
*
* @summary Move the XR rig using controller input
* @category XR
* @group Components
*/
export class XRControllerMovement extends Behaviour {
/** Movement speed in meters per second
* @default 1.5
*/
movementSpeed = 1.5;
/** How many degrees to rotate the XR rig when using the rotation trigger
* @default 30
*/
rotationStep = 30;
/** When enabled you can teleport using the right XR controller's thumbstick by pressing forward
* @default true
*/
useTeleport = true;
/**
* When enabled you can teleport by pinching the right XR controller's index finger tip in front of the hand
* @default true
*/
usePinchToTeleport = true;
/** Enable to only allow teleporting on objects with a TeleportTarget component (see {@link TeleportTarget})
* @default false
*/
useTeleportTarget = false;
/** Enable to fade out the scene when teleporting
* @default false
*/
useTeleportFade = false;
/** enable to visualize controller rays in the 3D scene
* @default true
*/
showRays = true;
/** enable to visualize pointer targets in the 3D scene
* @default false
*/
showHits = true;
isXRMovementHandler = true;
xrSessionMode = "immersive-vr";
_didApplyRotation = false;
_didTeleport = false;
onUpdateXR(args) {
const rig = args.xr.rig;
if (!rig?.gameObject)
return;
// in AR pass through mode we dont want to move the rig
if (args.xr.isPassThrough) {
return;
}
const movementController = args.xr.leftController;
const teleportController = args.xr.rightController;
if (movementController)
this.onHandleMovement(movementController, rig.gameObject);
if (teleportController) {
this.onHandleRotation(teleportController, rig.gameObject);
if (this.useTeleport)
this.onHandleTeleport(teleportController, rig.gameObject);
}
}
onLeaveXR(_) {
for (const line of this._lines) {
line.removeFromParent();
}
for (const disc of this._hitDiscs) {
disc?.removeFromParent();
}
}
onBeforeRender() {
if (this.context.xr?.running) {
if (this.showRays)
this.renderRays(this.context.xr);
if (this.showHits)
this.renderHits(this.context.xr);
}
}
onHandleMovement(controller, rig) {
const stick = controller.getStick("xr-standard-thumbstick");
if (stick.x != 0 || stick.y != 0) {
const vec = getTempVector(stick.x, 0, stick.y);
vec.multiplyScalar(this.context.time.deltaTimeUnscaled * this.movementSpeed);
const scale = getWorldScale(rig);
vec.multiplyScalar(scale.x);
vec.applyQuaternion(controller.xr.poseOrientation);
vec.y = 0;
vec.applyQuaternion(rig.worldQuaternion);
if (isDevEnvironment() && Number.isNaN(vec.x)) {
console.error("Stick movement resulted in NaN", { stick, vec });
}
rig.position.add(vec);
// if we dont do this here the XRControllerModel will be frame-delayed
// maybe we need to introduce a priority order for XR components
// TODO: would be better if this script would just run at the beginning of the frame
rig.updateWorldMatrix(false, false);
for (const ch of rig.children)
ch.updateWorldMatrix(false, false);
}
}
onHandleRotation(controller, rig) {
// WORKAROUND for QuestOS v69 and less where data from MX Ink comes as thumbstick motion
if (controller["_isMxInk"])
return;
const stick = controller.getStick("xr-standard-thumbstick");
const rotationInput = stick.x;
if (this._didApplyRotation) {
if (Math.abs(rotationInput) < .3) {
this._didApplyRotation = false;
}
}
else if (Math.abs(rotationInput) > .5) {
this._didApplyRotation = true;
const dir = rotationInput > 0 ? 1 : -1;
// store user worldpos
const start_worldpos = getWorldPosition(this.context.mainCamera).clone();
rig.rotateY(dir * Mathf.toRadians(this.rotationStep));
// apply user offset so we rotate around the user
const end_worldpos = getWorldPosition(this.context.mainCamera).clone();
const diff = end_worldpos.sub(start_worldpos);
diff.y = 0;
rig.position.sub(diff);
}
}
_teleportBuffer = new Array();
onHandleTeleport(controller, rig) {
let teleportInput = 0;
if (controller.hand && this.usePinchToTeleport && controller.isTeleportGesture) {
// prevent pinch teleport while the primary input is in use
const pointerId = controller.getPointerId("primary");
if (pointerId != undefined && this.context.input.getIsPointerIdInUse(pointerId)) {
return;
}
const pinch = controller.getGesture("pinch");
if (pinch) {
teleportInput = pinch.value;
}
}
else {
teleportInput = controller.getStick("xr-standard-thumbstick")?.y;
}
if (this._didTeleport) {
if (teleportInput >= 0 && teleportInput < .4) {
this._didTeleport = false;
}
else if (teleportInput < 0 && teleportInput > -.4) {
this._didTeleport = false;
}
}
else if (teleportInput > .8) {
this._didTeleport = true;
const hit = this.context.physics.raycastFromRay(controller.ray)[0];
if (hit && hit.object instanceof GroundedSkybox) {
const dot_up = hit.normal?.dot(getTempVector(0, 1, 0));
// Make sure we can only teleport on the ground / floor plane
if (dot_up !== undefined && dot_up < 0.4) {
return;
}
}
let point = hit?.point;
// If we didnt hit an object in the scene use the ground plane
if (!point && !this.useTeleportTarget) {
if (!this._plane) {
this._plane = new Plane(new Vector3(0, 1, 0), 0);
}
const currentPosition = rig.worldPosition;
this._plane.setFromNormalAndCoplanarPoint(new Vector3(0, 1, 0), currentPosition);
const ray = controller.ray;
point = currentPosition.clone();
this._plane.intersectLine(new Line3(ray.origin, getTempVector(ray.direction).multiplyScalar(10000).add(ray.origin)), point);
if (point.distanceTo(currentPosition) > rig.scale.x * 10) {
point = null;
}
}
if (point) {
if (this.useTeleportTarget) {
const teleportTarget = GameObject.getComponentInParent(hit.object, TeleportTarget);
if (!teleportTarget)
return;
}
const cloned = point.clone();
if (debug)
Gizmos.DrawSphere(point, .025, 0xff0000, 5);
// remove user XR rig position
const positionInRig = this.context.mainCamera?.position;
if (positionInRig) {
const vec = this.context.xr?.getUserOffsetInRig();
if (vec) {
vec.y = 0;
cloned.sub(vec);
if (debug)
Gizmos.DrawWireSphere(vec.add(cloned), .025, 0x00ff00, 5);
}
}
this._teleportBuffer.push(rig.matrix.clone());
if (this._teleportBuffer.length > 10) {
this._teleportBuffer.shift();
}
if (this.useTeleportFade) {
controller.xr.fadeTransition()?.then(() => {
rig.worldPosition = cloned;
});
}
else {
rig.worldPosition = cloned;
}
}
}
else if (teleportInput < -.8) {
this._didTeleport = true;
if (this._teleportBuffer.length > 0) {
// get latest teleport position
const prev = this._teleportBuffer.pop();
if (prev) {
prev.decompose(rig.position, rig.quaternion, rig.scale);
}
}
}
}
_plane = null;
_lines = [];
_hitDiscs = [];
_hitDistances = [];
_lastHitDistances = [];
renderRays(session) {
for (let i = 0; i < this._lines.length; i++) {
const line = this._lines[i];
if (line)
line.visible = false;
}
for (let i = 0; i < session.controllers.length; i++) {
const ctrl = session.controllers[i];
let line = this._lines[i];
if (!ctrl.connected || !ctrl.isTracking ||
!ctrl.ray || ctrl.targetRayMode === "transient-pointer" ||
!ctrl.hasSelectEvent) {
if (line)
line.visible = false;
continue;
}
if (!line) {
line = this.createRayLineObject();
line.scale.z = .5;
this._lines[i] = line;
}
ctrl.updateRayWorldPosition();
ctrl.updateRayWorldQuaternion();
const pos = ctrl.rayWorldPosition;
const rot = ctrl.rayWorldQuaternion;
line.position.copy(pos);
line.quaternion.copy(rot);
const scale = session.rigScale;
const forceShowRay = this.usePinchToTeleport && ctrl.isTeleportGesture;
const distance = this._lastHitDistances[i];
const hasHit = this._hitDistances[i] != null;
const dist = distance != null ? distance : scale;
line.scale.set(scale, scale, dist);
line.visible = true;
line.layers.disableAll();
line.layers.enable(2);
let targetOpacity = line.material.opacity;
if (forceShowRay) {
targetOpacity = 1;
}
else if (this.showHits && dist < session.rigScale * 0.5) {
targetOpacity = 0;
}
else if (ctrl.getButton("primary")?.pressed) {
targetOpacity = .5;
}
else {
targetOpacity = hasHit ? .2 : .1;
}
line.material.opacity = Mathf.lerp(line.material.opacity, targetOpacity, this.context.time.deltaTimeUnscaled / .1);
if (line.parent !== this.context.scene)
this.context.scene.add(line);
}
}
renderHits(session) {
for (const disc of this._hitDiscs) {
if (!disc)
continue;
const ctrl = disc["controller"];
if (!ctrl || !ctrl.connected || !ctrl.isTracking) {
disc.visible = false;
continue;
}
}
for (let i = 0; i < session.controllers.length; i++) {
const ctrl = session.controllers[i];
if (!ctrl.connected || !ctrl.isTracking || !ctrl.ray || !ctrl.hasSelectEvent)
continue;
let disc = this._hitDiscs[i];
let runRaycast = true;
// Check if the primary input button is in use
const pointerId = ctrl.getPointerId("primary");
if (pointerId != undefined) {
const isCurrentlyUsed = this.context.input.getIsPointerIdInUse(pointerId);
// if the input is being used then we hide the ray
if (isCurrentlyUsed) {
if (disc)
disc.visible = false;
this._hitDistances[i] = null;
this._lastHitDistances[i] = 0;
runRaycast = false;
}
}
// save performance by only raycasting every nth frame
const interval = this.context.time.smoothedFps >= 59 ? 1 : 10;
if ((this.context.time.frame + ctrl.index) % interval !== 0) {
runRaycast = false;
}
if (!runRaycast) {
const disc = this._hitDiscs[i];
// if the disc had a hit last frame, we can update it here
if (disc && disc.visible && disc["hit"]) {
this.updateHitPointerPosition(ctrl, disc, disc["hit"].distance);
}
continue;
}
const hits = this.context.physics.raycastFromRay(ctrl.ray, { testObject: this.hitPointRaycastFilter, precise: false });
let hit = hits.find(hit => {
if (this.usePinchToTeleport && ctrl.isTeleportGesture)
return true;
// Only render hits on interactable objects
return this.isObjectWithInteractiveComponent(hit.object);
});
// Fallback to use the first hit
if (!hit) {
hit = hits[0];
}
if (disc) // save the hit object on the disc
{
disc["controller"] = ctrl;
disc["hit"] = hit;
}
this._hitDistances[i] = hit?.distance || null;
if (hit) {
this._lastHitDistances[i] = hit.distance;
const rigScale = (session.rigScale ?? 1);
if (debug) {
Gizmos.DrawWireSphere(hit.point, .025 * rigScale, 0xff0000);
Gizmos.DrawLabel(getTempVector(0, .2, 0).add(hit.point), hit.object.name, .02, 0);
}
if (!disc) {
disc = this.createHitPointObject();
this._hitDiscs[i] = disc;
}
disc["hit"] = hit;
// hide the disc if the hit point is too close
disc.visible = hit.distance > rigScale * 0.05;
let size = (.01 * (rigScale + hit.distance));
const primaryPressed = ctrl.getButton("primary")?.pressed;
if (primaryPressed)
size *= 1.1;
disc.scale.set(size, size, size);
disc.layers.set(2);
let targetOpacity = disc.material.opacity;
if (primaryPressed) {
targetOpacity = 1;
}
else {
targetOpacity = hit.distance < .15 * rigScale ? .2 : .6;
}
disc.material.opacity = Mathf.lerp(disc.material.opacity, targetOpacity, this.context.time.deltaTimeUnscaled / .1);
if (disc.visible) {
if (hit.normal) {
this.updateHitPointerPosition(ctrl, disc, hit.distance);
const worldNormal = hit.normal.applyQuaternion(getWorldQuaternion(hit.object));
disc.quaternion.setFromUnitVectors(up, worldNormal);
}
else {
this.updateHitPointerPosition(ctrl, disc, hit.distance);
}
if (disc.parent !== this.context.scene) {
this.context.scene.add(disc);
}
}
}
else {
if (this._hitDiscs[i]) {
this._hitDiscs[i].visible = false;
}
}
}
}
isObjectWithInteractiveComponent(object, level = 0) {
if (hasPointerEventComponent(object) || (object["isUI"] === true))
return true;
if (object.isScene)
return false;
if (object.parent)
return this.isObjectWithInteractiveComponent(object.parent, level + 1);
return false;
}
updateHitPointerPosition(ctrl, pt, distance) {
const targetPos = getTempVector(ctrl.rayWorldPosition);
targetPos.add(getTempVector(0, 0, distance - .01).applyQuaternion(ctrl.rayWorldQuaternion));
pt.position.lerp(targetPos, this.context.time.deltaTimeUnscaled / .05);
}
hitPointRaycastFilter = (obj) => {
// by default dont raycast ont skinned meshes using the hit point raycast (because it is a big performance hit and only a visual indicator)
if (obj.type === "SkinnedMesh")
return "continue in children";
return true;
};
/** create an object to visualize hit points in the scene */
createHitPointObject() {
// var container = new Object3D();
const mesh = new Mesh(new SphereGeometry(.3, 6, 6), // new RingGeometry(.3, 0.5, 32).rotateX(- Math.PI / 2),
new MeshBasicMaterial({
color: 0xeeeeee,
opacity: .7,
transparent: true,
depthTest: false,
depthWrite: false,
side: DoubleSide,
}));
mesh.layers.disableAll();
mesh.layers.enable(2);
// container.add(disc);
// const disc2 = new Mesh(
// new RingGeometry(.43, 0.5, 32).rotateX(- Math.PI / 2),
// new MeshBasicMaterial({
// color: 0x000000,
// opacity: .2,
// transparent: true,
// side: DoubleSide,
// })
// );
// disc2.layers.disableAll();
// disc2.layers.enable(2);
// disc2.position.y = .01;
// container.add(disc2);
return mesh;
}
/** create an object to visualize controller rays */
createRayLineObject() {
const line = new Line2();
line.layers.disableAll();
line.layers.enable(2);
const geometry = new LineGeometry();
line.geometry = geometry;
const positions = new Float32Array(9);
positions.set([0, 0, .02, 0, 0, .4, 0, 0, 1]);
geometry.setPositions(positions);
const colors = new Float32Array(9);
colors.set([1, 1, 1, .1, .1, .1, 0, 0, 0]);
geometry.setColors(colors);
const mat = new LineMaterial({
color: 0xffffff,
vertexColors: true,
worldUnits: true,
linewidth: .004,
transparent: true,
depthWrite: false,
// TODO: this doesnt work with passthrough
blending: AdditiveBlending,
dashed: false,
// alphaToCoverage: true,
});
line.material = mat;
return line;
}
}
__decorate([
serializable()
], XRControllerMovement.prototype, "movementSpeed", void 0);
__decorate([
serializable()
], XRControllerMovement.prototype, "rotationStep", void 0);
__decorate([
serializable()
], XRControllerMovement.prototype, "useTeleport", void 0);
__decorate([
serializable()
], XRControllerMovement.prototype, "usePinchToTeleport", void 0);
__decorate([
serializable()
], XRControllerMovement.prototype, "useTeleportTarget", void 0);
__decorate([
serializable()
], XRControllerMovement.prototype, "useTeleportFade", void 0);
__decorate([
serializable()
], XRControllerMovement.prototype, "showRays", void 0);
__decorate([
serializable()
], XRControllerMovement.prototype, "showHits", void 0);
const up = new Vector3(0, 1, 0);
//# sourceMappingURL=XRControllerMovement.js.map