@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
551 lines (475 loc) • 21.6 kB
text/typescript
import { AdditiveBlending, BufferAttribute, Color, DoubleSide, Intersection, Line, Line3, Material, Matrix4, Mesh, MeshBasicMaterial, Object3D, Plane, RingGeometry, Scene, SphereGeometry, SubtractiveBlending, Vector3 } from "three";
import { Line2 } from "three/examples/jsm/lines/Line2.js";
import { LineGeometry } from "three/examples/jsm/lines/LineGeometry.js";
import { LineMaterial } from "three/examples/jsm/lines/LineMaterial.js";
import { GroundedSkybox } from "three/examples/jsm/objects/GroundedSkybox.js";
import { Gizmos } from "../../../engine/engine_gizmos.js";
import { Mathf } from "../../../engine/engine_math.js";
import type { RaycastTestObjectCallback } from "../../../engine/engine_physics.js";
import { serializable } from "../../../engine/engine_serialization.js"
import { getTempQuaternion, getTempVector, getWorldPosition, getWorldQuaternion, getWorldScale } from "../../../engine/engine_three_utils.js";
import type { IGameObject } from "../../../engine/engine_types.js";
import { getParam } from "../../../engine/engine_utils.js";
import { NeedleXRController, type NeedleXREventArgs, NeedleXRSession } from "../../../engine/engine_xr.js";
import { Behaviour, GameObject } from "../../Component.js"
import { hasPointerEventComponent } from "../../ui/PointerEvents.js";
import { TeleportTarget } from "../TeleportTarget.js";
import type { XRMovementBehaviour } from "../types.js";
const debug = getParam("debugwebxr");
declare type HitPointObject = Object3D & { material: Material & { opacity: number } }
/**
* XRControllerMovement is a component that allows to move the XR rig using the XR controller input.
* @category XR
* @group Components
*/
export class XRControllerMovement extends Behaviour implements XRMovementBehaviour {
/** Movement speed in meters per second
* @default 1.5
*/
movementSpeed = 1.5;
/** How many degrees to rotate the XR rig when using the rotation trigger
* @default 30
*/
rotationStep = 30;
/** When enabled you can teleport using the right XR controller's thumbstick by pressing forward
* @default true
*/
useTeleport: boolean = true;
/**
* When enabled you can teleport by pinching the right XR controller's index finger tip in front of the hand
* @default true
*/
usePinchToTeleport: boolean = true;
/** Enable to only allow teleporting on objects with a teleport target component
* @default false
*/
useTeleportTarget = false;
/** Enable to fade out the scene when teleporting
* @default false
*/
useTeleportFade = false;
/** enable to visualize controller rays in the 3D scene
* @default true
*/
showRays: boolean = true;
/** enable to visualize pointer targets in the 3D scene
* @default false
*/
showHits: boolean = true;
readonly isXRMovementHandler: true = true;
readonly xrSessionMode = "immersive-vr";
private _didApplyRotation = false;
private _didTeleport = false;
onUpdateXR(args: NeedleXREventArgs): void {
const rig = args.xr.rig;
if (!rig?.gameObject) return;
// in AR pass through mode we dont want to move the rig
if (args.xr.isPassThrough) {
return;
}
const movementController = args.xr.leftController;
const teleportController = args.xr.rightController;
if (movementController)
this.onHandleMovement(movementController, rig.gameObject);
if (teleportController) {
this.onHandleRotation(teleportController, rig.gameObject);
if (this.useTeleport)
this.onHandleTeleport(teleportController, rig.gameObject);
}
}
onLeaveXR(_: NeedleXREventArgs): void {
for (const line of this._lines) {
line.removeFromParent();
}
for (const disc of this._hitDiscs) {
disc?.removeFromParent();
}
}
onBeforeRender(): void {
if (this.context.xr?.running) {
if (this.showRays)
this.renderRays(this.context.xr);
if (this.showHits)
this.renderHits(this.context.xr);
}
}
protected onHandleMovement(controller: NeedleXRController, rig: IGameObject) {
const stick = controller.getStick("xr-standard-thumbstick");
if (stick.x != 0 || stick.y != 0) {
const vec = getTempVector(stick.x, 0, stick.y);
vec.multiplyScalar(this.context.time.deltaTimeUnscaled * this.movementSpeed);
const scale = getWorldScale(rig);
vec.multiplyScalar(scale.x);
vec.applyQuaternion(controller.xr.poseOrientation);
vec.y = 0;
vec.applyQuaternion(rig.worldQuaternion);
rig.position.add(vec);
// if we dont do this here the XRControllerModel will be frame-delayed
// maybe we need to introduce a priority order for XR components
// TODO: would be better if this script would just run at the beginning of the frame
rig.updateWorldMatrix(false, false);
for (const ch of rig.children) ch.updateWorldMatrix(false, false);
}
}
protected onHandleRotation(controller: NeedleXRController, rig: IGameObject) {
// WORKAROUND for QuestOS v69 and less where data from MX Ink comes as thumbstick motion
if (controller["_isMxInk"]) return;
const stick = controller.getStick("xr-standard-thumbstick");
const rotationInput = stick.x;
if (this._didApplyRotation) {
if (Math.abs(rotationInput) < .3) {
this._didApplyRotation = false;
}
}
else if (Math.abs(rotationInput) > .5) {
this._didApplyRotation = true;
const dir = rotationInput > 0 ? 1 : -1;
// store user worldpos
const start_worldpos = getWorldPosition(this.context.mainCamera!).clone();
rig.rotateY(dir * Mathf.toRadians(this.rotationStep));
// apply user offset so we rotate around the user
const end_worldpos = getWorldPosition(this.context.mainCamera!).clone();
const diff = end_worldpos.sub(start_worldpos);
diff.y = 0;
rig.position.sub(diff);
}
}
private readonly _teleportBuffer = new Array<Matrix4>();
protected onHandleTeleport(controller: NeedleXRController, rig: IGameObject) {
let teleportInput = 0;
if (controller.hand && this.usePinchToTeleport && controller.isTeleportGesture) {
// prevent pinch teleport while the primary input is in use
const pointerId = controller.getPointerId("primary");
if (pointerId != undefined && this.context.input.getIsPointerIdInUse(pointerId)) {
return;
}
const pinch = controller.getGesture("pinch");
if (pinch) {
teleportInput = pinch.value;
}
}
else {
teleportInput = controller.getStick("xr-standard-thumbstick")?.y;
}
if (this._didTeleport) {
if (teleportInput >= 0 && teleportInput < .4) {
this._didTeleport = false;
}
else if (teleportInput < 0 && teleportInput > -.4) {
this._didTeleport = false;
}
}
else if (teleportInput > .8) {
this._didTeleport = true;
const hit = this.context.physics.raycastFromRay(controller.ray)[0];
if (hit && hit.object instanceof GroundedSkybox) {
const dot_up = hit.normal?.dot(getTempVector(0, 1, 0));
// Make sure we can only teleport on the ground / floor plane
if (dot_up !== undefined && dot_up < 0.4) {
return;
}
}
let point: Vector3 | null = hit?.point;
// If we didnt hit an object in the scene use the ground plane
if (!point && !this.useTeleportTarget) {
if (!this._plane) {
this._plane = new Plane(new Vector3(0, 1, 0), 0);
}
const currentPosition = rig.worldPosition;
this._plane.setFromNormalAndCoplanarPoint(new Vector3(0, 1, 0), currentPosition);
const ray = controller.ray;
point = currentPosition.clone();
this._plane.intersectLine(new Line3(ray.origin, getTempVector(ray.direction).multiplyScalar(10000).add(ray.origin)), point);
if (point.distanceTo(currentPosition) > rig.scale.x * 10) {
point = null;
}
}
if (point) {
if (this.useTeleportTarget) {
const teleportTarget = GameObject.getComponentInParent(hit.object, TeleportTarget);
if (!teleportTarget) return;
}
const cloned = point.clone();
if (debug) Gizmos.DrawSphere(point, .025, 0xff0000, 5);
// remove user XR rig position
const positionInRig = this.context.mainCamera?.position;
if (positionInRig) {
const vec = this.context.xr?.getUserOffsetInRig();
if (vec) {
vec.y = 0;
cloned.sub(vec);
if (debug) Gizmos.DrawWireSphere(vec.add(cloned), .025, 0x00ff00, 5);
}
}
this._teleportBuffer.push(rig.matrix.clone());
if (this._teleportBuffer.length > 10) {
this._teleportBuffer.shift();
}
if (this.useTeleportFade) {
controller.xr.fadeTransition()?.then(() => {
rig.worldPosition = cloned;
})
}
else {
rig.worldPosition = cloned;
}
}
}
else if (teleportInput < -.8) {
this._didTeleport = true;
if (this._teleportBuffer.length > 0) {
// get latest teleport position
const prev = this._teleportBuffer.pop();
if (prev) {
prev.decompose(rig.position, rig.quaternion, rig.scale)
}
}
}
}
private _plane: Plane | null = null;
private readonly _lines: Line2[] = [];
private readonly _hitDiscs: HitPointObject[] = [];
private readonly _hitDistances: Array<number | null> = [];
private readonly _lastHitDistances: Array<number | null | undefined> = [];
protected renderRays(session: NeedleXRSession) {
for (let i = 0; i < this._lines.length; i++) {
const line = this._lines[i];
if (line) line.visible = false;
}
for (let i = 0; i < session.controllers.length; i++) {
const ctrl = session.controllers[i];
let line = this._lines[i];
if (!ctrl.connected || !ctrl.isTracking ||
!ctrl.ray || ctrl.targetRayMode === "transient-pointer" ||
!ctrl.hasSelectEvent
) {
if (line) line.visible = false;
continue;
}
if (!line) {
line = this.createRayLineObject();
line.scale.z = .5;
this._lines[i] = line;
}
ctrl.updateRayWorldPosition();
ctrl.updateRayWorldQuaternion();
const pos = ctrl.rayWorldPosition;
const rot = ctrl.rayWorldQuaternion;
line.position.copy(pos);
line.quaternion.copy(rot);
const scale = session.rigScale;
const forceShowRay = this.usePinchToTeleport && ctrl.isTeleportGesture;
const distance = this._lastHitDistances[i];
const hasHit = this._hitDistances[i] != null;
const dist = distance != null ? distance : scale;
line.scale.set(scale, scale, dist);
line.visible = true;
line.layers.disableAll();
line.layers.enable(2);
let targetOpacity = line.material.opacity;
if (forceShowRay) {
targetOpacity = 1;
}
else if (this.showHits && dist < session.rigScale * 0.5) {
targetOpacity = 0;
}
else if (ctrl.getButton("primary")?.pressed) {
targetOpacity = .5;
}
else {
targetOpacity = hasHit ? .2 : .1;
}
line.material.opacity = Mathf.lerp(line.material.opacity, targetOpacity, this.context.time.deltaTimeUnscaled / .1);
if (line.parent !== this.context.scene)
this.context.scene.add(line);
}
}
protected renderHits(session: NeedleXRSession) {
for (const disc of this._hitDiscs) {
if (!disc) continue;
const ctrl = disc["controller"];
if (!ctrl || !ctrl.connected || !ctrl.isTracking) {
disc.visible = false;
continue;
}
}
for (let i = 0; i < session.controllers.length; i++) {
const ctrl = session.controllers[i];
if (!ctrl.connected || !ctrl.isTracking || !ctrl.ray || !ctrl.hasSelectEvent) continue;
let disc = this._hitDiscs[i];
let runRaycast = true;
// Check if the primary input button is in use
const pointerId: number | undefined = ctrl.getPointerId("primary");
if (pointerId != undefined) {
const isCurrentlyUsed = this.context.input.getIsPointerIdInUse(pointerId);
// if the input is being used then we hide the ray
if (isCurrentlyUsed) {
if (disc) disc.visible = false;
this._hitDistances[i] = null;
this._lastHitDistances[i] = 0;
runRaycast = false;
}
}
// save performance by only raycasting every nth frame
const interval = this.context.time.smoothedFps >= 59 ? 1 : 10;
if ((this.context.time.frame + ctrl.index) % interval !== 0) {
runRaycast = false;
}
if (!runRaycast) {
const disc = this._hitDiscs[i];
// if the disc had a hit last frame, we can update it here
if (disc && disc.visible && disc["hit"]) {
this.updateHitPointerPosition(ctrl, disc, disc["hit"].distance);
}
continue;
}
const hits = this.context.physics.raycastFromRay(ctrl.ray, { testObject: this.hitPointRaycastFilter, precise: false });
let hit = hits.find(hit => {
if (this.usePinchToTeleport && ctrl.isTeleportGesture) return true;
// Only render hits on interactable objects
return this.isObjectWithInteractiveComponent(hit.object);
});
// Fallback to use the first hit
if (!hit) {
hit = hits[0];
}
if (disc) // save the hit object on the disc
{
disc["controller"] = ctrl;
disc["hit"] = hit;
}
this._hitDistances[i] = hit?.distance || null;
if (hit) {
this._lastHitDistances[i] = hit.distance;
const rigScale = (session.rigScale ?? 1);
if (debug) {
Gizmos.DrawWireSphere(hit.point, .025 * rigScale, 0xff0000);
Gizmos.DrawLabel(getTempVector(0, .2, 0).add(hit.point), hit.object.name, .02, 0);
}
if (!disc) {
disc = this.createHitPointObject();
this._hitDiscs[i] = disc;
}
disc["hit"] = hit;
// hide the disc if the hit point is too close
disc.visible = hit.distance > rigScale * 0.05;
let size = (.01 * (rigScale + hit.distance));
const primaryPressed = ctrl.getButton("primary")?.pressed;
if (primaryPressed) size *= 1.1;
disc.scale.set(size, size, size);
disc.layers.set(2);
let targetOpacity = disc.material.opacity;
if (primaryPressed) {
targetOpacity = 1;
}
else {
targetOpacity = hit.distance < .15 * rigScale ? .2 : .6;
}
disc.material.opacity = Mathf.lerp(disc.material.opacity, targetOpacity, this.context.time.deltaTimeUnscaled / .1);
if (disc.visible) {
if (hit.normal) {
this.updateHitPointerPosition(ctrl, disc, hit.distance);
const worldNormal = hit.normal.applyQuaternion(getWorldQuaternion(hit.object));
disc.quaternion.setFromUnitVectors(up, worldNormal);
}
else {
this.updateHitPointerPosition(ctrl, disc, hit.distance);
}
if (disc.parent !== this.context.scene) {
this.context.scene.add(disc);
}
}
}
else {
if (this._hitDiscs[i]) {
this._hitDiscs[i].visible = false;
}
}
}
}
private isObjectWithInteractiveComponent(object: Object3D, level: number = 0) {
if (hasPointerEventComponent(object) || (object["isUI"] === true)) return true;
if ((object as Scene).isScene) return false;
if (object.parent) return this.isObjectWithInteractiveComponent(object.parent, level + 1);
return false;
}
private updateHitPointerPosition(ctrl: NeedleXRController, pt: Object3D, distance: number) {
const targetPos = getTempVector(ctrl.rayWorldPosition);
targetPos.add(getTempVector(0, 0, distance - .01).applyQuaternion(ctrl.rayWorldQuaternion));
pt.position.lerp(targetPos, this.context.time.deltaTimeUnscaled / .05);
}
protected hitPointRaycastFilter: RaycastTestObjectCallback = (obj: Object3D) => {
// by default dont raycast ont skinned meshes using the hit point raycast (because it is a big performance hit and only a visual indicator)
if (obj.type === "SkinnedMesh") return "continue in children";
return true;
}
/** create an object to visualize hit points in the scene */
protected createHitPointObject(): HitPointObject {
// var container = new Object3D();
const mesh = new Mesh(
new SphereGeometry(.3, 6, 6),// new RingGeometry(.3, 0.5, 32).rotateX(- Math.PI / 2),
new MeshBasicMaterial({
color: 0xeeeeee,
opacity: .7,
transparent: true,
depthTest: false,
depthWrite: false,
side: DoubleSide,
})
);
mesh.layers.disableAll();
mesh.layers.enable(2);
// container.add(disc);
// const disc2 = new Mesh(
// new RingGeometry(.43, 0.5, 32).rotateX(- Math.PI / 2),
// new MeshBasicMaterial({
// color: 0x000000,
// opacity: .2,
// transparent: true,
// side: DoubleSide,
// })
// );
// disc2.layers.disableAll();
// disc2.layers.enable(2);
// disc2.position.y = .01;
// container.add(disc2);
return mesh;
}
/** create an object to visualize controller rays */
protected createRayLineObject() {
const line = new Line2();
line.layers.disableAll();
line.layers.enable(2);
const geometry = new LineGeometry();
line.geometry = geometry;
const positions = new Float32Array(9);
positions.set([0, 0, .02, 0, 0, .4, 0, 0, 1]);
geometry.setPositions(positions)
const colors = new Float32Array(9);
colors.set([1, 1, 1, .1, .1, .1, 0, 0, 0]);
geometry.setColors(colors);
const mat = new LineMaterial({
color: 0xffffff,
vertexColors: true,
worldUnits: true,
linewidth: .004,
transparent: true,
depthWrite: false,
// TODO: this doesnt work with passthrough
blending: AdditiveBlending,
dashed: false,
// alphaToCoverage: true,
});
line.material = mat;
return line;
}
}
const up = new Vector3(0, 1, 0);