@needle-tools/engine/src/engine-components/DragControls.ts

Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.

import { AxesHelper, Box3, BufferGeometry, Camera, Color, Line, LineBasicMaterial, Matrix4, Mesh, MeshBasicMaterial, Object3D, Plane, Quaternion, Ray, Raycaster, SphereGeometry, Vector3 } from "three";

import { Gizmos } from "../engine/engine_gizmos.js";
import { InstancingUtil } from "../engine/engine_instancing.js";
import { Mathf } from "../engine/engine_math.js";
import { RaycastOptions } from "../engine/engine_physics.js";
import { serializable } from "../engine/engine_serialization_decorator.js";
import { Context } from "../engine/engine_setup.js";
import { getBoundingBox, getTempVector, getWorldPosition, setWorldPosition } from "../engine/engine_three_utils.js";
import { type IGameObject } from "../engine/engine_types.js";
import { getParam } from "../engine/engine_utils.js";
import { NeedleXRSession } from "../engine/engine_xr.js";
import { Avatar_POI } from "./avatar/Avatar_Brain_LookAt.js";
import { Behaviour, GameObject } from "./Component.js";
import { UsageMarker } from "./Interactable.js";
import { Rigidbody } from "./RigidBody.js";
import { SyncedTransform } from "./SyncedTransform.js";
import type { IPointerEventHandler, PointerEventData } from "./ui/PointerEvents.js";
import { ObjectRaycaster } from "./ui/Raycaster.js";

/** Enable debug visualization and logging for DragControls by using the URL parameter `?debugdrag`. */
const debug = getParam("debugdrag");

/** Buffer to store currently active DragControls components */
const dragControlsBuffer: DragControls[] = [];

/**
 * The DragMode determines how an object is dragged around in the scene.
 */
export enum DragMode {
    /** Object stays at the same horizontal plane as it started. Commonly used for objects on the floor */
    XZPlane = 0,
    /** Object is dragged as if it was attached to the pointer. In 2D, that means it's dragged along the camera screen plane. In XR, it's dragged by the controller/hand. */
    Attached = 1,
    /** Object is dragged along the initial raycast hit normal. */
    HitNormal = 2,
    /** Combination of XZ and Screen based on the viewing angle. Low angles result in Screen dragging and higher angles in XZ dragging. */
    DynamicViewAngle = 3,
    /** The drag plane is snapped to surfaces in the scene while dragging. */
    SnapToSurfaces = 4,
    /** Don't allow dragging the object */
    None = 5,
}

/**
 * DragControls allows you to drag objects around in the scene. It can be used to move objects in 2D (screen space) or 3D (world space).
 * Debug mode can be enabled with the URL parameter `?debugdrag`, which shows visual helpers and logs drag operations.
 * 
 * @category Interactivity
 * @group Components
 */
export class DragControls extends Behaviour implements IPointerEventHandler {

    /**
     * Checks if any DragControls component is currently active with selected objects
     * @returns True if any DragControls component is currently active
     */
    public static get HasAnySelected(): boolean { return this._active > 0; }
    private static _active: number = 0;

    /** 
     * Retrieves a list of all DragControl components that are currently dragging objects.
     * @returns Array of currently active DragControls components
     */
    public static get CurrentlySelected() {
        dragControlsBuffer.length = 0;
        for (const dc of this._instances) {
            if (dc._isDragging) {
                dragControlsBuffer.push(dc);
            }
        }
        return dragControlsBuffer;
    }
    /** Registry of currently active and enabled DragControls components */
    private static _instances: DragControls[] = [];

    /** 
     * Determines how and where the object is dragged along. Different modes include
     * dragging along a plane, attached to the pointer, or following surface normals.
     */
    @serializable()
    public dragMode: DragMode = DragMode.DynamicViewAngle;

    /** 
     * Snaps dragged objects to a 3D grid with the specified resolution.
     * Set to 0 to disable snapping.
     */
    @serializable()
    public snapGridResolution: number = 0.0;

    /** 
     * When true, maintains the original rotation of the dragged object while moving it.
     * When false, allows the object to rotate freely during dragging.
     */
    @serializable()
    public keepRotation: boolean = true;

    /** 
     * Determines how and where the object is dragged along while dragging in XR.
     * Uses a separate setting from regular drag mode for better XR interaction.
     */
    @serializable()
    public xrDragMode: DragMode = DragMode.Attached;

    /** 
     * When true, maintains the original rotation of the dragged object during XR dragging.
     * When false, allows the object to rotate freely during XR dragging.
     */
    @serializable()
    public xrKeepRotation: boolean = false;

    /** 
     * Multiplier that affects how quickly objects move closer or further away when dragging in XR.
     * Higher values make distance changes more pronounced.
     * This is similar to mouse acceleration on a screen.
     */
    @serializable()
    public xrDistanceDragFactor: number = 1;

    /** 
     * When enabled, draws a visual line from the dragged object downwards to the next raycast hit,
     * providing visual feedback about the object's position relative to surfaces below it.
     */
    @serializable()
    public showGizmo: boolean = false;

    /** 
     * Returns the object currently being dragged by this DragControls component, if any.
     * @returns The object being dragged or null if no object is currently dragged
     */
    get draggedObject() {
        return this._targetObject;
    }

    /**
     * Updates the object that is being dragged by the DragControls.
     * This can be used to change the target during a drag operation.
     * @param obj The new object to drag, or null to stop dragging
     */
    setTargetObject(obj: Object3D | null) {
        this._targetObject = obj;
        for (const handler of this._dragHandlers.values()) {
            handler.setTargetObject(obj);
        }

        // If the object was kinematic we want to reset it
        const wasKinematicKey = "_rigidbody-was-kinematic";
        if (this._rigidbody?.[wasKinematicKey] === false) {
            this._rigidbody.isKinematic = false;
            this._rigidbody[wasKinematicKey] = undefined;

        }

        this._rigidbody = null;
        // If we have a object that is being dragged we want to get the Rigidbody component 
        // and we set kinematic to false while it's being dragged
        if (obj) {
            this._rigidbody = GameObject.getComponentInChildren(obj, Rigidbody);
            if (this._rigidbody?.isKinematic === false) {
                this._rigidbody.isKinematic = true;
                this._rigidbody[wasKinematicKey] = false;
            }
        }
    }
    
    private _rigidbody: Rigidbody | null = null;

    // future:
    // constraints?


    /** The object to be dragged – we pass this to handlers when they are created */
    private _targetObject: Object3D | null = null;
    private _dragHelper: LegacyDragVisualsHelper | null = null;
    private static lastHovered: Object3D;
    private _draggingRigidbodies: Rigidbody[] = [];
    private _potentialDragStartEvt: PointerEventData | null = null;
    private _dragHandlers: Map<Object3D, IDragHandler> = new Map();
    private _totalMovement: Vector3 = new Vector3();
    /** A marker is attached to components that are currently interacted with, to e.g. prevent them from being deleted. */
    private _marker: UsageMarker | null = null;
    private _isDragging: boolean = false;
    private _didDrag: boolean = false;

    /** @internal */
    awake() {
        // initialize all data that may be cloned incorrectly otherwise
        this._potentialDragStartEvt = null;
        this._dragHandlers = new Map();
        this._totalMovement = new Vector3();
        this._marker = null;
        this._isDragging = false;
        this._didDrag = false;
        this._dragHelper = null;
        this._draggingRigidbodies = [];
    }

    /** @internal */
    start() {
        if (!this.gameObject.getComponentInParent(ObjectRaycaster))
            this.gameObject.addComponent(ObjectRaycaster);
    }

    /** @internal */
    onEnable(): void {
        DragControls._instances.push(this);
    }
    /** @internal */
    onDisable(): void {
        DragControls._instances = DragControls._instances.filter(i => i !== this);
    }

    /**
     * Checks if editing is allowed for the current networking connection.
     * @param _obj Optional object to check edit permissions for
     * @returns True if editing is allowed
     */
    private allowEdit(_obj: Object3D | null = null) {
        return this.context.connection.allowEditing;
    }

    /** 
     * Handles pointer enter events. Sets the cursor style and tracks the hovered object.
     * @param evt Pointer event data containing information about the interaction
     * @internal
     */
    onPointerEnter?(evt: PointerEventData) {
        if (!this.allowEdit(this.gameObject)) return;
        if (evt.mode !== "screen") return;

        // get the drag mode and check if we need to abort early here
        const isSpatialInput = evt.event.mode === "tracked-pointer" || evt.event.mode === "transient-pointer";
        const dragMode = isSpatialInput ? this.xrDragMode : this.dragMode;
        if (dragMode === DragMode.None) return;

        const dc = GameObject.getComponentInParent(evt.object, DragControls);
        if (!dc || dc !== this) return;
        DragControls.lastHovered = evt.object;
        this.context.domElement.style.cursor = 'pointer';
    }

    /** 
     * Handles pointer movement events. Marks the event as used if dragging is active.
     * @param args Pointer event data containing information about the movement
     * @internal
     */
    onPointerMove?(args: PointerEventData) {
        if (this._isDragging || this._potentialDragStartEvt !== null) args.use();
    }

    /** 
     * Handles pointer exit events. Resets the cursor style when the pointer leaves a draggable object.
     * @param evt Pointer event data containing information about the interaction
     * @internal
     */
    onPointerExit?(evt: PointerEventData) {
        if (!this.allowEdit(this.gameObject)) return;
        if (evt.mode !== "screen") return;
        if (DragControls.lastHovered !== evt.object) return;
        this.context.domElement.style.cursor = 'auto';
    }

    /** 
     * Handles pointer down events. Initiates the potential drag operation if conditions are met.
     * @param args Pointer event data containing information about the interaction
     * @internal
     */
    onPointerDown(args: PointerEventData) {
        if (!this.allowEdit(this.gameObject)) return;
        if (args.used) return;

        // get the drag mode and check if we need to abort early here
        const isSpatialInput = args.mode === "tracked-pointer" || args.mode === "transient-pointer";
        const dragMode = isSpatialInput ? this.xrDragMode : this.dragMode;
        if (dragMode === DragMode.None) return;

        DragControls.lastHovered = args.object;

        if (args.button === 0) {
            if (this._dragHandlers.size === 0) {
                this._didDrag = false;
                this._totalMovement.set(0, 0, 0);
                this._potentialDragStartEvt = args;
            }
            if (!this._targetObject) {
                this.setTargetObject(this.gameObject);
            }

            DragControls._active += 1;

            const newDragHandler = new DragPointerHandler(this, this._targetObject!);
            this._dragHandlers.set(args.event.space, newDragHandler);

            newDragHandler.onDragStart(args);

            if (this._dragHandlers.size === 2) {
                const iterator = this._dragHandlers.values();
                const a = iterator.next().value;
                const b = iterator.next().value;
                if (a instanceof DragPointerHandler && b instanceof DragPointerHandler) {
                    const mtHandler = new MultiTouchDragHandler(this, this._targetObject!, a, b);
                    this._dragHandlers.set(this.gameObject, mtHandler);
                    mtHandler.onDragStart(args);
                }
                else {
                    console.error("Attempting to construct a MultiTouchDragHandler with invalid DragPointerHandlers. This is likely a bug.", { a, b });
                }
            }

            args.use();
        }
    }

    /** 
     * Handles pointer up events. Finalizes or cancels the drag operation.
     * @param args Pointer event data containing information about the interaction
     * @internal
     */
    onPointerUp(args: PointerEventData) {
        if (debug) Gizmos.DrawLabel(args.point ?? this.gameObject.worldPosition, "POINTERUP:" + args.pointerId + ", " + args.button, .03, 3);
        if (!this.allowEdit(this.gameObject)) return;
        if (args.button !== 0) return;
        this._potentialDragStartEvt = null;

        const handler = this._dragHandlers.get(args.event.space);
        const mtHandler = this._dragHandlers.get(this.gameObject) as MultiTouchDragHandler;
        if (mtHandler && (mtHandler.handlerA === handler || mtHandler.handlerB === handler)) {
            // any of the two handlers has been released, so we can remove the multi-touch handler
            this._dragHandlers.delete(this.gameObject);
            mtHandler.onDragEnd(args);
        }

        if (handler) {
            if (DragControls._active > 0)
                DragControls._active -= 1;

            this.setTargetObject(null);

            if (handler.onDragEnd) handler.onDragEnd(args);
            this._dragHandlers.delete(args.event.space);

            if (this._dragHandlers.size === 0) {
                this.onLastDragEnd(args);
            }
            args.use();
        }
    }

    /**
     * Updates the drag operation every frame. Processes pointer movement, accumulates drag distance
     * and triggers drag start once there's enough movement.
     * @internal
     */
    update(): void {
        for (const handler of this._dragHandlers.values()) {
            if (handler.collectMovementInfo) handler.collectMovementInfo();
            // TODO this doesn't make sense, we should instead just use the max here
            // or even better, each handler can decide on their own how to handle this
            if (handler.getTotalMovement) this._totalMovement.add(handler.getTotalMovement());
        }

        // drag start only after having dragged for some pixels
        if (this._potentialDragStartEvt) {
            if (!this._didDrag) {
                // this is so we can e.g. process clicks without having a drag change the position, e.g. a click to call a method.
                // TODO probably needs to be treated differently for spatial (3D motion) and screen (2D pixel motion) drags
                if (this._totalMovement.length() > 0.0003)
                    this._didDrag = true;
                else return;
            }
            const args = this._potentialDragStartEvt;
            this._potentialDragStartEvt = null;
            this.onFirstDragStart(args);
        }

        for (const handler of this._dragHandlers.values())
            if (handler.onDragUpdate) handler.onDragUpdate(this._dragHandlers.size);

        if (this._dragHelper && this._dragHelper.hasSelected)
            this.onAnyDragUpdate();
    }

    /** 
     * Called when the first pointer starts dragging on this object. 
     * Sets up network synchronization and marks rigidbodies for dragging.
     * Not called for subsequent pointers on the same object.
     * @param evt Pointer event data that initiated the drag
     */
    private onFirstDragStart(evt: PointerEventData) {
        if (!evt || !evt.object) return;

        const dc = GameObject.getComponentInParent(evt.object, DragControls);
        // if a DragControls is in parent (e.g. when we have nested DragControls) and the parent DragControls is currently active
        // then we will ignore this DragControls and not select it.
        // But if the parent DragControls isn't dragging then we allow this to run because we want to start networking
        if (!dc || (dc !== this && dc._isDragging)) return;

        const object = this._targetObject || this.gameObject;

        if (!object) return;

        this._isDragging = true;

        const sync = GameObject.getComponentInChildren(object, SyncedTransform);
        if (debug) console.log("DRAG START", sync, object);

        if (sync) {
            sync.fastMode = true;
            sync?.requestOwnership();
        }

        this._marker = GameObject.addComponent(object, UsageMarker);

        this._draggingRigidbodies.length = 0;
        const rbs = GameObject.getComponentsInChildren(object, Rigidbody);
        if (rbs)
            this._draggingRigidbodies.push(...rbs);
    }

    /** 
     * Called each frame as long as any pointer is dragging this object.
     * Updates visuals and keeps rigidbodies awake during the drag.
     */
    private onAnyDragUpdate() {
        if (!this._dragHelper) return;
        this._dragHelper.showGizmo = this.showGizmo;

        this._dragHelper.onUpdate(this.context);
        for (const rb of this._draggingRigidbodies) {
            rb.wakeUp();
            rb.resetVelocities();
            rb.resetForcesAndTorques();
        }

        const object = this._targetObject || this.gameObject;

        InstancingUtil.markDirty(object);
    }

    /** 
     * Called when the last pointer has been removed from this object.
     * Cleans up drag state and applies final velocities to rigidbodies.
     * @param evt Pointer event data for the last pointer that was lifted
     */
    private onLastDragEnd(evt: PointerEventData | null) {
        if (!this || !this._isDragging) return;
        this._isDragging = false;
        for (const rb of this._draggingRigidbodies) {
            rb.setVelocity(rb.smoothedVelocity);
        }
        this._draggingRigidbodies.length = 0;
        this._targetObject = null;
        if (evt?.object) {
            const sync = GameObject.getComponentInChildren(evt.object, SyncedTransform);
            if (sync) {
                sync.fastMode = false;
                // sync?.requestOwnership();
            }
        }
        if (this._marker)
            this._marker.destroy();

        if (!this._dragHelper) return;
        const selected = this._dragHelper.selected;
        if (debug) console.log("DRAG END", selected, selected?.visible)
        this._dragHelper.setSelected(null, this.context);
    }
}

/** 
 * Common interface for pointer handlers (single touch and multi touch).
 * Defines methods for tracking movement and managing target objects during drag operations.
 */
interface IDragHandler {
    /** Used to determine if a drag has happened for this handler */
    getTotalMovement?(): Vector3;
    /** Target object can change mid-flight (e.g. in Duplicatable), handlers should react properly to that */
    setTargetObject(obj: Object3D | null): void;

    /** Prewarms the drag – can already move internal points around here but should not move the object itself */
    collectMovementInfo?(): void;
    onDragStart?(args: PointerEventData): void;
    onDragEnd?(args: PointerEventData): void;
    /** The target object is moved around */
    onDragUpdate?(numberOfPointers: number): void;
}

/** 
 * Handles two touch points affecting one object. 
 * Enables multi-touch interactions that allow movement, scaling, and rotation of objects.
 */
class MultiTouchDragHandler implements IDragHandler {

    handlerA: DragPointerHandler;
    handlerB: DragPointerHandler;

    private context: Context;
    private settings: DragControls;
    private gameObject: Object3D;
    private _handlerAAttachmentPoint: Vector3 = new Vector3();
    private _handlerBAttachmentPoint: Vector3 = new Vector3();

    private _followObject: GameObject;
    private _manipulatorObject: GameObject;
    private _deviceMode!: XRTargetRayMode | "transient-pointer";
    private _followObjectStartWorldQuaternion: Quaternion = new Quaternion();

    constructor(dragControls: DragControls, gameObject: Object3D, pointerA: DragPointerHandler, pointerB: DragPointerHandler) {
        this.context = dragControls.context;
        this.settings = dragControls;
        this.gameObject = gameObject;
        this.handlerA = pointerA;
        this.handlerB = pointerB;

        this._followObject = new Object3D() as GameObject;
        this._manipulatorObject = new Object3D() as GameObject;

        this.context.scene.add(this._manipulatorObject);

        const rig = NeedleXRSession.active?.rig?.gameObject;

        if (!this.handlerA || !this.handlerB || !this.handlerA.hitPointInLocalSpace || !this.handlerB.hitPointInLocalSpace) {
            console.error("Invalid: MultiTouchDragHandler needs two valid DragPointerHandlers with hitPointInLocalSpace set.");
            return;
        }

        this._tempVec1.copy(this.handlerA.hitPointInLocalSpace);
        this._tempVec2.copy(this.handlerB.hitPointInLocalSpace);
        this.gameObject.localToWorld(this._tempVec1);
        this.gameObject.localToWorld(this._tempVec2);
        if (rig) {
            rig.worldToLocal(this._tempVec1);
            rig.worldToLocal(this._tempVec2);
        }
        this._initialDistance = this._tempVec1.distanceTo(this._tempVec2);

        if (this._initialDistance < 0.02) {
            if (debug) {
                console.log("Finding alternative drag attachment points since initial distance is too low: " + this._initialDistance.toFixed(2));
            }
            // We want two reasonable pointer attachment points here.
            // But if the hitPointInLocalSpace are very close to each other, we instead fall back to controller positions.
            this.handlerA.followObject.parent!.getWorldPosition(this._tempVec1);
            this.handlerB.followObject.parent!.getWorldPosition(this._tempVec2);
            this._handlerAAttachmentPoint.copy(this._tempVec1);
            this._handlerBAttachmentPoint.copy(this._tempVec2);
            this.gameObject.worldToLocal(this._handlerAAttachmentPoint);
            this.gameObject.worldToLocal(this._handlerBAttachmentPoint);
            this._initialDistance = this._tempVec1.distanceTo(this._tempVec2);

            if (this._initialDistance < 0.001) {
                console.warn("Not supported right now – controller drag points for multitouch are too close!");
                this._initialDistance = 1;
            }
        }
        else {
            this._handlerAAttachmentPoint.copy(this.handlerA.hitPointInLocalSpace);
            this._handlerBAttachmentPoint.copy(this.handlerB.hitPointInLocalSpace);
        }

        this._tempVec3.lerpVectors(this._tempVec1, this._tempVec2, 0.5);
        this._initialScale.copy(gameObject.scale);

        if (debug) {
            this._followObject.add(new AxesHelper(2));
            this._manipulatorObject.add(new AxesHelper(5));

            const formatVec = (v: Vector3) => `${v.x.toFixed(2)}, ${v.y.toFixed(2)}, ${v.z.toFixed(2)}`;
            Gizmos.DrawLine(this._tempVec1, this._tempVec2, 0x00ffff, 0, false);
            Gizmos.DrawLabel(this._tempVec3, "A:B " + this._initialDistance.toFixed(2) + "\n" + formatVec(this._tempVec1) + "\n" + formatVec(this._tempVec2), 0.03, 5);
        }
    }

    onDragStart(_args: PointerEventData): void {
        // align _followObject with the object we want to drag
        this.gameObject.add(this._followObject);
        this._followObject.matrixAutoUpdate = false;
        this._followObject.matrix.identity();
        this._deviceMode = _args.mode;
        this._followObjectStartWorldQuaternion.copy(this._followObject.worldQuaternion);

        // align _manipulatorObject in the same way it would if this was a drag update
        this.alignManipulator();

        // and then parent it to the space object so it follows along.
        this._manipulatorObject.attach(this._followObject);

        // store offsets in local space
        this._manipulatorPosOffset.copy(this._followObject.position);
        this._manipulatorRotOffset.copy(this._followObject.quaternion);
        this._manipulatorScaleOffset.copy(this._followObject.scale);
    }

    onDragEnd(_args: PointerEventData): void {
        if (!this.handlerA || !this.handlerB) {
            console.error("onDragEnd called on MultiTouchDragHandler without valid handlers. This is likely a bug.");
            return;
        }

        // we want to initialize the drag points for these handlers again.
        // one of them will be removed, but we don't know here which one
        this.handlerA.recenter();
        this.handlerB.recenter();

        // destroy helper objects
        this._manipulatorObject.removeFromParent();
        this._followObject.removeFromParent();
        this._manipulatorObject.destroy();
        this._followObject.destroy();
    }

    private _manipulatorPosOffset: Vector3 = new Vector3();
    private _manipulatorRotOffset: Quaternion = new Quaternion();
    private _manipulatorScaleOffset: Vector3 = new Vector3();

    private _tempVec1: Vector3 = new Vector3();
    private _tempVec2: Vector3 = new Vector3();
    private _tempVec3: Vector3 = new Vector3();
    private tempLookMatrix: Matrix4 = new Matrix4();
    private _initialScale: Vector3 = new Vector3();
    private _initialDistance: number = 0;

    private alignManipulator() {
        if (!this.handlerA || !this.handlerB) {
            console.error("alignManipulator called on MultiTouchDragHandler without valid handlers. This is likely a bug.", this);
            return;
        }

        if (!this.handlerA.followObject || !this.handlerB.followObject) {
            console.error("alignManipulator called on MultiTouchDragHandler without valid follow objects. This is likely a bug.", this.handlerA, this.handlerB);
            return;
        }

        this._tempVec1.copy(this._handlerAAttachmentPoint);
        this._tempVec2.copy(this._handlerBAttachmentPoint);
        this.handlerA.followObject.localToWorld(this._tempVec1);
        this.handlerB.followObject.localToWorld(this._tempVec2);
        this._tempVec3.lerpVectors(this._tempVec1, this._tempVec2, 0.5);

        this._manipulatorObject.position.copy(this._tempVec3);

        // - lookAt the second point on handlerB
        const camera = this.context.mainCamera;
        this.tempLookMatrix.lookAt(this._tempVec3, this._tempVec2, (camera as any as IGameObject).worldUp);
        this._manipulatorObject.quaternion.setFromRotationMatrix(this.tempLookMatrix);

        // - scale based on the distance between the two points
        const dist = this._tempVec1.distanceTo(this._tempVec2);
        this._manipulatorObject.scale.copy(this._initialScale).multiplyScalar(dist / this._initialDistance);

        this._manipulatorObject.updateMatrix();
        this._manipulatorObject.updateMatrixWorld(true);

        if (debug) {
            Gizmos.DrawLabel(this._tempVec3.clone().add(new Vector3(0, 0.2, 0)), "A:B " + dist.toFixed(2), 0.03);
            Gizmos.DrawLine(this._tempVec1, this._tempVec2, 0x00ff00, 0, false);

            // const wp = this._manipulatorObject.worldPosition;
            // Gizmos.DrawWireSphere(wp, this._initialScale.length() * dist / this._initialDistance, 0x00ff00, 0, false);
        }
    }

    onDragUpdate() {
        // At this point we've run both the other handlers, but their effects have been suppressed because they can't handle
        // two events at the same time. They're basically providing us with two Object3D's and we can combine these here
        // into a reasonable two-handed translation/rotation/scale.
        // One approach:
        // - position our control object on the center between the two pointer control objects

        // TODO close grab needs to be handled differently because there we don't have a hit point - 
        // Hit point is just the center of the object
        // So probably we should fix that close grab has a better hit point approximation (point on bounds?)

        this.alignManipulator();

        // apply (smoothed) to the gameObject
        const lerpStrength = 30;
        const lerpFactor = 1.0;

        this._followObject.position.copy(this._manipulatorPosOffset);
        this._followObject.quaternion.copy(this._manipulatorRotOffset);
        this._followObject.scale.copy(this._manipulatorScaleOffset);

        const draggedObject = this.gameObject;
        const targetObject = this._followObject;

        if (!draggedObject) {
            console.error("MultiTouchDragHandler has no dragged object. This is likely a bug.");
            return;
        }

        targetObject.updateMatrix();
        targetObject.updateMatrixWorld(true);

        const isSpatialInput = this._deviceMode === "tracked-pointer" || this._deviceMode === "transient-pointer";
        const keepRotation = isSpatialInput ? this.settings.xrKeepRotation : this.settings.keepRotation;

        // TODO refactor to a common place
        // apply constraints (position grid snap, rotation, ...)
        if (this.settings.snapGridResolution > 0) {
            const wp = this._followObject.worldPosition;
            const snap = this.settings.snapGridResolution;
            wp.x = Math.round(wp.x / snap) * snap;
            wp.y = Math.round(wp.y / snap) * snap;
            wp.z = Math.round(wp.z / snap) * snap;
            this._followObject.worldPosition = wp;
            this._followObject.updateMatrix();
        }
        if (keepRotation) {
            this._followObject.worldQuaternion = this._followObjectStartWorldQuaternion;
            this._followObject.updateMatrix();
        }

        // TODO refactor to a common place
        // TODO should use unscaled time here // some test for lerp speed depending on distance
        const t = Mathf.clamp01(this.context.time.deltaTime * lerpStrength * lerpFactor);// / (currentDist - 1 + 0.01));

        const wp = draggedObject.worldPosition;
        wp.lerp(targetObject.worldPosition, t);
        draggedObject.worldPosition = wp;

        const rot = draggedObject.worldQuaternion;
        rot.slerp(targetObject.worldQuaternion, t);
        draggedObject.worldQuaternion = rot;

        const scl = draggedObject.worldScale;
        scl.lerp(targetObject.worldScale, t);
        draggedObject.worldScale = scl;
    }

    setTargetObject(obj: Object3D | null): void {
        this.gameObject = obj as GameObject;
    }
}


/** 
 * Handles a single pointer on an object. 
 * DragPointerHandlers manage determining if a drag operation has started, tracking pointer movement,
 * and controlling object translation based on the drag mode.
 */
class DragPointerHandler implements IDragHandler {

    /** 
     * Returns the accumulated movement of the pointer in world units.
     * Used for determining if enough motion has occurred to start a drag.
     */
    getTotalMovement(): Vector3 { return this._totalMovement; }
    
    /** 
     * Returns the object that follows the pointer during dragging operations.
     */
    get followObject(): GameObject { return this._followObject; }
    
    /**
     * Returns the point where the pointer initially hit the object in local space.
     */
    get hitPointInLocalSpace(): Vector3 { return this._hitPointInLocalSpace; }

    private context: Context;
    private gameObject: Object3D | null;
    private settings: DragControls;
    private _lastRig: IGameObject | undefined = undefined;

    /** This object is placed at the pivot of the dragged object, and parented to the control space. */
    private _followObject: GameObject;
    private _totalMovement: Vector3 = new Vector3();
    /** Motion along the pointer ray. On screens this doesn't change. In XR it can be used to determine how much
     * effort someone is putting into moving an object closer or further away. */
    private _totalMovementAlongRayDirection: number = 0;
    /** Distance between _followObject and its parent at grab start, in local space */
    private _grabStartDistance: number = 0;
    private _deviceMode!: XRTargetRayMode | "transient-pointer";
    private _followObjectStartPosition: Vector3 = new Vector3();
    private _followObjectStartQuaternion: Quaternion = new Quaternion();
    private _followObjectStartWorldQuaternion: Quaternion = new Quaternion();
    private _lastDragPosRigSpace: Vector3 | undefined;
    private _tempVec: Vector3 = new Vector3();
    private _tempMat: Matrix4 = new Matrix4();

    private _hitPointInLocalSpace: Vector3 = new Vector3();
    private _hitNormalInLocalSpace: Vector3 = new Vector3();
    private _bottomCenter = new Vector3();
    private _backCenter = new Vector3();
    private _backBottomCenter = new Vector3();
    private _bounds = new Box3();
    private _dragPlane = new Plane(new Vector3(0, 1, 0));
    private _draggedOverObject: Object3D | null = null;
    private _draggedOverObjectLastSetUp: Object3D | null = null;
    private _draggedOverObjectLastNormal: Vector3 = new Vector3();
    private _draggedOverObjectDuration: number = 0;

    /** Allows overriding which object is dragged while a drag is already ongoing. Used for example by Duplicatable */
    setTargetObject(obj: Object3D | null) {
        this.gameObject = obj;
    }

    constructor(dragControls: DragControls, gameObject: Object3D) {
        this.settings = dragControls;
        this.context = dragControls.context;
        this.gameObject = gameObject;
        this._followObject = new Object3D() as GameObject;
    }

    recenter() {
        if (!this._followObject.parent) {
            console.warn("Error: space follow object doesn't have parent but recenter() is called. This is likely a bug");
            return;
        }
        if (!this.gameObject) {
            console.warn("Error: space follow object doesn't have a gameObject");
            return;
        }

        const p = this._followObject.parent as GameObject;

        this.gameObject.add(this._followObject);
        this._followObject.matrixAutoUpdate = false;

        this._followObject.position.set(0, 0, 0);
        this._followObject.quaternion.set(0, 0, 0, 1);
        this._followObject.scale.set(1, 1, 1);

        this._followObject.updateMatrix();
        this._followObject.updateMatrixWorld(true);

        p.attach(this._followObject);

        this._followObjectStartPosition.copy(this._followObject.position);
        this._followObjectStartQuaternion.copy(this._followObject.quaternion);
        this._followObjectStartWorldQuaternion.copy(this._followObject.worldQuaternion);

        this._followObject.updateMatrix();
        this._followObject.updateMatrixWorld(true);

        const hitPointWP = this._hitPointInLocalSpace.clone();
        this.gameObject.localToWorld(hitPointWP);
        this._grabStartDistance = hitPointWP.distanceTo(p.worldPosition);
        const rig = NeedleXRSession.active?.rig?.gameObject;
        const rigScale = rig?.worldScale.x || 1;
        this._grabStartDistance /= rigScale;

        this._totalMovementAlongRayDirection = 0;
        this._lastDragPosRigSpace = undefined;

        if (debug) {
            Gizmos.DrawLine(hitPointWP, p.worldPosition, 0x00ff00, 0.5, false);
            Gizmos.DrawLabel(p.worldPosition.add(new Vector3(0, 0.1, 0)), this._grabStartDistance.toFixed(2), 0.03, 0.5);
        }
    }

    onDragStart(args: PointerEventData) {
        if (!this.gameObject) {
            console.warn("Error: space follow object doesn't have a gameObject");
            return;
        }

        args.event.space.add(this._followObject);

        // prepare for drag, we will start dragging after an object has been dragged for a few centimeters
        this._lastDragPosRigSpace = undefined;

        if (args.point && args.normal) {
            this._hitPointInLocalSpace.copy(args.point);
            this.gameObject.worldToLocal(this._hitPointInLocalSpace);
            this._hitNormalInLocalSpace.copy(args.normal);
        }
        else if (args) {
            // can happen for e.g. close grabs; we can assume/guess a good hit point and normal based on the object's bounds or so
            // convert controller world position to local space instead and use that as hit point
            const controller = args.event.space as GameObject;
            const controllerWp = controller.worldPosition;
            this.gameObject.worldToLocal(controllerWp);
            this._hitPointInLocalSpace.copy(controllerWp);

            const controllerUp = controller.worldUp;
            this._tempMat.copy(this.gameObject.matrixWorld).invert();
            controllerUp.transformDirection(this._tempMat);
            this._hitNormalInLocalSpace.copy(controllerUp);
        }

        this.recenter();

        this._totalMovement.set(0, 0, 0);
        this._deviceMode = args.mode;


        const dragSource = this._followObject.parent as IGameObject;
        const rayDirection = dragSource.worldForward;

        const isSpatialInput = this._deviceMode === "tracked-pointer" || this._deviceMode === "transient-pointer";
        const dragMode = isSpatialInput ? this.settings.xrDragMode : this.settings.dragMode;

        // set up drag plane; we don't really know the normal yet but we can already set the point
        const hitWP = this._hitPointInLocalSpace.clone();
        this.gameObject.localToWorld(hitWP);

        switch (dragMode) {
            case DragMode.XZPlane:
                const up = new Vector3(0, 1, 0);
                if (this.gameObject.parent) {
                    // TODO in this case _dragPlane should be in parent space, not world space,
                    // otherwise dragging the parent and this object at the same time doesn't keep the plane constrained
                    up.transformDirection(this.gameObject.parent.matrixWorld.clone().invert());
                }
                this._dragPlane.setFromNormalAndCoplanarPoint(up, hitWP);
                break;
            case DragMode.HitNormal:
                const hitNormal = this._hitNormalInLocalSpace.clone();
                hitNormal.transformDirection(this.gameObject.matrixWorld);
                this._dragPlane.setFromNormalAndCoplanarPoint(hitNormal, hitWP);
                break;
            case DragMode.Attached:
                this._dragPlane.setFromNormalAndCoplanarPoint(rayDirection, hitWP);
                break;
            case DragMode.DynamicViewAngle: // At start (when nothing is hit yet) the drag plane should be aligned to the view
                this.setPlaneViewAligned(hitWP, true);
                break;
            case DragMode.SnapToSurfaces: // At start (when nothing is hit yet) the drag plane should be aligned to the view
                this.setPlaneViewAligned(hitWP, false);
                break;
            case DragMode.None:
                break;
        }

        // calculate bounding box and snapping points. We want to either snap the "back" point or the "bottom" point.
        // const bbox = new Box3();
        const p = this.gameObject.parent;
        const localP = this.gameObject.position.clone();
        const localQ = this.gameObject.quaternion.clone();
        const localS = this.gameObject.scale.clone();
        // save the original matrix world (because if some other script is doing a raycast at the same moment the matrix will not be correct anymore....)
        const matrixWorld = this.gameObject.matrixWorld.clone();

        if (p) p.remove(this.gameObject);
        this.gameObject.position.set(0, 0, 0);
        this.gameObject.quaternion.set(0, 0, 0, 1);
        this.gameObject.scale.set(1, 1, 1);
        const bbox = getBoundingBox([this.gameObject]);
        // we force the bbox to include our own point *because* the DragControls might be attached to an empty object (which isnt included in the bounding box call above)
        bbox.expandByPoint(this.gameObject.worldPosition);

        // console.log(this.gameObject.position.y - bbox.min.y)
        // bbox.min.y += (this.gameObject.position.y - bbox.min.y);

        // get front center point of the bbox. basically (0, 0, 1) in local space
        const bboxCenter = new Vector3();
        bbox.getCenter(bboxCenter);
        const bboxSize = new Vector3();
        bbox.getSize(bboxSize);

        // attachment points for dragging
        this._bottomCenter.copy(bboxCenter.clone().add(new Vector3(0, -bboxSize.y / 2, 0)));
        this._backCenter.copy(bboxCenter.clone().add(new Vector3(0, 0, bboxSize.z / 2)));
        this._backBottomCenter.copy(bboxCenter.clone().add(new Vector3(0, -bboxSize.y / 2, bboxSize.z / 2)));

        this._bounds.copy(bbox);

        // restore original transform
        if (p) p.add(this.gameObject);
        this.gameObject.position.copy(localP);
        this.gameObject.quaternion.copy(localQ);
        this.gameObject.scale.copy(localS);
        this.gameObject.matrixWorld.copy(matrixWorld);

        // surface snapping
        this._draggedOverObject = null;
        this._draggedOverObjectLastSetUp = null;
        this._draggedOverObjectLastNormal.set(0, 1, 0);
        this._draggedOverObjectDuration = 0;
    }

    collectMovementInfo() {
        // we're dragging - there is a controlling object
        if (!this._followObject.parent) return;

        // TODO This should all be handled properly per-pointer
        // and we want to have a chance to react to multiple pointers being on the same object.
        // some common stuff (calculating of movement offsets, etc) could be done by default
        // and then the main thing to override is the actual movement of the object based on N _followObjects

        const dragSource = this._followObject.parent as IGameObject;

        // modify _followObject with constraints, e.g.
        // - dragging on a plane, e.g. the floor (keeping the distance to the floor plane constant)
        /* TODO fix jump on drag start
        const p0 = this._followObject.parent as GameObject;
        const ray = new Ray(p0.worldPosition, p0.worldForward.multiplyScalar(-1));
        const p = new Vector3();
        const t0 = ray.intersectPlane(new Plane(new Vector3(0, 1, 0)), p);
        if (t0 !== null)
            this._followObject.worldPosition = t0;
        */

        this._followObject.updateMatrix();
        const dragPosRigSpace = dragSource.worldPosition;
        const rig = NeedleXRSession.active?.rig?.gameObject;
        if (rig)
            rig.worldToLocal(dragPosRigSpace);

        // sum up delta
        // TODO We need to do all/most of these calculations in Rig Space instead of world space
        // moving the rig while holding an object should not affect _rayDelta / _dragDelta
        if (this._lastDragPosRigSpace === undefined || rig != this._lastRig) {
            this._lastDragPosRigSpace = dragPosRigSpace.clone();
            this._lastRig = rig;
        }
        this._tempVec.copy(dragPosRigSpace).sub(this._lastDragPosRigSpace);

        const rayDirectionRigSpace = dragSource.worldForward;
        if (rig) {
            this._tempMat.copy(rig.matrixWorld).invert();
            rayDirectionRigSpace.transformDirection(this._tempMat);
        }
        // sum up delta movement along ray
        this._totalMovementAlongRayDirection += rayDirectionRigSpace.dot(this._tempVec);
        this._tempVec.x = Math.abs(this._tempVec.x);
        this._tempVec.y = Math.abs(this._tempVec.y);
        this._tempVec.z = Math.abs(this._tempVec.z);

        // sum up absolute total movement
        this._totalMovement.add(this._tempVec);
        this._lastDragPosRigSpace.copy(dragPosRigSpace);

        if (debug) {
            let wp = dragPosRigSpace;
            // ray direction of the input source object
            if (rig) {
                wp = wp.clone();
                wp.transformDirection(rig.matrixWorld);
            }
            Gizmos.DrawRay(wp, rayDirectionRigSpace, 0x0000ff);
        }
    }

    onDragUpdate(numberOfPointers: number) {

        // can only handle a single pointer
        // if there's more, we defer to multi-touch drag handlers
        if (numberOfPointers > 1) return;
        const draggedObject = this.gameObject as IGameObject | null;
        if (!draggedObject || !this._followObject) {
            console.warn("Warning: DragPointerHandler doesn't have a dragged object. This is likely a bug.");
            return;
        }
        const dragSource = this._followObject.parent as IGameObject | null;
        if (!dragSource) {
            console.warn("Warning: DragPointerHandler doesn't have a drag source. This is likely a bug.");
            return;
        }
        this._followObject.updateMatrix();
        const dragSourceWP = dragSource.worldPosition;
        const rayDirection = dragSource.worldForward;


        // Actually move and rotate draggedObject
        const isSpatialInput = this._deviceMode === "tracked-pointer" || this._deviceMode === "transient-pointer";
        const keepRotation = isSpatialInput ? this.settings.xrKeepRotation : this.settings.keepRotation;
        const dragMode = isSpatialInput ? this.settings.xrDragMode : this.settings.dragMode;

        if (dragMode === DragMode.None) return;

        const lerpStrength = 10;
        // - keeping rotation constant during dragging
        if (keepRotation) this._followObject.worldQuaternion = this._followObjectStartWorldQuaternion;
        this._followObject.updateMatrix();
        this._followObject.updateMatrixWorld(true);

        // Acceleration for moving the object - move followObject along the ray distance by _totalMovementAlongRayDirection
        let currentDist = 1.0;
        let lerpFactor = 2.0;
        if (isSpatialInput && this._grabStartDistance > 0.5) // hands and controllers, but not touches
        {
            const factor = 1 + this._totalMovementAlongRayDirection * (2 * this.settings.xrDistanceDragFactor);
            currentDist = Math.max(0.0, factor);
            currentDist = currentDist * currentDist * currentDist;
        }
        else if (this._grabStartDistance <= 0.5) {
            // TODO there's still a frame delay between dragged objects and the hand models
            lerpFactor = 3.0;
        }

        // reset _followObject to its original position and rotation
        this._followObject.position.copy(this._followObjectStartPosition);
        if (!keepRotation)
            this._followObject.quaternion.copy(this._followObjectStartQuaternion);

        // TODO restore previous functionality:
        // When distance dragging, the HIT POINT should move along the ray until it reaches the controller;
        // NOT the pivot point of the dragged object. E.g. grabbing a large cube and pulling towards you should at most 
        // move the grabbed point to your head and not slap the cube in your head.
        this._followObject.position.multiplyScalar(currentDist);
        this._followObject.updateMatrix();

        const didHaveSurfaceHitPointLastFrame = this._hasLastSurfaceHitPoint;
        this._hasLastSurfaceHitPoint = false;
        const ray = new Ray(dragSourceWP, rayDirection);
        let didHit = false;

        // Surface snapping.
        // Feels quite weird in VR right now!
        if (dragMode == DragMode.SnapToSurfaces) {
            // Idea: Do a sphere cast if we're still in the proximity of the current draggedObject.
            // This would allow dragging slightly out of the object's bounds and still continue snapping to it.
            // Do a regular raycast (without the dragged object) to determine if we should change what is dragged onto.
            const hits = this.context.physics.raycastFromRay(ray, {
                testObject: o => o !== this.followObject && o !== dragSource && o !== draggedObject// && !(o instanceof GroundedSkybox)
            });

            if (hits.length > 0) {
                const hit = hits[0];
                // if we're above the same surface for a specified time, adjust drag options:
                // - set that surface as the drag "plane". We will follow that object's surface instead now (raycast onto only that)
                // - if the drag plane is an object, we also want to 
                //   - calculate an initial rotation offset matching what surface/face the user originally started the drag on
                //   - rotate the dragged object to match the surface normal
                if (this._draggedOverObject === hit.object)
                    this._draggedOverObjectDuration += this.context.time.deltaTime;
                else {
                    this._draggedOverObject = hit.object;
                    this._draggedOve