@needle-tools/engine/src/engine-components/webxr/Avatar.ts

Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.

import { Mesh, Object3D, Quaternion, Vector3 } from "three";

import { AssetReference } from "../../engine/engine_addressables.js";
import { ObjectUtils, PrimitiveType } from "../../engine/engine_create_objects.js";
import { serializable } from "../../engine/engine_serialization_decorator.js";
import type { IGameObject } from "../../engine/engine_types.js";
import { getParam, PromiseAllWithErrors } from "../../engine/engine_utils.js";
import { setCustomVisibility } from "../../engine/js-extensions/Layers.js";
import { NeedleXRController, type NeedleXREventArgs, NeedleXRSession, NeedleXRUtils } from "../../engine/xr/api.js";
import { PlayerState } from "../../engine-components-experimental/networking/PlayerSync.js";
import { Behaviour, GameObject } from "../Component.js";
import { SyncedTransform } from "../SyncedTransform.js";
import { AvatarMarker } from "./WebXRAvatar.js";
import { XRFlag } from "./XRFlag.js";

const debug = getParam("debugwebxr");

const flipForwardQuaternion = new Quaternion().setFromAxisAngle(new Vector3(0, 1, 0), Math.PI);

/**
 * @category XR
 * @category Networking
 * @group Components
 */
export class Avatar extends Behaviour {

    @serializable(AssetReference)
    head?: AssetReference;

    @serializable(AssetReference)
    leftHand?: AssetReference;

    @serializable(AssetReference)
    rightHand?: AssetReference;

    private _leftHandMeshes?: Mesh[];
    private _rightHandMeshes?: Mesh[];

    private _syncTransforms?: SyncedTransform[];

    async onEnterXR(_args: NeedleXREventArgs) {
        if (!this.activeAndEnabled) return;
        if (debug) console.warn("AVATAR ENTER XR", this.guid, this.sourceId, this, this.activeAndEnabled)
        if (this._syncTransforms)
            this._syncTransforms.length = 0;
        await this.prepareAvatar();

        const playerstate = PlayerState.getFor(this);
        if (playerstate?.owner) {
            const marker = this.gameObject.addComponent(AvatarMarker)!;
            marker.avatar = this.gameObject;
            marker.connectionId = playerstate.owner;
        }
        else if (this.context.connection.isConnected) console.error("No player state found for avatar", this);
        // don't destroy the avatar when entering XR and not connected to a networking backend
        else if (playerstate && !this.context.connection.isConnected) playerstate.dontDestroy = true;
    }

    onLeaveXR(_args: NeedleXREventArgs): void {
        const marker = this.gameObject.getComponent(AvatarMarker);
        if (marker) {
            marker.destroy();
        }
    }

    onUpdateXR(args: NeedleXREventArgs): void {
        if (!this.activeAndEnabled) return;

        const isLocalPlayer = PlayerState.isLocalPlayer(this);
        if (!isLocalPlayer) return;

        const xr = args.xr;
        // make sure the avatar is inside the active rig
        if (xr.rig && xr.rig.gameObject !== this.gameObject.parent) {
            this.gameObject.position.set(0, 0, 0);
            this.gameObject.rotation.set(0, 0, 0);
            this.gameObject.scale.set(1, 1, 1);
            xr.rig.gameObject.add(this.gameObject);
        }
        // this.gameObject.position.copy(xr.rig!.gameObject.position);
        // this.gameObject.quaternion.copy(xr.rig!.gameObject.quaternion);
        // this.gameObject.scale.set(1, 1, 1);


        if (this._syncTransforms && isLocalPlayer) {
            for (const sync of this._syncTransforms) {
                sync.fastMode = true;
                if (!sync.isOwned())
                    sync.requestOwnership();
            }
        }


        // synchronize head
        if (this.head && this.context.mainCamera) {
            const headObj = this.head.asset as IGameObject;
            headObj.position.copy(this.context.mainCamera.position);
            headObj.position.x *= -1;
            headObj.position.z *= -1;
            headObj.quaternion.copy(this.context.mainCamera.quaternion);
            headObj.quaternion.x *= -1;

            // HACK: XRFlag limitation workaround to make sure first person user head is never rendered
            if (this.context.time.frameCount % 10 === 0 && this.head.asset) {
                const xrflags = GameObject.getComponentsInChildren(this.head.asset, XRFlag);
                for (const flag of xrflags) {
                    flag.enabled = false;
                    flag.gameObject.visible = false;
                }
            }
        }

        // synchronize hands
        const leftCtrl = args.xr.leftController;
        const leftObj = this.leftHand?.asset as Object3D;
        if (leftCtrl && leftObj) {
            leftObj.position.copy(leftCtrl.gripPosition);
            leftObj.quaternion.copy(leftCtrl.gripQuaternion);
            leftObj.quaternion.multiply(flipForwardQuaternion);
            leftObj.visible = leftCtrl.isTracking;
            this.updateHandVisibility(leftCtrl, leftObj, this._leftHandMeshes);
        }
        else if (leftObj && leftObj.visible) {
            leftObj.visible = false;
        }

        const right = args.xr.rightController;
        const rightObj = this.rightHand?.asset as Object3D;
        if (right && rightObj) {
            rightObj.position.copy(right.gripPosition);
            rightObj.quaternion.copy(right.gripQuaternion);
            rightObj.quaternion.multiply(flipForwardQuaternion);
            rightObj.visible = right.isTracking;
            this.updateHandVisibility(right, rightObj, this._rightHandMeshes);
        }
        else if (rightObj && rightObj.visible) {
            rightObj.visible = false;
        }
    }

    onBeforeRender(): void {
        if (this.context.xr) {
            if (this.context.time.frame % 10 === 0)
                this.updateRemoteAvatarVisibility();
        }
    }

    private updateHandVisibility(controller: NeedleXRController, avatarHand: Object3D, meshes: Mesh[] | undefined) {
        if (meshes) {
            // Hide the hand meshes for the local user if another model (e.g. the controller model) is being rendered
            // We don't set the visible flag here because it would also disable SyncedTransforms networking
            const hasOtherRenderingModel = controller.model && controller.model.visible && controller.model !== avatarHand;
            meshes.forEach(mesh => { setCustomVisibility(mesh, !hasOtherRenderingModel); });
        }
    }

    private updateRemoteAvatarVisibility() {
        if (this.context.connection.isConnected) {
            const state = PlayerState.getFor(this);
            if (state && state.isLocalPlayer == false) {

                const sync = NeedleXRSession.getXRSync(this.context);
                if (sync) {
                    if (sync.hasState(state.owner)) {
                        this.tryFindAvatarObjectsIfMissing();

                        const leftObj = this.leftHand?.asset as Object3D;
                        if (leftObj) {
                            leftObj.visible = sync?.isTracking(state.owner, "left") ?? false;
                        }
                        const rightObj = this.rightHand?.asset as Object3D;
                        if (rightObj) {
                            rightObj.visible = sync?.isTracking(state.owner, "right") ?? false;
                        }
                    }
                }

                // HACK: XRFlag limitation workaround to make sure first person user head of OTHER users is ALWAYS rendered
                if (this.head?.asset) {
                    const xrflags = GameObject.getComponentsInChildren(this.head.asset, XRFlag);
                    for (const flag of xrflags) {
                        flag.enabled = false;
                        flag.gameObject.visible = true;
                    }
                }
            }
        }
    }



    private tryFindAvatarObjectsIfMissing() {
        // if no avatar objects are set, try to find them
        if (!this.head || !this.leftHand || !this.rightHand) {
            const res = { head: this.head, leftHand: this.leftHand, rightHand: this.rightHand };
            NeedleXRUtils.tryFindAvatarObjects(this.gameObject, this.sourceId || "", res);
            if (res.head) this.head = res.head;
            if (res.leftHand) this.leftHand = res.leftHand;
            if (res.rightHand) this.rightHand = res.rightHand;
        }
    }

    private async prepareAvatar() {
        // if no avatar objects are set, try to find them
        this.tryFindAvatarObjectsIfMissing();

        if (!this.head) {
            const head = new Object3D();
            head.name = "Head";
            const cube = ObjectUtils.createPrimitive(PrimitiveType.Cube);
            head.add(cube);
            this.gameObject.add(head);
            this.head = new AssetReference("", this.sourceId, head);
            if (debug) console.log("Create head", head);
        }
        else if (this.head instanceof Object3D) {
            this.head = new AssetReference("", this.sourceId, this.head);
        }

        if (!this.rightHand) {
            const rightHand = new Object3D();
            rightHand.name = "Right Hand";
            this.gameObject.add(rightHand);
            this.rightHand = new AssetReference("", this.sourceId, rightHand);
            if (debug) console.log("Create right hand", rightHand);
        }
        else if (this.rightHand instanceof Object3D) {
            this.rightHand = new AssetReference("", this.sourceId, this.rightHand);
        }

        if (!this.leftHand) {
            const leftHand = new Object3D();
            leftHand.name = "Left Hand";
            this.gameObject.add(leftHand);
            this.leftHand = new AssetReference("", this.sourceId, leftHand);
            if (debug) console.log("Create left hand", leftHand);
        }
        else if (this.leftHand instanceof Object3D) {
            this.leftHand = new AssetReference("", this.sourceId, this.leftHand);
        }

        await this.loadAvatarObjects(this.head, this.leftHand, this.rightHand);

        this._leftHandMeshes = [];
        this.leftHand.asset?.traverse((obj) => { if ((obj as Mesh)?.isMesh) this._leftHandMeshes!.push(obj as Mesh); });
        this._rightHandMeshes = [];
        this.rightHand.asset?.traverse((obj) => { if ((obj as Mesh)?.isMesh) this._rightHandMeshes!.push(obj as Mesh); });

        if (PlayerState.isLocalPlayer(this.gameObject)) {
            this._syncTransforms = GameObject.getComponentsInChildren(this.gameObject, SyncedTransform);
        }
    }


    private async loadAvatarObjects(head: AssetReference, left: AssetReference, right: AssetReference) {
        const pHead = head.loadAssetAsync();
        const pHandLeft = left.loadAssetAsync();
        const pHandRight = right.loadAssetAsync();
        const promises = new Array<Promise<any>>();
        if (pHead) promises.push(pHead);
        if (pHandLeft) promises.push(pHandLeft);
        if (pHandRight) promises.push(pHandRight);
        const res = await PromiseAllWithErrors(promises);
        if (debug) console.log("Avatar loaded results:", res);
    }
}