@needle-tools/engine/src/engine/debug/debug_spatial_console.ts

Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.

import { Layers, Material, Matrix3, Matrix4, Mesh, Object3D, PerspectiveCamera, Quaternion, Texture, Vector2, Vector3, Vector4 } from "three";
import ThreeMeshUI, { Text } from "three-mesh-ui";
import type { Options } from "three-mesh-ui/build/types/core/elements/MeshUIBaseElement";

import { ContextRegistry } from "../engine_context_registry.js";
import { OneEuroFilterXYZ } from "../engine_math.js";
import { lookAtObject } from "../engine_three_utils.js";
import type { IContext, IGameObject } from "../engine_types.js";
import { isDevEnvironment } from "./debug.js";
import { onError } from "./debug_overlay.js";


let _isActive = false;

// enable the spatial console if we receive an error while in dev session and in XR
onError((...args: any[]) => {
    if (isDevEnvironment() && ContextRegistry.Current?.isInXR) {
        enableSpatialConsole(true);
        onLog("error", ...args);
    }
})


/** Enable a spatial debug console that follows the camera */
export function enableSpatialConsole(active: boolean) {
    if (active) {
        if (_isActive) return;
        _isActive = true;
        onEnable();

    } else {
        if (!_isActive) return;
        _isActive = false;
        onDisable();
    }
}


const originalConsoleMethods: { [key: string]: Function | undefined } = {
    "log": undefined,
    "warn": undefined,
    "error": undefined,
};




class SpatialMessagesHandler {

    private readonly familyName = "needle-xr";
    private root: ThreeMeshUI.Block | null = null;

    private context: IContext | null = null;
    private readonly defaultFontSize = .06;

    constructor() {
        this.ensureFont();
    }

    onEnable() {
        this.context = ContextRegistry.Current || ContextRegistry.All[0];
        this.context.pre_render_callbacks.push(this.onBeforeRender)
    }
    onDisable() {
        this.context?.pre_render_callbacks.splice(this.context?.pre_render_callbacks.indexOf(this.onBeforeRender), 1);
        this.root?.removeFromParent();
    }

    private readonly targetObject = new Object3D();
    /** this is a point in forward view of the user */
    private readonly userForwardViewPoint = new Vector3();
    private readonly oneEuroFilter = new OneEuroFilterXYZ(90, .8);
    private _lastElementRemoveTime = 0;

    private onBeforeRender = () => {
        const cam = this.context?.mainCamera as any as IGameObject;
        if (this.context && cam instanceof PerspectiveCamera) {
            const root = this.getRoot() as any as IGameObject;

            // TODO: need to figure out why this happens when entering VR (in the simulator at least)
            if (Number.isNaN(root.position.x))
                root.position.set(0, 0, 0);
            if (Number.isNaN(root.quaternion.x))
                root.quaternion.set(0, 0, 0, 1);

            this.context.scene.add(this.targetObject);

            const rigScale = this.context.xr?.rigScale ?? 1;

            const dist = 3.5 * rigScale;
            const forward = cam.worldForward;
            forward.y = 0;
            forward.normalize().multiplyScalar(dist);
            this.userForwardViewPoint.copy(cam.worldPosition).sub(forward);

            const distFromForwardView = this.targetObject.position.distanceTo(this.userForwardViewPoint);
            if (distFromForwardView > 2 * rigScale) {
                this.targetObject.position.copy(this.userForwardViewPoint);
                lookAtObject(this.targetObject, cam, true, true);
                this.targetObject.rotateY(Math.PI);
            }

            this.oneEuroFilter.filter(this.targetObject.position, root.position, this.context.time.time);
            const step = this.context.time.deltaTime;
            root.quaternion.slerp(this.targetObject.quaternion, step * 5);
            root.scale.setScalar(rigScale);

            this.targetObject.removeFromParent();
            this.context.scene.add(root);

            if (this.context.time.time - this._lastElementRemoveTime > .1) {
                this._lastElementRemoveTime = this.context.time.time;
                const now = Date.now();
                for (let i = 0; i < this._activeTexts.length; i++) {
                    const el = this._activeTexts[i];
                    if (el instanceof ThreeMeshUI.Text && now - el["_activatedTime"] > 20000) {
                        el.removeFromParent();
                        this._textBuffer.push(el);
                        this._activeTexts.splice(i, 1);
                        break;
                    }
                }
            }

        }
    }

    addLog(type: "log" | "warn" | "error", message: string) {
        const root = this.getRoot();

        const text = this.getText();

        let backgroundColor = 0xffffff;
        let fontColor = 0x000000;
        switch (type) {
            case "log":
                backgroundColor = 0xffffff;
                fontColor = 0x000000;
                break;
            case "warn":
                backgroundColor = 0xffee99;
                fontColor = 0x442200;
                break;
            case "error":
                backgroundColor = 0xffaaaa;
                fontColor = 0x770000;
                break;
        }

        if (message.length > 1000) message = message.substring(0, 1000) + "...";

        const minuteSecondMilliSecond = new Date().toISOString().split("T")[1].split(".")[0];
        text.textContent = "[" + minuteSecondMilliSecond + "] " + message;
        text.visible = true;
        text["_activatedTime"] = Date.now();
        root.add(text as any);
        this._activeTexts.push(text);
        if (this.context) this.context.scene.add(root as any);
        text.set({
            backgroundColor: backgroundColor,
            color: fontColor,
        });

        ThreeMeshUI.update();
    }

    private ensureFont() {
        let fontFamily = ThreeMeshUI.FontLibrary.getFontFamily(this.familyName);

        if (!fontFamily) {
            fontFamily = ThreeMeshUI.FontLibrary.addFontFamily(this.familyName);
            const variant = fontFamily.addVariant("normal", "normal", "./include/needle/arial-msdf.json", "./include/needle/arial.png") as any as ThreeMeshUI.FontVariant;
            /** @ts-ignore */
            variant?.addEventListener('ready', () => {
                ThreeMeshUI.update();
            });
        }
    }


    private readonly textOptions: Options = {
        fontSize: this.defaultFontSize,
        fontFamily: this.familyName,
        padding: .03,
        margin: .005,
        color: 0x000000,
        backgroundColor: 0xffffff,
        backgroundOpacity: .4,
        borderRadius: .03,
        offset: .025,
    };
    private readonly _textBuffer: ThreeMeshUI.Text[] = [];
    private readonly _activeTexts: ThreeMeshUI.Text[] = [];
    private getText(): Text {
        const root = this.getRoot();
        if (this._textBuffer.length > 0) {
            const text = this._textBuffer.pop() as any as ThreeMeshUI.Text;
            text.visible = true;
            setTimeout(() => this.disableDepthTestRecursive(text as any), 100);
            return text;
        }
        if (root.children.length > 20 && this._activeTexts.length > 0) {
            const active = this._activeTexts.shift();
            return active!;
        }
        const newText = new ThreeMeshUI.Text(this.textOptions);
        setTimeout(() => this.disableDepthTestRecursive(newText as any), 500);
        setTimeout(() => this.disableDepthTestRecursive(newText as any), 1500);
        return newText;
    }
    private disableDepthTestRecursive(obj: Object3D, level: number = 0) {
        for (let i = 0; i < obj.children.length; i++) {
            const child = obj.children[i];
            if (child instanceof Object3D) {
                this.disableDepthTestRecursive(child, level + 1);
            }
        }
        obj.renderOrder = 10 * level;
        obj.layers.set(2);
        // obj.position.z = .01 * level;
        const mat = (obj as Mesh).material as Material;
        if (mat) {
            mat.depthWrite = false;
            mat.depthTest = false;
            mat.transparent = true;
        }
        if (level === 0)
            ThreeMeshUI.update();
    }

    private getRoot() {
        if (this.root) {
            return this.root;
        }

        const fontSize = this.defaultFontSize;
        const defaultOptions: Options = {
            boxSizing: 'border-box',
            fontFamily: this.familyName,
            width: "2.6",
            fontSize: fontSize,
            color: 0x000000,
            lineHeight: 1,
            backgroundColor: 0xffffff,
            backgroundOpacity: 0,
            // borderColor: 0xffffff,
            // borderOpacity: .5,
            // borderWidth: 0.01,
            // padding: 0.01,
            whiteSpace: 'pre-wrap',
            flexDirection: 'column-reverse',
        };
        this.root = new ThreeMeshUI.Block(defaultOptions);

        return this.root;
    }
}



let messagesHandler: SpatialMessagesHandler | null = null;

function onEnable() {
    // create messages handler
    if (!messagesHandler) messagesHandler = new SpatialMessagesHandler();
    messagesHandler.onEnable();

    // save original console methods
    for (const key in originalConsoleMethods) {
        originalConsoleMethods[key] = console[key];
        let isLogging = false;
        console[key] = function () {
            // call original console method
            originalConsoleMethods[key]?.apply(console, arguments);
            // prevent circular calls
            if (isLogging) return;
            try {
                isLogging = true;
                onLog(key as "log" | "warn" | "error", ...arguments);
            } finally {
                isLogging = false;
            }
        };
    }
}
function onDisable() {
    messagesHandler?.onDisable();
    for (const key in originalConsoleMethods) {
        console[key] = originalConsoleMethods[key];
    }
}



const seen = new Map<string, any>();

function onLog(key: "log" | "warn" | "error", ...args: any[]) {
    try {
        seen.clear();
        switch (key) {
            case "log":
                messagesHandler?.addLog("log", getLogString());
                break;
            case "warn":
                messagesHandler?.addLog("warn", getLogString());
                break;
            case "error":
                messagesHandler?.addLog("error", getLogString());
                break;
        }
    }
    catch (err) {
        console.error("Error in spatial console", err);
    }
    finally {
        seen.clear();
    }


    function getLogString() {
        let str = "";
        for (let i = 0; i < args.length; i++) {
            const el = args[i];
            str += serialize(el);
            if (i < args.length - 1) str += ", ";
        }
        return str;
    }
    function serialize(value: any, level: number = 0): string {

        if (typeof value === "string") {
            return "\"" + value + "\"";
        }
        else if (typeof value === "number") {
            const hasDecimal = value % 1 !== 0;
            if (hasDecimal) {
                const str = value.toFixed(5);
                const dotIndex = str.indexOf(".");
                let i = str.length - 1;
                while (i > dotIndex && str[i] === "0") i--;
                return str.substring(0, i + 1);
            }
            return value.toString();
        }
        else if (Array.isArray(value)) {
            let res = "[";
            for (let i = 0; i < value.length; i++) {
                const val = value[i];
                res += serialize(val, level + 1);
                if (i < value.length - 1) res += ", ";
            }
            res += "]";
            return res;
        }
        else if (value === null) {
            return "null";
        }
        else if (value === undefined) {
            return "undefined";
        }
        else if (typeof value === "function") {
            return value.name + "()";
        }


        // if (value instanceof Object3D) {
        //     const name = value.name?.length > 0 ? value.name : ("object@" + (value["guid"] ?? value.uuid));;
        //     return name;
        // }
        if (value instanceof Vector2) return `(${serialize(value.x)}, ${serialize(value.y)})`;
        if (value instanceof Vector3) return `(${serialize(value.x)}, ${serialize(value.y)}, ${serialize(value.z)})`;
        if (value instanceof Vector4) return `(${serialize(value.x)}, ${serialize(value.y)}, ${serialize(value.z)}, ${serialize(value.w)})`;
        if (value instanceof Quaternion) return `(${serialize(value.x)}, ${serialize(value.y)}, ${serialize(value.z)}, ${serialize(value.w)})`;
        if (value instanceof Material) return value.name;
        if (value instanceof Texture) return value.name;
        if (value instanceof Matrix3) return `[${value.elements.join(", ")}]`;
        if (value instanceof Matrix4) return `[${value.elements.join(", ")}]`;
        if (value instanceof Layers) return value.mask.toString();

        if (typeof value === "object") {
            if (seen.has(value)) return "*";
            let res = "{\n";
            res += pad(level);
            const keys = Object.keys(value);
            let line = "";
            for (let i = 0; i < keys.length; i++) {
                const key = keys[i];
                const val = value[key];
                if (seen.has(val)) {
                    line += ""
                    continue;
                }
                seen.set(val, true);
                line += key + ":" + serialize(val, level + 1);
                if (i < keys.length - 1) line += ", ";
                if (line.length >= 60) {
                    line += "\n";
                    line += pad(level);
                    res += line;
                    line = "";
                }
            }
            res += line;
            res += "\n}";
            return res;
            // return JSON.stringify(value, (_key, value) => {
            //     if (seen.has(value)) return seen.get(value);
            //     seen.set(value, "-");
            //     const res = serialize(value);
            //     seen.set(value, res);
            //     return _key;
            // }, 1);
        }


        return value;
    }

    function pad(spaces: number) {
        let res = "";
        for (let i = 0; i < spaces; i++) {
            res += " ";
        }
        return res;
    }
}

// // this is just a hack - the spatial console should be enabled from the user or the NeedleXRSession
// if (getParam("debugwebxr") || getParam("console"))
//     setTimeout(() => enableSpatialConsole(true), 1000);