@needle-tools/engine/src/engine/engine_gltf_builtin_components.ts

Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.

import "./codegen/register_types.js";

import { Object3D } from "three";
import { type GLTF } from "three/examples/jsm/loaders/GLTFLoader.js";

import { LogType, showBalloonMessage } from "./debug/index.js";
import { addNewComponent } from "./engine_components.js";
import { builtinComponentKeyName, editorGuidKeyName } from "./engine_constants.js";
import { debugExtension } from "./engine_default_parameters.js";
import { InstantiateIdProvider } from "./engine_networking_instantiate.js"
import { isLocalNetwork } from "./engine_networking_utils.js";
import { deserializeObject, serializeObject } from "./engine_serialization.js";
import { assign, ImplementationInformation, type ISerializable, SerializationContext } from "./engine_serialization_core.js";
import { Context } from "./engine_setup.js";
import type { GuidsMap, ICamera, ICollider, IComponent, IGameObject, IRigidbody, SourceIdentifier, UIDProvider } from "./engine_types.js";
import { TypeStore } from "./engine_typestore.js";
import { getParam } from "./engine_utils.js";
import { NEEDLE_components } from "./extensions/NEEDLE_components.js";



const debug = debugExtension;
const debugTypeStore = getParam("debugtypestore");
if (debugTypeStore) console.log(TypeStore);

export function writeBuiltinComponentData(comp: IComponent, context: SerializationContext): object | null {

    // const fn = (comp as unknown as ISerializable)?.onBeforeSerialize;
    // if (fn) {
    //     const res = fn?.call(comp);
    //     if (res !== undefined) {
    //         res["name"] = comp.constructor.name;
    //         return res;
    //     }
    // }
    const serializable = comp as unknown as ISerializable;
    const data = serializeObject(serializable, context);
    // console.log(data);
    if (data !== undefined) return data;
    return null;
}

const typeImplementationInformation = new ImplementationInformation();
const $context_deserialize_queue = Symbol("deserialize-queue");

export async function createBuiltinComponents(context: Context, gltfId: SourceIdentifier, gltf: GLTF & { children?: Array<Object3D> }, seed: number | null | UIDProvider = null, extension?: NEEDLE_components) {
    if (!gltf) {
        console.debug("Can not create component instances: gltf is null");
        return;
    }
    const lateResolve: Array<(gltf: Object3D) => {}> = [];

    let idProvider: UIDProvider | null = seed as UIDProvider;
    if (typeof idProvider === "number") {
        idProvider = new InstantiateIdProvider(seed as number);
    }

    const idEnd = gltfId.indexOf("?");
    gltfId = idEnd === -1 ? gltfId : gltfId.substring(0, idEnd);

    const serializationContext = new SerializationContext(gltf.scene);
    serializationContext.gltfId = gltfId;
    serializationContext.context = context;
    serializationContext.gltf = gltf;
    serializationContext.nodeToObject = extension?.nodeToObjectMap;
    serializationContext.implementationInformation = typeImplementationInformation;

    // If we're loading multiple gltf files in one scene we need to make sure we deserialize all of them in one go
    // for that we collect them in one list per context
    let deserializeQueue = context[$context_deserialize_queue];
    if (!deserializeQueue) deserializeQueue = context[$context_deserialize_queue] = [];

    if (gltf.scenes) {
        for (const scene of gltf.scenes) {
            await onCreateBuiltinComponents(serializationContext, scene, deserializeQueue, lateResolve);
        }
    }
    // TODO: when is the gltf here an object3d?
    if (gltf.children) {
        for (const ch of gltf.children) {
            await onCreateBuiltinComponents(serializationContext, ch, deserializeQueue, lateResolve);
        }
    }

    context.new_scripts_pre_setup_callbacks.push(() => {
        // First deserialize ALL components that were loaded before pre setup
        // Down below they get new guids assigned so we have to do all of them first
        // E.g. in cases where we load multiple glb files on startup from one scene
        // and they might have cross-glb references
        const queue = context[$context_deserialize_queue];
        if (queue) {
            for (const des of queue) {
                handleDeserialization(des, serializationContext);
            }
            queue.length = 0;
        }
        // when dropping the same file multiple times we need to generate new guids
        // e.g. SyncedTransform sends its own guid to the server to know about ownership
        // so it requires a unique guid for a new instance
        // doing it here at the end of resolving of references should ensure that
        // and this should run before awake and onenable of newly created components
        if (idProvider) {
            // TODO: should we do this after setup callbacks now?
            const guidsMap: GuidsMap = {};
            const resolveGuids: IHasResolveGuids[] = [];
            // TODO: when is the gltf here an object3d?
            recursiveCreateGuids(gltf as any, idProvider, guidsMap, resolveGuids);
            for (const scene of gltf.scenes)
                recursiveCreateGuids(scene, idProvider, guidsMap, resolveGuids);
            // make sure to resolve all guids AFTER new guids have been assigned
            for (const res of resolveGuids) {
                res.resolveGuids(guidsMap);
            }
        }
    });
}

declare type IHasResolveGuids = {
    resolveGuids: (guidsMap: GuidsMap) => void;
}

const originalComponentNameKey = Symbol("original-component-name");

/** 
 * We want to create one id provider per component
 * If a component is used multiple times we want to create a new id provider for each instance
 * That way the order of components in the scene doesnt affect the result GUID
 */
// TODO: clear this when re-loading the context
const idProviderCache = new Map<string, InstantiateIdProvider>();

function recursiveCreateGuids(obj: Object3D, idProvider: UIDProvider | null, guidsMap: GuidsMap, resolveGuids: IHasResolveGuids[]) {
    if (idProvider === null) return;
    if (!obj) return;
    const prev = obj.guid;

    // we also want to use the idproviderCache for objects because they might be removed or re-ordered in the scene hierarchy
    // in which case we dont want to e.g. change the syncedInstantiate objects that get created because suddenly another object has that guid
    const idProviderKey = obj.guid;
    if (idProviderKey?.length) {
        if (!idProviderCache.has(idProviderKey)) {
            if (debug) console.log("Creating InstanceIdProvider with key \"" + idProviderKey + "\" for object " + obj.name);
            idProviderCache.set(idProviderKey, new InstantiateIdProvider(idProviderKey));
        }
    }
    const objectIdProvider = idProviderKey && idProviderCache.get(idProviderKey) || idProvider;

    obj.guid = objectIdProvider.generateUUID();
    if (prev && prev !== "invalid")
        guidsMap[prev] = obj.guid;

    // console.log(obj);
    if (obj && obj.userData && obj.userData.components) {
        for (const comp of obj.userData.components) {
            if (comp === null) continue;

            // by default we use the component guid as a key - order of the components in the scene doesnt matter with this approach
            // this is to prevent cases where multiple GLBs are loaded with the same component guid
            const idProviderKey = comp.guid;
            if (idProviderKey) {
                if (!idProviderCache.has(idProviderKey)) {
                    if (debug) console.log("Creating InstanceIdProvider with key \"" + idProviderKey + "\" for component " + comp[originalComponentNameKey]);
                    idProviderCache.set(idProviderKey, new InstantiateIdProvider(idProviderKey));
                }
            }
            else if (debug) console.warn("Can not create IdProvider: component " + comp[originalComponentNameKey] + " has no guid", comp.guid);
            const componentIdProvider = idProviderCache.get(idProviderKey) || idProvider

            const prev = comp.guid;
            comp.guid = componentIdProvider.generateUUID();
            if (prev && prev !== "invalid")
                guidsMap[prev] = comp.guid;
            if (comp.resolveGuids)
                resolveGuids.push(comp);
        }

    }
    if (obj.children) {
        for (const child of obj.children) {
            recursiveCreateGuids(child as IGameObject, idProvider, guidsMap, resolveGuids);
        }
    }
}

declare interface IGltfbuiltinComponent {
    name: string;
}

declare interface IGltfBuiltinComponentData {
    [builtinComponentKeyName]: IGltfbuiltinComponent[];
}

declare class DeserializeData {
    instance: any;
    compData: IGltfbuiltinComponent;
    obj: Object3D;
}

declare type LateResolveCallback = (gltf: Object3D) => void;

const unknownComponentsBuffer: Array<string> = [];


async function onCreateBuiltinComponents(context: SerializationContext, obj: Object3D,
    deserialize: DeserializeData[], lateResolve: LateResolveCallback[]) {
    if (!obj) return;

    // iterate injected data
    const data = obj.userData as IGltfBuiltinComponentData;
    if (data) {
        const components = data.builtin_components;
        if (components && components.length > 0) {
            // console.log(obj);
            for (const compData of components) {
                try {
                    if (compData === null) continue;
                    const type = TypeStore.get(compData.name);
                    // console.log(compData, compData.name, type, TypeStore);
                    if (type !== undefined && type !== null) {
                        const instance: IComponent = new type() as IComponent;
                        instance.sourceId = context.gltfId;

                        // assign basic fields
                        assign(instance, compData, context.implementationInformation);

                        // make sure we assign the Needle Engine Context because Context.Current is unreliable when loading multiple <needle-engine> elements at the same time due to async processes
                        instance.context = context.context;

                        // assign the guid of the original instance
                        if ("guid" in compData)
                            instance[editorGuidKeyName] = compData.guid;

                        // we store the original component name per component which will later be used to get or initialize the InstanceIdProvider
                        instance[originalComponentNameKey] = compData.name;

                        // Object.assign(instance, compData);
                        // dont call awake here because some references might not be resolved yet and components that access those fields in awake will throw
                        // for example Duplicatable reference to object might still be { node: id }
                        const callAwake = false;
                        addNewComponent(obj, instance, callAwake);
                        deserialize.push({ instance, compData, obj });

                        // if the component instance is a camera and we dont have a main camera yet
                        // we want to assign it to the context BEFORE any component becomes active (ensuring that in awake and onEnable the mainCamera is assigned)
                        // alternatively we could try to search for the mainCamera in the getter of the context when creating the engine for the first time
                        if ((instance as ICamera).isCamera && context.context) {
                            if (context.context.mainCamera === null && (instance.tag === "MainCamera"))
                                context.context.setCurrentCamera(instance as ICamera);
                        }

                        // if the component is a rigidbody or collider and the physics engine is not initialized yet
                        // initialize the physics engine right away
                        if (context.context?.physics?.engine?.isInitialized === false && ((instance as ICollider).isCollider || (instance as IRigidbody).isRigidbody)) {
                            context.context?.physics.engine?.initialize();
                        }
                    }
                    else {
                        if (debug)
                            console.debug("unknown component: " + compData.name);
                        if (!unknownComponentsBuffer.includes(compData.name))
                            unknownComponentsBuffer.push(compData.name);
                    }
                }
                catch (err: any) {
                    console.error(compData.name + " - " + err.message, err);
                }
            }
            // console.debug("finished adding gltf builtin components", obj);
        }
        if (unknownComponentsBuffer.length > 0) {
            const unknown = unknownComponentsBuffer.join(", ");
            console.warn("unknown components: " + unknown);
            unknownComponentsBuffer.length = 0;
            if (isLocalNetwork())
                showBalloonMessage(`<strong>Unknown components in scene</strong>:\n\n${unknown}\n\nThis could mean you forgot to add a npmdef to your ExportInfo\n<a href="https://engine.needle.tools/docs/project_structure.html#creating-and-installing-a-npmdef" target="_blank">documentation</a>`, LogType.Warn);
        }
    }

    if (obj.children) {
        for (const ch of obj.children) {
            await onCreateBuiltinComponents(context, ch, deserialize, lateResolve);
        }
    }
}

function handleDeserialization(data: DeserializeData, context: SerializationContext) {
    const { instance, compData, obj } = data;
    context.object = obj;
    context.target = instance;

    // const beforeFn = (instance as ISerializationCallbackReceiver)?.onBeforeDeserialize;
    // console.log(beforeFn, instance);
    // if (beforeFn) beforeFn.call(instance, data.compData);

    let deserialized: boolean = true;
    // console.log(instance, compData);
    // TODO: first build components and then deserialize data?
    // currently a component referencing another component can not find it if the referenced component hasnt been added
    // we should split this up in two steps then.
    deserialized = deserializeObject(instance, compData, context) === true;

    // if (!deserialized) {
    //     // now loop through data again and search for special reference types
    //     for (const key in compData) {
    //         const entry = compData[key];
    //         if (!entry) {
    //             instance[key] = null;
    //             continue;
    //         }

    //         const fn = (instance as ISerializationCallbackReceiver)?.onDeserialize;
    //         if (fn) {
    //             const res = fn.call(instance, key, entry);
    //             if (res !== undefined) {
    //                 instance[key] = res;
    //                 continue;
    //             }
    //         }

    //         // if (!resolve(instance, key, entry, lateResolve)) {
    //         // }
    //     }
    // }

    // console.log(instance);
    // const afterFn = (instance as ISerializationCallbackReceiver)?.onAfterDeserialize;
    // if (afterFn) afterFn.call(instance);
    if (debug)
        console.debug("add " + compData.name, compData, instance);
}

// // TODO: THIS should be legacy once we update unity builtin component exports
// function resolve(instance, key: string, entry, lateResolve: LateResolveCallback[]): boolean {

//     switch (entry["$type"]) {
//         default:
//             const type = entry["$type"];
//             if (type !== undefined) {
//                 const res = tryResolveType(type, entry);
//                 if (res !== undefined)
//                     instance[key] = res;
//                 return res !== undefined;
//             }
//             break;
//         // the thing is a reference
//         case "reference":
//             // we expect some identifier entry to use for finding the reference
//             const guid = entry["guid"];
//             lateResolve.push(async (gltf) => {
//                 instance[key] = findInGltf(guid, gltf);
//             });
//             return true;
//     }

//     if (Array.isArray(entry)) {
//         // the thing is an array
//         const targetArray = instance[key];
//         for (const index in entry) {
//             const val = entry[index];
//             if (val === null) {
//                 targetArray[index] = null;
//                 continue;
//             }
//             switch (val["$type"]) {
//                 default:
//                     const type = val["$type"];
//                     if (type !== undefined) {
//                         const res = tryResolveType(type, entry);
//                         if (res !== undefined) targetArray[index] = res;
//                     }
//                     break;
//                 case "reference":
//                     // this entry is a reference
//                     const guid = val["guid"];
//                     lateResolve.push(async (gltf) => {
//                         targetArray[index] = findInGltf(guid, gltf);
//                     });
//                     break;
//             }
//         }
//         return true;
//     }
//     return false;
// }

// function findInGltf(guid: string, gltf) {
//     let res = tools.tryFindScript(guid);
//     if (!res) res = tools.tryFindObject(guid, gltf, true);
//     return res;
// }

// function tryResolveType(type, entry): any | undefined {
//     switch (type) {
//         case "Vector2":
//             return new Vector2(entry.x, entry.y);
//         case "Vector3":
//             return new Vector3(entry.x, entry.y, entry.z);
//         case "Vector4":
//             return new Vector4(entry.x, entry.y, entry.z, entry.w);
//         case "Quaternion":
//             return new Quaternion(entry.x, entry.y, entry.z, entry.w);
//     }
//     return undefined;
// }