@needle-tools/engine/src/engine-components/export/usdz/extensions/behavior/BehavioursBuilder.ts

Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.

import { Object3D } from "three";

import { getParam } from "../../../../../engine/engine_utils.js";
import { makeNameSafeForUSD,USDDocument, USDObject, USDWriter } from "../../ThreeUSDZExporter.js";
import type { RegisteredAnimationInfo } from "./../Animation.js";
import { BehaviorExtension } from "./Behaviour.js";

const debug = getParam("debugusdz");

// TODO: rename to usdz element
export interface IBehaviorElement {
    id: string;
    writeTo(document: USDDocument, writer: USDWriter);
}

export class BehaviorModel {

    static global_id: number = 0;
    id: string;
    trigger: IBehaviorElement | IBehaviorElement[];
    action: IBehaviorElement;
    exclusive: boolean = false;

    makeExclusive(exclusive: boolean): BehaviorModel {
        this.exclusive = exclusive;
        return this;
    }

    constructor(id: string, trigger: IBehaviorElement | IBehaviorElement[], action: IBehaviorElement) {
        this.id = "Behavior_" + makeNameSafeForUSD(id) + "_" + BehaviorModel.global_id++;
        this.trigger = trigger;
        this.action = action;

        // Special case: SceneTransition enter triggers should never run multiple times, even if the stage loops.
        // There seems to have been a change in behavior in iOS, where SceneTransition enter runs even
        // when the stage length is reached...
        // BUT unfortunately it looks like child groups are also ignored then, and their own loops stop working
        // once any parent is set to "ignore", which is different than what's in the documentation.
        /*
        if (trigger instanceof TriggerModel) {
            if (trigger.tokenId === "SceneTransition" && trigger.type === "enter") {
                const wrapper = ActionBuilder.sequence(action);
                wrapper.multiplePerformOperation = "ignore";
                this.action = wrapper;
            }
        }
        */
        // Another idea: we let actions run forever by appending a waitAction to them
        // Also doesn't seem to work... the scene start action still runs again
        /*
        if (trigger instanceof TriggerModel) {
            if (trigger.tokenId === "SceneTransition" && trigger.type === "enter") {
                const wrapper = ActionBuilder.sequence(action, ActionBuilder.waitAction(9999999999));
                this.action = wrapper;
            }
        }
        */
    }

    writeTo(_ext: BehaviorExtension, document: USDDocument, writer: USDWriter) {
        if (!this.trigger || !this.action) return;
        writer.beginBlock(`def Preliminary_Behavior "${this.id}"`);
        let triggerString = "";
        if (Array.isArray(this.trigger)) {
            triggerString = "[";
            for (let i = 0; i < this.trigger.length; i++) {
                const tr = this.trigger[i];
                triggerString += "<" + tr.id + ">";
                if (i + 1 < this.trigger.length) triggerString += ", ";
            }
            triggerString += "]";
        }
        else
            triggerString = `<${this.trigger.id}>`;

        writer.appendLine(`rel triggers = ${triggerString}`);
        writer.appendLine(`rel actions = <${this.action.id}>`);
        writer.appendLine(`uniform bool exclusive = ${this.exclusive ? 1 : 0}`); // Apple uses 0 and 1 for bools
        writer.appendLine();
        if (Array.isArray(this.trigger)) {
            for (const trigger of this.trigger) {
                trigger.writeTo(document, writer);
                writer.appendLine();
            }
        }
        else
            this.trigger.writeTo(document, writer);
        writer.appendLine();
        this.action.writeTo(document, writer);
        writer.closeBlock();
    }
}


export type Target = USDObject | USDObject[] | Object3D | Object3D[];

const addedStrings = new Set<string>();
/** called to resolve target objects to usdz paths */
function resolve(targetObject: Target, document: USDDocument): string {
    let result: string = "";
    if (Array.isArray(targetObject)) {
        addedStrings.clear();
        let str = "[ ";
        for (let i = 0; i < targetObject.length; i++) {
            let obj = targetObject[i];
            if (!obj) {
                console.warn("Invalid target object in behavior", targetObject + ". Is the object exported?");
                continue;
            }
            if (typeof obj === "string") {
                if (addedStrings.has(obj)) continue;
                str += obj;
                addedStrings.add(obj);
            }
            else if (typeof obj === "object") {
                //@ts-ignore
                if (obj.isObject3D) {
                    //@ts-ignore
                    obj = document.findById(obj.uuid);
                    if (!obj) {
                        console.warn("Invalid target object in behavior", targetObject + ". Is the object exported?");
                        continue;
                    }
                }
                const res = (obj as any).getPath?.call(obj) as string;
                if (addedStrings.has(res)) continue;
                // console.log(str, res, addedStrings)
                str += res;
                addedStrings.add(res);
            }
            if (i + 1 < targetObject.length) str += ", ";
        }
        str += " ]";
        result = str;
        addedStrings.clear();
    }
    else if (typeof targetObject === "object") {
        const sourceObject = targetObject;
        //@ts-ignore
        if (sourceObject.isObject3D) {
            //@ts-ignore
            targetObject = document.findById(sourceObject.uuid);
        }
        if (!targetObject) {
            console.error("Invalid target object in behavior, the target object is likely missing from USDZ export. Is the object exported?", sourceObject);
            throw new Error(`Invalid target object in behavior, the target object is likely missing from USDZ export. Please report a bug. uuid: ${sourceObject.uuid}.`);
        }
        result = (targetObject as any).getPath?.call(targetObject) as string;
    }

    return result;
}

export class TriggerModel implements IBehaviorElement {
    static global_id: number = 0;

    id: string;
    targetId?: string | Target;
    tokenId?: string;
    type?: string;
    distance?: number;

    constructor(targetId?: string | Target, id?: string) {
        if (targetId) this.targetId = targetId;
        if (id) this.id = id;
        else this.id = "Trigger_" + TriggerModel.global_id++;
    }

    writeTo(document: USDDocument, writer: USDWriter) {
        writer.beginBlock(`def Preliminary_Trigger "${this.id}"`);
        if (this.targetId) {
            if (typeof this.targetId !== "string") this.targetId = resolve(this.targetId, document);
            writer.appendLine(`rel affectedObjects = ` + this.targetId);
        }
        if (this.tokenId)
            writer.appendLine(`token info:id = "${this.tokenId}"`);
        if (this.type)
            writer.appendLine(`token type = "${this.type}"`);
        if (typeof this.distance === "number")
            writer.appendLine(`double distance = ${this.distance}`);
        writer.closeBlock();
    }
}

/** Adds `RealityKit.InputTarget` child prim to enable Vision OS direct/indirect interactions
(by default, only indirect interactions are allowed) */
function addInputTargetComponent(model: USDObject, options: { direct: boolean, indirect: boolean } = { direct: true, indirect: true }) {
    const empty = USDObject.createEmpty();
    empty.name = "InputTarget_" + empty.name;
    empty.displayName = undefined;
    empty.type = "RealityKitComponent";
    empty.onSerialize = (writer: USDWriter) => {
        writer.appendLine("bool allowsDirectInput = " + (options.direct ? 1 : 0));
        writer.appendLine("bool allowsIndirectInput = " + (options.indirect ? 1 : 0));
        writer.appendLine('uniform token info:id = "RealityKit.InputTarget"');
    };
    model.add(empty);
}

export class TriggerBuilder {

    private static __sceneStartTrigger?: TriggerModel; 

    static sceneStartTrigger(): TriggerModel {
        if (this.__sceneStartTrigger !== undefined) return this.__sceneStartTrigger;
        const trigger = new TriggerModel(undefined, "SceneStart");
        trigger.tokenId = "SceneTransition";
        trigger.type = "enter";
        this.__sceneStartTrigger = trigger;
        return trigger;
    }

    /** Trigger that fires when an object has been tapped/clicked.
     * @param targetObject The object or list of objects that can be interacted with.
     * @param inputMode Input Mode (direct and/or indirect). Only available for USDObject targets. Only supported on Vision OS at the moment. */
    static tapTrigger(targetObject: Target, inputMode: { direct: boolean, indirect: boolean } = { direct: true, indirect: true }): TriggerModel {
        const trigger = new TriggerModel(targetObject);
        if (Array.isArray(targetObject) && targetObject.length > 1) {
            for (const obj of targetObject) {
                if (!(obj instanceof USDObject)) continue;
                addInputTargetComponent(obj, inputMode);
            }
        }
        else {
            if (targetObject instanceof USDObject) {
                addInputTargetComponent(targetObject, inputMode);
            }
        }
        trigger.tokenId = "TapGesture";
        return trigger;
    }

    static isTapTrigger(trigger?: TriggerModel) {
        return trigger?.tokenId === "TapGesture";
    }

    static proximityToCameraTrigger(targetObject: Target, distance: number): TriggerModel {
        const trigger = new TriggerModel(targetObject);
        trigger.tokenId = "ProximityToCamera";
        trigger.distance = distance;
        return trigger;
    }
}

export class GroupActionModel implements IBehaviorElement {

    static global_id: number = 0;
    static getId(): number {
        return this.global_id++;
    }

    id: string;
    actions: IBehaviorElement[];
    loops: number = 0;
    performCount: number = 1;
    type: string = "serial";
    multiplePerformOperation: MultiplePerformOperation | undefined = undefined;

    constructor(id: string, actions: IBehaviorElement[]) {
        this.id = id;
        this.actions = actions;
    }

    addAction(el: IBehaviorElement): GroupActionModel {
        this.actions.push(el);
        return this;
    }

    makeParallel(): GroupActionModel {
        this.type = "parallel";
        return this;
    }

    makeSequence(): GroupActionModel {
        this.type = "serial";
        return this;
    }

    makeLooping() {
        this.loops = 1;
        this.performCount = 0;
        return this;
    }

    makeRepeat(count: number) {
        this.performCount = count;
        return this;
    }

    writeTo(document: USDDocument, writer: USDWriter) {
        writer.beginBlock(`def Preliminary_Action "${this.id}"`);
        writer.beginArray("rel actions");
        for (const act of this.actions) {
            if (!act) continue;
            const isLast = act === this.actions[this.actions.length - 1];
            writer.appendLine("<" + act.id + ">" + (isLast ? "" : ", "));
        }
        writer.closeArray();
        writer.appendLine();

        writer.appendLine(`token info:id = "Group"`);
        writer.appendLine(`bool loops = ${this.loops}`);
        writer.appendLine(`int performCount = ${this.loops > 0 ? 0 : Math.max(0, this.performCount)}`);
        writer.appendLine(`token type = "${this.type}"`);
        if (typeof this.multiplePerformOperation === "string") {
            writer.appendLine(`token multiplePerformOperation = "${this.multiplePerformOperation}"`);
        }

        writer.appendLine();

        for (const act of this.actions) {
            if (!act) continue;
            act.writeTo(document, writer);
            writer.appendLine();
        }

        writer.closeBlock();
    }
}

/** @internal */
export type EmphasizeActionMotionType = "pop" | "blink" | "bounce" | "flip" | "float" | "jiggle" | "pulse" | "spin";

/** @internal */
export type VisibilityActionMotionType = "none" | "pop" | "scaleUp" | "scaleDown" | "moveLeft" | "moveRight" | "moveAbove" | "moveBelow" | "moveForward" | "moveBack";

/** @internal */
export type MotionStyle = "basic";

/** @internal */
export type Space = "relative" | "absolute";

/** @internal */
export type PlayAction = "play" | "pause" | "stop";

/** @internal */
export type AuralMode = "spatial" | "nonSpatial" | "ambient";

/** @internal */
export type VisibilityMode = "show" | "hide";

// https://developer.apple.com/documentation/arkit/usdz_schemas_for_ar/actions_and_triggers/preliminary_action/multipleperformoperation
/** @internal */
export type MultiplePerformOperation = "allow" | "ignore" | "stop";

/** @internal */
export type EaseType = "none" | "in" | "out" | "inout";

export class ActionModel implements IBehaviorElement {

    private static global_id: number = 0;

    id: string;
    tokenId?: "ChangeScene" | "Visibility" | "StartAnimation" | "Wait" | "LookAtCamera" | "Emphasize" | "Transform" | "Audio" | "Impulse";
    affectedObjects?: string | Target;
    easeType?: EaseType;;
    motionType: EmphasizeActionMotionType | VisibilityActionMotionType | undefined = undefined;
    duration?: number;
    moveDistance?: number;
    style?: MotionStyle;
    type?: Space | PlayAction | VisibilityMode; // combined types of different actions
    front?: Vec3;
    up?: Vec3;
    start?: number;
    animationSpeed?: number;
    reversed?: boolean;
    pingPong?: boolean;
    xFormTarget?: Target | string;
    audio?: string;
    gain?: number;
    auralMode?: AuralMode;
    multiplePerformOperation?: MultiplePerformOperation;
    velocity?: Vec3;

    // extra info written as comment at the beginning of the action
    comment?: string;
    animationName?: string;

    clone(): ActionModel {
        const copy = new ActionModel();
        const id = copy.id;
        Object.assign(copy, this);
        copy.id = id;
        return copy;
    }

    constructor(affectedObjects?: string | Target, id?: string) {
        if (affectedObjects) this.affectedObjects = affectedObjects;
        if (id) this.id = id;
        else this.id = "Action";
        this.id += "_" + ActionModel.global_id++;
    }

    writeTo(document: USDDocument, writer: USDWriter) {
        writer.beginBlock(`def Preliminary_Action "${this.id}"`);
        if (this.comment)
            writer.appendLine(`# ${this.comment}`);
        if (this.affectedObjects) {
            if (typeof this.affectedObjects !== "string") this.affectedObjects = resolve(this.affectedObjects, document);
            writer.appendLine('rel affectedObjects = ' + this.affectedObjects);
        }
        if (typeof this.duration === "number") {
            if (typeof this.animationSpeed === "number" && this.animationSpeed !== 1) {
                writer.appendLine(`double duration = ${this.duration / this.animationSpeed} `);
            }
            else {
                writer.appendLine(`double duration = ${this.duration} `);
            }
        }
        if (this.easeType)
            writer.appendLine(`token easeType = "${this.easeType}"`);
        if (this.tokenId)
            writer.appendLine(`token info:id = "${this.tokenId}"`);
        if (this.tokenId === "ChangeScene")
            writer.appendLine(`rel scene = </StageRoot/Scenes/Scene>`);
        if (this.motionType !== undefined)
            writer.appendLine(`token motionType = "${this.motionType}"`);
        if (typeof this.moveDistance === "number")
            writer.appendLine(`double moveDistance = ${this.moveDistance} `);
        if (this.style)
            writer.appendLine(`token style = "${this.style}"`);
        if (this.type)
            writer.appendLine(`token type = "${this.type}"`);
        if (this.front)
            writer.appendLine(`vector3d front = (${this.front.x}, ${this.front.y}, ${this.front.z})`);
        if (this.up)
            writer.appendLine(`vector3d upVector = (${this.up.x}, ${this.up.y}, ${this.up.z})`);
        if (typeof this.start === "number") {
            writer.appendLine(`double start = ${this.start} `);
        }
        if (typeof this.animationSpeed === "number") {
            writer.appendLine(`double animationSpeed = ${this.animationSpeed.toFixed(2)} `);
        }
        if (typeof this.reversed === "boolean") {
            writer.appendLine(`bool reversed = ${this.reversed}`)
        }
        if (typeof this.pingPong === "boolean") {
            writer.appendLine(`bool reverses = ${this.pingPong}`)
        }
        if (this.xFormTarget) {
            if (typeof this.xFormTarget !== "string")
                this.xFormTarget = resolve(this.xFormTarget, document);
            writer.appendLine(`rel xformTarget = ${this.xFormTarget}`)
        }
        if (typeof this.audio === "string") {
            writer.appendLine(`asset audio = @${this.audio}@`);
        }
        if (typeof this.gain ==="number") {
            writer.appendLine(`double gain = ${this.gain}`);
        }
        if (typeof this.auralMode === "string") {
            writer.appendLine(`token auralMode = "${this.auralMode}"`);
        }
        if (typeof this.multiplePerformOperation === "string") {
            writer.appendLine(`token multiplePerformOperation = "${this.multiplePerformOperation}"`);
        }
        if (typeof this.velocity === "object") {
            writer.appendLine(`vector3d velocity = (${this.velocity.x}, ${this.velocity.y}, ${this.velocity.z})`);
        }
        writer.closeBlock();
    }
}

class Vec3 {
    x: number = 0;
    y: number = 0;
    z: number = 0;

    constructor(x: number, y: number, z: number) {
        this.x = x;
        this.y = y;
        this.z = z;
    }

    static get up(): Vec3 {
        return new Vec3(0, 1, 0);
    }

    static get right(): Vec3 {
        return new Vec3(1, 0, 0);
    }

    static get forward(): Vec3 {
        return new Vec3(0, 0, 1);
    }

    static get back(): Vec3 {
        return new Vec3(0, 0, -1);
    }

    static get zero(): Vec3 {
        return new Vec3(0, 0, 0);
    }
}

export class ActionBuilder {

    static sequence(...params: IBehaviorElement[]) {
        const group = new GroupActionModel("Group_" + GroupActionModel.getId(), params);
        return group.makeSequence();
    }

    static parallel(...params: IBehaviorElement[]) {
        const group = new GroupActionModel("Group_" + GroupActionModel.getId(), params);
        return group.makeParallel();
    }

    static fadeAction(targetObject: Target, duration: number, show: boolean): ActionModel {
        const act = new ActionModel(targetObject);
        act.tokenId = "Visibility";
        act.type = show ? "show" : "hide";
        act.duration = duration;

        act.style = "basic";
        act.motionType = "none"; // only VisibilityActionMotionType allowed
        act.moveDistance = 0;
        act.easeType = "none";
        return act;
    }

    /**
     * creates an action that plays an animation
     * @param start offset in seconds!
     * @param duration in seconds! 0 means play to end
     */
    static startAnimationAction(targetObject: Target, anim: RegisteredAnimationInfo, reversed: boolean = false, pingPong: boolean = false): IBehaviorElement {
        const act = new ActionModel(targetObject);
        act.tokenId = "StartAnimation";
        
        /*
        if (targetObject instanceof USDObject) {
            act.cachedTargetObject = targetObject;
            // try to retarget the animation – this improves animation playback with overlapping roots.
            act.affectedObjects = anim.nearestAnimatedRoot;
        }
        */
        
        const start = anim.start;
        const duration = anim.duration;
        const animationSpeed = anim.speed;
        const animationName = anim.clipName;

        act.comment = `Animation: ${animationName}, start=${start * 60}, length=${duration * 60}, end=${(start + duration) * 60}`;
        act.animationName = animationName;

        // start is time in seconds, the documentation is not right here
        act.start = start;
        // duration of 0 is play to end
        act.duration = duration;
        act.animationSpeed = animationSpeed;
        act.reversed = reversed;
        act.pingPong = pingPong;
        act.multiplePerformOperation = "allow";
        if (reversed) {
            act.start -= duration;
            //console.warn("Reversed animation does currently not work. The resulting file will most likely not playback.", act.id, targetObject);
        }
        if (pingPong) {
            act.pingPong = false;
            const back = act.clone();
            back.reversed = !reversed;
            back.start = act.start;
            if (back.reversed) {
                back.start -= duration;
            }
            const group = ActionBuilder.sequence(act, back);
            return group;
        }
        // if (debug) console.log("Start Animation Action", act);
        return act;
    }

    static waitAction(duration: number): ActionModel {
        const act = new ActionModel();
        act.tokenId = "Wait";
        act.duration = duration;
        act.motionType = undefined;
        return act;
    }

    static lookAtCameraAction(targets: Target, duration?: number, front?: Vec3, up?: Vec3): ActionModel {
        const act = new ActionModel(targets);
        act.tokenId = "LookAtCamera";
        act.duration = duration === undefined ? 9999999999999 : duration;
        act.front = front ?? Vec3.forward;
        // 0,0,0 is a special case for "free look"
        // 0,1,0 is for "y-locked look-at"
        act.up = up ?? Vec3.up;
        return act;
    }

    static emphasize(targets: Target, duration: number, motionType: EmphasizeActionMotionType = "bounce", moveDistance: number = 1, style: MotionStyle = "basic") {
        const act = new ActionModel(targets);
        act.tokenId = "Emphasize";
        act.duration = duration;
        act.style = style ?? "basic";
        act.motionType = motionType;
        act.moveDistance = moveDistance;
        return act;
    }

    static transformAction(targets: Target, transformTarget: Target, duration: number, transformType: Space, easeType: EaseType = "inout") {
        const act = new ActionModel(targets);
        act.tokenId = "Transform";
        act.duration = duration;
        // Workaround for a bug in QuickLook: if duration is 0, loops stop somewhat randomly. FB13759712
        act.duration = Math.max(0.000001, duration);
        act.type = transformType;
        act.easeType = duration > 0 ? easeType : "none";
        if (Array.isArray(transformTarget)) {
            console.error("Transform target must not be an array", transformTarget);
        }
        act.xFormTarget = transformTarget;
        return act;
    }

    static playAudioAction(targets: Target, audio: string, type: PlayAction = "play", gain: number = 1, auralMode: AuralMode = "spatial") {
        const act = new ActionModel(targets);
        act.tokenId = "Audio";
        act.type = type;
        act.audio = audio;
        act.gain = gain;
        act.auralMode = auralMode;
        act.multiplePerformOperation = "allow";
        return act;
    }

    // Supported only on VisionOS, Preliminary Behaviours can affect RealityKit physics as well
    static impulseAction(targets: Target, velocity: Vec3) {
        const act = new ActionModel(targets);
        act.tokenId = "Impulse";
        act.velocity = velocity;
        return act;
    }

    // Currently doesn't work on VisionOS, see FB13761990
    /*
    static reloadSceneAction() {
        const act = new ActionModel();
        act.tokenId = "ChangeScene";
        // rel scene = ... is implicit since we only allow one scene right now
        return act;
    }
    */
}

export { Vec3 as USDVec3 }