@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
1,592 lines (1,178 loc) • 79.6 kB
text/typescript
import '../../../engine/engine_shims.js';
import {
AnimationClip,
Bone,
BufferAttribute,
BufferGeometry,
Color,
DoubleSide,
InterleavedBufferAttribute,
LinearFilter,
Material,
MathUtils,
Matrix4,
Mesh,
MeshBasicMaterial,
MeshPhysicalMaterial,
MeshStandardMaterial,
Object3D,
OrthographicCamera,
PerspectiveCamera,
PlaneGeometry,
Quaternion,
RGBAFormat,
Scene,
ShaderMaterial,
SkinnedMesh,
SRGBColorSpace,
Texture,
Uniform,
UnsignedByteType,
Vector3,
Vector4,
WebGLRenderer,
WebGLRenderTarget
} from 'three';
import * as fflate from 'three/examples/jsm/libs/fflate.module.js';
import { VERSION } from "../../../engine/engine_constants.js";
import type { OffscreenCanvasExt } from '../../../engine/engine_shims.js';
import { Progress } from '../../../engine/engine_time_utils.js';
import { BehaviorExtension } from '../../api.js';
import type { IUSDExporterExtension } from './Extension.js';
import type { AnimationExtension } from './extensions/Animation.js';
import type { PhysicsExtension } from './extensions/behavior/PhysicsExtension.js';
import {buildNodeMaterial} from './extensions/NodeMaterialConverter.js';
type MeshPhysicalNodeMaterial = import("three/src/materials/nodes/MeshPhysicalNodeMaterial.js").default;
function makeNameSafe( str ) {
// Remove characters that are not allowed in USD ASCII identifiers
str = str.replace( /[^a-zA-Z0-9_]/g, '' );
// If name doesn't start with a-zA-Z_, add _ to the beginning – required by USD
if ( !str.match( /^[a-zA-Z_]/ ) )
str = '_' + str;
return str;
}
function makeDisplayNameSafe(str) {
str = str.replace("\"", "\\\"");
return str;
}
// TODO check if this works when bones in the skeleton are completely unordered
function findCommonAncestor(objects: Object3D[]): Object3D | null {
if (objects.length === 0) return null;
const ancestors = objects.map((obj) => {
const objAncestors = new Array<Object3D>();
while (obj.parent) {
objAncestors.unshift(obj.parent);
obj = obj.parent;
}
return objAncestors;
});
//@ts-ignore – findLast seems to be missing in TypeScript types pre-5.x
const commonAncestor = ancestors[0].findLast((ancestor) => {
return ancestors.every((a) => a.includes(ancestor));
});
return commonAncestor || null;
}
function findStructuralNodesInBoneHierarchy(bones: Array<Object3D>) {
const commonAncestor = findCommonAncestor(bones);
// find all structural nodes – parents of bones that are not bones themselves
const structuralNodes = new Set<Object3D>();
for ( const bone of bones ) {
let current = bone.parent;
while ( current && current !== commonAncestor ) {
if ( !bones.includes(current) ) {
structuralNodes.add(current);
}
current = current.parent;
}
}
return structuralNodes;
}
declare type USDObjectTransform = {
position: Vector3 | null;
quaternion: Quaternion | null;
scale: Vector3 | null;
}
const PositionIdentity = new Vector3();
const QuaternionIdentity = new Quaternion();
const ScaleIdentity = new Vector3(1,1,1);
class USDObject {
static USDObject_export_id = 0;
uuid: string;
name: string;
/** If no type is provided, type is chosen automatically (Xform or Mesh) */
type?: string;
/** MaterialBindingAPI and SkelBindingAPI are handled automatically, extra schemas can be added here */
extraSchemas: string[] = [];
displayName?: string;
visibility?: "inherited" | "invisible"; // defaults to "inherited" in USD
getMatrix(): Matrix4 {
if (!this.transform) return new Matrix4();
const { position, quaternion, scale } = this.transform;
const matrix = new Matrix4();
matrix.compose(position || PositionIdentity, quaternion || QuaternionIdentity, scale || ScaleIdentity);
return matrix;
}
setMatrix( value ) {
if (!value || !(value instanceof Matrix4)) {
this.transform = null;
return;
}
const position = new Vector3();
const quaternion = new Quaternion();
const scale = new Vector3();
value.decompose(position, quaternion, scale);
this.transform = { position, quaternion, scale };
}
/** @deprecated Use `transform`, or `getMatrix()` if you really need the matrix */
get matrix() { return this.getMatrix(); }
/** @deprecated Use `transform`, or `setMatrix()` if you really need the matrix */
set matrix( value ) { this.setMatrix( value ); }
transform: USDObjectTransform | null = null;
private _isDynamic: boolean;
get isDynamic() { return this._isDynamic; }
private set isDynamic( value ) { this._isDynamic = value; }
geometry: BufferGeometry | null;
material: MeshStandardMaterial | MeshBasicMaterial | Material | MeshPhysicalNodeMaterial | null;
camera: PerspectiveCamera | OrthographicCamera | null;
parent: USDObject | null;
skinnedMesh: SkinnedMesh | null;
children: Array<USDObject | null> = [];
animations: AnimationClip[] | null;
_eventListeners: {};
// these are for tracking which xformops are needed
needsTranslate: boolean = false;
needsOrient: boolean = false;
needsScale: boolean = false;
static createEmptyParent( object: USDObject ) {
const emptyParent = new USDObject( MathUtils.generateUUID(), object.name + '_empty_' + ( USDObject.USDObject_export_id ++ ), object.transform );
const parent = object.parent;
if (parent) parent.add( emptyParent );
emptyParent.add( object );
emptyParent.isDynamic = true;
object.transform = null;
return emptyParent;
}
static createEmpty() {
const empty = new USDObject( MathUtils.generateUUID(), 'Empty_' + ( USDObject.USDObject_export_id ++ ) );
empty.isDynamic = true;
return empty;
}
constructor( id, name, transform: USDObjectTransform | null = null, mesh: BufferGeometry | null = null, material: MeshStandardMaterial | MeshBasicMaterial | MeshPhysicalNodeMaterial | Material | null = null, camera: PerspectiveCamera | OrthographicCamera | null = null, skinnedMesh: SkinnedMesh | null = null, animations: AnimationClip[] | null = null ) {
this.uuid = id;
this.name = makeNameSafe( name );
this.displayName = name;
if (!transform) this.transform = null;
else this.transform = {
position: transform.position?.clone() || null,
quaternion: transform.quaternion?.clone() || null,
scale: transform.scale?.clone() || null
};
this.geometry = mesh;
this.material = material;
this.camera = camera;
this.parent = null;
this.children = [];
this._eventListeners = {};
this._isDynamic = false;
this.skinnedMesh = skinnedMesh;
this.animations = animations;
}
is( obj ) {
if ( ! obj ) return false;
return this.uuid === obj.uuid;
}
isEmpty() {
return ! this.geometry;
}
clone() {
const clone = new USDObject( MathUtils.generateUUID(), this.name, this.transform, this.geometry, this.material );
clone.isDynamic = this.isDynamic;
return clone;
}
deepClone() {
const clone = this.clone();
for ( const child of this.children ) {
if ( !child ) continue;
clone.add( child.deepClone() );
}
return clone;
}
getPath() {
let current = this.parent;
let path = this.name;
while ( current ) {
// StageRoot has a special path right now since there's additional Xforms for encapsulation.
// Better would be to actually model them as part of our object graph, but they're written separately,
// so currently we don't and instead work around that here.
const currentName = current.parent ? current.name : (current.name + "/Scenes/Scene");
path = currentName + '/' + path;
current = current.parent;
}
return '</' + path + '>';
}
add( child ) {
if ( child.parent ) {
child.parent.remove( child );
}
child.parent = this;
this.children.push( child );
}
remove( child ) {
const index = this.children.indexOf( child );
if ( index >= 0 ) {
if ( child.parent === this ) child.parent = null;
this.children.splice( index, 1 );
}
}
addEventListener( evt, listener: ( writer: USDWriter, context: USDZExporterContext ) => void ) {
if ( ! this._eventListeners[ evt ] ) this._eventListeners[ evt ] = [];
this._eventListeners[ evt ].push( listener );
}
removeEventListener( evt, listener: ( writer: USDWriter, context: USDZExporterContext ) => void ) {
if ( ! this._eventListeners[ evt ] ) return;
const index = this._eventListeners[ evt ].indexOf( listener );
if ( index >= 0 ) {
this._eventListeners[ evt ].splice( index, 1 );
}
}
onSerialize( writer, context ) {
const listeners = this._eventListeners[ 'serialize' ];
if ( listeners ) listeners.forEach( listener => listener( writer, context ) );
}
}
class USDDocument extends USDObject {
stageLength: number;
get isDocumentRoot() {
return true;
}
get isDynamic() {
return false;
}
constructor() {
super(undefined, 'StageRoot', null, null, null, null);
this.children = [];
this.stageLength = 200;
}
add( child: USDObject ) {
child.parent = this;
this.children.push( child );
}
remove( child: USDObject ) {
const index = this.children.indexOf( child );
if ( index >= 0 ) {
if ( child.parent === this ) child.parent = null;
this.children.splice( index, 1 );
}
}
traverse( callback: ( object: USDObject ) => void, current: USDObject | null = null ) {
if ( current !== null ) callback( current );
else current = this;
if ( current.children ) {
for ( const child of current.children ) {
this.traverse( callback, child );
}
}
}
findById( uuid: string ) {
let found = false;
function search( current: USDObject ): USDObject | undefined {
if ( found ) return undefined;
if ( current.uuid === uuid ) {
found = true;
return current;
}
if ( current.children ) {
for ( const child of current.children ) {
if (!child) continue;
const res = search( child );
if ( res ) return res;
}
}
return undefined;
}
return search( this );
}
buildHeader( _context: USDZExporterContext ) {
const animationExtension = _context.extensions?.find( ext => ext?.extensionName === 'animation' ) as AnimationExtension | undefined;
const behaviorExtension = _context.extensions?.find( ext => ext?.extensionName === 'Behaviour' ) as BehaviorExtension | undefined;
const physicsExtension = _context.extensions?.find( ext => ext?.extensionName === 'Physics' ) as PhysicsExtension | undefined;
const startTimeCode = animationExtension?.getStartTimeCode() ?? 0;
const endTimeCode = animationExtension?.getEndTimeCode() ?? 0;
let comment = "";
const registeredClips = animationExtension?.registeredClips;
if (registeredClips) {
for ( const clip of registeredClips ) {
comment += `\t# Animation: ${clip.name}, start=${animationExtension.getStartTimeByClip(clip) * 60}, length=${clip.duration * 60}\n`;
}
}
const comments = comment;
return `#usda 1.0
(
customLayerData = {
string creator = "Needle Engine ${VERSION}"
dictionary Needle = {
bool animations = ${animationExtension ? 1 : 0}
bool interactive = ${behaviorExtension ? 1 : 0}
bool physics = ${physicsExtension ? 1 : 0}
bool quickLookCompatible = ${_context.quickLookCompatible ? 1 : 0}
}
}
defaultPrim = "${makeNameSafe( this.name )}"
metersPerUnit = 1
upAxis = "Y"
startTimeCode = ${startTimeCode}
endTimeCode = ${endTimeCode}
timeCodesPerSecond = 60
framesPerSecond = 60
doc = """Generated by Needle Engine USDZ Exporter ${VERSION}"""
${comments}
)
`;
}
}
const newLine = '\n';
const materialRoot = '</StageRoot/Materials';
class USDWriter {
str: string;
indent: number;
constructor() {
this.str = '';
this.indent = 0;
}
clear() {
this.str = '';
this.indent = 0;
}
beginBlock( str: string | undefined = undefined, char = '{', createNewLine = true ) {
if ( str !== undefined ) {
str = this.applyIndent( str );
this.str += str;
if ( createNewLine ) {
this.str += newLine;
this.str += this.applyIndent( char );
}
else {
this.str += " " + char;
}
}
else {
this.str += this.applyIndent( char );
}
this.str += newLine;
this.indent += 1;
}
closeBlock( char = '}' ) {
this.indent -= 1;
this.str += this.applyIndent( char ) + newLine;
}
beginArray( str ) {
str = this.applyIndent( str + ' = [' );
this.str += str;
this.str += newLine;
this.indent += 1;
}
closeArray() {
this.indent -= 1;
this.str += this.applyIndent( ']' ) + newLine;
}
appendLine( str = '' ) {
str = this.applyIndent( str );
this.str += str;
this.str += newLine;
}
toString() {
return this.str;
}
applyIndent( str ) {
let indents = '';
for ( let i = 0; i < this.indent; i ++ ) indents += '\t';
return indents + str;
}
}
declare type TextureMap = {[name: string]: {texture: Texture, scale?: Vector4}};
class USDZExporterContext {
root?: Object3D;
exporter: USDZExporter;
extensions: Array<IUSDExporterExtension> = [];
quickLookCompatible: boolean;
exportInvisible: boolean;
materials: Map<string, Material>;
textures: TextureMap;
files: { [path: string]: Uint8Array | [Uint8Array, fflate.ZipOptions] | null | any }
document: USDDocument;
output: string;
animations: AnimationClip[];
constructor( root: Object3D | undefined, exporter: USDZExporter, options: {
extensions?: Array<IUSDExporterExtension>,
quickLookCompatible: boolean,
exportInvisible: boolean,
} ) {
this.root = root;
this.exporter = exporter;
this.quickLookCompatible = options.quickLookCompatible;
this.exportInvisible = options.exportInvisible;
if ( options.extensions )
this.extensions = options.extensions;
this.materials = new Map();
this.textures = {};
this.files = {};
this.document = new USDDocument();
this.output = '';
this.animations = [];
}
}
/**[documentation](https://developer.apple.com/documentation/arkit/usdz_schemas_for_ar/preliminary_anchoringapi/preliminary_anchoring_type) */
export type Anchoring = "plane" | "image" | "face" | "none"
/**[documentation](https://developer.apple.com/documentation/arkit/usdz_schemas_for_ar/preliminary_anchoringapi/preliminary_planeanchoring_alignment) */
export type Alignment = "horizontal" | "vertical" | "any";
class USDZExporterOptions {
ar: {
anchoring: { type: Anchoring },
planeAnchoring: { alignment: Alignment },
} = {
anchoring: { type: 'plane' },
planeAnchoring: { alignment: 'horizontal' }
};
quickLookCompatible: boolean = false;
extensions: any[] = [];
maxTextureSize: number = 4096;
exportInvisible: boolean = false;
}
class USDZExporter {
debug: boolean;
pruneUnusedNodes: boolean;
sceneAnchoringOptions: USDZExporterOptions = new USDZExporterOptions();
extensions: Array<IUSDExporterExtension> = [];
keepObject?: (object: Object3D) => boolean;
beforeWritingDocument?: () => void;
constructor() {
this.debug = false;
this.pruneUnusedNodes = true;
}
async parse(scene: Object3D | undefined, options: USDZExporterOptions = new USDZExporterOptions()) {
options = Object.assign( new USDZExporterOptions(), options );
this.sceneAnchoringOptions = options;
const context = new USDZExporterContext( scene, this, options );
this.extensions = context.extensions;
const files = context.files;
const modelFileName = 'model.usda';
// model file should be first in USDZ archive so we init it here
files[ modelFileName ] = null;
const materials = context.materials;
const textures = context.textures;
Progress.report('export-usdz', "Invoking onBeforeBuildDocument");
await invokeAll( context, 'onBeforeBuildDocument' );
Progress.report('export-usdz', "Done onBeforeBuildDocument");
Progress.report('export-usdz', "Reparent bones to common ancestor");
// Find all skeletons and reparent them to their skelroot / armature / uppermost bone parent.
// This may not be correct in all cases.
const reparentings: Array<{ object: Object3D, originalParent: Object3D | null, newParent: Object3D }> = [];
const allReparentingObjects = new Set<string>();
scene?.traverse(object => {
if (!options.exportInvisible && !object.visible) return;
if (object instanceof SkinnedMesh) {
const bones = object.skeleton.bones as Bone[];
const commonAncestor = findCommonAncestor(bones);
if (commonAncestor) {
const newReparenting = { object, originalParent: object.parent, newParent: commonAncestor };
reparentings.push( newReparenting );
// keep track of which nodes are important for skeletal export consistency
allReparentingObjects.add(newReparenting.object.uuid);
if (newReparenting.newParent) allReparentingObjects.add(newReparenting.newParent.uuid);
if (newReparenting.originalParent) allReparentingObjects.add(newReparenting.originalParent.uuid);
}
}
});
for ( const reparenting of reparentings ) {
const { object, originalParent, newParent } = reparenting;
newParent.add( object );
}
Progress.report('export-usdz', "Traversing hierarchy");
if (scene) traverse( scene, context.document, context, this.keepObject);
// Root object should have identity matrix
// so that root transformations don't end up in the resulting file.
// if (context.document.children?.length > 0)
// context.document.children[0]?.matrix.identity(); //.multiply(new Matrix4().makeRotationY(Math.PI));
Progress.report('export-usdz', "Invoking onAfterBuildDocument");
await invokeAll( context, 'onAfterBuildDocument' );
// At this point, we know all animated objects, all skinned mesh objects, and all objects targeted by behaviors.
// We can prune all empty nodes (no geometry or material) depth-first.
// This avoids unnecessary export of e.g. animated bones as nodes when they have no children
// (for example, a sword attached to a hand still needs that entire hierarchy exported)
const behaviorExt = context.extensions.find( ext => ext.extensionName === 'Behaviour' ) as BehaviorExtension | undefined;
const allBehaviorTargets = behaviorExt?.getAllTargetUuids() ?? new Set<string>();
// Prune pass. Depth-first removal of nodes that don't affect the outcome of the scene.
if (this.pruneUnusedNodes) {
const options = {
allBehaviorTargets,
debug: false,
boneReparentings: allReparentingObjects,
quickLookCompatible: context.quickLookCompatible,
};
if (this.debug) logUsdHierarchy(context.document, "Hierarchy BEFORE pruning", options);
prune( context.document, options );
if (this.debug) logUsdHierarchy(context.document, "Hierarchy AFTER pruning");
}
else if (this.debug) {
console.log("Pruning of empty nodes is disabled. This may result in a larger USDZ file.");
}
Progress.report('export-usdz', { message: "Parsing document", autoStep: 10 });
await parseDocument( context, () => {
// injected after stageRoot.
// TODO property use context/writer instead of string concat
Progress.report('export-usdz', "Building materials");
const result = buildMaterials( materials, textures, options.quickLookCompatible );
return result;
} );
Progress.report("export-usdz", "Invoking onAfterSerialize");
await invokeAll( context, 'onAfterSerialize' );
// repair the parenting again
for ( const reparenting of reparentings ) {
const { object, originalParent, newParent } = reparenting;
if (originalParent)
originalParent.add( object );
}
// Moved into parseDocument callback for proper defaultPrim encapsulation
// context.output += buildMaterials( materials, textures, options.quickLookCompatible );
// callback for validating after all export has been done
context.exporter?.beforeWritingDocument?.();
const header = context.document.buildHeader( context );
const final = header + '\n' + context.output;
// full output file
if ( this.debug ) console.log( final );
files[ modelFileName ] = fflate.strToU8( final );
context.output = '';
Progress.report("export-usdz", { message: "Exporting textures", autoStep: 10 });
Progress.start("export-usdz-textures", { parentScope: "export-usdz", logTimings: false });
const decompressionRenderer = new WebGLRenderer( {
antialias: false,
alpha: true,
premultipliedAlpha: false,
preserveDrawingBuffer: true
} );
const textureCount = Object.keys(textures).length;
Progress.report("export-usdz-textures", { totalSteps: textureCount * 3, currentStep: 0 });
const convertTexture = async (id: string) => {
const textureData = textures[ id ];
const texture = textureData.texture;
const isRGBA = formatsWithAlphaChannel.includes( texture.format );
// Change: we need to always read back the texture now, otherwise the unpremultiplied workflow doesn't work.
let img: ImageReadbackResult = {
imageData: texture.image
};
Progress.report("export-usdz-textures", { message: "read back texture", autoStep: true });
const anyColorScale = textureData.scale !== undefined && textureData.scale.x !== 1 && textureData.scale.y !== 1 && textureData.scale.z !== 1 && textureData.scale.w !== 1;
// @ts-ignore
if ( texture.isCompressedTexture || texture.isRenderTargetTexture || anyColorScale ) {
img = await decompressGpuTexture( texture, options.maxTextureSize, decompressionRenderer, textureData.scale );
}
Progress.report("export-usdz-textures", { message: "convert texture to canvas", autoStep: true });
const canvas = await imageToCanvasUnpremultiplied( img.imageBitmap || img.imageData, options.maxTextureSize ).catch( err => {
console.error("Error converting texture to canvas", texture, err);
});
if ( canvas ) {
Progress.report("export-usdz-textures", { message: "convert canvas to blob", autoStep: true });
const blob = await canvas.convertToBlob( {type: isRGBA ? 'image/png' : 'image/jpeg', quality: 0.95 } );
files[ `textures/${id}.${isRGBA ? 'png' : 'jpg'}` ] = new Uint8Array( await blob.arrayBuffer() );
} else {
console.warn( 'Can`t export texture: ', texture );
}
};
for ( const id in textures ) {
await convertTexture( id );
}
decompressionRenderer.dispose();
// 64 byte alignment
// https://github.com/101arrowz/fflate/issues/39#issuecomment-777263109
Progress.end("export-usdz-textures");
let offset = 0;
for ( const filename in files ) {
const file = files[ filename ];
const headerSize = 34 + filename.length;
offset += headerSize;
const offsetMod64 = offset & 63;
if ( offsetMod64 !== 4 ) {
const padLength = 64 - offsetMod64;
const padding = new Uint8Array( padLength );
files[ filename ] = [ file, { extra: { 12345: padding } } ];
}
offset = file.length;
}
Progress.report("export-usdz", "zip archive");
return fflate.zipSync( files, { level: 0 } );
}
}
function traverse( object: Object3D, parentModel: USDObject, context: USDZExporterContext, keepObject?: (object: Object3D) => boolean ) {
if (!context.exportInvisible && !object.visible) return;
let model: USDObject | undefined = undefined;
let geometry: BufferGeometry | undefined = undefined;
let material: Material | Material[] | undefined = undefined;
const transform: USDObjectTransform = { position: object.position, quaternion: object.quaternion, scale: object.scale };
if (object.position.x === 0 && object.position.y === 0 && object.position.z === 0)
transform.position = null;
if (object.quaternion.x === 0 && object.quaternion.y === 0 && object.quaternion.z === 0 && object.quaternion.w === 1)
transform.quaternion = null;
if (object.scale.x === 1 && object.scale.y === 1 && object.scale.z === 1)
transform.scale = null;
if (object instanceof Mesh || object instanceof SkinnedMesh) {
geometry = object.geometry;
material = object.material;
}
// API for an explicit choice to discard this object – for example, a geometry that should not be exported,
// but childs should still be exported.
if (keepObject && !keepObject(object)) {
geometry = undefined;
material = undefined;
}
if ( (object instanceof Mesh || object instanceof SkinnedMesh) &&
material && typeof material === 'object' &&
(material instanceof MeshStandardMaterial ||
material instanceof MeshBasicMaterial ||
// material instanceof MeshPhysicalNodeMaterial ||
(material as any).isMeshPhysicalNodeMaterial ||
(material instanceof Material && material.type === "MeshLineMaterial"))) {
const name = getObjectId( object );
const skinnedMeshObject = object instanceof SkinnedMesh ? object : null;
model = new USDObject( object.uuid, name, transform, geometry, material as any, undefined, skinnedMeshObject, object.animations );
} else if ( object instanceof PerspectiveCamera || object instanceof OrthographicCamera ) {
const name = getObjectId( object );
model = new USDObject( object.uuid, name, transform, undefined, undefined, object );
} else {
const name = getObjectId( object );
model = new USDObject( object.uuid, name, transform, undefined, undefined, undefined, undefined, object.animations );
}
if ( model ) {
model.displayName = object.userData?.name || object.name;
model.visibility = object.visible ? undefined : "invisible";
if ( parentModel ) {
parentModel.add( model );
}
parentModel = model;
if ( context.extensions ) {
for ( const ext of context.extensions ) {
if ( ext.onExportObject ) ext.onExportObject.call( ext, object, model, context );
}
}
} else {
const name = getObjectId( object );
const empty = new USDObject( object.uuid, name, { position: object.position, quaternion: object.quaternion, scale: object.scale } );
if ( parentModel ) {
parentModel.add( empty );
}
parentModel = empty;
}
for ( const ch of object.children ) {
traverse( ch, parentModel, context, keepObject );
}
}
function logUsdHierarchy( object: USDObject, prefix: string, ...extraLogObjects: any[] ) {
const item = {};
let itemCount = 0;
function collectItem( object: USDObject, current) {
itemCount++;
let name = object.displayName || object.name;
name += " (" + object.uuid + ")";
const hasAny = object.geometry || object.material || object.camera || object.skinnedMesh;
if (hasAny) {
name += " (" + (object.geometry ? "geo, " : "") + (object.material ? "mat, " : "") + (object.camera ? "cam, " : "") + (object.skinnedMesh ? "skin, " : "") + ")";
}
current[name] = {};
const props = { object };
if (object.material) props['mat'] = true;
if (object.geometry) props['geo'] = true;
if (object.camera) props['cam'] = true;
if (object.skinnedMesh) props['skin'] = true;
current[name]._self = props;
for ( const child of object.children ) {
if (child) {
collectItem(child, current[name]);
}
}
}
collectItem(object, item);
console.log(prefix + " (" + itemCount + " nodes)", item, ...extraLogObjects);
}
function prune ( object: USDObject, options : {
allBehaviorTargets: Set<string>,
debug: boolean,
boneReparentings: Set<string>,
quickLookCompatible: boolean,
} ) {
let allChildsWerePruned = true;
const prunedChilds = new Array<USDObject>();
const keptChilds = new Array<USDObject>();
if (object.children.length === 0) {
allChildsWerePruned = true;
}
else {
const childs = [...object.children];
for ( const child of childs ) {
if (child) {
const childWasPruned = prune(child, options);
if (options.debug) {
if (childWasPruned) prunedChilds.push(child);
else keptChilds.push(child);
}
allChildsWerePruned = allChildsWerePruned && childWasPruned;
}
}
}
// check if this object is referenced by any behavior
const isBehaviorSourceOrTarget = options.allBehaviorTargets.has(object.uuid);
// check if this object has any material or geometry
const isVisible = object.geometry || object.material || (object.camera && !options.quickLookCompatible) || object.skinnedMesh || false;
// check if this object is part of any reparenting
const isBoneReparenting = options.boneReparentings.has(object.uuid);
const canBePruned = allChildsWerePruned && !isBehaviorSourceOrTarget && !isVisible && !isBoneReparenting;
if (canBePruned) {
if (options.debug) console.log("Pruned object:", (object.displayName || object.name) + " (" + object.uuid + ")", {
isVisible,
isBehaviorSourceOrTarget,
allChildsWerePruned,
isBoneReparenting,
object,
prunedChilds,
keptChilds
});
object.parent?.remove(object);
}
else {
if (options.debug) console.log("Kept object:", (object.displayName || object.name) + " (" + object.uuid + ")", {
isVisible,
isBehaviorSourceOrTarget,
allChildsWerePruned,
isBoneReparenting,
object,
prunedChilds,
keptChilds
});
}
// if it has no children and is not a behavior source or target, and is not visible, prune it
return canBePruned;
}
async function parseDocument( context: USDZExporterContext, afterStageRoot: () => string ) {
Progress.start("export-usdz-resources", "export-usdz");
const resources: Array<() => void> = [];
for ( const child of context.document.children ) {
addResources( child, context, resources );
}
// addResources now only collects promises for better progress reporting.
// We are resolving them here and reporting progress on that:
const total = resources.length;
for (let i = 0; i < total; i++) {
Progress.report("export-usdz-resources", { totalSteps: total, currentStep: i });
await new Promise<void>((resolve, _reject) => {
resources[i]();
resolve();
});
}
Progress.end("export-usdz-resources");
const writer = new USDWriter();
const arAnchoringOptions = context.exporter.sceneAnchoringOptions.ar;
writer.beginBlock( `def Xform "${context.document.name}"` );
writer.beginBlock( `def Scope "Scenes" (
kind = "sceneLibrary"
)` );
writer.beginBlock( `def Xform "Scene"`, '(', false);
writer.appendLine( `apiSchemas = ["Preliminary_AnchoringAPI"]` );
writer.appendLine( `customData = {`);
writer.appendLine( ` bool preliminary_collidesWithEnvironment = 0` );
writer.appendLine( ` string sceneName = "Scene"`);
writer.appendLine( `}` );
writer.appendLine( `sceneName = "Scene"` );
writer.closeBlock( ')' );
writer.beginBlock();
writer.appendLine( `token preliminary:anchoring:type = "${arAnchoringOptions.anchoring.type}"` );
if (arAnchoringOptions.anchoring.type === 'plane')
writer.appendLine( `token preliminary:planeAnchoring:alignment = "${arAnchoringOptions.planeAnchoring.alignment}"` );
// bit hacky as we don't have a callback here yet. Relies on the fact that the image is named identical in the ImageTracking extension.
if (arAnchoringOptions.anchoring.type === 'image')
writer.appendLine( `rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>` );
writer.appendLine();
const count = (object: USDObject | null) => {
if (!object) return 0;
let total = 1;
for ( const child of object.children ) total += count( child );
return total;
}
const totalXformCount = count(context.document);
Progress.start("export-usdz-xforms", "export-usdz");
Progress.report("export-usdz-xforms", { totalSteps: totalXformCount, currentStep: 1 });
for ( const child of context.document.children ) {
buildXform( child, writer, context );
}
Progress.end("export-usdz-xforms");
Progress.report("export-usdz", "invoke onAfterHierarchy");
invokeAll( context, 'onAfterHierarchy', writer );
writer.closeBlock();
writer.closeBlock();
writer.appendLine(afterStageRoot());
writer.closeBlock();
Progress.report("export-usdz", "write to string")
context.output += writer.toString();
}
function addResources( object: USDObject | null, context: USDZExporterContext, resources: Array<() => void>) {
if ( !object ) return;
const geometry = object.geometry;
const material = object.material;
if ( geometry ) {
if ( material && (
'isMeshStandardMaterial' in material && material.isMeshStandardMaterial ||
'isMeshBasicMaterial' in material && material.isMeshBasicMaterial ||
material.type === "MeshLineMaterial"
) ) { // TODO convert unlit to lit+emissive
const geometryFileName = 'geometries/' + getGeometryName(geometry, object.name) + '.usda';
if ( ! ( geometryFileName in context.files ) ) {
const action = () => {
const meshObject = buildMeshObject( geometry, object.skinnedMesh?.skeleton?.bones, context.quickLookCompatible );
context.files[ geometryFileName ] = buildUSDFileAsString( meshObject, context);
};
resources.push(action);
}
} else {
console.warn( 'NeedleUSDZExporter: Unsupported material type (USDZ only supports MeshStandardMaterial)', material?.name );
}
}
if ( material ) {
if ( ! ( material.uuid in context.materials ) ) {
context.materials[ material.uuid ] = material;
}
}
for ( const ch of object.children ) {
addResources( ch, context, resources );
}
}
async function invokeAll( context: USDZExporterContext, name: string, writer: USDWriter | null = null ) {
if ( context.extensions ) {
for ( const ext of context.extensions ) {
if ( !ext ) continue;
if ( typeof ext[ name ] === 'function' ) {
const method = ext[ name ];
const res = method.call( ext, context, writer );
if(res instanceof Promise) {
await res;
}
}
}
}
}
let _renderer: WebGLRenderer | null = null;
let renderTarget: WebGLRenderTarget | null = null;
let fullscreenQuadGeometry: PlaneGeometry | null;
let fullscreenQuadMaterial: ShaderMaterial | null;
let fullscreenQuad: Mesh | null;
declare type ImageReadbackResult = {
imageData: ImageData;
imageBitmap?: ImageBitmap;
}
/** Reads back a texture from the GPU (can be compressed, a render texture, or anything), optionally applies RGBA colorScale to it, and returns CPU data for further usage.
* Note that there are WebGL / WebGPU rules preventing some use of data between WebGL contexts.
*/
async function decompressGpuTexture( texture, maxTextureSize = Infinity, renderer: WebGLRenderer | null = null, colorScale: Vector4 | undefined = undefined): Promise<ImageReadbackResult> {
if ( ! fullscreenQuadGeometry ) fullscreenQuadGeometry = new PlaneGeometry( 2, 2, 1, 1 );
if ( ! fullscreenQuadMaterial ) fullscreenQuadMaterial = new ShaderMaterial( {
uniforms: {
blitTexture: new Uniform( texture ),
flipY: new Uniform( false ),
scale: new Uniform( new Vector4( 1, 1, 1, 1 ) ),
},
vertexShader: `
varying vec2 vUv;
uniform bool flipY;
void main(){
vUv = uv;
if (flipY)
vUv.y = 1. - vUv.y;
gl_Position = vec4(position.xy * 1.0,0.,.999999);
}`,
fragmentShader: `
uniform sampler2D blitTexture;
uniform vec4 scale;
varying vec2 vUv;
void main(){
gl_FragColor = vec4(vUv.xy, 0, 1);
#ifdef IS_SRGB
gl_FragColor = sRGBTransferOETF( texture2D( blitTexture, vUv) );
#else
gl_FragColor = texture2D( blitTexture, vUv);
#endif
gl_FragColor.rgba *= scale.rgba;
}`
} );
// update uniforms
const uniforms = fullscreenQuadMaterial.uniforms;
uniforms.blitTexture.value = texture;
uniforms.flipY.value = false;
uniforms.scale.value = new Vector4( 1, 1, 1, 1 );
if ( colorScale !== undefined ) uniforms.scale.value.copy( colorScale );
fullscreenQuadMaterial.defines.IS_SRGB = texture.colorSpace == SRGBColorSpace;
fullscreenQuadMaterial.needsUpdate = true;
if ( ! fullscreenQuad ) {
fullscreenQuad = new Mesh( fullscreenQuadGeometry, fullscreenQuadMaterial );
fullscreenQuad.frustumCulled = false;
}
const _camera = new PerspectiveCamera();
const _scene = new Scene();
_scene.add( fullscreenQuad );
if ( ! renderer ) {
renderer = _renderer = new WebGLRenderer( { antialias: false, alpha: true, premultipliedAlpha: false, preserveDrawingBuffer: true } );
}
const width = Math.min( texture.image.width, maxTextureSize );
const height = Math.min( texture.image.height, maxTextureSize );
// dispose render target if the size is wrong
if ( renderTarget && ( renderTarget.width !== width || renderTarget.height !== height ) ) {
renderTarget.dispose();
renderTarget = null;
}
if ( ! renderTarget ) {
renderTarget = new WebGLRenderTarget( width, height, { format: RGBAFormat, type: UnsignedByteType, minFilter: LinearFilter, magFilter: LinearFilter } );
}
renderer.setRenderTarget( renderTarget );
renderer.setSize( width, height );
renderer.clear();
renderer.render( _scene, _camera );
if ( _renderer ) {
_renderer.dispose();
_renderer = null;
}
const buffer = new Uint8ClampedArray( renderTarget.width * renderTarget.height * 4 );
renderer.readRenderTargetPixels( renderTarget, 0, 0, renderTarget.width, renderTarget.height, buffer );
const imageData = new ImageData( buffer, renderTarget.width, renderTarget.height, undefined );
const bmp = await createImageBitmap( imageData, { premultiplyAlpha: "none" } );
return {
imageData,
imageBitmap: bmp
};
}
/** Checks if the given image is of a type with readable data and width/height */
function isImageBitmap( image ) {
return ( typeof HTMLImageElement !== 'undefined' && image instanceof HTMLImageElement ) ||
( typeof HTMLCanvasElement !== 'undefined' && image instanceof HTMLCanvasElement ) ||
( typeof OffscreenCanvas !== 'undefined' && image instanceof OffscreenCanvas ) ||
( typeof ImageBitmap !== 'undefined' && image instanceof ImageBitmap );
}
/** This method uses a 'bitmaprenderer' context and doesn't do any pixel manipulation.
* This way, we can keep the alpha channel as it was, but we're losing the ability to do pixel manipulations or resize operations. */
async function imageToCanvasUnpremultiplied( image: ImageBitmapSource & { width: number, height: number }, maxTextureSize = 4096) {
const scale = maxTextureSize / Math.max( image.width, image.height );
const width = image.width * Math.min( 1, scale );
const height = image.height * Math.min( 1, scale );
const canvas = new OffscreenCanvas( width, height );
const settings: ImageBitmapOptions = { premultiplyAlpha: "none" };
if (image.width !== width) settings.resizeWidth = width;
if (image.height !== height) settings.resizeHeight = height;
const imageBitmap = await createImageBitmap(image, settings);
const ctx = canvas.getContext("bitmaprenderer") as ImageBitmapRenderingContext | null;
if (ctx) {
ctx.transferFromImageBitmap(imageBitmap);
}
return canvas as OffscreenCanvasExt;
}
/** This method uses a '2d' canvas context for pixel manipulation, and can apply a color scale or Y flip to the given image.
* Unfortunately, canvas always uses premultiplied data, and thus images with low alpha values (or multiplying by a=0) will result in black pixels.
*/
async function imageToCanvas( image: HTMLImageElement | HTMLCanvasElement | OffscreenCanvas | ImageBitmap, color: Vector4 | undefined = undefined, flipY = false, maxTextureSize = 4096 ) {
if ( isImageBitmap( image ) ) {
// max. canvas size on Safari is still 4096x4096
const scale = maxTextureSize / Math.max( image.width, image.height );
const canvas = new OffscreenCanvas( image.width * Math.min( 1, scale ), image.height * Math.min( 1, scale ) );
const context = canvas.getContext( '2d', { alpha: true, premultipliedAlpha: false } ) as OffscreenCanvasRenderingContext2D;
if (!context) throw new Error('Could not get canvas 2D context');
if ( flipY === true ) {
context.translate( 0, canvas.height );
context.scale( 1, - 1 );
}
context.drawImage( image, 0, 0, canvas.width, canvas.height );
// Currently only used to apply opacity scale since QuickLook and usdview don't support that yet
if ( color !== undefined ) {
const r = color.x;
const g = color.y;
const b = color.z;
const a = color.w;
const imagedata = context.getImageData( 0, 0, canvas.width, canvas.height );
const data = imagedata.data;
for ( let i = 0; i < data.length; i += 4 ) {
data[ i + 0 ] = data[ i + 0 ] * r;
data[ i + 1 ] = data[ i + 1 ] * g;
data[ i + 2 ] = data[ i + 2 ] * b;
data[ i + 3 ] = data[ i + 3 ] * a;
}
context.putImageData( imagedata, 0, 0 );
}
return canvas as OffscreenCanvasExt;
} else {
throw new Error( 'NeedleUSDZExporter: No valid image data found. Unable to process texture.' );
}
}
//
const PRECISION = 7;
function buildHeader() {
return `#usda 1.0
(
customLayerData = {
string creator = "Needle Engine USDZExporter"
}
metersPerUnit = 1
upAxis = "Y"
)
`;
}
function buildUSDFileAsString( dataToInsert, _context: USDZExporterContext ) {
let output = buildHeader();
output += dataToInsert;
return fflate.strToU8( output );
}
function getObjectId( object ) {
return object.name.replace( /[-<>\(\)\[\]§$%&\/\\\=\?\,\;]/g, '' ) + '_' + object.id;
}
function getBoneName(bone: Object3D) {
return makeNameSafe(bone.name || 'bone_' + bone.uuid);
}
function getGeometryName(geometry: BufferGeometry, _fallbackName: string) {
// Using object names here breaks instancing...
// So we removed fallbackName again. Downside: geometries don't have nice names...
// A better workaround would be that we actually name them on glTF import (so the geometry has a name, not the object)
return makeNameSafe(geometry.name || 'Geometry') + "_" + geometry.id;
}
function getMaterialName(material: Material) {
return makeNameSafe(material.name || 'Material') + "_" + material.id;
}
function getPathToSkeleton(bone: Object3D, assumedRoot: Object3D) {
let path = getBoneName(bone);
let current = bone.parent;
while ( current && current !== assumedRoot ) {
path = getBoneName(current) + '/' + path;
current = current.parent;
}
return path;
}
// Xform
export function buildXform( model: USDObject | null, writer: USDWriter, context: USDZExporterContext ) {
if ( model == null)
return;
Progress.report("export-usdz-xforms", { message: "buildXform " + model.displayName || model.name, autoStep: true });
// const matrix = model.matrix;
const transform = model.transform;
const geometry = model.geometry;
const material = model.material;
const camera = model.camera;
const name = model.name;
if ( model.animations ) {
for ( const animation of model.animations ) {
context.animations.push( animation )
}
}
// const transform = buildMatrix( matrix );
/*
if ( matrix.determinant() < 0 ) {
console.warn( 'NeedleUSDZExporter: USDZ does not support negative scales', name );
}
*/
const isSkinnedMesh = geometry && geometry.isBufferGeometry && geometry.attributes.skinIndex !== undefined && geometry.attributes.skinIndex.count > 0;
const objType = isSkinnedMesh ? 'SkelRoot' : 'Xform';
const _apiSchemas = new Array<string>();
// Specific case: the material is white unlit, the mesh has vertex colors, so we can
// export as displayColor directly
const isUnlitDisplayColor =
material && material instanceof MeshBasicMaterial &&
material.color && material.color.r === 1 && material.color.g === 1 && material.color.b === 1 &&
!material.map && material.opacity === 1 &&
geometry?.attributes.color;
if (geometry?.attributes.color && !isUnlitDisplayColor) {
console.warn("NeedleUSDZExporter: Geometry has vertex colors. Vertex colors will only be shown in QuickLook for unlit materials with white color and no texture. Otherwise, they will be ignored.", model.displayName);
}
writer.appendLine();
if ( geometry ) {
writer.beginBlock( `def ${objType} "${name}"`, "(", false );
// NE-4084: To use the doubleSided workaround with skeletal meshes we'd have to
// also emit extra data for jointIndices etc., so we're skipping skinned meshes here.
if (context.quickLookCompatible && material && material.side === DoubleSide && !isSkinnedMesh)
writer.appendLine(`prepend references = @./geometries/${getGeometryName(geometry, name)}.usda@</Geometry_doubleSided>`);
else
writer.appendLine(`prepend references = @./geometries/${getGeometryName(geometry, name)}.usda@</Geometry>`);
if (!isUnlitDisplayColor)
_apiSchemas.push("MaterialBindingAPI");
if (isSkinnedMesh)
_apiSchemas.push("SkelBindingAPI");
}
else if ( camera && !context.quickLookCompatible)
writer.beginBlock( `def Camera "${name}"`, "(", false );
else if ( model.type !== undefined)
writer.beginBlock( `def ${model.type} "${name}"` );
else // if (model.type === undefined)
writer.beginBlock( `def Xform "${name}"`, "(", false);
if (model.type === undefined) {
if (model.extraSchemas?.length)
_apiSchemas.push(...model.extraSchemas);
if (_apiSchemas.length)
writer.appendLine(`prepend apiSchemas = [${_apiSchemas.map(s => `"${s}"`).join(', ')}]`);
}
if (model.displayName)
writer.appendLine(`displayName = "${makeDisplayNameSafe(model.displayName)}"`);
if ( camera || model.type === undefined) {
writer.closeBlock( ")" );
writer.beginBlock();
}
if ( geometry && material ) {
if (!isUnlitDisplayColor) {
const materialName = getMaterialName(material);
writer.appendLine( `rel material:binding = </StageRoot/Materials/${materialName}>` );
}
// Turns out QuickLook / RealityKit doesn't support the doubleSided attribute, so we
// work around that by emitting additional indices above, and then we shouldn't emit the attribute either as geometry is
// already doubleSided then.
if (!context.quickLookCompatible && material.side === DoubleSide ) {
// double-sided is a mesh property in USD, we can apply it as `over` here
writer.beginBlock( `over "Geometry" `);
writer.appendLine( `uniform bool doubleSided = 1` );
writer.closeBlock();
}
}
let haveWrittenAnyXformOps = false;
if ( isSkinnedMesh ) {
writer.appendLine( `rel skel:skeleton = <Rig>` );
writer.appendLine( `rel skel:animationSource = <Rig/_anim>`);
haveWrittenAnyXformOps = false;
// writer.appendLine( `matrix4d xformOp:transform = ${buildMatrix(new Matrix4())}` ); // always identity / in world space
}
else if (model.type === undefined) {
if (transform) {
haveWrittenAnyXformOps = haveWrittenAnyXformOps || (transform.position !== null || transform.quaternion !== null || transform.scale !== null);
if (transform.position) {
model.needsTranslate = true;
writer.appendLine( `double3 xformOp:translate = (${fn(transform.position.x)}, ${fn(transform.position.y)}, ${fn(transform.position.z)})` );
}
if (transform.quaternion) {
model.needsOrient = true;
writer.appendLine( `quatf xformOp:orient = (${fn(transform.quaternion.w)}, ${fn(transform.quaternion.x)}, ${fn(transform.quaternion.y)}, ${fn(transform.quaternion.z)})` );
}
if (transform.scale) {
model.needsScale = true;
writer.appendLine( `double3 xformOp:scale = (${fn(transform.scale.x)}, ${fn(transform.scale.y)}, ${fn(transform.scale.z)})` );
}
}
}
if (model.visibility !== undefined)
writer.appendLine(`token visibility = "${model.visibility}"`);
if ( camera && !context.quickLookCompatible) {
if ( 'isOrthographicCamera' in camera && camera.isOrthographicCamera ) {
writer.appendLine( `float2 clippingRange = (${camera.near}, ${camera.far})` );
writer.appendLine( `float horizontalAperture = ${( ( Math.abs( camera.left ) + Math.abs( camera.right ) ) * 10 ).toPrecision( PRECISION )}` );
writer.appendLine( `float verticalAperture = ${( ( Math.abs( camera.top ) + Math.abs( camera.bottom ) ) * 10 ).toPrecision( PRECISION )}` );
writer.appendLine( 'token projection = "orthographic"' );
} else if ( 'isPerspectiveCamera' in camera && camera.isPerspectiveCamera) {
writer.appendLine( `float2 clippingRange = (${camera.near.toPrecision( PRECISION )}, ${camera.far.toPrecision( PRECISION )})` );
writer.appendLine( `float focalLength = ${camera.getFocalLength().toPrecision( PRECISION )}` );
writer.appendLine( `float focusDistance = ${camera.focus.toPrecision( PRECISION )}` );
writer.appendLine( `float horizontalAperture = ${camera.getFilmWidth().toPrecision( PRECISION )}` );
writer.appendLine( 'token projection = "perspective"' );
writer.appendLine( `float verticalAperture = ${camera.getFilmHeight().toPrecision( PRECISION )}` );
}
}
if ( model.onSerialize ) {
model.onSerialize( writer, context );
}
// after serialization, we know which xformops to actually define here:
if (model.type === undefined) {
// TODO only write the necessary ones – this isn't trivial though because we need to know
// if some of them are animated, and t