@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
1,206 lines (1,201 loc) • 80.6 kB
JavaScript
import '../../../engine/engine_shims.js';
import { BufferAttribute, Color, DoubleSide, LinearFilter, Material, MathUtils, Matrix4, Mesh, MeshBasicMaterial, MeshPhysicalMaterial, MeshStandardMaterial, OrthographicCamera, PerspectiveCamera, PlaneGeometry, Quaternion, RGBAFormat, Scene, ShaderMaterial, SkinnedMesh, SRGBColorSpace, Uniform, UnsignedByteType, Vector3, Vector4, WebGLRenderer, WebGLRenderTarget } from 'three';
import * as fflate from 'three/examples/jsm/libs/fflate.module.js';
import { VERSION } from "../../../engine/engine_constants.js";
import { Progress } from '../../../engine/engine_time_utils.js';
import { buildNodeMaterial } from './extensions/NodeMaterialConverter.js';
function makeNameSafe(str) {
// Remove characters that are not allowed in USD ASCII identifiers
str = str.replace(/[^a-zA-Z0-9_]/g, '');
// If name doesn't start with a-zA-Z_, add _ to the beginning – required by USD
if (!str.match(/^[a-zA-Z_]/))
str = '_' + str;
return str;
}
function makeDisplayNameSafe(str) {
str = str.replace("\"", "\\\"");
return str;
}
// TODO check if this works when bones in the skeleton are completely unordered
function findCommonAncestor(objects) {
if (objects.length === 0)
return null;
const ancestors = objects.map((obj) => {
const objAncestors = new Array();
while (obj.parent) {
objAncestors.unshift(obj.parent);
obj = obj.parent;
}
return objAncestors;
});
//@ts-ignore – findLast seems to be missing in TypeScript types pre-5.x
const commonAncestor = ancestors[0].findLast((ancestor) => {
return ancestors.every((a) => a.includes(ancestor));
});
return commonAncestor || null;
}
function findStructuralNodesInBoneHierarchy(bones) {
const commonAncestor = findCommonAncestor(bones);
// find all structural nodes – parents of bones that are not bones themselves
const structuralNodes = new Set();
for (const bone of bones) {
let current = bone.parent;
while (current && current !== commonAncestor) {
if (!bones.includes(current)) {
structuralNodes.add(current);
}
current = current.parent;
}
}
return structuralNodes;
}
const PositionIdentity = new Vector3();
const QuaternionIdentity = new Quaternion();
const ScaleIdentity = new Vector3(1, 1, 1);
class USDObject {
static USDObject_export_id = 0;
uuid;
name;
/** If no type is provided, type is chosen automatically (Xform or Mesh) */
type;
/** MaterialBindingAPI and SkelBindingAPI are handled automatically, extra schemas can be added here */
extraSchemas = [];
displayName;
visibility; // defaults to "inherited" in USD
getMatrix() {
if (!this.transform)
return new Matrix4();
const { position, quaternion, scale } = this.transform;
const matrix = new Matrix4();
matrix.compose(position || PositionIdentity, quaternion || QuaternionIdentity, scale || ScaleIdentity);
return matrix;
}
setMatrix(value) {
if (!value || !(value instanceof Matrix4)) {
this.transform = null;
return;
}
const position = new Vector3();
const quaternion = new Quaternion();
const scale = new Vector3();
value.decompose(position, quaternion, scale);
this.transform = { position, quaternion, scale };
}
/** @deprecated Use `transform`, or `getMatrix()` if you really need the matrix */
get matrix() { return this.getMatrix(); }
/** @deprecated Use `transform`, or `setMatrix()` if you really need the matrix */
set matrix(value) { this.setMatrix(value); }
transform = null;
_isDynamic;
get isDynamic() { return this._isDynamic; }
set isDynamic(value) { this._isDynamic = value; }
geometry;
material;
// usdMaterial?: USDMaterial;
camera;
parent;
skinnedMesh;
children = [];
animations;
_eventListeners;
// these are for tracking which xformops are needed
needsTranslate = false;
needsOrient = false;
needsScale = false;
static createEmptyParent(object) {
const emptyParent = new USDObject(MathUtils.generateUUID(), object.name + '_empty_' + (USDObject.USDObject_export_id++), object.transform);
const parent = object.parent;
if (parent)
parent.add(emptyParent);
emptyParent.add(object);
emptyParent.isDynamic = true;
object.transform = null;
return emptyParent;
}
static createEmpty() {
const empty = new USDObject(MathUtils.generateUUID(), 'Empty_' + (USDObject.USDObject_export_id++));
empty.isDynamic = true;
return empty;
}
constructor(id, name, transform = null, mesh = null, material = null, camera = null, skinnedMesh = null, animations = null) {
this.uuid = id;
this.name = makeNameSafe(name);
this.displayName = name;
if (!transform)
this.transform = null;
else
this.transform = {
position: transform.position?.clone() || null,
quaternion: transform.quaternion?.clone() || null,
scale: transform.scale?.clone() || null
};
this.geometry = mesh;
this.material = material;
this.camera = camera;
this.parent = null;
this.children = [];
this._eventListeners = {};
this._isDynamic = false;
this.skinnedMesh = skinnedMesh;
this.animations = animations;
}
is(obj) {
if (!obj)
return false;
return this.uuid === obj.uuid;
}
isEmpty() {
return !this.geometry;
}
clone() {
const clone = new USDObject(MathUtils.generateUUID(), this.name, this.transform, this.geometry, this.material);
clone.isDynamic = this.isDynamic;
return clone;
}
deepClone() {
const clone = this.clone();
for (const child of this.children) {
if (!child)
continue;
clone.add(child.deepClone());
}
return clone;
}
getPath() {
let current = this.parent;
let path = this.name;
while (current) {
// StageRoot has a special path right now since there's additional Xforms for encapsulation.
// Better would be to actually model them as part of our object graph, but they're written separately,
// so currently we don't and instead work around that here.
const currentName = current.parent ? current.name : (current.name + "/Scenes/Scene");
path = currentName + '/' + path;
current = current.parent;
}
return '</' + path + '>';
}
add(child) {
if (child.parent) {
child.parent.remove(child);
}
child.parent = this;
this.children.push(child);
}
remove(child) {
const index = this.children.indexOf(child);
if (index >= 0) {
if (child.parent === this)
child.parent = null;
this.children.splice(index, 1);
}
}
addEventListener(evt, listener) {
if (!this._eventListeners[evt])
this._eventListeners[evt] = [];
this._eventListeners[evt].push(listener);
}
removeEventListener(evt, listener) {
if (!this._eventListeners[evt])
return;
const index = this._eventListeners[evt].indexOf(listener);
if (index >= 0) {
this._eventListeners[evt].splice(index, 1);
}
}
onSerialize(writer, context) {
const listeners = this._eventListeners['serialize'];
if (listeners)
listeners.forEach(listener => listener(writer, context));
}
}
// #region USDMaterial
// class MaterialInput {
// name: string;
// }
class USDMaterial {
static USDMaterial_id = 0;
material;
id;
name;
isOverride = false;
isInstanceable = false;
constructor(material) {
this.material = material;
this.id = USDMaterial.USDMaterial_id++;
this.name = makeNameSafe(material.name || 'Material_' + this.id);
}
inputs = {};
// addInput( name: string, value: "" ) {
// }
serialize(writer, _context) {
const name = this.name;
writer.appendLine(`def Material "${this.name}" ${name ? `( displayName = "${makeDisplayNameSafe(name)}" )` : ''}`);
writer.beginBlock();
writer.closeBlock();
// def Material "${materialName}" ${material.name ?`(
// displayName = "${material.name}"
// )` : ''}
// {
// token outputs:mtlx:surface.connect = ${materialRoot}/${materialName}/Occlusion.outputs:out>
// def Shader "Occlusion"
// {
// uniform token info:id = "${mode}"
// token outputs:out
// }
// }`;
}
}
// #region USDDocument
class USDDocument extends USDObject {
stageLength;
get isDocumentRoot() {
return true;
}
get isDynamic() {
return false;
}
constructor() {
super(undefined, 'StageRoot', null, null, null, null);
this.children = [];
this.stageLength = 200;
}
add(child) {
child.parent = this;
this.children.push(child);
}
remove(child) {
const index = this.children.indexOf(child);
if (index >= 0) {
if (child.parent === this)
child.parent = null;
this.children.splice(index, 1);
}
}
traverse(callback, current = null) {
if (current !== null)
callback(current);
else
current = this;
if (current.children) {
for (const child of current.children) {
this.traverse(callback, child);
}
}
}
findById(uuid) {
let found = false;
function search(current) {
if (found)
return undefined;
if (current.uuid === uuid) {
found = true;
return current;
}
if (current.children) {
for (const child of current.children) {
if (!child)
continue;
const res = search(child);
if (res)
return res;
}
}
return undefined;
}
return search(this);
}
buildHeader(_context) {
const animationExtension = _context.extensions?.find(ext => ext?.extensionName === 'animation');
const behaviorExtension = _context.extensions?.find(ext => ext?.extensionName === 'Behaviour');
const physicsExtension = _context.extensions?.find(ext => ext?.extensionName === 'Physics');
const startTimeCode = animationExtension?.getStartTimeCode() ?? 0;
const endTimeCode = animationExtension?.getEndTimeCode() ?? 0;
let comment = "";
const registeredClips = animationExtension?.registeredClips;
if (registeredClips) {
for (const clip of registeredClips) {
comment += `\t# Animation: ${clip.name}, start=${animationExtension.getStartTimeByClip(clip) * 60}, length=${clip.duration * 60}\n`;
}
}
const comments = comment;
return `#usda 1.0
(
customLayerData = {
string creator = "Needle Engine ${VERSION}"
dictionary Needle = {
bool animations = ${animationExtension ? 1 : 0}
bool interactive = ${behaviorExtension ? 1 : 0}
bool physics = ${physicsExtension ? 1 : 0}
bool quickLookCompatible = ${_context.quickLookCompatible ? 1 : 0}
}
}
defaultPrim = "${makeNameSafe(this.name)}"
metersPerUnit = 1
upAxis = "Y"
startTimeCode = ${startTimeCode}
endTimeCode = ${endTimeCode}
timeCodesPerSecond = 60
framesPerSecond = 60
doc = """Generated by Needle Engine USDZ Exporter ${VERSION}"""
${comments}
)
`;
}
}
const newLine = '\n';
const materialRoot = '</StageRoot/Materials';
class USDWriter {
str;
indent;
constructor() {
this.str = '';
this.indent = 0;
}
clear() {
this.str = '';
this.indent = 0;
}
beginBlock(str = undefined, char = '{', createNewLine = true) {
if (str !== undefined) {
str = this.applyIndent(str);
this.str += str;
if (createNewLine) {
this.str += newLine;
this.str += this.applyIndent(char);
}
else {
this.str += " " + char;
}
}
else {
this.str += this.applyIndent(char);
}
this.str += newLine;
this.indent += 1;
}
closeBlock(char = '}') {
this.indent -= 1;
this.str += this.applyIndent(char) + newLine;
}
beginArray(str) {
str = this.applyIndent(str + ' = [');
this.str += str;
this.str += newLine;
this.indent += 1;
}
closeArray() {
this.indent -= 1;
this.str += this.applyIndent(']') + newLine;
}
appendLine(str = '') {
str = this.applyIndent(str);
this.str += str;
this.str += newLine;
}
toString() {
return this.str;
}
applyIndent(str) {
let indents = '';
for (let i = 0; i < this.indent; i++)
indents += '\t';
return indents + str;
}
}
// #region USDZExporterContext
class USDZExporterContext {
root;
exporter;
extensions = [];
quickLookCompatible;
exportInvisible;
materials;
textures;
files;
document;
output;
animations;
constructor(root, exporter, options) {
this.root = root || undefined;
this.exporter = exporter;
this.quickLookCompatible = options.quickLookCompatible;
this.exportInvisible = options.exportInvisible;
if (options.extensions)
this.extensions = options.extensions;
this.materials = new Map();
this.textures = {};
this.files = {};
this.document = new USDDocument();
this.output = '';
this.animations = [];
}
makeNameSafe(str) {
return makeNameSafe(str);
}
}
const getDefaultExporterOptions = () => {
return {
ar: {
anchoring: { type: 'plane' },
planeAnchoring: { alignment: 'horizontal' }
},
quickLookCompatible: false,
extensions: [],
maxTextureSize: 4096,
exportInvisible: false
};
};
// #region USDZExporter
class USDZExporter {
debug;
pruneUnusedNodes;
sceneAnchoringOptions = getDefaultExporterOptions();
extensions = [];
keepObject;
beforeWritingDocument;
constructor() {
this.debug = false;
this.pruneUnusedNodes = true;
}
async parse(scene, options = getDefaultExporterOptions()) {
// clone options to avoid modifying the original object
options = Object.assign({}, options);
this.sceneAnchoringOptions = options;
const context = new USDZExporterContext(scene, this, options);
this.extensions = context.extensions;
const files = context.files;
const modelFileName = 'model.usda';
// model file should be first in USDZ archive so we init it here
files[modelFileName] = null;
const materials = context.materials;
const textures = context.textures;
Progress.report('export-usdz', "Invoking onBeforeBuildDocument");
await invokeAll(context, 'onBeforeBuildDocument');
Progress.report('export-usdz', "Done onBeforeBuildDocument");
Progress.report('export-usdz', "Reparent bones to common ancestor");
// Find all skeletons and reparent them to their skelroot / armature / uppermost bone parent.
// This may not be correct in all cases.
const reparentings = [];
const allReparentingObjects = new Set();
scene?.traverse(object => {
if (!options.exportInvisible && !object.visible)
return;
if (object instanceof SkinnedMesh) {
const bones = object.skeleton.bones;
const commonAncestor = findCommonAncestor(bones);
if (commonAncestor) {
const newReparenting = { object, originalParent: object.parent, newParent: commonAncestor };
reparentings.push(newReparenting);
// keep track of which nodes are important for skeletal export consistency
allReparentingObjects.add(newReparenting.object.uuid);
if (newReparenting.newParent)
allReparentingObjects.add(newReparenting.newParent.uuid);
if (newReparenting.originalParent)
allReparentingObjects.add(newReparenting.originalParent.uuid);
}
}
});
for (const reparenting of reparentings) {
const { object, originalParent, newParent } = reparenting;
newParent.add(object);
}
Progress.report('export-usdz', "Traversing hierarchy");
if (scene)
traverse(scene, context.document, context, this.keepObject);
// Root object should have identity matrix
// so that root transformations don't end up in the resulting file.
// if (context.document.children?.length > 0)
// context.document.children[0]?.matrix.identity(); //.multiply(new Matrix4().makeRotationY(Math.PI));
Progress.report('export-usdz', "Invoking onAfterBuildDocument");
await invokeAll(context, 'onAfterBuildDocument');
// At this point, we know all animated objects, all skinned mesh objects, and all objects targeted by behaviors.
// We can prune all empty nodes (no geometry or material) depth-first.
// This avoids unnecessary export of e.g. animated bones as nodes when they have no children
// (for example, a sword attached to a hand still needs that entire hierarchy exported)
const behaviorExt = context.extensions.find(ext => ext.extensionName === 'Behaviour');
const allBehaviorTargets = behaviorExt?.getAllTargetUuids() ?? new Set();
// Prune pass. Depth-first removal of nodes that don't affect the outcome of the scene.
if (this.pruneUnusedNodes) {
const options = {
allBehaviorTargets,
debug: false,
boneReparentings: allReparentingObjects,
quickLookCompatible: context.quickLookCompatible,
};
if (this.debug)
logUsdHierarchy(context.document, "Hierarchy BEFORE pruning", options);
prune(context.document, options);
if (this.debug)
logUsdHierarchy(context.document, "Hierarchy AFTER pruning");
}
else if (this.debug) {
console.log("Pruning of empty nodes is disabled. This may result in a larger USDZ file.");
}
Progress.report('export-usdz', { message: "Parsing document", autoStep: 10 });
await parseDocument(context, options);
Progress.report("export-usdz", "Invoking onAfterSerialize");
await invokeAll(context, 'onAfterSerialize');
// repair the parenting again
for (const reparenting of reparentings) {
const { object, originalParent, newParent } = reparenting;
if (originalParent)
originalParent.add(object);
}
// Moved into parseDocument callback for proper defaultPrim encapsulation
// context.output += buildMaterials( materials, textures, options.quickLookCompatible );
// callback for validating after all export has been done
context.exporter?.beforeWritingDocument?.();
const header = context.document.buildHeader(context);
const final = header + '\n' + context.output;
// full output file
if (this.debug)
console.debug(final);
files[modelFileName] = fflate.strToU8(final);
context.output = '';
Progress.report("export-usdz", { message: "Exporting textures", autoStep: 10 });
Progress.start("export-usdz-textures", { parentScope: "export-usdz", logTimings: false });
const decompressionRenderer = new WebGLRenderer({
antialias: false,
alpha: true,
premultipliedAlpha: false,
preserveDrawingBuffer: true
});
const textureCount = Object.keys(textures).length;
Progress.report("export-usdz-textures", { totalSteps: textureCount * 3, currentStep: 0 });
const convertTexture = async (id) => {
const textureData = textures[id];
const texture = textureData.texture;
const isRGBA = formatsWithAlphaChannel.includes(texture.format);
// Change: we need to always read back the texture now, otherwise the unpremultiplied workflow doesn't work.
let img = {
imageData: texture.image
};
Progress.report("export-usdz-textures", { message: "read back texture", autoStep: true });
const anyColorScale = textureData.scale !== undefined && textureData.scale.x !== 1 && textureData.scale.y !== 1 && textureData.scale.z !== 1 && textureData.scale.w !== 1;
// @ts-ignore
if (texture.isCompressedTexture || texture.isRenderTargetTexture || anyColorScale) {
img = await decompressGpuTexture(texture, options.maxTextureSize, decompressionRenderer, textureData.scale);
}
Progress.report("export-usdz-textures", { message: "convert texture to canvas", autoStep: true });
const canvas = await imageToCanvasUnpremultiplied(img.imageBitmap || img.imageData, options.maxTextureSize).catch(err => {
console.error("Error converting texture to canvas", texture, err);
});
if (canvas) {
Progress.report("export-usdz-textures", { message: "convert canvas to blob", autoStep: true });
const blob = await canvas.convertToBlob({ type: isRGBA ? 'image/png' : 'image/jpeg', quality: 0.95 });
files[`textures/${id}.${isRGBA ? 'png' : 'jpg'}`] = new Uint8Array(await blob.arrayBuffer());
}
else {
console.warn('Can`t export texture: ', texture);
}
};
for (const id in textures) {
await convertTexture(id);
}
decompressionRenderer.dispose();
// 64 byte alignment
// https://github.com/101arrowz/fflate/issues/39#issuecomment-777263109
Progress.end("export-usdz-textures");
let offset = 0;
for (const filename in files) {
const file = files[filename];
const headerSize = 34 + filename.length;
offset += headerSize;
const offsetMod64 = offset & 63;
if (offsetMod64 !== 4) {
const padLength = 64 - offsetMod64;
const padding = new Uint8Array(padLength);
files[filename] = [file, { extra: { 12345: padding } }];
}
offset = file.length;
}
Progress.report("export-usdz", "zip archive");
return fflate.zipSync(files, { level: 0 });
}
}
// #endregion
// #region traverse
function traverse(object, parentModel, context, keepObject) {
if (!context.exportInvisible && !object.visible)
return;
let model = undefined;
let geometry = undefined;
let material = undefined;
const transform = { position: object.position, quaternion: object.quaternion, scale: object.scale };
if (object.position.x === 0 && object.position.y === 0 && object.position.z === 0)
transform.position = null;
if (object.quaternion.x === 0 && object.quaternion.y === 0 && object.quaternion.z === 0 && object.quaternion.w === 1)
transform.quaternion = null;
if (object.scale.x === 1 && object.scale.y === 1 && object.scale.z === 1)
transform.scale = null;
if (object instanceof Mesh || object instanceof SkinnedMesh) {
geometry = object.geometry;
material = object.material;
}
// API for an explicit choice to discard this object – for example, a geometry that should not be exported,
// but childs should still be exported.
if (keepObject && !keepObject(object)) {
geometry = undefined;
material = undefined;
}
if ((object instanceof Mesh || object instanceof SkinnedMesh) &&
material && typeof material === 'object' &&
(material instanceof MeshStandardMaterial ||
material instanceof MeshBasicMaterial ||
// material instanceof MeshPhysicalNodeMaterial ||
material.isMeshPhysicalNodeMaterial ||
(material instanceof Material && material.type === "MeshLineMaterial"))) {
const name = getObjectId(object);
const skinnedMeshObject = object instanceof SkinnedMesh ? object : null;
model = new USDObject(object.uuid, name, transform, geometry, material, undefined, skinnedMeshObject, object.animations);
}
else if (object instanceof PerspectiveCamera || object instanceof OrthographicCamera) {
const name = getObjectId(object);
model = new USDObject(object.uuid, name, transform, undefined, undefined, object);
}
else {
const name = getObjectId(object);
model = new USDObject(object.uuid, name, transform, undefined, undefined, undefined, undefined, object.animations);
}
if (model) {
model.displayName = object.userData?.name || object.name;
model.visibility = object.visible ? undefined : "invisible";
if (parentModel) {
parentModel.add(model);
}
parentModel = model;
if (context.extensions) {
for (const ext of context.extensions) {
if (ext.onExportObject)
ext.onExportObject.call(ext, object, model, context);
}
}
}
else {
const name = getObjectId(object);
const empty = new USDObject(object.uuid, name, { position: object.position, quaternion: object.quaternion, scale: object.scale });
if (parentModel) {
parentModel.add(empty);
}
parentModel = empty;
}
for (const ch of object.children) {
traverse(ch, parentModel, context, keepObject);
}
}
// #endregion
function logUsdHierarchy(object, prefix, ...extraLogObjects) {
const item = {};
let itemCount = 0;
function collectItem(object, current) {
itemCount++;
let name = object.displayName || object.name;
name += " (" + object.uuid + ")";
const hasAny = object.geometry || object.material || object.camera || object.skinnedMesh;
if (hasAny) {
name += " (" + (object.geometry ? "geo, " : "") + (object.material ? "mat, " : "") + (object.camera ? "cam, " : "") + (object.skinnedMesh ? "skin, " : "") + ")";
}
current[name] = {};
const props = { object };
if (object.material)
props['mat'] = true;
if (object.geometry)
props['geo'] = true;
if (object.camera)
props['cam'] = true;
if (object.skinnedMesh)
props['skin'] = true;
current[name]._self = props;
for (const child of object.children) {
if (child) {
collectItem(child, current[name]);
}
}
}
collectItem(object, item);
console.log(prefix + " (" + itemCount + " nodes)", item, ...extraLogObjects);
}
function prune(object, options) {
let allChildsWerePruned = true;
const prunedChilds = new Array();
const keptChilds = new Array();
if (object.children.length === 0) {
allChildsWerePruned = true;
}
else {
const childs = [...object.children];
for (const child of childs) {
if (child) {
const childWasPruned = prune(child, options);
if (options.debug) {
if (childWasPruned)
prunedChilds.push(child);
else
keptChilds.push(child);
}
allChildsWerePruned = allChildsWerePruned && childWasPruned;
}
}
}
// check if this object is referenced by any behavior
const isBehaviorSourceOrTarget = options.allBehaviorTargets.has(object.uuid);
// check if this object has any material or geometry
const isVisible = object.geometry || object.material || (object.camera && !options.quickLookCompatible) || object.skinnedMesh || false;
// check if this object is part of any reparenting
const isBoneReparenting = options.boneReparentings.has(object.uuid);
const canBePruned = allChildsWerePruned && !isBehaviorSourceOrTarget && !isVisible && !isBoneReparenting;
if (canBePruned) {
if (options.debug)
console.log("Pruned object:", (object.displayName || object.name) + " (" + object.uuid + ")", {
isVisible,
isBehaviorSourceOrTarget,
allChildsWerePruned,
isBoneReparenting,
object,
prunedChilds,
keptChilds
});
object.parent?.remove(object);
}
else {
if (options.debug)
console.log("Kept object:", (object.displayName || object.name) + " (" + object.uuid + ")", {
isVisible,
isBehaviorSourceOrTarget,
allChildsWerePruned,
isBoneReparenting,
object,
prunedChilds,
keptChilds
});
}
// if it has no children and is not a behavior source or target, and is not visible, prune it
return canBePruned;
}
// #region parseDocument
async function parseDocument(context, options) {
Progress.start("export-usdz-resources", "export-usdz");
const resources = [];
for (const child of context.document.children) {
addResources(child, context, resources);
}
// addResources now only collects promises for better progress reporting.
// We are resolving them here and reporting progress on that:
const total = resources.length;
for (let i = 0; i < total; i++) {
Progress.report("export-usdz-resources", { totalSteps: total, currentStep: i });
await new Promise((resolve, _reject) => {
resources[i]();
resolve();
});
}
Progress.end("export-usdz-resources");
const writer = new USDWriter();
const arAnchoringOptions = context.exporter.sceneAnchoringOptions.ar;
writer.beginBlock(`def Xform "${context.document.name}"`);
writer.beginBlock(`def Scope "Scenes" (
kind = "sceneLibrary"
)`);
writer.beginBlock(`def Xform "Scene"`, '(', false);
writer.appendLine(`apiSchemas = ["Preliminary_AnchoringAPI"]`);
writer.appendLine(`customData = {`);
writer.appendLine(` bool preliminary_collidesWithEnvironment = 0`);
writer.appendLine(` string sceneName = "Scene"`);
writer.appendLine(`}`);
writer.appendLine(`sceneName = "Scene"`);
writer.closeBlock(')');
writer.beginBlock();
writer.appendLine(`token preliminary:anchoring:type = "${arAnchoringOptions.anchoring.type}"`);
if (arAnchoringOptions.anchoring.type === 'plane')
writer.appendLine(`token preliminary:planeAnchoring:alignment = "${arAnchoringOptions.planeAnchoring.alignment}"`);
// bit hacky as we don't have a callback here yet. Relies on the fact that the image is named identical in the ImageTracking extension.
if (arAnchoringOptions.anchoring.type === 'image')
writer.appendLine(`rel preliminary:imageAnchoring:referenceImage = </${context.document.name}/Scenes/Scene/AnchoringReferenceImage>`);
writer.appendLine();
const count = (object) => {
if (!object)
return 0;
let total = 1;
for (const child of object.children)
total += count(child);
return total;
};
const totalXformCount = count(context.document);
Progress.start("export-usdz-xforms", "export-usdz");
Progress.report("export-usdz-xforms", { totalSteps: totalXformCount, currentStep: 1 });
for (const child of context.document.children) {
buildXform(child, writer, context);
}
Progress.end("export-usdz-xforms");
Progress.report("export-usdz", "invoke onAfterHierarchy");
await invokeAll(context, 'onAfterHierarchy', writer);
writer.closeBlock();
writer.closeBlock();
// TODO property use context/writer instead of string concat
Progress.report('export-usdz', "Building materials");
const result = buildMaterials(context.materials, context.textures, options.quickLookCompatible);
writer.appendLine(result);
writer.closeBlock();
Progress.report("export-usdz", "write to string");
context.output += writer.toString();
}
// #endregion
// #region addResources
function addResources(object, context, resources) {
if (!object)
return;
const geometry = object.geometry;
const material = object.material;
if (geometry) {
if (material && ('isMeshStandardMaterial' in material && material.isMeshStandardMaterial ||
'isMeshBasicMaterial' in material && material.isMeshBasicMaterial ||
material.type === "MeshLineMaterial")) { // TODO convert unlit to lit+emissive
const geometryFileName = 'geometries/' + getGeometryName(geometry, object.name) + '.usda';
if (!(geometryFileName in context.files)) {
const action = () => {
const meshObject = buildMeshObject(geometry, object.skinnedMesh?.skeleton?.bones, context.quickLookCompatible);
context.files[geometryFileName] = buildUSDFileAsString(meshObject, context);
};
resources.push(action);
}
}
else {
console.warn('NeedleUSDZExporter: Unsupported material type (USDZ only supports MeshStandardMaterial)', material?.name);
}
}
if (material) {
if (context.materials.get(material.uuid) === undefined) {
context.materials[material.uuid] = material;
}
}
for (const ch of object.children) {
addResources(ch, context, resources);
}
}
async function invokeAll(context, name, writer = null) {
if (context.extensions) {
for (const ext of context.extensions) {
if (!ext)
continue;
if (typeof ext[name] === 'function') {
const method = ext[name];
const res = method.call(ext, context, writer);
if (res instanceof Promise) {
await res;
}
}
}
}
}
// #endregion
// #region GPU utils
let _renderer = null;
let renderTarget = null;
let fullscreenQuadGeometry;
let fullscreenQuadMaterial;
let fullscreenQuad;
/** Reads back a texture from the GPU (can be compressed, a render texture, or anything), optionally applies RGBA colorScale to it, and returns CPU data for further usage.
* Note that there are WebGL / WebGPU rules preventing some use of data between WebGL contexts.
*/
async function decompressGpuTexture(texture, maxTextureSize = Infinity, renderer = null, colorScale = undefined) {
if (!fullscreenQuadGeometry)
fullscreenQuadGeometry = new PlaneGeometry(2, 2, 1, 1);
if (!fullscreenQuadMaterial)
fullscreenQuadMaterial = new ShaderMaterial({
uniforms: {
blitTexture: new Uniform(texture),
flipY: new Uniform(false),
scale: new Uniform(new Vector4(1, 1, 1, 1)),
},
vertexShader: `
varying vec2 vUv;
uniform bool flipY;
void main(){
vUv = uv;
if (flipY)
vUv.y = 1. - vUv.y;
gl_Position = vec4(position.xy * 1.0,0.,.999999);
}`,
fragmentShader: `
uniform sampler2D blitTexture;
uniform vec4 scale;
varying vec2 vUv;
void main(){
gl_FragColor = vec4(vUv.xy, 0, 1);
#ifdef IS_SRGB
gl_FragColor = sRGBTransferOETF( texture2D( blitTexture, vUv) );
#else
gl_FragColor = texture2D( blitTexture, vUv);
#endif
gl_FragColor.rgba *= scale.rgba;
}`
});
// update uniforms
const uniforms = fullscreenQuadMaterial.uniforms;
uniforms.blitTexture.value = texture;
uniforms.flipY.value = false;
uniforms.scale.value = new Vector4(1, 1, 1, 1);
if (colorScale !== undefined)
uniforms.scale.value.copy(colorScale);
fullscreenQuadMaterial.defines.IS_SRGB = texture.colorSpace == SRGBColorSpace;
fullscreenQuadMaterial.needsUpdate = true;
if (!fullscreenQuad) {
fullscreenQuad = new Mesh(fullscreenQuadGeometry, fullscreenQuadMaterial);
fullscreenQuad.frustumCulled = false;
}
const _camera = new PerspectiveCamera();
const _scene = new Scene();
_scene.add(fullscreenQuad);
if (!renderer) {
renderer = _renderer = new WebGLRenderer({ antialias: false, alpha: true, premultipliedAlpha: false, preserveDrawingBuffer: true });
}
const width = Math.min(texture.image.width, maxTextureSize);
const height = Math.min(texture.image.height, maxTextureSize);
// dispose render target if the size is wrong
if (renderTarget && (renderTarget.width !== width || renderTarget.height !== height)) {
renderTarget.dispose();
renderTarget = null;
}
if (!renderTarget) {
renderTarget = new WebGLRenderTarget(width, height, { format: RGBAFormat, type: UnsignedByteType, minFilter: LinearFilter, magFilter: LinearFilter });
}
renderer.setRenderTarget(renderTarget);
renderer.setSize(width, height);
renderer.clear();
renderer.render(_scene, _camera);
if (_renderer) {
_renderer.dispose();
_renderer = null;
}
const buffer = new Uint8ClampedArray(renderTarget.width * renderTarget.height * 4);
renderer.readRenderTargetPixels(renderTarget, 0, 0, renderTarget.width, renderTarget.height, buffer);
const imageData = new ImageData(buffer, renderTarget.width, renderTarget.height, undefined);
const bmp = await createImageBitmap(imageData, { premultiplyAlpha: "none" });
return {
imageData,
imageBitmap: bmp
};
}
/** Checks if the given image is of a type with readable data and width/height */
function isImageBitmap(image) {
return (typeof HTMLImageElement !== 'undefined' && image instanceof HTMLImageElement) ||
(typeof HTMLCanvasElement !== 'undefined' && image instanceof HTMLCanvasElement) ||
(typeof OffscreenCanvas !== 'undefined' && image instanceof OffscreenCanvas) ||
(typeof ImageBitmap !== 'undefined' && image instanceof ImageBitmap);
}
/** This method uses a 'bitmaprenderer' context and doesn't do any pixel manipulation.
* This way, we can keep the alpha channel as it was, but we're losing the ability to do pixel manipulations or resize operations. */
async function imageToCanvasUnpremultiplied(image, maxTextureSize = 4096) {
const scale = maxTextureSize / Math.max(image.width, image.height);
const width = image.width * Math.min(1, scale);
const height = image.height * Math.min(1, scale);
const canvas = new OffscreenCanvas(width, height);
const settings = { premultiplyAlpha: "none" };
if (image.width !== width)
settings.resizeWidth = width;
if (image.height !== height)
settings.resizeHeight = height;
const imageBitmap = await createImageBitmap(image, settings);
const ctx = canvas.getContext("bitmaprenderer");
if (ctx) {
ctx.transferFromImageBitmap(imageBitmap);
}
return canvas;
}
/** This method uses a '2d' canvas context for pixel manipulation, and can apply a color scale or Y flip to the given image.
* Unfortunately, canvas always uses premultiplied data, and thus images with low alpha values (or multiplying by a=0) will result in black pixels.
*/
async function imageToCanvas(image, color = undefined, flipY = false, maxTextureSize = 4096) {
if (isImageBitmap(image)) {
// max. canvas size on Safari is still 4096x4096
const scale = maxTextureSize / Math.max(image.width, image.height);
const canvas = new OffscreenCanvas(image.width * Math.min(1, scale), image.height * Math.min(1, scale));
const context = canvas.getContext('2d', { alpha: true, premultipliedAlpha: false });
if (!context)
throw new Error('Could not get canvas 2D context');
if (flipY === true) {
context.translate(0, canvas.height);
context.scale(1, -1);
}
context.drawImage(image, 0, 0, canvas.width, canvas.height);
// Currently only used to apply opacity scale since QuickLook and usdview don't support that yet
if (color !== undefined) {
const r = color.x;
const g = color.y;
const b = color.z;
const a = color.w;
const imagedata = context.getImageData(0, 0, canvas.width, canvas.height);
const data = imagedata.data;
for (let i = 0; i < data.length; i += 4) {
data[i + 0] = data[i + 0] * r;
data[i + 1] = data[i + 1] * g;
data[i + 2] = data[i + 2] * b;
data[i + 3] = data[i + 3] * a;
}
context.putImageData(imagedata, 0, 0);
}
return canvas;
}
else {
throw new Error('NeedleUSDZExporter: No valid image data found. Unable to process texture.');
}
}
//
const PRECISION = 7;
function buildHeader() {
return `#usda 1.0
(
customLayerData = {
string creator = "Needle Engine USDZExporter"
}
metersPerUnit = 1
upAxis = "Y"
)
`;
}
function buildUSDFileAsString(dataToInsert, _context) {
let output = buildHeader();
output += dataToInsert;
return fflate.strToU8(output);
}
function getObjectId(object) {
return object.name.replace(/[-<>\(\)\[\]§$%&\/\\\=\?\,\;]/g, '') + '_' + object.id;
}
function getBoneName(bone) {
return makeNameSafe(bone.name || 'bone_' + bone.uuid);
}
function getGeometryName(geometry, _fallbackName) {
// Using object names here breaks instancing...
// So we removed fallbackName again. Downside: geometries don't have nice names...
// A better workaround would be that we actually name them on glTF import (so the geometry has a name, not the object)
return makeNameSafe(geometry.name || 'Geometry') + "_" + geometry.id;
}
function getMaterialName(material) {
return makeNameSafe(material.name || 'Material') + "_" + material.id;
}
function getPathToSkeleton(bone, assumedRoot) {
let path = getBoneName(bone);
let current = bone.parent;
while (current && current !== assumedRoot) {
path = getBoneName(current) + '/' + path;
current = current.parent;
}
return path;
}
// #endregion
// #region XForm
export function buildXform(model, writer, context) {
if (model == null)
return;
Progress.report("export-usdz-xforms", { message: "buildXform " + model.displayName || model.name, autoStep: true });
// const matrix = model.matrix;
const transform = model.transform;
const geometry = model.geometry;
const material = model.material;
const camera = model.camera;
const name = model.name;
if (model.animations) {
for (const animation of model.animations) {
context.animations.push(animation);
}
}
// const transform = buildMatrix( matrix );
/*
if ( matrix.determinant() < 0 ) {
console.warn( 'NeedleUSDZExporter: USDZ does not support negative scales', name );
}
*/
const isSkinnedMesh = geometry && geometry.isBufferGeometry && geometry.attributes.skinIndex !== undefined && geometry.attributes.skinIndex.count > 0;
const objType = isSkinnedMesh ? 'SkelRoot' : 'Xform';
const _apiSchemas = new Array();
// Specific case: the material is white unlit, the mesh has vertex colors, so we can
// export as displayColor directly
const isUnlitDisplayColor = material && material instanceof MeshBasicMaterial &&
material.color && material.color.r === 1 && material.color.g === 1 && material.color.b === 1 &&
!material.map && material.opacity === 1 &&
geometry?.attributes.color;
if (geometry?.attributes.color && !isUnlitDisplayColor) {
console.warn("NeedleUSDZExporter: Geometry has vertex colors. Vertex colors will only be shown in QuickLook for unlit materials with white color and no texture. Otherwise, they will be ignored.", model.displayName);
}
writer.appendLine();
if (geometry) {
writer.beginBlock(`def ${objType} "${name}"`, "(", false);
// NE-4084: To use the doubleSided workaround with skeletal meshes we'd have to
// also emit extra data for jointIndices etc., so we're skipping skinned meshes here.
if (context.quickLookCompatible && material && material.side === DoubleSide && !isSkinnedMesh)
writer.appendLine(`prepend references = @./geometries/${getGeometryName(geometry, name)}.usda@</Geometry_doubleSided>`);
else
writer.appendLine(`prepend references = @./geometries/${getGeometryName(geometry, name)}.usda@</Geometry>`);
if (!isUnlitDisplayColor)
_apiSchemas.push("MaterialBindingAPI");
if (isSkinnedMesh)
_apiSchemas.push("SkelBindingAPI");
}
else if (camera && !context.quickLookCompatible)
writer.beginBlock(`def Camera "${name}"`, "(", false);
else if (model.type !== undefined)
writer.beginBlock(`def ${model.type} "${name}"`);
else // if (model.type === undefined)
writer.beginBlock(`def Xform "${name}"`, "(", false);
if (model.type === undefined) {
if (model.extraSchemas?.length)
_apiSchemas.push(...model.extraSchemas);
if (_apiSchemas.length)
writer.appendLine(`prepend apiSchemas = [${_apiSchemas.map(s => `"${s}"`).join(', ')}]`);
}
if (model.displayName)
writer.appendLine(`displayName = "${makeDisplayNameSafe(model.displayName)}"`);
if (camera || model.type === undefined) {
writer.closeBlock(")");
writer.beginBlock();
}
if (geometry && material) {
if (!isUnlitDisplayColor) {
const materialName = getMaterialName(material);
writer.appendLine(`rel material:binding = </StageRoot/Materials/${materialName}>`);
}
// Turns out QuickLook / RealityKit doesn't support the doubleSided attribute, so we
// work around that by emitting additional indices above, and then we shouldn't emit the attribute either as geometry is
// already doubleSided then.
if (!context.quickLookCompatible && material.side === DoubleSide) {
// double-sided is a mesh property in USD, we can apply it as `over` here
writer.beginBlock(`over "Geometry" `);
writer.appendLine(`uniform bool doubleSided = 1`);
writer.closeBlock();
}
}
let haveWrittenAnyXformOps = false;
if (isSkinnedMesh) {
writer.appendLine(`rel skel:skeleton = <Rig>`);
writer.appendLine(`rel skel:animationSource = <Rig/_anim>`);
haveWrittenAnyXformOps = false;
// writer.appendLine( `matrix4d xformOp:transform = ${buildMatrix(new Matrix4())}` ); // always identity / in world space
}
else if (model.type === undefined) {
if (transform) {
haveWrittenAnyXformOps = haveWrittenAnyXformOps || (transform.position !== null || transform.quaternion !== null || transform.scale !== null);
if (transform.position) {
model.needsTranslate = true;
writer.appendLine(`double3 xformOp:translate = (${fn(transform.position.x)}, ${fn(transform.position.y)}, ${fn(transform.position.z)})`);
}
if (transform.quaternion) {
model.needsOrient = true;
writer.appendLine(`quatf xformOp:orient = (${fn(transform.quaternion.w)}, ${fn(transform.quaternion.x)}, ${fn(transform.quaternion.y)}, ${fn(transform.quaternion.z)})`);
}
if (transform.scale) {
model.needsScale = true;
writer.appendLine(`double3 xformOp:scale = (${fn(transform.scale.x)}, ${fn(transform.scale.y)}, ${fn(transform.scale.z)})`);
}
}
}
if (model.visibility !== undefined)
writer.appendLine(`token visibility = "${model.visibility}"`);
if (camera && !context.quickLookCompatible) {
if ('isOrthographicCamera' in camera && camera.isOrthographicCamera) {
writer.appendLine(`float2 clippingRange = (${camera.near}, ${camera.far})`);
writer.appendLine(`float horizontalAperture = ${((Math.abs(camera.left) + Math.abs(camera.right)) * 10).toPrecision(PRECISION)}`);
writer.appendLine(`float verticalAperture = ${((Math.abs(camera.top) + Math.abs(camera.bottom)) * 10).toPrecision(PRECISION)}`);
writer.appendLine('token projection = "orthographic"');
}
else if ('isPerspectiveCamera' in camera && camera.isPerspectiveCamera) {
writer.appendLine(`float2 clippingRange = (${camera.near.toPrecision(PRECISION)}, ${c