@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
714 lines (638 loc) • 29.2 kB
text/typescript
import { AxesHelper, Box3, Cache, Object3D, Vector2, Vector3 } from "three";
import { isDevEnvironment } from "../engine/debug/index.js";
import { AnimationUtils } from "../engine/engine_animation.js";
import { addComponent } from "../engine/engine_components.js";
import { Context } from "../engine/engine_context.js";
import { destroy } from "../engine/engine_gameobject.js";
import { Gizmos } from "../engine/engine_gizmos.js";
import { getLoader } from "../engine/engine_gltf.js";
import { BlobStorage } from "../engine/engine_networking_blob.js";
import { PreviewHelper } from "../engine/engine_networking_files.js";
import { generateSeed, InstantiateIdProvider } from "../engine/engine_networking_instantiate.js";
import { serializable } from "../engine/engine_serialization_decorator.js";
import { fitObjectIntoVolume, getBoundingBox, placeOnSurface } from "../engine/engine_three_utils.js";
import { IGameObject, Model, Vec3 } from "../engine/engine_types.js";
import { getParam, setParamWithoutReload } from "../engine/engine_utils.js";
import { determineMimeTypeFromExtension } from "../engine/engine_utils_format.js";
import { Animation } from "./Animation.js";
import { Behaviour } from "./Component.js";
import { EventList } from "./EventList.js";
/**
* Debug mode can be enabled with the URL parameter `?debugdroplistener`, which
* logs additional information during drag and drop events and visualizes hit points.
*/
const debug = getParam("debugdroplistener");
/**
* Events dispatched by the DropListener component
* @enum {string}
*/
export enum DropListenerEvents {
/**
* Dispatched when a file is dropped into the scene. The detail of the event is the {@link File} that was dropped.
* The event is called once for each dropped file.
*/
FileDropped = "file-dropped",
/**
* Dispatched when a new object is added to the scene. The detail of the event contains {@link DropListenerOnDropArguments} for the content that was added.
*/
ObjectAdded = "object-added",
}
/**
* Context information for a drop operation
*/
declare type DropContext = {
/** Position where the file was dropped in screen coordinates */
screenposition: Vector2;
/** URL of the dropped content, if applicable */
url?: string,
/** File object of the dropped content, if applicable */
file?: File;
/** 3D position where the content should be placed */
point?: Vec3;
/** Size dimensions for the content */
size?: Vec3;
}
/**
* Network event arguments passed between clients when using the DropListener with networking
*/
export declare type DropListenerNetworkEventArguments = {
/** Unique identifier of the sender */
guid: string,
/** Name of the dropped object */
name: string,
/** URL or array of URLs to the dropped content */
url: string | string[],
/** Worldspace point where the object was placed in the scene */
point: Vec3;
/** Bounding box size */
size: Vec3;
/** MD5 hash of the content for verification */
contentMD5: string;
}
/**
* Arguments provided to handlers when an object is dropped or added to the scene
*/
export declare type DropListenerOnDropArguments = {
/** The DropListener component that processed the drop event */
sender: DropListener,
/** The root object added to the scene */
object: Object3D,
/** The complete model with all associated data */
model: Model,
/** MD5 hash of the content for verification */
contentMD5: string;
/** The original dropped URL or File object */
dropped: URL | File | undefined;
}
/**
* CustomEvent dispatched when an object is added to the scene via the DropListener
*/
class DropListenerAddedEvent<T extends DropListenerOnDropArguments> extends CustomEvent<T> {
/**
* Creates a new added event with the provided details
* @param detail Information about the added object
*/
constructor(detail: T) {
super(DropListenerEvents.ObjectAdded, { detail });
}
}
/**
* Key name used for blob storage parameters
*/
const blobKeyName = "blob";
/** The DropListener component is used to listen for drag and drop events in the browser and add the dropped files to the scene
* It can be used to allow users to drag and drop glTF files into the scene to add new objects.
*
* If {@link useNetworking} is enabled, the DropListener will automatically synchronize dropped files to other connected clients.
* Enable {@link fitIntoVolume} to automatically scale dropped objects to fit within the volume defined by {@link fitVolumeSize}.
*
* The following events are dispatched by the DropListener:
* - **object-added** - dispatched when a new object is added to the scene
* - **file-dropped** - dispatched when a file is dropped into the scene
*
* @example
* ```typescript
* import { DropListener, DropListenerEvents } from "@needle-tools/engine";
*
* const dropListener = new DropListener();
*
* gameObject.addComponent(dropListener);
* dropListener.on(DropListenerEvents.FileDropped, (evt) => {
* console.log("File dropped", evt.detail);
* const file = evt.detail as File;
* });
*
* dropListener.on(DropListenerEvents.ObjectAdded, (evt) => {
* console.log("Object added", evt.detail);
* const gltf = evt.detail as GLTF;
* });
* ```
*
* @category Asset Management
* @group Components
*/
export class DropListener extends Behaviour {
/**
* When enabled, the DropListener will automatically synchronize dropped files to other connected clients.
* When a file is dropped locally, it will be uploaded to blob storage and the URL will be shared with other clients.
*/
()
useNetworking: boolean = true;
/**
* When assigned, the DropListener will only accept files that are dropped on this specific object.
* This allows creating designated drop zones in your scene.
*/
(Object3D)
dropArea?: Object3D;
/**
* When enabled, dropped objects will be automatically scaled to fit within the volume defined by fitVolumeSize.
* Useful for ensuring dropped models appear at an appropriate scale.
* @default false
*/
()
fitIntoVolume: boolean = false;
/**
* Defines the dimensions of the volume that dropped objects will be scaled to fit within.
* Only used when fitIntoVolume is enabled.
*/
(Vector3)
fitVolumeSize = new Vector3(1, 1, 1);
/**
* When enabled, dropped objects will be positioned at the point where the cursor hit the scene.
* When disabled, objects will be placed at the origin of the DropListener.
* @default true
*/
()
placeAtHitPosition: boolean = true;
/**
* Event list that gets invoked after a file has been successfully added to the scene.
* Receives {@link DropListenerOnDropArguments} containing the added object and related information.
* @event object-added
* @example
* ```typescript
* dropListener.onDropped.addEventListener((evt) => {
* console.log("Object added", evt.model);
* });
*/
(EventList)
onDropped: EventList<DropListenerOnDropArguments> = new EventList();
/** @internal */
onEnable(): void {
this.context.renderer.domElement.addEventListener("dragover", this.onDrag);
this.context.renderer.domElement.addEventListener("drop", this.onDrop);
window.addEventListener("paste", this.handlePaste);
this.context.connection.beginListen("droplistener", this.onNetworkEvent)
}
/** @internal */
onDisable(): void {
this.context.renderer.domElement.removeEventListener("dragover", this.onDrag);
this.context.renderer.domElement.removeEventListener("drop", this.onDrop);
window.removeEventListener("paste", this.handlePaste);
this.context.connection.stopListen("droplistener", this.onNetworkEvent);
}
/**
* Loads a file from the given URL and adds it to the scene.
*/
loadFromURL(url: string, data?: { point?: Vec3, size?: Vec3 }) {
this.addFromUrl(url, { screenposition: new Vector2(), point: data?.point, size: data?.size, }, true);
}
/**
* Forgets all previously added objects.
* The droplistener will then not be able to remove previously added objects.
*/
forgetObjects() {
this.removePreviouslyAddedObjects(false);
}
/**
* Handles network events received from other clients containing information about dropped objects
* @param evt Network event data containing object information, position, and content URL
*/
private onNetworkEvent = (evt: DropListenerNetworkEventArguments) => {
if (!this.useNetworking) {
if (debug) console.debug("[DropListener] Ignoring networked event because networking is disabled", evt);
return;
}
if (evt.guid?.startsWith(this.guid)) {
const url = evt.url;
console.debug("[DropListener] Received networked event", evt);
if (url) {
if (Array.isArray(url)) {
for (const _url of url) {
this.addFromUrl(_url, { screenposition: new Vector2(), point: evt.point, size: evt.size, }, true);
}
}
else {
this.addFromUrl(url, { screenposition: new Vector2(), point: evt.point, size: evt.size }, true);
}
}
}
}
/**
* Handles clipboard paste events and processes them as potential URL drops
* Only URLs are processed by this handler, and only when editing is allowed
* @param evt The paste event
*/
private handlePaste = (evt: Event) => {
if (this.context.connection.allowEditing === false) return;
if (evt.defaultPrevented) return;
const clipboard = navigator.clipboard;
clipboard.readText()
.then(value => {
if (value) {
const isUrl = value.startsWith("http") || value.startsWith("https") || value.startsWith("blob");
if (isUrl) {
const ctx = { screenposition: new Vector2(this.context.input.mousePosition.x, this.context.input.mousePosition.y) };
if (this.testIfIsInDropArea(ctx))
this.addFromUrl(value, ctx, false);
}
}
})
.catch(console.warn);
}
/**
* Handles drag events over the renderer's canvas
* Prevents default behavior to enable drop events
* @param evt The drag event
*/
private onDrag = (evt: DragEvent) => {
if (this.context.connection.allowEditing === false) return;
// necessary to get drop event
evt.preventDefault();
}
/**
* Processes drop events to add files to the scene
* Handles both file drops and text/URL drops
* @param evt The drop event
*/
private onDrop = async (evt: DragEvent) => {
if (this.context.connection.allowEditing === false) return;
if (debug) console.log(evt);
if (!evt?.dataTransfer) return;
// If the event is marked as handled for droplisteners then ignore it
if (evt["droplistener:handled"]) return;
evt.preventDefault();
const ctx: DropContext = { screenposition: new Vector2(evt.offsetX, evt.offsetY) };
if (this.dropArea) {
const res = this.testIfIsInDropArea(ctx);
if (res === false) return;
}
// Don't stop propagation because this will break e.g. the RemoteSkybox drop
// evt.stopImmediatePropagation();
// Mark the event handled for droplisteners
evt["droplistener:handled"] = true;
const items = evt.dataTransfer.items;
if (!items) return;
const files: File[] = [];
for (const ite in items) {
const it = items[ite];
if (it.kind === "file") {
const file = it.getAsFile();
if (!file) continue;
files.push(file);
}
else if (it.kind === "string" && it.type == "text/plain") {
it.getAsString(str => {
this.addFromUrl(str, ctx, false);
});
}
}
if (files.length > 0) {
await this.addDroppedFiles(files, ctx);
}
}
/**
* Processes a dropped or pasted URL and tries to load it as a 3D model
* Handles special cases like GitHub URLs and Polyhaven asset URLs
* @param url The URL to process
* @param ctx Context information about where the drop occurred
* @param isRemote Whether this URL was shared from a remote client
* @returns The added object or null if loading failed
*/
private async addFromUrl(url: string, ctx: DropContext, isRemote: boolean) {
if (debug) console.log("dropped url", url);
try {
if (url.startsWith("https://github.com/")) {
// make raw.githubusercontent.com url
const parts = url.split("/");
const user = parts[3];
const repo = parts[4];
const branch = parts[6];
const path = parts.slice(7).join("/");
url = `https://raw.githubusercontent.com/${user}/${repo}/${branch}/${path}`;
}
else if (url.startsWith("https://polyhaven.com/a")) {
url = tryResolvePolyhavenAssetUrl(url);
}
if (!url) return null;
// Ignore dropped images
const lowercaseUrl = url.toLowerCase();
if (lowercaseUrl.endsWith(".hdr") || lowercaseUrl.endsWith(".hdri") || lowercaseUrl.endsWith(".exr") || lowercaseUrl.endsWith(".png") || lowercaseUrl.endsWith(".jpg") || lowercaseUrl.endsWith(".jpeg")) {
return null;
}
// TODO: if the URL is invalid this will become a problem
this.removePreviouslyAddedObjects();
// const binary = await fetch(url).then(res => res.arrayBuffer());
const res = await FileHelper.loadFileFromURL(new URL(url), {
guid: this.guid,
context: this.context,
parent: this.gameObject,
point: ctx.point,
size: ctx.size,
});
if (res && this._addedObjects.length <= 0) {
ctx.url = url;
const obj = this.addObject(res, ctx, isRemote);
return obj;
}
}
catch (_) {
console.warn("String is not a valid URL", url);
}
return null;
}
private _abort: AbortController | null = null;
/**
* Processes dropped files, loads them as 3D models, and handles networking if enabled
* Creates an abort controller to cancel previous uploads if new files are dropped
* @param fileList Array of dropped files
* @param ctx Context information about where the drop occurred
*/
private async addDroppedFiles(fileList: Array<File>, ctx: DropContext) {
if (debug) console.log("Add files", fileList)
if (!Array.isArray(fileList)) return;
if (!fileList.length) return;
this.deleteDropEvent();
this.removePreviouslyAddedObjects();
setParamWithoutReload(blobKeyName, null);
// Create an abort controller for the current drop operation
this._abort?.abort("New files dropped");
this._abort = new AbortController();
for (const file of fileList) {
if (!file) continue;
if (file.type.startsWith("image/")) {
// Ignore dropped images
if (debug) console.warn("Ignoring dropped image file", file.name, file.type);
continue;
}
else if (file.name.endsWith(".bin")) {
// Ignore dropped binary files
if (debug) console.warn("Ignoring dropped binary file", file.name, file.type);
continue;
}
console.debug("Load file " + file.name + " + " + file.type);
const res = await FileHelper.loadFile(file, this.context, { guid: this.guid });
if (res) {
this.dispatchEvent(new CustomEvent(DropListenerEvents.FileDropped, { detail: file }));
ctx.file = file;
const obj = this.addObject(res, ctx, false);
// handle uploading the dropped object and networking the event
if (obj && this.context.connection.isConnected && this.useNetworking) {
console.debug("Uploading dropped file to blob storage");
BlobStorage.upload(file, { abort: this._abort?.signal, })
.then(upload => {
// check if the upload was successful and if the object should still be visible
if (upload?.download_url && this._addedObjects.includes(obj)) {
// setParamWithoutReload(blobKeyName, upload.key);
this.sendDropEvent(upload.download_url, obj, res.contentMD5);
}
})
.catch(console.warn);
}
// we currently only support dropping one file
break;
}
}
}
/** Previously added objects */
private readonly _addedObjects = new Array<Object3D>();
private readonly _addedModels = new Array<Model>();
/**
* Removes all previously added objects from the scene
* @param doDestroy When true, destroys the objects; when false, just clears the references
*/
private removePreviouslyAddedObjects(doDestroy: boolean = true) {
if (doDestroy) {
for (const prev of this._addedObjects) {
if (prev.parent === this.gameObject) {
destroy(prev, true, true);
}
}
}
this._addedObjects.length = 0;
this._addedModels.length = 0;
}
/**
* Adds a loaded model to the scene with proper positioning and scaling.
* Handles placement based on component settings and raycasting.
* If {@link fitIntoVolume} is enabled, the object will be scaled to fit within the volume defined by {@link fitVolumeSize}.
* @param data The loaded model data and content hash
* @param ctx Context information about where the drop occurred
* @param isRemote Whether this object was shared from a remote client
* @returns The added object or null if adding failed
*/
private addObject(data: { model: Model, contentMD5: string }, ctx: DropContext, isRemote: boolean): Object3D | null {
const { model, contentMD5 } = data;
if (debug) console.log(`Dropped ${this.gameObject.name}`, model);
if (!model?.scene) {
console.warn("No object specified to add to scene", model);
return null;
}
this.removePreviouslyAddedObjects();
const obj = model.scene;
// use attach to ignore the DropListener scale (e.g. if the parent object scale is not uniform)
this.gameObject.attach(obj);
obj.position.set(0, 0, 0);
obj.quaternion.identity();
this._addedObjects.push(obj);
this._addedModels.push(model);
const volume = new Box3().setFromCenterAndSize(new Vector3(0, this.fitVolumeSize.y * .5, 0).add(this.gameObject.worldPosition), this.fitVolumeSize);
if (debug) Gizmos.DrawWireBox3(volume, 0x0000ff, 5);
if (this.fitIntoVolume) {
fitObjectIntoVolume(obj, volume, {
position: !this.placeAtHitPosition
});
}
if (this.placeAtHitPosition && ctx && ctx.screenposition) {
obj.visible = false; // < don't raycast on the placed object
const rc = this.context.physics.raycast({ screenPoint: this.context.input.convertScreenspaceToRaycastSpace(ctx.screenposition.clone()) });
obj.visible = true;
if (rc && rc.length > 0) {
for (const hit of rc) {
const pos = hit.point.clone();
if (debug) console.log("Place object at hit", hit);
placeOnSurface(obj, pos);
break;
}
}
}
AnimationUtils.assignAnimationsFromFile(model, {
createAnimationComponent: obj => addComponent(obj, Animation)
});
const evt = new DropListenerAddedEvent({
sender: this,
gltf: model,
model: model,
object: obj,
contentMD5: contentMD5,
dropped: ctx.file || (ctx.url ? new URL(ctx.url) : undefined),
});
this.dispatchEvent(evt);
this.onDropped?.invoke(evt.detail);
// send network event
if (!isRemote && ctx.url?.startsWith("http") && this.context.connection.isConnected && obj) {
this.sendDropEvent(ctx.url, obj, contentMD5);
}
return obj;
}
/**
* Sends a network event to other clients about a dropped object
* Only triggered when networking is enabled and the connection is established
* @param url The URL to the content that was dropped
* @param obj The object that was added to the scene
* @param contentmd5 The content hash for verification
*/
private async sendDropEvent(url: string, obj: Object3D, contentmd5: string) {
if (!this.useNetworking) {
if (debug) console.debug("[DropListener] Ignoring networked event because networking is disabled", url);
return;
}
if (this.context.connection.isConnected) {
console.debug("Sending drop event \"" + obj.name + "\"", url);
const bounds = getBoundingBox([obj]);
const evt: DropListenerNetworkEventArguments = {
name: obj.name,
guid: this.guid,
url,
point: obj.worldPosition.clone(),
size: bounds.getSize(new Vector3()),
contentMD5: contentmd5,
};
this.context.connection.send("droplistener", evt);
}
}
/**
* Deletes remote state for this DropListener's objects
* Called when new files are dropped to clean up previous state
*/
private deleteDropEvent() {
this.context.connection.sendDeleteRemoteState(this.guid);
}
/**
* Tests if a drop event occurred within the designated drop area if one is specified
* @param ctx The drop context containing screen position information
* @returns True if the drop is valid (either no drop area is set or the drop occurred inside it)
*/
private testIfIsInDropArea(ctx: DropContext): boolean {
if (this.dropArea) {
const screenPoint = this.context.input.convertScreenspaceToRaycastSpace(ctx.screenposition.clone());
const hits = this.context.physics.raycast({
targets: [this.dropArea],
screenPoint,
recursive: true,
testObject: obj => {
// Ignore hits on the already added objects, they don't count as part of the dropzone
if (this._addedObjects.includes(obj)) return false;
return true;
}
});
if (!hits.length) {
if (isDevEnvironment()) console.log(`Dropped outside of drop area for DropListener \"${this.name}\".`);
return false;
}
}
return true;
}
}
/**
* Attempts to convert a Polyhaven website URL to a direct glTF model download URL
* @param urlStr The original Polyhaven URL
* @returns The direct download URL for the glTF model if it's a valid Polyhaven asset URL, otherwise returns the original URL
*/
function tryResolvePolyhavenAssetUrl(urlStr: string) {
if (!urlStr.startsWith("https://polyhaven.com/")) return urlStr;
// Handle dropping polyhaven image url
const baseUrl = "https://dl.polyhaven.org/file/ph-assets/Models/gltf/4k/";
const url = new URL(urlStr);
const path = url.pathname;
const name = path.split("/").pop();
const assetUrl = `${baseUrl}${name}/${name}_4k.gltf`;
console.log("Resolved polyhaven asset url", urlStr, "→", assetUrl);
// TODO: need to resolve textures properly
return assetUrl;
}
/**
* Helper namespace for loading files and models from various sources
*/
namespace FileHelper {
/**
* Loads and processes a File object into a 3D model
* @param file The file to load (supported formats: gltf, glb, fbx, obj, usdz, vrm)
* @param context The application context
* @param args Additional arguments including a unique guid for instantiation
* @returns Promise containing the loaded model and its content hash, or null if loading failed
*/
export async function loadFile(file: File, context: Context, args: { guid: string }): Promise<{ model: Model, contentMD5: string } | null> {
// first load it locally
const seed = args.guid;
const prov = new InstantiateIdProvider(seed);
const blob = new Blob([file], { type: file.type || determineMimeTypeFromExtension(file.name) || undefined });
const objectUrl = URL.createObjectURL(blob);
const model = await getLoader().loadSync(context, objectUrl, file.name, prov).catch(err => {
console.error(`Failed to load file "${file.name}" (${file.type}):`, err);
return null;
})
URL.revokeObjectURL(objectUrl); // clean up the object URL
if (model) {
return new Promise((resolve, _reject) => {
const reader = new FileReader()
reader.readAsArrayBuffer(file);
reader.onloadend = async (_ev: ProgressEvent<FileReader>) => {
const content = reader.result as ArrayBuffer;
const hash = BlobStorage.hashMD5(content);
return resolve({ model, contentMD5: hash });
};
});
}
else {
console.warn(`Failed to load "${file.name}" (${file.type})`);
return null;
}
}
// return new Promise((resolve, _reject) => {
// });
// }
/**
* Loads a 3D model from a URL with progress visualization
* @param url The URL to load the model from
* @param args Arguments including context, parent object, and optional placement information
* @returns Promise containing the loaded model and its content hash, or null if loading failed
*/
export async function loadFileFromURL(url: URL, args: { guid: string, context: Context, parent: Object3D, point?: Vec3, size?: Vec3 }): Promise<{ model: Model, contentMD5: string } | null> {
return new Promise(async (resolve, _reject) => {
const prov = new InstantiateIdProvider(args.guid);
const urlStr = url.toString();
if (debug) Gizmos.DrawWireSphere(args.point!, .1, 0xff0000, 3);
const preview = PreviewHelper.addPreview({
guid: args.guid,
parent: args.parent,
position: args?.point,
size: args?.size,
});
const model = await getLoader().loadSync(args.context, urlStr, urlStr, prov, prog => {
preview.onProgress(prog.loaded / prog.total);
}).catch(console.warn);
if (model) {
const binary = await fetch(urlStr).then(res => res.arrayBuffer());
const hash = BlobStorage.hashMD5(binary);
if (debug) setTimeout(() => PreviewHelper.removePreview(args.guid), 3000);
else PreviewHelper.removePreview(args.guid);
resolve({ model, contentMD5: hash });
}
else {
if (debug) setTimeout(() => PreviewHelper.removePreview(args.guid), 3000);
else PreviewHelper.removePreview(args.guid);
console.warn("Unsupported file type: " + url.toString());
}
});
}
}