@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in
579 lines (505 loc) • 22.6 kB
text/typescript
import { AxesHelper, Box3, Cache, Object3D, Vector2, Vector3 } from "three";
import type { GLTF } from "three/examples/jsm/loaders/GLTFLoader.js";
import { isDevEnvironment } from "../engine/debug/index.js";
import { AnimationUtils } from "../engine/engine_animation.js";
import { addComponent } from "../engine/engine_components.js";
import { Context } from "../engine/engine_context.js";
import { destroy } from "../engine/engine_gameobject.js";
import { Gizmos } from "../engine/engine_gizmos.js";
import { getLoader } from "../engine/engine_gltf.js";
import { BlobStorage } from "../engine/engine_networking_blob.js";
import { PreviewHelper } from "../engine/engine_networking_files.js";
import { generateSeed, InstantiateIdProvider } from "../engine/engine_networking_instantiate.js";
import { serializable } from "../engine/engine_serialization_decorator.js";
import { fitObjectIntoVolume, getBoundingBox, placeOnSurface } from "../engine/engine_three_utils.js";
import { IGameObject, Model, Vec3 } from "../engine/engine_types.js";
import { getParam, setParamWithoutReload } from "../engine/engine_utils.js";
import { Animation } from "./Animation.js";
import { Behaviour } from "./Component.js";
import { EventList } from "./EventList.js";
const debug = getParam("debugdroplistener");
export enum DropListenerEvents {
/**
* Dispatched when a file is dropped into the scene. The detail of the event is the file that was dropped.
*/
FileDropped = "file-dropped",
/**
* Dispatched when a new object is added to the scene. The detail of the event is the glTF that was added.
*/
ObjectAdded = "object-added",
}
declare type DropContext = {
screenposition: Vector2;
url?: string,
file?: File;
point?: Vec3;
size?: Vec3;
}
/** Networking event arguments for the DropListener component */
export declare type DropListenerNetworkEventArguments = {
guid: string,
name: string,
url: string | string[],
/** Worldspace point where the object was placed in the scene */
point: Vec3;
/** Bounding box size */
size: Vec3;
contentMD5: string;
}
declare type AddedEventArguments = {
sender: DropListener,
/** the root object added to the scene */
object: Object3D,
/** The whole dropped model */
model: Model,
contentMD5: string;
dropped: URL | File | undefined;
}
/** Dispatched when an object is dropped/changed */
export class DropListenerAddedEvent<T extends AddedEventArguments> extends CustomEvent<T> {
constructor(detail: T) {
super(DropListenerEvents.ObjectAdded, { detail });
}
}
const blobKeyName = "blob";
/** The DropListener component is used to listen for drag and drop events in the browser and add the dropped files to the scene
* It can be used to allow users to drag and drop glTF files into the scene to add new objects.
*
* ## Events
* - **object-added** - dispatched when a new object is added to the scene
* - **file-dropped** - dispatched when a file is dropped into the scene
*
* @example
* ```typescript
* import { DropListener, DropListenerEvents } from "@needle-tools/engine";
*
* const dropListener = new DropListener();
*
* gameObject.addComponent(dropListener);
* dropListener.on(DropListenerEvents.FileDropped, (evt) => {
* console.log("File dropped", evt.detail);
* const file = evt.detail as File;
* });
*
* dropListener.on(DropListenerEvents.ObjectAdded, (evt) => {
* console.log("Object added", evt.detail);
* const gltf = evt.detail as GLTF;
* });
* ```
*
* @category Asset Management
* @group Components
*/
export class DropListener extends Behaviour {
/**
* When enabled the DropListener will automatically network dropped files to other clients.
*/
()
useNetworking: boolean = true;
/**
* When assigned the Droplistener will only accept files that are dropped on this object.
*/
(Object3D)
dropArea?: Object3D;
/**
* When enabled the object will be fitted into a volume. Use {@link fitVolumeSize} to specify the volume size.
* @default false
*/
()
fitIntoVolume: boolean = false;
/**
* The volume size will be used to fit the object into the volume. Use {@link fitIntoVolume} to enable this feature.
*/
(Vector3)
fitVolumeSize = new Vector3(1, 1, 1);
/** When enabled the object will be placed at the drop position (under the cursor)
* @default true
*/
()
placeAtHitPosition: boolean = true;
/**
* Invoked after a file has been **added** to the scene.
* Arguments are {@link AddedEventArguments}
* @event object-added
* @param {AddedEventArguments} evt
* @example
* ```typescript
* dropListener.onDropped.addEventListener((evt) => {
* console.log("Object added", evt.model);
* });
*/
(EventList)
onDropped: EventList<AddedEventArguments> = new EventList();
/** @internal */
onEnable(): void {
this.context.renderer.domElement.addEventListener("dragover", this.onDrag);
this.context.renderer.domElement.addEventListener("drop", this.onDrop);
window.addEventListener("paste", this.handlePaste);
this.context.connection.beginListen("droplistener", this.onNetworkEvent)
}
/** @internal */
onDisable(): void {
this.context.renderer.domElement.removeEventListener("dragover", this.onDrag);
this.context.renderer.domElement.removeEventListener("drop", this.onDrop);
window.removeEventListener("paste", this.handlePaste);
this.context.connection.stopListen("droplistener", this.onNetworkEvent);
}
/**
* Loads a file from the given URL and adds it to the scene.
*/
loadFromURL(url: string, data?: { point?: Vec3, size?: Vec3 }) {
this.addFromUrl(url, { screenposition: new Vector2(), point: data?.point, size: data?.size, }, true);
}
/**
* Forgets all previously added objects.
* The droplistener will then not be able to remove previously added objects.
*/
forgetObjects() {
this.removePreviouslyAddedObjects(false);
}
private onNetworkEvent = (evt: DropListenerNetworkEventArguments) => {
if (!this.useNetworking) {
if (debug) console.debug("[DropListener] Ignoring networked event because networking is disabled", evt);
return;
}
if (evt.guid?.startsWith(this.guid)) {
const url = evt.url;
console.debug("[DropListener] Received networked event", evt);
if (url) {
if (Array.isArray(url)) {
for (const _url of url) {
this.addFromUrl(_url, { screenposition: new Vector2(), point: evt.point, size: evt.size, }, true);
}
}
else {
this.addFromUrl(url, { screenposition: new Vector2(), point: evt.point, size: evt.size }, true);
}
}
}
}
private handlePaste = (evt: Event) => {
if (this.context.connection.allowEditing === false) return;
if (evt.defaultPrevented) return;
const clipboard = navigator.clipboard;
clipboard.readText()
.then(value => {
if (value) {
const isUrl = value.startsWith("http") || value.startsWith("https") || value.startsWith("blob");
if (isUrl) {
const ctx = { screenposition: new Vector2(this.context.input.mousePosition.x, this.context.input.mousePosition.y) };
if (this.testIfIsInDropArea(ctx))
this.addFromUrl(value, ctx, false);
}
}
})
.catch(console.warn);
}
private onDrag = (evt: DragEvent) => {
if (this.context.connection.allowEditing === false) return;
// necessary to get drop event
evt.preventDefault();
}
private onDrop = async (evt: DragEvent) => {
if (this.context.connection.allowEditing === false) return;
if (debug) console.log(evt);
if (!evt?.dataTransfer) return;
// If the event is marked as handled for droplisteners then ignore it
if (evt["droplistener:handled"]) return;
evt.preventDefault();
const ctx: DropContext = { screenposition: new Vector2(evt.offsetX, evt.offsetY) };
if (this.dropArea) {
const res = this.testIfIsInDropArea(ctx);
if (res === false) return;
}
// Don't stop propagation because this will break e.g. the RemoteSkybox drop
// evt.stopImmediatePropagation();
// Mark the event handled for droplisteners
evt["droplistener:handled"] = true;
const items = evt.dataTransfer.items;
if (!items) return;
const files: File[] = [];
for (const ite in items) {
const it = items[ite];
if (it.kind === "file") {
const file = it.getAsFile();
if (!file) continue;
files.push(file);
}
else if (it.kind === "string" && it.type == "text/plain") {
it.getAsString(str => {
this.addFromUrl(str, ctx, false);
});
}
}
if (files.length > 0) {
await this.addDroppedFiles(files, ctx);
}
}
private async addFromUrl(url: string, ctx: DropContext, isRemote: boolean) {
if (debug) console.log("dropped url", url);
try {
if (url.startsWith("https://github.com/")) {
// make raw.githubusercontent.com url
const parts = url.split("/");
const user = parts[3];
const repo = parts[4];
const branch = parts[6];
const path = parts.slice(7).join("/");
url = `https://raw.githubusercontent.com/${user}/${repo}/${branch}/${path}`;
}
else if (url.startsWith("https://polyhaven.com/a")) {
url = tryResolvePolyhavenAssetUrl(url);
}
if (!url) return null;
// Ignore dropped images
const lowercaseUrl = url.toLowerCase();
if (lowercaseUrl.endsWith(".hdr") || lowercaseUrl.endsWith(".hdri") || lowercaseUrl.endsWith(".exr") || lowercaseUrl.endsWith(".png") || lowercaseUrl.endsWith(".jpg") || lowercaseUrl.endsWith(".jpeg")) {
return null;
}
// TODO: if the URL is invalid this will become a problem
this.removePreviouslyAddedObjects();
// const binary = await fetch(url).then(res => res.arrayBuffer());
const res = await FileHelper.loadFileFromURL(new URL(url), {
guid: this.guid,
context: this.context,
parent: this.gameObject,
point: ctx.point,
size: ctx.size,
});
if (res && this._addedObjects.length <= 0) {
ctx.url = url;
const obj = this.addObject(res, ctx, isRemote);
return obj;
}
}
catch (_) {
console.warn("String is not a valid URL", url);
}
return null;
}
private _abort: AbortController | null = null;
private async addDroppedFiles(fileList: Array<File>, ctx: DropContext) {
if (debug) console.log("Add files", fileList)
if (!Array.isArray(fileList)) return;
if (!fileList.length) return;
this.deleteDropEvent();
this.removePreviouslyAddedObjects();
setParamWithoutReload(blobKeyName, null);
// Create an abort controller for the current drop operation
this._abort?.abort("New files dropped");
this._abort = new AbortController();
for (const file of fileList) {
if (!file) continue;
console.debug("Load file " + file.name);
const res = await FileHelper.loadFile(file, this.context, { guid: this.guid });
if (res) {
this.dispatchEvent(new CustomEvent(DropListenerEvents.FileDropped, { detail: file }));
ctx.file = file;
const obj = this.addObject(res, ctx, false);
// handle uploading the dropped object and networking the event
if (obj && this.context.connection.isConnected && this.useNetworking) {
console.debug("Uploading dropped file to blob storage");
BlobStorage.upload(file, { abort: this._abort?.signal, })
.then(upload => {
// check if the upload was successful and if the object should still be visible
if (upload?.download_url && this._addedObjects.includes(obj)) {
// setParamWithoutReload(blobKeyName, upload.key);
this.sendDropEvent(upload.download_url, obj, res.contentMD5);
}
})
.catch(console.warn);
}
// we currently only support dropping one file
break;
}
}
}
/** Previously added objects */
private readonly _addedObjects = new Array<Object3D>();
private readonly _addedModels = new Array<Model>();
/** Removes all previously added objects from the scene and removes those object references */
private removePreviouslyAddedObjects(doDestroy: boolean = true) {
if (doDestroy) {
for (const prev of this._addedObjects) {
if (prev.parent === this.gameObject) {
destroy(prev, true, true);
}
}
}
this._addedObjects.length = 0;
this._addedModels.length = 0;
}
/**
* Adds the object to the scene and fits it into the volume if {@link fitIntoVolume} is enabled.
*/
private addObject(data: { model: Model, contentMD5: string }, ctx: DropContext, isRemote: boolean): Object3D | null {
const { model, contentMD5 } = data;
if (debug) console.log(`Dropped ${this.gameObject.name}`, model);
if (!model?.scene) {
console.warn("No object specified to add to scene", model);
return null;
}
this.removePreviouslyAddedObjects();
const obj = model.scene;
// use attach to ignore the DropListener scale (e.g. if the parent object scale is not uniform)
this.gameObject.attach(obj);
obj.position.set(0, 0, 0);
obj.quaternion.identity();
this._addedObjects.push(obj);
this._addedModels.push(model);
const volume = new Box3().setFromCenterAndSize(new Vector3(0, this.fitVolumeSize.y * .5, 0).add(this.gameObject.worldPosition), this.fitVolumeSize);
if (debug) Gizmos.DrawWireBox3(volume, 0x0000ff, 5);
if (this.fitIntoVolume) {
fitObjectIntoVolume(obj, volume, {
position: !this.placeAtHitPosition
});
}
if (this.placeAtHitPosition && ctx && ctx.screenposition) {
obj.visible = false; // < don't raycast on the placed object
const rc = this.context.physics.raycast({ screenPoint: this.context.input.convertScreenspaceToRaycastSpace(ctx.screenposition.clone()) });
obj.visible = true;
if (rc && rc.length > 0) {
for (const hit of rc) {
const pos = hit.point.clone();
if (debug) console.log("Place object at hit", hit);
placeOnSurface(obj, pos);
break;
}
}
}
AnimationUtils.assignAnimationsFromFile(model, {
createAnimationComponent: obj => addComponent(obj, Animation)
});
const evt = new DropListenerAddedEvent({
sender: this,
gltf: model,
model: model,
object: obj,
contentMD5: contentMD5,
dropped: ctx.file || (ctx.url ? new URL(ctx.url) : undefined),
});
this.dispatchEvent(evt);
this.onDropped?.invoke(evt.detail);
// send network event
if (!isRemote && ctx.url?.startsWith("http") && this.context.connection.isConnected && obj) {
this.sendDropEvent(ctx.url, obj, contentMD5);
}
return obj;
}
private async sendDropEvent(url: string, obj: Object3D, contentmd5: string) {
if (!this.useNetworking) {
if (debug) console.debug("[DropListener] Ignoring networked event because networking is disabled", url);
return;
}
if (this.context.connection.isConnected) {
console.debug("Sending drop event \"" + obj.name + "\"", url);
const bounds = getBoundingBox([obj]);
const evt: DropListenerNetworkEventArguments = {
name: obj.name,
guid: this.guid,
url,
point: obj.worldPosition.clone(),
size: bounds.getSize(new Vector3()),
contentMD5: contentmd5,
};
this.context.connection.send("droplistener", evt);
}
}
private deleteDropEvent() {
this.context.connection.sendDeleteRemoteState(this.guid);
}
private testIfIsInDropArea(ctx: DropContext): boolean {
if (this.dropArea) {
const screenPoint = this.context.input.convertScreenspaceToRaycastSpace(ctx.screenposition.clone());
const hits = this.context.physics.raycast({
targets: [this.dropArea],
screenPoint,
recursive: true,
testObject: obj => {
// Ignore hits on the already added objects, they don't count as part of the dropzone
if (this._addedObjects.includes(obj)) return false;
return true;
}
});
if (!hits.length) {
if (isDevEnvironment()) console.log(`Dropped outside of drop area for DropListener \"${this.name}\".`);
return false;
}
}
return true;
}
}
function tryResolvePolyhavenAssetUrl(urlStr: string) {
if (!urlStr.startsWith("https://polyhaven.com/")) return urlStr;
// Handle dropping polyhaven image url
const baseUrl = "https://dl.polyhaven.org/file/ph-assets/Models/gltf/4k/";
const url = new URL(urlStr);
const path = url.pathname;
const name = path.split("/").pop();
const assetUrl = `${baseUrl}${name}/${name}_4k.gltf`;
console.log("Resolved polyhaven asset url", urlStr, "→", assetUrl);
// TODO: need to resolve textures properly
return assetUrl;
}
namespace FileHelper {
export async function loadFile(file: File, context: Context, args: { guid: string }): Promise<{ model: Model, contentMD5: string } | null> {
const name = file.name.toLowerCase();
if (name.endsWith(".gltf") ||
name.endsWith(".glb") ||
name.endsWith(".fbx") ||
name.endsWith(".obj") ||
name.endsWith(".usdz") ||
name.endsWith(".vrm") ||
file.type === "model/gltf+json" ||
file.type === "model/gltf-binary"
) {
return new Promise((resolve, _reject) => {
const reader = new FileReader()
reader.readAsArrayBuffer(file);
reader.onloadend = async (_ev: ProgressEvent<FileReader>) => {
const content = reader.result as ArrayBuffer;
// first load it locally
const seed = args.guid;
const prov = new InstantiateIdProvider(seed);
const model = await getLoader().parseSync(context, content, file.name, prov);
if (model) {
const hash = BlobStorage.hashMD5(content);
resolve({ model, contentMD5: hash });
}
};
});
}
else {
console.warn("Unsupported file type: " + name, file.type)
}
return null;
}
export async function loadFileFromURL(url: URL, args: { guid: string, context: Context, parent: Object3D, point?: Vec3, size?: Vec3 }): Promise<{ model: Model, contentMD5: string } | null> {
return new Promise(async (resolve, _reject) => {
const prov = new InstantiateIdProvider(args.guid);
const urlStr = url.toString();
if (debug) Gizmos.DrawWireSphere(args.point!, .1, 0xff0000, 3);
const preview = PreviewHelper.addPreview({
guid: args.guid,
parent: args.parent,
position: args?.point,
size: args?.size,
});
const model = await getLoader().loadSync(args.context, urlStr, urlStr, prov, prog => {
preview.onProgress(prog.loaded / prog.total);
}).catch(console.warn);
if (model) {
const binary = await fetch(urlStr).then(res => res.arrayBuffer());
const hash = BlobStorage.hashMD5(binary);
if (debug) setTimeout(() => PreviewHelper.removePreview(args.guid), 3000);
else PreviewHelper.removePreview(args.guid);
resolve({ model, contentMD5: hash });
}
else {
if (debug) setTimeout(() => PreviewHelper.removePreview(args.guid), 3000);
else PreviewHelper.removePreview(args.guid);
console.warn("Unsupported file type: " + url.toString());
}
});
}
}