UNPKG

@needle-tools/engine

Version:

Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.

751 lines (670 loc) 30.1 kB
import { CompressedCubeTexture, CubeTexture, CubeUVReflectionMapping, EquirectangularRefractionMapping, Texture } from "three" import { isDevEnvironment } from "../engine/debug/debug.js"; import { ContextEvent, ContextRegistry } from "../engine/engine_context_registry.js"; import { syncField } from "../engine/engine_networking_auto.js"; import { loadPMREM } from "../engine/engine_pmrem.js"; import { serializable } from "../engine/engine_serialization_decorator.js"; import { type IContext } from "../engine/engine_types.js"; import { addAttributeChangeCallback, getParam, toSourceId } from "../engine/engine_utils.js"; import { registerObservableAttribute } from "../engine/webcomponents/needle-engine.extras.js"; import { Camera } from "./Camera.js"; import { Behaviour, GameObject } from "./Component.js"; const debug = getParam("debugskybox"); export function initSkyboxAttributes() { registerObservableAttribute("background-image"); registerObservableAttribute("environment-image"); installSkyboxAttributeHandlers(); } // #region Attribute handlers type SlotState = { url: string | null; texture: Texture | null; loadId: number; }; type ElementHandlerState = { currentContext: IContext | null; bg: SlotState; env: SlotState; listenersInstalled: boolean; }; function makeSlot(): SlotState { return { url: null, texture: null, loadId: 0 }; } const elementHandlerState = new WeakMap<HTMLElement, ElementHandlerState>(); const promises = new Array<Promise<any>>(); const DEG2RAD = Math.PI / 180; function applyRotationAttribute(attr: string, target: { set(x: number, y: number, z: number): void }) { const parts = attr.trim().split(/\s+/); if (parts.length === 1) { const y = parseFloat(parts[0]); if (!isNaN(y)) target.set(0, y * DEG2RAD, 0); } else if (parts.length >= 3) { const x = parseFloat(parts[0]), y = parseFloat(parts[1]), z = parseFloat(parts[2]); if (!isNaN(x) && !isNaN(y) && !isNaN(z)) target.set(x * DEG2RAD, y * DEG2RAD, z * DEG2RAD); } } function applyTexture(context: IContext, texture: Texture, isBackground: boolean, isEnvironment: boolean) { if (!(texture instanceof CubeTexture || texture instanceof CompressedCubeTexture) && texture.mapping !== CubeUVReflectionMapping) { texture.mapping = EquirectangularRefractionMapping; texture.needsUpdate = true; } if (isEnvironment) { context.scene.environment = texture; } if (isBackground && !Camera.backgroundShouldBeTransparent(context)) { context.scene.background = texture; } const el = context.domElement; if (isBackground) { const blurriness = el.getAttribute("background-blurriness"); if (blurriness) { const v = parseFloat(blurriness); if (!isNaN(v)) context.scene.backgroundBlurriness = v; } else if (context.mainCameraComponent?.backgroundBlurriness !== undefined) { context.scene.backgroundBlurriness = context.mainCameraComponent.backgroundBlurriness; } const intensity = el.getAttribute("background-intensity"); if (intensity) { const v = parseFloat(intensity); if (!isNaN(v)) context.scene.backgroundIntensity = v; } const rotation = el.getAttribute("background-rotation"); if (rotation) { applyRotationAttribute(rotation, context.scene.backgroundRotation); } } if (isEnvironment) { const rotation = el.getAttribute("environment-rotation"); if (rotation) { applyRotationAttribute(rotation, context.scene.environmentRotation); } } } function restoreGltfDefaults(context: IContext, slot: SlotState, isBackground: boolean, isEnvironment: boolean) { const sourceId = slot.url ? toSourceId(slot.url) : undefined; slot.url = null; slot.texture = null; if (isEnvironment) { if (!sourceId || !context.sceneLighting.internalEnableReflection(sourceId)) { context.scene.environment = null; } } if (isBackground) { context.scene.background = sourceId ? context.lightmaps.tryGetSkybox(sourceId) : null; } } function applyAttributeValue( state: ElementHandlerState, slot: SlotState, attribute: "background-image" | "environment-image", isBackground: boolean, isEnvironment: boolean, rawValue: string | null, ): Promise<void> | null { const context = state.currentContext; if (!context) return null; const value = rawValue ? tryParseMagicSkyboxName(rawValue, isEnvironment, isBackground) : null; if (debug) console.log(`Skybox attribute [${attribute}]: raw="${rawValue}" → resolved="${value}"`); if (value && (value === "transparent" || value.startsWith("rgb") || value.startsWith("#"))) { console.warn(`Needle Engine: Invalid ${attribute} value (${value}). Did you mean to set background-color instead?`); return null; } const loadId = ++slot.loadId; if (!value) { restoreGltfDefaults(context, slot, isBackground, isEnvironment); return null; } if (slot.url === value && slot.texture) { if (debug) console.log(`Skybox attribute [${attribute}]: cache hit, re-applying`); applyTexture(context, slot.texture, isBackground, isEnvironment); return null; } slot.url = value; slot.texture = null; const promise = (async () => { if (debug) console.log(`Skybox attribute [${attribute}]: loading ${value}`); const texture = await loadPMREM(value, context.renderer); if (slot.loadId !== loadId) { if (debug) console.warn(`Skybox attribute [${attribute}]: loadId mismatch (stale load), skipping apply`); return; } if (!texture) { if (debug) console.warn(`Skybox attribute [${attribute}]: failed to load ${value}`); return; } if (debug) console.log(`Skybox attribute [${attribute}]: loaded, applying to scene`); slot.texture = texture; applyTexture(context, texture, isBackground, isEnvironment); context.domElement.dispatchEvent(new CustomEvent(`${attribute}-loaded`, { detail: { texture }, })); })(); return promise; } function getCurrentState(args: { context: IContext }): ElementHandlerState | null { const state = elementHandlerState.get(args.context.domElement); if (!state || state.currentContext !== args.context) return null; return state; } let _skyboxHandlersInstalled = false; function installSkyboxAttributeHandlers() { if (_skyboxHandlersInstalled) return; _skyboxHandlersInstalled = true; ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, (args) => { const context = args.context; const el = context.domElement; let state = elementHandlerState.get(el); if (!state) { state = { currentContext: context, bg: makeSlot(), env: makeSlot(), listenersInstalled: false }; elementHandlerState.set(el, state); } else { state.currentContext = context; } const bgLoad = applyAttributeValue(state, state.bg, "background-image", true, false, el.getAttribute("background-image")); if (bgLoad) promises.push(bgLoad); const envLoad = applyAttributeValue(state, state.env, "environment-image", false, true, el.getAttribute("environment-image")); if (envLoad) promises.push(envLoad); if (!state.listenersInstalled) { state.listenersInstalled = true; const stateRef = state; addAttributeChangeCallback(el, "background-image", (rawValue) => { const v = (typeof rawValue === "string" && rawValue.length > 0) ? rawValue : null; if (debug) console.log("background-image CHANGED TO", v); applyAttributeValue(stateRef, stateRef.bg, "background-image", true, false, v); }); addAttributeChangeCallback(el, "environment-image", (rawValue) => { const v = (typeof rawValue === "string" && rawValue.length > 0) ? rawValue : null; if (debug) console.log("environment-image CHANGED TO", v); applyAttributeValue(stateRef, stateRef.env, "environment-image", false, true, v); }); } }); ContextRegistry.registerCallback(ContextEvent.ContextCreationStart, () => { return Promise.all(promises).finally(() => { promises.length = 0; }); }); ContextRegistry.registerCallback(ContextEvent.ContextClearing, (args) => { const state = getCurrentState(args); if (!state) return; state.bg = makeSlot(); state.env = makeSlot(); }); ContextRegistry.registerCallback(ContextEvent.ContextDestroyed, (args) => { const state = getCurrentState(args); if (!state) return; state.currentContext = null; state.bg.loadId++; state.env.loadId++; }); } // #endregion // #region RemoteSkybox /** * The [RemoteSkybox](https://engine.needle.tools/docs/api/RemoteSkybox) component allows you to set the skybox or environment texture of a scene from a URL, a local file or a static skybox name. * It supports .hdr, .exr, .jpg, .png, and .ktx2 files. * * **HTML Attributes:** * You can control skybox and environment from HTML using `<needle-engine>` attributes: * - `background-image`: Sets the scene background/skybox image * - `environment-image`: Sets the scene environment map (for reflections and lighting) * * These attributes accept URLs or magic skybox names (see examples below). * * **Magic Skybox Names:** * Built-in optimized skyboxes hosted on Needle CDN: * - `"studio"` - Neutral studio lighting (default) * - `"blurred-skybox"` - Blurred environment * - `"quicklook"` - Apple QuickLook object mode style * - `"quicklook-ar"` - Apple QuickLook AR mode style * * ### Events * - `dropped-unknown-url`: Emitted when a file is dropped on the scene. The event detail contains the sender, the url and a function to apply the url. * * @example Using HTML attributes * ```html * <needle-engine * background-image="https://example.com/skybox.hdr" * environment-image="studio"> * </needle-engine> * ``` * * @example Using magic skybox names * ```html * <needle-engine background-image="studio"></needle-engine> * <needle-engine environment-image="quicklook"></needle-engine> * ``` * * @example Adding via code * ```ts * GameObject.addComponent(gameObject, RemoteSkybox, { * url: "https://example.com/skybox.hdr", * background: true, * environment: true * }); * ``` * * @example Handle custom dropped URL * ```ts * const skybox = GameObject.addComponent(gameObject, RemoteSkybox); * skybox.addEventListener("dropped-unknown-url", (evt) => { * let url = evt.detail.url; * console.log("User dropped file", url); * // change url or resolve it differently * url = "https://example.com/skybox.hdr"; * // apply the url * evt.detail.apply(url); * }); * ``` * * @example Update skybox at runtime * ```ts * skybox.setSkybox("https://example.com/skybox.hdr"); * // Or use a magic name: * skybox.setSkybox("studio"); * ``` * * @summary Sets the skybox or environment texture of a scene * @category Rendering * @group Components * @see {@link Camera} for clearFlags and background control * @link https://engine.needle.tools/docs/html.html#needle-engine-element */ export class RemoteSkybox extends Behaviour { /** * URL to a remote skybox. * To update the skybox/environment map use `setSkybox(url)`. * * The url can also be set to a magic skybox name. * Magic name options are: "quicklook", "quicklook-ar", "studio", "blurred-skybox". * These will resolve to built-in skyboxes hosted on the Needle CDN that are static, optimized for the web and will never change. * * @example * ```ts * skybox.url = "https://example.com/skybox.hdr"; * ``` */ @syncField(RemoteSkybox.prototype.urlChangedSyncField) @serializable(URL) url: MagicSkyboxName | AnyString = "studio"; /** * When enabled a user can drop a link to a skybox image on the scene to set the skybox. * @default true */ @serializable() allowDrop: boolean = true; /** * When enabled the skybox will be set as the background of the scene. * @default true */ @serializable() background: boolean = true; /** * When enabled the skybox will be set as the environment of the scene (to be used as environment map for reflections and lighting) * @default true */ @serializable() environment: boolean = true; /** * When enabled dropped skybox urls (or assigned skybox urls) will be networked to other users in the same networked room. * @default true */ @serializable() allowNetworking: boolean = true; private _prevUrl?: string; private _prevLoadedEnvironment?: Texture; /** Promise returned by the most recent in-flight {@link setSkybox} load. * The URL it's loading is `_prevUrl` (set synchronously before the * awaited work begins). When a second `setSkybox(sameUrl)` comes in * while `_prevLoadedEnvironment` hasn't been populated yet, we return * this promise instead of kicking off a duplicate `loadPMREM` — same * URL, same texture, no reason to race two loads. * * Without this, the create path fetches the same URL twice: * `createRemoteSkyboxComponent` calls `setSkybox(url)` eagerly to get * a promise for the ContextCreationStart barrel, AND the component's * `onEnable()` lifecycle also calls `setSkybox(this.url)` — both * before `_prevLoadedEnvironment` is populated, so the fast-path cache * check doesn't fire and `loadPMREM` runs twice. */ private _inFlightLoad?: Promise<boolean>; private _prevEnvironment: Texture | null = null; private _prevBackground: any = null; /** * If our texture is still installed in `scene.environment` / * `scene.background`, restore the snapshot {@link apply} took before it * wrote there. Slots owned by something else are left alone. * * @returns `true` if either slot was ours (and was reverted). */ private revertAppliedSceneState(): boolean { if (!this.context || !this._prevLoadedEnvironment) return false; let reverted = false; if (this.context.scene.environment === this._prevLoadedEnvironment) { this.context.scene.environment = this._prevEnvironment; reverted = true; } if (this.context.scene.background === this._prevLoadedEnvironment) { // Don't restore a stale concrete background when the camera config // wants transparency — leave the slot for the transparent path to // manage. if (!Camera.backgroundShouldBeTransparent(this.context)) { this.context.scene.background = this._prevBackground; } reverted = true; } return reverted; } /** * Discard the result of any in-flight {@link setSkybox} call AND revert * any already-applied skybox state from this RemoteSkybox so the scene * looks as if this component never ran. * * Automatically called when the `background-image` / `environment-image` * HTML attribute is cleared — callers normally don't need to invoke it * directly. * * Covers two races: * * 1. **In-flight load.** `setSkybox` re-checks `_prevUrl` after `await * loadPMREM(...)` and bails if it changed; resetting it here trips * that check on the in-flight call. * 2. **Already-applied load.** If `loadPMREM` resolved fast (HTTP cache * hit), `apply()` already ran. The in-flight check above doesn't * help — `revertAppliedSceneState` restores the prior snapshot. * * NOTE: this does NOT abort the underlying network fetch / PMREM * generation — `loadPMREM` has no cancellation. It only flips state so * `setSkybox` bails after the await and skips `apply()`. The texture * returned by `loadPMREM` in those bail-out branches (`_prevUrl !== url` * and disabled/destroyed) is still dropped without `dispose()` — a small * GPU leak that should be fixed at those sites. * @internal */ discardPendingLoad() { this._prevUrl = undefined; // Clear the in-flight tracking so a subsequent setSkybox(sameUrl) // doesn't latch onto the now-discarded load via the dedup path. The // running `await load` is not cancelled (loadPMREM has no cancel // surface) but its post-await `_prevUrl !== url` check trips and // short-circuits before apply(). this._inFlightLoad = undefined; if (!this.context) { // Context is gone — the cached texture has no owner. Release GPU // memory rather than orphan it. this._prevLoadedEnvironment?.dispose(); this._prevLoadedEnvironment = undefined; return; } this.revertAppliedSceneState(); this._prevLoadedEnvironment = undefined; } /** @internal */ onEnable() { this.setSkybox(this.url); this.registerDropEvents(); } /** @internal */ onDisable() { if (this.revertAppliedSceneState()) { this._prevLoadedEnvironment = undefined; } this.unregisterDropEvents(); // Re-apply the skybox/background settings of the main camera this.context.mainCameraComponent?.applyClearFlags(); } private urlChangedSyncField() { if (this.allowNetworking && this.url) { // omit local dragged files from being handled if (this.isRemoteTexture(this.url)) { this.setSkybox(this.url); } else if (debug) { console.warn(`RemoteSkybox: Not setting skybox: ${this.url} is not a remote texture. If you want to set a local texture, set allowNetworking to false.`); } } } /** * Set the skybox from a given url * @param url The url of the skybox image * @param name Define name of the file with extension if it isn't apart of the url * @returns Whether the skybox was successfully set */ async setSkybox(url: MagicSkyboxName | AnyString | undefined | null, name?: string) { if (!this.activeAndEnabled) return false; url = tryParseMagicSkyboxName(url, this.environment, this.background); if (!url) return false; name ??= url; if (!this.isValidTextureType(name)) { console.warn("Potentially invalid skybox URL: \"" + name + "\" on " + (this.name || this.gameObject?.name || "context")); } if (debug) console.log("Set RemoteSkybox url: " + url); if (this._prevUrl === url) { // Same URL as last/current setSkybox call. if (this._prevLoadedEnvironment) { // Load already completed — replay apply() against the cached texture. this.apply(); return true; } if (this._inFlightLoad) { // Load still running — return its promise instead of starting // a second loadPMREM. Same URL = same texture = nothing to gain // from racing two loads. See _inFlightLoad doc for the create- // path callers that hit this. return this._inFlightLoad; } // Same URL, no cached result, no in-flight load → fall through // (e.g. discardPendingLoad cleared _inFlightLoad but a stale // _prevUrl assignment is rare; treat as a fresh start). } // Different URL (or same URL with no live load) — release the previous // result and start fresh. this._prevLoadedEnvironment?.dispose(); this._prevLoadedEnvironment = undefined; this._prevUrl = url; const load = (async (): Promise<boolean> => { const texture = await loadPMREM(url, this.context.renderer); if (!texture) { if (debug) console.warn("RemoteSkybox: Failed to load texture from url", url); return false; } // Check if we're not disabled or destroyed if (!this.enabled || this.destroyed) { if (debug) console.warn("RemoteSkybox: Component is disabled or destroyed"); return false; } // Check if the url has changed while loading (covers both another // setSkybox(differentUrl) and discardPendingLoad clearing _prevUrl) if (this._prevUrl !== url) { if (debug) console.warn("RemoteSkybox: URL changed while loading texture, aborting setSkybox"); return false; } // Update the current url this.url = url; this._prevLoadedEnvironment = texture; this.apply(); return true; })(); this._inFlightLoad = load; try { return await load; } finally { // Only clear if we're still the active in-flight load — another // setSkybox call with a different URL may have replaced us, and // we shouldn't wipe that newer caller's tracking. if (this._inFlightLoad === load) { this._inFlightLoad = undefined; } } } private apply() { const envMap = this._prevLoadedEnvironment; if (!envMap) return; if ((envMap instanceof CubeTexture || envMap instanceof CompressedCubeTexture) || envMap.mapping == CubeUVReflectionMapping) { // Nothing to do } else { envMap.mapping = EquirectangularRefractionMapping; envMap.needsUpdate = true; } if (this.destroyed) return; if (!this.context) { console.warn("RemoteSkybox: Context is not available - can not apply skybox."); return; } // capture state if (this.context.scene.background !== envMap) this._prevBackground = this.context.scene.background; if (this.context.scene.environment !== envMap) this._prevEnvironment = this.context.scene.environment; if (debug) console.log("Set RemoteSkybox (" + ((this.environment && this.background) ? "environment and background" : this.environment ? "environment" : this.background ? "background" : "none") + ")", this.url, !Camera.backgroundShouldBeTransparent(this.context)); if (this.environment) this.context.scene.environment = envMap; if (this.background && !Camera.backgroundShouldBeTransparent(this.context)) this.context.scene.background = envMap; if (this.context.mainCameraComponent?.backgroundBlurriness !== undefined) this.context.scene.backgroundBlurriness = this.context.mainCameraComponent.backgroundBlurriness; } private readonly validProtocols = ["file:", "blob:", "data:"]; private readonly validTextureTypes = [".ktx2", ".hdr", ".exr", ".jpg", ".jpeg", ".png"]; private isRemoteTexture(url: string): boolean { return url.startsWith("http://") || url.startsWith("https://"); } private isValidTextureType(url: string): boolean { for (const type of this.validTextureTypes) { if (url.includes(type)) return true; } for (const protocol of this.validProtocols) { if (url.startsWith(protocol)) return true; } return false; } private registerDropEvents() { this.unregisterDropEvents(); this.context.domElement.addEventListener("dragover", this.onDragOverEvent); this.context.domElement.addEventListener("drop", this.onDrop); } private unregisterDropEvents() { this.context.domElement.removeEventListener("dragover", this.onDragOverEvent); this.context.domElement.removeEventListener("drop", this.onDrop); } private onDragOverEvent = (e: DragEvent) => { if (!this.allowDrop) return; if (!e.dataTransfer) return; for (const type of e.dataTransfer.types) { // in ondragover we dont get access to the content // but if we have a uri list we can assume // someone is maybe dragging a image file // so we want to capture this if (type === "text/uri-list" || type === "Files") { e.preventDefault(); } } }; private onDrop = (e: DragEvent) => { if (!this.allowDrop) return; if (!e.dataTransfer) return; for (const type of e.dataTransfer.types) { if (debug) console.log(type); if (type === "text/uri-list") { const url = e.dataTransfer.getData(type); if (debug) console.log(type, url); let name = new RegExp(/polyhaven.com\/asset_img\/.+?\/(?<name>.+)\.png/).exec(url)?.groups?.name; if (!name) { name = new RegExp(/polyhaven\.com\/a\/(?<name>.+)/).exec(url)?.groups?.name; } if (debug) console.log(name); if (name) { const skyboxurl = "https://dl.polyhaven.org/file/ph-assets/HDRIs/exr/1k/" + name + "_1k.exr"; console.log(`[Remote Skybox] Setting skybox from url: ${skyboxurl}`); e.preventDefault(); this.setSkybox(skyboxurl); break; } else if (this.isValidTextureType(url)) { console.log("[Remote Skybox] Setting skybox from url: " + url); e.preventDefault(); this.setSkybox(url); break; } else { console.warn(`[RemoteSkybox] Unknown url ${url}. If you want to load a skybox from a url, make sure it is a valid image url. Url must end with${this.validTextureTypes.join(", ")}.`); // emit custom event - users can listen to this event and handle the url themselves const evt = new CustomEvent("dropped-unknown-url", { detail: { sender: this, event: e, url, apply: (url: string) => { e.preventDefault(); this.setSkybox(url); } } }); this.dispatchEvent(evt); } } else if (type == "Files") { const file = e.dataTransfer.files.item(0); if (debug) console.log(type, file); if (!file) continue; if (!this.isValidTextureType(file.name)) { console.warn(`[RemoteSkybox]: File \"${file.name}\" is not supported. Supported files are ${this.validTextureTypes.join(", ")}`); return; } // if (tryGetPreviouslyLoadedTexture(file.name) === null) { // const blob = new Blob([file]); // const url = URL.createObjectURL(blob); // e.preventDefault(); // this.setSkybox(url, file.name); // } // else { e.preventDefault(); this.setSkybox(file.name); } break; } } }; } // #region Magic Skybox Names type MagicSkyboxName = "studio" | "blurred-skybox" | "quicklook-ar" | "quicklook"; const MagicSkyboxNames: Record<MagicSkyboxName, { url: string, url_low: string }> = { "studio": { url: "https://cdn.needle.tools/static/skybox/modelviewer-Neutral.pmrem4x4.ktx2?pmrem", url_low: "https://cdn.needle.tools/static/skybox/modelviewer-Neutral-small.pmrem4x4.ktx2?pmrem" }, "blurred-skybox": { url: "https://cdn.needle.tools/static/skybox/blurred-skybox.pmrem4x4.ktx2?pmrem", url_low: "https://cdn.needle.tools/static/skybox/blurred-skybox-small.pmrem4x4.ktx2?pmrem" }, "quicklook-ar": { url: "https://cdn.needle.tools/static/skybox/QuickLook-ARMode.pmrem4x4.ktx2?pmrem", url_low: "https://cdn.needle.tools/static/skybox/QuickLook-ARMode-small.pmrem4x4.ktx2?pmrem" }, "quicklook": { url: "https://cdn.needle.tools/static/skybox/QuickLook-ObjectMode.pmrem4x4.ktx2?pmrem", url_low: "https://cdn.needle.tools/static/skybox/QuickLook-ObjectMode-small.pmrem4x4.ktx2?pmrem" } } as const; type AnyString = string & { _brand?: never }; function tryParseMagicSkyboxName(str: string | null | undefined, environment: boolean, background: boolean): string | null { if (str === null || str === undefined) return null; const useLowRes = environment && !background; const value = MagicSkyboxNames[str.toLowerCase() as MagicSkyboxName]; if (value) { return useLowRes ? value.url_low : value.url; } else if (typeof str === "string" && str?.length && (isDevEnvironment() || debug)) { // Only warn if the string looks like it was meant to be a magic skybox name. // Strings that contain "/" or "." are paths or URLs, not magic names. const looksLikePath = str.includes("/") || str.includes("."); if(!looksLikePath) { console.warn(`RemoteSkybox: Unknown magic skybox name "${str}". Valid names are: ${Object.keys(MagicSkyboxNames).map(n => `"${n}"`).join(", ")}`); } } return str; }