@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
581 lines (519 loc) • 20.8 kB
text/typescript
import { Mesh, Object3D, TextureLoader, Vector4 } from "three";
import ThreeMeshUI from "three-mesh-ui";
import { addNewComponent } from "../../engine_components.js";
import { hasProLicense } from "../../engine_license.js";
import { OneEuroFilterXYZ } from "../../engine_math.js";
import type { Context } from "../../engine_setup.js";
import { lookAtObject } from "../../engine_three_utils.js";
import { IComponent, IContext, IGameObject } from "../../engine_types.js";
import { TypeStore } from "../../engine_typestore.js";
import { DeviceUtilities, getParam } from "../../engine_utils.js";
import { getIconTexture, isIconElement } from "../icons.js";
const debug = getParam("debugspatialmenu");
export class NeedleSpatialMenu {
private readonly _context: IContext;
private readonly needleMenu: HTMLElement;
private readonly htmlButtonsMap = new Map<HTMLElement, SpatialButton>();
private enabled: boolean = true;
constructor(context: IContext, menu: HTMLElement) {
this._context = context;
this._context.pre_render_callbacks.push(this.preRender);
this.needleMenu = menu;
const optionsContainer = this.needleMenu.shadowRoot?.querySelector(".options");
if (!optionsContainer) {
console.error("Could not find options container in needle menu");
}
else {
const watcher = new MutationObserver((mutations) => {
if (!this.enabled) return;
if (this._context.isInXR == false && !debug) return;
for (const mutation of mutations) {
if (mutation.type === "childList") {
mutation.addedNodes.forEach((node) => {
this.createButtonFromHTMLNode(node);
});
mutation.removedNodes.forEach((node) => {
const button = node as HTMLElement;
const spatialButton = this.htmlButtonsMap.get(button);
if (spatialButton) {
this.htmlButtonsMap.delete(button);
spatialButton.remove();
ThreeMeshUI.update();
}
});
}
}
});
watcher.observe(optionsContainer, { childList: true });
}
}
setEnabled(enabled: boolean) {
this.enabled = enabled;
if (!enabled)
this.menu?.removeFromParent();
}
private userRequestedMenu = false;
/** Bring up the spatial menu. This is typically invoked from a button click.
* The menu will show at a lower height to be easily accessible.
* @returns true if the menu was shown, false if it can't be shown because the menu has been disabled.
*/
setDisplay(display: boolean) {
if (!this.enabled) return false;
this.userRequestedMenu = display;
return true;
}
onDestroy() {
const index = this._context.pre_render_callbacks.indexOf(this.preRender);
if (index > -1) {
this._context.pre_render_callbacks.splice(index, 1);
}
}
private uiisDirty = false;
markDirty() {
this.uiisDirty = true;
}
private _showNeedleLogo: undefined | boolean;
showNeedleLogo(show: boolean) {
this._showNeedleLogo = show;
}
private _wasInXR = false;
private preRender = () => {
if (!this.enabled) {
this.menu?.removeFromParent();
return;
}
if (debug && DeviceUtilities.isDesktop()) {
this.updateMenu();
}
const xr = this._context.xr;
const isImmersiveXR = xr?.running && (xr?.isPassThrough || xr?.isVR)
if (!isImmersiveXR) {
if (this._wasInXR) {
this._wasInXR = false;
this.onExitXR();
}
return;
}
if (!this._wasInXR) {
this._wasInXR = true;
this.onEnterXR();
}
this.updateMenu();
}
private onEnterXR() {
const nodes = this.needleMenu.shadowRoot?.querySelector(".options");
if (nodes) {
nodes.childNodes.forEach((node) => {
this.createButtonFromHTMLNode(node);
});
}
}
private onExitXR() {
this.menu?.removeFromParent();
}
private createButtonFromHTMLNode(node: Node) {
const menu = this.getMenu();
const existing = this.htmlButtonsMap.get(node as HTMLElement);
if (existing) {
existing.add();
return;
}
if (node instanceof HTMLButtonElement) {
const spatialButton = this.createButton(menu, node);
this.htmlButtonsMap.set(node, spatialButton);
spatialButton.add();
}
else if (node instanceof HTMLSlotElement) {
node.assignedNodes().forEach((node) => {
this.createButtonFromHTMLNode(node);
});
}
}
private readonly _menuTarget: Object3D = new Object3D();
private readonly positionFilter = new OneEuroFilterXYZ(90, .5);
private updateMenu() {
//performance.mark('NeedleSpatialMenu updateMenu start');
const menu = this.getMenu();
this.handleNeedleWatermark();
this._context.scene.add(menu as any);
const camera = this._context.mainCamera as any as IGameObject;
const xr = this._context.xr;
const rigScale = xr?.rigScale || 1;
if (camera) {
const menuTargetPosition = camera.worldPosition;
const fwd = camera.worldForward.multiplyScalar(-1);
const showMenuThreshold = fwd.y > .6;
const hideMenuThreshold = fwd.y > .4;
const newVisibleState = (menu.visible ? hideMenuThreshold : showMenuThreshold) || this.userRequestedMenu;
const becomesVisible = !menu.visible && newVisibleState;
menu.visible = newVisibleState || (DeviceUtilities.isDesktop() && debug as boolean);
fwd.multiplyScalar(3 * rigScale);
menuTargetPosition.add(fwd);
const testBecomesVisible = false;// this._context.time.frame % 200 == 0;
if (becomesVisible || testBecomesVisible) {
menu.position.copy(this._menuTarget.position);
menu.position.y += 0.25;
this._menuTarget.position.copy(menu.position);
this.positionFilter.reset(menu.position);
menu.quaternion.copy(this._menuTarget.quaternion);
this.markDirty();
}
const distFromForwardView = this._menuTarget.position.distanceTo(menuTargetPosition);
if (becomesVisible || distFromForwardView > 1.5 * rigScale) {
this.ensureRenderOnTop(this.menu as any as Object3D);
this._menuTarget.position.copy(menuTargetPosition);
this._context.scene.add(this._menuTarget);
lookAtObject(this._menuTarget, this._context.mainCamera!, true, true);
this._menuTarget.removeFromParent();
}
this.positionFilter.filter(this._menuTarget.position, menu.position, this._context.time.time);
const step = 5;
this.menu?.quaternion.slerp(this._menuTarget.quaternion, this._context.time.deltaTime * step);
this.menu?.scale.setScalar(rigScale);
}
if (this.uiisDirty) {
//performance.mark('SpatialMenu.update.uiisDirty.start');
this.uiisDirty = false;
ThreeMeshUI.update();
//performance.mark('SpatialMenu.update.uiisDirty.end');
//performance.measure('SpatialMenu.update.uiisDirty', 'SpatialMenu.update.uiisDirty.start', 'SpatialMenu.update.uiisDirty.end');
}
//performance.mark('NeedleSpatialMenu updateMenu end');
//performance.measure('SpatialMenu.update', 'NeedleSpatialMenu updateMenu start', 'NeedleSpatialMenu updateMenu end');
}
private ensureRenderOnTop(obj: Object3D, level: number = 0) {
if (obj instanceof Mesh) {
obj.material.depthTest = false;
obj.material.depthWrite = false;
}
obj.renderOrder = 1000 + level * 2;
for (const child of obj.children) {
this.ensureRenderOnTop(child, level + 1);
}
}
private familyName = "Needle Spatial Menu";
private menu?: ThreeMeshUI.Block;
get isVisible() {
return this.menu?.visible;
}
private getMenu() {
if (this.menu) {
return this.menu;
}
this.ensureFont();
this.menu = new ThreeMeshUI.Block({
boxSizing: 'border-box',
fontFamily: this.familyName,
height: "auto",
fontSize: .1,
color: 0x000000,
lineHeight: 1,
backgroundColor: 0xffffff,
backgroundOpacity: .55,
borderRadius: 1.0,
whiteSpace: 'pre-wrap',
flexDirection: 'row',
alignItems: 'center',
padding: new Vector4(.0, .05, .0, .05),
borderColor: 0x000000,
borderOpacity: .05,
borderWidth: .005
});
// ensure the menu has a raycaster
const raycaster = TypeStore.get("ObjectRaycaster");
if (raycaster)
addNewComponent(this.menu as any, new raycaster())
return this.menu;
}
private _poweredByNeedleElement: ThreeMeshUI.Block | undefined;
private handleNeedleWatermark() {
if (!this._poweredByNeedleElement) {
this._poweredByNeedleElement = new ThreeMeshUI.Block({
width: "auto",
height: "auto",
fontSize: .05,
whiteSpace: 'pre-wrap',
flexDirection: 'row',
flexWrap: 'wrap',
justifyContent: 'center',
margin: 0.02,
borderRadius: .02,
padding: .02,
backgroundColor: 0xffffff,
backgroundOpacity: 1,
});
this._poweredByNeedleElement["needle:use_eventsystem"] = true;
const onClick = new OnClick(this._context, () => globalThis.open("https://needle.tools", "_self"));
addNewComponent(this._poweredByNeedleElement as any as Object3D, onClick as any as IComponent);
const firstLabel = new ThreeMeshUI.Text({
textContent: "Powered by",
width: "auto",
height: "auto",
});
const secondLabel = new ThreeMeshUI.Text({
textContent: "needle",
width: "auto",
height: "auto",
fontSize: .07,
margin: new Vector4(0, 0, 0, .02),
});
this._poweredByNeedleElement.add(firstLabel as any);
this._poweredByNeedleElement.add(secondLabel as any);
this.menu?.add(this._poweredByNeedleElement as any);
this.markDirty();
// const logoObject = needleLogoAsSVGObject();
// logoObject.position.y = 1;
// this._context.scene.add(logoObject);
const textureLoader = new TextureLoader();
textureLoader.load("https://cdn.needle.tools/static/branding/poweredbyneedle.webp", (texture) => {
if (texture) {
onClick.allowModifyUI = false;
firstLabel.removeFromParent();
secondLabel.removeFromParent();
const aspect = texture.image.width / texture.image.height;
this._poweredByNeedleElement?.set({
backgroundImage: texture,
backgroundOpacity: 1,
width: .1 * aspect,
height: .1
});
this.markDirty();
}
});
}
if (this.menu) {
const index = this.menu.children.indexOf(this._poweredByNeedleElement as any);
if (!this._showNeedleLogo && hasProLicense()) {
if (index >= 0) {
this._poweredByNeedleElement.removeFromParent();
this.markDirty();
}
}
else {
this._poweredByNeedleElement.visible = true;
this.menu.add(this._poweredByNeedleElement as any);
const newIndex = this.menu.children.indexOf(this._poweredByNeedleElement as any);
if (index !== newIndex) {
this.markDirty();
}
}
}
}
private ensureFont() {
let fontFamily = ThreeMeshUI.FontLibrary.getFontFamily(this.familyName);
if (!fontFamily) {
fontFamily = ThreeMeshUI.FontLibrary.addFontFamily(this.familyName);
const normal = fontFamily.addVariant(
"normal",
"normal",
"https://cdn.needle.tools/static/fonts/msdf/arial/arial-msdf.json",
"https://cdn.needle.tools/static/fonts/msdf/arial/arial.png") as any as ThreeMeshUI.FontVariant;
/** @ts-ignore */
normal?.addEventListener('ready', () => {
this.markDirty();
});
}
}
private createButton(menu: ThreeMeshUI.Block, htmlButton: HTMLButtonElement): SpatialButton {
const buttonParent = new ThreeMeshUI.Block({
width: "auto",
height: "auto",
whiteSpace: 'pre-wrap',
flexDirection: 'row',
flexWrap: 'wrap',
justifyContent: 'center',
backgroundColor: 0xffffff,
backgroundOpacity: 0,
padding: 0.02,
margin: 0.01,
borderRadius: 0.02,
cursor: 'pointer',
fontSize: 0.05,
});
const text = new ThreeMeshUI.Text({
textContent: "",
width: "auto",
justifyContent: 'center',
alignItems: 'center',
backgroundOpacity: 0,
backgroundColor: 0xffffff,
fontFamily: this.familyName,
color: 0x000000,
borderRadius: 0.02,
padding: .01,
});
buttonParent.add(text as any);
buttonParent["needle:use_eventsystem"] = true;
const onClick = new OnClick(this._context, () => htmlButton.click());
addNewComponent(buttonParent as any as Object3D, onClick as any as IComponent);
const spatialButton = new SpatialButton(this, menu, htmlButton, buttonParent, text);
return spatialButton;
}
}
class SpatialButton {
readonly menu: NeedleSpatialMenu;
readonly root: ThreeMeshUI.Block;
readonly htmlbutton: HTMLButtonElement;
readonly spatialContainer: ThreeMeshUI.Block;
readonly spatialText: ThreeMeshUI.Text;
private spatialIcon?: ThreeMeshUI.InlineBlock;
constructor(menu: NeedleSpatialMenu, root: ThreeMeshUI.Block, htmlbutton: HTMLButtonElement, buttonContainer: ThreeMeshUI.Block, buttonText: ThreeMeshUI.Text) {
this.menu = menu;
this.root = root;
this.htmlbutton = htmlbutton;
this.spatialContainer = buttonContainer;
this.spatialText = buttonText;
const styleObserver = new MutationObserver((mutations) => {
for (const mutation of mutations) {
if (mutation.type === "attributes") {
if (mutation.attributeName === "style") {
this.updateVisible();
}
}
else if (mutation.type === "childList") {
this.updateText();
}
}
});
// watch attributes and content
styleObserver.observe(htmlbutton, { attributes: true, childList: true });
this.updateText();
}
add() {
if (this.spatialContainer.parent != this.root as any) {
this.root.add(this.spatialContainer as any);
this.menu.markDirty();
this.updateVisible();
this.updateText();
}
}
remove() {
if (this.spatialContainer.parent) {
this.spatialContainer.removeFromParent();
this.menu.markDirty();
}
}
private updateVisible() {
const wasVisible = this.spatialContainer.visible;
this.spatialContainer.visible = this.htmlbutton.style.display !== "none";
if (wasVisible !== this.spatialContainer.visible) {
this.menu.markDirty();
}
}
private _lastText = "";
private updateText() {
let newText = "";
let iconToCreate = "";
this.htmlbutton.childNodes.forEach((child) => {
if (child.nodeType === Node.TEXT_NODE) {
newText += child.textContent;
}
else if (child instanceof HTMLElement && isIconElement(child) && child.textContent) {
iconToCreate = child.textContent;
}
});
if (this._lastText !== newText) {
this._lastText = newText;
this.spatialText.name = newText;
this.spatialText.set({ textContent: newText });
this.menu.markDirty();
}
if (newText.length <= 0) {
if (this.spatialText.parent) {
this.spatialText.removeFromParent();
this.menu.markDirty();
}
}
else {
if (!this.spatialText.parent) {
this.spatialContainer.add(this.spatialText as any);
this.menu.markDirty();
}
}
if (iconToCreate) {
this.createIcon(iconToCreate);
}
}
private _lastTexture?: string;
private async createIcon(str: string) {
if (!this.spatialIcon) {
const texture = await getIconTexture(str);
if (texture && !this.spatialIcon) {
const size = 0.08;
const icon = new ThreeMeshUI.Block({
width: size,
height: size,
backgroundColor: 0xffffff,
backgroundImage: texture,
backgroundOpacity: 1,
margin: new Vector4(0, .005, 0, 0),
});
this.spatialIcon = icon;
this.spatialContainer.add(icon as any);
this.menu.markDirty();
}
}
if (str != this._lastTexture) {
this._lastTexture = str;
const texture = await getIconTexture(str);
if (texture) {
this.spatialIcon?.set({ backgroundImage: texture });
this.menu.markDirty();
}
}
// make sure the icon is at the first index
const index = this.spatialContainer.children.indexOf(this.spatialIcon as any);
if (index > 0) {
this.spatialContainer.children.splice(index, 1);
this.spatialContainer.children.unshift(this.spatialIcon as any);
this.menu.markDirty();
}
}
}
// TODO: perhaps we should have a basic IComponent implementation in the engine folder to be able to write this more easily. OR possibly reduce the IComponent interface to the minimum
class OnClick implements Pick<IComponent, "__internalAwake"> {
readonly isComponent = true;
readonly enabled = true;
get activeAndEnabled() { return true; }
__internalAwake() { }
__internalEnable() { }
__internalDisable() { }
__internalStart() { }
onEnable() { }
onDisable() { }
gameObject!: IGameObject;
allowModifyUI = true;
get element() {
return this.gameObject as any as ThreeMeshUI.MeshUIBaseElement;
}
readonly context: Context;
readonly onclick: () => void;
constructor(context: Context, onclick: () => void) {
this.context = context;
this.onclick = onclick;
}
onPointerEnter() {
this.context.input.setCursor("pointer");
if (this.allowModifyUI) {
this.element.set({ backgroundOpacity: 1 });
ThreeMeshUI.update();
}
}
onPointerExit() {
this.context.input.unsetCursor("pointer");
if (this.allowModifyUI) {
this.element.set({ backgroundOpacity: 0 });
ThreeMeshUI.update();
}
}
onPointerDown(e) {
e.use();
}
onPointerUp(e) {
e.use();
}
onPointerClick(e) {
e.use();
this.onclick();
}
}