playcanvas
Version:
Open-source WebGL/WebGPU 3D engine for the web
1,414 lines (1,224 loc) • 58.1 kB
JavaScript
import { BUTTON_TRANSITION_MODE_TINT, Color, Entity, Quat, Script, Vec2, Vec3, Vec4, XRTARGETRAY_POINTER } from 'playcanvas';
/** @import { Asset, XrInputSource } from 'playcanvas' */
// Pre-allocated vectors for performance
const tmpVec3A = new Vec3();
const tmpVec3B = new Vec3();
const tmpVec3C = new Vec3();
const tmpVec3D = new Vec3();
const tmpQuat = new Quat();
// Finger joint IDs for extension detection (pre-allocated to avoid GC pressure)
const FINGER_JOINTS = [
{ tip: 'index-finger-tip', meta: 'index-finger-metacarpal' },
{ tip: 'middle-finger-tip', meta: 'middle-finger-metacarpal' },
{ tip: 'ring-finger-tip', meta: 'ring-finger-metacarpal' },
{ tip: 'pinky-finger-tip', meta: 'pinky-finger-metacarpal' }
];
/**
* Provides a hybrid WebXR menu system that supports Hand Tracking ("Palm Up" gesture),
* Controller button toggle, and an always-visible camera-anchored mode for debug HUDs. The
* menu automatically picks the right input mode and switches between hand-anchored,
* controller-anchored, and camera-anchored positioning.
*
* This script uses PlayCanvas' UI system (Screen, Element, Button components) for rendering
* the menu, providing proper text rendering and familiar button interaction patterns.
*
* This script should be attached to an entity in your scene. It creates menu buttons dynamically
* based on the `menuItems` configuration. When an interactive item is activated, it fires the
* corresponding app event.
*
* Items in `menuItems` come in two kinds:
*
* - **Interactive buttons** — declared with both `label` and `eventName`. Clickable; fire
* `app.fire(eventName)` when triggered.
* - **Labels** — declared with only `label` (no `eventName`). Non-interactive. Render in the
* same stack but skip click/hover, and their text is dimmed via {@link XrMenu#labelTextOpacity}.
* Useful for value readouts that can be updated at runtime with {@link XrMenu#setItemLabel}.
*
* Features:
*
* - **Hand Tracking** — detects "open palm facing camera" gesture to show the menu anchored
* to the palm.
* - **Controller mode** — toggle menu visibility with a configurable gamepad button, anchored
* to the controller.
* - **Always-visible mode** ({@link XrMenu#alwaysVisible}) — bypasses palm/button toggling,
* keeps the menu visible for the full XR session, and follows the camera at a configurable
* offset ({@link XrMenu#followDistance}, {@link XrMenu#followOffset}). Useful for debug HUDs.
* - **XR ray picking** — controller pointer rays drive hover via ray-vs-`worldCorners`
* intersection, and the trigger (gamepad button 0) fires clicks. Self-contained, does not
* rely on ElementInput's XR hover events.
* - **Finger touch** — hand-tracking index-fingertip pokes are also supported on interactive
* buttons.
* - **Runtime label updates** — {@link XrMenu#setItemLabel} changes the displayed text of any
* item (interactive or label-only) after the menu has been built. The new text is persisted
* into `menuItems` so it survives a future regeneration.
* - Smooth following with configurable dampening.
* - Fires `'xr:menu:active'` app event when menu visibility changes (for coordination with
* other scripts).
*
* @example
* // Configure menu items via script attributes — mix interactive buttons with read-only labels:
* menuItems: [
* { label: 'STATUS: idle' }, // label-only row
* { label: 'Teleport', eventName: 'menu:teleport' }, // interactive
* { label: 'Settings', eventName: 'menu:settings' },
* { label: 'Exit', eventName: 'xr:end' }
* ]
*
* @example
* // Update a label at runtime (e.g. show live values):
* const xrMenu = menuEntity.script.xrMenu;
* app.on('update', () => xrMenu.setItemLabel(0, `STATUS: ${currentStatus}`));
*
* @example
* // Always-visible debug HUD anchored to the right of the eye line:
* menuEntity.script.create(XrMenu, {
* properties: {
* menuItems: [ ... ],
* alwaysVisible: true,
* followDistance: 0.6,
* followOffset: new Vec2(0.25, -0.15)
* }
* });
* @category XR
*/
class XrMenu extends Script {
static scriptName = 'xrMenu';
/**
* Array of menu item definitions. Item kinds:
* - Interactive button: `{ label, eventName }` — fires `app.fire(eventName)` when activated.
* - Label (read-only): `{ label }` — non-interactive; update via {@link XrMenu#setItemLabel}.
* - Number row: `{ type: 'number', label, value, decEvent, incEvent }` — a single line with a
* `label: value` readout and '-' / '+' buttons firing `decEvent` / `incEvent`. Update the
* value via {@link XrMenu#setItemValue}.
*
* @type {Array<{label?: string, value?: string, eventName?: string, type?: string, decEvent?: string, incEvent?: string}>}
* @attribute
*/
menuItems = [];
/**
* Audio asset for button click sound.
*
* @type {Asset|null}
* @attribute
*/
clickSound = null;
/**
* Font asset for button text. Required for text rendering.
*
* @type {Asset|null}
* @attribute
*/
fontAsset = null;
/**
* Offset from the anchor point where the menu appears.
* For hand tracking: Z is distance from palm center along the palm normal.
* For controllers: Applied in controller-local space.
*
* @type {Vec3}
* @attribute
*/
menuOffset = new Vec3(0, 0, 0.06);
/**
* Vertical spacing between menu buttons in meters.
*
* @type {number}
* @attribute
* @range [0.001, 0.05]
* @precision 0.001
*/
buttonSpacing = 0.0025;
/**
* Width of each button in meters.
*
* @type {number}
* @attribute
* @range [0.02, 0.3]
* @precision 0.01
*/
buttonWidth = 0.075;
/**
* Height of each button in meters.
*
* @type {number}
* @attribute
* @range [0.01, 0.1]
* @precision 0.001
*/
buttonHeight = 0.015;
/**
* Font size for button text in UI pixels.
*
* @type {number}
* @attribute
* @range [4, 48]
*/
fontSize = 8;
/**
* Overall scale multiplier for the entire menu.
*
* @type {number}
* @attribute
* @range [0.5, 2]
* @precision 0.1
*/
menuScale = 1.0;
/**
* How quickly the menu follows the anchor point. Higher values = snappier movement.
*
* @type {number}
* @attribute
* @range [1, 30]
*/
followSpeed = 25;
/**
* Dot product threshold for detecting palm-up gesture. Higher values require the palm
* to face more directly toward the camera.
*
* @type {number}
* @attribute
* @range [0.3, 0.95]
* @precision 0.05
*/
palmUpThreshold = 0.6;
/**
* Gamepad button index used to toggle the menu in controller mode.
* Default is 4 (typically Y button on left controller, B on right).
*
* @type {number}
* @attribute
* @range [0, 10]
*/
toggleButtonIndex = 4;
/**
* Which hand the menu should be attached to ('left' or 'right').
*
* @type {string}
* @attribute
*/
preferredHand = 'left';
/**
* Distance threshold for finger touch hover in meters.
*
* @type {number}
* @attribute
* @range [0.01, 0.1]
* @precision 0.01
*/
touchDistance = 0.05;
/**
* Cooldown time after button press before another press is allowed (seconds).
*
* @type {number}
* @attribute
* @range [0.1, 1.0]
* @precision 0.1
*/
pressCooldown = 0.3;
/**
* Background color of menu buttons. Default is a dark slate so white text reads cleanly.
*
* @type {Color}
* @attribute
*/
buttonColor = new Color(0.12, 0.14, 0.18, 0.85);
/**
* Color of menu buttons when hovered. Default matches the typical XR pointer-ray cyan so
* "ray on button" reads as a single visual feedback loop.
*
* @type {Color}
* @attribute
*/
hoverColor = new Color(0.30, 0.65, 0.95, 1);
/**
* Color of menu buttons when pressed/activated.
*
* @type {Color}
* @attribute
*/
pressColor = new Color(0.55, 0.85, 1.0, 1);
/**
* Text color for button labels.
*
* @type {Color}
* @attribute
*/
textColor = new Color(0.95, 0.97, 1.0);
/**
* Opacity multiplier applied to text on label-only items (those declared without an
* `eventName`). Used so interactive buttons read as the primary affordance and label rows
* sit visually behind them. Range 0..1.
*
* @type {number}
* @attribute
* @range [0, 1]
* @precision 0.05
*/
labelTextOpacity = 0.65;
/**
* Optional texture asset for button backgrounds.
*
* @type {Asset|null}
* @attribute
*/
buttonTexture = null;
/**
* Duration of fade in/out animation in seconds.
*
* @type {number}
* @attribute
* @range [0, 1]
* @precision 0.05
*/
fadeDuration = 0.15;
/**
* When true, the menu is shown for the full duration of the XR session and follows the
* camera, rather than being toggled by the palm-up gesture or controller button. Hand-tracking
* finger-touch and controller-ray interaction still work. Useful for debug HUDs.
*
* @type {boolean}
* @attribute
*/
alwaysVisible = false;
/**
* Distance in meters in front of the camera to place the menu when {@link alwaysVisible} is on.
*
* @type {number}
* @attribute
* @range [0.2, 2]
* @precision 0.05
*/
followDistance = 0.6;
/**
* Lateral and vertical offset in meters applied in camera-local space when
* {@link alwaysVisible} is on. `x` is positive-right, `y` is positive-up.
*
* @type {Vec2}
* @attribute
*/
followOffset = new Vec2(0, -0.2);
// Internal state
/** @type {Entity|null} */
_menuContainer = null;
/** @type {Entity|null} */
_screenEntity = null;
/** @type {Entity[]} */
_buttons = [];
/**
* Per-line layout descriptors (one per generated menu row). A simple row is
* `{ kind: 'simple', entity }`; a number row is
* `{ kind: 'number', labelEntity, minus, plus, labelX, minusX, plusX, baseLabel }`.
*
* @type {Array<Object>}
*/
_rows = [];
/**
* Per-`menuItems` text-update targets, so {@link XrMenu#setItemLabel} / {@link XrMenu#setItemValue}
* stay indexed by the original `menuItems` position even when a row expands into multiple
* entities (number rows). Each entry is `{ entity, baseLabel }`.
*
* @type {Array<{entity: Entity, baseLabel: string|null}>}
*/
_itemTargets = [];
/** @type {Set<XrInputSource>} */
_inputSources = new Set();
/** @type {boolean} */
_menuVisible = false;
/** @type {boolean} */
_toggleButtonWasPressed = false;
/** @type {Vec3} */
_targetPosition = new Vec3();
/** @type {Quat} */
_targetRotation = new Quat();
/** @type {Entity|null} */
_hoveredButton = null;
/** @type {Entity|null} */
_pressedButton = null;
/** @type {number} */
_lastPressTime = 0;
/** @type {XrInputSource|null} */
_activeInputSource = null;
/** @type {Entity|null} */
_cameraEntity = null;
/** @type {number} */
_uiScale = 0.001; // Convert UI pixels to meters
/** @type {number} */
_currentOpacity = 0;
/** @type {number} */
_targetOpacity = 0;
/** @type {boolean} */
_followInitialized = false;
/**
* Per-input-source previous gamepad trigger (button 0) state, used to edge-detect "trigger pull"
* during XR ray picking. WeakMap so entries clean up when the input source goes away.
*
* @type {WeakMap<XrInputSource, boolean>}
* @private
*/
_triggerWasPressed = new WeakMap();
initialize() {
if (!this.app.xr) {
console.warn('XrMenu: XR is not available on this application');
return;
}
// Find camera entity for palm detection
this._cameraEntity = this.entity.findComponent('camera')?.entity || null;
if (!this._cameraEntity) {
// Try to find any camera in the scene
this._cameraEntity = this.app.root.findComponent('camera')?.entity || null;
}
// Set up click sound (non-positional for UI feedback)
if (this.clickSound) {
this.entity.addComponent('sound', {
positional: false
});
this.entity.sound?.addSlot('click', {
asset: this.clickSound.id,
volume: 0.5
});
}
// Create menu container and UI
this._createMenu();
// Hide menu initially
this._setMenuVisible(false);
// Listen for XR input sources
this.app.xr.input.on('add', this._onInputSourceAdd, this);
this.app.xr.input.on('remove', this._onInputSourceRemove, this);
// Listen for XR session end to clean up.
this.app.xr.on('end', this._onXrEnd, this);
// Drive clicks from the select/pinch event in addition to the gamepad-button polling in
// _updateRayInteraction. This is required on platforms (e.g. Apple Vision Pro) where a pinch
// arrives as a 'transient-pointer' select event with no pollable button on the hovering
// 'tracked-pointer' source.
this.app.xr.input.on('selectstart', this._onSelectStart, this);
this.on('destroy', () => {
this._onDestroy();
});
}
_onDestroy() {
if (this.app.xr) {
this.app.xr.input.off('add', this._onInputSourceAdd, this);
this.app.xr.input.off('remove', this._onInputSourceRemove, this);
this.app.xr.input.off('selectstart', this._onSelectStart, this);
this.app.xr.off('end', this._onXrEnd, this);
}
// Destroy menu container
if (this._menuContainer) {
this._menuContainer.destroy();
this._menuContainer = null;
}
this._buttons = [];
this._rows = [];
this._itemTargets = [];
this._inputSources.clear();
}
_onXrEnd() {
this._setMenuVisible(false);
this._inputSources.clear();
this._activeInputSource = null;
this._followInitialized = false;
}
/**
* Handler for the raw 'selectstart' (pinch/trigger press) input event. Clicks the currently
* hovered button. This is the click path for platforms where the pinch arrives as a separate
* 'transient-pointer' select source (e.g. Apple Vision Pro) that the per-frame gamepad-button
* pick loop can't see; the hovered button is already tracked via the 'tracked-pointer' ray.
* The {@link _onButtonClick} debounce prevents a double-fire on platforms (e.g. Quest) where the
* gamepad-button path also fires for the same press.
*
* @private
*/
_onSelectStart() {
// _hoveredButton is only ever an interactive button (labels are skipped during picking), and
// it reflects whatever the pointer ray is currently on.
if (this._hoveredButton) {
this._onButtonClick(this._hoveredButton);
}
}
/**
* Updates the displayed text of a menu item by index. Works for both interactive buttons
* and label-only items (those declared without an `eventName`). The caller controls casing —
* the text is written verbatim.
*
* @param {number} index - Index into the `menuItems` array passed at construction.
* @param {string} text - New display text.
* @returns {boolean} True if the item was found and updated, false otherwise.
*/
setItemLabel(index, text) {
// Persist the new text into menuItems so it survives a future _generateButtons call,
// and so the *first* _generateButtons (which may run after this method when
// setItemLabel is called synchronously after script.create) picks up the updated label
// instead of the placeholder originally passed in menuItems.
const item = this.menuItems[index];
if (item) item.label = text;
// Index by menuItems position via _itemTargets (robust to number rows expanding a single
// item into multiple entities); fall back to the flat _buttons list before generation.
const entity = this._itemTargets[index]?.entity ?? this._buttons[index];
if (!entity) return false;
// Prefer the direct ref captured in menuData; fall back to children[0] for items that
// were created before menuData.textElement was added (defensive — shouldn't happen).
// @ts-ignore - menuData is a custom property attached in _createButton
const textElement = entity.menuData?.textElement ?? entity.children[0]?.element;
if (!textElement) return false;
textElement.text = text;
// @ts-ignore
if (entity.menuData) entity.menuData.label = text;
return true;
}
/**
* Updates the value shown on a `number` row, recomposing the displayed text as `LABEL: value`
* from the row's static label. The new value is persisted into the `menuItems` entry so it
* survives regeneration.
*
* @param {number} index - Index into the `menuItems` array (the number row).
* @param {string} value - New value text (e.g. '1.5M').
* @returns {boolean} True if the item was found and updated, false otherwise.
*/
setItemValue(index, value) {
const item = this.menuItems[index];
if (item) item.value = value;
const target = this._itemTargets[index];
if (!target?.entity) return false;
const text = target.baseLabel != null ? `${target.baseLabel}: ${value}` : `${value}`;
// @ts-ignore - menuData is a custom property attached in _createButton
const textElement = target.entity.menuData?.textElement ?? target.entity.children[0]?.element;
if (!textElement) return false;
textElement.text = text;
return true;
}
/**
* Sets the overall menu scale at runtime, re-applying it to the menu screen immediately.
* Useful for compensating text readability when the XR framebuffer resolution is reduced
* (scale the menu up so it covers more framebuffer pixels).
*
* @param {number} value - New overall scale multiplier for the whole menu.
*/
setMenuScale(value) {
this.menuScale = value;
if (this._screenEntity) {
const scale = this._uiScale * value;
this._screenEntity.setLocalScale(scale, scale, scale);
}
}
/**
* Whether a pointer ray (controller or hand) is currently over an interactive menu button.
* Updated every frame from the ray-picking pass. Useful for coordinating with world
* interactors — e.g. a navigation script can skip teleporting when a select/pinch gesture
* lands on the menu rather than the world.
*
* @type {boolean}
*/
get isPointerOverMenu() {
return this._hoveredButton !== null;
}
/**
* @param {XrInputSource} inputSource - The input source that was added.
* @private
*/
_onInputSourceAdd(inputSource) {
this._inputSources.add(inputSource);
}
/**
* @param {XrInputSource} inputSource - The input source that was removed.
* @private
*/
_onInputSourceRemove(inputSource) {
this._inputSources.delete(inputSource);
if (this._activeInputSource === inputSource) {
this._activeInputSource = null;
this._setMenuVisible(false);
}
}
/**
* Creates the menu with PlayCanvas UI system.
*
* @private
*/
_createMenu() {
// Create a container entity for positioning
this._menuContainer = new Entity('XrMenuContainer');
this.app.root.addChild(this._menuContainer);
// Create a world-space screen for UI
this._screenEntity = new Entity('XrMenuScreen');
this._screenEntity.addComponent('screen', {
referenceResolution: new Vec2(1000, 1000),
screenSpace: false,
scaleBlend: 1
});
// Scale the screen to convert pixels to meters
const scale = this._uiScale * this.menuScale;
this._screenEntity.setLocalScale(scale, scale, scale);
this._menuContainer.addChild(this._screenEntity);
// Generate buttons from menuItems
this._generateButtons();
}
/**
* Generates menu buttons from the menuItems configuration.
*
* @private
*/
_generateButtons() {
if (!this._screenEntity) return;
// Clear existing buttons
for (const button of this._buttons) {
button.destroy();
}
this._buttons = [];
this._rows = [];
this._itemTargets = [];
// Convert meter sizes to UI pixels
const widthPx = this.buttonWidth / this._uiScale;
const heightPx = this.buttonHeight / this._uiScale;
const spacingPx = this.buttonSpacing / this._uiScale;
// Create buttons from menuItems
for (let i = 0; i < this.menuItems.length; i++) {
const item = this.menuItems[i];
if (!item || typeof item !== 'object') continue;
// Number row: a value readout plus '-' / '+' buttons on a single line.
if (item.type === 'number') {
const row = this._createNumberRow(item, i, widthPx, heightPx);
this._screenEntity.addChild(row.labelEntity);
this._screenEntity.addChild(row.minus);
this._screenEntity.addChild(row.plus);
// The label is non-interactive; minus/plus are interactive (picked + clicked like
// any other button). All three are tracked in _buttons for picking/opacity.
this._buttons.push(row.labelEntity, row.minus, row.plus);
this._rows.push(row);
this._itemTargets.push({ entity: row.labelEntity, baseLabel: row.baseLabel });
continue;
}
const label = item.label || `Button ${i}`;
const eventName = item.eventName || '';
const button = this._createButton(label, eventName, i, widthPx, heightPx);
if (button) {
this._screenEntity.addChild(button);
this._buttons.push(button);
this._rows.push({ kind: 'simple', entity: button });
this._itemTargets.push({ entity: button, baseLabel: null });
}
}
// Layout rows vertically
this._layoutButtons(heightPx, spacingPx);
}
/**
* Creates a single menu button using PlayCanvas UI.
*
* @param {string} label - Display text for the button.
* @param {string} eventName - Event to fire when button is activated.
* @param {number} index - Index of the button in the menu.
* @param {number} widthPx - Button width in pixels.
* @param {number} heightPx - Button height in pixels.
* @returns {Entity} The created button entity.
* @private
*/
_createButton(label, eventName, index, widthPx, heightPx) {
const isLabel = !eventName;
const button = new Entity(isLabel ? `MenuLabel_${index}` : `MenuButton_${index}`);
// Resting element color. Textured backgrounds only apply to interactive buttons (labels
// stay flat so they read as non-buttons), and a textured element must be tinted white to
// show the texture — so the "base" color to restore after hover/press is per-button, not
// always buttonColor.
const useTexture = !isLabel && !!this.buttonTexture?.resource;
const baseColor = useTexture ? new Color(1, 1, 1, this.buttonColor.a) : this.buttonColor;
// Add button component for interactivity (interactive items only). We keep TINT mode
// (the only auto-color mode the engine ships) but set all three tints equal to the
// button's base color, so the component's auto-tint is effectively a no-op — including
// when ElementInput's XR ray hover (selectenter/selectleave) drives it. Visuals are
// driven manually by _setHovered / _setButtonPress, which works reliably for both
// XR ray picking and finger touch even when ElementInput's XR hover events don't fire.
if (!isLabel) {
button.addComponent('button', {
active: true,
transitionMode: BUTTON_TRANSITION_MODE_TINT,
hoverTint: baseColor,
pressedTint: baseColor,
inactiveTint: baseColor
});
}
// Add element component for visual appearance (image type for button background)
/** @type {Object} */
const elementConfig = {
type: 'image',
anchor: new Vec4(0.5, 0.5, 0.5, 0.5),
pivot: new Vec2(0.5, 0.5),
width: widthPx,
height: heightPx,
color: baseColor,
opacity: this.buttonColor.a,
useInput: !isLabel,
layers: [this.app.scene.layers.getLayerByName('UI')?.id ?? 0]
};
if (useTexture) {
elementConfig.textureAsset = this.buttonTexture.id;
}
button.addComponent('element', elementConfig);
// Store metadata
// @ts-ignore - Adding custom property
button.menuData = {
label: label,
eventName: eventName,
index: index,
isLabel: isLabel,
baseColor: baseColor,
/** @type {import('playcanvas').ElementComponent|null} */
textElement: null // populated after the text child is created below
};
// Handle button click (interactive items only). We keep the click event as a fallback
// for input paths that still work through ElementInput (e.g. desktop mouse during
// pre-XR debugging). _onButtonClick has a cooldown guard, so double-firing with our
// own ray-picking click is harmless.
//
// We deliberately do NOT subscribe to hoverstart/hoverend here: ElementInput's XR hover
// events are unreliable in practice, and if they fire they'd update _hoveredButton
// without applying any visual — fighting the ray-picking path that drives both. Hover
// state in this script is owned entirely by _updateRayInteraction and _checkFingerTouch.
if (!isLabel && button.button) {
button.button.on('click', () => {
this._onButtonClick(button);
});
}
// Create text label as child
const textEntity = new Entity('ButtonText');
textEntity.addComponent('element', {
type: 'text',
text: label,
anchor: new Vec4(0, 0, 1, 1),
pivot: new Vec2(0.5, 0.5),
margin: new Vec4(4, 4, 4, 4),
fontSize: this.fontSize,
color: this.textColor,
fontAsset: this.fontAsset?.id ?? this._getDefaultFontAsset()?.id,
autoWidth: false,
autoHeight: false,
wrapLines: false,
alignment: new Vec2(0.5, 0.5)
});
button.addChild(textEntity);
// Direct reference — avoids relying on children[0] indexing (cheap to mis-assume if a
// future change adds a sibling element).
// @ts-ignore - menuData is a custom property attached above
button.menuData.textElement = textEntity.element;
return button;
}
/**
* Creates a single-line "number" row: a value readout on the left and small '-' / '+' buttons
* on the right. The readout is a non-interactive label; the buttons fire `decEvent` / `incEvent`.
* Returns the three entities plus their precomputed horizontal positions for {@link XrMenu#_layoutButtons}.
*
* @param {Object} item - The menu item ({ type:'number', label, value, decEvent, incEvent }).
* @param {number} index - Index of the item in `menuItems`.
* @param {number} widthPx - Full row width in pixels.
* @param {number} heightPx - Row height in pixels.
* @returns {Object} Row descriptor.
* @private
*/
_createNumberRow(item, index, widthPx, heightPx) {
const gap = heightPx * 0.25;
const btn = heightPx; // square +/- buttons
const labelWidth = Math.max(1, widthPx - 2 * btn - 2 * gap);
// Centers, with the row centered at x = 0 (total width = widthPx).
const left = -widthPx / 2;
const labelX = left + labelWidth / 2;
const minusX = left + labelWidth + gap + btn / 2;
const plusX = left + labelWidth + gap + btn + gap + btn / 2;
const baseLabel = item.label ?? '';
const value = item.value ?? '';
// Reuse _createButton for all three cells (label = non-interactive, minus/plus = buttons).
const labelEntity = this._createButton(`${baseLabel}: ${value}`, '', index, labelWidth, heightPx);
const minus = this._createButton('-', item.decEvent || '', index, btn, heightPx);
const plus = this._createButton('+', item.incEvent || '', index, btn, heightPx);
return { kind: 'number', labelEntity, minus, plus, labelX, minusX, plusX, baseLabel };
}
/**
* Gets or creates a default font asset.
*
* @returns {Asset|null} The default font asset.
* @private
*/
_getDefaultFontAsset() {
// Try to find an existing font in the asset registry
const fonts = this.app.assets.filter(asset => asset.type === 'font');
if (fonts.length > 0) {
return fonts[0];
}
return null;
}
/**
* Whether the press debounce cooldown has elapsed since the last accepted press.
*
* @returns {boolean} True when a new press may be accepted.
* @private
*/
_cooldownElapsed() {
return Date.now() / 1000 - this._lastPressTime >= this.pressCooldown;
}
/**
* Handles button click with visual feedback.
*
* @param {Entity} button - The clicked button.
* @private
*/
_onButtonClick(button) {
// @ts-ignore
const menuData = button.menuData;
if (!menuData) return;
// Debounce: avoid double-firing if multiple input paths (ray picking + ElementInput
// click + finger touch) all detect the same press within pressCooldown.
if (!this._cooldownElapsed()) return;
this._lastPressTime = Date.now() / 1000;
// Play click sound
if (this.entity.sound) {
this.entity.sound.play('click');
}
// Visual feedback - flash press color and scale
this._setButtonPress(button, true);
// Reset visual after short delay
setTimeout(() => {
this._setButtonPress(button, false);
}, 150);
// Fire the event
if (menuData.eventName) {
this.app.fire(menuData.eventName);
}
}
/**
* Applies the visual (color + scale) matching a button's current press/hover state.
* Single source of truth for button styling - press and hover transitions both funnel
* through here.
*
* @param {Entity} button - The button to restyle.
* @private
*/
_applyButtonVisual(button) {
if (!button.element) return;
// @ts-ignore - _isPressed is a custom property attached in _setButtonPress
if (button._isPressed) {
button.element.color = this.pressColor;
button.setLocalScale(0.95, 0.95, 1);
} else if (this._hoveredButton === button) {
button.element.color = this.hoverColor;
button.setLocalScale(1.05, 1.05, 1);
} else {
// @ts-ignore - menuData is a custom property attached in _createButton
button.element.color = button.menuData?.baseColor ?? this.buttonColor;
button.setLocalScale(1, 1, 1);
}
}
/**
* Sets press state on a button and refreshes its visual.
*
* @param {Entity} button - The button.
* @param {boolean} pressed - Whether the button is pressed.
* @private
*/
_setButtonPress(button, pressed) {
// @ts-ignore - Adding custom property
button._isPressed = pressed;
this._applyButtonVisual(button);
}
/**
* Transitions hover to a new button (or null). Single owner of {@link XrMenu#_hoveredButton}:
* updates the state, then refreshes the visuals of the buttons that changed.
*
* @param {Entity|null} button - The newly hovered button, or null to clear hover.
* @private
*/
_setHovered(button) {
const prev = this._hoveredButton;
if (prev === button) return;
this._hoveredButton = button;
if (prev) this._applyButtonVisual(prev);
if (button) this._applyButtonVisual(button);
}
/**
* Lays out buttons vertically in the menu.
*
* @param {number} heightPx - Button height in pixels.
* @param {number} spacingPx - Spacing between buttons in pixels.
* @private
*/
_layoutButtons(heightPx, spacingPx) {
// One line per row (a number row's value + '-' + '+' share a single line).
const rows = this._rows;
const totalHeight = (rows.length - 1) * (heightPx + spacingPx) + heightPx;
const startY = totalHeight / 2 - heightPx / 2;
for (let r = 0; r < rows.length; r++) {
const y = startY - r * (heightPx + spacingPx);
const row = rows[r];
if (row.kind === 'number') {
row.labelEntity.setLocalPosition(row.labelX, y, 0);
row.minus.setLocalPosition(row.minusX, y, 0);
row.plus.setLocalPosition(row.plusX, y, 0);
} else {
row.entity.setLocalPosition(0, y, 0);
}
}
}
/**
* Sets menu visibility and fires the appropriate event.
*
* @param {boolean} visible - Whether the menu should be visible.
* @private
*/
_setMenuVisible(visible) {
if (this._menuVisible === visible) return;
this._menuVisible = visible;
this._targetOpacity = visible ? 1 : 0;
// Enable container immediately when showing (opacity will fade in)
if (visible && this._menuContainer) {
this._menuContainer.enabled = true;
// Snap to current anchor position immediately (don't lerp from old position)
if (this._activeInputSource) {
const anchor = this._getAnchor(this._activeInputSource);
if (anchor) {
this._menuContainer.setPosition(anchor.position);
this._menuContainer.setRotation(anchor.rotation);
}
}
}
// Fire event for other scripts to coordinate (e.g., disable navigation while menu is open)
this.app.fire('xr:menu:active', visible);
// Reset hover state when hiding
if (!visible) {
this._setHovered(null);
this._pressedButton = null;
}
}
/**
* Updates the opacity of all menu elements.
*
* @param {number} opacity - Opacity value from 0 to 1.
* @private
*/
_updateMenuOpacity(opacity) {
for (const button of this._buttons) {
if (button.element) {
button.element.opacity = opacity * this.buttonColor.a;
}
// Also update text opacity. Label-only items get a dim multiplier so the eye is drawn
// to interactive buttons.
// @ts-ignore - menuData is a custom property attached in _createButton
const menuData = button.menuData;
if (menuData?.textElement) {
menuData.textElement.opacity = opacity * (menuData.isLabel ? this.labelTextOpacity : 1);
}
}
}
/**
* Finds the preferred input source based on handedness setting.
*
* @returns {XrInputSource|null} The preferred input source or null.
* @private
*/
_findPreferredInput() {
for (const inputSource of this._inputSources) {
if (inputSource.handedness === this.preferredHand) {
return inputSource;
}
}
// Fallback to any available input
for (const inputSource of this._inputSources) {
if (inputSource.handedness !== 'none') {
return inputSource;
}
}
return null;
}
/**
* Checks if the fingers are extended (open hand).
* Measures the distance from fingertip to metacarpal (knuckle) -
* when extended this is large (~8-10cm), when curled it's small (~3-5cm).
*
* @param {XrInputSource} inputSource - The hand input source.
* @returns {boolean} True if fingers are extended.
* @private
*/
_areFingersExtended(inputSource) {
const hand = inputSource.hand;
if (!hand || !hand.tracking) return false;
let extendedCount = 0;
for (const finger of FINGER_JOINTS) {
const tip = hand.getJointById(finger.tip);
const meta = hand.getJointById(finger.meta);
if (!tip || !meta) continue;
// Distance from metacarpal (knuckle) to fingertip
// Extended finger: ~8-10cm (0.08-0.10m)
// Curled finger: ~3-5cm (0.03-0.05m)
const tipToMeta = tip.getPosition().distance(meta.getPosition());
// Threshold: finger is extended if tip is more than 6cm from knuckle
if (tipToMeta > 0.06) {
extendedCount++;
}
}
// Require at least 3 fingers extended for "open hand"
return extendedCount >= 3;
}
/**
* Checks if the palm is facing the camera with an open hand gesture.
*
* @param {XrInputSource} inputSource - The hand input source.
* @returns {boolean} True if palm is facing camera with open hand.
* @private
*/
_isPalmFacingCamera(inputSource) {
// First check if fingers are extended (open hand)
if (!this._areFingersExtended(inputSource)) {
return false;
}
// Get palm normal using shared calculation
const palmNormal = this._getPalmNormal(inputSource);
if (!palmNormal) return false;
// Get camera forward direction
if (!this._cameraEntity) return false;
const cameraForward = this._cameraEntity.forward;
// Check if palm normal faces roughly toward camera (negative dot product)
// We want the palm facing the user, so the normal should point toward the camera
const dot = palmNormal.dot(cameraForward);
// Negative dot means palm is facing camera
return dot < -this.palmUpThreshold;
}
/**
* Calculates the palm normal vector (pointing away from palm surface).
*
* @param {XrInputSource} inputSource - The hand input source.
* @returns {Vec3|null} The palm normal or null.
* @private
*/
_getPalmNormal(inputSource) {
const hand = inputSource.hand;
if (!hand || !hand.tracking) return null;
const wrist = hand.wrist;
const middleMeta = hand.getJointById('middle-finger-metacarpal');
const indexMeta = hand.getJointById('index-finger-metacarpal');
const pinkyMeta = hand.getJointById('pinky-finger-metacarpal');
if (!wrist || !middleMeta || !indexMeta || !pinkyMeta) return null;
const wristPos = wrist.getPosition();
const middlePos = middleMeta.getPosition();
const indexPos = indexMeta.getPosition();
const pinkyPos = pinkyMeta.getPosition();
// Vector from wrist to middle finger base
tmpVec3A.sub2(middlePos, wristPos);
// Vector from index to pinky (across the palm)
tmpVec3B.sub2(pinkyPos, indexPos);
// Cross product gives palm normal
tmpVec3C.cross(tmpVec3A, tmpVec3B).normalize();
// Flip normal for left hand so it always points away from palm surface
if (inputSource.handedness === 'left') {
tmpVec3C.mulScalar(-1);
}
return tmpVec3C;
}
/**
* Gets the palm anchor position and rotation for menu placement.
*
* Note: Returns references to reused internal Vec3/Quat objects for performance.
* Callers must use the values immediately or copy them - do not store the references.
*
* @param {XrInputSource} inputSource - The hand input source.
* @returns {{position: Vec3, rotation: Quat}|null} Anchor transform or null.
* @private
*/
_getPalmAnchor(inputSource) {
const hand = inputSource.hand;
if (!hand || !hand.tracking) return null;
// Use middle-finger-phalanx-proximal (first knuckle) for positioning closer to palm center
const middleProximal = hand.getJointById('middle-finger-phalanx-proximal');
const middleMeta = hand.getJointById('middle-finger-metacarpal');
if (!middleProximal || !middleMeta) return null;
// Get palm normal (pointing away from palm surface, toward camera when palm is up)
const palmNormal = this._getPalmNormal(inputSource);
if (!palmNormal) return null;
// Position at center of palm (halfway between metacarpal and proximal)
this._targetPosition.lerp(middleMeta.getPosition(), middleProximal.getPosition(), 0.5);
// Offset the menu along the palm normal (in front of palm)
tmpVec3A.copy(palmNormal).mulScalar(this.menuOffset.z);
this._targetPosition.add(tmpVec3A);
// Menu should face the camera (full look-at, not just Y rotation)
if (this._cameraEntity) {
const cameraPos = this._cameraEntity.getPosition();
tmpVec3A.sub2(cameraPos, this._targetPosition);
if (tmpVec3A.lengthSq() > 0.001) {
tmpVec3A.normalize();
// Calculate yaw (Y rotation)
const yaw = Math.atan2(tmpVec3A.x, tmpVec3A.z) * (180 / Math.PI);
// Calculate pitch (X rotation) - tilt to face camera
const pitch = -Math.asin(tmpVec3A.y) * (180 / Math.PI);
this._targetRotation.setFromEulerAngles(pitch, yaw, 0);
}
}
return {
position: this._targetPosition,
rotation: this._targetRotation
};
}
/**
* Gets the controller anchor position and rotation for menu placement.
*
* Note: Returns references to reused internal Vec3/Quat objects for performance.
* Callers must use the values immediately or copy them - do not store the references.
*
* @param {XrInputSource} inputSource - The controller input source.
* @returns {{position: Vec3, rotation: Quat}|null} Anchor transform or null.
* @private
*/
_getControllerAnchor(inputSource) {
if (!inputSource.grip) return null;
const position = inputSource.getPosition();
const rotation = inputSource.getRotation();
if (!position || !rotation) return null;
// Apply offset in controller-local space
this._targetPosition.copy(position);
tmpVec3A.copy(this.menuOffset);
rotation.transformVector(tmpVec3A, tmpVec3A);
this._targetPosition.add(tmpVec3A);
// Menu faces outward from controller
this._targetRotation.copy(rotation);
return {
position: this._targetPosition,
rotation: this._targetRotation
};
}
/**
* Gets the menu anchor transform for an input source - the palm for tracked hands, the
* controller grip otherwise.
*
* Note: Returns references to reused internal Vec3/Quat objects for performance.
* Callers must use the values immediately or copy them - do not store the references.
*
* @param {XrInputSource} inputSource - The input source.
* @returns {{position: Vec3, rotation: Quat}|null} Anchor transform or null.
* @private
*/
_getAnchor(inputSource) {
return inputSource.hand ?
this._getPalmAnchor(inputSource) :
this._getControllerAnchor(inputSource);
}
/**
* Smoothly moves the menu container toward a target transform at
* {@link XrMenu#followSpeed}.
*
* @param {Vec3} position - Target world position.
* @param {Quat} rotation - Target world rotation.
* @param {number} dt - Delta time.
* @private
*/
_smoothFollow(position, rotation, dt) {
const container = this._menuContainer;
const t = Math.min(1, this.followSpeed * dt);
tmpVec3B.lerp(container.getPosition(), position, t);
container.setPosition(tmpVec3B);
tmpQuat.slerp(container.getRotation(), rotation, t);
container.setRotation(tmpQuat);
}
/**
* While the menu is visible (or still fading out), smoothly follows the anchor transform
* of the given input source.
*
* @param {XrInputSource} inputSource - The input source anchoring the menu.
* @param {number} dt - Delta time.
* @private
*/
_followAnchor(inputSource, dt) {
if ((!this._menuVisible && this._currentOpacity <= 0) || !this._menuContainer) return;
const anchor = this._getAnchor(inputSource);
if (anchor) {
this._smoothFollow(anchor.position, anchor.rotation, dt);
}
}
/**
* Checks for finger touch interaction with buttons.
*
* @param {XrInputSource} inputSource - The hand input source.
* @private
*/
_checkFingerTouch(inputSource) {
const hand = inputSource.hand;
if (!hand || !hand.tracking) return;
// Get index finger tip position (using the opposite hand for interaction)
// Find the other hand to use for touching
let touchHand = null;
for (const source of this._inputSources) {
if (source !== inputSource && source.hand && source.hand.tracking) {
touchHand = source.hand;
break;
}
}
if (!touchHand) return;
const indexTip = touchHand.getJointById('index-finger-tip');
if (!indexTip) return;
const fingerPos = indexTip.getPosition();
let closestButton = null;
let closestDist = this.touchDistance;
for (const button of this._buttons) {
// @ts-ignore - menuData is a custom property attached in _createButton
if (button.menuData?.isLabel) continue;
const buttonPos = button.getPosition();
const dist = fingerPos.distance(buttonPos);
if (dist < closestDist) {
closestDist = dist;
closestButton = button;
}
}
const pressDist = this.touchDistance * 0.6; // Press threshold
if (closestButton) {
this._setHovered(closestButton);
// Check for press (finger moving into button)
// Only allow press if: within press distance, not already pressed, and cooldown elapsed
if (closestDist < pressDist) {
if (!this._pressedButton && this._cooldownElapsed()) {
this._pressedButton = closestButton;
// Note: _lastPressTime is owned by _onButtonClick — setting it here would
// trip _onButtonClick's own debounce guard and swallow the click entirely.
this._onButtonClick(closestButton);
}
} else if (this._pressedButton === closestButton && closestDist >= pressDist) {
// Finger moved out of press threshold but is still hovering - clear pressed state
this._pressedButton = null;
}
} else {
// Finger fully exited hover zone - clear states and allow re-press
this._setHovered(null);
this._pressedButton = null;
}
}
/**
* Updates hand tracking mode.
*
* @param {XrInputSource} inputSource - The hand input source.
* @param {number} dt - Delta time.
* @private
*/
_updateHandMode(inputSource, dt) {
// Palm-up gesture shows the menu; dropping the gesture hides it
const palmFacing = this._isPalmFacingCamera(inputSource);
this._setMenuVisible(palmFacing);
// Check for finger touch interaction
if (palmFacing) {
this._checkFingerTouch(inputSource);
}
this._followAnchor(inputSource, dt);
}
/**
* Updates controller mode.
*
* @param {XrInputSource} inputSource - The controller input source.
* @param {number} dt - Delta time.
* @private
*/
_updateControllerMode(inputSource, dt) {
// Edge-detect the menu toggle button
const gamepad = inputSource.gamepad;
if (gamepad?.buttons?.[this.toggleButtonIndex]) {
const pressed = gamepad.buttons[this.toggleButtonIndex].pressed;
if (pressed && !this._toggleButtonWasPressed) {
this._setMenuVisible(!this._menuVisible);
}
this._toggleButtonWasPressed = pressed;
} else {
// Reset toggle state if the gamepad or button is unavailable
this._toggleButtonWasPressed = false;
}
this._followAnchor(inputSource, dt);
}
update(dt) {
if (