@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.
558 lines (509 loc) • 23.8 kB
text/typescript
import { isDevEnvironment, showBalloonError, showBalloonWarning } from "../engine/debug/index.js";
import { Application } from "../engine/engine_application.js";
import { RoomEvents } from "../engine/engine_networking.js";
import { disposeStream, NetworkedStreamEvents, NetworkedStreams, StreamEndedEvent, StreamReceivedEvent } from "../engine/engine_networking_streams.js"
import { serializable } from "../engine/engine_serialization_decorator.js";
import { delay, DeviceUtilities, getParam } from "../engine/engine_utils.js";
import { getIconElement } from "../engine/webcomponents/icons.js";
import { Behaviour } from "./Component.js";
import { EventList } from "./EventList.js";
export const noVoip = "noVoip";
const debugParam = getParam("debugvoip");
/**
* [Voip](https://engine.needle.tools/docs/api/Voip) Voice over IP (VoIP) component for real-time audio communication between users.
* Allows sending and receiving audio streams in networked rooms.
*
* **Requirements:**
* - Active network connection (via {@link SyncedRoom} or manual connection)
* - User permission for microphone access (requested automatically)
* - HTTPS connection (required for WebRTC)
*
* **Features:**
* - Automatic connection when joining rooms (`autoConnect`)
* - Background audio support (`runInBackground`)
* - Optional UI toggle button (`createMenuButton`)
* - Mute/unmute control
*
* **Debug:** Use `?debugvoip` URL parameter or set `debug = true` for logging.
* Press 'v' to toggle mute, 'c' to connect/disconnect when debugging.
*
* @example Enable VoIP in your scene
* ```ts
* const voip = myObject.addComponent(Voip);
* voip.autoConnect = true;
* voip.createMenuButton = true;
*
* // Manual control
* voip.connect(); // Start sending your microphone
* voip.disconnect(); // Stop sending your microphone
* voip.setMuted(true); // Mute incoming audio (silence other users)
* ```
*
* @summary Voice over IP for networked audio communication
* @category Networking
* @group Components
* @see {@link SyncedRoom} for room management
* @see {@link NetworkedStreams} for the underlying streaming
* @see {@link ScreenCapture} for video streaming
* @link https://engine.needle.tools/docs/networking.html
*/
export class Voip extends Behaviour {
/** When enabled, VoIP will start when a room is joined or when this component is enabled while already in a room.
* @default true
*/
autoConnect: boolean = true;
/**
* When enabled, VoIP will stay connected even when the browser tab is not focused/active anymore.
* @default true
*/
runInBackground: boolean = true;
/**
* When enabled, a menu button will be created to allow the user to toggle VoIP on and off.
*/
createMenuButton: boolean = true;
/**
* When enabled debug messages will be printed to the console. This is useful for debugging audio issues. You can also append ?debugvoip to the URL to enable this.
*/
debug: boolean = false;
private _volume: number = 1;
/**
* Volume for incoming audio streams (0 = silent, 1 = full volume).
* Changes apply immediately to all active incoming streams.
*/
get volume(): number { return this._volume; }
set volume(val: number) {
// HTMLMediaElement.volume throws IndexSizeError outside [0,1] — clamp before assigning.
// Reject NaN so we don't poison _volume and break serialization.
if (Number.isNaN(val)) return;
const clamped = Math.max(0, Math.min(1, val));
this._volume = clamped;
for (const audio of this._incomingStreams.values()) {
audio.volume = clamped;
}
}
/**
* Get the incoming audio element for a specific user.
* Use this to route audio through the Web Audio API for spatial audio, effects, or analysis.
* @param userId The connection ID of the remote user
* @returns The HTMLAudioElement for this user's stream, or undefined if not connected
* @example
* ```ts
* const audioEl = voip.getAudioElement(userId);
* if (audioEl) {
* const audioCtx = new AudioContext();
* const source = audioCtx.createMediaElementSource(audioEl);
* const panner = audioCtx.createPanner();
* source.connect(panner).connect(audioCtx.destination);
* }
* ```
*/
getAudioElement(userId: string): HTMLAudioElement | undefined {
return this._incomingStreams.get(userId);
}
/**
* Get all active incoming audio streams as a Map of userId → HTMLAudioElement.
* Useful for iterating over all connected voice users.
*/
get incomingStreams(): ReadonlyMap<string, HTMLAudioElement> {
return this._incomingStreams;
}
/**
* Normalized amplitude threshold for speaking detection (0–1). When a user's average
* audio amplitude exceeds this, they are considered "speaking". Default is 0.1.
*/
speakingThreshold: number = 0.1;
/**
* Event fired when a user's speaking state changes.
* Passes `{ userId: string, isSpeaking: boolean, volume: number }`.
*/
onSpeakingChanged: EventList = new EventList();
private _speakingStates = new Map<string, boolean>();
private _analysers = new Map<string, { source: MediaStreamAudioSourceNode, analyser: AnalyserNode, data: Uint8Array }>();
// Single shared AudioContext for all remote-user analysers. Browsers cap live AudioContexts at ~6 per tab,
// so one-per-user would break voice rooms of 7+ participants. Lazily created on first setupAnalyser.
private _sharedAudioContext?: AudioContext;
private _lastSpeakingPollMs = 0;
private _net?: NetworkedStreams;
private _menubutton?: HTMLElement;
/** @internal */
awake() {
if (debugParam) this.debug = true;
if (this.debug) {
console.log("VOIP debugging: press 'v' to toggle incoming mute, 'c' to toggle connect/disconnect");
window.addEventListener("keydown", async (evt) => {
const key = evt.key.toLowerCase();
switch (key) {
case "v":
console.log("VOIP: toggle incoming mute → ", !this.isMuted);
this.setMuted(!this.isMuted);
break;
case "c":
if (this.isSending) this.disconnect();
else this.connect();
break;
}
});
}
}
/** @internal */
onEnable(): void {
if (!this._net) this._net = NetworkedStreams.create(this);
if (this.debug) this._net.debug = true;
this._net.addEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream);
this._net.addEventListener(NetworkedStreamEvents.StreamEnded, this.onStreamEnded);
this._net.enable();
if (this.autoConnect) {
if (this.context.connection.isConnected)
this.connect();
}
this.context.connection.beginListen(RoomEvents.JoinedRoom, this.onJoinedRoom);
this.context.connection.beginListen(RoomEvents.LeftRoom, this.onLeftRoom);
this.onEnabledChanged();
this.updateButton();
window.addEventListener("visibilitychange", this.onVisibilityChanged);
}
/** @internal */
onDisable(): void {
if (this._net) {
this._net.stopSendingStream(this._outputStream);
//@ts-ignore
this._net.removeEventListener(NetworkedStreamEvents.StreamReceived, this.onReceiveStream);
//@ts-ignore
this._net.removeEventListener(NetworkedStreamEvents.StreamEnded, this.onStreamEnded)
this._net?.disable();
}
this.context.connection.stopListen(RoomEvents.JoinedRoom, this.onJoinedRoom);
this.context.connection.stopListen(RoomEvents.LeftRoom, this.onLeftRoom);
this.onEnabledChanged();
this.updateButton();
window.removeEventListener("visibilitychange", this.onVisibilityChanged);
// Clean up analysers and the shared AudioContext (lazily recreated on next setupAnalyser).
for (const userId of [...this._analysers.keys()]) {
this.cleanupAnalyser(userId);
}
this.closeSharedAudioContext();
}
/** @internal */
onDestroy(): void {
this._menubutton?.remove();
this._menubutton = undefined;
// Clean up all streams and analysers
for (const userId of [...this._analysers.keys()]) {
this.cleanupAnalyser(userId);
}
this.closeSharedAudioContext();
for (const incoming of this._incomingStreams.values()) {
disposeStream(incoming.srcObject as MediaStream);
}
this._incomingStreams.clear();
this._speakingStates.clear();
}
/** Set via the mic button (e.g. when the websocket connection closes and rejoins but the user was muted before we don't want to enable VOIP again automatically) */
private _allowSending = true;
private _outputStream: MediaStream | null = null;
// Tracks an in-flight connect() so concurrent callers coalesce onto the same promise
// instead of each running getUserMedia in parallel (which would leak a MediaStream and
// briefly transmit then kill the first acquired stream).
private _connectInFlight?: Promise<boolean>;
/**
* @returns true if the component is currently sending audio
*/
get isSending() { return this._outputStream != null && this._outputStream.active; }
/** Start sending audio. */
connect(audioSource?: MediaTrackConstraints): Promise<boolean> {
// Coalesce concurrent callers. Without this, two near-simultaneous connect() calls
// each call getUserMedia in parallel, the first stream gets disposed mid-broadcast
// by the second, and a MediaStream leaks.
if (this._connectInFlight) return this._connectInFlight;
this._connectInFlight = this._connectImpl(audioSource).finally(() => {
this._connectInFlight = undefined;
});
return this._connectInFlight;
}
private async _connectImpl(audioSource?: MediaTrackConstraints): Promise<boolean> {
if (!this._net) {
console.error("Cannot connect to voice chat - NetworkedStreams not initialized. Make sure the component is enabled before calling this method.");
return false;
}
if (!this.context.connection.isConnected) {
console.error("Cannot connect to voice chat - not connected to server");
this.updateButton();
return false;
}
else if (!await DeviceUtilities.microphonePermissionsGranted()) {
console.error("Cannot connect to voice chat - microphone permissions not granted");
this.updateButton();
return false;
}
this._allowSending = true;
this._net?.stopSendingStream(this._outputStream);
disposeStream(this._outputStream);
this._outputStream = await this.getAudioStream(audioSource);
if (this._outputStream) {
if (this.debug) console.log("VOIP: Got audio stream");
this._net?.startSendingStream(this._outputStream);
this.updateButton();
return true;
}
else {
this.updateButton();
if (!await DeviceUtilities.microphonePermissionsGranted()) {
showBalloonError("Microphone permissions not granted: Please grant microphone permissions to use voice chat");
}
else console.error("VOIP: Could not get audio stream - please make sure to connect an audio device and grant microphone permissions");
}
if (this.debug || isDevEnvironment()) console.log("VOIP: Failed to get audio stream");
return false;
}
/** Stop sending audio (muting your own microphone) */
disconnect(opts?: { remember: boolean }) {
if (opts?.remember) {
this._allowSending = false;
}
this._net?.stopSendingStream(this._outputStream);
disposeStream(this._outputStream);
this._outputStream = null;
this.updateButton();
}
/**
* Mute or unmute the audio you hear from other users (incoming streams).
* This does NOT mute your own microphone — use {@link disconnect} to stop sending your microphone.
*/
setMuted(mute: boolean) {
for (const audio of this._incomingStreams.values()) {
audio.muted = mute;
}
}
/**
* Returns true if incoming audio is currently muted (you can't hear other users).
* When there are no incoming streams, returns false.
*/
get isMuted() {
if (this._incomingStreams.size === 0) return false;
for (const audio of this._incomingStreams.values()) {
if (!audio.muted) return false;
}
return true;
}
private async updateButton() {
if (this.createMenuButton) {
if (!this._menubutton) {
this._menubutton = document.createElement("button");
this._menubutton.addEventListener("click", () => {
if (this.isSending) {
this.disconnect({ remember: true });
}
else this.connect();
DeviceUtilities.microphonePermissionsGranted().then(res => {
if (!res) showBalloonWarning("<strong>Microphone permissions not granted</strong>. Please allow your browser to use the microphone to be able to talk. Click on the button on the left side of your browser's address bar to allow microphone permissions.");
})
});
}
if (this._menubutton) {
this.context.menu.appendChild(this._menubutton);
if (this.activeAndEnabled) {
this._menubutton.style.display = "";
}
else {
this._menubutton.style.display = "none";
}
this._menubutton.title = this.isSending ? "Click to disable your microphone" : "Click to enable your microphone";
let label = this.isSending ? "" : "";
let icon = this.isSending ? "mic" : "mic_off";
const hasPermission = await DeviceUtilities.microphonePermissionsGranted();
if (!hasPermission) {
label = "No Permission";
icon = "mic_off";
this._menubutton.title = "Microphone permissions not granted. Please allow your browser to use the microphone to be able to talk. This can usually be done in the addressbar of the webpage.";
}
this._menubutton.innerText = label;
this._menubutton.prepend(getIconElement(icon));
if (this.context.connection.isConnected == false)
this._menubutton.setAttribute("disabled", "");
else
this._menubutton.removeAttribute("disabled");
}
}
else if (!this.activeAndEnabled) {
this._menubutton?.remove();
}
}
// private _analyzer?: AudioAnalyser;
/** @deprecated */
public getFrequency(_userId: string | null): number | null {
if (!this["unsupported_getfrequency"]) {
this["unsupported_getfrequency"] = true;
if (isDevEnvironment()) showBalloonWarning("VOIP: getFrequency is currently not supported");
console.warn("VOIP: getFrequency is currently not supported");
}
// null is get the first with some data
// if (userId === null) {
// for (const c in this._incomingStreams) {
// const call = this._incomingStreams[c];
// if (call && call.currentAnalyzer) return call.currentAnalyzer.getAverageFrequency();
// }
// return null;
// }
// const call = this._incomingStreams.get(userId);
// if (call && call.currentAnalyzer) return call.currentAnalyzer.getAverageFrequency();
return null;
}
private async getAudioStream(audio?: MediaTrackConstraints) {
if (!navigator.mediaDevices.getUserMedia) {
console.error("No getDisplayMedia support");
return null;
}
const getUserMedia = async (constraints?: MediaTrackConstraints): Promise<MediaStream | null> => {
return await navigator.mediaDevices.getUserMedia({ audio: constraints ?? true, video: false })
.catch((err) => {
console.warn("VOIP failed getting audio stream", err);
return null;
});
}
const stream = await getUserMedia(audio);
if (!stream) return null;
// NE-5445, on iOS after calling `getUserMedia` it automatically switches the audio to the built-in microphone and speakers even if headphones are connected
// if there's no device selected explictly we will try to automatically select an external device
if (DeviceUtilities.isiOS() && audio?.deviceId === undefined) {
const devices = await navigator.mediaDevices.enumerateDevices();
// select anything that doesn't have "iPhone" is likely "AirPods" or other bluetooth headphones
const nonBuiltInAudioSource = devices.find((device) => (device.kind === "audioinput" || device.kind === "audiooutput") && !device.label.includes("iPhone"));
if (nonBuiltInAudioSource) {
const constraints = Object.assign({}, audio);
constraints.deviceId = nonBuiltInAudioSource.deviceId;
const externalStream = await getUserMedia(constraints);
if (externalStream) {
// Release the built-in mic stream we grabbed first — otherwise its tracks stay live.
disposeStream(stream);
return externalStream;
}
// External device acquisition failed — keep the original stream rather than returning null.
}
}
return stream;
}
// we have to wait for the user to connect to a room when "auto connect" is enabled
private onJoinedRoom = async () => {
if (this.debug) console.log("VOIP: Joined room");
// Wait a moment for user list to be populated
await delay(300)
if (this.autoConnect && !this.isSending && this._allowSending) {
this.connect();
}
}
private onLeftRoom = () => {
if (this.debug) console.log("VOIP: Left room");
this.disconnect();
for (const incoming of this._incomingStreams.values()) {
disposeStream(incoming.srcObject as MediaStream);
}
this._incomingStreams.clear();
for (const userId of this._analysers.keys()) {
this.cleanupAnalyser(userId);
}
}
private _incomingStreams: Map<string, HTMLAudioElement> = new Map();
/** @internal */
update() {
// Only run speaking detection if someone is listening
if (!this.onSpeakingChanged || this.onSpeakingChanged.listenerCount <= 0) return;
// Rate-limit analysis to ~10Hz. Speaking-state is a coarse UI signal; running FFT per
// remote user every frame (60Hz) is wasteful and scales linearly with participant count.
const now = performance.now();
if (now - this._lastSpeakingPollMs < 100) return;
this._lastSpeakingPollMs = now;
for (const [userId, info] of this._analysers) {
info.analyser.getByteFrequencyData(info.data as Uint8Array<ArrayBuffer>);
// Average amplitude normalized to 0–1
let sum = 0;
for (let i = 0; i < info.data.length; i++) sum += info.data[i];
const volume = sum / info.data.length / 255;
const wasSpeaking = this._speakingStates.get(userId) ?? false;
const isSpeaking = volume > this.speakingThreshold;
if (isSpeaking !== wasSpeaking) {
this._speakingStates.set(userId, isSpeaking);
this.onSpeakingChanged.invoke({ userId, isSpeaking, volume });
}
}
}
private setupAnalyser(userId: string, _audioElement: HTMLAudioElement, stream: MediaStream) {
// Only set up if someone is listening or might listen
if (this._analysers.has(userId)) return;
try {
if (!this._sharedAudioContext) {
this._sharedAudioContext = new AudioContext();
}
const ctx = this._sharedAudioContext;
const source = ctx.createMediaStreamSource(stream);
const analyser = ctx.createAnalyser();
analyser.fftSize = 256;
source.connect(analyser);
const data = new Uint8Array(analyser.frequencyBinCount);
this._analysers.set(userId, { source, analyser, data });
}
catch (err) {
if (this.debug) console.warn("VOIP: Failed to create analyser for", userId, err);
}
}
private cleanupAnalyser(userId: string) {
const info = this._analysers.get(userId);
if (info) {
info.source.disconnect();
info.analyser.disconnect();
this._analysers.delete(userId);
}
this._speakingStates.delete(userId);
}
private closeSharedAudioContext() {
if (this._sharedAudioContext) {
this._sharedAudioContext.close();
this._sharedAudioContext = undefined;
}
}
private onReceiveStream = (evt: StreamReceivedEvent) => {
const userId = evt.target.userId;
const stream = evt.stream;
let audioElement = this._incomingStreams.get(userId);
if (!audioElement) {
audioElement = new Audio()
this._incomingStreams.set(userId, audioElement);
}
audioElement.srcObject = stream;
audioElement.volume = this._volume;
audioElement.setAttribute("autoplay", "true");
this.setupAnalyser(userId, audioElement, stream);
// for mobile we need to wait for user interaction to play audio. Auto play doesnt work on android when the page is refreshed
Application.registerWaitForInteraction(() => {
audioElement?.play().catch((err) => {
console.error("VOIP: Failed to play audio", err);
});
})
}
private onStreamEnded = (evt: StreamEndedEvent) => {
const existing = this._incomingStreams.get(evt.userId);
disposeStream(existing?.srcObject as MediaStream);
this._incomingStreams.delete(evt.userId);
this.cleanupAnalyser(evt.userId);
}
private onEnabledChanged = () => {
for (const key of this._incomingStreams) {
const element = key[1];
element.muted = !this.enabled;
}
}
private onVisibilityChanged = () => {
if (this.runInBackground) return;
const muted = document.visibilityState !== "visible";
// Mute incoming so we don't hear other users while tab is hidden.
this.setMuted(muted);
// Also disable our outgoing mic tracks (cheaper than disconnect/reconnect — keeps the mic permission).
const tracks = this._outputStream?.getAudioTracks();
if (tracks) for (const t of tracks) t.enabled = !muted;
};
}