@needle-tools/engine/src/engine-components/AudioSource.ts

Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in.

import { AudioLoader, PositionalAudio } from "three";
import { PositionalAudioHelper } from 'three/examples/jsm/helpers/PositionalAudioHelper.js';

import { isDevEnvironment } from "../engine/debug/index.js";
import { Application, ApplicationEvents } from "../engine/engine_application.js";
import { findObjectOfType } from "../engine/engine_components.js";
import { serializable } from "../engine/engine_serialization_decorator.js";
import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
import { AudioListener } from "./AudioListener.js";
import { Behaviour, GameObject } from "./Component.js";


const debug = getParam("debugaudio");

/**
 * Defines how audio volume attenuates over distance from the listener.
 */
export enum AudioRolloffMode {
    /**
     * Logarithmic rolloff provides a natural, real-world attenuation where volume decreases
     * exponentially with distance.
     */
    Logarithmic = 0,

    /**
     * Linear rolloff provides a straightforward volume reduction that decreases at a constant
     * rate with distance.
     */
    Linear = 1,

    /**
     * Custom rolloff allows for defining specialized distance-based attenuation curves.
     * Note: Custom rolloff is not fully implemented in this version.
     */
    Custom = 2,
}


/**
 * Plays audio clips in the scene with support for spatial (3D) positioning.  
 *
 * **Browser autoplay policies:**  
 * Web browsers require user interaction before playing audio. Use  
 * `AudioSource.userInteractionRegistered` to check if playback is allowed,  
 * or `registerWaitForAllowAudio()` to queue playback until interaction occurs.  
 *
 * **Spatial audio:**  
 * Set `spatialBlend` to 1 for full 3D positioning, or 0 for 2D (non-spatial).  
 * Requires an {@link AudioListener} in the scene (typically on the camera).  
 *
 * **Visibility handling:**  
 * Audio automatically pauses when the tab is hidden unless `playInBackground = true`.  
 * On mobile, audio always pauses in background regardless of this setting.  
 *
 * @example Play audio on button click
 * ```ts
 * onClick() {
 *   const audio = this.getComponent(AudioSource);
 *   audio.play();
 * }
 * ```
 *
 * @example Wait for user interaction
 * ```ts
 * AudioSource.registerWaitForAllowAudio(() => {
 *   this.getComponent(AudioSource)?.play();
 * });
 * ```
 *
 * @summary Plays audio clips from files or media streams
 * @category Multimedia
 * @group Components
 * @see {@link AudioListener} for the audio receiver component
 * @see {@link AudioRolloffMode} for distance attenuation options
 * @see {@link Voip} for voice communication
 * @see {@link PlayableDirector} for timeline-based audio  
 * @link https://engine.needle.tools/samples/?overlay=samples&tag=audio
 * @link https://spatial-audio-zubckswmztj.needle.run/
 */
export class AudioSource extends Behaviour {

    /**
     * Checks if the user has interacted with the page to allow audio playback.
     * Audio playback often requires a user gesture first due to browser autoplay policies.
     * This is the same as calling {@link Application.userInteractionRegistered}.
     * 
     * @returns Whether user interaction has been registered to allow audio playback
     */
    public static get userInteractionRegistered(): boolean {
        return Application.userInteractionRegistered;
    }

    /**
     * Registers a callback that will be executed once the user has interacted with the page,
     * allowing audio playback to begin.
     * This is the same as calling {@link Application.registerWaitForInteraction}.
     * 
     * @param cb - The callback function to execute when user interaction is registered
     */
    public static registerWaitForAllowAudio(cb: Function) {
        Application.registerWaitForInteraction(cb);
    }

    /**
     * The audio clip to play. Can be a URL string pointing to an audio file or a {@link MediaStream} object.
     */
    @serializable(URL)
    clip: string | MediaStream = "";

    /**
     * When true, the audio will automatically start playing when the component is enabled.
     * When false, you must call play() manually to start audio playback.
     * @default false
     */
    @serializable()
    playOnAwake: boolean = false;

    /**
     * When true, the audio clip will be loaded during initialization rather than when play() is called.
     * This can reduce playback delay but increases initial loading time.
     * @default true
     */
    @serializable()
    preload: boolean = true;

    /**
     * When true, audio will continue playing when the browser tab loses focus.
     * When false, audio will pause when the tab is minimized or not active.
     * @default true
     */
    @serializable()
    playInBackground: boolean = true;

    /**
     * Indicates whether the audio is currently playing.
     * 
     * @returns True if the audio is playing, false otherwise
     */
    get isPlaying(): boolean { return this.sound?.isPlaying ?? false; }

    /**
     * The total duration of the currently loaded audio clip in seconds.
     * 
     * @returns Duration in seconds or undefined if no clip is loaded
     * @remarks For MediaStream clips, duration is not directly available and will return undefined. If the audio clip has not started loading or is still loading, duration may also be undefined until the audio buffer is ready.
     */
    get duration() {
        if (this.sound?.buffer?.duration) {
            return this.sound.buffer.duration;
        }
        else if (this._audioElement?.duration) {
            return this._audioElement.duration;
        }
        else if (this._loadedClip instanceof MediaStream) {
            // MediaStream duration is not directly available; return undefined or estimate if possible
            return undefined;
        }
        return undefined;
    }

    /**
     * The current playback position as a normalized value between 0 and 1.
     * Can be set to seek to a specific position in the audio.
     */
    get time01() {
        const duration = this.duration;
        if (duration && this.sound) {
            return this.sound?.context.currentTime / duration;
        }
        return 0;
    }
    set time01(val: number) {
        const duration = this.duration;
        if (duration && this.sound) {
            this.time = val * duration;
        }
    }

    /**
     * The current playback position in seconds.
     * Can be set to seek to a specific time in the audio.
     */
    get time(): number { return this.sound?.source ? (this.sound.source?.context.currentTime - this._lastContextTime + this.sound.offset) : 0; }
    set time(val: number) {
        if (this.sound) {
            if (val === this.sound.offset) return;
            const wasPlaying = this.isPlaying;
            this.stop();
            this.sound.offset = val;
            if (wasPlaying)
                this.play();
        }
    }

    /**
     * When true, the audio will repeat after reaching the end.
     * When false, audio will play once and stop.
     * @default false
     */
    @serializable()
    get loop(): boolean {
        if (this.sound) this._loop = this.sound.getLoop();
        return this._loop;
    }
    set loop(val: boolean) {
        this._loop = val;
        if (this.sound) this.sound.setLoop(val);
    }

    /**
     * Controls how the audio is positioned in space.
     * Values range from 0 (2D, non-positional) to 1 (fully 3D positioned).
     * Internally uses a dual-path audio graph to crossfade between a spatialized (PannerNode)
     * and a non-spatialized (direct) signal path.
     */
    @serializable()
    get spatialBlend(): number {
        return this._spatialBlend;
    }
    set spatialBlend(val: number) {
        if (val === this._spatialBlend) return;
        this._spatialBlend = val;
        this._needUpdateSpatialDistanceSettings = true;
    }

    /**
     * The minimum distance from the audio source at which the volume starts to attenuate.
     * Within this radius, the audio plays at full volume regardless of distance.
     */
    @serializable()
    get minDistance(): number {
        return this._minDistance;
    }
    set minDistance(val: number) {
        if (this._minDistance === val) return;
        this._minDistance = val;
        this._needUpdateSpatialDistanceSettings = true;
    }

    /**
     * The maximum distance from the audio source beyond which the volume no longer decreases.
     * This defines the outer limit of the attenuation curve.
     */
    @serializable()
    get maxDistance(): number {
        return this._maxDistance;
    }
    set maxDistance(val: number) {
        if (this._maxDistance === val) return;
        this._maxDistance = val;
        this._needUpdateSpatialDistanceSettings = true;
    }

    private _spatialBlend: number = 0;
    private _minDistance: number = 1;
    private _maxDistance: number = 100;

    /**
     * Controls the overall volume/loudness of the audio.
     * Values range from 0 (silent) to 1 (full volume).
     * @default 1
     */
    @serializable()
    get volume(): number { return this._volume; }
    set volume(val: number) {
        this._volume = val;
        if (this.sound && !this.context.application.muted) {
            if (debug) console.log(this.name, "audio set volume", val);
            this.sound.setVolume(val);
        }
    }
    private _volume: number = 1;

    /**
     * Controls the playback rate (speed) of the audio.
     * Values greater than 1 increase speed, values less than 1 decrease it.
     * This affects both speed and pitch of the audio.
     * @default 1
     */
    @serializable()
    set pitch(val: number) {
        if (this.sound) this.sound.setPlaybackRate(val);
    }
    get pitch(): number {
        return this.sound ? this.sound.getPlaybackRate() : 1;
    }

    /**
     * Determines how audio volume decreases with distance from the listener.
     * @default AudioRolloffMode.Logarithmic
     * @see {@link AudioRolloffMode}
     */
    @serializable()
    rollOffMode: AudioRolloffMode = 0;


    private _loop: boolean = false;
    private sound: PositionalAudio | null = null;
    private helper: PositionalAudioHelper | null = null;
    private wasPlaying = false;
    private audioLoader: AudioLoader | null = null;
    private shouldPlay: boolean = false;
    // set this from audio context time, used to set clip offset when setting "time" property
    // there is maybe a better way to set a audio clip current time?!
    private _lastClipStartedLoading: string | MediaStream | null = null;
    private _loadedClip: string | MediaStream | null = null;
    private _audioElement: HTMLAudioElement | null = null;

    // Spatial blend dual-path audio nodes
    private _entryNode: GainNode | null = null;
    private _spatialGain: GainNode | null = null;
    private _bypassGain: GainNode | null = null;

    /**
     * Returns the underlying {@link PositionalAudio} object, creating it if necessary.
     * The audio source needs a user interaction to be initialized due to browser autoplay policies.
     *
     * @returns The three.js PositionalAudio object or null if unavailable
     */
    public get Sound(): PositionalAudio | null {
        if (!this.sound && AudioSource.userInteractionRegistered) {
            // Get or create an audiolistener in the scene
            let listener = this.gameObject.getComponent(AudioListener) // AudioListener on AudioSource?
                ?? this.context.mainCamera.getComponent(AudioListener) // AudioListener on current main camera?
                ?? findObjectOfType(AudioListener, this.context, false); // Active AudioListener in scene?

            if (!listener && this.context.mainCamera) listener = this.context.mainCamera.addComponent(AudioListener);

            if (listener?.listener) {
                this.sound = new PositionalAudio(listener.listener);
                this.gameObject?.add(this.sound);
            }
            else if (debug) console.warn("No audio listener found in scene - can not play audio");
        }
        return this.sound;
    }

    /**
     * Indicates whether the audio source is queued to play when possible.
     * This may be true before user interaction has been registered.
     * 
     * @returns Whether the audio source intends to play
     */
    public get ShouldPlay(): boolean { return this.shouldPlay; }

    /**
     * Returns the Web Audio API context associated with this audio source.
     * 
     * @returns The {@link AudioContext} or null if not available
     */
    public get audioContext() {
        return this.sound?.context;
    }

    /** @internal */
    awake() {
        if (debug) console.log("[AudioSource]", this);

        this.audioLoader = new AudioLoader();
        if (this.playOnAwake) this.shouldPlay = true;

        if (this.preload) {
            if (typeof this.clip === "string" && this.clip.length > 0) {
                this.audioLoader.load(this.clip, this.createAudio, () => { }, e => {
                    console.error(`[AudioSource] Error preloading audio clip "${this.clip}":`, e);
                });
            }
        }
    }

    /** @internal */
    onEnable(): void {
        if (this.sound)
            this.gameObject.add(this.sound);

        if (!AudioSource.userInteractionRegistered) {
            AudioSource.registerWaitForAllowAudio(() => {
                if (this.enabled && !this.destroyed && this.shouldPlay)
                    this.onNewClip(this.clip);
            });
        }
        else if (this.playOnAwake && this.context.application.isVisible) {
            this.play();
        }
        globalThis.addEventListener('visibilitychange', this.onVisibilityChanged);
        this.context.application.addEventListener(ApplicationEvents.MuteChanged, this.onApplicationMuteChanged);
    }

    /** @internal */
    onDisable() {
        globalThis.removeEventListener('visibilitychange', this.onVisibilityChanged);
        this.context.application.removeEventListener(ApplicationEvents.MuteChanged, this.onApplicationMuteChanged);
        this.pause();
    }

    private onVisibilityChanged = () => {
        switch (document.visibilityState) {
            case "hidden":
                if (this.playInBackground === false || DeviceUtilities.isMobileDevice()) {
                    this.wasPlaying = this.isPlaying;
                    if (this.isPlaying) {
                        this.pause();
                    }
                }
                break;
            case "visible":
                if (debug) console.log("visible", this.enabled, this.playOnAwake, !this.isPlaying, AudioSource.userInteractionRegistered, this.wasPlaying);
                if (this.enabled && this.playOnAwake && !this.isPlaying && AudioSource.userInteractionRegistered && this.wasPlaying) {
                    this.play();
                }
                break;
        }
    }

    private onApplicationMuteChanged = () => {
        if (this.context.application.muted)
            this.sound?.setVolume(0);
        else
            this.sound?.setVolume(this.volume);
    }

    private createAudio = (buffer?: AudioBuffer) => {
        if (this.destroyed) {
            if (debug) console.warn("AudioSource destroyed, not creating audio", this.name);
            return;
        }

        if (debug) console.log("AudioBuffer finished loading", buffer);

        const sound = this.Sound;
        if (!sound) {
            if (debug) console.warn("Failed getting sound?", this.name);
            return;
        }

        if (sound.isPlaying)
            sound.stop();

        if (buffer) sound.setBuffer(buffer);
        this._loadedClip = this.clip;
        sound.loop = this._loop;
        if (this.context.application.muted) sound.setVolume(0);
        else sound.setVolume(this.volume);
        sound.autoplay = this.shouldPlay && AudioSource.userInteractionRegistered;
        this.setupSpatialBlendNodes();
        this.applySpatialDistanceSettings();

        if (sound.isPlaying)
            sound.stop();

        // make sure we only play the sound if the user has interacted with the page
        AudioSource.registerWaitForAllowAudio(this.__onAllowAudioCallback);
    }
    private __onAllowAudioCallback = () => {
        if (this.shouldPlay)
            this.play();
    }

    /**
     * Sets up the dual-path audio graph for spatial blend crossfading.
     * Creates two parallel signal paths from source to output:
     * - 3D path: entry → panner → spatialGain → gain (spatialized)
     * - 2D path: entry → bypassGain → gain (non-spatialized)
     */
    private setupSpatialBlendNodes() {
        const sound = this.sound;
        if (!sound || this._entryNode) return;

        const ctx = sound.context;
        this._entryNode = ctx.createGain();
        this._spatialGain = ctx.createGain();
        this._bypassGain = ctx.createGain();

        // Disconnect the default panner → gain connection set up by three.js
        try { sound.panner.disconnect(sound.gain); } catch { /* not connected */ }

        // 3D path: entry → panner → spatialGain → gain
        this._entryNode.connect(sound.panner);
        sound.panner.connect(this._spatialGain);
        this._spatialGain.connect(sound.gain);

        // 2D path: entry → bypassGain → gain
        this._entryNode.connect(this._bypassGain);
        this._bypassGain.connect(sound.gain);

        // Override getOutput so three.js connects source → entryNode instead of panner
        sound.getOutput = () => this._entryNode! as unknown as PannerNode;

        // Wrap connect() to undo PositionalAudio's redundant panner → gain reconnection
        const origConnect = sound.connect;
        sound.connect = function () {
            origConnect.call(sound);
            try { sound.panner.disconnect(sound.gain); } catch { /* expected */ }
            return sound;
        };

        // Wrap disconnect() to prevent error when panner → gain doesn't exist
        const origDisconnect = sound.disconnect;
        sound.disconnect = function () {
            try { origDisconnect.call(sound); } catch { /* panner → gain already removed */ }
            return sound;
        };

        this.updateSpatialBlendGains();
    }

    /** Updates the spatial/bypass gain values based on the current spatialBlend. */
    private updateSpatialBlendGains() {
        if (!this._spatialGain || !this._bypassGain || !this.sound) return;
        const t = this.sound.context.currentTime;
        // Use setTargetAtTime for smooth transitions (avoids clicks/pops)
        this._spatialGain.gain.setTargetAtTime(this._spatialBlend, t, 0.01);
        this._bypassGain.gain.setTargetAtTime(1 - this._spatialBlend, t, 0.01);
    }

    private applySpatialDistanceSettings() {
        const sound = this.sound;
        if (!sound) return;
        this._needUpdateSpatialDistanceSettings = false;

        const ref = Math.max(0.0001, this._minDistance);
        const max = Math.max(0.01, this._maxDistance);
        if (debug) console.log(this.name, ref, max, this.spatialBlend);
        sound.setRefDistance(ref);
        sound.setMaxDistance(max);

        // https://developer.mozilla.org/en-US/docs/Web/API/PannerNode/distanceModel
        switch (this.rollOffMode) {
            case AudioRolloffMode.Logarithmic:
                // Unity's logarithmic rolloff matches Web Audio's 'inverse' model:
                // gain = refDistance / (refDistance + rolloffFactor * (distance - refDistance))
                // We compute rolloffFactor so gain reaches ~0.01 (-40dB) at maxDistance,
                // matching Unity's behavior of effective silence at maxDistance.
                sound.setDistanceModel('inverse');
                sound.setRolloffFactor(99 * ref / Math.max(0.001, max - ref));
                break;
            case AudioRolloffMode.Linear:
                // gain = 1 - rolloffFactor * (distance - refDistance) / (maxDistance - refDistance)
                // With rolloffFactor=1 this reaches exactly 0 at maxDistance.
                sound.setDistanceModel('linear');
                sound.setRolloffFactor(1);
                break;
            case AudioRolloffMode.Custom:
                console.warn("Custom rolloff for AudioSource is not supported: " + this.name);
                break;
        }

        this.updateSpatialBlendGains();

        if (this.spatialBlend > 0) {
            if (debug && !this.helper) {
                this.helper = new PositionalAudioHelper(sound, sound.getRefDistance());
                sound.add(this.helper);
            }
        }
        else if (this.helper && this.helper.parent) {
            this.helper.removeFromParent();
        }
    }

    private async onNewClip(clip?: string | MediaStream) {
        if (clip) this.clip = clip;
        if (typeof clip === "string") {
            if (debug)
                console.log(clip);
            if (clip.endsWith(".mp3") || clip.endsWith(".wav") || clip.endsWith(".ogg") || clip.endsWith(".flac") || clip.endsWith(".aac") || clip.endsWith(".webm")) {
                if (!this.audioLoader)
                    this.audioLoader = new AudioLoader();
                this.shouldPlay = true;
                if (this._lastClipStartedLoading === clip) {
                    if (debug) console.log("Is currently loading:", this._lastClipStartedLoading, this)
                    return;
                }
                this._lastClipStartedLoading = clip;
                if (debug)
                    console.log("load audio", clip);
                const buffer = await this.audioLoader.loadAsync(clip).catch(e => {
                    console.error(`[AudioSource] Error loading audio clip "${clip}":`, e);
                    return null;
                });
                if (this.destroyed) return;
                if (this._lastClipStartedLoading === clip) this._lastClipStartedLoading = null;
                if (buffer) this.createAudio(buffer);
            }
            else console.warn("Unsupported audio clip type", clip)
        }
        else {
            this.shouldPlay = true;
            this.createAudio();
        }
    }

    /**
     * Plays the audio clip or media stream.
     * If no argument is provided, plays the currently assigned clip.
     * 
     * @param clip - Optional audio clip or {@link MediaStream} to play
     * @returns A promise that resolves when playback starts successfully, or rejects on error
     */
    async play(clip: string | MediaStream | undefined = undefined): Promise<boolean> {
        // use audio source's clip when no clip is passed in
        if (!clip && this.clip)
            clip = this.clip;

        // We only support strings and media stream
        // TODO: maybe we should return here if an invalid value is passed in
        if (clip !== undefined && typeof clip !== "string" && !(clip instanceof MediaStream)) {
            // e.g. when a AudioSource.Play is called from SpatialTrigger onEnter this event is called with the TriggerReceiver... to still make this work we *re-use* our already assigned clip. Because otherwise calling `play` would not play the clip...
            clip = this.clip;
            if (isDevEnvironment()) console.warn("[AudioSource] Called play on AudioSource with unknown argument type:", clip + "\nUsing the assigned clip instead:", this.clip)
        }

        // Load if the sound hasn't been created yet or if the clip changed since last load
        let needsLoading = !this.sound || (clip && clip !== this._loadedClip);
        if (typeof clip === "string" && !this.audioLoader) needsLoading = true;
        if (clip instanceof MediaStream || typeof clip === "string")
            this.clip = clip;
        if (needsLoading) {
            this.shouldPlay = true;
            return this.onNewClip(clip).then(() => true).catch(() => false);
        }

        this.shouldPlay = true;
        this._hasEnded = false;

        if (debug) console.log("[AudioSource] play", this.sound?.getVolume(), this.sound);
        // If a different clip was passed in, needsLoading above would be true and onNewClip handles it.
        // This guard prevents double-playing the same already-playing sound (which would throw in three.js).
        if (this.sound && !this.sound.isPlaying) {
            const muted = this.context.application.muted;
            if (muted) this.sound.setVolume(0);
            this.gameObject?.add(this.sound);

            if (this.clip instanceof MediaStream) {

                // We have to set the audio element source to the mediastream as well
                // otherwise it will not play for some reason...
                this.sound.setMediaStreamSource(this.clip);

                if (!this._audioElement) {
                    this._audioElement = document.createElement('audio');
                    this._audioElement.style.display = "none";
                }
                if (!this._audioElement.parentNode)
                    this.context.domElement.shadowRoot?.append(this._audioElement);
                this._audioElement.srcObject = this.clip;
                this._audioElement.autoplay = false;
                return true;

            }
            else {
                if (this._audioElement) this._audioElement.remove();
                this.sound.play(muted ? .1 : 0);
                return true;
            }
        }

        return false;
    }

    /**
     * Pauses audio playback while maintaining the current position.
     * Use play() to resume from the paused position.
     */
    pause() {
        if (debug) console.log("[AudioSource] pause", this);
        this._hasEnded = true;
        this.shouldPlay = false;
        if (this.sound && this.sound.isPlaying && this.sound.source) {
            this._lastContextTime = this.sound?.context.currentTime;
            this.sound.pause();
        }
        this._audioElement?.remove();
    }

    /**
     * Stops audio playback completely and resets the playback position to the beginning.
     * Unlike pause(), calling play() after stop() will start from the beginning.
     */
    stop() {
        if (debug) console.log("[AudioSource] stop", this);
        this._hasEnded = true;
        this.shouldPlay = false;
        if (this.sound && this.sound.source) {
            this._lastContextTime = this.sound?.context.currentTime;
            if (debug)
                console.log("[AudioSource] lastContextTime", this._lastContextTime);
            this.sound.stop();
        }
        this._audioElement?.remove();
    }

    private _lastContextTime: number = 0;

    private _hasEnded: boolean = true;
    private _needUpdateSpatialDistanceSettings: boolean = false;

    /** @internal */
    update() {
        if (this.helper) {
            if (this.isPlaying)
                this.helper.update();
            this.helper.visible = this.isPlaying;
        }

        if (this._needUpdateSpatialDistanceSettings) {
            this.applySpatialDistanceSettings();
        }

        if (this.sound && !this.sound.isPlaying && this.shouldPlay && !this._hasEnded) {
            this._hasEnded = true;
            if (debug) console.log("[AudioSource] Audio clip ended", this.clip);
            this.dispatchEvent(new CustomEvent("ended", { detail: this }));
        }

        // this.gameObject.position.x = Math.sin(time.time) * 2;
        // this.gameObject.position.z = Math.cos(time.time * .5) * 2;
    }
}