@needle-tools/engine
Version:
Needle Engine is a web-based runtime for 3D apps. It runs on your machine for development with great integrations into editors like Unity or Blender - and can be deployed onto any device! It is flexible, extensible and networking and XR are built-in
538 lines • 21.8 kB
JavaScript
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
import { AudioLoader, PositionalAudio } from "three";
import { PositionalAudioHelper } from 'three/examples/jsm/helpers/PositionalAudioHelper.js';
import { isDevEnvironment } from "../engine/debug/index.js";
import { Application, ApplicationEvents } from "../engine/engine_application.js";
import { Mathf } from "../engine/engine_math.js";
import { serializable } from "../engine/engine_serialization_decorator.js";
import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
import { AudioListener } from "./AudioListener.js";
import { Behaviour, GameObject } from "./Component.js";
const debug = getParam("debugaudio");
/**
* The AudioRolloffMode enum describes different ways that audio can attenuate with distance.
*/
export var AudioRolloffMode;
(function (AudioRolloffMode) {
/// <summary>
/// <para>Use this mode when you want a real-world rolloff.</para>
/// </summary>
AudioRolloffMode[AudioRolloffMode["Logarithmic"] = 0] = "Logarithmic";
/// <summary>
/// <para>Use this mode when you want to lower the volume of your sound over the distance.</para>
/// </summary>
AudioRolloffMode[AudioRolloffMode["Linear"] = 1] = "Linear";
/// <summary>
/// <para>Use this when you want to use a custom rolloff.</para>
/// </summary>
AudioRolloffMode[AudioRolloffMode["Custom"] = 2] = "Custom";
})(AudioRolloffMode || (AudioRolloffMode = {}));
/** The AudioSource can be used to play audio in the scene.
* Use `clip` to set the audio file to play.
* @category Multimedia
* @group Components
*/
export class AudioSource extends Behaviour {
/** Check if the user has interacted with the page to allow audio playback.
* Internally calling {@link Application.userInteractionRegistered}
*/
static get userInteractionRegistered() {
return Application.userInteractionRegistered;
}
/** Register a callback that is called when the user has interacted with the page to allow audio playback.
* Internally calling {@link Application.registerWaitForInteraction}
*/
static registerWaitForAllowAudio(cb) {
Application.registerWaitForInteraction(cb);
}
/**
* The audio clip to play. Can be a string (URL) or a MediaStream.
*/
clip = "";
/**
* If true, the audio source will start playing as soon as the scene starts.
* If false, you can call play() to start the audio.
* @default false
*/
playOnAwake = false;
/**
* If true, the audio source will start loading the audio clip as soon as the scene starts.
* If false, the audio clip will be loaded when play() is called.
* @default false
*/
preload = false;
/**
* When true, the audio will play in the background. This means it will continue playing if the browser tab is not focused/active or minimized
* @default true
*/
playInBackground = true;
/**
* If true, the audio is currently playing.
*/
get isPlaying() { return this.sound?.isPlaying ?? false; }
/** The duration of the audio clip in seconds. */
get duration() {
return this.sound?.buffer?.duration;
}
/** The current time of the audio clip in 0-1 range. */
get time01() {
const duration = this.duration;
if (duration && this.sound) {
return this.sound?.context.currentTime / duration;
}
return 0;
}
set time01(val) {
const duration = this.duration;
if (duration && this.sound) {
this.time = val * duration;
}
}
/**
* The current time of the audio clip in seconds.
*/
get time() { return this.sound?.source ? (this.sound.source?.context.currentTime - this._lastContextTime + this.sound.offset) : 0; }
set time(val) {
if (this.sound) {
if (val === this.sound.offset)
return;
const wasPlaying = this.isPlaying;
this.stop();
this.sound.offset = val;
if (wasPlaying)
this.play();
}
}
/**
* If true, the audio source will loop the audio clip.
* If false, the audio clip will play once.
* @default false
*/
get loop() {
if (this.sound)
this._loop = this.sound.getLoop();
return this._loop;
}
set loop(val) {
this._loop = val;
if (this.sound)
this.sound.setLoop(val);
}
/** Can be used to play the audio clip in 2D or 3D space.
* 2D Playback is currently not fully supported.
* 0 = 2D, 1 = 3D
* */
get spatialBlend() {
return this._spatialBlend;
}
set spatialBlend(val) {
if (val === this._spatialBlend)
return;
this._spatialBlend = val;
this._needUpdateSpatialDistanceSettings = true;
}
get minDistance() {
return this._minDistance;
}
set minDistance(val) {
if (this._minDistance === val)
return;
this._minDistance = val;
this._needUpdateSpatialDistanceSettings = true;
}
get maxDistance() {
return this._maxDistance;
}
set maxDistance(val) {
if (this._maxDistance === val)
return;
this._maxDistance = val;
this._needUpdateSpatialDistanceSettings = true;
}
_spatialBlend = 0;
_minDistance = 1;
_maxDistance = 100;
get volume() { return this._volume; }
set volume(val) {
this._volume = val;
if (this.sound && !this.context.application.muted) {
if (debug)
console.log(this.name, "audio set volume", val);
this.sound.setVolume(val);
}
}
_volume = 1;
set pitch(val) {
if (this.sound)
this.sound.setPlaybackRate(val);
}
get pitch() {
return this.sound ? this.sound.getPlaybackRate() : 1;
}
rollOffMode = 0;
_loop = false;
sound = null;
helper = null;
wasPlaying = false;
audioLoader = null;
shouldPlay = false;
// set this from audio context time, used to set clip offset when setting "time" property
// there is maybe a better way to set a audio clip current time?!
_lastClipStartedLoading = null;
_audioElement = null;
get Sound() {
if (!this.sound && AudioSource.userInteractionRegistered) {
let listener = GameObject.getComponent(this.context.mainCamera, AudioListener) ?? GameObject.findObjectOfType(AudioListener, this.context);
if (!listener && this.context.mainCamera)
listener = GameObject.addComponent(this.context.mainCamera, AudioListener);
if (listener?.listener) {
this.sound = new PositionalAudio(listener.listener);
this.gameObject?.add(this.sound);
// this._listener = listener;
// this._originalSoundMatrixWorldFunction = this.sound.updateMatrixWorld;
// this.sound.updateMatrixWorld = this._onSoundMatrixWorld;
}
else if (debug)
console.warn("No audio listener found in scene - can not play audio");
}
return this.sound;
}
// This is a hacky workaround to get the PositionalAudio behave like a 2D audio source
// private _listener: AudioListener | null = null;
// private _originalSoundMatrixWorldFunction: Function | null = null;
// private _onSoundMatrixWorld = (force: boolean) => {
// if (this._spatialBlend > .05) {
// if (this._originalSoundMatrixWorldFunction) {
// this._originalSoundMatrixWorldFunction.call(this.sound, force);
// }
// }
// else {
// // we use another object's matrix world function (but bound to the positional audio)
// // this is just a little trick to prevent calling the PositionalAudio's updateMatrixWorld function
// this.gameObject.updateMatrixWorld?.call(this.sound, force);
// if (this.sound && this._listener) {
// this.sound.gain.connect(this._listener.listener.getInput());
// // const pos = getTempVector().setFromMatrixPosition(this._listener.gameObject.matrixWorld);
// // const ctx = this.sound.context;
// // const delay = this._listener.listener.timeDelta;
// // const time = ctx.currentTime ;
// // this.sound.panner.positionX.setValueAtTime(pos.x, time);
// // this.sound.panner.positionY.setValueAtTime(pos.y, time);
// // this.sound.panner.positionZ.setValueAtTime(pos.z, time);
// // this.sound.panner.orientationX.setValueAtTime(0, time);
// // this.sound.panner.orientationY.setValueAtTime(0, time);
// // this.sound.panner.orientationZ.setValueAtTime(-1, time);
// }
// }
// }
get ShouldPlay() { return this.shouldPlay; }
/** Get the audio context from the Sound */
get audioContext() {
return this.sound?.context;
}
/** @internal */
awake() {
if (debug)
console.log(this);
this.audioLoader = new AudioLoader();
if (this.playOnAwake)
this.shouldPlay = true;
if (this.preload) {
if (typeof this.clip === "string") {
this.audioLoader.load(this.clip, this.createAudio, () => { }, console.error);
}
}
}
/** @internal */
onEnable() {
if (this.sound)
this.gameObject.add(this.sound);
if (!AudioSource.userInteractionRegistered) {
AudioSource.registerWaitForAllowAudio(() => {
if (this.enabled && !this.destroyed && this.shouldPlay)
this.onNewClip(this.clip);
});
}
else if (this.playOnAwake && this.context.application.isVisible) {
this.play();
}
globalThis.addEventListener('visibilitychange', this.onVisibilityChanged);
this.context.application.addEventListener(ApplicationEvents.MuteChanged, this.onApplicationMuteChanged);
}
/** @internal */
onDisable() {
globalThis.removeEventListener('visibilitychange', this.onVisibilityChanged);
this.context.application.removeEventListener(ApplicationEvents.MuteChanged, this.onApplicationMuteChanged);
this.pause();
}
onVisibilityChanged = () => {
switch (document.visibilityState) {
case "hidden":
if (this.playInBackground === false || DeviceUtilities.isMobileDevice()) {
this.wasPlaying = this.isPlaying;
if (this.isPlaying) {
this.pause();
}
}
break;
case "visible":
if (debug)
console.log("visible", this.enabled, this.playOnAwake, !this.isPlaying, AudioSource.userInteractionRegistered, this.wasPlaying);
if (this.enabled && this.playOnAwake && !this.isPlaying && AudioSource.userInteractionRegistered && this.wasPlaying) {
this.play();
}
break;
}
};
onApplicationMuteChanged = () => {
if (this.context.application.muted)
this.sound?.setVolume(0);
else
this.sound?.setVolume(this.volume);
};
createAudio = (buffer) => {
if (debug)
console.log("AudioBuffer finished loading", buffer);
const sound = this.Sound;
if (!sound) {
if (debug)
console.warn("Failed getting sound?", this.name);
return;
}
if (sound.isPlaying)
sound.stop();
if (buffer)
sound.setBuffer(buffer);
sound.loop = this._loop;
if (this.context.application.muted)
sound.setVolume(0);
else
sound.setVolume(this.volume);
sound.autoplay = this.shouldPlay && AudioSource.userInteractionRegistered;
this.applySpatialDistanceSettings();
if (sound.isPlaying)
sound.stop();
// const src = sound.context.createBufferSource();
// src.buffer = sound.buffer;
// src.connect(sound.panner);
// src.start(this.audioContext?.currentTime);
// const gain = sound.context.createGain();
// gain.gain.value = 1 - this.spatialBlend;
// src.connect(gain);
// make sure we only play the sound if the user has interacted with the page
AudioSource.registerWaitForAllowAudio(this.__onAllowAudioCallback);
};
__onAllowAudioCallback = () => {
if (this.shouldPlay)
this.play();
};
applySpatialDistanceSettings() {
const sound = this.sound;
if (!sound)
return;
this._needUpdateSpatialDistanceSettings = false;
const dist = Mathf.lerp(10 * this._maxDistance / Math.max(0.0001, this.spatialBlend), this._minDistance, this.spatialBlend);
if (debug)
console.log(this.name, this._minDistance, this._maxDistance, this.spatialBlend, "Ref distance=" + dist);
sound.setRefDistance(dist);
sound.setMaxDistance(Math.max(0.01, this._maxDistance));
// sound.setRolloffFactor(this.spatialBlend);
// sound.panner.positionZ.automationRate
// https://developer.mozilla.org/en-US/docs/Web/API/PannerNode/distanceModel
switch (this.rollOffMode) {
case AudioRolloffMode.Logarithmic:
sound.setDistanceModel('exponential');
break;
case AudioRolloffMode.Linear:
sound.setDistanceModel('linear');
break;
case AudioRolloffMode.Custom:
console.warn("Custom rolloff for AudioSource is not supported: " + this.name);
break;
}
if (this.spatialBlend > 0) {
if (debug && !this.helper) {
this.helper = new PositionalAudioHelper(sound, sound.getRefDistance());
sound.add(this.helper);
}
}
else if (this.helper && this.helper.parent) {
this.helper.removeFromParent();
}
}
async onNewClip(clip) {
if (clip)
this.clip = clip;
if (typeof clip === "string") {
if (debug)
console.log(clip);
if (clip.endsWith(".mp3") || clip.endsWith(".wav")) {
if (!this.audioLoader)
this.audioLoader = new AudioLoader();
this.shouldPlay = true;
if (this._lastClipStartedLoading === clip) {
if (debug)
console.log("Is currently loading:", this._lastClipStartedLoading, this);
return;
}
this._lastClipStartedLoading = clip;
if (debug)
console.log("load audio", clip);
const buffer = await this.audioLoader.loadAsync(clip).catch(console.error);
this._lastClipStartedLoading = null;
if (buffer)
this.createAudio(buffer);
}
else
console.warn("Unsupported audio clip type", clip);
}
else {
this.shouldPlay = true;
this.createAudio();
}
}
/** Play a audioclip or mediastream */
play(clip = undefined) {
// use audio source's clip when no clip is passed in
if (!clip && this.clip)
clip = this.clip;
// We only support strings and media stream
// TODO: maybe we should return here if an invalid value is passed in
if (clip !== undefined && typeof clip !== "string" && !(clip instanceof MediaStream)) {
if (isDevEnvironment())
console.warn("Called play on AudioSource with unknown argument type:", clip + "\nUsing the assigned clip instead:", this.clip);
// e.g. when a AudioSource.Play is called from SpatialTrigger onEnter this event is called with the TriggerReceiver... to still make this work we *re-use* our already assigned clip. Because otherwise calling `play` would not play the clip...
clip = this.clip;
}
// Check if we need to call load first
let needsLoading = !this.sound || (clip && clip !== this.clip);
if (typeof clip === "string" && !this.audioLoader)
needsLoading = true;
if (clip instanceof MediaStream || typeof clip === "string")
this.clip = clip;
if (needsLoading) {
this.shouldPlay = true;
this.onNewClip(clip);
return;
}
this.shouldPlay = true;
this._hasEnded = false;
if (debug)
console.log("play", this.sound?.getVolume(), this.sound);
if (this.sound && !this.sound.isPlaying) {
const muted = this.context.application.muted;
if (muted)
this.sound.setVolume(0);
this.gameObject?.add(this.sound);
if (this.clip instanceof MediaStream) {
// We have to set the audio element source to the mediastream as well
// otherwise it will not play for some reason...
this.sound.setMediaStreamSource(this.clip);
if (!this._audioElement) {
this._audioElement = document.createElement('audio');
this._audioElement.style.display = "none";
}
if (!this._audioElement.parentNode)
this.context.domElement.shadowRoot?.append(this._audioElement);
this._audioElement.srcObject = this.clip;
this._audioElement.autoplay = false;
}
else {
if (this._audioElement)
this._audioElement.remove();
this.sound.play(muted ? .1 : 0);
}
}
}
/**
* Pause the audio
*/
pause() {
if (debug)
console.log("Pause", this);
this._hasEnded = true;
this.shouldPlay = false;
if (this.sound && this.sound.isPlaying && this.sound.source) {
this._lastContextTime = this.sound?.context.currentTime;
this.sound.pause();
}
this._audioElement?.remove();
}
/**
* Stop the audio and reset the time to 0
*/
stop() {
if (debug)
console.log("Pause", this);
this._hasEnded = true;
this.shouldPlay = false;
if (this.sound && this.sound.source) {
this._lastContextTime = this.sound?.context.currentTime;
if (debug)
console.log(this._lastContextTime);
this.sound.stop();
}
this._audioElement?.remove();
}
_lastContextTime = 0;
_hasEnded = true;
_needUpdateSpatialDistanceSettings = false;
/** @internal */
update() {
if (this.helper) {
if (this.isPlaying)
this.helper.update();
this.helper.visible = this.isPlaying;
}
if (this._needUpdateSpatialDistanceSettings) {
this.applySpatialDistanceSettings();
}
if (this.sound && !this.sound.isPlaying && this.shouldPlay && !this._hasEnded) {
this._hasEnded = true;
if (debug)
console.log("Audio clip ended", this.clip);
this.dispatchEvent(new CustomEvent("ended", { detail: this }));
}
// this.gameObject.position.x = Math.sin(time.time) * 2;
// this.gameObject.position.z = Math.cos(time.time * .5) * 2;
}
}
__decorate([
serializable(URL)
], AudioSource.prototype, "clip", void 0);
__decorate([
serializable()
], AudioSource.prototype, "playOnAwake", void 0);
__decorate([
serializable()
], AudioSource.prototype, "preload", void 0);
__decorate([
serializable()
], AudioSource.prototype, "playInBackground", void 0);
__decorate([
serializable()
], AudioSource.prototype, "loop", null);
__decorate([
serializable()
], AudioSource.prototype, "spatialBlend", null);
__decorate([
serializable()
], AudioSource.prototype, "minDistance", null);
__decorate([
serializable()
], AudioSource.prototype, "maxDistance", null);
__decorate([
serializable()
], AudioSource.prototype, "volume", null);
__decorate([
serializable()
], AudioSource.prototype, "pitch", null);
__decorate([
serializable()
], AudioSource.prototype, "rollOffMode", void 0);
//# sourceMappingURL=AudioSource.js.map