babylon-mmd
Version:
babylon.js mmd loader and runtime
748 lines (747 loc) • 28.8 kB
JavaScript
import { Logger } from "@babylonjs/core/Misc/logger";
import { Observable } from "@babylonjs/core/Misc/observable";
import { MmdMesh } from "./mmdMesh";
import { MmdModel } from "./mmdModel";
import { MmdRuntimeShared } from "./mmdRuntimeShared";
/**
* MMD runtime orchestrates several MMD components (models, camera, audio)
*
* MMD runtime handles updates and synchronization of MMD components
*
* It can also create and remove runtime components
*/
export class MmdRuntime {
_physics;
_models;
_animatables;
_audioPlayer;
/**
* Whether to automatically initialize rigid bodies transform and velocity (default: true)
*
* auto physics initialization is triggered when
* - animation seek is far from current frame time (more than 2 seconds)
* - browser tab is stop rendering and resumed
* - animation is played from the frame 0
*/
autoPhysicsInitialization;
_loggingEnabled;
/** @internal */
log;
/** @internal */
warn;
/** @internal */
error;
_isRegistered;
/**
* This observable is notified when animation duration is changed
*/
onAnimationDurationChangedObservable;
/**
* This observable is notified when animation is played
*/
onPlayAnimationObservable;
/**
* This observable is notified when animation is paused
*/
onPauseAnimationObservable;
/**
* This observable is notified when animation is seeked
*/
onSeekAnimationObservable;
/**
* This observable is notified when animation is evaluated (usually every frame)
*/
onAnimationTickObservable;
_currentFrameTime;
_animationTimeScale;
_animationPaused;
_animationFrameTimeDuration;
_useManualAnimationDuration;
_needToInitializePhysicsModels;
_beforePhysicsBinded;
_afterPhysicsBinded;
_bindedDispose;
_disposeObservableObject;
/**
* Creates a new MMD runtime
* @param scene Objects that limit the lifetime of this instance
* @param physics Physics builder
*/
constructor(scene = null, physics = null) {
this._physics = physics;
this._models = [];
this._animatables = [];
this._audioPlayer = null;
this.autoPhysicsInitialization = true;
this._loggingEnabled = false;
this.log = this._logDisabled;
this.warn = this._warnDisabled;
this.error = this._errorDisabled;
this._isRegistered = false;
this.onAnimationDurationChangedObservable = new Observable();
this.onPlayAnimationObservable = new Observable();
this.onPauseAnimationObservable = new Observable();
this.onSeekAnimationObservable = new Observable();
this.onAnimationTickObservable = new Observable();
this._currentFrameTime = 0;
this._animationTimeScale = 1;
this._animationPaused = true;
this._animationFrameTimeDuration = 0;
this._useManualAnimationDuration = false;
this._needToInitializePhysicsModels = new Set();
this._beforePhysicsBinded = null;
this._afterPhysicsBinded = this.afterPhysics.bind(this);
if (scene !== null) {
this._bindedDispose = () => this.dispose(scene);
this._disposeObservableObject = scene;
if (this._disposeObservableObject !== null) {
this._disposeObservableObject.onDisposeObservable.add(this._bindedDispose);
}
}
else {
this._bindedDispose = null;
this._disposeObservableObject = null;
}
}
/**
* Dispose MMD runtime
*
* Destroy all MMD models and unregister this runtime from scene
* @param scene Scene
*/
dispose(scene) {
for (let i = 0; i < this._models.length; ++i)
this._models[i]._dispose();
this._models.length = 0;
this._animatables.length = 0;
this.setAudioPlayer(null);
this.onAnimationDurationChangedObservable.clear();
this.onPlayAnimationObservable.clear();
this.onPauseAnimationObservable.clear();
this.onSeekAnimationObservable.clear();
this.onAnimationTickObservable.clear();
this._needToInitializePhysicsModels.clear();
this.unregister(scene);
if (this._disposeObservableObject !== null && this._bindedDispose !== null) {
this._disposeObservableObject.onDisposeObservable.removeCallback(this._bindedDispose);
}
}
/**
* Create MMD model from mesh that has MMD metadata
*
* The skeletons in the mesh where the MmdModel was created no longer follow the usual matrix update policy
* @param mmdSkinnedMesh MmdSkinnedMesh
* @param options Creation options
* @returns MMD model
* @throws {Error} if mesh is not `MmdSkinnedMesh`
*/
createMmdModel(mmdSkinnedMesh, options = {}) {
if (!MmdMesh.isMmdSkinnedMesh(mmdSkinnedMesh))
throw new Error("Mesh validation failed.");
return this.createMmdModelFromSkeleton(mmdSkinnedMesh, mmdSkinnedMesh.metadata.skeleton, options);
}
/**
* Create MMD model from humanoid mesh and virtual skeleton
*
* this method is useful for supporting humanoid models, usually used by `HumanoidMmd`
* @param mmdSkinnedMesh MmdSkinnedMesh
* @param skeleton Skeleton or Virtualized skeleton
* @param options Creation options
*/
createMmdModelFromSkeleton(mmdSkinnedMesh, skeleton, options = {}) {
if (options.materialProxyConstructor === undefined) {
options.materialProxyConstructor = MmdRuntimeShared.MaterialProxyConstructor;
if (options.materialProxyConstructor === null) {
this.log("Material proxy constructor is null. Material morphing features will be disabled for this model. Please provide a valid materialProxyConstructor in IMmdModelCreationOptions if you require material morphing support.");
}
}
if (options.buildPhysics === undefined) {
options.buildPhysics = true;
}
if (options.trimMetadata === undefined) {
options.trimMetadata = true;
}
const model = new MmdModel(mmdSkinnedMesh, skeleton, options.materialProxyConstructor, options.buildPhysics
? this._physics !== null
? {
physicsImpl: this._physics,
physicsOptions: typeof options.buildPhysics === "boolean"
? null
: options.buildPhysics
}
: null
: null, options.trimMetadata, this);
this._models.push(model);
this._needToInitializePhysicsModels.add(model);
model.onAnimationDurationChangedObservable.add(this._onAnimationDurationChanged);
return model;
}
/**
* Destroy MMD model
*
* Dispose all resources used at MMD runtime and restores the skeleton to the usual matrix update policy
*
* After calling the `destroyMmdModel` once, the mesh is no longer able to `createMmdModel` because the metadata is lost
* @param mmdModel MMD model to destroy
* @throws {Error} if model is not found
*/
destroyMmdModel(mmdModel) {
mmdModel._dispose();
const models = this._models;
const index = models.indexOf(mmdModel);
if (index < 0)
throw new Error("Model not found.");
models.splice(index, 1);
}
/**
* Queue MMD model to initialize physics
*
* Actual physics initialization is done by the before physics stage
* @param mmdModel MMD model
*/
initializeMmdModelPhysics(mmdModel) {
this._needToInitializePhysicsModels.add(mmdModel);
}
/**
* Queue all MMD models to initialize physics
*
* Actual physics initialization is done by the before physics stage
*
* If you set onlyAnimated true, it only initializes physics for animated models
*/
initializeAllMmdModelsPhysics(onlyAnimated) {
const models = this._models;
if (onlyAnimated) {
for (let i = 0; i < models.length; ++i) {
const model = models[i];
if (model.currentAnimation !== null) {
this._needToInitializePhysicsModels.add(model);
}
}
}
else {
for (let i = 0; i < models.length; ++i) {
this._needToInitializePhysicsModels.add(models[i]);
}
}
}
/**
* Add Animatable object to the runtime
*
* Usually this is MMD camera, but you can add any object that implements `IMmdRuntimeAnimatable`
* @param animatable Animatable object
* @return true if the animatable is added, false if the animatable is already added
*/
addAnimatable(animatable) {
if (this._animatables.includes(animatable)) {
return false;
}
animatable.onAnimationDurationChangedObservable?.add(this._onAnimationDurationChanged);
this._animatables.push(animatable);
this._onAnimationDurationChanged(animatable.animationFrameTimeDuration);
return true;
}
/**
* Remove Animatable object from the runtime
* @param animatable Animatable object
* @return true if the animatable is removed, false if the animatable is not found
*/
removeAnimatable(animatable) {
const index = this._animatables.indexOf(animatable);
if (index === -1) {
return false;
}
animatable.onAnimationDurationChangedObservable?.removeCallback(this._onAnimationDurationChanged);
this._animatables.splice(index, 1);
this._onAnimationDurationChanged(0);
return true;
}
_setAudioPlayerLastValue = null;
/**
* Set audio player to sync with animation
*
* If you set up audio Player while playing an animation, it try to play the audio from the current animation time
* And returns Promise because this operation is asynchronous. In most cases, you don't have to await this Promise
* @param audioPlayer Audio player
* @returns Promise
*/
// eslint-disable-next-line @typescript-eslint/naming-convention
async setAudioPlayer(audioPlayer) {
if (this._audioPlayer === audioPlayer)
return;
this._setAudioPlayerLastValue = audioPlayer;
if (this._audioPlayer !== null) {
this._audioPlayer.onDurationChangedObservable.removeCallback(this._onAudioDurationChanged);
this._audioPlayer.onPlaybackRateChangedObservable.removeCallback(this._onAudioPlaybackRateChanged);
this._audioPlayer.onPlayObservable.removeCallback(this._onAudioPlay);
this._audioPlayer.onPauseObservable.removeCallback(this._onAudioPause);
this._audioPlayer.onSeekObservable.removeCallback(this._onAudioSeek);
this._audioPlayer.pause();
}
this._audioPlayer = null;
if (audioPlayer === null) {
this._onAudioDurationChanged();
return;
}
// sync audio player time with current frame time
const audioFrameTimeDuration = audioPlayer.duration * 30;
if (this._currentFrameTime < audioFrameTimeDuration) {
audioPlayer.currentTime = this._currentFrameTime / 30;
}
// sync audio player pause state with animation pause state
if (this._animationPaused !== audioPlayer.paused) {
if (this._animationPaused) {
audioPlayer.pause();
}
else {
if (audioFrameTimeDuration <= this._currentFrameTime) {
await audioPlayer.play();
// for handle race condition
if (this._setAudioPlayerLastValue !== audioPlayer) {
audioPlayer.pause();
return;
}
}
}
}
this._audioPlayer = audioPlayer;
this._onAudioDurationChanged();
audioPlayer.onDurationChangedObservable.add(this._onAudioDurationChanged);
audioPlayer.onPlaybackRateChangedObservable.add(this._onAudioPlaybackRateChanged);
audioPlayer.onPlayObservable.add(this._onAudioPlay);
audioPlayer.onPauseObservable.add(this._onAudioPause);
audioPlayer.onSeekObservable.add(this._onAudioSeek);
audioPlayer._setPlaybackRateWithoutNotify(this._animationTimeScale);
}
/**
* Register MMD runtime to scene
*
* register `beforePhysics` and `afterPhysics` to scene Observables
*
* If you need a more complex update method you can call `beforePhysics` and `afterPhysics` manually
* @param scene Scene
*/
register(scene) {
if (this._isRegistered)
return;
this._isRegistered = true;
this._beforePhysicsBinded = () => this.beforePhysics(scene.getEngine().getDeltaTime());
scene.onBeforeAnimationsObservable.add(this._beforePhysicsBinded);
scene.onBeforeRenderObservable.add(this._afterPhysicsBinded);
}
/**
* Unregister MMD runtime from scene
* @param scene Scene
*/
unregister(scene) {
if (!this._isRegistered)
return;
this._isRegistered = false;
scene.onBeforeAnimationsObservable.removeCallback(this._beforePhysicsBinded);
scene.onBeforeRenderObservable.removeCallback(this._afterPhysicsBinded);
this._beforePhysicsBinded = null;
}
/**
* Before the physics stage, update animations and run MMD runtime solvers
*
* @param deltaTime Delta time in milliseconds
*/
beforePhysics(deltaTime) {
if (!this._animationPaused) {
if (this._audioPlayer !== null && !this._audioPlayer.paused) { // sync animation time with audio time
const audioPlayerCurrentTime = this._audioPlayer.currentTime;
const timeDiff = audioPlayerCurrentTime - this._currentFrameTime / 30;
const timeDiffAbs = Math.abs(timeDiff);
if (timeDiffAbs < 0.05) { // synced
this._currentFrameTime += deltaTime / 1000 * 30 * this._animationTimeScale;
}
else if (timeDiffAbs < 0.5) {
if (timeDiff < 0) { // animation is faster than audio
this._currentFrameTime += deltaTime / 1000 * 30 * this._animationTimeScale * 0.9;
}
else { // animation is slower than audio
this._currentFrameTime += deltaTime / 1000 * 30 * this._animationTimeScale * 1.1;
}
}
else {
if (this.autoPhysicsInitialization) {
if (2 * 30 < Math.abs(audioPlayerCurrentTime - this._currentFrameTime)) {
const needToInitializePhysicsModels = this._needToInitializePhysicsModels;
for (let i = 0; i < this._models.length; ++i) {
const model = this._models[i];
if (model.currentAnimation !== null) {
needToInitializePhysicsModels.add(model);
}
}
}
}
this._currentFrameTime = audioPlayerCurrentTime * 30;
}
}
else { // only use delta time to calculate animation time
this._currentFrameTime += deltaTime / 1000 * 30 * this._animationTimeScale;
}
const elapsedFrameTime = this._currentFrameTime;
if (this._animationFrameTimeDuration <= elapsedFrameTime) {
this._animationPaused = true;
this._currentFrameTime = this._animationFrameTimeDuration;
if (this._audioPlayer !== null && !this._audioPlayer.paused) {
this._audioPlayer.pause();
}
else {
this.onPauseAnimationObservable.notifyObservers();
}
}
const models = this._models;
for (let i = 0; i < models.length; ++i) {
models[i].beforePhysics(elapsedFrameTime);
}
const animatables = this._animatables;
for (let i = 0; i < animatables.length; ++i) {
animatables[i].animate(elapsedFrameTime);
}
this.onAnimationTickObservable.notifyObservers();
}
else {
const models = this._models;
for (let i = 0; i < models.length; ++i) {
models[i].beforePhysics(null);
}
}
const needToInitializePhysicsModels = this._needToInitializePhysicsModels;
for (const model of needToInitializePhysicsModels) {
model.initializePhysics();
}
needToInitializePhysicsModels.clear();
}
/**
* After the physics stage, update physics and run MMD runtime solvers
*/
afterPhysics() {
const models = this._models;
for (let i = 0; i < models.length; ++i) {
models[i].afterPhysics();
}
}
_onAnimationDurationChanged = (newAnimationFrameTimeDuration) => {
if (this._useManualAnimationDuration)
return;
if (this._animationFrameTimeDuration < newAnimationFrameTimeDuration) {
this._animationFrameTimeDuration = newAnimationFrameTimeDuration;
}
else if (newAnimationFrameTimeDuration < this._animationFrameTimeDuration) {
this._animationFrameTimeDuration = this._computeAnimationDuration();
}
this.onAnimationDurationChangedObservable.notifyObservers();
};
_computeAnimationDuration() {
let duration = 0;
const models = this._models;
for (let i = 0; i < models.length; ++i) {
const model = models[i];
if (model.currentAnimation !== null) {
duration = Math.max(duration, model.currentAnimation.animation.endFrame);
}
}
const animatables = this._animatables;
for (let i = 0; i < animatables.length; ++i) {
const animatable = animatables[i];
duration = Math.max(duration, animatable.animationFrameTimeDuration);
}
if (this._audioPlayer !== null) {
duration = Math.max(duration, this._audioPlayer.duration * 30);
}
return duration;
}
_onAudioDurationChanged = () => {
if (!this._animationPaused && this._audioPlayer !== null) {
const audioPlayer = this._audioPlayer;
const currentTime = this._currentFrameTime / 30;
if (currentTime < audioPlayer.duration) {
audioPlayer._setCurrentTimeWithoutNotify(currentTime);
audioPlayer.play().then(() => {
if (this._setAudioPlayerLastValue !== audioPlayer) {
audioPlayer.pause();
return;
}
});
}
}
if (this._useManualAnimationDuration)
return;
const audioFrameTimeDuration = this._audioPlayer !== null
? this._audioPlayer.duration * 30
: 0;
if (this._animationFrameTimeDuration < audioFrameTimeDuration) {
this._animationFrameTimeDuration = audioFrameTimeDuration;
}
else {
this._animationFrameTimeDuration = this._computeAnimationDuration();
}
this.onAnimationDurationChangedObservable.notifyObservers();
};
_onAudioPlaybackRateChanged = () => {
this._animationTimeScale = this._audioPlayer.playbackRate;
};
_onAudioPlay = () => {
this._playAnimationInternal();
};
_onAudioPause = () => {
if (this._audioPlayer.currentTime === this._audioPlayer.duration)
return;
this._animationPaused = true;
this.onPauseAnimationObservable.notifyObservers();
};
_onAudioSeek = () => {
this._seekAnimationInternal(this._audioPlayer.currentTime * 30, this._animationPaused);
};
_playAnimationInternal() {
if (!this._animationPaused)
return;
this._animationPaused = false;
if (this.autoPhysicsInitialization) {
if (this._currentFrameTime === 0) {
const models = this._models;
const needToInitializePhysicsModels = this._needToInitializePhysicsModels;
for (let i = 0; i < models.length; ++i) {
needToInitializePhysicsModels.add(models[i]);
}
}
}
this.onPlayAnimationObservable.notifyObservers();
}
/**
* Play animation from the current animation time
*
* If audio player is set, it try to play the audio from the current animation time
*
* It returns Promise because playing audio is asynchronous
* @returns Promise
*/
// eslint-disable-next-line @typescript-eslint/naming-convention
async playAnimation() {
if (this._audioPlayer !== null && this._currentFrameTime < this._audioPlayer.duration * 30) {
try {
const currentTime = this._currentFrameTime / 30;
if (0.05 < Math.abs(this._audioPlayer.currentTime - currentTime)) {
this._audioPlayer._setCurrentTimeWithoutNotify(currentTime);
}
await this._audioPlayer.play();
}
catch (e) {
if (e instanceof DOMException && e.name === "NotSupportedError") {
this.error("Failed to play audio.");
this._playAnimationInternal();
}
else {
throw e;
}
}
}
else {
this._playAnimationInternal();
}
}
/**
* Pause animation
*/
pauseAnimation() {
if (this._audioPlayer !== null && !this._audioPlayer.paused) {
this._audioPlayer.pause();
this._animationPaused = true;
}
else {
this._animationPaused = true;
this.onPauseAnimationObservable.notifyObservers();
}
}
_seekAnimationInternal(frameTime, forceEvaluate) {
if (this.autoPhysicsInitialization) {
if (2 * 30 < Math.abs(frameTime - this._currentFrameTime)) {
const needToInitializePhysicsModels = this._needToInitializePhysicsModels;
for (let i = 0; i < this._models.length; ++i) {
const model = this._models[i];
if (model.currentAnimation !== null) {
needToInitializePhysicsModels.add(model);
}
}
}
}
this._currentFrameTime = frameTime;
if (forceEvaluate) {
const models = this._models;
for (let i = 0; i < models.length; ++i) {
const model = models[i];
if (model.currentAnimation !== null) {
model.currentAnimation.animate(frameTime);
}
}
const animatables = this._animatables;
for (let i = 0; i < animatables.length; ++i) {
animatables[i].animate(frameTime);
}
this.onAnimationTickObservable.notifyObservers();
}
this.onSeekAnimationObservable.notifyObservers();
}
/**
* Seek animation to the specified frame time
*
* If you set forceEvaluate true, the animation is evaluated even if the animation is not playing
*
* If audio player is set and not paused, it try to play the audio from the seek time so it returns Promise
* @param frameTime Time in 30fps frame
* @param forceEvaluate Whether to force evaluate animation
* @returns Promise
*/
// eslint-disable-next-line @typescript-eslint/naming-convention
async seekAnimation(frameTime, forceEvaluate = false) {
frameTime = Math.max(0, Math.min(frameTime, this._animationFrameTimeDuration));
if (this._audioPlayer !== null) {
if (!this._audioPlayer.paused) {
this._audioPlayer.currentTime = frameTime / 30;
}
else if (!this._animationPaused && // animation playing but audio paused
this._audioPlayer.currentTime * 30 < this._animationFrameTimeDuration && // is player exausted
frameTime < this._audioPlayer.duration * 30 // is seek time in audio duration
) {
try {
this._audioPlayer._setCurrentTimeWithoutNotify(frameTime / 30);
await this._audioPlayer.play();
}
catch (e) {
if (e instanceof DOMException && e.name === "NotSupportedError") {
this.error("Failed to play audio.");
this._seekAnimationInternal(frameTime, forceEvaluate);
}
else {
throw e;
}
}
}
else {
this._seekAnimationInternal(frameTime, forceEvaluate);
this._audioPlayer?._setCurrentTimeWithoutNotify(frameTime / 30);
}
}
else {
this._seekAnimationInternal(frameTime, forceEvaluate);
}
}
/**
* Whether animation is playing
*/
get isAnimationPlaying() {
return !this._animationPaused;
}
/**
* MMD models created by this runtime
*/
get models() {
return this._models;
}
/**
* Animatable objects that added to this runtime
*/
get animatables() {
return this._animatables;
}
/**
* Audio player
*/
get audioPlayer() {
return this._audioPlayer;
}
/**
* Current animation time scale (default: 1)
*/
get timeScale() {
return this._animationTimeScale;
}
set timeScale(value) {
this._animationTimeScale = value;
if (this._audioPlayer !== null) {
this._audioPlayer._setPlaybackRateWithoutNotify(value);
}
}
/**
* Current animation time in 30fps frame
*/
get currentFrameTime() {
return this._currentFrameTime;
}
/**
* Current animation time in seconds
*/
get currentTime() {
return this._currentFrameTime / 30;
}
/**
* Current animation duration in 30fps frame
*/
get animationFrameTimeDuration() {
return this._animationFrameTimeDuration;
}
/**
* Current animation duration in seconds
*/
get animationDuration() {
return this._animationFrameTimeDuration / 30;
}
/**
* Set animation duration manually
*
* When the difference between the length of the song and the length of the animation is large, it can be helpful to adjust the animation duration manually
* @param frameTimeDuration Time in 30fps frame
*/
setManualAnimationDuration(frameTimeDuration) {
if (frameTimeDuration === null && !this._useManualAnimationDuration)
return;
if (frameTimeDuration === null) {
this._useManualAnimationDuration = false;
this._animationFrameTimeDuration = this._computeAnimationDuration();
}
else {
this._useManualAnimationDuration = true;
this._animationFrameTimeDuration = frameTimeDuration;
}
this.onAnimationDurationChangedObservable.notifyObservers();
}
/**
* Enable or disable debug logging (default: false)
*/
get loggingEnabled() {
return this._loggingEnabled;
}
set loggingEnabled(value) {
this._loggingEnabled = value;
if (value) {
this.log = this._logEnabled;
this.warn = this._warnEnabled;
this.error = this._errorEnabled;
}
else {
this.log = this._logDisabled;
this.warn = this._warnDisabled;
this.error = this._errorDisabled;
}
}
_logEnabled(message) {
Logger.Log(message);
}
_logDisabled() {
// do nothing
}
_warnEnabled(message) {
Logger.Warn(message);
}
_warnDisabled() {
// do nothing
}
_errorEnabled(message) {
Logger.Error(message);
}
_errorDisabled() {
// do nothing
}
}