v3d-web-realbits
Version:
Single camera motion-tracking in browser
333 lines (330 loc) • 14.1 kB
JavaScript
/*
Copyright (C) 2021 The v3d Authors.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import * as Comlink from "comlink";
// import "@babylonjs/core/Loading/loadingScreen";
// Register plugins (side effect)
// import "@babylonjs/core/Loading/Plugins/babylonFileLoader";
// import "@babylonjs/core/Materials";
// import "@babylonjs/loaders/glTF/glTFFileLoader";
import { Engine, } from "@babylonjs/core";
import { FPS } from "@mediapipe/control_utils";
import { Holistic } from "@mediapipe/holistic";
//* TODO: Mobile patch.
import { FaceMesh, } from "@mediapipe/face_mesh";
import { createScene, updateBuffer, updatePose, updateSpringBones, } from "./core";
import { createControlPanel, InitHolisticOptions, onResults, setHolisticOptions,
//* TODO: Mobile patch.
InitFaceMeshOptions, setFaceMeshOptions, onFMResults, } from "./mediapipe";
import { CustomLoadingScreen } from "./helper/utils";
export class V3DWeb {
constructor(vrmFilePath, videoElement, webglCanvasElement, videoCanvasElement, controlsElement, holisticConfig, loadingDiv, useMotionUpdate, useFaceMesh, afterInitCallback) {
// console.log("call constructor()");
this.videoElement = videoElement;
this.webglCanvasElement = webglCanvasElement;
this.videoCanvasElement = videoCanvasElement;
this.controlsElement = controlsElement;
this.holisticConfig = holisticConfig;
this.loadingDiv = loadingDiv;
this.useMotionUpdate = useMotionUpdate;
this.useFaceMesh = useFaceMesh;
this.workerPose = null;
this.boneState = {
boneRotations: null,
bonesNeedUpdate: false,
};
this._boneOptions = {
blinkLinkLR: false,
expression: "Neutral",
irisLockX: false,
lockFinger: false,
lockArm: false,
lockLeg: false,
resetInvisible: true,
};
this._updateBufferCallback = Comlink.proxy((data) => {
updateBuffer(data, this.boneState);
});
this.customLoadingScreen = null;
this._v3DCore = null;
this._vrmManager = null;
this.fpsControl = this.controlsElement ? new FPS() : null;
this.holistic = new Holistic(this.holisticConfig);
this.holisticState = {
ready: false,
activeEffect: "mask",
holisticUpdate: false,
};
this._holisticOptions = Object.assign({}, InitHolisticOptions);
//* TODO: Mobile patch.
this.faceMesh = new FaceMesh({
locateFile: (file) => {
return `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`;
},
});
this.faceMeshState = {
ready: false,
activeEffect: "mask",
faceMeshUpdate: false,
};
this._faceMeshOptions = Object.assign({}, InitFaceMeshOptions);
this._cameraList = [];
let globalInit = false;
if (!this.videoElement || !this.webglCanvasElement) {
globalInit = true;
this.videoElement = document.getElementsByClassName("input_video")[0];
this.videoCanvasElement = document.getElementById("video-canvas");
this.webglCanvasElement = document.getElementById("webgl-canvas");
this.controlsElement = document.getElementsByClassName("control-panel")[0];
}
if (!this.videoElement || !this.webglCanvasElement)
throw Error("Canvas or Video elements not found!");
this._vrmFile = vrmFilePath;
/**
* Babylonjs
*/
if (Engine.isSupported()) {
this.engine = new Engine(this.webglCanvasElement, true);
// engine.loadingScreen = new CustomLoadingScreen(webglCanvasElement);
}
else {
throw Error("WebGL is not supported in this browser!");
}
// Loading screen
if (this.loadingDiv) {
this.customLoadingScreen = new CustomLoadingScreen(this.webglCanvasElement, this.loadingDiv);
}
this.customLoadingScreen?.displayLoadingUI();
/**
* Comlink/workers
*/
this.worker = new Worker(new URL("./worker/pose-processing", import.meta.url), { type: "module" });
const posesRemote = Comlink.wrap(this.worker);
//* TODO: Mobile patch.
//* TODO: Fix iris bone rotation bug in case of face mesh.
if (useFaceMesh) {
this._boneOptions.irisLockX = true;
}
const Poses = new posesRemote.poses(this.boneOptions, this._updateBufferCallback);
Poses.then((v) => {
if (!v)
throw Error("Worker start failed!");
this.workerPose = v;
createScene(this.engine, this.workerPose, this.boneState, this.boneOptions, this.holistic, this.holisticState, this.faceMesh, this.faceMeshState, this._vrmFile, this.videoElement, this.useMotionUpdate, this.useFaceMesh).then((value) => {
if (!value)
throw Error("VRM Manager initialization failed!");
const [v3DCore, vrmManager] = value;
this._v3DCore = v3DCore;
this._vrmManager = vrmManager;
// Camera
this.getVideoDevices().then((devices) => {
if (devices.length < 1) {
throw Error("No camera found!");
}
else {
this._cameraList = devices;
this.getCamera(0).then(() => {
/**
* MediaPipe
*/
//* TODO: Mobile patch.
if (useFaceMesh) {
const mainOnResults = (results) => {
// console.log("call mainOnResults()");
onFMResults(results, vrmManager, this.videoCanvasElement, this.workerPose, this.faceMeshState.activeEffect, this._updateBufferCallback, this.fpsControl);
};
this.faceMesh.initialize().then(() => {
setFaceMeshOptions(this.faceMeshOptions, this.videoElement, this.faceMesh);
this.faceMesh.onResults(mainOnResults);
this.faceMeshState.ready = true;
this.customLoadingScreen?.hideLoadingUI();
if (afterInitCallback)
afterInitCallback();
});
}
else {
const mainOnResults = (results) => {
// console.log("call mainOnResults()");
if (results?.ea &&
results.poseLandmarks) {
onResults(results, vrmManager, this.videoCanvasElement, this.workerPose, this.holisticState.activeEffect, this._updateBufferCallback, this.fpsControl);
}
};
this.holistic.initialize().then(() => {
// Set initial options
setHolisticOptions(this.holisticOptions, this.videoElement, this.holisticState.activeEffect, this.holistic);
this.holistic.onResults(mainOnResults);
this.holisticState.ready = true;
this.customLoadingScreen?.hideLoadingUI();
if (afterInitCallback)
afterInitCallback();
});
}
});
}
});
});
});
// Present a control panel through which the user can manipulate the solution
// options.
if (this.controlsElement) {
createControlPanel(this.holistic, this.videoElement, this.controlsElement, this.holisticState.activeEffect, this.fpsControl);
}
}
get boneOptions() {
return this._boneOptions;
}
set boneOptions(value) {
this._boneOptions = value;
this.workerPose?.updateBoneOptions(this._boneOptions);
}
get v3DCore() {
return this._v3DCore;
}
get vrmManager() {
return this._vrmManager;
}
set vrmFile(value) {
this._vrmFile = value;
this.switchModel();
}
get holisticOptions() {
return this._holisticOptions;
}
set holisticOptions(value) {
this._holisticOptions = value;
setHolisticOptions(value, this.videoElement, this.holisticState.activeEffect, this.holistic);
}
get faceMeshOptions() {
return this._faceMeshOptions;
}
set faceMeshOptions(value) {
this._faceMeshOptions = value;
setFaceMeshOptions(value, this.videoElement, this.faceMesh);
}
get cameraList() {
return this._cameraList;
}
async getVideoDevices() {
// Ask permission
await navigator.mediaDevices.getUserMedia({ video: true });
const devices = await navigator.mediaDevices.enumerateDevices();
return devices.filter((device) => device.kind === "videoinput");
}
async getCamera(idx) {
// console.log("call getCamera()");
// console.log("idx: ", idx);
// const resultGetUserMedia = await navigator.mediaDevices.getUserMedia({
// audio: true,
// video: true,
// });
// // console.log("result: ", resultGetUserMedia);
// if (!this.videoElement) throw Error("Video Element not found!");
// console.log("this.videoElement: ", this.videoElement);
// this.videoElement.srcObject = resultGetUserMedia;
// this.videoElement.play();
// return resultGetUserMedia;
await navigator.mediaDevices
.getUserMedia({
//* TODO: Mobile patch.
// audio: true,
// video: true,
video: {
// width: 640,
// height: 480,
width: 320,
height: 240,
deviceId: {
exact: this.cameraList[idx].deviceId,
},
},
})
.then((stream) => {
// console.log("stream: ", stream);
if (!this.videoElement)
throw Error("Video Element not found!");
this.videoElement.srcObject = stream;
// let source = document.createElement('source');
//
// source.setAttribute('src', 'testfiles/dance5.webm');
// source.setAttribute('type', 'video/webm');
//
// this.videoElement.appendChild(source);
this.videoElement.play();
});
// .catch(e => console.error(e));
}
/**
* Close and dispose the application (BabylonJS and MediaPipe)
*/
close() {
this.holisticState.ready = false;
this.holistic.close().then(() => {
this.worker.terminate();
this._updateBufferCallback = null;
this.videoElement.srcObject = null;
});
this._vrmManager?.dispose();
this.engine.dispose();
}
/**
* Reset poses and holistic
*/
reset() {
this.workerPose?.resetBoneRotations(true);
this.holistic.reset();
}
switchSource(idx) {
if (idx >= this.cameraList.length)
return;
this.holisticState.ready = false;
this.getCamera(idx).then(() => {
this.reset();
this.holisticState.ready = true;
});
}
async switchModel() {
if (!this.v3DCore || !this.workerPose)
return;
this.v3DCore.updateAfterRenderFunction(() => { });
this.vrmManager?.dispose();
this._vrmManager = null;
await this.v3DCore.AppendAsync("", this._vrmFile);
this._vrmManager = this.v3DCore.getVRMManagerByURI(this._vrmFile.name
? this._vrmFile.name
: this._vrmFile);
if (!this._vrmManager)
throw Error("VRM model loading failed!");
// Reset camera
// const mainCamera = this.v3DCore.mainCamera as ArcRotateCamera;
// mainCamera.setPosition(new Vector3(0, 1.05, 4.5));
// mainCamera.setTarget(
// this._vrmManager.rootMesh
// .getWorldMatrix()
// .getTranslation()
// .subtractFromFloats(0, -1.25, 0)
// );
await this.workerPose.setBonesHierarchyTree(this._vrmManager.transformNodeTree, true);
this.workerPose.resetBoneRotations();
this.v3DCore.updateAfterRenderFunction(() => {
if (this.boneState.bonesNeedUpdate) {
updatePose(this._vrmManager, this.boneState, this.boneOptions);
updateSpringBones(this._vrmManager);
this.boneState.bonesNeedUpdate = false;
}
});
// this._vrmManager.rootMesh.rotationQuaternion =
// Quaternion.RotationYawPitchRoll(0, 0, 0);
}
}
export default V3DWeb;
//# sourceMappingURL=v3d-web.js.map