v3d-web-realbits
Version:
Single camera motion-tracking in browser
307 lines (304 loc) • 13.8 kB
JavaScript
/*
Copyright (C) 2021 The v3d Authors.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { Mesh, MeshBuilder, StandardMaterial, Vector3 } from "@babylonjs/core";
import { Color3 } from "@babylonjs/core/Maths";
import { initArray, } from "./utils";
import chroma from "chroma-js";
import { POSE_LANDMARKS } from "@mediapipe/holistic";
import { Poses } from "../worker/pose-processing";
import { cloneableQuaternionToQuaternion } from "./quaternion";
import { HAND_LANDMARK_LENGTH, HAND_LANDMARKS, normalizedLandmarkToVector, POSE_LANDMARK_LENGTH } from "./landmark";
export class Arrow3D {
constructor(scene, arrowRadius = 0.5, n = 30, arrowHeadLength = 1.5, arrowHeadMaxSize = 1.5, arrowLength = 10, arrowStart, arrowDirection, color) {
this.scene = scene;
//Shape profile in XY plane
this.myShape = [];
this.myPath = [];
this.arrowInstance = null;
this._arrowRadius = arrowRadius;
this._n = n;
this._arrowHeadLength = arrowHeadLength;
this._arrowHeadMaxSize = arrowHeadMaxSize;
this._arrowLength = arrowLength;
this._arrowStart = arrowStart;
this._arrowDirection = arrowDirection;
this.updateTopShape();
this.updatePath();
this._material = new StandardMaterial("sphereMaterial", scene);
if (color) {
if (typeof color === 'number')
color = `#${color.toString(16)}`;
this._material.diffuseColor = Color3.FromHexString(color);
}
}
get arrowRadius() {
return this._arrowRadius;
}
set arrowRadius(value) {
this._arrowRadius = value;
this.updateTopShape();
}
get n() {
return this._n;
}
set n(value) {
this._n = value;
this.updateTopShape();
}
get arrowHeadLength() {
return this._arrowHeadLength;
}
set arrowHeadLength(value) {
this._arrowHeadLength = value;
this.updatePath();
}
get arrowStart() {
return this._arrowStart;
}
set arrowStart(value) {
this._arrowStart = value;
this.updatePath();
}
get arrowLength() {
return this._arrowLength;
}
set arrowLength(value) {
this._arrowLength = value;
this.updatePath();
}
get arrowHeadMaxSize() {
return this._arrowHeadMaxSize;
}
set arrowHeadMaxSize(value) {
this._arrowHeadMaxSize = value;
this.updatePath();
}
get arrowDirection() {
return this._arrowDirection;
}
set arrowDirection(value) {
this._arrowDirection = value;
this.updatePath();
}
get material() {
return this._material;
}
updateTopShape() {
const deltaAngle = 2 * Math.PI / this.n;
this.myShape.length = 0;
for (let i = 0; i <= this.n; i++) {
this.myShape.push(new Vector3(this.arrowRadius * Math.cos(i * deltaAngle), this.arrowRadius * Math.sin(i * deltaAngle), 0));
}
this.myShape.push(this.myShape[0]); //close profile
}
updatePath() {
const arrowBodyLength = this.arrowLength - this.arrowHeadLength;
this.arrowDirection.normalize();
const arrowBodyEnd = this.arrowStart.add(this.arrowDirection.scale(arrowBodyLength));
const arrowHeadEnd = arrowBodyEnd.add(this.arrowDirection.scale(this.arrowHeadLength));
this.myPath.length = 0;
this.myPath.push(this.arrowStart);
this.myPath.push(arrowBodyEnd);
this.myPath.push(arrowBodyEnd);
this.myPath.push(arrowHeadEnd);
if (!this.arrowInstance)
this.arrowInstance = MeshBuilder.ExtrudeShapeCustom("arrow", {
shape: this.myShape,
path: this.myPath,
updatable: true,
scaleFunction: this.scaling.bind(this),
sideOrientation: Mesh.DOUBLESIDE
}, this.scene);
else
this.arrowInstance = MeshBuilder.ExtrudeShapeCustom("arrow", {
shape: this.myShape,
path: this.myPath,
scaleFunction: this.scaling.bind(this),
instance: this.arrowInstance
}, this.scene);
this.arrowInstance.material = this.material;
}
scaling(index, distance) {
switch (index) {
case 0:
case 1:
return 1;
case 2:
return this.arrowHeadMaxSize / this.arrowRadius;
case 3:
return 0;
default:
return 1;
}
}
updateStartAndDirection(arrowStart, arrowDirection) {
this._arrowStart = arrowStart;
this._arrowDirection = arrowDirection.length() === 0 ?
Vector3.One() : arrowDirection;
this.updatePath();
}
}
export function makeSphere(scene, pos, color, options) {
const sphere = MeshBuilder.CreateSphere("sphere", options || {
diameterX: 1, diameterY: 0.5, diameterZ: 0.5
}, scene);
const material = new StandardMaterial("sphereMaterial", scene);
if (color) {
if (typeof color === 'number')
color = `#${color.toString(16)}`;
material.diffuseColor = Color3.FromHexString(color);
}
sphere.material = material;
if (pos)
sphere.position = pos;
return sphere;
}
export function quaternionToDirectionVector(base, resultQuaternion) {
const quaternion = cloneableQuaternionToQuaternion(resultQuaternion);
const result = Vector3.Zero();
base.rotateByQuaternionToRef(quaternion, result);
return result.normalize();
}
export class DebugInfo {
constructor(scene) {
this.scene = scene;
this.faceMeshLandmarkSpheres = null;
this.poseLandmarkSpheres = this.initLandmarks(POSE_LANDMARK_LENGTH);
this.faceNormalArrows = this.initNormalArrows(1);
this.leftHandLandmarkSpheres = this.initLandmarks(HAND_LANDMARK_LENGTH, '#ff0000');
this.rightHandLandmarkSpheres = this.initLandmarks(HAND_LANDMARK_LENGTH, '#0022ff');
this.irisNormalArrows = this.initIrisNormalArrows();
this.poseNormalArrows = this.initNormalArrows(3);
this.leftHandNormalArrow = new Arrow3D(this.scene, 0.02, 32, 0.06, 0.06, 0.5, Vector3.Zero(), Vector3.One());
this.rightHandNormalArrow = new Arrow3D(this.scene, 0.02, 32, 0.06, 0.06, 0.5, Vector3.Zero(), Vector3.One());
this.leftHandNormalArrows = this.initNormalArrows(6);
this.rightHandNormalArrows = this.initNormalArrows(6);
if (!scene.debugLayer.isVisible()) {
scene.debugLayer.show({
globalRoot: document.getElementById('wrapper'),
handleResize: true,
});
}
}
initLandmarks(length, color) {
const colors = chroma.scale('Spectral')
.colors(length, 'hex');
return initArray(length, (i) => makeSphere(this.scene, Vector3.One(), colors[i], { diameter: 0.03 }));
}
initNormalArrows(length) {
const colors = chroma.scale('Spectral')
.colors(length / 2, 'hex');
return initArray(length, // Temp magical number
(i) => new Arrow3D(this.scene, 0.01, 32, 0.02, 0.02, 0.2, Vector3.Zero(), Vector3.One(), colors[i % (length / 2)]));
}
initIrisNormalArrows() {
return initArray(2, () => new Arrow3D(this.scene, 0.01, 32, 0.04, 0.04, 0.25, Vector3.Zero(), Vector3.One()));
}
initFaceMeshLandmarks(indexList) {
return initArray(indexList.length, (i) => {
return initArray(indexList[i].length, ((() => {
const colors = chroma.scale('Spectral')
.colors(indexList[i].length, 'hex');
return (i) => {
return makeSphere(this.scene, Vector3.One(), colors[i], { diameter: 0.01 });
};
})()));
});
}
updatePoseLandmarkSpheres(resultPoseLandmarks) {
if (resultPoseLandmarks.length !== POSE_LANDMARK_LENGTH)
return;
for (let i = 0; i < POSE_LANDMARK_LENGTH; ++i) {
if ([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 17, 18, 19, 20, 21, 22].includes(i))
continue;
this.poseLandmarkSpheres[i].position.set(resultPoseLandmarks[i].x, resultPoseLandmarks[i].y, resultPoseLandmarks[i].z);
}
}
updateHandLandmarkSpheres(resultHandLandmarks, left) {
const landmarkSpheres = left ?
this.leftHandLandmarkSpheres :
this.rightHandLandmarkSpheres;
if (resultHandLandmarks.length !== HAND_LANDMARK_LENGTH)
return;
for (let i = 0; i < HAND_LANDMARK_LENGTH; ++i) {
landmarkSpheres[i].position.set(resultHandLandmarks[i].x, resultHandLandmarks[i].y, resultHandLandmarks[i].z);
}
}
updateFaceNormalArrows(resultFaceNormals, resultPoseLandmarks) {
if (resultFaceNormals.length !== this.faceNormalArrows.length)
return;
for (let i = 0; i < this.faceNormalArrows.length; ++i) {
this.faceNormalArrows[i].updateStartAndDirection(normalizedLandmarkToVector(resultPoseLandmarks[POSE_LANDMARKS.NOSE]), normalizedLandmarkToVector(resultFaceNormals[i]));
}
}
updateIrisQuaternionArrows(resultIrisQuaternions, resultPoseLandmarks, resultFaceNormal) {
if (resultIrisQuaternions.length !== 2 && resultIrisQuaternions.length !== 3)
return;
this.irisNormalArrows[0].updateStartAndDirection(normalizedLandmarkToVector(resultPoseLandmarks[POSE_LANDMARKS.LEFT_EYE]), quaternionToDirectionVector(normalizedLandmarkToVector(resultFaceNormal), resultIrisQuaternions[0]));
this.irisNormalArrows[1].updateStartAndDirection(normalizedLandmarkToVector(resultPoseLandmarks[POSE_LANDMARKS.RIGHT_EYE]), quaternionToDirectionVector(normalizedLandmarkToVector(resultFaceNormal), resultIrisQuaternions[1]));
}
updateFaceMeshLandmarkSpheres(resultFaceMeshIndexLandmarks, resultFaceMeshLandmarks) {
const toShow = [
[0], [0],
[9, 4, 15, 12], [0, 4, 8, 12],
[0], [0],
[24, 25, 26, 34, 35, 36],
[0, 6, 18, 30]
];
if (resultFaceMeshIndexLandmarks.length !== 0 && !this.faceMeshLandmarkSpheres)
this.faceMeshLandmarkSpheres = this.initFaceMeshLandmarks(resultFaceMeshIndexLandmarks);
if (!this.faceMeshLandmarkSpheres ||
resultFaceMeshLandmarks.length !== this.faceMeshLandmarkSpheres.length)
return;
for (let i = 0; i < this.faceMeshLandmarkSpheres.length; ++i) {
for (let j = 0; j < this.faceMeshLandmarkSpheres[i].length; ++j) {
if (toShow[i].length > 0 && !toShow[i].includes(j))
continue;
this.faceMeshLandmarkSpheres[i][j].position.set(resultFaceMeshLandmarks[i][j].x, resultFaceMeshLandmarks[i][j].y, resultFaceMeshLandmarks[i][j].z);
}
}
}
updateHandWristNormalArrows(resultLeftHandBoneRotations, resultRightHandBoneRotations, resultPoseLandmarks) {
this.leftHandNormalArrow.updateStartAndDirection(normalizedLandmarkToVector(resultPoseLandmarks[POSE_LANDMARKS.LEFT_WRIST]), quaternionToDirectionVector(Poses.HAND_BASE_ROOT_NORMAL, resultLeftHandBoneRotations[HAND_LANDMARKS.WRIST]));
this.rightHandNormalArrow.updateStartAndDirection(normalizedLandmarkToVector(resultPoseLandmarks[POSE_LANDMARKS.RIGHT_WRIST]), quaternionToDirectionVector(Poses.HAND_BASE_ROOT_NORMAL, resultRightHandBoneRotations[HAND_LANDMARKS.WRIST]));
}
updateHandNormalArrows(resultLeftHandNormals, resultRightHandNormals, resultPoseLandmarks) {
if (resultLeftHandNormals) {
for (let i = 0; i < Math.min(this.leftHandNormalArrows.length, resultLeftHandNormals.length); ++i) {
this.leftHandNormalArrows[i].updateStartAndDirection(
// normalizedLandmarkToVector(
// resultPoseLandmarks[POSE_LANDMARKS.LEFT_WRIST]),
i < 3 ? Vector3.Zero() : new Vector3(0, 1, 0), normalizedLandmarkToVector(resultLeftHandNormals[i]));
}
}
if (resultRightHandNormals) {
for (let i = 0; i < Math.min(this.rightHandNormalArrows.length, resultRightHandNormals.length); ++i) {
this.rightHandNormalArrows[i].updateStartAndDirection(
// normalizedLandmarkToVector(
// resultPoseLandmarks[POSE_LANDMARKS.RIGHT_WRIST]),
i < 3 ? Vector3.One() : new Vector3(0, 1, 0), normalizedLandmarkToVector(resultRightHandNormals[i]));
}
}
}
updatePoseNormalArrows(resultPoseNormals, resultPoseLandmarks) {
if (resultPoseNormals.length !== this.poseNormalArrows.length)
return;
for (let i = 0; i < this.poseNormalArrows.length; ++i) {
this.poseNormalArrows[i].updateStartAndDirection(
// normalizedLandmarkToVector(
// resultPoseLandmarks[POSE_LANDMARKS.NOSE]),
Vector3.Zero(), normalizedLandmarkToVector(resultPoseNormals[i]));
}
}
}
//# sourceMappingURL=debug.js.map