handsfree
Version:
A Face Pointer and Pose Estimator for interacting with pages, desktops, robots, and more via gestures
1,602 lines (1,365 loc) • 281 kB
JavaScript
(function (global, factory) {
typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() :
typeof define === 'function' && define.amd ? define(factory) :
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, global.Handsfree = factory());
}(this, (function () { 'use strict';
class BaseModel {
constructor (handsfree, config) {
this.handsfree = handsfree;
this.config = config;
this.data = {};
// Whether we've loaded dependencies or not
this.dependenciesLoaded = false;
// Whether the model is enabled or not
this.enabled = config.enabled;
// Collection of plugins
this.plugins = [];
setTimeout(() => {
const getData = this.getData;
this.getData = async () => {
const data = await getData.apply(this, arguments);
this.runPlugins();
return data
};
}, 0);
}
// Implement in the model class
loadDependencies (callback) {}
updateData () {}
/**
* Enable model
* @param {*} handleLoad If true then it'll also attempt to load,
* otherwise you'll need to handle it yourself. This is mostly used internally
* to prevent the .update() method from double loading
*/
enable (handleLoad = true) {
this.handsfree.config[this.name] = this.config;
this.handsfree.config[this.name].enabled = this.enabled = true;
document.body.classList.add(`handsfree-model-${this.name}`);
if (handleLoad && !this.dependenciesLoaded) {
this.loadDependencies();
}
// Weboji uses a webgl context
if (this.name === 'weboji') {
this.handsfree.debug.$canvas.weboji.style.display = 'block';
}
}
disable () {
this.handsfree.config[this.name] = this.config;
this.handsfree.config[this.name].enabled = this.enabled = false;
document.body.classList.remove(`handsfree-model-${this.name}`);
setTimeout(() => {
// Weboji uses a webgl context so let's just hide it
if (this.name === 'weboji') {
this.handsfree.debug.$canvas.weboji.style.display = 'none';
} else {
this.handsfree.debug.context[this.name].clearRect(0, 0, this.handsfree.debug.$canvas[this.name].width, this.handsfree.debug.$canvas[this.name].height);
}
}, 0);
}
/**
* Loads a script and runs a callback
* @param {string} src The absolute path of the source file
* @param {*} callback The callback to call after the file is loaded
* @param {boolean} skip Whether to skip loading the dependency and just call the callback
*/
loadDependency (src, callback, skip = false) {
// Skip and run callback
if (skip) {
callback && callback();
return
}
// Inject script into DOM
const $script = document.createElement('script');
$script.async = true;
$script.onload = () => {
callback && callback();
};
$script.onerror = () => {
this.handsfree.emit('modelError', `Error loading ${src}`);
};
$script.src = src;
document.body.appendChild($script);
}
/**
* Run all the plugins attached to this model
*/
runPlugins () {
// Exit if no data
if (!this.data || (this.name === 'handpose' && !this.data.annotations)) {
return
}
if (Object.keys(this.data).length) {
this.plugins.forEach(name => {
this.handsfree.plugin[name].enabled && this.handsfree.plugin[name]?.onFrame(this.handsfree.data);
});
}
}
}
class HandsModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'hands';
this.palmPoints = [0, 1, 2, 5, 9, 13, 17];
}
loadDependencies (callback) {
// Just load utils on client
if (this.handsfree.config.isClient) {
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, () => {
this.onWarmUp(callback);
}, !!window.drawConnectors);
return
}
// Load hands
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/hands/hands.js`, () => {
// Configure model
this.api = new window.Hands({locateFile: file => {
return `${this.handsfree.config.assetsPath}/@mediapipe/hands/${file}`
}});
this.api.setOptions(this.handsfree.config.hands);
this.api.onResults(results => this.dataReceived(results));
// Load the media stream
this.handsfree.getUserMedia(() => {
// Warm up before using in loop
if (!this.handsfree.mediapipeWarmups.isWarmingUp) {
this.warmUp(callback);
} else {
this.handsfree.on('mediapipeWarmedUp', () => {
if (!this.handsfree.mediapipeWarmups.isWarmingUp && !this.handsfree.mediapipeWarmups[this.name]) {
this.warmUp(callback);
}
});
}
});
// Load the hands camera module
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, null, !!window.drawConnectors);
});
}
/**
* Warms up the model
*/
warmUp (callback) {
this.handsfree.mediapipeWarmups[this.name] = true;
this.handsfree.mediapipeWarmups.isWarmingUp = true;
this.api.send({image: this.handsfree.debug.$video}).then(() => {
this.handsfree.mediapipeWarmups.isWarmingUp = false;
this.onWarmUp(callback);
});
}
/**
* Called after the model has been warmed up
* - If we don't do this there will be too many initial hits and cause an error
*/
onWarmUp (callback) {
this.dependenciesLoaded = true;
document.body.classList.add('handsfree-model-hands');
this.handsfree.emit('modelReady', this);
this.handsfree.emit('handsModelReady', this);
this.handsfree.emit('mediapipeWarmedUp', this);
callback && callback(this);
}
/**
* Get data
*/
async getData () {
this.dependenciesLoaded && await this.api.send({image: this.handsfree.debug.$video});
}
// Called through this.api.onResults
dataReceived (results) {
this.data = results;
this.handsfree.data.hands = results;
if (this.handsfree.isDebugging) {
this.debug(results);
}
}
/**
* Debugs the hands model
*/
debug (results) {
// Bail if drawing helpers haven't loaded
if (typeof drawConnectors === 'undefined') return
// Clear the canvas
this.handsfree.debug.context.hands.clearRect(0, 0, this.handsfree.debug.$canvas.hands.width, this.handsfree.debug.$canvas.hands.height);
// Draw skeletons
if (results.multiHandLandmarks) {
for (const landmarks of results.multiHandLandmarks) {
drawConnectors(this.handsfree.debug.context.hands, landmarks, HAND_CONNECTIONS, {color: '#00FF00', lineWidth: 5});
drawLandmarks(this.handsfree.debug.context.hands, landmarks, {color: '#FF0000', lineWidth: 2});
}
}
}
}
class FacemeshModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'facemesh';
this.isWarmedUp = false;
}
loadDependencies (callback) {
// Just load utils on client
if (this.handsfree.config.isClient) {
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, () => {
this.onWarmUp(callback);
}, !!window.drawConnectors);
return
}
// Load facemesh
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/face_mesh/face_mesh.js`, () => {
// Configure model
this.api = new window.FaceMesh({locateFile: file => {
return `${this.handsfree.config.assetsPath}/@mediapipe/face_mesh/${file}`
}});
this.api.setOptions(this.handsfree.config.facemesh);
this.api.onResults(results => this.dataReceived(results));
// Load the media stream
this.handsfree.getUserMedia(() => {
// Warm up before using in loop
if (!this.handsfree.mediapipeWarmups.isWarmingUp) {
this.warmUp(callback);
} else {
this.handsfree.on('mediapipeWarmedUp', () => {
if (!this.handsfree.mediapipeWarmups.isWarmingUp && !this.handsfree.mediapipeWarmups[this.name]) {
this.warmUp(callback);
}
});
}
});
// Load the hands camera module
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, null, !!window.drawConnectors);
});
}
/**
* Warms up the model
*/
warmUp (callback) {
this.handsfree.mediapipeWarmups[this.name] = true;
this.handsfree.mediapipeWarmups.isWarmingUp = true;
this.api.send({image: this.handsfree.debug.$video}).then(() => {
this.handsfree.mediapipeWarmups.isWarmingUp = false;
this.onWarmUp(callback);
});
}
/**
* Called after the model has been warmed up
* - If we don't do this there will be too many initial hits and cause an error
*/
onWarmUp (callback) {
this.dependenciesLoaded = true;
document.body.classList.add('handsfree-model-facemesh');
this.handsfree.emit('modelReady', this);
this.handsfree.emit('facemeshModelReady', this);
this.handsfree.emit('mediapipeWarmedUp', this);
callback && callback(this);
}
/**
* Get data
*/
async getData () {
this.dependenciesLoaded && await this.api.send({image: this.handsfree.debug.$video});
}
// Called through this.api.onResults
dataReceived (results) {
this.data = results;
this.handsfree.data.facemesh = results;
if (this.handsfree.isDebugging) {
this.debug(results);
}
}
/**
* Debugs the facemesh model
*/
debug (results) {
// Bail if drawing helpers haven't loaded
if (typeof drawConnectors === 'undefined') return
this.handsfree.debug.context.facemesh.clearRect(0, 0, this.handsfree.debug.$canvas.facemesh.width, this.handsfree.debug.$canvas.facemesh.height);
if (results.multiFaceLandmarks) {
for (const landmarks of results.multiFaceLandmarks) {
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_TESSELATION, {color: '#C0C0C070', lineWidth: 1});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_RIGHT_EYE, {color: '#FF3030'});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_RIGHT_EYEBROW, {color: '#FF3030'});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_LEFT_EYE, {color: '#30FF30'});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_LEFT_EYEBROW, {color: '#30FF30'});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_FACE_OVAL, {color: '#E0E0E0'});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_LIPS, {color: '#E0E0E0'});
}
}
}
}
class PoseModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'pose';
// Without this the loading event will happen before the first frame
this.hasLoadedAndRun = false;
this.palmPoints = [0, 1, 2, 5, 9, 13, 17];
}
loadDependencies (callback) {
// Just load utils on client
if (this.handsfree.config.isClient) {
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, () => {
this.onWarmUp(callback);
}, !!window.drawConnectors);
return
}
// Load pose
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/pose/pose.js`, () => {
this.api = new window.Pose({locateFile: file => {
return `${this.handsfree.config.assetsPath}/@mediapipe/pose/${file}`
}});
this.api.setOptions(this.handsfree.config.pose);
this.api.onResults(results => this.dataReceived(results));
// Load the media stream
this.handsfree.getUserMedia(() => {
// Warm up before using in loop
if (!this.handsfree.mediapipeWarmups.isWarmingUp) {
this.warmUp(callback);
} else {
this.handsfree.on('mediapipeWarmedUp', () => {
if (!this.handsfree.mediapipeWarmups.isWarmingUp && !this.handsfree.mediapipeWarmups[this.name]) {
this.warmUp(callback);
}
});
}
});
// Load the hands camera module
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, null, !!window.drawConnectors);
});
}
/**
* Warms up the model
*/
warmUp (callback) {
this.handsfree.mediapipeWarmups[this.name] = true;
this.handsfree.mediapipeWarmups.isWarmingUp = true;
this.api.send({image: this.handsfree.debug.$video}).then(() => {
this.handsfree.mediapipeWarmups.isWarmingUp = false;
this.onWarmUp(callback);
});
}
/**
* Called after the model has been warmed up
* - If we don't do this there will be too many initial hits and cause an error
*/
onWarmUp (callback) {
this.dependenciesLoaded = true;
document.body.classList.add('handsfree-model-pose');
this.handsfree.emit('modelReady', this);
this.handsfree.emit('poseModelReady', this);
this.handsfree.emit('mediapipeWarmedUp', this);
callback && callback(this);
}
/**
* Get data
*/
async getData () {
this.dependenciesLoaded && await this.api.send({image: this.handsfree.debug.$video});
}
// Called through this.api.onResults
dataReceived (results) {
this.data = results;
this.handsfree.data.pose = results;
if (this.handsfree.isDebugging) {
this.debug(results);
}
}
/**
* Debugs the pose model
*/
debug (results) {
this.handsfree.debug.context.pose.clearRect(0, 0, this.handsfree.debug.$canvas.pose.width, this.handsfree.debug.$canvas.pose.height);
if (results.poseLandmarks) {
drawConnectors(this.handsfree.debug.context.pose, results.poseLandmarks, POSE_CONNECTIONS, {color: '#00FF00', lineWidth: 4});
drawLandmarks(this.handsfree.debug.context.pose, results.poseLandmarks, {color: '#FF0000', lineWidth: 2});
}
}
}
class HolisticModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'holistic';
// Without this the loading event will happen before the first frame
this.hasLoadedAndRun = false;
this.palmPoints = [0, 1, 2, 5, 9, 13, 17];
}
loadDependencies (callback) {
// Just load utils on client
if (this.handsfree.config.isClient) {
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, () => {
this.onWarmUp(callback);
}, !!window.drawConnectors);
return
}
// Load holistic
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/holistic/holistic.js`, () => {
this.api = new window.Holistic({locateFile: file => {
return `${this.handsfree.config.assetsPath}/@mediapipe/holistic/${file}`
}});
this.api.setOptions(this.handsfree.config.holistic);
this.api.onResults(results => this.dataReceived(results));
// Load the media stream
this.handsfree.getUserMedia(() => {
// Warm up before using in loop
if (!this.handsfree.mediapipeWarmups.isWarmingUp) {
this.warmUp(callback);
} else {
this.handsfree.on('mediapipeWarmedUp', () => {
if (!this.handsfree.mediapipeWarmups.isWarmingUp && !this.handsfree.mediapipeWarmups[this.name]) {
this.warmUp(callback);
}
});
}
});
// Load the hands camera module
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, null, !!window.drawConnectors);
});
}
/**
* Warms up the model
*/
warmUp (callback) {
this.handsfree.mediapipeWarmups[this.name] = true;
this.handsfree.mediapipeWarmups.isWarmingUp = true;
this.api.send({image: this.handsfree.debug.$video}).then(() => {
this.handsfree.mediapipeWarmups.isWarmingUp = false;
this.onWarmUp(callback);
});
}
/**
* Called after the model has been warmed up
* - If we don't do this there will be too many initial hits and cause an error
*/
onWarmUp (callback) {
this.dependenciesLoaded = true;
document.body.classList.add('handsfree-model-holistic');
this.handsfree.emit('modelReady', this);
this.handsfree.emit('holisticModelReady', this);
this.handsfree.emit('mediapipeWarmedUp', this);
callback && callback(this);
}
/**
* Get data
*/
async getData () {
this.dependenciesLoaded && await this.api.send({image: this.handsfree.debug.$video});
}
// Called through this.api.onResults
dataReceived (results) {
this.data = results;
this.handsfree.data.holistic = results;
if (this.handsfree.isDebugging) {
this.debug(results);
}
}
/**
* Debugs the holistic model
*/
debug (results) {
this.handsfree.debug.context.holistic.clearRect(0, 0, this.handsfree.debug.$canvas.holistic.width, this.handsfree.debug.$canvas.holistic.height);
drawConnectors(this.handsfree.debug.context.holistic, results.poseLandmarks, POSE_CONNECTIONS, {
color: '#0f0',
lineWidth: 4
});
drawLandmarks(this.handsfree.debug.context.holistic, results.poseLandmarks, {
color: '#f00',
lineWidth: 2
});
drawConnectors(this.handsfree.debug.context.holistic, results.faceLandmarks, FACEMESH_TESSELATION, {
color: '#f0f',
lineWidth: 1
});
drawConnectors(this.handsfree.debug.context.holistic, results.leftHandLandmarks, HAND_CONNECTIONS, {
color: '#0f0',
lineWidth: 5
});
drawLandmarks(this.handsfree.debug.context.holistic, results.leftHandLandmarks, {
color: '#f0f',
lineWidth: 2
});
drawConnectors(this.handsfree.debug.context.holistic, results.rightHandLandmarks, HAND_CONNECTIONS, {
color: '#0f0',
lineWidth: 5
});
drawLandmarks(this.handsfree.debug.context.holistic, results.rightHandLandmarks, {
color: '#f0f',
lineWidth: 2
});
}
}
/**
* 🚨 This model is not currently active
*/
class HandposeModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'handpose';
// Various THREE variables
this.three = {
scene: null,
camera: null,
renderer: null,
meshes: []
};
// landmark indices that represent the palm
// 8 = Index finger tip
// 12 = Middle finger tip
this.palmPoints = [0, 1, 2, 5, 9, 13, 17];
}
loadDependencies (callback) {
this.loadDependency(`${this.handsfree.config.assetsPath}/three/three.min.js`, () => {
this.loadDependency(`${this.handsfree.config.assetsPath}/@tensorflow/tf-core.js`, () => {
this.loadDependency(`${this.handsfree.config.assetsPath}/@tensorflow/tf-converter.js`, () => {
this.loadDependency(`${this.handsfree.config.assetsPath}/@tensorflow/tf-backend-${this.handsfree.config.handpose.backend}.js`, () => {
this.loadDependency(`${this.handsfree.config.assetsPath}/@tensorflow-models/handpose/handpose.js`, () => {
this.handsfree.getUserMedia(async () => {
await window.tf.setBackend(this.handsfree.config.handpose.backend);
this.api = await handpose.load(this.handsfree.config.handpose.model);
this.setup3D();
callback && callback(this);
this.dependenciesLoaded = true;
this.handsfree.emit('modelReady', this);
this.handsfree.emit('handposeModelReady', this);
document.body.classList.add('handsfree-model-handpose');
});
});
});
});
}, !!window.tf);
}, !!window.THREE);
}
/**
* Runs inference and sets up other data
*/
async getData () {
if (!this.handsfree.debug.$video) return
const predictions = await this.api.estimateHands(this.handsfree.debug.$video);
this.data = {
...predictions[0],
meshes: this.three.meshes
};
if (predictions[0]) {
this.updateMeshes(this.data);
}
this.three.renderer.render(this.three.scene, this.three.camera);
return this.data
}
/**
* Sets up the 3D environment
*/
setup3D () {
// Setup Three
this.three = {
scene: new window.THREE.Scene(),
camera: new window.THREE.PerspectiveCamera(90, window.outerWidth / window.outerHeight, 0.1, 1000),
renderer: new THREE.WebGLRenderer({
alpha: true,
canvas: this.handsfree.debug.$canvas.handpose
}),
meshes: []
};
this.three.renderer.setSize(window.outerWidth, window.outerHeight);
this.three.camera.position.z = this.handsfree.debug.$video.videoWidth / 4;
this.three.camera.lookAt(new window.THREE.Vector3(0, 0, 0));
// Camera plane
this.three.screen = new window.THREE.Mesh(
new window.THREE.BoxGeometry(window.outerWidth, window.outerHeight, 1),
new window.THREE.MeshNormalMaterial()
);
this.three.screen.position.z = 300;
this.three.scene.add(this.three.screen);
// Camera raycaster
this.three.raycaster = new window.THREE.Raycaster();
this.three.arrow = new window.THREE.ArrowHelper(this.three.raycaster.ray.direction, this.three.raycaster.ray.origin, 300, 0xff0000);
this.three.scene.add(this.three.arrow);
// Create model representations (one for each keypoint)
for (let i = 0; i < 21; i++){
const {isPalm} = this.getLandmarkProperty(i);
const obj = new window.THREE.Object3D(); // a parent object to facilitate rotation/scaling
// we make each bone a cylindrical shape, but you can use your own models here too
const geometry = new window.THREE.CylinderGeometry(isPalm ? 5 : 10, 5, 1);
let material = new window.THREE.MeshNormalMaterial();
const mesh = new window.THREE.Mesh(geometry, material);
mesh.rotation.x = Math.PI / 2;
obj.add(mesh);
this.three.scene.add(obj);
this.three.meshes.push(obj);
// uncomment this to help identify joints
// if (i === 4) {
// mesh.material.transparent = true
// mesh.material.opacity = 0
// }
}
// Create center of palm
const obj = new window.THREE.Object3D();
const geometry = new window.THREE.CylinderGeometry(5, 5, 1);
let material = new window.THREE.MeshNormalMaterial();
const mesh = new window.THREE.Mesh(geometry, material);
mesh.rotation.x = Math.PI / 2;
this.three.centerPalmObj = obj;
obj.add(mesh);
this.three.scene.add(obj);
this.three.meshes.push(obj);
this.three.screen.visible = false;
}
// compute some metadata given a landmark index
// - is the landmark a palm keypoint or a finger keypoint?
// - what's the next landmark to connect to if we're drawing a bone?
getLandmarkProperty (i) {
const palms = [0, 1, 2, 5, 9, 13, 17]; //landmark indices that represent the palm
const idx = palms.indexOf(i);
const isPalm = idx != -1;
let next; // who to connect with?
if (!isPalm) { // connect with previous finger landmark if it's a finger landmark
next = i - 1;
}else { // connect with next palm landmark if it's a palm landmark
next = palms[(idx + 1) % palms.length];
}
return {isPalm, next}
}
/**
* update threejs object position and orientation from the detected hand pose
* threejs has a "scene" model, so we don't have to specify what to draw each frame,
* instead we put objects at right positions and threejs renders them all
* @param {*} hand
*/
updateMeshes (hand) {
for (let i = 0; i < this.three.meshes.length - 1 /* palmbase */; i++) {
const {next} = this.getLandmarkProperty(i);
const p0 = this.webcam2space(...hand.landmarks[i]); // one end of the bone
const p1 = this.webcam2space(...hand.landmarks[next]); // the other end of the bone
// compute the center of the bone (midpoint)
const mid = p0.clone().lerp(p1, 0.5);
this.three.meshes[i].position.set(mid.x, mid.y, mid.z);
// compute the length of the bone
this.three.meshes[i].scale.z = p0.distanceTo(p1);
// compute orientation of the bone
this.three.meshes[i].lookAt(p1);
if (i === 8) {
this.three.arrow.position.set(mid.x, mid.y, mid.z);
const direction = new window.THREE.Vector3().subVectors(p0, mid);
this.three.arrow.setDirection(direction.normalize());
this.three.arrow.setLength(800);
this.three.arrow.direction = direction;
}
}
this.updateCenterPalmMesh(hand);
}
/**
* Update the palm
*/
updateCenterPalmMesh (hand) {
let points = [];
let mid = {
x: 0,
y: 0,
z: 0
};
// Get position for the palm
this.palmPoints.forEach((i, n) => {
points.push(this.webcam2space(...hand.landmarks[i]));
mid.x += points[n].x;
mid.y += points[n].y;
mid.z += points[n].z;
});
mid.x = mid.x / this.palmPoints.length;
mid.y = mid.y / this.palmPoints.length;
mid.z = mid.z / this.palmPoints.length;
this.three.centerPalmObj.position.set(mid.x, mid.y, mid.z);
this.three.centerPalmObj.scale.z = 10;
this.three.centerPalmObj.rotation.x = this.three.meshes[12].rotation.x - Math.PI / 2;
this.three.centerPalmObj.rotation.y = -this.three.meshes[12].rotation.y;
this.three.centerPalmObj.rotation.z = this.three.meshes[12].rotation.z;
}
// transform webcam coordinates to threejs 3d coordinates
webcam2space (x, y, z) {
return new window.THREE.Vector3(
(x-this.handsfree.debug.$video.videoWidth / 2),
-(y-this.handsfree.debug.$video.videoHeight / 2), // in threejs, +y is up
-z
)
}
}
class WebojiModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'weboji';
}
loadDependencies (callback) {
// Just load utils on client
if (this.handsfree.config.isClient) {
this.onReady(callback);
return
}
// Load weboji
this.loadDependency(`${this.handsfree.config.assetsPath}/jeeliz/jeelizFaceTransfer.js`, () => {
const url = this.handsfree.config.assetsPath + '/jeeliz/jeelizFaceTransferNNC.json';
this.api = window.JEEFACETRANSFERAPI;
fetch(url)
.then(model => model.json())
// Next, let's initialize the weboji tracker API
.then(model => {
this.api.init({
canvasId: `handsfree-canvas-weboji-${this.handsfree.id}`,
NNC: JSON.stringify(model),
videoSettings: this.handsfree.config.weboji.videoSettings,
callbackReady: () => this.onReady(callback)
});
})
.catch((ev) => {
console.log(ev);
console.error(`Couldn't load weboji tracking model at ${url}`);
this.handsfree.emit('modelError', ev);
});
});
}
onReady (callback) {
this.dependenciesLoaded = true;
this.handsfree.emit('modelReady', this);
this.handsfree.emit('webojiModelReady', this);
document.body.classList.add('handsfree-model-weboji');
callback && callback(this);
}
getData () {
// Core
this.data.rotation = this.api.get_rotationStabilized();
this.data.translation = this.api.get_positionScale();
this.data.morphs = this.api.get_morphTargetInfluencesStabilized();
// Helpers
this.data.state = this.getStates();
this.data.degree = this.getDegrees();
this.data.isDetected = this.api.is_detected();
this.handsfree.data.weboji = this.data;
return this.data
}
/**
* Helpers for getting degrees
*/
getDegrees () {
return [
this.data.rotation[0] * 180 / Math.PI,
this.data.rotation[1] * 180 / Math.PI,
this.data.rotation[2] * 180 / Math.PI
]
}
/**
* Sets some stateful helpers
*/
getStates() {
/**
* Handles extra calculations for weboji morphs
*/
const morphs = this.data.morphs;
const state = this.data.state || {};
// Smiles
state.smileRight =
morphs[0] > this.handsfree.config.weboji.morphs.threshold.smileRight;
state.smileLeft =
morphs[1] > this.handsfree.config.weboji.morphs.threshold.smileLeft;
state.smile = state.smileRight && state.smileLeft;
state.smirk =
(state.smileRight && !state.smileLeft) ||
(!state.smileRight && state.smileLeft);
state.pursed =
morphs[7] > this.handsfree.config.weboji.morphs.threshold.mouthRound;
// Eyebrows
state.browLeftUp =
morphs[4] > this.handsfree.config.weboji.morphs.threshold.browLeftUp;
state.browRightUp =
morphs[5] > this.handsfree.config.weboji.morphs.threshold.browRightUp;
state.browsUp =
morphs[4] > this.handsfree.config.weboji.morphs.threshold.browLeftUp &&
morphs[5] > this.handsfree.config.weboji.morphs.threshold.browLeftUp;
state.browLeftDown =
morphs[2] > this.handsfree.config.weboji.morphs.threshold.browLeftDown;
state.browRightDown =
morphs[3] > this.handsfree.config.weboji.morphs.threshold.browRightDown;
state.browsDown =
morphs[2] > this.handsfree.config.weboji.morphs.threshold.browLeftDown &&
morphs[3] > this.handsfree.config.weboji.morphs.threshold.browLeftDown;
state.browsUpDown =
(state.browLeftDown && state.browRightUp) ||
(state.browRightDown && state.browLeftUp);
// Eyes
state.eyeLeftClosed =
morphs[8] > this.handsfree.config.weboji.morphs.threshold.eyeLeftClosed;
state.eyeRightClosed =
morphs[9] > this.handsfree.config.weboji.morphs.threshold.eyeRightClosed;
state.eyesClosed = state.eyeLeftClosed && state.eyeRightClosed;
// Mouth
state.mouthClosed = morphs[6] === 0;
state.mouthOpen =
morphs[6] > this.handsfree.config.weboji.morphs.threshold.mouthOpen;
return state
}
}
/**
* Removes all key-value entries from the list cache.
*
* @private
* @name clear
* @memberOf ListCache
*/
function listCacheClear() {
this.__data__ = [];
this.size = 0;
}
var _listCacheClear = listCacheClear;
/**
* Performs a
* [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero)
* comparison between two values to determine if they are equivalent.
*
* @static
* @memberOf _
* @since 4.0.0
* @category Lang
* @param {*} value The value to compare.
* @param {*} other The other value to compare.
* @returns {boolean} Returns `true` if the values are equivalent, else `false`.
* @example
*
* var object = { 'a': 1 };
* var other = { 'a': 1 };
*
* _.eq(object, object);
* // => true
*
* _.eq(object, other);
* // => false
*
* _.eq('a', 'a');
* // => true
*
* _.eq('a', Object('a'));
* // => false
*
* _.eq(NaN, NaN);
* // => true
*/
function eq(value, other) {
return value === other || (value !== value && other !== other);
}
var eq_1 = eq;
/**
* Gets the index at which the `key` is found in `array` of key-value pairs.
*
* @private
* @param {Array} array The array to inspect.
* @param {*} key The key to search for.
* @returns {number} Returns the index of the matched value, else `-1`.
*/
function assocIndexOf(array, key) {
var length = array.length;
while (length--) {
if (eq_1(array[length][0], key)) {
return length;
}
}
return -1;
}
var _assocIndexOf = assocIndexOf;
/** Used for built-in method references. */
var arrayProto = Array.prototype;
/** Built-in value references. */
var splice = arrayProto.splice;
/**
* Removes `key` and its value from the list cache.
*
* @private
* @name delete
* @memberOf ListCache
* @param {string} key The key of the value to remove.
* @returns {boolean} Returns `true` if the entry was removed, else `false`.
*/
function listCacheDelete(key) {
var data = this.__data__,
index = _assocIndexOf(data, key);
if (index < 0) {
return false;
}
var lastIndex = data.length - 1;
if (index == lastIndex) {
data.pop();
} else {
splice.call(data, index, 1);
}
--this.size;
return true;
}
var _listCacheDelete = listCacheDelete;
/**
* Gets the list cache value for `key`.
*
* @private
* @name get
* @memberOf ListCache
* @param {string} key The key of the value to get.
* @returns {*} Returns the entry value.
*/
function listCacheGet(key) {
var data = this.__data__,
index = _assocIndexOf(data, key);
return index < 0 ? undefined : data[index][1];
}
var _listCacheGet = listCacheGet;
/**
* Checks if a list cache value for `key` exists.
*
* @private
* @name has
* @memberOf ListCache
* @param {string} key The key of the entry to check.
* @returns {boolean} Returns `true` if an entry for `key` exists, else `false`.
*/
function listCacheHas(key) {
return _assocIndexOf(this.__data__, key) > -1;
}
var _listCacheHas = listCacheHas;
/**
* Sets the list cache `key` to `value`.
*
* @private
* @name set
* @memberOf ListCache
* @param {string} key The key of the value to set.
* @param {*} value The value to set.
* @returns {Object} Returns the list cache instance.
*/
function listCacheSet(key, value) {
var data = this.__data__,
index = _assocIndexOf(data, key);
if (index < 0) {
++this.size;
data.push([key, value]);
} else {
data[index][1] = value;
}
return this;
}
var _listCacheSet = listCacheSet;
/**
* Creates an list cache object.
*
* @private
* @constructor
* @param {Array} [entries] The key-value pairs to cache.
*/
function ListCache(entries) {
var index = -1,
length = entries == null ? 0 : entries.length;
this.clear();
while (++index < length) {
var entry = entries[index];
this.set(entry[0], entry[1]);
}
}
// Add methods to `ListCache`.
ListCache.prototype.clear = _listCacheClear;
ListCache.prototype['delete'] = _listCacheDelete;
ListCache.prototype.get = _listCacheGet;
ListCache.prototype.has = _listCacheHas;
ListCache.prototype.set = _listCacheSet;
var _ListCache = ListCache;
/**
* Removes all key-value entries from the stack.
*
* @private
* @name clear
* @memberOf Stack
*/
function stackClear() {
this.__data__ = new _ListCache;
this.size = 0;
}
var _stackClear = stackClear;
/**
* Removes `key` and its value from the stack.
*
* @private
* @name delete
* @memberOf Stack
* @param {string} key The key of the value to remove.
* @returns {boolean} Returns `true` if the entry was removed, else `false`.
*/
function stackDelete(key) {
var data = this.__data__,
result = data['delete'](key);
this.size = data.size;
return result;
}
var _stackDelete = stackDelete;
/**
* Gets the stack value for `key`.
*
* @private
* @name get
* @memberOf Stack
* @param {string} key The key of the value to get.
* @returns {*} Returns the entry value.
*/
function stackGet(key) {
return this.__data__.get(key);
}
var _stackGet = stackGet;
/**
* Checks if a stack value for `key` exists.
*
* @private
* @name has
* @memberOf Stack
* @param {string} key The key of the entry to check.
* @returns {boolean} Returns `true` if an entry for `key` exists, else `false`.
*/
function stackHas(key) {
return this.__data__.has(key);
}
var _stackHas = stackHas;
var commonjsGlobal = typeof globalThis !== 'undefined' ? globalThis : typeof window !== 'undefined' ? window : typeof global !== 'undefined' ? global : typeof self !== 'undefined' ? self : {};
function createCommonjsModule(fn, basedir, module) {
return module = {
path: basedir,
exports: {},
require: function (path, base) {
return commonjsRequire(path, (base === undefined || base === null) ? module.path : base);
}
}, fn(module, module.exports), module.exports;
}
function commonjsRequire () {
throw new Error('Dynamic requires are not currently supported by @rollup/plugin-commonjs');
}
/** Detect free variable `global` from Node.js. */
var freeGlobal = typeof commonjsGlobal == 'object' && commonjsGlobal && commonjsGlobal.Object === Object && commonjsGlobal;
var _freeGlobal = freeGlobal;
/** Detect free variable `self`. */
var freeSelf = typeof self == 'object' && self && self.Object === Object && self;
/** Used as a reference to the global object. */
var root = _freeGlobal || freeSelf || Function('return this')();
var _root = root;
/** Built-in value references. */
var Symbol = _root.Symbol;
var _Symbol = Symbol;
/** Used for built-in method references. */
var objectProto = Object.prototype;
/** Used to check objects for own properties. */
var hasOwnProperty = objectProto.hasOwnProperty;
/**
* Used to resolve the
* [`toStringTag`](http://ecma-international.org/ecma-262/7.0/#sec-object.prototype.tostring)
* of values.
*/
var nativeObjectToString = objectProto.toString;
/** Built-in value references. */
var symToStringTag = _Symbol ? _Symbol.toStringTag : undefined;
/**
* A specialized version of `baseGetTag` which ignores `Symbol.toStringTag` values.
*
* @private
* @param {*} value The value to query.
* @returns {string} Returns the raw `toStringTag`.
*/
function getRawTag(value) {
var isOwn = hasOwnProperty.call(value, symToStringTag),
tag = value[symToStringTag];
try {
value[symToStringTag] = undefined;
var unmasked = true;
} catch (e) {}
var result = nativeObjectToString.call(value);
if (unmasked) {
if (isOwn) {
value[symToStringTag] = tag;
} else {
delete value[symToStringTag];
}
}
return result;
}
var _getRawTag = getRawTag;
/** Used for built-in method references. */
var objectProto$1 = Object.prototype;
/**
* Used to resolve the
* [`toStringTag`](http://ecma-international.org/ecma-262/7.0/#sec-object.prototype.tostring)
* of values.
*/
var nativeObjectToString$1 = objectProto$1.toString;
/**
* Converts `value` to a string using `Object.prototype.toString`.
*
* @private
* @param {*} value The value to convert.
* @returns {string} Returns the converted string.
*/
function objectToString(value) {
return nativeObjectToString$1.call(value);
}
var _objectToString = objectToString;
/** `Object#toString` result references. */
var nullTag = '[object Null]',
undefinedTag = '[object Undefined]';
/** Built-in value references. */
var symToStringTag$1 = _Symbol ? _Symbol.toStringTag : undefined;
/**
* The base implementation of `getTag` without fallbacks for buggy environments.
*
* @private
* @param {*} value The value to query.
* @returns {string} Returns the `toStringTag`.
*/
function baseGetTag(value) {
if (value == null) {
return value === undefined ? undefinedTag : nullTag;
}
return (symToStringTag$1 && symToStringTag$1 in Object(value))
? _getRawTag(value)
: _objectToString(value);
}
var _baseGetTag = baseGetTag;
/**
* Checks if `value` is the
* [language type](http://www.ecma-international.org/ecma-262/7.0/#sec-ecmascript-language-types)
* of `Object`. (e.g. arrays, functions, objects, regexes, `new Number(0)`, and `new String('')`)
*
* @static
* @memberOf _
* @since 0.1.0
* @category Lang
* @param {*} value The value to check.
* @returns {boolean} Returns `true` if `value` is an object, else `false`.
* @example
*
* _.isObject({});
* // => true
*
* _.isObject([1, 2, 3]);
* // => true
*
* _.isObject(_.noop);
* // => true
*
* _.isObject(null);
* // => false
*/
function isObject(value) {
var type = typeof value;
return value != null && (type == 'object' || type == 'function');
}
var isObject_1 = isObject;
/** `Object#toString` result references. */
var asyncTag = '[object AsyncFunction]',
funcTag = '[object Function]',
genTag = '[object GeneratorFunction]',
proxyTag = '[object Proxy]';
/**
* Checks if `value` is classified as a `Function` object.
*
* @static
* @memberOf _
* @since 0.1.0
* @category Lang
* @param {*} value The value to check.
* @returns {boolean} Returns `true` if `value` is a function, else `false`.
* @example
*
* _.isFunction(_);
* // => true
*
* _.isFunction(/abc/);
* // => false
*/
function isFunction(value) {
if (!isObject_1(value)) {
return false;
}
// The use of `Object#toString` avoids issues with the `typeof` operator
// in Safari 9 which returns 'object' for typed arrays and other constructors.
var tag = _baseGetTag(value);
return tag == funcTag || tag == genTag || tag == asyncTag || tag == proxyTag;
}
var isFunction_1 = isFunction;
/** Used to detect overreaching core-js shims. */
var coreJsData = _root['__core-js_shared__'];
var _coreJsData = coreJsData;
/** Used to detect methods masquerading as native. */
var maskSrcKey = (function() {
var uid = /[^.]+$/.exec(_coreJsData && _coreJsData.keys && _coreJsData.keys.IE_PROTO || '');
return uid ? ('Symbol(src)_1.' + uid) : '';
}());
/**
* Checks if `func` has its source masked.
*
* @private
* @param {Function} func The function to check.
* @returns {boolean} Returns `true` if `func` is masked, else `false`.
*/
function isMasked(func) {
return !!maskSrcKey && (maskSrcKey in func);
}
var _isMasked = isMasked;
/** Used for built-in method references. */
var funcProto = Function.prototype;
/** Used to resolve the decompiled source of functions. */
var funcToString = funcProto.toString;
/**
* Converts `func` to its source code.
*
* @private
* @param {Function} func The function to convert.
* @returns {string} Returns the source code.
*/
function toSource(func) {
if (func != null) {
try {
return funcToString.call(func);
} catch (e) {}
try {
return (func + '');
} catch (e) {}
}
return '';
}
var _toSource = toSource;
/**
* Used to match `RegExp`
* [syntax characters](http://ecma-international.org/ecma-262/7.0/#sec-patterns).
*/
var reRegExpChar = /[\\^$.*+?()[\]{}|]/g;
/** Used to detect host constructors (Safari). */
var reIsHostCtor = /^\[object .+?Constructor\]$/;
/** Used for built-in method references. */
var funcProto$1 = Function.prototype,
objectProto$2 = Object.prototype;
/** Used to resolve the decompiled source of functions. */
var funcToString$1 = funcProto$1.toString;
/** Used to check objects for own properties. */
var hasOwnProperty$1 = objectProto$2.hasOwnProperty;
/** Used to detect if a method is native. */
var reIsNative = RegExp('^' +
funcToString$1.call(hasOwnProperty$1).replace(reRegExpChar, '\\$&')
.replace(/hasOwnProperty|(function).*?(?=\\\()| for .+?(?=\\\])/g, '$1.*?') + '$'
);
/**
* The base implementation of `_.isNative` without bad shim checks.
*
* @private
* @param {*} value The value to check.
* @returns {boolean} Returns `true` if `value` is a native function,
* else `false`.
*/
function baseIsNative(value) {
if (!isObject_1(value) || _isMasked(value)) {
return false;
}
var pattern = isFunction_1(value) ? reIsNative : reIsHostCtor;
return pattern.test(_toSource(value));
}
var _baseIsNative = baseIsNative;
/**
* Gets the value at `key` of `object`.
*
* @private
* @param {Object} [object] The object to query.
* @param {string} key The key of the property to get.
* @returns {*} Returns the property value.
*/
function getValue(object, key) {
return object == null ? undefined : object[key];
}
var _getValue = getValue;
/**
* Gets the native function at `key` of `object`.
*
* @private
* @param {Object} object The object to query.
* @param {string} key The key of the method to get.
* @returns {*} Returns the function if it's native, else `undefined`.
*/
function getNative(object, key) {
var value = _getValue(object, key);
return _baseIsNative(value) ? value : undefined;
}
var _getNative = getNative;
/* Built-in method references that are verified to be native. */
var Map = _getNative(_root, 'Map');
var _Map = Map;
/* Built-in method references that are verified to be native. */
var nativeCreate = _getNative(Object, 'create');
var _nativeCreate = nativeCreate;
/**
* Removes all key-value entries from the hash.
*
* @private
* @name clear
* @memberOf Hash
*/
function hashClear() {
this.__data__ = _nativeCreate ? _nativeCreate(null) : {};
this.size = 0;
}
var _hashClear = hashClear;
/**
* Removes `key` and its value from the hash.
*
* @private
* @name delete
* @memberOf Hash
* @param {Object} hash The hash to modify.
* @param {string} key The key of the value to remove.
* @returns {boolean} Returns `true` if the entry was removed, else `false`.
*/
function hashDelete(key) {
var result = this.has(key) && delete this.__data__[key];
this.size -= result ? 1 : 0;
return result;
}
var _hashDelete = hashDelete;
/** Used to stand-in for `undefined` hash values. */
var HASH_UNDEFINED = '__lodash_hash_undefined__';
/** Used for built-in method references. */
var objectProto$3 = Object.prototype;
/** Used to check objects for own properties. */
var hasOwnProperty$2 = objectProto$3.hasOwnProperty;
/**
* Gets the hash value for `key`.
*
* @private
* @name get
* @memberOf Hash
* @param {string} key The key of the value to get.
* @returns {*} Returns the entry value.
*/
function hashGet(key) {
var data = this.__data__;
if (_nativeCreate) {
var result = data[key];
return result === HASH_UNDEFINED ? undefined : result;
}
return hasOwnProperty$2.call(data, key) ? data[key] : undefined;
}
var _hashGet = hashGet;
/** Used for built-in method references. */
var objectProto$4 = Object.prototype;
/** Used to check objects for own properties. */
var hasOwnProperty$3 = objectProto$4.hasOwnProperty;
/**
* Checks if a hash value for `key` exists.
*
* @private
* @name has
* @memberOf Hash
* @param {string} key The key of the entry to check.
* @returns {boolean} Returns `true` if an entry for `key` exists, else `false`.
*/
function hashHas(key) {
var data = this.__data__;
return _nativeCreate ? (data[key] !== undefined) : hasOwnProperty$3.call(data, key);
}
var _hashHas = hashHas;
/** Used to stand-in for `undefined` hash values. */
var HASH_UNDEFINED$1 = '__lodash_hash_undefined__';
/**
* Sets the hash `key` to `value`.
*
* @private
* @name set
* @memberOf Hash
* @param {string} key The key of the va