handsfree
Version:
Quickly integrate face, hand, and/or pose tracking to your frontend projects in a snap ✨👌
1,043 lines (891 loc) • 320 kB
JavaScript
(function (global, factory) {
typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() :
typeof define === 'function' && define.amd ? define(factory) :
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, global.Handsfree = factory());
}(this, (function () { 'use strict';
class BaseModel {
constructor (handsfree, config) {
this.handsfree = handsfree;
this.config = config;
this.data = {};
// Whether we've loaded dependencies or not
this.dependenciesLoaded = false;
// Whether the model is enabled or not
this.enabled = config.enabled;
// Collection of plugins and gestures
this.plugins = [];
this.gestures = [];
this.gestureEstimator = null;
setTimeout(() => {
// Get data
const getData = this.getData;
this.getData = async () => {
let data = await getData.apply(this, arguments) || {};
data.gesture = this.getGesture();
this.runPlugins();
return data
};
// Get gesture
let getGesture = this.getGesture;
this.getGesture = () => {
if (!getGesture) {
getGesture = function () {};
}
return getGesture.apply(this, arguments)
};
}, 0);
}
// Implement in the model class
loadDependencies () {}
updateData () {}
updateGestureEstimator () {}
/**
* Enable model
* @param {*} handleLoad If true then it'll also attempt to load,
* otherwise you'll need to handle it yourself. This is mostly used internally
* to prevent the .update() method from double loading
*/
enable (handleLoad = true) {
this.handsfree.config[this.name] = this.config;
this.handsfree.config[this.name].enabled = this.enabled = true;
document.body.classList.add(`handsfree-model-${this.name}`);
if (handleLoad && !this.dependenciesLoaded) {
this.loadDependencies();
}
// Weboji uses a webgl context
if (this.name === 'weboji') {
this.handsfree.debug.$canvas.weboji.style.display = 'block';
}
}
disable () {
this.handsfree.config[this.name] = this.config;
this.handsfree.config[this.name].enabled = this.enabled = false;
document.body.classList.remove(`handsfree-model-${this.name}`);
setTimeout(() => {
// Weboji uses a webgl context so let's just hide it
if (this.name === 'weboji') {
this.handsfree.debug.$canvas.weboji.style.display = 'none';
} else {
this.handsfree.debug.context[this.name]?.clearRect && this.handsfree.debug.context[this.name].clearRect(0, 0, this.handsfree.debug.$canvas[this.name].width, this.handsfree.debug.$canvas[this.name].height);
}
// Stop if all models have been stopped
let hasRunningModels = Object.keys(this.handsfree.model).some(model => this.handsfree.model[model].enabled);
if (!hasRunningModels) {
this.handsfree.stop();
}
}, 0);
}
/**
* Loads a script and runs a callback
* @param {string} src The absolute path of the source file
* @param {*} callback The callback to call after the file is loaded
* @param {boolean} skip Whether to skip loading the dependency and just call the callback
*/
loadDependency (src, callback, skip = false) {
// Skip and run callback
if (skip) {
callback && callback();
return
}
// Inject script into DOM
const $script = document.createElement('script');
$script.async = true;
$script.onload = () => {
callback && callback();
};
$script.onerror = () => {
this.handsfree.emit('modelError', `Error loading ${src}`);
};
$script.src = src;
document.body.appendChild($script);
}
/**
* Run all the plugins attached to this model
*/
runPlugins () {
// Exit if no data
if (!this.data || (this.name === 'handpose' && !this.data.annotations)) {
return
}
if (Object.keys(this.data).length) {
this.plugins.forEach(name => {
this.handsfree.plugin[name].enabled && this.handsfree.plugin[name]?.onFrame(this.handsfree.data);
});
}
}
}
var commonjsGlobal = typeof globalThis !== 'undefined' ? globalThis : typeof window !== 'undefined' ? window : typeof global !== 'undefined' ? global : typeof self !== 'undefined' ? self : {};
function getDefaultExportFromCjs (x) {
return x && x.__esModule && Object.prototype.hasOwnProperty.call(x, 'default') ? x['default'] : x;
}
function createCommonjsModule(fn, basedir, module) {
return module = {
path: basedir,
exports: {},
require: function (path, base) {
return commonjsRequire(path, (base === undefined || base === null) ? module.path : base);
}
}, fn(module, module.exports), module.exports;
}
function commonjsRequire () {
throw new Error('Dynamic requires are not currently supported by @rollup/plugin-commonjs');
}
var fingerpose = createCommonjsModule(function (module, exports) {
!function(t,e){module.exports=e();}("undefined"!=typeof self?self:commonjsGlobal,(function(){return function(t){var e={};function n(r){if(e[r])return e[r].exports;var i=e[r]={i:r,l:!1,exports:{}};return t[r].call(i.exports,i,i.exports,n),i.l=!0,i.exports}return n.m=t,n.c=e,n.d=function(t,e,r){n.o(t,e)||Object.defineProperty(t,e,{enumerable:!0,get:r});},n.r=function(t){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0});},n.t=function(t,e){if(1&e&&(t=n(t)),8&e)return t;if(4&e&&"object"==typeof t&&t&&t.__esModule)return t;var r=Object.create(null);if(n.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:t}),2&e&&"string"!=typeof t)for(var i in t)n.d(r,i,function(e){return t[e]}.bind(null,i));return r},n.n=function(t){var e=t&&t.__esModule?function(){return t.default}:function(){return t};return n.d(e,"a",e),e},n.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},n.p="",n(n.s=0)}([function(t,e,n){n.r(e);var r={};function i(t){return (i="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}n.r(r),n.d(r,"VictoryGesture",(function(){return C})),n.d(r,"ThumbsUpGesture",(function(){return j}));var o={Thumb:0,Index:1,Middle:2,Ring:3,Pinky:4,all:[0,1,2,3,4],nameMapping:{0:"Thumb",1:"Index",2:"Middle",3:"Ring",4:"Pinky"},pointsMapping:{0:[[0,1],[1,2],[2,3],[3,4]],1:[[0,5],[5,6],[6,7],[7,8]],2:[[0,9],[9,10],[10,11],[11,12]],3:[[0,13],[13,14],[14,15],[15,16]],4:[[0,17],[17,18],[18,19],[19,20]]},getName:function(t){return void 0!==i(this.nameMapping[t])&&this.nameMapping[t]},getPoints:function(t){return void 0!==i(this.pointsMapping[t])&&this.pointsMapping[t]}},a={NoCurl:0,HalfCurl:1,FullCurl:2,nameMapping:{0:"No Curl",1:"Half Curl",2:"Full Curl"},getName:function(t){return void 0!==i(this.nameMapping[t])&&this.nameMapping[t]}},l={VerticalUp:0,VerticalDown:1,HorizontalLeft:2,HorizontalRight:3,DiagonalUpRight:4,DiagonalUpLeft:5,DiagonalDownRight:6,DiagonalDownLeft:7,nameMapping:{0:"Vertical Up",1:"Vertical Down",2:"Horizontal Left",3:"Horizontal Right",4:"Diagonal Up Right",5:"Diagonal Up Left",6:"Diagonal Down Right",7:"Diagonal Down Left"},getName:function(t){return void 0!==i(this.nameMapping[t])&&this.nameMapping[t]}};function u(t){if("undefined"==typeof Symbol||null==t[Symbol.iterator]){if(Array.isArray(t)||(t=function(t,e){if(!t)return;if("string"==typeof t)return c(t,e);var n=Object.prototype.toString.call(t).slice(8,-1);"Object"===n&&t.constructor&&(n=t.constructor.name);if("Map"===n||"Set"===n)return Array.from(n);if("Arguments"===n||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n))return c(t,e)}(t))){var e=0,n=function(){};return {s:n,n:function(){return e>=t.length?{done:!0}:{done:!1,value:t[e++]}},e:function(t){throw t},f:n}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var r,i,o=!0,a=!1;return {s:function(){r=t[Symbol.iterator]();},n:function(){var t=r.next();return o=t.done,t},e:function(t){a=!0,i=t;},f:function(){try{o||null==r.return||r.return();}finally{if(a)throw i}}}}function c(t,e){(null==e||e>t.length)&&(e=t.length);for(var n=0,r=new Array(e);n<e;n++)r[n]=t[n];return r}function f(t,e){var n=Object.keys(t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(t);e&&(r=r.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),n.push.apply(n,r);}return n}function s(t,e,n){return e in t?Object.defineProperty(t,e,{value:n,enumerable:!0,configurable:!0,writable:!0}):t[e]=n,t}function h(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,r.key,r);}}var d=function(){function t(e){!function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}(this,t),this.options=function(t){for(var e=1;e<arguments.length;e++){var n=null!=arguments[e]?arguments[e]:{};e%2?f(Object(n),!0).forEach((function(e){s(t,e,n[e]);})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(n)):f(Object(n)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(n,e));}));}return t}({},{HALF_CURL_START_LIMIT:60,NO_CURL_START_LIMIT:130,DISTANCE_VOTE_POWER:1.1,SINGLE_ANGLE_VOTE_POWER:.9,TOTAL_ANGLE_VOTE_POWER:1.6},{},e);}var e,n;return e=t,(n=[{key:"estimate",value:function(t){var e,n=[],r=[],i=u(o.all);try{for(i.s();!(e=i.n()).done;){var a,l=e.value,c=o.getPoints(l),f=[],s=[],h=u(c);try{for(h.s();!(a=h.n()).done;){var d=a.value,p=t[d[0]],y=t[d[1]],g=this.getSlopes(p,y),v=g[0],m=g[1];f.push(v),s.push(m);}}catch(t){h.e(t);}finally{h.f();}n.push(f),r.push(s);}}catch(t){i.e(t);}finally{i.f();}var b,D=[],w=[],O=u(o.all);try{for(O.s();!(b=O.n()).done;){var M=b.value,S=M==o.Thumb?1:0,T=o.getPoints(M),C=t[T[S][0]],R=t[T[S+1][1]],A=t[T[3][1]],L=this.estimateFingerCurl(C,R,A),_=this.calculateFingerDirection(C,R,A,n[M].slice(S));D[M]=L,w[M]=_;}}catch(t){O.e(t);}finally{O.f();}return {curls:D,directions:w}}},{key:"getSlopes",value:function(t,e){var n=this.calculateSlope(t[0],t[1],e[0],e[1]);return 2==t.length?n:[n,this.calculateSlope(t[1],t[2],e[1],e[2])]}},{key:"angleOrientationAt",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:1,n=0,r=0,i=0;return t>=75&&t<=105?n=1*e:t>=25&&t<=155?r=1*e:i=1*e,[n,r,i]}},{key:"estimateFingerCurl",value:function(t,e,n){var r=t[0]-e[0],i=t[0]-n[0],o=e[0]-n[0],l=t[1]-e[1],u=t[1]-n[1],c=e[1]-n[1],f=t[2]-e[2],s=t[2]-n[2],h=e[2]-n[2],d=Math.sqrt(r*r+l*l+f*f),p=Math.sqrt(i*i+u*u+s*s),y=Math.sqrt(o*o+c*c+h*h),g=(y*y+d*d-p*p)/(2*y*d);g>1?g=1:g<-1&&(g=-1);var v=Math.acos(g);return (v=57.2958*v%180)>this.options.NO_CURL_START_LIMIT?a.NoCurl:v>this.options.HALF_CURL_START_LIMIT?a.HalfCurl:a.FullCurl}},{key:"estimateHorizontalDirection",value:function(t,e,n,r){return r==Math.abs(t)?t>0?l.HorizontalLeft:l.HorizontalRight:r==Math.abs(e)?e>0?l.HorizontalLeft:l.HorizontalRight:n>0?l.HorizontalLeft:l.HorizontalRight}},{key:"estimateVerticalDirection",value:function(t,e,n,r){return r==Math.abs(t)?t<0?l.VerticalDown:l.VerticalUp:r==Math.abs(e)?e<0?l.VerticalDown:l.VerticalUp:n<0?l.VerticalDown:l.VerticalUp}},{key:"estimateDiagonalDirection",value:function(t,e,n,r,i,o,a,u){var c=this.estimateVerticalDirection(t,e,n,r),f=this.estimateHorizontalDirection(i,o,a,u);return c==l.VerticalUp?f==l.HorizontalLeft?l.DiagonalUpLeft:l.DiagonalUpRight:f==l.HorizontalLeft?l.DiagonalDownLeft:l.DiagonalDownRight}},{key:"calculateFingerDirection",value:function(t,e,n,r){var i=t[0]-e[0],o=t[0]-n[0],a=e[0]-n[0],l=t[1]-e[1],c=t[1]-n[1],f=e[1]-n[1],s=Math.max(Math.abs(i),Math.abs(o),Math.abs(a)),h=Math.max(Math.abs(l),Math.abs(c),Math.abs(f)),d=0,p=0,y=0,g=h/(s+1e-5);g>1.5?d+=this.options.DISTANCE_VOTE_POWER:g>.66?p+=this.options.DISTANCE_VOTE_POWER:y+=this.options.DISTANCE_VOTE_POWER;var v=Math.sqrt(i*i+l*l),m=Math.sqrt(o*o+c*c),b=Math.sqrt(a*a+f*f),D=Math.max(v,m,b),w=t[0],O=t[1],M=n[0],S=n[1];D==v?(M=n[0],S=n[1]):D==b&&(w=e[0],O=e[1]);var T=[w,O],C=[M,S],R=this.getSlopes(T,C),A=this.angleOrientationAt(R,this.options.TOTAL_ANGLE_VOTE_POWER);d+=A[0],p+=A[1],y+=A[2];var L,_=u(r);try{for(_.s();!(L=_.n()).done;){var j=L.value,E=this.angleOrientationAt(j,this.options.SINGLE_ANGLE_VOTE_POWER);d+=E[0],p+=E[1],y+=E[2];}}catch(t){_.e(t);}finally{_.f();}return d==Math.max(d,p,y)?this.estimateVerticalDirection(c,l,f,h):y==Math.max(p,y)?this.estimateHorizontalDirection(o,i,a,s):this.estimateDiagonalDirection(c,l,f,h,o,i,a,s)}},{key:"calculateSlope",value:function(t,e,n,r){var i=(e-r)/(t-n),o=180*Math.atan(i)/Math.PI;return o<=0?o=-o:o>0&&(o=180-o),o}}])&&h(e.prototype,n),t}();function p(t){if("undefined"==typeof Symbol||null==t[Symbol.iterator]){if(Array.isArray(t)||(t=function(t,e){if(!t)return;if("string"==typeof t)return y(t,e);var n=Object.prototype.toString.call(t).slice(8,-1);"Object"===n&&t.constructor&&(n=t.constructor.name);if("Map"===n||"Set"===n)return Array.from(n);if("Arguments"===n||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n))return y(t,e)}(t))){var e=0,n=function(){};return {s:n,n:function(){return e>=t.length?{done:!0}:{done:!1,value:t[e++]}},e:function(t){throw t},f:n}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var r,i,o=!0,a=!1;return {s:function(){r=t[Symbol.iterator]();},n:function(){var t=r.next();return o=t.done,t},e:function(t){a=!0,i=t;},f:function(){try{o||null==r.return||r.return();}finally{if(a)throw i}}}}function y(t,e){(null==e||e>t.length)&&(e=t.length);for(var n=0,r=new Array(e);n<e;n++)r[n]=t[n];return r}function g(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function v(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,r.key,r);}}var m=function(){function t(e){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};g(this,t),this.estimator=new d(n),this.gestures=e;}var e,n;return e=t,(n=[{key:"estimate",value:function(t,e){var n,r=[],i=this.estimator.estimate(t),u=[],c=p(o.all);try{for(c.s();!(n=c.n()).done;){var f=n.value;u.push([o.getName(f),a.getName(i.curls[f]),l.getName(i.directions[f])]);}}catch(t){c.e(t);}finally{c.f();}var s,h=p(this.gestures);try{for(h.s();!(s=h.n()).done;){var d=s.value,y=d.matchAgainst(i.curls,i.directions);y>=e&&r.push({name:d.name,confidence:y});}}catch(t){h.e(t);}finally{h.f();}return {poseData:u,gestures:r}}}])&&v(e.prototype,n),t}();function b(t,e){return function(t){if(Array.isArray(t))return t}(t)||function(t,e){if("undefined"==typeof Symbol||!(Symbol.iterator in Object(t)))return;var n=[],r=!0,i=!1,o=void 0;try{for(var a,l=t[Symbol.iterator]();!(r=(a=l.next()).done)&&(n.push(a.value),!e||n.length!==e);r=!0);}catch(t){i=!0,o=t;}finally{try{r||null==l.return||l.return();}finally{if(i)throw o}}return n}(t,e)||w(t,e)||function(){throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}()}function D(t){if("undefined"==typeof Symbol||null==t[Symbol.iterator]){if(Array.isArray(t)||(t=w(t))){var e=0,n=function(){};return {s:n,n:function(){return e>=t.length?{done:!0}:{done:!1,value:t[e++]}},e:function(t){throw t},f:n}}throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}var r,i,o=!0,a=!1;return {s:function(){r=t[Symbol.iterator]();},n:function(){var t=r.next();return o=t.done,t},e:function(t){a=!0,i=t;},f:function(){try{o||null==r.return||r.return();}finally{if(a)throw i}}}}function w(t,e){if(t){if("string"==typeof t)return O(t,e);var n=Object.prototype.toString.call(t).slice(8,-1);return "Object"===n&&t.constructor&&(n=t.constructor.name),"Map"===n||"Set"===n?Array.from(n):"Arguments"===n||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)?O(t,e):void 0}}function O(t,e){(null==e||e>t.length)&&(e=t.length);for(var n=0,r=new Array(e);n<e;n++)r[n]=t[n];return r}function M(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,r.key,r);}}var S=function(){function t(e){!function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}(this,t),this.name=e,this.curls={},this.directions={},this.weights=[1,1,1,1,1],this.weightsRelative=[1,1,1,1,1];}var e,n;return e=t,(n=[{key:"addCurl",value:function(t,e,n){void 0===this.curls[t]&&(this.curls[t]=[]),this.curls[t].push([e,n]);}},{key:"addDirection",value:function(t,e,n){void 0===this.directions[t]&&(this.directions[t]=[]),this.directions[t].push([e,n]);}},{key:"setWeight",value:function(t,e){this.weights[t]=e;var n=this.weights.reduce((function(t,e){return t+e}),0);this.weightsRelative=this.weights.map((function(t){return 5*t/n}));}},{key:"matchAgainst",value:function(t,e){var n=0;for(var r in t){var i=t[r],o=this.curls[r];if(void 0!==o){var a,l=D(o);try{for(l.s();!(a=l.n()).done;){var u=b(a.value,2),c=u[0],f=u[1];if(i==c){n+=f*this.weightsRelative[r];break}}}catch(t){l.e(t);}finally{l.f();}}else n+=this.weightsRelative[r];}for(var s in e){var h=e[s],d=this.directions[s];if(void 0!==d){var p,y=D(d);try{for(y.s();!(p=y.n()).done;){var g=b(p.value,2),v=g[0],m=g[1];if(h==v){n+=m*this.weightsRelative[s];break}}}catch(t){y.e(t);}finally{y.f();}}else n+=this.weightsRelative[s];}return n}}])&&M(e.prototype,n),t}(),T=new S("victory");T.addCurl(o.Thumb,a.HalfCurl,.5),T.addCurl(o.Thumb,a.NoCurl,.5),T.addDirection(o.Thumb,l.VerticalUp,1),T.addDirection(o.Thumb,l.DiagonalUpLeft,1),T.addCurl(o.Index,a.NoCurl,1),T.addDirection(o.Index,l.VerticalUp,.75),T.addDirection(o.Index,l.DiagonalUpLeft,1),T.addCurl(o.Middle,a.NoCurl,1),T.addDirection(o.Middle,l.VerticalUp,1),T.addDirection(o.Middle,l.DiagonalUpLeft,.75),T.addCurl(o.Ring,a.FullCurl,1),T.addDirection(o.Ring,l.VerticalUp,.2),T.addDirection(o.Ring,l.DiagonalUpLeft,1),T.addDirection(o.Ring,l.HorizontalLeft,.2),T.addCurl(o.Pinky,a.FullCurl,1),T.addDirection(o.Pinky,l.VerticalUp,.2),T.addDirection(o.Pinky,l.DiagonalUpLeft,1),T.addDirection(o.Pinky,l.HorizontalLeft,.2),T.setWeight(o.Index,2),T.setWeight(o.Middle,2);var C=T,R=new S("thumbs_up");R.addCurl(o.Thumb,a.NoCurl,1),R.addDirection(o.Thumb,l.VerticalUp,1),R.addDirection(o.Thumb,l.DiagonalUpLeft,.25),R.addDirection(o.Thumb,l.DiagonalUpRight,.25);for(var A=0,L=[o.Index,o.Middle,o.Ring,o.Pinky];A<L.length;A++){var _=L[A];R.addCurl(_,a.FullCurl,1),R.addDirection(_,l.HorizontalLeft,1),R.addDirection(_,l.HorizontalRight,1);}var j=R;e.default={GestureEstimator:m,GestureDescription:S,Finger:o,FingerCurl:a,FingerDirection:l,Gestures:r};}]).default}));
});
var fingerpose$1 = /*@__PURE__*/getDefaultExportFromCjs(fingerpose);
class HandsModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'hands';
this.palmPoints = [0, 5, 9, 13, 17];
this.gestureEstimator = new fingerpose$1.GestureEstimator([]);
}
loadDependencies (callback) {
// Just load utils on client
if (this.handsfree.config.isClient) {
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, () => {
this.onWarmUp(callback);
}, !!window.drawConnectors);
return
}
// Load hands
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/hands/hands.js`, () => {
// Configure model
this.api = new window.Hands({locateFile: file => {
return `${this.handsfree.config.assetsPath}/@mediapipe/hands/${file}`
}});
this.api.setOptions(this.handsfree.config.hands);
this.api.onResults(results => this.dataReceived(results));
// Load the media stream
this.handsfree.getUserMedia(() => {
// Warm up before using in loop
if (!this.handsfree.mediapipeWarmups.isWarmingUp) {
this.warmUp(callback);
} else {
this.handsfree.on('mediapipeWarmedUp', () => {
if (!this.handsfree.mediapipeWarmups.isWarmingUp && !this.handsfree.mediapipeWarmups[this.name]) {
this.warmUp(callback);
}
});
}
});
// Load the hands camera module
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, null, !!window.drawConnectors);
});
}
/**
* Warms up the model
*/
warmUp (callback) {
this.handsfree.mediapipeWarmups[this.name] = true;
this.handsfree.mediapipeWarmups.isWarmingUp = true;
this.api.send({image: this.handsfree.debug.$video}).then(() => {
this.handsfree.mediapipeWarmups.isWarmingUp = false;
this.onWarmUp(callback);
});
}
/**
* Called after the model has been warmed up
* - If we don't do this there will be too many initial hits and cause an error
*/
onWarmUp (callback) {
this.dependenciesLoaded = true;
document.body.classList.add('handsfree-model-hands');
this.handsfree.emit('modelReady', this);
this.handsfree.emit('handsModelReady', this);
this.handsfree.emit('mediapipeWarmedUp', this);
callback && callback(this);
}
/**
* Get data
*/
async getData () {
this.dependenciesLoaded && await this.api.send({image: this.handsfree.debug.$video});
return this.data
}
// Called through this.api.onResults
dataReceived (results) {
// Get center of palm
if (results.multiHandLandmarks) {
results = this.getCenterOfPalm(results);
}
// Force handedness
results = this.forceHandedness(results);
// Update and debug
this.data = results;
this.handsfree.data.hands = results;
if (this.handsfree.isDebugging) {
this.debug(results);
}
}
/**
* Forces the hands to always be in the same index
*/
forceHandedness (results) {
// Empty landmarks
results.landmarks = [[], [], [], []];
results.landmarksVisible = [false, false, false, false];
if (!results.multiHandLandmarks) {
return results
}
// Store landmarks in the correct index
results.multiHandLandmarks.forEach((landmarks, n) => {
let hand;
if (n < 2) {
hand = results.multiHandedness[n].label === 'Right' ? 0 : 1;
} else {
hand = results.multiHandedness[n].label === 'Right' ? 2 : 3;
}
results.landmarks[hand] = landmarks;
results.landmarksVisible[hand] = true;
});
return results
}
/**
* Calculates the center of the palm
*/
getCenterOfPalm (results) {
results.multiHandLandmarks.forEach((hand, n) => {
let x = 0;
let y = 0;
this.palmPoints.forEach(i => {
x += hand[i].x;
y += hand[i].y;
});
x /= this.palmPoints.length;
y /= this.palmPoints.length;
results.multiHandLandmarks[n][21] = {x, y};
});
return results
}
/**
* Debugs the hands model
*/
debug (results) {
// Bail if drawing helpers haven't loaded
if (typeof drawConnectors === 'undefined') return
// Clear the canvas
this.handsfree.debug.context.hands.clearRect(0, 0, this.handsfree.debug.$canvas.hands.width, this.handsfree.debug.$canvas.hands.height);
// Draw skeletons
if (results.multiHandLandmarks) {
for (const landmarks of results.multiHandLandmarks) {
drawConnectors(this.handsfree.debug.context.hands, landmarks, HAND_CONNECTIONS, {color: '#00FF00', lineWidth: 5});
drawLandmarks(this.handsfree.debug.context.hands, landmarks, {color: '#FF0000', lineWidth: 2});
}
}
}
/**
* Updates the gesture estimator
*/
updateGestureEstimator () {
const activeGestures = [];
const gestureDescriptions = [];
// Build the gesture descriptions
this.gestures.forEach(name => {
if (!this.handsfree.gesture[name].enabled) return
activeGestures.push(name);
// Loop through the description and compile it
if (!this.handsfree.gesture[name].compiledDescription && this.handsfree.gesture[name].enabled) {
const description = new fingerpose$1.GestureDescription(name);
this.handsfree.gesture[name].description.forEach(pose => {
// Build the description
switch (pose[0]) {
case 'addCurl':
description[pose[0]](
fingerpose$1.Finger[pose[1]],
fingerpose$1.FingerCurl[pose[2]],
pose[3]
);
break
case 'addDirection':
description[pose[0]](
fingerpose$1.Finger[pose[1]],
fingerpose$1.FingerDirection[pose[2]],
pose[3]
);
break
case 'setWeight':
description[pose[0]](
fingerpose$1.Finger[pose[1]],
pose[2]
);
break
}
});
this.handsfree.gesture[name].compiledDescription = description;
}
});
// Create the gesture estimator
activeGestures.forEach(gesture => {
gestureDescriptions.push(this.handsfree.gesture[gesture].compiledDescription);
});
if (activeGestures.length) {
this.gestureEstimator = new fingerpose$1.GestureEstimator(gestureDescriptions);
}
}
/**
* Gets current gesture
*/
getGesture () {
let gestures = [null, null, null, null];
this.data.landmarks.forEach((landmarksObj, hand) => {
if (this.data.landmarksVisible[hand]) {
// Convert object to array
const landmarks = [];
for (let i = 0; i < 21; i++) {
landmarks.push([landmarksObj[i].x * window.outerWidth, landmarksObj[i].y * window.outerHeight, 0]);
}
// Estimate
const estimate = this.gestureEstimator.estimate(landmarks, 7.5);
if (estimate.gestures.length) {
gestures[hand] = estimate.gestures.reduce((p, c) => {
const requiredConfidence = this.handsfree.gesture[c.name].confidence;
return (c.confidence >= requiredConfidence && c.confidence > p.confidence) ? c : p
});
} else {
gestures[hand] = {
name: '',
confidence: 0
};
}
// Must pass confidence
if (gestures[hand].name) {
const requiredConfidence = this.handsfree.gesture[gestures[hand].name].confidence;
if (gestures[hand].confidence < requiredConfidence) {
gestures[hand] = {
name: '',
confidence: 0
};
}
}
gestures[hand].pose = estimate.poseData;
}
});
return gestures
}
}
class FacemeshModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'facemesh';
this.isWarmedUp = false;
}
loadDependencies (callback) {
// Just load utils on client
if (this.handsfree.config.isClient) {
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, () => {
this.onWarmUp(callback);
}, !!window.drawConnectors);
return
}
// Load facemesh
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/face_mesh/face_mesh.js`, () => {
// Configure model
this.api = new window.FaceMesh({locateFile: file => {
return `${this.handsfree.config.assetsPath}/@mediapipe/face_mesh/${file}`
}});
this.api.setOptions(this.handsfree.config.facemesh);
this.api.onResults(results => this.dataReceived(results));
// Load the media stream
this.handsfree.getUserMedia(() => {
// Warm up before using in loop
if (!this.handsfree.mediapipeWarmups.isWarmingUp) {
this.warmUp(callback);
} else {
this.handsfree.on('mediapipeWarmedUp', () => {
if (!this.handsfree.mediapipeWarmups.isWarmingUp && !this.handsfree.mediapipeWarmups[this.name]) {
this.warmUp(callback);
}
});
}
});
// Load the hands camera module
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, null, !!window.drawConnectors);
});
}
/**
* Warms up the model
*/
warmUp (callback) {
this.handsfree.mediapipeWarmups[this.name] = true;
this.handsfree.mediapipeWarmups.isWarmingUp = true;
this.api.send({image: this.handsfree.debug.$video}).then(() => {
this.handsfree.mediapipeWarmups.isWarmingUp = false;
this.onWarmUp(callback);
});
}
/**
* Called after the model has been warmed up
* - If we don't do this there will be too many initial hits and cause an error
*/
onWarmUp (callback) {
this.dependenciesLoaded = true;
document.body.classList.add('handsfree-model-facemesh');
this.handsfree.emit('modelReady', this);
this.handsfree.emit('facemeshModelReady', this);
this.handsfree.emit('mediapipeWarmedUp', this);
callback && callback(this);
}
/**
* Get data
*/
async getData () {
this.dependenciesLoaded && await this.api.send({image: this.handsfree.debug.$video});
}
// Called through this.api.onResults
dataReceived (results) {
this.data = results;
this.handsfree.data.facemesh = results;
if (this.handsfree.isDebugging) {
this.debug(results);
}
}
/**
* Debugs the facemesh model
*/
debug (results) {
// Bail if drawing helpers haven't loaded
if (typeof drawConnectors === 'undefined') return
this.handsfree.debug.context.facemesh.clearRect(0, 0, this.handsfree.debug.$canvas.facemesh.width, this.handsfree.debug.$canvas.facemesh.height);
if (results.multiFaceLandmarks) {
for (const landmarks of results.multiFaceLandmarks) {
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_TESSELATION, {color: '#C0C0C070', lineWidth: 1});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_RIGHT_EYE, {color: '#FF3030'});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_RIGHT_EYEBROW, {color: '#FF3030'});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_LEFT_EYE, {color: '#30FF30'});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_LEFT_EYEBROW, {color: '#30FF30'});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_FACE_OVAL, {color: '#E0E0E0'});
drawConnectors(this.handsfree.debug.context.facemesh, landmarks, FACEMESH_LIPS, {color: '#E0E0E0'});
}
}
}
}
class PoseModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'pose';
// Without this the loading event will happen before the first frame
this.hasLoadedAndRun = false;
this.palmPoints = [0, 1, 2, 5, 9, 13, 17];
}
loadDependencies (callback) {
// Just load utils on client
if (this.handsfree.config.isClient) {
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, () => {
this.onWarmUp(callback);
}, !!window.drawConnectors);
return
}
// Load pose
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/pose/pose.js`, () => {
this.api = new window.Pose({locateFile: file => {
return `${this.handsfree.config.assetsPath}/@mediapipe/pose/${file}`
}});
this.api.setOptions(this.handsfree.config.pose);
this.api.onResults(results => this.dataReceived(results));
// Load the media stream
this.handsfree.getUserMedia(() => {
// Warm up before using in loop
if (!this.handsfree.mediapipeWarmups.isWarmingUp) {
this.warmUp(callback);
} else {
this.handsfree.on('mediapipeWarmedUp', () => {
if (!this.handsfree.mediapipeWarmups.isWarmingUp && !this.handsfree.mediapipeWarmups[this.name]) {
this.warmUp(callback);
}
});
}
});
// Load the hands camera module
this.loadDependency(`${this.handsfree.config.assetsPath}/@mediapipe/drawing_utils.js`, null, !!window.drawConnectors);
});
}
/**
* Warms up the model
*/
warmUp (callback) {
this.handsfree.mediapipeWarmups[this.name] = true;
this.handsfree.mediapipeWarmups.isWarmingUp = true;
this.api.send({image: this.handsfree.debug.$video}).then(() => {
this.handsfree.mediapipeWarmups.isWarmingUp = false;
this.onWarmUp(callback);
});
}
/**
* Called after the model has been warmed up
* - If we don't do this there will be too many initial hits and cause an error
*/
onWarmUp (callback) {
this.dependenciesLoaded = true;
document.body.classList.add('handsfree-model-pose');
this.handsfree.emit('modelReady', this);
this.handsfree.emit('poseModelReady', this);
this.handsfree.emit('mediapipeWarmedUp', this);
callback && callback(this);
}
/**
* Get data
*/
async getData () {
this.dependenciesLoaded && await this.api.send({image: this.handsfree.debug.$video});
}
// Called through this.api.onResults
dataReceived (results) {
this.data = results;
this.handsfree.data.pose = results;
if (this.handsfree.isDebugging) {
this.debug(results);
}
}
/**
* Debugs the pose model
*/
debug (results) {
this.handsfree.debug.context.pose.clearRect(0, 0, this.handsfree.debug.$canvas.pose.width, this.handsfree.debug.$canvas.pose.height);
if (results.poseLandmarks) {
drawConnectors(this.handsfree.debug.context.pose, results.poseLandmarks, POSE_CONNECTIONS, {color: '#00FF00', lineWidth: 4});
drawLandmarks(this.handsfree.debug.context.pose, results.poseLandmarks, {color: '#FF0000', lineWidth: 2});
}
}
}
/**
* 🚨 This model is not currently active
*/
class HandposeModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'handpose';
// Various THREE variables
this.three = {
scene: null,
camera: null,
renderer: null,
meshes: []
};
this.normalized = [];
// landmark indices that represent the palm
// 8 = Index finger tip
// 12 = Middle finger tip
this.palmPoints = [0, 1, 2, 5, 9, 13, 17];
this.gestureEstimator = new fingerpose$1.GestureEstimator([]);
}
loadDependencies (callback) {
this.loadDependency(`${this.handsfree.config.assetsPath}/three/three.min.js`, () => {
this.loadDependency(`${this.handsfree.config.assetsPath}/@tensorflow/tf-core.js`, () => {
this.loadDependency(`${this.handsfree.config.assetsPath}/@tensorflow/tf-converter.js`, () => {
this.loadDependency(`${this.handsfree.config.assetsPath}/@tensorflow/tf-backend-${this.handsfree.config.handpose.backend}.js`, () => {
this.loadDependency(`${this.handsfree.config.assetsPath}/@tensorflow-models/handpose/handpose.js`, () => {
this.handsfree.getUserMedia(async () => {
await window.tf.setBackend(this.handsfree.config.handpose.backend);
this.api = await handpose.load(this.handsfree.config.handpose.model);
this.setup3D();
callback && callback(this);
this.dependenciesLoaded = true;
this.handsfree.emit('modelReady', this);
this.handsfree.emit('handposeModelReady', this);
document.body.classList.add('handsfree-model-handpose');
});
});
});
});
}, !!window.tf);
}, !!window.THREE);
}
/**
* Runs inference and sets up other data
*/
async getData () {
if (!this.handsfree.debug.$video) return
const predictions = await this.api.estimateHands(this.handsfree.debug.$video);
this.handsfree.data.handpose = this.data = {
...predictions[0],
normalized: this.normalized,
meshes: this.three.meshes
};
if (predictions[0]) {
this.updateMeshes(this.data);
}
this.three.renderer.render(this.three.scene, this.three.camera);
return this.data
}
/**
* Sets up the 3D environment
*/
setup3D () {
// Setup Three
this.three = {
scene: new window.THREE.Scene(),
camera: new window.THREE.PerspectiveCamera(90, window.outerWidth / window.outerHeight, 0.1, 1000),
renderer: new THREE.WebGLRenderer({
alpha: true,
canvas: this.handsfree.debug.$canvas.handpose
}),
meshes: []
};
this.three.renderer.setSize(window.outerWidth, window.outerHeight);
this.three.camera.position.z = this.handsfree.debug.$video.videoWidth / 4;
this.three.camera.lookAt(new window.THREE.Vector3(0, 0, 0));
// Camera plane
this.three.screen = new window.THREE.Mesh(
new window.THREE.BoxGeometry(window.outerWidth, window.outerHeight, 1),
new window.THREE.MeshNormalMaterial()
);
this.three.screen.position.z = 300;
this.three.scene.add(this.three.screen);
// Camera raycaster
this.three.raycaster = new window.THREE.Raycaster();
this.three.arrow = new window.THREE.ArrowHelper(this.three.raycaster.ray.direction, this.three.raycaster.ray.origin, 300, 0xff0000);
this.three.scene.add(this.three.arrow);
// Create model representations (one for each keypoint)
for (let i = 0; i < 21; i++){
const {isPalm} = this.getLandmarkProperty(i);
const obj = new window.THREE.Object3D(); // a parent object to facilitate rotation/scaling
// we make each bone a cylindrical shape, but you can use your own models here too
const geometry = new window.THREE.CylinderGeometry(isPalm ? 5 : 10, 5, 1);
let material = new window.THREE.MeshNormalMaterial();
const mesh = new window.THREE.Mesh(geometry, material);
mesh.rotation.x = Math.PI / 2;
obj.add(mesh);
this.three.scene.add(obj);
this.three.meshes.push(obj);
// uncomment this to help identify joints
// if (i === 4) {
// mesh.material.transparent = true
// mesh.material.opacity = 0
// }
}
// Create center of palm
const obj = new window.THREE.Object3D();
const geometry = new window.THREE.CylinderGeometry(5, 5, 1);
let material = new window.THREE.MeshNormalMaterial();
const mesh = new window.THREE.Mesh(geometry, material);
mesh.rotation.x = Math.PI / 2;
this.three.centerPalmObj = obj;
obj.add(mesh);
this.three.scene.add(obj);
this.three.meshes.push(obj);
this.three.screen.visible = false;
}
// compute some metadata given a landmark index
// - is the landmark a palm keypoint or a finger keypoint?
// - what's the next landmark to connect to if we're drawing a bone?
getLandmarkProperty (i) {
const palms = [0, 1, 2, 5, 9, 13, 17]; //landmark indices that represent the palm
const idx = palms.indexOf(i);
const isPalm = idx != -1;
let next; // who to connect with?
if (!isPalm) { // connect with previous finger landmark if it's a finger landmark
next = i - 1;
}else { // connect with next palm landmark if it's a palm landmark
next = palms[(idx + 1) % palms.length];
}
return {isPalm, next}
}
/**
* update threejs object position and orientation from the detected hand pose
* threejs has a "scene" model, so we don't have to specify what to draw each frame,
* instead we put objects at right positions and threejs renders them all
* @param {*} hand
*/
updateMeshes (hand) {
for (let i = 0; i < this.three.meshes.length - 1 /* palmbase */; i++) {
const {next} = this.getLandmarkProperty(i);
const p0 = this.webcam2space(...hand.landmarks[i]); // one end of the bone
const p1 = this.webcam2space(...hand.landmarks[next]); // the other end of the bone
// compute the center of the bone (midpoint)
const mid = p0.clone().lerp(p1, 0.5);
this.three.meshes[i].position.set(mid.x, mid.y, mid.z);
this.normalized[i] = [
this.handsfree.normalize(p0.x, this.handsfree.debug.$video.videoWidth / -2, this.handsfree.debug.$video.videoWidth / 2),
this.handsfree.normalize(p0.y, this.handsfree.debug.$video.videoHeight / -2, this.handsfree.debug.$video.videoHeight / 2),
this.three.meshes[i].position.z
];
// compute the length of the bone
this.three.meshes[i].scale.z = p0.distanceTo(p1);
// compute orientation of the bone
this.three.meshes[i].lookAt(p1);
if (i === 8) {
this.three.arrow.position.set(mid.x, mid.y, mid.z);
const direction = new window.THREE.Vector3().subVectors(p0, mid);
this.three.arrow.setDirection(direction.normalize());
this.three.arrow.setLength(800);
this.three.arrow.direction = direction;
}
}
this.updateCenterPalmMesh(hand);
}
/**
* Update the palm
*/
updateCenterPalmMesh (hand) {
let points = [];
let mid = {
x: 0,
y: 0,
z: 0
};
// Get position for the palm
this.palmPoints.forEach((i, n) => {
points.push(this.webcam2space(...hand.landmarks[i]));
mid.x += points[n].x;
mid.y += points[n].y;
mid.z += points[n].z;
});
mid.x = mid.x / this.palmPoints.length;
mid.y = mid.y / this.palmPoints.length;
mid.z = mid.z / this.palmPoints.length;
this.three.centerPalmObj.position.set(mid.x, mid.y, mid.z);
this.three.centerPalmObj.scale.z = 10;
this.three.centerPalmObj.rotation.x = this.three.meshes[12].rotation.x - Math.PI / 2;
this.three.centerPalmObj.rotation.y = -this.three.meshes[12].rotation.y;
this.three.centerPalmObj.rotation.z = this.three.meshes[12].rotation.z;
}
// transform webcam coordinates to threejs 3d coordinates
webcam2space (x, y, z) {
return new window.THREE.Vector3(
(x-this.handsfree.debug.$video.videoWidth / 2),
-(y-this.handsfree.debug.$video.videoHeight / 2), // in threejs, +y is up
-z
)
}
/**
* Updates the gesture estimator
*/
updateGestureEstimator () {
const activeGestures = [];
const gestureDescriptions = [];
// Build the gesture descriptions
this.gestures.forEach(name => {
this.handsfree.gesture[name].enabled && activeGestures.push(name);
// Loop through the description and compile it
if (!this.handsfree.gesture[name].compiledDescription && this.handsfree.gesture[name].enabled) {
const description = new fingerpose$1.GestureDescription(name);
this.handsfree.gesture[name].description.forEach(pose => {
// Build the description
switch (pose[0]) {
case 'addCurl':
description[pose[0]](
fingerpose$1.Finger[pose[1]],
fingerpose$1.FingerCurl[pose[2]],
pose[3]
);
break
case 'addDirection':
description[pose[0]](
fingerpose$1.Finger[pose[1]],
fingerpose$1.FingerDirection[pose[2]],
pose[3]
);
break
case 'setWeight':
description[pose[0]](
fingerpose$1.Finger[pose[1]],
pose[2]
);
break
}
});
this.handsfree.gesture[name].compiledDescription = description;
}
});
// Create the gesture estimator
activeGestures.forEach(gesture => {
gestureDescriptions.push(this.handsfree.gesture[gesture].compiledDescription);
});
if (activeGestures.length) {
this.gestureEstimator = new fingerpose$1.GestureEstimator(gestureDescriptions);
}
}
/**
* Gets current gesture
*/
getGesture () {
let gesture = null;
if (this.data.landmarks && this.gestureEstimator) {
const estimate = this.gestureEstimator.estimate(this.data.landmarks, 7.5);
if (estimate.gestures.length) {
gesture = estimate.gestures.reduce((p, c) => {
return (p.confidence > c.confidence) ? p : c
});
}
}
return gesture
}
}
class WebojiModel extends BaseModel {
constructor (handsfree, config) {
super(handsfree, config);
this.name = 'weboji';
}
loadDependencies (callback) {
// Just load utils on client
if (this.handsfree.config.isClient) {
this.onReady(callback);
return
}
// Load weboji
this.loadDependency(`${this.handsfree.config.assetsPath}/jeeliz/jeelizFaceTransfer.js`, () => {
const url = this.handsfree.config.assetsPath + '/jeeliz/jeelizFaceTransferNNC.json';
this.api = window.JEEFACETRANSFERAPI;
fetch(url)
.then(model => model.json())
// Next, let's initialize the weboji tracker API
.then(model => {
this.api.init({
canvasId: `handsfree-canvas-weboji-${this.handsfree.id}`,
NNC: JSON.stringify(model),
videoSettings: this.handsfree.config.weboji.videoSettings,
callbackReady: () => this.onReady(callback)
});
})
.catch((ev) => {
console.log(ev);
console.error(`Couldn't load weboji tracking model at ${url}`);
this.handsfree.emit('modelError', ev);
});
});
}
onReady (callback) {
this.dependenciesLoaded = true;
this.handsfree.emit('modelReady', this);
this.handsfree.emit('webojiModelReady', this);
document.body.classList.add('handsfree-model-weboji');
callback && callback(this);
}
getData () {
// Core
this.data.rotation = this.api.get_rotationStabilized();
this.data.translation = this.api.get_positionScale();
this.data.morphs = this.api.get_morphTargetInfluencesStabilized();
// Helpers
this.data.state = this.getStates();
this.data.degree = this.getDegrees();
this.data.isDetected = this.api.is_detected();
this.handsfree.data.weboji = this.data;
return this.data
}
/**
* Helpers for getting degrees
*/
getDegrees () {
return [
this.data.rotation[0] * 180 / Math.PI,
this.data.rotation[1] * 180 / Math.PI,
this.data.rotation[2] * 180 / Math.PI
]
}
/**
* Sets some stateful helpers
*/
getStates() {
/**
* Handles extra calculations for weboji morphs
*/
const morphs = this.data.morphs;
const state = this.data.state || {};
// Smiles
state.smileRight =
morphs[0] > this.handsfree.config.weboji.morphs.threshold.smileRight;
state.smileLeft =
morphs[1] > this.handsfree.config.weboji.morphs.threshold.