@glyph-cat/ml-helpers/lib/es/index.mjs

Helper functions to use in conjunction with machine learning outputs such as mediapipe.

import{LazyValue as t,createEnumToStringConverter as s,reflect1D as n,getDistance2DByCoordinates as i,fullyEnumerate as e,degToRad as o,getAngleFromPointsIn3D as r,NumericDataSet as c,hasProperty as a,isInRange as h}from"@glyph-cat/swiss-army-knife";import{FilesetResolver as I,PoseLandmarker as E,FaceLandmarker as _,HandLandmarker as u,GestureRecognizer as f}from"@mediapipe/tasks-vision";import{SimpleStateManager as T,SimpleFiniteStateManager as l}from"cotton-box";var R,N,F,P,d,G,m;!function(t){t[t.CREATED=0]="CREATED",t[t.INITIALIZING=1]="INITIALIZING",t[t.STANDBY=2]="STANDBY",t[t.ACTIVE=3]="ACTIVE",t[t.DISPOSED=4]="DISPOSED"}(R||(R={}));class y{static t=!1;static i="/mediapipe/wasm";static get(){return this.i}static set(t){if(this.t)throw new Error("WASM path cannot be changed because the fileset has been loaded");this.i=t}}!function(t){t.detectForVideo="detectForVideo",t.recognizeForVideo="recognizeForVideo"}(N||(N={}));class H{videoElement;getTaskRunner;detectionMethodName;static o=new t(()=>(y.t=!0,I.forVisionTasks(y.get())));static async getVision(){return this.o.value}taskRunner;lastRequestedAnimationFrame;result;state;constructor(t,n,i,e,o,r){this.videoElement=t,this.getTaskRunner=i,this.detectionMethodName=e,this.initialize=this.initialize.bind(this),this.start=this.start.bind(this),this.stop=this.stop.bind(this),this.performAnalysis=this.performAnalysis.bind(this),this.getProcessedResult=this.getProcessedResult.bind(this),this.dispose=this.dispose.bind(this),this.result=new T(n),this.state=new l(R.CREATED,[[R.CREATED,R.INITIALIZING],[R.CREATED,R.DISPOSED],[R.INITIALIZING,R.STANDBY],[R.STANDBY,R.ACTIVE],[R.STANDBY,R.DISPOSED],[R.ACTIVE,R.STANDBY],[R.ACTIVE,R.DISPOSED]],{name:o,serializeState:s(R)}),r?.initializeImmediately&&this.initialize()}async initialize(){this.state.set(R.INITIALIZING),this.taskRunner=await this.getTaskRunner(),this.state.set(t=>t===R.INITIALIZING?R.STANDBY:t)}async start(){let t=!1;this.state.get()===R.INITIALIZING&&(t=!0,await this.state.wait(t=>t>R.INITIALIZING)),t&&this.state.get()!==R.STANDBY||(this.state.set(R.ACTIVE),this.lastRequestedAnimationFrame=requestAnimationFrame(this.performAnalysis))}async stop(){await this.state.wait(t=>t===R.CREATED||t>R.STANDBY),this.h(),this.state.get()!==R.DISPOSED&&this.state.set(R.STANDBY)}async dispose(){await this.state.wait(t=>t!==R.INITIALIZING),cancelAnimationFrame(this.lastRequestedAnimationFrame),this.result.dispose(),this.state.trySet(R.DISPOSED),this.state.dispose()}h(){cancelAnimationFrame(this.lastRequestedAnimationFrame)}async performAnalysis(){if(this.state.get()!==R.ACTIVE)return;const t=this.taskRunner[this.detectionMethodName](this.videoElement,performance.now()),s=this.getProcessedResult(t);s&&this.result.set(s),this.lastRequestedAnimationFrame=requestAnimationFrame(this.performAnalysis)}}class L extends H{constructor(t,s,n,i,e){super(t,s,n,N.detectForVideo,i,e)}}class M extends L{options;static I={};constructor(t,s,n){super(t,new Array(Object.keys(F).length/2).fill({x:0,y:0,z:0,visibility:0}),async()=>{const t=JSON.stringify(s);return M.I[t]||(M.I[t]=E.createFromOptions(await L.getVision(),{...s,numPoses:1})),M.I[t]},"OnePersonBodyPoseAnalyzer",n),this.options=n}getProcessedResult(t){const s=[];if(t.landmarks.length<=0)return;const i=t.landmarks[0],e=this.options?.flipHorizontally;for(const t of i)s.push({...t,x:e?n(t.x,.5):t.x,visibility:1});return s}}!function(t){t[t.NOSE=0]="NOSE",t[t.LEFT_EYE_INNER=1]="LEFT_EYE_INNER",t[t.LEFT_EYE=2]="LEFT_EYE",t[t.LEFT_EYE_OUTER=3]="LEFT_EYE_OUTER",t[t.RIGHT_EYE_INNER=4]="RIGHT_EYE_INNER",t[t.RIGHT_EYE=5]="RIGHT_EYE",t[t.RIGHT_EYE_OUTER=6]="RIGHT_EYE_OUTER",t[t.LEFT_EAR=7]="LEFT_EAR",t[t.RIGHT_EAR=8]="RIGHT_EAR",t[t.MOUTH_LEFT=9]="MOUTH_LEFT",t[t.MOUTH_RIGHT=10]="MOUTH_RIGHT",t[t.LEFT_SHOULDER=11]="LEFT_SHOULDER",t[t.RIGHT_SHOULDER=12]="RIGHT_SHOULDER",t[t.LEFT_ELBOW=13]="LEFT_ELBOW",t[t.RIGHT_ELBOW=14]="RIGHT_ELBOW",t[t.LEFT_WRIST=15]="LEFT_WRIST",t[t.RIGHT_WRIST=16]="RIGHT_WRIST",t[t.LEFT_PINKY=17]="LEFT_PINKY",t[t.RIGHT_PINKY=18]="RIGHT_PINKY",t[t.LEFT_INDEX=19]="LEFT_INDEX",t[t.RIGHT_INDEX=20]="RIGHT_INDEX",t[t.LEFT_THUMB=21]="LEFT_THUMB",t[t.RIGHT_THUMB=22]="RIGHT_THUMB",t[t.LEFT_HIP=23]="LEFT_HIP",t[t.RIGHT_HIP=24]="RIGHT_HIP",t[t.LEFT_KNEE=25]="LEFT_KNEE",t[t.RIGHT_KNEE=26]="RIGHT_KNEE",t[t.LEFT_ANKLE=27]="LEFT_ANKLE",t[t.RIGHT_ANKLE=28]="RIGHT_ANKLE",t[t.LEFT_HEEL=29]="LEFT_HEEL",t[t.RIGHT_HEEL=30]="RIGHT_HEEL",t[t.LEFT_FOOT_INDEX=31]="LEFT_FOOT_INDEX",t[t.RIGHT_FOOT_INDEX=32]="RIGHT_FOOT_INDEX"}(F||(F={}));class p extends L{options;static I={};constructor(t,s,n){super(t,new Array(478).fill({x:0,y:0,z:0,visibility:0}),async()=>{const t=JSON.stringify(s);return p.I[t]||(p.I[t]=_.createFromOptions(await L.getVision(),{...s,numFaces:1})),p.I[t]},"OnePersonFaceMeshAnalyzer",n),this.options=n}getProcessedResult(t){const s=[];if(t.faceLandmarks?.length<=0)return;const i=this.options?.flipHorizontally,e=t.faceLandmarks[0];for(const t of e)s.push({...t,x:i?n(t.x,.5):t.x,visibility:1});return s}}function D(t,s,n){return i(t,s)<i(t,n)?"L":"R"}class O extends L{bodyPoseAnalyzer;options;static I={};constructor(t,s,n){super(t.videoElement,{},async()=>{const t=JSON.stringify(s);return O.I[t]||(O.I[t]=u.createFromOptions(await L.getVision(),{...s,numHands:2})),O.I[t]},"OnePersonHandPoseAnalyzer",n),this.bodyPoseAnalyzer=t,this.options=n}getProcessedResult(t){const s=this.bodyPoseAnalyzer.result.get(),i={},e=this.options?.flipHorizontally;for(const o of t.landmarks){const t=[];let r;for(let s=0;s<o.length;s++){const i=o[s],c={...i,x:e?n(i.x,.5):i.x,visibility:1};s===P.WRIST&&(r=c),t.push(c)}i[D(r,s[F.LEFT_WRIST],s[F.RIGHT_WRIST])]=t}return i}}!function(t){t[t.WRIST=0]="WRIST",t[t.THUMB_CMC=1]="THUMB_CMC",t[t.THUMB_MCP=2]="THUMB_MCP",t[t.THUMB_IP=3]="THUMB_IP",t[t.THUMB_TIP=4]="THUMB_TIP",t[t.INDEX_FINGER_MCP=5]="INDEX_FINGER_MCP",t[t.INDEX_FINGER_PIP=6]="INDEX_FINGER_PIP",t[t.INDEX_FINGER_DIP=7]="INDEX_FINGER_DIP",t[t.INDEX_FINGER_TIP=8]="INDEX_FINGER_TIP",t[t.MIDDLE_FINGER_MCP=9]="MIDDLE_FINGER_MCP",t[t.MIDDLE_FINGER_PIP=10]="MIDDLE_FINGER_PIP",t[t.MIDDLE_FINGER_DIP=11]="MIDDLE_FINGER_DIP",t[t.MIDDLE_FINGER_TIP=12]="MIDDLE_FINGER_TIP",t[t.RING_FINGER_MCP=13]="RING_FINGER_MCP",t[t.RING_FINGER_PIP=14]="RING_FINGER_PIP",t[t.RING_FINGER_DIP=15]="RING_FINGER_DIP",t[t.RING_FINGER_TIP=16]="RING_FINGER_TIP",t[t.PINKY_FINGER_MCP=17]="PINKY_FINGER_MCP",t[t.PINKY_FINGER_PIP=18]="PINKY_FINGER_PIP",t[t.PINKY_FINGER_DIP=19]="PINKY_FINGER_DIP",t[t.PINKY_FINGER_TIP=20]="PINKY_FINGER_TIP"}(P||(P={}));class A extends H{bodyPoseAnalyzer;static I={};constructor(t,s,n){super(t.videoElement,{},async()=>{const t=JSON.stringify(s);return A.I[t]||(A.I[t]=f.createFromOptions(await H.getVision(),{...s,numHands:2})),A.I[t]},N.recognizeForVideo,"OnePersonHandGestureAnalyzer",n),this.bodyPoseAnalyzer=t}getProcessedResult(t){const s=this.bodyPoseAnalyzer.result.get(),i={};for(const e in t.landmarks){const o=t.landmarks[e],r=[];let c;for(let t=0;t<o.length;t++){const s=o[t],i={...s,x:n(s.x,.5),visibility:1};t===P.WRIST&&(c=i),r.push(i)}i[D(c,s[F.LEFT_WRIST],s[F.RIGHT_WRIST])]=[t.gestures[e][0].categoryName??d.NONE,r]}return i}}!function(t){t.NONE="None",t.CLOSED_FIST="Closed_Fist",t.OPEN_PALM="Open_Palm",t.POINTING_UP="Pointing_Up",t.THUMB_DOWN="Thumb_Down",t.THUMB_UP="Thumb_Up",t.VICTORY="Victory",t.I_LOVE_YOU="ILoveYou"}(d||(d={})),function(t){t.THUMB="T",t.INDEX="I",t.MIDDLE="M",t.RING="R",t.PINKY="P"}(G||(G={})),e(G),function(t){t[t.STRAIGHT=1]="STRAIGHT",t[t.HALF=2]="HALF",t[t.FULL=3]="FULL"}(m||(m={}));class w{static getFingerCurlAngles(t,s){const n=x[s],i=[];for(let s=0;s<n.length-2;s++){const e=t[n[s]],o=t[n[s+1]],c=t[n[s+2]];i.push(r(e,o,c,o))}return i}static determineFingerCurl(t,s){const n=new c(this.getFingerCurlAngles(t,s));return s!==G.THUMB&&n.mean>=S||n.mean>=b?m.STRAIGHT:function(t,s,n){for(let i=0;i<t.length;i++)if(Math.abs(t[i]-s[i])>n)return!1;return!0}(n.values,g[s],10)?m.FULL:m.HALF}_;constructor(t){const s={};for(const n in t){const i=t[n];a(i,"is")?s[n]={curlStates:new Set([i.is]),is:!0}:a(i,"isOneOf")?s[n]={curlStates:new Set([...i.isOneOf]),is:!0}:a(i,"isNot")?s[n]={curlStates:new Set([i.isNot]),is:!1}:a(i,"isNotOneOf")&&(s[n]={curlStates:new Set([...i.isNotOneOf]),is:!1})}this._=s}isMatchedBy(t){for(const s in this._){if(!a(this._,s))continue;const n=w.determineFingerCurl(t,s),i=this._[s];if(i?.is){if(!i.curlStates.has(n))return!1}else if(i?.curlStates.has(n))return!1}return!0}}const S=o(170),b=o(150),g={[G.THUMB]:[o(150),o(160),o(140)],[G.INDEX]:[o(155),o(40),o(175)],[G.MIDDLE]:[o(155),o(40),o(130)],[G.RING]:[o(145),o(40),o(125)],[G.PINKY]:[o(150),o(35),o(145)]};const x={[G.THUMB]:[P.WRIST,P.THUMB_CMC,P.THUMB_MCP,P.THUMB_IP,P.THUMB_TIP],[G.INDEX]:[P.WRIST,P.INDEX_FINGER_MCP,P.INDEX_FINGER_PIP,P.INDEX_FINGER_DIP,P.INDEX_FINGER_TIP],[G.MIDDLE]:[P.WRIST,P.MIDDLE_FINGER_MCP,P.MIDDLE_FINGER_PIP,P.MIDDLE_FINGER_DIP,P.MIDDLE_FINGER_TIP],[G.RING]:[P.WRIST,P.RING_FINGER_MCP,P.RING_FINGER_PIP,P.RING_FINGER_DIP,P.RING_FINGER_TIP],[G.PINKY]:[P.WRIST,P.PINKY_FINGER_MCP,P.PINKY_FINGER_PIP,P.PINKY_FINGER_DIP,P.PINKY_FINGER_TIP]},U="4d46b09a99c0294205b422ab416bab0b429e4c70",v=process.env.PACKAGE_BUILD_TYPE,C="1.3.0",Y=[[0,1],[1,2],[2,3],[3,4],[0,5],[5,6],[6,7],[7,8],[5,9],[9,10],[10,11],[11,12],[9,13],[13,14],[14,15],[15,16],[13,17],[0,17],[17,18],[18,19],[19,20]],B={color:"#ffffff",lineWidth:1};function z(t,s,n,i){const e={...B,...i};for(const i of n){const[n,o]=i,r=s[n],c=s[o];t.beginPath(),t.moveTo(r.x*t.canvas.width,r.y*t.canvas.height),t.lineTo(c.x*t.canvas.width,c.y*t.canvas.height),t.lineWidth=e.lineWidth,t.strokeStyle=e.color,t.stroke()}}const K={color:"#ffffff",radius:3};function $(t,s,n){const i={...K,...n};for(const n of s){const{x:s,y:e}=n;t.beginPath(),t.arc(s*t.canvas.width,e*t.canvas.height,i.radius,0,2*Math.PI),t.fillStyle=i.color,t.fill()}}class X{static getFingerCurlAngles(t,s){const n=W[s],i={};for(let s=0;s<n.length-2;s++){const e=n[s],o=n[s+1],c=n[s+2],a=[e,o,c].join("-"),h=t[e],I=t[o],E=t[c];i[a]=r(h,I,E,I)}return i}processedPoints;constructor(t){const s={};for(const n of t)for(const t in W){const i=X.getFingerCurlAngles(n,t);for(const t in i)s[t]||(s[t]=[]),s[t].push(i[t])}const n={};for(const t in s){const i=new c(s[t]);n[t]={mean:i.mean,stdDev:i.stddev}}this.processedPoints=n}isMatchedBy(t,s=2){for(const n in W){const i=X.getFingerCurlAngles(t,n);for(const t in i){const{mean:n,stdDev:e}=this.processedPoints[t];if(!h(i[t],n-e*s,n+e*s))return!1}}return!0}}const W={[G.THUMB]:[P.WRIST,P.THUMB_CMC,P.THUMB_MCP,P.THUMB_IP,P.THUMB_TIP],[G.INDEX]:[P.WRIST,P.INDEX_FINGER_MCP,P.INDEX_FINGER_PIP,P.INDEX_FINGER_DIP,P.INDEX_FINGER_TIP],[G.MIDDLE]:[P.WRIST,P.MIDDLE_FINGER_MCP,P.MIDDLE_FINGER_PIP,P.MIDDLE_FINGER_DIP,P.MIDDLE_FINGER_TIP],[G.RING]:[P.WRIST,P.RING_FINGER_MCP,P.RING_FINGER_PIP,P.RING_FINGER_DIP,P.RING_FINGER_TIP],[G.PINKY]:[P.WRIST,P.PINKY_FINGER_MCP,P.PINKY_FINGER_PIP,P.PINKY_FINGER_DIP,P.PINKY_FINGER_TIP]};export{U as BUILD_HASH,v as BUILD_TYPE,L as BaseLandmarkAnalyzer,H as BaseVisionAnalyzer,F as BodyPoseLandmark,w as ComplexHandGesture,N as DetectionMethod,y as FilesetResolverPath,G as Finger,m as FingerCurl,Y as HAND_CONNECTIONS,d as HandGesture,X as HandGestureSnapshot,P as HandPoseLandmark,M as OnePersonBodyPoseAnalyzer,p as OnePersonFaceMeshAnalyzer,A as OnePersonHandGestureAnalyzer,O as OnePersonHandPoseAnalyzer,C as VERSION,R as VisionAnalyzerState,z as drawConnectors,$ as drawLandmarks};
//# sourceMappingURL=index.mjs.map