@glyph-cat/ml-helpers/lib/es/index.mjs

Helper functions to use in conjunction with machine learning outputs such as mediapipe.

import{createEnumToStringConverter as t,NotImplementedError as s,LazyValue as e,reflect1D as n,getDistance2DByCoordinates as i,fullyEnumerate as o,degToRad as r,getAngleFromPointsIn3D as c,NumericDataSet as a,hasProperty as I,isInRange as E}from"@glyph-cat/swiss-army-knife";import{FilesetResolver as _,PoseLandmarker as u,HandLandmarker as h,GestureRecognizer as l}from"@mediapipe/tasks-vision";import{SimpleStateManager as f,SimpleFiniteStateManager as T}from"cotton-box";const R=[[0,1],[1,2],[2,3],[3,4],[0,5],[5,6],[6,7],[7,8],[5,9],[9,10],[10,11],[11,12],[9,13],[13,14],[14,15],[15,16],[13,17],[0,17],[17,18],[18,19],[19,20]];var P,d,N,F,G,m;!function(t){t[t.CREATED=0]="CREATED",t[t.INITIALIZING=1]="INITIALIZING",t[t.STANDBY=2]="STANDBY",t[t.ACTIVE=3]="ACTIVE",t[t.DISPOSED=4]="DISPOSED"}(P||(P={}));class H{videoElement;detectionMethodName;static _vision;static async getVision(){return this._vision||(this._vision=_.forVisionTasks("/mediapipe/wasm")),this._vision}taskRunner;lastRequestedAnimationFrame;result;state;constructor(s,e,n,i,o){this.videoElement=s,this.detectionMethodName=i,this.start=this.start.bind(this),this.stop=this.stop.bind(this),this.getProcessedResult=this.getProcessedResult.bind(this),this.dispose=this.dispose.bind(this),this.result=new f(e),this.state=new T(P.CREATED,[[P.CREATED,P.INITIALIZING],[P.CREATED,P.DISPOSED],[P.INITIALIZING,P.STANDBY],[P.ACTIVE,P.STANDBY],[P.STANDBY,P.ACTIVE],[P.STANDBY,P.DISPOSED]],{name:"VisionAnalyzer",serializeState:t(P)});(async()=>{this.state.set(P.INITIALIZING),this.taskRunner=await n.value,this.state.set((t=>t===P.INITIALIZING?P.STANDBY:t))})()}async start(){if(this.state.get()<P.STANDBY)await this.state.wait((t=>t>P.STANDBY));else if(this.state.get()!==P.STANDBY)return;this.state.set(P.ACTIVE),this.lastRequestedAnimationFrame=requestAnimationFrame(this.performAnalysis)}async stop(){await this.state.wait((t=>t===P.CREATED||t>P.STANDBY)),this.state.get()!==P.DISPOSED&&(cancelAnimationFrame(this.lastRequestedAnimationFrame),this.state.set(P.STANDBY))}async dispose(){await this.state.wait((t=>t!==P.INITIALIZING)),this.result.dispose(),this.state.set(P.DISPOSED),this.state.dispose()}performAnalysis=async()=>{if(this.state.get()!==P.ACTIVE)return;const t=this.taskRunner[this.detectionMethodName](this.videoElement,performance.now()),s=this.getProcessedResult(t);s&&this.result.set(s),this.lastRequestedAnimationFrame=requestAnimationFrame(this.performAnalysis)};getProcessedResult(t){throw new s}}class L extends H{constructor(t,s,e,n){super(t,s,e,"detectForVideo",n)}}class p extends L{static t=new e((async()=>u.createFromOptions(await L.getVision(),{baseOptions:{modelAssetPath:"/mediapipe/models/pose_landmarker_lite.task",delegate:"GPU"},numPoses:1,runningMode:"VIDEO"})));constructor(t){super(t,new Array(Object.keys(d).length/2).fill({x:0,y:0,z:0,visibility:0}),p.t,"OnePersonBodyPoseAnalyzer")}getProcessedResult(t){const s=[];if(t.landmarks.length<=0)return;const e=t.landmarks[0];for(const t of e)s.push({...t,x:n(t.x,.5),visibility:1});return s}}function D(t,s,e){return i(t,s)<i(t,e)?"L":"R"}!function(t){t[t.NOSE=0]="NOSE",t[t.LEFT_EYE_INNER=1]="LEFT_EYE_INNER",t[t.LEFT_EYE=2]="LEFT_EYE",t[t.LEFT_EYE_OUTER=3]="LEFT_EYE_OUTER",t[t.RIGHT_EYE_INNER=4]="RIGHT_EYE_INNER",t[t.RIGHT_EYE=5]="RIGHT_EYE",t[t.RIGHT_EYE_OUTER=6]="RIGHT_EYE_OUTER",t[t.LEFT_EAR=7]="LEFT_EAR",t[t.RIGHT_EAR=8]="RIGHT_EAR",t[t.MOUTH_LEFT=9]="MOUTH_LEFT",t[t.MOUTH_RIGHT=10]="MOUTH_RIGHT",t[t.LEFT_SHOULDER=11]="LEFT_SHOULDER",t[t.RIGHT_SHOULDER=12]="RIGHT_SHOULDER",t[t.LEFT_ELBOW=13]="LEFT_ELBOW",t[t.RIGHT_ELBOW=14]="RIGHT_ELBOW",t[t.LEFT_WRIST=15]="LEFT_WRIST",t[t.RIGHT_WRIST=16]="RIGHT_WRIST",t[t.LEFT_PINKY=17]="LEFT_PINKY",t[t.RIGHT_PINKY=18]="RIGHT_PINKY",t[t.LEFT_INDEX=19]="LEFT_INDEX",t[t.RIGHT_INDEX=20]="RIGHT_INDEX",t[t.LEFT_THUMB=21]="LEFT_THUMB",t[t.RIGHT_THUMB=22]="RIGHT_THUMB",t[t.LEFT_HIP=23]="LEFT_HIP",t[t.RIGHT_HIP=24]="RIGHT_HIP",t[t.LEFT_KNEE=25]="LEFT_KNEE",t[t.RIGHT_KNEE=26]="RIGHT_KNEE",t[t.LEFT_ANKLE=27]="LEFT_ANKLE",t[t.RIGHT_ANKLE=28]="RIGHT_ANKLE",t[t.LEFT_HEEL=29]="LEFT_HEEL",t[t.RIGHT_HEEL=30]="RIGHT_HEEL",t[t.LEFT_FOOT_INDEX=31]="LEFT_FOOT_INDEX",t[t.RIGHT_FOOT_INDEX=32]="RIGHT_FOOT_INDEX"}(d||(d={}));class y extends L{bodyPoseAnalyzer;static t=new e((async()=>h.createFromOptions(await L.getVision(),{baseOptions:{modelAssetPath:"/mediapipe/models/hand_landmarker.task",delegate:"GPU"},numHands:2,runningMode:"VIDEO"})));constructor(t){super(t.videoElement,{},y.t,"OnePersonHandPoseAnalyzer"),this.bodyPoseAnalyzer=t}getProcessedResult(t){const s=this.bodyPoseAnalyzer.result.get(),e={};for(const i of t.landmarks){const t=[];let o;for(let s=0;s<i.length;s++){const e=i[s],r={...e,x:n(e.x,.5),visibility:1};s===N.WRIST&&(o=r),t.push(r)}e[D(o,s[d.LEFT_WRIST],s[d.RIGHT_WRIST])]=t}return e}}!function(t){t[t.WRIST=0]="WRIST",t[t.THUMB_CMC=1]="THUMB_CMC",t[t.THUMB_MCP=2]="THUMB_MCP",t[t.THUMB_IP=3]="THUMB_IP",t[t.THUMB_TIP=4]="THUMB_TIP",t[t.INDEX_FINGER_MCP=5]="INDEX_FINGER_MCP",t[t.INDEX_FINGER_PIP=6]="INDEX_FINGER_PIP",t[t.INDEX_FINGER_DIP=7]="INDEX_FINGER_DIP",t[t.INDEX_FINGER_TIP=8]="INDEX_FINGER_TIP",t[t.MIDDLE_FINGER_MCP=9]="MIDDLE_FINGER_MCP",t[t.MIDDLE_FINGER_PIP=10]="MIDDLE_FINGER_PIP",t[t.MIDDLE_FINGER_DIP=11]="MIDDLE_FINGER_DIP",t[t.MIDDLE_FINGER_TIP=12]="MIDDLE_FINGER_TIP",t[t.RING_FINGER_MCP=13]="RING_FINGER_MCP",t[t.RING_FINGER_PIP=14]="RING_FINGER_PIP",t[t.RING_FINGER_DIP=15]="RING_FINGER_DIP",t[t.RING_FINGER_TIP=16]="RING_FINGER_TIP",t[t.PINKY_FINGER_MCP=17]="PINKY_FINGER_MCP",t[t.PINKY_FINGER_PIP=18]="PINKY_FINGER_PIP",t[t.PINKY_FINGER_DIP=19]="PINKY_FINGER_DIP",t[t.PINKY_FINGER_TIP=20]="PINKY_FINGER_TIP"}(N||(N={}));class M extends H{bodyPoseAnalyzer;static t=new e((async()=>l.createFromOptions(await H.getVision(),{baseOptions:{modelAssetPath:"/mediapipe/models/gesture_recognizer.task",delegate:"GPU"},numHands:2,runningMode:"VIDEO"})));constructor(t){super(t.videoElement,{},M.t,"recognizeForVideo","OnePersonHandGestureAnalyzer"),this.bodyPoseAnalyzer=t}getProcessedResult(t){const s=this.bodyPoseAnalyzer.result.get(),e={};for(const i in t.landmarks){const o=t.landmarks[i],r=[];let c;for(let t=0;t<o.length;t++){const s=o[t],e={...s,x:n(s.x,.5),visibility:1};t===N.WRIST&&(c=e),r.push(e)}e[D(c,s[d.LEFT_WRIST],s[d.RIGHT_WRIST])]=[t.gestures[i][0].categoryName??F.NONE,r]}return e}}!function(t){t.NONE="None",t.CLOSED_FIST="Closed_Fist",t.OPEN_PALM="Open_Palm",t.POINTING_UP="Pointing_Up",t.THUMB_DOWN="Thumb_Down",t.THUMB_UP="Thumb_Up",t.VICTORY="Victory",t.I_LOVE_YOU="ILoveYou"}(F||(F={})),function(t){t.THUMB="T",t.INDEX="I",t.MIDDLE="M",t.RING="R",t.PINKY="P"}(G||(G={})),o(G),function(t){t[t.STRAIGHT=1]="STRAIGHT",t[t.HALF=2]="HALF",t[t.FULL=3]="FULL"}(m||(m={}));class O{static getFingerCurlAngles(t,s){const e=S[s],n=[];for(let s=0;s<e.length-2;s++){const i=t[e[s]],o=t[e[s+1]],r=t[e[s+2]];n.push(c(i,o,r,o))}return n}static determineFingerCurl(t,s){const e=new a(this.getFingerCurlAngles(t,s));return s!==G.THUMB&&e.mean>=A||e.mean>=g?m.STRAIGHT:function(t,s,e){for(let n=0;n<t.length;n++)if(Math.abs(t[n]-s[n])>e)return!1;return!0}(e.values,w[s],10)?m.FULL:m.HALF}i;constructor(t){const s={};for(const e in t){const n=t[e];I(n,"is")?s[e]={curlStates:new Set([n.is]),is:!0}:I(n,"isOneOf")?s[e]={curlStates:new Set([...n.isOneOf]),is:!0}:I(n,"isNot")?s[e]={curlStates:new Set([n.isNot]),is:!1}:I(n,"isNotOneOf")&&(s[e]={curlStates:new Set([...n.isNotOneOf]),is:!1})}this.i=s}isMatchedBy(t){for(const s in this.i){if(!I(this.i,s))continue;const e=O.determineFingerCurl(t,s),n=this.i[s];if(n.is){if(!n.curlStates.has(e))return!1}else if(n.curlStates.has(e))return!1}return!0}}const A=r(170),g=r(150),w={[G.THUMB]:[r(150),r(160),r(140)],[G.INDEX]:[r(155),r(40),r(175)],[G.MIDDLE]:[r(155),r(40),r(130)],[G.RING]:[r(145),r(40),r(125)],[G.PINKY]:[r(150),r(35),r(145)]};const S={[G.THUMB]:[N.WRIST,N.THUMB_CMC,N.THUMB_MCP,N.THUMB_IP,N.THUMB_TIP],[G.INDEX]:[N.WRIST,N.INDEX_FINGER_MCP,N.INDEX_FINGER_PIP,N.INDEX_FINGER_DIP,N.INDEX_FINGER_TIP],[G.MIDDLE]:[N.WRIST,N.MIDDLE_FINGER_MCP,N.MIDDLE_FINGER_PIP,N.MIDDLE_FINGER_DIP,N.MIDDLE_FINGER_TIP],[G.RING]:[N.WRIST,N.RING_FINGER_MCP,N.RING_FINGER_PIP,N.RING_FINGER_DIP,N.RING_FINGER_TIP],[G.PINKY]:[N.WRIST,N.PINKY_FINGER_MCP,N.PINKY_FINGER_PIP,N.PINKY_FINGER_DIP,N.PINKY_FINGER_TIP]},U={color:"#ffffff",lineWidth:1};function C(t,s,e,n){const i={...U,...n};for(const n of e){const[e,o]=n,r=s[e],c=s[o];t.beginPath(),t.moveTo(r.x*t.canvas.width,r.y*t.canvas.height),t.lineTo(c.x*t.canvas.width,c.y*t.canvas.height),t.lineWidth=i.lineWidth,t.strokeStyle=i.color,t.stroke()}}const Y={color:"#ffffff",radius:3};function b(t,s,e){const n={...Y,...e};for(const i of s){const{x:s,y:o}=i;t.beginPath(),t.arc(s*t.canvas.width,o*t.canvas.height,e.radius,0,2*Math.PI),t.fillStyle=n.color,t.fill()}}class x{static getFingerCurlAngles(t,s){const e=B[s],n={};for(let s=0;s<e.length-2;s++){const i=e[s],o=e[s+1],r=e[s+2],a=[i,o,r].join("-"),I=t[i],E=t[o],_=t[r];n[a]=c(I,E,_,E)}return n}processedPoints;constructor(t){const s={};for(const e of t)for(const t in B){const n=x.getFingerCurlAngles(e,t);for(const t in n)s[t]||(s[t]=[]),s[t].push(n[t])}const e={};for(const t in s){const n=new a(s[t]);e[t]={mean:n.mean,stdDev:n.stddev}}this.processedPoints=e}isMatchedBy(t,s=2){for(const e in B){const n=x.getFingerCurlAngles(t,e);for(const t in n){const{mean:e,stdDev:i}=this.processedPoints[t];if(!E(n[t],e-i*s,e+i*s))return!1}}return!0}}const B={[G.THUMB]:[N.WRIST,N.THUMB_CMC,N.THUMB_MCP,N.THUMB_IP,N.THUMB_TIP],[G.INDEX]:[N.WRIST,N.INDEX_FINGER_MCP,N.INDEX_FINGER_PIP,N.INDEX_FINGER_DIP,N.INDEX_FINGER_TIP],[G.MIDDLE]:[N.WRIST,N.MIDDLE_FINGER_MCP,N.MIDDLE_FINGER_PIP,N.MIDDLE_FINGER_DIP,N.MIDDLE_FINGER_TIP],[G.RING]:[N.WRIST,N.RING_FINGER_MCP,N.RING_FINGER_PIP,N.RING_FINGER_DIP,N.RING_FINGER_TIP],[G.PINKY]:[N.WRIST,N.PINKY_FINGER_MCP,N.PINKY_FINGER_PIP,N.PINKY_FINGER_DIP,N.PINKY_FINGER_TIP]};export{L as BaseLandmarkAnalyzer,H as BaseVisionAnalyzer,d as BodyPoseLandmark,O as ComplexHandGesture,G as Finger,m as FingerCurl,R as HAND_CONNECTIONS,F as HandGesture,x as HandGestureSnapshot,N as HandPoseLandmark,p as OnePersonBodyPoseAnalyzer,M as OnePersonHandGestureAnalyzer,y as OnePersonHandPoseAnalyzer,P as VisionAnalyzerState,C as drawConnectors,b as drawLandmarks};
//# sourceMappingURL=index.mjs.map