@glyph-cat/ml-helpers/lib/cjs/index.js

Helper functions to use in conjunction with machine learning outputs such as mediapipe.

"use strict";var t=require("@glyph-cat/swiss-army-knife"),s=require("@mediapipe/tasks-vision"),e=require("cotton-box");var o,r,n,i,p,x;exports.VisionAnalyzerState=void 0,(o=exports.VisionAnalyzerState||(exports.VisionAnalyzerState={}))[o.CREATED=0]="CREATED",o[o.INITIALIZING=1]="INITIALIZING",o[o.STANDBY=2]="STANDBY",o[o.ACTIVE=3]="ACTIVE",o[o.DISPOSED=4]="DISPOSED";class c{videoElement;detectionMethodName;static _vision;static async getVision(){return this._vision||(this._vision=s.FilesetResolver.forVisionTasks("/mediapipe/wasm")),this._vision}taskRunner;lastRequestedAnimationFrame;result;state;constructor(s,o,r,n,i){this.videoElement=s,this.detectionMethodName=n,this.start=this.start.bind(this),this.stop=this.stop.bind(this),this.getProcessedResult=this.getProcessedResult.bind(this),this.dispose=this.dispose.bind(this),this.result=new e.SimpleStateManager(o),this.state=new e.SimpleFiniteStateManager(exports.VisionAnalyzerState.CREATED,[[exports.VisionAnalyzerState.CREATED,exports.VisionAnalyzerState.INITIALIZING],[exports.VisionAnalyzerState.CREATED,exports.VisionAnalyzerState.DISPOSED],[exports.VisionAnalyzerState.INITIALIZING,exports.VisionAnalyzerState.STANDBY],[exports.VisionAnalyzerState.ACTIVE,exports.VisionAnalyzerState.STANDBY],[exports.VisionAnalyzerState.STANDBY,exports.VisionAnalyzerState.ACTIVE],[exports.VisionAnalyzerState.STANDBY,exports.VisionAnalyzerState.DISPOSED]],{name:"VisionAnalyzer",serializeState:t.createEnumToStringConverter(exports.VisionAnalyzerState)});(async()=>{this.state.set(exports.VisionAnalyzerState.INITIALIZING),this.taskRunner=await r.value,this.state.set((t=>t===exports.VisionAnalyzerState.INITIALIZING?exports.VisionAnalyzerState.STANDBY:t))})()}async start(){if(this.state.get()<exports.VisionAnalyzerState.STANDBY)await this.state.wait((t=>t>exports.VisionAnalyzerState.STANDBY));else if(this.state.get()!==exports.VisionAnalyzerState.STANDBY)return;this.state.set(exports.VisionAnalyzerState.ACTIVE),this.lastRequestedAnimationFrame=requestAnimationFrame(this.performAnalysis)}async stop(){await this.state.wait((t=>t===exports.VisionAnalyzerState.CREATED||t>exports.VisionAnalyzerState.STANDBY)),this.state.get()!==exports.VisionAnalyzerState.DISPOSED&&(cancelAnimationFrame(this.lastRequestedAnimationFrame),this.state.set(exports.VisionAnalyzerState.STANDBY))}async dispose(){await this.state.wait((t=>t!==exports.VisionAnalyzerState.INITIALIZING)),this.result.dispose(),this.state.set(exports.VisionAnalyzerState.DISPOSED),this.state.dispose()}performAnalysis=async()=>{if(this.state.get()!==exports.VisionAnalyzerState.ACTIVE)return;const t=this.taskRunner[this.detectionMethodName](this.videoElement,performance.now()),s=this.getProcessedResult(t);s&&this.result.set(s),this.lastRequestedAnimationFrame=requestAnimationFrame(this.performAnalysis)};getProcessedResult(s){throw new t.NotImplementedError}}class a extends c{constructor(t,s,e,o){super(t,s,e,"detectForVideo",o)}}class I extends a{static t=new t.LazyValue((async()=>s.PoseLandmarker.createFromOptions(await a.getVision(),{baseOptions:{modelAssetPath:"/mediapipe/models/pose_landmarker_lite.task",delegate:"GPU"},numPoses:1,runningMode:"VIDEO"})));constructor(t){super(t,new Array(Object.keys(exports.BodyPoseLandmark).length/2).fill({x:0,y:0,z:0,visibility:0}),I.t,"OnePersonBodyPoseAnalyzer")}getProcessedResult(s){const e=[];if(s.landmarks.length<=0)return;const o=s.landmarks[0];for(const s of o)e.push({...s,x:t.reflect1D(s.x,.5),visibility:1});return e}}function E(s,e,o){return t.getDistance2DByCoordinates(s,e)<t.getDistance2DByCoordinates(s,o)?"L":"R"}exports.BodyPoseLandmark=void 0,(r=exports.BodyPoseLandmark||(exports.BodyPoseLandmark={}))[r.NOSE=0]="NOSE",r[r.LEFT_EYE_INNER=1]="LEFT_EYE_INNER",r[r.LEFT_EYE=2]="LEFT_EYE",r[r.LEFT_EYE_OUTER=3]="LEFT_EYE_OUTER",r[r.RIGHT_EYE_INNER=4]="RIGHT_EYE_INNER",r[r.RIGHT_EYE=5]="RIGHT_EYE",r[r.RIGHT_EYE_OUTER=6]="RIGHT_EYE_OUTER",r[r.LEFT_EAR=7]="LEFT_EAR",r[r.RIGHT_EAR=8]="RIGHT_EAR",r[r.MOUTH_LEFT=9]="MOUTH_LEFT",r[r.MOUTH_RIGHT=10]="MOUTH_RIGHT",r[r.LEFT_SHOULDER=11]="LEFT_SHOULDER",r[r.RIGHT_SHOULDER=12]="RIGHT_SHOULDER",r[r.LEFT_ELBOW=13]="LEFT_ELBOW",r[r.RIGHT_ELBOW=14]="RIGHT_ELBOW",r[r.LEFT_WRIST=15]="LEFT_WRIST",r[r.RIGHT_WRIST=16]="RIGHT_WRIST",r[r.LEFT_PINKY=17]="LEFT_PINKY",r[r.RIGHT_PINKY=18]="RIGHT_PINKY",r[r.LEFT_INDEX=19]="LEFT_INDEX",r[r.RIGHT_INDEX=20]="RIGHT_INDEX",r[r.LEFT_THUMB=21]="LEFT_THUMB",r[r.RIGHT_THUMB=22]="RIGHT_THUMB",r[r.LEFT_HIP=23]="LEFT_HIP",r[r.RIGHT_HIP=24]="RIGHT_HIP",r[r.LEFT_KNEE=25]="LEFT_KNEE",r[r.RIGHT_KNEE=26]="RIGHT_KNEE",r[r.LEFT_ANKLE=27]="LEFT_ANKLE",r[r.RIGHT_ANKLE=28]="RIGHT_ANKLE",r[r.LEFT_HEEL=29]="LEFT_HEEL",r[r.RIGHT_HEEL=30]="RIGHT_HEEL",r[r.LEFT_FOOT_INDEX=31]="LEFT_FOOT_INDEX",r[r.RIGHT_FOOT_INDEX=32]="RIGHT_FOOT_INDEX";class _ extends a{bodyPoseAnalyzer;static t=new t.LazyValue((async()=>s.HandLandmarker.createFromOptions(await a.getVision(),{baseOptions:{modelAssetPath:"/mediapipe/models/hand_landmarker.task",delegate:"GPU"},numHands:2,runningMode:"VIDEO"})));constructor(t){super(t.videoElement,{},_.t,"OnePersonHandPoseAnalyzer"),this.bodyPoseAnalyzer=t}getProcessedResult(s){const e=this.bodyPoseAnalyzer.result.get(),o={};for(const r of s.landmarks){const s=[];let n;for(let e=0;e<r.length;e++){const o=r[e],i={...o,x:t.reflect1D(o.x,.5),visibility:1};e===exports.HandPoseLandmark.WRIST&&(n=i),s.push(i)}o[E(n,e[exports.BodyPoseLandmark.LEFT_WRIST],e[exports.BodyPoseLandmark.RIGHT_WRIST])]=s}return o}}exports.HandPoseLandmark=void 0,(n=exports.HandPoseLandmark||(exports.HandPoseLandmark={}))[n.WRIST=0]="WRIST",n[n.THUMB_CMC=1]="THUMB_CMC",n[n.THUMB_MCP=2]="THUMB_MCP",n[n.THUMB_IP=3]="THUMB_IP",n[n.THUMB_TIP=4]="THUMB_TIP",n[n.INDEX_FINGER_MCP=5]="INDEX_FINGER_MCP",n[n.INDEX_FINGER_PIP=6]="INDEX_FINGER_PIP",n[n.INDEX_FINGER_DIP=7]="INDEX_FINGER_DIP",n[n.INDEX_FINGER_TIP=8]="INDEX_FINGER_TIP",n[n.MIDDLE_FINGER_MCP=9]="MIDDLE_FINGER_MCP",n[n.MIDDLE_FINGER_PIP=10]="MIDDLE_FINGER_PIP",n[n.MIDDLE_FINGER_DIP=11]="MIDDLE_FINGER_DIP",n[n.MIDDLE_FINGER_TIP=12]="MIDDLE_FINGER_TIP",n[n.RING_FINGER_MCP=13]="RING_FINGER_MCP",n[n.RING_FINGER_PIP=14]="RING_FINGER_PIP",n[n.RING_FINGER_DIP=15]="RING_FINGER_DIP",n[n.RING_FINGER_TIP=16]="RING_FINGER_TIP",n[n.PINKY_FINGER_MCP=17]="PINKY_FINGER_MCP",n[n.PINKY_FINGER_PIP=18]="PINKY_FINGER_PIP",n[n.PINKY_FINGER_DIP=19]="PINKY_FINGER_DIP",n[n.PINKY_FINGER_TIP=20]="PINKY_FINGER_TIP";class u extends c{bodyPoseAnalyzer;static t=new t.LazyValue((async()=>s.GestureRecognizer.createFromOptions(await c.getVision(),{baseOptions:{modelAssetPath:"/mediapipe/models/gesture_recognizer.task",delegate:"GPU"},numHands:2,runningMode:"VIDEO"})));constructor(t){super(t.videoElement,{},u.t,"recognizeForVideo","OnePersonHandGestureAnalyzer"),this.bodyPoseAnalyzer=t}getProcessedResult(s){const e=this.bodyPoseAnalyzer.result.get(),o={};for(const r in s.landmarks){const n=s.landmarks[r],i=[];let p;for(let s=0;s<n.length;s++){const e=n[s],o={...e,x:t.reflect1D(e.x,.5),visibility:1};s===exports.HandPoseLandmark.WRIST&&(p=o),i.push(o)}o[E(p,e[exports.BodyPoseLandmark.LEFT_WRIST],e[exports.BodyPoseLandmark.RIGHT_WRIST])]=[s.gestures[r][0].categoryName??exports.HandGesture.NONE,i]}return o}}exports.HandGesture=void 0,(i=exports.HandGesture||(exports.HandGesture={})).NONE="None",i.CLOSED_FIST="Closed_Fist",i.OPEN_PALM="Open_Palm",i.POINTING_UP="Pointing_Up",i.THUMB_DOWN="Thumb_Down",i.THUMB_UP="Thumb_Up",i.VICTORY="Victory",i.I_LOVE_YOU="ILoveYou",exports.Finger=void 0,(p=exports.Finger||(exports.Finger={})).THUMB="T",p.INDEX="I",p.MIDDLE="M",p.RING="R",p.PINKY="P",t.fullyEnumerate(exports.Finger),exports.FingerCurl=void 0,(x=exports.FingerCurl||(exports.FingerCurl={}))[x.STRAIGHT=1]="STRAIGHT",x[x.HALF=2]="HALF",x[x.FULL=3]="FULL";class h{static getFingerCurlAngles(s,e){const o=d[e],r=[];for(let e=0;e<o.length-2;e++){const n=s[o[e]],i=s[o[e+1]],p=s[o[e+2]];r.push(t.getAngleFromPointsIn3D(n,i,p,i))}return r}static determineFingerCurl(s,e){const o=new t.NumericDataSet(this.getFingerCurlAngles(s,e));return e!==exports.Finger.THUMB&&o.mean>=l||o.mean>=T?exports.FingerCurl.STRAIGHT:function(t,s,e){for(let o=0;o<t.length;o++)if(Math.abs(t[o]-s[o])>e)return!1;return!0}(o.values,R[e],10)?exports.FingerCurl.FULL:exports.FingerCurl.HALF}o;constructor(s){const e={};for(const o in s){const r=s[o];t.hasProperty(r,"is")?e[o]={curlStates:new Set([r.is]),is:!0}:t.hasProperty(r,"isOneOf")?e[o]={curlStates:new Set([...r.isOneOf]),is:!0}:t.hasProperty(r,"isNot")?e[o]={curlStates:new Set([r.isNot]),is:!1}:t.hasProperty(r,"isNotOneOf")&&(e[o]={curlStates:new Set([...r.isNotOneOf]),is:!1})}this.o=e}isMatchedBy(s){for(const e in this.o){if(!t.hasProperty(this.o,e))continue;const o=h.determineFingerCurl(s,e),r=this.o[e];if(r.is){if(!r.curlStates.has(o))return!1}else if(r.curlStates.has(o))return!1}return!0}}const l=t.degToRad(170),T=t.degToRad(150),R={[exports.Finger.THUMB]:[t.degToRad(150),t.degToRad(160),t.degToRad(140)],[exports.Finger.INDEX]:[t.degToRad(155),t.degToRad(40),t.degToRad(175)],[exports.Finger.MIDDLE]:[t.degToRad(155),t.degToRad(40),t.degToRad(130)],[exports.Finger.RING]:[t.degToRad(145),t.degToRad(40),t.degToRad(125)],[exports.Finger.PINKY]:[t.degToRad(150),t.degToRad(35),t.degToRad(145)]};const d={[exports.Finger.THUMB]:[exports.HandPoseLandmark.WRIST,exports.HandPoseLandmark.THUMB_CMC,exports.HandPoseLandmark.THUMB_MCP,exports.HandPoseLandmark.THUMB_IP,exports.HandPoseLandmark.THUMB_TIP],[exports.Finger.INDEX]:[exports.HandPoseLandmark.WRIST,exports.HandPoseLandmark.INDEX_FINGER_MCP,exports.HandPoseLandmark.INDEX_FINGER_PIP,exports.HandPoseLandmark.INDEX_FINGER_DIP,exports.HandPoseLandmark.INDEX_FINGER_TIP],[exports.Finger.MIDDLE]:[exports.HandPoseLandmark.WRIST,exports.HandPoseLandmark.MIDDLE_FINGER_MCP,exports.HandPoseLandmark.MIDDLE_FINGER_PIP,exports.HandPoseLandmark.MIDDLE_FINGER_DIP,exports.HandPoseLandmark.MIDDLE_FINGER_TIP],[exports.Finger.RING]:[exports.HandPoseLandmark.WRIST,exports.HandPoseLandmark.RING_FINGER_MCP,exports.HandPoseLandmark.RING_FINGER_PIP,exports.HandPoseLandmark.RING_FINGER_DIP,exports.HandPoseLandmark.RING_FINGER_TIP],[exports.Finger.PINKY]:[exports.HandPoseLandmark.WRIST,exports.HandPoseLandmark.PINKY_FINGER_MCP,exports.HandPoseLandmark.PINKY_FINGER_PIP,exports.HandPoseLandmark.PINKY_FINGER_DIP,exports.HandPoseLandmark.PINKY_FINGER_TIP]},f={color:"#ffffff",lineWidth:1};const P={color:"#ffffff",radius:3};class N{static getFingerCurlAngles(s,e){const o=F[e],r={};for(let e=0;e<o.length-2;e++){const n=o[e],i=o[e+1],p=o[e+2],x=[n,i,p].join("-"),c=s[n],a=s[i],I=s[p];r[x]=t.getAngleFromPointsIn3D(c,a,I,a)}return r}processedPoints;constructor(s){const e={};for(const t of s)for(const s in F){const o=N.getFingerCurlAngles(t,s);for(const t in o)e[t]||(e[t]=[]),e[t].push(o[t])}const o={};for(const s in e){const r=new t.NumericDataSet(e[s]);o[s]={mean:r.mean,stdDev:r.stddev}}this.processedPoints=o}isMatchedBy(s,e=2){for(const o in F){const r=N.getFingerCurlAngles(s,o);for(const s in r){const{mean:o,stdDev:n}=this.processedPoints[s];if(!t.isInRange(r[s],o-n*e,o+n*e))return!1}}return!0}}const F={[exports.Finger.THUMB]:[exports.HandPoseLandmark.WRIST,exports.HandPoseLandmark.THUMB_CMC,exports.HandPoseLandmark.THUMB_MCP,exports.HandPoseLandmark.THUMB_IP,exports.HandPoseLandmark.THUMB_TIP],[exports.Finger.INDEX]:[exports.HandPoseLandmark.WRIST,exports.HandPoseLandmark.INDEX_FINGER_MCP,exports.HandPoseLandmark.INDEX_FINGER_PIP,exports.HandPoseLandmark.INDEX_FINGER_DIP,exports.HandPoseLandmark.INDEX_FINGER_TIP],[exports.Finger.MIDDLE]:[exports.HandPoseLandmark.WRIST,exports.HandPoseLandmark.MIDDLE_FINGER_MCP,exports.HandPoseLandmark.MIDDLE_FINGER_PIP,exports.HandPoseLandmark.MIDDLE_FINGER_DIP,exports.HandPoseLandmark.MIDDLE_FINGER_TIP],[exports.Finger.RING]:[exports.HandPoseLandmark.WRIST,exports.HandPoseLandmark.RING_FINGER_MCP,exports.HandPoseLandmark.RING_FINGER_PIP,exports.HandPoseLandmark.RING_FINGER_DIP,exports.HandPoseLandmark.RING_FINGER_TIP],[exports.Finger.PINKY]:[exports.HandPoseLandmark.WRIST,exports.HandPoseLandmark.PINKY_FINGER_MCP,exports.HandPoseLandmark.PINKY_FINGER_PIP,exports.HandPoseLandmark.PINKY_FINGER_DIP,exports.HandPoseLandmark.PINKY_FINGER_TIP]};exports.BaseLandmarkAnalyzer=a,exports.BaseVisionAnalyzer=c,exports.ComplexHandGesture=h,exports.HAND_CONNECTIONS=[[0,1],[1,2],[2,3],[3,4],[0,5],[5,6],[6,7],[7,8],[5,9],[9,10],[10,11],[11,12],[9,13],[13,14],[14,15],[15,16],[13,17],[0,17],[17,18],[18,19],[19,20]],exports.HandGestureSnapshot=N,exports.OnePersonBodyPoseAnalyzer=I,exports.OnePersonHandGestureAnalyzer=u,exports.OnePersonHandPoseAnalyzer=_,exports.drawConnectors=function(t,s,e,o){const r={...f,...o};for(const o of e){const[e,n]=o,i=s[e],p=s[n];t.beginPath(),t.moveTo(i.x*t.canvas.width,i.y*t.canvas.height),t.lineTo(p.x*t.canvas.width,p.y*t.canvas.height),t.lineWidth=r.lineWidth,t.strokeStyle=r.color,t.stroke()}},exports.drawLandmarks=function(t,s,e){const o={...P,...e};for(const r of s){const{x:s,y:n}=r;t.beginPath(),t.arc(s*t.canvas.width,n*t.canvas.height,e.radius,0,2*Math.PI),t.fillStyle=o.color,t.fill()}};