wawa-lipsync/dist/wawa-lipsync.umd.js

A simple and easy-to-use library for real-time lipsyncing with JS and web audio API.

(function(r,o){typeof exports=="object"&&typeof module<"u"?o(exports):typeof define=="function"&&define.amd?define(["exports"],o):(r=typeof globalThis<"u"?globalThis:r||self,o(r["Wawa-lipsync"]={}))})(this,function(r){"use strict";var D=Object.defineProperty;var S=(r,o,e)=>o in r?D(r,o,{enumerable:!0,configurable:!0,writable:!0,value:e}):r[o]=e;var d=(r,o,e)=>S(r,typeof o!="symbol"?o+"":o,e);function o(n){return n.length?n.reduce((t,s)=>t+s,0)/n.length:0}var e=(n=>(n.sil="viseme_sil",n.PP="viseme_PP",n.FF="viseme_FF",n.TH="viseme_TH",n.DD="viseme_DD",n.kk="viseme_kk",n.CH="viseme_CH",n.SS="viseme_SS",n.nn="viseme_nn",n.RR="viseme_RR",n.aa="viseme_aa",n.E="viseme_E",n.I="viseme_I",n.O="viseme_O",n.U="viseme_U",n))(e||{});const _={[e.sil]:"silence",[e.PP]:"plosive",[e.FF]:"fricative",[e.TH]:"fricative",[e.DD]:"plosive",[e.kk]:"plosive",[e.CH]:"fricative",[e.SS]:"fricative",[e.nn]:"plosive",[e.RR]:"fricative",[e.aa]:"vowel",[e.E]:"vowel",[e.I]:"vowel",[e.O]:"vowel",[e.U]:"vowel"};class C{constructor(t={fftSize:2048,historySize:10}){d(this,"features",null);d(this,"viseme",e.sil);d(this,"audioContext");d(this,"analyser");d(this,"dataArray");d(this,"history");d(this,"historySize");d(this,"sampleRate");d(this,"binWidth");d(this,"bands");d(this,"audioSource");d(this,"state","silence");const{fftSize:s=2048,historySize:a=10}=t;this.audioContext=new(window.AudioContext||window.webkitAudioContext),this.analyser=this.audioContext.createAnalyser(),this.analyser.fftSize=s,this.dataArray=new Uint8Array(this.analyser.frequencyBinCount),this.history=[],this.historySize=a,this.sampleRate=this.audioContext.sampleRate,this.binWidth=this.sampleRate/s,this.bands=[{start:50,end:200},{start:200,end:400},{start:400,end:800},{start:800,end:1500},{start:1500,end:2500},{start:2500,end:4e3},{start:4e3,end:8e3}]}connectAudio(t){if(this.audioContext.resume(),this.history=[],this.features=null,this.state="silence",this.audioSource===t)return;if(this.audioSource=t,!t.src){console.warn("An audio source must be set before connecting");return}this.audioContext.createMediaElementSource(t).connect(this.analyser),this.analyser.connect(this.audioContext.destination)}async connectMicrophone(){try{const t=await navigator.mediaDevices.getUserMedia({audio:!0}),s=this.audioContext.createMediaStreamSource(t);return s.connect(this.analyser),this.analyser.connect(this.audioContext.destination),s}catch(t){throw console.error("Error accessing microphone:",t),t}}extractFeatures(){this.analyser.getByteFrequencyData(this.dataArray);const t=this.bands.map(({start:h,end:m})=>{const u=Math.round(h/this.binWidth),g=Math.min(Math.round(m/this.binWidth),this.dataArray.length-1);return o(Array.from(this.dataArray.slice(u,g)))/255});let s=0,a=0;for(let h=0;h<this.dataArray.length;h++){const m=h*this.binWidth,u=this.dataArray[h]/255;s+=u,a+=m*u}const c=s>0?a/s:0,i=o(t),f=t.map((h,m)=>{if(this.history.length<2)return 0;const u=this.history[this.history.length-2].bands[m];return h-u}),b={bands:t,deltaBands:f,volume:i,centroid:c};return s>0&&(this.history.push(b),this.history.length>this.historySize&&this.history.shift()),b}getAveragedFeatures(){const t=this.history.length,s={volume:0,centroid:0,bands:Array(this.bands.length).fill(0)};for(const c of this.history)s.volume+=c.volume,s.centroid+=c.centroid,c.bands.forEach((i,f)=>s.bands[f]+=i);const a=s.bands.map(c=>c/t);return{volume:s.volume/t,centroid:s.centroid/t,bands:a,deltaBands:a}}detectState(){const t=this.history[this.history.length-1];if(!t){this.state="silence",this.viseme=e.sil;return}const s=this.getAveragedFeatures(),a=t.volume-s.volume,c=t.centroid-s.centroid,i=this.computeVisemeScores(t,s,a,c),f=this.adjustScoresForConsistency(i);let b=-1/0,h=e.sil;for(const u in f)f[u]>b&&(b=f[u],h=u);let m=_[h];this.state=m,this.viseme=h}computeVisemeScores(t,s,a,c){const i={[e.sil]:0,[e.PP]:0,[e.FF]:0,[e.TH]:0,[e.DD]:0,[e.kk]:0,[e.CH]:0,[e.SS]:0,[e.nn]:0,[e.RR]:0,[e.aa]:0,[e.E]:0,[e.I]:0,[e.O]:0,[e.U]:0},[f,b,h,m,u,g,w]=t.bands;if(s.volume<.2&&t.volume<.2&&(i[e.sil]=1),Object.entries(_).forEach(([p,y])=>{y==="plosive"&&(a<.01&&(i[p]-=.5),s.volume<.2&&(i[p]+=.2),c>1e3&&(i[p]+=.2))}),t.centroid>1e3&&t.centroid<8e3&&(t.centroid>7e3?i[e.DD]+=.6:t.centroid>5e3?i[e.kk]+=.6:t.centroid>4e3?(i[e.PP]+=1,w>.25&&t.centroid<6e3&&(i[e.DD]+=1.4)):i[e.nn]+=.6),c>1e3&&t.centroid>6e3&&s.centroid>5e3&&t.bands[6]>.4&&s.bands[6]>.3&&(i[e.FF]=.7),s.volume>.1&&s.centroid<6e3&&t.centroid<6e3){const[p,y,l,v,A]=s.bands,x=Math.abs(p-y),F=Math.max(Math.abs(y-l),Math.abs(y-v),Math.abs(l-v));(l>.1||v>.1)&&(v>l&&(i[e.aa]=.8,l>y&&(i[e.aa]+=.2)),l>y&&l>v&&(i[e.I]=.7),x<.25&&(i[e.U]=.7),F<.25&&(i[e.O]=.9),y>l&&l>v&&(i[e.E]=1),l<.2&&v>.3&&(i[e.I]=.7),l>.25&&A>.25&&(i[e.O]=.7),l<.15&&A<.15&&(i[e.U]=.7))}return i}adjustScoresForConsistency(t){const s={...t};if(this.viseme&&this.state)for(const a in s)a===this.viseme&&(s[a]*=1.3);return s}processAudio(){this.features=this.extractFeatures(),this.detectState()}}r.Lipsync=C,r.VISEMES=e,Object.defineProperty(r,Symbol.toStringTag,{value:"Module"})});