@millisai/web-sdk/dist/index.js

Millis AI's web sdk to build voice agents on web and browser extension.

"use strict";var N=Object.create;var h=Object.defineProperty;var P=Object.getOwnPropertyDescriptor;var T=Object.getOwnPropertyNames,C=Object.getOwnPropertySymbols,U=Object.getPrototypeOf,x=Object.prototype.hasOwnProperty,F=Object.prototype.propertyIsEnumerable;var _=(s,i,t)=>i in s?h(s,i,{enumerable:!0,configurable:!0,writable:!0,value:t}):s[i]=t,p=(s,i)=>{for(var t in i||(i={}))x.call(i,t)&&_(s,t,i[t]);if(C)for(var t of C(i))F.call(i,t)&&_(s,t,i[t]);return s};var D=(s,i)=>{for(var t in i)h(s,t,{get:i[t],enumerable:!0})},E=(s,i,t,e)=>{if(i&&typeof i=="object"||typeof i=="function")for(let a of T(i))!x.call(s,a)&&a!==t&&h(s,a,{get:()=>i[a],enumerable:!(e=P(i,a))||e.enumerable});return s};var M=(s,i,t)=>(t=s!=null?N(U(s)):{},E(i||!s||!s.__esModule?h(t,"default",{value:s,enumerable:!0}):t,s)),W=s=>E(h({},"__esModule",{value:!0}),s);var d=(s,i,t)=>new Promise((e,a)=>{var n=r=>{try{u(t.next(r))}catch(c){a(c)}},o=r=>{try{u(t.throw(r))}catch(c){a(c)}},u=r=>r.done?e(r.value):Promise.resolve(r.value).then(n,o);u((t=t.apply(s,i)).next())});var z={};D(z,{AgentState:()=>S,default:()=>B});module.exports=W(z);var A={};D(A,{createClient:()=>G,createMicrophoneAudioTrack:()=>b});var R=require("eventemitter3");var I=M(require("eventemitter3"));var k=`
class captureAndPlaybackProcessor extends AudioWorkletProcessor {
    audioData = [];
    index = 0;
    pause = false;

    constructor() {
      super();
      //set listener to receive audio data, data is float32 array.
      this.port.onmessage = (e) => {
        if (e.data === "clear") {
          // Clear all buffer.
          this.audioData = [];
          this.index = 0;
        } else if (e.data === "pause") {
          this.pause = true;
        } else if (e.data === "unpause") {
          this.pause = false;
        } else if (e.data.length > 0) {
          this.audioData.push(this.convertUint8ToFloat32(e.data));
        }
      };
    }

    convertUint8ToFloat32(array) {
      const targetArray = new Float32Array(array.byteLength / 2);

      // A DataView is used to read our 16-bit little-endian samples out of the Uint8Array buffer
      const sourceDataView = new DataView(array.buffer);

      // Loop through, get values, and divide by 32,768
      for (let i = 0; i < targetArray.length; i++) {
        targetArray[i] = sourceDataView.getInt16(i * 2, true) / Math.pow(2, 16 - 1);
      }
      return targetArray;
    }

    convertFloat32ToUint8(array) {
      const buffer = new ArrayBuffer(array.length * 2);
      const view = new DataView(buffer);

      for (let i = 0; i < array.length; i++) {
        const value = array[i] * 32768;
        view.setInt16(i * 2, value, true); // true for little-endian
      }

      return new Uint8Array(buffer);
    }

    process(inputs, outputs, parameters) {
      // Capture
      const input = inputs[0];
      const inputChannel1 = input[0];
      const inputChannel2 = input[1];
      this.port.postMessage(this.convertFloat32ToUint8(inputChannel1));

      // Playback
      const output = outputs[0];
      const outputChannel1 = output[0];
      const outputChannel2 = output[1];
      // start playback.
      for (let i = 0; i < outputChannel1.length; ++i) {
        if (this.audioData.length > 0 && !this.pause) {
          outputChannel1[i] = this.audioData[0][this.index];
          outputChannel2[i] = this.audioData[0][this.index];
          this.index++;
          if (this.index == this.audioData[0].length) {
            this.audioData.shift();
            this.index = 0;
            if (this.audioData.length == 0) {
              this.port.postMessage("playback_finished");
            }
          }
        } else {
          outputChannel1[i] = 0;
          outputChannel2[i] = 0;
        }
      }

      return true;
    }
  }

  registerProcessor(
    "capture-and-playback-processor",
    captureAndPlaybackProcessor,
  );
`;var f=require("@alexanderolsen/libsamplerate-js");var L=16e3,g=class extends I.default{constructor(t){super();this.audioContext=null;this.stream=null;this.audioNode=null;this.captureNode=null;this.audioData=[];this.audioDataIndex=0;this.pause=!1;this.needResample=!1;this.resamplerCreated=!1;this.config=t}print(t){this.config.debug&&console.log("[millis audio service]",t)}start(){return d(this,null,function*(){let t=L;this.isFirefox()?(this.audioContext=new AudioContext({latencyHint:"interactive"}),this.needResample=!0):(this.needResample=!1,this.audioContext=new AudioContext({latencyHint:"interactive",sampleRate:t})),this.print("starting audio service, firefox: "+this.isFirefox()+", need resample: "+this.needResample);try{this.print("requesting microphone permission"),this.stream=yield b(t,this.config),this.print("microphone permission granted")}catch(e){throw this.print("microphone permission denied"),new Error("User didn't give microphone permission")}if(this.print("starting audio capture and playback processor, worklet: "+this.isAudioWorkletSupported()),this.isAudioWorkletSupported()){this.print("Starting audio worklet"),this.audioContext.resume();let e=new Blob([k],{type:"application/javascript"}),a=URL.createObjectURL(e);yield this.audioContext.audioWorklet.addModule(a),this.print("Audio worklet loaded"),this.audioNode=new AudioWorkletNode(this.audioContext,"capture-and-playback-processor"),this.print("Audio worklet setup"),this.audioNode.port.onmessage=r=>{r.data instanceof Uint8Array?this.emit("data",r.data):r.data==="playback_finished"&&this.onPlaybackFinished()};let n=this.audioContext.createMediaStreamSource(this.stream);n.connect(this.audioNode),this.audioNode.connect(this.audioContext.destination);let o=this.audioContext.createAnalyser();n.connect(o),this.emit("useraudioready",{analyser:o,stream:this.stream});let u=this.audioContext.createAnalyser();this.audioNode.connect(u),u.connect(this.audioContext.destination),this.emit("analyzer",u)}else{this.print("Starting audio capture node");let e=this.audioContext.createMediaStreamSource(this.stream);this.captureNode=this.audioContext.createScriptProcessor(2048,1,1),this.captureNode.onaudioprocess=o=>{if(this.captureNode&&this.audioContext){if(this.needResample&&!this.resamplerCreated){let l=f.ConverterType.SRC_SINC_FASTEST,w=1;this.resamplerCreated=!0,(0,f.create)(w,this.audioContext.sampleRate,t,{converterType:l}).then(v=>{this.inputResampler=v}),(0,f.create)(w,t,this.audioContext.sampleRate,{converterType:l}).then(v=>{this.outputResampler=v})}var u=o.inputBuffer.getChannelData(0),r=null;this.inputResampler!=null?r=this.inputResampler.full(u):r=u;let c=V(r);this.emit("data",c);let y=o.outputBuffer.getChannelData(0);for(let l=0;l<y.length;++l)this.audioData.length>0&&!this.pause?(y[l]=this.audioData[0][this.audioDataIndex++],this.audioDataIndex===this.audioData[0].length&&(this.audioData.shift(),this.audioDataIndex=0,this.audioData.length==0&&this.onPlaybackFinished())):y[l]=0}},e.connect(this.captureNode),this.captureNode.connect(this.audioContext.destination),this.print("Audio capture node setup");let a=this.audioContext.createAnalyser();e.connect(a),this.emit("useraudioready",{analyser:a,stream:this.stream});let n=this.audioContext.createAnalyser();this.captureNode.connect(n),n.connect(this.audioContext.destination),this.emit("analyzer",n)}})}stop(){return d(this,null,function*(){var t,e,a,n;(t=this.audioContext)==null||t.suspend(),(e=this.audioContext)==null||e.close(),this.isAudioWorkletSupported()?((a=this.audioNode)==null||a.disconnect(),this.audioNode=null):this.captureNode&&(this.captureNode.disconnect(),this.captureNode.onaudioprocess=null,this.captureNode=null,this.audioData=[],this.audioDataIndex=0),(n=this.stream)==null||n.getTracks().forEach(o=>o.stop()),this.audioContext=null,this.stream=null})}play(t){var a;if(this.isAudioWorkletSupported())(a=this.audioNode)==null||a.port.postMessage(t);else{let n=O(t);var e=null;this.outputResampler!=null?e=this.outputResampler.full(n):e=n,this.audioData.push(e)}}setpause(t){var e;this.isAudioWorkletSupported()?(e=this.audioNode)==null||e.port.postMessage(t?"pause":"unpause"):this.pause=t}reset(){var t;this.isAudioWorkletSupported()?(t=this.audioNode)==null||t.port.postMessage("clear"):(this.audioData=[],this.audioDataIndex=0)}onPlaybackFinished(){this.emit("playback_finished")}isAudioWorkletSupported(){return/Chrome/.test(navigator.userAgent)&&/Google Inc/.test(navigator.vendor)}isFirefox(){return/Firefox/.test(navigator.userAgent)}};function O(s){let i=new Float32Array(s.byteLength/2),t=new DataView(s.buffer);for(let e=0;e<i.length;e++)i[e]=t.getInt16(e*2,!0)/Math.pow(2,15);return i}function V(s){let i=new ArrayBuffer(s.length*2),t=new DataView(i);for(let e=0;e<s.length;e++){let a=s[e]*32768;t.setInt16(e*2,a,!0)}return new Uint8Array(i)}var S=(a=>(a.IDLE="idle",a.PREPARE_ANSWER="prepare_answer",a.ANSWER="answer",a.PAUSE="pause",a))(S||{});var j="wss://api-west.millis.ai:8080/millis";var m=class extends R.EventEmitter{constructor(t){super();this.startAnswering=0;this.count=0;this.agentState="idle";this.state=0;this.muted=!1;this.readyPayload=void 0;this.ws=null,this.audioService=null,this.latencyEstimator=null;let e={debug:!1};this.config=p(p({},e),t),this.print("init")}reset(){this.state=0,this.agentState="idle",this.count=0,this.startAnswering=0,this.muted=!1}print(t){this.config.debug&&console.log("[millis]",t)}connect(t){this.print("starting websocket"),this.ws=new WebSocket(this.config.endPoint||j),this.ws.binaryType="arraybuffer",this.ws.onopen=()=>{this.print("websocket connected, sending initiate message"),this.emit("onopen"),this.send(JSON.stringify({method:"initiate",data:{public_key:this.config.publicKey,agent:t.agent,metadata:t.metadata,include_metadata_in_prompt:t.include_metadata_in_prompt,session_continuation:t.session_continuation}}))},this.ws.onmessage=e=>{var a;if(e.data instanceof ArrayBuffer){if(this.print("audio data received"),this.startAnswering>0&&(this.emit("onlatency",Date.now()-this.startAnswering),this.print("latency: "+(Date.now()-this.startAnswering)),this.startAnswering=0,this.switchState("answer")),!this.config.isBenchmark){let n=new Uint8Array(e.data);(a=this.audioService)==null||a.play(n),this.emit("onaudio",n)}}else{let n=JSON.parse(e.data);if(n.method==="onready"){this.onready(n.payload);return}this.handle(n)||this.emit(n.method,n.data,n.payload),this.print(`received ${n.method}`)}},this.ws.onclose=e=>{this.stop(),this.emit("onclose",e),this.print("websocket disconnected"),this.switchState("idle")},this.ws.onerror=e=>{this.stop(),this.emit("onerror",e),this.print("websocket error: "+e)}}send(t){var e;((e=this.ws)==null?void 0:e.readyState)===1&&this.ws.send(t)}handle(t){var e,a,n;switch(t.method){case"start_answering":return this.startAnswering=Date.now(),this.switchState("prepare_answer"),!0;case"clear":return(e=this.audioService)==null||e.reset(),this.switchState("idle"),!0;case"pause":return(a=this.audioService)==null||a.setpause(!0),this.switchState("pause"),!0;case"unpause":return(n=this.audioService)==null||n.setpause(!1),this.switchState("answer"),!0;case"pong":return this.print("Ping rtt "+(Date.now()-Number(t.data))),!0}return!1}onready(t){this.readyPayload=t,this.switchState("idle"),this.switchConnectionState(2)}start(t,e,a){return d(this,null,function*(){this.print("starting conversation"),this.audioService=new g(this.config),this.audioService.on("data",o=>{this.muted?o.fill(0):this.print("sending audio data"),this.send(o),this.count++,this.count%1e3==0&&(this.print("sending ping"),this.send(JSON.stringify({method:"ping",data:Date.now().toString()}))),this.switchConnectionState(3)}),this.audioService.on("analyzer",o=>{this.emit("analyzer",o)}),this.audioService.on("useraudioready",o=>{this.print("user audio ready"),this.emit("useraudioready",o)}),this.audioService.on("playback_finished",()=>{this.agentState==="answer"&&this.switchState("idle")}),this.print("starting audio service"),yield this.audioService.start(),this.print("audio service started");let n;typeof t=="object"&&("agent"in t||"call_id"in t)?n=t:(n={agent:typeof t=="string"?{agent_id:t}:{agent_config:t}},e&&(n.metadata=e),a&&(n.include_metadata_in_prompt=a)),this.connect(n)})}stop(){return d(this,null,function*(){var t;this.audioService&&(this.print("stopping audio service"),this.audioService.stop(),this.audioService=null,this.print("audio service stopped")),(t=this.ws)==null||t.close(),this.reset()})}mute(){this.muted=!0}unmute(){this.muted=!1}switchState(t){this.agentState=t,this.emit("onagentstate",t)}switchConnectionState(t){this.state===0?this.state=t:(this.state===3&&t===2||this.state===2&&t===3)&&(this.state=1,this.emit("onready",this.readyPayload))}};function G(s){return new m(s)}function b(s,i){return d(this,null,function*(){var e,a;let t=p({sampleRate:s,echoCancellation:!0,noiseSuppression:!0,channelCount:1,autoGainControl:!0,latency:0},((e=i==null?void 0:i.audioSource)==null?void 0:e.constraints)||{});return(a=i==null?void 0:i.audioSource)!=null&&a.deviceId&&(t.deviceId=i.audioSource.deviceId),navigator.mediaDevices.getUserMedia({audio:t})})}var B=A;0&&(module.exports={AgentState});
//# sourceMappingURL=index.js.map