@speechmatics/flow-client/dist/index.js

Javascript client for the Speechmatics Flow API

'use strict';

var ws = require('ws');
var typescriptEventTarget = require('typescript-event-target');

class AgentAudioEvent extends Event {
  constructor(data) {
    super("agentAudio");
    this.data = data;
  }
}
class FlowIncomingMessageEvent extends Event {
  constructor(data) {
    super("message");
    this.data = data;
  }
}

var __defProp$1 = Object.defineProperty;
var __defNormalProp$1 = (obj, key, value) => key in obj ? __defProp$1(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __publicField$1 = (obj, key, value) => __defNormalProp$1(obj, key + "" , value);
const FLUSH = "flush";
class FlushEvent extends Event {
  constructor(data) {
    super(FLUSH);
    this.data = data;
  }
}
class JitterBuffer extends typescriptEventTarget.TypedEventTarget {
  constructor(maxByteLength) {
    super();
    this.maxByteLength = maxByteLength;
    __publicField$1(this, "buffer", []);
  }
  get byteLength() {
    return this.buffer.reduce((sum, curr) => sum + curr.byteLength, 0);
  }
  enqueue(data) {
    this.buffer.push(data);
    if (this.byteLength >= this.maxByteLength) {
      this.flush();
    }
  }
  flush() {
    this.dispatchTypedEvent(FLUSH, new FlushEvent(this.buffer));
    this.buffer = [];
  }
}

var __defProp = Object.defineProperty;
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
class FlowClient extends typescriptEventTarget.TypedEventTarget {
  constructor(server, {
    appId,
    audioBufferingMs = 10,
    websocketBinaryType = "blob"
  }) {
    super();
    this.server = server;
    __publicField(this, "appId");
    __publicField(this, "audioBufferingMs");
    __publicField(this, "websocketBinaryType");
    // active websocket
    __publicField(this, "ws", null);
    __publicField(this, "serverSeqNo", 0);
    // tracks sequence of server sent audio
    __publicField(this, "clientSeqNo", 0);
    // tracks sequence of client sent audio
    __publicField(this, "jitterBuffer", null);
    this.appId = appId;
    this.audioBufferingMs = audioBufferingMs;
    this.websocketBinaryType = websocketBinaryType;
  }
  get socketState() {
    if (!this.ws) return void 0;
    return {
      [ws.WebSocket.CONNECTING]: "connecting",
      [ws.WebSocket.OPEN]: "open",
      [ws.WebSocket.CLOSING]: "closing",
      [ws.WebSocket.CLOSED]: "closed"
    }[this.ws.readyState];
  }
  async connect(jwt, timeoutMs = 1e4) {
    const socketState = this.socketState;
    if (socketState && socketState !== "closed") {
      throw new SpeechmaticsFlowError(
        "SocketNotClosed",
        `Cannot start connection while socket is ${socketState}`
      );
    }
    const waitForConnect = new Promise((resolve, reject) => {
      const wsUrl = new URL("/v1/flow", this.server);
      wsUrl.searchParams.append("jwt", jwt);
      wsUrl.searchParams.append("sm-app", this.appId);
      this.ws = new ws.WebSocket(wsUrl.toString());
      this.ws.binaryType = this.websocketBinaryType;
      this.dispatchTypedEvent(
        "socketInitialized",
        new Event("socketInitialized")
      );
      this.setupSocketEventListeners();
      this.addEventListener("socketOpen", resolve, { once: true });
      this.addEventListener(
        "socketError",
        (e) => {
          reject(
            new SpeechmaticsFlowError(
              "SocketError",
              "Error opening websocket",
              { cause: e }
            )
          );
        },
        { once: true }
      );
    });
    const { timeout, cancelTimeout } = rejectAfter(
      timeoutMs,
      "websocket connect"
    );
    await Promise.race([waitForConnect, timeout]);
    cancelTimeout();
  }
  setupSocketEventListeners() {
    if (!this.ws)
      throw new SpeechmaticsFlowError("SocketError", "socket not initialized!");
    this.ws.addEventListener("open", () => {
      this.dispatchTypedEvent("socketOpen", new Event("socketOpen"));
    });
    this.ws.addEventListener(
      "close",
      () => this.dispatchTypedEvent("socketClose", new Event("socketClose"))
    );
    this.ws.addEventListener(
      "error",
      (e) => this.dispatchTypedEvent("socketError", new Event("socketError", e))
    );
    this.ws.addEventListener("message", ({ data }) => {
      if (data instanceof Blob || data instanceof ArrayBuffer) {
        this.handleWebsocketAudio(data);
      } else if (typeof data === "string") {
        this.handleWebsocketMessage(data);
      } else {
        throw new SpeechmaticsFlowError(
          "UnexpectedMessage",
          `Unexpected message type: ${data}`
        );
      }
    });
  }
  handleWebsocketAudio(data) {
    this.sendWebsocketMessage({
      message: "AudioReceived",
      seq_no: ++this.serverSeqNo,
      buffering: this.audioBufferingMs / 1e3
    });
    if (data instanceof Blob && this.websocketBinaryType === "blob") {
      data.arrayBuffer().then((b) => this.jitterBuffer?.enqueue(new Int16Array(b))).catch((e) => {
        throw new SpeechmaticsFlowError(
          "BadBinaryMessage",
          "Failed to decode array buffer",
          { cause: e }
        );
      });
    } else if (data instanceof ArrayBuffer && this.websocketBinaryType === "arraybuffer") {
      this.jitterBuffer?.enqueue(new Int16Array(data));
    } else {
      throw new SpeechmaticsFlowError(
        "BadBinaryMessage",
        `Could not process audio: expecting audio to be ${this.websocketBinaryType} but got ${data.constructor.name}`
      );
    }
  }
  handleWebsocketMessage(message) {
    let data;
    try {
      data = JSON.parse(message);
    } catch (e) {
      throw new SpeechmaticsFlowError(
        "UnexpectedMessage",
        "Failed to parse message as JSON",
        { cause: e }
      );
    }
    if (data.message === "AudioAdded") {
      this.clientSeqNo = data.seq_no;
    }
    if (data.message === "ResponseCompleted" || data.message === "ResponseInterrupted") {
      this.jitterBuffer?.flush();
    }
    this.dispatchTypedEvent("message", new FlowIncomingMessageEvent(data));
  }
  sendWebsocketMessage(message) {
    if (this.socketState === "open") {
      this.ws?.send(JSON.stringify(message));
    }
  }
  sendAudio(pcmData) {
    if (this.socketState !== "open") return;
    this.ws?.send(pcmData);
  }
  async startConversation(jwt, {
    config,
    audioFormat
  }) {
    await this.connect(jwt);
    const conversation_config = {
      ...config,
      template_variables: {
        timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
        ...config.template_variables
      }
    };
    const startMessage = {
      message: "StartConversation",
      conversation_config,
      audio_format: audioFormat ?? DEFAULT_AUDIO_FORMAT
    };
    const waitForConversationStarted = new Promise((resolve, reject) => {
      const client = this;
      this.addEventListener("message", function onStart({ data }) {
        if (data.message === "ConversationStarted") {
          resolve();
          client.removeEventListener("message", onStart);
        } else if (data.message === "Error") {
          reject(
            new SpeechmaticsFlowError(
              "ServerError",
              "Error waiting for conversation start",
              {
                cause: data
              }
            )
          );
        }
      });
      this.sendWebsocketMessage(startMessage);
    });
    const rejectOnSocketClose = new Promise((_, reject) => {
      this.addEventListener(
        "socketClose",
        () => reject(
          new SpeechmaticsFlowError(
            "SocketClosedPrematurely",
            "Socket closed before conversation started"
          )
        ),
        { once: true }
      );
    });
    const { timeout, cancelTimeout } = rejectAfter(
      1e4,
      "conversation start"
    );
    this.jitterBuffer = new JitterBuffer(
      TTS_BYTES_PER_MS * this.audioBufferingMs
    );
    this.jitterBuffer.addEventListener("flush", ({ data }) => {
      for (const buffer of data) {
        this.dispatchTypedEvent("agentAudio", new AgentAudioEvent(buffer));
      }
    });
    try {
      await Promise.race([
        waitForConversationStarted,
        rejectOnSocketClose,
        timeout
      ]);
    } finally {
      cancelTimeout();
    }
  }
  endConversation() {
    this.sendWebsocketMessage({
      message: "AudioEnded",
      last_seq_no: this.clientSeqNo
    });
    this.disconnectSocket();
  }
  disconnectSocket() {
    this.dispatchTypedEvent("socketClosing", new Event("socketClosing"));
    this.ws?.close();
  }
}
function rejectAfter(timeoutMs, key) {
  let timeoutId = void 0;
  let resolve = void 0;
  const timeout = new Promise((_resolve, reject) => {
    resolve = _resolve;
    timeoutId = setTimeout(
      () => reject(
        new SpeechmaticsFlowError(
          "Timeout",
          `Timed out after ${timeoutMs}ms waiting for ${key}`
        )
      ),
      timeoutMs
    );
  });
  const cancel = () => {
    if (typeof timeoutId !== "undefined") {
      clearTimeout(timeoutId);
    }
    resolve?.();
  };
  return { timeout, cancelTimeout: cancel };
}
class SpeechmaticsFlowError extends Error {
  constructor(type, message, options) {
    super(message, options);
    this.type = type;
    this.name = "SpeechmaticsFlowError";
  }
}
const DEFAULT_AUDIO_FORMAT = {
  type: "raw",
  encoding: "pcm_s16le",
  sample_rate: 16e3
};
const TTS_SAMPLE_RATE = 16e3;
const TTS_BYTES_PER_SAMPLE = 2;
const TTS_BYTES_PER_MS = TTS_SAMPLE_RATE * TTS_BYTES_PER_SAMPLE / 1e3;

async function fetchPersonas() {
  const resp = await fetch(
    "https://flow.api.speechmatics.com/v1/discovery/templates"
  );
  const json = await resp.json();
  return json.templates;
}

const AddTranscriptMessageEnum = {
  AddTranscript: "AddTranscript"
};

const RecognitionResultTypeEnum = {
  Word: "word",
  Punctuation: "punctuation"
};
const RecognitionResultAttachesToEnum = {
  Next: "next",
  Previous: "previous",
  None: "none",
  Both: "both"
};

const RecognitionDisplayDirectionEnum = {
  Ltr: "ltr",
  Rtl: "rtl"
};

exports.AddTranscriptMessageEnum = AddTranscriptMessageEnum;
exports.AgentAudioEvent = AgentAudioEvent;
exports.FlowClient = FlowClient;
exports.FlowIncomingMessageEvent = FlowIncomingMessageEvent;
exports.RecognitionDisplayDirectionEnum = RecognitionDisplayDirectionEnum;
exports.RecognitionResultAttachesToEnum = RecognitionResultAttachesToEnum;
exports.RecognitionResultTypeEnum = RecognitionResultTypeEnum;
exports.SpeechmaticsFlowError = SpeechmaticsFlowError;
exports.fetchPersonas = fetchPersonas;