@comsultia/speech2text/src/ws.service.ts

Google Speech to Text plugin using websockets to communicate with backend

import Logger from './log.service';
import { Config } from './config.interface';

declare global {
  interface Window {
    AudioContext: typeof AudioContext;
    webkitAudioContext: typeof AudioContext;
  }
}


class WsSpeechToText {
  
  debug: Logger;
  socket = null;
  readonly elementStart: NodeListOf<Element>;
  readonly elementStop: NodeListOf<Element>;
  readonly elementResult: NodeListOf<Element>;
  audioContext = null;
  context = null;
  processor = null;
  bufferSize: number = 1024;
  globalStream = null;
  input = null;
  recognizeStream = null;
  timer: number = 0;
  defaultTimeout: number = 4000;
  

  constructor(private config: Config) {
    this.debug = new Logger(this.config.debug);

    if(document.querySelector(this.config.elementStart) == null) {
      this.debug.warn('element Start not found');
      return;
    }
    if(document.querySelector(this.config.elementStop) == null) {
      this.debug.warn('element Stop not found');
      return;
    }
    if(document.querySelector(this.config.elementResult) == null) {
      this.debug.warn('element Result not found');
      return;
    }

    this.elementStart = document.querySelectorAll(this.config.elementStart);
    this.elementStop = document.querySelectorAll(this.config.elementStop);
    this.elementResult = document.querySelectorAll(this.config.elementResult);

    this.elementStart.forEach(element => {
      element.addEventListener('click', (): void => this.init());
    });

    this.elementStop.forEach(element => {
      element.addEventListener('click', (): void => this.closeWebSocket());
    });
    document.addEventListener('keydown', (): void => this.closeWebSocket());

    if(this.config.defaultTimeout) {
      this.defaultTimeout = this.config.defaultTimeout;
    }
  }

  init = (): void => {
    this.socket = new WebSocket(this.config.url);
    this.openWebSocket();
  }

  initRecording = (): void => {
    const _this = this;

    this.audioContext = window.AudioContext || window.webkitAudioContext;
    this.context = new this.audioContext();

    if (this.context.createJavaScriptNode) {
      this.processor = this.context.createJavaScriptNode(this.bufferSize, 1, 1);
    } else if (this.context.createScriptProcessor) {
      this.processor = this.context.createScriptProcessor(this.bufferSize, 1, 1);
    } else {
      if(this.config.onNotSupported) {
        this.config.onNotSupported();
      }
    }

    this.processor.connect(this.context.destination);
    this.context.resume();

    let handleSuccess = (stream) => {
      _this.globalStream = stream;

      _this.elementStart.forEach(element => {
        element.setAttribute('disabled', 'disabled');
      });
      _this.elementStop.forEach(element => {
        element.removeAttribute('disabled');
      });

      
      if(_this.context != null) {
        _this.input = _this.context.createMediaStreamSource(stream);
        _this.input.connect(_this.processor);

        if(_this.config.onStart) {
          _this.config.onStart();
        }

        // if browser automatically suspended Web Audio context, resume recording
        if(_this.context.state === 'suspended') {
          this.context.resume();
        }

        _this.processor.onaudioprocess = (e):void => {
          if(_this.socket.readyState === WebSocket.OPEN) {
            _this.microphoneProcess(e);
          }
        }
      } else {
        let track = _this.globalStream.getTracks()[0];
        track.stop();
      }
    };
  
    <any>navigator.mediaDevices.getUserMedia({ audio: true, video: false })
      .then(handleSuccess, err => {
        if(this.config.onBlock) {
          this.config.onBlock();
          this.closeWebSocket();
        }
      });

    this.messageWebSocket();
  }

  openWebSocket = (): void => {
    const _this = this;

    if(this.config.onConnecting) {
      this.config.onConnecting();
    }

    this.socket.onopen = () => {
      _this.debug.success('WebSocket open');

      if(this.config.onConnection) {
        this.config.onConnection();
      }

      _this.initRecording();
    };

    this.messageWebSocket();
  }

  closeWebSocket = (): void => {
    const _this = this;

    this.elementStart.forEach(element => {
      element.removeAttribute('disabled');
    });
    this.elementStop.forEach(element => {
      element.setAttribute('disabled', 'disabled');
    });

    if(this.socket != null && this.socket.readyState === WebSocket.OPEN) {
        clearTimeout(_this.timer);

        this.debug.log('Closing...');
        this.socket.close();
        this.socket.onclose = () => {
          _this.debug.result('WebSocket closed');

          if(_this.config.onEnd) {
            _this.config.onEnd();
          }
        };
      }

    // stop microphone recorder
    if(this.globalStream != null && this.globalStream != undefined && typeof this.globalStream != 'undefined') {
      let track = this.globalStream.getTracks()[0];
      track.stop();
    }

    if(this.input != null && this.input != undefined && typeof this.input != 'undefined') {
      this.input.disconnect(this.processor);
    }

    if(this.processor != null && this.processor != undefined && typeof this.processor != 'undefined') {
      this.processor.disconnect(this.context.destination);
    }

    // reset microphone
    if(this.context != null && this.context != undefined && typeof this.context != 'undefined') {
      this.context.close().then(() => {
        _this.input = null;
        _this.processor = null;
        _this.context = null;
        _this.audioContext = null;
      });
    }

  }

  messageWebSocket = (): void => {
    const _this = this;

    this.socket.onmessage = (e) => {
      
      clearTimeout(_this.timer);
      _this.timer = <any>setTimeout((): void => _this.closeWebSocket(), _this.defaultTimeout);
      
      if(e.data === 'error') {
        _this.debug.warn('Error from Google Speech API:');
        _this.debug.object(e);
      } else {
        _this.debug.info('WebSocket is receiving message...');
        _this.debug.log(`message: ${e.data}`);

        // write result into html output
        this.elementResult.forEach(element => {
          element.innerHTML = `<span>${e.data}</span>`;
        })
      };
      
    };

    this.socket.onerror = (e): void => {
      _this.debug.warn('WebSocket error:');
      _this.debug.object(e);

      if(_this.config.onError) {
        _this.config.onError();
      }
    };
  }

  microphoneProcess = (e): void => {
    let left: Array<number> = e.inputBuffer.getChannelData(0);
    let left16 = <ArrayBuffer>this.downsampleBuffer(left, 44100, 16000);
    this.socket.send(left16);
  }

  // helper method
  downsampleBuffer = (buffer: Array<number>, sampleRate: number, outSampleRate: number) => {
    if (outSampleRate == sampleRate) {
      return buffer;
    }
    if (outSampleRate > sampleRate) {
      this.debug.warn('downsampling rate show be smaller than original sample rate');
    }

    let sampleRateRatio: number = sampleRate / outSampleRate;
    let newLength: number = Math.round(buffer.length / sampleRateRatio);
    let result = <Int16Array>new Int16Array(newLength);
    let offsetResult: number = 0;
    let offsetBuffer: number = 0;
    
    while (offsetResult < result.length) {
      let nextOffsetBuffer: number = Math.round((offsetResult + 1) * sampleRateRatio);
      let accum: number = 0;
      let count: number = 0;
      for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
        accum += buffer[i];
        count++;
      }
      result[offsetResult] = Math.min(1, accum / count) * 0x7FFF;
      offsetResult++;
      offsetBuffer = nextOffsetBuffer;
    }
    return result.buffer;
  }

}

export default WsSpeechToText;