apphouse/dist/components/media/audiox/models/SpeechRecognitionRecorder.ts

Component library for React that uses observable state management and theme-able components.

import { action, computed, makeObservable, observable } from "mobx";
import { Recorder } from "./Recorder";
import Microphone from "./Microphone";
import Visualizer from "./Visualizer";
import AudioSource from "./AudioSource";
import Player from "./Player";
import Speech, { GrammarItemType } from "./Speech";

interface NonMatchedMemos {
  text: string;
  url: string;
}
export default class SpeechRecognitionRecorder extends Speech {
  matchingId?: string;
  recorder: Recorder;
  microphone: Microphone;
  audioSource: AudioSource;
  recordedAudioBlob: Blob | undefined;
  recordedAudioUrl: string;
  visualizer: Visualizer;
  player: Player;
  voiceMemos: Record<string, string[]>;
  otherMemos: NonMatchedMemos[];
  spokenText?: string;
  visualizerId: string;

  constructor(grammar: GrammarItemType[], visualizerId: string) {
    super(grammar);
    this.audioSource = new AudioSource();
    this.matchingId = undefined;
    this.microphone = new Microphone();
    this.otherMemos = [];
    this.player = new Player();
    if (typeof window != "undefined") {
      this.recordedAudioBlob = new Blob();
    } else {
      this.recordedAudioBlob = undefined;
    }
    this.recordedAudioUrl = "";
    this.recorder = new Recorder(this.onRecordingComplete);
    this.spokenText = undefined;
    this.visualizer = new Visualizer();
    this.visualizerId = visualizerId;
    this.voiceMemos = {};

    makeObservable(this, {
      audioSource: observable,
      currentVoiceMemos: computed,
      matchingId: observable,
      onRecordingComplete: action,
      onResult: action,
      otherMemos: observable,
      recordedAudioBlob: observable,
      recordedAudioUrl: observable,
      recorder: observable,
      setRecordedAudioUrl: action,
      spokenText: observable,
      startRecording: action,
      stopRecording: action,
      visualizer: observable,
      visualizerId: observable,
      voiceMemos: observable,
    });
  }

  get currentVoiceMemos() {
    return this.voiceMemos;
  }

  get audioUrl() {
    return this.recordedAudioUrl;
  }

  init = () => {
    this.recorder.init(this.microphone.stream);
  };

  setRecordedAudioUrl = (url: string) => {
    this.recordedAudioUrl = url;
  };

  onRecordingComplete = (audio: Blob) => {
    this.recordedAudioBlob = audio;
    const url = URL.createObjectURL(audio);
    this.setRecordedAudioUrl(url);
    if (this.matchingId) {
      if (this.voiceMemos[this.matchingId]) {
        this.voiceMemos[this.matchingId] = [
          ...this.voiceMemos[this.matchingId],
          url,
        ];
      } else {
        this.voiceMemos[this.matchingId] = [url];
      }
    } else {
      if (this.spokenText) {
        this.otherMemos = [
          ...this.otherMemos,
          { text: this.spokenText, url },
        ];
      }
    }
  };

  start = () => {
    this.recognition.onresult = this.onResult;
    this.recognition.start();
    this.startRecording();
  };

  startRecording = () => {
    const started = this.microphone.startStream();
    if (started) {
      started.then(() => {
        this.audioSource.initContext();
        if (this.microphone.isReady) {
          console.log("Microphone is ready");
          const stream = this.microphone.stream;
          if (stream) {
            const source = this.audioSource.initWithStream(stream);
            source.then(() => {
              if (
                this.audioSource.analyzer &&
                this.audioSource.hasSource
              ) {
                this.recorder.init(stream);
                this.recorder.start();
                const canvas = document.getElementById(
                  this.visualizerId,
                ) as HTMLCanvasElement;
                if (canvas) {
                  this.visualizer.setCanvas(canvas);
                  this.visualizer.visualizeFreqBar(
                    this.audioSource.analyzer,
                  );
                }
              }
            });
          }
        }
      });
    }
  };

  onResult = (event: any) => {
    // The SpeechRecognitionEvent results property returns a SpeechRecognitionResultList object
    // The SpeechRecognitionResultList object contains SpeechRecognitionResult objects.
    // It has a getter so it can be accessed like an array
    // The first [0] returns the SpeechRecognitionResult at the last position.
    // Each SpeechRecognitionResult object contains SpeechRecognitionAlternative objects that contain individual results.
    // These also have getters so they can be accessed like arrays.
    // The second [0] returns the SpeechRecognitionAlternative at position 0.
    // We then return the transcript property of the SpeechRecognitionAlternative object
    var result = Speech.cleanKey(event.results[0][0].transcript);
    console.log({ result });
    let matchingId;
    this.spokenText = result;
    if (this.gramar[result]) {
      matchingId = this.gramar[result].id;
    }
    this.stopRecording(matchingId);
    if (result)
      console.log("Confidence: " + event.results[0][0].confidence);
  };

  stopRecording = (matchedItemId?: string) => {
    this.matchingId = matchedItemId;
    this.recorder.stop();
    this.microphone.stopStreaming();
  };
}