apphouse/dist/components/media/audiox/models/Speech.ts

Component library for React that uses observable state management and theme-able components.

import { action, makeObservable, observable } from 'mobx';
import { audioLog } from './AudioLog';

export interface GrammarItemType {
  text: string;
  id: string;
}
export default class Speech {
  recognition: SpeechRecognition;
  gramar: { [text: string]: GrammarItemType };
  dictionaryLookupTable: { [word: string]: string[] };

  constructor(grammar?: GrammarItemType[]) {
    //@ts-ignore
    const SpeechRecognition = webkitSpeechRecognition;
    //@ts-ignore
    const SpeechGrammarList = webkitSpeechGrammarList;
    this.recognition = new SpeechRecognition();
    const speechRecognitionList = new SpeechGrammarList();
    this.gramar = {};
    this.dictionaryLookupTable = {};

    if (grammar) {
      grammar.forEach((item) => {
        speechRecognitionList.addFromString(item.text, 1);
        this.gramar[Speech.cleanKey(item.text)] = item;
      });

      this.dictionaryLookupTable = Speech.createGrammerLookupTable(grammar);
    }

    this.recognition.grammars = speechRecognitionList;
    this.recognition.continuous = false;
    this.recognition.lang = 'en-US';
    this.recognition.interimResults = false;
    this.recognition.maxAlternatives = 1;
    this.recognition.onresult = this.onResult;
    this.recognition.onnomatch = this.onNoMatch;
    this.recognition.onspeechend = this.onSpeechEnd;
    this.recognition.onerror = this.onError;

    makeObservable(this, {
      dictionaryLookupTable: observable,
      gramar: observable,
      recognition: observable,
      onSpeechEnd: action,
      onNoMatch: action,
      onError: action,
      start: action,
      setGrammar: action
    });
  }

  start = () => {
    this.recognition.continuous = false;
    this.recognition.start();
  };

  onResult = (event: any) => {
    // The SpeechRecognitionEvent results property returns a SpeechRecognitionResultList object
    // The SpeechRecognitionResultList object contains SpeechRecognitionResult objects.
    // It has a getter so it can be accessed like an array
    // The first [0] returns the SpeechRecognitionResult at the last position.
    // Each SpeechRecognitionResult object contains SpeechRecognitionAlternative objects that contain individual results.
    // These also have getters so they can be accessed like arrays.
    // The second [0] returns the SpeechRecognitionAlternative at position 0.
    // We then return the transcript property of the SpeechRecognitionAlternative object
    const transcription = event.results[0][0].transcript;
    audioLog.onResult(transcription);

    const result = Speech.cleanKey(transcription);
    const match = this.gramar[result];
    if (match) {
      audioLog.onSpeechGrammarMatch(true, match, result);
    } else {
      // try to find closest match
      const closestMatchId = Speech.findClosestMatch(
        this.dictionaryLookupTable,
        result
      );
      audioLog.onSpeechGrammarMatch(false, closestMatchId, result);
    }
    if (result) console.log('Confidence: ' + event.results[0][0].confidence);
  };

  onSpeechEnd = () => {
    audioLog.onSpeechEnd();
    this.recognition.stop();
  };

  onNoMatch = () => {
    console.log('no match');
  };

  onError = (event: any) => {
    console.log('Error occurred in recognition: ' + event.error);
  };

  setGrammar = (grammar: GrammarItemType[]) => {
    //@ts-ignore
    const SpeechRecognition = webkitSpeechRecognition;
    //@ts-ignore
    const SpeechGrammarList = webkitSpeechGrammarList;
    this.recognition = new SpeechRecognition();

    const speechRecognitionList = new SpeechGrammarList();
    this.gramar = {};
    grammar.forEach((item) => {
      speechRecognitionList.addFromString(item.text, 1);
      this.gramar[Speech.cleanKey(item.text)] = item;
    });

    this.recognition.grammars = speechRecognitionList;
    this.recognition.continuous = false;
    this.recognition.lang = 'en-US';
    this.recognition.interimResults = false;
    this.recognition.maxAlternatives = 1;
    this.recognition.onresult = this.onResult;
    this.recognition.onnomatch = this.onNoMatch;
    this.recognition.onspeechend = this.onSpeechEnd;
    this.recognition.onerror = this.onError;
    this.dictionaryLookupTable = Speech.createGrammerLookupTable(grammar);
  };
  /**
   * Should return a string withough punctuation and all lowercase.
   * @param text the text with punctuation
   */
  static cleanKey = (text: string): string => {
    return text
      .replace(/[^A-Za-z0-9\s]/g, '')
      .replace(/\s{2,}/g, ' ')
      .toLocaleLowerCase()
      .trim();
  };

  static cleanAlphaNumeric = (text: string): string => {
    return text
      .replace(/[^A-Za-z\s]/g, '')
      .replace(/\s{2,}/g, ' ')
      .toLocaleLowerCase()
      .trim();
  };

  static findClosestMatch = (
    lookupTable: { [word: string]: string[] },
    spokenSentence: string
  ) => {
    const matchCounts = {};
    const words = spokenSentence.split(' ');
    words.forEach((word) => {
      const lookup = lookupTable[word];
      if (lookup) {
        Object.keys(lookup).forEach((key) => {
          const matchedId = lookup[key];
          if (matchCounts[matchedId]) {
            matchCounts[matchedId] = matchCounts[matchedId] + 1;
          } else {
            matchCounts[matchedId] = 1;
          }
        });
      }
    });
    let max = {
      id: 'undefined',
      count: 0
    };
    Object.keys(matchCounts).forEach((key) => {
      const count = matchCounts[key];
      if (max.count < count) {
        max = {
          id: key,
          count
        };
      }
    });

    return max.id;
  };

  /**
   * Find closest match for a given sentence based on a lookup table
   * @param lookupTable { [word: string]: string[] } a hashed key value pair where the key is a word and they value is an array of ids of the object where this word occurs
   * @param spokenSentence the sentence we try to match
   */
  static createGrammerLookupTable = (
    grammar: GrammarItemType[]
  ): { [word: string]: string[] } => {
    const lookup: { [word: string]: string[] } = {};
    grammar.forEach((entry) => {
      const words = entry.text.split(' ');
      words.forEach((w) => {
        const word = Speech.cleanAlphaNumeric(w);
        if (word !== '') {
          if (lookup[word]) {
            lookup[word] = [...lookup[word], entry.id];
          } else {
            lookup[word] = [entry.id];
          }
        }
      });
    });
    return lookup;
  };
}