@broadreachalliance/q-assistant
Version:
A React-based voice assistant
348 lines (344 loc) • 12.3 kB
JavaScript
"use client";
import { useEffect, useState, useRef } from "react";
import axios from "axios";
import config from "../config";
import { engagingVoices, errorMessages, listeningStoppedVoices, listeningVoices } from "./listening-voices";
// import { usePathname, useSearchParams } from 'next/navigation'
const useAssistant = () => {
const finalTranscriptRef = useRef("");
const speechRecognitionref = useRef(null);
const [state, setState] = useState("idle"); // idle | listening | responding | processing
const [message, setMessage] = useState(); // text message to be displayed in the UI
const messageTimeoutRef = useRef(null);
const idleTimeoutRef = useRef(null);
const restartTimeoutRef = useRef(null);
const manualTurnoffRef = useRef(false);
const audioContextRef = useRef(null);
const currentSourceRef = useRef(null);
const hasRefreshed = useRef();
// const pathname = usePathname();
// const searchParams = useSearchParams();
const stateRef = useRef("idle");
const greetingRegEx = /^(?:\s*(hi|hello|bye|good\s+(morning|night)|how\s+are\s+you|thank\s*you|thanks|ok(?:ay)?)\s*[!.]?\s*){1,5}$/i;
useEffect(() => {
const initInteractions = () => {
const sttSupported = "SpeechRecognition" in window || "webkitSpeechRecognition" in window;
if (sttSupported) {
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
speechRecognitionref.current = new SpeechRecognition();
speechRecognitionref.current.continuous = true;
speechRecognitionref.current.interimResults = true;
speechRecognitionref.current.lang = "en-US";
speechRecognitionref.current.onresult = event => {
if (stateRef.current != "listening") return;
handleRecognitionResult(event);
};
speechRecognitionref.current.onerror = event => {
console.error("Speech recognition error", event.error);
if (event.error === "no-speech" || event.error === "audio-capture") {
if (manualTurnoffRef.current == false) {
softRestart();
}
}
};
speechRecognitionref.current.onend = () => {
if (manualTurnoffRef.current == false) {
softRestart();
}
};
} else {
console.error("Speech recognition not supported in this browser.");
}
return () => {
if (speechRecognitionref.current) {
speechRecognitionref.current.stop();
}
};
};
if (typeof window !== "undefined") {
initInteractions();
}
return () => {
stopListening();
};
}, []);
// useEffect(() => {
// // console.log(pathname)
// // if (pathname === '/dashboard') {
// // if (hasRefreshed.current == false) {
// // hasRefreshed.current = true; // Mark it so we don't refresh again
// window.location.reload();
// console.log("Navigated to Home screen");
// console.log("Refreshing page");
// // }
// // } else {
// // hasRefreshed.current = false;
// // }
// }, [pathname, searchParams])
useEffect(() => {
stateRef.current = state;
}, [state]);
const unlockAudio = async () => {
const AudioContext = window.AudioContext || window.webkitAudioContext;
if (!audioContextRef.current || audioContextRef.current.state === "closed") {
console.log("Creating Audio Context");
const audioCtx = new AudioContext();
await audioCtx.resume().then(() => {
audioContextRef.current = audioCtx;
console.log('Audio Context Created');
});
}
try {
console.log("Checking Audio Context State");
if (audioContextRef.current.state === "suspended") {
console.log("Resuming Audio Context");
await audioContextRef.current.resume();
console.log("Audio context resumed!");
}
} catch (e) {
console.warn("Audio context resume failed", e);
}
};
const softRestart = () => {
clearTimeout(restartTimeoutRef.current);
restartTimeoutRef.current = setTimeout(() => {
try {
speechRecognitionref.current.abort();
speechRecognitionref.current.start();
} catch (e) {
console.error("Restart failed", e);
}
}, 200);
};
const startListening = async intro => {
if (typeof window !== "undefined") {
console.log("Listening Started.");
console.log("Checking Audio Context");
if (!audioContextRef.current) {
console.log("Audio Context not Available");
await unlockAudio();
}
await unlockAudioContext();
if (!process.env.NEXT_PUBLIC_AI_ASSISTANT_HOST) {
const voice = errorMessages[0];
playAudioFromBase64(voice, false, true, true);
stopListening();
return null;
} else if (speechRecognitionref.current) {
if (intro !== false) {
const voice = getRandomVoice(listeningVoices);
playAudioFromBase64(voice, true, false, true);
// setTimeout(() => {
// speechRecognitionref.current.start();
// }, 1000)
} else {
speechRecognitionref.current.start();
}
manualTurnoffRef.current = false;
if (idleTimeoutRef.current) {
clearTimeout(idleTimeoutRef.current);
}
idleTimeoutRef.current = setTimeout(() => {
if (!finalTranscriptRef.current.trim() && manualTurnoffRef.current == false) {
const voice = listeningStoppedVoices[0];
manualTurnoffRef.current = true;
playAudioFromBase64(voice, false, true);
stopListening();
}
}, 20000);
setState("listening");
}
}
};
const stopListening = systemCall => {
if (speechRecognitionref.current) {
if (systemCall != true) {
manualTurnoffRef.current = true;
}
speechRecognitionref.current.stop();
setState("idle");
}
};
const expandAbbreviations = text => {
const abbreviations = {
'St\\.': 'Street',
'Ave\\.': 'Avenue',
'Rd\\.': 'Road',
'Dr\\.': 'Drive',
'Blvd\\.': 'Boulevard',
'Ln\\.': 'Lane',
'Ct\\.': 'Court',
'Pl\\.': 'Place',
'Ter\\.': 'Terrace',
'Pkwy\\.': 'Parkway',
'Cir\\.': 'Circle',
'Hwy\\.': 'Highway',
'Trl\\.': 'Trail',
'Sq\\.': 'Square',
'Aly\\.': 'Alley',
'Cres\\.': 'Crescent',
'Rte\\.': 'Route'
};
for (const [abbr, full] of Object.entries(abbreviations)) {
const regex = new RegExp(`\\b${abbr}\\b`, 'gi');
text = text.replace(regex, full);
}
return text;
};
const handleRecognitionResult = event => {
for (let i = event.resultIndex; i < event.results.length; i++) {
const currentTranscript = event.results[i][0].transcript;
if (event.results[i].isFinal) {
finalTranscriptRef.current += ". " + currentTranscript;
}
}
if (messageTimeoutRef.current) {
clearTimeout(messageTimeoutRef.current);
}
messageTimeoutRef.current = setTimeout(() => {
sendMessage();
}, 2000);
if (idleTimeoutRef.current) {
clearTimeout(idleTimeoutRef.current);
}
idleTimeoutRef.current = setTimeout(() => {
if (!finalTranscriptRef.current.trim() && manualTurnoffRef.current == false) {
const voice = listeningStoppedVoices[0];
playAudioFromBase64(voice, false, true);
stopListening();
}
}, 20000);
};
const unlockAudioContext = async () => {
console.log("Context Status >> ", audioContextRef.current?.state);
if (audioContextRef.current?.state !== "running") {
console.log("Unlocking Audio");
await audioContextRef.current.resume();
}
};
const getRandomVoice = voices => {
let index = Math.floor(Math.random() * voices.length);
return voices[index];
};
const base64ToArrayBuffer = base64 => {
const binaryString = atob(base64); // decode base64 to binary
const len = binaryString.length;
const bytes = new Uint8Array(len);
for (let i = 0; i < len; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return bytes.buffer;
};
const handlePlaybackEnd = (continueListening, listeningStopped, changeState) => {
if (continueListening == true && manualTurnoffRef.current != true) {
startListening(false);
}
if (changeState == true) {
if (listeningStopped == true) {
setState("idle");
} else {
setTimeout(() => {
setState("listening");
if (idleTimeoutRef.current) {
clearTimeout(idleTimeoutRef.current);
}
idleTimeoutRef.current = setTimeout(async () => {
if (!finalTranscriptRef.current.trim() && manualTurnoffRef.current == false) {
const voice = listeningStoppedVoices[0];
await playAudioFromBase64(voice, false, true);
stopListening();
}
}, 20000);
}, 200);
}
}
};
const playAudioFromBase64 = async (audio_base64, continueListening, listeningStopped, changeState = false) => {
if (!audioContextRef.current) return;
unlockAudioContext();
if (currentSourceRef.current) {
try {
console.log("Clearing Existing Audios");
await currentSourceRef.current.stop();
await currentSourceRef.current.disconnect();
} catch (e) {
console.warn("Error stopping previous audio:", e);
}
currentSourceRef.current = null;
}
try {
const arrayBuffer = base64ToArrayBuffer(audio_base64);
const audioBuffer = await audioContextRef.current.decodeAudioData(arrayBuffer);
const source = audioContextRef.current.createBufferSource();
source.buffer = audioBuffer;
await source.connect(audioContextRef.current.destination);
currentSourceRef.current = source;
console.log("Calling Delay");
setTimeout(() => {
console.log("Playing Audio");
source.start(0);
}, 1000);
if (changeState == true) {
setState("responding");
}
source.addEventListener("ended", () => {
currentSourceRef.current = null;
handlePlaybackEnd(continueListening, listeningStopped, changeState);
});
} catch (err) {
console.error("Failed to play audio:", err);
}
};
const sendMessage = async () => {
if (idleTimeoutRef.current) {
clearTimeout(idleTimeoutRef.current);
}
if (typeof window !== "undefined") {
let messageToSend = finalTranscriptRef.current;
messageToSend = expandAbbreviations(messageToSend);
setMessage(messageToSend);
if (!messageToSend.trim()) return;
// stopListening(true);
finalTranscriptRef.current = "";
const userData = localStorage.getItem(config.USER_DETAIL_STORAGE_KEY);
try {
setState("processing");
const voiceDelay = new Promise(resolve => {
setTimeout(() => {
if (messageToSend?.length > 25 || greetingRegEx.test(messageToSend) == true) {
const voice = getRandomVoice(engagingVoices);
playAudioFromBase64(voice, false, false);
resolve();
}
}, 100);
});
const request = await axios.post(process.env.NEXT_PUBLIC_AI_ASSISTANT_HOST, {
messages: messageToSend,
thread_id: JSON.parse(userData)?.userID ?? "new_user",
debug: false
}, {
headers: {
Authorization: `Bearer ${localStorage.getItem("token")?.replace(/"/g, '')}`,
"Content-Type": "application/json"
}
});
const response = await Promise.race([request, voiceDelay]);
const finalResponse = response?.data ? response : await request;
let audio_base64 = finalResponse.data.audio;
// speechRecognitionref.current?.stop();
playAudioFromBase64(audio_base64, false, false, true);
} catch (error) {
console.error("Error:", error);
const voice = errorMessages[0];
playAudioFromBase64(voice, false, false, true);
}
}
};
return {
startListening,
stopListening,
state,
message
};
};
export default useAssistant;