@broadreachalliance/q-assistant
Version:
A React-based voice assistant
291 lines (281 loc) • 10.2 kB
JavaScript
"use client";
import { useEffect, useState, useRef } from "react";
import axios from "axios";
import config from "../config";
const useAssistant = () => {
const finalTranscriptRef = useRef("");
const speechRecognitionref = useRef(null);
const [state, setState] = useState("idle"); // idle | listening | responding | processing
const [message, setMessage] = useState(); // text message to be displayed in the UI
const messageTimeoutRef = useRef(null);
const idleTimeoutRef = useRef(null);
const restartTimeoutRef = useRef(null);
const manualTurnoffRef = useRef(false);
const audioUnlockedRef = useRef(false);
useEffect(() => {
const initInteractions = () => {
const sttSupported = "SpeechRecognition" in window || "webkitSpeechRecognition" in window;
if (sttSupported) {
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
speechRecognitionref.current = new SpeechRecognition();
speechRecognitionref.current.continuous = true;
speechRecognitionref.current.interimResults = true;
speechRecognitionref.current.lang = "en-US";
speechRecognitionref.current.onresult = event => {
handleRecognitionResult(event);
};
speechRecognitionref.current.onerror = event => {
console.error("Speech recognition error", event.error);
if (event.error === "no-speech" || event.error === "audio-capture") {
if (manualTurnoffRef.current == false) {
softRestart();
}
}
// if (event.error == "no-speech") {
// setState("idle");
// }
};
// speechRecognitionref.current.onend = (event) => {
// console.error("Ended >>>>> ", event);
// softRestart();
// }
// speechRecognitionref.current.onend = () => {
// setTimeout(() => {
// if (state == "listening" || state == "idle") {
// setState("idle");
// }
// }, 500);
// };
} else {
console.error("Speech recognition not supported in this browser.");
}
return () => {
if (speechRecognitionref.current) {
speechRecognitionref.current.stop();
}
};
};
if (typeof window !== "undefined") {
initInteractions();
}
return () => {
stopListening();
};
}, []);
const handleUnlock = () => {
const AudioContext = window.AudioContext || window.webkitAudioContext;
const ctx = new AudioContext();
// Create an empty (silent) buffer and play it
const buffer = ctx.createBuffer(1, 1, 22050);
const source = ctx.createBufferSource();
source.buffer = buffer;
source.connect(ctx.destination);
source.start(0);
// Resume context to fully unlock audio system
ctx.resume().then(() => {
console.log('Audio unlocked on iOS!');
});
};
const softRestart = () => {
clearTimeout(restartTimeoutRef.current);
restartTimeoutRef.current = setTimeout(() => {
try {
speechRecognitionref.current.abort(); // Smoothly reset instead of stop/start
speechRecognitionref.current.start();
} catch (e) {
console.error("Restart failed", e);
}
}, 200); // small buffer to avoid immediate failure
};
const startListening = intro => {
if (typeof window !== "undefined") {
if (!process.env.NEXT_PUBLIC_AI_ASSISTANT_HOST) {
speakText("Unable to process your request.", false);
stopListening();
return null;
} else if (speechRecognitionref.current) {
handleUnlock();
if (intro !== false) {
speakText("I am listening");
}
manualTurnoffRef.current = false;
speechRecognitionref.current.start();
if (idleTimeoutRef.current) {
clearTimeout(idleTimeoutRef.current);
}
idleTimeoutRef.current = setTimeout(() => {
if (!finalTranscriptRef.current.trim() && manualTurnoffRef.current == false) {
speakText("Listening stopped.", false);
stopListening();
}
}, 20000);
setState("listening");
}
}
};
const stopListening = systemCall => {
if (speechRecognitionref.current) {
if (systemCall != true) {
manualTurnoffRef.current = true;
}
speechRecognitionref.current.stop();
setState("idle");
}
};
const handleRecognitionResult = event => {
for (let i = event.resultIndex; i < event.results.length; i++) {
const currentTranscript = event.results[i][0].transcript;
if (event.results[i].isFinal) {
finalTranscriptRef.current += ". " + currentTranscript;
}
}
if (messageTimeoutRef.current) {
clearTimeout(messageTimeoutRef.current);
}
messageTimeoutRef.current = setTimeout(() => {
sendMessage();
}, 2000);
if (idleTimeoutRef.current) {
clearTimeout(idleTimeoutRef.current);
}
idleTimeoutRef.current = setTimeout(() => {
if (!finalTranscriptRef.current.trim() && manualTurnoffRef.current == false) {
speakText("Listening stopped.", false);
stopListening();
}
}, 20000);
};
// const unlockAudio = () => {
// if (audioUnlockedRef.current == true) return;
// const silence = `UklGRiQAAABXQVZFZm10IBAAAAABAAEAIlYAAESsAAACABAAZGF0YQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA`
// const audioSrc = `data:audio/mpeg;base64,${silence}`
// const audio = new Audio(audioSrc);
// audio.muted = true;
// const onPlay = () => {
// console.log("Audio unlocked via event!");
// audioUnlockedRef.current = true;
// audio.removeEventListener('playing', onPlay);
// };
// audio.addEventListener('playing', onPlay);
// try {
// audio.play();
// } catch (e) {
// console.warn("Play failed immediately:", e);
// }
// }
function playBase64Audio(audio_base64) {
const byteCharacters = atob(audio_base64);
const byteNumbers = new Array(byteCharacters.length);
for (let i = 0; i < byteCharacters.length; i++) {
byteNumbers[i] = byteCharacters.charCodeAt(i);
}
const byteArray = new Uint8Array(byteNumbers);
const blob = new Blob([byteArray], {
type: 'audio/mpeg'
});
const blobUrl = URL.createObjectURL(blob);
const audio = new Audio(blobUrl);
audio.play();
audio.onplay = () => {
setState("responding");
};
audio.onended = () => {
if (manualTurnoffRef.current != true) {
startListening(false);
} else {
setState("idle");
}
};
}
const sendMessage = async () => {
if (idleTimeoutRef.current) {
clearTimeout(idleTimeoutRef.current);
}
if (typeof window !== "undefined") {
const messageToSend = finalTranscriptRef.current;
setMessage(messageToSend);
// unlockAudio();
if (!messageToSend.trim()) return;
stopListening(true);
finalTranscriptRef.current = "";
const userData = localStorage.getItem(config.USER_DETAIL_STORAGE_KEY);
try {
setState("processing");
const response = await axios.post(process.env.NEXT_PUBLIC_AI_ASSISTANT_HOST, {
messages: messageToSend,
thread_id: JSON.parse(userData)?.userID ?? "new_user",
debug: false
}, {
headers: {
// Authorization: `Bearer ${localStorage.getItem("token")?.replace(/"/g, '')}`,
"Content-Type": "application/json"
}
});
let audio_base64 = response.data.audio;
// lines = lines.replace(/\s*\(.*?\)\s*/g, " ").trim();
// lines = lines.replaceAll("**", " ");
// lines = lines.replace(/\b([aApP])\.(m|M)\b/g, "$1$2");
speechRecognitionref.current?.stop();
playBase64Audio(audio_base64);
// speakText(lines, true);
} catch (error) {
console.error("Error:", error);
speakText("Sorry, there was an error processing your message.", true);
}
}
};
const getDefaultVoice = availableVoices => {
let defaultVoice = availableVoices.find(voice => voice.name === "Eddy" && voice.lang === "en-US");
if (!defaultVoice) {
defaultVoice = availableVoices.find(voice => voice.name === "Google UK English Male" && voice.lang === "en-GB");
}
if (!defaultVoice) {
defaultVoice = availableVoices.find(voice => voice.name === "Microsoft Mark - English (United States)" && voice.lang === "en-US");
}
if (!defaultVoice) {
defaultVoice = availableVoices?.[0];
}
return defaultVoice;
};
const speakText = (text, continueListening) => {
if (typeof window !== "undefined") {
text = text.replace(/\s*\(.*?\)\s*/g, ' ').trim();
if (!window.speechSynthesis) return;
window.speechSynthesis.cancel();
const utterance = new SpeechSynthesisUtterance(text);
const selectedVoice = localStorage.getItem("selectedVoice");
const availableVoices = window.speechSynthesis.getVoices();
if (selectedVoice) {
const voiceParts = selectedVoice?.split("$");
const IdentifiedVoice = availableVoices.find(voice => voice.name === voiceParts?.[0] && voice.lang === voiceParts?.[1]);
utterance.voice = IdentifiedVoice;
} else {
const defaultVoice = getDefaultVoice(availableVoices);
utterance.voice = defaultVoice;
}
utterance.rate = 1;
utterance.onstart = () => {
setState("responding");
};
utterance.onend = () => {
if (continueListening == true) {
startListening(false);
} else {
if (text == "Unable to process your request." || text == "Listening stopped.") {
setState("idle");
} else {
setState("listening");
}
}
};
window.speechSynthesis.speak(utterance);
}
};
return {
startListening,
stopListening,
state,
message
};
};
export default useAssistant;