sinsintro-ai-assistant-dh/src/frontend/VoiceAssistant.jsx

A single npm package providing a React-based voice assistant with OpenAI function-calling, purely client-side.

// src/components/VoiceAssistant.jsx

import React, { useState, useRef, useEffect } from "react";
import { createAssistant } from "../backend/index.js";
import "./VoiceAssistant.css";

const SpeechRecognition =
  window.SpeechRecognition || window.webkitSpeechRecognition;

const BASE_URL = "https://digital-human-server-2yshz.ondigitalocean.app/digital-human";

export default function VoiceAssistant({ apiKey, functions = [], dbId }) {
  const [isListening, setIsListening] = useState(false);
  const [isProcessing, setIsProcessing] = useState(false);
  const [interimText, setInterimText] = useState("");

  const assistantRef = useRef(null);
  const recognitionRef = useRef(null);
  const pendingSpeechRef = useRef("");
  const finalDebounceRef = useRef(null);
  const manualStopRef = useRef(false);
  const counterRef = useRef(true);
  const firstRunRef = useRef(true);

  // 1) On mount, init assistant and do first poll
  useEffect(() => {
    (async () => {
      assistantRef.current = await createAssistant({ apiKey, functions });
      // if (counterRef.current) {
      //   pollReady();
      //   counterRef.current = false;
      // }
    })();
    return () => {
      stopListening();
      clearTimeout(finalDebounceRef.current);
    };
  }, []);

  useEffect(() => {
  }, [isListening]);

  // 2) Poll until digital-human says ready, then start listening
  async function pollReady() {
    try {
      const { ready } = await fetchReady();
      if (ready) {
        startListening();
      } else {
        if(isListening || recognitionRef.current) {
          stopListening();
        }
        setTimeout(pollReady, 1000);
      }
    } catch (error) {

      if(!error.message.includes("recognition has already started")) {
        setTimeout(pollReady, 1000);
      }
    }
  }

  // 3) Recognition setup
  function initRecognition() {
    const rec = new SpeechRecognition();
    rec.continuous = true;
    rec.interimResults = true;
    rec.lang = "ar-SA";

    rec.onresult = handleResult;
    rec.onerror = (e) => {console.warn("Speech error:", e.error);};
    rec.onend = () => {handleError()};

    return rec;
  }

  function startListening() {
    if (isListening || isProcessing || manualStopRef.current) return;
    if (!recognitionRef.current) {
      recognitionRef.current = initRecognition();
    }
    recognitionRef.current.start();
    setIsListening(true);
    setInterimText("");
  }

  function stopListening() {
    if (recognitionRef.current) {
      recognitionRef.current.stop();
    }
    setIsListening(false);
    pendingSpeechRef.current = "";
    counterRef.current = true;
    clearTimeout(finalDebounceRef.current);
  }

  function handleError() {
    stopListening();
    setTimeout(()=>{
      pollReady();
    }, 1000);
  }

  // 4) Debounce final chunks at 200 ms after you pause
  function handleResult(event) {
    let interim = "";
    for (let i = event.resultIndex; i < event.results.length; i++) {
      const res = event.results[i];
      if (res.isFinal) {
        pendingSpeechRef.current += res[0].transcript + " ";
      } else {
        interim += res[0].transcript;
      }
    }
    setInterimText(interim.trim());

    if (pendingSpeechRef.current) {
      clearTimeout(finalDebounceRef.current);
      finalDebounceRef.current = setTimeout(() => {
        const phrase = pendingSpeechRef.current.trim();
        pendingSpeechRef.current = "";
        if (phrase) commitSpeech(phrase);
      }, 5);
    }
  }

  // 5) Send to AI + Digital-Human, then poll again
  async function commitSpeech(text) {
    stopListening();
    setIsProcessing(true);

    try {
      const aiReply = await assistantRef.current.chatWithFunctions(text);

      // send to digital‐human
      await postText(aiReply);
      await setFlag(true);
      await setReady(false);

      // only now do we start polling again
      if (counterRef.current) {
        counterRef.current = false;
        setTimeout(() => {
          pollReady();
        }, 1000);
      }
    } catch (err) {
      console.error("Pipeline error:", err);
      if (counterRef.current) {
        counterRef.current = false;
        setTimeout(() => {
          pollReady();
        }, 1000);
      }
    } finally {
      setIsProcessing(false);
    }
  }

  // 6) Manual toggle
  function toggleCircle() {
    if(firstRunRef.current){
      firstRunRef.current = false;
      pollReady();
    }
  }

  // 7) Helpers
  async function fetchReady() {
    const res = await fetch(`${BASE_URL}/${dbId}/ready`);
    if (!res.ok) throw new Error(res.statusText);
    return res.json();
  }

  async function postText(text) {
    const res = await fetch(`${BASE_URL}/${dbId}/text`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ text }),
    });
    if (!res.ok) throw new Error(res.statusText);
  }

  async function setFlag(flag) {
    const res = await fetch(`${BASE_URL}/${dbId}/flag`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ flag }),
    });
    if (!res.ok) throw new Error(res.statusText);
  }

  async function setReady(ready) {
    const res = await fetch(`${BASE_URL}/${dbId}/ready`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ ready }),
    });
    if (!res.ok) throw new Error(res.statusText);
  }

  // 8) Render
  const circleClass = isProcessing
    ? "voice-assistant-circle processing"
    : isListening
    ? "voice-assistant-circle active"
    : "voice-assistant-circle idle";

  return (
    <div className="voice-assistant-container">
      {/* <div className="transcript">
        {interimText || (isProcessing ? "…processing" : "Say something")}
      </div> */}
      <div
        className={circleClass}
        onClick={toggleCircle}
        title={isListening ? "Click to stop" : "Click to start"}
      />
    </div>
  );
}