@copilotkit/react-ui
Version:
<div align="center"> <a href="https://copilotkit.ai" target="_blank"> <img src="https://github.com/copilotkit/copilotkit/raw/main/assets/banner.png" alt="CopilotKit Logo"> </a>
1 lines • 8.93 kB
Source Map (JSON)
{"version":3,"sources":["../src/hooks/use-push-to-talk.tsx"],"sourcesContent":["import { useCopilotContext, useCopilotMessagesContext } from \"@copilotkit/react-core\";\nimport { Message, TextMessage } from \"@copilotkit/runtime-client-gql\";\nimport { MutableRefObject, useEffect, useRef, useState } from \"react\";\n\nexport const checkMicrophonePermission = async () => {\n try {\n const permissionStatus = await navigator.permissions.query({\n name: \"microphone\" as PermissionName,\n });\n if (permissionStatus.state === \"granted\") {\n return true;\n } else {\n return false;\n }\n } catch (err) {\n console.error(\"Error checking microphone permission\", err);\n }\n};\n\nexport const requestMicAndPlaybackPermission = async () => {\n try {\n const stream = await navigator.mediaDevices.getUserMedia({ audio: true });\n const audioContext = new window.AudioContext();\n await audioContext.resume();\n return { stream, audioContext };\n } catch (err) {\n console.error(\"Error requesting microphone and playback permissions\", err);\n return null;\n }\n};\n\nconst startRecording = async (\n mediaStreamRef: MutableRefObject<MediaStream | null>,\n mediaRecorderRef: MutableRefObject<MediaRecorder | null>,\n audioContextRef: MutableRefObject<AudioContext | null>,\n recordedChunks: Blob[],\n onStop: () => void,\n) => {\n if (!mediaStreamRef.current || !audioContextRef.current) {\n mediaStreamRef.current = await navigator.mediaDevices.getUserMedia({ audio: true });\n audioContextRef.current = new window.AudioContext();\n await audioContextRef.current.resume();\n }\n\n mediaRecorderRef.current = new MediaRecorder(mediaStreamRef.current!);\n mediaRecorderRef.current.start(1000);\n mediaRecorderRef.current.ondataavailable = (event) => {\n recordedChunks.push(event.data);\n };\n mediaRecorderRef.current.onstop = onStop;\n};\n\nconst stopRecording = (mediaRecorderRef: MutableRefObject<MediaRecorder | null>) => {\n if (mediaRecorderRef.current && mediaRecorderRef.current.state !== \"inactive\") {\n mediaRecorderRef.current.stop();\n }\n};\n\nconst transcribeAudio = async (recordedChunks: Blob[], transcribeAudioUrl: string) => {\n const completeBlob = new Blob(recordedChunks, { type: \"audio/mp4\" });\n const formData = new FormData();\n formData.append(\"file\", completeBlob, \"recording.mp4\");\n\n const response = await fetch(transcribeAudioUrl, {\n method: \"POST\",\n body: formData,\n });\n\n if (!response.ok) {\n throw new Error(`Error: ${response.statusText}`);\n }\n\n const transcription = await response.json();\n return transcription.text;\n};\n\nconst playAudioResponse = (text: string, textToSpeechUrl: string, audioContext: AudioContext) => {\n const encodedText = encodeURIComponent(text);\n const url = `${textToSpeechUrl}?text=${encodedText}`;\n\n fetch(url)\n .then((response) => response.arrayBuffer())\n .then((arrayBuffer) => audioContext.decodeAudioData(arrayBuffer))\n .then((audioBuffer) => {\n const source = audioContext.createBufferSource();\n source.buffer = audioBuffer;\n source.connect(audioContext.destination);\n source.start(0);\n })\n .catch((error) => {\n console.error(\"Error with decoding audio data\", error);\n });\n};\n\nexport type PushToTalkState = \"idle\" | \"recording\" | \"transcribing\";\n\nexport type SendFunction = (text: string) => Promise<Message>;\n\nexport const usePushToTalk = ({\n sendFunction,\n inProgress,\n}: {\n sendFunction: SendFunction;\n inProgress: boolean;\n}) => {\n const [pushToTalkState, setPushToTalkState] = useState<PushToTalkState>(\"idle\");\n const mediaStreamRef = useRef<MediaStream | null>(null);\n const audioContextRef = useRef<AudioContext | null>(null);\n const mediaRecorderRef = useRef<MediaRecorder | null>(null);\n const recordedChunks = useRef<Blob[]>([]);\n const generalContext = useCopilotContext();\n const messagesContext = useCopilotMessagesContext();\n const context = { ...generalContext, ...messagesContext };\n const [startReadingFromMessageId, setStartReadingFromMessageId] = useState<string | null>(null);\n\n useEffect(() => {\n if (pushToTalkState === \"recording\") {\n startRecording(\n mediaStreamRef,\n mediaRecorderRef,\n audioContextRef,\n recordedChunks.current,\n () => {\n setPushToTalkState(\"transcribing\");\n },\n );\n } else {\n stopRecording(mediaRecorderRef);\n if (pushToTalkState === \"transcribing\") {\n transcribeAudio(recordedChunks.current, context.copilotApiConfig.transcribeAudioUrl!).then(\n async (transcription) => {\n recordedChunks.current = [];\n setPushToTalkState(\"idle\");\n const message = await sendFunction(transcription);\n setStartReadingFromMessageId(message.id);\n },\n );\n }\n }\n\n return () => {\n stopRecording(mediaRecorderRef);\n };\n }, [pushToTalkState]);\n\n useEffect(() => {\n if (inProgress === false && startReadingFromMessageId) {\n const lastMessageIndex = context.messages.findIndex(\n (message) => message.id === startReadingFromMessageId,\n );\n\n const messagesAfterLast = context.messages\n .slice(lastMessageIndex + 1)\n .filter(\n (message) => message.isTextMessage() && message.role === \"assistant\",\n ) as TextMessage[];\n\n const text = messagesAfterLast.map((message) => message.content).join(\"\\n\");\n playAudioResponse(text, context.copilotApiConfig.textToSpeechUrl!, audioContextRef.current!);\n\n setStartReadingFromMessageId(null);\n }\n }, [startReadingFromMessageId, inProgress]);\n\n return { pushToTalkState, setPushToTalkState };\n};\n"],"mappings":";;;;;;AAAA,SAAS,mBAAmB,iCAAiC;AAE7D,SAA2B,WAAW,QAAQ,gBAAgB;AAEvD,IAAM,4BAA4B,MAAY;AACnD,MAAI;AACF,UAAM,mBAAmB,MAAM,UAAU,YAAY,MAAM;AAAA,MACzD,MAAM;AAAA,IACR,CAAC;AACD,QAAI,iBAAiB,UAAU,WAAW;AACxC,aAAO;AAAA,IACT,OAAO;AACL,aAAO;AAAA,IACT;AAAA,EACF,SAAS,KAAP;AACA,YAAQ,MAAM,wCAAwC,GAAG;AAAA,EAC3D;AACF;AAEO,IAAM,kCAAkC,MAAY;AACzD,MAAI;AACF,UAAM,SAAS,MAAM,UAAU,aAAa,aAAa,EAAE,OAAO,KAAK,CAAC;AACxE,UAAM,eAAe,IAAI,OAAO,aAAa;AAC7C,UAAM,aAAa,OAAO;AAC1B,WAAO,EAAE,QAAQ,aAAa;AAAA,EAChC,SAAS,KAAP;AACA,YAAQ,MAAM,wDAAwD,GAAG;AACzE,WAAO;AAAA,EACT;AACF;AAEA,IAAM,iBAAiB,CACrB,gBACA,kBACA,iBACA,gBACA,WACG;AACH,MAAI,CAAC,eAAe,WAAW,CAAC,gBAAgB,SAAS;AACvD,mBAAe,UAAU,MAAM,UAAU,aAAa,aAAa,EAAE,OAAO,KAAK,CAAC;AAClF,oBAAgB,UAAU,IAAI,OAAO,aAAa;AAClD,UAAM,gBAAgB,QAAQ,OAAO;AAAA,EACvC;AAEA,mBAAiB,UAAU,IAAI,cAAc,eAAe,OAAQ;AACpE,mBAAiB,QAAQ,MAAM,GAAI;AACnC,mBAAiB,QAAQ,kBAAkB,CAAC,UAAU;AACpD,mBAAe,KAAK,MAAM,IAAI;AAAA,EAChC;AACA,mBAAiB,QAAQ,SAAS;AACpC;AAEA,IAAM,gBAAgB,CAAC,qBAA6D;AAClF,MAAI,iBAAiB,WAAW,iBAAiB,QAAQ,UAAU,YAAY;AAC7E,qBAAiB,QAAQ,KAAK;AAAA,EAChC;AACF;AAEA,IAAM,kBAAkB,CAAO,gBAAwB,uBAA+B;AACpF,QAAM,eAAe,IAAI,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnE,QAAM,WAAW,IAAI,SAAS;AAC9B,WAAS,OAAO,QAAQ,cAAc,eAAe;AAErD,QAAM,WAAW,MAAM,MAAM,oBAAoB;AAAA,IAC/C,QAAQ;AAAA,IACR,MAAM;AAAA,EACR,CAAC;AAED,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,IAAI,MAAM,UAAU,SAAS,YAAY;AAAA,EACjD;AAEA,QAAM,gBAAgB,MAAM,SAAS,KAAK;AAC1C,SAAO,cAAc;AACvB;AAEA,IAAM,oBAAoB,CAAC,MAAc,iBAAyB,iBAA+B;AAC/F,QAAM,cAAc,mBAAmB,IAAI;AAC3C,QAAM,MAAM,GAAG,wBAAwB;AAEvC,QAAM,GAAG,EACN,KAAK,CAAC,aAAa,SAAS,YAAY,CAAC,EACzC,KAAK,CAAC,gBAAgB,aAAa,gBAAgB,WAAW,CAAC,EAC/D,KAAK,CAAC,gBAAgB;AACrB,UAAM,SAAS,aAAa,mBAAmB;AAC/C,WAAO,SAAS;AAChB,WAAO,QAAQ,aAAa,WAAW;AACvC,WAAO,MAAM,CAAC;AAAA,EAChB,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,YAAQ,MAAM,kCAAkC,KAAK;AAAA,EACvD,CAAC;AACL;AAMO,IAAM,gBAAgB,CAAC;AAAA,EAC5B;AAAA,EACA;AACF,MAGM;AACJ,QAAM,CAAC,iBAAiB,kBAAkB,IAAI,SAA0B,MAAM;AAC9E,QAAM,iBAAiB,OAA2B,IAAI;AACtD,QAAM,kBAAkB,OAA4B,IAAI;AACxD,QAAM,mBAAmB,OAA6B,IAAI;AAC1D,QAAM,iBAAiB,OAAe,CAAC,CAAC;AACxC,QAAM,iBAAiB,kBAAkB;AACzC,QAAM,kBAAkB,0BAA0B;AAClD,QAAM,UAAU,kCAAK,iBAAmB;AACxC,QAAM,CAAC,2BAA2B,4BAA4B,IAAI,SAAwB,IAAI;AAE9F,YAAU,MAAM;AACd,QAAI,oBAAoB,aAAa;AACnC;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA,eAAe;AAAA,QACf,MAAM;AACJ,6BAAmB,cAAc;AAAA,QACnC;AAAA,MACF;AAAA,IACF,OAAO;AACL,oBAAc,gBAAgB;AAC9B,UAAI,oBAAoB,gBAAgB;AACtC,wBAAgB,eAAe,SAAS,QAAQ,iBAAiB,kBAAmB,EAAE;AAAA,UACpF,CAAO,kBAAkB;AACvB,2BAAe,UAAU,CAAC;AAC1B,+BAAmB,MAAM;AACzB,kBAAM,UAAU,MAAM,aAAa,aAAa;AAChD,yCAA6B,QAAQ,EAAE;AAAA,UACzC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,MAAM;AACX,oBAAc,gBAAgB;AAAA,IAChC;AAAA,EACF,GAAG,CAAC,eAAe,CAAC;AAEpB,YAAU,MAAM;AACd,QAAI,eAAe,SAAS,2BAA2B;AACrD,YAAM,mBAAmB,QAAQ,SAAS;AAAA,QACxC,CAAC,YAAY,QAAQ,OAAO;AAAA,MAC9B;AAEA,YAAM,oBAAoB,QAAQ,SAC/B,MAAM,mBAAmB,CAAC,EAC1B;AAAA,QACC,CAAC,YAAY,QAAQ,cAAc,KAAK,QAAQ,SAAS;AAAA,MAC3D;AAEF,YAAM,OAAO,kBAAkB,IAAI,CAAC,YAAY,QAAQ,OAAO,EAAE,KAAK,IAAI;AAC1E,wBAAkB,MAAM,QAAQ,iBAAiB,iBAAkB,gBAAgB,OAAQ;AAE3F,mCAA6B,IAAI;AAAA,IACnC;AAAA,EACF,GAAG,CAAC,2BAA2B,UAAU,CAAC;AAE1C,SAAO,EAAE,iBAAiB,mBAAmB;AAC/C;","names":[]}