@copilotkit/runtime

Version:

github.com/CopilotKit/CopilotKit

CopilotKit/CopilotKit

1 lines • 11.2 kB

Source Map (JSON)

View Raw

{"version":3,"file":"handle-transcribe.mjs","names":[],"sources":["../../../../src/v2/runtime/handlers/handle-transcribe.ts"],"sourcesContent":["import type { CopilotRuntimeLike } from \"../core/runtime\";\nimport {\n TranscriptionErrorCode,\n TranscriptionErrors,\n type TranscriptionErrorResponse,\n} from \"@copilotkit/shared\";\n\n/**\n * HTTP status codes for transcription error codes\n */\nconst ERROR_STATUS_CODES: Record<TranscriptionErrorCode, number> = {\n [TranscriptionErrorCode.SERVICE_NOT_CONFIGURED]: 503,\n [TranscriptionErrorCode.INVALID_AUDIO_FORMAT]: 400,\n [TranscriptionErrorCode.AUDIO_TOO_LONG]: 400,\n [TranscriptionErrorCode.AUDIO_TOO_SHORT]: 400,\n [TranscriptionErrorCode.RATE_LIMITED]: 429,\n [TranscriptionErrorCode.AUTH_FAILED]: 401,\n [TranscriptionErrorCode.PROVIDER_ERROR]: 500,\n [TranscriptionErrorCode.NETWORK_ERROR]: 502,\n [TranscriptionErrorCode.INVALID_REQUEST]: 400,\n};\n\ninterface HandleTranscribeParameters {\n runtime: CopilotRuntimeLike;\n request: Request;\n}\n\ninterface Base64AudioInput {\n audio: string; // base64-encoded audio data\n mimeType: string;\n filename?: string;\n}\n\nconst VALID_AUDIO_TYPES = [\n \"audio/mpeg\",\n \"audio/mp3\",\n \"audio/mp4\",\n \"audio/wav\",\n \"audio/webm\",\n \"audio/ogg\",\n \"audio/flac\",\n \"audio/aac\",\n];\n\nfunction isValidAudioType(type: string): boolean {\n // Extract base MIME type (before semicolon) to handle types like \"audio/webm; codecs=opus\"\n const baseType = type.split(\";\")[0]?.trim() ?? \"\";\n return (\n VALID_AUDIO_TYPES.includes(baseType) ||\n baseType === \"\" ||\n baseType === \"application/octet-stream\"\n );\n}\n\nfunction createErrorResponse(\n errorResponse: TranscriptionErrorResponse,\n): Response {\n const status = ERROR_STATUS_CODES[errorResponse.error] ?? 500;\n return new Response(JSON.stringify(errorResponse), {\n status,\n headers: { \"Content-Type\": \"application/json\" },\n });\n}\n\nfunction base64ToFile(\n base64: string,\n mimeType: string,\n filename: string,\n): File {\n // Remove data URL prefix if present (e.g., \"data:audio/webm;base64,\")\n const base64Data = base64.includes(\",\")\n ? (base64.split(\",\")[1] ?? base64)\n : base64;\n\n // Decode base64 to binary\n const binaryString = atob(base64Data);\n const bytes = new Uint8Array(binaryString.length);\n for (let i = 0; i < binaryString.length; i++) {\n bytes[i] = binaryString.charCodeAt(i);\n }\n\n // Create File object\n return new File([bytes], filename, { type: mimeType });\n}\n\nasync function extractAudioFromFormData(\n request: Request,\n): Promise<{ file: File } | { error: Response }> {\n const formData = await request.formData();\n let audioFile = formData.get(\"audio\") as File | null;\n\n if (!audioFile || !(audioFile instanceof File)) {\n const err = TranscriptionErrors.invalidRequest(\n \"No audio file found in form data. Please include an 'audio' field.\",\n );\n return { error: createErrorResponse(err) };\n }\n\n if (!isValidAudioType(audioFile.type)) {\n const err = TranscriptionErrors.invalidAudioFormat(\n audioFile.type,\n VALID_AUDIO_TYPES,\n );\n return { error: createErrorResponse(err) };\n }\n\n // Browser MediaRecorder Blobs often arrive at the server with their\n // `type` field empty (and `application/octet-stream` is also accepted by\n // isValidAudioType for compatibility). OpenAI Whisper rejects files\n // without a recognizable MIME / extension with a 502 \"Invalid file\n // format\" — even though the bytes themselves are valid WebM Opus.\n // Stamp `audio/webm` (the standard MediaRecorder default in Chromium /\n // Firefox / Safari) so Whisper can pick the right decoder.\n if (!audioFile.type || audioFile.type === \"application/octet-stream\") {\n audioFile = new File([audioFile], audioFile.name || \"recording.webm\", {\n type: \"audio/webm\",\n });\n }\n\n return { file: audioFile };\n}\n\nasync function extractAudioFromJson(\n request: Request,\n): Promise<{ file: File } | { error: Response }> {\n let body: Base64AudioInput;\n\n try {\n body = await request.json();\n } catch {\n const err = TranscriptionErrors.invalidRequest(\n \"Request body must be valid JSON\",\n );\n return { error: createErrorResponse(err) };\n }\n\n if (!body.audio || typeof body.audio !== \"string\") {\n const err = TranscriptionErrors.invalidRequest(\n \"Request must include 'audio' field with base64-encoded audio data\",\n );\n return { error: createErrorResponse(err) };\n }\n\n if (!body.mimeType || typeof body.mimeType !== \"string\") {\n const err = TranscriptionErrors.invalidRequest(\n \"Request must include 'mimeType' field (e.g., 'audio/webm')\",\n );\n return { error: createErrorResponse(err) };\n }\n\n if (!isValidAudioType(body.mimeType)) {\n const err = TranscriptionErrors.invalidAudioFormat(\n body.mimeType,\n VALID_AUDIO_TYPES,\n );\n return { error: createErrorResponse(err) };\n }\n\n try {\n const filename = body.filename || \"recording.webm\";\n const file = base64ToFile(body.audio, body.mimeType, filename);\n return { file };\n } catch {\n const err = TranscriptionErrors.invalidRequest(\n \"Failed to decode base64 audio data\",\n );\n return { error: createErrorResponse(err) };\n }\n}\n\n/**\n * Categorize provider errors into appropriate transcription error responses.\n */\nfunction categorizeProviderError(error: unknown): TranscriptionErrorResponse {\n const message =\n error instanceof Error ? error.message : \"Unknown error occurred\";\n const errorStr = String(error).toLowerCase();\n\n // Check for rate limiting\n if (\n errorStr.includes(\"rate\") ||\n errorStr.includes(\"429\") ||\n errorStr.includes(\"too many\")\n ) {\n return TranscriptionErrors.rateLimited();\n }\n\n // Check for auth errors\n if (\n errorStr.includes(\"auth\") ||\n errorStr.includes(\"401\") ||\n errorStr.includes(\"api key\") ||\n errorStr.includes(\"unauthorized\")\n ) {\n return TranscriptionErrors.authFailed();\n }\n\n // Check for audio too long\n if (\n errorStr.includes(\"too long\") ||\n errorStr.includes(\"duration\") ||\n errorStr.includes(\"length\")\n ) {\n return TranscriptionErrors.audioTooLong();\n }\n\n // Default to provider error\n return TranscriptionErrors.providerError(message);\n}\n\nexport async function handleTranscribe({\n runtime,\n request,\n}: HandleTranscribeParameters) {\n try {\n // Check if transcription service is configured\n if (!runtime.transcriptionService) {\n const err = TranscriptionErrors.serviceNotConfigured();\n return createErrorResponse(err);\n }\n\n // Determine input type based on content-type header\n const contentType = request.headers.get(\"content-type\") || \"\";\n\n let extractResult: { file: File } | { error: Response };\n\n if (contentType.includes(\"multipart/form-data\")) {\n // Handle multipart/form-data (REST mode)\n extractResult = await extractAudioFromFormData(request);\n } else if (contentType.includes(\"application/json\")) {\n // Handle JSON with base64 audio (single-endpoint mode)\n extractResult = await extractAudioFromJson(request);\n } else {\n const err = TranscriptionErrors.invalidRequest(\n \"Request must be multipart/form-data or application/json with base64 audio\",\n );\n return createErrorResponse(err);\n }\n\n // Check for extraction errors\n if (\"error\" in extractResult) {\n return extractResult.error;\n }\n\n const audioFile = extractResult.file;\n\n // Transcribe the audio file\n const transcription = await runtime.transcriptionService.transcribeFile({\n audioFile,\n mimeType: audioFile.type,\n size: audioFile.size,\n });\n\n return new Response(\n JSON.stringify({\n text: transcription,\n size: audioFile.size,\n type: audioFile.type,\n }),\n {\n status: 200,\n headers: { \"Content-Type\": \"application/json\" },\n },\n );\n } catch (error) {\n // Categorize the error for better client-side handling\n return createErrorResponse(categorizeProviderError(error));\n }\n}\n"],"mappings":";;;;;;;AAUA,MAAM,qBAA6D;EAChE,uBAAuB,yBAAyB;EAChD,uBAAuB,uBAAuB;EAC9C,uBAAuB,iBAAiB;EACxC,uBAAuB,kBAAkB;EACzC,uBAAuB,eAAe;EACtC,uBAAuB,cAAc;EACrC,uBAAuB,iBAAiB;EACxC,uBAAuB,gBAAgB;EACvC,uBAAuB,kBAAkB;CAC3C;AAaD,MAAM,oBAAoB;CACxB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAED,SAAS,iBAAiB,MAAuB;CAE/C,MAAM,WAAW,KAAK,MAAM,IAAI,CAAC,IAAI,MAAM,IAAI;AAC/C,QACE,kBAAkB,SAAS,SAAS,IACpC,aAAa,MACb,aAAa;;AAIjB,SAAS,oBACP,eACU;CACV,MAAM,SAAS,mBAAmB,cAAc,UAAU;AAC1D,QAAO,IAAI,SAAS,KAAK,UAAU,cAAc,EAAE;EACjD;EACA,SAAS,EAAE,gBAAgB,oBAAoB;EAChD,CAAC;;AAGJ,SAAS,aACP,QACA,UACA,UACM;CAEN,MAAM,aAAa,OAAO,SAAS,IAAI,GAClC,OAAO,MAAM,IAAI,CAAC,MAAM,SACzB;CAGJ,MAAM,eAAe,KAAK,WAAW;CACrC,MAAM,QAAQ,IAAI,WAAW,aAAa,OAAO;AACjD,MAAK,IAAI,IAAI,GAAG,IAAI,aAAa,QAAQ,IACvC,OAAM,KAAK,aAAa,WAAW,EAAE;AAIvC,QAAO,IAAI,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;;AAGxD,eAAe,yBACb,SAC+C;CAE/C,IAAI,aADa,MAAM,QAAQ,UAAU,EAChB,IAAI,QAAQ;AAErC,KAAI,CAAC,aAAa,EAAE,qBAAqB,MAIvC,QAAO,EAAE,OAAO,oBAHJ,oBAAoB,eAC9B,qEACD,CACuC,EAAE;AAG5C,KAAI,CAAC,iBAAiB,UAAU,KAAK,CAKnC,QAAO,EAAE,OAAO,oBAJJ,oBAAoB,mBAC9B,UAAU,MACV,kBACD,CACuC,EAAE;AAU5C,KAAI,CAAC,UAAU,QAAQ,UAAU,SAAS,2BACxC,aAAY,IAAI,KAAK,CAAC,UAAU,EAAE,UAAU,QAAQ,kBAAkB,EACpE,MAAM,cACP,CAAC;AAGJ,QAAO,EAAE,MAAM,WAAW;;AAG5B,eAAe,qBACb,SAC+C;CAC/C,IAAI;AAEJ,KAAI;AACF,SAAO,MAAM,QAAQ,MAAM;SACrB;AAIN,SAAO,EAAE,OAAO,oBAHJ,oBAAoB,eAC9B,kCACD,CACuC,EAAE;;AAG5C,KAAI,CAAC,KAAK,SAAS,OAAO,KAAK,UAAU,SAIvC,QAAO,EAAE,OAAO,oBAHJ,oBAAoB,eAC9B,oEACD,CACuC,EAAE;AAG5C,KAAI,CAAC,KAAK,YAAY,OAAO,KAAK,aAAa,SAI7C,QAAO,EAAE,OAAO,oBAHJ,oBAAoB,eAC9B,6DACD,CACuC,EAAE;AAG5C,KAAI,CAAC,iBAAiB,KAAK,SAAS,CAKlC,QAAO,EAAE,OAAO,oBAJJ,oBAAoB,mBAC9B,KAAK,UACL,kBACD,CACuC,EAAE;AAG5C,KAAI;EACF,MAAM,WAAW,KAAK,YAAY;AAElC,SAAO,EAAE,MADI,aAAa,KAAK,OAAO,KAAK,UAAU,SAAS,EAC/C;SACT;AAIN,SAAO,EAAE,OAAO,oBAHJ,oBAAoB,eAC9B,qCACD,CACuC,EAAE;;;;;;AAO9C,SAAS,wBAAwB,OAA4C;CAC3E,MAAM,UACJ,iBAAiB,QAAQ,MAAM,UAAU;CAC3C,MAAM,WAAW,OAAO,MAAM,CAAC,aAAa;AAG5C,KACE,SAAS,SAAS,OAAO,IACzB,SAAS,SAAS,MAAM,IACxB,SAAS,SAAS,WAAW,CAE7B,QAAO,oBAAoB,aAAa;AAI1C,KACE,SAAS,SAAS,OAAO,IACzB,SAAS,SAAS,MAAM,IACxB,SAAS,SAAS,UAAU,IAC5B,SAAS,SAAS,eAAe,CAEjC,QAAO,oBAAoB,YAAY;AAIzC,KACE,SAAS,SAAS,WAAW,IAC7B,SAAS,SAAS,WAAW,IAC7B,SAAS,SAAS,SAAS,CAE3B,QAAO,oBAAoB,cAAc;AAI3C,QAAO,oBAAoB,cAAc,QAAQ;;AAGnD,eAAsB,iBAAiB,EACrC,SACA,WAC6B;AAC7B,KAAI;AAEF,MAAI,CAAC,QAAQ,qBAEX,QAAO,oBADK,oBAAoB,sBAAsB,CACvB;EAIjC,MAAM,cAAc,QAAQ,QAAQ,IAAI,eAAe,IAAI;EAE3D,IAAI;AAEJ,MAAI,YAAY,SAAS,sBAAsB,CAE7C,iBAAgB,MAAM,yBAAyB,QAAQ;WAC9C,YAAY,SAAS,mBAAmB,CAEjD,iBAAgB,MAAM,qBAAqB,QAAQ;MAKnD,QAAO,oBAHK,oBAAoB,eAC9B,4EACD,CAC8B;AAIjC,MAAI,WAAW,cACb,QAAO,cAAc;EAGvB,MAAM,YAAY,cAAc;EAGhC,MAAM,gBAAgB,MAAM,QAAQ,qBAAqB,eAAe;GACtE;GACA,UAAU,UAAU;GACpB,MAAM,UAAU;GACjB,CAAC;AAEF,SAAO,IAAI,SACT,KAAK,UAAU;GACb,MAAM;GACN,MAAM,UAAU;GAChB,MAAM,UAAU;GACjB,CAAC,EACF;GACE,QAAQ;GACR,SAAS,EAAE,gBAAgB,oBAAoB;GAChD,CACF;UACM,OAAO;AAEd,SAAO,oBAAoB,wBAAwB,MAAM,CAAC"}