@websolutespa/payload-plugin-bowl-llm/dist/api/python/pythonStream.handler.js

LLM plugin for Bowl PayloadCms plugin

import { ResponseStream, withCors } from '@websolutespa/payload-utils/server';
import { mergeHeaders } from 'payload';
import { MessageDecoder } from '../message.service';
export const textToChunksLastUnparsed = {
    text: ''
};
/**
 * Executes the ROBOT endpoint returning a stream response.
 *
 * @param request - The request to be sent to the ROBOT endpoint.
 * @param callback - Optional callback function to handle the stream response.
 * @returns The payload handler function.
 */ export const pythonStreamHandler = (request, raw, callback)=>{
    return async (req)=>{
        let robotEndpoint = `${process.env.ROBOT_HOST}/api/llm/stream`;
        if (raw) {
            robotEndpoint += '/raw';
        }
        const response = await fetch(robotEndpoint, {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'Authorization': `Basic ${btoa(`${process.env.ROBOT_USER}:${process.env.ROBOT_PASSWORD}`)}`
            },
            body: JSON.stringify(request)
        });
        /*
    console.log('Bowl headers', Object.fromEntries(response.headers));
    console.log('Bowl messageId', response.headers.get('x-msg-id') || '');
    console.log('Bowl threadId', response.headers.get('x-thread-id') || '');
    console.log('Bowl poweredBy', response.headers.get('x-powered-by') || '');
    */ if (!response.ok) {
            const error = new Error(`Error while fetching from ROBOT endpoint '${robotEndpoint}'. Status: ${response.status} - ${response.statusText}`);
            error.details = await response.text();
            throw error;
        }
        if (!response.body) {
            throw new Error('Response is null');
        }
        // forward ROBOT headers to the client
        const messageId = response.headers.get('x-msg-id') || '';
        const threadId = response.headers.get('x-thread-id') || '';
        const chunks = [];
        const reader = response.body.getReader();
        const textDecoder = new TextDecoder();
        const headers = withCors({
            req,
            cors: '*'
        });
        const stream = ResponseStream(async ({ aborted, controller })=>{
            if (controller) {
                try {
                    let isDone = false;
                    while(!isDone && !aborted){
                        const { done, value } = await reader.read();
                        if (done) {
                            isDone = true;
                            if (typeof callback === 'function') {
                                const decoder = new MessageDecoder();
                                if (raw) {
                                    const response = {
                                        messageId: messageId,
                                        chunks: [
                                            {
                                                type: 'string',
                                                content: chunks.join('')
                                            }
                                        ]
                                    };
                                    callback(response);
                                } else {
                                    const response = decoder.decodeChunks(chunks);
                                    response.messageId = messageId;
                                    callback(response);
                                }
                            }
                            continue;
                        }
                        const stringChunk = textDecoder.decode(value, {
                            stream: true
                        });
                        controller.enqueue(stringChunk);
                        chunks.push(stringChunk);
                    }
                    if (!aborted) {
                        controller.close();
                    }
                } catch (error) {
                    console.error('Error in response stream:', error);
                    throw error;
                // return ResponseError('Error streaming data');
                } finally{
                    reader.releaseLock();
                }
            }
        }, {
            headers: mergeHeaders(new Headers({
                'Content-Type': 'application/octet-stream; charset=utf-8',
                'Transfer-Encoding': 'chunked',
                'x-msg-id': messageId,
                'x-thread-id': threadId,
                'Access-Control-Expose-Headers': '*'
            }), headers)
        });
        return stream;
    /*
    try {
      let isDone = false;
      while (!isDone) {
        const { done, value } = await reader.read();
        if (done) {
          isDone = true;
          res.end();
          if (typeof callback === 'function') {
            const decoder = new MessageDecoder();
            if (raw) {
              const response: PythonStreamResponse = {
                messageId: messageId,
                chunks: [{ type: 'string', content: chunks.join('') }],
              };
              callback(response);
            } else {
              const response: PythonStreamResponse = decoder.decodeChunks(chunks);
              response.messageId = messageId;
              callback(response);
            }
          }
          continue;
        }
        const stringChunk = textDecoder.decode(value, { stream: true });
        res.write(stringChunk);
        chunks.push(stringChunk);
      }
    } catch (error) {
      console.error('Error in response stream:', error);
      return ResponseError('Error streaming data');
    } finally {
      reader.releaseLock();
    }
    */ };
};

//# sourceMappingURL=pythonStream.handler.js.map