@nomyx/assistant/dist/ai/providers/VertexProvider/streaming.js

A powerful assistant library and cli for your AI projects. works with Vertex AI (Claude and Gemini)

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.streamChat = streamChat;
const axios_1 = __importDefault(require("axios"));
const tools_1 = require("./tools");
async function* streamChat(config, messages, options, tools) {
    const url = config.getApiUrl();
    const modelType = config.getModelType();
    const requestBody = {
        instances: [
            {
                messages: (0, tools_1.convertMessages)(messages, modelType),
            },
        ],
        parameters: {
            temperature: options.temperature || 0.7,
            maxOutputTokens: options.maxTokens || 1024,
            topP: 0.95,
            topK: 40,
        },
    };
    if (tools && tools.length > 0) {
        requestBody.instances[0].context = `You have access to the following tools: ${tools.map(tool => tool.name).join(', ')}. Use them when necessary.`;
    }
    const response = await axios_1.default.post(url, requestBody, {
        headers: {
            'Content-Type': 'application/json',
            'Authorization': `Bearer ${await getAccessToken(config)}`,
        },
        responseType: 'stream',
    });
    let buffer = '';
    for await (const chunk of response.data) {
        buffer += chunk.toString();
        let newlineIndex;
        while ((newlineIndex = buffer.indexOf('\n')) !== -1) {
            const line = buffer.slice(0, newlineIndex).trim();
            buffer = buffer.slice(newlineIndex + 1);
            if (line.startsWith('data: ')) {
                const data = JSON.parse(line.slice(6));
                yield createStreamingProviderResponse(data, modelType);
            }
        }
    }
}
function createStreamingProviderResponse(data, modelType) {
    const response = {
        content: '',
        toolCalls: [],
        usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 }, // Vertex AI doesn't provide token usage
    };
    if (data.predictions && data.predictions.length > 0) {
        const prediction = data.predictions[0];
        if (prediction.candidates && prediction.candidates.length > 0) {
            response.content = prediction.candidates[0].content;
        }
    }
    response.toolCalls = (0, tools_1.extractToolCalls)(response.content, modelType);
    return response;
}
async function getAccessToken(config) {
    // Implement logic to get access token from Google Cloud credentials
    // This might involve using the Google Auth Library or a similar mechanism
    // For now, we'll return a placeholder
    return 'placeholder_access_token';
}