jorel/dist/providers/ollama/ollama-provider.js

The easiest way to use LLMs, including streams, images, documents, tools and various agent scenarios.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.OllamaProvider = void 0;
const ollama_1 = require("ollama");
const providers_1 = require("../../providers");
const shared_1 = require("../../shared");
const convert_inputs_1 = require("./convert-inputs");
const convert_llm_message_1 = require("./convert-llm-message");
/** Provides access to local Ollama server */
class OllamaProvider {
    get client() {
        return ollama_1.default;
    }
    constructor({ name } = {}) {
        this.name = name || OllamaProvider.defaultName;
    }
    async generateResponse(model, messages, config = {}) {
        const start = Date.now();
        const temperature = config.temperature ?? undefined;
        const response = await ollama_1.default.chat({
            model,
            messages: await (0, convert_llm_message_1.convertLlmMessagesToOllamaMessages)(messages),
            format: (0, convert_inputs_1.jsonResponseToOllama)(config.json),
            tools: (0, convert_inputs_1.toolsToOllama)(config.tools),
            options: {
                temperature,
            },
        });
        const durationMs = Date.now() - start;
        const inputTokens = response.prompt_eval_count; // Somewhat undocumented at the moment
        const outputTokens = response.eval_count; // Somewhat undocumented at the moment
        const message = response.message;
        const toolCalls = message.tool_calls?.map((call) => ({
            id: (0, shared_1.generateUniqueId)(),
            request: {
                id: (0, shared_1.generateRandomId)(),
                function: {
                    name: call.function.name,
                    arguments: call.function.arguments,
                },
            },
            approvalState: config.tools?.getTool(call.function.name)?.requiresConfirmation
                ? "requiresApproval"
                : "noApprovalRequired",
            executionState: "pending",
            result: null,
            error: null,
        }));
        const provider = this.name;
        return {
            ...(0, providers_1.generateAssistantMessage)(message.content, toolCalls),
            meta: {
                model,
                provider,
                temperature,
                durationMs,
                inputTokens,
                outputTokens,
            },
        };
    }
    async *generateResponseStream(model, messages, config = {}) {
        const start = Date.now();
        const temperature = config.temperature ?? undefined;
        const stream = await ollama_1.default.chat({
            model,
            messages: await (0, convert_llm_message_1.convertLlmMessagesToOllamaMessages)(messages),
            stream: true,
            format: (0, convert_inputs_1.jsonResponseToOllama)(config.json),
            tools: (0, convert_inputs_1.toolsToOllama)(config.tools),
            options: {
                temperature,
            },
        });
        const _toolCalls = [];
        let inputTokens = undefined;
        let outputTokens = undefined;
        let content = "";
        for await (const chunk of stream) {
            const contentChunk = chunk.message.content;
            if (contentChunk) {
                content += contentChunk;
                yield { type: "chunk", content: contentChunk };
            }
            if (chunk.message.tool_calls) {
                for (const toolCall of chunk.message.tool_calls) {
                    if (!_toolCalls.some((t) => t.function.name === toolCall.function.name)) {
                        _toolCalls.push(toolCall);
                    }
                }
            }
            if (chunk.prompt_eval_count) {
                inputTokens = (inputTokens ?? 0) + chunk.prompt_eval_count;
            }
            if (chunk.eval_count) {
                outputTokens = (outputTokens ?? 0) + chunk.eval_count;
            }
        }
        const durationMs = Date.now() - start;
        const provider = this.name;
        const meta = {
            model,
            provider,
            temperature,
            durationMs,
            inputTokens,
            outputTokens,
        };
        if (_toolCalls.length > 0) {
            const toolCalls = _toolCalls.map((call) => ({
                id: (0, shared_1.generateUniqueId)(),
                request: {
                    id: (0, shared_1.generateRandomId)(),
                    function: {
                        name: call.function.name,
                        arguments: call.function.arguments,
                    },
                },
                approvalState: config.tools?.getTool(call.function.name)?.requiresConfirmation
                    ? "requiresApproval"
                    : "noApprovalRequired",
                executionState: "pending",
                result: null,
                error: null,
            }));
            yield {
                type: "response",
                role: "assistant_with_tools",
                content,
                toolCalls: toolCalls,
                meta,
            };
        }
        else {
            yield {
                type: "response",
                role: "assistant",
                content,
                meta,
            };
        }
    }
    async getAvailableModels() {
        const { models } = await ollama_1.default.ps();
        return models.map((model) => model.name);
    }
    async createEmbedding(model, text) {
        const response = await ollama_1.default.embeddings({
            model,
            prompt: text,
        });
        return response.embedding;
    }
}
exports.OllamaProvider = OllamaProvider;
OllamaProvider.defaultName = "ollama";