jorel
Version:
A unified wrapper for working with LLMs from multiple providers, including streams, images, documents & automatic tool use.
213 lines (212 loc) • 8.28 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.OllamaProvider = void 0;
const ollama_1 = require("ollama");
const providers_1 = require("../../providers");
const shared_1 = require("../../shared");
const convert_inputs_1 = require("./convert-inputs");
const convert_llm_message_1 = require("./convert-llm-message");
/** Provides access to local Ollama server */
class OllamaProvider {
get client() {
return ollama_1.default;
}
constructor({ name } = {}) {
this.name = name || OllamaProvider.defaultName;
}
async generateResponse(model, messages, config = {}) {
const start = Date.now();
const temperature = config.temperature ?? undefined;
// Note: Ollama SDK doesn't support AbortSignal for non-streaming requests
// Check for cancellation before making the request
if (config.abortSignal?.aborted) {
throw new shared_1.JorElAbortError("Request was aborted");
}
let response;
try {
response = await ollama_1.default.chat({
model,
messages: await (0, convert_llm_message_1.convertLlmMessagesToOllamaMessages)(messages),
format: (0, convert_inputs_1.jsonResponseToOllama)(config.json),
tools: (0, convert_inputs_1.toolsToOllama)(config.tools),
options: {
temperature,
},
});
}
catch (error) {
if (error.name === "AbortError" || (error.message && error.message.toLowerCase().includes("aborted"))) {
throw new shared_1.JorElAbortError("Request was aborted");
}
throw error;
}
const durationMs = Date.now() - start;
const inputTokens = response.prompt_eval_count; // Somewhat undocumented at the moment
const outputTokens = response.eval_count; // Somewhat undocumented at the moment
const message = response.message;
const toolCalls = message.tool_calls?.map((call) => ({
id: (0, shared_1.generateUniqueId)(),
request: {
id: (0, shared_1.generateRandomId)(),
function: {
name: call.function.name,
arguments: call.function.arguments,
},
},
approvalState: config.tools?.getTool(call.function.name)?.requiresConfirmation
? "requiresApproval"
: "noApprovalRequired",
executionState: "pending",
result: null,
error: null,
}));
const provider = this.name;
return {
...(0, providers_1.generateAssistantMessage)(message.content, message.thinking ?? null, toolCalls),
meta: {
model,
provider,
temperature,
durationMs,
inputTokens,
outputTokens,
},
};
}
async *generateResponseStream(model, messages, config = {}) {
const start = Date.now();
const temperature = config.temperature ?? undefined;
// Check for cancellation before making the request
if (config.abortSignal?.aborted) {
throw new shared_1.JorElAbortError("Request was aborted");
}
let stream;
try {
stream = await ollama_1.default.chat({
model,
messages: await (0, convert_llm_message_1.convertLlmMessagesToOllamaMessages)(messages),
stream: true,
format: (0, convert_inputs_1.jsonResponseToOllama)(config.json),
tools: (0, convert_inputs_1.toolsToOllama)(config.tools),
options: {
temperature,
},
});
}
catch (error) {
if (error.name === "AbortError" || (error.message && error.message.toLowerCase().includes("aborted"))) {
throw new shared_1.JorElAbortError("Request was aborted");
}
throw error;
}
// Set up cancellation listener for Ollama's native abort support
const abortListener = () => {
stream.abort();
};
if (config.abortSignal) {
config.abortSignal.addEventListener("abort", abortListener);
}
const _toolCalls = [];
let inputTokens = undefined;
let outputTokens = undefined;
let content = "";
let reasoningContent = "";
try {
for await (const chunk of stream) {
const contentChunk = chunk.message.content;
if (contentChunk && typeof contentChunk === "string") {
content += contentChunk;
const chunkId = (0, shared_1.generateUniqueId)();
yield { type: "chunk", content: contentChunk, chunkId };
}
if (chunk.message.thinking) {
reasoningContent += chunk.message.thinking;
const chunkId = (0, shared_1.generateUniqueId)();
yield { type: "reasoningChunk", content: chunk.message.thinking, chunkId };
}
if (chunk.message.tool_calls) {
for (const toolCall of chunk.message.tool_calls) {
if (!_toolCalls.some((t) => t.function.name === toolCall.function.name)) {
_toolCalls.push(toolCall);
}
}
}
if (chunk.prompt_eval_count) {
inputTokens = (inputTokens ?? 0) + chunk.prompt_eval_count;
}
if (chunk.eval_count) {
outputTokens = (outputTokens ?? 0) + chunk.eval_count;
}
}
}
finally {
// Clean up the abort listener
if (config.abortSignal) {
config.abortSignal.removeEventListener("abort", abortListener);
}
}
const durationMs = Date.now() - start;
const provider = this.name;
const meta = {
model,
provider,
temperature,
durationMs,
inputTokens,
outputTokens,
};
if (_toolCalls.length > 0) {
const toolCalls = _toolCalls.map((call) => ({
id: (0, shared_1.generateUniqueId)(),
request: {
id: (0, shared_1.generateRandomId)(),
function: {
name: call.function.name,
arguments: call.function.arguments,
},
},
approvalState: config.tools?.getTool(call.function.name)?.requiresConfirmation
? "requiresApproval"
: "noApprovalRequired",
executionState: "pending",
result: null,
error: null,
}));
yield {
type: "response",
role: "assistant_with_tools",
content,
reasoningContent: reasoningContent || null,
toolCalls: toolCalls,
meta,
};
}
else {
yield {
type: "response",
role: "assistant",
content,
reasoningContent: reasoningContent || null,
meta,
};
}
}
async getAvailableModels() {
const { models } = await ollama_1.default.ps();
return models.map((model) => model.name);
}
async createEmbedding(model, text, abortSignal) {
// Note: Ollama SDK doesn't support AbortSignal for embeddings
// Check for cancellation before making the request
if (abortSignal?.aborted) {
throw new shared_1.JorElAbortError("Request was aborted");
}
const response = await ollama_1.default.embeddings({
model,
prompt: text,
});
return response.embedding;
}
}
exports.OllamaProvider = OllamaProvider;
OllamaProvider.defaultName = "ollama";