capsule-ai-cli
Version:
The AI Model Orchestrator - Intelligent multi-model workflows with device-locked licensing
514 lines • 19 kB
JavaScript
import { BaseProvider } from './base.js';
import { localModelsService } from '../services/local-models.js';
export class LocalProvider extends BaseProvider {
name = 'local';
supportsStreaming = true;
supportsTools = true;
serverType = 'unknown';
get models() {
const models = localModelsService.getAvailableModels();
if (models.length > 0) {
return models;
}
return [
'llama3.3:latest',
'qwen2.5-coder:latest',
'deepseek-r1:latest',
'mistral:latest',
'phi3:latest',
'gemma2:latest',
'codellama:latest'
];
}
constructor(baseUrl) {
super('', baseUrl);
if (!this.baseUrl) {
if (this.name === 'ollama') {
this.baseUrl = 'http://localhost:11434';
}
else if (this.name === 'lmstudio') {
this.baseUrl = 'http://localhost:1234';
}
else {
this.baseUrl = this.detectLocalServer();
}
}
this.detectServerType();
}
detectLocalServer() {
const commonPorts = [
{ port: 11434, type: 'ollama' },
{ port: 1234, type: 'lmstudio' },
{ port: 8080, type: 'llamacpp' }
];
return 'http://localhost:11434';
}
async detectServerType() {
try {
const ollamaResponse = await fetch(`${this.baseUrl}/api/version`);
if (ollamaResponse.ok) {
this.serverType = 'ollama';
return;
}
}
catch {
}
try {
const openaiResponse = await fetch(`${this.baseUrl}/v1/models`);
if (openaiResponse.ok) {
this.serverType = 'openai';
return;
}
}
catch {
}
this.serverType = 'openai';
}
async complete(messages, options = {}) {
this.validateMessages(messages);
const model = options.model || this.models[0];
if (this.serverType === 'ollama') {
return this.completeOllama(messages, model, options);
}
else {
return this.completeOpenAI(messages, model, options);
}
}
async completeOllama(messages, model, options) {
const formattedMessages = this.formatMessagesForOllama(messages);
const body = {
model: model,
messages: formattedMessages,
stream: false,
options: {
temperature: options.temperature ?? 0.7,
num_predict: options.maxTokens
}
};
if (options.tools && options.tools.length > 0) {
body.tools = options.tools.map((tool) => ({
type: 'function',
function: tool.function || {
name: tool.name,
description: tool.description,
parameters: tool.parameters
}
}));
}
const response = await fetch(`${this.baseUrl}/api/chat`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
});
if (!response.ok) {
const error = await response.text();
throw new Error(`Ollama API error ${response.status}: ${error}`);
}
const data = await response.json();
return this.parseOllamaResponse(data, model);
}
async completeOpenAI(messages, model, options) {
const formattedMessages = this.formatMessagesForOpenAI(messages);
const body = {
model,
messages: formattedMessages,
temperature: options.temperature,
};
if (options.maxTokens) {
body.max_tokens = options.maxTokens;
}
if (options.tools && options.tools.length > 0) {
body.tools = options.tools;
body.tool_choice = 'auto';
}
const response = await fetch(`${this.baseUrl}/v1/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
});
if (!response.ok) {
const error = await response.text();
throw new Error(`Local model API error ${response.status}: ${error}`);
}
const data = await response.json();
return this.parseOpenAIResponse(data, model);
}
async *stream(messages, options = {}) {
this.validateMessages(messages);
const model = options.model || this.models[0];
if (this.serverType === 'ollama') {
yield* this.streamOllama(messages, model, options);
}
else {
yield* this.streamOpenAI(messages, model, options);
}
}
async *streamOllama(messages, model, options) {
const formattedMessages = this.formatMessagesForOllama(messages);
const body = {
model: model,
messages: formattedMessages,
stream: true,
options: {
temperature: options.temperature ?? 0.7,
num_predict: options.maxTokens
}
};
if (options.tools && options.tools.length > 0) {
body.tools = options.tools.map((tool) => ({
type: 'function',
function: tool.function || {
name: tool.name,
description: tool.description,
parameters: tool.parameters
}
}));
}
const response = await fetch(`${this.baseUrl}/api/chat`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
});
if (!response.ok || !response.body) {
const error = await response.text();
throw new Error(`Ollama stream error ${response.status}: ${error}`);
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = '';
while (true) {
const { value, done } = await reader.read();
if (done)
break;
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split('\n');
buffer = lines.pop() || '';
for (const line of lines) {
if (!line.trim())
continue;
try {
const chunk = JSON.parse(line);
if (chunk.message?.content) {
yield { delta: chunk.message.content };
}
if (chunk.message?.tool_calls) {
for (const toolCall of chunk.message.tool_calls) {
yield {
delta: '',
toolCall: {
id: `call_${Date.now()}`,
name: toolCall.function.name,
arguments: toolCall.function.arguments,
},
};
}
}
if (chunk.done && chunk.eval_count) {
yield {
delta: '',
usage: {
promptTokens: chunk.prompt_eval_count || 0,
completionTokens: chunk.eval_count || 0,
totalTokens: (chunk.prompt_eval_count || 0) + (chunk.eval_count || 0),
},
};
}
}
catch (e) {
}
}
}
}
async *streamOpenAI(messages, model, options) {
const formattedMessages = this.formatMessagesForOpenAI(messages);
const body = {
model,
messages: formattedMessages,
temperature: options.temperature,
stream: true,
};
if (options.maxTokens) {
body.max_tokens = options.maxTokens;
}
if (options.tools && options.tools.length > 0) {
body.tools = options.tools;
body.tool_choice = 'auto';
}
const response = await fetch(`${this.baseUrl}/v1/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
});
if (!response.ok || !response.body) {
const error = await response.text();
throw new Error(`Local model stream error ${response.status}: ${error}`);
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = '';
const toolCallsInProgress = new Map();
while (true) {
const { value, done } = await reader.read();
if (done)
break;
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split('\n');
buffer = lines.pop() || '';
for (const line of lines) {
if (line.startsWith('data: ')) {
const data = line.slice(6);
if (data === '[DONE]')
return;
try {
const chunk = JSON.parse(data);
const delta = chunk.choices?.[0]?.delta;
if (delta?.content) {
yield { delta: delta.content };
}
if (delta?.tool_calls) {
for (const toolCallDelta of delta.tool_calls) {
const index = toolCallDelta.index || 0;
if (!toolCallsInProgress.has(index)) {
toolCallsInProgress.set(index, {});
}
const toolCall = toolCallsInProgress.get(index);
if (toolCallDelta.id) {
toolCall.id = toolCallDelta.id;
}
if (toolCallDelta.function?.name) {
toolCall.name = toolCallDelta.function.name;
}
if (toolCallDelta.function?.arguments) {
toolCall.arguments = (toolCall.arguments || '') + toolCallDelta.function.arguments;
}
if (toolCall.id && toolCall.name && toolCall.arguments) {
try {
const parsedArgs = JSON.parse(toolCall.arguments);
yield {
delta: '',
toolCall: {
id: toolCall.id,
name: toolCall.name,
arguments: parsedArgs,
},
};
toolCallsInProgress.delete(index);
}
catch (e) {
}
}
}
}
if (chunk.usage) {
yield {
delta: '',
usage: {
promptTokens: chunk.usage.prompt_tokens,
completionTokens: chunk.usage.completion_tokens,
totalTokens: chunk.usage.total_tokens,
},
};
}
}
catch (e) {
}
}
}
}
}
calculateCost(usage, _model) {
return {
amount: 0,
currency: 'USD',
breakdown: {
prompt: 0,
completion: 0,
},
};
}
formatMessagesForOllama(messages) {
const formatted = [];
for (const msg of messages) {
if (msg.role === 'tool_result') {
formatted.push({
role: 'tool',
content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content)
});
continue;
}
const formattedMsg = {
role: msg.role,
content: ''
};
if (msg.role === 'assistant' && msg.tool_calls) {
formattedMsg.content = '';
formattedMsg.tool_calls = msg.tool_calls.map(tc => ({
function: {
name: tc.name,
arguments: tc.arguments
}
}));
}
else if (typeof msg.content === 'string') {
formattedMsg.content = msg.content;
}
else if (Array.isArray(msg.content)) {
const textParts = [];
const images = [];
for (const part of msg.content) {
if (part.type === 'text') {
textParts.push(part.text);
}
else if (part.type === 'image') {
const imagePart = part;
images.push(imagePart.source.data);
}
}
formattedMsg.content = textParts.join(' ');
if (images.length > 0) {
formattedMsg.images = images;
}
}
formatted.push(formattedMsg);
}
return formatted;
}
formatMessagesForOpenAI(messages) {
const formatted = [];
for (const msg of messages) {
if (msg.role === 'tool_result') {
formatted.push({
role: 'tool',
content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content),
tool_call_id: msg.tool_call_id,
name: msg.name || 'unknown'
});
continue;
}
const formattedMsg = {
role: msg.role,
content: ''
};
if (msg.role === 'assistant' && msg.tool_calls) {
formattedMsg.content = null;
formattedMsg.tool_calls = msg.tool_calls.map(tc => ({
id: tc.id,
type: 'function',
function: {
name: tc.name,
arguments: JSON.stringify(tc.arguments)
}
}));
}
else if (typeof msg.content === 'string') {
formattedMsg.content = msg.content;
}
else if (Array.isArray(msg.content)) {
formattedMsg.content = msg.content.map((part) => {
if (part.type === 'text') {
return { type: 'text', text: part.text };
}
else if (part.type === 'image') {
const imagePart = part;
return {
type: 'image_url',
image_url: {
url: `data:${imagePart.source.media_type};base64,${imagePart.source.data}`,
},
};
}
return { type: part.type };
});
}
formatted.push(formattedMsg);
}
return formatted;
}
parseOllamaResponse(data, model) {
const toolCalls = [];
if (data.message?.tool_calls) {
for (const tc of data.message.tool_calls) {
toolCalls.push({
id: `call_${Date.now()}`,
name: tc.function.name,
arguments: tc.function.arguments,
});
}
}
const usage = {
promptTokens: data.prompt_eval_count || 0,
completionTokens: data.eval_count || 0,
totalTokens: (data.prompt_eval_count || 0) + (data.eval_count || 0),
};
return {
provider: this.name,
model,
content: data.message?.content || '',
usage,
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
};
}
parseOpenAIResponse(data, model) {
const choice = data.choices?.[0];
if (!choice) {
throw new Error('No response from local model');
}
const message = choice.message;
const toolCalls = [];
if (message.tool_calls) {
for (const tc of message.tool_calls) {
toolCalls.push({
id: tc.id,
name: tc.function.name,
arguments: JSON.parse(tc.function.arguments || '{}'),
});
}
}
const usage = {
promptTokens: data.usage?.prompt_tokens || 0,
completionTokens: data.usage?.completion_tokens || 0,
totalTokens: data.usage?.total_tokens || 0,
};
return {
provider: this.name,
model,
content: message.content || '',
usage,
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
};
}
async isAvailable() {
try {
const endpoints = [
`${this.baseUrl}/api/version`,
`${this.baseUrl}/v1/models`,
];
for (const endpoint of endpoints) {
try {
const response = await fetch(endpoint, {
method: 'GET',
signal: AbortSignal.timeout(1000),
});
if (response.ok) {
return true;
}
}
catch (e) {
}
}
return false;
}
catch {
return false;
}
}
}
//# sourceMappingURL=local.js.map