multi-llm
Version:
A unified TypeScript/JavaScript package to use LLMs across ALL platforms with support for 17 major providers, streaming, MCP tools, and intelligent response parsing
198 lines • 8.29 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DeepInfraProvider = void 0;
const axios_1 = __importDefault(require("axios"));
const provider_1 = require("../provider");
const llm_1 = require("../llm");
const parser_1 = require("../utils/parser");
class DeepInfraProvider extends provider_1.Provider {
constructor(apiKey, baseUrl) {
super(apiKey, baseUrl);
this.baseUrl = baseUrl || 'https://api.deepinfra.com/v1/openai';
}
async getModels() {
try {
const response = await axios_1.default.get(`${this.baseUrl}/models`, {
headers: {
'Authorization': `Bearer ${this.apiKey}`,
'Content-Type': 'application/json'
}
});
return response.data.data.map((model) => ({
id: model.id,
name: model.id,
contextWindow: this.getContextWindow(model.id),
maxOutputTokens: this.getMaxOutputTokens(model.id),
pricing: this.getPricing(model.id)
}));
}
catch (error) {
// Return known popular models if API call fails
return this.getKnownModels();
}
}
createLLM(modelId) {
return new llm_1.LLM(this, modelId);
}
async chat(modelId, messages, options, streamCallback) {
try {
const payload = {
model: modelId,
messages: messages.map(msg => ({ role: msg.role, content: msg.content })),
temperature: options.temperature,
max_tokens: options.maxTokens,
top_p: options.topP,
stream: !!streamCallback,
repetition_penalty: options.repetition_penalty || 1.0,
stop: options.stop
};
if (streamCallback) {
return this.streamChat(payload, streamCallback);
}
else {
const response = await axios_1.default.post(`${this.baseUrl}/chat/completions`, payload, {
headers: {
'Authorization': `Bearer ${this.apiKey}`,
'Content-Type': 'application/json'
}
});
const content = response.data.choices[0].message.content;
const parsed = parser_1.ResponseParser.parseResponse(content);
return {
raw: response.data,
parsed,
usage: {
inputTokens: response.data.usage?.prompt_tokens || 0,
outputTokens: response.data.usage?.completion_tokens || 0,
totalTokens: response.data.usage?.total_tokens || 0
}
};
}
}
catch (error) {
throw new Error(`DeepInfra chat failed: ${error}`);
}
}
async streamChat(payload, streamCallback) {
return new Promise((resolve, reject) => {
let fullContent = '';
let rawResponse = null;
const source = axios_1.default.post(`${this.baseUrl}/chat/completions`, payload, {
headers: {
'Authorization': `Bearer ${this.apiKey}`,
'Content-Type': 'application/json'
},
responseType: 'stream'
});
source.then(response => {
response.data.on('data', (chunk) => {
const lines = chunk.toString().split('\n');
for (const line of lines) {
if (line.startsWith('data: ')) {
const data = line.slice(6);
if (data === '[DONE]') {
const parsed = parser_1.ResponseParser.parseResponse(fullContent);
resolve({
raw: rawResponse,
parsed,
usage: undefined
});
return;
}
try {
const parsed = JSON.parse(data);
rawResponse = parsed;
const content = parsed.choices[0]?.delta?.content || '';
if (content) {
fullContent += content;
streamCallback(content);
}
}
catch (e) {
// Ignore parsing errors for partial chunks
}
}
}
});
response.data.on('error', reject);
}).catch(reject);
});
}
getKnownModels() {
return [
{
id: 'meta-llama/Llama-3.2-3B-Instruct',
name: 'Llama 3.2 3B Instruct',
contextWindow: 131072,
maxOutputTokens: 4096,
pricing: { input: 0.055, output: 0.055, currency: 'USD' }
},
{
id: 'meta-llama/Llama-3.2-1B-Instruct',
name: 'Llama 3.2 1B Instruct',
contextWindow: 131072,
maxOutputTokens: 4096,
pricing: { input: 0.035, output: 0.035, currency: 'USD' }
},
{
id: 'meta-llama/Meta-Llama-3.1-70B-Instruct',
name: 'Llama 3.1 70B Instruct',
contextWindow: 131072,
maxOutputTokens: 4096,
pricing: { input: 0.52, output: 0.75, currency: 'USD' }
},
{
id: 'meta-llama/Meta-Llama-3.1-8B-Instruct',
name: 'Llama 3.1 8B Instruct',
contextWindow: 131072,
maxOutputTokens: 4096,
pricing: { input: 0.055, output: 0.055, currency: 'USD' }
},
{
id: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
name: 'Mixtral 8x7B Instruct',
contextWindow: 32768,
maxOutputTokens: 4096,
pricing: { input: 0.24, output: 0.24, currency: 'USD' }
},
{
id: 'mistralai/Mixtral-8x22B-Instruct-v0.1',
name: 'Mixtral 8x22B Instruct',
contextWindow: 65536,
maxOutputTokens: 4096,
pricing: { input: 0.65, output: 0.65, currency: 'USD' }
}
];
}
getContextWindow(modelId) {
const contextWindows = {
'meta-llama/Llama-3.2-3B-Instruct': 131072,
'meta-llama/Llama-3.2-1B-Instruct': 131072,
'meta-llama/Meta-Llama-3.1-70B-Instruct': 131072,
'meta-llama/Meta-Llama-3.1-8B-Instruct': 131072,
'mistralai/Mixtral-8x7B-Instruct-v0.1': 32768,
'mistralai/Mixtral-8x22B-Instruct-v0.1': 65536,
};
return contextWindows[modelId] || 8192;
}
getMaxOutputTokens(modelId) {
return 4096; // Standard for most DeepInfra models
}
getPricing(modelId) {
const pricing = {
'meta-llama/Llama-3.2-3B-Instruct': { input: 0.055, output: 0.055 },
'meta-llama/Llama-3.2-1B-Instruct': { input: 0.035, output: 0.035 },
'meta-llama/Meta-Llama-3.1-70B-Instruct': { input: 0.52, output: 0.75 },
'meta-llama/Meta-Llama-3.1-8B-Instruct': { input: 0.055, output: 0.055 },
'mistralai/Mixtral-8x7B-Instruct-v0.1': { input: 0.24, output: 0.24 },
'mistralai/Mixtral-8x22B-Instruct-v0.1': { input: 0.65, output: 0.65 },
};
const modelPricing = pricing[modelId];
return modelPricing ? { ...modelPricing, currency: 'USD' } : undefined;
}
}
exports.DeepInfraProvider = DeepInfraProvider;
//# sourceMappingURL=deepinfra.js.map