@rocketnew/llm-sdk
Version:
Universal LLM SDK for JavaScript/TypeScript - OpenAI, Anthropic, Gemini, Perplexity and more
1,430 lines (1,425 loc) • 1.75 MB
JavaScript
import {
ElevenLabsException,
ElevenLabsTextToSpeechConfig,
init_common_utils,
init_transformation as init_transformation2
} from "./chunk-K6KIZTYF.mjs";
import {
BinaryResponseContent,
init_responses
} from "./chunk-W4XUZXUW.mjs";
import {
APIConnectionError,
APIError,
AuthenticationError,
BadGatewayError,
BadRequestError,
BaseLLMHTTPHandler,
ConfigurationError,
ContentPolicyViolationError,
ContextWindowExceededError,
HTTPHandler,
ImageFetchError,
InternalServerError,
NotFoundError,
PermissionDeniedError,
ProviderNotFoundError,
ROCKETLLM_EXCEPTION_TYPES,
RateLimitError,
ServiceUnavailableError,
Timeout,
UnprocessableEntityError,
UnsupportedParamsError,
init_exceptions,
init_http_handler,
init_llm_http_handler
} from "./chunk-OCYOJAOQ.mjs";
import {
BaseConfig,
BaseLLMException,
init_transformation
} from "./chunk-EMCXL27T.mjs";
import {
ANTHROPIC_ADVISOR_TOOL_TYPE,
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_EFFORT_BETA_HEADER,
ANTHROPIC_HOSTED_TOOLS,
ANTHROPIC_SKILLS_API_BETA_VERSION,
ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES,
COMPUTER_TOOL_BETA_MAPPING,
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS,
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
OPENAI_CHAT_COMPLETION_PARAMS,
OPENAI_COMPATIBLE_PROVIDERS,
OPENAI_TRANSCRIPTION_PARAMS,
RESPONSE_FORMAT_TOOL_NAME,
__esm,
__export,
__toCommonJS,
all_rocketllm_params,
init_constants
} from "./chunk-DDRQKMV4.mjs";
// src/types/utils.ts
import { randomUUID } from "crypto";
function generateId() {
return "chatcmpl-" + randomUUID();
}
var Usage, RocketLLMProviders;
var init_utils = __esm({
"src/types/utils.ts"() {
"use strict";
Usage = class {
constructor(params) {
// Private attributes for caching (Anthropic compatibility)
this._cache_creation_input_tokens = 0;
this._cache_read_input_tokens = 0;
let _completionTokensDetails = void 0;
if (params.completion_tokens_details) {
_completionTokensDetails = { ...params.completion_tokens_details };
}
let effectiveReasoningTokens = params.reasoning_tokens;
if (effectiveReasoningTokens === void 0) {
if (_completionTokensDetails?.reasoning_tokens !== void 0 && _completionTokensDetails.reasoning_tokens > 0) {
effectiveReasoningTokens = _completionTokensDetails.reasoning_tokens;
}
}
if (effectiveReasoningTokens) {
if (_completionTokensDetails === void 0) {
_completionTokensDetails = {};
}
if (_completionTokensDetails.reasoning_tokens === void 0) {
_completionTokensDetails.reasoning_tokens = effectiveReasoningTokens;
}
if (_completionTokensDetails.text_tokens === void 0 && params.completion_tokens !== void 0) {
let calculatedTextTokens = params.completion_tokens - effectiveReasoningTokens;
if (_completionTokensDetails.image_tokens) {
calculatedTextTokens -= _completionTokensDetails.image_tokens;
}
if (_completionTokensDetails.audio_tokens) {
calculatedTextTokens -= _completionTokensDetails.audio_tokens;
}
_completionTokensDetails.text_tokens = Math.max(0, calculatedTextTokens);
}
}
let _promptTokensDetails = void 0;
if (params.prompt_tokens_details) {
_promptTokensDetails = { ...params.prompt_tokens_details };
}
if (_promptTokensDetails && _promptTokensDetails.text_tokens === void 0) {
const cachedTokens = _promptTokensDetails.cached_tokens || 0;
const audioTokens = _promptTokensDetails.audio_tokens || 0;
if (cachedTokens > 0 || audioTokens > 0) {
const calculatedTextTokens = (params.prompt_tokens || 0) - cachedTokens - audioTokens;
_promptTokensDetails.text_tokens = Math.max(0, calculatedTextTokens);
}
}
if (params.prompt_cache_hit_tokens !== void 0 && typeof params.prompt_cache_hit_tokens === "number") {
if (_promptTokensDetails === void 0) {
_promptTokensDetails = { cached_tokens: params.prompt_cache_hit_tokens };
} else {
_promptTokensDetails.cached_tokens = params.prompt_cache_hit_tokens;
}
}
if (params.cache_read_input_tokens !== void 0 && typeof params.cache_read_input_tokens === "number") {
if (_promptTokensDetails === void 0) {
_promptTokensDetails = { cached_tokens: params.cache_read_input_tokens };
} else {
_promptTokensDetails.cached_tokens = params.cache_read_input_tokens;
}
}
if (params.cache_creation_input_tokens !== void 0 && typeof params.cache_creation_input_tokens === "number") {
if (_promptTokensDetails === void 0) {
_promptTokensDetails = { cache_creation_tokens: params.cache_creation_input_tokens };
} else {
_promptTokensDetails.cache_creation_tokens = params.cache_creation_input_tokens;
}
}
this.prompt_tokens = params.prompt_tokens || 0;
this.completion_tokens = params.completion_tokens || 0;
this.total_tokens = params.total_tokens || 0;
if (_completionTokensDetails !== void 0) {
this.completion_tokens_details = _completionTokensDetails;
}
if (_promptTokensDetails !== void 0) {
this.prompt_tokens_details = _promptTokensDetails;
}
if (params.server_tool_use !== void 0 && params.server_tool_use !== null) {
this.server_tool_use = params.server_tool_use;
}
if (params.cost !== void 0 && params.cost !== null) {
this.cost = params.cost;
}
if (params.cache_creation_input_tokens !== void 0 && typeof params.cache_creation_input_tokens === "number") {
this._cache_creation_input_tokens = params.cache_creation_input_tokens;
}
if (params.cache_read_input_tokens !== void 0 && typeof params.cache_read_input_tokens === "number") {
this._cache_read_input_tokens = params.cache_read_input_tokens;
}
if (params.prompt_cache_hit_tokens !== void 0 && typeof params.prompt_cache_hit_tokens === "number") {
this._cache_read_input_tokens = params.prompt_cache_hit_tokens;
}
for (const [key, value] of Object.entries(params)) {
if (![
"prompt_tokens",
"completion_tokens",
"total_tokens",
"reasoning_tokens",
"prompt_tokens_details",
"completion_tokens_details",
"server_tool_use",
"cost",
"cache_creation_input_tokens",
"cache_read_input_tokens",
"prompt_cache_hit_tokens"
].includes(key)) {
this[key] = value;
}
}
}
/**
* Check if a key exists (for 'in' operator compatibility)
*/
has(key) {
return key in this;
}
/**
* Get a value with default (for .get() compatibility)
*/
get(key, defaultValue) {
return this[key] ?? defaultValue;
}
/**
* Convert to plain object for JSON serialization
*/
toJSON() {
const result = {
prompt_tokens: this.prompt_tokens,
completion_tokens: this.completion_tokens,
total_tokens: this.total_tokens
};
if (this.prompt_tokens_details !== void 0) {
result.prompt_tokens_details = this.prompt_tokens_details;
}
if (this.completion_tokens_details !== void 0) {
result.completion_tokens_details = this.completion_tokens_details;
}
if (this.server_tool_use !== void 0) {
result.server_tool_use = this.server_tool_use;
}
if (this.cost !== void 0) {
result.cost = this.cost;
}
return result;
}
};
RocketLLMProviders = /* @__PURE__ */ ((RocketLLMProviders2) => {
RocketLLMProviders2["OPENAI"] = "openai";
RocketLLMProviders2["ANTHROPIC"] = "anthropic";
RocketLLMProviders2["GEMINI"] = "gemini";
RocketLLMProviders2["PERPLEXITY"] = "perplexity";
RocketLLMProviders2["ELEVENLABS"] = "elevenlabs";
RocketLLMProviders2["ROCKETLLM_PROXY"] = "rocketllm_proxy";
return RocketLLMProviders2;
})(RocketLLMProviders || {});
}
});
// src/rocketllm_core_utils/prompt_templates/image_handling.ts
async function convertUrlToBase64(url) {
const cached = inMemoryCache.get(url);
if (cached) {
return cached;
}
for (let attempt = 0; attempt < 3; attempt++) {
try {
const response = await fetch(url, {
method: "GET",
redirect: "follow"
});
if (!response.ok) {
throw new ImageFetchError(
`Error: Unable to fetch image from URL. Status code: ${response.status}, url=${url}`
);
}
const bytes = await response.arrayBuffer();
const base64 = Buffer.from(bytes).toString("base64");
let contentType = response.headers.get("Content-Type");
if (!contentType) {
const ext = url.split(".").pop()?.toLowerCase();
const typeMap = {
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"png": "image/png",
"gif": "image/gif",
"webp": "image/webp",
"pdf": "application/pdf"
};
contentType = typeMap[ext || ""] || "application/octet-stream";
}
const result = `data:${contentType};base64,${base64}`;
inMemoryCache.set(url, result);
return result;
} catch (error) {
if (error instanceof ImageFetchError) {
if (attempt === 2) {
throw error;
}
continue;
}
if (attempt === 2) {
throw new ImageFetchError(
`Error: Unable to fetch image from URL after 3 attempts. url=${url}`
);
}
}
}
throw new ImageFetchError(
`Error: Unable to fetch image from URL after 3 attempts. url=${url}`
);
}
var InMemoryCache, inMemoryCache;
var init_image_handling = __esm({
"src/rocketllm_core_utils/prompt_templates/image_handling.ts"() {
"use strict";
init_exceptions();
InMemoryCache = class {
constructor(maxSize = 10) {
this.cache = /* @__PURE__ */ new Map();
this.maxSize = maxSize;
}
get(key) {
return this.cache.get(key) || null;
}
set(key, value) {
if (this.cache.size >= this.maxSize) {
const firstKey = this.cache.keys().next().value;
if (firstKey !== void 0) {
this.cache.delete(firstKey);
}
}
this.cache.set(key, value);
}
};
inMemoryCache = new InMemoryCache(10);
}
});
// src/rocketllm_core_utils/prompt_templates/common_utils.ts
function getCompletionMessages(messages, assistantContinueMessage, userContinueMessage, ensureAlternatingRoles = false) {
if (!ensureAlternatingRoles) {
return deepCopyMessages(messages);
}
let result = insertUserContinueMessage(
messages,
userContinueMessage,
ensureAlternatingRoles
);
result = insertAssistantContinueMessage(
result,
assistantContinueMessage,
ensureAlternatingRoles
);
return result;
}
function insertUserContinueMessage(messages, userContinueMessage, ensureAlternatingRoles = false) {
if (!messages || messages.length === 0) {
return messages;
}
const resultMessages = deepCopyMessages(messages);
const continueMessage = userContinueMessage || DEFAULT_USER_CONTINUE_MESSAGE;
if (resultMessages[0].role === "assistant") {
resultMessages.unshift(continueMessage);
}
let i = 1;
while (i < resultMessages.length) {
const currMessage = resultMessages[i];
const prevMessage = resultMessages[i - 1];
if (currMessage.role === "assistant" && prevMessage.role === "assistant") {
resultMessages.splice(i, 0, continueMessage);
i += 2;
} else {
i += 1;
}
}
if (resultMessages[resultMessages.length - 1].role === "assistant" && ensureAlternatingRoles) {
resultMessages.push(continueMessage);
}
return resultMessages;
}
function insertAssistantContinueMessage(messages, assistantContinueMessage, ensureAlternatingRoles = true) {
if (!ensureAlternatingRoles || messages.length <= 1) {
return messages;
}
const modifiedMessages = [];
const continueMessage = assistantContinueMessage || DEFAULT_ASSISTANT_CONTINUE_MESSAGE;
for (let i = 0; i < messages.length; i++) {
const message = messages[i];
modifiedMessages.push(message);
if (i < messages.length - 1 && message.role === "user" && messages[i + 1].role === "user") {
modifiedMessages.push(continueMessage);
}
}
return modifiedMessages;
}
function deepCopyMessages(messages) {
return JSON.parse(JSON.stringify(messages));
}
function filterValueFromDict(obj, key, depth = 0) {
if (depth > 10) return obj;
if (Array.isArray(obj)) {
return obj.map(
(item) => typeof item === "object" ? filterValueFromDict(item, key, depth + 1) : item
);
} else if (obj && typeof obj === "object") {
const result = {};
for (const [k, v] of Object.entries(obj)) {
if (k !== key) {
result[k] = typeof v === "object" ? filterValueFromDict(v, key, depth + 1) : v;
}
}
return result;
}
return obj;
}
function unpackDefs(schema, defs, refChain = /* @__PURE__ */ new Set()) {
if (!schema || typeof schema !== "object") return;
for (const key of Object.keys(schema)) {
const value = schema[key];
if (!value || typeof value !== "object") continue;
if (Array.isArray(value)) {
for (const item of value) {
if (item && typeof item === "object") {
unpackDefs(item, defs, refChain);
}
}
continue;
}
const valueRecord = value;
const ref = valueRecord.$ref;
if (typeof ref === "string") {
const refName = ref.split("/").pop();
if (!refName || refChain.has(refName)) {
delete valueRecord.$ref;
continue;
}
const target = defs[refName];
if (target && typeof target === "object") {
const resolved = JSON.parse(JSON.stringify(target));
const nextChain = new Set(refChain);
nextChain.add(refName);
unpackDefs(resolved, defs, nextChain);
delete valueRecord.$ref;
for (const [rk, rv] of Object.entries(resolved)) {
if (!(rk in valueRecord)) {
valueRecord[rk] = rv;
}
}
} else {
delete valueRecord.$ref;
}
} else {
unpackDefs(valueRecord, defs, refChain);
}
}
}
var DEFAULT_USER_CONTINUE_MESSAGE, DEFAULT_USER_CONTINUE_MESSAGE_TYPED, DEFAULT_ASSISTANT_CONTINUE_MESSAGE;
var init_common_utils2 = __esm({
"src/rocketllm_core_utils/prompt_templates/common_utils.ts"() {
"use strict";
DEFAULT_USER_CONTINUE_MESSAGE = {
role: "user",
content: "continue"
};
DEFAULT_USER_CONTINUE_MESSAGE_TYPED = {
role: "user",
content: "Please continue."
};
DEFAULT_ASSISTANT_CONTINUE_MESSAGE = {
role: "assistant",
content: ""
};
}
});
// src/llms/openai/common_utils.ts
function getOpenaiApiBase(apiBase) {
return apiBase || process.env.OPENAI_BASE_URL || process.env.OPENAI_API_BASE || OPENAI_DEFAULT_API_BASE;
}
function getOpenaiApiKey(apiKey) {
return apiKey || process.env.OPENAI_API_KEY;
}
function getOpenaiOrganization(organization) {
return organization || process.env.OPENAI_ORGANIZATION;
}
function dropParamsFromUnprocessableEntityError(error, data) {
const invalidParams = [];
let errorBody;
if (error.response) {
try {
const parsed = typeof error.response === "string" ? JSON.parse(error.response) : error.response;
errorBody = parsed && typeof parsed === "object" ? parsed.error ?? parsed : parsed;
} catch {
errorBody = { message: error.message };
}
} else {
errorBody = error.body || { message: error.message };
}
if (errorBody && typeof errorBody === "object" && errorBody.message) {
let message = errorBody.message;
if (typeof message === "string") {
try {
message = JSON.parse(message);
} catch {
message = { detail: message };
}
}
const detail = message?.detail;
if (Array.isArray(detail) && detail.length > 0) {
for (const errorDict of detail) {
if (errorDict && typeof errorDict === "object" && errorDict.loc && Array.isArray(errorDict.loc) && errorDict.loc.length === 2) {
invalidParams.push(errorDict.loc[1]);
}
}
}
}
const newData = {};
for (const [key, value] of Object.entries(data)) {
if (!invalidParams.includes(key)) {
newData[key] = value;
}
}
return newData;
}
var OPENAI_DEFAULT_API_BASE, OpenAIError;
var init_common_utils3 = __esm({
"src/llms/openai/common_utils.ts"() {
"use strict";
init_transformation();
OPENAI_DEFAULT_API_BASE = "https://api.openai.com/v1";
OpenAIError = class extends BaseLLMException {
constructor(statusCode, message, headers, body) {
const defaultRequest = { method: "POST", url: "https://api.openai.com/v1" };
super({
statusCode,
message,
headers,
request: defaultRequest,
response: void 0,
body
});
this.name = "OpenAIError";
}
};
}
});
// model_prices_and_context_window.json
var model_prices_and_context_window_default;
var init_model_prices_and_context_window = __esm({
"model_prices_and_context_window.json"() {
model_prices_and_context_window_default = {
sample_spec: {
code_interpreter_cost_per_session: 0,
computer_use_input_cost_per_1k_tokens: 0,
computer_use_output_cost_per_1k_tokens: 0,
deprecation_date: "date when the model becomes deprecated in the format YYYY-MM-DD",
file_search_cost_per_1k_calls: 0,
file_search_cost_per_gb_per_day: 0,
input_cost_per_audio_token: 0,
input_cost_per_token: 0,
litellm_provider: "one of https://docs.litellm.ai/docs/providers",
max_input_tokens: "max input tokens, if the provider specifies it. if not default to max_tokens",
max_output_tokens: "max output tokens, if the provider specifies it. if not default to max_tokens",
max_tokens: "LEGACY parameter. set to max_output_tokens if provider specifies it. IF not set to max_input_tokens, if provider specifies it.",
mode: "one of: chat, embedding, completion, image_generation, audio_transcription, audio_speech, image_generation, moderation, rerank, search",
output_cost_per_reasoning_token: 0,
output_cost_per_token: 0,
search_context_cost_per_query: {
search_context_size_high: 0,
search_context_size_low: 0,
search_context_size_medium: 0
},
supported_regions: [
"global",
"us-west-2",
"eu-west-1",
"ap-southeast-1",
"ap-northeast-1"
],
supports_audio_input: true,
supports_audio_output: true,
supports_function_calling: true,
supports_parallel_function_calling: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_system_messages: true,
supports_vision: true,
supports_web_search: true,
vector_store_cost_per_gb_per_day: 0
},
"1024-x-1024/50-steps/bedrock/amazon.nova-canvas-v1:0": {
litellm_provider: "bedrock",
max_input_tokens: 2600,
mode: "image_generation",
output_cost_per_image: 0.06
},
"1024-x-1024/50-steps/stability.stable-diffusion-xl-v1": {
litellm_provider: "bedrock",
max_input_tokens: 77,
max_tokens: 77,
mode: "image_generation",
output_cost_per_image: 0.04
},
"1024-x-1024/dall-e-2": {
input_cost_per_pixel: 19e-9,
litellm_provider: "openai",
mode: "image_generation",
output_cost_per_pixel: 0
},
"1024-x-1024/max-steps/stability.stable-diffusion-xl-v1": {
litellm_provider: "bedrock",
max_input_tokens: 77,
max_tokens: 77,
mode: "image_generation",
output_cost_per_image: 0.08
},
"256-x-256/dall-e-2": {
input_cost_per_pixel: 24414e-11,
litellm_provider: "openai",
mode: "image_generation",
output_cost_per_pixel: 0
},
"512-x-512/50-steps/stability.stable-diffusion-xl-v0": {
litellm_provider: "bedrock",
max_input_tokens: 77,
max_tokens: 77,
mode: "image_generation",
output_cost_per_image: 0.018
},
"512-x-512/dall-e-2": {
input_cost_per_pixel: 686e-10,
litellm_provider: "openai",
mode: "image_generation",
output_cost_per_pixel: 0
},
"512-x-512/max-steps/stability.stable-diffusion-xl-v0": {
litellm_provider: "bedrock",
max_input_tokens: 77,
max_tokens: 77,
mode: "image_generation",
output_cost_per_image: 0.036
},
"ai21.j2-mid-v1": {
input_cost_per_token: 125e-7,
litellm_provider: "bedrock",
max_input_tokens: 8191,
max_output_tokens: 8191,
max_tokens: 8191,
mode: "chat",
output_cost_per_token: 125e-7
},
"ai21.j2-ultra-v1": {
input_cost_per_token: 188e-7,
litellm_provider: "bedrock",
max_input_tokens: 8191,
max_output_tokens: 8191,
max_tokens: 8191,
mode: "chat",
output_cost_per_token: 188e-7
},
"ai21.jamba-1-5-large-v1:0": {
input_cost_per_token: 2e-6,
litellm_provider: "bedrock",
max_input_tokens: 256e3,
max_output_tokens: 256e3,
max_tokens: 256e3,
mode: "chat",
output_cost_per_token: 8e-6
},
"ai21.jamba-1-5-mini-v1:0": {
input_cost_per_token: 2e-7,
litellm_provider: "bedrock",
max_input_tokens: 256e3,
max_output_tokens: 256e3,
max_tokens: 256e3,
mode: "chat",
output_cost_per_token: 4e-7
},
"ai21.jamba-instruct-v1:0": {
input_cost_per_token: 5e-7,
litellm_provider: "bedrock",
max_input_tokens: 7e4,
max_output_tokens: 4096,
max_tokens: 4096,
mode: "chat",
output_cost_per_token: 7e-7,
supports_system_messages: true
},
"aiml/dall-e-2": {
litellm_provider: "aiml",
metadata: {
notes: "DALL-E 2 via AI/ML API - Reliable text-to-image generation"
},
mode: "image_generation",
output_cost_per_image: 0.026,
source: "https://docs.aimlapi.com/",
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/dall-e-3": {
litellm_provider: "aiml",
metadata: {
notes: "DALL-E 3 via AI/ML API - High-quality text-to-image generation"
},
mode: "image_generation",
output_cost_per_image: 0.052,
source: "https://docs.aimlapi.com/",
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/flux-pro": {
litellm_provider: "aiml",
metadata: {
notes: "Flux Dev - Development version optimized for experimentation"
},
mode: "image_generation",
output_cost_per_image: 0.065,
source: "https://docs.aimlapi.com/",
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/flux-pro/v1.1": {
litellm_provider: "aiml",
mode: "image_generation",
output_cost_per_image: 0.052,
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/flux-pro/v1.1-ultra": {
litellm_provider: "aiml",
mode: "image_generation",
output_cost_per_image: 0.063,
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/flux-realism": {
litellm_provider: "aiml",
metadata: {
notes: "Flux Pro - Professional-grade image generation model"
},
mode: "image_generation",
output_cost_per_image: 0.046,
source: "https://docs.aimlapi.com/",
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/flux/dev": {
litellm_provider: "aiml",
metadata: {
notes: "Flux Dev - Development version optimized for experimentation"
},
mode: "image_generation",
output_cost_per_image: 0.033,
source: "https://docs.aimlapi.com/",
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/flux/kontext-max/text-to-image": {
litellm_provider: "aiml",
metadata: {
notes: "Flux Pro v1.1 - Enhanced version with improved capabilities and 6x faster inference speed"
},
mode: "image_generation",
output_cost_per_image: 0.104,
source: "https://docs.aimlapi.com/",
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/flux/kontext-pro/text-to-image": {
litellm_provider: "aiml",
metadata: {
notes: "Flux Pro v1.1 - Enhanced version with improved capabilities and 6x faster inference speed"
},
mode: "image_generation",
output_cost_per_image: 0.052,
source: "https://docs.aimlapi.com/",
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/flux/schnell": {
litellm_provider: "aiml",
metadata: {
notes: "Flux Schnell - Fast generation model optimized for speed"
},
mode: "image_generation",
output_cost_per_image: 4e-3,
source: "https://docs.aimlapi.com/",
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/google/imagen-4.0-ultra-generate-001": {
litellm_provider: "aiml",
metadata: {
notes: "Imagen 4.0 Ultra Generate API - Photorealistic image generation with precise text rendering"
},
mode: "image_generation",
output_cost_per_image: 0.078,
source: "https://docs.aimlapi.com/api-references/image-models/google/imagen-4-ultra-generate",
supported_endpoints: [
"/v1/images/generations"
]
},
"aiml/google/nano-banana-pro": {
litellm_provider: "aiml",
metadata: {
notes: "Gemini 3 Pro Image (Nano Banana Pro) - Advanced text-to-image generation with reasoning and 4K resolution support"
},
mode: "image_generation",
output_cost_per_image: 0.195,
source: "https://docs.aimlapi.com/api-references/image-models/google/gemini-3-pro-image-preview",
supported_endpoints: [
"/v1/images/generations"
]
},
"amazon.nova-canvas-v1:0": {
litellm_provider: "bedrock",
max_input_tokens: 2600,
mode: "image_generation",
output_cost_per_image: 0.06,
supports_nova_canvas_image_edit: true
},
"us.amazon.nova-canvas-v1:0": {
litellm_provider: "bedrock",
max_input_tokens: 2600,
mode: "image_generation",
output_cost_per_image: 0.06,
supports_nova_canvas_image_edit: true
},
"us.writer.palmyra-x4-v1:0": {
input_cost_per_token: 25e-7,
litellm_provider: "bedrock_converse",
max_input_tokens: 128e3,
max_output_tokens: 8192,
max_tokens: 8192,
mode: "chat",
output_cost_per_token: 1e-5,
supports_function_calling: true,
supports_pdf_input: true
},
"us.writer.palmyra-x5-v1:0": {
input_cost_per_token: 6e-7,
litellm_provider: "bedrock_converse",
max_input_tokens: 1e6,
max_output_tokens: 8192,
max_tokens: 8192,
mode: "chat",
output_cost_per_token: 6e-6,
supports_function_calling: true,
supports_pdf_input: true
},
"writer.palmyra-x4-v1:0": {
input_cost_per_token: 25e-7,
litellm_provider: "bedrock_converse",
max_input_tokens: 128e3,
max_output_tokens: 8192,
max_tokens: 8192,
mode: "chat",
output_cost_per_token: 1e-5,
supports_function_calling: true,
supports_pdf_input: true
},
"writer.palmyra-x5-v1:0": {
input_cost_per_token: 6e-7,
litellm_provider: "bedrock_converse",
max_input_tokens: 1e6,
max_output_tokens: 8192,
max_tokens: 8192,
mode: "chat",
output_cost_per_token: 6e-6,
supports_function_calling: true,
supports_pdf_input: true
},
"amazon.nova-lite-v1:0": {
input_cost_per_token: 6e-8,
litellm_provider: "bedrock_converse",
max_input_tokens: 3e5,
max_output_tokens: 1e4,
max_tokens: 1e4,
mode: "chat",
output_cost_per_token: 24e-8,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_response_schema: true,
supports_vision: true
},
"amazon.nova-2-lite-v1:0": {
cache_read_input_token_cost: 75e-9,
input_cost_per_token: 3e-7,
litellm_provider: "bedrock_converse",
max_input_tokens: 1e6,
max_output_tokens: 64e3,
max_tokens: 64e3,
mode: "chat",
output_cost_per_token: 25e-7,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_video_input: true,
supports_vision: true
},
"amazon.nova-2-pro-preview-20251202-v1:0": {
cache_read_input_token_cost: 546875e-12,
input_cost_per_token: 21875e-10,
input_cost_per_image_token: 21875e-10,
input_cost_per_audio_token: 21875e-10,
litellm_provider: "bedrock_converse",
max_input_tokens: 1e6,
max_output_tokens: 64e3,
max_tokens: 64e3,
mode: "chat",
output_cost_per_token: 175e-7,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_video_input: true,
supports_vision: true
},
"apac.amazon.nova-2-lite-v1:0": {
cache_read_input_token_cost: 825e-10,
input_cost_per_token: 33e-8,
litellm_provider: "bedrock_converse",
max_input_tokens: 1e6,
max_output_tokens: 64e3,
max_tokens: 64e3,
mode: "chat",
output_cost_per_token: 275e-8,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_video_input: true,
supports_vision: true
},
"apac.amazon.nova-2-pro-preview-20251202-v1:0": {
cache_read_input_token_cost: 546875e-12,
input_cost_per_token: 21875e-10,
input_cost_per_image_token: 21875e-10,
input_cost_per_audio_token: 21875e-10,
litellm_provider: "bedrock_converse",
max_input_tokens: 1e6,
max_output_tokens: 64e3,
max_tokens: 64e3,
mode: "chat",
output_cost_per_token: 175e-7,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_video_input: true,
supports_vision: true
},
"eu.amazon.nova-2-lite-v1:0": {
cache_read_input_token_cost: 825e-10,
input_cost_per_token: 33e-8,
litellm_provider: "bedrock_converse",
max_input_tokens: 1e6,
max_output_tokens: 64e3,
max_tokens: 64e3,
mode: "chat",
output_cost_per_token: 275e-8,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_video_input: true,
supports_vision: true
},
"eu.amazon.nova-2-pro-preview-20251202-v1:0": {
cache_read_input_token_cost: 546875e-12,
input_cost_per_token: 21875e-10,
input_cost_per_image_token: 21875e-10,
input_cost_per_audio_token: 21875e-10,
litellm_provider: "bedrock_converse",
max_input_tokens: 1e6,
max_output_tokens: 64e3,
max_tokens: 64e3,
mode: "chat",
output_cost_per_token: 175e-7,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_video_input: true,
supports_vision: true
},
"us.amazon.nova-2-lite-v1:0": {
cache_read_input_token_cost: 825e-10,
input_cost_per_token: 33e-8,
litellm_provider: "bedrock_converse",
max_input_tokens: 1e6,
max_output_tokens: 64e3,
max_tokens: 64e3,
mode: "chat",
output_cost_per_token: 275e-8,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_video_input: true,
supports_vision: true
},
"us.amazon.nova-2-pro-preview-20251202-v1:0": {
cache_read_input_token_cost: 546875e-12,
input_cost_per_token: 21875e-10,
input_cost_per_image_token: 21875e-10,
input_cost_per_audio_token: 21875e-10,
litellm_provider: "bedrock_converse",
max_input_tokens: 1e6,
max_output_tokens: 64e3,
max_tokens: 64e3,
mode: "chat",
output_cost_per_token: 175e-7,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_video_input: true,
supports_vision: true
},
"amazon.nova-2-multimodal-embeddings-v1:0": {
litellm_provider: "bedrock",
max_input_tokens: 8172,
max_tokens: 8172,
mode: "embedding",
input_cost_per_token: 135e-9,
input_cost_per_image: 6e-5,
input_cost_per_video_per_second: 7e-4,
input_cost_per_audio_per_second: 14e-5,
output_cost_per_token: 0,
output_vector_size: 3072,
source: "https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/model-catalog/serverless/amazon.nova-2-multimodal-embeddings-v1:0",
supports_embedding_image_input: true,
supports_image_input: true,
supports_video_input: true,
supports_audio_input: true
},
"amazon.nova-micro-v1:0": {
input_cost_per_token: 35e-9,
litellm_provider: "bedrock_converse",
max_input_tokens: 128e3,
max_output_tokens: 1e4,
max_tokens: 1e4,
mode: "chat",
output_cost_per_token: 14e-8,
supports_function_calling: true,
supports_prompt_caching: true,
supports_response_schema: true
},
"amazon.nova-pro-v1:0": {
input_cost_per_token: 8e-7,
litellm_provider: "bedrock_converse",
max_input_tokens: 3e5,
max_output_tokens: 1e4,
max_tokens: 1e4,
mode: "chat",
output_cost_per_token: 32e-7,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_response_schema: true,
supports_vision: true
},
"amazon.rerank-v1:0": {
input_cost_per_query: 1e-3,
input_cost_per_token: 0,
litellm_provider: "bedrock",
max_document_chunks_per_query: 100,
max_input_tokens: 32e3,
max_output_tokens: 32e3,
max_query_tokens: 32e3,
max_tokens: 32e3,
max_tokens_per_document_chunk: 512,
mode: "rerank",
output_cost_per_token: 0
},
"amazon.titan-embed-image-v1": {
input_cost_per_image: 6e-5,
input_cost_per_token: 8e-7,
litellm_provider: "bedrock",
max_input_tokens: 128,
max_tokens: 128,
metadata: {
notes: "'supports_image_input' is a deprecated field. Use 'supports_embedding_image_input' instead."
},
mode: "embedding",
output_cost_per_token: 0,
output_vector_size: 1024,
source: "https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/providers?model=amazon.titan-image-generator-v1",
supports_embedding_image_input: true,
supports_image_input: true
},
"amazon.titan-embed-text-v1": {
input_cost_per_token: 1e-7,
litellm_provider: "bedrock",
max_input_tokens: 8192,
max_tokens: 8192,
mode: "embedding",
output_cost_per_token: 0,
output_vector_size: 1536
},
"amazon.titan-embed-text-v2:0": {
input_cost_per_token: 2e-7,
litellm_provider: "bedrock",
max_input_tokens: 8192,
max_tokens: 8192,
mode: "embedding",
output_cost_per_token: 0,
output_vector_size: 1024
},
"amazon.titan-image-generator-v1": {
input_cost_per_image: 0,
output_cost_per_image: 8e-3,
output_cost_per_image_premium_image: 0.01,
output_cost_per_image_above_512_and_512_pixels: 0.01,
output_cost_per_image_above_512_and_512_pixels_and_premium_image: 0.012,
litellm_provider: "bedrock",
mode: "image_generation"
},
"amazon.titan-image-generator-v2": {
input_cost_per_image: 0,
output_cost_per_image: 8e-3,
output_cost_per_image_premium_image: 0.01,
output_cost_per_image_above_1024_and_1024_pixels: 0.01,
output_cost_per_image_above_1024_and_1024_pixels_and_premium_image: 0.012,
litellm_provider: "bedrock",
mode: "image_generation"
},
"amazon.titan-image-generator-v2:0": {
input_cost_per_image: 0,
output_cost_per_image: 8e-3,
output_cost_per_image_premium_image: 0.01,
output_cost_per_image_above_1024_and_1024_pixels: 0.01,
output_cost_per_image_above_1024_and_1024_pixels_and_premium_image: 0.012,
litellm_provider: "bedrock",
mode: "image_generation"
},
"twelvelabs.marengo-embed-2-7-v1:0": {
input_cost_per_token: 7e-5,
litellm_provider: "bedrock",
max_input_tokens: 77,
max_tokens: 77,
mode: "embedding",
output_cost_per_token: 0,
output_vector_size: 1024,
supports_embedding_image_input: true,
supports_image_input: true
},
"us.twelvelabs.marengo-embed-2-7-v1:0": {
input_cost_per_token: 7e-5,
input_cost_per_video_per_second: 7e-4,
input_cost_per_audio_per_second: 14e-5,
input_cost_per_image: 1e-4,
litellm_provider: "bedrock",
max_input_tokens: 77,
max_tokens: 77,
mode: "embedding",
output_cost_per_token: 0,
output_vector_size: 1024,
supports_embedding_image_input: true,
supports_image_input: true
},
"eu.twelvelabs.marengo-embed-2-7-v1:0": {
input_cost_per_token: 7e-5,
input_cost_per_video_per_second: 7e-4,
input_cost_per_audio_per_second: 14e-5,
input_cost_per_image: 1e-4,
litellm_provider: "bedrock",
max_input_tokens: 77,
max_tokens: 77,
mode: "embedding",
output_cost_per_token: 0,
output_vector_size: 1024,
supports_embedding_image_input: true,
supports_image_input: true
},
"twelvelabs.pegasus-1-2-v1:0": {
input_cost_per_video_per_second: 49e-5,
output_cost_per_token: 75e-7,
litellm_provider: "bedrock",
mode: "chat",
supports_video_input: true
},
"us.twelvelabs.pegasus-1-2-v1:0": {
input_cost_per_video_per_second: 49e-5,
output_cost_per_token: 75e-7,
litellm_provider: "bedrock",
mode: "chat",
supports_video_input: true
},
"eu.twelvelabs.pegasus-1-2-v1:0": {
input_cost_per_video_per_second: 49e-5,
output_cost_per_token: 75e-7,
litellm_provider: "bedrock",
mode: "chat",
supports_video_input: true
},
"amazon.titan-text-express-v1": {
input_cost_per_token: 13e-7,
litellm_provider: "bedrock",
max_input_tokens: 42e3,
max_output_tokens: 8e3,
max_tokens: 8e3,
mode: "chat",
output_cost_per_token: 17e-7
},
"amazon.titan-text-lite-v1": {
input_cost_per_token: 3e-7,
litellm_provider: "bedrock",
max_input_tokens: 42e3,
max_output_tokens: 4e3,
max_tokens: 4e3,
mode: "chat",
output_cost_per_token: 4e-7
},
"amazon.titan-text-premier-v1:0": {
input_cost_per_token: 5e-7,
litellm_provider: "bedrock",
max_input_tokens: 42e3,
max_output_tokens: 32e3,
max_tokens: 32e3,
mode: "chat",
output_cost_per_token: 15e-7
},
"anthropic.claude-3-5-haiku-20241022-v1:0": {
cache_creation_input_token_cost: 1e-6,
cache_read_input_token_cost: 8e-8,
input_cost_per_token: 8e-7,
litellm_provider: "bedrock",
max_input_tokens: 2e5,
max_output_tokens: 8192,
max_tokens: 8192,
mode: "chat",
output_cost_per_token: 4e-6,
supports_assistant_prefill: true,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_response_schema: true,
supports_tool_choice: true
},
"anthropic.claude-haiku-4-5-20251001-v1:0": {
cache_creation_input_token_cost: 125e-8,
cache_read_input_token_cost: 1e-7,
input_cost_per_token: 1e-6,
litellm_provider: "bedrock_converse",
max_input_tokens: 2e5,
max_output_tokens: 64e3,
max_tokens: 64e3,
mode: "chat",
output_cost_per_token: 5e-6,
source: "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
supports_assistant_prefill: true,
supports_computer_use: true,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_tool_choice: true,
supports_vision: true,
tool_use_system_prompt_tokens: 346,
supports_native_structured_output: true
},
"anthropic.claude-haiku-4-5@20251001": {
cache_creation_input_token_cost: 125e-8,
cache_read_input_token_cost: 1e-7,
input_cost_per_token: 1e-6,
litellm_provider: "bedrock_converse",
max_input_tokens: 2e5,
max_output_tokens: 64e3,
max_tokens: 64e3,
mode: "chat",
output_cost_per_token: 5e-6,
source: "https://aws.amazon.com/about-aws/whats-new/2025/10/claude-4-5-haiku-anthropic-amazon-bedrock",
supports_assistant_prefill: true,
supports_computer_use: true,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_tool_choice: true,
supports_vision: true,
tool_use_system_prompt_tokens: 346,
supports_native_streaming: true,
supports_native_structured_output: true
},
"anthropic.claude-3-5-sonnet-20240620-v1:0": {
input_cost_per_token: 3e-6,
litellm_provider: "bedrock",
max_input_tokens: 1e6,
max_output_tokens: 4096,
max_tokens: 4096,
mode: "chat",
output_cost_per_token: 15e-6,
supports_function_calling: true,
supports_pdf_input: true,
supports_response_schema: true,
supports_tool_choice: true,
supports_vision: true,
input_cost_per_token_above_200k_tokens: 6e-6,
output_cost_per_token_above_200k_tokens: 3e-5,
cache_creation_input_token_cost_above_200k_tokens: 75e-7,
cache_read_input_token_cost_above_200k_tokens: 6e-7,
cache_creation_input_token_cost_above_1hr: 75e-7,
cache_creation_input_token_cost_above_1hr_above_200k_tokens: 15e-6,
cache_creation_input_token_cost: 375e-8,
cache_read_input_token_cost: 3e-7
},
"anthropic.claude-3-5-sonnet-20241022-v2:0": {
cache_creation_input_token_cost: 375e-8,
cache_read_input_token_cost: 3e-7,
input_cost_per_token: 3e-6,
litellm_provider: "bedrock",
max_input_tokens: 1e6,
max_output_tokens: 8192,
max_tokens: 8192,
mode: "chat",
output_cost_per_token: 15e-6,
supports_assistant_prefill: true,
supports_computer_use: true,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_response_schema: true,
supports_tool_choice: true,
supports_vision: true,
input_cost_per_token_above_200k_tokens: 6e-6,
output_cost_per_token_above_200k_tokens: 3e-5,
cache_creation_input_token_cost_above_200k_tokens: 75e-7,
cache_read_input_token_cost_above_200k_tokens: 6e-7,
cache_creation_input_token_cost_above_1hr: 75e-7,
cache_creation_input_token_cost_above_1hr_above_200k_tokens: 15e-6
},
"anthropic.claude-3-7-sonnet-20240620-v1:0": {
cache_creation_input_token_cost: 45e-7,
cache_read_input_token_cost: 36e-8,
input_cost_per_token: 36e-7,
litellm_provider: "bedrock",
max_input_tokens: 2e5,
max_output_tokens: 8192,
max_tokens: 8192,
mode: "chat",
output_cost_per_token: 18e-6,
supports_assistant_prefill: true,
supports_computer_use: true,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_tool_choice: true,
supports_vision: true
},
"anthropic.claude-3-7-sonnet-20250219-v1:0": {
cache_creation_input_token_cost: 375e-8,
cache_read_input_token_cost: 3e-7,
input_cost_per_token: 3e-6,
litellm_provider: "bedrock_converse",
max_input_tokens: 2e5,
max_output_tokens: 8192,
max_tokens: 8192,
mode: "chat",
output_cost_per_token: 15e-6,
supports_assistant_prefill: true,
supports_computer_use: true,
supports_function_calling: true,
supports_pdf_input: true,
supports_prompt_caching: true,
supports_reasoning: true,
supports_response_schema: true,
supports_tool_choice: true,
supports_vision: true
},
"anthropic.claude-3-haiku-20240307-v1:0": {
input_cost_per_token: 25e-8,
litellm_provider: "bedrock",
max_input_tokens: 2e5,
max_output_tokens: 4096,
max_tokens: 4096,
mode: "chat",
output_cost_per_token: 125e-8,
supports_function_calling: true,
supports_pdf_input: true,
supports_response_schema: true,
supports_tool_choice: true,
supports_vision: true,
cache_read_input_token_cost: 25e-9,
cache_creation_input_token_cost: 3125e-10
},
"anthropic.claude-3-opus-20240229-v1:0": {
input_cost_per_token: 15e-6,
litellm_provider: "bedrock",
max_input_tokens: 2e5,
max_output_tokens: 4096,
max_tokens: 4096,
mode: "chat",
output_cost_per_token: 75e-6,
supports_function_calling: true,
supports_response_schema: true,
supports_tool_choice: true,
supports_vision: true,
cache_read_input_token_cost: 15e-7,
cache_creation_input_token_cost: 1875e-8
},
"anthropic.claude-3-sonnet-20240229-v1:0": {
input_cost_per_token: 3e-6,
litellm_provider: "bedrock",
max_input_tokens: 2e5,
max_output_tokens: 4096,
max_tokens: 4096,
mode: "chat",
output_cost_per_token: 15e-6,
supports_function_calling: true,
supports_pdf_input: true,
supports_response_schema: true,
supports_tool_choice: true,
supports_vision: true,
cache_read_input_token_cost: 3e-7,
cache_creation_input_token_cost: 375e-8
},
"anthropic.claude-instant-v1": {
input_cost_per_token: 8e-7,
litellm_provider: "bedrock",
max_input_tokens: 1e5,
max_output_tokens: 8191,
max_tokens: 8191,
mode: "chat",
output_cost_per_token: 24e-7,
supports_tool_choice: true
},
"anthropic.claude-opus-4-1-20250805-v1:0": {
cache_creation_input_token_cost: 1875e-8,
cache_read_input_token_