judgeval
Version:
Judgment SDK for TypeScript/JavaScript
341 lines • 28.4 kB
JavaScript
import dotenv from 'dotenv';
// Load environment variables from .env file
dotenv.config();
/**
* Collection of proprietary scorers implemented by Judgment.
*
* These are ready-made evaluation scorers that can be used to evaluate
* Examples via the Judgment API.
*/
export var APIScorer;
(function (APIScorer) {
APIScorer["FAITHFULNESS"] = "faithfulness";
APIScorer["ANSWER_RELEVANCY"] = "answer_relevancy";
APIScorer["ANSWER_CORRECTNESS"] = "answer_correctness";
APIScorer["HALLUCINATION"] = "hallucination";
APIScorer["SUMMARIZATION"] = "summarization";
APIScorer["CONTEXTUAL_RECALL"] = "contextual_recall";
APIScorer["CONTEXTUAL_RELEVANCY"] = "contextual_relevancy";
APIScorer["CONTEXTUAL_PRECISION"] = "contextual_precision";
APIScorer["INSTRUCTION_ADHERENCE"] = "instruction_adherence";
APIScorer["EXECUTION_ORDER"] = "execution_order";
APIScorer["JSON_CORRECTNESS"] = "json_correctness";
APIScorer["COMPARISON"] = "comparison";
APIScorer["GROUNDEDNESS"] = "groundedness";
})(APIScorer || (APIScorer = {}));
// Note: The Python version checks for case-insensitivity via _missing_.
// TypeScript enums don't have a direct equivalent built-in.
// Access needs to be case-sensitive, e.g., APIScorer.FAITHFULNESS.
// scorers whose scores are not bounded between 0-1
export const UNBOUNDED_SCORERS = new Set([
APIScorer.COMPARISON,
]);
// API URLs
// Define ROOT_API directly as a constant string
export const ROOT_API = 'https://api.judgmentlabs.ai';
// Dataset API URLs (derive from ROOT_API)
export const JUDGMENT_DATASETS_PUSH_API_URL = `${ROOT_API}/datasets/push/`;
export const JUDGMENT_DATASETS_APPEND_API_URL = `${ROOT_API}/datasets/insert_examples/`;
export const JUDGMENT_DATASETS_PULL_API_URL = `${ROOT_API}/datasets/pull/`;
export const JUDGMENT_DATASETS_DELETE_API_URL = `${ROOT_API}/datasets/delete/`;
export const JUDGMENT_DATASETS_EXPORT_JSONL_API_URL = `${ROOT_API}/datasets/export_jsonl/`;
export const JUDGMENT_DATASETS_PROJECT_STATS_API_URL = `${ROOT_API}/datasets/fetch_stats_by_project/`;
export const JUDGMENT_DATASETS_INSERT_API_URL = `${ROOT_API}/datasets/insert_examples/`;
// Evaluation API URLs (derive from ROOT_API)
export const JUDGMENT_EVAL_FETCH_API_URL = `${ROOT_API}/fetch_eval_results/`;
export const JUDGMENT_EVAL_API_URL = `${ROOT_API}/evaluate/`;
export const JUDGMENT_EVAL_LOG_API_URL = `${ROOT_API}/log_eval_results/`;
export const JUDGMENT_EVAL_DELETE_API_URL = `${ROOT_API}/delete_eval_results_by_project_and_run_names/`;
export const JUDGMENT_EVAL_DELETE_PROJECT_API_URL = `${ROOT_API}/delete_eval_results_by_project/`;
export const JUDGMENT_PROJECT_DELETE_API_URL = `${ROOT_API}/projects/delete/`;
export const JUDGMENT_PROJECT_CREATE_API_URL = `${ROOT_API}/projects/add/`;
export const JUDGMENT_TRACES_FETCH_API_URL = `${ROOT_API}/traces/fetch/`;
export const JUDGMENT_TRACES_SAVE_API_URL = `${ROOT_API}/traces/save/`;
export const JUDGMENT_TRACES_DELETE_API_URL = `${ROOT_API}/traces/delete/`;
export const JUDGMENT_CALCULATE_TOKEN_COSTS_API_URL = `${ROOT_API}/calculate-token-costs`;
export const JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL = `${ROOT_API}/add_to_run_eval_queue/`;
// RabbitMQ
export const RABBITMQ_HOST = process.env.RABBITMQ_HOST || 'localhost';
export const RABBITMQ_PORT = process.env.RABBITMQ_PORT ? parseInt(process.env.RABBITMQ_PORT, 10) : 5672;
export const RABBITMQ_QUEUE = process.env.RABBITMQ_QUEUE || 'task_queue';
// Models
// TODO: Review litellm equivalent in JS/TS.
// The Python code uses `litellm.model_list`.
// A JS library exists (@skadefro/litellm), but doesn't seem to expose model_list directly.
// For now, using an empty set.
export const LITELLM_SUPPORTED_MODELS = new Set([
'omni-moderation-latest', 'omni-moderation-latest-intents', 'omni-moderation-2024-09-26', 'gpt-4', 'gpt-4o',
'gpt-4o-search-preview-2025-03-11', 'gpt-4o-search-preview', 'gpt-4.5-preview', 'gpt-4.5-preview-2025-02-27',
'gpt-4o-audio-preview', 'gpt-4o-audio-preview-2024-12-17', 'gpt-4o-audio-preview-2024-10-01', 'gpt-4o-mini-audio-preview-2024-12-17',
'gpt-4o-mini', 'gpt-4o-mini-search-preview-2025-03-11', 'gpt-4o-mini-search-preview', 'gpt-4o-mini-2024-07-18',
'o1-pro', 'o1-pro-2025-03-19', 'o1', 'o1-mini', 'o3-mini', 'o3-mini-2025-01-31', 'o1-mini-2024-09-12',
'o1-preview', 'o1-preview-2024-09-12', 'o1-2024-12-17', 'chatgpt-4o-latest', 'gpt-4o-2024-05-13', 'gpt-4o-2024-08-06',
'gpt-4o-2024-11-20', 'gpt-4o-realtime-preview-2024-10-01', 'gpt-4o-realtime-preview', 'gpt-4o-realtime-preview-2024-12-17',
'gpt-4o-mini-realtime-preview', 'gpt-4o-mini-realtime-preview-2024-12-17', 'gpt-4-turbo-preview', 'gpt-4-0314',
'gpt-4-0613', 'gpt-4-32k', 'gpt-4-32k-0314', 'gpt-4-32k-0613', 'gpt-4-turbo', 'gpt-4-turbo-2024-04-09',
'gpt-4-1106-preview', 'gpt-4-0125-preview', 'gpt-4-vision-preview', 'gpt-4-1106-vision-preview', 'gpt-3.5-turbo',
'gpt-3.5-turbo-0301', 'gpt-3.5-turbo-0613', 'gpt-3.5-turbo-1106', 'gpt-3.5-turbo-0125', 'gpt-3.5-turbo-16k',
'gpt-3.5-turbo-16k-0613', 'text-embedding-3-large', 'text-embedding-3-small', 'text-embedding-ada-002',
'text-embedding-ada-002-v2', 'text-moderation-stable', 'text-moderation-007', 'text-moderation-latest',
'256-x-256/dall-e-2', '512-x-512/dall-e-2', '1024-x-1024/dall-e-2', 'hd/1024-x-1792/dall-e-3',
'hd/1792-x-1024/dall-e-3', 'hd/1024-x-1024/dall-e-3', 'standard/1024-x-1792/dall-e-3',
'standard/1792-x-1024/dall-e-3', 'standard/1024-x-1024/dall-e-3', 'gpt-4o-transcribe',
'gpt-4o-mini-transcribe', 'whisper-1', 'tts-1', 'tts-1-hd', 'ft:davinci-002', 'ft:babbage-002',
'babbage-002', 'davinci-002', 'gpt-3.5-turbo-instruct', 'gpt-3.5-turbo-instruct-0914',
'command-nightly', 'command', 'rerank-v3.5', 'rerank-english-v3.0', 'rerank-multilingual-v3.0',
'rerank-english-v2.0', 'rerank-multilingual-v2.0', 'embed-english-light-v3.0', 'embed-multilingual-v3.0',
'embed-english-v2.0', 'embed-english-light-v2.0', 'embed-multilingual-v2.0', 'embed-english-v3.0',
'command-r', 'command-r-08-2024', 'command-r7b-12-2024', 'command-light', 'command-r-plus',
'command-r-plus-08-2024', 'claude-instant-1', 'claude-instant-1.2', 'claude-2', 'claude-2.1',
'claude-3-haiku-20240307', 'claude-3-5-haiku-20241022', 'claude-3-5-haiku-latest', 'claude-3-opus-latest',
'claude-3-opus-20240229', 'claude-3-sonnet-20240229', 'claude-3-5-sonnet-latest', 'claude-3-5-sonnet-20240620',
'claude-3-7-sonnet-latest', 'claude-3-7-sonnet-20250219', 'claude-3-5-sonnet-20241022',
'replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf',
'a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52',
'meta/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db',
'replicate/vicuna-13b:6282abe6a492de4145dbb601023762212f9ddbbe78278bd6771c8b3b2f2a13b',
'joehoover/instructblip-vicuna13b:c4c54e3c8c97cd50c2d2fec9be3b6065563ccf7d43787fb99f84151b867178fe',
'daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f',
'replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5',
'replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad',
'openrouter/deepseek/deepseek-r1', 'openrouter/deepseek/deepseek-chat', 'openrouter/deepseek/deepseek-coder',
'openrouter/microsoft/wizardlm-2-8x22b:nitro', 'openrouter/google/gemini-pro-1.5', 'openrouter/google/gemini-2.0-flash-001',
'openrouter/mistralai/mixtral-8x22b-instruct', 'openrouter/cohere/command-r-plus', 'openrouter/databricks/dbrx-instruct',
'openrouter/anthropic/claude-3-haiku', 'openrouter/anthropic/claude-3-5-haiku', 'openrouter/anthropic/claude-3-haiku-20240307',
'openrouter/anthropic/claude-3-5-haiku-20241022', 'openrouter/anthropic/claude-3.5-sonnet', 'openrouter/anthropic/claude-3.5-sonnet:beta',
'openrouter/anthropic/claude-3.7-sonnet', 'openrouter/anthropic/claude-3.7-sonnet:beta', 'openrouter/anthropic/claude-3-sonnet',
'openrouter/mistralai/mistral-large', 'mistralai/mistral-small-3.1-24b-instruct', 'openrouter/cognitivecomputations/dolphin-mixtral-8x7b',
'openrouter/google/gemini-pro-vision', 'openrouter/fireworks/firellava-13b', 'openrouter/meta-llama/llama-3-8b-instruct:free',
'openrouter/meta-llama/llama-3-8b-instruct:extended', 'openrouter/meta-llama/llama-3-70b-instruct:nitro',
'openrouter/meta-llama/llama-3-70b-instruct', 'openrouter/openai/o1', 'openrouter/openai/o1-mini',
'openrouter/openai/o1-mini-2024-09-12', 'openrouter/openai/o1-preview', 'openrouter/openai/o1-preview-2024-09-12',
'openrouter/openai/o3-mini', 'openrouter/openai/o3-mini-high', 'openrouter/openai/gpt-4o',
'openrouter/openai/gpt-4o-2024-05-13', 'openrouter/openai/gpt-4-vision-preview', 'openrouter/openai/gpt-3.5-turbo',
'openrouter/openai/gpt-3.5-turbo-16k', 'openrouter/openai/gpt-4', 'openrouter/anthropic/claude-instant-v1',
'openrouter/anthropic/claude-2', 'openrouter/anthropic/claude-3-opus', 'openrouter/google/palm-2-chat-bison',
'openrouter/google/palm-2-codechat-bison', 'openrouter/meta-llama/llama-2-13b-chat', 'openrouter/meta-llama/llama-2-70b-chat',
'openrouter/meta-llama/codellama-34b-instruct', 'openrouter/nousresearch/nous-hermes-llama2-13b',
'openrouter/mancer/weaver', 'openrouter/gryphe/mythomax-l2-13b', 'openrouter/jondurbin/airoboros-l2-70b-2.1',
'openrouter/undi95/remm-slerp-l2-13b', 'openrouter/pygmalionai/mythalion-13b', 'openrouter/mistralai/mistral-7b-instruct',
'openrouter/mistralai/mistral-7b-instruct:free', 'openrouter/qwen/qwen-2.5-coder-32b-instruct',
'meta-llama/Llama-2-7b-hf', 'meta-llama/Llama-2-7b-chat-hf', 'meta-llama/Llama-2-13b-hf',
'meta-llama/Llama-2-13b-chat-hf', 'meta-llama/Llama-2-70b-hf', 'meta-llama/Llama-2-70b-chat-hf',
'meta-llama/Llama-2-7b', 'meta-llama/Llama-2-7b-chat', 'meta-llama/Llama-2-13b', 'meta-llama/Llama-2-13b-chat',
'meta-llama/Llama-2-70b', 'meta-llama/Llama-2-70b-chat', 'chat-bison', 'chat-bison@001', 'chat-bison@002',
'chat-bison-32k', 'chat-bison-32k@002', 'text-bison', 'text-bison@001', 'text-bison@002', 'text-bison32k',
'text-bison32k@002', 'text-unicorn', 'text-unicorn@001', 'j2-ultra', 'j2-mid', 'j2-light', 'jamba-1.5-mini@001',
'jamba-1.5-large@001', 'jamba-1.5', 'jamba-1.5-mini', 'jamba-1.5-large', 'jamba-large-1.6', 'jamba-mini-1.6',
'togethercomputer/llama-2-70b-chat', 'togethercomputer/llama-2-70b', 'togethercomputer/LLaMA-2-7B-32K',
'togethercomputer/Llama-2-7B-32K-Instruct', 'togethercomputer/llama-2-7b', 'togethercomputer/falcon-40b-instruct',
'togethercomputer/falcon-7b-instruct', 'togethercomputer/alpaca-7b', 'HuggingFaceH4/starchat-alpha',
'togethercomputer/CodeLlama-34b', 'togethercomputer/CodeLlama-34b-Instruct', 'togethercomputer/CodeLlama-34b-Python',
'defog/sqlcoder', 'NumbersStation/nsql-llama-2-7B', 'WizardLM/WizardCoder-15B-V1.0',
'WizardLM/WizardCoder-Python-34B-V1.0', 'NousResearch/Nous-Hermes-Llama2-13b', 'Austism/chronos-hermes-13b',
'upstage/SOLAR-0-70b-16bit', 'WizardLM/WizardLM-70B-V1.0', 'qvv0xeq', 'q841o8w', '31dxrj3', 'luminous-base',
'luminous-base-control', 'luminous-extended', 'luminous-extended-control', 'luminous-supreme', 'luminous-supreme-control',
'dolphin', 'chatdolphin', 'llama2', 'ai21.j2-mid-v1', 'ai21.j2-ultra-v1', 'ai21.jamba-instruct-v1:0',
'ai21.jamba-1-5-large-v1:0', 'ai21.jamba-1-5-mini-v1:0', 'amazon.rerank-v1:0', 'amazon.titan-text-lite-v1',
'amazon.titan-text-express-v1', 'amazon.titan-text-premier-v1:0', 'amazon.titan-embed-text-v1',
'amazon.titan-embed-text-v2:0', 'amazon.titan-embed-image-v1', 'mistral.mistral-7b-instruct-v0:2',
'mistral.mixtral-8x7b-instruct-v0:1', 'mistral.mistral-large-2402-v1:0', 'mistral.mistral-large-2407-v1:0',
'mistral.mistral-small-2402-v1:0', '1024-x-1024/50-steps/bedrock/amazon.nova-canvas-v1:0',
'anthropic.claude-3-sonnet-20240229-v1:0', 'anthropic.claude-3-5-sonnet-20240620-v1:0',
'anthropic.claude-3-5-sonnet-20241022-v2:0', 'anthropic.claude-3-haiku-20240307-v1:0',
'anthropic.claude-3-5-haiku-20241022-v1:0', 'anthropic.claude-3-opus-20240229-v1:0',
'us.anthropic.claude-3-sonnet-20240229-v1:0', 'us.anthropic.claude-3-5-sonnet-20240620-v1:0',
'us.anthropic.claude-3-5-sonnet-20241022-v2:0', 'us.anthropic.claude-3-haiku-20240307-v1:0',
'us.anthropic.claude-3-5-haiku-20241022-v1:0', 'us.anthropic.claude-3-opus-20240229-v1:0',
'eu.anthropic.claude-3-sonnet-20240229-v1:0', 'eu.anthropic.claude-3-5-sonnet-20240620-v1:0',
'eu.anthropic.claude-3-5-sonnet-20241022-v2:0', 'eu.anthropic.claude-3-haiku-20240307-v1:0',
'eu.anthropic.claude-3-5-haiku-20241022-v1:0', 'eu.anthropic.claude-3-opus-20240229-v1:0',
'anthropic.claude-v1', 'anthropic.claude-v2', 'anthropic.claude-v2:1', 'anthropic.claude-instant-v1',
'cohere.rerank-v3-5:0', 'cohere.command-text-v14', 'cohere.command-light-text-v14',
'cohere.command-r-plus-v1:0', 'cohere.command-r-v1:0', 'cohere.embed-english-v3', 'cohere.embed-multilingual-v3',
'meta.llama2-13b-chat-v1', 'meta.llama2-70b-chat-v1', 'meta.llama3-8b-instruct-v1:0',
'meta.llama3-70b-instruct-v1:0', 'meta.llama3-1-8b-instruct-v1:0', 'us.meta.llama3-1-8b-instruct-v1:0',
'meta.llama3-1-70b-instruct-v1:0', 'us.meta.llama3-1-70b-instruct-v1:0', 'meta.llama3-1-405b-instruct-v1:0',
'us.meta.llama3-1-405b-instruct-v1:0', 'meta.llama3-2-1b-instruct-v1:0', 'us.meta.llama3-2-1b-instruct-v1:0',
'eu.meta.llama3-2-1b-instruct-v1:0', 'meta.llama3-2-3b-instruct-v1:0', 'us.meta.llama3-2-3b-instruct-v1:0',
'eu.meta.llama3-2-3b-instruct-v1:0', 'meta.llama3-2-11b-instruct-v1:0', 'us.meta.llama3-2-11b-instruct-v1:0',
'meta.llama3-2-90b-instruct-v1:0', 'us.meta.llama3-2-90b-instruct-v1:0',
'512-x-512/50-steps/stability.stable-diffusion-xl-v0', '512-x-512/max-steps/stability.stable-diffusion-xl-v0',
'max-x-max/50-steps/stability.stable-diffusion-xl-v0', 'max-x-max/max-steps/stability.stable-diffusion-xl-v0',
'1024-x-1024/50-steps/stability.stable-diffusion-xl-v1', '1024-x-1024/max-steps/stability.stable-diffusion-xl-v1',
'stability.sd3-large-v1:0', 'stability.sd3-5-large-v1:0', 'stability.stable-image-core-v1:0',
'stability.stable-image-core-v1:1', 'stability.stable-image-ultra-v1:0', 'stability.stable-image-ultra-v1:1',
'deepinfra/lizpreciatior/lzlv_70b_fp16_hf', 'deepinfra/Gryphe/MythoMax-L2-13b',
'deepinfra/mistralai/Mistral-7B-Instruct-v0.1', 'deepinfra/meta-llama/Llama-2-70b-chat-hf',
'deepinfra/cognitivecomputations/dolphin-2.6-mixtral-8x7b', 'deepinfra/codellama/CodeLlama-34b-Instruct-hf',
'deepinfra/deepinfra/mixtral', 'deepinfra/Phind/Phind-CodeLlama-34B-v2',
'deepinfra/mistralai/Mixtral-8x7B-Instruct-v0.1', 'deepinfra/deepinfra/airoboros-70b',
'deepinfra/01-ai/Yi-34B-Chat', 'deepinfra/01-ai/Yi-6B-200K', 'deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1',
'deepinfra/meta-llama/Llama-2-13b-chat-hf', 'deepinfra/amazon/MistralLite',
'deepinfra/meta-llama/Llama-2-7b-chat-hf', 'deepinfra/meta-llama/Meta-Llama-3-8B-Instruct',
'deepinfra/meta-llama/Meta-Llama-3-70B-Instruct', 'deepinfra/meta-llama/Meta-Llama-3.1-405B-Instruct',
'deepinfra/01-ai/Yi-34B-200K', 'deepinfra/openchat/openchat_3.5', 'perplexity/codellama-34b-instruct',
'perplexity/codellama-70b-instruct', 'perplexity/llama-3.1-70b-instruct', 'perplexity/llama-3.1-8b-instruct',
'perplexity/llama-3.1-sonar-huge-128k-online', 'perplexity/llama-3.1-sonar-large-128k-online',
'perplexity/llama-3.1-sonar-large-128k-chat', 'perplexity/llama-3.1-sonar-small-128k-chat',
'perplexity/llama-3.1-sonar-small-128k-online', 'perplexity/sonar', 'perplexity/sonar-pro',
'perplexity/pplx-7b-chat', 'perplexity/pplx-70b-chat', 'perplexity/pplx-7b-online', 'perplexity/pplx-70b-online',
'perplexity/llama-2-70b-chat', 'perplexity/mistral-7b-instruct', 'perplexity/mixtral-8x7b-instruct',
'perplexity/sonar-small-chat', 'perplexity/sonar-small-online', 'perplexity/sonar-medium-chat',
'perplexity/sonar-medium-online', 'maritalk', 'gemini-pro', 'gemini-1.0-pro', 'gemini-1.0-pro-001',
'gemini-1.0-ultra', 'gemini-1.0-ultra-001', 'gemini-1.0-pro-002', 'gemini-1.5-pro', 'gemini-1.5-pro-002',
'gemini-1.5-pro-001', 'gemini-1.5-pro-preview-0514', 'gemini-1.5-pro-preview-0215', 'gemini-1.5-pro-preview-0409',
'gemini-1.5-flash', 'gemini-1.5-flash-exp-0827', 'gemini-1.5-flash-002', 'gemini-1.5-flash-001',
'gemini-1.5-flash-preview-0514', 'gemini-pro-experimental', 'gemini-flash-experimental', 'medlm-medium',
'medlm-large', 'gemini-2.5-pro-exp-03-25', 'gemini-2.0-pro-exp-02-05', 'gemini-2.0-flash-exp',
'gemini-2.0-flash-001', 'gemini-2.0-flash-thinking-exp', 'gemini-2.0-flash-thinking-exp-01-21', 'gemini-2.0-flash',
'gemini-2.0-flash-lite', 'gemini-2.0-flash-lite-001', 'watsonx/ibm/granite-3-8b-instruct',
'gemini/gemini-2.0-pro-exp-02-05', 'gemini/gemini-2.0-flash', 'gemini/gemini-2.0-flash-lite',
'gemini/gemini-2.0-flash-001', 'gemini/gemini-2.5-pro-preview-03-25', 'gemini/gemini-2.0-flash-exp',
'gemini/gemini-2.0-flash-lite-preview-02-05', 'gemini/gemini-2.0-flash-thinking-exp',
'gemini/gemini-2.0-flash-thinking-exp-01-21', 'gemini/gemma-3-27b-it', 'gemini/learnlm-1.5-pro-experimental',
'gemini/gemini-1.5-flash-002', 'gemini/gemini-1.5-flash-001', 'gemini/gemini-1.5-flash',
'gemini/gemini-1.5-flash-latest', 'gemini/gemini-1.5-flash-8b', 'gemini/gemini-1.5-flash-8b-exp-0924',
'gemini/gemini-exp-1114', 'gemini/gemini-exp-1206', 'gemini/gemini-1.5-flash-exp-0827',
'gemini/gemini-1.5-flash-8b-exp-0827', 'gemini/gemini-pro', 'gemini/gemini-1.5-pro', 'gemini/gemini-1.5-pro-002',
'gemini/gemini-1.5-pro-001', 'gemini/gemini-1.5-pro-exp-0801', 'gemini/gemini-1.5-pro-exp-0827',
'gemini/gemini-1.5-pro-latest', 'gemini/gemini-pro-vision', 'gemini/gemini-gemma-2-27b-it',
'gemini/gemini-gemma-2-9b-it', 'text-completion-codestral/codestral-latest', 'text-completion-codestral/codestral-2405',
'xai/grok-beta', 'xai/grok-2-vision-1212', 'xai/grok-2-vision-latest', 'xai/grok-2-vision', 'xai/grok-vision-beta',
'xai/grok-2-1212', 'xai/grok-2', 'xai/grok-2-latest', 'deepseek/deepseek-reasoner', 'deepseek/deepseek-chat',
'deepseek/deepseek-coder', 'azure_ai/deepseek-r1', 'azure_ai/deepseek-v3', 'azure_ai/jamba-instruct',
'azure_ai/mistral-nemo', 'azure_ai/mistral-large', 'azure_ai/mistral-small', 'azure_ai/mistral-small-2503',
'azure_ai/mistral-large-2407', 'azure_ai/mistral-large-latest', 'azure_ai/ministral-3b',
'azure_ai/Llama-3.2-11B-Vision-Instruct', 'azure_ai/Llama-3.3-70B-Instruct', 'azure_ai/Llama-3.2-90B-Vision-Instruct',
'azure_ai/Meta-Llama-3-70B-Instruct', 'azure_ai/Meta-Llama-3.1-8B-Instruct', 'azure_ai/Meta-Llama-3.1-70B-Instruct',
'azure_ai/Meta-Llama-3.1-405B-Instruct', 'azure_ai/Phi-4-mini-instruct', 'azure_ai/Phi-4-multimodal-instruct',
'azure_ai/Phi-4', 'azure_ai/Phi-3.5-mini-instruct', 'azure_ai/Phi-3.5-vision-instruct', 'azure_ai/Phi-3.5-MoE-instruct',
'azure_ai/Phi-3-mini-4k-instruct', 'azure_ai/Phi-3-mini-128k-instruct', 'azure_ai/Phi-3-small-8k-instruct',
'azure_ai/Phi-3-small-128k-instruct', 'azure_ai/Phi-3-medium-4k-instruct', 'azure_ai/Phi-3-medium-128k-instruct',
'azure_ai/cohere-rerank-v3-multilingual', 'azure_ai/cohere-rerank-v3-english', 'azure_ai/Cohere-embed-v3-english',
'azure_ai/Cohere-embed-v3-multilingual', 'voyage/voyage-01', 'voyage/voyage-lite-01', 'voyage/voyage-large-2',
'voyage/voyage-finance-2', 'voyage/voyage-lite-02-instruct', 'voyage/voyage-law-2', 'voyage/voyage-code-2',
'voyage/voyage-2', 'voyage/voyage-3-large', 'voyage/voyage-3', 'voyage/voyage-3-lite', 'voyage/voyage-code-3',
'voyage/voyage-multimodal-3', 'voyage/rerank-2', 'voyage/rerank-2-lite',
'databricks/databricks-claude-3-7-sonnet', 'databricks/databricks-meta-llama-3-1-405b-instruct',
'databricks/databricks-meta-llama-3-1-70b-instruct', 'databricks/databricks-meta-llama-3-3-70b-instruct',
'databricks/databricks-dbrx-instruct', 'databricks/databricks-meta-llama-3-70b-instruct',
'databricks/databricks-llama-2-70b-chat', 'databricks/databricks-mixtral-8x7b-instruct',
'databricks/databricks-mpt-30b-instruct', 'databricks/databricks-mpt-7b-instruct',
'databricks/databricks-bge-large-en', 'databricks/databricks-gte-large-en',
'cloudflare/@cf/meta/llama-2-7b-chat-fp16', 'cloudflare/@cf/meta/llama-2-7b-chat-int8',
'cloudflare/@cf/mistral/mistral-7b-instruct-v0.1', 'cloudflare/@hf/thebloke/codellama-7b-instruct-awq',
'codestral/codestral-latest', 'codestral/codestral-2405', 'friendliai/meta-llama-3.1-8b-instruct',
'friendliai/meta-llama-3.1-70b-instruct', 'palm/chat-bison', 'palm/chat-bison-001', 'palm/text-bison',
'palm/text-bison-001', 'palm/text-bison-safety-off', 'palm/text-bison-safety-recitation-off',
'groq/deepseek-r1-distill-llama-70b', 'groq/llama-3.3-70b-versatile', 'groq/llama-3.3-70b-specdec',
'groq/llama2-70b-4096', 'groq/llama3-8b-8192', 'groq/llama-3.2-1b-preview', 'groq/llama-3.2-3b-preview',
'groq/llama-3.2-11b-text-preview', 'groq/llama-3.2-11b-vision-preview', 'groq/llama-3.2-90b-text-preview',
'groq/llama-3.2-90b-vision-preview', 'groq/llama3-70b-8192', 'groq/llama-3.1-8b-instant',
'groq/llama-3.1-70b-versatile', 'groq/llama-3.1-405b-reasoning', 'groq/mixtral-8x7b-32768',
'groq/gemma-7b-it', 'groq/gemma2-9b-it', 'groq/llama3-groq-70b-8192-tool-use-preview',
'groq/llama3-groq-8b-8192-tool-use-preview', 'groq/whisper-large-v3', 'groq/whisper-large-v3-turbo',
'groq/distil-whisper-large-v3-en', 'azure/gpt-4o-mini-realtime-preview-2024-12-17',
'azure/eu/gpt-4o-mini-realtime-preview-2024-12-17', 'azure/us/gpt-4o-mini-realtime-preview-2024-12-17',
'azure/gpt-4o-realtime-preview-2024-10-01', 'azure/us/gpt-4o-realtime-preview-2024-10-01',
'azure/eu/gpt-4o-realtime-preview-2024-10-01', 'azure/o3-mini-2025-01-31', 'azure/us/o3-mini-2025-01-31',
'azure/eu/o3-mini-2025-01-31', 'azure/tts-1', 'azure/tts-1-hd', 'azure/whisper-1', 'azure/o3-mini', 'azure/o1-mini',
'azure/o1-mini-2024-09-12', 'azure/us/o1-mini-2024-09-12', 'azure/eu/o1-mini-2024-09-12', 'azure/o1',
'azure/o1-2024-12-17', 'azure/us/o1-2024-12-17', 'azure/eu/o1-2024-12-17', 'azure/o1-preview',
'azure/o1-preview-2024-09-12', 'azure/us/o1-preview-2024-09-12', 'azure/eu/o1-preview-2024-09-12',
'azure/gpt-4.5-preview', 'azure/gpt-4o', 'azure/global/gpt-4o-2024-11-20', 'azure/gpt-4o-2024-08-06',
'azure/global/gpt-4o-2024-08-06', 'azure/gpt-4o-2024-11-20', 'azure/us/gpt-4o-2024-11-20',
'azure/eu/gpt-4o-2024-11-20', 'azure/gpt-4o-2024-05-13', 'azure/global-standard/gpt-4o-2024-08-06',
'azure/us/gpt-4o-2024-08-06', 'azure/eu/gpt-4o-2024-08-06', 'azure/global-standard/gpt-4o-2024-11-20',
'azure/global-standard/gpt-4o-mini', 'azure/gpt-4o-mini', 'azure/gpt-4o-mini-2024-07-18',
'azure/us/gpt-4o-mini-2024-07-18', 'azure/eu/gpt-4o-mini-2024-07-18', 'azure/gpt-4-turbo-2024-04-09',
'azure/gpt-4-0125-preview', 'azure/gpt-4-1106-preview', 'azure/gpt-4-0613', 'azure/gpt-4-32k-0613',
'azure/gpt-4-32k', 'azure/gpt-4', 'azure/gpt-4-turbo', 'azure/gpt-4-turbo-vision-preview',
'azure/gpt-35-turbo-16k-0613', 'azure/gpt-35-turbo-1106', 'azure/gpt-35-turbo-0613',
'azure/gpt-35-turbo-0301', 'azure/gpt-35-turbo-0125', 'azure/gpt-3.5-turbo-0125', 'azure/gpt-35-turbo-16k',
'azure/gpt-35-turbo', 'azure/gpt-3.5-turbo', 'azure/mistral-large-latest', 'azure/mistral-large-2402',
'azure/command-r-plus', 'azure/ada', 'azure/text-embedding-ada-002', 'azure/text-embedding-3-large',
'azure/text-embedding-3-small', 'azure/standard/1024-x-1024/dall-e-3', 'azure/hd/1024-x-1024/dall-e-3',
'azure/standard/1024-x-1792/dall-e-3', 'azure/standard/1792-x-1024/dall-e-3', 'azure/hd/1024-x-1792/dall-e-3',
'azure/hd/1792-x-1024/dall-e-3', 'azure/standard/1024-x-1024/dall-e-2',
'anyscale/mistralai/Mistral-7B-Instruct-v0.1', 'anyscale/mistralai/Mixtral-8x7B-Instruct-v0.1',
'anyscale/mistralai/Mixtral-8x22B-Instruct-v0.1', 'anyscale/HuggingFaceH4/zephyr-7b-beta',
'anyscale/google/gemma-7b-it', 'anyscale/meta-llama/Llama-2-7b-chat-hf', 'anyscale/meta-llama/Llama-2-13b-chat-hf',
'anyscale/meta-llama/Llama-2-70b-chat-hf', 'anyscale/codellama/CodeLlama-34b-Instruct-hf',
'anyscale/codellama/CodeLlama-70b-Instruct-hf', 'anyscale/meta-llama/Meta-Llama-3-8B-Instruct',
'anyscale/meta-llama/Meta-Llama-3-70B-Instruct', 'cerebras/llama3.1-8b', 'cerebras/llama3.1-70b',
'cerebras/llama3.3-70b', 'azure/gpt-3.5-turbo-instruct-0914', 'azure/gpt-35-turbo-instruct',
'azure/gpt-35-turbo-instruct-0914', 'assemblyai/nano', 'assemblyai/best',
'jina-reranker-v2-base-multilingual', 'snowflake/deepseek-r1', 'snowflake/snowflake-arctic',
'snowflake/claude-3-5-sonnet', 'snowflake/mistral-large', 'snowflake/mistral-large2', 'snowflake/reka-flash',
'snowflake/reka-core', 'snowflake/jamba-instruct', 'snowflake/jamba-1.5-mini', 'snowflake/jamba-1.5-large',
'snowflake/mixtral-8x7b', 'snowflake/llama2-70b-chat', 'snowflake/llama3-8b', 'snowflake/llama3-70b',
'snowflake/llama3.1-8b', 'snowflake/llama3.1-70b', 'snowflake/llama3.3-70b', 'snowflake/snowflake-llama-3.3-70b',
'snowflake/llama3.1-405b', 'snowflake/snowflake-llama-3.1-405b', 'snowflake/llama3.2-1b', 'snowflake/llama3.2-3b',
'snowflake/mistral-7b', 'snowflake/gemma-7b'
]);
export const TOGETHER_SUPPORTED_MODELS = [
"meta-llama/Meta-Llama-3-70B-Instruct-Turbo",
"Qwen/Qwen2-VL-72B-Instruct",
"meta-llama/Llama-Vision-Free",
"Gryphe/MythoMax-L2-13b",
"Qwen/Qwen2.5-72B-Instruct-Turbo",
"meta-llama/Llama-3.3-70B-Instruct-Turbo-Free",
"deepseek-ai/DeepSeek-R1",
"meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo",
"meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo",
"google/gemma-2-27b-it",
"mistralai/Mistral-Small-24B-Instruct-2501",
"mistralai/Mixtral-8x22B-Instruct-v0.1",
"meta-llama/Meta-Llama-3-8B-Instruct-Turbo",
"NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
"meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo-classifier",
"deepseek-ai/DeepSeek-V3",
"Qwen/Qwen2-72B-Instruct",
"meta-llama/Meta-Llama-3-8B-Instruct-Lite",
"deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
"upstage/SOLAR-10.7B-Instruct-v1.0",
"togethercomputer/MoA-1",
"Qwen/QwQ-32B-Preview",
"meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo",
"meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
"mistralai/Mistral-7B-Instruct-v0.2",
"databricks/dbrx-instruct",
"meta-llama/Llama-3-8b-chat-hf",
"google/gemma-2b-it",
"meta-llama/Meta-Llama-3-70B-Instruct-Lite",
"google/gemma-2-9b-it",
"meta-llama/Llama-3.3-70B-Instruct-Turbo",
"meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo-p",
"meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
"Gryphe/MythoMax-L2-13b-Lite",
"meta-llama/Llama-2-7b-chat-hf",
"deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
"meta-llama/Llama-2-13b-chat-hf",
"scb10x/scb10x-llama3-typhoon-v1-5-8b-instruct",
"scb10x/scb10x-llama3-typhoon-v1-5x-4f316",
"nvidia/Llama-3.1-Nemotron-70B-Instruct-HF",
"Qwen/Qwen2.5-Coder-32B-Instruct",
"microsoft/WizardLM-2-8x22B",
"mistralai/Mistral-7B-Instruct-v0.3",
"scb10x/scb10x-llama3-1-typhoon2-60256",
"Qwen/Qwen2.5-7B-Instruct-Turbo",
"scb10x/scb10x-llama3-1-typhoon-18370",
"meta-llama/Llama-3.2-3B-Instruct-Turbo",
"meta-llama/Llama-3-70b-chat-hf",
"mistralai/Mixtral-8x7B-Instruct-v0.1",
"togethercomputer/MoA-1-Turbo",
"deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free",
"deepseek-ai/DeepSeek-R1-Distill-Qwen-14B",
"mistralai/Mistral-7B-Instruct-v0.1"
];
export const JUDGMENT_SUPPORTED_MODELS = new Set(["osiris-large", "osiris-mini", "osiris"]);
// Combine model sets
export const ACCEPTABLE_MODELS = new Set([
...LITELLM_SUPPORTED_MODELS,
...TOGETHER_SUPPORTED_MODELS,
...JUDGMENT_SUPPORTED_MODELS,
]);
// System settings
export const MAX_WORKER_THREADS = 10;
// Maximum number of concurrent operations for evaluation runs
export const MAX_CONCURRENT_EVALUATIONS = 50; // Adjust based on system capabilities
//# sourceMappingURL=constants.js.map