judgeval
Version:
Judgment SDK for TypeScript/JavaScript
236 lines • 11.2 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.TogetherJudge = exports.DefaultJudge = void 0;
exports.createJudge = createJudge;
const axios_1 = __importDefault(require("axios"));
const logger_js_1 = require("../common/logger.js");
const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const path = __importStar(require("path"));
const child_process_1 = require("child_process");
/**
* Default judge implementation using OpenAI API
*/
class DefaultJudge {
constructor(modelName = 'gpt-3.5-turbo', apiKey, user) {
this.modelName = modelName;
this.apiKey = apiKey || process.env.OPENAI_API_KEY;
this.user = user;
if (!this.apiKey) {
(0, logger_js_1.warn)('No API key provided for DefaultJudge. Set OPENAI_API_KEY environment variable or pass apiKey to constructor.');
}
}
generate(prompt) {
// For synchronous generation, we need to block until we get a response
// This is similar to how the Python SDK's fetch_litellm_api_response works
if (!this.apiKey) {
throw new Error('No API key provided for DefaultJudge');
}
try {
// Create a temporary file for the request and response
const tempDir = os.tmpdir();
const requestFile = path.join(tempDir, `openai-request-${Date.now()}.json`);
const responseFile = path.join(tempDir, `openai-response-${Date.now()}.json`);
// Write request data to file
fs.writeFileSync(requestFile, JSON.stringify(Object.assign({ model: this.modelName, messages: [{ role: 'user', content: prompt }], temperature: 0.0 }, (this.user ? { user: this.user } : {}))));
// Make the request using curl
const curlCommand = `curl -s -X POST https://api.openai.com/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${this.apiKey}" \
-d @${requestFile} > ${responseFile}`;
(0, child_process_1.execSync)(curlCommand);
// Read the response
const responseData = JSON.parse(fs.readFileSync(responseFile, 'utf8'));
// Clean up temporary files
fs.unlinkSync(requestFile);
fs.unlinkSync(responseFile);
// Check if the response has the expected structure
if (!responseData || !responseData.choices || !responseData.choices[0] || !responseData.choices[0].message) {
(0, logger_js_1.error)(`Invalid response structure: ${JSON.stringify(responseData)}`);
throw new Error('Invalid response structure from OpenAI API');
}
// Return the content
return responseData.choices[0].message.content;
}
catch (e) {
(0, logger_js_1.error)(`Error in synchronous generate: ${e.message}`);
throw new Error(`Failed to generate text: ${e.message}`);
}
}
aGenerate(prompt) {
return __awaiter(this, void 0, void 0, function* () {
if (!this.apiKey) {
throw new Error('No API key provided for DefaultJudge');
}
try {
const response = yield axios_1.default.post('https://api.openai.com/v1/chat/completions', Object.assign({ model: this.modelName, messages: [{ role: 'user', content: prompt }], temperature: 0.0 }, (this.user ? { user: this.user } : {})), {
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`
}
});
// Check if the response has the expected structure
if (!response.data || !response.data.choices || !response.data.choices[0] || !response.data.choices[0].message) {
(0, logger_js_1.error)(`Invalid response structure: ${JSON.stringify(response.data)}`);
throw new Error('Invalid response structure from OpenAI API');
}
return response.data.choices[0].message.content;
}
catch (e) {
(0, logger_js_1.error)(`Error generating text with OpenAI API: ${e.message}`);
throw new Error(`Failed to generate text: ${e.message}`);
}
});
}
getModelName() {
return this.modelName;
}
}
exports.DefaultJudge = DefaultJudge;
/**
* Together AI judge implementation
*/
class TogetherJudge {
constructor(modelName = 'meta-llama/Meta-Llama-3-8B-Instruct-Turbo', apiKey) {
this.modelName = modelName;
this.apiKey = apiKey || process.env.TOGETHER_API_KEY;
if (!this.apiKey) {
(0, logger_js_1.warn)('No API key provided for TogetherJudge. Set TOGETHER_API_KEY environment variable or pass apiKey to constructor.');
}
}
generate(prompt) {
// For synchronous generation, we need to block until we get a response
if (!this.apiKey) {
throw new Error('No API key provided for TogetherJudge');
}
try {
// In Node.js, we can use a synchronous HTTP request via child_process
const tempDir = os.tmpdir();
const requestFile = path.join(tempDir, `together-request-${Date.now()}.json`);
const responseFile = path.join(tempDir, `together-response-${Date.now()}.json`);
// Write request data to file
fs.writeFileSync(requestFile, JSON.stringify({
model: this.modelName,
prompt: prompt,
temperature: 0.0,
max_tokens: 1024
}));
// Make the request using curl
const curlCommand = `curl -s -X POST https://api.together.xyz/v1/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${this.apiKey}" \
-d @${requestFile} > ${responseFile}`;
(0, child_process_1.execSync)(curlCommand);
// Read the response
const responseData = JSON.parse(fs.readFileSync(responseFile, 'utf8'));
// Clean up temporary files
fs.unlinkSync(requestFile);
fs.unlinkSync(responseFile);
// Check if the response has the expected structure
if (!responseData || !responseData.choices || !responseData.choices[0] || !responseData.choices[0].text) {
(0, logger_js_1.error)(`Invalid response structure: ${JSON.stringify(responseData)}`);
throw new Error('Invalid response structure from Together API');
}
// Return the content
return responseData.choices[0].text;
}
catch (e) {
(0, logger_js_1.error)(`Error in synchronous generate: ${e.message}`);
throw new Error(`Failed to generate text: ${e.message}`);
}
}
aGenerate(prompt) {
return __awaiter(this, void 0, void 0, function* () {
if (!this.apiKey) {
throw new Error('No API key provided for TogetherJudge');
}
try {
const response = yield axios_1.default.post('https://api.together.xyz/v1/completions', {
model: this.modelName,
prompt: prompt,
temperature: 0.0,
max_tokens: 1024
}, {
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.apiKey}`
}
});
// Check if the response has the expected structure
if (!response.data || !response.data.choices || !response.data.choices[0] || !response.data.choices[0].text) {
(0, logger_js_1.error)(`Invalid response structure: ${JSON.stringify(response.data)}`);
throw new Error('Invalid response structure from Together API');
}
return response.data.choices[0].text;
}
catch (e) {
(0, logger_js_1.error)(`Error generating text with Together API: ${e.message}`);
throw new Error(`Failed to generate text: ${e.message}`);
}
});
}
getModelName() {
return this.modelName;
}
}
exports.TogetherJudge = TogetherJudge;
/**
* Create a judge instance
* @param model Model name or Judge instance
* @param user Optional user identifier
* @returns Judge instance and whether it's a native model
*/
function createJudge(model, user) {
if (!model) {
return { judge: new DefaultJudge(undefined, undefined, user), usingNativeModel: true };
}
if (typeof model === 'string') {
// Check if it's a Together AI model
if (model.startsWith('together/') ||
model.startsWith('meta-llama/') ||
model.startsWith('mistralai/') ||
model.includes('llama')) {
return { judge: new TogetherJudge(model), usingNativeModel: true };
}
// Default to OpenAI
return { judge: new DefaultJudge(model, undefined, user), usingNativeModel: true };
}
// It's already a Judge instance
return { judge: model, usingNativeModel: false };
}
//# sourceMappingURL=index.js.map