judgeval
Version:
Judgment SDK for TypeScript/JavaScript
405 lines • 19.7 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.EvalDataset = void 0;
const example_js_1 = require("../example.js"); // Import class and options
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const papaparse_1 = __importDefault(require("papaparse")); // Added PapaParse
const js_yaml_1 = __importDefault(require("js-yaml")); // Added js-yaml
const date_fns_1 = require("date-fns"); // For default filename timestamp
const ACCEPTABLE_FILE_TYPES = ['json', 'csv', 'yaml'];
// Define keys that represent lists potentially split by a delimiter in CSV
// Use camelCase names from Example class/ExampleOptions
const LIST_LIKE_KEYS = [
'context',
'retrievalContext',
'toolsCalled',
'expectedTools',
];
class EvalDataset {
// These might not be needed directly in the TS class if client handles auth
// public judgment_api_key: string;
// public organization_id: string;
constructor(examples = []) {
console.debug(`Initializing EvalDataset with ${examples.length} examples`);
this.examples = examples;
this._alias = null;
this._id = null;
// Assuming API keys are handled by the client, not stored in the dataset object itself
// this.judgment_api_key = judgment_api_key || process.env.JUDGMENT_API_KEY || '';
// this.organization_id = organization_id || process.env.JUDGMENT_ORG_ID || '';
// if (!this.judgment_api_key) {
// console.warn("No judgment_api_key provided");
// }
}
addExample(e) {
this.examples.push(e);
}
get length() {
return this.examples.length;
}
get alias() {
return this._alias;
}
set alias(value) {
this._alias = value;
}
get id() {
return this._id;
}
set id(value) {
this._id = value;
}
/**
* Adds examples from a JSON file.
* Assumes the JSON file has a top-level key "examples" containing an array of example objects.
* @param filePath Path to the JSON file.
*/
addFromJson(filePath) {
console.debug(`Loading dataset from JSON file: ${filePath}`);
try {
const fileContent = fs.readFileSync(filePath, 'utf-8');
const payload = JSON.parse(fileContent);
const examplesFromJson = payload === null || payload === void 0 ? void 0 : payload.examples;
if (!Array.isArray(examplesFromJson)) {
throw new Error('Invalid JSON format: "examples" key not found or not an array.');
}
console.info(`Adding ${examplesFromJson.length} examples from JSON`);
examplesFromJson.forEach((e) => {
// Map snake_case from file to camelCase for ExampleOptions
const options = {
input: e.input,
actualOutput: e.actual_output,
expectedOutput: e.expected_output,
context: e.context,
retrievalContext: e.retrieval_context,
additionalMetadata: e.additional_metadata,
toolsCalled: e.tools_called,
expectedTools: e.expected_tools,
name: e.name,
exampleId: e.example_id,
exampleIndex: e.example_index,
timestamp: e.timestamp,
traceId: e.trace_id,
example: e.example,
};
// TODO: Add handling for fields not in ExampleOptions (name, comments, etc.) if needed
this.addExample(new example_js_1.Example(options));
});
}
catch (error) {
if (error.code === 'ENOENT') {
console.error(`JSON file not found: ${filePath}`);
throw new Error(`The file ${filePath} was not found.`);
}
else if (error instanceof SyntaxError) {
console.error(`Invalid JSON file: ${filePath}`, error);
throw new Error(`The file ${filePath} is not a valid JSON file.`);
}
else {
console.error(`Error loading dataset from JSON: ${filePath}`, error);
throw error; // Re-throw other errors
}
}
}
/**
* Adds examples from a YAML file.
* Assumes the YAML file has a top-level key "examples" containing an array of example objects.
* @param filePath Path to the YAML file.
*/
addFromYaml(filePath) {
console.debug(`Loading dataset from YAML file: ${filePath}`);
try {
const fileContent = fs.readFileSync(filePath, 'utf-8');
const payload = js_yaml_1.default.load(fileContent);
if (!payload) {
throw new Error('The YAML file is empty.');
}
const examplesFromYaml = payload === null || payload === void 0 ? void 0 : payload.examples;
if (!Array.isArray(examplesFromYaml)) {
throw new Error('Invalid YAML format: "examples" key not found or not an array.');
}
console.info(`Adding ${examplesFromYaml.length} examples from YAML`);
examplesFromYaml.forEach((e) => {
// Map snake_case from file to camelCase for ExampleOptions
const options = {
input: e.input,
actualOutput: e.actual_output,
expectedOutput: e.expected_output,
context: e.context,
retrievalContext: e.retrieval_context,
additionalMetadata: e.additional_metadata,
toolsCalled: e.tools_called,
expectedTools: e.expected_tools,
name: e.name,
exampleId: e.example_id,
exampleIndex: e.example_index,
timestamp: e.timestamp,
traceId: e.trace_id,
example: e.example,
};
this.addExample(new example_js_1.Example(options));
});
}
catch (error) {
if (error.code === 'ENOENT') {
console.error(`YAML file not found: ${filePath}`);
throw new Error(`The file ${filePath} was not found.`);
}
else if (error instanceof js_yaml_1.default.YAMLException) {
console.error(`Invalid YAML file: ${filePath}`, error);
throw new Error(`The file ${filePath} is not a valid YAML file.`);
}
else {
console.error(`Error loading dataset from YAML: ${filePath}`, error);
throw error; // Re-throw other errors
}
}
}
/**
* Adds examples from a CSV file.
* @param filePath Path to the CSV file.
* @param headerMapping Dictionary mapping Example headers (keys) to custom headers in the CSV (values).
* @param primaryDelimiter Main delimiter used in CSV file. Defaults to ",".
* @param secondaryDelimiter Secondary delimiter for list fields (context, retrieval_context, etc.). Defaults to ";".
*/
addFromCsv(filePath,
// headerMapping values are CSV headers, keys need to be ExampleOptions properties (camelCase)
headerMapping, primaryDelimiter = ',', secondaryDelimiter = ';') {
console.debug(`Loading dataset from CSV file: ${filePath}`);
try {
const fileContent = fs.readFileSync(filePath, 'utf-8');
// Invert headerMapping for easier lookup: CSV Header -> ExampleOption Key
const csvHeaderToExampleKeyMap = Object.entries(headerMapping).reduce((acc, [exampleKey, csvHeader]) => {
if (csvHeader)
acc[csvHeader] = exampleKey;
return acc;
}, {});
const parseResult = papaparse_1.default.parse(fileContent, {
header: true,
delimiter: primaryDelimiter,
skipEmptyLines: true,
dynamicTyping: (field) => {
const exampleKey = csvHeaderToExampleKeyMap[field];
// Prevent auto-typing traceId
if (exampleKey === 'traceId')
return false;
return true;
},
});
if (parseResult.errors.length > 0) {
console.error('CSV parsing errors:', parseResult.errors);
// throw new Error(`Failed to parse CSV file: ${filePath}`);
}
const csvData = parseResult.data;
const newExamples = [];
csvData.forEach((row, index) => {
var _a;
const exampleOptions = {};
// Need a way to handle the 'example' boolean column if it's not in ExampleOptions
let isExampleRow = true; // Assume true unless 'example' column says otherwise
const exampleColumnCsvHeader = headerMapping['example']; // Assuming 'example' key exists in mapping for the boolean
for (const csvHeader in row) {
const exampleKey = csvHeaderToExampleKeyMap[csvHeader];
if (!exampleKey)
continue;
let value = row[csvHeader];
if (value === null || value === undefined || value === '') {
value = exampleKey === 'additionalMetadata' ? {} : null;
}
else {
value = String(value);
if (LIST_LIKE_KEYS.includes(exampleKey)) {
value = value
.split(secondaryDelimiter)
.map((item) => item.trim())
.filter((item) => item !== '');
}
else if (exampleKey === 'additionalMetadata') {
try {
value = JSON.parse(value);
}
catch (e) {
console.warn(`Row ${index + 2}: Could not parse additionalMetadata "${value}" as JSON. Storing as string.`);
}
}
else if (exampleColumnCsvHeader && csvHeader === exampleColumnCsvHeader) {
// Check if current CSV header matches the one mapped to the 'example' concept
const lowerVal = value.toLowerCase();
isExampleRow = !['false', '0', 'no', ''].includes(lowerVal);
// We don't store this boolean directly on ExampleOptions unless it has an 'example' field
continue; // Don't assign this value to exampleOptions map
}
else if (exampleKey === 'example') {
// Check key directly
const lowerVal = value.toLowerCase();
isExampleRow = !['false', '0', 'no', ''].includes(lowerVal);
// Assign to options IF the key exists in headerMapping
if (headerMapping.example) {
exampleOptions[exampleKey] = isExampleRow;
}
// Don't continue here, let assignment happen below
}
else if (exampleKey === 'actualOutput' || exampleKey === 'expectedOutput') {
// If it contains the delimiter, treat as array, otherwise string
if (value.includes(secondaryDelimiter)) {
value = value
.split(secondaryDelimiter)
.map((item) => item.trim())
.filter((item) => item !== '');
} // else leave as string
}
}
exampleOptions[exampleKey] = value; // Assign using the camelCase key
}
// Only add if it's marked as an example and has required fields
if (isExampleRow) {
// Check required fields using camelCase
if (exampleOptions.input === undefined ||
exampleOptions.input === null
// actualOutput is optional in ExampleOptions, so don't check here
) {
console.warn(`Row ${index + 2}: Skipping example due to missing 'input'.`);
}
else {
// Add trace_id if missing and mapped
if (headerMapping.traceId && !exampleOptions.traceId) {
exampleOptions.traceId = String((_a = row[headerMapping.traceId]) !== null && _a !== void 0 ? _a : null); // Ensure it's a string
}
newExamples.push(new example_js_1.Example(exampleOptions));
}
}
});
console.info(`Adding ${newExamples.length} examples from CSV`);
newExamples.forEach((e) => this.addExample(e));
}
catch (error) {
if (error.code === 'ENOENT') {
console.error(`CSV file not found: ${filePath}`);
throw new Error(`The file ${filePath} was not found.`);
}
else {
console.error(`Error loading dataset from CSV: ${filePath}`, error);
throw error; // Re-throw other errors
}
}
}
/**
* Saves the dataset as a file.
* @param fileType The file type to save as ('json', 'csv', 'yaml').
* @param dirPath The directory path to save the file to.
* @param saveName Optional: The name of the file (without extension). Defaults to a timestamp.
* @param secondaryDelimiter Optional: The delimiter used for joining list fields in CSV output. Defaults to ";".
*/
saveAs(fileType, dirPath, saveName, secondaryDelimiter = ';') {
if (!ACCEPTABLE_FILE_TYPES.includes(fileType)) {
throw new TypeError(`Invalid file type: ${fileType}. Please choose from ${ACCEPTABLE_FILE_TYPES.join(', ')}`);
}
try {
// Ensure directory exists
if (!fs.existsSync(dirPath)) {
fs.mkdirSync(dirPath, { recursive: true });
}
const fileName = saveName !== null && saveName !== void 0 ? saveName : (0, date_fns_1.format)(new Date(), 'yyyyMMdd_HHmmss');
const completePath = path.join(dirPath, `${fileName}.${fileType}`);
// Use the Example class's toJSON method which outputs snake_case keys
const examplesForOutput = this.examples.map(e => e.toJSON());
console.info(`Saving dataset to ${completePath}`);
if (fileType === 'json') {
const jsonContent = JSON.stringify({ examples: examplesForOutput }, null, 4);
fs.writeFileSync(completePath, jsonContent, 'utf-8');
}
else if (fileType === 'yaml') {
// The toJSON output is already structured correctly for YAML
const yamlContent = js_yaml_1.default.dump({ examples: examplesForOutput }, { noRefs: true, lineWidth: -1 });
fs.writeFileSync(completePath, yamlContent, 'utf-8');
}
else if (fileType === 'csv') {
// Define CSV headers using the snake_case keys expected in the output
const csvHeaders = [
'input',
'actual_output',
'expected_output',
'context',
'retrieval_context',
'additional_metadata',
'tools_called',
'expected_tools',
'name',
'example_id',
'example_index',
'timestamp',
'trace_id',
'example',
];
// Prepare data for PapaParse using the snake_case keys from toJSON()
const csvData = examplesForOutput.map((e_json) => {
const row = {};
csvHeaders.forEach((header) => {
const value = e_json[header];
// Check original camelCase list keys for joining arrays
if ((header === 'context' ||
header === 'retrieval_context' ||
header === 'tools_called' ||
header === 'expected_tools') && Array.isArray(value)) {
row[header] = value.join(secondaryDelimiter);
}
else if (header === 'additional_metadata' && typeof value === 'object' && value !== null) {
try {
row[header] = JSON.stringify(value);
}
catch (_a) {
row[header] = String(value);
}
}
else if (header === 'example') {
row[header] = value !== null && value !== void 0 ? value : true; // Default to true if null/undefined
}
else {
row[header] = value === undefined ? null : value;
}
});
return row;
});
const csvContent = papaparse_1.default.unparse(csvData, {
columns: csvHeaders,
header: true,
delimiter: ',',
});
fs.writeFileSync(completePath, csvContent, 'utf-8');
}
}
catch (error) {
console.error(`Error saving dataset to ${fileType}:`, error);
throw error; // Re-throw error
}
}
}
exports.EvalDataset = EvalDataset;
//# sourceMappingURL=eval-dataset.js.map