judgeval
Version:
Judgment SDK for TypeScript/JavaScript
374 lines • 16.1 kB
JavaScript
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
import { JudgevalScorer } from '../../base-scorer.js';
import { APIScorer } from '../../../constants.js';
import { info } from '../../../common/logger.js';
import { InstructionAdherenceTemplate, InstructionsSchema, VerdictsSchema } from './prompts.js';
import { createJudge } from '../../../judges/index.js';
// Required parameters for this scorer
const required_params = ['input', 'actualOutput'];
/**
* InstructionAdherenceScorer evaluates how well an LLM follows instructions
* by extracting instructions from the input and checking if they are followed in the output.
*
* The score is the average of scores for each instruction (1 = followed, 0.5 = partially followed, 0 = not followed).
*/
export class InstructionAdherenceScorer extends JudgevalScorer {
/**
* Create a new InstructionAdherenceScorer
*
* @param threshold - Success threshold (default: 0.5)
* @param model - Model to use for evaluation (default: DefaultJudge)
* @param include_reason - Whether to include a reason for the score (default: true)
* @param async_mode - Whether to use async mode (default: false)
* @param strict_mode - Whether to use strict mode (default: false)
* @param verbose_mode - Whether to include verbose logs (default: false)
*/
constructor(threshold = 0.5, model = undefined, include_reason = true, async_mode = false, strict_mode = false, verbose_mode = false) {
super(APIScorer.INSTRUCTION_ADHERENCE, strict_mode ? 1 : threshold, undefined, include_reason, async_mode, strict_mode, verbose_mode);
this._instructions = [];
this._verdicts = [];
const { judge, usingNativeModel } = createJudge(model);
this.model = judge;
this.using_native_model = usingNativeModel;
this.evaluation_model = this.model.getModelName();
this.requiredFields = ['input', 'actualOutput'];
}
/**
* Extract instructions from input text
*/
_aGetInstructions(input) {
return __awaiter(this, void 0, void 0, function* () {
const prompt = InstructionAdherenceTemplate.getInstructions(input);
if (this.using_native_model) {
const res = yield this.model.aGenerate(prompt);
try {
const data = JSON.parse(res);
return data.instructions || [];
}
catch (error) {
throw new Error(`Failed to parse response: ${error}`);
}
}
else {
try {
// Create a parser function to validate the response
const parseInstructionsResponse = (response) => {
const parsed = JSON.parse(response);
const result = InstructionsSchema.safeParse(parsed);
if (result.success) {
return result.data;
}
throw new Error(`Invalid response format: ${result.error}`);
};
const res = yield this.model.aGenerate(prompt);
return parseInstructionsResponse(res).instructions;
}
catch (error) {
const res = yield this.model.aGenerate(prompt);
try {
const data = JSON.parse(res);
return data.instructions || [];
}
catch (parseError) {
throw new Error(`Failed to parse response: ${parseError}`);
}
}
}
});
}
/**
* Extract instructions from input text (synchronous)
*/
_getInstructions(input) {
const prompt = InstructionAdherenceTemplate.getInstructions(input);
if (this.using_native_model) {
const res = this.model.generate(prompt);
try {
const data = JSON.parse(res);
return data.instructions || [];
}
catch (error) {
throw new Error(`Failed to parse response: ${error}`);
}
}
else {
try {
// Create a parser function to validate the response
const parseInstructionsResponse = (response) => {
const parsed = JSON.parse(response);
const result = InstructionsSchema.safeParse(parsed);
if (result.success) {
return result.data;
}
throw new Error(`Invalid response format: ${result.error}`);
};
const res = this.model.generate(prompt);
return parseInstructionsResponse(res).instructions;
}
catch (error) {
const res = this.model.generate(prompt);
try {
const data = JSON.parse(res);
return data.instructions || [];
}
catch (parseError) {
throw new Error(`Failed to parse response: ${parseError}`);
}
}
}
}
/**
* Generate verdicts for each instruction
*/
_aGetVerdicts(instructions, actualOutput) {
return __awaiter(this, void 0, void 0, function* () {
if (instructions.length === 0) {
return [];
}
const prompt = InstructionAdherenceTemplate.generateVerdicts(instructions, actualOutput);
if (this.using_native_model) {
const res = yield this.model.aGenerate(prompt);
try {
const data = JSON.parse(res);
return data.verdicts || [];
}
catch (error) {
throw new Error(`Failed to parse response: ${error}`);
}
}
else {
try {
// Create a parser function to validate the response
const parseVerdictsResponse = (response) => {
const parsed = JSON.parse(response);
const result = VerdictsSchema.safeParse(parsed);
if (result.success) {
return result.data;
}
throw new Error(`Invalid response format: ${result.error}`);
};
const res = yield this.model.aGenerate(prompt);
return parseVerdictsResponse(res).verdicts;
}
catch (error) {
const res = yield this.model.aGenerate(prompt);
try {
const data = JSON.parse(res);
return data.verdicts || [];
}
catch (parseError) {
throw new Error(`Failed to parse response: ${parseError}`);
}
}
}
});
}
/**
* Generate verdicts for each instruction (synchronous)
*/
_getVerdicts(instructions, actualOutput) {
if (instructions.length === 0) {
return [];
}
const prompt = InstructionAdherenceTemplate.generateVerdicts(instructions, actualOutput);
if (this.using_native_model) {
const res = this.model.generate(prompt);
try {
const data = JSON.parse(res);
return data.verdicts || [];
}
catch (error) {
throw new Error(`Failed to parse response: ${error}`);
}
}
else {
try {
// Create a parser function to validate the response
const parseVerdictsResponse = (response) => {
const parsed = JSON.parse(response);
const result = VerdictsSchema.safeParse(parsed);
if (result.success) {
return result.data;
}
throw new Error(`Invalid response format: ${result.error}`);
};
const res = this.model.generate(prompt);
return parseVerdictsResponse(res).verdicts;
}
catch (error) {
const res = this.model.generate(prompt);
try {
const data = JSON.parse(res);
return data.verdicts || [];
}
catch (parseError) {
throw new Error(`Failed to parse response: ${parseError}`);
}
}
}
}
/**
* Calculate the instruction adherence score
*/
_computeScore() {
if (this._verdicts.length === 0) {
return 1;
}
let totalScore = 0;
for (const verdict of this._verdicts) {
totalScore += verdict.score;
}
return totalScore / this._verdicts.length;
}
/**
* Create verbose logs for debugging
*/
_createVerboseLogs() {
if (!this.verbose_mode) {
return null;
}
const steps = [
`Instructions:\n${JSON.stringify(this._instructions, null, 2)}`,
`Score: ${this.score}\nReason: ${this.reason || "No reason provided"}`
];
return steps.join('\n\n');
}
/**
* Score an example synchronously
*/
syncScoreExample(example) {
info("Starting example scoring (sync mode)");
try {
// Check required parameters
this._checkExampleParams(example);
// Process example
this._instructions = this._getInstructions(example.input);
this._verdicts = this._getVerdicts(this._instructions, example.actualOutput);
// Add instructions and verdicts to additional metadata
const additional_metadata = {
instructions: this._instructions,
verdicts: this._verdicts
};
this.score = this._computeScore();
this.reason = this._verdicts.length > 0 ? JSON.stringify(this._verdicts) : 'No instructions found';
this.success = this._successCheck();
const verbose_logs = this._createVerboseLogs();
// Calculate evaluation cost
const promptTokens = 700; // Estimate - in a real implementation, track actual tokens
const completionTokens = 250; // Estimate - in a real implementation, track actual tokens
this.evaluation_cost = this._calculateTokenCosts(this.evaluation_model || 'gpt-3.5-turbo', promptTokens, completionTokens);
info(`Scoring completed with score: ${this.score}`);
// Ensure all fields match the ScorerData interface
return {
name: this.type,
threshold: this.threshold,
success: this.success,
score: this.score,
reason: this.reason || "",
strict_mode: this.strict_mode,
evaluation_model: this.evaluation_model || null,
error: null,
evaluation_cost: this.evaluation_cost || null,
verbose_logs: verbose_logs,
additional_metadata: additional_metadata
};
}
catch (error) {
// Handle errors
const errorMessage = error instanceof Error ? error.message : String(error);
this.error = errorMessage;
return {
name: this.type,
threshold: this.threshold,
success: false,
score: 0,
reason: `Error during scoring: ${errorMessage}`,
strict_mode: this.strict_mode,
evaluation_model: this.evaluation_model || null,
error: errorMessage,
evaluation_cost: null,
verbose_logs: null,
additional_metadata: {}
};
}
}
/**
* Score an example asynchronously
*/
scoreExample(example) {
return __awaiter(this, void 0, void 0, function* () {
if (!this.async_mode) {
return this.syncScoreExample(example);
}
info("Starting example scoring (async mode)");
try {
// Check required parameters
this._checkExampleParams(example);
// Process example
this._instructions = yield this._aGetInstructions(example.input);
this._verdicts = yield this._aGetVerdicts(this._instructions, example.actualOutput);
// Add instructions and verdicts to additional metadata
const additional_metadata = {
instructions: this._instructions,
verdicts: this._verdicts
};
this.score = this._computeScore();
this.reason = this._verdicts.length > 0 ? JSON.stringify(this._verdicts) : 'No instructions found';
this.success = this._successCheck();
const verbose_logs = this._createVerboseLogs();
// Calculate evaluation cost
const promptTokens = 700; // Estimate - in a real implementation, track actual tokens
const completionTokens = 250; // Estimate - in a real implementation, track actual tokens
this.evaluation_cost = this._calculateTokenCosts(this.evaluation_model || 'gpt-3.5-turbo', promptTokens, completionTokens);
info(`Scoring completed with score: ${this.score}`);
// Ensure all fields match the ScorerData interface
return {
name: this.type,
threshold: this.threshold,
success: this.success,
score: this.score,
reason: this.reason || "",
strict_mode: this.strict_mode,
evaluation_model: this.evaluation_model || null,
error: null,
evaluation_cost: this.evaluation_cost || null,
verbose_logs: verbose_logs,
additional_metadata: additional_metadata
};
}
catch (error) {
// Handle errors
const errorMessage = error instanceof Error ? error.message : String(error);
this.error = errorMessage;
return {
name: this.type,
threshold: this.threshold,
success: false,
score: 0,
reason: `Error during scoring: ${errorMessage}`,
strict_mode: this.strict_mode,
evaluation_model: this.evaluation_model || null,
error: errorMessage,
evaluation_cost: null,
verbose_logs: null,
additional_metadata: {}
};
}
});
}
/**
* Get the name of the scorer
*/
get name() {
return "Instruction Adherence";
}
}
//# sourceMappingURL=instruction-adherence.js.map