judgeval
Version:
Judgment SDK for TypeScript/JavaScript
512 lines • 24.2 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.FaithfulnessScorer = void 0;
const base_scorer_js_1 = require("../../base-scorer.js");
const constants_js_1 = require("../../../constants.js");
const logger_js_1 = require("../../../common/logger.js");
const prompts_js_1 = require("./prompts.js");
const index_js_1 = require("../../../judges/index.js");
// Required parameters for this scorer
const requiredParams = ['input', 'actualOutput', 'retrievalContext'];
/**
* FaithfulnessScorer evaluates how well the actual output is supported by the retrieval context
* by extracting claims from the output and checking if each claim is supported by the context.
*/
class FaithfulnessScorer extends base_scorer_js_1.JudgevalScorer {
/**
* Constructor for FaithfulnessScorer
* @param threshold Minimum score to consider the evaluation successful (default: 0.5)
* @param model LLM to use for evaluation (string or Judge instance)
* @param include_reason Whether to generate a reason for the score
* @param async_mode Whether to use asynchronous evaluation
* @param strict_mode If true, sets threshold to 1.0 (requiring perfect match)
* @param verbose_mode Enables detailed logging
* @param user Optional user identifier for the LLM
* @param additional_metadata Additional metadata to include in the result
*/
constructor(threshold = 0.5, model, include_reason = true, async_mode = true, strict_mode = false, verbose_mode = true, user, additional_metadata) {
super(constants_js_1.APIScorer.FAITHFULNESS, strict_mode ? 1.0 : threshold, additional_metadata, include_reason, async_mode, strict_mode, verbose_mode);
(0, logger_js_1.info)(`Initializing FaithfulnessScorer with threshold=${this.threshold}, model=${model}, strict_mode=${strict_mode}`);
const { judge, usingNativeModel } = (0, index_js_1.createJudge)(model, user);
this.model = judge;
this.usingNativeModel = usingNativeModel;
this.evaluation_model = this.model.getModelName();
(0, logger_js_1.log)(`Using model: ${this.evaluation_model}`);
// Set required fields for this scorer
this.requiredFields = ['input', 'actualOutput', 'retrievalContext'];
}
/**
* Generate claims from actual output asynchronously
*/
_aGenerateClaims(actualOutput_1) {
return __awaiter(this, arguments, void 0, function* (actualOutput, allClaims = false) {
(0, logger_js_1.log)("Generating claims asynchronously");
// Handle string array
const actualOutputStr = Array.isArray(actualOutput) ? actualOutput.join('\n') : actualOutput;
const prompt = prompts_js_1.FaithfulnessTemplate.findClaims(actualOutputStr, allClaims);
try {
const response = yield this.model.aGenerate(prompt);
// Parse the response
try {
const jsonResponse = JSON.parse(response);
const parsed = prompts_js_1.ClaimsSchema.safeParse(jsonResponse);
if (parsed.success) {
this.claimsWithQuotes = parsed.data.claims;
return parsed.data.claims.map(c => c.claim);
}
else {
// Fallback to direct access if schema validation fails
(0, logger_js_1.warn)("Schema validation failed, falling back to raw response parsing");
if (jsonResponse.claims && Array.isArray(jsonResponse.claims)) {
this.claimsWithQuotes = jsonResponse.claims;
return jsonResponse.claims.map((c) => c.claim);
}
}
}
catch (parseError) {
(0, logger_js_1.warn)(`Error parsing JSON response: ${parseError}`);
// Try to extract JSON from the response text
const jsonMatch = response.match(/\{[\s\S]*\}/);
if (jsonMatch) {
try {
const extractedJson = JSON.parse(jsonMatch[0]);
if (extractedJson.claims && Array.isArray(extractedJson.claims)) {
this.claimsWithQuotes = extractedJson.claims;
return extractedJson.claims.map((c) => c.claim);
}
}
catch (e) {
(0, logger_js_1.error)(`Failed to extract JSON from response: ${e}`);
}
}
}
// If all parsing attempts fail, return empty array
(0, logger_js_1.error)("Failed to parse claims from model response");
return [];
}
catch (e) {
(0, logger_js_1.error)(`Error generating claims: ${e}`);
return [];
}
});
}
/**
* Generate claims from actual output synchronously
*/
_generateClaims(actualOutput, allClaims = false) {
// Handle string array
const actualOutputStr = Array.isArray(actualOutput) ? actualOutput.join('\n') : actualOutput;
const prompt = prompts_js_1.FaithfulnessTemplate.findClaims(actualOutputStr, allClaims);
try {
const response = this.model.generate(prompt);
// Parse the response
try {
const jsonResponse = JSON.parse(response);
const parsed = prompts_js_1.ClaimsSchema.safeParse(jsonResponse);
if (parsed.success) {
this.claimsWithQuotes = parsed.data.claims;
return parsed.data.claims.map(c => c.claim);
}
else {
// Fallback to direct access if schema validation fails
(0, logger_js_1.warn)("Schema validation failed, falling back to raw response parsing");
if (jsonResponse.claims && Array.isArray(jsonResponse.claims)) {
this.claimsWithQuotes = jsonResponse.claims;
return jsonResponse.claims.map((c) => c.claim);
}
}
}
catch (parseError) {
(0, logger_js_1.warn)(`Error parsing JSON response: ${parseError}`);
// Try to extract JSON from the response text
const jsonMatch = response.match(/\{[\s\S]*\}/);
if (jsonMatch) {
try {
const extractedJson = JSON.parse(jsonMatch[0]);
if (extractedJson.claims && Array.isArray(extractedJson.claims)) {
this.claimsWithQuotes = extractedJson.claims;
return extractedJson.claims.map((c) => c.claim);
}
}
catch (e) {
(0, logger_js_1.error)(`Failed to extract JSON from response: ${e}`);
}
}
}
// If all parsing attempts fail, return empty array
(0, logger_js_1.error)("Failed to parse claims from model response");
return [];
}
catch (e) {
(0, logger_js_1.error)(`Error generating claims: ${e}`);
return [];
}
}
/**
* Generate verdicts for claims against retrieval context asynchronously
*/
_aGenerateVerdicts(retrievalContext) {
return __awaiter(this, void 0, void 0, function* () {
(0, logger_js_1.log)("Generating verdicts asynchronously");
if (!this.claims || this.claims.length === 0) {
(0, logger_js_1.warn)("No claims to evaluate");
return [];
}
// Handle string array
const contextStr = Array.isArray(retrievalContext) ? retrievalContext.join('\n') : retrievalContext;
const prompt = prompts_js_1.FaithfulnessTemplate.generateVerdicts(this.claims, contextStr);
try {
const response = yield this.model.aGenerate(prompt);
// Parse the response
try {
const jsonResponse = JSON.parse(response);
const parsed = prompts_js_1.VerdictsSchema.safeParse(jsonResponse);
if (parsed.success) {
return parsed.data.verdicts;
}
else {
// Fallback to direct access if schema validation fails
(0, logger_js_1.warn)("Schema validation failed, falling back to raw response parsing");
if (jsonResponse.verdicts && Array.isArray(jsonResponse.verdicts)) {
return jsonResponse.verdicts.map((v) => ({
verdict: v.verdict,
reason: v.reason
}));
}
}
}
catch (parseError) {
(0, logger_js_1.warn)(`Error parsing JSON response: ${parseError}`);
// Try to extract JSON from the response text
const jsonMatch = response.match(/\{[\s\S]*\}/);
if (jsonMatch) {
try {
const extractedJson = JSON.parse(jsonMatch[0]);
if (extractedJson.verdicts && Array.isArray(extractedJson.verdicts)) {
return extractedJson.verdicts.map((v) => ({
verdict: v.verdict,
reason: v.reason
}));
}
}
catch (e) {
(0, logger_js_1.error)(`Failed to extract JSON from response: ${e}`);
}
}
}
// If all parsing attempts fail, return empty array
(0, logger_js_1.error)("Failed to parse verdicts from model response");
return [];
}
catch (e) {
(0, logger_js_1.error)(`Error generating verdicts: ${e}`);
return [];
}
});
}
/**
* Generate verdicts for claims against retrieval context synchronously
*/
_generateVerdicts(retrievalContext) {
if (!this.claims || this.claims.length === 0) {
(0, logger_js_1.warn)("No claims to evaluate");
return [];
}
// Handle string array
const contextStr = Array.isArray(retrievalContext) ? retrievalContext.join('\n') : retrievalContext;
const prompt = prompts_js_1.FaithfulnessTemplate.generateVerdicts(this.claims, contextStr);
try {
const response = this.model.generate(prompt);
// Parse the response
try {
const jsonResponse = JSON.parse(response);
const parsed = prompts_js_1.VerdictsSchema.safeParse(jsonResponse);
if (parsed.success) {
return parsed.data.verdicts;
}
else {
// Fallback to direct access if schema validation fails
(0, logger_js_1.warn)("Schema validation failed, falling back to raw response parsing");
if (jsonResponse.verdicts && Array.isArray(jsonResponse.verdicts)) {
return jsonResponse.verdicts.map((v) => ({
verdict: v.verdict,
reason: v.reason
}));
}
}
}
catch (parseError) {
(0, logger_js_1.warn)(`Error parsing JSON response: ${parseError}`);
// Try to extract JSON from the response text
const jsonMatch = response.match(/\{[\s\S]*\}/);
if (jsonMatch) {
try {
const extractedJson = JSON.parse(jsonMatch[0]);
if (extractedJson.verdicts && Array.isArray(extractedJson.verdicts)) {
return extractedJson.verdicts.map((v) => ({
verdict: v.verdict,
reason: v.reason
}));
}
}
catch (e) {
(0, logger_js_1.error)(`Failed to extract JSON from response: ${e}`);
}
}
}
// If all parsing attempts fail, return empty array
(0, logger_js_1.error)("Failed to parse verdicts from model response");
return [];
}
catch (e) {
(0, logger_js_1.error)(`Error generating verdicts: ${e}`);
return [];
}
}
/**
* Generate reason for the score asynchronously
*/
_aGenerateReason() {
return __awaiter(this, void 0, void 0, function* () {
var _a;
if (!this.include_reason) {
return undefined;
}
if (!this.verdicts || this.verdicts.length === 0) {
return undefined;
}
try {
// Generate reason
const prompt = prompts_js_1.FaithfulnessTemplate.generateReason(this.verdicts, ((_a = this.score) === null || _a === void 0 ? void 0 : _a.toString()) || "0");
const reasonText = yield this.model.aGenerate(prompt);
const parsedReason = prompts_js_1.ReasonSchema.safeParse(JSON.parse(reasonText));
if (!parsedReason.success) {
(0, logger_js_1.error)(`Failed to parse reason: ${parsedReason.error}`);
return undefined;
}
return parsedReason.data.reason;
}
catch (err) {
(0, logger_js_1.error)(`Error getting reason: ${err}`);
return undefined;
}
});
}
/**
* Generate reason for the score synchronously
*/
_generateReason() {
var _a;
if (!this.include_reason) {
return undefined;
}
if (!this.verdicts || this.verdicts.length === 0) {
return undefined;
}
try {
// Generate reason
const prompt = prompts_js_1.FaithfulnessTemplate.generateReason(this.verdicts, ((_a = this.score) === null || _a === void 0 ? void 0 : _a.toString()) || "0");
const reasonText = this.model.generate(prompt);
const parsedReason = prompts_js_1.ReasonSchema.safeParse(JSON.parse(reasonText));
if (!parsedReason.success) {
(0, logger_js_1.error)(`Failed to parse reason: ${parsedReason.error}`);
return undefined;
}
return parsedReason.data.reason;
}
catch (err) {
(0, logger_js_1.error)(`Error getting reason: ${err}`);
return undefined;
}
}
/**
* Compute score based on verdicts
*/
_computeScore() {
(0, logger_js_1.log)("Computing score");
// If we have no claims or verdicts due to API errors, return 0
if (!this.claims || this.claims.length === 0) {
return 0;
}
if (!this.verdicts || this.verdicts.length === 0) {
return 0;
}
let supportedCount = 0;
let partialCount = 0;
for (const verdict of this.verdicts) {
const verdictLower = verdict.verdict.trim().toLowerCase();
if (verdictLower === "yes") {
supportedCount++;
}
else if (verdictLower === "partially") {
partialCount += 0.5;
}
}
const score = (supportedCount + partialCount) / this.verdicts.length;
// Match Python implementation's handling of strict_mode
return this.strict_mode && score < this.threshold ? 0 : score;
}
/**
* Create verbose logs for debugging
*/
_createVerboseLogs() {
if (!this.verbose_mode) {
return '';
}
const steps = [
`Claims:\n${JSON.stringify(this.claims, null, 2)}`,
`Verdicts:\n${JSON.stringify(this.verdicts, null, 2)}`,
`Score: ${this.score}\nReason: ${this.reason}`
];
return steps.join('\n\n');
}
/**
* Score an example synchronously
*/
syncScoreExample(example, allClaims = false) {
(0, logger_js_1.info)("Starting example scoring (sync mode)");
try {
// Check required parameters
this._checkExampleParams(example);
// Process example
if (this.async_mode) {
throw new Error("Cannot use synchronous scoreExample with async_mode=true. Use async scoreExample instead.");
}
this.claims = this._generateClaims(example.actualOutput, allClaims);
// Add claims to additional metadata
if (!this.additional_metadata) {
this.additional_metadata = {};
}
this.additional_metadata.claims = this.claims;
this.additional_metadata.claimsWithQuotes = this.claimsWithQuotes;
this.verdicts = this._generateVerdicts(example.retrievalContext);
this.additional_metadata.verdicts = this.verdicts;
this.score = this._computeScore();
this.reason = this._generateReason();
this.success = this._successCheck();
this.verbose_logs = this._createVerboseLogs();
// Calculate evaluation cost
const promptTokens = 800; // Estimate - in a real implementation, track actual tokens
const completionTokens = 300; // Estimate - in a real implementation, track actual tokens
this.evaluation_cost = this._calculateTokenCosts(this.evaluation_model || 'gpt-3.5-turbo', promptTokens, completionTokens);
(0, logger_js_1.info)(`Scoring completed with score: ${this.score}`);
// Ensure all fields match the ScorerData interface
return {
name: this.type,
threshold: this.threshold,
success: this.success || false,
score: this.score || 0,
reason: this.reason !== undefined ? this.reason : null,
strict_mode: this.strict_mode || false,
evaluation_model: this.evaluation_model || null,
error: null,
evaluation_cost: this.evaluation_cost || null,
verbose_logs: this.verbose_logs ? this.verbose_logs : null,
additional_metadata: this.additional_metadata || {}
};
}
catch (error) {
// Handle errors
const errorMessage = error instanceof Error ? error.message : String(error);
this.error = errorMessage;
this.success = false;
return {
name: this.type,
threshold: this.threshold,
success: false,
score: 0,
reason: `Error during scoring: ${errorMessage}`,
strict_mode: this.strict_mode || false,
evaluation_model: this.evaluation_model || null,
error: errorMessage,
evaluation_cost: null,
verbose_logs: null,
additional_metadata: this.additional_metadata || {}
};
}
}
/**
* Score an example asynchronously
*/
scoreExample(example_1) {
return __awaiter(this, arguments, void 0, function* (example, allClaims = false) {
if (!this.async_mode) {
return this.syncScoreExample(example, allClaims);
}
(0, logger_js_1.info)("Starting example scoring (async mode)");
try {
// Check required parameters
this._checkExampleParams(example);
// Process example
this.claims = yield this._aGenerateClaims(example.actualOutput, allClaims);
// Add claims to additional metadata
if (!this.additional_metadata) {
this.additional_metadata = {};
}
this.additional_metadata.claims = this.claims;
this.additional_metadata.claimsWithQuotes = this.claimsWithQuotes;
this.verdicts = yield this._aGenerateVerdicts(example.retrievalContext);
this.additional_metadata.verdicts = this.verdicts;
this.score = this._computeScore();
this.reason = yield this._aGenerateReason();
this.success = this._successCheck();
this.verbose_logs = this._createVerboseLogs();
// Calculate evaluation cost
const promptTokens = 800; // Estimate - in a real implementation, track actual tokens
const completionTokens = 300; // Estimate - in a real implementation, track actual tokens
this.evaluation_cost = this._calculateTokenCosts(this.evaluation_model || 'gpt-3.5-turbo', promptTokens, completionTokens);
(0, logger_js_1.info)(`Scoring completed with score: ${this.score}`);
// Ensure all fields match the ScorerData interface
return {
name: this.type,
threshold: this.threshold,
success: this.success || false,
score: this.score || 0,
reason: this.reason !== undefined ? this.reason : null,
strict_mode: this.strict_mode || false,
evaluation_model: this.evaluation_model || null,
error: null,
evaluation_cost: this.evaluation_cost || null,
verbose_logs: this.verbose_logs ? this.verbose_logs : null,
additional_metadata: this.additional_metadata || {}
};
}
catch (error) {
// Handle errors
const errorMessage = error instanceof Error ? error.message : String(error);
this.error = errorMessage;
this.success = false;
return {
name: this.type,
threshold: this.threshold,
success: false,
score: 0,
reason: `Error during scoring: ${errorMessage}`,
strict_mode: this.strict_mode || false,
evaluation_model: this.evaluation_model || null,
error: errorMessage,
evaluation_cost: null,
verbose_logs: null,
additional_metadata: this.additional_metadata || {}
};
}
});
}
/**
* Get the name of the scorer
*/
get name() {
return "Faithfulness";
}
}
exports.FaithfulnessScorer = FaithfulnessScorer;
//# sourceMappingURL=faithfulness.js.map