judgeval/dist/cjs/scorers/metrics/hallucination/hallucination.js

Judgment SDK for TypeScript/JavaScript

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.HallucinationScorer = void 0;
const base_scorer_js_1 = require("../../base-scorer.js");
const constants_js_1 = require("../../../constants.js");
const logger_js_1 = require("../../../common/logger.js");
const prompts_js_1 = require("./prompts.js");
const index_js_1 = require("../../../judges/index.js");
// Required parameters for this scorer
const required_params = ['actualOutput', 'context'];
/**
 * HallucinationScorer evaluates whether an LLM's output contains hallucinations
 * by comparing it against provided context.
 *
 * The score is the fraction of context segments that contradict the output.
 * Lower scores are better (0 = no hallucinations, 1 = all contexts contradict the output).
 */
class HallucinationScorer extends base_scorer_js_1.JudgevalScorer {
    /**
     * Create a new HallucinationScorer
     *
     * @param threshold - Success threshold (default: 0.5)
     * @param model - Model to use for evaluation (default: DefaultJudge)
     * @param include_reason - Whether to include a reason for the score (default: true)
     * @param async_mode - Whether to use async mode (default: false)
     * @param strict_mode - Whether to use strict mode (default: false)
     * @param verbose_mode - Whether to include verbose logs (default: false)
     */
    constructor(threshold = 0.5, model = undefined, include_reason = true, async_mode = false, strict_mode = false, verbose_mode = false) {
        super(constants_js_1.APIScorer.HALLUCINATION, strict_mode ? 1 : threshold, undefined, include_reason, async_mode, strict_mode, verbose_mode);
        this._verdicts = [];
        const { judge, usingNativeModel } = (0, index_js_1.createJudge)(model);
        this.model = judge;
        this.using_native_model = usingNativeModel;
        this.evaluation_model = this.model.getModelName();
        this.requiredFields = ['actualOutput', 'context'];
    }
    /**
     * Generate verdicts for each context
     */
    _aGenerateVerdicts(actualOutput, contexts) {
        return __awaiter(this, void 0, void 0, function* () {
            const prompt = prompts_js_1.HallucinationTemplate.generateVerdicts(actualOutput, contexts);
            if (this.using_native_model) {
                const res = yield this.model.aGenerate(prompt);
                try {
                    const data = JSON.parse(res);
                    return data.verdicts.map((item) => ({
                        verdict: item.verdict,
                        reason: item.reason
                    }));
                }
                catch (error) {
                    throw new Error(`Failed to parse response: ${error}`);
                }
            }
            else {
                try {
                    const parseVerdictsResponse = (response) => {
                        const parsed = JSON.parse(response);
                        const result = prompts_js_1.VerdictsSchema.safeParse(parsed);
                        if (result.success) {
                            return result.data;
                        }
                        throw new Error(`Invalid response format: ${result.error}`);
                    };
                    const res = yield this.model.aGenerate(prompt);
                    return parseVerdictsResponse(res).verdicts;
                }
                catch (error) {
                    const res = yield this.model.aGenerate(prompt);
                    try {
                        const data = JSON.parse(res);
                        return data.verdicts.map((item) => ({
                            verdict: item.verdict,
                            reason: item.reason
                        }));
                    }
                    catch (parseError) {
                        throw new Error(`Failed to parse response: ${parseError}`);
                    }
                }
            }
        });
    }
    /**
     * Generate verdicts for each context (synchronous)
     */
    _generateVerdicts(actualOutput, contexts) {
        const prompt = prompts_js_1.HallucinationTemplate.generateVerdicts(actualOutput, contexts);
        if (this.using_native_model) {
            const res = this.model.generate(prompt);
            try {
                const data = JSON.parse(res);
                return data.verdicts.map((item) => ({
                    verdict: item.verdict,
                    reason: item.reason
                }));
            }
            catch (error) {
                throw new Error(`Failed to parse response: ${error}`);
            }
        }
        else {
            try {
                const parseVerdictsResponse = (response) => {
                    const parsed = JSON.parse(response);
                    const result = prompts_js_1.VerdictsSchema.safeParse(parsed);
                    if (result.success) {
                        return result.data;
                    }
                    throw new Error(`Invalid response format: ${result.error}`);
                };
                const res = this.model.generate(prompt);
                return parseVerdictsResponse(res).verdicts;
            }
            catch (error) {
                const res = this.model.generate(prompt);
                try {
                    const data = JSON.parse(res);
                    return data.verdicts.map((item) => ({
                        verdict: item.verdict,
                        reason: item.reason
                    }));
                }
                catch (parseError) {
                    throw new Error(`Failed to parse response: ${parseError}`);
                }
            }
        }
    }
    /**
     * Generate a reason for the score
     */
    _aGenerateReason(actualOutput, contexts) {
        return __awaiter(this, void 0, void 0, function* () {
            if (!this.include_reason) {
                return "No reason provided (include_reason is false)";
            }
            const prompt = prompts_js_1.HallucinationTemplate.generateReason(actualOutput, contexts);
            if (this.using_native_model) {
                const res = yield this.model.aGenerate(prompt);
                try {
                    const data = JSON.parse(res);
                    return data.reason || "No reason provided in response";
                }
                catch (error) {
                    throw new Error(`Failed to parse response: ${error}`);
                }
            }
            else {
                try {
                    const parseReasonResponse = (response) => {
                        const parsed = JSON.parse(response);
                        const result = prompts_js_1.ReasonSchema.safeParse(parsed);
                        if (result.success) {
                            return result.data;
                        }
                        throw new Error(`Invalid response format: ${result.error}`);
                    };
                    const res = yield this.model.aGenerate(prompt);
                    return parseReasonResponse(res).reason;
                }
                catch (error) {
                    const res = yield this.model.aGenerate(prompt);
                    try {
                        const data = JSON.parse(res);
                        return data.reason || "No reason provided in response";
                    }
                    catch (parseError) {
                        throw new Error(`Failed to parse response: ${parseError}`);
                    }
                }
            }
        });
    }
    /**
     * Generate a reason for the score (synchronous)
     */
    _generateReason(actualOutput, contexts) {
        if (!this.include_reason) {
            return "No reason provided (include_reason is false)";
        }
        const prompt = prompts_js_1.HallucinationTemplate.generateReason(actualOutput, contexts);
        if (this.using_native_model) {
            const res = this.model.generate(prompt);
            try {
                const data = JSON.parse(res);
                return data.reason || "No reason provided in response";
            }
            catch (error) {
                throw new Error(`Failed to parse response: ${error}`);
            }
        }
        else {
            try {
                const parseReasonResponse = (response) => {
                    const parsed = JSON.parse(response);
                    const result = prompts_js_1.ReasonSchema.safeParse(parsed);
                    if (result.success) {
                        return result.data;
                    }
                    throw new Error(`Invalid response format: ${result.error}`);
                };
                const res = this.model.generate(prompt);
                return parseReasonResponse(res).reason;
            }
            catch (error) {
                const res = this.model.generate(prompt);
                try {
                    const data = JSON.parse(res);
                    return data.reason || "No reason provided in response";
                }
                catch (parseError) {
                    throw new Error(`Failed to parse response: ${parseError}`);
                }
            }
        }
    }
    /**
     * Calculate the hallucination score
     */
    _computeScore() {
        if (this._verdicts.length === 0) {
            return 0;
        }
        let contradictions = 0;
        for (const verdict of this._verdicts) {
            if (verdict.verdict.trim().toLowerCase() === "no") {
                contradictions += 1;
            }
        }
        return contradictions / this._verdicts.length;
    }
    /**
     * Create verbose logs for debugging
     */
    _createVerboseLogs() {
        if (!this.verbose_mode) {
            return null;
        }
        const steps = [
            `Verdicts:\n${JSON.stringify(this._verdicts, null, 2)}`,
            `Score: ${this.score}\nReason: ${this.reason || "No reason provided"}`
        ];
        return steps.join('\n\n');
    }
    /**
     * Score an example synchronously
     */
    syncScoreExample(example) {
        (0, logger_js_1.info)("Starting example scoring (sync mode)");
        try {
            // Check required parameters
            this._checkExampleParams(example);
            // Process example
            const contexts = Array.isArray(example.context) ? example.context : [example.context || ''];
            this._verdicts = this._generateVerdicts(example.actualOutput, contexts);
            // Calculate score
            this.score = this._computeScore();
            this.reason = this._generateReason(example.actualOutput, contexts) || '';
            this.success = this._successCheck();
            // Create verbose logs if enabled
            const verbose_logs = this._createVerboseLogs();
            // Calculate evaluation cost
            const promptTokens = 600; // Estimate - in a real implementation, track actual tokens
            const completionTokens = 200; // Estimate - in a real implementation, track actual tokens
            this.evaluation_cost = this._calculateTokenCosts(this.evaluation_model || 'gpt-3.5-turbo', promptTokens, completionTokens);
            (0, logger_js_1.info)(`Scoring completed with score: ${this.score}`);
            // Return ScorerData object
            return {
                name: this.type,
                threshold: this.threshold,
                success: this.success,
                score: this.score,
                reason: this.reason,
                strict_mode: this.strict_mode,
                evaluation_model: this.evaluation_model || null,
                error: null,
                evaluation_cost: this.evaluation_cost || null,
                verbose_logs: verbose_logs,
                additional_metadata: {
                    verdicts: this._verdicts
                }
            };
        }
        catch (error) {
            // Handle errors
            const errorMessage = error instanceof Error ? error.message : String(error);
            this.error = errorMessage;
            return {
                name: this.type,
                threshold: this.threshold,
                success: false,
                score: 0,
                reason: `Error during scoring: ${errorMessage}`,
                strict_mode: this.strict_mode,
                evaluation_model: this.evaluation_model || null,
                error: errorMessage,
                evaluation_cost: null,
                verbose_logs: null,
                additional_metadata: {}
            };
        }
    }
    /**
     * Score an example asynchronously
     */
    scoreExample(example) {
        return __awaiter(this, void 0, void 0, function* () {
            if (!this.async_mode) {
                return this.syncScoreExample(example);
            }
            (0, logger_js_1.info)("Starting example scoring (async mode)");
            try {
                // Check required parameters
                this._checkExampleParams(example);
                // Process example
                const contexts = Array.isArray(example.context) ? example.context : [example.context || ''];
                this._verdicts = yield this._aGenerateVerdicts(example.actualOutput, contexts);
                // Calculate score
                this.score = this._computeScore();
                this.reason = (yield this._aGenerateReason(example.actualOutput, contexts)) || '';
                this.success = this._successCheck();
                // Create verbose logs if enabled
                const verbose_logs = this._createVerboseLogs();
                // Calculate evaluation cost
                const promptTokens = 600; // Estimate - in a real implementation, track actual tokens
                const completionTokens = 200; // Estimate - in a real implementation, track actual tokens
                this.evaluation_cost = this._calculateTokenCosts(this.evaluation_model || 'gpt-3.5-turbo', promptTokens, completionTokens);
                (0, logger_js_1.info)(`Scoring completed with score: ${this.score}`);
                // Return ScorerData object
                return {
                    name: this.type,
                    threshold: this.threshold,
                    success: this.success,
                    score: this.score,
                    reason: this.reason,
                    strict_mode: this.strict_mode,
                    evaluation_model: this.evaluation_model || null,
                    error: null,
                    evaluation_cost: this.evaluation_cost || null,
                    verbose_logs: verbose_logs,
                    additional_metadata: {
                        verdicts: this._verdicts
                    }
                };
            }
            catch (error) {
                // Handle errors
                const errorMessage = error instanceof Error ? error.message : String(error);
                this.error = errorMessage;
                return {
                    name: this.type,
                    threshold: this.threshold,
                    success: false,
                    score: 0,
                    reason: `Error during scoring: ${errorMessage}`,
                    strict_mode: this.strict_mode,
                    evaluation_model: this.evaluation_model || null,
                    error: errorMessage,
                    evaluation_cost: null,
                    verbose_logs: null,
                    additional_metadata: {}
                };
            }
        });
    }
    /**
     * Get the name of the scorer
     */
    get name() {
        return "Hallucination";
    }
}
exports.HallucinationScorer = HallucinationScorer;
//# sourceMappingURL=hallucination.js.map