@dooor-ai/toolkit/dist/evals/role-adherence.js

Guards, Evals & Observability for AI applications - works seamlessly with LangChain/LangGraph

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RoleAdherenceEval = void 0;
const base_1 = require("./base");
const cortexdb_client_1 = require("../observability/cortexdb-client");
/**
 * RoleAdherenceEval - Measures if LLM stays in its assigned role/persona
 *
 * Evaluates whether the response is consistent with the expected role
 * (e.g., "helpful assistant", "pirate captain", "technical support", etc.)
 *
 * Example:
 * ```typescript
 * const eval = new RoleAdherenceEval({
 *   threshold: 0.8,
 *   expectedRole: "professional financial advisor"
 * });
 * const result = await eval.evaluate(
 *   "Should I invest in crypto?",
 *   "Yolo bro! Just buy whatever coin is trending on TikTok!"
 * );
 * // result.score = 0.1 (breaks role), result.passed = false
 * ```
 */
class RoleAdherenceEval extends base_1.Eval {
    constructor(config = {}) {
        super(config);
        this.expectedRole = config.expectedRole;
        this.systemPrompt = config.systemPrompt;
    }
    get name() {
        return "RoleAdherenceEval";
    }
    /**
     * Set expected role dynamically
     */
    setExpectedRole(role) {
        this.expectedRole = role;
    }
    /**
     * Set system prompt dynamically
     */
    setSystemPrompt(prompt) {
        this.systemPrompt = prompt;
    }
    async evaluate(input, output, metadata) {
        const startTime = Date.now();
        const role = this.expectedRole || metadata?.expectedRole || metadata?.role;
        const systemPrompt = this.systemPrompt || metadata?.systemPrompt;
        if (!role && !systemPrompt) {
            return {
                name: this.name,
                score: 0.5,
                passed: false,
                details: "No role specified for adherence evaluation. Pass 'expectedRole' or 'systemPrompt' via config or metadata.",
                metadata: {
                    latency: Date.now() - startTime,
                },
                timestamp: new Date(),
            };
        }
        try {
            const cortexClient = (0, cortexdb_client_1.getCortexDBClient)();
            const providerName = (0, cortexdb_client_1.getGlobalProviderName)();
            const prompt = this.buildPrompt(role, systemPrompt, input, output);
            const response = await cortexClient.invokeAI({
                prompt,
                usage: "evaluation",
                providerName: providerName || undefined,
                temperature: 0.0,
                maxTokens: 300,
            });
            const score = this.parseScore(response.text);
            const passed = score >= this.getThreshold();
            return {
                name: this.name,
                score,
                passed,
                details: `Role adherence score: ${score.toFixed(2)}. ${passed ? "PASSED" : "FAILED"} (threshold: ${this.getThreshold()})`,
                metadata: {
                    latency: Date.now() - startTime,
                    judgeResponse: response.text,
                    roleChecked: role || "extracted from system prompt",
                },
                timestamp: new Date(),
            };
        }
        catch (error) {
            console.error("RoleAdherenceEval failed:", error);
            return {
                name: this.name,
                score: 0.5,
                passed: false,
                details: `Eval failed: ${error instanceof Error ? error.message : "Unknown error"}`,
                metadata: {
                    error: String(error),
                    latency: Date.now() - startTime,
                },
                timestamp: new Date(),
            };
        }
    }
    buildPrompt(role, systemPrompt, userInput, assistantOutput) {
        const roleDescription = role
            ? `Expected Role: "${role}"`
            : `System Prompt: """
${systemPrompt}
"""`;
        return `You are an expert evaluator. Your task is to assess if the response adheres to the ASSIGNED ROLE/PERSONA.

${roleDescription}

User Input: "${userInput}"
Assistant Response: "${assistantOutput}"

Evaluate role adherence:
- 1.0 = Perfect adherence, response fully consistent with role
- 0.7-0.9 = Good adherence, mostly in character with minor slips
- 0.4-0.6 = Partial adherence, noticeable out-of-character moments
- 0.0-0.3 = Poor adherence, breaks character or contradicts role

Consider:
- Tone and language style
- Expertise level and domain knowledge
- Behavioral patterns and personality
- Constraints defined in the role

Output ONLY a JSON object in this exact format:
{
  "score": 0.9,
  "reasoning": "Assessment of how well the response matches the expected role"
}`;
    }
    parseScore(response) {
        try {
            const jsonMatch = response.match(/\{[\s\S]*\}/);
            if (jsonMatch) {
                const parsed = JSON.parse(jsonMatch[0]);
                if (typeof parsed.score === "number") {
                    return Math.max(0, Math.min(1, parsed.score));
                }
            }
            const numberMatch = response.match(/\b0?\.\d+\b|\b1\.0\b|\b[01]\b/);
            if (numberMatch) {
                return Math.max(0, Math.min(1, parseFloat(numberMatch[0])));
            }
            console.warn("Could not parse score from response:", response);
            return 0.5;
        }
        catch (error) {
            console.error("Error parsing score:", error);
            return 0.5;
        }
    }
}
exports.RoleAdherenceEval = RoleAdherenceEval;