@vfarcic/dot-ai/dist/evaluation/evaluators/policy-comparative.js

AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance

"use strict";
/**
 * Policy Comparative Evaluator
 *
 * Compares multiple AI models on Kubernetes organizational policy intent management scenarios
 * Groups by interaction_id (e.g., policy_create_workflow) and evaluates
 * quality of policy creation, validation, and enforcement recommendations across different models
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.PolicyComparativeEvaluator = void 0;
const base_comparative_js_1 = require("./base-comparative.js");
class PolicyComparativeEvaluator extends base_comparative_js_1.BaseComparativeEvaluator {
    name = 'policy-comparative';
    description = 'Compares AI models on Kubernetes organizational policy intent management quality';
    promptFileName = 'policy-comparative.md';
    toolName = 'policy';
    constructor(datasetDir) {
        super(datasetDir);
        this.initializePrompt();
    }
    async evaluateAllScenarios() {
        try {
            const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
            const results = [];
            console.log(`Found ${scenarios.length} policy scenarios with multiple models for comparative evaluation`);
            for (const scenario of scenarios) {
                try {
                    const result = await this.evaluateScenario(scenario);
                    results.push(result);
                }
                catch (error) {
                    console.error(`Failed to evaluate scenario ${scenario.interaction_id}:`, error);
                }
            }
            return results;
        }
        catch (error) {
            console.error(`Policy comparative evaluation failed:`, error);
            return [{
                    key: `${this.name}_error`,
                    score: 0,
                    comment: `Evaluation error: ${error instanceof Error ? error.message : String(error)}`,
                    confidence: 0,
                    modelRankings: [],
                    bestModel: 'unknown',
                    modelCount: 0
                }];
        }
    }
    /**
     * Build the evaluation prompt - uses base class reliability context with policy-specific template
     */
    buildEvaluationPrompt(scenario, modelResponsesText, modelList) {
        // Use the base class's properly formatted model responses which include:
        // - Reliability Status (✅ Completed successfully OR ⚠️ TIMEOUT FAILURE)
        // - Performance metrics
        // - All model responses
        return this.promptTemplate
            .replace('{scenario_name}', scenario.interaction_id)
            .replace('{model_responses}', modelResponsesText)
            .replace('{models}', modelList);
    }
    /**
     * Get detailed breakdown of evaluation phases available
     */
    getEvaluationPhases() {
        const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
        const phaseGroups = new Map();
        // Group scenarios by phase type
        for (const scenario of scenarios) {
            const phase = scenario.interaction_id;
            if (!phaseGroups.has(phase)) {
                phaseGroups.set(phase, {
                    models: new Set(),
                    count: 0
                });
            }
            const group = phaseGroups.get(phase);
            scenario.models.forEach(model => group.models.add(model.model));
            group.count++;
        }
        // Convert to structured output with descriptions
        const phaseDescriptions = {
            'policy_create_workflow': 'Policy Creation Workflow - How well each model guides users through creating organizational policies',
            'policy_validation': 'Policy Validation Phase - How well each model validates policy intent correctness and enforceability',
            'policy_enforcement_recommendations': 'Policy Enforcement Recommendations - How well each model provides enforcement strategies',
            'policy_compliance_analysis': 'Policy Compliance Analysis - How well each model analyzes compliance with existing policies'
        };
        return Array.from(phaseGroups.entries()).map(([phase, data]) => ({
            phase,
            description: phaseDescriptions[phase] || `${phase} phase evaluation`,
            availableModels: Array.from(data.models).sort(),
            scenarioCount: data.count
        }));
    }
}
exports.PolicyComparativeEvaluator = PolicyComparativeEvaluator;