@vfarcic/dot-ai/dist/evaluation/evaluators/recommendation-comparative.js

AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance

"use strict";
/**
 * Recommendation Comparative Evaluator
 *
 * Compares multiple AI models on Kubernetes recommendation scenarios
 * Uses dynamic model inclusion based on available datasets
 * Follows reference-free comparative evaluation methodology
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.RecommendationComparativeEvaluator = void 0;
const base_comparative_js_1 = require("./base-comparative.js");
class RecommendationComparativeEvaluator extends base_comparative_js_1.BaseComparativeEvaluator {
    name = 'recommendation_comparative';
    description = 'Compares multiple AI models on Kubernetes deployment recommendation scenarios';
    promptFileName = 'recommendation-comparative.md';
    toolName = 'recommend';
    constructor(datasetDir) {
        super(datasetDir);
        this.initializePrompt();
    }
    /**
     * Get detailed breakdown of evaluation phases available
     */
    getEvaluationPhases() {
        const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
        const phaseGroups = new Map();
        // Group scenarios by phase type
        for (const scenario of scenarios) {
            const phase = scenario.interaction_id;
            if (!phaseGroups.has(phase)) {
                phaseGroups.set(phase, {
                    models: new Set(),
                    count: 0
                });
            }
            const group = phaseGroups.get(phase);
            scenario.models.forEach(model => group.models.add(model.model));
            group.count++;
        }
        // Convert to structured output with descriptions
        const phaseDescriptions = {
            'clarification_phase': 'Intent Analysis Phase - How well each model analyzes user intents and identifies missing context',
            'question_generation': 'Question Generation Phase - How well each model generates clarifying questions to enhance requirements',
            'solution_assembly': 'Solution Assembly Phase - How well each model selects appropriate Kubernetes resources and deployment patterns',
            'generate_manifests_phase': 'Manifest Generation Phase - How well each model generates production-ready Kubernetes manifests'
        };
        return Array.from(phaseGroups.entries()).map(([phase, data]) => ({
            phase,
            description: phaseDescriptions[phase] || `${phase} phase evaluation`,
            availableModels: Array.from(data.models).sort(),
            scenarioCount: data.count
        }));
    }
}
exports.RecommendationComparativeEvaluator = RecommendationComparativeEvaluator;