@vfarcic/dot-ai
Version:
AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance
55 lines (54 loc) • 2.37 kB
JavaScript
;
/**
* Remediation Comparative Evaluator
*
* Compares multiple AI models on Kubernetes troubleshooting scenarios
* Uses dynamic model inclusion based on available datasets
* Follows reference-free comparative evaluation methodology
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.RemediationComparativeEvaluator = void 0;
const base_comparative_js_1 = require("./base-comparative.js");
class RemediationComparativeEvaluator extends base_comparative_js_1.BaseComparativeEvaluator {
name = 'remediation_comparative';
description = 'Compares multiple AI models on Kubernetes troubleshooting scenarios';
promptFileName = 'remediation-comparative.md';
toolName = 'remediate';
constructor(datasetDir) {
super(datasetDir);
this.initializePrompt();
}
/**
* Get detailed breakdown of evaluation phases available
*/
getEvaluationPhases() {
const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
const phaseGroups = new Map();
// Group scenarios by phase type
for (const scenario of scenarios) {
const phase = scenario.interaction_id;
if (!phaseGroups.has(phase)) {
phaseGroups.set(phase, {
models: new Set(),
count: 0
});
}
const group = phaseGroups.get(phase);
scenario.models.forEach(model => group.models.add(model.model));
group.count++;
}
// Convert to structured output with descriptions
const phaseDescriptions = {
'manual_analyze': 'Manual Investigation Phase - How well each model investigates and diagnoses issues',
'manual_execute': 'Manual Execution Phase - How well each model validates and confirms fixes worked',
'automatic_analyze_execute': 'Automatic Full Workflow - End-to-end troubleshooting in single automated workflow'
};
return Array.from(phaseGroups.entries()).map(([phase, data]) => ({
phase,
description: phaseDescriptions[phase] || `${phase} phase evaluation`,
availableModels: Array.from(data.models).sort(),
scenarioCount: data.count
}));
}
}
exports.RemediationComparativeEvaluator = RemediationComparativeEvaluator;