@vfarcic/dot-ai
Version:
AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance
105 lines (104 loc) • 4.58 kB
JavaScript
;
/**
* Capability Comparative Evaluator
*
* Compares multiple AI models on Kubernetes capability inference scenarios
* Groups by interaction_id (e.g., auto_scan, crud_auto_scan) and evaluates
* quality of capability analyses across different models
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.CapabilityComparativeEvaluator = void 0;
const base_comparative_js_1 = require("./base-comparative.js");
class CapabilityComparativeEvaluator extends base_comparative_js_1.BaseComparativeEvaluator {
name = 'capability-comparative';
description = 'Compares AI models on Kubernetes capability inference quality';
promptFileName = 'capability-comparative.md';
toolName = 'capability';
constructor(datasetDir) {
super(datasetDir);
this.initializePrompt();
}
async evaluateAllScenarios() {
try {
const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
const results = [];
console.log(`Found ${scenarios.length} capability scenarios with multiple models for comparative evaluation`);
for (const scenario of scenarios) {
try {
const result = await this.evaluateScenario(scenario);
results.push(result);
}
catch (error) {
console.error(`Failed to evaluate scenario ${scenario.interaction_id}:`, error);
}
}
return results;
}
catch (error) {
console.error(`Capability comparative evaluation failed:`, error);
return [{
key: `${this.name}_error`,
score: 0,
comment: `Evaluation error: ${error instanceof Error ? error.message : String(error)}`,
confidence: 0,
modelRankings: [],
bestModel: 'unknown',
modelCount: 0
}];
}
}
/**
* Build the evaluation prompt - uses base class reliability context with capability-specific template
*/
buildEvaluationPrompt(scenario, modelResponsesText, modelList) {
// Use the base class's properly formatted model responses which include:
// - Reliability Status (✅ Completed successfully OR ⚠️ TIMEOUT FAILURE)
// - Performance metrics
// - All model responses
return this.promptTemplate
.replace('{scenario_name}', scenario.interaction_id)
.replace('{model_responses}', modelResponsesText)
.replace('{models}', modelList);
}
extractResourceName(input) {
if (input?.issue && typeof input.issue === 'string') {
const match = input.issue.match(/resource: (.+)/);
return match ? match[1] : 'unknown';
}
return 'unknown';
}
/**
* Get detailed breakdown of evaluation phases available
*/
getEvaluationPhases() {
const scenarios = this.datasetAnalyzer.groupByScenario(this.toolName);
const phaseGroups = new Map();
// Group scenarios by phase type
for (const scenario of scenarios) {
const phase = scenario.interaction_id;
if (!phaseGroups.has(phase)) {
phaseGroups.set(phase, {
models: new Set(),
count: 0
});
}
const group = phaseGroups.get(phase);
scenario.models.forEach(model => group.models.add(model.model));
group.count++;
}
// Convert to structured output with descriptions
const phaseDescriptions = {
'auto_scan': 'Auto Scan Phase - How well each model analyzes cluster resource capabilities automatically',
'crud_auto_scan': 'CRUD Auto Scan Phase - How well each model handles capability analysis with CRUD operations',
'list_auto_scan': 'List Auto Scan Phase - How well each model handles capability listing and organization',
'search_auto_scan': 'Search Auto Scan Phase - How well each model handles capability search and filtering'
};
return Array.from(phaseGroups.entries()).map(([phase, data]) => ({
phase,
description: phaseDescriptions[phase] || `${phase} phase evaluation`,
availableModels: Array.from(data.models).sort(),
scenarioCount: data.count
}));
}
}
exports.CapabilityComparativeEvaluator = CapabilityComparativeEvaluator;