@fromsvenwithlove/devops-issues-cli/bin/test-model-config.js

AI-powered CLI tool and library for Azure DevOps work item management with Claude agents

#!/usr/bin/env node

/**
 * Test script for Agent Model Configuration System
 * Demonstrates cost optimization and model selection
 */

import { agents, getCostOptimizationSummary, getAgentModel, getModelForComplexity, getResearchModel, getResearchOptimizationReport, MODEL_TIERS } from '../agents/index.js';

console.log('🤖 Azure DevOps CLI - Agent Model Configuration Test\n');

// Test cost optimization summary
console.log('💰 Cost Optimization Analysis:');
const summary = getCostOptimizationSummary();
let totalCostSavings = 0;

Object.entries(summary).forEach(([tier, info]) => {
  const costLevel = tier === 'opus' ? 'HIGH' : tier === 'sonnet' ? 'MEDIUM' : 'LOW';
  console.log(`\n${tier.toUpperCase()} Tier (${costLevel} Cost):`);
  console.log(`  📊 Agent Count: ${info.count}`);
  console.log(`  🎯 Agents: ${info.agents.join(', ')}`);
  console.log(`  📝 Use Case: ${info.description}`);
  
  // Estimate cost savings (using Haiku vs Opus baseline)
  if (tier === 'haiku') {
    const savings = info.count * 85; // ~85% savings vs Opus
    totalCostSavings += savings;
    console.log(`  💵 Est. Cost Savings: ~${savings}% vs using Opus for all`);
  }
});

console.log(`\n💡 Total Estimated Cost Reduction: ~${Math.round(totalCostSavings / Object.keys(agents).length)}% average\n`);

// Test complexity-based model selection
console.log('🎯 Complexity-Based Model Selection:');
const testComplexities = [2, 4, 6, 8, 10];
testComplexities.forEach(complexity => {
  const recommendedModel = getModelForComplexity(complexity);
  const modelTier = Object.keys(MODEL_TIERS).find(key => MODEL_TIERS[key] === recommendedModel);
  console.log(`  Complexity ${complexity}/10 → ${modelTier.toUpperCase()} (${recommendedModel})`);
});

console.log('\n📋 Individual Agent Assignments:');
Object.entries(agents).forEach(([key, agent]) => {
  const modelTier = Object.keys(MODEL_TIERS).find(tier => MODEL_TIERS[tier] === agent.recommendedModel);
  const costIcon = modelTier === 'opus' ? '💎' : modelTier === 'sonnet' ? '⚡' : '💚';
  console.log(`  ${costIcon} ${agent.name}: ${modelTier.toUpperCase()} (${agent.complexity}/10) - ${agent.reasoning}`);
});

console.log('\n🔬 Research Agent Dynamic Model Selection:');
const researchTests = [
  { type: 'Documentation lookup', complexity: 2 },
  { type: 'Multi-technology comparative analysis', complexity: 8 },
  { type: 'Version comparison', complexity: 3 },
  { type: 'Strategic technology recommendations', complexity: 9 }
];

researchTests.forEach(test => {
  const model = getResearchModel(test.type, test.complexity);
  const tier = Object.keys(MODEL_TIERS).find(k => MODEL_TIERS[k] === model);
  const costIcon = tier === 'HAIKU' ? '💚' : '⚡';
  console.log(`  ${costIcon} "${test.type}" (${test.complexity}/10) → ${tier}`);
});

const researchReport = getResearchOptimizationReport();
console.log(`\n📊 Research Optimization Potential: ${researchReport.costSavingsOpportunity.percentageOfTasksSuitable} of tasks suitable for Haiku`);
console.log(`💰 Research Cost Savings: ${researchReport.costSavingsOpportunity.haikuSavings}`);

console.log('\n✅ Model configuration system ready for deployment!');
console.log('🔧 Configuration stored in: .claude/settings.json');
console.log('📖 Documentation updated in: CLAUDE.md');
console.log('⚙️  Agent metadata enhanced in: agents/index.js');
console.log('🔬 Research agent optimized for dynamic model selection');