@hivetechs/hive-ai
Version:
Real-time streaming AI consensus platform with HTTP+SSE MCP integration for Claude Code, VS Code, Cursor, and Windsurf - powered by OpenRouter's unified API
447 lines • 20.5 kB
JavaScript
/**
* Dynamic Model Selector - The Brain of Intelligent Consensus
*
* This is the intelligent engine that makes flagship profiles truly dynamic.
* It combines OpenRouter rankings, cost intelligence, performance metrics,
* and user preferences to select optimal models for each consensus stage.
*
* Features:
* - Real-time model selection based on current rankings
* - Cost-aware routing with budget optimization
* - Performance-based filtering and fallbacks
* - Stage-specific model optimization
* - Learning from user feedback and results
*/
import { getDatabase } from '../storage/unified-database.js';
// ===== DYNAMIC MODEL SELECTOR CLASS =====
export class DynamicModelSelector {
/**
* Master function: Select optimal model for a specific stage and criteria
*/
async selectOptimalModel(criteria, conversationId) {
console.log(`🎯 Selecting optimal model for ${criteria.stage} stage (${criteria.questionComplexity} complexity)`);
try {
// 1. Get available models from rankings and database
const candidates = await this.getCandidateModels(criteria);
if (candidates.length === 0) {
console.warn('⚠️ No candidate models found, falling back to defaults');
return await this.getFallbackModel(criteria.stage);
}
// 2. Apply filtering based on constraints
const filteredCandidates = await this.applyConstraintFiltering(candidates, criteria);
// 3. Score models based on suitability
const scoredCandidates = await this.scoreModelSuitability(filteredCandidates, criteria);
// 4. Select top model
const selectedModel = scoredCandidates[0];
if (selectedModel) {
console.log(`✅ Selected ${selectedModel.openrouterId} (score: ${selectedModel.suitabilityScore.toFixed(3)}) for ${criteria.stage}`);
// 5. Record selection for learning
await this.recordModelSelection(selectedModel, criteria, conversationId);
return selectedModel;
}
return await this.getFallbackModel(criteria.stage);
}
catch (error) {
console.error(`❌ Model selection failed for ${criteria.stage}:`, error);
return await this.getFallbackModel(criteria.stage);
}
}
/**
* Get complete model lineup for all 4 consensus stages
*/
async selectCompleteLineup(questionComplexity, questionCategory, options = {}, conversationId) {
console.log(`🎯 Selecting complete model lineup for ${questionComplexity} ${questionCategory} question`);
// Select models for each stage with stage-specific optimization
const generator = await this.selectOptimalModel({
stage: 'generator',
questionComplexity,
questionCategory,
...options,
// Generator stage: prioritize creativity and capability
performanceTargets: { ...options.performanceTargets, prioritizeQuality: true }
}, conversationId);
const refiner = await this.selectOptimalModel({
stage: 'refiner',
questionComplexity,
questionCategory,
...options,
// Refiner stage: balance quality and cost
performanceTargets: { ...options.performanceTargets, prioritizeQuality: true }
}, conversationId);
const validator = await this.selectOptimalModel({
stage: 'validator',
questionComplexity,
questionCategory,
...options,
// Validator stage: prioritize reliability and speed
performanceTargets: { ...options.performanceTargets, prioritizeSpeed: true }
}, conversationId);
const curator = await this.selectOptimalModel({
stage: 'curator',
questionComplexity,
questionCategory,
...options,
// Curator stage: highest quality for final output
performanceTargets: { ...options.performanceTargets, prioritizeQuality: true }
}, conversationId);
// Fallback to defaults if any selection failed
const finalLineup = {
generator: generator || await this.getFallbackModel('generator'),
refiner: refiner || await this.getFallbackModel('refiner'),
validator: validator || await this.getFallbackModel('validator'),
curator: curator || await this.getFallbackModel('curator')
};
// Ensure all models are available
if (!finalLineup.generator || !finalLineup.refiner || !finalLineup.validator || !finalLineup.curator) {
throw new Error('Unable to select complete model lineup. Please check your configuration.');
}
const totalEstimatedCost = Object.values(finalLineup).reduce((sum, model) => sum + (model?.estimatedCost || 0), 0);
const selectionReasoning = this.generateSelectionReasoning(finalLineup, questionComplexity, options);
console.log(`✅ Complete lineup selected - Total estimated cost: $${totalEstimatedCost.toFixed(4)}`);
return {
generator: finalLineup.generator,
refiner: finalLineup.refiner,
validator: finalLineup.validator,
curator: finalLineup.curator,
totalEstimatedCost,
selectionReasoning
};
}
/**
* Get candidate models from rankings and database
*/
async getCandidateModels(criteria) {
const db = await getDatabase();
const candidates = [];
try {
// Get models from rankings (top performers) combined with database data
const query = `
SELECT DISTINCT
om.internal_id,
om.openrouter_id,
om.provider_name,
om.pricing_input,
om.pricing_output,
om.context_window,
om.capabilities,
COALESCE(mr.rank_position, 999) as rank_position,
COALESCE(mr.relative_score, 0.1) as relative_score,
COALESCE(pp.avg_latency_ms, 2000) as avg_latency_ms,
COALESCE(pp.success_rate, 0.95) as success_rate
FROM openrouter_models om
LEFT JOIN model_rankings mr ON om.internal_id = mr.model_internal_id
AND mr.ranking_source = 'openrouter_programming_weekly'
AND mr.collected_at >= date('now', '-7 days')
LEFT JOIN provider_performance pp ON om.internal_id = pp.model_internal_id
AND pp.measured_at >= date('now', '-24 hours')
WHERE om.is_active = 1
AND om.pricing_input > 0 -- Exclude models without pricing data
ORDER BY
CASE WHEN mr.rank_position IS NOT NULL THEN mr.rank_position ELSE 999 END,
om.pricing_input ASC
LIMIT 50
`;
const rows = await db.all(query);
for (const row of rows) {
// 🛡️ FINAL VALIDATION: Reject any pseudo-models that might still exist
const pseudoModelPatterns = [
'openrouter/auto', 'openrouter/best', '/auto', '/best',
'/router', '/routing', 'auto-select', 'best-select'
];
if (pseudoModelPatterns.some(pattern => row.openrouter_id.toLowerCase().includes(pattern))) {
console.log(`🛡️ REJECTED pseudo-model in dynamic selection: ${row.openrouter_id}`);
continue;
}
// Estimate cost for typical consensus stage (500 input + 1000 output tokens)
const estimatedCost = (row.pricing_input * 500 + row.pricing_output * 1000) / 1000000;
const candidate = {
internalId: row.internal_id,
openrouterId: row.openrouter_id,
provider: row.provider_name,
rankPosition: row.rank_position !== 999 ? row.rank_position : undefined,
relativeScore: row.relative_score,
estimatedCost,
estimatedLatency: row.avg_latency_ms,
successRate: row.success_rate,
features: JSON.parse(row.capabilities || '[]'),
contextWindow: row.context_window || 4096,
suitabilityScore: 0 // Will be calculated later
};
candidates.push(candidate);
}
console.log(`📊 Found ${candidates.length} candidate models for ${criteria.stage} stage`);
return candidates;
}
catch (error) {
console.error('Failed to get candidate models:', error);
return [];
}
}
/**
* Apply filtering based on constraints
*/
async applyConstraintFiltering(candidates, criteria) {
let filtered = [...candidates];
// Budget constraints
if (criteria.budgetConstraints?.maxCostPerStage) {
filtered = filtered.filter(c => c.estimatedCost <= criteria.budgetConstraints.maxCostPerStage);
console.log(`💰 Budget filter: ${filtered.length} models under $${criteria.budgetConstraints.maxCostPerStage}`);
}
// Performance constraints
if (criteria.performanceTargets?.maxLatency) {
filtered = filtered.filter(c => c.estimatedLatency <= criteria.performanceTargets.maxLatency);
console.log(`⚡ Latency filter: ${filtered.length} models under ${criteria.performanceTargets.maxLatency}ms`);
}
if (criteria.performanceTargets?.minSuccessRate) {
filtered = filtered.filter(c => c.successRate >= criteria.performanceTargets.minSuccessRate);
console.log(`✅ Success rate filter: ${filtered.length} models above ${criteria.performanceTargets.minSuccessRate}`);
}
// User preferences
if (criteria.userPreferences?.preferredProviders?.length) {
filtered = filtered.filter(c => criteria.userPreferences.preferredProviders.includes(c.provider));
console.log(`👤 Preferred providers filter: ${filtered.length} models`);
}
if (criteria.userPreferences?.avoidedProviders?.length) {
filtered = filtered.filter(c => !criteria.userPreferences.avoidedProviders.includes(c.provider));
console.log(`🚫 Avoided providers filter: ${filtered.length} models`);
}
if (criteria.userPreferences?.modelBlacklist?.length) {
filtered = filtered.filter(c => !criteria.userPreferences.modelBlacklist.includes(c.openrouterId));
console.log(`🚫 Blacklist filter: ${filtered.length} models`);
}
return filtered;
}
/**
* Score models based on suitability for the specific stage and criteria
*/
async scoreModelSuitability(candidates, criteria) {
const scoredCandidates = candidates.map(candidate => {
let score = 0;
let maxScore = 0;
// 1. Ranking Score (40% weight) - Higher rank = higher score
if (candidate.rankPosition) {
const rankScore = Math.max(0, (51 - candidate.rankPosition) / 50); // Top 50 models
score += rankScore * 0.4;
}
else {
score += (candidate.relativeScore || 0.1) * 0.4; // Use relative score if no ranking
}
maxScore += 0.4;
// 2. Cost Efficiency Score (25% weight) - Better value = higher score
if (criteria.budgetConstraints?.prioritizeCost) {
const costScore = Math.max(0, 1 - (candidate.estimatedCost / 0.01)); // Normalize to $0.01 max
score += costScore * 0.25;
}
else {
// Standard cost consideration
const costScore = Math.max(0, 1 - (candidate.estimatedCost / 0.005)); // Normalize to $0.005 max
score += costScore * 0.25;
}
maxScore += 0.25;
// 3. Performance Score (20% weight)
const latencyScore = Math.max(0, 1 - (candidate.estimatedLatency / 5000)); // Normalize to 5s max
const reliabilityScore = candidate.successRate;
const performanceScore = (latencyScore + reliabilityScore) / 2;
if (criteria.performanceTargets?.prioritizeSpeed) {
score += latencyScore * 0.2;
}
else if (criteria.performanceTargets?.prioritizeQuality) {
score += reliabilityScore * 0.2;
}
else {
score += performanceScore * 0.2;
}
maxScore += 0.2;
// 4. Stage-Specific Optimization (15% weight)
const stageScore = this.calculateStageSpecificScore(candidate, criteria);
score += stageScore * 0.15;
maxScore += 0.15;
// Normalize score to 0-1 range
candidate.suitabilityScore = maxScore > 0 ? score / maxScore : 0;
return candidate;
});
// Sort by suitability score (highest first)
return scoredCandidates.sort((a, b) => b.suitabilityScore - a.suitabilityScore);
}
/**
* Calculate stage-specific scoring
*/
calculateStageSpecificScore(candidate, criteria) {
let score = 0.5; // Base score
switch (criteria.stage) {
case 'generator':
// Generator: Prefer creative, capable models with good context windows
if (candidate.contextWindow >= 8000)
score += 0.2;
if (candidate.provider === 'anthropic' || candidate.provider === 'openai')
score += 0.2;
if (candidate.openrouterId.includes('claude-3.5-sonnet') || candidate.openrouterId.includes('gpt-4o'))
score += 0.1;
break;
case 'refiner':
// Refiner: Balance of quality and speed
if (candidate.estimatedLatency < 2000)
score += 0.2;
if (candidate.successRate > 0.95)
score += 0.2;
if (candidate.rankPosition && candidate.rankPosition <= 10)
score += 0.1;
break;
case 'validator':
// Validator: Fast, reliable models for validation tasks
if (candidate.estimatedLatency < 1500)
score += 0.3;
if (candidate.successRate > 0.98)
score += 0.2;
break;
case 'curator':
// Curator: Highest quality models for final output
if (candidate.rankPosition && candidate.rankPosition <= 5)
score += 0.3;
if (candidate.provider === 'anthropic' || candidate.provider === 'openai')
score += 0.2;
break;
}
// Complexity-based adjustments
if (criteria.questionComplexity === 'production') {
// Production: Prefer top-tier models
if (candidate.rankPosition && candidate.rankPosition <= 3)
score += 0.2;
}
else if (criteria.questionComplexity === 'minimal') {
// Minimal: Prefer cost-effective models
if (candidate.estimatedCost < 0.001)
score += 0.2;
}
return Math.min(1.0, score);
}
/**
* Get fallback model when selection fails
*/
async getFallbackModel(stage) {
const fallbackModels = {
generator: 'anthropic/claude-3.5-sonnet',
refiner: 'openai/gpt-4o-mini',
validator: 'openai/gpt-4o-mini',
curator: 'anthropic/claude-3.5-sonnet'
};
const fallbackId = fallbackModels[stage];
if (!fallbackId)
return null;
try {
const db = await getDatabase();
const result = await db.get('SELECT internal_id, openrouter_id, provider_name, pricing_input, pricing_output FROM openrouter_models WHERE openrouter_id = ?', [fallbackId]);
if (result) {
return {
internalId: result.internal_id,
openrouterId: result.openrouter_id,
provider: result.provider_name,
estimatedCost: (result.pricing_input * 500 + result.pricing_output * 1000) / 1000000,
estimatedLatency: 2000,
successRate: 0.95,
features: [],
contextWindow: 4096,
suitabilityScore: 0.5
};
}
}
catch (error) {
console.error('Failed to get fallback model:', error);
}
return null;
}
/**
* Record model selection for learning and optimization
*/
async recordModelSelection(model, criteria, conversationId) {
try {
// Only record if we have a valid conversation ID
if (!conversationId) {
return; // Skip recording if no conversation context
}
const db = await getDatabase();
await db.run(`
INSERT INTO model_selection_history
(conversation_id, selection_criteria, selected_models, selection_reasoning, created_at)
VALUES (?, ?, ?, ?, ?)
`, [
conversationId,
JSON.stringify(criteria),
JSON.stringify({ [criteria.stage]: model.openrouterId }),
`Selected ${model.openrouterId} for ${criteria.stage} stage with suitability score ${model.suitabilityScore.toFixed(3)}`,
new Date().toISOString()
]);
}
catch (error) {
console.warn('Failed to record model selection:', error);
}
}
/**
* Generate human-readable selection reasoning
*/
generateSelectionReasoning(lineup, complexity, options) {
const parts = [];
parts.push(`Selected models for ${complexity} complexity question:`);
if (lineup.generator) {
parts.push(`🎯 Generator: ${lineup.generator.openrouterId} (rank #${lineup.generator.rankPosition || 'N/A'}) - Creative foundation`);
}
if (lineup.refiner) {
parts.push(`🔧 Refiner: ${lineup.refiner.openrouterId} (rank #${lineup.refiner.rankPosition || 'N/A'}) - Quality enhancement`);
}
if (lineup.validator) {
parts.push(`✅ Validator: ${lineup.validator.openrouterId} (rank #${lineup.validator.rankPosition || 'N/A'}) - Fast validation`);
}
if (lineup.curator) {
parts.push(`👑 Curator: ${lineup.curator.openrouterId} (rank #${lineup.curator.rankPosition || 'N/A'}) - Final polish`);
}
if (options.budgetConstraints?.prioritizeCost) {
parts.push('💰 Optimized for cost efficiency');
}
else if (options.performanceTargets?.prioritizeSpeed) {
parts.push('⚡ Optimized for speed');
}
else if (options.performanceTargets?.prioritizeQuality) {
parts.push('🎯 Optimized for quality');
}
return parts.join('\n');
}
}
// ===== CONVENIENCE FUNCTIONS =====
/**
* Quick selection for a single stage
*/
export async function selectModelForStage(stage, questionComplexity = 'basic') {
const selector = new DynamicModelSelector();
const result = await selector.selectOptimalModel({
stage,
questionComplexity,
questionCategory: 'general'
});
return result?.openrouterId || null;
}
/**
* Quick lineup selection with cost limit
*/
export async function selectBudgetOptimizedLineup(maxTotalCost, questionComplexity = 'basic') {
const selector = new DynamicModelSelector();
return await selector.selectCompleteLineup(questionComplexity, 'general', {
budgetConstraints: {
maxTotalCost,
prioritizeCost: true
}
});
}
/**
* Quick lineup selection optimized for speed
*/
export async function selectSpeedOptimizedLineup(questionComplexity = 'basic') {
const selector = new DynamicModelSelector();
return await selector.selectCompleteLineup(questionComplexity, 'general', {
performanceTargets: {
prioritizeSpeed: true,
maxLatency: 2000
}
});
}
export default DynamicModelSelector;
//# sourceMappingURL=dynamic-model-selector.js.map