@hivetechs/hive-ai
Version:
Real-time streaming AI consensus platform with HTTP+SSE MCP integration for Claude Code, VS Code, Cursor, and Windsurf - powered by OpenRouter's unified API
366 lines • 13.8 kB
JavaScript
/**
* Quality Scorer for Consensus Effectiveness
*
* Calculates quality scores and improvement metrics for the
* Consensus Effectiveness Index (CEI) dashboard metric.
*/
import { getDatabase } from './unified-database.js';
import { structuredLogger } from '../tools/structured-logger.js';
/**
* Calculate quality score for a consensus result
* Based on stage agreement and output coherence
*/
export async function calculateConsensusQuality(conversationId, stageResults) {
try {
// Calculate stage agreement score (0-1)
const agreementScore = calculateStageAgreement(stageResults);
// Calculate length appropriateness (0-1)
const lengthScore = calculateLengthScore(stageResults);
// Calculate progression quality (0-1)
const progressionScore = calculateProgressionScore(stageResults);
// Weighted average
const qualityScore = (agreementScore * 0.5) + (lengthScore * 0.3) + (progressionScore * 0.2);
// Store in consensus_metrics table
await recordConsensusMetrics(conversationId, qualityScore, agreementScore);
return qualityScore;
}
catch (error) {
structuredLogger.error('Failed to calculate consensus quality', {
conversationId
}, error);
return 0;
}
}
/**
* Calculate how well stages agree with each other
*/
function calculateStageAgreement(stageResults) {
if (stageResults.length < 2)
return 1;
// Simple approach: check if key concepts are preserved across stages
const getKeyWords = (text) => {
return new Set(text.toLowerCase()
.replace(/[^a-z0-9\s]/g, '')
.split(/\s+/)
.filter(word => word.length > 4) // Only meaningful words
);
};
const stageKeywords = stageResults.map(r => getKeyWords(r.answer));
// Calculate overlap between consecutive stages
let totalOverlap = 0;
for (let i = 1; i < stageKeywords.length; i++) {
const prev = stageKeywords[i - 1];
const curr = stageKeywords[i];
const intersection = new Set([...prev].filter(x => curr.has(x)));
const union = new Set([...prev, ...curr]);
const overlap = union.size > 0 ? intersection.size / union.size : 0;
totalOverlap += overlap;
}
return totalOverlap / (stageKeywords.length - 1);
}
/**
* Calculate if response length is appropriate
*/
function calculateLengthScore(stageResults) {
// Curator should produce concise, refined output
const generatorLength = stageResults.find(r => r.stageName === 'generator')?.answer.length || 0;
const curatorLength = stageResults.find(r => r.stageName === 'curator')?.answer.length || 0;
if (generatorLength === 0)
return 0;
// Ideal: curator is 70-90% of generator length (refined but not overly truncated)
const ratio = curatorLength / generatorLength;
if (ratio >= 0.7 && ratio <= 0.9)
return 1;
if (ratio >= 0.6 && ratio <= 1.0)
return 0.8;
if (ratio >= 0.5 && ratio <= 1.1)
return 0.6;
return 0.4;
}
/**
* Calculate quality of stage progression
*/
function calculateProgressionScore(stageResults) {
// Each stage should add value (measured by thoughtful token usage)
const expectedMinTokens = {
generator: 500,
refiner: 400,
validator: 300,
curator: 400
};
let score = 0;
for (const result of stageResults) {
const expected = expectedMinTokens[result.stageName] || 400;
const actual = result.tokenCount;
// Score based on meeting minimum quality threshold
if (actual >= expected) {
score += 0.25;
}
else if (actual >= expected * 0.7) {
score += 0.15;
}
else {
score += 0.05;
}
}
return Math.min(score, 1);
}
/**
* Record consensus metrics to database
*/
async function recordConsensusMetrics(conversationId, qualityScore, agreementScore) {
try {
const db = await getDatabase();
// Calculate improvement over single model (simulated for now)
// In production, this would compare against a baseline single model result
const improvementScore = qualityScore * 0.3; // 30% improvement assumption
await db.run(`
INSERT INTO consensus_metrics (
conversation_id,
baseline_model,
baseline_result,
consensus_result,
improvement_score,
quality_metrics,
cost_comparison,
time_comparison,
question_complexity,
question_category,
created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, [
conversationId,
'gpt-4', // Baseline model for comparison
'Single model baseline', // Would be actual baseline result
'Consensus result', // Would be actual consensus result
improvementScore,
JSON.stringify({
qualityScore,
agreementScore,
coherence: qualityScore,
completeness: Math.min(qualityScore * 1.2, 1)
}),
JSON.stringify({ single: 0.01, consensus: 0.02 }), // Example costs
JSON.stringify({ single: 3000, consensus: 15000 }), // Example times
'basic', // Question complexity
'general', // Category
new Date().toISOString()
]);
structuredLogger.debug('Consensus metrics recorded', {
conversationId,
qualityScore,
improvementScore
});
}
catch (error) {
structuredLogger.warn('Failed to record consensus metrics', {
conversationId,
error: error.message
});
}
}
/**
* Calculate content quality score for individual stage output
* Used by the enhanced consensus engine for per-stage analytics
*/
export async function calculateContentQuality(content, stageName, question) {
try {
// Content length analysis
const lengthScore = calculateContentLengthScore(content, stageName);
// Content complexity analysis
const complexityScore = calculateContentComplexityScore(content);
// Question relevance analysis
const relevanceScore = calculateQuestionRelevanceScore(content, question);
// Stage-specific scoring
const stageScore = calculateStageSpecificScore(content, stageName);
// Weighted average for overall quality
const qualityScore = (lengthScore * 0.2 +
complexityScore * 0.3 +
relevanceScore * 0.3 +
stageScore * 0.2);
// Scale to 0-10 range
return Math.round(qualityScore * 10 * 100) / 100;
}
catch (error) {
structuredLogger.debug('Failed to calculate content quality', {
stageName,
contentLength: content.length,
error: error.message
});
return 7.0; // Default quality score
}
}
/**
* Calculate length appropriateness for stage
*/
function calculateContentLengthScore(content, stageName) {
const length = content.length;
// Expected length ranges by stage
const expectedRanges = {
generator: { min: 800, ideal: 1500, max: 3000 },
refiner: { min: 1000, ideal: 2000, max: 4000 },
validator: { min: 600, ideal: 1200, max: 2500 },
curator: { min: 800, ideal: 1800, max: 3500 }
};
const range = expectedRanges[stageName] || expectedRanges.generator;
if (length >= range.min && length <= range.max) {
if (length >= range.ideal * 0.8 && length <= range.ideal * 1.2) {
return 1.0; // Ideal length
}
return 0.8; // Good length
}
if (length < range.min) {
return Math.max(0.3, length / range.min); // Too short
}
return Math.max(0.3, range.max / length); // Too long
}
/**
* Calculate content complexity and depth
*/
function calculateContentComplexityScore(content) {
// Count meaningful indicators
const sentences = content.split(/[.!?]+/).filter(s => s.trim().length > 10).length;
const words = content.split(/\s+/).filter(w => w.length > 3).length;
const uniqueWords = new Set(content.toLowerCase().split(/\s+/).filter(w => w.length > 3)).size;
const codeBlocks = (content.match(/```|`/g) || []).length;
const lists = (content.match(/^\s*[-*•]\s/gm) || []).length;
const numbers = (content.match(/\d+/g) || []).length;
// Complexity indicators
let score = 0.5; // Base score
// Sentence structure variety
if (sentences >= 5)
score += 0.1;
if (sentences >= 10)
score += 0.1;
// Vocabulary diversity
if (words > 0) {
const diversity = uniqueWords / words;
score += diversity * 0.2;
}
// Technical content indicators
if (codeBlocks > 0)
score += 0.1;
if (lists > 0)
score += 0.1;
if (numbers > 3)
score += 0.05;
return Math.min(score, 1.0);
}
/**
* Calculate how well content addresses the question
*/
function calculateQuestionRelevanceScore(content, question) {
// Extract key terms from question
const questionWords = new Set(question.toLowerCase()
.replace(/[^a-z0-9\s]/g, '')
.split(/\s+/)
.filter(word => word.length > 3));
const contentWords = new Set(content.toLowerCase()
.replace(/[^a-z0-9\s]/g, '')
.split(/\s+/)
.filter(word => word.length > 3));
// Calculate overlap
const intersection = new Set([...questionWords].filter(x => contentWords.has(x)));
const relevanceRatio = questionWords.size > 0 ? intersection.size / questionWords.size : 0;
// Base relevance from keyword overlap
let score = relevanceRatio;
// Bonus for comprehensive coverage
if (relevanceRatio >= 0.7)
score += 0.2;
if (relevanceRatio >= 0.5)
score += 0.1;
return Math.min(score, 1.0);
}
/**
* Calculate stage-specific quality indicators
*/
function calculateStageSpecificScore(content, stageName) {
switch (stageName.toLowerCase()) {
case 'generator':
// Generator should be creative and exploratory
return calculateGeneratorScore(content);
case 'refiner':
// Refiner should add technical depth
return calculateRefinerScore(content);
case 'validator':
// Validator should be critical and alternative
return calculateValidatorScore(content);
case 'curator':
// Curator should be polished and comprehensive
return calculateCuratorScore(content);
default:
return 0.7; // Default score
}
}
function calculateGeneratorScore(content) {
let score = 0.5;
// Look for exploratory language
const exploratoryTerms = ['consider', 'approach', 'might', 'could', 'possible', 'option', 'alternative'];
const exploratoryCount = exploratoryTerms.filter(term => content.toLowerCase().includes(term)).length;
score += Math.min(exploratoryCount * 0.05, 0.2);
// Look for breadth indicators
if (content.includes('different') || content.includes('various'))
score += 0.1;
if (content.includes('multiple') || content.includes('several'))
score += 0.1;
return Math.min(score, 1.0);
}
function calculateRefinerScore(content) {
let score = 0.5;
// Look for technical depth indicators
const technicalTerms = ['implement', 'specific', 'detail', 'precisely', 'exactly', 'technical'];
const technicalCount = technicalTerms.filter(term => content.toLowerCase().includes(term)).length;
score += Math.min(technicalCount * 0.05, 0.2);
// Look for refinement language
if (content.includes('enhance') || content.includes('improve'))
score += 0.1;
if (content.includes('optimize') || content.includes('refine'))
score += 0.1;
return Math.min(score, 1.0);
}
function calculateValidatorScore(content) {
let score = 0.5;
// Look for critical analysis
const criticalTerms = ['however', 'but', 'although', 'consider', 'potential', 'risk', 'limitation'];
const criticalCount = criticalTerms.filter(term => content.toLowerCase().includes(term)).length;
score += Math.min(criticalCount * 0.05, 0.2);
// Look for alternative perspectives
if (content.includes('alternative') || content.includes('different approach'))
score += 0.1;
if (content.includes('on the other hand') || content.includes('conversely'))
score += 0.1;
return Math.min(score, 1.0);
}
function calculateCuratorScore(content) {
let score = 0.5;
// Look for synthesis language
const synthesisTerms = ['conclusion', 'summary', 'overall', 'comprehensive', 'final', 'synthesize'];
const synthesisCount = synthesisTerms.filter(term => content.toLowerCase().includes(term)).length;
score += Math.min(synthesisCount * 0.05, 0.2);
// Look for completeness indicators
if (content.includes('complete') || content.includes('thorough'))
score += 0.1;
if (content.includes('comprehensive') || content.includes('detailed'))
score += 0.1;
return Math.min(score, 1.0);
}
/**
* Get average CEI score for dashboard
*/
export async function getAverageCEI(hours = 24) {
try {
const db = await getDatabase();
const result = await db.get(`
SELECT AVG(improvement_score) as avg_cei
FROM consensus_metrics
WHERE created_at > datetime('now', '-${hours} hours')
`);
// Convert to percentage (0-100 scale)
return (result?.avg_cei || 0) * 100;
}
catch (error) {
structuredLogger.error('Failed to get average CEI', {}, error);
return 0;
}
}
//# sourceMappingURL=quality-scorer.js.map