@spaik/mcp-server-roi
Version:
MCP server for AI ROI prediction and tracking with Monte Carlo simulations
505 lines • 19.7 kB
JavaScript
import { z } from 'zod';
import { createLogger } from '../utils/logger.js';
/**
* Metadata Enricher Service
*
* Enhances responses with confidence scores, data quality metrics,
* assumption tracking, and contextual metadata for AI agents.
*/
// Metadata schemas
export const DataQualityMetricsSchema = z.object({
completeness: z.number().min(0).max(1),
accuracy: z.number().min(0).max(1),
consistency: z.number().min(0).max(1),
timeliness: z.number().min(0).max(1),
overall: z.enum(['low', 'medium', 'high'])
});
export const ConfidenceMetricsSchema = z.object({
overall: z.number().min(0).max(1),
breakdown: z.object({
data_quality: z.number().min(0).max(1),
model_accuracy: z.number().min(0).max(1),
assumption_validity: z.number().min(0).max(1),
benchmark_alignment: z.number().min(0).max(1)
}),
factors: z.array(z.object({
factor: z.string(),
impact: z.enum(['positive', 'negative']),
weight: z.number()
}))
});
export const AssumptionTrackingSchema = z.object({
assumptions: z.array(z.object({
id: z.string(),
category: z.string(),
description: z.string(),
confidence: z.number().min(0).max(1),
impact: z.enum(['low', 'medium', 'high']),
sensitivity: z.number().optional(),
validation_method: z.string().optional()
})),
overall_impact: z.enum(['low', 'medium', 'high']),
key_dependencies: z.array(z.string())
});
export const ContextualMetadataSchema = z.object({
industry_context: z.object({
industry: z.string(),
market_maturity: z.enum(['emerging', 'growing', 'mature', 'declining']),
competitive_intensity: z.enum(['low', 'medium', 'high']),
regulatory_complexity: z.enum(['low', 'medium', 'high'])
}).optional(),
organization_context: z.object({
size: z.enum(['small', 'medium', 'large', 'enterprise']),
ai_maturity: z.enum(['beginner', 'intermediate', 'advanced', 'leader']),
change_readiness: z.enum(['low', 'medium', 'high']),
resource_availability: z.enum(['constrained', 'adequate', 'abundant'])
}).optional(),
temporal_context: z.object({
analysis_date: z.string().datetime(),
data_freshness: z.enum(['real-time', 'recent', 'historical']),
projection_horizon: z.string(),
seasonality_considered: z.boolean()
}),
calculation_context: z.object({
methodology: z.string(),
key_parameters: z.record(z.any()),
sensitivity_tested: z.boolean(),
scenario_count: z.number()
})
});
export class MetadataEnricher {
logger = createLogger({ component: 'MetadataEnricher' });
/**
* Enrich response with comprehensive metadata
*/
async enrichResponse(response, tool, executionContext) {
this.logger.debug('Enriching response metadata', { tool });
const confidence = await this.calculateConfidence(response, tool, executionContext);
const dataQuality = this.assessDataQuality(response, executionContext);
const assumptions = this.trackAssumptions(response, tool);
const context = this.gatherContextualMetadata(response, tool, executionContext);
const provenance = this.generateProvenance(tool, executionContext);
return {
confidence,
dataQuality,
assumptions,
context,
provenance
};
}
/**
* Calculate comprehensive confidence metrics
*/
async calculateConfidence(response, tool, context) {
const breakdown = {
data_quality: 0.85, // Default, will be calculated
model_accuracy: 0.9, // Based on tool type
assumption_validity: 0.8, // Based on assumptions
benchmark_alignment: 0.75 // Based on benchmark usage
};
// Tool-specific confidence adjustments
switch (tool) {
case 'predict_roi':
breakdown.model_accuracy = this.assessROIModelAccuracy(response);
breakdown.benchmark_alignment = context.enable_benchmarks ? 0.9 : 0.7;
break;
case 'compare_projects':
breakdown.model_accuracy = response.projects?.length > 2 ? 0.95 : 0.85;
breakdown.benchmark_alignment = this.assessComparativeBenchmarks(response);
break;
}
// Calculate data quality impact
const dataCompleteness = this.assessDataCompleteness(response);
breakdown.data_quality = dataCompleteness;
// Assess assumption validity
breakdown.assumption_validity = this.assessAssumptionValidity(response, context);
// Calculate overall confidence
const weights = {
data_quality: 0.3,
model_accuracy: 0.3,
assumption_validity: 0.25,
benchmark_alignment: 0.15
};
const overall = Object.entries(breakdown).reduce((sum, [key, value]) => {
return sum + value * weights[key];
}, 0);
// Identify confidence factors
const factors = this.identifyConfidenceFactors(breakdown, response, context);
return {
overall,
breakdown,
factors
};
}
/**
* Assess data quality metrics
*/
assessDataQuality(response, context) {
const metrics = {
completeness: 1.0,
accuracy: 0.9,
consistency: 0.95,
timeliness: 0.85
};
// Completeness check
if (response.use_cases) {
const missingFields = response.use_cases.filter((uc) => !uc.monthly_benefit || !uc.category || !uc.name).length;
metrics.completeness = 1 - (missingFields / response.use_cases.length);
}
// Accuracy assessment (based on calculation methods)
if (response.metadata?.calculated_with_benchmarks) {
metrics.accuracy = 0.95;
}
else if (context.confidence_level && context.confidence_level > 0.9) {
metrics.accuracy = 0.9;
}
// Consistency check
if (response.financial_metrics) {
const hasConsistentMetrics = this.checkMetricConsistency(response.financial_metrics);
metrics.consistency = hasConsistentMetrics ? 0.95 : 0.7;
}
// Timeliness (based on data freshness)
if (context.benchmark_config?.enable_real_time) {
metrics.timeliness = 0.95;
}
else if (response.metadata?.calculation_timestamp) {
const ageHours = (Date.now() - new Date(response.metadata.calculation_timestamp).getTime()) / 3600000;
metrics.timeliness = Math.max(0.5, 1 - (ageHours / 168)); // Decay over a week
}
// Calculate overall quality
const avgScore = Object.values(metrics).reduce((a, b) => a + b, 0) / 4;
const overall = avgScore >= 0.85 ? 'high' : avgScore >= 0.7 ? 'medium' : 'low';
return {
...metrics,
overall
};
}
/**
* Track and analyze assumptions
*/
trackAssumptions(response, tool) {
const assumptions = [];
// Common assumptions across all tools
assumptions.push({
id: 'A001',
category: 'Financial',
description: 'Discount rate remains stable at 10% annually',
confidence: 0.9,
impact: 'medium',
sensitivity: 0.15,
validation_method: 'Industry standard rates'
});
// Tool-specific assumptions
switch (tool) {
case 'predict_roi':
assumptions.push({
id: 'A002',
category: 'Implementation',
description: 'Linear ramp-up over 6-month period',
confidence: 0.8,
impact: 'high',
sensitivity: 0.25,
validation_method: 'Historical project data'
}, {
id: 'A003',
category: 'Adoption',
description: 'User adoption follows typical S-curve',
confidence: 0.85,
impact: 'medium',
sensitivity: 0.2,
validation_method: 'Industry benchmarks'
});
break;
case 'compare_projects':
assumptions.push({
id: 'A004',
category: 'Comparison',
description: 'Projects evaluated under similar market conditions',
confidence: 0.75,
impact: 'medium',
validation_method: 'Temporal alignment check'
});
break;
}
// Add response-specific assumptions
if (response.metadata?.assumptions) {
response.metadata.assumptions.forEach((assumption, index) => {
assumptions.push({
id: `A${(100 + index).toString().padStart(3, '0')}`,
category: assumption.category || 'General',
description: assumption.description,
confidence: assumption.confidence || 0.75,
impact: assumption.impact || 'medium',
validation_method: 'User provided'
});
});
}
// Calculate overall impact
const highImpactCount = assumptions.filter(a => a.impact === 'high').length;
const overall_impact = highImpactCount > assumptions.length / 3 ? 'high' :
highImpactCount > 0 ? 'medium' : 'low';
// Identify key dependencies
const key_dependencies = assumptions
.filter(a => a.impact === 'high' && a.confidence < 0.8)
.map(a => a.description);
return {
assumptions,
overall_impact,
key_dependencies
};
}
/**
* Gather contextual metadata
*/
gatherContextualMetadata(response, tool, context) {
// Industry context
const industry_context = context.industry ? {
industry: context.industry,
market_maturity: this.assessMarketMaturity(context.industry),
competitive_intensity: 'medium', // Could be enhanced with real data
regulatory_complexity: this.assessRegulatoryComplexity(context.industry)
} : undefined;
// Organization context
const organization_context = context.company_size ? {
size: context.company_size,
ai_maturity: this.inferAIMaturity(response, context),
change_readiness: 'medium', // Default, could be assessed
resource_availability: this.inferResourceAvailability(context)
} : undefined;
// Temporal context
const temporal_context = {
analysis_date: new Date().toISOString(),
data_freshness: (context.benchmark_config?.enable_real_time ? 'real-time' : 'recent'),
projection_horizon: `${context.timeline_months || 60} months`,
seasonality_considered: false // Could be enhanced
};
// Calculation context
const calculation_context = {
methodology: this.getMethodologyName(tool),
key_parameters: {
discount_rate: 0.1,
confidence_level: context.confidence_level || 0.95,
simulation_iterations: context.simulation_iterations || 10000
},
sensitivity_tested: tool === 'predict_roi',
scenario_count: this.countScenarios(response)
};
return {
industry_context,
organization_context,
temporal_context,
calculation_context
};
}
/**
* Generate provenance information
*/
generateProvenance(tool, context) {
return {
tool_name: tool,
tool_version: '2.0',
execution_id: this.generateExecutionId(),
timestamp: new Date().toISOString(),
data_sources: this.identifyDataSources(context),
calculation_methods: this.identifyCalculationMethods(tool),
external_apis_used: this.identifyExternalAPIs(context),
caching_used: false, // Could track actual cache usage
processing_location: 'server'
};
}
// Helper methods
assessROIModelAccuracy(response) {
// Factors: calculation completeness, use case detail, projection quality
let accuracy = 0.9;
if (response.use_cases?.length > 5) {
accuracy -= 0.05; // More complexity, slightly less accurate
}
if (response.financial_metrics?.conservative && response.financial_metrics?.optimistic) {
accuracy += 0.05; // Multiple scenarios increase confidence
}
return Math.min(0.95, Math.max(0.7, accuracy));
}
assessComparativeBenchmarks(response) {
const projectsWithBenchmarks = response.projects?.filter((p) => p.benchmark_comparison?.available).length || 0;
const totalProjects = response.projects?.length || 1;
const benchmarkCoverage = projectsWithBenchmarks / totalProjects;
return 0.6 + (benchmarkCoverage * 0.35); // Base 0.6, up to 0.95
}
assessDataCompleteness(response) {
const requiredFields = [
'summary',
'financial_metrics',
'use_cases',
'metadata'
];
const presentFields = requiredFields.filter(field => response[field] && Object.keys(response[field]).length > 0).length;
return presentFields / requiredFields.length;
}
assessAssumptionValidity(response, context) {
let validity = 0.8; // Base validity
// Adjust based on data sources
if (context.enable_benchmarks || context.use_industry_benchmarks) {
validity += 0.1;
}
// Adjust based on confidence level
if (context.confidence_level && context.confidence_level > 0.9) {
validity += 0.05;
}
// Adjust based on complexity
if (response.use_cases?.length > 10) {
validity -= 0.1; // More assumptions with more use cases
}
return Math.min(0.95, Math.max(0.6, validity));
}
identifyConfidenceFactors(breakdown, response, context) {
const factors = [];
// Positive factors
if (breakdown.benchmark_alignment > 0.8) {
factors.push({
factor: 'Strong benchmark alignment',
impact: 'positive',
weight: 0.2
});
}
if (response.use_cases?.length >= 3 && response.use_cases?.length <= 7) {
factors.push({
factor: 'Optimal use case portfolio size',
impact: 'positive',
weight: 0.15
});
}
// Negative factors
if (breakdown.data_quality < 0.8) {
factors.push({
factor: 'Data quality concerns',
impact: 'negative',
weight: 0.25
});
}
if (!context.enable_benchmarks && !context.use_industry_benchmarks) {
factors.push({
factor: 'No external benchmarks used',
impact: 'negative',
weight: 0.15
});
}
return factors;
}
checkMetricConsistency(financialMetrics) {
if (!financialMetrics.conservative || !financialMetrics.expected || !financialMetrics.optimistic) {
return false;
}
// Check logical ordering
const conservative = financialMetrics.conservative.total_monthly_benefit;
const expected = financialMetrics.expected.total_monthly_benefit;
const optimistic = financialMetrics.optimistic.total_monthly_benefit;
return conservative <= expected && expected <= optimistic;
}
assessMarketMaturity(industry) {
const maturityMap = {
'technology': 'growing',
'financial_services': 'mature',
'healthcare': 'growing',
'retail': 'mature',
'manufacturing': 'mature',
'education': 'emerging',
'government': 'emerging'
};
return maturityMap[industry] || 'growing';
}
assessRegulatoryComplexity(industry) {
const complexityMap = {
'financial_services': 'high',
'healthcare': 'high',
'government': 'high',
'education': 'medium',
'technology': 'low',
'retail': 'low',
'manufacturing': 'medium'
};
return complexityMap[industry] || 'medium';
}
inferAIMaturity(response, context) {
const useCaseCount = response.use_cases?.length || 0;
const investment = response.summary?.total_investment || 0;
if (useCaseCount > 10 && investment > 1000000) {
return 'leader';
}
else if (useCaseCount > 5 || investment > 500000) {
return 'advanced';
}
else if (useCaseCount > 2) {
return 'intermediate';
}
return 'beginner';
}
inferResourceAvailability(context) {
const size = context.company_size;
if (size === 'enterprise' || size === 'large') {
return 'abundant';
}
else if (size === 'medium') {
return 'adequate';
}
return 'constrained';
}
getMethodologyName(tool) {
const methodologies = {
'predict_roi': 'DCF with Monte Carlo simulation',
'compare_projects': 'Multi-criteria portfolio analysis'
};
return methodologies[tool] || 'Standard financial analysis';
}
countScenarios(response) {
if (response.scenarios) {
return Object.keys(response.scenarios).length;
}
else if (response.financial_metrics) {
return Object.keys(response.financial_metrics).length;
}
return 1;
}
generateExecutionId() {
return `exec_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
identifyDataSources(context) {
const sources = ['Internal calculations'];
if (context.enable_benchmarks || context.use_industry_benchmarks) {
sources.push('Industry benchmarks');
}
if (context.benchmark_config?.enable_real_time) {
sources.push('Real-time market data');
}
return sources;
}
identifyCalculationMethods(tool) {
const methods = {
'predict_roi': [
'Discounted Cash Flow (DCF)',
'Net Present Value (NPV)',
'Internal Rate of Return (IRR)',
'Monte Carlo simulation'
],
'compare_projects': [
'Comparative analysis',
'Portfolio optimization',
'Risk-return assessment',
'ML pattern detection'
],
};
return methods[tool] || ['Standard analysis'];
}
identifyExternalAPIs(context) {
const apis = [];
if (context.benchmark_config?.sonar_api_key) {
apis.push('Perplexity Sonar API');
}
if (context.benchmark_config?.fmp_api_key) {
apis.push('Financial Modeling Prep API');
}
return apis;
}
}
// Export singleton instance
export const metadataEnricher = new MetadataEnricher();
//# sourceMappingURL=metadata-enricher.js.map