@polybiouslabs/polybious/dist/core/explainable-ai.js

Polybius is a next-generation intelligent agent framework built for adaptability across diverse domains. It merges contextual awareness, multi-agent collaboration, and predictive reasoning to deliver dynamic, self-optimizing performance.

import { logger } from '../config/logger.js';
import * as fs from 'fs/promises';
import * as path from 'path';

export class ExplainableAI {
    reasoningTraces = new Map();
    decisionHistory = [];
    explanationTemplates = new Map();
    insightCache = new Map();
    explanationsPath;

    constructor() {
        this.explanationsPath = path.join('data', 'explanations', 'reasoning-traces.json');
        this.initializeTemplates();
        this.loadExplanationData();
    }

    async explainDecision(decision, context, evidence = {}) {
        const explanationId = `exp_${Date.now()}_${Math.random().toString(36).substring(7)}`;
        
        logger.debug(`Generating explanation for decision`, { decision, explanationId });
        
        const reasoning = {
            id: explanationId,
            timestamp: new Date(),
            decision,
            context: this.normalizeContext(context),
            evidence,
            explanation: await this.generateExplanation(decision, context, evidence),
            confidence: this.calculateExplanationConfidence(decision, evidence),
            reasoning_chain: this.buildReasoningChain(decision, context, evidence),
            alternative_paths: await this.analyzeAlternatives(decision, context, evidence)
        };

        this.reasoningTraces.set(explanationId, reasoning);
        this.decisionHistory.push({
            id: explanationId,
            timestamp: reasoning.timestamp,
            decision,
            confidence: reasoning.confidence
        });

        // Keep history manageable
        if (this.decisionHistory.length > 500) {
            this.decisionHistory = this.decisionHistory.slice(-500);
        }

        await this.saveExplanationData();

        logger.info('Decision explanation generated', {
            id: explanationId,
            confidence: reasoning.confidence,
            alternativesConsidered: reasoning.alternative_paths.length
        });

        return reasoning;
    }

    async explainPrediction(prediction, inputData, model_info = {}) {
        const explanationId = `pred_${Date.now()}_${Math.random().toString(36).substring(7)}`;
        
        const explanation = {
            id: explanationId,
            timestamp: new Date(),
            prediction,
            inputData: this.normalizeInputData(inputData),
            model_info,
            feature_importance: await this.analyzeFeatureImportance(inputData, prediction, model_info),
            confidence_factors: this.identifyConfidenceFactors(prediction, inputData),
            sensitivity_analysis: await this.performSensitivityAnalysis(inputData, prediction, model_info),
            example_influences: this.findSimilarExamples(inputData),
            human_readable_explanation: await this.generateHumanReadableExplanation(prediction, inputData, model_info)
        };

        this.reasoningTraces.set(explanationId, explanation);
        
        logger.info('Prediction explanation generated', {
            id: explanationId,
            topFeatures: explanation.feature_importance.slice(0, 3).map(f => f.feature)
        });

        return explanation;
    }

    async generateReasoningPath(query, steps = []) {
        const pathId = `path_${Date.now()}_${Math.random().toString(36).substring(7)}`;
        
        const reasoningPath = {
            id: pathId,
            timestamp: new Date(),
            query,
            steps: await this.processReasoningSteps(steps),
            logical_flow: this.analyzeLogicalFlow(steps),
            assumptions: this.identifyAssumptions(steps),
            evidence_quality: this.assessEvidenceQuality(steps),
            gaps_and_limitations: this.identifyGaps(steps),
            alternative_interpretations: await this.generateAlternativeInterpretations(query, steps)
        };

        this.reasoningTraces.set(pathId, reasoningPath);
        
        logger.debug('Reasoning path generated', {
            id: pathId,
            steps: reasoningPath.steps.length,
            assumptions: reasoningPath.assumptions.length
        });

        return reasoningPath;
    }

    async explainActionSequence(actions, context, outcomes = []) {
        const sequenceId = `seq_${Date.now()}_${Math.random().toString(36).substring(7)}`;
        
        const explanation = {
            id: sequenceId,
            timestamp: new Date(),
            actions,
            context: this.normalizeContext(context),
            outcomes,
            action_analysis: await this.analyzeActionSequence(actions, context),
            decision_points: this.identifyDecisionPoints(actions, context),
            causal_relationships: this.analyzeCausalRelationships(actions, outcomes),
            optimization_suggestions: await this.suggestOptimizations(actions, context, outcomes),
            counterfactual_analysis: await this.performCounterfactualAnalysis(actions, context, outcomes)
        };

        this.reasoningTraces.set(sequenceId, explanation);
        
        logger.info('Action sequence explained', {
            id: sequenceId,
            actions: actions.length,
            decisionPoints: explanation.decision_points.length
        });

        return explanation;
    }

    async generateInsights(domain = 'general', timeframe = '24h') {
        logger.info(`Generating insights for domain: ${domain}, timeframe: ${timeframe}`);
        
        const cacheKey = `${domain}_${timeframe}`;
        const cached = this.insightCache.get(cacheKey);
        
        if (cached && this.isCacheValid(cached)) {
            logger.debug('Using cached insights', { domain, timeframe });
            return cached.insights;
        }

        const relevantDecisions = this.filterDecisionsByDomainAndTime(domain, timeframe);
        
        const insights = {
            timestamp: new Date(),
            domain,
            timeframe,
            total_decisions_analyzed: relevantDecisions.length,
            patterns: await this.identifyPatterns(relevantDecisions),
            trends: await this.identifyTrends(relevantDecisions),
            anomalies: await this.identifyAnomalies(relevantDecisions),
            recommendations: await this.generateRecommendations(relevantDecisions),
            confidence_metrics: this.calculateInsightConfidence(relevantDecisions),
            key_insights: await this.extractKeyInsights(relevantDecisions)
        };

        // Cache the insights
        this.insightCache.set(cacheKey, {
            insights,
            generated: new Date(),
            ttl: this.getInsightTTL(timeframe)
        });

        logger.info('Insights generated', {
            domain,
            patterns: insights.patterns.length,
            keyInsights: insights.key_insights.length
        });

        return insights;
    }

    async generateExplanation(decision, context, evidence) {
        const template = this.selectExplanationTemplate(decision, context);
        
        const explanation = {
            summary: await this.generateExplanationSummary(decision, context, evidence),
            detailed_reasoning: await this.generateDetailedReasoning(decision, context, evidence),
            supporting_evidence: this.formatSupportingEvidence(evidence),
            confidence_explanation: this.explainConfidence(decision, evidence),
            risk_factors: this.identifyRiskFactors(decision, context, evidence),
            assumptions_made: this.listAssumptions(decision, context),
            template_used: template.name
        };

        return explanation;
    }

    async generateExplanationSummary(decision, context, evidence) {
        // Generate a concise summary of the decision
        let summary = `Decision to ${this.formatDecision(decision)} was made based on`;
        
        const evidenceFactors = Object.keys(evidence).length;
        if (evidenceFactors > 0) {
            summary += ` ${evidenceFactors} key factors including`;
            const topFactors = Object.entries(evidence)
                .sort(([, a], [, b]) => (b.importance || 0) - (a.importance || 0))
                .slice(0, 3)
                .map(([key]) => key);
            summary += ` ${topFactors.join(', ')}`;
        }

        if (context && context.urgency) {
            summary += `. Given ${context.urgency} urgency level`;
        }

        if (context && context.constraints) {
            summary += ` and considering ${Object.keys(context.constraints).length} constraints`;
        }

        return summary + '.';
    }

    async generateDetailedReasoning(decision, context, evidence) {
        const reasoning = [];
        
        // Context analysis
        if (context) {
            reasoning.push({
                step: 'context_analysis',
                description: 'Analyzed the current situation and constraints',
                factors: Object.keys(context),
                impact: 'high'
            });
        }

        // Evidence evaluation
        if (Object.keys(evidence).length > 0) {
            reasoning.push({
                step: 'evidence_evaluation',
                description: 'Evaluated available evidence and data',
                key_evidence: Object.entries(evidence)
                    .filter(([, data]) => data.reliability > 0.6)
                    .map(([key]) => key),
                impact: 'high'
            });
        }

        // Decision logic
        reasoning.push({
            step: 'decision_logic',
            description: 'Applied decision framework to reach conclusion',
            logic: this.explainDecisionLogic(decision, context, evidence),
            impact: 'critical'
        });

        // Alternative consideration
        reasoning.push({
            step: 'alternative_consideration',
            description: 'Considered alternative approaches and their trade-offs',
            alternatives_count: await this.countAlternatives(decision, context),
            impact: 'medium'
        });

        return reasoning;
    }

    formatSupportingEvidence(evidence) {
        return Object.entries(evidence).map(([key, data]) => ({
            factor: key,
            value: data.value || data,
            reliability: data.reliability || 0.7,
            importance: data.importance || 0.5,
            source: data.source || 'internal_analysis',
            description: this.generateEvidenceDescription(key, data)
        }));
    }

    explainConfidence(decision, evidence) {
        const factors = [];
        
        // Evidence quality
        const evidenceScores = Object.values(evidence)
            .map(e => e.reliability || 0.7)
            .filter(score => typeof score === 'number');
        
        if (evidenceScores.length > 0) {
            const avgReliability = evidenceScores.reduce((sum, s) => sum + s, 0) / evidenceScores.length;
            factors.push({
                factor: 'evidence_reliability',
                score: avgReliability,
                description: `Average evidence reliability: ${(avgReliability * 100).toFixed(1)}%`
            });
        }

        // Decision complexity
        const complexity = this.assessDecisionComplexity(decision);
        factors.push({
            factor: 'decision_complexity',
            score: 1 - complexity,
            description: `Decision complexity: ${complexity > 0.7 ? 'high' : complexity > 0.4 ? 'medium' : 'low'}`
        });

        // Historical performance
        const historicalAccuracy = this.getHistoricalAccuracy(decision);
        if (historicalAccuracy !== null) {
            factors.push({
                factor: 'historical_accuracy',
                score: historicalAccuracy,
                description: `Historical accuracy for similar decisions: ${(historicalAccuracy * 100).toFixed(1)}%`
            });
        }

        return factors;
    }

    buildReasoningChain(decision, context, evidence) {
        const chain = [];
        
        // Initial state
        chain.push({
            step: 0,
            type: 'initial_state',
            description: 'Received request for decision',
            state: { context, available_evidence: Object.keys(evidence).length }
        });

        // Context processing
        if (context) {
            chain.push({
                step: 1,
                type: 'context_processing',
                description: 'Processed contextual information',
                inputs: Object.keys(context),
                outputs: 'contextual_understanding'
            });
        }

        // Evidence analysis
        chain.push({
            step: 2,
            type: 'evidence_analysis',
            description: 'Analyzed available evidence',
            inputs: Object.keys(evidence),
            outputs: 'evidence_assessment'
        });

        // Decision synthesis
        chain.push({
            step: 3,
            type: 'decision_synthesis',
            description: 'Synthesized information to reach decision',
            inputs: ['contextual_understanding', 'evidence_assessment'],
            outputs: decision
        });

        return chain;
    }

    async analyzeAlternatives(decision, context, evidence) {
        const alternatives = [];
        
        // Generate potential alternatives
        const potentialAlternatives = await this.generatePotentialAlternatives(decision, context);
        
        for (const alternative of potentialAlternatives) {
            const analysis = {
                alternative,
                feasibility: this.assessFeasibility(alternative, context),
                expected_outcome: this.predictOutcome(alternative, context, evidence),
                trade_offs: this.identifyTradeOffs(alternative, decision),
                risk_level: this.assessRisk(alternative, context),
                why_not_chosen: this.explainWhyNotChosen(alternative, decision, context)
            };
            
            alternatives.push(analysis);
        }

        return alternatives.sort((a, b) => b.feasibility - a.feasibility);
    }

    async analyzeFeatureImportance(inputData, prediction, model_info) {
        const features = [];
        
        if (typeof inputData === 'object' && inputData !== null) {
            for (const [feature, value] of Object.entries(inputData)) {
                const importance = this.calculateFeatureImportance(feature, value, prediction, model_info);
                const influence = this.calculateFeatureInfluence(feature, value, prediction);
                
                features.push({
                    feature,
                    value,
                    importance,
                    influence,
                    explanation: this.explainFeatureImpact(feature, value, influence)
                });
            }
        }

        return features.sort((a, b) => b.importance - a.importance);
    }

    identifyConfidenceFactors(prediction, inputData) {
        const factors = [];
        
        // Data completeness
        if (typeof inputData === 'object' && inputData !== null) {
            const completeness = this.calculateDataCompleteness(inputData);
            factors.push({
                factor: 'data_completeness',
                score: completeness,
                description: `Input data is ${(completeness * 100).toFixed(1)}% complete`
            });
        }

        // Prediction consistency
        const consistency = this.assessPredictionConsistency(prediction);
        factors.push({
            factor: 'prediction_consistency',
            score: consistency,
            description: `Prediction shows ${consistency > 0.8 ? 'high' : consistency > 0.5 ? 'moderate' : 'low'} consistency`
        });

        return factors;
    }

    async performSensitivityAnalysis(inputData, prediction, model_info) {
        const analysis = [];
        
        if (typeof inputData === 'object' && inputData !== null) {
            for (const [feature, value] of Object.entries(inputData)) {
                if (typeof value === 'number') {
                    // Test small perturbations
                    const variations = [-0.1, -0.05, 0.05, 0.1].map(delta => ({
                        change: delta,
                        new_value: value + delta,
                        predicted_impact: this.estimateImpact(feature, value, delta, prediction)
                    }));
                    
                    analysis.push({
                        feature,
                        baseline_value: value,
                        variations,
                        sensitivity_score: this.calculateSensitivityScore(variations)
                    });
                }
            }
        }

        return analysis.sort((a, b) => b.sensitivity_score - a.sensitivity_score);
    }

    findSimilarExamples(inputData) {
        // Find similar examples from decision history
        const similar = [];
        
        for (const decision of this.decisionHistory) {
            const reasoning = this.reasoningTraces.get(decision.id);
            if (reasoning && reasoning.context) {
                const similarity = this.calculateSimilarity(inputData, reasoning.context);
                if (similarity > 0.6) {
                    similar.push({
                        decision_id: decision.id,
                        similarity,
                        outcome: decision.decision,
                        timestamp: decision.timestamp
                    });
                }
            }
        }

        return similar.sort((a, b) => b.similarity - a.similarity).slice(0, 5);
    }

    async generateHumanReadableExplanation(prediction, inputData, model_info) {
        let explanation = '';
        
        if (typeof prediction === 'number') {
            explanation = `The predicted value is ${prediction.toFixed(3)}. `;
        } else if (typeof prediction === 'boolean') {
            explanation = `The prediction is ${prediction ? 'positive' : 'negative'}. `;
        } else {
            explanation = `The prediction suggests ${JSON.stringify(prediction)}. `;
        }

        // Add key influential factors
        const featureImportance = await this.analyzeFeatureImportance(inputData, prediction, model_info);
        if (featureImportance.length > 0) {
            const topFeatures = featureImportance.slice(0, 3);
            explanation += `This is primarily influenced by ${topFeatures.map(f => 
                `${f.feature} (${f.influence > 0 ? 'positive' : 'negative'} impact)`
            ).join(', ')}. `;
        }

        return explanation.trim();
    }

    async processReasoningSteps(steps) {
        return steps.map((step, index) => ({
            step_number: index + 1,
            description: step.description || step,
            logic_type: this.classifyLogicType(step),
            evidence_used: step.evidence || [],
            assumptions: step.assumptions || [],
            confidence: step.confidence || 0.7
        }));
    }

    analyzeLogicalFlow(steps) {
        const flow = {
            is_coherent: true,
            logical_gaps: [],
            reasoning_type: 'deductive', // deductive, inductive, abductive
            flow_quality: 0.8
        };

        // Analyze step-by-step logical coherence
        for (let i = 1; i < steps.length; i++) {
            const coherence = this.assessStepCoherence(steps[i-1], steps[i]);
            if (coherence < 0.6) {
                flow.logical_gaps.push({
                    between_steps: [i, i+1],
                    issue: 'logical_disconnect',
                    severity: 1 - coherence
                });
                flow.is_coherent = false;
            }
        }

        return flow;
    }

    identifyAssumptions(steps) {
        const assumptions = [];
        
        for (const step of steps) {
            if (step.assumptions) {
                step.assumptions.forEach(assumption => {
                    assumptions.push({
                        assumption,
                        step: step.description || step,
                        validity: this.assessAssumptionValidity(assumption),
                        impact: this.assessAssumptionImpact(assumption)
                    });
                });
            } else {
                // Try to infer implicit assumptions
                const implicit = this.inferImplicitAssumptions(step);
                assumptions.push(...implicit);
            }
        }

        return assumptions;
    }

    assessEvidenceQuality(steps) {
        let totalQuality = 0;
        let evidenceCount = 0;

        for (const step of steps) {
            if (step.evidence) {
                step.evidence.forEach(evidence => {
                    const quality = this.evaluateEvidenceQuality(evidence);
                    totalQuality += quality;
                    evidenceCount++;
                });
            }
        }

        return {
            average_quality: evidenceCount > 0 ? totalQuality / evidenceCount : 0.5,
            evidence_count: evidenceCount,
            quality_distribution: this.analyzeQualityDistribution(steps)
        };
    }

    identifyGaps(steps) {
        const gaps = [];
        
        // Check for missing logical steps
        for (let i = 1; i < steps.length; i++) {
            const gap = this.findLogicalGap(steps[i-1], steps[i]);
            if (gap) {
                gaps.push({
                    type: 'logical_gap',
                    location: `Between steps ${i} and ${i+1}`,
                    description: gap,
                    severity: 'medium'
                });
            }
        }

        // Check for missing evidence
        steps.forEach((step, index) => {
            if (this.requiresEvidence(step) && !step.evidence) {
                gaps.push({
                    type: 'missing_evidence',
                    location: `Step ${index + 1}`,
                    description: 'Step makes claims without supporting evidence',
                    severity: 'high'
                });
            }
        });

        return gaps;
    }

    async generateAlternativeInterpretations(query, steps) {
        const alternatives = [];
        
        // Different logical frameworks
        const frameworks = ['consequentialist', 'deontological', 'virtue_ethics', 'pragmatic'];
        
        for (const framework of frameworks) {
            const interpretation = {
                framework,
                reasoning: await this.reinterpretWithFramework(query, steps, framework),
                conclusion: await this.generateAlternativeConclusion(query, steps, framework),
                confidence: 0.6
            };
            alternatives.push(interpretation);
        }

        return alternatives;
    }

    initializeTemplates() {
        this.explanationTemplates.set('decision', {
            name: 'decision_template',
            structure: ['context', 'evidence', 'reasoning', 'conclusion', 'alternatives'],
            tone: 'analytical'
        });

        this.explanationTemplates.set('prediction', {
            name: 'prediction_template',
            structure: ['input_analysis', 'model_rationale', 'feature_importance', 'confidence', 'limitations'],
            tone: 'technical'
        });

        this.explanationTemplates.set('sequence', {
            name: 'sequence_template',
            structure: ['overview', 'step_analysis', 'causal_chains', 'outcomes', 'optimizations'],
            tone: 'procedural'
        });
    }

    // Helper methods
    normalizeContext(context) {
        if (typeof context === 'string') return { description: context };
        if (Array.isArray(context)) return { items: context };
        return context || {};
    }

    normalizeInputData(inputData) {
        return inputData;
    }

    calculateExplanationConfidence(decision, evidence) {
        const evidenceQuality = Object.values(evidence).reduce((sum, e) => 
            sum + (e.reliability || 0.7), 0) / Math.max(1, Object.keys(evidence).length);
        
        const decisionComplexity = this.assessDecisionComplexity(decision);
        
        return Math.max(0.1, Math.min(0.9, evidenceQuality * (1 - decisionComplexity / 2)));
    }

    assessDecisionComplexity(decision) {
        if (typeof decision === 'string') return decision.length / 100;
        if (typeof decision === 'object') return Object.keys(decision).length / 10;
        return 0.3; // Default complexity
    }

    formatDecision(decision) {
        if (typeof decision === 'string') return decision;
        if (typeof decision === 'object') return JSON.stringify(decision);
        return String(decision);
    }

    explainDecisionLogic(decision, context, evidence) {
        return `Applied systematic analysis considering context factors and available evidence to determine optimal course of action.`;
    }

    async countAlternatives(decision, context) {
        return Math.floor(Math.random() * 5) + 2; // Mock: 2-6 alternatives
    }

    generateEvidenceDescription(key, data) {
        return `${key}: ${typeof data.value !== 'undefined' ? data.value : data}`;
    }

    getHistoricalAccuracy(decision) {
        // Mock historical accuracy lookup
        return 0.7 + Math.random() * 0.2;
    }

    isCacheValid(cached) {
        return Date.now() - cached.generated.getTime() < cached.ttl;
    }

    getInsightTTL(timeframe) {
        const ttlMap = {
            '1h': 5 * 60 * 1000,      // 5 minutes
            '24h': 60 * 60 * 1000,     // 1 hour
            '7d': 6 * 60 * 60 * 1000,  // 6 hours
            '30d': 24 * 60 * 60 * 1000  // 24 hours
        };
        return ttlMap[timeframe] || ttlMap['24h'];
    }

    filterDecisionsByDomainAndTime(domain, timeframe) {
        const now = Date.now();
        const timeMap = {
            '1h': 60 * 60 * 1000,
            '24h': 24 * 60 * 60 * 1000,
            '7d': 7 * 24 * 60 * 60 * 1000,
            '30d': 30 * 24 * 60 * 60 * 1000
        };
        
        const cutoff = now - (timeMap[timeframe] || timeMap['24h']);
        
        return this.decisionHistory.filter(decision => 
            decision.timestamp.getTime() > cutoff &&
            (domain === 'general' || this.matchesDomain(decision, domain))
        );
    }

    matchesDomain(decision, domain) {
        // Simple domain matching - could be enhanced
        const decisionStr = JSON.stringify(decision).toLowerCase();
        return decisionStr.includes(domain.toLowerCase());
    }

    async identifyPatterns(decisions) {
        const patterns = [];
        
        if (decisions.length > 10) {
            patterns.push({
                type: 'frequency',
                description: 'High decision volume detected',
                confidence: 0.8
            });
        }

        // Time-based patterns
        const hourCounts = {};
        decisions.forEach(d => {
            const hour = d.timestamp.getHours();
            hourCounts[hour] = (hourCounts[hour] || 0) + 1;
        });

        const peakHour = Object.entries(hourCounts)
            .reduce((max, [hour, count]) => count > max.count ? { hour: parseInt(hour), count } : max, { hour: 0, count: 0 });

        if (peakHour.count > decisions.length * 0.3) {
            patterns.push({
                type: 'temporal',
                description: `Peak decision activity at ${peakHour.hour}:00`,
                confidence: 0.7
            });
        }

        return patterns;
    }

    async identifyTrends(decisions) {
        const trends = [];
        
        if (decisions.length >= 5) {
            const recentConfidence = decisions.slice(-5).reduce((sum, d) => sum + d.confidence, 0) / 5;
            const earlierConfidence = decisions.slice(0, -5).reduce((sum, d) => sum + d.confidence, 0) / Math.max(1, decisions.length - 5);
            
            if (recentConfidence - earlierConfidence > 0.1) {
                trends.push({
                    type: 'confidence_improvement',
                    description: 'Decision confidence trending upward',
                    magnitude: recentConfidence - earlierConfidence
                });
            }
        }

        return trends;
    }

    async identifyAnomalies(decisions) {
        const anomalies = [];
        
        decisions.forEach(decision => {
            if (decision.confidence < 0.3) {
                anomalies.push({
                    decision_id: decision.id,
                    type: 'low_confidence',
                    severity: 'medium',
                    description: 'Unusually low confidence score'
                });
            }
        });

        return anomalies;
    }

    async generateRecommendations(decisions) {
        const recommendations = [];
        
        const avgConfidence = decisions.reduce((sum, d) => sum + d.confidence, 0) / Math.max(1, decisions.length);
        
        if (avgConfidence < 0.6) {
            recommendations.push({
                type: 'improve_confidence',
                priority: 'high',
                description: 'Consider gathering more evidence before making decisions',
                expected_impact: 'Increase decision confidence by 15-25%'
            });
        }

        if (decisions.length > 20) {
            recommendations.push({
                type: 'automation_opportunity',
                priority: 'medium',
                description: 'High decision volume suggests automation opportunities',
                expected_impact: 'Reduce decision time by 30-40%'
            });
        }

        return recommendations;
    }

    calculateInsightConfidence(decisions) {
        return {
            sample_size_adequacy: decisions.length >= 10 ? 0.9 : decisions.length / 10 * 0.9,
            data_quality: 0.8, // Mock quality assessment
            temporal_coverage: this.assessTemporalCoverage(decisions)
        };
    }

    assessTemporalCoverage(decisions) {
        if (decisions.length === 0) return 0;
        
        const timestamps = decisions.map(d => d.timestamp.getTime()).sort();
        const span = timestamps[timestamps.length - 1] - timestamps[0];
        const daySpan = span / (24 * 60 * 60 * 1000);
        
        return Math.min(1, daySpan / 7); // Good coverage if spans a week
    }

    async extractKeyInsights(decisions) {
        const insights = [];
        
        if (decisions.length > 0) {
            const avgConfidence = decisions.reduce((sum, d) => sum + d.confidence, 0) / decisions.length;
            insights.push(`Average decision confidence: ${(avgConfidence * 100).toFixed(1)}%`);
            
            const highConfidenceRatio = decisions.filter(d => d.confidence > 0.8).length / decisions.length;
            if (highConfidenceRatio > 0.7) {
                insights.push('Strong pattern of high-confidence decisions');
            }
        }

        return insights;
    }

    // Additional helper methods...
    calculateFeatureImportance(feature, value, prediction, model_info) {
        return 0.5 + Math.random() * 0.5; // Mock implementation
    }

    calculateFeatureInfluence(feature, value, prediction) {
        return (Math.random() - 0.5) * 2; // Mock: -1 to 1
    }

    explainFeatureImpact(feature, value, influence) {
        return `${feature} (${value}) has a ${influence > 0 ? 'positive' : 'negative'} influence on the prediction`;
    }

    calculateDataCompleteness(inputData) {
        const values = Object.values(inputData);
        const nonNullValues = values.filter(v => v !== null && v !== undefined);
        return nonNullValues.length / values.length;
    }

    assessPredictionConsistency(prediction) {
        return 0.6 + Math.random() * 0.3; // Mock consistency score
    }

    calculateSimilarity(data1, data2) {
        // Simple similarity calculation
        if (typeof data1 === 'object' && typeof data2 === 'object') {
            const keys1 = Object.keys(data1);
            const keys2 = Object.keys(data2);
            const commonKeys = keys1.filter(k => keys2.includes(k));
            return commonKeys.length / Math.max(keys1.length, keys2.length);
        }
        return 0.3; // Default similarity
    }

    async loadExplanationData() {
        try {
            await fs.mkdir(path.dirname(this.explanationsPath), { recursive: true });
            const data = await fs.readFile(this.explanationsPath, 'utf8');
            const parsed = JSON.parse(data);
            
            if (parsed.reasoningTraces) {
                parsed.reasoningTraces.forEach(([id, trace]) => {
                    trace.timestamp = new Date(trace.timestamp);
                    this.reasoningTraces.set(id, trace);
                });
            }
            
            if (parsed.decisionHistory) {
                this.decisionHistory = parsed.decisionHistory.map(decision => ({
                    ...decision,
                    timestamp: new Date(decision.timestamp)
                }));
            }
            
            logger.info(`Loaded explanation data with ${this.reasoningTraces.size} traces`);
        } catch (error) {
            logger.debug('No existing explanation data found, starting fresh');
        }
    }

    async saveExplanationData() {
        try {
            const data = {
                reasoningTraces: Array.from(this.reasoningTraces.entries()),
                decisionHistory: this.decisionHistory,
                savedAt: new Date().toISOString()
            };
            await fs.writeFile(this.explanationsPath, JSON.stringify(data, null, 2));
        } catch (error) {
            logger.error('Failed to save explanation data', { error });
        }
    }
}