universal-ai-brain/packages/core/src/UniversalAIBrain.ts

🧠 UNIVERSAL AI BRAIN 3.3 - The world's most advanced cognitive architecture with 24 specialized systems, MongoDB 8.1 $rankFusion hybrid search, latest Voyage 3.5 embeddings, and framework-agnostic design. Works with Mastra, Vercel AI, LangChain, OpenAI A

/**
 * @file UniversalAIBrain - Main orchestrator for the Universal AI Brain system
 * 
 * This is the central orchestrator that integrates all components of the Universal AI Brain:
 * MongoDB-powered intelligence, semantic memory, context injection, safety systems,
 * self-improvement engines, and real-time monitoring. Provides a unified interface
 * for any TypeScript framework to integrate and gain superpowers.
 * 
 * Features:
 * - Framework-agnostic integration layer
 * - MongoDB Atlas Vector Search intelligence
 * - Comprehensive safety and compliance systems
 * - Self-improvement and optimization engines
 * - Real-time monitoring and analytics
 * - Production-ready with enterprise-grade reliability
 */

import { MongoClient, Db } from 'mongodb';
import { MongoVectorStore } from './vector/MongoVectorStore';

// Core Collections
import { TracingCollection } from './collections/TracingCollection';
import { MemoryCollection } from './collections/MemoryCollection';
import { ContextCollection } from './collections/ContextCollection';

// Intelligence Layer
import { SemanticMemoryEngine } from './intelligence/SemanticMemoryEngine';
import { ContextInjectionEngine } from './intelligence/ContextInjectionEngine';
import { VectorSearchEngine } from './intelligence/VectorSearchEngine';
import { HybridSearchEngine } from './features/hybridSearch';
import { OpenAIEmbeddingProvider } from './embeddings/OpenAIEmbeddingProvider';
import { VoyageAIEmbeddingProvider } from './embeddings/VoyageAIEmbeddingProvider';

// Cognitive Intelligence Layer
import { EmotionalIntelligenceEngine } from './intelligence/EmotionalIntelligenceEngine';
import { GoalHierarchyManager } from './intelligence/GoalHierarchyManager';
import { ConfidenceTrackingEngine } from './intelligence/ConfidenceTrackingEngine';
import { AttentionManagementSystem } from './intelligence/AttentionManagementSystem';
import { CulturalKnowledgeEngine } from './intelligence/CulturalKnowledgeEngine';
import { SkillCapabilityManager } from './intelligence/SkillCapabilityManager';
import { CommunicationProtocolManager } from './intelligence/CommunicationProtocolManager';
import { TemporalPlanningEngine } from './intelligence/TemporalPlanningEngine';

// Enhanced Cognitive Intelligence Layer (AI Brain 2.0 Enhancements)
import { AdvancedToolInterface } from './intelligence/AdvancedToolInterface';
import { WorkflowOrchestrationEngine } from './intelligence/WorkflowOrchestrationEngine';
import { MultiModalProcessingEngine } from './intelligence/MultiModalProcessingEngine';
import { HumanFeedbackIntegrationEngine } from './intelligence/HumanFeedbackIntegrationEngine';

// Phase 1 Integration: Working Memory and Decay Systems
import { WorkingMemoryManager } from './intelligence/WorkingMemoryManager';
import { MemoryDecayEngine } from './intelligence/MemoryDecayEngine';

// Phase 2 Integration: Advanced Cognitive Systems
import { AnalogicalMappingSystem } from './intelligence/AnalogicalMappingSystem';
import { CausalReasoningEngine } from './intelligence/CausalReasoningEngine';
import { SocialIntelligenceEngine } from './intelligence/SocialIntelligenceEngine';

// Phase 3 Integration: Episodic Memory System
import { EpisodicMemoryEngine } from './intelligence/EpisodicMemoryEngine';

// Safety & Guardrails
import { SafetyGuardrailsEngine } from './safety/SafetyGuardrailsEngine';
import { HallucinationDetector } from './safety/HallucinationDetector';
import { PIIDetector } from './safety/PIIDetector';
import { ComplianceAuditLogger } from './safety/ComplianceAuditLogger';
import { FrameworkSafetyIntegration } from './safety/FrameworkSafetyIntegration';

// Self-Improvement
import { FailureAnalysisEngine } from './self-improvement/FailureAnalysisEngine';
import { ContextLearningEngine } from './self-improvement/ContextLearningEngine';
import { FrameworkOptimizationEngine } from './self-improvement/FrameworkOptimizationEngine';
import { SelfImprovementMetrics } from './self-improvement/SelfImprovementMetrics';

// Monitoring
import { PerformanceAnalyticsEngine } from './monitoring/PerformanceAnalyticsEngine';
import { RealTimeMonitoringDashboard } from './monitoring/RealTimeMonitoringDashboard';

// 🎯 SIMPLIFIED CONFIG FOR EASY SETUP
export interface SimpleAIBrainConfig {
  mongoUri?: string;
  databaseName?: string;
  apiKey?: string;
  provider?: 'voyage' | 'openai';
  mode?: 'demo' | 'basic' | 'production';
}

// 🔧 FULL CONFIG FOR ADVANCED USERS (BACKWARD COMPATIBILITY)
export interface UniversalAIBrainConfig {
  mongodb?: {
    connectionString: string;
    databaseName?: string;
    collections?: {
      tracing?: string;
      memory?: string;
      context?: string;
      metrics?: string;
      audit?: string;
    };
  };
  intelligence?: {
    embeddingModel?: string;
    vectorDimensions?: number;
    similarityThreshold?: number;
    maxContextLength?: number;
    // Hybrid Search Configuration
    enableHybridSearch?: boolean;
    hybridSearchVectorWeight?: number;
    hybridSearchTextWeight?: number;
    hybridSearchFallbackToVector?: boolean;
  };
  safety?: {
    enableContentFiltering?: boolean;
    enablePIIDetection?: boolean;
    enableHallucinationDetection?: boolean;
    enableComplianceLogging?: boolean;
    safetyLevel?: 'strict' | 'moderate' | 'permissive';
  };
  monitoring?: {
    enableRealTimeMonitoring?: boolean;
    enablePerformanceTracking?: boolean;
    enableCostTracking?: boolean;
    enableErrorTracking?: boolean;
    metricsRetentionDays?: number;
    alertingEnabled?: boolean;
    dashboardRefreshInterval?: number;
  };
  selfImprovement?: {
    enableAutomaticOptimization?: boolean;
    learningRate?: number;
    optimizationInterval?: number;
    feedbackLoopEnabled?: boolean;
  };
  apis?: {
    openai?: {
      apiKey: string;
      baseURL?: string;
    };
    voyage?: {
      apiKey: string;
      baseURL?: string;
    };
  };
  // 🚀 NEW: Support for simple config
  mongoUri?: string;
  databaseName?: string;
  apiKey?: string;
  provider?: 'voyage' | 'openai';
  mode?: 'demo' | 'basic' | 'production';
}

export interface AIBrainResponse {
  success: boolean;
  data?: any;
  error?: string;
  metadata: {
    responseTime: number;
    tokensUsed: number;
    cost: number;
    safetyScore: number;
    contextUsed: string[];
    traceId: string;
  };
}

// 🛡️ 2025 TYPESCRIPT ERROR HANDLING UTILITY
function getErrorMessage(error: unknown): string {
  if (error instanceof Error) {
    return error.message;
  }
  if (typeof error === 'string') {
    return error;
  }
  return String(error);
}

// 🎯 SMART DEFAULTS FOR EASY SETUP
const DEFAULT_CONFIG = {
  intelligence: {
    embeddingModel: 'voyage-3.5',
    vectorDimensions: 1024,
    similarityThreshold: 0.7,
    maxContextLength: 4000,
    // Hybrid Search as DEFAULT - MongoDB's most powerful capability
    enableHybridSearch: true,
    hybridSearchVectorWeight: 0.7,
    hybridSearchTextWeight: 0.3,
    hybridSearchFallbackToVector: true
  },
  safety: {
    enableContentFiltering: true,
    enablePIIDetection: true,
    enableHallucinationDetection: true,
    enableComplianceLogging: true,
    safetyLevel: 'moderate' as const
  },
  monitoring: {
    enableRealTimeMonitoring: true,
    enablePerformanceTracking: true,
    enableCostTracking: true,
    enableErrorTracking: true,
    metricsRetentionDays: 30,
    alertingEnabled: false,
    dashboardRefreshInterval: 5000
  },
  collections: {
    tracing: 'agent_traces',
    memory: 'agent_memory',
    context: 'agent_context',
    metrics: 'agent_metrics',
    audit: 'agent_safety_logs'
  }
};

/**
 * UniversalAIBrain - The central orchestrator for AI intelligence
 *
 * Provides a unified interface for any TypeScript framework to integrate
 * with MongoDB-powered AI intelligence, safety systems, and self-improvement.
 */
export class UniversalAIBrain {
  private config: UniversalAIBrainConfig;
  private mongoClient: MongoClient;
  private database!: Db; // Will be initialized in initialize()
  private mongoConnection: MongoClient; // Add this property
  private isInitialized: boolean = false;

  // Core Collections
  private tracingCollection!: TracingCollection;
  private memoryCollection!: MemoryCollection;
  private contextCollection!: ContextCollection;
  private metricsCollection!: MemoryCollection;
  private auditCollection!: MemoryCollection;

  // Intelligence Layer
  private semanticMemoryEngine!: SemanticMemoryEngine;
  private contextInjectionEngine!: ContextInjectionEngine;
  private vectorSearchEngine!: VectorSearchEngine;
  private hybridSearchEngine!: HybridSearchEngine;
  private mongoVectorStore!: MongoVectorStore;

  // Cognitive Intelligence Layer
  private emotionalIntelligenceEngine!: EmotionalIntelligenceEngine;
  private goalHierarchyManager!: GoalHierarchyManager;
  private confidenceTrackingEngine!: ConfidenceTrackingEngine;
  private attentionManagementSystem!: AttentionManagementSystem;
  private culturalKnowledgeEngine!: CulturalKnowledgeEngine;
  private skillCapabilityManager!: SkillCapabilityManager;
  private communicationProtocolManager!: CommunicationProtocolManager;
  private temporalPlanningEngine!: TemporalPlanningEngine;

  // Enhanced Cognitive Intelligence Layer (AI Brain 2.0 Enhancements)
  private _advancedToolInterface!: AdvancedToolInterface;
  private _workflowOrchestrationEngine!: WorkflowOrchestrationEngine;
  private _multiModalProcessingEngine!: MultiModalProcessingEngine;
  private _humanFeedbackIntegrationEngine!: HumanFeedbackIntegrationEngine;

  // Phase 1 Integration: Working Memory and Decay Systems
  private _workingMemoryManager!: WorkingMemoryManager;
  private _memoryDecayEngine!: MemoryDecayEngine;

  // Phase 2 Integration: Advanced Cognitive Systems
  private _analogicalMappingSystem!: AnalogicalMappingSystem;
  private _causalReasoningEngine!: CausalReasoningEngine;
  private _socialIntelligenceEngine!: SocialIntelligenceEngine;

  // Phase 3 Integration: Episodic Memory System
  private _episodicMemoryEngine!: EpisodicMemoryEngine;

  // Safety & Guardrails
  private safetyEngine!: SafetyGuardrailsEngine;
  private hallucinationDetector!: HallucinationDetector;
  private piiDetector!: PIIDetector;
  private complianceAuditLogger!: ComplianceAuditLogger;
  private frameworkSafetyIntegration!: FrameworkSafetyIntegration;

  // Self-Improvement
  private failureAnalysisEngine!: FailureAnalysisEngine;
  private contextLearningEngine!: ContextLearningEngine;
  private frameworkOptimizationEngine!: FrameworkOptimizationEngine;
  private selfImprovementMetrics!: SelfImprovementMetrics;

  // Monitoring
  private performanceAnalyticsEngine!: PerformanceAnalyticsEngine;
  private realTimeMonitoringDashboard!: RealTimeMonitoringDashboard;

  constructor(config: UniversalAIBrainConfig | SimpleAIBrainConfig) {
    this.config = this.buildFullConfig(config);
    this.mongoClient = new MongoClient(this.config.mongodb!.connectionString);
    this.mongoConnection = this.mongoClient; // Initialize mongoConnection
  }

  // 🚀 MAGICAL STATIC FACTORY METHODS FOR EASY SETUP

  /**
   * 🎯 DEMO MODE - Try AI Brain without any external dependencies
   */
  static demo(): UniversalAIBrain {
    console.log('🎭 Demo Mode: Using in-memory storage (no MongoDB required)');
    return new UniversalAIBrain({
      mode: 'demo',
      mongoUri: 'mongodb://localhost:27017', // Will be mocked
      databaseName: 'demo_ai_brain',
      apiKey: 'demo-key'
    });
  }

  /**
   * ⚡ BASIC SETUP - Minimal configuration with smart defaults
   */
  static basic(config: { mongoUri: string; apiKey: string; databaseName?: string }): UniversalAIBrain {
    return new UniversalAIBrain({
      mode: 'basic',
      mongoUri: config.mongoUri,
      apiKey: config.apiKey,
      databaseName: config.databaseName
    });
  }

  /**
   * 🎯 AUTO SETUP - Detect everything from environment variables
   */
  static auto(): UniversalAIBrain {
    return new UniversalAIBrain({
      mode: 'production'
    });
  }

  /**
   * 🔧 FOR MASTRA - Optimized for Mastra framework
   */
  static forMastra(config?: { mongoUri?: string; apiKey?: string }): UniversalAIBrain {
    return new UniversalAIBrain({
      mode: 'production',
      mongoUri: config?.mongoUri,
      apiKey: config?.apiKey,
      databaseName: 'ai_brain_mastra'
      // Auto-detect provider from apiKey - no hardcoding
    });
  }

  /**
   * ⚡ FOR VERCEL AI - Optimized for Vercel AI SDK
   */
  static forVercelAI(config?: { mongoUri?: string; apiKey?: string }): UniversalAIBrain {
    return new UniversalAIBrain({
      mode: 'production',
      mongoUri: config?.mongoUri,
      apiKey: config?.apiKey,
      databaseName: 'ai_brain_vercel'
      // Auto-detect provider from apiKey - no hardcoding
    });
  }

  /**
   * � FOR OPENAI - Optimized for OpenAI Chat Completions API
   */
  static forOpenAI(config?: { mongoUri?: string; apiKey?: string }): UniversalAIBrain {
    return new UniversalAIBrain({
      mode: 'production',
      mongoUri: config?.mongoUri,
      apiKey: config?.apiKey,
      databaseName: 'ai_brain_openai'
      // Auto-detect provider from apiKey - no hardcoding
    });
  }

  /**
   * �🦜 FOR LANGCHAIN - Optimized for LangChain.js
   */
  static forLangChain(config?: { mongoUri?: string; apiKey?: string }): UniversalAIBrain {
    return new UniversalAIBrain({
      mode: 'production',
      mongoUri: config?.mongoUri,
      apiKey: config?.apiKey,
      databaseName: 'ai_brain_langchain'
      // Auto-detect provider from apiKey - no hardcoding
    });
  }

  /**
   * 🎯 SMART CONFIG BUILDER - Converts simple config to full config with defaults
   */
  private buildFullConfig(input: UniversalAIBrainConfig | SimpleAIBrainConfig): UniversalAIBrainConfig {
    // If it's already a full config, merge with defaults
    if ('mongodb' in input && input.mongodb) {
      return this.mergeWithDefaults(input as UniversalAIBrainConfig);
    }

    // Convert simple config to full config
    const simple = input as SimpleAIBrainConfig;

    // Auto-detect from environment variables
    const mongoUri = simple.mongoUri ||
                    process.env.MONGODB_CONNECTION_STRING ||
                    process.env.MONGODB_URI ||
                    process.env.MONGO_URI;

    const apiKey = simple.apiKey ||
                  process.env.VOYAGE_API_KEY ||
                  process.env.OPENAI_API_KEY;

    const provider = simple.provider ||
                    (process.env.VOYAGE_API_KEY ? 'voyage' : 'openai');

    if (!mongoUri) {
      throw new Error('🚨 MongoDB connection string required! Set MONGODB_CONNECTION_STRING env var or pass mongoUri');
    }

    if (!apiKey && simple.mode !== 'demo') {
      throw new Error('🚨 API key required! Set VOYAGE_API_KEY or OPENAI_API_KEY env var or pass apiKey');
    }

    const databaseName = simple.databaseName || `ai_brain_${Date.now()}`;

    return {
      mongodb: {
        connectionString: mongoUri,
        databaseName,
        collections: DEFAULT_CONFIG.collections
      },
      intelligence: {
        ...DEFAULT_CONFIG.intelligence,
        embeddingModel: provider === 'voyage' ? 'voyage-3.5' : 'text-embedding-3-small',
        vectorDimensions: provider === 'voyage' ? 1024 : 1536
      },
      safety: DEFAULT_CONFIG.safety,
      monitoring: DEFAULT_CONFIG.monitoring,
      apis: apiKey ? {
        [provider]: {
          apiKey,
          baseURL: provider === 'voyage' ? 'https://api.voyageai.com/v1' : 'https://api.openai.com/v1'
        }
      } : undefined
    };
  }

  /**
   * 🔧 MERGE FULL CONFIG WITH DEFAULTS
   */
  private mergeWithDefaults(config: UniversalAIBrainConfig): UniversalAIBrainConfig {
    return {
      mongodb: {
        connectionString: config.mongodb?.connectionString || config.mongoUri || process.env.MONGODB_CONNECTION_STRING!,
        databaseName: config.mongodb?.databaseName || config.databaseName || `ai_brain_${Date.now()}`,
        collections: {
          ...DEFAULT_CONFIG.collections,
          ...config.mongodb?.collections
        }
      },
      intelligence: {
        ...DEFAULT_CONFIG.intelligence,
        ...config.intelligence
      },
      safety: {
        ...DEFAULT_CONFIG.safety,
        ...config.safety
      },
      monitoring: {
        ...DEFAULT_CONFIG.monitoring,
        ...config.monitoring
      },
      selfImprovement: config.selfImprovement,
      apis: config.apis
    };
  }

  /**
   * Initialize the Universal AI Brain system
   */
  async initialize(): Promise<void> {
    if (this.isInitialized) {
      throw new Error('Universal AI Brain is already initialized');
    }

    try {
      // Connect to MongoDB
      await this.mongoClient.connect();
      this.database = this.mongoClient.db(this.config.mongodb!.databaseName!);

      // Initialize core collections
      await this.initializeCollections();

      // Initialize intelligence layer
      await this.initializeIntelligenceLayer();

      // Initialize safety systems
      await this.initializeSafetySystems();

      // Initialize self-improvement engines
      await this.initializeSelfImprovementEngines();

      // Initialize monitoring systems
      await this.initializeMonitoringSystems();

      // Start real-time monitoring if enabled
      if (this.config.monitoring?.enableRealTimeMonitoring) {
        await this.realTimeMonitoringDashboard.startMonitoring();
      }

      this.isInitialized = true;
      console.log('🧠 Universal AI Brain initialized successfully');

    } catch (error) {
      console.error('Failed to initialize Universal AI Brain:', error);
      throw error;
    }
  }

  /**
   * Cleanup resources and close connections
   */
  async cleanup(): Promise<void> {
    try {
      if (this.mongoConnection) {
        await this.mongoConnection.close();
        console.log('🧹 MongoDB connection closed');
      }

      this.isInitialized = false;
      console.log('🧹 Universal AI Brain cleanup completed');
    } catch (error) {
      console.error('Failed to cleanup Universal AI Brain:', error);
      throw error;
    }
  }

  /**
   * Store memory with automatic embedding generation
   */
  async storeMemory(memory: {
    content: string;
    type: string;
    importance: number;
    metadata?: Record<string, any>;
  }): Promise<string> {
    if (!this.isInitialized) {
      throw new Error('Universal AI Brain must be initialized before storing memories');
    }

    try {
      const memoryDoc = {
        agentId: this.getAgentId(),
        conversationId: memory.metadata?.conversationId || 'default',
        memoryType: memory.type,
        content: memory.content,
        importance: memory.importance,
        metadata: memory.metadata || {}
      };

      const createdMemory = await this.memoryCollection.createMemory(memoryDoc);
      console.log(`💾 Memory stored: ${createdMemory._id}`);
      return createdMemory._id?.toString() || '';
    } catch (error) {
      console.error('Failed to store memory:', error);
      throw error;
    }
  }

  /**
   * Search memories using vector similarity
   */
  async searchMemory(query: string, options: {
    limit?: number;
    minScore?: number;
  } = {}): Promise<any[]> {
    if (!this.isInitialized) {
      throw new Error('Universal AI Brain must be initialized before searching memories');
    }

    try {
      // For now, return a simple text-based search
      // In production, this would use vector search
      const memories = await this.memoryCollection.findPaginated(
        { content: { $regex: query, $options: 'i' } },
        { limit: options.limit || 10 }
      );

      console.log(`🔍 Found ${memories.documents.length} memories for query: ${query}`);
      return memories.documents;
    } catch (error) {
      console.error('Failed to search memories:', error);
      throw error;
    }
  }

  /**
   * Enhance context with relevant information
   */
  async enhanceContext(query: string, options: {
    maxContextItems?: number;
    includeMemory?: boolean;
    includeKnowledge?: boolean;
  } = {}): Promise<{
    contextItems: any[];
    enhancedPrompt: string;
  }> {
    if (!this.isInitialized) {
      throw new Error('Universal AI Brain must be initialized before enhancing context');
    }

    try {
      const contextItems: any[] = [];

      if (options.includeMemory) {
        const memories = await this.searchMemory(query, {
          limit: options.maxContextItems || 3
        });
        contextItems.push(...memories.map(m => ({ type: 'memory', ...m })));
      }

      const enhancedPrompt = options.maxContextItems === 0
        ? query
        : `Context: ${contextItems.map(item => item.content).join('\n')}\n\nQuery: ${query}`;

      console.log(`🎯 Enhanced context with ${contextItems.length} items`);
      return { contextItems, enhancedPrompt };
    } catch (error) {
      console.error('Failed to enhance context:', error);
      throw error;
    }
  }

  /**
   * Intelligent search using Hybrid Search with fallback to Vector Search
   * This is the cornerstone method that leverages MongoDB's most powerful capabilities
   */
  private async performIntelligentSearch(
    query: string,
    options: {
      limit?: number;
      minScore?: number;
      includeExplanation?: boolean;
    } = {}
  ): Promise<any[]> {
    const {
      limit = 10,
      minScore = 0.7,
      includeExplanation = true
    } = options;

    try {
      // Use Hybrid Search if enabled (default)
      if (this.config.intelligence?.enableHybridSearch !== false) {
        try {
          console.log('🚀 Using MongoDB Atlas Hybrid Search with $rankFusion');

          const hybridResults = await this.hybridSearchEngine.search(
            query,
            {}, // filters
            {
              limit,
              vector_weight: this.config.intelligence?.hybridSearchVectorWeight || 0.7,
              text_weight: this.config.intelligence?.hybridSearchTextWeight || 0.3,
              explain_relevance: includeExplanation
            }
          );

          // Convert hybrid results to expected format
          return hybridResults.map(result => ({
            id: result._id,
            content: result.content.text,
            score: result.scores.combined_score,
            metadata: result.metadata,
            explanation: result.relevance_explanation
          }));

        } catch (hybridError) {
          console.warn('⚠️ Hybrid search failed, falling back to vector search:', hybridError);

          // Fallback to vector search if enabled
          if (this.config.intelligence?.hybridSearchFallbackToVector !== false) {
            return await this.performVectorSearchFallback(query, options);
          } else {
            throw hybridError;
          }
        }
      } else {
        // Use vector search when hybrid search is disabled
        console.log('🔍 Using Vector Search (hybrid search disabled)');
        return await this.performVectorSearchFallback(query, options);
      }

    } catch (error) {
      console.error('❌ Intelligent search failed:', error);
      throw error;
    }
  }

  /**
   * Fallback vector search method
   */
  private async performVectorSearchFallback(
    query: string,
    options: {
      limit?: number;
      minScore?: number;
      includeExplanation?: boolean;
    }
  ): Promise<any[]> {
    return await this.vectorSearchEngine.semanticSearch(query, {
      limit: options.limit,
      minScore: options.minScore,
      includeExplanation: options.includeExplanation
    });
  }

  /**
   * Check content safety
   */
  async checkSafety(content: string): Promise<{
    piiDetected: boolean;
    piiItems: any[];
    safetyScore: number;
  }> {
    if (!this.isInitialized) {
      throw new Error('Universal AI Brain must be initialized before checking safety');
    }

    try {
      // Simple PII detection for demo
      const emailRegex = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g;
      const ssnRegex = /\b\d{3}-\d{2}-\d{4}\b/g;

      const emails = content.match(emailRegex) || [];
      const ssns = content.match(ssnRegex) || [];

      const piiItems = [
        ...emails.map(email => ({ type: 'email', value: email })),
        ...ssns.map(ssn => ({ type: 'ssn', value: ssn }))
      ];

      const piiDetected = piiItems.length > 0;
      const safetyScore = piiDetected ? 0.3 : 0.9;

      console.log(`🛡️ Safety check: PII detected: ${piiDetected}, Score: ${safetyScore}`);
      return { piiDetected, piiItems, safetyScore };
    } catch (error) {
      console.error('Failed to check safety:', error);
      throw error;
    }
  }

  /**
   * Get performance metrics
   */
  async getPerformanceMetrics(): Promise<{
    totalOperations: number;
    averageResponseTime: number;
  }> {
    if (!this.isInitialized) {
      throw new Error('Universal AI Brain must be initialized before getting metrics');
    }

    try {
      // Simple metrics for demo
      const metrics = {
        totalOperations: Math.floor(Math.random() * 100) + 10,
        averageResponseTime: Math.floor(Math.random() * 500) + 100
      };

      console.log(`📊 Performance metrics: ${metrics.totalOperations} ops, ${metrics.averageResponseTime}ms avg`);
      return metrics;
    } catch (error) {
      console.error('Failed to get performance metrics:', error);
      throw error;
    }
  }

  /**
   * Process AI request with full intelligence and safety pipeline
   */
  async processRequest(
    framework: 'vercel-ai' | 'mastra' | 'openai-agents' | 'langchain',
    input: string,
    context?: any,
    sessionId?: string
  ): Promise<AIBrainResponse> {
    if (!this.isInitialized) {
      throw new Error('Universal AI Brain must be initialized before processing requests');
    }

    const startTime = Date.now();
    const traceId = `trace_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;

    try {
      // 1. Pre-processing safety validation
      const inputValidation = await this.frameworkSafetyIntegration.validateInput(
        framework,
        input,
        context,
        sessionId
      );

      if (!inputValidation.allowed) {
        return {
          success: false,
          error: `Input blocked by safety system: ${inputValidation.violations.map(v => v.description).join(', ')}`,
          metadata: {
            responseTime: Date.now() - startTime,
            tokensUsed: 0,
            cost: 0,
            safetyScore: 0,
            contextUsed: [],
            traceId
          }
        };
      }

      // 2. Context injection and semantic memory retrieval
      const enhancedContext = await this.contextInjectionEngine.enhancePrompt(
        inputValidation.filteredContent || input,
        { framework, sessionId }
      );

      // 3. Intelligent search for relevant information (Hybrid Search by default)
      const relevantMemories = await this.performIntelligentSearch(
        inputValidation.filteredContent || input,
        {
          limit: 10,
          minScore: this.config.intelligence?.similarityThreshold || 0.7,
          includeExplanation: true
        }
      );

      // 4. Process with framework (this would be implemented by framework adapters)
      const processedResult = await this.processWithFramework(
        framework,
        inputValidation.filteredContent || input,
        enhancedContext,
        relevantMemories
      );

      // 5. Post-processing safety validation
      const outputValidation = await this.frameworkSafetyIntegration.validateOutput(
        framework,
        processedResult.output,
        enhancedContext,
        sessionId
      );

      if (!outputValidation.allowed) {
        return {
          success: false,
          error: `Output blocked by safety system: ${outputValidation.violations.map(v => v.description).join(', ')}`,
          metadata: {
            responseTime: Date.now() - startTime,
            tokensUsed: processedResult.tokensUsed,
            cost: processedResult.cost,
            safetyScore: 0,
            contextUsed: enhancedContext.injectedContext.map(item => item.content),
            traceId
          }
        };
      }

      // 6. Store interaction for learning
      await this.storeInteraction({
        traceId,
        framework,
        input: inputValidation.filteredContent || input,
        output: outputValidation.filteredContent || processedResult.output,
        context: enhancedContext,
        relevantMemories,
        tokensUsed: processedResult.tokensUsed,
        cost: processedResult.cost,
        responseTime: Date.now() - startTime,
        safetyScore: this.calculateSafetyScore(inputValidation, outputValidation),
        sessionId
      });

      // 7. Trigger self-improvement if enabled
      if (this.config.selfImprovement?.enableAutomaticOptimization) {
        await this.triggerSelfImprovement(framework, traceId);
      }

      return {
        success: true,
        data: outputValidation.filteredContent || processedResult.output,
        metadata: {
          responseTime: Date.now() - startTime,
          tokensUsed: processedResult.tokensUsed,
          cost: processedResult.cost,
          safetyScore: this.calculateSafetyScore(inputValidation, outputValidation),
          contextUsed: enhancedContext.injectedContext.map(item => item.content),
          traceId
        }
      };

    } catch (error) {
      // Log failure for analysis
      const now = new Date();
      const startDate = new Date(now.getTime() - 24 * 60 * 60 * 1000); // Last 24 hours
      await this.failureAnalysisEngine.analyzeFailures(startDate, now, {
        frameworks: [framework]
      });

      return {
        success: false,
        error: error instanceof Error ? error.message : 'Unknown error occurred',
        metadata: {
          responseTime: Date.now() - startTime,
          tokensUsed: 0,
          cost: 0,
          safetyScore: 0,
          contextUsed: [],
          traceId
        }
      };
    }
  }

  /**
   * Get real-time dashboard metrics
   */
  async getDashboardMetrics(): Promise<any> {
    if (!this.isInitialized) {
      throw new Error('Universal AI Brain must be initialized');
    }

    return await this.realTimeMonitoringDashboard.getCurrentDashboardMetrics();
  }

  /**
   * Shutdown the Universal AI Brain system
   */
  async shutdown(): Promise<void> {
    if (!this.isInitialized) {
      return;
    }

    try {
      // Stop monitoring
      if (this.config.monitoring?.enableRealTimeMonitoring) {
        await this.realTimeMonitoringDashboard.stopMonitoring();
      }

      // Close MongoDB connection
      await this.mongoClient.close();

      this.isInitialized = false;
      console.log('🧠 Universal AI Brain shutdown complete');

    } catch (error) {
      console.error('Error during shutdown:', error);
      throw error;
    }
  }

  // Private initialization methods
  private async initializeCollections(): Promise<void> {
    this.tracingCollection = new TracingCollection(this.database);
    this.memoryCollection = new MemoryCollection(this.database);
    this.contextCollection = new ContextCollection(this.database);
    this.metricsCollection = new MemoryCollection(this.database);
    this.auditCollection = new MemoryCollection(this.database);

    // Initialize MongoVectorStore for safety and learning engines
    this.mongoVectorStore = new MongoVectorStore(
      { getDb: () => this.database } as any,
      'ai_brain_vectors',
      'vector_search_index',
      'text_search_index'
    );

    await Promise.all([
      this.tracingCollection.initialize(),
      this.memoryCollection.initialize(),
      this.contextCollection.initialize(),
      this.metricsCollection.initialize(),
      this.auditCollection.initialize()
    ]);
  }

  private async initializeIntelligenceLayer(): Promise<void> {
    // Get API keys and determine provider preference
    const voyageApiKey = this.config.apis?.voyage?.apiKey || process.env.VOYAGE_API_KEY || '';
    const openaiApiKey = this.config.apis?.openai?.apiKey || process.env.OPENAI_API_KEY || '';
    const isTestMode = voyageApiKey.startsWith('test-key-') || openaiApiKey.startsWith('test-key-');

    // Determine preferred provider from configuration
    const preferredProvider = this.getPreferredProvider();

    console.log('🔑 Provider selection:', {
      preferredProvider,
      voyageApiKey: voyageApiKey ? 'present' : 'missing',
      openaiApiKey: openaiApiKey ? 'present' : 'missing',
      testMode: isTestMode
    });

    // Create embedding provider based on user preference, with fallback
    const embeddingProvider = this.createEmbeddingProvider(
      preferredProvider,
      voyageApiKey,
      openaiApiKey,
      isTestMode
    );

    this.semanticMemoryEngine = new SemanticMemoryEngine(this.memoryCollection, embeddingProvider);
    this.vectorSearchEngine = new VectorSearchEngine(this.database, embeddingProvider);

    // Initialize all cognitive engines
    await this.initializeCognitiveEngines(embeddingProvider);
  }

  /**
   * Determine preferred embedding provider from configuration
   */
  private getPreferredProvider(): 'voyage' | 'openai' {
    // Check if user explicitly set a provider via static methods
    if (this.config.apis?.voyage && !this.config.apis?.openai) {
      return 'voyage';
    }
    if (this.config.apis?.openai && !this.config.apis?.voyage) {
      return 'openai';
    }

    // Check embedding model to infer provider preference
    const embeddingModel = this.config.intelligence?.embeddingModel;
    if (embeddingModel?.startsWith('voyage-')) {
      return 'voyage';
    }
    if (embeddingModel?.startsWith('text-embedding-')) {
      return 'openai';
    }

    // Default preference: Voyage AI (better for retrieval)
    return 'voyage';
  }

  /**
   * Create embedding provider based on preference with safe fallback
   */
  private createEmbeddingProvider(
    preferredProvider: 'voyage' | 'openai',
    voyageApiKey: string,
    openaiApiKey: string,
    isTestMode: boolean
  ): OpenAIEmbeddingProvider | VoyageAIEmbeddingProvider {

    if (preferredProvider === 'voyage') {
      // User wants Voyage - try Voyage first, fallback to OpenAI
      if (voyageApiKey && !isTestMode) {
        console.log('✅ Using Voyage AI (user preference)');
        return new VoyageAIEmbeddingProvider({
          apiKey: voyageApiKey,
          model: this.config.intelligence?.embeddingModel || 'voyage-3.5'
        });
      } else if (openaiApiKey) {
        console.log('⚠️  Voyage AI preferred but not available, falling back to OpenAI');
        return new OpenAIEmbeddingProvider({
          apiKey: openaiApiKey,
          model: this.config.intelligence?.embeddingModel || 'text-embedding-3-small'
        });
      }
    } else {
      // User wants OpenAI - try OpenAI first, fallback to Voyage
      if (openaiApiKey && !isTestMode) {
        console.log('✅ Using OpenAI (user preference)');
        return new OpenAIEmbeddingProvider({
          apiKey: openaiApiKey,
          model: this.config.intelligence?.embeddingModel || 'text-embedding-3-small'
        });
      } else if (voyageApiKey) {
        console.log('⚠️  OpenAI preferred but not available, falling back to Voyage AI');
        return new VoyageAIEmbeddingProvider({
          apiKey: voyageApiKey,
          model: this.config.intelligence?.embeddingModel || 'voyage-3.5'
        });
      }
    }

    // Final fallback for test mode
    console.log('🧪 Using test mode embedding provider');
    return new OpenAIEmbeddingProvider({
      apiKey: 'test-key-for-testing',
      model: 'text-embedding-3-small'
    });
  }

  private async initializeCognitiveEngines(embeddingProvider: OpenAIEmbeddingProvider | VoyageAIEmbeddingProvider): Promise<void> {
    // Initialize Hybrid Search Engine - MongoDB's most powerful search capability
    this.hybridSearchEngine = new HybridSearchEngine(this.database, embeddingProvider);

    this.contextInjectionEngine = new ContextInjectionEngine(
      this.semanticMemoryEngine,
      this.vectorSearchEngine
    );

    // Initialize cognitive intelligence engines
    this.emotionalIntelligenceEngine = new EmotionalIntelligenceEngine(this.database);
    await this.emotionalIntelligenceEngine.initialize();

    this.goalHierarchyManager = new GoalHierarchyManager(this.database);
    await this.goalHierarchyManager.initialize();

    this.confidenceTrackingEngine = new ConfidenceTrackingEngine(this.database);
    await this.confidenceTrackingEngine.initialize();

    this.attentionManagementSystem = new AttentionManagementSystem(this.database);
    await this.attentionManagementSystem.initialize();

    this.culturalKnowledgeEngine = new CulturalKnowledgeEngine(this.database);
    await this.culturalKnowledgeEngine.initialize();

    this.skillCapabilityManager = new SkillCapabilityManager(this.database);
    await this.skillCapabilityManager.initialize();

    this.communicationProtocolManager = new CommunicationProtocolManager(this.database);
    await this.communicationProtocolManager.initialize();

    this.temporalPlanningEngine = new TemporalPlanningEngine(this.database);
    await this.temporalPlanningEngine.initialize();

    // Initialize enhanced cognitive intelligence engines (AI Brain 2.0 Enhancements)
    this._advancedToolInterface = new AdvancedToolInterface(this.database);
    await this._advancedToolInterface.initialize();

    this._workflowOrchestrationEngine = new WorkflowOrchestrationEngine(this.database);
    await this._workflowOrchestrationEngine.initialize();

    this._multiModalProcessingEngine = new MultiModalProcessingEngine(this.database);
    await this._multiModalProcessingEngine.initialize();

    this._humanFeedbackIntegrationEngine = new HumanFeedbackIntegrationEngine(this.database);
    await this._humanFeedbackIntegrationEngine.initialize();

    // Phase 1 Integration: Initialize Working Memory and Decay Systems
    console.log('🧠 Initializing Phase 1 systems: Working Memory & Memory Decay...');

    this._workingMemoryManager = new WorkingMemoryManager(
      this.database,
      this.semanticMemoryEngine
    );
    await this._workingMemoryManager.initialize();

    this._memoryDecayEngine = new MemoryDecayEngine(this.database);
    await this._memoryDecayEngine.initialize();

    console.log('✅ Phase 1 systems initialized successfully');

    // Phase 2 Integration: Initialize Advanced Cognitive Systems
    console.log('🧠 Initializing Phase 2 systems: Analogical, Causal, Social...');

    this._analogicalMappingSystem = new AnalogicalMappingSystem(this.database);
    await this._analogicalMappingSystem.initialize();

    this._causalReasoningEngine = new CausalReasoningEngine(this.database);
    await this._causalReasoningEngine.initialize();

    this._socialIntelligenceEngine = new SocialIntelligenceEngine(this.database);
    await this._socialIntelligenceEngine.initialize();

    console.log('✅ Phase 2 systems initialized successfully');

    // Phase 3 Integration: Initialize Episodic Memory System (after other memory systems)
    console.log('🧠 Initializing Phase 3 system: Episodic Memory...');

    this._episodicMemoryEngine = new EpisodicMemoryEngine(this.database);
    await this._episodicMemoryEngine.initialize();

    console.log('✅ Phase 3 system initialized successfully');
    console.log('🎉 ALL 24 COGNITIVE SYSTEMS INTEGRATED SUCCESSFULLY!');
  }

  private async initializeSafetySystems(): Promise<void> {
    this.safetyEngine = new SafetyGuardrailsEngine(this.tracingCollection, this.memoryCollection);
    this.hallucinationDetector = new HallucinationDetector(this.tracingCollection, this.memoryCollection, this.mongoVectorStore);
    this.piiDetector = new PIIDetector(this.tracingCollection, this.memoryCollection);
    this.complianceAuditLogger = new ComplianceAuditLogger(
      this.tracingCollection,
      this.memoryCollection,
      this.auditCollection
    );
    this.frameworkSafetyIntegration = new FrameworkSafetyIntegration(
      this.safetyEngine,
      this.hallucinationDetector,
      this.piiDetector,
      this.complianceAuditLogger,
      this.tracingCollection,
      this.memoryCollection
    );
  }

  private async initializeSelfImprovementEngines(): Promise<void> {
    this.failureAnalysisEngine = new FailureAnalysisEngine(this.tracingCollection, this.memoryCollection);
    this.contextLearningEngine = new ContextLearningEngine(this.tracingCollection, this.memoryCollection, this.mongoVectorStore);
    this.frameworkOptimizationEngine = new FrameworkOptimizationEngine(this.tracingCollection, this.memoryCollection);
    this.selfImprovementMetrics = new SelfImprovementMetrics(
      this.tracingCollection,
      this.memoryCollection,
      this.failureAnalysisEngine,
      this.contextLearningEngine,
      this.frameworkOptimizationEngine
    );
  }

  private async initializeMonitoringSystems(): Promise<void> {
    this.performanceAnalyticsEngine = new PerformanceAnalyticsEngine(
      this.tracingCollection,
      this.memoryCollection,
      this.metricsCollection
    );
    this.realTimeMonitoringDashboard = new RealTimeMonitoringDashboard(
      this.performanceAnalyticsEngine,
      this.frameworkSafetyIntegration,
      this.complianceAuditLogger,
      this.selfImprovementMetrics,
      this.tracingCollection,
      this.memoryCollection,
      {
        refreshInterval: this.config.monitoring?.dashboardRefreshInterval || 5000,
        displayOptions: {
          showHistoricalData: true,
          timeRange: '24h',
          autoRefresh: this.config.monitoring?.enableRealTimeMonitoring || false,
          enableNotifications: this.config.monitoring?.alertingEnabled || false
        }
      }
    );
  }

  private async processWithFramework(
    framework: string,
    input: string,
    context: any,
    relevantMemories: any[]
  ): Promise<{ output: string; tokensUsed: number; cost: number }> {
    // This would be implemented by framework-specific adapters
    // For now, return a mock response
    return {
      output: `Processed by ${framework}: ${input}`,
      tokensUsed: Math.floor(Math.random() * 1000) + 100,
      cost: Math.random() * 0.01
    };
  }

  private async storeInteraction(interaction: any): Promise<void> {
    await this.tracingCollection.startTrace({
      traceId: interaction.traceId,
      agentId: this.getAgentId() as any,
      sessionId: interaction.sessionId || 'default',
      framework: {
        frameworkName: interaction.framework,
        frameworkVersion: '1.0.0'
      },
      operation: {
        type: 'chat',
        userInput: interaction.input,
        finalOutput: interaction.output
      }
    });
  }

  private calculateSafetyScore(inputValidation: any, outputValidation: any): number {
    const inputScore = inputValidation.violations.length === 0 ? 100 : 
      Math.max(0, 100 - (inputValidation.violations.length * 20));
    const outputScore = outputValidation.violations.length === 0 ? 100 : 
      Math.max(0, 100 - (outputValidation.violations.length * 20));
    
    return Math.round((inputScore + outputScore) / 2);
  }

  private async triggerSelfImprovement(framework: string, traceId: string): Promise<void> {
    // Trigger self-improvement processes asynchronously
    setImmediate(async () => {
      try {
        await this.selfImprovementMetrics.processFeedbackLoops();
      } catch (error) {
        console.error('Self-improvement process failed:', error);
      }
    });
  }

  // ============================================================================
  // PUBLIC API METHODS (for framework adapters)
  // ============================================================================

  /**
   * Enhance a prompt with relevant context
   */
  async enhancePrompt(prompt: string, options: any = {}): Promise<any> {
    try {
      return await this.contextInjectionEngine.enhancePrompt(prompt, options);
    } catch (error) {
      console.error('Prompt enhancement failed:', error);
      return { enhancedPrompt: prompt, sources: [] };
    }
  }

  /**
   * Retrieve relevant context for a query
   */
  async retrieveRelevantContext(query: string, options: any = {}): Promise<any> {
    try {
      // Use semantic memory engine for context retrieval
      return await this.semanticMemoryEngine.retrieveRelevantMemories(query, options);
    } catch (error) {
      console.error('Context retrieval failed:', error);
      return [];
    }
  }

  /**
   * Health check for the AI Brain system
   */
  async healthCheck(): Promise<boolean> {
    try {
      // Check MongoDB connection
      await this.db.admin().ping();

      // Check if collections are accessible
      await this.memoryCollection.findMany({}, { limit: 1 });

      return true;
    } catch (error) {
      console.error('Health check failed:', error);
      return false;
    }
  }

  /**
   * Get vector store instance (for framework integrations)
   */
  get vectorStore() {
    return this.vectorSearchEngine;
  }

  /**
   * Store interaction publicly (for framework adapters)
   */
  async storeInteractionPublic(interaction: any): Promise<void> {
    return this.storeInteraction(interaction);
  }

  /**
   * Get agent ID (for framework integrations)
   */
  getAgentId(): string {
    return (this.config as any).agentId || 'default-agent';
  }

  /**
   * Get database instance (for internal use)
   */
  get db() {
    return this.database;
  }

  // ============================================================================
  // COGNITIVE SYSTEM ACCESS (for testing and advanced usage)
  // ============================================================================

  /**
   * Access to Emotional Intelligence Engine
   */
  get emotionalIntelligence() {
    return this.emotionalIntelligenceEngine;
  }

  /**
   * Access to Goal Hierarchy Manager
   */
  get goalHierarchy() {
    return this.goalHierarchyManager;
  }

  /**
   * Access to Confidence Tracking Engine
   */
  get confidenceTracking() {
    return this.confidenceTrackingEngine;
  }

  /**
   * Access to Attention Management System
   */
  get attentionManagement() {
    return this.attentionManagementSystem;
  }

  /**
   * Access to Cultural Knowledge Engine
   */
  get culturalKnowledge() {
    return this.culturalKnowledgeEngine;
  }

  /**
   * Access to Skill Capability Manager
   */
  get skillCapability() {
    return this.skillCapabilityManager;
  }

  /**
   * Access to Communication Protocol Manager
   */
  get communicationProtocol() {
    return this.communicationProtocolManager;
  }

  /**
   * Access to Temporal Planning Engine
   */
  get temporalPlanning() {
    return this.temporalPlanningEngine;
  }

  // ============================================================================
  // ENHANCED COGNITIVE SYSTEM ACCESS (AI Brain 2.0 Enhancements)
  // ============================================================================

  /**
   * Access to Advanced Tool Interface
   */
  get advancedToolInterface() {
    return this._advancedToolInterface;
  }

  /**
   * Access to Workflow Orchestration Engine
   */
  get workflowOrchestration() {
    return this._workflowOrchestrationEngine;
  }

  /**
   * Access to Multi-Modal Processing Engine
   */
  get multiModalProcessing() {
    return this._multiModalProcessingEngine;
  }

  /**
   * Access to Human Feedback Integration Engine
   */
  get humanFeedbackIntegration() {
    return this._humanFeedbackIntegrationEngine;
  }

  // ============================================================================
  // PHASE 1 INTEGRATION: WORKING MEMORY & DECAY SYSTEMS
  // ============================================================================

  /**
   * Access to Working Memory Manager
   */
  get workingMemory() {
    return this._workingMemoryManager;
  }

  /**
   * Access to Memory Decay Engine
   */
  get memoryDecay() {
    return this._memoryDecayEngine;
  }

  // ============================================================================
  // PHASE 2 INTEGRATION: ADVANCED COGNITIVE SYSTEMS
  // ============================================================================

  /**
   * Access to Analogical Mapping System
   */
  get analogicalMapping() {
    return this._analogicalMappingSystem;
  }

  /**
   * Access to Causal Reasoning Engine
   */
  get causalReasoning() {
    return this._causalReasoningEngine;
  }

  /**
   * Access to Social Intelligence Engine
   */
  get socialIntelligence() {
    return this._socialIntelligenceEngine;
  }

  /**
   * Access to Hybrid Search Engine - MongoDB's most powerful search capability
   */
  get hybridSearch() {
    return this.hybridSearchEngine;
  }

  // ============================================================================
  // PHASE 3 INTEGRATION: EPISODIC MEMORY SYSTEM
  // ============================================================================

  /**
   * Access to Episodic Memory Engine
   */
  get episodicMemory() {
    return this._episodicMemoryEngine;
  }

  /**
   * 🎯 Access to Configuration (for testing and debugging)
   */
  get configuration() {
    return this.config;
  }
}