sf-agent-framework/tools/lib/context-memory-system.js

AI Agent Orchestration Framework for Salesforce Development - Two-phase architecture with 70% context reduction

/**
 * Context Memory System
 * Provides persistent context storage across sessions with learning capabilities
 */

const fs = require('fs').promises;
const path = require('path');
const crypto = require('crypto');

class ContextMemorySystem {
  constructor(rootDir = process.cwd()) {
    this.rootDir = rootDir;
    this.memoryDir = path.join(rootDir, '.sf-agent-memory');
    this.memoryIndex = null;
    this.currentSession = null;
    this.shortTermMemory = new Map();
    this.workingMemory = new Map();
  }

  /**
   * Initialize the memory system
   */
  async initialize() {
    // Ensure memory directory exists
    await fs.mkdir(this.memoryDir, { recursive: true });

    // Load memory index
    await this.loadMemoryIndex();

    // Start new session
    this.currentSession = {
      id: this.generateSessionId(),
      started_at: new Date().toISOString(),
      contexts: [],
      learned_patterns: [],
      agent_interactions: [],
    };
  }

  /**
   * Load or create memory index
   */
  async loadMemoryIndex() {
    const indexPath = path.join(this.memoryDir, 'memory-index.json');

    try {
      const content = await fs.readFile(indexPath, 'utf8');
      this.memoryIndex = JSON.parse(content);
    } catch (error) {
      // Create new index if doesn't exist
      this.memoryIndex = {
        version: '1.0.0',
        created_at: new Date().toISOString(),
        sessions: [],
        patterns: {},
        knowledge_base: {},
        statistics: {
          total_sessions: 0,
          total_contexts: 0,
          patterns_learned: 0,
        },
      };
      await this.saveMemoryIndex();
    }
  }

  /**
   * Save memory index
   */
  async saveMemoryIndex() {
    const indexPath = path.join(this.memoryDir, 'memory-index.json');
    await fs.writeFile(indexPath, JSON.stringify(this.memoryIndex, null, 2));
  }

  /**
   * Store context in memory
   */
  async storeContext(context) {
    const contextId = this.generateContextId();

    const memoryContext = {
      id: contextId,
      session_id: this.currentSession.id,
      timestamp: new Date().toISOString(),
      type: context.type || 'general',
      agent: context.agent,
      content: context.content,
      metadata: context.metadata || {},
      tokens: context.tokens || 0,
      importance: this.calculateImportance(context),
      embeddings: await this.generateEmbeddings(context.content),
    };

    // Store in short-term memory
    this.shortTermMemory.set(contextId, memoryContext);

    // Add to current session
    this.currentSession.contexts.push(contextId);

    // Check for patterns
    const patterns = await this.extractPatterns(memoryContext);
    if (patterns.length > 0) {
      await this.learnPatterns(patterns);
    }

    // Consolidate if needed
    if (this.shortTermMemory.size > 100) {
      await this.consolidateMemory();
    }

    return contextId;
  }

  /**
   * Retrieve relevant context
   */
  async retrieveContext(query, options = {}) {
    const limit = options.limit || 10;
    const minImportance = options.minImportance || 0.5;
    const sessionOnly = options.sessionOnly || false;

    const contexts = [];

    // Search in working memory first
    for (const [id, context] of this.workingMemory) {
      if (this.isRelevant(context, query) && context.importance >= minImportance) {
        contexts.push(context);
      }
    }

    // Search in short-term memory
    for (const [id, context] of this.shortTermMemory) {
      if (this.isRelevant(context, query) && context.importance >= minImportance) {
        contexts.push(context);
      }
    }

    // Search in long-term memory if not session-only
    if (!sessionOnly) {
      const longTermContexts = await this.searchLongTermMemory(query, limit);
      contexts.push(...longTermContexts);
    }

    // Sort by relevance and importance
    contexts.sort((a, b) => {
      const scoreA = this.calculateRelevanceScore(a, query) * a.importance;
      const scoreB = this.calculateRelevanceScore(b, query) * b.importance;
      return scoreB - scoreA;
    });

    return contexts.slice(0, limit);
  }

  /**
   * Learn from interactions
   */
  async learnFromInteraction(interaction) {
    const learning = {
      timestamp: new Date().toISOString(),
      type: interaction.type,
      input: interaction.input,
      output: interaction.output,
      success: interaction.success,
      feedback: interaction.feedback,
    };

    // Extract patterns from successful interactions
    if (interaction.success) {
      const pattern = {
        trigger: interaction.input,
        response: interaction.output,
        context: interaction.context,
        frequency: 1,
      };

      await this.learnPatterns([pattern]);
    }

    // Store in current session
    this.currentSession.agent_interactions.push(learning);

    // Update statistics
    this.memoryIndex.statistics.total_contexts++;
  }

  /**
   * Extract patterns from context
   */
  async extractPatterns(context) {
    const patterns = [];

    // Look for common sequences
    if (context.type === 'workflow') {
      const workflowPattern = {
        type: 'workflow_sequence',
        steps: context.content.steps,
        success_rate: context.metadata.success_rate || 1.0,
      };
      patterns.push(workflowPattern);
    }

    // Look for agent collaboration patterns
    if (context.type === 'handoff') {
      const handoffPattern = {
        type: 'agent_collaboration',
        from_agent: context.metadata.from_agent,
        to_agent: context.metadata.to_agent,
        artifact_types: context.metadata.artifacts,
      };
      patterns.push(handoffPattern);
    }

    // Look for solution patterns
    if (context.type === 'solution') {
      const solutionPattern = {
        type: 'problem_solution',
        problem: context.metadata.problem,
        solution: context.content,
        effectiveness: context.metadata.effectiveness || 1.0,
      };
      patterns.push(solutionPattern);
    }

    return patterns;
  }

  /**
   * Learn and store patterns
   */
  async learnPatterns(patterns) {
    for (const pattern of patterns) {
      const patternKey = this.generatePatternKey(pattern);

      if (this.memoryIndex.patterns[patternKey]) {
        // Update existing pattern
        this.memoryIndex.patterns[patternKey].frequency++;
        this.memoryIndex.patterns[patternKey].last_seen = new Date().toISOString();
        this.memoryIndex.patterns[patternKey].instances.push({
          session_id: this.currentSession.id,
          timestamp: new Date().toISOString(),
        });
      } else {
        // Create new pattern
        this.memoryIndex.patterns[patternKey] = {
          pattern,
          frequency: 1,
          first_seen: new Date().toISOString(),
          last_seen: new Date().toISOString(),
          instances: [
            {
              session_id: this.currentSession.id,
              timestamp: new Date().toISOString(),
            },
          ],
        };
        this.memoryIndex.statistics.patterns_learned++;
      }
    }

    // Save patterns periodically
    await this.saveMemoryIndex();
  }

  /**
   * Consolidate short-term memory to long-term
   */
  async consolidateMemory() {
    const consolidated = [];

    // Group related contexts
    const groups = this.groupRelatedContexts();

    for (const group of groups) {
      const consolidatedContext = {
        id: this.generateContextId(),
        type: 'consolidated',
        timestamp: new Date().toISOString(),
        contexts: group.map((c) => c.id),
        summary: await this.summarizeContexts(group),
        importance: Math.max(...group.map((c) => c.importance)),
        tokens: Math.floor(group.reduce((sum, c) => sum + c.tokens, 0) / 2), // Assume 50% compression
      };

      consolidated.push(consolidatedContext);

      // Save to long-term storage
      await this.saveLongTermMemory(consolidatedContext);
    }

    // Clear consolidated contexts from short-term memory
    for (const context of consolidated) {
      for (const contextId of context.contexts) {
        this.shortTermMemory.delete(contextId);
      }
    }

    return consolidated;
  }

  /**
   * Group related contexts for consolidation
   */
  groupRelatedContexts() {
    const groups = [];
    const processed = new Set();

    for (const [id, context] of this.shortTermMemory) {
      if (processed.has(id)) continue;

      const group = [context];
      processed.add(id);

      // Find related contexts
      for (const [otherId, otherContext] of this.shortTermMemory) {
        if (processed.has(otherId)) continue;

        if (this.areContextsRelated(context, otherContext)) {
          group.push(otherContext);
          processed.add(otherId);
        }
      }

      if (group.length > 1) {
        groups.push(group);
      }
    }

    return groups;
  }

  /**
   * Check if two contexts are related
   */
  areContextsRelated(context1, context2) {
    // Same agent
    if (context1.agent === context2.agent) return true;

    // Same type
    if (context1.type === context2.type) return true;

    // Similar content (simplified similarity check)
    const similarity = this.calculateSimilarity(context1.content, context2.content);
    if (similarity > 0.7) return true;

    // Time proximity (within 5 minutes)
    const timeDiff = Math.abs(new Date(context1.timestamp) - new Date(context2.timestamp));
    if (timeDiff < 5 * 60 * 1000) return true;

    return false;
  }

  /**
   * Summarize multiple contexts
   */
  async summarizeContexts(contexts) {
    // Simple summarization - in production, use LLM
    const summary = {
      agent: contexts[0].agent,
      type: contexts[0].type,
      main_points: contexts.map((c) => c.content.substring(0, 100)),
      context_count: contexts.length,
      time_span: {
        start: contexts[0].timestamp,
        end: contexts[contexts.length - 1].timestamp,
      },
    };

    return summary;
  }

  /**
   * Save to long-term memory
   */
  async saveLongTermMemory(context) {
    const sessionDir = path.join(this.memoryDir, this.currentSession.id);
    await fs.mkdir(sessionDir, { recursive: true });

    const contextPath = path.join(sessionDir, `${context.id}.json`);
    await fs.writeFile(contextPath, JSON.stringify(context, null, 2));

    // Update index
    if (!this.memoryIndex.sessions.includes(this.currentSession.id)) {
      this.memoryIndex.sessions.push(this.currentSession.id);
    }
  }

  /**
   * Search long-term memory
   */
  async searchLongTermMemory(query, limit = 10) {
    const results = [];

    // Search through all sessions
    for (const sessionId of this.memoryIndex.sessions) {
      const sessionDir = path.join(this.memoryDir, sessionId);

      try {
        const files = await fs.readdir(sessionDir);

        for (const file of files) {
          if (!file.endsWith('.json')) continue;

          const contextPath = path.join(sessionDir, file);
          const content = await fs.readFile(contextPath, 'utf8');
          const context = JSON.parse(content);

          if (this.isRelevant(context, query)) {
            results.push(context);

            if (results.length >= limit * 2) {
              break; // Get more than needed for better sorting
            }
          }
        }
      } catch (error) {
        console.warn(`Failed to search session ${sessionId}:`, error.message);
      }
    }

    return results;
  }

  /**
   * Check if context is relevant to query
   */
  isRelevant(context, query) {
    const queryLower = query.toLowerCase();
    const contentStr = JSON.stringify(context).toLowerCase();

    // Simple keyword matching - in production, use embeddings
    const keywords = queryLower.split(' ').filter((k) => k.length > 2);
    const matches = keywords.filter((k) => contentStr.includes(k));

    return matches.length > keywords.length / 2;
  }

  /**
   * Calculate relevance score
   */
  calculateRelevanceScore(context, query) {
    // Simplified scoring - in production, use vector similarity
    const queryLower = query.toLowerCase();
    const contentStr = JSON.stringify(context).toLowerCase();

    const keywords = queryLower.split(' ').filter((k) => k.length > 2);
    const matches = keywords.filter((k) => contentStr.includes(k));

    return matches.length / keywords.length;
  }

  /**
   * Calculate context importance
   */
  calculateImportance(context) {
    let importance = 0.5; // Base importance

    // Adjust based on type
    const typeWeights = {
      solution: 0.9,
      error: 0.8,
      decision: 0.8,
      handoff: 0.7,
      workflow: 0.7,
      general: 0.5,
    };

    importance = typeWeights[context.type] || importance;

    // Adjust based on metadata
    if (context.metadata?.critical) importance = Math.max(importance, 0.9);
    if (context.metadata?.success === false) importance = Math.max(importance, 0.8);

    return importance;
  }

  /**
   * Calculate similarity between two texts
   */
  calculateSimilarity(text1, text2) {
    // Simplified Jaccard similarity
    const words1 = new Set(text1.toLowerCase().split(' '));
    const words2 = new Set(text2.toLowerCase().split(' '));

    const intersection = new Set([...words1].filter((x) => words2.has(x)));
    const union = new Set([...words1, ...words2]);

    return intersection.size / union.size;
  }

  /**
   * Generate embeddings for content (placeholder)
   */
  async generateEmbeddings(content) {
    // In production, use actual embedding model
    // For now, return simplified feature vector
    const words = content.toLowerCase().split(' ');
    const features = {
      length: content.length,
      word_count: words.length,
      unique_words: new Set(words).size,
      hash: crypto.createHash('md5').update(content).digest('hex'),
    };

    return features;
  }

  /**
   * Generate pattern key
   */
  generatePatternKey(pattern) {
    const keyData = `${pattern.type}-${JSON.stringify(pattern)}`;
    return crypto.createHash('md5').update(keyData).digest('hex');
  }

  /**
   * End session and save
   */
  async endSession() {
    if (!this.currentSession) return;

    this.currentSession.ended_at = new Date().toISOString();

    // Save session metadata
    const sessionPath = path.join(this.memoryDir, `${this.currentSession.id}`, 'session.json');

    await fs.mkdir(path.dirname(sessionPath), { recursive: true });
    await fs.writeFile(sessionPath, JSON.stringify(this.currentSession, null, 2));

    // Consolidate remaining memory
    await this.consolidateMemory();

    // Update statistics
    this.memoryIndex.statistics.total_sessions++;
    await this.saveMemoryIndex();

    // Clear working memory
    this.workingMemory.clear();
    this.shortTermMemory.clear();

    return this.currentSession;
  }

  /**
   * Get memory statistics
   */
  getMemoryStats() {
    return {
      current_session: this.currentSession?.id,
      working_memory_size: this.workingMemory.size,
      short_term_memory_size: this.shortTermMemory.size,
      total_sessions: this.memoryIndex.statistics.total_sessions,
      total_contexts: this.memoryIndex.statistics.total_contexts,
      patterns_learned: this.memoryIndex.statistics.patterns_learned,
      pattern_types: Object.keys(this.memoryIndex.patterns).length,
    };
  }

  /**
   * Get learned patterns
   */
  getLearnedPatterns(type = null) {
    const patterns = [];

    for (const [key, data] of Object.entries(this.memoryIndex.patterns)) {
      if (!type || data.pattern.type === type) {
        patterns.push({
          key,
          ...data,
        });
      }
    }

    // Sort by frequency
    patterns.sort((a, b) => b.frequency - a.frequency);

    return patterns;
  }

  /**
   * Apply learned patterns to new situation
   */
  async applyPatterns(situation) {
    const applicablePatterns = [];

    for (const [key, data] of Object.entries(this.memoryIndex.patterns)) {
      if (this.isPatternApplicable(data.pattern, situation)) {
        applicablePatterns.push(data.pattern);
      }
    }

    return applicablePatterns;
  }

  /**
   * Check if pattern is applicable
   */
  isPatternApplicable(pattern, situation) {
    // Match pattern type with situation type
    if (pattern.type !== situation.type) return false;

    // Additional checks based on pattern type
    switch (pattern.type) {
      case 'workflow_sequence':
        return situation.workflow === pattern.workflow;

      case 'agent_collaboration':
        return situation.from_agent === pattern.from_agent;

      case 'problem_solution':
        return this.calculateSimilarity(situation.problem, pattern.problem) > 0.7;

      default:
        return false;
    }
  }

  /**
   * Generate unique IDs
   */
  generateSessionId() {
    return `session-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`;
  }

  generateContextId() {
    return `context-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`;
  }
}

module.exports = ContextMemorySystem;