@boundless-oss/atlas/src/modules/memory-management/memory-manager.ts

Atlas - MCP Server for comprehensive startup project management

import { ConfigManager } from '../../config/config-manager.js';
import { 
  MemoryEntry, 
  MemoryQuery, 
  MemorySearchResult, 
  MemoryStats,
  MemoryExportOptions,
  MemoryExportResult,
  MemoryClearOptions,
  SemanticIndex,
  MemoryGraph,
  MemoryInsight
} from './types.js';
import { promises as fs } from 'fs';
import path from 'path';
import { randomUUID } from 'crypto';

export class MemoryManager {
  private configManager: ConfigManager;
  private memoryPath: string;
  private indexPath: string;
  private memories: Map<string, MemoryEntry>;
  private semanticIndex: SemanticIndex;
  private memoryGraph: MemoryGraph;

  constructor(configManager: ConfigManager) {
    this.configManager = configManager;
    this.memoryPath = '';
    this.indexPath = '';
    this.memories = new Map();
    this.semanticIndex = {
      entries: new Map(),
      lastUpdated: new Date().toISOString(),
    };
    this.memoryGraph = {
      nodes: new Map(),
      edges: new Map(),
    };
  }

  async init(): Promise<void> {
    return this.initialize();
  }

  async initialize(): Promise<void> {
    const storageManager = this.configManager.getStorageManager();
    const location = await storageManager.getStorageLocation();
    
    this.memoryPath = path.join(location.data, 'memory', 'entries.json');
    this.indexPath = path.join(location.data, 'memory', 'index.json');

    // Ensure memory directory exists
    await fs.mkdir(path.dirname(this.memoryPath), { recursive: true });

    // Load existing memories
    await this.loadMemories();
    await this.loadIndex();
  }

  async store(entry: Omit<MemoryEntry, 'id' | 'timestamp'>): Promise<string> {
    const id = randomUUID();
    const memoryEntry: MemoryEntry = {
      ...entry,
      id,
      timestamp: new Date().toISOString(),
    };

    // Store in memory
    this.memories.set(id, memoryEntry);

    // Generate embedding for semantic search
    await this.generateEmbedding(memoryEntry);

    // Update graph relationships
    await this.updateGraph(memoryEntry);

    // Save to disk
    await this.saveMemories();
    await this.saveIndex();

    return id;
  }

  async search(query: MemoryQuery): Promise<MemorySearchResult[]> {
    const results: MemorySearchResult[] = [];

    // Simple text-based search (can be enhanced with embeddings)
    for (const [id, memory] of this.memories) {
      let score = 0;

      // Calculate relevance score
      score += this.calculateTextRelevance(query.query, memory.content);
      
      // Type filter
      if (query.type && memory.type !== query.type) {
        continue;
      }

      // Tags filter
      if (query.tags && query.tags.length > 0) {
        const tagMatch = query.tags.some(tag => memory.tags.includes(tag));
        if (!tagMatch) continue;
        score += 0.2; // Bonus for tag match
      }

      // Importance boost
      const importanceBoost = {
        critical: 0.4,
        high: 0.3,
        medium: 0.2,
        low: 0.1,
      };
      score += importanceBoost[memory.importance];

      // Recency boost (more recent = higher score)
      const ageInDays = (Date.now() - new Date(memory.timestamp).getTime()) / (1000 * 60 * 60 * 24);
      const recencyBoost = Math.max(0, 0.2 - (ageInDays * 0.01));
      score += recencyBoost;

      if (score > (query.minScore || 0.1)) {
        results.push({
          id,
          content: memory.content,
          type: memory.type,
          tags: memory.tags,
          importance: memory.importance,
          timestamp: memory.timestamp,
          score,
          context: query.includeContext ? this.getContext(memory) : undefined,
        });
      }
    }

    // Sort by score and limit results
    results.sort((a, b) => b.score - a.score);
    return results.slice(0, query.limit || 10);
  }

  async getStats(): Promise<MemoryStats> {
    const memories = Array.from(this.memories.values());
    
    const byType: Record<string, number> = {};
    const byImportance: Record<string, number> = {};
    const tagCounts: Record<string, number> = {};

    memories.forEach(memory => {
      byType[memory.type] = (byType[memory.type] || 0) + 1;
      byImportance[memory.importance] = (byImportance[memory.importance] || 0) + 1;
      
      memory.tags.forEach(tag => {
        tagCounts[tag] = (tagCounts[tag] || 0) + 1;
      });
    });

    const now = new Date();
    const today = new Date(now.getFullYear(), now.getMonth(), now.getDate());
    const thisWeek = new Date(today.getTime() - (7 * 24 * 60 * 60 * 1000));

    const todayCount = memories.filter(m => new Date(m.timestamp) >= today).length;
    const thisWeekCount = memories.filter(m => new Date(m.timestamp) >= thisWeek).length;

    const topTags = Object.entries(tagCounts)
      .sort(([, a], [, b]) => b - a)
      .slice(0, 10)
      .map(([name, count]) => ({ name, count }));

    const storageSize = this.calculateStorageSize();

    return {
      totalEntries: memories.length,
      storageSize,
      lastUpdated: new Date().toISOString(),
      byType,
      byImportance,
      recentActivity: {
        today: todayCount,
        thisWeek: thisWeekCount,
        mostActiveDay: this.getMostActiveDay(memories),
      },
      topTags,
    };
  }

  async clear(options: MemoryClearOptions): Promise<number> {
    let deletedCount = 0;
    const toDelete: string[] = [];

    for (const [id, memory] of this.memories) {
      let shouldDelete = true;

      if (options.type && memory.type !== options.type) {
        shouldDelete = false;
      }

      if (options.olderThan) {
        const cutoffDate = new Date(options.olderThan);
        const memoryDate = new Date(memory.timestamp);
        if (memoryDate >= cutoffDate) {
          shouldDelete = false;
        }
      }

      if (shouldDelete) {
        toDelete.push(id);
      }
    }

    // Delete memories
    for (const id of toDelete) {
      this.memories.delete(id);
      this.semanticIndex.entries.delete(id);
      this.memoryGraph.nodes.delete(id);
      this.memoryGraph.edges.delete(id);
      deletedCount++;
    }

    // Save changes
    await this.saveMemories();
    await this.saveIndex();

    return deletedCount;
  }

  async export(options: MemoryExportOptions): Promise<MemoryExportResult> {
    const memories = Array.from(this.memories.values());
    let data: any;
    let content: string;

    switch (options.format) {
      case 'json':
        data = {
          memories: options.includeMetadata ? memories : memories.map(m => ({
            id: m.id,
            content: m.content,
            type: m.type,
            tags: m.tags,
            timestamp: m.timestamp,
          })),
          exported: new Date().toISOString(),
          totalEntries: memories.length,
        };
        content = JSON.stringify(data, null, 2);
        break;

      case 'markdown':
        content = this.generateMarkdownExport(memories, options.includeMetadata);
        break;

      case 'csv':
        content = this.generateCSVExport(memories, options.includeMetadata);
        break;

      default:
        throw new Error(`Unsupported export format: ${options.format}`);
    }

    const size = Buffer.byteLength(content, 'utf8');

    if (options.outputPath) {
      await fs.writeFile(options.outputPath, content);
      return {
        outputPath: options.outputPath,
        entryCount: memories.length,
        size,
      };
    } else {
      return {
        data,
        entryCount: memories.length,
        size,
      };
    }
  }

  async suggestRelated(options: {
    currentContext: string;
    type?: string;
    limit?: number;
  }): Promise<MemorySearchResult[]> {
    const query: MemoryQuery = {
      query: options.currentContext,
      type: options.type,
      limit: options.limit || 5,
      minScore: 0.3, // Higher threshold for suggestions
    };

    return this.search(query);
  }

  async generateInsights(): Promise<MemoryInsight[]> {
    const insights: MemoryInsight[] = [];
    const memories = Array.from(this.memories.values());

    // Pattern detection
    const patterns = this.detectPatterns(memories);
    insights.push(...patterns);

    // Gap analysis
    const gaps = this.detectGaps(memories);
    insights.push(...gaps);

    // Clustering analysis
    const clusters = this.detectClusters(memories);
    insights.push(...clusters);

    return insights;
  }

  private async loadMemories(): Promise<void> {
    try {
      const data = await fs.readFile(this.memoryPath, 'utf-8');
      const memoriesArray: MemoryEntry[] = JSON.parse(data);
      
      this.memories.clear();
      for (const memory of memoriesArray) {
        this.memories.set(memory.id, memory);
      }
    } catch (error) {
      // File doesn't exist or is invalid, start with empty memories
      this.memories.clear();
    }
  }

  private async saveMemories(): Promise<void> {
    const memoriesArray = Array.from(this.memories.values());
    await fs.writeFile(this.memoryPath, JSON.stringify(memoriesArray, null, 2));
  }

  private async loadIndex(): Promise<void> {
    try {
      const data = await fs.readFile(this.indexPath, 'utf-8');
      const indexData = JSON.parse(data);
      
      this.semanticIndex = {
        entries: new Map(indexData.entries || []),
        lastUpdated: indexData.lastUpdated || new Date().toISOString(),
      };
    } catch (error) {
      // Index doesn't exist, start fresh
    }
  }

  private async saveIndex(): Promise<void> {
    const indexData = {
      entries: Array.from(this.semanticIndex.entries.entries()),
      lastUpdated: this.semanticIndex.lastUpdated,
    };
    
    await fs.writeFile(this.indexPath, JSON.stringify(indexData, null, 2));
  }

  private async generateEmbedding(memory: MemoryEntry): Promise<void> {
    // Simple word-based embedding (in real implementation, use proper embeddings)
    const words = memory.content.toLowerCase().split(/\s+/);
    const embedding = new Array(100).fill(0);
    
    // Simple hash-based embedding for demonstration
    for (let i = 0; i < words.length; i++) {
      const hash = this.simpleHash(words[i]);
      const index = Math.abs(hash) % embedding.length;
      embedding[index] += 1;
    }
    
    // Normalize
    const norm = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
    if (norm > 0) {
      for (let i = 0; i < embedding.length; i++) {
        embedding[i] /= norm;
      }
    }
    
    this.semanticIndex.entries.set(memory.id, embedding);
    this.semanticIndex.lastUpdated = new Date().toISOString();
  }

  private async updateGraph(memory: MemoryEntry): Promise<void> {
    // Update memory graph with new node
    this.memoryGraph.nodes.set(memory.id, {
      id: memory.id,
      type: memory.type,
      importance: this.getImportanceScore(memory.importance),
      connections: 0,
      lastAccessed: memory.timestamp,
    });

    // Find and create edges to related memories
    const related = await this.findRelatedMemories(memory);
    const edges: any[] = [];
    
    for (const relatedId of related) {
      edges.push({
        from: memory.id,
        to: relatedId,
        relationship: 'relates_to',
        strength: 0.5, // Can be calculated based on similarity
      });
    }
    
    if (edges.length > 0) {
      this.memoryGraph.edges.set(memory.id, edges);
    }
  }

  private calculateTextRelevance(query: string, content: string): number {
    const queryWords = query.toLowerCase().split(/\s+/);
    const contentWords = content.toLowerCase().split(/\s+/);
    
    let matches = 0;
    for (const queryWord of queryWords) {
      if (contentWords.some(word => word.includes(queryWord) || queryWord.includes(word))) {
        matches++;
      }
    }
    
    return matches / queryWords.length;
  }

  private getContext(memory: MemoryEntry): string {
    // Get related memories as context
    const related = this.memoryGraph.edges.get(memory.id) || [];
    if (related.length === 0) return '';
    
    const relatedMemories = related
      .slice(0, 3)
      .map(edge => this.memories.get(edge.to))
      .filter(Boolean)
      .map(m => m!.content.substring(0, 100))
      .join(' ... ');
    
    return `Related context: ${relatedMemories}`;
  }

  private calculateStorageSize(): number {
    return Buffer.byteLength(JSON.stringify(Array.from(this.memories.values())), 'utf8');
  }

  private getMostActiveDay(memories: MemoryEntry[]): string {
    const dayTallies: Record<string, number> = {};
    
    memories.forEach(memory => {
      const day = new Date(memory.timestamp).toDateString();
      dayTallies[day] = (dayTallies[day] || 0) + 1;
    });
    
    const mostActive = Object.entries(dayTallies)
      .sort(([, a], [, b]) => b - a)[0];
    
    return mostActive ? mostActive[0] : 'No activity yet';
  }

  private generateMarkdownExport(memories: MemoryEntry[], includeMetadata: boolean): string {
    let content = '# Memory Export\n\n';
    content += `Exported: ${new Date().toISOString()}\n`;
    content += `Total Entries: ${memories.length}\n\n`;

    memories.forEach((memory, index) => {
      content += `## ${index + 1}. ${memory.type.toUpperCase()} - ${memory.importance}\n\n`;
      content += `${memory.content}\n\n`;
      
      if (memory.tags.length > 0) {
        content += `**Tags:** ${memory.tags.join(', ')}\n\n`;
      }
      
      if (includeMetadata) {
        content += `**Timestamp:** ${memory.timestamp}\n`;
        content += `**ID:** ${memory.id}\n`;
        if (Object.keys(memory.metadata).length > 0) {
          content += `**Metadata:** ${JSON.stringify(memory.metadata)}\n`;
        }
      }
      
      content += '\n---\n\n';
    });

    return content;
  }

  private generateCSVExport(memories: MemoryEntry[], includeMetadata: boolean): string {
    const headers = ['ID', 'Type', 'Content', 'Tags', 'Importance', 'Timestamp'];
    if (includeMetadata) {
      headers.push('Metadata');
    }
    
    let content = headers.join(',') + '\n';
    
    memories.forEach(memory => {
      const row = [
        memory.id,
        memory.type,
        `"${memory.content.replace(/"/g, '""')}"`,
        `"${memory.tags.join(', ')}"`,
        memory.importance,
        memory.timestamp,
      ];
      
      if (includeMetadata) {
        row.push(`"${JSON.stringify(memory.metadata).replace(/"/g, '""')}"`);
      }
      
      content += row.join(',') + '\n';
    });
    
    return content;
  }

  private async findRelatedMemories(memory: MemoryEntry): Promise<string[]> {
    const related: string[] = [];
    
    // Find memories with shared tags
    for (const [id, otherMemory] of this.memories) {
      if (id === memory.id) continue;
      
      const sharedTags = memory.tags.filter(tag => otherMemory.tags.includes(tag));
      if (sharedTags.length > 0) {
        related.push(id);
      }
    }
    
    return related.slice(0, 5); // Limit to 5 related memories
  }

  private getImportanceScore(importance: string): number {
    const scores = { low: 1, medium: 2, high: 3, critical: 4 };
    return scores[importance as keyof typeof scores] || 2;
  }

  private simpleHash(str: string): number {
    let hash = 0;
    for (let i = 0; i < str.length; i++) {
      const char = str.charCodeAt(i);
      hash = ((hash << 5) - hash) + char;
      hash = hash & hash; // Convert to 32-bit integer
    }
    return hash;
  }

  private detectPatterns(memories: MemoryEntry[]): MemoryInsight[] {
    const insights: MemoryInsight[] = [];
    
    // Detect frequent types
    const typeCounts = memories.reduce((acc, memory) => {
      acc[memory.type] = (acc[memory.type] || 0) + 1;
      return acc;
    }, {} as Record<string, number>);
    
    const dominantType = Object.entries(typeCounts)
      .sort(([, a], [, b]) => b - a)[0];
    
    if (dominantType && dominantType[1] > memories.length * 0.5) {
      insights.push({
        type: 'pattern',
        title: `Dominant Memory Type: ${dominantType[0]}`,
        description: `${dominantType[1]} out of ${memories.length} memories are of type "${dominantType[0]}"`,
        confidence: 0.8,
        evidence: [`${Math.round((dominantType[1] / memories.length) * 100)}% of memories are ${dominantType[0]} type`],
        recommendations: [`Consider diversifying memory types`, `Create templates for ${dominantType[0]} entries`],
      });
    }
    
    return insights;
  }

  private detectGaps(memories: MemoryEntry[]): MemoryInsight[] {
    const insights: MemoryInsight[] = [];
    
    // Check for missing types
    const expectedTypes = ['code', 'documentation', 'decision', 'learning', 'context'];
    const existingTypes = new Set(memories.map(m => m.type));
    const missingTypes = expectedTypes.filter((type: string) => !existingTypes.has(type as 'code' | 'documentation' | 'decision' | 'learning' | 'context'));
    
    if (missingTypes.length > 0) {
      insights.push({
        type: 'gap',
        title: 'Missing Memory Types',
        description: `The following memory types are not represented: ${missingTypes.join(', ')}`,
        confidence: 0.9,
        evidence: [`No memories found of types: ${missingTypes.join(', ')}`],
        recommendations: missingTypes.map(type => `Consider storing ${type} related information`),
      });
    }
    
    return insights;
  }

  private detectClusters(memories: MemoryEntry[]): MemoryInsight[] {
    const insights: MemoryInsight[] = [];
    
    // Simple tag-based clustering
    const tagGroups: Record<string, string[]> = {};
    
    memories.forEach(memory => {
      memory.tags.forEach(tag => {
        if (!tagGroups[tag]) tagGroups[tag] = [];
        tagGroups[tag].push(memory.id);
      });
    });
    
    const largeClusters = Object.entries(tagGroups)
      .filter(([, ids]) => ids.length >= 3)
      .sort(([, a], [, b]) => b.length - a.length);
    
    if (largeClusters.length > 0) {
      const [tag, ids] = largeClusters[0];
      insights.push({
        type: 'cluster',
        title: `Large Memory Cluster: ${tag}`,
        description: `${ids.length} memories are tagged with "${tag}"`,
        confidence: 0.7,
        evidence: [`${ids.length} memories share the tag "${tag}"`],
        recommendations: [`Review and organize memories tagged with "${tag}"`, `Consider creating sub-categories for this cluster`],
      });
    }
    
    return insights;
  }
}