@claude-vector/claude-tools/src/context-manager.js

Claude integration tools for AI-powered development assistance

/**
 * Context Manager for Claude Vector Search
 * Manages context window optimization and token calculation
 * ESM version converted from CommonJS original
 */

import { EmbeddingGenerator, VectorSearchEngine } from '@claude-vector/core';

/**
 * Simple LRU Cache implementation
 */
class LRUCache {
  constructor(maxSize = 1000) {
    this.maxSize = maxSize;
    this.cache = new Map();
  }
  
  get(key) {
    if (!this.cache.has(key)) return null;
    
    // Move to end (most recently used)
    const value = this.cache.get(key);
    this.cache.delete(key);
    this.cache.set(key, value);
    return value;
  }
  
  set(key, value) {
    // Remove if exists
    if (this.cache.has(key)) {
      this.cache.delete(key);
    }
    
    // Add to end
    this.cache.set(key, value);
    
    // Remove oldest if over limit
    if (this.cache.size > this.maxSize) {
      const firstKey = this.cache.keys().next().value;
      this.cache.delete(firstKey);
    }
  }
  
  has(key) {
    return this.cache.has(key);
  }
  
  clear() {
    this.cache.clear();
  }
  
  get size() {
    return this.cache.size;
  }
}

export class ContextManager {
  constructor(maxTokens = 150000) {
    this.maxTokens = maxTokens;
    this.reservedTokens = Math.floor(maxTokens * 0.2); // Reserve 20% for safety
    this.usedTokens = 0;
    this.contextItems = [];
    this.taskDescription = null;
    this.priorities = {
      critical: 10,
      high: 7,
      medium: 5,
      low: 3,
      optional: 1
    };
    
    // Long session configuration
    this.sessionConfig = {
      maxItems: 5000, // Maximum items to keep in memory
      targetResponseTime: 50, // Target response time in ms
      autoCleanupThreshold: 0.9, // Start cleanup when 90% full
      retentionPolicy: {
        minAge: 300000, // 5 minutes minimum retention
        maxAge: 7200000, // 2 hours maximum retention
        criticalMaxAge: 14400000, // 4 hours for critical items
        recentAccessBoost: 1800000 // 30 minutes boost for recently accessed
      },
      cleanupBatchSize: 100, // Items to process per cleanup cycle
      streamingChunkSize: 50 // Items per streaming chunk
    };
    
    // Smart merge configuration
    this.mergeConfig = {
      threshold: 0.85, // 85% similarity for merge
      embeddingCache: new Map(), // Cache embeddings to avoid re-computation
      mergeHistory: [], // Track merge operations
      batchSize: 100 // For batch embedding generation
    };
    
    // Dynamic priority configuration
    this.priorityConfig = {
      accessTracking: new Map(), // Track access frequency per item
      decayInterval: 3600000, // 1 hour in milliseconds
      decayFactor: 0.9, // Reduce priority by 10% per hour
      contextWeights: {
        development: { code: 1.2, error: 1.5, task: 1.3, general: 1.0 },
        debugging: { error: 2.0, code: 1.5, task: 1.1, general: 0.8 },
        research: { general: 1.3, code: 1.1, task: 1.0, error: 0.9 },
        default: { code: 1.0, error: 1.0, task: 1.0, general: 1.0 }
      },
      learningRate: 0.1, // For user behavior learning
      feedbackHistory: [] // Store user feedback for learning
    };
    
    // Initialize embedding generator if API key is available
    if (process.env.OPENAI_API_KEY) {
      this.embeddingGenerator = new EmbeddingGenerator();
    }
    
    // Search integration configuration
    this.searchConfig = {
      autoIntegrate: true,
      relevanceThreshold: 0.7,
      maxSearchResults: 10,
      searchContextWindow: 5, // Number of recent searches to consider
      searchHistory: [],
      relatedItemsCache: new Map()
    };
    
    // Initialize search engine reference
    this.searchEngine = null;
    
    // Performance optimization
    this.performanceConfig = {
      lruCacheSize: 1000,
      indexUpdateInterval: 5000, // Update index every 5 seconds
      enableAsyncProcessing: true
    };
    
    // Initialize LRU cache
    this.lruCache = new LRUCache(this.performanceConfig.lruCacheSize);
    
    // Initialize in-memory indexes
    this.indexes = {
      byType: new Map(),
      byFile: new Map(),
      byPriority: new Map(),
      byTimestamp: new Map()
    };
    
    // Build initial indexes
    this.buildInMemoryIndex();
    
    // Start decay timer
    this.startDecayTimer();
    
    // Start index update timer
    this.startIndexUpdateTimer();
  }

  /**
   * Estimate token count for text
   * Japanese: ~2 chars per token
   * English: ~4 chars per token
   */
  estimateTokens(text) {
    if (!text) return 0;
    
    const japaneseRegex = /[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\u4e00-\u9faf\u3400-\u4dbf]/g;
    const japaneseChars = (text.match(japaneseRegex) || []).length;
    const otherChars = text.length - japaneseChars;
    
    // More accurate estimation
    const japaneseTokens = japaneseChars / 2;
    const otherTokens = otherChars / 4;
    
    return Math.ceil(japaneseTokens + otherTokens);
  }

  /**
   * Add item to context with priority
   */
  addItem(item, priority = 'medium') {
    // Check if we need cleanup before adding
    if (this.contextItems.length >= this.sessionConfig.maxItems * this.sessionConfig.autoCleanupThreshold) {
      this.performAutoCleanup();
    }
    
    const priorityValue = typeof priority === 'number' 
      ? priority 
      : this.priorities[priority] || this.priorities.medium;
    
    const tokens = this.estimateTokens(item.content);
    
    const newItem = {
      id: item.id || this.generateId(),
      type: item.type || 'general',
      content: item.content,
      metadata: item.metadata || {},
      tokens,
      priority: priorityValue,
      timestamp: Date.now(),
      source: item.source || 'unknown',
      lastAccessed: Date.now(),
      accessCount: 0
    };
    
    this.contextItems.push(newItem);
    
    // Add to indexes
    this.addToIndexes(newItem);
    
    // Add to cache
    this.lruCache.set(newItem.id, newItem);
    
    this.optimize();
    
    return this.getStats();
  }

  /**
   * Add multiple items at once
   */
  addItems(items) {
    for (const item of items) {
      const priority = item.priority || 'medium';
      this.addItem(item, priority);
    }
    return this.getStats();
  }

  /**
   * Remove item by ID
   */
  removeItem(itemId) {
    const item = this.contextItems.find(i => i.id === itemId);
    if (item) {
      this.removeFromIndexes(item);
    }
    
    this.contextItems = this.contextItems.filter(item => item.id !== itemId);
    this.recalculateTokens();
    return this.getStats();
  }

  /**
   * Clear all items
   */
  clear() {
    this.contextItems = [];
    this.usedTokens = 0;
    return this.getStats();
  }

  /**
   * Optimize context to fit within token limits
   */
  optimize() {
    // Calculate priority score (priority per token)
    const scoredItems = this.contextItems.map(item => ({
      ...item,
      score: this.calculateItemScore(item)
    }));
    
    // Sort by score (highest first)
    scoredItems.sort((a, b) => b.score - a.score);
    
    // Select items that fit within token limit
    let totalTokens = 0;
    const optimized = [];
    const availableTokens = this.maxTokens - this.reservedTokens;
    
    for (const item of scoredItems) {
      if (totalTokens + item.tokens <= availableTokens) {
        optimized.push(item);
        totalTokens += item.tokens;
      } else if (this.canSplitItem(item)) {
        // Try to fit partial content
        const remainingTokens = availableTokens - totalTokens;
        const partialItem = this.splitItem(item, remainingTokens);
        if (partialItem) {
          optimized.push(partialItem);
          totalTokens += partialItem.tokens;
        }
      }
    }
    
    this.contextItems = optimized;
    this.usedTokens = totalTokens;
  }

  /**
   * Calculate item score for optimization
   */
  calculateItemScore(item) {
    const priorityWeight = 0.5;
    const recencyWeight = 0.3;
    const typeWeight = 0.2;
    
    // Priority score
    const priorityScore = item.priority / 10;
    
    // Recency score (newer items score higher)
    const ageInMinutes = (Date.now() - item.timestamp) / (1000 * 60);
    const recencyScore = Math.max(0, 1 - (ageInMinutes / 60)); // Decay over 1 hour
    
    // Type score (some types are more important)
    const typeScores = {
      task: 1.0,
      error: 0.9,
      code: 0.8,
      general: 0.7,
      context: 0.6
    };
    const typeScore = typeScores[item.type] || 0.5;
    
    // Combine scores
    const combinedScore = (
      priorityScore * priorityWeight +
      recencyScore * recencyWeight +
      typeScore * typeWeight
    ) * (item.priority / item.tokens); // Efficiency: priority per token
    
    return combinedScore;
  }

  /**
   * Check if item can be split
   */
  canSplitItem(item) {
    return item.type === 'code' || item.type === 'general' || item.content.length > 200;
  }

  /**
   * Split item to fit remaining tokens
   */
  splitItem(item, remainingTokens) {
    if (remainingTokens < 50) return null; // Minimum viable content
    
    const charPerToken = item.content.match(/[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\u4e00-\u9faf\u3400-\u4dbf]/g) ? 2 : 4;
    const maxChars = Math.floor(remainingTokens * charPerToken * 0.9); // 90% safety margin
    
    if (maxChars >= item.content.length) return item;
    
    // Find good split point (prefer line breaks)
    let splitPoint = maxChars;
    for (let i = maxChars; i > maxChars * 0.7; i--) {
      if (item.content[i] === '\n') {
        splitPoint = i;
        break;
      }
    }
    
    const truncatedContent = item.content.substring(0, splitPoint) + '\n...';
    
    return {
      ...item,
      content: truncatedContent,
      tokens: this.estimateTokens(truncatedContent),
      metadata: {
        ...item.metadata,
        truncated: true,
        originalTokens: item.tokens,
        originalLength: item.content.length
      }
    };
  }

  /**
   * Generate unique ID
   */
  generateId() {
    return `ctx_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
  }

  /**
   * Recalculate total tokens
   */
  recalculateTokens() {
    this.usedTokens = this.contextItems.reduce((total, item) => total + item.tokens, 0);
  }

  /**
   * Get context statistics
   */
  getStats() {
    const totalItems = this.contextItems.length;
    const availableTokens = this.maxTokens - this.reservedTokens;
    const utilizationRate = this.usedTokens / availableTokens;
    
    const typeDistribution = {};
    const priorityDistribution = {};
    
    for (const item of this.contextItems) {
      typeDistribution[item.type] = (typeDistribution[item.type] || 0) + 1;
      priorityDistribution[item.priority] = (priorityDistribution[item.priority] || 0) + 1;
    }
    
    return {
      totalItems,
      usedTokens: this.usedTokens,
      maxTokens: this.maxTokens,
      availableTokens,
      reservedTokens: this.reservedTokens,
      remainingTokens: availableTokens - this.usedTokens,
      utilizationRate: Math.round(utilizationRate * 100),
      typeDistribution,
      priorityDistribution,
      isOptimized: this.usedTokens <= availableTokens
    };
  }

  /**
   * Set task description for context
   */
  setTaskDescription(description) {
    this.taskDescription = description;
    if (description) {
      this.addItem({
        type: 'task',
        content: `Task: ${description}`,
        source: 'system'
      }, 'critical');
    }
  }

  /**
   * Get formatted context for Claude
   */
  getFormattedContext() {
    if (this.contextItems.length === 0) {
      return 'No context available.';
    }
    
    let formatted = '';
    
    if (this.taskDescription) {
      formatted += `# Current Task\n${this.taskDescription}\n\n`;
    }
    
    formatted += '# Context Information\n\n';
    
    // Group by type for better organization
    const groupedItems = {};
    for (const item of this.contextItems) {
      if (!groupedItems[item.type]) {
        groupedItems[item.type] = [];
      }
      groupedItems[item.type].push(item);
    }
    
    // Sort types by importance
    const typeOrder = ['task', 'error', 'code', 'general', 'context'];
    for (const type of typeOrder) {
      if (groupedItems[type]) {
        formatted += `## ${type.charAt(0).toUpperCase() + type.slice(1)} Information\n\n`;
        for (const item of groupedItems[type]) {
          formatted += this.formatItem(item) + '\n\n';
        }
      }
    }
    
    // Add remaining types
    for (const [type, items] of Object.entries(groupedItems)) {
      if (!typeOrder.includes(type)) {
        formatted += `## ${type.charAt(0).toUpperCase() + type.slice(1)} Information\n\n`;
        for (const item of items) {
          formatted += this.formatItem(item) + '\n\n';
        }
      }
    }
    
    const stats = this.getStats();
    formatted += `---\n*Context Statistics: ${stats.totalItems} items, ${stats.usedTokens}/${stats.availableTokens} tokens (${stats.utilizationRate}% utilization)*`;
    
    return formatted;
  }

  /**
   * Format individual context item
   */
  formatItem(item) {
    let formatted = '';
    
    if (item.metadata && Object.keys(item.metadata).length > 0) {
      const relevantMetadata = ['file', 'score', 'type', 'language'];
      for (const key of relevantMetadata) {
        if (item.metadata[key]) {
          formatted += `**${key}**: ${item.metadata[key]}\n`;
        }
      }
    }
    
    if (formatted) {
      formatted += '\n';
    }
    
    formatted += item.content;
    
    if (item.metadata && item.metadata.truncated) {
      formatted += `\n*[Truncated from ${item.metadata.originalTokens} tokens]*`;
    }
    
    return formatted;
  }

  /**
   * Calculate cosine similarity between two vectors
   */
  cosineSimilarity(a, b) {
    if (!a || !b || a.length !== b.length) return 0;
    
    let dotProduct = 0;
    let normA = 0;
    let normB = 0;
    
    for (let i = 0; i < a.length; i++) {
      dotProduct += a[i] * b[i];
      normA += a[i] * a[i];
      normB += b[i] * b[i];
    }
    
    normA = Math.sqrt(normA);
    normB = Math.sqrt(normB);
    
    if (normA === 0 || normB === 0) return 0;
    
    return dotProduct / (normA * normB);
  }

  /**
   * Get or generate embedding for content
   */
  async getEmbedding(content) {
    if (!this.embeddingGenerator) {
      throw new Error('OpenAI API key not configured. Set OPENAI_API_KEY environment variable.');
    }
    
    // Check cache first
    const cacheKey = content.substring(0, 100); // Use first 100 chars as key
    if (this.mergeConfig.embeddingCache.has(cacheKey)) {
      return this.mergeConfig.embeddingCache.get(cacheKey);
    }
    
    // Generate embedding
    const embedding = await this.embeddingGenerator.generate(content);
    
    // Cache it
    this.mergeConfig.embeddingCache.set(cacheKey, embedding);
    
    // Limit cache size
    if (this.mergeConfig.embeddingCache.size > 1000) {
      const firstKey = this.mergeConfig.embeddingCache.keys().next().value;
      this.mergeConfig.embeddingCache.delete(firstKey);
    }
    
    return embedding;
  }

  /**
   * Detect duplicates for a given item
   */
  async detectDuplicates(item) {
    const duplicates = [];
    const nearDuplicates = [];
    
    if (!item.content) return { duplicates, nearDuplicates };
    
    // Get embedding for the new item
    const itemEmbedding = await this.getEmbedding(item.content);
    
    // Compare with existing items
    for (const existingItem of this.contextItems) {
      // Skip if it's the same item
      if (existingItem.id === item.id) continue;
      
      // Check file path match for same file detection
      const sameFile = item.metadata?.file && 
                      existingItem.metadata?.file && 
                      item.metadata.file === existingItem.metadata.file;
      
      // Get embedding for existing item
      const existingEmbedding = await this.getEmbedding(existingItem.content);
      
      // Calculate similarity
      const similarity = this.cosineSimilarity(itemEmbedding, existingEmbedding);
      
      if (similarity >= 0.99) {
        // Nearly identical content
        duplicates.push({
          item: existingItem,
          similarity,
          sameFile
        });
      } else if (similarity >= this.mergeConfig.threshold) {
        // Similar enough to consider merging
        nearDuplicates.push({
          item: existingItem,
          similarity,
          sameFile
        });
      }
    }
    
    return { duplicates, nearDuplicates };
  }

  /**
   * Calculate similarity between two items
   */
  async calculateSimilarity(item1, item2) {
    if (!item1.content || !item2.content) return 0;
    
    const embedding1 = await this.getEmbedding(item1.content);
    const embedding2 = await this.getEmbedding(item2.content);
    
    return this.cosineSimilarity(embedding1, embedding2);
  }

  /**
   * Merge similar items intelligently
   */
  async mergeItems(items) {
    if (!items || items.length < 2) return items[0];
    
    // Sort by timestamp (newest first) and priority (highest first)
    items.sort((a, b) => {
      if (b.timestamp !== a.timestamp) {
        return b.timestamp - a.timestamp;
      }
      return b.priority - a.priority;
    });
    
    // Use the newest item as base
    const mergedItem = {
      ...items[0],
      metadata: {
        ...items[0].metadata,
        mergedFrom: items.map(item => item.id),
        mergeCount: items.length,
        originalContents: []
      }
    };
    
    // Keep highest priority
    mergedItem.priority = Math.max(...items.map(item => item.priority));
    
    // For same file, different versions - keep diffs
    const sameFile = items.every(item => 
      item.metadata?.file === items[0].metadata?.file
    );
    
    if (sameFile) {
      // Store diffs instead of full content for older versions
      for (let i = 1; i < items.length; i++) {
        const diff = this.computeDiff(items[i].content, items[0].content);
        mergedItem.metadata.originalContents.push({
          id: items[i].id,
          timestamp: items[i].timestamp,
          diff
        });
      }
    } else {
      // Different files or no file info - merge content
      const contents = [];
      for (const item of items) {
        const header = item.metadata?.file 
          ? `\n--- ${item.metadata.file} ---\n`
          : `\n--- Item ${item.id} ---\n`;
        contents.push(header + item.content);
      }
      mergedItem.content = contents.join('\n');
    }
    
    // Recalculate tokens
    mergedItem.tokens = this.estimateTokens(mergedItem.content);
    
    return mergedItem;
  }

  /**
   * Compute simple diff between two contents
   */
  computeDiff(oldContent, newContent) {
    // Simple diff: store what was removed and what was added
    const oldLines = oldContent.split('\n');
    const newLines = newContent.split('\n');
    
    const diff = {
      removed: [],
      added: [],
      similarity: 0
    };
    
    // Find removed lines
    oldLines.forEach((line, index) => {
      if (!newLines.includes(line)) {
        diff.removed.push({ line: index, content: line });
      }
    });
    
    // Find added lines
    newLines.forEach((line, index) => {
      if (!oldLines.includes(line)) {
        diff.added.push({ line: index, content: line });
      }
    });
    
    // Calculate similarity
    const commonLines = oldLines.filter(line => newLines.includes(line)).length;
    diff.similarity = commonLines / Math.max(oldLines.length, newLines.length);
    
    return diff;
  }

  /**
   * Track merge history
   */
  trackMergeHistory(merged, originals) {
    const historyEntry = {
      timestamp: Date.now(),
      mergedId: merged.id,
      originalIds: originals.map(item => item.id),
      originalTokens: originals.reduce((sum, item) => sum + item.tokens, 0),
      mergedTokens: merged.tokens,
      tokensSaved: originals.reduce((sum, item) => sum + item.tokens, 0) - merged.tokens,
      reason: 'similarity_threshold',
      similarity: merged.metadata?.averageSimilarity || null
    };
    
    this.mergeConfig.mergeHistory.push(historyEntry);
    
    // Limit history size
    if (this.mergeConfig.mergeHistory.length > 100) {
      this.mergeConfig.mergeHistory.shift();
    }
    
    return historyEntry;
  }

  /**
   * Smart add item with automatic duplicate detection and merging
   */
  async addItemSmart(item, priority = 'medium') {
    // First detect duplicates
    const { duplicates, nearDuplicates } = await this.detectDuplicates(item);
    
    if (duplicates.length > 0) {
      // Exact duplicate found - update existing
      const duplicate = duplicates[0];
      const existingIndex = this.contextItems.findIndex(
        i => i.id === duplicate.item.id
      );
      
      // Update timestamp and priority
      this.contextItems[existingIndex].timestamp = Date.now();
      this.contextItems[existingIndex].priority = Math.max(
        this.contextItems[existingIndex].priority,
        typeof priority === 'number' ? priority : this.priorities[priority] || 5
      );
      
      this.optimize();
      return {
        action: 'updated',
        item: this.contextItems[existingIndex],
        stats: this.getStats()
      };
    }
    
    if (nearDuplicates.length > 0 && nearDuplicates[0].similarity >= this.mergeConfig.threshold) {
      // Merge with similar items
      const itemsToMerge = [item, ...nearDuplicates.map(nd => nd.item)];
      const mergedItem = await this.mergeItems(itemsToMerge);
      
      // Remove original items
      nearDuplicates.forEach(nd => {
        this.contextItems = this.contextItems.filter(i => i.id !== nd.item.id);
      });
      
      // Add merged item
      this.contextItems.push(mergedItem);
      
      // Track merge
      this.trackMergeHistory(mergedItem, itemsToMerge);
      
      this.optimize();
      return {
        action: 'merged',
        item: mergedItem,
        mergedCount: itemsToMerge.length,
        stats: this.getStats()
      };
    }
    
    // No duplicates - add normally
    return {
      action: 'added',
      item: this.addItem(item, priority),
      stats: this.getStats()
    };
  }

  /**
   * Track item access for dynamic priority
   */
  trackAccess(itemId) {
    const access = this.priorityConfig.accessTracking.get(itemId) || {
      count: 0,
      lastAccessed: 0,
      totalScore: 0
    };
    
    access.count++;
    access.lastAccessed = Date.now();
    access.totalScore += 1;
    
    this.priorityConfig.accessTracking.set(itemId, access);
    
    // Update item's dynamic priority
    const item = this.contextItems.find(i => i.id === itemId);
    if (item) {
      this.updateDynamicPriority(item);
    }
  }

  /**
   * Adjust priority based on task context
   */
  adjustPriorityByContext(item, taskContext = 'default') {
    const weights = this.priorityConfig.contextWeights[taskContext] || 
                   this.priorityConfig.contextWeights.default;
    
    const typeWeight = weights[item.type] || 1.0;
    
    // Calculate adjusted priority
    const basePriority = item.priority;
    const adjustedPriority = basePriority * typeWeight;
    
    // Store both base and dynamic priority
    item.basePriority = basePriority;
    item.dynamicPriority = adjustedPriority;
    
    return adjustedPriority;
  }

  /**
   * Apply time-based decay to priorities
   */
  applyTimeDecay() {
    const now = Date.now();
    
    for (const item of this.contextItems) {
      const age = now - item.timestamp;
      const decayPeriods = Math.floor(age / this.priorityConfig.decayInterval);
      
      if (decayPeriods > 0 && !item.noDecay) {
        // Apply decay factor for each period
        const decayMultiplier = Math.pow(this.priorityConfig.decayFactor, decayPeriods);
        
        // Update dynamic priority with decay
        const basePriority = item.basePriority || item.priority;
        item.dynamicPriority = (item.dynamicPriority || basePriority) * decayMultiplier;
        
        // Apply access frequency boost to counter decay
        const access = this.priorityConfig.accessTracking.get(item.id);
        if (access) {
          const recencyBoost = this.calculateRecencyBoost(access.lastAccessed);
          const frequencyBoost = Math.log(access.count + 1) * 0.2;
          item.dynamicPriority *= (1 + recencyBoost + frequencyBoost);
        }
      }
    }
    
    // Re-optimize after decay
    this.optimize();
  }

  /**
   * Calculate recency boost based on last access time
   */
  calculateRecencyBoost(lastAccessed) {
    const timeSinceAccess = Date.now() - lastAccessed;
    const hoursAgo = timeSinceAccess / 3600000;
    
    // Exponential decay for recency boost
    return Math.exp(-hoursAgo / 24); // Half-life of 24 hours
  }

  /**
   * Learn from user behavior and feedback
   */
  learnFromUserBehavior(feedback) {
    // Store feedback
    this.priorityConfig.feedbackHistory.push({
      timestamp: Date.now(),
      itemId: feedback.itemId,
      action: feedback.action, // 'helpful', 'not_helpful', 'accessed', 'ignored'
      context: feedback.context,
      metadata: feedback.metadata
    });
    
    // Limit history size
    if (this.priorityConfig.feedbackHistory.length > 1000) {
      this.priorityConfig.feedbackHistory = 
        this.priorityConfig.feedbackHistory.slice(-500);
    }
    
    // Update item priority based on feedback
    const item = this.contextItems.find(i => i.id === feedback.itemId);
    if (item) {
      const adjustment = this.calculateFeedbackAdjustment(feedback);
      item.feedbackScore = (item.feedbackScore || 0) + adjustment;
      
      // Update dynamic priority
      this.updateDynamicPriority(item);
    }
    
    // Update context weights if enough feedback
    if (this.priorityConfig.feedbackHistory.length >= 50) {
      this.updateContextWeights();
    }
  }

  /**
   * Calculate priority adjustment based on feedback
   */
  calculateFeedbackAdjustment(feedback) {
    const adjustments = {
      helpful: 0.5,
      not_helpful: -0.3,
      accessed: 0.2,
      ignored: -0.1
    };
    
    return adjustments[feedback.action] || 0;
  }

  /**
   * Update context weights based on accumulated feedback
   */
  updateContextWeights() {
    // Group feedback by context and type
    const contextStats = {};
    
    for (const feedback of this.priorityConfig.feedbackHistory) {
      if (!feedback.context) continue;
      
      if (!contextStats[feedback.context]) {
        contextStats[feedback.context] = {};
      }
      
      const item = this.contextItems.find(i => i.id === feedback.itemId);
      if (!item) continue;
      
      const type = item.type;
      if (!contextStats[feedback.context][type]) {
        contextStats[feedback.context][type] = {
          helpful: 0,
          total: 0
        };
      }
      
      contextStats[feedback.context][type].total++;
      if (feedback.action === 'helpful' || feedback.action === 'accessed') {
        contextStats[feedback.context][type].helpful++;
      }
    }
    
    // Update weights based on success rate
    for (const [context, types] of Object.entries(contextStats)) {
      if (!this.priorityConfig.contextWeights[context]) {
        this.priorityConfig.contextWeights[context] = { ...this.priorityConfig.contextWeights.default };
      }
      
      for (const [type, stats] of Object.entries(types)) {
        const successRate = stats.helpful / stats.total;
        const currentWeight = this.priorityConfig.contextWeights[context][type] || 1.0;
        
        // Gradually adjust weight based on success rate
        const targetWeight = 0.5 + successRate * 1.5; // Range: 0.5 - 2.0
        const newWeight = currentWeight + (targetWeight - currentWeight) * this.priorityConfig.learningRate;
        
        this.priorityConfig.contextWeights[context][type] = newWeight;
      }
    }
  }

  /**
   * Update dynamic priority for an item
   */
  updateDynamicPriority(item) {
    const basePriority = item.basePriority || item.priority;
    let dynamicPriority = basePriority;
    
    // Apply access frequency boost
    const access = this.priorityConfig.accessTracking.get(item.id);
    if (access) {
      const recencyBoost = this.calculateRecencyBoost(access.lastAccessed);
      const frequencyBoost = Math.log(access.count + 1) * 0.2;
      dynamicPriority *= (1 + recencyBoost + frequencyBoost);
    }
    
    // Apply feedback score
    if (item.feedbackScore) {
      dynamicPriority *= (1 + item.feedbackScore * 0.1);
    }
    
    // Apply time decay
    const age = Date.now() - item.timestamp;
    const decayPeriods = Math.floor(age / this.priorityConfig.decayInterval);
    if (decayPeriods > 0 && !item.noDecay) {
      const decayMultiplier = Math.pow(this.priorityConfig.decayFactor, decayPeriods);
      dynamicPriority *= decayMultiplier;
    }
    
    item.dynamicPriority = dynamicPriority;
  }

  /**
   * Start the decay timer
   */
  startDecayTimer() {
    // Apply decay every 10 minutes
    this.decayTimer = setInterval(() => {
      this.applyTimeDecay();
    }, 600000); // 10 minutes
  }

  /**
   * Stop the decay timer
   */
  stopDecayTimer() {
    if (this.decayTimer) {
      clearInterval(this.decayTimer);
      this.decayTimer = null;
    }
  }

  /**
   * Get context with dynamic priorities
   */
  getContextWithDynamicPriorities(taskContext = 'default') {
    // Apply context-based adjustments
    for (const item of this.contextItems) {
      this.adjustPriorityByContext(item, taskContext);
    }
    
    // Return formatted context
    return this.getFormattedContext();
  }

  /**
   * Override calculateItemScore to use dynamic priority
   */
  calculateItemScore(item) {
    // Use dynamic priority if available
    const priority = item.dynamicPriority || item.priority;
    
    const priorityWeight = 0.5;
    const recencyWeight = 0.3;
    const typeWeight = 0.2;
    
    // Priority score
    const priorityScore = priority / 10;
    
    // Recency score (newer items score higher)
    const ageInMinutes = (Date.now() - item.timestamp) / (1000 * 60);
    const recencyScore = Math.max(0, 1 - (ageInMinutes / 60)); // Decay over 1 hour
    
    // Type score (some types are more important)
    const typeScores = {
      task: 1.0,
      error: 0.9,
      code: 0.8,
      general: 0.7,
      context: 0.6
    };
    const typeScore = typeScores[item.type] || 0.5;
    
    // Combine scores
    const combinedScore = (
      priorityScore * priorityWeight +
      recencyScore * recencyWeight +
      typeScore * typeWeight
    ) * (priority / item.tokens); // Efficiency: priority per token
    
    return combinedScore;
  }

  /**
   * Set search engine for integration
   */
  setSearchEngine(searchEngine) {
    if (searchEngine instanceof VectorSearchEngine) {
      this.searchEngine = searchEngine;
    } else {
      throw new Error('Invalid search engine instance');
    }
  }

  /**
   * Integrate search results into context
   */
  async integrateSearchResults(results, query, options = {}) {
    if (!results || !results.results) return;
    
    // Store search in history
    this.searchConfig.searchHistory.push({
      query,
      timestamp: Date.now(),
      resultCount: results.results.length,
      integrated: []
    });
    
    // Limit search history
    if (this.searchConfig.searchHistory.length > this.searchConfig.searchContextWindow) {
      this.searchConfig.searchHistory.shift();
    }
    
    const integrated = [];
    const relevanceThreshold = options.threshold || this.searchConfig.relevanceThreshold;
    
    for (const result of results.results) {
      if (result.score >= relevanceThreshold) {
        const contextItem = await this.autoContextualize(result, query);
        
        // Use smart add to handle duplicates
        const addResult = await this.addItemSmart(contextItem, this.calculateSearchPriority(result.score));
        
        integrated.push({
          itemId: addResult.item.id || contextItem.id,
          action: addResult.action,
          score: result.score
        });
      }
    }
    
    // Update search history with integrated items
    const lastSearch = this.searchConfig.searchHistory[this.searchConfig.searchHistory.length - 1];
    lastSearch.integrated = integrated;
    
    return {
      query,
      integratedCount: integrated.length,
      totalResults: results.results.length,
      actions: integrated.map(i => i.action),
      stats: this.getStats()
    };
  }

  /**
   * Auto-contextualize search result
   */
  async autoContextualize(searchResult) {
    const { chunk, score } = searchResult;
    
    // Create context item from search result
    const contextItem = {
      type: this.inferItemType(chunk),
      content: chunk.content,
      metadata: {
        file: chunk.file,
        startLine: chunk.startLine,
        endLine: chunk.endLine,
        language: chunk.language || 'unknown',
        searchScore: score,
        source: 'search',
        ...chunk.metadata
      },
      source: 'search'
    };
    
    // Add surrounding context if available
    if (this.searchEngine && chunk.file) {
      const relatedContext = await this.getRelatedContext(chunk);
      if (relatedContext) {
        contextItem.content = this.mergeWithContext(chunk.content, relatedContext);
        contextItem.metadata.hasExtendedContext = true;
      }
    }
    
    return contextItem;
  }

  /**
   * Get related context for a chunk
   */
  async getRelatedContext(chunk) {
    const cacheKey = `${chunk.file}:${chunk.startLine}-${chunk.endLine}`;
    
    // Check cache
    if (this.searchConfig.relatedItemsCache.has(cacheKey)) {
      return this.searchConfig.relatedItemsCache.get(cacheKey);
    }
    
    // Find related chunks in the same file
    const relatedChunks = [];
    
    // This would typically query the search engine for nearby chunks
    // For now, we'll return null and let the chunk stand alone
    // In a full implementation, this would fetch surrounding code
    
    this.searchConfig.relatedItemsCache.set(cacheKey, relatedChunks);
    
    // Limit cache size
    if (this.searchConfig.relatedItemsCache.size > 100) {
      const firstKey = this.searchConfig.relatedItemsCache.keys().next().value;
      this.searchConfig.relatedItemsCache.delete(firstKey);
    }
    
    return relatedChunks.length > 0 ? relatedChunks : null;
  }

  /**
   * Merge chunk content with surrounding context
   */
  mergeWithContext(mainContent, relatedContext) {
    // Simple merge strategy - could be more sophisticated
    if (!relatedContext || relatedContext.length === 0) {
      return mainContent;
    }
    
    // Add context markers
    let merged = '// === Main Content ===\n';
    merged += mainContent;
    merged += '\n\n// === Related Context ===\n';
    merged += relatedContext.map(ctx => ctx.content).join('\n\n');
    
    return merged;
  }

  /**
   * Rank items by relevance to query
   */
  async rankByRelevance(items, query) {
    if (!this.embeddingGenerator) {
      // Fallback to simple text matching
      return this.rankByTextMatch(items, query);
    }
    
    // Get query embedding
    const queryEmbedding = await this.getEmbedding(query);
    
    // Calculate relevance scores
    const rankedItems = [];
    for (const item of items) {
      const itemEmbedding = await this.getEmbedding(item.content);
      const relevance = this.cosineSimilarity(queryEmbedding, itemEmbedding);
      
      rankedItems.push({
        item,
        relevance,
        originalPriority: item.priority
      });
    }
    
    // Sort by relevance
    rankedItems.sort((a, b) => b.relevance - a.relevance);
    
    return rankedItems;
  }

  /**
   * Simple text-based ranking fallback
   */
  rankByTextMatch(items, query) {
    const queryTerms = query.toLowerCase().split(/\s+/);
    
    return items.map(item => {
      const content = item.content.toLowerCase();
      let matchScore = 0;
      
      for (const term of queryTerms) {
        if (content.includes(term)) {
          matchScore += 1;
          // Bonus for exact matches
          const regex = new RegExp(`\\b${term}\\b`, 'g');
          const exactMatches = (content.match(regex) || []).length;
          matchScore += exactMatches * 0.5;
        }
      }
      
      return {
        item,
        relevance: matchScore / queryTerms.length,
        originalPriority: item.priority
      };
    }).sort((a, b) => b.relevance - a.relevance);
  }

  /**
   * Calculate priority based on search score
   */
  calculateSearchPriority(score) {
    if (score >= 0.9) return 'high';
    if (score >= 0.8) return 'medium';
    if (score >= 0.7) return 'low';
    return 'optional';
  }

  /**
   * Infer item type from chunk
   */
  inferItemType(chunk) {
    const content = chunk.content.toLowerCase();
    const file = chunk.file || '';
    
    // Check for error patterns
    if (content.includes('error') || content.includes('exception') || 
        content.includes('traceback') || content.includes('stack')) {
      return 'error';
    }
    
    // Check for task/todo patterns
    if (content.includes('todo') || content.includes('fixme') || 
        content.includes('task:')) {
      return 'task';
    }
    
    // Check file extension
    const ext = file.split('.').pop();
    const codeExtensions = ['js', 'ts', 'py', 'java', 'cpp', 'c', 'go', 'rs', 'rb'];
    if (codeExtensions.includes(ext)) {
      return 'code';
    }
    
    return 'general';
  }

  /**
   * Get search-aware context
   */
  async getSearchAwareContext(query, taskContext = 'default') {
    // First, get items ranked by relevance to the query
    const rankedItems = await this.rankByRelevance(this.contextItems, query);
    
    // Apply dynamic priorities with search relevance boost
    for (const ranked of rankedItems) {
      const { item, relevance } = ranked;
      
      // Boost priority based on relevance
      const relevanceBoost = relevance * 2; // Up to 2x boost for perfect match
      const currentPriority = item.dynamicPriority || item.priority;
      item.searchBoostPriority = currentPriority * (1 + relevanceBoost);
    }
    
    // Temporarily use search-boosted priorities for optimization
    const originalPriorities = this.contextItems.map(item => ({
      id: item.id,
      priority: item.priority,
      dynamicPriority: item.dynamicPriority
    }));
    
    // Apply search boost
    for (const item of this.contextItems) {
      if (item.searchBoostPriority) {
        item.priority = item.searchBoostPriority;
      }
    }
    
    // Optimize with boosted priorities
    this.optimize();
    const context = this.getFormattedContext();
    
    // Restore original priorities
    for (const original of originalPriorities) {
      const item = this.contextItems.find(i => i.id === original.id);
      if (item) {
        item.priority = original.priority;
        item.dynamicPriority = original.dynamicPriority;
        delete item.searchBoostPriority;
      }
    }
    
    return context;
  }

  /**
   * Search and integrate in one operation
   */
  async searchAndIntegrate(query, options = {}) {
    if (!this.searchEngine) {
      throw new Error('Search engine not configured. Use setSearchEngine() first.');
    }
    
    // Perform search
    const results = await this.searchEngine.search(query, {
      threshold: options.threshold || this.searchConfig.relevanceThreshold,
      maxResults: options.maxResults || this.searchConfig.maxSearchResults,
      ...options
    });
    
    // Integrate results
    const integration = await this.integrateSearchResults(results, query, options);
    
    // Get search-aware context
    const context = await this.getSearchAwareContext(query, options.taskContext);
    
    return {
      searchResults: results,
      integration,
      context,
      stats: this.getStats()
    };
  }

  /**
   * Build in-memory indexes for fast access
   */
  buildInMemoryIndex() {
    // Clear existing indexes
    for (const index of Object.values(this.indexes)) {
      index.clear();
    }
    
    // Build indexes
    for (const item of this.contextItems) {
      this.addToIndexes(item);
    }
  }

  /**
   * Add item to indexes
   */
  addToIndexes(item) {
    // Index by type
    if (!this.indexes.byType.has(item.type)) {
      this.indexes.byType.set(item.type, new Set());
    }
    this.indexes.byType.get(item.type).add(item.id);
    
    // Index by file
    if (item.metadata?.file) {
      if (!this.indexes.byFile.has(item.metadata.file)) {
        this.indexes.byFile.set(item.metadata.file, new Set());
      }
      this.indexes.byFile.get(item.metadata.file).add(item.id);
    }
    
    // Index by priority range
    const priorityBucket = Math.floor(item.priority / 2) * 2; // Bucket by 2s
    if (!this.indexes.byPriority.has(priorityBucket)) {
      this.indexes.byPriority.set(priorityBucket, new Set());
    }
    this.indexes.byPriority.get(priorityBucket).add(item.id);
    
    // Index by timestamp range (hourly buckets)
    const hourBucket = Math.floor(item.timestamp / 3600000) * 3600000;
    if (!this.indexes.byTimestamp.has(hourBucket)) {
      this.indexes.byTimestamp.set(hourBucket, new Set());
    }
    this.indexes.byTimestamp.get(hourBucket).add(item.id);
  }

  /**
   * Remove item from indexes
   */
  removeFromIndexes(item) {
    // Remove from type index
    if (this.indexes.byType.has(item.type)) {
      this.indexes.byType.get(item.type).delete(item.id);
      if (this.indexes.byType.get(item.type).size === 0) {
        this.indexes.byType.delete(item.type);
      }
    }
    
    // Remove from file index
    if (item.metadata?.file && this.indexes.byFile.has(item.metadata.file)) {
      this.indexes.byFile.get(item.metadata.file).delete(item.id);
      if (this.indexes.byFile.get(item.metadata.file).size === 0) {
        this.indexes.byFile.delete(item.metadata.file);
      }
    }
    
    // Remove from priority index
    const priorityBucket = Math.floor(item.priority / 2) * 2;
    if (this.indexes.byPriority.has(priorityBucket)) {
      this.indexes.byPriority.get(priorityBucket).delete(item.id);
      if (this.indexes.byPriority.get(priorityBucket).size === 0) {
        this.indexes.byPriority.delete(priorityBucket);
      }
    }
    
    // Remove from timestamp index
    const hourBucket = Math.floor(item.timestamp / 3600000) * 3600000;
    if (this.indexes.byTimestamp.has(hourBucket)) {
      this.indexes.byTimestamp.get(hourBucket).delete(item.id);
      if (this.indexes.byTimestamp.get(hourBucket).size === 0) {
        this.indexes.byTimestamp.delete(hourBucket);
      }
    }
  }

  /**
   * Get items by type using index
   */
  getItemsByType(type) {
    const itemIds = this.indexes.byType.get(type);
    if (!itemIds) return [];
    
    return Array.from(itemIds)
      .map(id => this.getItemById(id))
      .filter(item => item !== null);
  }

  /**
   * Get items by file using index
   */
  getItemsByFile(file) {
    const itemIds = this.indexes.byFile.get(file);
    if (!itemIds) return [];
    
    return Array.from(itemIds)
      .map(id => this.getItemById(id))
      .filter(item => item !== null);
  }

  /**
   * Get item by ID with LRU cache
   */
  getItemById(id) {
    // Check cache first
    const cached = this.lruCache.get(id);
    if (cached) return cached;
    
    // Find in context items
    const item = this.contextItems.find(i => i.id === id);
    if (item) {
      // Cache it
      this.lruCache.set(id, item);
    }
    
    return item || null;
  }


  /**
   * Start index update timer
   */
  startIndexUpdateTimer() {
    this.indexUpdateTimer = setInterval(() => {
      this.buildInMemoryIndex();
    }, this.performanceConfig.indexUpdateInterval);
  }

  /**
   * Stop index update timer
   */
  stopIndexUpdateTimer() {
    if (this.indexUpdateTimer) {
      clearInterval(this.indexUpdateTimer);
      this.indexUpdateTimer = null;
    }
  }

  /**
   * Async process for heavy operations
   */
  async asyncProcess(operation, ...args) {
    if (!this.performanceConfig.enableAsyncProcessing) {
      return operation.apply(this, args);
    }
    
    return new Promise((resolve, reject) => {
      // Use setImmediate for true async processing
      setImmediate(() => {
        try {
          const result = operation.apply(this, args);
          resolve(result);
        } catch (error) {
          reject(error);
        }
      });
    });
  }

  /**
   * Monitor memory usage
   */
  monitorMemoryUsage() {
    const usage = process.memoryUsage();
    const heapUsedMB = Math.round(usage.heapUsed / 1024 / 1024);
    const heapTotalMB = Math.round(usage.heapTotal / 1024 / 1024);
    
    return {
      heapUsedMB,
      heapTotalMB,
      contextItemsCount: this.contextItems.length,
      cacheSize: this.lruCache.size,
      embeddingCacheSize: this.mergeConfig.embeddingCache.size,
      indexSizes: {
        byType: this.indexes.byType.size,
        byFile: this.indexes.byFile.size,
        byPriority: this.indexes.byPriority.size,
        byTimestamp: this.indexes.byTimestamp.size
      }
    };
  }

  /**
   * Get performance statistics
   */
  getPerformanceStats() {
    const memory = this.monitorMemoryUsage();
    const stats = this.getStats();
    
    return {
      ...stats,
      memory,
      performance: {
        cacheHitRate: this.lruCache.size > 0 ? 
          (this.lruCache.size / this.performanceConfig.lruCacheSize) : 0,
        indexUpdateInterval: this.performanceConfig.indexUpdateInterval,
        asyncProcessingEnabled: this.performanceConfig.enableAsyncProcessing
      }
    };
  }

  /**
   * Cleanup method to stop all timers
   */
  cleanup() {
    this.stopDecayTimer();
    this.stopIndexUpdateTimer();
    this.lruCache.clear();
    for (const index of Object.values(this.indexes)) {
      index.clear();
    }
  }

  /**
   * Perform automatic cleanup for long sessions
   * Intelligently removes items based on relevance, age, and access patterns
   */
  performAutoCleanup() {
    const startTime = Date.now();
    const itemsToRemove = [];
    const now = Date.now();
    
    // Calculate relevance scores for all items
    const scoredItems = this.contextItems.map(item => {
      const score = this.calculateRetentionScore(item, now);
      return { item, score };
    });
    
    // Sort by score (lowest scores will be removed first)
    scoredItems.sort((a, b) => a.score - b.score);
    
    // Determine how many items to remove
    const targetCount = Math.floor(this.sessionConfig.maxItems * 0.8); // Keep 80% after cleanup
    const removeCount = Math.max(0, this.contextItems.length - targetCount);
    
    // Select items for removal
    for (let i = 0; i < removeCount && i < scoredItems.length; i++) {
      const { item, score } = scoredItems[i];
      
      // Never remove critical items less than minAge old
      if (item.priority >= this.priorities.critical && 
          (now - item.timestamp) < this.sessionConfig.retentionPolicy.minAge) {
        continue;
      }
      
      itemsToRemove.push(item.id);
    }
    
    // Remove selected items
    for (const itemId of itemsToRemove) {
      this.removeItem(itemId);
    }
    
    // Log cleanup performance
    const duration = Date.now() - startTime;
    if (this.debugMode) {
      console.log(`Auto cleanup: removed ${itemsToRemove.length} items in ${duration}ms`);
    }
    
    return {
      removed: itemsToRemove.length,
      duration,
      remaining: this.contextItems.length
    };
  }

  /**
   * Calculate retention score for an item
   * Higher scores mean the item is more likely to be kept
   */
  calculateRetentionScore(item, now) {
    const age = now - item.timestamp;
    const timeSinceAccess = now - (item.lastAccessed || item.timestamp);
    
    // Base score from priority
    let score = item.priority || this.priorities.medium;
    
    // Boost for recently accessed items
    if (timeSinceAccess < this.sessionConfig.retentionPolicy.recentAccessBoost) {
      score *= 2;
    }
    
    // Boost for frequently accessed items
    if (item.accessCount > 0) {
      score *= (1 + Math.log(item.accessCount + 1));
    }
    
    // Penalty for old items
    const maxAge = item.priority >= this.priorities.critical 
      ? this.sessionConfig.retentionPolicy.criticalMaxAge 
      : this.sessionConfig.retentionPolicy.maxAge;
    
    if (age > maxAge) {
      score *= 0.1; // Severe penalty for very old items
    } else if (age > maxAge / 2) {
      score *= 0.5; // Moderate penalty for aging items
    }
    
    // Boost for items with high search relevance
    if (item.metadata?.searchScore) {
      score *= (1 + item.metadata.searchScore);
    }
    
    // Boost for items that were helpful (from feedback)
    if (item.feedbackScore && item.feedbackScore > 0) {
      score *= (1 + item.feedbackScore);
    }
    
    // Type-based adjustments
    const typeBoosts = {
      error: 1.5,    // Errors are often referenced
      task: 1.3,     // Tasks define context
      code: 1.0,     // Standard code
      general: 0.8,  // General info less critical
      context: 0.7   // Context can be regenerated
    };
    score *= typeBoosts[item.type] || 1.0;
    
    return score;
  }

  /**
   * Fast search for relevant items (target: <50ms)
   */
  async fastSearch(query, options = {}) {
    const startTime = Date.now();
    const maxResults = options.maxResults || 10;
    const threshold = options.threshold || 0.7;
    
    try {
      // Use cached embeddings if available
      const queryKey = query.substring(0, 50);
      let queryEmbedding;
      
      if (this.mergeConfig.embeddingCache.has(queryKey)) {
        queryEmbedding = this.mergeConfig.embeddingCache.get(queryKey);
      } else if (this.embed