codecrucible-synth
Version:
Production-Ready AI Development Platform with Multi-Voice Synthesis, Smithery MCP Integration, Enterprise Security, and Zero-Timeout Reliability
677 lines • 24.9 kB
JavaScript
import { logger } from '../logger.js';
import { readFile, writeFile, mkdir } from 'fs/promises';
import { join } from 'path';
import { existsSync } from 'fs';
import { registerShutdownHandler, createManagedInterval, clearManagedInterval } from '../agent.js';
/**
* Enhanced Context Manager
*
* Provides sophisticated context management with:
* - Long-term memory persistence
* - Intelligent context pruning
* - Relevance-based retrieval
* - Cross-session context retention
* - Smart context compression
*/
export class EnhancedContextManager {
contextStore = new Map();
maxContextSize;
persistencePath;
currentSessionId;
lastPersistTime = 0;
persistenceInterval = 5 * 60 * 1000; // 5 minutes
persistenceTimer = null;
constructor(maxContextSize = 50 * 1024 * 1024, // 50MB
persistencePath = join(process.cwd(), '.codecrucible', 'context')) {
this.maxContextSize = maxContextSize;
this.persistencePath = persistencePath;
this.currentSessionId = this.generateSessionId();
this.initializePersistence();
this.startPeriodicPersistence();
// Register for shutdown
registerShutdownHandler(() => this.cleanup());
}
/**
* Add context with automatic relevance scoring and metadata
*/
async addContext(key, value, options = {}) {
const now = Date.now();
const serializedValue = this.serializeValue(value);
const size = this.estimateSize(serializedValue);
// Calculate relevance score
const relevanceScore = this.calculateRelevanceScore(key, value, options);
const contextItem = {
key,
value: serializedValue,
timestamp: now,
size,
relevanceScore,
sessionId: this.currentSessionId,
type: options.type || this.inferType(key, value),
metadata: {
tags: options.tags || [],
source: options.source || 'unknown',
priority: options.priority || 'medium',
expiresAt: options.expiresIn ? now + options.expiresIn : undefined,
},
};
this.contextStore.set(key, contextItem);
// Trigger pruning if needed
await this.pruneContextIfNeeded();
logger.debug(`Context added: ${key} (${size} bytes, relevance: ${relevanceScore.toFixed(2)})`);
}
/**
* Get context with smart relevance-based retrieval
*/
async getContext(key) {
const item = this.contextStore.get(key);
if (!item) {
return null;
}
// Check expiration
if (item.metadata.expiresAt && Date.now() > item.metadata.expiresAt) {
this.contextStore.delete(key);
return null;
}
// Update relevance score based on access
item.relevanceScore += 0.1;
item.timestamp = Date.now();
return this.deserializeValue(item.value);
}
/**
* Get context for a specific task with intelligent filtering
*/
async getContextForTask(task) {
const keywords = this.extractKeywords(task.description);
const query = {
keywords,
maxAge: 24 * 60 * 60 * 1000, // 24 hours
minRelevanceScore: 0.3,
};
const relevantItems = await this.queryContext(query);
// Categorize results
const categorized = {
relevantFiles: relevantItems.filter(item => item.type === 'file'),
conversationHistory: relevantItems.filter(item => item.type === 'conversation'),
executionHistory: relevantItems.filter(item => item.type === 'execution'),
projectContext: relevantItems.filter(item => item.type === 'project'),
suggestions: this.generateContextSuggestions(task, relevantItems),
};
return categorized;
}
/**
* Query context with sophisticated filtering
*/
async queryContext(query) {
let results = Array.from(this.contextStore.values());
// Filter by type
if (query.type) {
results = results.filter(item => item.type === query.type);
}
// Filter by session
if (query.sessionId) {
results = results.filter(item => item.sessionId === query.sessionId);
}
// Filter by tags
if (query.tags && query.tags.length > 0) {
results = results.filter(item => query.tags.some(tag => item.metadata.tags.includes(tag)));
}
// Filter by time range
if (query.timeRange) {
results = results.filter(item => item.timestamp >= query.timeRange.start && item.timestamp <= query.timeRange.end);
}
// Filter by max age
if (query.maxAge) {
const cutoff = Date.now() - query.maxAge;
results = results.filter(item => item.timestamp >= cutoff);
}
// Filter by relevance score
if (query.minRelevanceScore) {
results = results.filter(item => item.relevanceScore >= query.minRelevanceScore);
}
// Filter by keywords
if (query.keywords && query.keywords.length > 0) {
results = results.filter(item => this.matchesKeywords(item, query.keywords));
}
// Sort by relevance score (descending)
results.sort((a, b) => b.relevanceScore - a.relevanceScore);
return results;
}
/**
* Get context summary and statistics
*/
getContextSummary() {
const items = Array.from(this.contextStore.values());
if (items.length === 0) {
return {
totalItems: 0,
totalSize: 0,
typeBreakdown: {},
oldestItem: 0,
newestItem: 0,
avgRelevanceScore: 0,
};
}
const totalSize = items.reduce((sum, item) => sum + item.size, 0);
const totalRelevance = items.reduce((sum, item) => sum + item.relevanceScore, 0);
const typeBreakdown = items.reduce((acc, item) => {
acc[item.type] = (acc[item.type] || 0) + 1;
return acc;
}, {});
const timestamps = items.map(item => item.timestamp);
return {
totalItems: items.length,
totalSize,
typeBreakdown,
oldestItem: Math.min(...timestamps),
newestItem: Math.max(...timestamps),
avgRelevanceScore: totalRelevance / items.length,
};
}
/**
* Intelligent context pruning with multiple strategies
*/
async pruneContextIfNeeded() {
const totalSize = this.getTotalSize();
if (totalSize <= this.maxContextSize) {
return;
}
logger.info(`Context size ${totalSize} exceeds limit ${this.maxContextSize}, starting pruning`);
const items = Array.from(this.contextStore.values());
const targetSize = this.maxContextSize * 0.8; // Target 80% of max size
// Strategy 1: Remove expired items
await this.removeExpiredItems();
if (this.getTotalSize() <= targetSize) {
return;
}
// Strategy 2: Remove low-relevance temporary items
await this.removeLowRelevanceItems(0.2);
if (this.getTotalSize() <= targetSize) {
return;
}
// Strategy 3: Age-based pruning with relevance weighting
await this.ageBasedPruning(targetSize);
logger.info(`Context pruning complete. New size: ${this.getTotalSize()}`);
}
/**
* Compress context to optimize memory usage
*/
async compressContext() {
const items = Array.from(this.contextStore.values());
let compressionSavings = 0;
for (const item of items) {
if (item.type === 'conversation' || item.type === 'execution') {
const originalSize = item.size;
const compressed = this.compressLongText(item.value);
if (compressed.length < item.value.length) {
item.value = compressed;
item.size = this.estimateSize(compressed);
compressionSavings += originalSize - item.size;
// Update relevance slightly down due to compression
item.relevanceScore *= 0.95;
}
}
}
if (compressionSavings > 0) {
logger.info(`Context compression saved ${compressionSavings} bytes`);
}
}
/**
* Merge similar context items to reduce redundancy
*/
async mergeSimilarContext() {
const items = Array.from(this.contextStore.values());
const groups = this.groupSimilarItems(items);
for (const group of groups) {
if (group.length > 1) {
const merged = this.mergeContextItems(group);
// Remove original items
group.forEach(item => this.contextStore.delete(item.key));
// Add merged item
this.contextStore.set(merged.key, merged);
logger.debug(`Merged ${group.length} similar context items into ${merged.key}`);
}
}
}
/**
* Cross-session context retention
*/
async loadPreviousSessionContext(sessionId) {
try {
const contextFile = join(this.persistencePath, `context-${sessionId || 'latest'}.json`);
if (!existsSync(contextFile)) {
return 0;
}
const data = await readFile(contextFile, 'utf8');
const savedContext = JSON.parse(data);
let loadedCount = 0;
for (const item of savedContext.items || []) {
// Only load non-temporary items from previous sessions
if (item.type !== 'temporary') {
// Reduce relevance for old session data
item.relevanceScore *= 0.7;
item.metadata.tags.push('previous-session');
this.contextStore.set(item.key, item);
loadedCount++;
}
}
logger.info(`Loaded ${loadedCount} context items from previous session`);
return loadedCount;
}
catch (error) {
logger.warn('Failed to load previous session context:', error);
return 0;
}
}
/**
* Save context to persistent storage
*/
async saveContext() {
try {
// Ensure directory exists
if (!existsSync(this.persistencePath)) {
await mkdir(this.persistencePath, { recursive: true });
}
const contextFile = join(this.persistencePath, `context-${this.currentSessionId}.json`);
const latestFile = join(this.persistencePath, 'context-latest.json');
const exportData = {
sessionId: this.currentSessionId,
timestamp: Date.now(),
items: Array.from(this.contextStore.values()),
summary: this.getContextSummary(),
};
const jsonData = JSON.stringify(exportData, null, 2);
// Save both session-specific and latest
await writeFile(contextFile, jsonData, 'utf8');
await writeFile(latestFile, jsonData, 'utf8');
this.lastPersistTime = Date.now();
logger.debug(`Context saved to ${contextFile}`);
}
catch (error) {
logger.error('Failed to save context:', error);
}
}
/**
* Private helper methods
*/
calculateRelevanceScore(key, value, options) {
let score = 0.5; // Base score
// Priority boost
if (options.priority === 'high')
score += 0.3;
else if (options.priority === 'medium')
score += 0.1;
// Type-based scoring
const typeScores = {
file: 0.8,
project: 0.9,
conversation: 0.6,
execution: 0.7,
temporary: 0.3,
};
const type = options.type || this.inferType(key, value);
score += typeScores[type] || 0.5;
// Size penalty for very large items
const size = this.estimateSize(value);
if (size > 1024 * 1024) {
// 1MB
score -= 0.2;
}
// Tag boost
if (options.tags && options.tags.length > 0) {
score += Math.min(options.tags.length * 0.1, 0.3);
}
// Manual relevance boost
if (options.relevanceBoost) {
score += options.relevanceBoost;
}
return Math.max(0, Math.min(1, score));
}
inferType(key, value) {
if (key.includes('file') || key.includes('path'))
return 'file';
if (key.includes('conversation') || key.includes('chat'))
return 'conversation';
if (key.includes('execution') || key.includes('task'))
return 'execution';
if (key.includes('project') || key.includes('structure'))
return 'project';
return 'temporary';
}
extractKeywords(text) {
// Simple keyword extraction - could be enhanced with NLP
const words = text
.toLowerCase()
.replace(/[^\w\s]/g, ' ')
.split(/\s+/)
.filter(word => word.length > 3)
.filter(word => !this.isStopWord(word));
// Remove duplicates and return top keywords
return [...new Set(words)].slice(0, 10);
}
isStopWord(word) {
const stopWords = new Set([
'the',
'and',
'for',
'are',
'but',
'not',
'you',
'all',
'can',
'had',
'her',
'was',
'one',
'our',
'out',
'day',
'get',
'has',
'him',
'his',
'how',
'its',
'may',
'new',
'now',
'old',
'see',
'two',
'who',
'boy',
'did',
'has',
'let',
'put',
'say',
'she',
'too',
'use',
]);
return stopWords.has(word);
}
matchesKeywords(item, keywords) {
const text = JSON.stringify(item.value).toLowerCase();
const itemKeywords = this.extractKeywords(text);
// Check for keyword overlap
const overlap = keywords.filter(keyword => itemKeywords.some(itemKeyword => itemKeyword.includes(keyword) || keyword.includes(itemKeyword)));
return overlap.length > 0;
}
generateContextSuggestions(task, relevantItems) {
const suggestions = [];
// File-based suggestions
const fileItems = relevantItems.filter(item => item.type === 'file');
if (fileItems.length > 0) {
suggestions.push(`Consider reviewing ${fileItems.length} relevant files from your project`);
}
// Conversation history suggestions
const conversationItems = relevantItems.filter(item => item.type === 'conversation');
if (conversationItems.length > 0) {
suggestions.push(`Reference ${conversationItems.length} previous conversations on similar topics`);
}
// Execution history suggestions
const executionItems = relevantItems.filter(item => item.type === 'execution');
if (executionItems.length > 0) {
suggestions.push(`Learn from ${executionItems.length} previous similar task executions`);
}
// Pattern-based suggestions
const commonTags = this.extractCommonTags(relevantItems);
if (commonTags.length > 0) {
suggestions.push(`Common patterns found: ${commonTags.slice(0, 3).join(', ')}`);
}
return suggestions;
}
extractCommonTags(items) {
const tagCounts = new Map();
items.forEach(item => {
item.metadata.tags.forEach(tag => {
tagCounts.set(tag, (tagCounts.get(tag) || 0) + 1);
});
});
return Array.from(tagCounts.entries())
.filter(([, count]) => count > 1)
.sort((a, b) => b[1] - a[1])
.map(([tag]) => tag);
}
serializeValue(value) {
if (typeof value === 'string')
return value;
return JSON.stringify(value);
}
deserializeValue(value) {
if (typeof value === 'string') {
try {
return JSON.parse(value);
}
catch {
return value;
}
}
return value;
}
estimateSize(value) {
if (typeof value === 'string') {
return value.length * 2; // Assume UTF-16
}
return JSON.stringify(value).length * 2;
}
getTotalSize() {
return Array.from(this.contextStore.values()).reduce((sum, item) => sum + item.size, 0);
}
async removeExpiredItems() {
const now = Date.now();
const toRemove = [];
for (const [key, item] of this.contextStore) {
if (item.metadata.expiresAt && now > item.metadata.expiresAt) {
toRemove.push(key);
}
}
toRemove.forEach(key => this.contextStore.delete(key));
if (toRemove.length > 0) {
logger.debug(`Removed ${toRemove.length} expired context items`);
}
}
async removeLowRelevanceItems(threshold) {
const toRemove = [];
for (const [key, item] of this.contextStore) {
if (item.type === 'temporary' && item.relevanceScore < threshold) {
toRemove.push(key);
}
}
toRemove.forEach(key => this.contextStore.delete(key));
if (toRemove.length > 0) {
logger.debug(`Removed ${toRemove.length} low-relevance items`);
}
}
async ageBasedPruning(targetSize) {
const items = Array.from(this.contextStore.values());
// Sort by age and relevance score (older and less relevant first)
items.sort((a, b) => {
const ageScore = a.timestamp - b.timestamp; // Older items have lower score
const relevanceScore = (b.relevanceScore - a.relevanceScore) * 100000; // Higher relevance preferred
return ageScore + relevanceScore;
});
let currentSize = this.getTotalSize();
let removedCount = 0;
for (const item of items) {
if (currentSize <= targetSize)
break;
// Don't remove high-priority items unless absolutely necessary
if (item.metadata.priority === 'high' && currentSize > targetSize * 1.1) {
continue;
}
this.contextStore.delete(item.key);
currentSize -= item.size;
removedCount++;
}
if (removedCount > 0) {
logger.debug(`Age-based pruning removed ${removedCount} items`);
}
}
compressLongText(text) {
if (typeof text !== 'string' || text.length < 1000) {
return text;
}
// Simple compression: remove excessive whitespace and summarize if very long
let compressed = text.replace(/\s+/g, ' ').trim();
if (compressed.length > 5000) {
// Keep first and last parts, summarize middle
const start = compressed.substring(0, 2000);
const end = compressed.substring(compressed.length - 2000);
compressed = start + '\n\n[... content compressed ...]\n\n' + end;
}
return compressed;
}
groupSimilarItems(items) {
const groups = [];
const used = new Set();
for (const item of items) {
if (used.has(item))
continue;
const similarItems = [item];
used.add(item);
// Find similar items (simplified similarity check)
for (const otherItem of items) {
if (used.has(otherItem) || item === otherItem)
continue;
if (this.areItemsSimilar(item, otherItem)) {
similarItems.push(otherItem);
used.add(otherItem);
}
}
groups.push(similarItems);
}
return groups;
}
areItemsSimilar(item1, item2) {
// Check if items are similar enough to merge
if (item1.type !== item2.type)
return false;
// Check tag overlap
const tagOverlap = item1.metadata.tags.filter(tag => item2.metadata.tags.includes(tag)).length;
return (tagOverlap > 0 &&
tagOverlap >= Math.min(item1.metadata.tags.length, item2.metadata.tags.length) * 0.5);
}
mergeContextItems(items) {
if (items.length === 1)
return items[0];
// Sort by relevance (highest first)
items.sort((a, b) => b.relevanceScore - a.relevanceScore);
const primary = items[0];
const merged = {
key: `merged_${primary.key}_${items.length}`,
value: {
primary: primary.value,
additional: items.slice(1).map(item => ({
key: item.key,
value: item.value,
timestamp: item.timestamp,
})),
},
timestamp: Math.max(...items.map(item => item.timestamp)),
size: items.reduce((sum, item) => sum + item.size, 0) * 0.8, // Assume 20% compression
relevanceScore: items.reduce((sum, item) => sum + item.relevanceScore, 0) / items.length,
sessionId: primary.sessionId,
type: primary.type,
metadata: {
tags: [...new Set(items.flatMap(item => item.metadata.tags))],
source: 'merged',
priority: items.some(item => item.metadata.priority === 'high') ? 'high' : 'medium',
},
};
return merged;
}
async initializePersistence() {
try {
if (!existsSync(this.persistencePath)) {
await mkdir(this.persistencePath, { recursive: true });
}
// Try to load previous context
await this.loadPreviousSessionContext();
}
catch (error) {
logger.warn('Failed to initialize context persistence:', error);
}
}
startPeriodicPersistence() {
this.persistenceTimer = createManagedInterval(async () => {
const timeSinceLastPersist = Date.now() - this.lastPersistTime;
if (timeSinceLastPersist >= this.persistenceInterval) {
await this.saveContext();
}
}, 60000); // Check every minute
}
/**
* Shutdown context manager and cleanup resources
*/
async shutdown() {
if (this.persistenceTimer) {
clearManagedInterval(this.persistenceTimer);
this.persistenceTimer = null;
}
// Save final state before shutdown
await this.saveContext();
this.contextStore.clear();
logger.info('✅ EnhancedContextManager shut down successfully');
}
generateSessionId() {
return `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
/**
* Public utility methods
*/
/**
* Clear all context
*/
clearContext() {
this.contextStore.clear();
logger.info('Context cleared');
}
/**
* Get current session ID
*/
getCurrentSessionId() {
return this.currentSessionId;
}
/**
* Start new session
*/
startNewSession() {
this.currentSessionId = this.generateSessionId();
logger.info(`Started new session: ${this.currentSessionId}`);
return this.currentSessionId;
}
/**
* Export context for debugging or analysis
*/
exportContext() {
return {
sessionId: this.currentSessionId,
timestamp: Date.now(),
items: Array.from(this.contextStore.values()),
summary: this.getContextSummary(),
};
}
/**
* Cleanup resources and save state
*/
async cleanup() {
try {
// Stop the persistence timer
if (this.persistenceTimer) {
clearInterval(this.persistenceTimer);
this.persistenceTimer = null;
}
// Save current state
await this.saveContext();
// Clear the context store
this.contextStore.clear();
}
catch (error) {
console.error('Error during context manager cleanup:', error);
}
}
}
//# sourceMappingURL=enhanced-context-manager.js.map