@versatil/sdlc-framework
Version:
🚀 AI-Native SDLC framework with 11-MCP ecosystem, RAG memory, OPERA orchestration, and 6 specialized agents achieving ZERO CONTEXT LOSS. Features complete CI/CD pipeline with 7 GitHub workflows (MCP testing, security scanning, performance benchmarking),
778 lines • 29.5 kB
JavaScript
/**
* Intelligent Context Caching Layer
* Provides 10x faster environmental analyses through smart caching strategies
*
* Features:
* - Multi-layer caching (memory, disk, distributed)
* - Intelligent invalidation based on file changes
* - Context similarity detection for cache hits
* - Learning from cache patterns for optimization
* - Cross-project knowledge sharing
* - Predictive pre-caching
*/
import { EventEmitter } from 'events';
import { promises as fs } from 'fs';
import { join, resolve } from 'path';
import { createHash } from 'crypto';
import { watch } from 'chokidar';
export class IntelligentContextCache extends EventEmitter {
constructor(config = {}) {
super();
this.memoryCache = new Map();
this.watchers = new Map();
this.learningData = new Map();
this.similarityThreshold = 0.85;
this.compressionLevel = 6;
this.config = {
memoryLimit: 500 * 1024 * 1024, // 500MB
diskLimit: 2 * 1024 * 1024 * 1024, // 2GB
ttl: 24 * 60 * 60 * 1000, // 24 hours
maxEntries: 10000,
persistentStorage: true,
distributedMode: false,
learningEnabled: true,
preloadPatterns: ['package.json', 'tsconfig.json', '*.config.*'],
compressionEnabled: true,
...config
};
this.diskCachePath = join(process.cwd(), '.versatil', 'cache');
this.stats = this.initializeStats();
this.initialize();
}
initializeStats() {
return {
hitRate: 0,
missRate: 0,
totalRequests: 0,
totalHits: 0,
totalMisses: 0,
averageResponseTime: 0,
memoryUsage: 0,
diskUsage: 0,
entriesCount: 0,
topPatterns: [],
recentActivity: []
};
}
async initialize() {
try {
await fs.mkdir(this.diskCachePath, { recursive: true });
if (this.config.persistentStorage) {
await this.loadPersistedCache();
}
if (this.config.learningEnabled) {
await this.loadLearningData();
}
this.startCacheMaintenance();
this.emit('initialized', { cacheSize: this.memoryCache.size });
}
catch (error) {
this.emit('error', {
phase: 'initialization',
error: error instanceof Error ? error.message : String(error)
});
}
}
async get(key, projectPath) {
const startTime = Date.now();
this.stats.totalRequests++;
try {
// Try memory cache first
const memoryEntry = this.memoryCache.get(key);
if (memoryEntry && !this.isExpired(memoryEntry)) {
memoryEntry.metadata.accessCount++;
memoryEntry.metadata.lastAccessed = Date.now();
this.recordHit(key, startTime);
return memoryEntry.data;
}
// Try similarity matching for intelligent cache hits
if (projectPath) {
const similarMatch = await this.findSimilarContext(key, projectPath);
if (similarMatch && similarMatch.confidence > this.similarityThreshold) {
this.recordSimilarityHit(key, similarMatch, startTime);
return this.adaptContextForProject(similarMatch.entry.data, projectPath);
}
}
// Try disk cache
const diskEntry = await this.loadFromDisk(key);
if (diskEntry && !this.isExpired(diskEntry)) {
this.memoryCache.set(key, diskEntry);
diskEntry.metadata.accessCount++;
diskEntry.metadata.lastAccessed = Date.now();
this.recordHit(key, startTime);
return diskEntry.data;
}
this.recordMiss(key, startTime);
return null;
}
catch (error) {
this.emit('error', {
operation: 'get',
key,
error: error instanceof Error ? error.message : String(error)
});
this.recordMiss(key, startTime);
return null;
}
}
async set(key, data, metadata = {}, invalidationRules = []) {
try {
const entry = {
id: this.generateId(),
key,
data,
metadata: {
projectPath: process.cwd(),
filePatterns: [],
dependencies: [],
timestamp: Date.now(),
accessCount: 0,
lastAccessed: Date.now(),
size: this.calculateSize(data),
tags: [],
similarity: 0,
...metadata
},
expiry: Date.now() + this.config.ttl,
invalidationRules
};
// Memory cache
this.memoryCache.set(key, entry);
// Disk cache
if (this.config.persistentStorage) {
await this.saveToDisk(entry);
}
// Set up file watchers for invalidation
this.setupInvalidationWatchers(entry);
// Learning
if (this.config.learningEnabled) {
this.updateLearningData(entry);
}
// Maintain cache limits
await this.enforceLimits();
this.stats.entriesCount = this.memoryCache.size;
this.emit('cached', { key, size: entry.metadata.size });
}
catch (error) {
this.emit('error', {
operation: 'set',
key,
error: error instanceof Error ? error.message : String(error)
});
}
}
async invalidate(key) {
try {
this.memoryCache.delete(key);
await this.removeFromDisk(key);
this.removeWatcher(key);
this.stats.entriesCount = this.memoryCache.size;
this.emit('invalidated', { key });
}
catch (error) {
this.emit('error', {
operation: 'invalidate',
key,
error: error instanceof Error ? error.message : String(error)
});
}
}
async clear() {
try {
this.memoryCache.clear();
this.watchers.forEach(watcher => watcher.close());
this.watchers.clear();
if (this.config.persistentStorage) {
await fs.rmdir(this.diskCachePath, { recursive: true });
await fs.mkdir(this.diskCachePath, { recursive: true });
}
this.stats = this.initializeStats();
this.emit('cleared');
}
catch (error) {
this.emit('error', {
operation: 'clear',
error: error instanceof Error ? error.message : String(error)
});
}
}
async warmup(projectPath) {
try {
const patterns = this.config.preloadPatterns;
const preloadTasks = patterns.map(pattern => this.preloadPattern(pattern, projectPath));
await Promise.all(preloadTasks);
this.emit('warmed_up', { projectPath, patterns: patterns.length });
}
catch (error) {
this.emit('error', {
operation: 'warmup',
projectPath,
error: error instanceof Error ? error.message : String(error)
});
}
}
async scanAndCache(projectPath) {
const scanStartTime = Date.now();
try {
const cacheKey = this.generateProjectCacheKey(projectPath);
// Check cache first
const cached = await this.get(cacheKey, projectPath);
if (cached) {
this.emit('scan_cache_hit', { projectPath, cacheKey });
return cached;
}
// Perform full scan
const scanResult = await this.performContextScan(projectPath);
scanResult.scanDuration = Date.now() - scanStartTime;
// Cache the result
await this.set(cacheKey, scanResult, {
projectPath,
filePatterns: await this.getProjectFilePatterns(projectPath),
dependencies: await this.getProjectDependencies(projectPath),
tags: ['context_scan', 'project_analysis']
}, [
{ type: 'file_change', pattern: 'package.json' },
{ type: 'file_change', pattern: 'tsconfig.json' },
{ type: 'time_based', maxAge: this.config.ttl }
]);
this.emit('scan_completed', {
projectPath,
cacheKey,
duration: scanResult.scanDuration
});
return scanResult;
}
catch (error) {
this.emit('error', {
operation: 'scanAndCache',
projectPath,
error: error instanceof Error ? error.message : String(error)
});
throw error;
}
}
getStats() {
this.updateStats();
return { ...this.stats };
}
async exportCache(filePath) {
try {
const exportData = {
version: '1.0.0',
timestamp: Date.now(),
config: this.config,
stats: this.stats,
entries: Array.from(this.memoryCache.entries()),
learningData: Array.from(this.learningData.entries())
};
await fs.writeFile(filePath, JSON.stringify(exportData, null, 2));
this.emit('exported', { filePath, entriesCount: this.memoryCache.size });
}
catch (error) {
this.emit('error', {
operation: 'export',
filePath,
error: error instanceof Error ? error.message : String(error)
});
}
}
async importCache(filePath) {
try {
const data = await fs.readFile(filePath, 'utf-8');
const importData = JSON.parse(data);
// Validate version compatibility
if (importData.version !== '1.0.0') {
throw new Error(`Incompatible cache version: ${importData.version}`);
}
// Import entries
for (const [key, entry] of importData.entries) {
if (!this.isExpired(entry)) {
this.memoryCache.set(key, entry);
}
}
// Import learning data
if (importData.learningData) {
for (const [key, data] of importData.learningData) {
this.learningData.set(key, data);
}
}
this.stats.entriesCount = this.memoryCache.size;
this.emit('imported', {
filePath,
entriesCount: importData.entries.length,
validEntries: this.memoryCache.size
});
}
catch (error) {
this.emit('error', {
operation: 'import',
filePath,
error: error instanceof Error ? error.message : String(error)
});
}
}
async findSimilarContext(key, projectPath) {
try {
const projectSignature = await this.generateProjectSignature(projectPath);
let bestMatch = null;
for (const [cacheKey, entry] of this.memoryCache) {
if (cacheKey === key || this.isExpired(entry))
continue;
const similarity = await this.calculateSimilarity(projectSignature, entry);
if (similarity > this.similarityThreshold &&
(!bestMatch || similarity > bestMatch.similarity)) {
bestMatch = {
entry,
similarity,
confidence: this.calculateConfidence(similarity, entry),
reasons: this.getSimilarityReasons(projectSignature, entry)
};
}
}
return bestMatch;
}
catch (error) {
this.emit('error', {
operation: 'findSimilarContext',
error: error instanceof Error ? error.message : String(error)
});
return null;
}
}
async generateProjectSignature(projectPath) {
try {
const signature = {
packageJson: await this.safeReadJson(join(projectPath, 'package.json')),
tsConfig: await this.safeReadJson(join(projectPath, 'tsconfig.json')),
fileStructure: await this.getFileStructureHash(projectPath),
dependencies: await this.getProjectDependencies(projectPath),
patterns: await this.getProjectFilePatterns(projectPath)
};
return signature;
}
catch (error) {
return {};
}
}
async calculateSimilarity(signature, entry) {
let score = 0;
let factors = 0;
// Package.json similarity
if (signature.packageJson && entry.metadata.tags.includes('project_analysis')) {
const depSimilarity = this.compareDependencies(signature.packageJson.dependencies || {}, signature.packageJson.devDependencies || {});
score += depSimilarity * 0.4;
factors += 0.4;
}
// File structure similarity
if (signature.fileStructure && entry.metadata.filePatterns.length > 0) {
const structureSimilarity = this.compareFileStructures(signature.patterns, entry.metadata.filePatterns);
score += structureSimilarity * 0.3;
factors += 0.3;
}
// Technology stack similarity
const techSimilarity = this.compareTechnologyStacks(signature, entry);
score += techSimilarity * 0.3;
factors += 0.3;
return factors > 0 ? score / factors : 0;
}
calculateConfidence(similarity, entry) {
const ageFactor = Math.max(0, 1 - (Date.now() - entry.metadata.timestamp) / this.config.ttl);
const accessFactor = Math.min(1, entry.metadata.accessCount / 10);
const sizeFactor = entry.metadata.size > 0 ? 1 : 0.5;
return (similarity * 0.6 + ageFactor * 0.2 + accessFactor * 0.1 + sizeFactor * 0.1);
}
getSimilarityReasons(signature, entry) {
const reasons = [];
if (signature.packageJson) {
reasons.push('Similar package.json dependencies');
}
if (signature.patterns && entry.metadata.filePatterns.length > 0) {
reasons.push('Similar file structure patterns');
}
if (entry.metadata.tags.includes('project_analysis')) {
reasons.push('Previous project analysis available');
}
return reasons;
}
adaptContextForProject(cachedData, projectPath) {
// Adapt cached context data for the specific project
const adapted = JSON.parse(JSON.stringify(cachedData));
if (adapted.projectStructure) {
adapted.projectStructure.path = projectPath;
adapted.projectStructure.name = projectPath.split('/').pop();
}
if (adapted.agentRecommendations) {
adapted.agentRecommendations.adapted = true;
adapted.agentRecommendations.originalProject = cachedData.projectStructure?.path;
}
adapted.cached = true;
adapted.adaptedAt = Date.now();
return adapted;
}
async performContextScan(projectPath) {
// Simulate comprehensive context scanning
// In real implementation, this would integrate with existing scanning logic
return {
projectStructure: await this.analyzeProjectStructure(projectPath),
dependencies: await this.analyzeDependencies(projectPath),
configurations: await this.analyzeConfigurations(projectPath),
codeMetrics: await this.analyzeCodeMetrics(projectPath),
agentRecommendations: await this.generateAgentRecommendations(projectPath),
patterns: await this.analyzePatterns(projectPath),
timestamp: Date.now(),
scanDuration: 0 // Will be set by caller
};
}
async analyzeProjectStructure(projectPath) {
// Placeholder for project structure analysis
return {
path: projectPath,
name: projectPath.split('/').pop(),
type: 'web-application',
framework: 'typescript',
structure: 'standard'
};
}
async analyzeDependencies(projectPath) {
const packageJson = await this.safeReadJson(join(projectPath, 'package.json'));
return {
dependencies: packageJson?.dependencies || {},
devDependencies: packageJson?.devDependencies || {},
peerDependencies: packageJson?.peerDependencies || {}
};
}
async analyzeConfigurations(projectPath) {
return {
typescript: await this.safeReadJson(join(projectPath, 'tsconfig.json')),
eslint: await this.safeReadJson(join(projectPath, '.eslintrc.json')),
prettier: await this.safeReadJson(join(projectPath, '.prettierrc')),
jest: await this.safeReadJson(join(projectPath, 'jest.config.js'))
};
}
async analyzeCodeMetrics(projectPath) {
// Placeholder for code metrics analysis
return {
files: 0,
lines: 0,
complexity: 'medium',
testCoverage: 0
};
}
async generateAgentRecommendations(projectPath) {
// Placeholder for agent recommendations
return {
recommended: ['maria-qa', 'james-frontend', 'marcus-backend'],
confidence: 0.9,
reasons: ['TypeScript project', 'Web application structure']
};
}
async analyzePatterns(projectPath) {
return {
architectural: ['mvc', 'component-based'],
testing: ['unit', 'integration'],
deployment: ['npm', 'node']
};
}
generateProjectCacheKey(projectPath) {
const normalized = resolve(projectPath);
return `project_scan:${this.hashString(normalized)}`;
}
async getProjectFilePatterns(projectPath) {
// Simplified file pattern detection
const patterns = [];
try {
const files = await fs.readdir(projectPath);
for (const file of files) {
if (file.endsWith('.json') || file.endsWith('.js') || file.endsWith('.ts')) {
patterns.push(file);
}
}
}
catch (error) {
// Ignore errors
}
return patterns;
}
async getProjectDependencies(projectPath) {
const packageJson = await this.safeReadJson(join(projectPath, 'package.json'));
if (!packageJson)
return [];
return [
...Object.keys(packageJson.dependencies || {}),
...Object.keys(packageJson.devDependencies || {})
];
}
compareDependencies(deps1, deps2) {
const set1 = new Set(Object.keys(deps1));
const set2 = new Set(Object.keys(deps2));
const intersection = new Set([...set1].filter(x => set2.has(x)));
const union = new Set([...set1, ...set2]);
return union.size > 0 ? intersection.size / union.size : 0;
}
compareFileStructures(patterns1, patterns2) {
const set1 = new Set(patterns1);
const set2 = new Set(patterns2);
const intersection = new Set([...set1].filter(x => set2.has(x)));
const union = new Set([...set1, ...set2]);
return union.size > 0 ? intersection.size / union.size : 0;
}
compareTechnologyStacks(signature, entry) {
// Simplified technology comparison
let matches = 0;
let total = 0;
const technologies = ['typescript', 'react', 'vue', 'node', 'express'];
for (const tech of technologies) {
total++;
if (signature.packageJson?.dependencies?.[tech] &&
entry.metadata.tags.includes(tech)) {
matches++;
}
}
return total > 0 ? matches / total : 0;
}
async safeReadJson(filePath) {
try {
const content = await fs.readFile(filePath, 'utf-8');
return JSON.parse(content);
}
catch (error) {
return null;
}
}
async getFileStructureHash(projectPath) {
try {
const files = await fs.readdir(projectPath);
const structure = files.sort().join(',');
return this.hashString(structure);
}
catch (error) {
return '';
}
}
hashString(str) {
return createHash('md5').update(str).digest('hex');
}
generateId() {
return createHash('md5').update(`${Date.now()}-${Math.random()}`).digest('hex');
}
calculateSize(data) {
return JSON.stringify(data).length;
}
isExpired(entry) {
if (!entry.expiry)
return false;
return Date.now() > entry.expiry;
}
recordHit(key, startTime) {
this.stats.totalHits++;
this.stats.hitRate = this.stats.totalHits / this.stats.totalRequests;
this.updateResponseTime(startTime);
this.addRecentActivity('hit', key);
}
recordMiss(key, startTime) {
this.stats.totalMisses++;
this.stats.missRate = this.stats.totalMisses / this.stats.totalRequests;
this.updateResponseTime(startTime);
this.addRecentActivity('miss', key);
}
recordSimilarityHit(key, match, startTime) {
this.stats.totalHits++;
this.stats.hitRate = this.stats.totalHits / this.stats.totalRequests;
this.updateResponseTime(startTime);
this.addRecentActivity('similarity_hit', key);
this.emit('similarity_match', {
key,
originalKey: match.entry.key,
similarity: match.similarity,
confidence: match.confidence,
reasons: match.reasons
});
}
updateResponseTime(startTime) {
const responseTime = Date.now() - startTime;
this.stats.averageResponseTime =
(this.stats.averageResponseTime * (this.stats.totalRequests - 1) + responseTime) /
this.stats.totalRequests;
}
addRecentActivity(operation, key) {
this.stats.recentActivity.unshift({
timestamp: Date.now(),
operation,
key
});
// Keep only last 100 activities
if (this.stats.recentActivity.length > 100) {
this.stats.recentActivity = this.stats.recentActivity.slice(0, 100);
}
}
updateStats() {
this.stats.entriesCount = this.memoryCache.size;
this.stats.memoryUsage = Array.from(this.memoryCache.values())
.reduce((total, entry) => total + entry.metadata.size, 0);
}
setupInvalidationWatchers(entry) {
for (const rule of entry.invalidationRules) {
if (rule.type === 'file_change' && rule.pattern) {
const watchPath = join(entry.metadata.projectPath, rule.pattern);
const watcher = watch(watchPath, { ignoreInitial: true });
watcher.on('change', () => {
this.invalidate(entry.key);
});
this.watchers.set(`${entry.key}:${rule.pattern}`, watcher);
}
}
}
removeWatcher(key) {
for (const [watchKey, watcher] of this.watchers) {
if (watchKey.startsWith(`${key}:`)) {
watcher.close();
this.watchers.delete(watchKey);
}
}
}
async loadPersistedCache() {
try {
const cacheFiles = await fs.readdir(this.diskCachePath);
for (const file of cacheFiles) {
if (file.endsWith('.cache.json')) {
const entry = await this.loadFromDisk(file.replace('.cache.json', ''));
if (entry && !this.isExpired(entry)) {
this.memoryCache.set(entry.key, entry);
}
}
}
}
catch (error) {
// Cache directory doesn't exist or is empty
}
}
async loadFromDisk(key) {
try {
const filePath = join(this.diskCachePath, `${key}.cache.json`);
const content = await fs.readFile(filePath, 'utf-8');
return JSON.parse(content);
}
catch (error) {
return null;
}
}
async saveToDisk(entry) {
try {
const filePath = join(this.diskCachePath, `${entry.key}.cache.json`);
await fs.writeFile(filePath, JSON.stringify(entry, null, 2));
}
catch (error) {
this.emit('error', {
operation: 'saveToDisk',
key: entry.key,
error: error instanceof Error ? error.message : String(error)
});
}
}
async removeFromDisk(key) {
try {
const filePath = join(this.diskCachePath, `${key}.cache.json`);
await fs.unlink(filePath);
}
catch (error) {
// File doesn't exist
}
}
async enforceLimits() {
// Memory limit enforcement
const memoryUsage = Array.from(this.memoryCache.values())
.reduce((total, entry) => total + entry.metadata.size, 0);
if (memoryUsage > this.config.memoryLimit ||
this.memoryCache.size > this.config.maxEntries) {
await this.evictLeastUsed();
}
}
async evictLeastUsed() {
const entries = Array.from(this.memoryCache.values())
.sort((a, b) => a.metadata.lastAccessed - b.metadata.lastAccessed);
const toEvict = Math.ceil(entries.length * 0.1); // Evict 10%
for (let i = 0; i < toEvict && i < entries.length; i++) {
this.memoryCache.delete(entries[i].key);
this.removeWatcher(entries[i].key);
}
this.emit('evicted', { count: toEvict });
}
async preloadPattern(pattern, projectPath) {
try {
const glob = await import('glob');
const files = await glob.glob(pattern, { cwd: projectPath });
for (const file of files) {
const cacheKey = `preload:${file}`;
const exists = this.memoryCache.has(cacheKey);
if (!exists) {
const content = await this.safeReadJson(join(projectPath, file));
if (content) {
await this.set(cacheKey, content, {
projectPath,
filePatterns: [file],
tags: ['preload', pattern]
});
}
}
}
}
catch (error) {
// Pattern not found or error reading files
}
}
async loadLearningData() {
try {
const learningPath = join(this.diskCachePath, 'learning.json');
const content = await fs.readFile(learningPath, 'utf-8');
const data = JSON.parse(content);
for (const [key, value] of Object.entries(data)) {
this.learningData.set(key, value);
}
}
catch (error) {
// Learning data doesn't exist yet
}
}
updateLearningData(entry) {
const pattern = entry.metadata.tags.join(',');
const current = this.learningData.get(pattern) || { count: 0, avgSize: 0, avgAccess: 0 };
current.count++;
current.avgSize = (current.avgSize + entry.metadata.size) / 2;
current.lastSeen = Date.now();
this.learningData.set(pattern, current);
}
startCacheMaintenance() {
// Run maintenance every hour
setInterval(async () => {
await this.runMaintenance();
}, 60 * 60 * 1000);
}
async runMaintenance() {
try {
// Remove expired entries
for (const [key, entry] of this.memoryCache) {
if (this.isExpired(entry)) {
await this.invalidate(key);
}
}
// Persist learning data
if (this.config.learningEnabled) {
const learningPath = join(this.diskCachePath, 'learning.json');
const data = Object.fromEntries(this.learningData);
await fs.writeFile(learningPath, JSON.stringify(data, null, 2));
}
this.emit('maintenance_completed', {
entriesCount: this.memoryCache.size,
learningDataSize: this.learningData.size
});
}
catch (error) {
this.emit('error', {
operation: 'maintenance',
error: error instanceof Error ? error.message : String(error)
});
}
}
}
export default IntelligentContextCache;
//# sourceMappingURL=intelligent-context-cache.js.map