memory-engineering
Version:
Advanced Memory-Aware Code Context System with claude-flow-inspired architecture, showcasing MongoDB + Voyage AI strategic positioning
272 lines ā¢ 12.3 kB
JavaScript
/**
* Memory Codebase Embed Tool
* Using claude-flow-inspired MemoryManager for conflict-free operations
* Superior to code-context-master - no synchronizer bottlenecks
*/
import { generateMemoryId } from '../utils/id-generator.js';
import { logger } from '../utils/logger.js';
import { promises as fs } from 'fs';
import path from 'path';
import crypto from 'crypto';
import { glob } from 'glob';
export const memoryCodebaseEmbedTool = {
name: 'memory_codebase_embed',
description: 'Index codebase files for memory-aware context (superior to code-context-master)',
inputSchema: {
type: 'object',
properties: {
projectPath: { type: 'string', description: 'Project path to index' },
patterns: {
type: 'array',
items: { type: 'string' },
description: 'File patterns to include (default: common code files)',
default: ['**/*.ts', '**/*.js', '**/*.py', '**/*.rs', '**/*.go', '**/*.java', '**/*.md']
},
maxFileSize: { type: 'number', description: 'Max file size in KB (default: 100)', default: 100 },
forceReindex: { type: 'boolean', description: 'Force reindex all files', default: false }
},
required: ['projectPath']
}
};
export function createMemoryCodebaseEmbedHandler(memoryManager) {
return async (args) => {
try {
const { projectPath, patterns = ['**/*.ts', '**/*.js', '**/*.py', '**/*.rs', '**/*.go', '**/*.java', '**/*.md'], maxFileSize = 100, forceReindex = false } = args;
const stats = {
filesProcessed: 0,
filesSkipped: 0,
totalSize: 0,
errors: []
};
// Get existing hashes to enable incremental indexing (superior to code-context-master)
const existingEntries = await memoryManager.query({
projectPath,
memoryType: 'codebase'
});
const existingHashes = new Map();
for (const entry of existingEntries) {
if (entry.context.filePath && entry.context.hash) {
existingHashes.set(entry.context.filePath, entry.context.hash);
}
}
// Use efficient glob patterns instead of recursive traversal (unlike code-context-master)
const allFiles = [];
for (const pattern of patterns) {
const files = await glob(pattern, {
cwd: projectPath,
ignore: ['**/node_modules/**', '**/dist/**', '**/build/**', '**/.git/**', '**/target/**'],
absolute: false
});
allFiles.push(...files);
}
const uniqueFiles = [...new Set(allFiles)];
for (const filePath of uniqueFiles) {
try {
const fullPath = path.join(projectPath, filePath);
// Single fs.stat call (not double like code-context-master)
const fileStats = await fs.stat(fullPath);
if (!fileStats.isFile()) {
stats.filesSkipped++;
continue;
}
if (fileStats.size > maxFileSize * 1024) {
stats.filesSkipped++;
continue;
}
const content = await fs.readFile(fullPath, 'utf-8');
const hash = crypto.createHash('sha256').update(content).digest('hex');
// Skip if unchanged (incremental indexing)
if (existingHashes.get(filePath) === hash && !forceReindex) {
stats.filesSkipped++;
continue;
}
const ext = path.extname(filePath).toLowerCase();
const language = getLanguageFromExtension(ext);
// Advanced content analysis (inspired by code-context-master)
const lines = content.split('\n');
const nonEmptyLines = lines.filter(line => line.trim().length > 0);
const summary = content.substring(0, 500).trim();
// Extract key code patterns
const codePatterns = extractCodePatterns(content, language);
const imports = extractImports(content, language);
const exports = extractExports(content, language);
const memoryId = generateMemoryId(`${projectPath}:${filePath}`, 'codebase');
const existing = await memoryManager.retrieve(memoryId);
const memoryEntry = {
id: memoryId,
projectPath,
memoryType: 'codebase',
content: {
code: content,
summary,
language,
filePath,
size: fileStats.size,
// Enhanced analysis (inspired by code-context-master)
lineCount: lines.length,
nonEmptyLineCount: nonEmptyLines.length,
codePatterns,
imports,
exports
},
context: {
filePath,
hash,
language,
size: fileStats.size,
lastModified: fileStats.mtime,
// Advanced metrics
complexity: calculateComplexity(content, language),
dependencies: [...imports, ...exports].length,
isTestFile: isTestFile(filePath),
isConfigFile: isConfigFile(filePath)
},
timestamp: new Date(),
tags: ['codebase', 'file', language],
version: existing ? (existing.version + 1) : 1,
metadata: {
created: existing?.metadata?.created || new Date(),
lastUpdated: new Date(),
},
searchableText: `${filePath} ${language} ${summary}`
};
await memoryManager.store(memoryEntry);
stats.filesProcessed++;
stats.totalSize += fileStats.size;
}
catch (error) {
stats.errors.push(`${filePath}: ${error instanceof Error ? error.message : 'Unknown error'}`);
}
}
logger.info('Codebase indexing completed', {
projectPath,
filesProcessed: stats.filesProcessed,
filesSkipped: stats.filesSkipped,
totalSizeKB: Math.round(stats.totalSize / 1024),
errorCount: stats.errors.length
});
return {
content: [{
type: 'text',
text: `ā
Codebase indexed successfully!\n\nš **Statistics:**\n- Files processed: ${stats.filesProcessed}\n- Files skipped: ${stats.filesSkipped}\n- Total size: ${Math.round(stats.totalSize / 1024)} KB\n- Errors: ${stats.errors.length}\n\nš **Superior to code-context-master:**\n- No synchronizer bottlenecks\n- Incremental hashing\n- Single fs.stat calls\n- Efficient glob patterns\n\nš” Use 'memory_search' to find specific code patterns.`
}],
isError: false
};
}
catch (error) {
logger.error('Failed to index codebase', error);
return {
content: [{ type: 'text', text: `ā Indexing error: ${error instanceof Error ? error.message : 'Unknown error'}` }],
isError: true
};
}
};
}
function getLanguageFromExtension(ext) {
const languageMap = {
'.ts': 'typescript',
'.js': 'javascript',
'.py': 'python',
'.rs': 'rust',
'.go': 'go',
'.java': 'java',
'.md': 'markdown',
'.json': 'json',
'.yml': 'yaml',
'.yaml': 'yaml'
};
return languageMap[ext] || 'text';
}
// Advanced code analysis functions (inspired by code-context-master)
function extractCodePatterns(content, language) {
const patterns = [];
switch (language) {
case 'typescript':
case 'javascript':
// Extract class names, function names, interfaces
const classMatches = content.match(/(?:class|interface|type)\s+(\w+)/g);
const functionMatches = content.match(/(?:function|const|let|var)\s+(\w+)/g);
if (classMatches)
patterns.push(...classMatches);
if (functionMatches)
patterns.push(...functionMatches);
break;
case 'python':
const pyClassMatches = content.match(/class\s+(\w+)/g);
const pyFunctionMatches = content.match(/def\s+(\w+)/g);
if (pyClassMatches)
patterns.push(...pyClassMatches);
if (pyFunctionMatches)
patterns.push(...pyFunctionMatches);
break;
default:
// Generic pattern extraction
const genericMatches = content.match(/\b[A-Z][a-zA-Z0-9_]*\b/g);
if (genericMatches)
patterns.push(...genericMatches.slice(0, 10));
}
return [...new Set(patterns)].slice(0, 20); // Limit to 20 unique patterns
}
function extractImports(content, language) {
const imports = [];
switch (language) {
case 'typescript':
case 'javascript':
const importMatches = content.match(/import.*from\s+['"`]([^'"`]+)['"`]/g);
const requireMatches = content.match(/require\(['"`]([^'"`]+)['"`]\)/g);
if (importMatches)
imports.push(...importMatches);
if (requireMatches)
imports.push(...requireMatches);
break;
case 'python':
const pyImportMatches = content.match(/(?:from\s+(\w+)|import\s+(\w+))/g);
if (pyImportMatches)
imports.push(...pyImportMatches);
break;
}
return [...new Set(imports)].slice(0, 10);
}
function extractExports(content, language) {
const exports = [];
switch (language) {
case 'typescript':
case 'javascript':
const exportMatches = content.match(/export\s+(?:default\s+)?(?:class|function|const|let|var|interface|type)\s+(\w+)/g);
if (exportMatches)
exports.push(...exportMatches);
break;
}
return [...new Set(exports)].slice(0, 10);
}
function calculateComplexity(content, language) {
// Simple complexity calculation based on control structures
const controlStructures = [
'if', 'else', 'for', 'while', 'switch', 'case', 'try', 'catch', 'finally'
];
let complexity = 1; // Base complexity
for (const structure of controlStructures) {
const regex = new RegExp(`\\b${structure}\\b`, 'g');
const matches = content.match(regex);
if (matches)
complexity += matches.length;
}
return Math.min(complexity, 50); // Cap at 50
}
function isTestFile(filePath) {
return /\.(test|spec)\.(ts|js|py|rs|go|java)$/.test(filePath) ||
filePath.includes('/test/') ||
filePath.includes('/__tests__/');
}
function isConfigFile(filePath) {
const configFiles = [
'package.json', 'tsconfig.json', 'webpack.config.js', 'babel.config.js',
'.eslintrc', '.prettierrc', 'Dockerfile', 'docker-compose.yml',
'requirements.txt', 'Cargo.toml', 'go.mod', 'pom.xml'
];
const fileName = path.basename(filePath);
return configFiles.some(config => fileName.includes(config)) ||
fileName.startsWith('.') ||
filePath.includes('/config/');
}
//# sourceMappingURL=memory-codebase-embed.js.map