@opichi/smartcode
Version:
Universal code intelligence MCP server - analyze any codebase with TypeScript excellence and multi-language support
271 lines • 10.7 kB
JavaScript
import { promises as fs } from 'fs';
import path from 'path';
import chokidar from 'chokidar';
import { CodeParser } from './parser.js';
import { CodeEmbedder } from './embedder.js';
import { CodeVectorStore } from './qdrant.js';
export class CodebaseIndexer {
parser;
embedder;
vectorStore;
watcher = null;
isIndexing = false;
constructor(qdrantUrl) {
this.parser = new CodeParser();
this.embedder = new CodeEmbedder();
this.vectorStore = new CodeVectorStore(qdrantUrl);
}
async initialize() {
console.log('Initializing codebase indexer...');
await Promise.all([
this.embedder.initialize(),
this.vectorStore.initialize()
]);
console.log('Codebase indexer initialized successfully');
}
async indexProject(projectPath = '.') {
if (this.isIndexing) {
console.log('Indexing already in progress...');
return;
}
this.isIndexing = true;
console.log(`Starting to index project: ${projectPath}`);
try {
// Clear existing data for this project
await this.vectorStore.clear();
// Detect project structure
const projectStructure = await this.detectProjectStructure(projectPath);
console.log(`Detected project type: ${projectStructure.type} (${projectStructure.framework || 'unknown framework'})`);
// Get all code files
const codeFiles = await this.getCodeFiles(projectPath, projectStructure);
console.log(`Found ${codeFiles.length} code files to index`);
// Parse all files
const allNodes = [];
for (const filePath of codeFiles) {
try {
const nodes = this.parser.parseFile(filePath);
allNodes.push(...nodes);
}
catch (error) {
console.warn(`Failed to parse ${filePath}:`, error);
}
}
console.log(`Extracted ${allNodes.length} code nodes`);
// Generate embeddings
const embeddedNodes = await this.embedder.embedNodes(allNodes);
console.log(`Generated embeddings for ${embeddedNodes.filter(n => n.embedding).length} nodes`);
// Index in vector store
await this.vectorStore.indexNodes(embeddedNodes);
console.log('Indexing completed successfully');
}
catch (error) {
console.error('Failed to index project:', error);
throw error;
}
finally {
this.isIndexing = false;
}
}
async startWatching(projectPath = '.') {
if (this.watcher) {
await this.stopWatching();
}
console.log('Starting file watcher...');
this.watcher = chokidar.watch(projectPath, {
ignored: [
'**/node_modules/**',
'**/dist/**',
'**/build/**',
'**/.git/**',
'**/coverage/**',
'**/*.log'
],
persistent: true,
ignoreInitial: true
});
this.watcher
.on('change', (filePath) => this.handleFileChange(filePath))
.on('add', (filePath) => this.handleFileAdd(filePath))
.on('unlink', (filePath) => this.handleFileDelete(filePath));
console.log('File watcher started');
}
async stopWatching() {
if (this.watcher) {
await this.watcher.close();
this.watcher = null;
console.log('File watcher stopped');
}
}
async handleFileChange(filePath) {
if (!this.isCodeFile(filePath))
return;
console.log(`File changed: ${filePath}`);
await this.reindexFile(filePath);
}
async handleFileAdd(filePath) {
if (!this.isCodeFile(filePath))
return;
console.log(`File added: ${filePath}`);
await this.reindexFile(filePath);
}
async handleFileDelete(filePath) {
if (!this.isCodeFile(filePath))
return;
console.log(`File deleted: ${filePath}`);
await this.vectorStore.deleteByFile(filePath);
}
async reindexFile(filePath) {
try {
// Remove existing nodes for this file
await this.vectorStore.deleteByFile(filePath);
// Parse and index the updated file
const nodes = this.parser.parseFile(filePath);
if (nodes.length > 0) {
const embeddedNodes = await this.embedder.embedNodes(nodes);
await this.vectorStore.indexNodes(embeddedNodes);
console.log(`Reindexed ${embeddedNodes.length} nodes from ${filePath}`);
}
}
catch (error) {
console.error(`Failed to reindex ${filePath}:`, error);
}
}
async detectProjectStructure(projectPath) {
const structure = {
type: 'unknown',
entryPoints: [],
configFiles: [],
testDirectories: [],
buildDirectories: []
};
try {
const files = await fs.readdir(projectPath);
// Check for Node.js project
if (files.includes('package.json')) {
structure.type = 'nodejs';
structure.configFiles.push('package.json');
try {
const packageJson = JSON.parse(await fs.readFile(path.join(projectPath, 'package.json'), 'utf8'));
const deps = { ...packageJson.dependencies, ...packageJson.devDependencies };
if (deps.react)
structure.framework = 'react';
else if (deps.vue)
structure.framework = 'vue';
else if (deps.angular)
structure.framework = 'angular';
else if (deps.next)
structure.framework = 'nextjs';
else if (deps.express)
structure.framework = 'express';
else if (deps.fastify)
structure.framework = 'fastify';
else
structure.framework = 'vanilla';
if (files.includes('yarn.lock'))
structure.packageManager = 'yarn';
else if (files.includes('pnpm-lock.yaml'))
structure.packageManager = 'pnpm';
else
structure.packageManager = 'npm';
}
catch (error) {
console.warn('Failed to parse package.json');
}
}
// Check for Python project
else if (files.includes('requirements.txt') || files.includes('pyproject.toml')) {
structure.type = 'python';
if (files.includes('manage.py'))
structure.framework = 'django';
else if (files.includes('app.py') || files.includes('main.py'))
structure.framework = 'flask';
}
// Check for Ruby project
else if (files.includes('Gemfile')) {
structure.type = 'ruby';
if (files.includes('config') && files.includes('app'))
structure.framework = 'rails';
}
// Detect common directories
if (files.includes('test'))
structure.testDirectories.push('test');
if (files.includes('tests'))
structure.testDirectories.push('tests');
if (files.includes('spec'))
structure.testDirectories.push('spec');
if (files.includes('__tests__'))
structure.testDirectories.push('__tests__');
if (files.includes('dist'))
structure.buildDirectories.push('dist');
if (files.includes('build'))
structure.buildDirectories.push('build');
if (files.includes('.next'))
structure.buildDirectories.push('.next');
}
catch (error) {
console.warn('Failed to detect project structure:', error);
}
return structure;
}
async getCodeFiles(projectPath, structure) {
const codeFiles = [];
const extensions = this.getRelevantExtensions(structure);
const walkDir = async (dir) => {
try {
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
// Skip certain directories
if (this.shouldSkipDirectory(entry.name, structure)) {
continue;
}
await walkDir(fullPath);
}
else if (entry.isFile()) {
const ext = path.extname(entry.name).slice(1);
if (extensions.includes(ext)) {
codeFiles.push(fullPath);
}
}
}
}
catch (error) {
console.warn(`Failed to read directory ${dir}:`, error);
}
};
await walkDir(projectPath);
return codeFiles;
}
getRelevantExtensions(structure) {
const baseExtensions = ['js', 'ts', 'jsx', 'tsx'];
switch (structure.type) {
case 'nodejs':
return [...baseExtensions, 'mjs', 'cjs'];
case 'python':
return ['py', 'pyx'];
case 'ruby':
return ['rb', 'rake'];
case 'php':
return ['php'];
case 'java':
return ['java'];
default:
return baseExtensions;
}
}
shouldSkipDirectory(dirName, structure) {
const skipDirs = [
'node_modules', '.git', 'dist', 'build', 'coverage',
'.next', '.nuxt', '__pycache__', '.pytest_cache',
'vendor', 'tmp', 'temp', 'logs'
];
return skipDirs.includes(dirName) || dirName.startsWith('.');
}
isCodeFile(filePath) {
const ext = path.extname(filePath).slice(1);
const codeExtensions = ['js', 'ts', 'jsx', 'tsx', 'py', 'rb', 'php', 'java', 'mjs', 'cjs'];
return codeExtensions.includes(ext);
}
}
//# sourceMappingURL=index.js.map