mcp-repl/src/js-vector-indexer.js

MCP REPL with code execution and semantic code search

#!/usr/bin/env node
// Pure JavaScript implementation of code indexing and vector search
// This implementation avoids native dependencies for better Windows compatibility

// Force use of WASM backend to avoid onnxruntime-node dependency
process.env.TFJS_BACKEND = 'wasm';

import fs from 'fs/promises';
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
import path from 'path';
import { pipeline, env } from '@xenova/transformers';

// Configure transformers to use WASM backend
env.backends.onnx.wasm.numThreads = 1;

// Configuration constants
const INDEX_DIR = './code_search_index';
const DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2';
const DEFAULT_DIM = 384; // Dimension size for the chosen model
const DEFAULT_EXTS = ['js', 'ts'];
const DEFAULT_IGNORES = ['node_modules'];
const INDEX_FILE = 'code_index.json';
const VECTOR_INDEX_FILE = 'vector_index.json';

// Global state
let embedder;
let codeChunks = [];
let chunkIds = [];
let isInitialized = false;

// Helper to calculate cosine similarity between two vectors
function cosineSimilarity(vecA, vecB) {
  const dotProduct = vecA.reduce((sum, a, i) => sum + a * vecB[i], 0);
  const magnitudeA = Math.sqrt(vecA.reduce((sum, a) => sum + a * a, 0));
  const magnitudeB = Math.sqrt(vecB.reduce((sum, b) => sum + b * b, 0));
  return dotProduct / (magnitudeA * magnitudeB);
}

// Parse .gitignore file and get ignore patterns
function parseGitignore(rootDir) {
  const gitignorePath = path.join(rootDir, '.gitignore');
  const patterns = [];
  
  if (existsSync(gitignorePath)) {
    try {
      const content = readFileSync(gitignorePath, 'utf8');
      const lines = content.split('\n');
      
      for (const line of lines) {
        const trimmedLine = line.trim();
        // Skip empty lines and comments
        if (trimmedLine && !trimmedLine.startsWith('#')) {
          patterns.push(trimmedLine);
        }
      }
    } catch (error) {
      // Silently handle errors
    }
  }
  
  return patterns;
}

// Check if a path should be ignored based on .gitignore patterns
function shouldIgnorePath(filePath, ignorePatterns, rootDir) {
  // Normalize the path relative to the root directory
  const normalizedPath = path.relative(rootDir, filePath);
  // Ensure consistent path separators (use forward slashes for matching)
  const standardPath = normalizedPath.replace(/\\/g, '/');
  
  for (const pattern of ignorePatterns) {
    // Handle exact matches
    if (standardPath === pattern || standardPath === pattern.replace(/\/$/, '')) {
      return true;
    }
    
    // Handle directory wildcards (e.g., dir/**)
    if (pattern.endsWith('/**') && standardPath.startsWith(pattern.slice(0, -2))) {
      return true;
    }
    
    // Handle file wildcards (e.g., *.log)
    if (pattern.startsWith('*.') && standardPath.endsWith(pattern.slice(1))) {
      return true;
    }
    
    // Handle simple directory patterns (e.g., node_modules/)
    if (pattern.endsWith('/') && standardPath.startsWith(pattern)) {
      return true;
    }
    
    // Handle direct file matches
    if (standardPath === pattern) {
      return true;
    }
  }
  
  return false;
}

// Initialize the in-memory index and embedding model
export async function initialize(indexDir = INDEX_DIR) {
  if (isInitialized) return true;

  try {
    // Create index directory if it doesn't exist
    if (!existsSync(indexDir)) {
      mkdirSync(indexDir, { recursive: true });
    }

    // Initialize embedding model with WASM backend
    try {
      embedder = await pipeline('feature-extraction', DEFAULT_MODEL, {
        env: {
          backends: {
            onnx: {
              wasm: {
                numThreads: 1
              }
            }
          }
        }
      });
    } catch (modelError) {
      // Fallback to a model that works well with Xenova
      embedder = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {
        env: {
          backends: {
            onnx: {
              wasm: {
                numThreads: 1
              }
            }
          }
        }
      });
    }

    // Load existing index if available
    const indexPath = path.join(indexDir, INDEX_FILE);
    
    if (existsSync(indexPath)) {
      try {
        const data = readFileSync(indexPath, 'utf8');
        codeChunks = JSON.parse(data);
        chunkIds = codeChunks.map(chunk => chunk.id);
      } catch (error) {
        codeChunks = [];
        chunkIds = [];
      }
    } else {
      codeChunks = [];
      chunkIds = [];
    }

    isInitialized = true;
    return true;
  } catch (error) {
    return false;
  }
}

// Gather files for indexing
export async function gatherFiles(dir, exts = DEFAULT_EXTS, ignores = DEFAULT_IGNORES) {
  const results = [];
  
  try {
    // Parse .gitignore in the root directory
    const gitignorePatterns = parseGitignore(dir);
    const allIgnores = [...ignores, ...gitignorePatterns];
    
    const entries = await fs.readdir(dir, { withFileTypes: true });
    
    for (const entry of entries) {
      const full = path.join(dir, entry.name);
      
      // Check if the path should be ignored based on standard ignores or .gitignore
      const isIgnoredByPattern = ignores.some(p => full.includes(p));
      const isIgnoredByGitignore = shouldIgnorePath(full, gitignorePatterns, dir);
      
      if (isIgnoredByPattern || isIgnoredByGitignore) {
        continue;
      }
      
      if (entry.isDirectory()) {
        const subDirFiles = await gatherFiles(full, exts, ignores);
        results.push(...subDirFiles);
      } else if (exts.includes(path.extname(entry.name).slice(1))) {
        results.push(full);
      }
    }
  } catch (error) {
    // Silently handle errors
  }
  
  return results;
}

// Extract comment above an element
function extractDocComment(content, position) {
  let docComment = '';
  const linesBefore = content.substring(0, position).split('\n');
  let i = linesBefore.length - 1;
  
  while (i >= 0 && i >= linesBefore.length - 5) {
    const line = linesBefore[i].trim();
    if (line.startsWith('//') || line.startsWith('/*') || line.startsWith('*')) {
      docComment = line.replace(/^\/\/|\*\/|\*|\/\*\*?/g, '').trim() + ' ' + docComment;
    } else if (line === '') {
      i--;
      continue;
    } else {
      break;
    }
    i--;
  }
  
  return docComment.trim();
}

// Extract parameters from function or method signature
function extractParameters(signature) {
  const paramMatch = signature.match(/\((.*?)\)/);
  if (!paramMatch || !paramMatch[1]) return [];
  
  const paramString = paramMatch[1].trim();
  if (!paramString) return [];
  
  return paramString.split(',')
    .map(param => {
      // Handle destructuring or complex params
      const cleanParam = param.trim().replace(/[{}[\]]/g, '');
      const parts = cleanParam.split('='); // Handle default values
      const nameWithType = parts[0].trim();
      
      // Try to separate type annotations (for TypeScript)
      const typeSplit = nameWithType.split(':');
      const name = typeSplit[0].trim();
      const type = typeSplit.length > 1 ? typeSplit[1].trim() : '';
      
      return { name, type };
    })
    .filter(p => p.name && p.name !== '');
}

// Extract return type from function signature (TypeScript)
function extractReturnType(signature, code) {
  // Check for TypeScript return type annotation
  const returnTypeMatch = signature.match(/\)(?:\s*:\s*([^{]+))?/);
  if (returnTypeMatch && returnTypeMatch[1]) {
    return returnTypeMatch[1].trim();
  }
  
  // Try to infer from return statements
  const returnMatches = code.match(/return\s+([^;]+)/g);
  if (returnMatches && returnMatches.length > 0) {
    // Just indicate there are returns but don't try to infer type
    return 'inferred';
  }
  
  return '';
}

// Extract exported status
function isExported(content, position) {
  const linesBefore = content.substring(0, position).split('\n');
  const currentLine = linesBefore[linesBefore.length - 1];
  return currentLine.includes('export ');
}

// Extract code structure from a file
export async function extractChunks(filePath) {
  try {
    const content = await fs.readFile(filePath, 'utf-8');
    const stat = await fs.stat(filePath);
    const chunks = [];
    const fileName = path.basename(filePath);
    const fileScope = { 
      id: Buffer.from(`file-${filePath}`).toString('base64').replace(/[^a-zA-Z0-9]/g, '').substring(0, 16),
      type: 'file',
      name: fileName,
      path: filePath,
      children: [],
      exports: []
    };
    
    // Map of element IDs to their relationship data
    const relationships = new Map();
    
    // Extract functions
    const funcRegex = /(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\([^)]*\)\s*(?::\s*[^{]+)?\s*{/g;
    let funcMatch;
    
    while ((funcMatch = funcRegex.exec(content)) !== null) {
      const funcName = funcMatch[1];
      const funcStart = funcMatch.index;
      
      // Find the function end
      let openBraces = 1;
      let funcEnd = funcStart + funcMatch[0].length;
      
      for (let i = funcEnd; i < content.length; i++) {
        if (content[i] === '{') openBraces++;
        else if (content[i] === '}') openBraces--;
        
        if (openBraces === 0) {
          funcEnd = i + 1;
          break;
        }
      }
      
      const funcCode = content.substring(funcStart, funcEnd);
      const lines = funcCode.split('\n').length;
      const startPos = content.substring(0, funcStart).split('\n').length - 1;
      const endPos = startPos + lines - 1;
      const docComment = extractDocComment(content, funcStart);
      const isExportedFunc = isExported(content, funcStart);
      const parameters = extractParameters(funcMatch[0]);
      const returnType = extractReturnType(funcMatch[0], funcCode);
      
      const funcChunk = {
        id: Buffer.from(`function-${funcName}-${filePath}`).toString('base64').replace(/[^a-zA-Z0-9]/g, '').substring(0, 16),
        type: 'function',
        name: funcName,
        qualifiedName: funcName,
        file: filePath,
        startLine: startPos,
        endLine: endPos,
        lines,
        code: funcCode,
        mtime: stat.mtimeMs,
        doc: docComment,
        isExported: isExportedFunc,
        parameters,
        returnType,
        complexity: calculateComplexity(funcCode)
      };
      
      chunks.push(funcChunk);
      fileScope.children.push(funcChunk.id);
      if (isExportedFunc) {
        fileScope.exports.push(funcChunk.id);
      }
      
      // Store relationships
      relationships.set(funcChunk.id, {
        calls: extractFunctionCalls(funcCode),
        dependencies: extractDependencies(funcCode)
      });
    }
    
    // Extract classes
    const classRegex = /(?:export\s+)?class\s+(\w+)(?:\s+extends\s+(\w+))?\s*{/g;
    let classMatch;
    
    while ((classMatch = classRegex.exec(content)) !== null) {
      const className = classMatch[1];
      const extendedClass = classMatch[2] || null;
      const classStart = classMatch.index;
      
      // Find the class end
      let openBraces = 1;
      let classEnd = classStart + classMatch[0].length;
      
      for (let i = classEnd; i < content.length; i++) {
        if (content[i] === '{') openBraces++;
        else if (content[i] === '}') openBraces--;
        
        if (openBraces === 0) {
          classEnd = i + 1;
          break;
        }
      }
      
      const classCode = content.substring(classStart, classEnd);
      const lines = classCode.split('\n').length;
      const startPos = content.substring(0, classStart).split('\n').length - 1;
      const endPos = startPos + lines - 1;
      const docComment = extractDocComment(content, classStart);
      const isExportedClass = isExported(content, classStart);
      
      const classChunk = {
        id: Buffer.from(`class-${className}-${filePath}`).toString('base64').replace(/[^a-zA-Z0-9]/g, '').substring(0, 16),
        type: 'class',
        name: className,
        qualifiedName: className,
        parentClass: extendedClass,
        file: filePath,
        startLine: startPos,
        endLine: endPos,
        lines,
        code: classCode,
        mtime: stat.mtimeMs,
        doc: docComment,
        isExported: isExportedClass,
        methods: [],
        properties: []
      };
      
      chunks.push(classChunk);
      fileScope.children.push(classChunk.id);
      if (isExportedClass) {
        fileScope.exports.push(classChunk.id);
      }
      
      // Extract class methods
      const methodRegex = /(?:async\s+)?(?:static\s+)?(?:get|set)?\s*(\w+)\s*\([^)]*\)\s*(?::\s*[^{]+)?\s*{/g;
      let methodMatch;
      const methodIds = [];
      
      while ((methodMatch = methodRegex.exec(classCode)) !== null) {
        const methodName = methodMatch[1];
        // Skip constructor and private methods
        if (methodName === 'constructor' || !methodName.match(/^[a-zA-Z]/) || methodName.startsWith('_')) continue;
        
        const methodStart = classStart + methodMatch.index;
        
        // Find the method end
        let openBraces = 1;
        let methodEnd = methodStart + methodMatch[0].length;
        
        for (let i = methodEnd; i < classEnd; i++) {
          if (content[i] === '{') openBraces++;
          else if (content[i] === '}') openBraces--;
          
          if (openBraces === 0) {
            methodEnd = i + 1;
            break;
          }
          
          // Don't go past the class boundary
          if (i >= classEnd - 1) {
            methodEnd = classEnd - 1;
            break;
          }
        }
        
        const methodCode = content.substring(methodStart, methodEnd);
        const methodLines = methodCode.split('\n').length;
        const methodStartPos = content.substring(0, methodStart).split('\n').length - 1;
        const methodEndPos = methodStartPos + methodLines - 1;
        const methodDocComment = extractDocComment(content, methodStart);
        const parameters = extractParameters(methodMatch[0]);
        const returnType = extractReturnType(methodMatch[0], methodCode);
        const isStatic = methodMatch[0].includes('static ');
        
        const methodId = Buffer.from(`method-${className}-${methodName}-${filePath}`).toString('base64').replace(/[^a-zA-Z0-9]/g, '').substring(0, 16);
        methodIds.push(methodId);
        
        const methodChunk = {
          id: methodId,
          type: 'method',
          name: methodName,
          qualifiedName: `${className}.${methodName}`,
          parentClass: className,
          parentClassId: classChunk.id,
          file: filePath,
          startLine: methodStartPos,
          endLine: methodEndPos,
          lines: methodLines,
          code: methodCode,
          mtime: stat.mtimeMs,
          doc: methodDocComment,
          parameters,
          returnType,
          isStatic,
          complexity: calculateComplexity(methodCode)
        };
        
        chunks.push(methodChunk);
        classChunk.methods.push(methodId);
        
        // Store relationships
        relationships.set(methodId, {
          calls: extractFunctionCalls(methodCode),
          dependencies: extractDependencies(methodCode)
        });
      }
      
      // Extract class properties
      const propertyRegex = /(?:static\s+)?(?:readonly\s+)?(\w+)\s*(?::\s*([^;=]+))?\s*(?:=|;)/g;
      let propertyMatch;
      
      while ((propertyMatch = propertyRegex.exec(classCode)) !== null) {
        const propName = propertyMatch[1];
        // Skip private properties
        if (propName.startsWith('_') || !propName.match(/^[a-zA-Z]/)) continue;
        
        const propStart = classStart + propertyMatch.index;
        let propEnd = propStart + propertyMatch[0].length;
        
        // Find property end (could be a complex assignment)
        if (content[propEnd - 1] !== ';') {
          for (let i = propEnd; i < classEnd; i++) {
            if (content[i] === ';') {
              propEnd = i + 1;
              break;
            }
          }
        }
        
        const propCode = content.substring(propStart, propEnd);
        const propLines = propCode.split('\n').length;
        const propStartPos = content.substring(0, propStart).split('\n').length - 1;
        const propEndPos = propStartPos + propLines - 1;
        const propType = propertyMatch[2] ? propertyMatch[2].trim() : '';
        const isStatic = propertyMatch[0].includes('static ');
        
        const propId = Buffer.from(`property-${className}-${propName}-${filePath}`).toString('base64').replace(/[^a-zA-Z0-9]/g, '').substring(0, 16);
        
        const propChunk = {
          id: propId,
          type: 'property',
          name: propName,
          qualifiedName: `${className}.${propName}`,
          parentClass: className,
          parentClassId: classChunk.id,
          file: filePath,
          startLine: propStartPos,
          endLine: propEndPos,
          lines: propLines,
          code: propCode,
          mtime: stat.mtimeMs,
          propertyType: propType,
          isStatic
        };
        
        chunks.push(propChunk);
        classChunk.properties.push(propId);
      }
      
      // Add inheritance relationships
      if (extendedClass) {
        relationships.set(classChunk.id, {
          inheritsFrom: extendedClass,
          methods: methodIds
        });
      }
    }
    
    // Extract imports/exports
    const importExportRegex = /(import|export)[\s\S]+?;/g;
    let importMatch;
    
    while ((importMatch = importExportRegex.exec(content)) !== null) {
      const code = importMatch[0];
      const type = importMatch[1] === 'import' ? 'import' : 'export';
      const lines = code.split('\n').length;
      const startPos = content.substring(0, importMatch.index).split('\n').length - 1;
      const endPos = startPos + lines - 1;
      
      // Extract imported/exported elements and module
      let modulePath = '';
      let elements = [];
      
      if (type === 'import') {
        const moduleMatch = code.match(/from\s+['"]([^'"]+)['"]/);
        if (moduleMatch) {
          modulePath = moduleMatch[1];
        }
        
        const elementsMatch = code.match(/{\s*([^}]+)\s*}/);
        if (elementsMatch) {
          elements = elementsMatch[1].split(',').map(e => e.trim());
        } else {
          // Default import
          const defaultMatch = code.match(/import\s+(\w+)/);
          if (defaultMatch) {
            elements = [defaultMatch[1] + ' (default)'];
          }
        }
      } else {
        // Export
        const namedExport = code.match(/{\s*([^}]+)\s*}/);
        if (namedExport) {
          elements = namedExport[1].split(',').map(e => e.trim());
        } else {
          const defaultExport = code.match(/export\s+default\s+(\w+)/);
          if (defaultExport) {
            elements = [defaultExport[1] + ' (default)'];
          }
        }
      }
      
      const chunk = {
        id: Buffer.from(`${type}-${importMatch.index}-${filePath}`).toString('base64').replace(/[^a-zA-Z0-9]/g, '').substring(0, 16),
        type,
        qualifiedName: code.trim(),
        file: filePath,
        startLine: startPos,
        endLine: endPos,
        lines,
        code,
        mtime: stat.mtimeMs,
        doc: '',
        modulePath,
        elements
      };
      
      chunks.push(chunk);
      
      if (type === 'import') {
        // Add dependency relationships
        relationships.set(chunk.id, {
          dependsOn: modulePath,
          imports: elements
        });
      }
    }
    
    // Add file metadata chunk as the first item
    const fileChunk = {
      id: fileScope.id,
      type: 'file',
      name: fileName,
      qualifiedName: filePath,
      file: filePath,
      startLine: 0,
      endLine: content.split('\n').length - 1,
      lines: content.split('\n').length,
      code: content.substring(0, Math.min(150, content.length)) + '...',
      mtime: stat.mtimeMs,
      doc: extractFileHeader(content),
      children: fileScope.children,
      exports: fileScope.exports
    };
    
    chunks.unshift(fileChunk);
    
    // Add relationships data to chunks
    for (const chunk of chunks) {
      if (relationships.has(chunk.id)) {
        chunk.relationships = relationships.get(chunk.id);
      }
    }
    
    return chunks;
  } catch (error) {
    return [];
  }
}

// Calculate code complexity (simplified)
function calculateComplexity(code) {
  let complexity = 1; // Base complexity
  
  // Count control flow statements
  const controlFlow = (code.match(/if|else|for|while|switch|case|catch|try|return|throw/g) || []).length;
  complexity += controlFlow * 0.5;
  
  // Count logical operators
  const logicalOps = (code.match(/&&|\|\|/g) || []).length;
  complexity += logicalOps * 0.3;
  
  return parseFloat(complexity.toFixed(1));
}

// Extract function calls from code
function extractFunctionCalls(code) {
  const calls = [];
  const callRegex = /(\w+)\s*\(/g;
  let callMatch;
  
  while ((callMatch = callRegex.exec(code)) !== null) {
    const calledFunc = callMatch[1];
    // Filter out common keywords that can appear before parentheses
    if (!['if', 'for', 'while', 'switch', 'catch', 'function'].includes(calledFunc)) {
      calls.push(calledFunc);
    }
  }
  
  return [...new Set(calls)]; // Remove duplicates
}

// Extract dependencies from code
function extractDependencies(code) {
  // Simple regex to find variable usage
  const dependencies = [];
  const varRegex = /(\b\w+\b)(?!\s*\(|:)/g;
  let varMatch;
  
  while ((varMatch = varRegex.exec(code)) !== null) {
    const varName = varMatch[1];
    // Filter out keywords and common primitives
    if (!['let', 'const', 'var', 'function', 'class', 'if', 'else', 'return', 
         'true', 'false', 'null', 'undefined', 'this', 'super'].includes(varName)) {
      dependencies.push(varName);
    }
  }
  
  return [...new Set(dependencies)]; // Remove duplicates
}

// Extract file header comments
function extractFileHeader(content) {
  const headerLines = [];
  const lines = content.split('\n');
  
  for (let i = 0; i < Math.min(10, lines.length); i++) {
    const line = lines[i].trim();
    if (line.startsWith('//') || line.startsWith('/*') || line.startsWith('*')) {
      headerLines.push(line.replace(/^\/\/|\*\/|\*|\/\*\*?/g, '').trim());
    } else if (headerLines.length > 0 && line === '') {
      continue;
    } else if (headerLines.length > 0) {
      break;
    }
  }
  
  return headerLines.join(' ');
}

// Create text representation of a chunk for embedding
function createEmbeddingText(chunk) {
  const parts = [];
  
  parts.push(`${chunk.type}: ${chunk.name || chunk.qualifiedName || ''}`);
  
  if (chunk.doc) {
    parts.push(`Documentation: ${chunk.doc}`);
  }
  
  if (chunk.parentClass) {
    parts.push(`In class: ${chunk.parentClass}`);
  }
  
  // Add structural info
  if (chunk.parameters) {
    const paramText = chunk.parameters
      .map(p => p.type ? `${p.name}: ${p.type}` : p.name)
      .join(', ');
    parts.push(`Parameters: ${paramText}`);
  }
  
  if (chunk.returnType) {
    parts.push(`Returns: ${chunk.returnType}`);
  }
  
  if (chunk.complexity) {
    parts.push(`Complexity: ${chunk.complexity}`);
  }
  
  if (chunk.isExported) {
    parts.push('Exported: true');
  }
  
  if (chunk.relationships) {
    if (chunk.relationships.calls && chunk.relationships.calls.length > 0) {
      parts.push(`Calls: ${chunk.relationships.calls.join(', ')}`);
    }
    
    if (chunk.relationships.inheritsFrom) {
      parts.push(`Inherits from: ${chunk.relationships.inheritsFrom}`);
    }
  }
  
  if (chunk.code) {
    // Clean up code to focus on semantics
    const cleanCode = chunk.code
      .replace(/[{};,=()[\]]/g, ' ')
      .replace(/\s+/g, ' ')
      .trim();
    
    parts.push(`Code: ${cleanCode}`);
  }
  
  return parts.join(' ');
}

// Generate embedding for a text
async function generateEmbedding(text) {
  try {
    if (!embedder) {
      return null;
    }
    
    const output = await embedder(text, { 
      pooling: 'mean',
      normalize: true 
    });
    
    return Array.from(output.data);
  } catch (error) {
    return null;
  }
}

// Save index to disk
async function saveIndex(indexDir = INDEX_DIR) {
  try {
    // Save code chunks
    const indexPath = path.join(indexDir, INDEX_FILE);
    await fs.writeFile(indexPath, JSON.stringify(codeChunks));
    
    return true;
  } catch (error) {
    return false;
  }
}

// Synchronize the index with the file system
export async function syncIndex(folders, exts = DEFAULT_EXTS, ignores = DEFAULT_IGNORES) {
  if (!isInitialized) {
    await initialize();
  }
  
  // Gather all files
  const files = [];
  for (const folder of folders) {
    const folderFiles = await gatherFiles(folder, exts, ignores);
    files.push(...folderFiles);
  }
  
  // Process files and extract chunks
  let newChunksCount = 0;
  const allNewChunks = [];
  const updatedChunkIds = new Set();
  
  for (const file of files) {
    try {
      const fileChunks = await extractChunks(file);
      
      for (const chunk of fileChunks) {
        updatedChunkIds.add(chunk.id);
        
        // Check if chunk exists with the same mtime
        const existingIndex = chunkIds.indexOf(chunk.id);
        if (existingIndex !== -1 && codeChunks[existingIndex].mtime === chunk.mtime) {
          continue;
        }
        
        // Generate embedding for the chunk
        const text = createEmbeddingText(chunk);
        chunk.embedding = await generateEmbedding(text);
        
        allNewChunks.push(chunk);
        newChunksCount++;
      }
    } catch (error) {
      // Silently handle errors
    }
  }
  
  // Find chunks to delete (chunks not in updated files)
  const chunksToDelete = codeChunks.filter(chunk => !updatedChunkIds.has(chunk.id));
  
  // Update the in-memory index
  if (allNewChunks.length > 0 || chunksToDelete.length > 0) {
    // Remove deleted chunks
    for (const chunk of chunksToDelete) {
      const index = chunkIds.indexOf(chunk.id);
      if (index !== -1) {
        // Remove from code chunks array
        codeChunks.splice(index, 1);
        chunkIds.splice(index, 1);
      }
    }
    
    // Add new chunks
    for (const chunk of allNewChunks) {
      const existingIndex = chunkIds.indexOf(chunk.id);
      if (existingIndex !== -1) {
        // Update existing chunk
        codeChunks[existingIndex] = chunk;
      } else {
        // Add new chunk
        codeChunks.push(chunk);
        chunkIds.push(chunk.id);
      }
    }
    
    // Save the updated index
    await saveIndex();
  }
  
  return { 
    total: codeChunks.length, 
    new: newChunksCount,
    deleted: chunksToDelete.length
  };
}

// Calculate text match score based on keyword presence
function textMatchScore(query, chunk) {
  // Normalize query and text
  const normalizedQuery = query.toLowerCase();
  const normalizedCode = chunk.code ? chunk.code.toLowerCase() : '';
  const normalizedName = chunk.name ? chunk.name.toLowerCase() : '';
  const normalizedQualifiedName = chunk.qualifiedName ? chunk.qualifiedName.toLowerCase() : '';
  const normalizedDoc = chunk.doc ? chunk.doc.toLowerCase() : '';
  
  let score = 0;
  
  // Check exact matches in name (highest weight)
  if (normalizedName === normalizedQuery || normalizedQualifiedName === normalizedQuery) {
    score += 1.0;
  }
  // Check if name contains query
  else if (normalizedName.includes(normalizedQuery) || normalizedQualifiedName.includes(normalizedQuery)) {
    score += 0.8;
  }
  
  // Check if doc contains query
  if (normalizedDoc.includes(normalizedQuery)) {
    score += 0.5;
  }
  
  // Check if code contains query
  if (normalizedCode.includes(normalizedQuery)) {
    score += 0.3;
  }
  
  // Check individual words
  const queryWords = normalizedQuery.split(/\s+/);
  for (const word of queryWords) {
    if (word.length < 3) continue; // Skip very short words
    
    if (normalizedName.includes(word) || normalizedQualifiedName.includes(word)) {
      score += 0.2;
    }
    if (normalizedDoc.includes(word)) {
      score += 0.1;
    }
    if (normalizedCode.includes(word)) {
      score += 0.05;
    }
  }
  
  return Math.min(score, 1.0); // Cap at 1.0
}

// Query the index
export async function queryIndex(query, topK = 8) {
  if (!isInitialized) {
    await initialize();
  }
  
  try {
    if (codeChunks.length === 0) {
      return [];
    }
    
    // Generate embedding for the query
    const queryEmbedding = await generateEmbedding(query);
    
    let scoredResults = [];
    
    // If we have a valid embedding, do vector search
    if (queryEmbedding) {
      try {
        // Score all chunks using vector similarity
        scoredResults = codeChunks.map(chunk => {
          // Calculate vector similarity if chunk has embedding
          let similarityScore = 0;
          if (chunk.embedding) {
            similarityScore = cosineSimilarity(queryEmbedding, chunk.embedding);
          }
          
          // Get text match score
          const textScore = textMatchScore(query, chunk);
          
          // Combine scores (70% vector similarity, 30% text matching)
          const combinedScore = (similarityScore * 0.7) + (textScore * 0.3);
          
          return {
            score: combinedScore,
            chunk
          };
        });
      } catch (error) {
        // Fall back to text-based search
        scoredResults = codeChunks.map(chunk => ({
          score: textMatchScore(query, chunk),
          chunk
        }));
      }
    } else {
      // Use text-based search only
      scoredResults = codeChunks.map(chunk => ({
        score: textMatchScore(query, chunk),
        chunk
      }));
    }
    
    // Filter out zero scores and sort by score descending
    const filteredResults = scoredResults
      .filter(result => result.score > 0)
      .sort((a, b) => b.score - a.score)
      .slice(0, topK);
    
    // Format results for display
    const results = filteredResults.map(result => {
      const chunk = result.chunk;
      
      // Base result structure
      const formattedResult = {
        score: parseFloat(result.score.toFixed(3)),
        file: chunk.file,
        startLine: chunk.startLine + 1,
        endLine: chunk.endLine + 1,
        type: chunk.type,
        name: chunk.name || '',
        qualifiedName: chunk.qualifiedName || '',
        lines: chunk.lines,
        doc: chunk.doc || '',
        code: chunk.code ? (chunk.code.length > 140 ? 
          chunk.code.replace(/\s+/g, ' ').slice(0, 140) + '...' : 
          chunk.code.replace(/\s+/g, ' ')
        ) : ''
      };
      
      // Add structural metadata based on chunk type
      switch (chunk.type) {
        case 'file':
          formattedResult.structure = {
            childCount: chunk.children ? chunk.children.length : 0,
            exportCount: chunk.exports ? chunk.exports.length : 0
          };
          break;
          
        case 'function':
          formattedResult.structure = {
            isExported: chunk.isExported || false,
            complexity: chunk.complexity || 1,
            parameters: chunk.parameters || [],
            returnType: chunk.returnType || '',
            calls: chunk.relationships?.calls || []
          };
          break;
          
        case 'class':
          formattedResult.structure = {
            isExported: chunk.isExported || false,
            methodCount: chunk.methods ? chunk.methods.length : 0,
            propertyCount: chunk.properties ? chunk.properties.length : 0,
            parentClass: chunk.parentClass || null,
            inheritsFrom: chunk.relationships?.inheritsFrom || null
          };
          break;
          
        case 'method':
          formattedResult.structure = {
            parentClass: chunk.parentClass || '',
            isStatic: chunk.isStatic || false,
            complexity: chunk.complexity || 1,
            parameters: chunk.parameters || [],
            returnType: chunk.returnType || '',
            calls: chunk.relationships?.calls || []
          };
          break;
          
        case 'property':
          formattedResult.structure = {
            parentClass: chunk.parentClass || '',
            isStatic: chunk.isStatic || false,
            propertyType: chunk.propertyType || ''
          };
          break;
          
        case 'import':
          formattedResult.structure = {
            modulePath: chunk.modulePath || '',
            importedElements: chunk.elements || []
          };
          break;
          
        case 'export':
          formattedResult.structure = {
            exportedElements: chunk.elements || []
          };
          break;
      }
      
      // Add relationships data if available
      if (chunk.relationships) {
        formattedResult.relationships = {};
        
        if (chunk.relationships.calls && chunk.relationships.calls.length > 0) {
          formattedResult.relationships.calls = chunk.relationships.calls;
        }
        
        if (chunk.relationships.dependencies && chunk.relationships.dependencies.length > 0) {
          formattedResult.relationships.dependencies = chunk.relationships.dependencies;
        }
        
        if (chunk.relationships.inheritsFrom) {
          formattedResult.relationships.inheritsFrom = chunk.relationships.inheritsFrom;
        }
        
        if (chunk.relationships.dependsOn) {
          formattedResult.relationships.dependsOn = chunk.relationships.dependsOn;
        }
      }
      
      return formattedResult;
    });
    
    return results;
  } catch (error) {
    return [];
  }
}