cntx-ui/lib/treesitter-semantic-chunker.js

File context management tool with web UI and MCP server for AI development workflows - bundle project files for LLM consumption

/**
 * Treesitter-based Semantic Chunker for JavaScript/TypeScript Files
 * Uses tree-sitter for true AST-based code analysis and semantic chunking
 * Supports JS/TS/JSX/TSX with equal treatment
 * Node ecosystem focus: React components, Express APIs, CLI tools, utilities
 */

import { readFileSync, existsSync } from 'fs'
import { extname, basename, dirname, relative, join } from 'path'
import glob from 'glob'
import { promisify } from 'util'
import Parser from 'tree-sitter'
import JavaScript from 'tree-sitter-javascript'
import TypeScript from 'tree-sitter-typescript'

const globAsync = promisify(glob)

class TreesitterSemanticChunker {
  constructor(options = {}) {
    this.options = {
      includeImports: true,
      includeExports: true,
      detectComponentTypes: true,
      groupRelatedFiles: true,
      minChunkSize: 100,
      maxChunkSize: 50000,
      namingStrategy: 'domain-based', // domain-based, pattern-based, graph-based
      ...options
    }
    
    // Initialize parsers for different languages
    this.parsers = {}
    this.initializeParsers()
    
    // Semantic patterns for Node ecosystem
    this.semanticPatterns = {
      reactComponent: this.isReactComponent.bind(this),
      reactHook: this.isReactHook.bind(this),
      expressRoute: this.isExpressRoute.bind(this),
      expressMiddleware: this.isExpressMiddleware.bind(this),
      cliCommand: this.isCliCommand.bind(this),
      utilityFunction: this.isUtilityFunction.bind(this),
      apiHandler: this.isApiHandler.bind(this),
      typeDefinition: this.isTypeDefinition.bind(this),
      configModule: this.isConfigModule.bind(this)
    }
  }

  /**
   * Initialize tree-sitter parsers for different languages
   */
  initializeParsers() {
    // JavaScript parser
    this.parsers.javascript = new Parser()
    this.parsers.javascript.setLanguage(JavaScript)
    
    // TypeScript parser  
    this.parsers.typescript = new Parser()
    this.parsers.typescript.setLanguage(TypeScript.typescript)
    
    // TSX parser
    this.parsers.tsx = new Parser()
    this.parsers.tsx.setLanguage(TypeScript.tsx)
  }

  /**
   * Get appropriate parser for file extension
   */
  getParser(filePath) {
    const ext = extname(filePath)
    switch (ext) {
      case '.ts': return this.parsers.typescript
      case '.tsx': return this.parsers.tsx
      case '.js':
      case '.jsx':
      default: return this.parsers.javascript
    }
  }

  /**
   * Main entry point - analyze files and create semantic chunks
   */
  async analyzeProject(projectPath, patterns = ['**/*.{js,jsx,ts,tsx}']) {
    console.log('🔍 Starting treesitter-based semantic analysis...')
    
    const files = await this.findFiles(projectPath, patterns)
    console.log(`📁 Found ${files.length} files to analyze`)
    
    const analysis = await this.analyzeFiles(files, projectPath)
    const successfulFiles = Object.keys(analysis).filter(f => !analysis[f].error)
    console.log(`✅ Analyzed ${Object.keys(analysis).length} files (${successfulFiles.length} successful)`)
    if (successfulFiles.length > 0) {
      console.log('📝 Sample successful files:', successfulFiles.slice(0, 5))
    }
    
    const relationshipGraph = this.buildRelationshipGraph(analysis)
    console.log(`🔗 Built relationship graph with ${Object.keys(relationshipGraph).length} nodes`)
    
    const chunks = await this.createSmartChunks(analysis, relationshipGraph)
    console.log(`📦 Created ${chunks.length} semantic chunks`)
    
    return {
      summary: this.generateSummary(analysis, chunks),
      files: analysis,
      chunks: chunks,
      relationshipGraph,
      recommendations: this.generateRecommendations(analysis, chunks)
    }
  }

  /**
   * Find files matching patterns
   */
  async findFiles(projectPath, patterns) {
    const files = []
    
    for (const pattern of patterns) {
      const matches = await globAsync(pattern, {
        cwd: projectPath,
        ignore: [
          'node_modules/**', 'dist/**', 'build/**', '.git/**',
          '*.test.*', '*.spec.*', '**/test/**', '**/tests/**',
          '**/*.min.js', '**/*.bundle.js', '**/coverage/**',
          '**/.next/**', '**/.cache/**', '**/tmp/**', '**/temp/**'
        ]
      })
      
      // Extra filter to ensure no node_modules files get through
      const filteredMatches = matches.filter(file =>
        !file.includes('node_modules') &&
        !file.includes('dist/') &&
        !file.includes('.min.') &&
        !file.includes('.bundle.')
      )
      
      files.push(...filteredMatches)
    }
    
    return [...new Set(files)] // Remove duplicates
  }

  /**
   * Analyze all files using treesitter
   */
  async analyzeFiles(filePaths, projectPath) {
    const analysis = {}
    
    for (const relativePath of filePaths) {
      // Bulletproof check to skip node_modules
      if (relativePath.includes('node_modules')) {
        console.log(`Skipping node_modules file: ${relativePath}`);
        continue;
      }

      const fullPath = join(projectPath, relativePath)
      if (!existsSync(fullPath)) continue
      
      try {
        const content = readFileSync(fullPath, 'utf8')
        const fileAnalysis = await this.analyzeFile(fullPath, content)
        fileAnalysis.path = relativePath // Store relative path
        analysis[relativePath] = fileAnalysis
      } catch (error) {
        // Silently skip files that can't be parsed - they won't be included in semantic analysis
        // This is normal for complex files or unsupported syntax patterns
        analysis[relativePath] = { error: error.message, path: relativePath }
      }
    }
    
    return analysis
  }

  /**
   * Analyze a single file using treesitter AST
   */
  async analyzeFile(filePath, content) {
    const parser = this.getParser(filePath)
    
    // Skip files that are too large or have syntax errors
    if (content.length > 500000) { // Skip files > 500KB
      throw new Error('File too large')
    }
    
    let tree, rootNode
    try {
      // Use simple string parsing (confirmed working in tests)
      tree = parser.parse(content)
      rootNode = tree.rootNode
      
      // Check for parse errors
      if (rootNode.hasError()) {
        throw new Error('Parse error in file')
      }
    } catch (error) {
      throw new Error(`Tree-sitter parse failed: ${error.message}`)
    }
    
    const analysis = {
      path: filePath,
      fileName: basename(filePath),
      dirName: basename(dirname(filePath)),
      extension: extname(filePath),
      size: content.length,
      lines: content.split('\n').length,
      
      // AST-based analysis
      ast: {
        functions: this.extractFunctions(rootNode, content),
        classes: this.extractClasses(rootNode, content),
        imports: this.extractImports(rootNode, content),
        exports: this.extractExports(rootNode, content),
        variables: this.extractVariables(rootNode, content),
        jsxElements: this.extractJsxElements(rootNode, content),
        typeDefinitions: this.extractTypeDefinitions(rootNode, content)
      },
      
      // Semantic classification
      semanticType: this.classifyFileSemantics(rootNode, content, filePath),
      businessDomain: this.extractBusinessDomain(rootNode, content, filePath),
      technicalPatterns: this.identifyTechnicalPatterns(rootNode, content),
      
      // Relationships
      dependencies: this.analyzeDependencies(rootNode, content),
      complexity: this.calculateAstComplexity(rootNode),
      
      // Metadata
      codeSignature: this.generateCodeSignature(rootNode, content)
    }
    
    // Generate semantic tags based on AST analysis
    analysis.semanticTags = this.generateSemanticTags(analysis)
    
    return analysis
  }

  /**
   * Extract function declarations from AST
   */
  extractFunctions(rootNode, content) {
    const functions = []
    
    // Function declarations
    const functionDeclarations = this.queryNode(rootNode, '(function_declaration name: (identifier) @name)')
    functions.push(...functionDeclarations.map(capture => ({
      name: this.getNodeText(capture.node, content),
      type: 'function_declaration',
      startPosition: capture.node.startPosition,
      endPosition: capture.node.endPosition,
      isExported: this.isNodeExported(capture.node)
    })))
    
    // Arrow functions
    const arrowFunctions = this.queryNode(rootNode, '(variable_declarator name: (identifier) @name value: (arrow_function))')
    functions.push(...arrowFunctions.map(capture => ({
      name: this.getNodeText(capture.node, content),
      type: 'arrow_function',
      startPosition: capture.node.startPosition,
      endPosition: capture.node.endPosition,
      isExported: this.isNodeExported(capture.node.parent.parent)
    })))
    
    // Method definitions
    const methods = this.queryNode(rootNode, '(method_definition name: (property_name) @name)')
    functions.push(...methods.map(capture => ({
      name: this.getNodeText(capture.node, content),
      type: 'method',
      startPosition: capture.node.startPosition,
      endPosition: capture.node.endPosition,
      isExported: false // methods are part of classes
    })))
    
    return functions
  }

  /**
   * Extract class declarations from AST
   */
  extractClasses(rootNode, content) {
    const classes = []
    
    const classDeclarations = this.queryNode(rootNode, '(class_declaration name: (identifier) @name)')
    classes.push(...classDeclarations.map(capture => ({
      name: this.getNodeText(capture.node, content),
      type: 'class',
      startPosition: capture.node.startPosition,
      endPosition: capture.node.endPosition,
      isExported: this.isNodeExported(capture.node.parent),
      methods: this.extractClassMethods(capture.node.parent, content)
    })))
    
    return classes
  }

  /**
   * Extract class methods
   */
  extractClassMethods(classNode, content) {
    const methods = []
    
    try {
      const methodNodes = this.queryNode(classNode, '(method_definition)')
      methods.push(...methodNodes.map(capture => ({
        name: this.getNodeText(capture.node, content),
        type: 'method',
        startPosition: capture.node.startPosition,
        endPosition: capture.node.endPosition
      })))
    } catch (error) {
      // Handle case where method extraction fails
    }
    
    return methods
  }

  /**
   * Extract import statements from AST
   */
  extractImports(rootNode, content) {
    const imports = []
    
    const importStatements = this.queryNode(rootNode, '(import_statement source: (string) @source)')
    imports.push(...importStatements.map(capture => {
      const source = this.getNodeText(capture.node, content).replace(/['"]/g, '')
      return {
        source,
        statement: this.getNodeText(capture.node.parent, content),
        isRelative: source.startsWith('.'),
        isExternal: !source.startsWith('.') && !source.startsWith('/'),
        importedNames: this.extractImportedNames(capture.node.parent, content)
      }
    }))
    
    return imports
  }

  /**
   * Extract export statements from AST  
   */
  extractExports(rootNode, content) {
    const exports = []
    
    // Export declarations
    const exportDeclarations = this.queryNode(rootNode, '(export_statement)')
    exports.push(...exportDeclarations.map(capture => {
      const exportNode = capture.node
      const declaration = exportNode.namedChild(0)
      
      if (declaration) {
        return {
          type: declaration.type === 'export_clause' ? 'named' : 'declaration',
          name: this.extractExportName(declaration, content),
          statement: this.getNodeText(exportNode, content),
          isDefault: this.getNodeText(exportNode, content).includes('default')
        }
      }
      return null
    }).filter(Boolean))
    
    return exports
  }

  /**
   * Extract variable declarations from AST
   */
  extractVariables(rootNode, content) {
    const variables = []
    
    const variableDeclarations = this.queryNode(rootNode, '(variable_declarator name: (identifier) @name)')
    variables.push(...variableDeclarations.map(capture => ({
      name: this.getNodeText(capture.node, content),
      type: 'variable',
      startPosition: capture.node.startPosition,
      endPosition: capture.node.endPosition,
      isExported: this.isNodeExported(capture.node.parent.parent),
      declarationType: capture.node.parent.parent.type // const, let, var
    })))
    
    return variables
  }

  /**
   * Extract JSX elements from AST (for React components)
   */
  extractJsxElements(rootNode, content) {
    const jsxElements = []
    
    try {
      const jsxNodes = this.queryNode(rootNode, '(jsx_element)')
      jsxElements.push(...jsxNodes.map(capture => ({
        elementName: this.extractJsxElementName(capture.node, content),
        startPosition: capture.node.startPosition,
        endPosition: capture.node.endPosition
      })))
    } catch (error) {
      // JSX might not be available in JavaScript parser
    }
    
    return jsxElements
  }

  /**
   * Extract TypeScript type definitions from AST
   */
  extractTypeDefinitions(rootNode, content) {
    const types = []
    
    try {
      // Interface declarations
      const interfaces = this.queryNode(rootNode, '(interface_declaration name: (type_identifier) @name)')
      types.push(...interfaces.map(capture => ({
        name: this.getNodeText(capture.node, content),
        type: 'interface',
        startPosition: capture.node.startPosition,
        endPosition: capture.node.endPosition,
        isExported: this.isNodeExported(capture.node.parent)
      })))
      
      // Type alias declarations  
      const typeAliases = this.queryNode(rootNode, '(type_alias_declaration name: (type_identifier) @name)')
      types.push(...typeAliases.map(capture => ({
        name: this.getNodeText(capture.node, content),
        type: 'type_alias',
        startPosition: capture.node.startPosition,
        endPosition: capture.node.endPosition,
        isExported: this.isNodeExported(capture.node.parent)
      })))
    } catch (error) {
      // TypeScript types might not be available in JavaScript parser
    }
    
    return types
  }

  /**
   * Classify file semantics based on AST patterns
   */
  classifyFileSemantics(rootNode, content, filePath) {
    const classifications = []
    
    // Test each semantic pattern
    for (const [patternName, patternFn] of Object.entries(this.semanticPatterns)) {
      if (patternFn(rootNode, content, filePath)) {
        classifications.push(patternName)
      }
    }
    
    // Return primary classification (most specific first)
    const priority = ['reactComponent', 'reactHook', 'expressRoute', 'expressMiddleware',
                     'cliCommand', 'apiHandler', 'typeDefinition', 'configModule', 'utilityFunction']
    
    for (const pattern of priority) {
      if (classifications.includes(pattern)) {
        return pattern
      }
    }
    
    return 'module'
  }

  /**
   * Semantic pattern: React Component
   */
  isReactComponent(rootNode, content, filePath) {
    // Check for JSX elements
    const hasJsx = this.queryNode(rootNode, '(jsx_element)').length > 0
    
    // Check for React imports
    const hasReactImport = content.includes("import React") || content.includes("from 'react'")
    
    // Check for component naming pattern
    const fileName = basename(filePath, extname(filePath))
    const hasComponentName = fileName[0] === fileName[0].toUpperCase()
    
    // Check for function that returns JSX
    const functions = this.extractFunctions(rootNode, content)
    const hasComponentFunction = functions.some(fn =>
      fn.isExported && fn.name[0] === fn.name[0].toUpperCase()
    )
    
    return (hasJsx && (hasReactImport || hasComponentName)) ||
           (hasComponentFunction && hasReactImport)
  }

  /**
   * Semantic pattern: React Hook
   */
  isReactHook(rootNode, content, filePath) {
    const fileName = basename(filePath, extname(filePath))
    const hasHookName = fileName.startsWith('use') && fileName[3] === fileName[3].toUpperCase()
    
    const functions = this.extractFunctions(rootNode, content)
    const hasHookFunction = functions.some(fn =>
      fn.name.startsWith('use') && fn.name[3] === fn.name[3].toUpperCase() && fn.isExported
    )
    
    const hasReactHookImports = content.includes("from 'react'") &&
                                (content.includes('useState') || content.includes('useEffect'))
    
    return hasHookName || (hasHookFunction && hasReactHookImports)
  }

  /**
   * Semantic pattern: Express Route
   */
  isExpressRoute(rootNode, content, filePath) {
    const hasExpressImport = content.includes("from 'express'") || content.includes("require('express')")
    const hasRouterMethods = /\.(get|post|put|delete|patch)\s*\(/.test(content)
    const hasRoutePattern = /['"`]\/[^'"`]*['"`]/.test(content)
    
    return hasExpressImport && hasRouterMethods && hasRoutePattern
  }

  /**
   * Semantic pattern: Express Middleware
   */
  isExpressMiddleware(rootNode, content, filePath) {
    const hasMiddlewarePattern = /\(req,\s*res,\s*next\)|function\s*\([^)]*req[^)]*res[^)]*next/.test(content)
    const hasExpressImport = content.includes("from 'express'") || content.includes("require('express')")
    const fileName = basename(filePath).toLowerCase()
    
    return (hasMiddlewarePattern && hasExpressImport) || fileName.includes('middleware')
  }

  /**
   * Semantic pattern: CLI Command
   */
  isCliCommand(rootNode, content, filePath) {
    const hasCommanderImport = content.includes('commander') || content.includes('yargs')
    const hasProcessArgv = content.includes('process.argv')
    const hasCliPatterns = content.includes('.command(') || content.includes('.option(')
    const fileName = basename(filePath).toLowerCase()
    
    return hasCommanderImport || (hasProcessArgv && hasCliPatterns) || fileName.includes('cli')
  }

  /**
   * Semantic pattern: Utility Function
   */
  isUtilityFunction(rootNode, content, filePath) {
    const functions = this.extractFunctions(rootNode, content)
    const hasMultipleExportedFunctions = functions.filter(fn => fn.isExported).length > 1
    
    const fileName = basename(filePath).toLowerCase()
    const hasUtilityName = fileName.includes('util') || fileName.includes('helper') || fileName.includes('lib')
    
    const hasNoDomSpecificImports = !content.includes('react') && !content.includes('express')
    
    return hasUtilityName || (hasMultipleExportedFunctions && hasNoDomSpecificImports)
  }

  /**
   * Semantic pattern: API Handler
   */
  isApiHandler(rootNode, content, filePath) {
    const hasApiPattern = /api|handler|controller/i.test(filePath)
    const hasFetchPattern = content.includes('fetch(') || content.includes('axios')
    const hasHttpMethods = /\b(GET|POST|PUT|DELETE|PATCH)\b/.test(content)
    
    return hasApiPattern || (hasFetchPattern && hasHttpMethods)
  }

  /**
   * Semantic pattern: Type Definition
   */
  isTypeDefinition(rootNode, content, filePath) {
    const types = this.extractTypeDefinitions(rootNode, content)
    const hasTypeDefinitions = types.length > 0
    
    const fileName = basename(filePath).toLowerCase()
    const hasTypeFileName = fileName.includes('type') || fileName.includes('.d.ts')
    
    const hasOnlyTypes = hasTypeDefinitions &&
                        this.extractFunctions(rootNode, content).length === 0 &&
                        this.extractClasses(rootNode, content).length === 0
    
    return hasTypeFileName || hasOnlyTypes
  }

  /**
   * Semantic pattern: Config Module
   */
  isConfigModule(rootNode, content, filePath) {
    const fileName = basename(filePath).toLowerCase()
    const hasConfigName = fileName.includes('config') || fileName.includes('setting')
    
    const hasConfigPatterns = content.includes('module.exports') || content.includes('export default')
    const hasConfigObject = /\{[\s\S]*\}/.test(content) && !/function|class/.test(content)
    
    return hasConfigName && (hasConfigPatterns || hasConfigObject)
  }

  /**
   * Extract business domain terms from code
   */
  extractBusinessDomain(rootNode, content, filePath) {
    const domains = []
    
    // Focus on meaningful path segments instead of generic business terms
    const pathSegments = filePath.split('/').filter(s => s && s !== 'src' && s !== 'lib' && s !== 'components')
    const fileName = basename(filePath, extname(filePath))
    
    // Extract domain from directory structure (more reliable than keywords)
    if (pathSegments.length > 0) {
      const relevantSegments = pathSegments.slice(-2) // Last 2 directories
      domains.push(...relevantSegments.map(s => s.toLowerCase()))
    }
    
    // Add meaningful file-based domains
    if (fileName.toLowerCase().includes('config')) domains.push('configuration')
    if (fileName.toLowerCase().includes('test')) domains.push('testing')
    if (fileName.toLowerCase().includes('util')) domains.push('utilities')
    if (fileName.toLowerCase().includes('api')) domains.push('api')
    if (fileName.toLowerCase().includes('ui') || fileName.toLowerCase().includes('component')) {
      domains.push('user-interface')
    }
    
    // Only return meaningful, non-generic domains
    return [...new Set(domains)].filter(domain =>
      domain.length > 2 && !['web', 'src', 'ts', 'js', 'tsx', 'jsx'].includes(domain)
    )
  }

  /**
   * Identify technical patterns in the code
   */
  identifyTechnicalPatterns(rootNode, content) {
    const patterns = []
    
    // Framework patterns
    if (content.includes('react')) patterns.push('react')
    if (content.includes('express')) patterns.push('express')
    if (content.includes('typescript')) patterns.push('typescript')
    
    // Architecture patterns
    if (content.includes('async') && content.includes('await')) patterns.push('async-await')
    if (content.includes('Promise')) patterns.push('promises')
    if (content.includes('class') && content.includes('extends')) patterns.push('inheritance')
    
    // Design patterns
    const functions = this.extractFunctions(rootNode, content)
    if (functions.some(f => f.name.includes('Factory'))) patterns.push('factory-pattern')
    if (functions.some(f => f.name.includes('Observer'))) patterns.push('observer-pattern')
    
    return patterns
  }

  /**
   * Build relationship graph between files
   */
  buildRelationshipGraph(analysis) {
    const graph = {}
    
    for (const [filePath, fileAnalysis] of Object.entries(analysis)) {
      if (fileAnalysis.error) continue
      
      graph[filePath] = {
        imports: [],
        importedBy: [],
        semanticSimilarity: {},
        businessDomainOverlap: {},
        technicalPatternOverlap: {}
      }
    }
    
    // Build import relationships
    for (const [filePath, fileAnalysis] of Object.entries(analysis)) {
      if (fileAnalysis.error) continue
      
      for (const imp of fileAnalysis.ast.imports) {
        if (imp.isRelative) {
          // Resolve relative import to actual file path
          const importPath = this.resolveImportPath(filePath, imp.source)
          if (graph[importPath]) {
            graph[filePath].imports.push(importPath)
            graph[importPath].importedBy.push(filePath)
          }
        }
      }
    }
    
    // Calculate semantic similarities
    for (const [fileA, analysisA] of Object.entries(analysis)) {
      if (analysisA.error) continue
      
      for (const [fileB, analysisB] of Object.entries(analysis)) {
        if (analysisB.error || fileA === fileB) continue
        
        // Semantic type similarity
        const semanticSimilarity = analysisA.semanticType === analysisB.semanticType ? 1.0 : 0.0
        
        // Business domain overlap
        const domainOverlap = this.calculateOverlap(analysisA.businessDomain, analysisB.businessDomain)
        
        // Technical pattern overlap
        const patternOverlap = this.calculateOverlap(analysisA.technicalPatterns, analysisB.technicalPatterns)
        
        if (semanticSimilarity > 0 || domainOverlap > 0 || patternOverlap > 0) {
          graph[fileA].semanticSimilarity[fileB] = semanticSimilarity
          graph[fileA].businessDomainOverlap[fileB] = domainOverlap
          graph[fileA].technicalPatternOverlap[fileB] = patternOverlap
        }
      }
    }
    
    return graph
  }

  /**
   * Create smart chunks using clustering algorithms
   */
  async createSmartChunks(analysis, relationshipGraph) {
    
    // Start with individual files as nodes
    const nodes = Object.keys(analysis).filter(path => !analysis[path].error)
    console.log(`🧩 Starting with ${nodes.length} nodes for clustering`)
    
    // Apply different clustering strategies
    const strategies = [
      this.clusterBySemanticType.bind(this),
      this.clusterByBusinessDomain.bind(this),
      this.clusterByDependencyGraph.bind(this),
      this.clusterByDirectoryStructure.bind(this)
    ]
    
    let clusters = nodes.map(node => [node]) // Start with individual nodes
    
    // Apply clustering strategies
    for (const strategy of strategies) {
      clusters = strategy(clusters, analysis, relationshipGraph)
      console.log(`📦 After ${strategy.name}: ${clusters.length} clusters`)
    }
    
    // Convert clusters to named chunks
    const chunks = []
    const usedNames = new Set()
    
    for (const cluster of clusters) {
      if (cluster.length === 0) continue
      
      let chunkName = await this.generateChunkName(cluster, analysis)
      
      // Ensure unique names
      let uniqueName = chunkName
      let counter = 1
      while (usedNames.has(uniqueName)) {
        uniqueName = `${chunkName}-${counter}`
        counter++
      }
      usedNames.add(uniqueName)
      
      const chunk = {
        name: uniqueName,
        type: this.determineChunkType(cluster, analysis),
        files: cluster,
        size: cluster.reduce((sum, file) => sum + analysis[file].size, 0),
        complexity: this.calculateClusterComplexity(cluster, analysis),
        dependencies: this.calculateClusterDependencies(cluster, analysis),
        businessDomains: this.extractClusterBusinessDomains(cluster, analysis),
        technicalPatterns: this.extractClusterTechnicalPatterns(cluster, analysis),
        purpose: this.determineClusterPurpose(cluster, analysis),
        cohesion: this.calculateClusterCohesion(cluster, relationshipGraph),
        recommendations: this.generateClusterRecommendations(cluster, analysis),
        tags: this.generateTags(cluster, analysis)
      }
      
      chunks.push(chunk)
    }
    
    return chunks.sort((a, b) => b.cohesion - a.cohesion) // Sort by cohesion (best chunks first)
  }

  /**
   * Cluster files by semantic type
   */
  clusterBySemanticType(clusters, analysis, relationshipGraph) {
    const semanticGroups = {}
    
    for (const cluster of clusters) {
      for (const file of cluster) {
        const semanticType = analysis[file].semanticType
        if (!semanticGroups[semanticType]) {
          semanticGroups[semanticType] = []
        }
        semanticGroups[semanticType].push(file)
      }
    }
    
    return Object.values(semanticGroups).filter(group => group.length > 0)
  }

  /**
   * Cluster files by business domain
   */
  clusterByBusinessDomain(clusters, analysis, relationshipGraph) {
    const domainGroups = {}
    
    for (const cluster of clusters) {
      for (const file of cluster) {
        const domains = analysis[file].businessDomain
        
        if (domains.length === 0) {
          // Files with no clear domain go to 'general' group
          if (!domainGroups.general) domainGroups.general = []
          domainGroups.general.push(file)
        } else {
          // Files go to their primary domain group
          const primaryDomain = domains[0]
          if (!domainGroups[primaryDomain]) domainGroups[primaryDomain] = []
          domainGroups[primaryDomain].push(file)
        }
      }
    }
    
    return Object.values(domainGroups).filter(group => group.length > 0)
  }

  /**
   * Cluster files by dependency relationships
   */
  clusterByDependencyGraph(clusters, analysis, relationshipGraph) {
    const dependencyGroups = []
    const visited = new Set()
    
    for (const cluster of clusters) {
      for (const file of cluster) {
        if (visited.has(file)) continue
        
        // Find all files connected to this file through imports
        const connected = this.findConnectedFiles(file, relationshipGraph, new Set())
        
        // Filter to only files in current clusters
        const relevantConnected = connected.filter(f =>
          clusters.some(cluster => cluster.includes(f))
        )
        
        if (relevantConnected.length > 1) {
          dependencyGroups.push(relevantConnected)
          relevantConnected.forEach(f => visited.add(f))
        } else {
          // Isolated file becomes its own group
          dependencyGroups.push([file])
          visited.add(file)
        }
      }
    }
    
    return dependencyGroups.filter(group => group.length > 0)
  }

  /**
   * Cluster files by directory structure
   */
  clusterByDirectoryStructure(clusters, analysis, relationshipGraph) {
    const directoryGroups = {}
    
    for (const cluster of clusters) {
      for (const file of cluster) {
        const dir = dirname(file)
        if (!directoryGroups[dir]) {
          directoryGroups[dir] = []
        }
        directoryGroups[dir].push(file)
      }
    }
    
    return Object.values(directoryGroups).filter(group => group.length > 0)
  }

  /**
   * Generate intelligent chunk name
   */
  async generateChunkName(files, analysis) {
    const namingStrategies = {
      domainBased: this.generateDomainBasedName.bind(this),
      patternBased: this.generatePatternBasedName.bind(this),
      functionalityBased: this.generateFunctionalityBasedName.bind(this)
    }
    
    const names = {}
    
    for (const [strategy, generator] of Object.entries(namingStrategies)) {
      try {
        names[strategy] = generator(files, analysis)
      } catch (error) {
        names[strategy] = 'unnamed-chunk'
      }
    }
    
    // Choose best name based on strategy preference - prefer pattern-based for better names
    const strategy = 'patternBased' // Force pattern-based naming
    return names[strategy] || names.patternBased || names.functionalityBased || names.domainBased || 'unknown-chunk'
  }

  /**
   * Generate domain-based chunk name
   */
  generateDomainBasedName(files, analysis) {
    // Always fallback to pattern-based naming since domain extraction is unreliable
    return this.generatePatternBasedName(files, analysis)
  }

  /**
   * Generate pattern-based chunk name
   */
  generatePatternBasedName(files, analysis) {
    const semanticTypes = files.map(file => analysis[file].semanticType)
    const mostCommon = this.getMostCommon(semanticTypes)
    
    // Look at actual file names and directories for context
    const commonPath = this.findCommonPath(files)
    const dirName = commonPath ? basename(dirname(commonPath)) : null
    
    const typeNames = {
      reactComponent: 'ui-components',
      reactHook: 'react-hooks',
      expressRoute: 'server-routes',
      expressMiddleware: 'server-middleware',
      utilityFunction: 'utility-functions',
      typeDefinition: 'type-definitions',
      configModule: 'configuration',
      cliCommand: 'cli-tools',
      apiHandler: 'api-endpoints',
      module: 'shared-modules'
    }
    
    let baseName = typeNames[mostCommon] || 'mixed-files'
    
    // Add more specific context based on file paths
    if (commonPath) {
      if (commonPath.includes('/components/ui/')) {
        baseName = 'ui-library-components'
      } else if (commonPath.includes('/components/')) {
        baseName = 'application-components'
      } else if (commonPath.includes('/hooks/')) {
        baseName = 'custom-hooks'
      } else if (commonPath.includes('/lib/')) {
        baseName = 'core-utilities'
      } else if (commonPath.includes('/utils/')) {
        baseName = 'helper-utilities'
      } else if (dirName && dirName !== 'src' && dirName !== 'components' && dirName !== 'lib') {
        baseName = `${dirName}-${baseName}`
      }
    }
    
    return baseName
  }

  /**
   * Generate functionality-based chunk name
   */
  generateFunctionalityBasedName(files, analysis) {
    // Extract function names and find common themes
    const allFunctions = files.flatMap(file =>
      analysis[file].ast.functions.map(fn => fn.name.toLowerCase())
    )
    
    const commonWords = this.extractCommonWords(allFunctions)
    
    if (commonWords.length > 0) {
      return commonWords.slice(0, 2).join('-') + '-logic'
    }
    
    // Fallback to directory-based naming
    const dirs = files.map(file => basename(dirname(file)))
    const commonDir = this.getMostCommon(dirs)
    
    return commonDir + '-module'
  }

  /**
   * Helper methods for AST analysis
   */

  queryNode(node, query) {
    // Simplified query implementation
    // In a full implementation, you'd use tree-sitter's query language
    const results = []
    
    const traverse = (currentNode) => {
      // Match based on node type for now
      if (query.includes(currentNode.type)) {
        results.push({ node: currentNode })
      }
      
      for (let i = 0; i < currentNode.namedChildCount; i++) {
        traverse(currentNode.namedChild(i))
      }
    }
    
    traverse(node)
    return results
  }

  getNodeText(node, content) {
    return content.slice(node.startIndex, node.endIndex)
  }

  isNodeExported(node) {
    // Check if node is part of an export statement
    let parent = node.parent
    while (parent) {
      if (parent.type === 'export_statement') {
        return true
      }
      parent = parent.parent
    }
    return false
  }

  calculateOverlap(arrayA, arrayB) {
    const setA = new Set(arrayA)
    const setB = new Set(arrayB)
    const intersection = new Set([...setA].filter(x => setB.has(x)))
    const union = new Set([...setA, ...setB])
    
    return union.size === 0 ? 0 : intersection.size / union.size
  }

  getMostCommon(arr) {
    const counts = {}
    for (const item of arr) {
      counts[item] = (counts[item] || 0) + 1
    }
    
    return Object.entries(counts)
      .sort(([,a], [,b]) => b - a)[0]?.[0] || 'unknown'
  }

  generateSemanticTags(analysis) {
    const tags = []
    
    tags.push(analysis.semanticType)
    tags.push(...analysis.businessDomain)
    tags.push(...analysis.technicalPatterns)
    
    if (analysis.complexity.level) {
      tags.push(`complexity-${analysis.complexity.level}`)
    }
    
    if (analysis.lines < 50) tags.push('small')
    else if (analysis.lines < 200) tags.push('medium')  
    else tags.push('large')
    
    return [...new Set(tags)]
  }

  calculateAstComplexity(rootNode) {
    let complexity = 1
    
    const complexityNodes = ['if_statement', 'while_statement', 'for_statement',
                            'switch_statement', 'try_statement', 'catch_clause']
    
    const traverse = (node) => {
      if (complexityNodes.includes(node.type)) {
        complexity++
      }
      
      for (let i = 0; i < node.namedChildCount; i++) {
        traverse(node.namedChild(i))
      }
    }
    
    traverse(rootNode)
    
    return {
      score: complexity,
      level: complexity < 5 ? 'low' : complexity < 15 ? 'medium' : 'high'
    }
  }

  /**
   * Analyze dependencies from AST
   */
  analyzeDependencies(rootNode, content) {
    const dependencies = {
      internal: [],
      external: [],
      relative: []
    }
    
    const imports = this.extractImports(rootNode, content)
    
    for (const imp of imports) {
      if (imp.isRelative) {
        dependencies.relative.push(imp.source)
      } else if (imp.isExternal) {
        dependencies.external.push(imp.source)
      } else {
        dependencies.internal.push(imp.source)
      }
    }
    
    return dependencies
  }

  /**
   * Resolve relative import path to absolute path
   */
  resolveImportPath(fromFile, importPath) {
    // Simplified path resolution
    const dir = dirname(fromFile)
    return join(dir, importPath)
  }

  /**
   * Find all files connected through imports
   */
  findConnectedFiles(startFile, relationshipGraph, visited = new Set()) {
    if (visited.has(startFile)) return []
    
    visited.add(startFile)
    const connected = [startFile]
    
    if (relationshipGraph[startFile]) {
      // Follow imports
      for (const importedFile of relationshipGraph[startFile].imports) {
        connected.push(...this.findConnectedFiles(importedFile, relationshipGraph, visited))
      }
      
      // Follow files that import this one
      for (const importingFile of relationshipGraph[startFile].importedBy) {
        connected.push(...this.findConnectedFiles(importingFile, relationshipGraph, visited))
      }
    }
    
    return [...new Set(connected)]
  }

  /**
   * Extract imported names from import statement
   */
  extractImportedNames(importNode, content) {
    const names = []
    // Simplified implementation - would need more complex parsing
    const importText = this.getNodeText(importNode, content)
    const match = importText.match(/import\s+(?:\{([^}]+)\}|(\w+))/i)
    if (match) {
      if (match[1]) {
        // Named imports
        names.push(...match[1].split(',').map(n => n.trim()))
      } else if (match[2]) {
        // Default import
        names.push(match[2])
      }
    }
    return names
  }

  /**
   * Extract export name from export declaration
   */
  extractExportName(declaration, content) {
    const text = this.getNodeText(declaration, content)
    const match = text.match(/(?:function|class|const|let|var)\s+(\w+)/)
    return match ? match[1] : 'unnamed'
  }

  /**
   * Extract JSX element name
   */
  extractJsxElementName(jsxNode, content) {
    try {
      const openingElement = jsxNode.namedChild(0)
      if (openingElement) {
        const nameNode = openingElement.namedChild(0)
        return nameNode ? this.getNodeText(nameNode, content) : 'unknown'
      }
    } catch (error) {
      return 'unknown'
    }
    return 'unknown'
  }

  /**
   * Determine chunk type based on files
   */
  determineChunkType(files, analysis) {
    const semanticTypes = files.map(file => analysis[file].semanticType)
    const mostCommon = this.getMostCommon(semanticTypes)
    
    const typeMapping = {
      reactComponent: 'ui-components',
      reactHook: 'custom-hooks',
      expressRoute: 'api-routes',
      expressMiddleware: 'middleware',
      utilityFunction: 'utilities',
      typeDefinition: 'type-definitions',
      configModule: 'configuration',
      cliCommand: 'cli-commands',
      apiHandler: 'api-handlers'
    }
    
    return typeMapping[mostCommon] || 'mixed-module'
  }

  /**
   * Calculate cluster complexity
   */
  calculateClusterComplexity(files, analysis) {
    const complexities = files.map(file => analysis[file].complexity.score)
    const total = complexities.reduce((sum, c) => sum + c, 0)
    const average = total / files.length
    
    return {
      total,
      average,
      level: average < 5 ? 'low' : average < 15 ? 'medium' : 'high'
    }
  }

  /**
   * Calculate cluster dependencies
   */
  calculateClusterDependencies(files, analysis) {
    const allDeps = {
      internal: new Set(),
      external: new Set(),
      relative: new Set()
    }
    
    for (const file of files) {
      const deps = analysis[file].dependencies
      deps.internal.forEach(dep => allDeps.internal.add(dep))
      deps.external.forEach(dep => allDeps.external.add(dep))
      deps.relative.forEach(dep => allDeps.relative.add(dep))
    }
    
    return {
      internal: Array.from(allDeps.internal),
      external: Array.from(allDeps.external),
      relative: Array.from(allDeps.relative),
      totalCount: allDeps.internal.size + allDeps.external.size + allDeps.relative.size
    }
  }

  /**
   * Extract cluster business domains
   */
  extractClusterBusinessDomains(files, analysis) {
    const allDomains = files.flatMap(file => analysis[file].businessDomain)
    return [...new Set(allDomains)]
  }

  /**
   * Extract cluster technical patterns
   */
  extractClusterTechnicalPatterns(files, analysis) {
    const allPatterns = files.flatMap(file => analysis[file].technicalPatterns)
    return [...new Set(allPatterns)]
  }

  /**
   * Determine cluster purpose
   */
  determineClusterPurpose(files, analysis) {
    const semanticTypes = files.map(file => analysis[file].semanticType)
    const mostCommon = this.getMostCommon(semanticTypes)
    
    const purposeMapping = {
      reactComponent: 'User interface components and React elements',
      reactHook: 'Custom React hooks for state and logic sharing',
      expressRoute: 'API routes and endpoint handlers',
      expressMiddleware: 'Express middleware and request processing',
      utilityFunction: 'Utility functions and helper libraries',
      typeDefinition: 'TypeScript type definitions and interfaces',
      configModule: 'Configuration files and settings',
      cliCommand: 'Command-line interface and CLI tools',
      apiHandler: 'API client and data fetching logic'
    }
    
    return purposeMapping[mostCommon] || 'Mixed functionality module'
  }

  /**
   * Calculate cluster cohesion
   */
  calculateClusterCohesion(files, relationshipGraph) {
    if (files.length <= 1) return 1.0
    
    let connections = 0
    let totalPossible = files.length * (files.length - 1)
    
    for (const fileA of files) {
      for (const fileB of files) {
        if (fileA !== fileB && relationshipGraph[fileA]) {
          if (relationshipGraph[fileA].imports.includes(fileB) ||
              relationshipGraph[fileA].importedBy.includes(fileB) ||
              relationshipGraph[fileA].semanticSimilarity[fileB] > 0.5) {
            connections++
          }
        }
      }
    }
    
    return totalPossible > 0 ? connections / totalPossible : 0
  }

  /**
   * Generate cluster recommendations
   */
  generateClusterRecommendations(files, analysis) {
    const recommendations = []
    
    const totalSize = files.reduce((sum, file) => sum + analysis[file].size, 0)
    const avgComplexity = files.reduce((sum, file) => sum + analysis[file].complexity.score, 0) / files.length
    
    if (totalSize > 100000) {
      recommendations.push({
        type: 'warning',
        message: 'Large cluster - consider splitting by functionality'
      })
    }
    
    if (avgComplexity > 20) {
      recommendations.push({
        type: 'warning',
        message: 'High complexity cluster - review for refactoring opportunities'
      })
    }
    
    if (files.length === 1) {
      recommendations.push({
        type: 'info',
        message: 'Single file cluster - consider grouping with related files'
      })
    }
    
    return recommendations
  }

  /**
   * Extract common words from function names
   */
  extractCommonWords(functionNames) {
    const words = functionNames.flatMap(name =>
      name.split(/(?=[A-Z])|_|-/).filter(word => word.length > 2)
    )
    
    const wordCounts = {}
    for (const word of words) {
      wordCounts[word] = (wordCounts[word] || 0) + 1
    }
    
    return Object.entries(wordCounts)
      .filter(([, count]) => count > 1)
      .sort(([, a], [, b]) => b - a)
      .slice(0, 3)
      .map(([word]) => word)
  }

  /**
   * Generate code signature for caching and similarity comparison
   */
  generateCodeSignature(rootNode, content) {
    const functions = this.extractFunctions(rootNode, content)
    const classes = this.extractClasses(rootNode, content)
    const imports = this.extractImports(rootNode, content)
    const exports = this.extractExports(rootNode, content)
    
    return {
      functionCount: functions.length,
      classCount: classes.length,
      importCount: imports.length,
      exportCount: exports.length,
      exportedFunctions: functions.filter(f => f.isExported).map(f => f.name),
      importSources: imports.map(i => i.source),
      hasJsx: this.extractJsxElements(rootNode, content).length > 0,
      contentHash: this.simpleHash(content)
    }
  }

  /**
   * Simple hash function for content comparison
   */
  simpleHash(str) {
    let hash = 0
    for (let i = 0; i < str.length; i++) {
      const char = str.charCodeAt(i)
      hash = ((hash << 5) - hash) + char
      hash = hash & hash // Convert to 32bit integer
    }
    return hash
  }
  
  generateSummary(analysis, chunks) {
    const files = Object.values(analysis).filter(f => !f.error)
    
    return {
      totalFiles: files.length,
      totalSize: files.reduce((sum, f) => sum + f.size, 0),
      totalLines: files.reduce((sum, f) => sum + f.lines, 0),
      semanticTypes: this.countByProperty(files, 'semanticType'),
      businessDomains: this.countDomains(files),
      technicalPatterns: this.countPatterns(files),
      totalChunks: chunks.length,
      averageChunkSize: chunks.reduce((sum, c) => sum + c.size, 0) / chunks.length,
      chunkTypes: this.countByProperty(chunks, 'type')
    }
  }

  generateRecommendations(analysis, chunks) {
    const recommendations = []
    
    // Add specific recommendations based on treesitter analysis
    
    return recommendations
  }

  countByProperty(items, property) {
    const counts = {}
    for (const item of items) {
      const value = typeof property === 'function' ? property(item) : item[property]
      counts[value] = (counts[value] || 0) + 1
    }
    return counts
  }

  countDomains(files) {
    const allDomains = files.flatMap(f => f.businessDomain)
    return this.countByProperty(allDomains, d => d)
  }

  countPatterns(files) {
    const allPatterns = files.flatMap(f => f.technicalPatterns)
    return this.countByProperty(allPatterns, p => p)
  }

  /**
   * Generate tags for a chunk based on its characteristics
   */
  generateTags(files, analysis) {
    const tags = new Set()
    
    // Add semantic type tags
    const semanticTypes = files.map(file => analysis[file].semanticType)
    for (const type of semanticTypes) {
      if (type === 'reactComponent') tags.add('react-component')
      if (type === 'reactHook') tags.add('react-hook')
      if (type === 'utilityFunction') tags.add('utility')
      if (type === 'expressRoute') tags.add('api')
      if (type === 'configModule') tags.add('config')
    }
    
    // Add directory-based tags
    const commonPath = this.findCommonPath(files)
    if (commonPath) {
      if (commonPath.includes('/components/')) tags.add('component')
      if (commonPath.includes('/hooks/')) tags.add('hook')
      if (commonPath.includes('/lib/')) tags.add('library')
      if (commonPath.includes('/utils/')) tags.add('utility')
      if (commonPath.includes('/ui/')) tags.add('ui-library')
    }
    
    // Add complexity tags
    const avgComplexity = files.reduce((sum, file) => sum + analysis[file].complexity.score, 0) / files.length
    if (avgComplexity > 15) tags.add('complex')
    if (avgComplexity < 5) tags.add('simple')
    
    return Array.from(tags)
  }

  /**
   * Find common path prefix for a group of files
   */
  findCommonPath(files) {
    if (files.length === 0) return null
    if (files.length === 1) return files[0]
    
    const pathParts = files.map(file => file.split('/'))
    const commonParts = []
    
    for (let i = 0; i < Math.min(...pathParts.map(p => p.length)); i++) {
      const part = pathParts[0][i]
      if (pathParts.every(p => p[i] === part)) {
        commonParts.push(part)
      } else {
        break
      }
    }
    
    return commonParts.length > 0 ? commonParts.join('/') : null
  }
}

export default TreesitterSemanticChunker