project-indexer/src/fileValidator.js

Local project indexer with browser trigger support

import fs from 'fs';
import {globby} from 'globby';
import ignore from 'ignore'; // For parsing .gitignore files
import path from 'path';

// Default file extensions that are considered important project files
export const DEFAULT_EXTENSIONS = [
  // Web development
  '.js', '.ts', '.tsx', '.jsx', '.html', '.css', '.scss', '.less', '.sass',
  // Backend development
  '.py', '.java', '.rb', '.php', '.go', '.rs', '.cs', '.cpp', '.c', '.h', '.hpp',
  // Data and configuration
  '.json', '.xml', '.yml', '.yaml', '.toml', '.ini', '.env.example', '.template',
  // Documentation
  '.md', '.txt', '.rst', '.adoc',
  // Templates
  '.ejs', '.hbs', '.pug', '.mustache', '.twig',
  // Shell scripts
  '.sh', '.bash', '.zsh', '.fish', '.bat', '.cmd', '.ps1',
  // Project-specific files
  'Dockerfile', 'docker-compose.yml', 'Makefile', 'Jenkinsfile', 'Procfile'
];

// Universal ignore patterns for any project
export const UNIVERSAL_IGNORE_PATTERNS = [
  // Dependency directories
  'node_modules/**',
  'bower_components/**',
  'jspm_packages/**',
  'package-lock.json',
  'yarn.lock',
  'pnpm-lock.yaml',
  '.pnpm-store/**',
  'vendor/**',
  '.bundle/**',
  'composer.lock',
  'Gemfile.lock',
  'requirements.txt.lock',
  'poetry.lock',
  'Pipfile.lock',
  'cargo.lock',
  
  // Build outputs and cache
  'build/**',
  'dist/**',
  'out/**',
  'output/**',
  'target/**',
  'bin/**',
  'obj/**',
  'lib/**',
  'libs/**',
  'debug/**',
  'release/**',
  'coverage/**',
  'public/build/**',
  '.svelte-kit/**',
  '.next/**',
  '.nuxt/**',
  '.cache/**',
  '.parcel-cache/**',
  '.webpack/**',
  '.serverless/**',
  '.gradle/**',
  '.eggs/**',
  '__pycache__/**',
  '*.py[cod]',
  '*$py.class',
  '*.so',
  '*.dylib',
  '*.dll',
  '*.exe',
  '*.out',
  '*.app',
  '*.jar',
  '*.war',
  '*.nar',
  '*.ear',
  '*.zip',
  '*.tar.gz',
  '*.rar',
  '.vscode',
  
  // IDE and editor configurations
  '.vscode/**',
  '.idea/**',
  '.eclipse/**',
  '.settings/**',
  '.project',
  '.classpath',
  '.factorypath',
  '*.sublime-*',
  '.atom/**',
  '.nova/**',
  '.vs/**',
  '.editorconfig',
  '.prettierrc*',
  '.eslintrc*',
  '.stylelintrc*',
  '.jshintrc',
  '*.code-workspace',
  '*.swp',
  '*.swo',
  '*~',
  '.history/**',
  
  // OS and environment files
  '.DS_Store',
  '.AppleDouble',
  '.LSOverride',
  'Icon\r',
  'Thumbs.db',
  'ehthumbs.db',
  'Desktop.ini',
  '*.lnk',
  '*.url',
  '.env',
  '.env.*',
  '!.env.example',
  '!.env.template',
  '*.log',
  '*.tmp',
  '*.temp',
  '*.pid',
  '*.seed',
  '*.bak',
  '*.swp',
  '*.dump',
  'npm-debug.log*',
  'yarn-debug.log*',
  'yarn-error.log*',
  'lerna-debug.log*',
  '.pnpm-debug.log*',
  
  // Version control
  '.git/**',
  '.gitattributes',
  '.gitmodules',
  '.svn/**',
  '.hg/**',
  '.bzr/**',
  'CVS/**',
  
  // CI/CD and deployment
  '.circleci/**',
  '.github/**',
  '.gitlab/**',
  '.gitlab-ci.yml',
  '.travis.yml',
  'appveyor.yml',
  '.azure-pipelines/**',
  '.jenkins/**',
  'Jenkinsfile',
  '.wercker/**',
  '.semaphore/**',
  '.drone.yml',
  '.codeship/**',
  
  // Docker
  '.docker/**',
  'docker-compose.override.yml',
  
  // Testing
  'coverage/**',
  '.nyc_output/**',
  '.cypress/**',
  '.jest-cache/**',
  
  // Framework-specific
  '.angular/**',
  '.turbo/**',
  '.meteor/**',
  
  // TypeScript
  '*.tsbuildinfo',
  
  // Misc
  '*.iml',
  '.sass-cache/**',
  '.terraform/**',
  '*.tfstate',
  '*.tfstate.*',
  '.vagrant/**',
];

/**
 * Attempts to read and parse all .gitignore files in the project
 * @param {string} baseDir - The base directory of the project
 * @returns {string[]} - Array of gitignore patterns
 */
export async function getGitignorePatterns(baseDir) {
  try {
    // Find all .gitignore files in the project
    const gitignoreFiles = await globby(['**/.gitignore'], {
      cwd: baseDir,
      absolute: true,
      ignore: ['**/node_modules/**'],
      dot: true,
    });
    
    let allPatterns = [];
    
    // Read each .gitignore file and collect patterns
    for (const gitignoreFile of gitignoreFiles) {
      try {
        const content = fs.readFileSync(gitignoreFile, 'utf8');
        // Get directory containing this .gitignore
        const gitignoreDir = path.dirname(gitignoreFile);
        const relativeDir = path.relative(baseDir, gitignoreDir);
        
        // Process each line in the .gitignore file
        const patterns = content
          .split('\n')
          .map(line => line.trim())
          .filter(line => line && !line.startsWith('#'))
          .map(pattern => {
            // Make patterns relative to the project root
            if (relativeDir && pattern.startsWith('/')) {
              return path.join(relativeDir, pattern.substring(1)).replace(/\\/g, '/');
            } else if (relativeDir && !pattern.startsWith('!')) {
              return path.join(relativeDir, pattern).replace(/\\/g, '/');
            }
            return pattern;
          });
        
        allPatterns = [...allPatterns, ...patterns];
      } catch (err) {
        console.warn(`Error reading gitignore file ${gitignoreFile}: ${err.message}`);
      }
    }
    
    return allPatterns;
  } catch (err) {
    console.warn(`Could not process gitignore files: ${err.message}`);
    return [];
  }
}

/**
 * Get project-type-specific ignore patterns based on detected project files
 * @param {string} baseDir - The base directory of the project
 * @returns {string[]} - Additional ignore patterns specific to the project type
 */
export async function getProjectSpecificIgnores(baseDir) {
  const projectFiles = await globby([
    'package.json',
    'pom.xml',
    'build.gradle',
    'requirements.txt',
    'Gemfile',
    'Cargo.toml',
    'go.mod',
    'composer.json',
    'project.clj',
    'build.sbt',
    'mix.exs',
    'rebar.config',
    'pubspec.yaml',
    'stackage.yaml',
    'Pipfile',
    'pyproject.toml',
  ], {
    cwd: baseDir,
    deep: 1, // Only look at the root level
    absolute: false, // We just need the names
  });
  
  const additionalIgnores = [];
  
  // Node.js/JavaScript projects
  if (projectFiles.includes('package.json')) {
    additionalIgnores.push(
      'coverage/**',
      '.nyc_output/**',
      '.storybook-out/**',
      '.tern-port',
      '.storybook-static/**',
      'storybook-static/**'
    );
  }
  
  // Java projects
  if (projectFiles.includes('pom.xml') || projectFiles.includes('build.gradle')) {
    additionalIgnores.push(
      'target/**',
      'build/**',
      '.gradle/**',
      'out/**',
      '*.class',
      '*.jar',
      'gradle-app.setting',
      '.gradletasknamecache',
      '*.war',
      '*.ear',
      '*.hprof',
      '.jrebel',
      'rebel.xml',
      '.apt_generated/**',
      '.apt_generated_tests/**',
      '.settings/**',
      '.metadata/**'
    );
  }
  
  // Python projects
  if (projectFiles.includes('requirements.txt') || 
      projectFiles.includes('Pipfile') || 
      projectFiles.includes('pyproject.toml')) {
    additionalIgnores.push(
      '__pycache__/**',
      '*.py[cod]',
      '*$py.class',
      '*.so',
      '.Python',
      'env/**',
      'venv/**',
      'ENV/**',
      'env.bak/**',
      'venv.bak/**',
      '.pytest_cache/**',
      '.coverage',
      'htmlcov/**',
      '.tox/**',
      '.nox/**',
      '.hypothesis/**',
      '.pytype/**',
      'cython_debug/**',
      '*.egg-info/**',
      '.installed.cfg',
      '*.egg',
      'celerybeat-schedule',
      'celerybeat.pid',
      '*.sage.py',
      '.ipynb_checkpoints/**'
    );
  }
  
  // Ruby projects
  if (projectFiles.includes('Gemfile')) {
    additionalIgnores.push(
      '*.gem',
      '*.rbc',
      '/.config',
      '/coverage/',
      '/InstalledFiles',
      '/pkg/',
      '/spec/reports/',
      '/spec/examples.txt',
      '/test/tmp/',
      '/test/version_tmp/',
      '/tmp/',
      '.byebug_history',
      '.dat*',
      '.repl_history',
      '*.bridgesupport',
      'build-iPhoneOS/',
      'build-iPhoneSimulator/'
    );
  }
  
  // Rust projects
  if (projectFiles.includes('Cargo.toml')) {
    additionalIgnores.push(
      '/target/',
      '**/*.rs.bk',
      'Cargo.lock'
    );
  }
  
  // Go projects
  if (projectFiles.includes('go.mod')) {
    additionalIgnores.push(
      '*.test',
      '*.out',
      '/vendor/',
      '/Godeps/'
    );
  }
  
  // PHP projects
  if (projectFiles.includes('composer.json')) {
    additionalIgnores.push(
      '/vendor/',
      'composer.phar',
      '/phpunit.xml',
      '/.phpunit.result.cache'
    );
  }
  
  return additionalIgnores;
}

/**
 * Gets files to be indexed based on provided extensions and ignore patterns
 * @param {string} baseDir - The base directory of the project
 * @param {string[]} extensions - Array of file extensions to include
 * @returns {Promise<string[]>} - Array of file paths
 */
export async function getFocusedFiles(baseDir, extensions) {
  // Prepare include patterns from extensions
  const includePatterns = extensions.map(ext => {
    // Handle extensions that start with dot and those that don't
    return ext.startsWith('.') ? `**/*${ext}` : `**/*.${ext}`;
  });
  
  // Add exact-match filenames without extensions that we want to include
  includePatterns.push(
    '**/Dockerfile',
    '**/docker-compose.yml',
    '**/Makefile',
    '**/LICENSE',
    '**/README*',
    '**/CONTRIBUTING*',
    '**/Jenkinsfile'
  );
  
  // Collect all ignore patterns
  const gitignorePatterns = await getGitignorePatterns(baseDir);
  const projectSpecificIgnores = await getProjectSpecificIgnores(baseDir);
  
  const allIgnorePatterns = [
    ...UNIVERSAL_IGNORE_PATTERNS,
    ...gitignorePatterns,
    ...projectSpecificIgnores
  ];
  
  // Use the ignore package to create a proper ignore filter
  const ig = ignore().add(allIgnorePatterns);
  
  // First, get all files that match the include patterns
  const allMatchingFiles = await globby(includePatterns, {
    cwd: baseDir,
    absolute: true,
    dot: true, // Include dotfiles
  });
  
  // Then filter out ignored files
  return allMatchingFiles.filter(absPath => {
    const relativePath = path.relative(baseDir, absPath).replace(/\\/g, '/');
    return !ig.ignores(relativePath);
  });
}