@boilerbuilder/deps-analyzer/lib/index.js

CLI tool to analyze dependency evolution and release frequency

const fs = require('fs');
const path = require('path');
const glob = require('fast-glob');

// Import utility functions
const parseYarnLock = require('../utils/parseYarnLock');
const parsePnpmLock = require('../utils/parsePnpmLock');
const findWorkspaceRoot = require('../utils/findWorkspaceRoot');
const filterDependenciesByScope = require('../utils/filterDependenciesByScope');

/**
 * Load max-deps.json file if it exists
 * @param {string} searchPath - Path to search for max-deps.json
 * @param {string|null} customConfigPath - Custom path to max-deps.json file
 * @returns {Object|null} Max dependencies constraints or null
 */
function loadMaxDeps(searchPath, customConfigPath = null) {
  try {
    // Try custom config path first (highest priority)
    if (customConfigPath) {
      const resolvedCustomPath = path.isAbsolute(customConfigPath) 
        ? customConfigPath 
        : path.resolve(customConfigPath);
        
      if (fs.existsSync(resolvedCustomPath)) {
        console.log(`📋 Loading custom max-deps config: ${resolvedCustomPath}`);
        const content = fs.readFileSync(resolvedCustomPath, 'utf8');
        return JSON.parse(content);
      } else {
        throw new Error(`Custom max-deps config file not found: ${resolvedCustomPath}`);
      }
    }
    
    // Try current working directory first
    const cwdPath = path.join(process.cwd(), 'max-deps.json');
    if (fs.existsSync(cwdPath)) {
      const content = fs.readFileSync(cwdPath, 'utf8');
      return JSON.parse(content);
    }
    
    // Try next to the search path
    if (searchPath) {
      const searchDir = path.dirname(searchPath);
      const searchPath2 = path.join(searchDir, 'max-deps.json');
      if (fs.existsSync(searchPath2)) {
        const content = fs.readFileSync(searchPath2, 'utf8');
        return JSON.parse(content);
      }
    }
    
    return null;
  } catch (error) {
    console.log(`⚠️  Error loading max-deps.json: ${error.message}`);
    return null;
  }
}

/**
 * Find package.json files based on pattern
 * @param {string} pattern - File pattern or directory path
 * @returns {Promise<string[]>} Array of package.json file paths
 */
async function findPackageJsonFiles(pattern) {
  try {
    // Case 1: Direct package.json file
    if (pattern.endsWith('package.json')) {
      if (fs.existsSync(pattern)) {
        return [pattern];
      }
      throw new Error(`File not found: ${pattern}`);
    }
    
    // Case 2: Directory path
    if (fs.existsSync(pattern) && fs.statSync(pattern).isDirectory()) {
      const patterns = [
        path.join(pattern, 'package.json'), // Direct package.json in folder
        path.join(pattern, '*/package.json') // Only first-level subdirectories
      ];
      
      const files = await glob(patterns, {
        ignore: ['**/node_modules/**', '**/public/**', '**/dist/**', '**/build/**'],
        absolute: true
      });
      
      return files;
    }
    
    // Case 3: Glob pattern
    const files = await glob(pattern, {
      ignore: ['**/node_modules/**'],
      absolute: true
    });
    
    return files.filter(file => file.endsWith('package.json'));
    
  } catch (error) {
    throw new Error(`Error finding package.json files: ${error.message}`);
  }
}


/**
 * Extract installed versions from lock files or node_modules (supports workspaces)
 * @param {string} packageJsonPath - Path to package.json
 * @param {Object} dependencies - Dependencies object
 * @returns {Object} Map of dependency -> installedVersion
 */
function extractInstalledVersions(packageJsonPath, dependencies) {
  const projectDir = path.dirname(packageJsonPath);
  const installedVersions = {};
  
  // Find workspace root (for monorepos)
  const workspaceRoot = findWorkspaceRoot(projectDir);
  const searchDirs = workspaceRoot ? [projectDir, workspaceRoot] : [projectDir];
  
  console.log(`🔍 Searching for installed versions in: ${searchDirs.map(d => path.relative(process.cwd(), d) || '.').join(', ')}`);
  
  // Method 1: Try lock files (most reliable) - search in all possible locations
  for (const searchDir of searchDirs) {
    // Try package-lock.json (npm)
    const packageLockPath = path.join(searchDir, 'package-lock.json');
    if (fs.existsSync(packageLockPath)) {
      try {
        const lockData = JSON.parse(fs.readFileSync(packageLockPath, 'utf8'));
        
        // Handle different package-lock.json formats
        if (lockData.packages) {
          // npm v7+ format
          Object.keys(dependencies).forEach(depName => {
            if (!installedVersions[depName]) {
              const packageKey = `node_modules/${depName}`;
              if (lockData.packages[packageKey]?.version) {
                installedVersions[depName] = lockData.packages[packageKey].version;
              }
            }
          });
        } else if (lockData.dependencies) {
          // npm v6 format
          Object.keys(dependencies).forEach(depName => {
            if (!installedVersions[depName] && lockData.dependencies[depName]?.version) {
              installedVersions[depName] = lockData.dependencies[depName].version;
            }
          });
        }
        
        const lockLocation = searchDir === projectDir ? 'local' : 'workspace root';
        const foundInThisLock = Object.keys(dependencies).filter(dep => 
          installedVersions[dep] && !Object.keys(installedVersions).slice(0, Object.keys(installedVersions).indexOf(dep)).includes(dep)
        ).length;
        
        if (foundInThisLock > 0) {
          console.log(`📦 Found package-lock.json (${lockLocation}): extracted ${foundInThisLock} versions`);
        }
        
      } catch (error) {
        console.log(`⚠️ Failed to read package-lock.json in ${path.relative(process.cwd(), searchDir)}: ${error.message}`);
      }
      continue; // Skip other lock files if package-lock.json exists
    }
    
    // Try yarn.lock (yarn)
    const yarnLockPath = path.join(searchDir, 'yarn.lock');
    if (fs.existsSync(yarnLockPath)) {
      const yarnVersions = parseYarnLock(yarnLockPath, dependencies);
      Object.assign(installedVersions, yarnVersions);
      
      const lockLocation = searchDir === projectDir ? 'local' : 'workspace root';
      const foundInYarnLock = Object.keys(yarnVersions).length;
      
      if (foundInYarnLock > 0) {
        console.log(`🧶 Found yarn.lock (${lockLocation}): extracted ${foundInYarnLock} versions`);
      }
    }
    
    // Try pnpm-lock.yaml (pnpm)
    const pnpmLockPath = path.join(searchDir, 'pnpm-lock.yaml');
    if (fs.existsSync(pnpmLockPath)) {
      const pnpmVersions = parsePnpmLock(pnpmLockPath, dependencies);
      Object.assign(installedVersions, pnpmVersions);
      
      const lockLocation = searchDir === projectDir ? 'local' : 'workspace root';
      const foundInPnpmLock = Object.keys(pnpmVersions).length;
      
      if (foundInPnpmLock > 0) {
        console.log(`📦 Found pnpm-lock.yaml (${lockLocation}): extracted ${foundInPnpmLock} versions`);
      }
    }
  }
  
  // Method 2: Fallback to node_modules (if lock files failed or incomplete)
  for (const searchDir of searchDirs) {
    Object.keys(dependencies).forEach(depName => {
      if (!installedVersions[depName]) {
        const nodeModulePath = path.join(searchDir, 'node_modules', depName, 'package.json');
        if (fs.existsSync(nodeModulePath)) {
          try {
            const depPackage = JSON.parse(fs.readFileSync(nodeModulePath, 'utf8'));
            installedVersions[depName] = depPackage.version;
          } catch (error) {
            // Silent fail for individual packages
          }
        }
      }
    });
  }
  
  const foundCount = Object.keys(installedVersions).length;
  const missingCount = Object.keys(dependencies).length - foundCount;
  
  if (foundCount > 0) {
    console.log(`📋 Installed versions found: ${foundCount}/${Object.keys(dependencies).length}`);
  }
  
  if (missingCount > 0) {
    console.log(`⚠️ Missing installed versions: ${missingCount} (likely not installed or in different workspace)`);
  }
  
  return installedVersions;
}

/**
 * Extract dependencies from a package.json file
 * @param {string} packageJsonPath - Path to package.json
 * @returns {Object} Project info with dependencies
 */
function extractDependencies(packageJsonPath) {
  try {
    const content = fs.readFileSync(packageJsonPath, 'utf8');
    const packageData = JSON.parse(content);
    
    const packageName = packageData.name || path.basename(path.dirname(packageJsonPath));
    
    // Extract workspace from path to create unique project names
    const pathParts = packageJsonPath.split(path.sep);
    let workspace = null;
    
    // Look for workspace patterns like npm-design-system, npm-design-blocks
    for (let i = pathParts.length - 1; i >= 0; i--) {
      if (pathParts[i].startsWith('npm-')) {
        workspace = pathParts[i].replace('npm-', '');
        break;
      }
    }
    
    // Create unique project name: workspace-packageName or just packageName
    const projectName = workspace ? `${workspace}-${packageName}` : packageName;
    
    const dependencies = {
      ...packageData.dependencies || {},
      ...packageData.devDependencies || {}
    };
    
    // Extract installed versions
    const installedVersions = extractInstalledVersions(packageJsonPath, dependencies);
    
    return {
      projectName,
      packagePath: packageJsonPath,
      dependencies,
      installedVersions,
      dependencyCount: Object.keys(dependencies).length
    };
    
  } catch (error) {
    throw new Error(`Error reading ${packageJsonPath}: ${error.message}`);
  }
}

/**
 * Main function to orchestrate the analysis
 * @param {string|string[]} patterns - Search pattern(s)
 * @param {Object} options - Configuration options
 * @returns {Promise<Object>} Analysis results
 */
async function main(patterns, options = {}) {
  const { analyzeNpm = true, months = 24, output = 'dependency-report', useNpmCli = true, concurrency = 8, ignoreFilters = [], internalLibs = [], scope = 'all', maxDepsConfig = null } = options;
  
  // Normalize patterns to array
  const patternArray = Array.isArray(patterns) ? patterns : [patterns];
  
  console.log(`🔍 Searching for package.json files in ${patternArray.length} pattern${patternArray.length > 1 ? 's' : ''}:`);
  patternArray.forEach(p => console.log(`   - ${p}`));
  
  try {
    // Phase 1: Discovery
    let allPackageFiles = [];
    for (const pattern of patternArray) {
      const packageFiles = await findPackageJsonFiles(pattern);
      allPackageFiles = [...allPackageFiles, ...packageFiles];
    }
    
    // Remove duplicates (in case same file is found by multiple patterns)
    allPackageFiles = [...new Set(allPackageFiles)];
    
    console.log(`📦 Found ${allPackageFiles.length} package.json files total`);
    
    if (allPackageFiles.length === 0) {
      throw new Error('No package.json files found');
    }
    
    // Phase 2: Extract dependencies
    const projects = [];
    for (const file of allPackageFiles) {
      const projectInfo = extractDependencies(file);
      projects.push(projectInfo);
      console.log(`✅ ${projectInfo.projectName}: ${projectInfo.dependencyCount} dependencies`);
    }
    
    // Phase 3: NPM Analysis (optional)
    if (analyzeNpm) {
      const npmClient = require('./npm-client');
      const libyear = require('./libyear');
      
      // Load max-deps.json constraints
      const maxDeps = loadMaxDeps(patternArray[0], maxDepsConfig);
      if (maxDeps) {
        console.log(`📋 Loaded max-deps.json with ${Object.keys(maxDeps).length} version constraints`);
        Object.entries(maxDeps).forEach(([dep, version]) => {
          console.log(`   - ${dep}: ${version}`);
        });
      }
      
      console.log(`\n🌐 Starting NPM registry analysis...`);
      
      for (const project of projects) {
        // Apply scope filtering first
        const scopeFilterResult = filterDependenciesByScope(project.dependencies, scope, internalLibs);
        
        if (scope !== 'all') {
          console.log(`🔍 ${project.projectName}: Scope filter (${scope}) - ${scopeFilterResult.metadata.filteredCount} of ${scopeFilterResult.metadata.originalCount} dependencies match`);
          
          // Update project dependencies with filtered ones
          project.dependencies = scopeFilterResult.filteredDependencies;
          project.dependencyCount = scopeFilterResult.metadata.filteredCount;
          project.scopeFilter = scopeFilterResult.metadata;
        }
        
        let dependencyNames = Object.keys(project.dependencies);
        
        // Filter out dependencies that are already "handled" in max-deps.json
        if (maxDeps) {
          const originalCount = dependencyNames.length;
          dependencyNames = dependencyNames.filter(dep => !maxDeps.hasOwnProperty(dep));
          const filteredCount = originalCount - dependencyNames.length;
          
          if (filteredCount > 0) {
            console.log(`🔒 ${project.projectName}: Filtered out ${filteredCount} dependencies from max-deps.json (considered up-to-date)`);
          }
        }
        
        if (dependencyNames.length > 0) {
          console.log(`\n📊 Analyzing ${project.projectName} (${dependencyNames.length} dependencies)...`);
          
          const { results, errors } = await npmClient.analyzePackages(dependencyNames, {
            periodMonths: months,
            useNpmCli,
            concurrency,
            maxDepsConstraints: maxDeps,
            ignoreFilters,
            internalLibs
          });

          // Calculate drift/pulse for each dependency
          console.log(`\n🕐 Calculating drift/pulse metrics...`);
          let totalDrift = 0;
          let totalPulse = 0;
          let driftPulseCount = 0;
          let constrainedDependencies = 0;

          for (const [depName, depData] of Object.entries(results)) {
            const specifiedVersion = project.dependencies[depName];
            const installedVersion = project.installedVersions[depName];
            
            if (specifiedVersion) {
              // Add version information to results
              results[depName].specifiedVersion = specifiedVersion;
              results[depName].installedVersion = installedVersion || 'unknown';
              
              // Get NPM data for drift/pulse calculation
              try {
                const npmData = await npmClient.fetchPackageWithNpmCli(depName);
                const maxVersion = maxDeps ? maxDeps[depName] : null;
                
                // Use installedVersion for drift calculation if available, otherwise fall back to specifiedVersion
                let versionForDrift = installedVersion || specifiedVersion;
                
                // If using specifiedVersion and it's a range (^, ~, etc), resolve to best matching version
                if (!installedVersion && specifiedVersion && npmData.time) {
                  const availableVersions = Object.keys(npmData.time).filter(v => v !== 'created' && v !== 'modified');
                  const semver = require('semver');
                  
                  try {
                    // Find the highest version that satisfies the range
                    const satisfyingVersion = semver.maxSatisfying(availableVersions, specifiedVersion);
                    if (satisfyingVersion) {
                      versionForDrift = satisfyingVersion;
                      // Update installedVersion for display in report (fix "unknown" issue)
                      results[depName].installedVersion = satisfyingVersion;
                      console.log(`🔄 ${depName}: Resolved range ${specifiedVersion} to specific version ${satisfyingVersion}`);
                    }
                  } catch (error) {
                    console.log(`⚠️ ${depName}: Failed to resolve range ${specifiedVersion}, using as-is`);
                  }
                }
                
                const driftPulse = libyear.calculateDependencyDriftPulse(npmData, versionForDrift, maxVersion);
                
                if (!driftPulse.error) {
                  results[depName].drift = parseFloat(driftPulse.drift.toFixed(3));
                  results[depName].pulse = parseFloat(driftPulse.pulse.toFixed(3));
                  results[depName].targetVersion = driftPulse.targetVersion;
                  results[depName].constrainedByMaxVersion = driftPulse.constrainedByMaxVersion;
                  
                  totalDrift += driftPulse.drift;
                  totalPulse += driftPulse.pulse;
                  driftPulseCount++;
                  
                  if (driftPulse.constrainedByMaxVersion) {
                    constrainedDependencies++;
                  }
                } else {
                  results[depName].drift = 0;
                  results[depName].pulse = 0;
                  console.log(`⚠️ ${depName}: ${driftPulse.error}`);
                }
              } catch (error) {
                results[depName].drift = 0;
                results[depName].pulse = 0;
                console.log(`⚠️ ${depName}: Failed to calculate drift/pulse - ${error.message}`);
              }
            }
          }
          
          project.dependencyAnalysis = results;
          project.analysisErrors = errors;
          
          // Add drift/pulse summary to project
          project.driftPulseSummary = {
            totalDrift: parseFloat(totalDrift.toFixed(3)),
            totalPulse: parseFloat(totalPulse.toFixed(3)),
            avgDriftPerDep: driftPulseCount > 0 ? parseFloat((totalDrift / driftPulseCount).toFixed(3)) : 0,
            avgPulsePerDep: driftPulseCount > 0 ? parseFloat((totalPulse / driftPulseCount).toFixed(3)) : 0,
            dependenciesWithDriftPulse: driftPulseCount,
            constrainedDependencies: constrainedDependencies
          };
          
          const successCount = Object.keys(results).length;
          console.log(`✅ ${project.projectName}: ${successCount}/${dependencyNames.length} dependencies analyzed`);
          console.log(`📊 Drift/Pulse: ${project.driftPulseSummary.totalDrift}y drift, ${project.driftPulseSummary.totalPulse}y pulse`);
          
          if (constrainedDependencies > 0) {
            console.log(`🔒 ${constrainedDependencies} dependencies constrained by max-deps.json`);
          }
          
          if (errors.length > 0) {
            console.log(`⚠️  ${errors.length} dependencies had errors`);
          }
        }
      }
    }
    
    // Phase 4: Generate reports if output is specified
    if (output && analyzeNpm) {
      const reporter = require('./reporter');
      
      console.log(`\n📄 Generating reports...`);
      const analysisData = {
        metadata: {
          generatedAt: new Date().toISOString(),
          patterns: patternArray,
          totalProjects: projects.length,
          options: { months, analyzeNpm }
        },
        projects,
        options: { months }
      };
      
      await reporter.writeReports(analysisData, output);
    }
    
    return {
      metadata: {
        generatedAt: new Date().toISOString(),
        patterns: patternArray,
        totalProjects: projects.length,
        options: { months, analyzeNpm, output, useNpmCli, concurrency, ignoreFilters, internalLibs, scope }
      },
      projects
    };
    
  } catch (error) {
    console.error('❌ Error:', error.message);
    throw error;
  }
}

module.exports = {
  findPackageJsonFiles,
  extractDependencies,
  main
};