@boilerbuilder/deps-analyzer/lib/npm-client.js

CLI tool to analyze dependency evolution and release frequency

const https = require('https');
const { exec } = require('child_process');
const { promisify } = require('util');
const semver = require('semver');

const execAsync = promisify(exec);

// Import utility functions
const findMaxAllowedVersion = require('../utils/findMaxAllowedVersion');
const versionSatisfiesConstraint = require('../utils/versionSatisfiesConstraint');
const chunkArray = require('../utils/chunkArray');
const isInternalLib = require('../utils/isInternalLib');


/**
 * Fetch package information from NPM registry
 * @param {string} packageName - NPM package name
 * @returns {Promise<Object>} Package data from registry
 */
async function fetchPackageReleases(packageName) {
  return new Promise((resolve, reject) => {
    const url = `https://registry.npmjs.org/${encodeURIComponent(packageName)}`;
    
    const req = https.get(url, (res) => {
      let data = '';
      
      res.on('data', (chunk) => {
        data += chunk;
      });
      
      res.on('end', () => {
        try {
          if (res.statusCode === 404) {
            reject(new Error(`Package not found: ${packageName}`));
            return;
          }
          
          if (res.statusCode !== 200) {
            reject(new Error(`HTTP ${res.statusCode}: ${packageName}`));
            return;
          }
          
          const packageData = JSON.parse(data);
          
          if (!packageData.time) {
            reject(new Error(`No release data found for: ${packageName}`));
            return;
          }
          
          resolve({
            name: packageData.name,
            time: packageData.time,
            latestVersion: packageData['dist-tags']?.latest,
            totalVersions: Object.keys(packageData.versions || {}).length
          });
          
        } catch (error) {
          reject(new Error(`Failed to parse response for ${packageName}: ${error.message}`));
        }
      });
    });
    
    req.on('error', (error) => {
      reject(new Error(`Network error for ${packageName}: ${error.message}`));
    });
    
    req.setTimeout(10000, () => {
      req.destroy();
      reject(new Error(`Timeout fetching ${packageName}`));
    });
  });
}

/**
 * Generate dynamic periods based on total months
 * @param {number} totalMonths - Total period in months
 * @returns {Array} Array of period objects with start, end, label, cutoffDate
 */
function generateDynamicPeriods(totalMonths) {
  const now = new Date();
  const periods = [];
  
  for (let i = 0; i < totalMonths; i += 12) {
    const start = i;
    const end = Math.min(i + 12, totalMonths);
    const label = start === 0 ? `${end}m` : `${start}-${end}m`;
    const cutoffDate = new Date(now.getFullYear(), now.getMonth() - end, now.getDate());
    
    periods.push({ start, end, label, cutoffDate });
  }
  
  return periods;
}

/**
 * Filter releases by time period (dynamic periods)
 * @param {Object} timeData - Time data from NPM registry
 * @param {number} periodMonths - Period in months (default: 24)
 * @param {string} packageName - Package name for debugging (default: 'unknown')
 * @param {string|null} maxVersionConstraint - Max version constraint (e.g., "^18.0.0")
 * @param {Array<string>} ignoreFilters - Filters to ignore (e.g., ['alpha', 'beta'])
 * @param {Array<string>} internalLibs - Internal libraries patterns (e.g., ['@akad/*'])
 * @returns {Object} Release counts by period
 */
function filterReleasesByPeriod(timeData, periodMonths = 24, packageName = 'unknown', maxVersionConstraint = null, ignoreFilters = [], internalLibs = []) {
  const periods = generateDynamicPeriods(periodMonths);
  const now = new Date();
  
  const releases = {
    total: 0,
    monthlyBreakdown: {},
    periods: []
  };
  
  // Filter out special entries and development versions
  const versions = Object.entries(timeData).filter(([version]) => {
    // Skip metadata entries
    if (version === 'created' || version === 'modified') return false;
    
    // Skip pre-release and development versions (configurable with ignoreFilters and internalLibs)
    const isInternal = isInternalLib(packageName, internalLibs);
    
    // If it's an internal library, allow alpha/beta versions (don't filter them)
    if (!isInternal) {
      const skipAlpha = !ignoreFilters.includes('alpha') && version.includes('-alpha');
      const skipBeta = !ignoreFilters.includes('beta') && version.includes('-beta');
      const skipRc = !ignoreFilters.includes('rc') && version.includes('-rc');
      const skipCanary = !ignoreFilters.includes('canary') && version.includes('-canary');
      const skipExperimental = !ignoreFilters.includes('experimental') && version.includes('-experimental');
      const skipDev = !ignoreFilters.includes('dev') && (version.includes('-dev') || version.includes('.dev.'));
      const skipNightly = !ignoreFilters.includes('nightly') && version.includes('-nightly');
      const skipPre = !ignoreFilters.includes('pre') && version.includes('-pre');
      const skipZeroVersion = !ignoreFilters.includes('zero') && version.startsWith('0.0.0-');
      
      if (skipAlpha || skipBeta || skipRc || skipCanary || skipExperimental || 
          skipDev || skipNightly || skipPre || skipZeroVersion) {
        return false;
      }
    } else {
      // Log when we're including alpha/beta versions for internal libs
      if (version.includes('-alpha') || version.includes('-beta')) {
        console.log(`🔓 ${packageName}: Including ${version} (internal library)`);
      }
    }
    
    // Apply max-deps constraint filter
    if (maxVersionConstraint && !versionSatisfiesConstraint(version, maxVersionConstraint)) {
      return false;
    }
    
    // Keep only stable versions
    return true;
  });
  
  // Log constraint filtering for debugging
  if (maxVersionConstraint && packageName !== 'unknown') {
    const allVersions = Object.keys(timeData).filter(v => v !== 'created' && v !== 'modified');
    const filteredOut = allVersions.length - versions.length;
    if (filteredOut > 0) {
      console.log(`🔒 ${packageName}: Filtered out ${filteredOut} versions exceeding constraint ${maxVersionConstraint}`);
    }
  }
  
  // Sort by timestamp to calculate average days between releases and semver analysis
  const sortedReleases = versions
    .map(([version, timestamp]) => ({ version, date: new Date(timestamp) }))
    .filter(release => !isNaN(release.date.getTime()))
    .sort((a, b) => a.date - b.date);
  
  // Initialize dynamic periods structure
  const releasesByPeriod = {};
  periods.forEach(period => {
    releasesByPeriod[period.label] = [];
  });
  releasesByPeriod.older = [];
  
  // Group releases by dynamic time periods
  for (const release of sortedReleases) {
    releases.total++;
    
    let assigned = false;
    
    // Check which period this release belongs to
    for (const period of periods) {
      const periodStartDate = new Date(now.getFullYear(), now.getMonth() - period.start, now.getDate());
      const periodEndDate = period.cutoffDate;
      
      if (release.date >= periodEndDate && release.date < periodStartDate) {
        releasesByPeriod[period.label].push(release);
        assigned = true;
        
        // Monthly breakdown with version details (only for most recent period)
        if (period === periods[0]) {
          const monthKey = `${release.date.getFullYear()}-${String(release.date.getMonth() + 1).padStart(2, '0')}`;
          releases.monthlyBreakdown[monthKey] = releases.monthlyBreakdown[monthKey] || [];
          releases.monthlyBreakdown[monthKey].push(release.version);
        }
        break;
      }
    }
    
    // If not assigned to any period, it's older
    if (!assigned) {
      releasesByPeriod.older.push(release);
    }
  }
  
  // Build periods array for response
  periods.forEach(period => {
    const periodReleases = releasesByPeriod[period.label];
    releases.periods.push({
      label: period.label,
      releases: periodReleases.length,
      avgReleasesPerMonth: parseFloat((periodReleases.length / 12).toFixed(2)),
      semver: { major: 0, minor: 0, patch: 0 } // Will be calculated below
    });
  });
  
  // SIMPLIFIED semver analysis - analyze all stable versions chronologically
  const analyzeSemverChanges = (periodReleases, contextReleases = [], packageName = 'unknown') => {
    const counts = { major: 0, minor: 0, patch: 0 };
    
    if (periodReleases.length === 0) return counts;
    
    // DEBUG: Log for specific packages we're debugging
    const shouldLog = packageName.includes('types/react') || packageName.includes('types/node') || 
                     packageName.includes('typescript') || packageName.includes('vite') || packageName.includes('next');
    
    if (shouldLog) {
      console.log(`\n🔍 DEBUG: Analyzing ${packageName}`);
      console.log(`📦 Period releases (${periodReleases.length}):`, periodReleases.map(r => `${r.version} (${r.date.toISOString().substr(0,10)})`));
    }
    
    // Filter out pre-release and development versions from all releases
    const filterStableVersions = (releases) => {
      return releases.filter(release => {
        const version = release.version;
        // Skip pre-release versions (alpha, beta, rc, canary, experimental, dev, nightly, pre)
        if (version.includes('-alpha') || version.includes('-beta') || 
            version.includes('-rc') || version.includes('-canary') ||
            version.includes('-experimental') || version.includes('-dev') ||
            version.includes('-nightly') || version.includes('-pre') ||
            version.includes('.dev.') || version.startsWith('0.0.0-')) {
          return false;
        }
        // Keep stable versions
        return true;
      });
    };
    
    // Get all stable releases and sort chronologically
    const allStableReleases = [
      ...filterStableVersions(periodReleases), 
      ...filterStableVersions(contextReleases || [])
    ].sort((a, b) => a.date - b.date);
    
    const periodStableReleases = filterStableVersions(periodReleases)
      .sort((a, b) => a.date - b.date);
    
    if (shouldLog) {
      console.log(`✅ Stable period releases (${periodStableReleases.length}):`, periodStableReleases.map(r => `${r.version} (${r.date.toISOString().substr(0,10)})`));
      console.log(`🗂️ Context releases (${contextReleases.length}):`, contextReleases.length > 10 ? `${contextReleases.length} releases` : contextReleases.map(r => `${r.version} (${r.date.toISOString().substr(0,10)})`));
    }
    
    if (periodStableReleases.length === 0) {
      // No stable releases in period - count all as patches
      counts.patch = periodReleases.length;
      if (shouldLog) console.log(`⚠️ No stable releases, counting ${periodReleases.length} as patches`);
      return counts;
    }
    
    // Find context: latest stable release before the period
    let prevVersion = null;
    const periodStartDate = Math.min(...periodStableReleases.map(r => r.date.getTime()));
    
    for (const release of allStableReleases) {
      if (release.date.getTime() < periodStartDate) {
        prevVersion = release.version;
      } else {
        break;
      }
    }
    
    if (shouldLog) {
      console.log(`🎯 Context version: ${prevVersion || 'NONE'}`);
      console.log(`📅 Period start date: ${new Date(periodStartDate).toISOString().substr(0,10)}`);
    }
    
    // Analyze each period release
    for (const currentRelease of periodStableReleases) {
      if (prevVersion) {
        try {
          // Check for downgrades (version republishing/backports)
          const isDowngrade = semver.lt(currentRelease.version, prevVersion);
          
          if (isDowngrade) {
            // Skip downgrades - they're usually backports or republishing
            if (shouldLog) {
              console.log(`⬇️ SKIP DOWNGRADE: ${prevVersion} → ${currentRelease.version} (likely backport)`);
            }
            continue;
          }
          
          const diff = semver.diff(prevVersion, currentRelease.version);
          if (shouldLog) {
            console.log(`🔄 ${prevVersion} → ${currentRelease.version} = ${diff}`);
          }
          
          if (diff === 'major') {
            counts.major++;
          } else if (diff === 'minor' || diff === 'preminor') {
            counts.minor++;
          } else if (diff === 'patch' || diff === 'prepatch') {
            counts.patch++;
          } else {
            // For any other diff type, count as patch
            counts.patch++;
            if (shouldLog) console.log(`⚠️ Unknown diff type "${diff}", counting as patch`);
          }
          
          prevVersion = currentRelease.version;
        } catch (error) {
          // If semver.diff fails, count as patch
          counts.patch++;
          if (shouldLog) console.log(`❌ semver.diff failed: ${error.message}, counting as patch`);
          prevVersion = currentRelease.version;
        }
      } else {
        // First release without context - count as patch
        counts.patch++;
        if (shouldLog) console.log(`🆕 First release ${currentRelease.version} without context, counting as patch`);
        prevVersion = currentRelease.version;
      }
    }
    
    // Add non-stable releases as patches (pre-releases, etc.)
    const nonStableCount = periodReleases.length - periodStableReleases.length;
    counts.patch += nonStableCount;
    
    if (shouldLog) {
      console.log(`📊 Final counts: Major: ${counts.major}, Minor: ${counts.minor}, Patch: ${counts.patch}`);
      console.log(`🎯 Non-stable releases added as patches: ${nonStableCount}`);
    }
    
    return counts;
  };
  
  // Analyze semver changes for each period with proper context (dynamic)
  for (let i = 0; i < periods.length; i++) {
    const period = periods[i];
    const periodReleases = releasesByPeriod[period.label];
    
    // Build context from older periods
    let contextReleases = releasesByPeriod.older;
    for (let j = i + 1; j < periods.length; j++) {
      contextReleases = [...contextReleases, ...releasesByPeriod[periods[j].label]];
    }
    
    const semverAnalysis = analyzeSemverChanges(periodReleases, contextReleases, packageName);
    
    // Update the period object with semver data
    releases.periods[i].semver = semverAnalysis;
  }
  
  // Calculate average days between releases (for all releases in period)
  if (sortedReleases.length > 1) {
    const totalDays = (sortedReleases[sortedReleases.length - 1].date - sortedReleases[0].date) / (1000 * 60 * 60 * 24);
    releases.avgDaysBetweenReleases = parseFloat((totalDays / (sortedReleases.length - 1)).toFixed(1));
  } else {
    releases.avgDaysBetweenReleases = 0;
  }
  
  return releases;
}

/**
 * Fetch package information using npm CLI (much faster)
 * @param {string} packageName - NPM package name
 * @returns {Promise<Object>} Package data
 */
async function fetchPackageWithNpmCli(packageName) {
  try {
    const { stdout } = await execAsync(`npm view ${packageName} time --json`);
    const timeData = JSON.parse(stdout);
    
    // Also get latest version
    const { stdout: versionOut } = await execAsync(`npm view ${packageName} version --json`);
    const latestVersion = JSON.parse(versionOut);
    
    return {
      name: packageName,
      time: timeData,
      latestVersion,
      totalVersions: Object.keys(timeData).filter(v => v !== 'created' && v !== 'modified').length
    };
    
  } catch (error) {
    throw new Error(`NPM CLI error for ${packageName}: ${error.message}`);
  }
}


/**
 * Process multiple packages with parallel npm CLI calls (MUCH FASTER!)
 * @param {Array<string>} packageNames - Array of package names
 * @param {Object} options - Options including concurrency and periodMonths
 * @returns {Promise<Object>} Analysis results for all packages
 */
async function analyzePackages(packageNames, options = {}) {
  const { concurrency = 8, periodMonths = 24, useNpmCli = true, maxDepsConstraints = null, ignoreFilters = [], internalLibs = [] } = options;
  const results = {};
  const errors = [];
  
  console.log(`🔍 Analyzing ${packageNames.length} packages with ${useNpmCli ? 'NPM CLI' : 'HTTP'} (${concurrency} concurrent)...`);
  
  if (useNpmCli) {
    // Use fast npm CLI approach with parallelization
    const chunks = chunkArray(packageNames, concurrency);
    let processed = 0;
    
    for (const chunk of chunks) {
      const promises = chunk.map(async (packageName) => {
        try {
          console.log(`📦 [${processed + 1}/${packageNames.length}] Fetching ${packageName}...`);
          
          const packageData = await fetchPackageWithNpmCli(packageName);
          const maxConstraint = maxDepsConstraints ? maxDepsConstraints[packageName] : null;
          const releaseAnalysis = filterReleasesByPeriod(packageData.time, periodMonths, packageName, maxConstraint, ignoreFilters, internalLibs);
          
          // Apply max-deps constraints to currentVersion if available
          let currentVersion = packageData.latestVersion;
          if (maxConstraint && packageData.time) {
            const availableVersions = Object.keys(packageData.time).filter(v => v !== 'created' && v !== 'modified');
            const maxAllowedVersion = findMaxAllowedVersion(maxConstraint, availableVersions);
            if (maxAllowedVersion) {
              currentVersion = maxAllowedVersion;
              console.log(`🔒 ${packageName}: Limited to ${maxAllowedVersion} (constraint: ${maxConstraint}) instead of ${packageData.latestVersion}`);
            }
          }
          
          results[packageName] = {
            currentVersion: currentVersion,
            totalVersions: packageData.totalVersions,
            ...releaseAnalysis
          };
          
          // Log period summary dynamically
          const periodSummary = releaseAnalysis.periods.map(p => `${p.releases} (${p.label})`).join(', ');
          console.log(`✅ ${packageName}: ${periodSummary}`);
          processed++;
          
        } catch (error) {
          console.log(`❌ ${packageName}: ${error.message}`);
          errors.push({ package: packageName, error: error.message });
          processed++;
        }
      });
      
      await Promise.allSettled(promises);
      
      // Small delay between chunks to be nice to npm
      if (chunks.indexOf(chunk) < chunks.length - 1) {
        await new Promise(resolve => setTimeout(resolve, 50));
      }
    }
    
  } else {
    // Fallback to HTTP method (original implementation)
    for (let i = 0; i < packageNames.length; i++) {
      const packageName = packageNames[i];
      
      try {
        console.log(`📦 [${i + 1}/${packageNames.length}] Fetching ${packageName}...`);
        
        const packageData = await fetchPackageReleases(packageName);
        const maxConstraint = maxDepsConstraints ? maxDepsConstraints[packageName] : null;
        const releaseAnalysis = filterReleasesByPeriod(packageData.time, periodMonths, packageName, maxConstraint, ignoreFilters, internalLibs);
        
        // Apply max-deps constraints to currentVersion if available
        let currentVersion = packageData.latestVersion;
        if (maxConstraint && packageData.time) {
          const availableVersions = Object.keys(packageData.time).filter(v => v !== 'created' && v !== 'modified');
          const maxAllowedVersion = findMaxAllowedVersion(maxConstraint, availableVersions);
          if (maxAllowedVersion) {
            currentVersion = maxAllowedVersion;
            console.log(`🔒 ${packageName}: Limited to ${maxAllowedVersion} (constraint: ${maxConstraint}) instead of ${packageData.latestVersion}`);
          }
        }
        
        results[packageName] = {
          currentVersion: currentVersion,
          totalVersions: packageData.totalVersions,
          ...releaseAnalysis
        };
        
        // Log period summary dynamically
        const periodSummary = releaseAnalysis.periods.map(p => `${p.releases} (${p.label})`).join(', ');
        console.log(`✅ ${packageName}: ${periodSummary}`);
        
        // Rate limiting for HTTP
        if (i < packageNames.length - 1) {
          await new Promise(resolve => setTimeout(resolve, 100));
        }
        
      } catch (error) {
        console.log(`❌ ${packageName}: ${error.message}`);
        errors.push({ package: packageName, error: error.message });
        
        if (i < packageNames.length - 1) {
          await new Promise(resolve => setTimeout(resolve, 100));
        }
      }
    }
  }
  
  return { results, errors };
}

module.exports = {
  fetchPackageReleases,
  fetchPackageWithNpmCli,
  filterReleasesByPeriod,
  analyzePackages
};