rugby/src/lib/script-network-capture.js

A Node.js CLI to collect npm package tarballs and assets (including those fetched by pre/postinstall scripts) for offline Artifactory use.

"use strict";

const path = require("path");
const fs = require("fs-extra");
const os = require("os");
const child_process = require("child_process");
const which = require("which");
const crypto = require("crypto");

// Strategy:
// - Run `npm ci --ignore-scripts` to materialize node_modules deterministically without running scripts.
// - Then run install lifecycle scripts in a sandbox by invoking `npm rebuild --foreground-scripts` so that
//   only lifecycle scripts execute. We wrap spawn with env vars that point HTTP(S)_PROXY to our local proxy
//   recorder (implemented via Node fetch proxy) which logs all external requests and also downloads those assets
//   into outDir for later upload. We also record file reads of URLs passed to curl/wget by patching PATH to
//   shim curl/wget.

async function collectNetworkDuringScripts({ projectDir, outDir, verbose, npmPath, npmArgs = "" }) {
  await fs.mkdirp(outDir);

  // On Windows, prefer npm.cmd over npm.ps1 to avoid PowerShell execution policy issues
  const npm = npmPath || (process.platform === "win32" 
    ? (fs.existsSync("C:\\Program Files\\nodejs\\npm.cmd") 
      ? "C:\\Program Files\\nodejs\\npm.cmd" 
      : which.sync("npm"))
    : which.sync("npm"));
  const realNode = which.sync("node");

  const env = { ...process.env };
  const logPath = path.join(outDir, "network.log");
  const assetsDir = path.join(outDir, "assets-temp");
  await fs.mkdirp(assetsDir);

  // Generate shims for curl and wget that log and download files via Node
  const shimDir = path.join(outDir, "shims");
  await fs.mkdirp(shimDir);
  
  // On Windows, create batch file shims; on Unix, create bash shims
  if (process.platform === "win32") {
    // Windows batch file shims
    const curlShim = windowsShim("curl", assetsDir, logPath);
    const wgetShim = windowsShim("wget", assetsDir, logPath);
    const nodeShim = windowsNodeShim(realNode);
    await fs.writeFile(path.join(shimDir, "curl.bat"), curlShim);
    await fs.writeFile(path.join(shimDir, "wget.bat"), wgetShim);
    await fs.writeFile(path.join(shimDir, "node.bat"), nodeShim);
  } else {
    // Unix bash shims
    const curlShim = `#!/usr/bin/env bash\n${bashShim("curl", assetsDir)}\n`;
    const wgetShim = `#!/usr/bin/env bash\n${bashShim("wget", assetsDir)}\n`;
    const nodeShim = `#!/usr/bin/env bash\n${nodeShimScript(realNode)}\n`;
    await fs.writeFile(path.join(shimDir, "curl"), curlShim, { mode: 0o755 });
    await fs.writeFile(path.join(shimDir, "wget"), wgetShim, { mode: 0o755 });
    await fs.writeFile(path.join(shimDir, "node"), nodeShim, { mode: 0o755 });
  }

  // Prepend our shim directory to PATH so pre/postinstall scripts use it
  const pathSeparator = process.platform === "win32" ? ";" : ":";
  env.PATH = `${shimDir}${pathSeparator}${env.PATH || ""}`;
  env.RUGBY_ASSETS_DIR = assetsDir;
  env.RUGBY_LOG = logPath;
  
  // Skip NODE_OPTIONS hook on Windows if path contains spaces - it causes issues with argument parsing
  // TODO: Find a better solution for Windows paths with spaces
  const httpHookPath = path.join(__dirname, "http-hook.js");
  if (process.platform !== "win32" || !httpHookPath.includes(' ')) {
    env.NODE_OPTIONS = `${env.NODE_OPTIONS ? env.NODE_OPTIONS + " " : ""}--require ${httpHookPath}`;
  } else {
    console.warn("Warning: Skipping http hook on Windows due to path containing spaces. Some network capture features may not work.");
  }

  // 1) Ensure node_modules are present without scripts
  try {
    spawnSyncLogged(npm, ["ci", "--ignore-scripts", ...splitArgs(npmArgs)], { cwd: projectDir, env, verbose });
  } catch (e) {
    // fallback to install if ci fails (e.g., no lockfile v2)
    spawnSyncLogged(npm, ["install", "--ignore-scripts", ...splitArgs(npmArgs)], { cwd: projectDir, env, verbose });
  }

  // Baseline node_modules snapshot
  const baseline = await scanNodeModules(path.join(projectDir, "node_modules"));

  // 2) Run lifecycle scripts only
  spawnSyncLogged(npm, ["rebuild", "--foreground-scripts", "--ignore-scripts=false", ...splitArgs(npmArgs)], {
    cwd: projectDir,
    env,
    verbose,
  });

  // Post-run node_modules snapshot and diff
  const after = await scanNodeModules(path.join(projectDir, "node_modules"));
  const scriptPackages = [];
  for (const [pkgName, version] of after.entries()) {
    if (!baseline.has(pkgName)) {
      scriptPackages.push({ name: pkgName, version });
    }
  }

  // Collect assets list by scanning assetsDir
  const assets = [];
  const files = await fs.readdir(assetsDir);
  for (const f of files) {
    assets.push({ filename: f, tempPath: path.join(assetsDir, f) });
  }

  // Parse log for URL list as well
  if (await fs.pathExists(logPath)) {
    const content = await fs.readFile(logPath, "utf8");
    const urlRegex = /(https?:\/[\w\-._~:/?#[\]@!$&'()*+,;=%]+)/g;
    let m;
    while ((m = urlRegex.exec(content))) {
      assets.push({ url: m[1] });
    }
  }

  return { assets, logPath, scriptPackages };
}

function splitArgs(str) {
  if (!str) return [];
  // naive split respecting quotes
  const re = /\s+([^\s\"]+|\"[^\"]*\")/g;
  const parts = [];
  let m;
  const s = ` ${str}`;
  while ((m = re.exec(s))) {
    const p = m[1];
    parts.push(p.startsWith('"') && p.endsWith('"') ? p.slice(1, -1) : p);
  }
  return parts;
}

function spawnSyncLogged(cmd, args, { cwd, env, verbose }) {
  if (verbose) console.log("$", cmd, ...args);
  
  // On Windows, we need to handle .cmd files specially
  let spawnCmd = cmd;
  let spawnArgs = args;
  let spawnOptions = {
    cwd,
    env,
    stdio: "inherit",
  };
  
  if (process.platform === "win32") {
    // Use cmd.exe to run .cmd files on Windows to avoid PowerShell execution policy issues
    if (cmd.endsWith('.cmd') || cmd.endsWith('.CMD')) {
      spawnCmd = process.env.ComSpec || 'C:\\Windows\\System32\\cmd.exe';
      spawnArgs = ['/c', cmd, ...args];
    } else {
      spawnOptions.shell = true;
    }
  }
  
  const res = child_process.spawnSync(spawnCmd, spawnArgs, spawnOptions);
  if (res.error) throw res.error;
  if (res.status !== 0) throw new Error(`${cmd} ${args.join(" ")} failed with code ${res.status}`);
  return res;
}

function bashShim(tool, assetsDir) {
  return `
set -euo pipefail
LOG="${assetsDir}/network.log"
ARGS=("$@")
URL=""
for a in "${'$'}{ARGS[@]}"; do
  if [[ "${'$'}a" =~ ^https?:// ]]; then URL="${'$'}a"; break; fi
done
if [ -n "${'$'}URL" ]; then
  echo "[shim:${tool}] ${'$'}URL" >> "${'$'}LOG" 2>/dev/null || true
  # Opportunistic fetch for capture; ignore failures
  node -e 'const fs=require("fs"),path=require("path"),http=require("http"),https=require("https");
const url=process.argv[1];
const fn=Buffer.from(url).toString("base64").replace(/[^a-zA-Z0-9_-]/g,"")+".bin";
const dest=path.join(process.env.RUGBY_ASSETS_DIR||"${assetsDir}",fn);
const proto=url.startsWith("https")?https:http;
proto.get(url,(res)=>{if(res.statusCode>=300&&res.statusCode<400&&res.headers.location){return require("http");}
const out=fs.createWriteStream(dest);res.pipe(out);out.on("finish",()=>out.close());}).on("error",()=>{});' "${'$'}URL" >/dev/null 2>&1 || true
fi
exec -a ${tool} $(command -v -- ${tool}) "${'$'}@"
`;
}

function nodeShimScript(realNodePath) {
  return `
set -euo pipefail
HOOK="${path.join(__dirname, 'http-hook.js').replace(/"/g, '\\"')}"
exec "${realNodePath}" -r "${'$'}HOOK" "${'$'}@"
`;
}

function windowsShim(tool, assetsDir, logPath) {
  // Windows batch file shim for curl/wget
  return `@echo off
set LOG=${logPath}
set ASSETS_DIR=${assetsDir}
set URL=
set OUTPUT=

:parseargs
if "%~1"=="" goto :run
if "%~1"=="-o" (
  set OUTPUT=%~2
  shift
  shift
  goto :parseargs
)
if "%~1"=="-O" (
  set OUTPUT=%%~nx1
  shift
  goto :parseargs
)
set URL=%~1
shift
goto :parseargs

:run
if not "%URL%"=="" (
  echo [%DATE% %TIME%] ${tool} %URL% >> "%LOG%"
  if not "%OUTPUT%"=="" (
    echo Downloading %URL% to %OUTPUT%
    powershell -Command "Invoke-WebRequest -Uri '%URL%' -OutFile '%ASSETS_DIR%\\%OUTPUT%'"
  )
)
exit /b 0
`;
}

function windowsNodeShim(realNodePath) {
  // Windows batch file shim for node
  const hookPath = path.join(__dirname, 'http-hook.js').replace(/\\/g, '\\\\');
  return `@echo off
"${realNodePath}" -r "${hookPath}" %*
`;
}

module.exports = { collectNetworkDuringScripts };

async function scanNodeModules(root) {
  const results = new Map();
  async function walk(dir) {
    let entries = [];
    try {
      entries = await fs.readdir(dir, { withFileTypes: true });
    } catch {
      return;
    }
    for (const ent of entries) {
      if (ent.name.startsWith(".")) continue;
      const full = path.join(dir, ent.name);
      if (ent.isDirectory()) {
        if (ent.name.startsWith("@")) {
          await walk(full);
          continue;
        }
        const pkgJsonPath = path.join(full, "package.json");
        if (await fs.pathExists(pkgJsonPath)) {
          try {
            const pkg = await fs.readJson(pkgJsonPath);
            if (pkg && pkg.name && pkg.version) {
              results.set(pkg.name, pkg.version);
            }
          } catch {}
        }
        const subNodeModules = path.join(full, "node_modules");
        if (await fs.pathExists(subNodeModules)) {
          await walk(subNodeModules);
        }
      }
    }
  }
  await walk(root);
  return results;
}