npminstall
Version:
Make npm install fast and handy.
781 lines (709 loc) • 23.3 kB
JavaScript
const debug = require('node:util').debuglog('npminstall:utils');
const fs = require('node:fs/promises');
const { accessSync } = require('node:fs');
const path = require('node:path');
const cp = require('node:child_process');
const { promisify } = require('node:util');
const { parse: urlparse } = require('node:url');
const url = require('node:url');
const querystring = require('node:querystring');
const zlib = require('node:zlib');
const chalk = require('chalk');
const globby = require('globby');
const tar = require('tar');
const { command } = require('execa');
const homedir = require('node-homedir');
const fse = require('fs-extra');
const destroy = require('destroy');
const normalizeData = require('normalize-package-data');
const semver = require('semver');
const globalConfig = require('./config');
const get = require('./get');
exports.exists = async filepath => {
try {
await fs.access(filepath);
return true;
} catch {
return false;
}
};
exports.existsSync = filepath => {
try {
accessSync(filepath);
return true;
} catch {
return false;
}
};
exports.hasOwnProp = (target, key) => target.hasOwnProperty(key);
/**
*
* @param {String} filepath cwd package.json path
* @param {String} depName dependency name
* @description clear removed pkg info from package.json
*/
exports.pruneJSON = async (filepath, depName) => {
const pkg = await this.readJSON(filepath);
const depMap = {};
const depKeys = [ 'dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies' ];
for (const key of depKeys) {
pkg[key] && (depMap[key] = pkg[key]);
}
for (const dep of Object.values(depMap)) {
if (this.hasOwnProp(dep, depName)) {
Reflect.deleteProperty(dep, depName);
}
}
this.addMetaToJSONFile(filepath, depMap);
};
exports.readJSON = async filepath => {
if (!(await exports.exists(filepath))) {
return {};
}
const content = await fs.readFile(filepath, 'utf8');
try {
return JSON.parse(content.trim());
} catch (err) {
err.message += ` (file: ${filepath})`;
console.error('content buffer: %j', await fs.readFile(filepath));
throw err;
}
};
exports.readPackageJSON = async root => {
const pkg = await exports.readJSON(path.join(root, 'package.json'));
normalizeData(pkg);
return pkg;
};
const INSTALL_DONE_KEY = '__npminstall_done';
// 设置 pkg 安装完成的标记
exports.setInstallDone = async pkgRoot => {
await exports.addMetaToJSONFile(path.join(pkgRoot, 'package.json'), {
[INSTALL_DONE_KEY]: true,
});
};
exports.unsetInstallDone = async pkgRoot => {
await exports.addMetaToJSONFile(path.join(pkgRoot, 'package.json'), {
[INSTALL_DONE_KEY]: false,
});
};
exports.removeInstallDone = async pkgRoot => {
const pkgFile = path.join(pkgRoot, 'package.json');
if (!(await exports.exists(pkgFile))) return;
const pkg = await exports.readJSON(pkgFile);
if (!(INSTALL_DONE_KEY in pkg)) return;
await exports.addMetaToJSONFile(pkgFile, {
[INSTALL_DONE_KEY]: undefined,
});
};
// 判断 pkg 是否已经安装完成
exports.isInstallDone = async pkgRoot => {
const pkg = await exports.readJSON(path.join(pkgRoot, 'package.json'));
return !!pkg[INSTALL_DONE_KEY];
};
exports.addMetaToJSONFile = async (filepath, meta) => {
await fs.chmod(filepath, '644');
const pkg = await exports.readJSON(filepath);
for (const key in meta) {
pkg[key] = meta[key];
}
await fs.writeFile(filepath, JSON.stringify(pkg, null, 2) + '\n');
};
exports.mkdirp = async dir => {
await fs.mkdir(dir, { recursive: true });
};
exports.rimraf = async dest => {
await fs.rm(dest, { force: true, recursive: true });
};
exports.relative = (src, dest) => {
// Windows don't support relative path
if (process.platform === 'win32') return src;
return path.relative(path.dirname(dest), src);
};
exports.forceSymlink = async (src, dest, type) => {
const relative = exports.relative(src, dest);
type = type || 'junction';
// cleanup dest
try {
const linkString = await fs.readlink(dest);
// already linked
if (linkString === relative) {
return relative;
}
} catch (err) {
// ignore error, will always cleanup dest
}
const destDir = path.dirname(dest);
// check if destDir is not exist
if (!(await exports.exists(destDir))) {
await exports.mkdirp(destDir);
}
await exports.rimraf(dest);
await fs.symlink(relative, dest, type);
return relative;
};
function setNpmPackageEnv(env, key, value) {
const t = typeof value;
if (t === 'string' || t === 'number' || t === 'boolean') {
env[`npm_package_${key}`] = value;
} else if (value === null) {
env[`npm_package_${key}`] = 'null';
} else if (value) {
for (const subkey in value) {
setNpmPackageEnv(env, `${key}_${subkey}`, value[subkey]);
}
}
}
exports.formatPackageUrl = (registry, name) => {
if (name[0] === '@') {
// dont encodeURIComponent @ char, it will be 405
// https://registry.npmjs.com/%40rstacruz%2Ftap-spec/%3E%3D4.1.1
name = '@' + encodeURIComponent(name.substring(1));
}
const parsed = url.parse(registry);
if (parsed.pathname.endsWith('/')) {
parsed.pathname += name;
} else {
parsed.pathname += `/${name}`;
}
return url.format(parsed);
};
exports.parseTarballUrls = tarball => {
const urls = [ tarball ];
const parsed = urlparse(tarball);
const query = parsed.query && querystring.parse(parsed.query);
if (query && query.other_urls) {
const otherUrls = query.other_urls.split(',');
for (const url of otherUrls) {
urls.push(url);
}
}
return urls;
};
/*
* Runs an npm script.
*/
exports.runScript = async (pkgDir, script, globalOptions, runInForeground = false) => {
// merge config.env <= process.env <= options.env
const env = {};
for (const key in globalConfig.env) {
env[key] = globalConfig.env[key];
}
for (const key in process.env) {
// ignore `Path` env on Windows
if (/^path$/i.test(key)) {
continue;
}
env[key] = process.env[key];
}
for (const key in globalOptions.env) {
// ignore `Path` env on Windows
if (/^path$/i.test(key)) {
continue;
}
env[key] = globalOptions.env[key];
}
// set npm_package_* env from package.json
const pkg = await exports.readJSON(path.join(pkgDir, 'package.json'));
for (const key in pkg) {
setNpmPackageEnv(env, key, pkg[key]);
}
env.PATH = [
path.join(__dirname, '../node-gyp-bin'),
path.join(globalOptions.root, 'node_modules', '.bin'),
path.join(pkgDir, 'node_modules', '.bin'),
process.env.PATH,
].join(path.delimiter);
// replace `npm install xxx` to `npminstall xxx`
const NPM_INSTALL_RE = /^npm (i|install) /;
if (NPM_INSTALL_RE.test(script)) {
const npminstall = path.join(__dirname, '../bin/install.js');
const newScript = script.replace(NPM_INSTALL_RE, `${process.execPath} ${npminstall} `);
globalOptions.console.info('[npminstall:runScript] replace %j to %j', script, newScript);
script = newScript;
}
// ignore npm ls error
// e.g.: npm ERR! extraneous: base64-js@1.1.2
let ignoreError = false;
if (/^npm (ls|list)$/.test(script)) {
ignoreError = true;
}
try {
return await command(script, {
cwd: pkgDir,
env,
stdio: runInForeground ? 'inherit' : 'ignore',
shell: true,
});
} catch (err) {
if (ignoreError) {
globalOptions.console.info('[npminstall:runScript] ignore runscript error: %s', err);
} else {
throw err;
}
}
};
exports.getMaxRange = spec => {
// >=1.0.0 <2.0.0
const r = /^>=.*?<(.*?)$/.exec(spec);
if (r) {
return r[1];
}
};
const semverRangeCacheMap = new Map();
function getCacheSemverRange(range) {
let semverRange = semverRangeCacheMap.get(range);
if (semverRange === undefined) {
try {
semverRange = new semver.Range(range, { loose: true, includePrerelease: false });
} catch {
semverRange = null;
}
semverRangeCacheMap.set(range, semverRange);
}
return semverRange;
}
// faster semver.satisfies with Range Instance cache
// https://github.com/cnpm/npminstall/issues/453
// https://github.com/pnpm/pnpm/pull/6336
exports.fastSemverSatisfies = (version, range) => {
const semverRange = getCacheSemverRange(range);
if (semverRange) {
try {
return semverRange.test(new semver.SemVer(version, { loose: true, includePrerelease: false }));
} catch {
return false;
}
}
return false;
};
// https://github.com/npm/node-semver/blob/09c69e23cdf6c69c51f83635482fff89ab2574e3/ranges/max-satisfying.js#L4
exports.fastSemverMaxSatisfying = (versions, range) => {
const semverRange = getCacheSemverRange(range);
if (!semverRange) return null;
let max = null;
let maxSV = null;
for (const version of versions) {
const semVersion = new semver.SemVer(version, { loose: true, includePrerelease: false });
if (semverRange.test(semVersion)) {
if (!max || maxSV.compare(semVersion) === -1) {
max = version;
maxSV = semVersion;
}
}
}
return max;
};
exports.findMaxSatisfyingVersion = (spec, distTags, allVersions) => {
// try tag first
let realPkgVersion = distTags[spec];
if (!realPkgVersion) {
const version = semver.valid(spec);
const range = semver.validRange(spec, true);
if (exports.fastSemverSatisfies(distTags.latest, spec)) {
realPkgVersion = distTags.latest;
} else if (version) {
// use the valid version
realPkgVersion = version;
} else if (range) {
realPkgVersion = exports.fastSemverMaxSatisfying(allVersions, range);
if (realPkgVersion) {
// try to use latest-{major} tag version on range
// ^1.0.1 =range=> get 1.0.3 in (1.0.2, 1.0.3), but latest-1 tag is 1.0.2
// finnaly we should use 1.0.2 on ^1.0.1
const major = semver.major(realPkgVersion);
if (major) {
const latestMajorVersion = distTags[`latest-${major}`];
if (latestMajorVersion && exports.fastSemverSatisfies(latestMajorVersion, spec)) {
realPkgVersion = latestMajorVersion;
}
}
}
}
}
return realPkgVersion;
};
exports.getPackageStorePath = (storeDir, pkg, globalOptions) => {
// if workspace enable, install packages to `<workspaceRoot>/node_modules`
if (globalOptions.enableWorkspace) {
storeDir = path.join(globalOptions.workspaceRoot, 'node_modules');
}
// https://github.com/npm/rfcs/blob/main/accepted/0042-isolated-mode.md
// https://github.com/npm/cli/pull/5492
return path.join(storeDir, `.store/${pkg.name.replace('/', '+')}@${pkg.version}/node_modules/${pkg.name}`);
};
exports.unpack = (readstream, target, pkg) => {
return new Promise((resolve, reject) => {
const extracter = tar.extract({
cwd: target,
strip: 1,
onentry(entry) {
if (entry.type.toLowerCase() === 'file') {
/* eslint-disable no-bitwise */
entry.mode = (entry.mode || 0) | 0o644;
}
if (entry.type.toLowerCase() === 'directory') {
/* eslint-disable no-bitwise */
entry.mode = (entry.mode || 0) | 0o755;
}
},
});
const gunzip = zlib.createGunzip();
const name = pkg.name || pkg.displayName || 'unknown package';
// just support gzip tarball and nacked tarball
readstream
.on('data', function ondata(data) {
// detect what it is.
// Then, depending on that, we'll figure out whether it's
// gzipped tarball or naked tarball.
// gzipped files all start with 1f8b08
if (data[0] === 0x1F &&
data[1] === 0x8B &&
data[2] === 0x08) {
readstream.pipe(gunzip).pipe(extracter);
} else {
readstream.pipe(extracter);
}
// re-emit
readstream.removeListener('data', ondata);
readstream.emit('data', data);
});
extracter.on('end', handleCallback);
readstream.on('error', handleCallback);
gunzip.on('error', handleCallback);
extracter.on('error', handleCallback);
let ended = false;
function handleCallback(err) {
if (ended) {
return;
}
ended = true;
if (err) {
debug(`failed to unpack ${name}: ${err}`);
reject(err);
} else {
debug(`unpacked ${name}`);
resolve();
}
}
});
};
exports.copyInstall = async (src, options) => {
// 1. make sure source folder has package.json, and package.json contains name
// 2. get the target directory: $storeDir/${pkg.name}/${pkg.version}
// 3. check if this package has been installed, and make sure only copy once.
// 4. if already installed, return with exists = true
// 5. if not installed, copy and return with exists = false
const pkgpath = path.join(src, 'package.json');
if (!(await exports.exists(pkgpath))) {
throw new Error(`package.json missed(${pkgpath})`);
}
const realPkg = await exports.readPackageJSON(src);
if (!realPkg.name || !realPkg.version) {
throw new Error(`package.json must contains name and version(${pkgpath})`);
}
const targetdir = options.ungzipDir || exports.getPackageStorePath(options.storeDir, realPkg, options);
const key = `copy:${targetdir}`;
const result = {
dir: targetdir,
package: realPkg,
exists: true,
};
if (options.cache[key]) {
options.console.log('exist cache: %j %j', key, options.cache[key]);
if (options.cache[key].done) {
return result;
}
// wait copy finish
await options.events.await(key);
return result;
}
options.cache[key] = {
done: false,
};
if (!(await exports.isInstallDone(targetdir))) {
await fse.emptyDir(targetdir);
await fse.copy(src, targetdir);
await exports.setInstallDone(targetdir);
result.exists = false;
}
options.cache[key].done = true;
options.events.emit(key);
return result;
};
exports.getPkgFromPaths = async (name, paths) => {
for (const p of paths) {
const tryPath = path.join(p, name, 'package.json');
const pkg = await exports.readJSON(tryPath);
if (pkg.name && pkg.version) {
pkg.installPath = path.join(p, name);
return pkg;
}
}
return null;
};
exports.getTarballStream = async (url, options) => {
const result = await get(url, {
timeout: options.streamingTimeout || options.timeout,
followRedirect: true,
streaming: true,
}, options);
if (result.status !== 200) {
destroy(result.res);
throw new Error(`Download ${url} status: ${result.status} error, should be 200`);
}
return result.res;
};
async function getRemotePackage(name, registry, globalOptions) {
let lastErr;
let pkg;
const cachePkgFileName = `manifests/${name}/latest/package.json`;
let cachePkgFile;
if (globalOptions?.cacheDir) {
cachePkgFile = path.join(globalOptions.cacheDir, cachePkgFileName);
}
if (globalOptions?.offline) {
if (cachePkgFile && await exports.exists(cachePkgFile)) {
pkg = await exports.readJSON(cachePkgFile);
}
} else {
const registries = [ registry ].concat([
'https://registry.npmmirror.com',
'https://r.cnpmjs.org',
'https://registry.npmjs.com',
]);
for (const registry of registries) {
const binaryMirrorUrl = exports.formatPackageUrl(registry, `${name}/latest`);
try {
const res = await get(binaryMirrorUrl, {
dataType: 'json',
followRedirect: true,
// don't retry
retry: 0,
}, globalOptions);
pkg = res.data;
if (cachePkgFile) {
await exports.mkdirp(path.dirname(cachePkgFile));
await fs.writeFile(cachePkgFile, JSON.stringify(pkg));
}
break;
} catch (err) {
lastErr = err;
}
}
// try to read from cache file
if (!pkg) {
if (cachePkgFile && await exports.exists(cachePkgFile)) {
pkg = await exports.readJSON(cachePkgFile);
}
}
}
if (!pkg || process.env.NPMINSTALL_TEST_LOCAL_PKG) {
console.warn('Get /%s/latest from %s error: %s', name, registry, lastErr);
pkg = require(name + '/package.json');
}
return pkg;
}
exports.getBinaryMirrors = async (registry, globalOptions) => {
const pkg = await getRemotePackage('binary-mirror-config', registry, globalOptions);
return pkg.mirrors.china;
};
exports.getBugVersions = async (registry, globalOptions) => {
const pkg = await getRemotePackage('bug-versions', registry, globalOptions);
return pkg.config['bug-versions'];
};
// match platform, arch or libc
// see https://docs.npmjs.com/cli/v7/configuring-npm/package-json#os
exports.matchPlatform = (current, osNames) => {
if (!Array.isArray(osNames) || osNames.length === 0) {
return true;
}
let hasAnti = false;
for (const name of osNames) {
if (name === current) {
return true;
}
if (name[0] === '!') {
hasAnti = true;
if (name.substring(1) === current) {
return false;
}
}
}
return hasAnti;
};
exports.isSudo = () => {
const effectiveUser = process.env.USER;
const actualUser = process.env.SUDO_USER || process.env.USER;
return effectiveUser === 'root' && actualUser !== 'root';
};
exports.sleep = ms => {
return new Promise(resolve => {
setTimeout(resolve, ms);
});
};
exports.formatPath = pathname => {
if (pathname[0] === '~') {
// convert '~/foo/path' => '$HOME/foo/path'
pathname = homedir() + pathname.substring(1);
}
return pathname;
};
exports.fork = (moduleFile, args, options) => {
options = options || {};
options.stdio = options.stdio || [
process.stdin,
process.stdout,
process.stderr,
'ipc',
];
return new Promise((resolve, reject) => {
const child = cp.fork(moduleFile, args, options);
child.on('exit', code => {
if (code !== 0) {
return reject(new Error(`Run ${moduleFile} ${args.join(' ')} exit ${code}`));
}
resolve();
});
});
};
exports.getGlobalPrefix = prefix => {
if (!prefix && globalConfig.npmrc.prefix) {
prefix = globalConfig.npmrc.prefix;
}
if (!prefix) {
try {
prefix = cp.execSync('npm config get prefix').toString().trim();
} catch (err) {
throw new Error(`exec npm config get prefix ERROR: ${err.message}`);
}
}
return exports.formatPath(prefix);
};
exports.getGlobalInstallMeta = prefix => {
prefix = exports.getGlobalPrefix(prefix);
const meta = {
targetDir: prefix,
binDir: prefix,
};
if (process.platform !== 'win32') {
meta.targetDir = path.join(prefix, 'lib');
meta.binDir = path.join(prefix, 'bin');
}
return meta;
};
exports.endsWithX = version => typeof version === 'string' && !!version.match(/^\d+\.(x|\d+\.x)$/);
exports.getDisplayName = (pkg, ancestors) => {
return ancestors
.map(ancestor => ancestor.displayName || ancestor)
.concat([ `${pkg.name}@${pkg.version}` ])
.join(' › ');
};
exports.exec = promisify(cp.exec);
exports.formatWorkspaceNames = argv => {
let workspaceNames = argv.workspace || [];
if (!argv.workspaces && workspaceNames && typeof workspaceNames === 'string') {
workspaceNames = [ workspaceNames ];
}
return workspaceNames;
};
exports.readWorkspaces = async root => {
const workspaceInfos = [];
const rootPkgFile = path.join(root, 'package.json');
const rootPkg = await exports.readJSON(rootPkgFile);
if (Array.isArray(rootPkg.workspaces) && rootPkg.workspaces.length > 0) {
// should contains package.json
const patterns = rootPkg.workspaces.map(workspace => {
// 'packages/*', 'packages/*/'
return workspace + (workspace.endsWith('/') ? '' : '/') + 'package.json';
});
const workspacePkgFiles = await globby(patterns, {
cwd: root,
gitignore: true,
});
debug('[readWorkspaces] glob %o => %o', patterns, workspacePkgFiles);
for (const workspacePkgName of workspacePkgFiles) {
const workspacePkgFile = path.join(root, workspacePkgName);
const workspacePkg = await exports.readJSON(workspacePkgFile);
if (!workspacePkg.name) {
console.warn(chalk.yellow('npminstall WARN: workspace(%s) not found or missing `name` property'), workspacePkgFile);
continue;
}
const workspaceRoot = path.dirname(workspacePkgFile);
workspaceInfos.push({
root: workspaceRoot,
package: workspacePkg,
});
}
}
// sort by dependencies/devDependencies/peerDependencies relations
const beforeSortNames = workspaceInfos.map(info => info.package.name);
workspaceInfos.sort((a, b) => {
if (a.package.dependencies?.[b.package.name] ||
a.package.devDependencies?.[b.package.name] ||
a.package.peerDependencies?.[b.package.name]) {
return 1;
}
return -1;
});
const afterSortNames = workspaceInfos.map(info => info.package.name);
debug('workspaces sort %j => %j', beforeSortNames, afterSortNames);
const workspaceRoots = [];
const workspacesMap = new Map();
for (const info of workspaceInfos) {
workspaceRoots.push(info.root);
workspacesMap.set(info.package.name, info);
}
return {
workspaceRoots,
workspacesMap,
};
};
exports.getWorkspaceInfos = async (root, workspaceNameOrPaths, workspacesMap = null) => {
if (!workspacesMap) {
const rootInfo = await exports.readWorkspaces(root);
workspacesMap = rootInfo.workspacesMap;
}
const workspaceInfos = [];
const existsRootsSet = new Set();
for (const workspaceNameOrPath of new Set(workspaceNameOrPaths)) {
let workspaceInfo = workspacesMap.get(workspaceNameOrPath);
if (!workspaceInfo) {
// try to use `<workspaceNameOrPath>/package.json`
const workspacePkg = await exports.readJSON(path.join(root, workspaceNameOrPath, 'package.json'));
workspaceInfo = workspacePkg.name && workspacesMap.get(workspacePkg.name);
}
if (!workspaceInfo) {
// try to use `<workspaceNameOrPath>/*/package.json`
const patterns = [
workspaceNameOrPath + (workspaceNameOrPath.endsWith('/') ? '' : '/') + '*/package.json',
];
const workspacePkgFiles = await globby(patterns, {
cwd: root,
gitignore: true,
});
debug('[getWorkspaceInfo] glob %o => %o', patterns, workspacePkgFiles);
for (const workspacePkgName of workspacePkgFiles) {
const workspacePkgFile = path.join(root, workspacePkgName);
const workspacePkg = await exports.readJSON(workspacePkgFile);
workspaceInfo = workspacePkg.name && workspacesMap.get(workspacePkg.name);
if (workspaceInfo && !existsRootsSet.has(workspaceInfo.root)) {
existsRootsSet.add(workspaceInfo.root);
workspaceInfos.push(workspaceInfo);
}
}
} else {
if (!existsRootsSet.has(workspaceInfo.root)) {
existsRootsSet.add(workspaceInfo.root);
workspaceInfos.push(workspaceInfo);
}
}
}
return workspaceInfos;
};
exports.exitWithError = (cmd, err, code = 1) => {
console.error(chalk.red(err.stack));
console.error(chalk.yellow(`${cmd} version: %s`), require('../package.json').version);
console.error(chalk.yellow(`${cmd} argv: %s`), process.argv.join(' '));
console.log('');
process.exit(code);
};