@pnpm/plugin-commands-rebuild
Version:
Commands for rebuilding dependencies
397 lines • 19.5 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.rebuildSelectedPkgs = rebuildSelectedPkgs;
exports.rebuildProjects = rebuildProjects;
const assert_1 = __importDefault(require("assert"));
const path_1 = __importDefault(require("path"));
const util_1 = __importDefault(require("util"));
const store_cafs_1 = require("@pnpm/store.cafs");
const calc_dep_state_1 = require("@pnpm/calc-dep-state");
const constants_1 = require("@pnpm/constants");
const core_loggers_1 = require("@pnpm/core-loggers");
const error_1 = require("@pnpm/error");
const get_context_1 = require("@pnpm/get-context");
const lifecycle_1 = require("@pnpm/lifecycle");
const link_bins_1 = require("@pnpm/link-bins");
const lockfile_utils_1 = require("@pnpm/lockfile.utils");
const lockfile_walker_1 = require("@pnpm/lockfile.walker");
const logger_1 = require("@pnpm/logger");
const modules_yaml_1 = require("@pnpm/modules-yaml");
const store_connection_manager_1 = require("@pnpm/store-connection-manager");
const builder_policy_1 = require("@pnpm/builder.policy");
const exec_pkg_requires_build_1 = require("@pnpm/exec.pkg-requires-build");
const dp = __importStar(require("@pnpm/dependency-path"));
const read_package_json_1 = require("@pnpm/read-package-json");
const worker_1 = require("@pnpm/worker");
const load_json_file_1 = __importDefault(require("load-json-file"));
const run_groups_1 = __importDefault(require("run-groups"));
const deps_graph_sequencer_1 = require("@pnpm/deps.graph-sequencer");
const npm_package_arg_1 = __importDefault(require("@pnpm/npm-package-arg"));
const p_limit_1 = __importDefault(require("p-limit"));
const semver_1 = __importDefault(require("semver"));
const extendRebuildOptions_1 = require("./extendRebuildOptions");
function findPackages(packages, searched, opts) {
return Object.keys(packages)
.filter((relativeDepPath) => {
const pkgLockfile = packages[relativeDepPath];
const pkgInfo = (0, lockfile_utils_1.nameVerFromPkgSnapshot)(relativeDepPath, pkgLockfile);
if (!pkgInfo.name) {
logger_1.logger.warn({
message: `Skipping ${relativeDepPath} because cannot get the package name from ${constants_1.WANTED_LOCKFILE}.
Try to run run \`pnpm update --depth 100\` to create a new ${constants_1.WANTED_LOCKFILE} with all the necessary info.`,
prefix: opts.prefix,
});
return false;
}
return matches(searched, pkgInfo);
});
}
// TODO: move this logic to separate package as this is also used in dependencies-hierarchy
function matches(searched, manifest) {
return searched.some((searchedPkg) => {
if (typeof searchedPkg === 'string') {
return manifest.name === searchedPkg;
}
return searchedPkg.name === manifest.name && !!manifest.version &&
semver_1.default.satisfies(manifest.version, searchedPkg.range);
});
}
async function rebuildSelectedPkgs(projects, pkgSpecs, maybeOpts) {
const reporter = maybeOpts?.reporter;
if ((reporter != null) && typeof reporter === 'function') {
logger_1.streamParser.on('data', reporter);
}
const opts = await (0, extendRebuildOptions_1.extendRebuildOptions)(maybeOpts);
const ctx = await (0, get_context_1.getContext)({ ...opts, allProjects: projects });
if (ctx.currentLockfile?.packages == null)
return;
const packages = ctx.currentLockfile.packages;
const searched = pkgSpecs.map((arg) => {
const { fetchSpec, name, raw, type } = (0, npm_package_arg_1.default)(arg);
if (raw === name) {
return name;
}
if (type !== 'version' && type !== 'range') {
throw new Error(`Invalid argument - ${arg}. Rebuild can only select by version or range`);
}
return {
name,
range: fetchSpec,
};
});
let pkgs = [];
for (const { rootDir } of projects) {
pkgs = [
...pkgs,
...findPackages(packages, searched, { prefix: rootDir }),
];
}
const { ignoredPkgs } = await _rebuild({
pkgsToRebuild: new Set(pkgs),
...ctx,
}, opts);
await (0, modules_yaml_1.writeModulesManifest)(ctx.rootModulesDir, {
prunedAt: new Date().toUTCString(),
...ctx.modulesFile,
hoistedDependencies: ctx.hoistedDependencies,
hoistPattern: ctx.hoistPattern,
included: ctx.include,
ignoredBuilds: ignoredPkgs,
layoutVersion: constants_1.LAYOUT_VERSION,
packageManager: `${opts.packageManager.name}@${opts.packageManager.version}`,
pendingBuilds: ctx.pendingBuilds,
publicHoistPattern: ctx.publicHoistPattern,
registries: ctx.registries,
skipped: Array.from(ctx.skipped),
storeDir: ctx.storeDir,
virtualStoreDir: ctx.virtualStoreDir,
virtualStoreDirMaxLength: ctx.virtualStoreDirMaxLength,
});
}
async function rebuildProjects(projects, maybeOpts) {
const reporter = maybeOpts?.reporter;
if ((reporter != null) && typeof reporter === 'function') {
logger_1.streamParser.on('data', reporter);
}
const opts = await (0, extendRebuildOptions_1.extendRebuildOptions)(maybeOpts);
const ctx = await (0, get_context_1.getContext)({ ...opts, allProjects: projects });
let idsToRebuild = [];
if (opts.pending) {
idsToRebuild = ctx.pendingBuilds;
}
else if ((ctx.currentLockfile?.packages) != null) {
idsToRebuild = Object.keys(ctx.currentLockfile.packages);
}
const { pkgsThatWereRebuilt, ignoredPkgs } = await _rebuild({
pkgsToRebuild: new Set(idsToRebuild),
...ctx,
}, opts);
ctx.pendingBuilds = ctx.pendingBuilds.filter((depPath) => !pkgsThatWereRebuilt.has(depPath));
const store = await (0, store_connection_manager_1.createOrConnectStoreController)(opts);
const scriptsOpts = {
extraBinPaths: ctx.extraBinPaths,
extraNodePaths: ctx.extraNodePaths,
extraEnv: opts.extraEnv,
preferSymlinkedExecutables: opts.preferSymlinkedExecutables,
rawConfig: opts.rawConfig,
scriptsPrependNodePath: opts.scriptsPrependNodePath,
scriptShell: opts.scriptShell,
shellEmulator: opts.shellEmulator,
storeController: store.ctrl,
unsafePerm: opts.unsafePerm || false,
};
await (0, lifecycle_1.runLifecycleHooksConcurrently)(['preinstall', 'install', 'postinstall', 'prepublish', 'prepare'], Object.values(ctx.projects), opts.childConcurrency || 5, scriptsOpts);
for (const { id, manifest } of Object.values(ctx.projects)) {
if (((manifest?.scripts) != null) && (!opts.pending || ctx.pendingBuilds.includes(id))) {
ctx.pendingBuilds.splice(ctx.pendingBuilds.indexOf(id), 1);
}
}
await (0, modules_yaml_1.writeModulesManifest)(ctx.rootModulesDir, {
prunedAt: new Date().toUTCString(),
...ctx.modulesFile,
hoistedDependencies: ctx.hoistedDependencies,
hoistPattern: ctx.hoistPattern,
included: ctx.include,
ignoredBuilds: ignoredPkgs,
layoutVersion: constants_1.LAYOUT_VERSION,
packageManager: `${opts.packageManager.name}@${opts.packageManager.version}`,
pendingBuilds: ctx.pendingBuilds,
publicHoistPattern: ctx.publicHoistPattern,
registries: ctx.registries,
skipped: Array.from(ctx.skipped),
storeDir: ctx.storeDir,
virtualStoreDir: ctx.virtualStoreDir,
virtualStoreDirMaxLength: ctx.virtualStoreDirMaxLength,
});
}
function getSubgraphToBuild(step, nodesToBuildAndTransitive, opts) {
let currentShouldBeBuilt = false;
for (const { depPath, next } of step.dependencies) {
if (nodesToBuildAndTransitive.has(depPath)) {
currentShouldBeBuilt = true;
}
const childShouldBeBuilt = getSubgraphToBuild(next(), nodesToBuildAndTransitive, opts) ||
opts.pkgsToRebuild.has(depPath);
if (childShouldBeBuilt) {
nodesToBuildAndTransitive.add(depPath);
currentShouldBeBuilt = true;
}
}
for (const depPath of step.missing) {
// It might make sense to fail if the depPath is not in the skipped list from .modules.yaml
// However, the skipped list currently contains package IDs, not dep paths.
logger_1.logger.debug({ message: `No entry for "${depPath}" in ${constants_1.WANTED_LOCKFILE}` });
}
return currentShouldBeBuilt;
}
const limitLinking = (0, p_limit_1.default)(16);
async function _rebuild(ctx, opts) {
const depGraph = (0, calc_dep_state_1.lockfileToDepGraph)(ctx.currentLockfile);
const depsStateCache = {};
const pkgsThatWereRebuilt = new Set();
const graph = new Map();
const pkgSnapshots = ctx.currentLockfile.packages ?? {};
const nodesToBuildAndTransitive = new Set();
getSubgraphToBuild((0, lockfile_walker_1.lockfileWalker)(ctx.currentLockfile, Object.values(ctx.projects).map(({ id }) => id), {
include: {
dependencies: opts.production,
devDependencies: opts.development,
optionalDependencies: opts.optional,
},
}).step, nodesToBuildAndTransitive, { pkgsToRebuild: ctx.pkgsToRebuild });
const nodesToBuildAndTransitiveArray = Array.from(nodesToBuildAndTransitive);
for (const depPath of nodesToBuildAndTransitiveArray) {
const pkgSnapshot = pkgSnapshots[depPath];
graph.set(depPath, Object.entries({ ...pkgSnapshot.dependencies, ...pkgSnapshot.optionalDependencies })
.map(([pkgName, reference]) => dp.refToRelative(reference, pkgName))
.filter((childRelDepPath) => childRelDepPath && nodesToBuildAndTransitive.has(childRelDepPath)));
}
const graphSequencerResult = (0, deps_graph_sequencer_1.graphSequencer)(graph, nodesToBuildAndTransitiveArray);
const chunks = graphSequencerResult.chunks;
const warn = (message) => {
logger_1.logger.info({ message, prefix: opts.dir });
};
const ignoredPkgs = [];
const _allowBuild = (0, builder_policy_1.createAllowBuildFunction)(opts) ?? (() => true);
const allowBuild = (pkgName) => {
if (_allowBuild(pkgName))
return true;
ignoredPkgs.push(pkgName);
return false;
};
const builtDepPaths = new Set();
const groups = chunks.map((chunk) => chunk.filter((depPath) => ctx.pkgsToRebuild.has(depPath) && !ctx.skipped.has(depPath)).map((depPath) => async () => {
const pkgSnapshot = pkgSnapshots[depPath];
const pkgInfo = (0, lockfile_utils_1.nameVerFromPkgSnapshot)(depPath, pkgSnapshot);
const pkgRoots = opts.nodeLinker === 'hoisted'
? (ctx.modulesFile?.hoistedLocations?.[depPath] ?? []).map((hoistedLocation) => path_1.default.join(opts.lockfileDir, hoistedLocation))
: [path_1.default.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength), 'node_modules', pkgInfo.name)];
if (pkgRoots.length === 0) {
if (pkgSnapshot.optional)
return;
throw new error_1.PnpmError('MISSING_HOISTED_LOCATIONS', `${depPath} is not found in hoistedLocations inside node_modules/.modules.yaml`, {
hint: 'If you installed your node_modules with pnpm older than v7.19.0, you may need to remove it and run "pnpm install"',
});
}
const pkgRoot = pkgRoots[0];
try {
const extraBinPaths = ctx.extraBinPaths;
if (opts.nodeLinker !== 'hoisted') {
const modules = path_1.default.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength), 'node_modules');
const binPath = path_1.default.join(pkgRoot, 'node_modules', '.bin');
await (0, link_bins_1.linkBins)(modules, binPath, { extraNodePaths: ctx.extraNodePaths, warn });
}
else {
extraBinPaths.push(...binDirsInAllParentDirs(pkgRoot, opts.lockfileDir));
}
const resolution = pkgSnapshot.resolution;
let sideEffectsCacheKey;
const pkgId = `${pkgInfo.name}@${pkgInfo.version}`;
if (opts.skipIfHasSideEffectsCache && resolution.integrity) {
const filesIndexFile = (0, store_cafs_1.getIndexFilePathInCafs)(opts.storeDir, resolution.integrity.toString(), pkgId);
const pkgFilesIndex = await (0, load_json_file_1.default)(filesIndexFile);
sideEffectsCacheKey = (0, calc_dep_state_1.calcDepState)(depGraph, depsStateCache, depPath, {
includeDepGraphHash: true,
});
if (pkgFilesIndex.sideEffects?.[sideEffectsCacheKey]) {
pkgsThatWereRebuilt.add(depPath);
return;
}
}
let requiresBuild = true;
const pgkManifest = await (0, read_package_json_1.safeReadPackageJsonFromDir)(pkgRoot);
if (pgkManifest != null) {
// This won't return the correct result for packages with binding.gyp as we don't pass the filesIndex to the function.
// However, currently rebuild doesn't work for such packages at all, which should be fixed.
requiresBuild = (0, exec_pkg_requires_build_1.pkgRequiresBuild)(pgkManifest, {});
}
const hasSideEffects = requiresBuild && allowBuild(pkgInfo.name) && await (0, lifecycle_1.runPostinstallHooks)({
depPath,
extraBinPaths,
extraEnv: opts.extraEnv,
optional: pkgSnapshot.optional === true,
pkgRoot,
rawConfig: opts.rawConfig,
rootModulesDir: ctx.rootModulesDir,
scriptsPrependNodePath: opts.scriptsPrependNodePath,
shellEmulator: opts.shellEmulator,
unsafePerm: opts.unsafePerm || false,
});
if (hasSideEffects && (opts.sideEffectsCacheWrite ?? true) && resolution.integrity) {
builtDepPaths.add(depPath);
const filesIndexFile = (0, store_cafs_1.getIndexFilePathInCafs)(opts.storeDir, resolution.integrity.toString(), pkgId);
try {
if (!sideEffectsCacheKey) {
sideEffectsCacheKey = (0, calc_dep_state_1.calcDepState)(depGraph, depsStateCache, depPath, {
includeDepGraphHash: true,
});
}
await opts.storeController.upload(pkgRoot, {
sideEffectsCacheKey,
filesIndexFile,
});
}
catch (err) {
(0, assert_1.default)(util_1.default.types.isNativeError(err));
if ('statusCode' in err && err.statusCode === 403) {
logger_1.logger.warn({
message: `The store server disabled upload requests, could not upload ${pkgRoot}`,
prefix: opts.lockfileDir,
});
}
else {
logger_1.logger.warn({
error: err,
message: `An error occurred while uploading ${pkgRoot}`,
prefix: opts.lockfileDir,
});
}
}
}
pkgsThatWereRebuilt.add(depPath);
}
catch (err) {
(0, assert_1.default)(util_1.default.types.isNativeError(err));
if (pkgSnapshot.optional) {
// TODO: add parents field to the log
core_loggers_1.skippedOptionalDependencyLogger.debug({
details: err.toString(),
package: {
id: pkgSnapshot.id ?? depPath,
name: pkgInfo.name,
version: pkgInfo.version,
},
prefix: opts.dir,
reason: 'build_failure',
});
return;
}
throw err;
}
if (pkgRoots.length > 1) {
await (0, worker_1.hardLinkDir)(pkgRoot, pkgRoots.slice(1));
}
}));
await (0, run_groups_1.default)(opts.childConcurrency || 5, groups);
if (builtDepPaths.size > 0) {
// It may be optimized because some bins were already linked before running lifecycle scripts
await Promise.all(Object
.keys(pkgSnapshots)
.filter((depPath) => !(0, lockfile_utils_1.packageIsIndependent)(pkgSnapshots[depPath]))
.map(async (depPath) => limitLinking(async () => {
const pkgSnapshot = pkgSnapshots[depPath];
const pkgInfo = (0, lockfile_utils_1.nameVerFromPkgSnapshot)(depPath, pkgSnapshot);
const modules = path_1.default.join(ctx.virtualStoreDir, dp.depPathToFilename(depPath, opts.virtualStoreDirMaxLength), 'node_modules');
const binPath = path_1.default.join(modules, pkgInfo.name, 'node_modules', '.bin');
return (0, link_bins_1.linkBins)(modules, binPath, { warn });
})));
await Promise.all(Object.values(ctx.projects).map(async ({ rootDir }) => limitLinking(async () => {
const modules = path_1.default.join(rootDir, 'node_modules');
const binPath = path_1.default.join(modules, '.bin');
return (0, link_bins_1.linkBins)(modules, binPath, {
allowExoticManifests: true,
warn,
});
})));
}
return { pkgsThatWereRebuilt, ignoredPkgs };
}
function binDirsInAllParentDirs(pkgRoot, lockfileDir) {
const binDirs = [];
let dir = pkgRoot;
do {
if (!(path_1.default.dirname(dir)[0] === '@')) {
binDirs.push(path_1.default.join(dir, 'node_modules/.bin'));
}
dir = path_1.default.dirname(dir);
} while (path_1.default.relative(dir, lockfileDir) !== '');
binDirs.push(path_1.default.join(lockfileDir, 'node_modules/.bin'));
return binDirs;
}
//# sourceMappingURL=index.js.map