webpack
Version:
Packs CommonJs/AMD modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jsx, es7, css, less, ... and your custom stuff.
1,660 lines (1,560 loc) • 103 kB
JavaScript
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
const { create: createResolver } = require("enhanced-resolve");
const asyncLib = require("neo-async");
const AsyncQueue = require("./util/AsyncQueue");
const StackedCacheMap = require("./util/StackedCacheMap");
const createHash = require("./util/createHash");
const { join, dirname, relative, lstatReadlinkAbsolute } = require("./util/fs");
const makeSerializable = require("./util/makeSerializable");
const processAsyncTree = require("./util/processAsyncTree");
/** @typedef {import("./WebpackError")} WebpackError */
/** @typedef {import("./logging/Logger").Logger} Logger */
/** @typedef {typeof import("./util/Hash")} Hash */
/** @typedef {import("./util/fs").IStats} IStats */
/** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
const supportsEsm = +process.versions.modules >= 83;
let FS_ACCURACY = 2000;
const EMPTY_SET = new Set();
const RBDT_RESOLVE_CJS = 0;
const RBDT_RESOLVE_ESM = 1;
const RBDT_RESOLVE_DIRECTORY = 2;
const RBDT_RESOLVE_CJS_FILE = 3;
const RBDT_RESOLVE_CJS_FILE_AS_CHILD = 4;
const RBDT_RESOLVE_ESM_FILE = 5;
const RBDT_DIRECTORY = 6;
const RBDT_FILE = 7;
const RBDT_DIRECTORY_DEPENDENCIES = 8;
const RBDT_FILE_DEPENDENCIES = 9;
const INVALID = Symbol("invalid");
/**
* @typedef {Object} FileSystemInfoEntry
* @property {number} safeTime
* @property {number=} timestamp
*/
/**
* @typedef {Object} ResolvedContextFileSystemInfoEntry
* @property {number} safeTime
* @property {string=} timestampHash
*/
/**
* @typedef {Object} ContextFileSystemInfoEntry
* @property {number} safeTime
* @property {string=} timestampHash
* @property {ResolvedContextFileSystemInfoEntry=} resolved
* @property {Set<string>=} symlinks
*/
/**
* @typedef {Object} TimestampAndHash
* @property {number} safeTime
* @property {number=} timestamp
* @property {string} hash
*/
/**
* @typedef {Object} ResolvedContextTimestampAndHash
* @property {number} safeTime
* @property {string=} timestampHash
* @property {string} hash
*/
/**
* @typedef {Object} ContextTimestampAndHash
* @property {number} safeTime
* @property {string=} timestampHash
* @property {string} hash
* @property {ResolvedContextTimestampAndHash=} resolved
* @property {Set<string>=} symlinks
*/
/**
* @typedef {Object} ContextHash
* @property {string} hash
* @property {string=} resolved
* @property {Set<string>=} symlinks
*/
/**
* @typedef {Object} SnapshotOptimizationEntry
* @property {Snapshot} snapshot
* @property {number} shared
* @property {Set<string>} snapshotContent
* @property {Set<SnapshotOptimizationEntry>} children
*/
/**
* @typedef {Object} ResolveBuildDependenciesResult
* @property {Set<string>} files list of files
* @property {Set<string>} directories list of directories
* @property {Set<string>} missing list of missing entries
* @property {Map<string, string | false>} resolveResults stored resolve results
* @property {Object} resolveDependencies dependencies of the resolving
* @property {Set<string>} resolveDependencies.files list of files
* @property {Set<string>} resolveDependencies.directories list of directories
* @property {Set<string>} resolveDependencies.missing list of missing entries
*/
const DONE_ITERATOR_RESULT = new Set().keys().next();
// cspell:word tshs
// Tsh = Timestamp + Hash
// Tshs = Timestamp + Hash combinations
class SnapshotIterator {
constructor(next) {
this.next = next;
}
}
class SnapshotIterable {
constructor(snapshot, getMaps) {
this.snapshot = snapshot;
this.getMaps = getMaps;
}
[Symbol.iterator]() {
let state = 0;
/** @type {IterableIterator<string>} */
let it;
/** @type {(Snapshot) => (Map<string, any> | Set<string>)[]} */
let getMaps;
/** @type {(Map<string, any> | Set<string>)[]} */
let maps;
/** @type {Snapshot} */
let snapshot;
let queue;
return new SnapshotIterator(() => {
for (;;) {
switch (state) {
case 0:
snapshot = this.snapshot;
getMaps = this.getMaps;
maps = getMaps(snapshot);
state = 1;
/* falls through */
case 1:
if (maps.length > 0) {
const map = maps.pop();
if (map !== undefined) {
it = map.keys();
state = 2;
} else {
break;
}
} else {
state = 3;
break;
}
/* falls through */
case 2: {
const result = it.next();
if (!result.done) return result;
state = 1;
break;
}
case 3: {
const children = snapshot.children;
if (children !== undefined) {
if (children.size === 1) {
// shortcut for a single child
// avoids allocation of queue
for (const child of children) snapshot = child;
maps = getMaps(snapshot);
state = 1;
break;
}
if (queue === undefined) queue = [];
for (const child of children) {
queue.push(child);
}
}
if (queue !== undefined && queue.length > 0) {
snapshot = queue.pop();
maps = getMaps(snapshot);
state = 1;
break;
} else {
state = 4;
}
}
/* falls through */
case 4:
return DONE_ITERATOR_RESULT;
}
}
});
}
}
class Snapshot {
constructor() {
this._flags = 0;
/** @type {number | undefined} */
this.startTime = undefined;
/** @type {Map<string, FileSystemInfoEntry | null> | undefined} */
this.fileTimestamps = undefined;
/** @type {Map<string, string | null> | undefined} */
this.fileHashes = undefined;
/** @type {Map<string, TimestampAndHash | string | null> | undefined} */
this.fileTshs = undefined;
/** @type {Map<string, ResolvedContextFileSystemInfoEntry | null> | undefined} */
this.contextTimestamps = undefined;
/** @type {Map<string, string | null> | undefined} */
this.contextHashes = undefined;
/** @type {Map<string, ResolvedContextTimestampAndHash | null> | undefined} */
this.contextTshs = undefined;
/** @type {Map<string, boolean> | undefined} */
this.missingExistence = undefined;
/** @type {Map<string, string> | undefined} */
this.managedItemInfo = undefined;
/** @type {Set<string> | undefined} */
this.managedFiles = undefined;
/** @type {Set<string> | undefined} */
this.managedContexts = undefined;
/** @type {Set<string> | undefined} */
this.managedMissing = undefined;
/** @type {Set<Snapshot> | undefined} */
this.children = undefined;
}
hasStartTime() {
return (this._flags & 1) !== 0;
}
setStartTime(value) {
this._flags = this._flags | 1;
this.startTime = value;
}
setMergedStartTime(value, snapshot) {
if (value) {
if (snapshot.hasStartTime()) {
this.setStartTime(Math.min(value, snapshot.startTime));
} else {
this.setStartTime(value);
}
} else {
if (snapshot.hasStartTime()) this.setStartTime(snapshot.startTime);
}
}
hasFileTimestamps() {
return (this._flags & 2) !== 0;
}
setFileTimestamps(value) {
this._flags = this._flags | 2;
this.fileTimestamps = value;
}
hasFileHashes() {
return (this._flags & 4) !== 0;
}
setFileHashes(value) {
this._flags = this._flags | 4;
this.fileHashes = value;
}
hasFileTshs() {
return (this._flags & 8) !== 0;
}
setFileTshs(value) {
this._flags = this._flags | 8;
this.fileTshs = value;
}
hasContextTimestamps() {
return (this._flags & 0x10) !== 0;
}
setContextTimestamps(value) {
this._flags = this._flags | 0x10;
this.contextTimestamps = value;
}
hasContextHashes() {
return (this._flags & 0x20) !== 0;
}
setContextHashes(value) {
this._flags = this._flags | 0x20;
this.contextHashes = value;
}
hasContextTshs() {
return (this._flags & 0x40) !== 0;
}
setContextTshs(value) {
this._flags = this._flags | 0x40;
this.contextTshs = value;
}
hasMissingExistence() {
return (this._flags & 0x80) !== 0;
}
setMissingExistence(value) {
this._flags = this._flags | 0x80;
this.missingExistence = value;
}
hasManagedItemInfo() {
return (this._flags & 0x100) !== 0;
}
setManagedItemInfo(value) {
this._flags = this._flags | 0x100;
this.managedItemInfo = value;
}
hasManagedFiles() {
return (this._flags & 0x200) !== 0;
}
setManagedFiles(value) {
this._flags = this._flags | 0x200;
this.managedFiles = value;
}
hasManagedContexts() {
return (this._flags & 0x400) !== 0;
}
setManagedContexts(value) {
this._flags = this._flags | 0x400;
this.managedContexts = value;
}
hasManagedMissing() {
return (this._flags & 0x800) !== 0;
}
setManagedMissing(value) {
this._flags = this._flags | 0x800;
this.managedMissing = value;
}
hasChildren() {
return (this._flags & 0x1000) !== 0;
}
setChildren(value) {
this._flags = this._flags | 0x1000;
this.children = value;
}
addChild(child) {
if (!this.hasChildren()) {
this.setChildren(new Set());
}
this.children.add(child);
}
serialize({ write }) {
write(this._flags);
if (this.hasStartTime()) write(this.startTime);
if (this.hasFileTimestamps()) write(this.fileTimestamps);
if (this.hasFileHashes()) write(this.fileHashes);
if (this.hasFileTshs()) write(this.fileTshs);
if (this.hasContextTimestamps()) write(this.contextTimestamps);
if (this.hasContextHashes()) write(this.contextHashes);
if (this.hasContextTshs()) write(this.contextTshs);
if (this.hasMissingExistence()) write(this.missingExistence);
if (this.hasManagedItemInfo()) write(this.managedItemInfo);
if (this.hasManagedFiles()) write(this.managedFiles);
if (this.hasManagedContexts()) write(this.managedContexts);
if (this.hasManagedMissing()) write(this.managedMissing);
if (this.hasChildren()) write(this.children);
}
deserialize({ read }) {
this._flags = read();
if (this.hasStartTime()) this.startTime = read();
if (this.hasFileTimestamps()) this.fileTimestamps = read();
if (this.hasFileHashes()) this.fileHashes = read();
if (this.hasFileTshs()) this.fileTshs = read();
if (this.hasContextTimestamps()) this.contextTimestamps = read();
if (this.hasContextHashes()) this.contextHashes = read();
if (this.hasContextTshs()) this.contextTshs = read();
if (this.hasMissingExistence()) this.missingExistence = read();
if (this.hasManagedItemInfo()) this.managedItemInfo = read();
if (this.hasManagedFiles()) this.managedFiles = read();
if (this.hasManagedContexts()) this.managedContexts = read();
if (this.hasManagedMissing()) this.managedMissing = read();
if (this.hasChildren()) this.children = read();
}
/**
* @param {function(Snapshot): (ReadonlyMap<string, any> | ReadonlySet<string>)[]} getMaps first
* @returns {Iterable<string>} iterable
*/
_createIterable(getMaps) {
return new SnapshotIterable(this, getMaps);
}
/**
* @returns {Iterable<string>} iterable
*/
getFileIterable() {
return this._createIterable(s => [
s.fileTimestamps,
s.fileHashes,
s.fileTshs,
s.managedFiles
]);
}
/**
* @returns {Iterable<string>} iterable
*/
getContextIterable() {
return this._createIterable(s => [
s.contextTimestamps,
s.contextHashes,
s.contextTshs,
s.managedContexts
]);
}
/**
* @returns {Iterable<string>} iterable
*/
getMissingIterable() {
return this._createIterable(s => [s.missingExistence, s.managedMissing]);
}
}
makeSerializable(Snapshot, "webpack/lib/FileSystemInfo", "Snapshot");
const MIN_COMMON_SNAPSHOT_SIZE = 3;
/**
* @template T
*/
class SnapshotOptimization {
/**
* @param {function(Snapshot): boolean} has has value
* @param {function(Snapshot): Map<string, T> | Set<string>} get get value
* @param {function(Snapshot, Map<string, T> | Set<string>): void} set set value
* @param {boolean=} useStartTime use the start time of snapshots
* @param {boolean=} isSet value is an Set instead of a Map
*/
constructor(has, get, set, useStartTime = true, isSet = false) {
this._has = has;
this._get = get;
this._set = set;
this._useStartTime = useStartTime;
this._isSet = isSet;
/** @type {Map<string, SnapshotOptimizationEntry>} */
this._map = new Map();
this._statItemsShared = 0;
this._statItemsUnshared = 0;
this._statSharedSnapshots = 0;
this._statReusedSharedSnapshots = 0;
}
getStatisticMessage() {
const total = this._statItemsShared + this._statItemsUnshared;
if (total === 0) return undefined;
return `${
this._statItemsShared && Math.round((this._statItemsShared * 100) / total)
}% (${this._statItemsShared}/${total}) entries shared via ${
this._statSharedSnapshots
} shared snapshots (${
this._statReusedSharedSnapshots + this._statSharedSnapshots
} times referenced)`;
}
clear() {
this._map.clear();
this._statItemsShared = 0;
this._statItemsUnshared = 0;
this._statSharedSnapshots = 0;
this._statReusedSharedSnapshots = 0;
}
/**
* @param {Snapshot} newSnapshot snapshot
* @param {Set<string>} capturedFiles files to snapshot/share
* @returns {void}
*/
optimize(newSnapshot, capturedFiles) {
/**
* @param {SnapshotOptimizationEntry} entry optimization entry
* @returns {void}
*/
const increaseSharedAndStoreOptimizationEntry = entry => {
if (entry.children !== undefined) {
entry.children.forEach(increaseSharedAndStoreOptimizationEntry);
}
entry.shared++;
storeOptimizationEntry(entry);
};
/**
* @param {SnapshotOptimizationEntry} entry optimization entry
* @returns {void}
*/
const storeOptimizationEntry = entry => {
for (const path of entry.snapshotContent) {
const old = this._map.get(path);
if (old.shared < entry.shared) {
this._map.set(path, entry);
}
capturedFiles.delete(path);
}
};
/** @type {SnapshotOptimizationEntry} */
let newOptimizationEntry = undefined;
const capturedFilesSize = capturedFiles.size;
/** @type {Set<SnapshotOptimizationEntry> | undefined} */
const optimizationEntries = new Set();
for (const path of capturedFiles) {
const optimizationEntry = this._map.get(path);
if (optimizationEntry === undefined) {
if (newOptimizationEntry === undefined) {
newOptimizationEntry = {
snapshot: newSnapshot,
shared: 0,
snapshotContent: undefined,
children: undefined
};
}
this._map.set(path, newOptimizationEntry);
continue;
} else {
optimizationEntries.add(optimizationEntry);
}
}
optimizationEntries: for (const optimizationEntry of optimizationEntries) {
const snapshot = optimizationEntry.snapshot;
if (optimizationEntry.shared > 0) {
// It's a shared snapshot
// We can't change it, so we can only use it when all files match
// and startTime is compatible
if (
this._useStartTime &&
newSnapshot.startTime &&
(!snapshot.startTime || snapshot.startTime > newSnapshot.startTime)
) {
continue;
}
const nonSharedFiles = new Set();
const snapshotContent = optimizationEntry.snapshotContent;
const snapshotEntries = this._get(snapshot);
for (const path of snapshotContent) {
if (!capturedFiles.has(path)) {
if (!snapshotEntries.has(path)) {
// File is not shared and can't be removed from the snapshot
// because it's in a child of the snapshot
continue optimizationEntries;
}
nonSharedFiles.add(path);
continue;
}
}
if (nonSharedFiles.size === 0) {
// The complete snapshot is shared
// add it as child
newSnapshot.addChild(snapshot);
increaseSharedAndStoreOptimizationEntry(optimizationEntry);
this._statReusedSharedSnapshots++;
} else {
// Only a part of the snapshot is shared
const sharedCount = snapshotContent.size - nonSharedFiles.size;
if (sharedCount < MIN_COMMON_SNAPSHOT_SIZE) {
// Common part it too small
continue optimizationEntries;
}
// Extract common timestamps from both snapshots
let commonMap;
if (this._isSet) {
commonMap = new Set();
for (const path of /** @type {Set<string>} */ (snapshotEntries)) {
if (nonSharedFiles.has(path)) continue;
commonMap.add(path);
snapshotEntries.delete(path);
}
} else {
commonMap = new Map();
const map = /** @type {Map<string, T>} */ (snapshotEntries);
for (const [path, value] of map) {
if (nonSharedFiles.has(path)) continue;
commonMap.set(path, value);
snapshotEntries.delete(path);
}
}
// Create and attach snapshot
const commonSnapshot = new Snapshot();
if (this._useStartTime) {
commonSnapshot.setMergedStartTime(newSnapshot.startTime, snapshot);
}
this._set(commonSnapshot, commonMap);
newSnapshot.addChild(commonSnapshot);
snapshot.addChild(commonSnapshot);
// Create optimization entry
const newEntry = {
snapshot: commonSnapshot,
shared: optimizationEntry.shared + 1,
snapshotContent: new Set(commonMap.keys()),
children: undefined
};
if (optimizationEntry.children === undefined)
optimizationEntry.children = new Set();
optimizationEntry.children.add(newEntry);
storeOptimizationEntry(newEntry);
this._statSharedSnapshots++;
}
} else {
// It's a unshared snapshot
// We can extract a common shared snapshot
// with all common files
const snapshotEntries = this._get(snapshot);
if (snapshotEntries === undefined) {
// Incomplete snapshot, that can't be used
continue optimizationEntries;
}
let commonMap;
if (this._isSet) {
commonMap = new Set();
const set = /** @type {Set<string>} */ (snapshotEntries);
if (capturedFiles.size < set.size) {
for (const path of capturedFiles) {
if (set.has(path)) commonMap.add(path);
}
} else {
for (const path of set) {
if (capturedFiles.has(path)) commonMap.add(path);
}
}
} else {
commonMap = new Map();
const map = /** @type {Map<string, T>} */ (snapshotEntries);
for (const path of capturedFiles) {
const ts = map.get(path);
if (ts === undefined) continue;
commonMap.set(path, ts);
}
}
if (commonMap.size < MIN_COMMON_SNAPSHOT_SIZE) {
// Common part it too small
continue optimizationEntries;
}
// Create and attach snapshot
const commonSnapshot = new Snapshot();
if (this._useStartTime) {
commonSnapshot.setMergedStartTime(newSnapshot.startTime, snapshot);
}
this._set(commonSnapshot, commonMap);
newSnapshot.addChild(commonSnapshot);
snapshot.addChild(commonSnapshot);
// Remove files from snapshot
for (const path of commonMap.keys()) snapshotEntries.delete(path);
const sharedCount = commonMap.size;
this._statItemsUnshared -= sharedCount;
this._statItemsShared += sharedCount;
// Create optimization entry
storeOptimizationEntry({
snapshot: commonSnapshot,
shared: 2,
snapshotContent: new Set(commonMap.keys()),
children: undefined
});
this._statSharedSnapshots++;
}
}
const unshared = capturedFiles.size;
this._statItemsUnshared += unshared;
this._statItemsShared += capturedFilesSize - unshared;
}
}
/* istanbul ignore next */
/**
* @param {number} mtime mtime
*/
const applyMtime = mtime => {
if (FS_ACCURACY > 1 && mtime % 2 !== 0) FS_ACCURACY = 1;
else if (FS_ACCURACY > 10 && mtime % 20 !== 0) FS_ACCURACY = 10;
else if (FS_ACCURACY > 100 && mtime % 200 !== 0) FS_ACCURACY = 100;
else if (FS_ACCURACY > 1000 && mtime % 2000 !== 0) FS_ACCURACY = 1000;
};
/**
* @template T
* @template K
* @param {Map<T, K>} a source map
* @param {Map<T, K>} b joining map
* @returns {Map<T, K>} joined map
*/
const mergeMaps = (a, b) => {
if (!b || b.size === 0) return a;
if (!a || a.size === 0) return b;
const map = new Map(a);
for (const [key, value] of b) {
map.set(key, value);
}
return map;
};
/**
* @template T
* @template K
* @param {Set<T, K>} a source map
* @param {Set<T, K>} b joining map
* @returns {Set<T, K>} joined map
*/
const mergeSets = (a, b) => {
if (!b || b.size === 0) return a;
if (!a || a.size === 0) return b;
const map = new Set(a);
for (const item of b) {
map.add(item);
}
return map;
};
/**
* Finding file or directory to manage
* @param {string} managedPath path that is managing by {@link FileSystemInfo}
* @param {string} path path to file or directory
* @returns {string|null} managed item
* @example
* getManagedItem(
* '/Users/user/my-project/node_modules/',
* '/Users/user/my-project/node_modules/package/index.js'
* ) === '/Users/user/my-project/node_modules/package'
* getManagedItem(
* '/Users/user/my-project/node_modules/',
* '/Users/user/my-project/node_modules/package1/node_modules/package2'
* ) === '/Users/user/my-project/node_modules/package1/node_modules/package2'
* getManagedItem(
* '/Users/user/my-project/node_modules/',
* '/Users/user/my-project/node_modules/.bin/script.js'
* ) === null // hidden files are disallowed as managed items
* getManagedItem(
* '/Users/user/my-project/node_modules/',
* '/Users/user/my-project/node_modules/package'
* ) === '/Users/user/my-project/node_modules/package'
*/
const getManagedItem = (managedPath, path) => {
let i = managedPath.length;
let slashes = 1;
let startingPosition = true;
loop: while (i < path.length) {
switch (path.charCodeAt(i)) {
case 47: // slash
case 92: // backslash
if (--slashes === 0) break loop;
startingPosition = true;
break;
case 46: // .
// hidden files are disallowed as managed items
// it's probably .yarn-integrity or .cache
if (startingPosition) return null;
break;
case 64: // @
if (!startingPosition) return null;
slashes++;
break;
default:
startingPosition = false;
break;
}
i++;
}
if (i === path.length) slashes--;
// return null when path is incomplete
if (slashes !== 0) return null;
// if (path.slice(i + 1, i + 13) === "node_modules")
if (
path.length >= i + 13 &&
path.charCodeAt(i + 1) === 110 &&
path.charCodeAt(i + 2) === 111 &&
path.charCodeAt(i + 3) === 100 &&
path.charCodeAt(i + 4) === 101 &&
path.charCodeAt(i + 5) === 95 &&
path.charCodeAt(i + 6) === 109 &&
path.charCodeAt(i + 7) === 111 &&
path.charCodeAt(i + 8) === 100 &&
path.charCodeAt(i + 9) === 117 &&
path.charCodeAt(i + 10) === 108 &&
path.charCodeAt(i + 11) === 101 &&
path.charCodeAt(i + 12) === 115
) {
// if this is the end of the path
if (path.length === i + 13) {
// return the node_modules directory
// it's special
return path;
}
const c = path.charCodeAt(i + 13);
// if next symbol is slash or backslash
if (c === 47 || c === 92) {
// Managed subpath
return getManagedItem(path.slice(0, i + 14), path);
}
}
return path.slice(0, i);
};
/**
* @template {ContextFileSystemInfoEntry | ContextTimestampAndHash} T
* @param {T} entry entry
* @returns {T["resolved"] | undefined} the resolved entry
*/
const getResolvedTimestamp = entry => {
if (entry === null) return null;
if (entry.resolved !== undefined) return entry.resolved;
return entry.symlinks === undefined ? entry : undefined;
};
/**
* @param {ContextHash} entry entry
* @returns {string | undefined} the resolved entry
*/
const getResolvedHash = entry => {
if (entry === null) return null;
if (entry.resolved !== undefined) return entry.resolved;
return entry.symlinks === undefined ? entry.hash : undefined;
};
const addAll = (source, target) => {
for (const key of source) target.add(key);
};
/**
* Used to access information about the filesystem in a cached way
*/
class FileSystemInfo {
/**
* @param {InputFileSystem} fs file system
* @param {Object} options options
* @param {Iterable<string | RegExp>=} options.managedPaths paths that are only managed by a package manager
* @param {Iterable<string | RegExp>=} options.immutablePaths paths that are immutable
* @param {Logger=} options.logger logger used to log invalid snapshots
* @param {string | Hash=} options.hashFunction the hash function to use
*/
constructor(
fs,
{
managedPaths = [],
immutablePaths = [],
logger,
hashFunction = "md4"
} = {}
) {
this.fs = fs;
this.logger = logger;
this._remainingLogs = logger ? 40 : 0;
this._loggedPaths = logger ? new Set() : undefined;
this._hashFunction = hashFunction;
/** @type {WeakMap<Snapshot, boolean | (function(WebpackError=, boolean=): void)[]>} */
this._snapshotCache = new WeakMap();
this._fileTimestampsOptimization = new SnapshotOptimization(
s => s.hasFileTimestamps(),
s => s.fileTimestamps,
(s, v) => s.setFileTimestamps(v)
);
this._fileHashesOptimization = new SnapshotOptimization(
s => s.hasFileHashes(),
s => s.fileHashes,
(s, v) => s.setFileHashes(v),
false
);
this._fileTshsOptimization = new SnapshotOptimization(
s => s.hasFileTshs(),
s => s.fileTshs,
(s, v) => s.setFileTshs(v)
);
this._contextTimestampsOptimization = new SnapshotOptimization(
s => s.hasContextTimestamps(),
s => s.contextTimestamps,
(s, v) => s.setContextTimestamps(v)
);
this._contextHashesOptimization = new SnapshotOptimization(
s => s.hasContextHashes(),
s => s.contextHashes,
(s, v) => s.setContextHashes(v),
false
);
this._contextTshsOptimization = new SnapshotOptimization(
s => s.hasContextTshs(),
s => s.contextTshs,
(s, v) => s.setContextTshs(v)
);
this._missingExistenceOptimization = new SnapshotOptimization(
s => s.hasMissingExistence(),
s => s.missingExistence,
(s, v) => s.setMissingExistence(v),
false
);
this._managedItemInfoOptimization = new SnapshotOptimization(
s => s.hasManagedItemInfo(),
s => s.managedItemInfo,
(s, v) => s.setManagedItemInfo(v),
false
);
this._managedFilesOptimization = new SnapshotOptimization(
s => s.hasManagedFiles(),
s => s.managedFiles,
(s, v) => s.setManagedFiles(v),
false,
true
);
this._managedContextsOptimization = new SnapshotOptimization(
s => s.hasManagedContexts(),
s => s.managedContexts,
(s, v) => s.setManagedContexts(v),
false,
true
);
this._managedMissingOptimization = new SnapshotOptimization(
s => s.hasManagedMissing(),
s => s.managedMissing,
(s, v) => s.setManagedMissing(v),
false,
true
);
/** @type {StackedCacheMap<string, FileSystemInfoEntry | "ignore" | null>} */
this._fileTimestamps = new StackedCacheMap();
/** @type {Map<string, string>} */
this._fileHashes = new Map();
/** @type {Map<string, TimestampAndHash | string>} */
this._fileTshs = new Map();
/** @type {StackedCacheMap<string, ContextFileSystemInfoEntry | "ignore" | null>} */
this._contextTimestamps = new StackedCacheMap();
/** @type {Map<string, ContextHash>} */
this._contextHashes = new Map();
/** @type {Map<string, ContextTimestampAndHash>} */
this._contextTshs = new Map();
/** @type {Map<string, string>} */
this._managedItems = new Map();
/** @type {AsyncQueue<string, string, FileSystemInfoEntry | null>} */
this.fileTimestampQueue = new AsyncQueue({
name: "file timestamp",
parallelism: 30,
processor: this._readFileTimestamp.bind(this)
});
/** @type {AsyncQueue<string, string, string | null>} */
this.fileHashQueue = new AsyncQueue({
name: "file hash",
parallelism: 10,
processor: this._readFileHash.bind(this)
});
/** @type {AsyncQueue<string, string, ContextFileSystemInfoEntry | null>} */
this.contextTimestampQueue = new AsyncQueue({
name: "context timestamp",
parallelism: 2,
processor: this._readContextTimestamp.bind(this)
});
/** @type {AsyncQueue<string, string, ContextHash | null>} */
this.contextHashQueue = new AsyncQueue({
name: "context hash",
parallelism: 2,
processor: this._readContextHash.bind(this)
});
/** @type {AsyncQueue<string, string, ContextTimestampAndHash | null>} */
this.contextTshQueue = new AsyncQueue({
name: "context hash and timestamp",
parallelism: 2,
processor: this._readContextTimestampAndHash.bind(this)
});
/** @type {AsyncQueue<string, string, string | null>} */
this.managedItemQueue = new AsyncQueue({
name: "managed item info",
parallelism: 10,
processor: this._getManagedItemInfo.bind(this)
});
/** @type {AsyncQueue<string, string, Set<string>>} */
this.managedItemDirectoryQueue = new AsyncQueue({
name: "managed item directory info",
parallelism: 10,
processor: this._getManagedItemDirectoryInfo.bind(this)
});
this.managedPaths = Array.from(managedPaths);
this.managedPathsWithSlash = /** @type {string[]} */ (
this.managedPaths.filter(p => typeof p === "string")
).map(p => join(fs, p, "_").slice(0, -1));
this.managedPathsRegExps = /** @type {RegExp[]} */ (
this.managedPaths.filter(p => typeof p !== "string")
);
this.immutablePaths = Array.from(immutablePaths);
this.immutablePathsWithSlash = /** @type {string[]} */ (
this.immutablePaths.filter(p => typeof p === "string")
).map(p => join(fs, p, "_").slice(0, -1));
this.immutablePathsRegExps = /** @type {RegExp[]} */ (
this.immutablePaths.filter(p => typeof p !== "string")
);
this._cachedDeprecatedFileTimestamps = undefined;
this._cachedDeprecatedContextTimestamps = undefined;
this._warnAboutExperimentalEsmTracking = false;
this._statCreatedSnapshots = 0;
this._statTestedSnapshotsCached = 0;
this._statTestedSnapshotsNotCached = 0;
this._statTestedChildrenCached = 0;
this._statTestedChildrenNotCached = 0;
this._statTestedEntries = 0;
}
logStatistics() {
const logWhenMessage = (header, message) => {
if (message) {
this.logger.log(`${header}: ${message}`);
}
};
this.logger.log(`${this._statCreatedSnapshots} new snapshots created`);
this.logger.log(
`${
this._statTestedSnapshotsNotCached &&
Math.round(
(this._statTestedSnapshotsNotCached * 100) /
(this._statTestedSnapshotsCached +
this._statTestedSnapshotsNotCached)
)
}% root snapshot uncached (${this._statTestedSnapshotsNotCached} / ${
this._statTestedSnapshotsCached + this._statTestedSnapshotsNotCached
})`
);
this.logger.log(
`${
this._statTestedChildrenNotCached &&
Math.round(
(this._statTestedChildrenNotCached * 100) /
(this._statTestedChildrenCached + this._statTestedChildrenNotCached)
)
}% children snapshot uncached (${this._statTestedChildrenNotCached} / ${
this._statTestedChildrenCached + this._statTestedChildrenNotCached
})`
);
this.logger.log(`${this._statTestedEntries} entries tested`);
this.logger.log(
`File info in cache: ${this._fileTimestamps.size} timestamps ${this._fileHashes.size} hashes ${this._fileTshs.size} timestamp hash combinations`
);
logWhenMessage(
`File timestamp snapshot optimization`,
this._fileTimestampsOptimization.getStatisticMessage()
);
logWhenMessage(
`File hash snapshot optimization`,
this._fileHashesOptimization.getStatisticMessage()
);
logWhenMessage(
`File timestamp hash combination snapshot optimization`,
this._fileTshsOptimization.getStatisticMessage()
);
this.logger.log(
`Directory info in cache: ${this._contextTimestamps.size} timestamps ${this._contextHashes.size} hashes ${this._contextTshs.size} timestamp hash combinations`
);
logWhenMessage(
`Directory timestamp snapshot optimization`,
this._contextTimestampsOptimization.getStatisticMessage()
);
logWhenMessage(
`Directory hash snapshot optimization`,
this._contextHashesOptimization.getStatisticMessage()
);
logWhenMessage(
`Directory timestamp hash combination snapshot optimization`,
this._contextTshsOptimization.getStatisticMessage()
);
logWhenMessage(
`Missing items snapshot optimization`,
this._missingExistenceOptimization.getStatisticMessage()
);
this.logger.log(
`Managed items info in cache: ${this._managedItems.size} items`
);
logWhenMessage(
`Managed items snapshot optimization`,
this._managedItemInfoOptimization.getStatisticMessage()
);
logWhenMessage(
`Managed files snapshot optimization`,
this._managedFilesOptimization.getStatisticMessage()
);
logWhenMessage(
`Managed contexts snapshot optimization`,
this._managedContextsOptimization.getStatisticMessage()
);
logWhenMessage(
`Managed missing snapshot optimization`,
this._managedMissingOptimization.getStatisticMessage()
);
}
_log(path, reason, ...args) {
const key = path + reason;
if (this._loggedPaths.has(key)) return;
this._loggedPaths.add(key);
this.logger.debug(`${path} invalidated because ${reason}`, ...args);
if (--this._remainingLogs === 0) {
this.logger.debug(
"Logging limit has been reached and no further logging will be emitted by FileSystemInfo"
);
}
}
clear() {
this._remainingLogs = this.logger ? 40 : 0;
if (this._loggedPaths !== undefined) this._loggedPaths.clear();
this._snapshotCache = new WeakMap();
this._fileTimestampsOptimization.clear();
this._fileHashesOptimization.clear();
this._fileTshsOptimization.clear();
this._contextTimestampsOptimization.clear();
this._contextHashesOptimization.clear();
this._contextTshsOptimization.clear();
this._missingExistenceOptimization.clear();
this._managedItemInfoOptimization.clear();
this._managedFilesOptimization.clear();
this._managedContextsOptimization.clear();
this._managedMissingOptimization.clear();
this._fileTimestamps.clear();
this._fileHashes.clear();
this._fileTshs.clear();
this._contextTimestamps.clear();
this._contextHashes.clear();
this._contextTshs.clear();
this._managedItems.clear();
this._managedItems.clear();
this._cachedDeprecatedFileTimestamps = undefined;
this._cachedDeprecatedContextTimestamps = undefined;
this._statCreatedSnapshots = 0;
this._statTestedSnapshotsCached = 0;
this._statTestedSnapshotsNotCached = 0;
this._statTestedChildrenCached = 0;
this._statTestedChildrenNotCached = 0;
this._statTestedEntries = 0;
}
/**
* @param {ReadonlyMap<string, FileSystemInfoEntry | "ignore" | null>} map timestamps
* @param {boolean=} immutable if 'map' is immutable and FileSystemInfo can keep referencing it
* @returns {void}
*/
addFileTimestamps(map, immutable) {
this._fileTimestamps.addAll(map, immutable);
this._cachedDeprecatedFileTimestamps = undefined;
}
/**
* @param {ReadonlyMap<string, FileSystemInfoEntry | "ignore" | null>} map timestamps
* @param {boolean=} immutable if 'map' is immutable and FileSystemInfo can keep referencing it
* @returns {void}
*/
addContextTimestamps(map, immutable) {
this._contextTimestamps.addAll(map, immutable);
this._cachedDeprecatedContextTimestamps = undefined;
}
/**
* @param {string} path file path
* @param {function(WebpackError=, (FileSystemInfoEntry | "ignore" | null)=): void} callback callback function
* @returns {void}
*/
getFileTimestamp(path, callback) {
const cache = this._fileTimestamps.get(path);
if (cache !== undefined) return callback(null, cache);
this.fileTimestampQueue.add(path, callback);
}
/**
* @param {string} path context path
* @param {function(WebpackError=, (ResolvedContextFileSystemInfoEntry | "ignore" | null)=): void} callback callback function
* @returns {void}
*/
getContextTimestamp(path, callback) {
const cache = this._contextTimestamps.get(path);
if (cache !== undefined) {
if (cache === "ignore") return callback(null, "ignore");
const resolved = getResolvedTimestamp(cache);
if (resolved !== undefined) return callback(null, resolved);
return this._resolveContextTimestamp(cache, callback);
}
this.contextTimestampQueue.add(path, (err, entry) => {
if (err) return callback(err);
const resolved = getResolvedTimestamp(entry);
if (resolved !== undefined) return callback(null, resolved);
this._resolveContextTimestamp(entry, callback);
});
}
/**
* @param {string} path context path
* @param {function(WebpackError=, (ContextFileSystemInfoEntry | "ignore" | null)=): void} callback callback function
* @returns {void}
*/
_getUnresolvedContextTimestamp(path, callback) {
const cache = this._contextTimestamps.get(path);
if (cache !== undefined) return callback(null, cache);
this.contextTimestampQueue.add(path, callback);
}
/**
* @param {string} path file path
* @param {function(WebpackError=, string=): void} callback callback function
* @returns {void}
*/
getFileHash(path, callback) {
const cache = this._fileHashes.get(path);
if (cache !== undefined) return callback(null, cache);
this.fileHashQueue.add(path, callback);
}
/**
* @param {string} path context path
* @param {function(WebpackError=, string=): void} callback callback function
* @returns {void}
*/
getContextHash(path, callback) {
const cache = this._contextHashes.get(path);
if (cache !== undefined) {
const resolved = getResolvedHash(cache);
if (resolved !== undefined) return callback(null, resolved);
return this._resolveContextHash(cache, callback);
}
this.contextHashQueue.add(path, (err, entry) => {
if (err) return callback(err);
const resolved = getResolvedHash(entry);
if (resolved !== undefined) return callback(null, resolved);
this._resolveContextHash(entry, callback);
});
}
/**
* @param {string} path context path
* @param {function(WebpackError=, ContextHash=): void} callback callback function
* @returns {void}
*/
_getUnresolvedContextHash(path, callback) {
const cache = this._contextHashes.get(path);
if (cache !== undefined) return callback(null, cache);
this.contextHashQueue.add(path, callback);
}
/**
* @param {string} path context path
* @param {function(WebpackError=, ResolvedContextTimestampAndHash=): void} callback callback function
* @returns {void}
*/
getContextTsh(path, callback) {
const cache = this._contextTshs.get(path);
if (cache !== undefined) {
const resolved = getResolvedTimestamp(cache);
if (resolved !== undefined) return callback(null, resolved);
return this._resolveContextTsh(cache, callback);
}
this.contextTshQueue.add(path, (err, entry) => {
if (err) return callback(err);
const resolved = getResolvedTimestamp(entry);
if (resolved !== undefined) return callback(null, resolved);
this._resolveContextTsh(entry, callback);
});
}
/**
* @param {string} path context path
* @param {function(WebpackError=, ContextTimestampAndHash=): void} callback callback function
* @returns {void}
*/
_getUnresolvedContextTsh(path, callback) {
const cache = this._contextTshs.get(path);
if (cache !== undefined) return callback(null, cache);
this.contextTshQueue.add(path, callback);
}
_createBuildDependenciesResolvers() {
const resolveContext = createResolver({
resolveToContext: true,
exportsFields: [],
fileSystem: this.fs
});
const resolveCjs = createResolver({
extensions: [".js", ".json", ".node"],
conditionNames: ["require", "node"],
exportsFields: ["exports"],
fileSystem: this.fs
});
const resolveCjsAsChild = createResolver({
extensions: [".js", ".json", ".node"],
conditionNames: ["require", "node"],
exportsFields: [],
fileSystem: this.fs
});
const resolveEsm = createResolver({
extensions: [".js", ".json", ".node"],
fullySpecified: true,
conditionNames: ["import", "node"],
exportsFields: ["exports"],
fileSystem: this.fs
});
return { resolveContext, resolveEsm, resolveCjs, resolveCjsAsChild };
}
/**
* @param {string} context context directory
* @param {Iterable<string>} deps dependencies
* @param {function(Error=, ResolveBuildDependenciesResult=): void} callback callback function
* @returns {void}
*/
resolveBuildDependencies(context, deps, callback) {
const { resolveContext, resolveEsm, resolveCjs, resolveCjsAsChild } =
this._createBuildDependenciesResolvers();
/** @type {Set<string>} */
const files = new Set();
/** @type {Set<string>} */
const fileSymlinks = new Set();
/** @type {Set<string>} */
const directories = new Set();
/** @type {Set<string>} */
const directorySymlinks = new Set();
/** @type {Set<string>} */
const missing = new Set();
/** @type {Set<string>} */
const resolveFiles = new Set();
/** @type {Set<string>} */
const resolveDirectories = new Set();
/** @type {Set<string>} */
const resolveMissing = new Set();
/** @type {Map<string, string | false>} */
const resolveResults = new Map();
const invalidResolveResults = new Set();
const resolverContext = {
fileDependencies: resolveFiles,
contextDependencies: resolveDirectories,
missingDependencies: resolveMissing
};
const expectedToString = expected => {
return expected ? ` (expected ${expected})` : "";
};
const jobToString = job => {
switch (job.type) {
case RBDT_RESOLVE_CJS:
return `resolve commonjs ${job.path}${expectedToString(
job.expected
)}`;
case RBDT_RESOLVE_ESM:
return `resolve esm ${job.path}${expectedToString(job.expected)}`;
case RBDT_RESOLVE_DIRECTORY:
return `resolve directory ${job.path}`;
case RBDT_RESOLVE_CJS_FILE:
return `resolve commonjs file ${job.path}${expectedToString(
job.expected
)}`;
case RBDT_RESOLVE_ESM_FILE:
return `resolve esm file ${job.path}${expectedToString(
job.expected
)}`;
case RBDT_DIRECTORY:
return `directory ${job.path}`;
case RBDT_FILE:
return `file ${job.path}`;
case RBDT_DIRECTORY_DEPENDENCIES:
return `directory dependencies ${job.path}`;
case RBDT_FILE_DEPENDENCIES:
return `file dependencies ${job.path}`;
}
return `unknown ${job.type} ${job.path}`;
};
const pathToString = job => {
let result = ` at ${jobToString(job)}`;
job = job.issuer;
while (job !== undefined) {
result += `\n at ${jobToString(job)}`;
job = job.issuer;
}
return result;
};
processAsyncTree(
Array.from(deps, dep => ({
type: RBDT_RESOLVE_CJS,
context,
path: dep,
expected: undefined,
issuer: undefined
})),
20,
(job, push, callback) => {
const { type, context, path, expected } = job;
const resolveDirectory = path => {
const key = `d\n${context}\n${path}`;
if (resolveResults.has(key)) {
return callback();
}
resolveResults.set(key, undefined);
resolveContext(context, path, resolverContext, (err, _, result) => {
if (err) {
if (expected === false) {
resolveResults.set(key, false);
return callback();
}
invalidResolveResults.add(key);
err.message += `\nwhile resolving '${path}' in ${context} to a directory`;
return callback(err);
}
const resultPath = result.path;
resolveResults.set(key, resultPath);
push({
type: RBDT_DIRECTORY,
context: undefined,
path: resultPath,
expected: undefined,
issuer: job
});
callback();
});
};
const resolveFile = (path, symbol, resolve) => {
const key = `${symbol}\n${context}\n${path}`;
if (resolveResults.has(key)) {
return callback();
}
resolveResults.set(key, undefined);
resolve(context, path, resolverContext, (err, _, result) => {
if (typeof expected === "string") {
if (!err && result && result.path === expected) {
resolveResults.set(key, result.path);
} else {
invalidResolveResults.add(key);
this.logger.warn(
`Resolving '${path}' in ${context} for build dependencies doesn't lead to expected result '${expected}', but to '${
err || (result && result.path)
}' instead. Resolving dependencies are ignored for this path.\n${pathToString(
job
)}`
);
}
} else {
if (err) {
if (expected === false) {
resolveResults.set(key, false);
return callback();
}
invalidResolveResults.add(key);
err.message += `\nwhile resolving '${path}' in ${context} as file\n${pathToString(
job
)}`;
return callback(err);
}
const resultPath = result.path;
resolveResults.set(key, resultPath);
push({
type: RBDT_FILE,
context: undefined,
path: resultPath,
expected: undefined,
issuer: job
});
}
callback();
});
};
switch (type) {
case RBDT_RESOLVE_CJS: {
const isDirectory = /[\\/]$/.test(path);
if (isDirectory) {
resolveDirectory(path.slice(0, path.length - 1));
} else {
resolveFile(path, "f", resolveCjs);
}
break;
}
case RBDT_RESOLVE_ESM: {
const isDirectory = /[\\/]$/.test(path);
if (isDirectory) {
resolveDirectory(path.slice(0, path.length - 1));
} else {
resolveFile(path);
}
break;
}
case RBDT_RESOLVE_DIRECTORY: {
resolveDirectory(path);
break;
}
case RBDT_RESOLVE_CJS_FILE: {
resolveFile(path, "f", resolveCjs);
break;
}
case RBDT_RESOLVE_CJS_FILE_AS_CHILD: {
resolveFile(path, "c", resolveCjsAsChild);
break;
}
case RBDT_RESOLVE_ESM_FILE: {
resolveFile(path, "e", resolveEsm);
break;
}
case RBDT_FILE: {
if (files.has(path)) {
callback();
break;
}
files.add(path);
this.fs.realpath(path, (err, _realPath) => {
if (err) return callback(err);
const realPath = /** @type {string} */ (_realPath);
if (realPath !== path) {
fileSymlinks.add(path);
resolveFiles.add(path);
if (files.has(realPath)) return callback();
files.add(realPath);
}
push({
type: RBDT_FILE_DEPENDENCIES,
context: undefined,
path: realPath,
expected: undefined,
issuer: job
});
callback();
});
break;
}
case RBDT_DIRECTORY: {
if (directories.has(path)) {
callback();
break;
}
directories.add(path);
this.fs.realpath(path, (err, _realPath) => {
if (err) return callback(err);
const realPath = /** @type {string} */ (_realPath);
if (realPath !== path) {
directorySymlinks.add(path);
resolveFiles.add(path);
if (directories.has(realPath)) return callback();
directories.add(realPath);
}
push({
type: RBDT_DIRECTORY_DEPENDENCIES,
context: undefined,
path: realPath,
expected: undefined,
issuer: job
});
callback();
});
break;
}
case RBDT_FILE_DEPENDENCIES: {
// Check for known files without dependencies
if (/\.json5?$|\.yarn-integrity$|yarn\.lock$|\.ya?ml/.test(path)) {
process.nextTick(callback);
break;
}
// Check commonjs cache for the module
/** @type {NodeModule} */
const module = require.cache[path];
if (module && Array.isArray(module.children)) {
children: for (const child of module.children) {
let childPath = child.filename;
if (childPath) {
push({
type: RBDT_FILE,
context: undefined,
path: childPath,
expected: undefined,
issuer: job
});
const context = dirname(this.fs, path);
for (const modulePath of module.paths) {
if (childPath.startsWith(modulePath)) {
let subPath = childPath.slice(modulePath.length + 1);
const packageMatch = /^(@[^\\/]+[\\/])[^\\/]+/.exec(
subPath
);
if (packageMatch) {
push({
type: RBDT_FILE,
context: undefined,
path:
modulePath +
childPath[modulePath.length] +
packageMatch[0] +
childPath[modulePath.length] +
"package.json",
expected: false,
issuer: job
});
}
let request = subPath.replace(/\\/g, "/");
if (request.endsWith(".js"))
request = request.slice(0, -3);
push({
type: RBDT_RESOLVE_CJS_FILE_AS_CHILD,
context,
path: request,
expected: child.filename,
issuer: job
});
continue children;
}
}
let request = relative(this.fs, context, childPath);
if (request.endsWith(".js")) request = request.slice(0, -3);
request = request.replace(/\\/g, "/");
if (!request.startsWith("../")) request = `./${request}`;
push({
type: RBDT_RESOLVE_CJS_FILE,
context,
path: request,
expected: child.filename,
issuer: job
});
}
}
} else if (supportsEsm && /\.m?js$/.test(path)) {
if (!this._warnAboutExperimentalEsmTracking) {
this.logger.log(
"Node.js doesn't offer a (nice) way to introspect the ESM dependency graph yet.\n" +
"Until a full solution is available webpack uses an experimental ESM tracking based on parsing.\n" +
"As best effort webpack parses the ESM files to guess dependencies. But this can lead to expensive and incorrect tracking."
);
this._warnAboutExperimentalEsmTracking = true;
}
const lexer = require("es-module-lexer");
lexer.init.then(() => {
this.fs.readFile(path, (err, content) => {
if (err) return callback(err);
try {
const context = dirname(this.fs, path);
const source = content.toString();
const [imports] = lexer.parse(source);
for (const imp of imports) {
try {
let dependency;
if (imp.d === -1) {
// import ... from "..."
dependency = J