hard-source-webpack-plugin
Version:
Hard cache the source of modules in webpack.
307 lines (271 loc) • 9.19 kB
JavaScript
const { fork: cpFork } = require('child_process');
const { cpus } = require('os');
const { resolve } = require('path');
const logMessages = require('./util/log-messages');
const pluginCompat = require('./util/plugin-compat');
const webpackBin = () => {
try {
return require.resolve('webpack-cli');
} catch (e) {}
try {
return require.resolve('webpack-command');
} catch (e) {}
throw new Error('webpack cli tool not installed or discoverable');
};
const configPath = compiler => {
try {
return require.resolve(
resolve(compiler.options.context || process.cwd(), 'webpack.config'),
);
} catch (e) {}
try {
return require.resolve(resolve(process.cwd(), 'webpack.config'));
} catch (e) {}
throw new Error('config not in obvious location');
};
class ParallelModulePlugin {
constructor(options) {
this.options = options;
}
apply(compiler) {
try {
require('webpack/lib/JavascriptGenerator');
} catch (e) {
logMessages.parallelRequireWebpack4(compiler);
return;
}
const options = this.options || {};
const fork =
options.fork ||
((fork, compiler, webpackBin) =>
fork(webpackBin(compiler), ['--config', configPath(compiler)], {
silent: true,
}));
const numWorkers = options.numWorkers
? typeof options.numWorkers === 'function'
? options.numWorkers
: () => options.numWorkers
: () => cpus().length;
const minModules =
typeof options.minModules === 'number' ? options.minModules : 10;
const compilerHooks = pluginCompat.hooks(compiler);
let freeze, thaw;
compilerHooks._hardSourceMethods.tap('ParallelModulePlugin', methods => {
freeze = methods.freeze;
thaw = methods.thaw;
});
compilerHooks.thisCompilation.tap(
'ParallelModulePlugin',
(compilation, params) => {
const compilationHooks = pluginCompat.hooks(compilation);
const nmfHooks = pluginCompat.hooks(params.normalModuleFactory);
const doMaster = () => {
const jobs = {};
const readyJobs = {};
const workers = [];
let nextWorkerIndex = 0;
let start = 0;
let started = false;
let configMismatch = false;
let modules = 0;
const startWorkers = () => {
const _numWorkers = numWorkers();
logMessages.parallelStartWorkers(compiler, {
numWorkers: _numWorkers,
});
for (let i = 0; i < _numWorkers; i++) {
const worker = fork(cpFork, compiler, webpackBin);
workers.push(worker);
worker.on('message', _result => {
if (configMismatch) {
return;
}
if (_result.startsWith('ready:')) {
const configHash = _result.split(':')[1];
if (configHash !== compiler.__hardSource_configHash) {
logMessages.parallelConfigMismatch(compiler, {
outHash: compiler.__hardSource_configHash,
theirHash: configHash,
});
configMismatch = true;
killWorkers();
for (const id in jobs) {
jobs[id].cb({ error: true });
delete readyJobs[id];
delete jobs[id];
}
return;
}
}
if (Object.values(readyJobs).length) {
const id = Object.keys(readyJobs)[0];
worker.send(
JSON.stringify({
id,
data: readyJobs[id].data,
}),
);
delete readyJobs[id];
} else {
worker.ready = true;
}
if (_result.startsWith('ready:')) {
start = Date.now();
return;
}
const result = JSON.parse(_result);
jobs[result.id].cb(result);
delete [result.id];
});
}
};
const killWorkers = () => {
Object.values(workers).forEach(worker => worker.kill());
};
const doJob = (module, cb) => {
if (configMismatch) {
cb({ error: new Error('config mismatch') });
return;
}
const id = 'xxxxxxxx-xxxxxxxx'.replace(/x/g, () =>
Math.random()
.toString(16)
.substring(2, 3),
);
jobs[id] = {
id,
data: freeze('Module', null, module, {
id: module.identifier(),
compilation,
}),
cb,
};
const worker = Object.values(workers).find(worker => worker.ready);
if (worker) {
worker.ready = false;
worker.send(
JSON.stringify({
id,
data: jobs[id].data,
}),
);
} else {
readyJobs[id] = jobs[id];
}
if (!started) {
started = true;
startWorkers();
}
};
const _create = params.normalModuleFactory.create;
params.normalModuleFactory.create = (data, cb) => {
_create.call(params.normalModuleFactory, data, (err, module) => {
if (err) {
return cb(err);
}
if (module.constructor.name === 'NormalModule') {
const build = module.build;
module.build = (
options,
compilation,
resolver,
fs,
callback,
) => {
if (modules < minModules) {
build.call(
module,
options,
compilation,
resolver,
fs,
callback,
);
modules += 1;
return;
}
try {
doJob(module, result => {
if (result.error) {
build.call(
module,
options,
compilation,
resolver,
fs,
callback,
);
} else {
thaw('Module', module, result.module, {
compilation,
normalModuleFactory: params.normalModuleFactory,
contextModuleFactory: params.contextModuleFactory,
});
callback();
}
});
} catch (e) {
logMessages.parallelErrorSendingJob(compiler, e);
build.call(
module,
options,
compilation,
resolver,
fs,
callback,
);
}
};
cb(null, module);
} else {
cb(err, module);
}
});
};
compilationHooks.seal.tap('ParallelModulePlugin', () => {
killWorkers();
});
};
const doChild = () => {
const _create = params.normalModuleFactory.create;
params.normalModuleFactory.create = (data, cb) => {};
process.send('ready:' + compiler.__hardSource_configHash);
process.on('message', _job => {
const job = JSON.parse(_job);
const module = thaw('Module', null, job.data, {
compilation,
normalModuleFactory: params.normalModuleFactory,
contextModuleFactory: params.contextModuleFactory,
});
module.build(
compilation.options,
compilation,
compilation.resolverFactory.get('normal', module.resolveOptions),
compilation.inputFileSystem,
error => {
process.send(
JSON.stringify({
id: job.id,
error: error,
module:
module &&
freeze('Module', null, module, {
id: module.identifier(),
compilation,
}),
}),
);
},
);
});
};
if (!process.send) {
doMaster();
} else {
doChild();
}
},
);
}
}
module.exports = ParallelModulePlugin;