bytenode
Version:
A minimalist bytecode compiler for Node.js
621 lines (523 loc) • 22.4 kB
JavaScript
;
const { ok } = require('node:assert/strict');
const { spawn } = require('node:child_process');
const fs = require('node:fs');
const Module = require('node:module');
const path = require('node:path');
const vm = require('node:vm');
const v8 = require('node:v8');
const { brotliCompressSync, brotliDecompressSync } = require('node:zlib');
v8.setFlagsFromString('--no-lazy');
if (Number.parseInt(process.versions.node, 10) >= 12) {
v8.setFlagsFromString('--no-flush-bytecode'); // Thanks to A-Parser (@a-parser)
}
function isBuffer (obj) {
return obj != null && obj.constructor != null &&
typeof obj.constructor.isBuffer === 'function' && obj.constructor.isBuffer(obj)
}
const COMPILED_EXTNAME = '.jsc';
const MAGIC_NUMBER = Buffer.from([0xde, 0xc0]);
const ZERO_LENGTH_EXTERNAL_REFERENCE_TABLE = Buffer.alloc(2);
const sheBangRegex = /^#!.*/;
// Set BYTENODE_DEBUG=1 to print the V8 cached-data header layout at load time.
// This is the key diagnostic for SIGTRAP crashes on newer V8 (e.g. Electron 42 / V8 14.8):
// it lets us compare, field by field, the .jsc compiled at build time against a dummy
// compiled in the *current* runtime process, so we can see exactly which header field
// (version hash / flag hash / read-only snapshot checksum) is mismatched.
const DEBUG = !!process.env.BYTENODE_DEBUG;
// Modern V8 SerializedCodeData header (uint32_t LE entries):
// [0] magic number offset 0
// [1] version hash offset 4
// [2] source hash (== source length) offset 8
// [3] flag hash offset 12
// [4] read-only snapshot checksum offset 16
// [5] payload length offset 20
// [6] payload checksum offset 24
function dumpBytecodeHeader (label, buffer) {
const names = [
'magic @0 ',
'versionHash @4 ',
'sourceHash @8 ',
'flagHash @12',
'roChecksum @16',
'payloadLen @20',
'checksum @24'
];
const lines = [`[bytenode] ${label} (buffer length = ${buffer.length})`];
for (let i = 0; i < names.length; i++) {
const off = i * 4;
if (off + 4 > buffer.length) break;
const v = buffer.readUInt32LE(off);
lines.push(` ${names[i]} = 0x${v.toString(16).padStart(8, '0')} (${v})`);
}
console.error(lines.join('\n'));
}
function generateScript (cachedData, filename) {
if (!isBufferV8Bytecode(cachedData)) {
// Try to decompress as Brotli
cachedData = brotliDecompressSync(cachedData);
ok(isBufferV8Bytecode(cachedData), 'Invalid bytecode buffer');
}
if (DEBUG) {
console.error(`[bytenode] loading ${filename || '<buffer>'}`);
console.error(`[bytenode] runtime: node ${process.version}` +
(process.versions.electron ? `, electron ${process.versions.electron}` : '') +
`, v8 ${process.versions.v8}`);
dumpBytecodeHeader('on-disk .jsc header (before fixBytecode)', cachedData);
// A dummy compiled in THIS runtime process: its build-dependent header fields
// (version hash / flag hash / ro snapshot checksum) are what V8 will expect here.
try {
dumpBytecodeHeader('runtime dummy header (what V8 expects)', compileCode('"\u0ca0_\u0ca0"'));
} catch (err) {
console.error('[bytenode] failed to compile runtime dummy:', err && err.message);
}
}
fixBytecode(cachedData);
if (DEBUG) {
dumpBytecodeHeader('on-disk .jsc header (after fixBytecode)', cachedData);
}
const length = readSourceHash(cachedData);
let dummyCode = '';
if (length > 1) {
dummyCode = '"' + '\u200b'.repeat(length - 2) + '"'; // "\u200b" Zero width space
}
if (DEBUG) {
console.error(`[bytenode] sourceHash/dummy length = ${length}; about to call new vm.Script(...)`);
}
const script = new vm.Script(dummyCode, { cachedData, filename });
if (script.cachedDataRejected) {
throw new Error('Invalid or incompatible cached data (cachedDataRejected)');
}
if (DEBUG) {
console.error('[bytenode] vm.Script created, cachedData accepted.');
}
return script;
}
function isBufferV8Bytecode (buffer) {
return (
isBuffer(buffer) &&
!buffer.subarray(0, 2).equals(ZERO_LENGTH_EXTERNAL_REFERENCE_TABLE) &&
buffer.subarray(2, 4).equals(MAGIC_NUMBER)
);
// TODO: check that code start + payload size = buffer length. See
// https://github.com/bytenode/bytenode/issues/210#issuecomment-1605691369
}
/**
* Generates v8 bytecode buffer.
* @param {string} javascriptCode JavaScript source that will be compiled to bytecode.
* @param {boolean} compress Compress the bytecode.
* @returns {Buffer} The generated bytecode.
*/
const compileCode = function (javascriptCode, compress) {
if (typeof javascriptCode !== 'string') {
throw new Error(`javascriptCode must be string. ${typeof javascriptCode} was given.`);
}
const script = new vm.Script(javascriptCode, {
produceCachedData: true
});
let bytecodeBuffer = (script.createCachedData && script.createCachedData.call)
? script.createCachedData()
: script.cachedData;
if (compress) bytecodeBuffer = brotliCompressSync(bytecodeBuffer);
return bytecodeBuffer;
};
/**
* This function runs the compileCode() function (above)
* via a child process using Electron as Node
* @param {string} javascriptCode
* @param {object} [options] - optional options object
* @param {string} [options.electronPath] - optional path to Electron executable, defaults to the installed node_modules/electron
* @param {boolean} [options.compress]
* @returns {Promise<Buffer>} - returns a Promise which resolves in the generated bytecode.
*/
const compileElectronCode = function (javascriptCode, options) {
return new Promise((resolve, reject) => {
function onEnd () {
if (options.compress) data = brotliCompressSync(data);
resolve(data);
}
options = options || {};
let data = Buffer.from([]);
// Resolve `electron` lazily and only when no explicit path was given. This lets
// consumers (e.g. a linked/forked bytenode living outside the project's node_modules,
// where `require('electron')` would not resolve) pass `electronPath` instead.
const electronPath = options.electronPath
? path.normalize(options.electronPath)
: /** @type {string} */ (require('electron'));
if (!fs.existsSync(electronPath)) {
throw new Error('Electron not found');
}
const bytenodePath = path.join(__dirname, 'cli.js');
// create a subprocess in which we run Electron as our Node and V8 engine
// running Bytenode to compile our code through stdin/stdout
const child = spawn(electronPath, [bytenodePath, '--compile', '--no-module', '-'], {
env: { ELECTRON_RUN_AS_NODE: '1' },
stdio: ['pipe', 'pipe', 'pipe', 'ipc']
});
if (child.stdin) {
child.stdin.write(javascriptCode);
child.stdin.end();
}
if (child.stdout) {
child.stdout.on('data', (chunk) => {
data = Buffer.concat([data, chunk]);
});
child.stdout.on('error', (err) => {
console.error(err);
});
child.stdout.on('end', onEnd);
}
if (child.stderr) {
child.stderr.on('data', (chunk) => {
console.error('Error: ', chunk.toString());
});
child.stderr.on('error', (err) => {
console.error('Error: ', err);
});
}
child.addListener('message', (message) => console.log(message));
child.addListener('error', err => console.error(err));
child.on('error', (err) => reject(err));
child.on('exit', onEnd);
});
};
/**
* Compiles code to bytecode inside an actual Electron *main* process (NOT
* ELECTRON_RUN_AS_NODE). This is required on Electron >= 42 (V8 >= 14.8): the
* main/browser process boots from Chromium's `v8_context_snapshot`, whose
* read-only heap differs from the Node default snapshot used by
* ELECTRON_RUN_AS_NODE. Bytecode carries a read-only-snapshot checksum
* (header offset 16); if it was produced under run-as-node it will not match
* the main process at load time, and V8 14.8 hard-aborts (SIGTRAP) while
* deserializing instead of cleanly rejecting. Compiling here, in the same
* process type that will later load the .jsc, makes the checksum match.
* @param {string} javascriptCode
* @param {object} [options]
* @param {string} [options.electronPath]
* @param {boolean} [options.compress]
* @returns {Promise<Buffer>}
*/
const compileElectronMainCode = function (javascriptCode, options) {
return new Promise((resolve, reject) => {
options = options || {};
const os = require('node:os');
const electronPath = options.electronPath
? path.normalize(options.electronPath)
: /** @type {string} */ (require('electron'));
if (!fs.existsSync(electronPath)) {
throw new Error('Electron not found');
}
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bytenode-electron-main-'));
const inFile = path.join(tmpDir, 'input.js');
const outFile = path.join(tmpDir, 'output.jsc');
const compilerScript = path.join(tmpDir, 'compiler.js');
fs.writeFileSync(inFile, javascriptCode);
// Minimal Electron main-process entry: wait for app ready, compile, write,
// then quit. No BrowserWindow is created. The V8 flags must match those set
// by bytenode when the .jsc is later loaded, so the flag hash agrees.
const compilerSource = [
"const { app } = require('electron');",
"const fs = require('fs');",
"const vm = require('vm');",
"const v8 = require('v8');",
"v8.setFlagsFromString('--no-lazy');",
"v8.setFlagsFromString('--no-flush-bytecode');",
// No renderer/window is created, so the Chromium sandbox is irrelevant here.
// Disabling it avoids failures when Electron is launched as root in CI containers.
"if (app.commandLine && app.commandLine.appendSwitch) app.commandLine.appendSwitch('no-sandbox');",
'if (typeof app.disableHardwareAcceleration === "function") app.disableHardwareAcceleration();',
'function run () {',
' try {',
' const code = fs.readFileSync(' + JSON.stringify(inFile) + ", 'utf-8');",
' const script = new vm.Script(code, { produceCachedData: true });',
' const buf = (script.createCachedData && script.createCachedData.call) ? script.createCachedData() : script.cachedData;',
' fs.writeFileSync(' + JSON.stringify(outFile) + ', buf);',
' process.exitCode = 0;',
' } catch (err) {',
' process.stderr.write(String((err && err.stack) || err) + "\\n");',
' process.exitCode = 1;',
' } finally {',
' app.quit();',
' }',
'}',
'app.whenReady().then(run);'
].join('\n');
fs.writeFileSync(compilerScript, compilerSource);
const cleanup = () => {
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) { /* ignore */ }
};
const child = spawn(electronPath, [compilerScript], {
// Deliberately NOT setting ELECTRON_RUN_AS_NODE: we want the browser/main process.
env: process.env,
stdio: ['ignore', 'inherit', 'inherit']
});
child.on('error', (err) => { cleanup(); reject(err); });
child.on('exit', (code) => {
if (code !== 0) {
cleanup();
reject(new Error('Electron main-process bytecode compilation failed (exit code ' + code + ')'));
return;
}
try {
let data = fs.readFileSync(outFile);
if (options.compress) data = brotliCompressSync(data);
cleanup();
resolve(data);
} catch (err) {
cleanup();
reject(err);
}
});
});
};
// TODO: rewrite this function
const fixBytecode = function (bytecodeBuffer) {
if (!isBuffer(bytecodeBuffer)) {
throw new Error('bytecodeBuffer must be a buffer object.');
}
const dummyBytecode = compileCode('"ಠ_ಠ"');
const version = parseFloat(process.version.slice(1, 5));
if (process.version.startsWith('v8.8') || process.version.startsWith('v8.9')) {
// Node is v8.8.x or v8.9.x
if (DEBUG) console.error('[bytenode] fixBytecode branch: legacy v8.8/v8.9 (patch 16-20, 20-24)');
dummyBytecode.subarray(16, 20).copy(bytecodeBuffer, 16);
dummyBytecode.subarray(20, 24).copy(bytecodeBuffer, 20);
} else if (version >= 12) {
// Node >= 12 (modern V8 layout). Only the flag hash (offset 12) is patched.
//
// The previous upper bound (`version <= 23`) accidentally pushed Node 24+ into the
// legacy branch below, which ALSO overwrites bytes 16-20 (the read-only snapshot
// checksum) with the runtime dummy's value. On Electron 42 / V8 14.8 that masks a
// real checksum mismatch and makes V8 hard-abort (SIGTRAP) inside vm.Script instead
// of cleanly reporting cachedDataRejected. Patching only the flag hash is the safe,
// version-stable behavior shared with Node 12-23.
if (DEBUG) console.error('[bytenode] fixBytecode branch: modern node>=12 (patch flag hash 12-16 only)');
dummyBytecode.subarray(12, 16).copy(bytecodeBuffer, 12);
} else {
// Node 9/10/11 (older header layout).
if (DEBUG) console.error('[bytenode] fixBytecode branch: node 9/10/11 (patch 12-16, 16-20)');
dummyBytecode.subarray(12, 16).copy(bytecodeBuffer, 12);
dummyBytecode.subarray(16, 20).copy(bytecodeBuffer, 16);
}
};
// TODO: rewrite this function
const readSourceHash = function (bytecodeBuffer) {
if (!isBuffer(bytecodeBuffer)) {
throw new Error('bytecodeBuffer must be a buffer object.');
}
if (process.version.startsWith('v8.8') || process.version.startsWith('v8.9')) {
// Node is v8.8.x or v8.9.x
// eslint-disable-next-line no-return-assign
return bytecodeBuffer.subarray(12, 16).reduce((sum, number, power) => sum += number * Math.pow(256, power), 0);
} else {
// eslint-disable-next-line no-return-assign
return bytecodeBuffer.subarray(8, 12).reduce((sum, number, power) => sum += number * Math.pow(256, power), 0);
}
};
/**
* Runs v8 bytecode buffer and returns the result.
* @param {Buffer} bytecodeBuffer The buffer object that was created using compileCode function.
* @returns {any} The result of the very last statement executed in the script.
*/
const runBytecode = function (bytecodeBuffer) {
if (!isBuffer(bytecodeBuffer)) {
throw new Error('bytecodeBuffer must be a buffer object.');
}
const script = generateScript(bytecodeBuffer);
return script.runInThisContext();
};
/**
* Compiles JavaScript file to .jsc file.
* @param {object|string} args
* @param {string} args.filename The JavaScript source file that will be compiled
* @param {boolean} [args.compileAsModule=true] If true, the output will be a commonjs module
* @param {boolean} [args.compress=false] If true, compress the output bytecode
* @param {string} [args.output=filename.jsc] The output filename. Defaults to the same path and name of the original file, but with `.jsc` extension.
* @param {boolean} [args.electron=false] If true, compile code for Electron.
* @param {string} [args.electronPath] (optional) path to Electron executable. When present, the `electron` argument is ignored.
* @param {boolean|string} [args.createLoader=false] If true, create a CommonJS loader file. As a string, select between 'module' or 'commonjs' loader.
* @param {boolean} [args.loaderFilename='%.loader.js'] Filename or pattern for generated loader files. Defaults to originalFilename.loader.js. Use % as a substitute for originalFilename.
* @param {string} [output] The output filename. (Deprecated: use args.output instead)
* @returns {Promise<string>} A Promise which returns the compiled filename
*/
const compileFile = async function (args, output) {
let filename, compileAsModule, compress, electron, electronMain, createLoader, loaderFilename, electronPath;
if (typeof args === 'string') {
filename = args;
compileAsModule = true;
compress = false;
electron = false;
electronMain = false;
createLoader = false;
} else if (typeof args === 'object') {
filename = args.filename;
compileAsModule = args.compileAsModule !== false;
compress = args.compress;
electron = args.electron || !!args.electronPath;
// Compile inside an Electron main process (required for Electron >= 42 / V8 >= 14.8).
electronMain = args.electronMain;
electronPath = args.electronPath;
createLoader = args.createLoader;
loaderFilename = args.loaderFilename;
if (loaderFilename && !createLoader) createLoader = true;
}
if (typeof filename !== 'string') {
throw new Error(`filename must be a string. ${typeof filename} was given.`);
}
if (createLoader && typeof createLoader !== 'string') {
createLoader = 'commonjs';
}
// @ts-ignore
const compiledFilename = args.output || output || filename.slice(0, -path.extname(filename).length) + COMPILED_EXTNAME;
if (typeof compiledFilename !== 'string') {
throw new Error(`output must be a string. ${typeof compiledFilename} was given.`);
}
const javascriptCode = fs.readFileSync(filename, 'utf-8');
const sheBang = javascriptCode.match(sheBangRegex);
let code = javascriptCode.replace(sheBangRegex, '');
if (compileAsModule) {
code = Module.wrap(code);
}
let bytecodeBuffer;
if (electronMain) {
bytecodeBuffer = await compileElectronMainCode(code, { compress, electronPath });
} else if (electron) {
bytecodeBuffer = await compileElectronCode(code, { compress, electronPath });
} else {
bytecodeBuffer = compileCode(code, compress);
}
fs.writeFileSync(compiledFilename, bytecodeBuffer);
if (createLoader) {
addLoaderFile(compiledFilename, loaderFilename, createLoader, sheBang);
}
return compiledFilename;
};
/**
* Runs .jsc file and returns the result.
* @param {string} filename
* @returns {any} The result of the very last statement executed in the script.
*/
const runBytecodeFile = function (filename) {
if (typeof filename !== 'string') {
throw new Error(`filename must be a string. ${typeof filename} was given.`);
}
const bytecodeBuffer = fs.readFileSync(filename);
return runBytecode(bytecodeBuffer);
};
Module._extensions[COMPILED_EXTNAME] = function (fileModule, filename) {
const bytecodeBuffer = fs.readFileSync(filename);
const script = generateScript(bytecodeBuffer, filename);
/*
This part is based on:
https://github.com/zertosh/v8-compile-cache/blob/7182bd0e30ab6f6421365cee0a0c4a8679e9eb7c/v8-compile-cache.js#L158-L178
*/
function require (id) {
return fileModule.require(id);
}
require.resolve = function (request, options) {
// @ts-ignore
return Module._resolveFilename(request, fileModule, false, options);
};
if (process.main) {
require.main = process.main;
}
// @ts-ignore
require.extensions = Module._extensions;
// @ts-ignore
require.cache = Module._cache;
const compiledWrapper = script.runInThisContext({
filename: filename,
lineOffset: 0,
columnOffset: 0,
displayErrors: true
});
const dirname = path.dirname(filename);
const args = [
fileModule.exports, require, fileModule, filename, dirname, process, global
];
return compiledWrapper.apply(fileModule.exports, args);
};
/**
* Add a loader file for a given .jsc file
* @param {String} fileToLoad path of the .jsc file we're loading
* @param {String} loaderFilename - optional pattern or name of the file to write - defaults to filename.loader.js. Patterns: "%" represents the root name of .jsc file.
* @param {string} type select between 'module' or 'commonjs' loader.
*/
const addLoaderFile = function (fileToLoad, loaderFilename, type, sheBang) {
let loaderFilePath;
if (typeof loaderFilename === 'boolean' || loaderFilename === undefined || loaderFilename === '') {
loaderFilePath = fileToLoad.replace(COMPILED_EXTNAME, '.loader.js');
} else {
loaderFilename = loaderFilename.replace('%', path.parse(fileToLoad).name);
loaderFilePath = path.join(path.dirname(fileToLoad), loaderFilename);
}
const loaderCode = type === 'module' ? loaderCodeModule : loaderCodeCommonJS;
const relativePath = path.relative(path.dirname(loaderFilePath), fileToLoad);
const code = loaderCode('./' + relativePath, sheBang, loaderFilePath);
fs.writeFileSync(loaderFilePath, code);
};
const loaderCodeCommonJS = function (targetPath, sheBang) {
const lines = [
`require('bytenode')`,
``,
`module.exports = require('${targetPath}')`
];
if (sheBang) {
lines.unshift(sheBang, '');
}
return lines.join('\n');
};
const loaderCodeModule = function (targetPath, sheBang, loaderFilePath) {
const lines = [
`import { createRequire } from 'node:module'`,
``,
`import 'bytenode'`,
``,
``,
`const require = createRequire(import.meta.url)`,
``
];
if (sheBang) {
lines.unshift(sheBang, '');
lines.push(`require('${targetPath}')`);
} else {
// Only `require()` the module when not having a shebang, so we can be
// somewhat sure it's not an executable, to prevent running it. Also, an
// executable having exports is a code smell, they should be two different
// modules, using the executable the exported one as a library
let { default: defaultExport, ...namedExports } = require(loaderFilePath);
defaultExport = defaultExport ? 'default: defaultExport' : '';
namedExports = Object.keys(namedExports);
let exports = [];
if (defaultExport) {
exports.push(defaultExport);
}
exports = exports.concat(namedExports).join(', ');
if (!exports) {
lines.push(`require('${targetPath}')`);
}
else {
lines.push(`const {${exports}} = require('${targetPath}')`, ``, ``);
if (defaultExport) {
lines.push('export default defaultExport');
}
if (namedExports.length) {
lines.push(`export { ${namedExports} }`);
}
}
}
return lines.join('\n');
};
global.bytenode = {
compileCode,
compileFile,
compileElectronCode,
compileElectronMainCode,
runBytecode,
runBytecodeFile,
addLoaderFile,
loaderCode: loaderCodeCommonJS,
loaderCodeCommonJS,
loaderCodeModule
};
module.exports = global.bytenode;