to-esm
Version:
Tool to convert Commonjs files into ESM
1,634 lines (1,400 loc) • 171 kB
JavaScript
/**
* This file is to convert a Commonjs file into an ESM one.
*/
// ===========================================================================
// Imports
// ---------------------------------------------------------------------------
const path = require("path");
const fs = require("fs");
const process = require("process");
const glob = require("glob");
let crypto = require("crypto");
const {anaLogger} = require("analogger");
const UglifyJS = require("uglify-js");
const acorn = require("acorn");
const escodegen = require("escodegen");
const {hideText, restoreText, beforeReplace, resetAll} = require("before-replace");
const {stripStrings, stripComments, stripRegexes, clearStrings, parseString} = require("strip-comments-strings");
const {
resolvePath,
joinPath,
normalisePath,
generateTempName,
isArgsDir,
normaliseDirPath,
importLowerCaseOptions,
calculateCommon,
writeFileContent
} = require("@thimpat/libutils");
const {Readable} = require("stream");
const toAnsi = require("to-ansi");
const {findPackageEntryPoint} = require("find-entry-point");
const espree = require("espree");
const estraverse = require("estraverse");
const esbuild = require("esbuild");
const toEsmPackageJson = require("../package.json");
const REGEXES = {};
const SOME_REPLACER = "****";
// ===========================================================================
// Constants
// ---------------------------------------------------------------------------
// Value for parsable code
let commentMasks = {
COMMENT_MASK_START: "🥽👕🧥",
COMMENT_MASK_END : "🥾👑🩳",
};
let sourceExtractedComments = [];
let sourceExtractedStrings = [];
let sourceExtractedRegexes = [];
const blockMaskIn = "👉";
const blockMaskOut = "👈";
let strSheBang = "";
let dumpCounter = 0;
let DEBUG_MODE = (process.env.TO_ESM_DEBUG_MODE === "true") || false;
let eslint = null;
const TARGET = {
BROWSER: "browser",
ESM : "esm",
CJS : "cjs",
PACKAGE: "package",
ALL : "all"
};
const ESM_EXTENSION = ".mjs";
const CJS_EXTENSION = ".cjs";
const JSON_EXTENSION = ".json";
const COMMENT_MASK = "❖✎🔏❉";
const STRING_MASK_START = "❖✎❉";
const STRING_MASK_END = "❉✎❖";
const REGEX_MASK_START = "✋⛽⚒";
const REGEX_MASK_END = "⚒⛽✋";
const EOL = require("os").EOL;
const IMPORT_MASK_START = EOL + "/** to-esm: import-start **/" + EOL;
const IMPORT_MASK_END = EOL + "/** to-esm: import-end **/" + EOL;
const EXPORT_KEYWORD_MASK = "🦊";
const DEBUG_DIR = "./debug/";
const GENERATED_ROOT_FOLDER_NAME = "_root";
let indexGeneratedTempVariable = 1;
const DEFAULT_PREFIX_TEMP = ".tmp-toesm";
const ORIGIN_ADDING_TO_INDEX = {
START : "START",
RESOLVE_RELATIVE_IMPORT: "RESOLVE_RELATIVE_IMPORT",
RESOLVE_THIRD_PARTY : "RESOLVE_THIRD_PARTY",
RESOLVE_ABSOLUTE : "RESOLVE_ABSOLUTE"
};
/**
* module and exports can be redeclared in a block. They are not protected keywords.
* @type {[{search: RegExp, original: string, replace: string},{search: RegExp, original: string, replace: string}]}
*/
const AMBIGUOUS = [
{
search : /\bmodule\b/gm,
replace : "❖❖❖❖❖❖",
original: "module"
},
{
search : /\bexports\b/gm,
replace : "❉❉❉❉❉❉❉",
original: "exports"
}
];
const AMBIGUOUS_VAR_NAMES = ["module", "exports"];
const nativeModules = Object.keys(process.binding("natives"));
// ===========================================================================
// Globals
// ---------------------------------------------------------------------------
// The whole list of files to convert
let cjsList = [];
// ===========================================================================
// Static Locals
// ---------------------------------------------------------------------------
/**
* Register a module name
* If the module is already registered, returns false, otherwise add it to the list and returns true
* @note Add here modules that has been converted to ESM during a parsing
* @param moduleName
* @returns {boolean}
*/
const displayWarningOncePerModule = (function ()
{
let convertedModuleList = {};
return function (moduleName, message)
{
if (convertedModuleList[moduleName])
{
return false;
}
convertedModuleList[moduleName] = true;
console.warn({
lid : 1236,
color : "yellow",
symbol: "exclamation_mark"
}, message);
return true;
};
})();
// ===========================================================================
// Cores
// ---------------------------------------------------------------------------
const normaliseString = (content) =>
{
content = content.replace(/\r\n/gm, "\n").replace(/\n/gm, EOL);
return content;
};
const setupConsole = (Console = anaLogger) =>
{
try
{
Console.setOptions({silent: false, hideError: false, hideHookMessage: true, lidLenMax: 4});
Console.overrideConsole();
Console.overrideError();
Console.log({lid: 1012}, "Console is set up");
return Console;
}
catch (e)
{
console.error({lid: 3008}, e.message);
}
return null;
};
/**
* Build target directory.
* Ignore, if the directory already exist
* @param {string} targetDir Directory to build
* @test Some parts are ignored for the coverage (needs to simulate conditions
* linked to filesystem like root access or bad hard drive)
*/
const buildTargetDir = (targetDir) =>
{
try
{
if (fs.existsSync(targetDir))
{
return true;
}
fs.mkdirSync(targetDir, {recursive: true});
return true;
}
catch (e)
{
/* istanbul ignore next */
console.error({lid: 3010}, "", e.message);
}
/* istanbul ignore next */
return false;
};
const identifyExportedFunctionsThenTransform = function (converted, source, detectedExported = [])
{
try
{
}
catch (e)
{
console.error({lid: "TE6541"}, e.message);
}
return converted;
};
/**
* Execute some non-trivial transformations that require multiple passes
* @param {string} converted String to perform transformations onto
* @param source
* @param detectedExported
* @returns {*}
*/
const convertNonTrivialExportsWithAST = (converted, source, detectedExported = []) =>
{
let converted0, subst;
converted = identifyExportedFunctionsThenTransform(converted, source, detectedExported);
converted = hideKeyElementCode(converted, source);
for (let i = 0; i < detectedExported.length; ++i)
{
const item = detectedExported[i];
if (!item.funcname)
{
continue;
}
let isFunctionExportedAlready = false;
let isNameExportedAlready = false;
let nonMatchingExport = false;
const exportedAlreadyRegex = new RegExp(`export\\s+\\w+\\s+${item.namedExport}`, "gm");
if (exportedAlreadyRegex.test(converted))
{
isNameExportedAlready = true;
}
const functionAlreadyExported = new RegExp(`export\\s+\\w+\\s+${item.funcname}`, "gm");
if (functionAlreadyExported.test(converted))
{
isFunctionExportedAlready = true;
}
if (item.funcname && item.namedExport !== item.funcname)
{
nonMatchingExport = true;
}
// Both function and name export have already been exported. Likely a bug, we ignore the export
if (isFunctionExportedAlready && isNameExportedAlready)
{
continue;
}
if (isFunctionExportedAlready && !isNameExportedAlready)
{
// The function is already exported
if (nonMatchingExport)
{
const regexSentence =
// eslint-disable-next-line max-len
`(?:module\\.)?exports\\.\\b${item.namedExport}\\b\\s*=\\s*\\b${item.funcname}\\b\\s*;?`;
const regexp =
new RegExp(regexSentence, "gm");
subst = `export const ${item.namedExport} = ${item.funcname};`;
converted = converted.replace(regexp, subst);
continue;
}
}
const regexSentence =
// eslint-disable-next-line max-len
`(class|const|let|var|class|(?:\\basync \\s*)?function\\s*\\*?)\\s*\\b${item.funcname}\\b([\\S\\s]*?)(?:module\\.)?exports\\.\\b${item.namedExport}\\b\\s*=\\s*\\b${item.funcname}\\b\\s*;?`;
const regexp =
new RegExp(regexSentence, "gm");
if (isFunctionExportedAlready)
{
subst = `$1 ${item.namedExport} $2`;
}
else
{
subst = `export $1 ${item.namedExport} $2`;
}
if (item.funcname && item.namedExport !== item.funcname)
{
subst = subst + "\n" + `$1 ${item.funcname} = ${item.namedExport};`;
}
converted0 = converted;
converted = converted0.replace(regexp, subst);
dumpData(converted, source, `convertNonTrivialExportsWithAST in loop - ${i}`);
}
converted = restoreKeyElementCode(converted);
return converted;
};
/**
* Execute some non-trivial transformations that require multiple passes
* @param {string} converted String to perform transformations onto
* @param source
* @returns {*}
*/
const convertNonTrivial = (converted, source) =>
{
let converted0;
let regex = /((?<!export\s+)(?:const|let|var|class|function\s*\*?)\s+)(\w+)(\s+=.*\b(?:module\.)?exports\s*=\s*{[^}]*\2\b)/sgm;
let subst = "export $1$2$3";
converted0 = converted;
converted = converted0.replaceAll(regex, subst);
dumpData(converted, source, "convertNonTrivial - p1");
regex = /(?:const|let|var|class|function\s*\*?)\b\s+\b([\w]+)\b([\s\S]*)\1\s*=\s*require\(([^)]+.js[^)])\)/sgm;
subst = "import $1 from $3$2";
converted0 = converted;
converted = converted0.replaceAll(regex, subst);
dumpData(converted, source, "convertNonTrivial - p2");
return converted;
};
/**
* Check whether the given text has a valid JavaScript syntax
* @param str
* @param syntaxType
* @returns {boolean}
*/
const validateSyntax = (str, syntaxType = "commonjs") =>
{
try
{
str = str.replaceAll("\n", "");
espree.parse(
str, {
sourceType : syntaxType,
ecmaVersion : "latest",
allowReserved: false,
loc : false,
range : false,
tokens : false,
comment : false
}
);
return true;
}
/* istanbul ignore next */
catch (e)
{
console.log({lid: 1317}, e);
}
return false;
};
function expandRequireDeclarations(code) {
try {
const ast = acorn.parse(code, {ecmaVersion: "latest"});
const newBody = [];
let changed = false;
ast.body.forEach(node => {
if (node.type === "VariableDeclaration" && node.declarations.length > 1) {
const declarationsToExpand = [];
const standaloneDeclarations = [];
node.declarations.forEach(declarator => {
if (declarator.init && declarator.init.type === "CallExpression" && declarator.init.callee.type === "Identifier" && declarator.init.callee.name === "require") {
declarationsToExpand.push(declarator);
} else {
standaloneDeclarations.push(declarator);
}
});
if (declarationsToExpand.length > 0) {
changed = true;
declarationsToExpand.forEach(declarator => {
newBody.push({
type: "VariableDeclaration",
kind: node.kind,
declarations: [declarator]
});
});
standaloneDeclarations.forEach(declarator => {
newBody.push({
type: "VariableDeclaration",
kind: node.kind,
declarations: [declarator]
});
});
} else {
newBody.push(node);
}
} else {
newBody.push(node);
}
});
if (changed) {
ast.body = newBody;
return escodegen.generate(ast);
}
} catch (e) {
console.error({lid: 1317}, e.message);
}
return code;
}
/**
* May use this in the next version
* @next
* @returns {Promise<*>}
*/
async function initLinter() {
const { ESLint } = require("eslint");
const configPath = joinPath(__dirname, "../eslint-runtime.js");
if (!fs.existsSync(configPath)) {
console.log({lid: 3207}, "Missing dependency");
return code;
}
eslint = new ESLint({
overrideConfigFile: configPath,
fix: true,
});
return eslint;
}
/**
* May use this in the next version
* @param eslint
* @returns {Promise<*>}
*/
async function lintCode(eslint) {
try {
const results = await eslint.lintText(code);
if (results.length > 0 && results[0].output)
{
return results[0].output;
}
} catch (error) {
console.error({lid: "6541"}, `Error linting code: ${error}`);
return code; // Return original code on error
}
return code;
}
/**
* If source finishes with a "/", it's a folder,
* otherwise, it's not.
* @returns {boolean}
*/
const isConventionalFolder = (source) =>
{
if (!source)
{
return false;
}
return source.charAt(source.length - 1) === "/";
};
/**
* Returns the path of a relative path relative to source.
* @param source File that contains the require or import
* @param requiredPath Relative path inside the require or import
* @todo Change function name to more appropriate name
*/
const concatenatePaths = (source, requiredPath) =>
{
source = normalisePath(source);
let sourceDir = isArgsDir(source) ? source : path.parse(source).dir;
sourceDir = normaliseDirPath(sourceDir);
let pkgImportPath = joinPath(sourceDir, requiredPath);
return normalisePath(pkgImportPath);
};
/**
* Calculate the relative path from a source to another path.
* For instance, when doing a require() or import, the target
* path needs to be resolved for file1 to correctly require file2.
* ------> /some/path/to/file1
* ------> /some/other/path/to/file2
* Resolution on file1: require("../../other/path/to/file2")
* @param sourcePath
* @param targetPath
* @returns {string}
*/
const calculateRelativePath = (sourcePath, targetPath) =>
{
sourcePath = normalisePath(sourcePath);
targetPath = normalisePath(targetPath);
if (!isConventionalFolder(sourcePath))
{
sourcePath = path.parse(sourcePath).dir + "/";
}
const relativePath = path.relative(sourcePath, targetPath);
return normalisePath(relativePath);
};
/**
* Third-Party Module path starting with ./node_modules/ + relative path to the entry point
* @param moduleName
* @param targetDir
* @param target
* @returns {string|null}
*/
const getModuleEntryPointPath = (moduleName, targetDir = "", target = "") =>
{
try
{
let isCjs = target === TARGET.CJS;
let entryPoint = findPackageEntryPoint(moduleName, targetDir, {
isCjs,
isBrowser : target === TARGET.BROWSER,
useNativeResolve: false,
noAnsi : true
});
/* istanbul ignore next */
if (entryPoint === null)
{
console.error({lid: 3013}, ` Could not find entry point for module ${moduleName}.`);
return null;
}
entryPoint = normalisePath(entryPoint);
const nodeModulesPos = entryPoint.indexOf("node_modules");
/* istanbul ignore next */
if (nodeModulesPos === -1)
{
console.error({lid: 3015}, ` The module [${moduleName}] is located in a non-node_modules directory.`);
}
entryPoint = "./" + entryPoint.substring(nodeModulesPos);
return entryPoint;
}
catch (e)
{
/* istanbul ignore next */
console.info({lid: 1147}, ` Checking [${moduleName}] package.json`, e.message);
}
/* istanbul ignore next */
return null;
};
const getCJSModuleEntryPath = (moduleName, targetDir = "") =>
{
return getModuleEntryPointPath(moduleName, targetDir, TARGET.CJS);
};
const getESMModuleEntryPath = (moduleName, targetDir, target) =>
{
return getModuleEntryPointPath(moduleName, targetDir, target);
};
// ---------------------------------------------------
// NEW STUFF
// ---------------------------------------------------
const dumpData = (converted, source, title = "") =>
{
try
{
if (!DEBUG_MODE)
{
return;
}
++dumpCounter;
const name = path.parse(source).name;
if (title)
{
title = "-" + title;
}
const indexCounter = dumpCounter.toString().padStart(4, "0");
fs.writeFileSync(joinPath(DEBUG_DIR, `dump-${indexCounter}-${name}-${title}.js`), converted, "utf-8");
}
catch (e)
{
/* istanbul ignore next */
console.error({lid: 3014}, e.message);
}
};
/**
* Convert path like:
* C:/a/b/c/d => /a/b/c/d
* So, they can be created as subdirectory
* @param wholePath
* @returns {*|string[]}
*/
const convertToSubRootDir = (wholePath) =>
{
wholePath = normalisePath(wholePath);
const arr = wholePath.split("/");
arr.shift();
return arr.join("/");
};
/**
* Remove part of path by subtracting a given directory from a whole path
* TODO: Re-Check this function goal
* TODO: Use path.relative and swap parameters
* TODO: Try to not use this function at all. Remove it as soon as possible
* @param wholePath Full File Path
* @param pathToSubtract Subdirectory to remove from path
* @returns {*}
*/
const subtractPath = (wholePath, pathToSubtract) =>
{
let subPath, subDir;
// Get mapped path by subtracting rootDir
wholePath = normalisePath(wholePath);
pathToSubtract = normalisePath(pathToSubtract);
if (wholePath.length < pathToSubtract.length)
{
console.error({lid: 3016}, "" + "Path subtraction will not work here. " +
"The subtracting path is bigger than the whole path");
return {
subPath: wholePath
};
}
if (pathToSubtract === "./")
{
subPath = convertToSubRootDir(wholePath);
subDir = path.parse(subPath).dir;
subDir = normaliseDirPath(subDir);
return {
subDir, subPath
};
}
else if (wholePath.indexOf(pathToSubtract) === -1)
{
console.error({lid: 3018}, "" + "Path subtraction will not work here. " +
"The subtracting path is not part of the whole path");
return {
subPath: wholePath
};
}
if (pathToSubtract.charAt(pathToSubtract.length - 1) !== "/")
{
pathToSubtract = pathToSubtract + "/";
}
let subPaths = wholePath.split(pathToSubtract);
subPath = subPaths[1];
subPath = normalisePath(subPath);
subDir = path.parse(subPath).dir;
subDir = normaliseDirPath(subDir);
return {
subDir, subPath
};
};
/**
* Look up for a path in the glob list
* @param requiredPath
* @param list
* @returns {{}|*}
*/
const getTranslatedPath = (requiredPath, list) =>
{
requiredPath = normalisePath(requiredPath);
for (let i = 0; i < list.length; ++i)
{
const item = list[i];
const source = normalisePath(item.source);
if (requiredPath === source)
{
return item;
}
}
return {};
};
/**
* Calculate for a given source (usually a .cjs), its supposed destination path after a conversion
* i.e.
* ./my/cjs/path/item.cjs => ./output/item.mjs
* @param source
* @param rootDir
* @param outputDir
* @returns {{}|{projectedPath: (*), subDir: *, projectedDir: (*), subPath: *, sourcePath: (*)}}
*/
const getProjectedPathAll = ({source, outputDir, subRootDir = ""} = {}) =>
{
try
{
if (subRootDir && source.indexOf(subRootDir) === 0)
{
source = source.substring(subRootDir.length);
// source = joinPath(outputDir, source);
}
let projectedPath = joinPath(outputDir, source);
projectedPath = normalisePath(projectedPath);
return {
projectedPath,
};
}
catch (e)
{
console.error({lid: 3020}, "", e.message);
}
return {};
};
/**
* Change the given path extension to .mjs
* @param filepath
* @param extension
* @returns {string}
*/
const changePathExtensionToESM = (filepath, {esmExtension = ESM_EXTENSION} = {}) =>
{
const parsed = path.parse(filepath);
const renamed = joinPath(parsed.dir, parsed.name + esmExtension);
return normalisePath(renamed);
};
/**
* @todo Investigate
* @param match
* @returns {*}
*/
const reviewEntryImportMaps = (match/*, requestedRequired, moreOptions*/) =>
{
try
{
// projectedRequiredPath = moduleName;
// if (requiredPath.indexOf("node_modules") > -1)
// {
// requiredPath = "./node_modules" + requiredPath.split("node_modules")[1];
// }
// importMaps[moduleName] = requiredPath;
// match = `from "${moduleName}"`;
}
catch (e)
{
}
return match;
};
/**
*
* @obsolete
* @param {string} sourcePath Path to the file that does the require/import
* @param {string} requiredPath Original required path
* @param list
* @param outputDir
* @param esmExtension
* @returns {string}
*/
const calculateRequiredPath = (
{
sourcePath, requiredPath, list, outputDir, esmExtension = ESM_EXTENSION
} = {}) =>
{
let projectedRequiredPath;
// Projected path of required path
const requiredPathProperties = getTranslatedPath(requiredPath, list);
const target = requiredPathProperties.target;
if (target)
{
// The relative path of the two projected paths above (projectedPath + target)
projectedRequiredPath = calculateRelativePath(sourcePath, target);
}
else
{
const newPath = concatenatePaths(outputDir, requiredPath);
projectedRequiredPath = calculateRelativePath(sourcePath, newPath);
projectedRequiredPath = changePathExtensionToESM(projectedRequiredPath, {esmExtension});
}
return projectedRequiredPath;
};
/**
* Determine if the required third party required needs to be resolved
* and calculate its value
* @param text
* @param list
* @param {string} source Relative path to the source file being parsed
* @param rootDir
* @param workingDir
* @param regexRequiredPath
* @param nonHybridModuleMap
* @param importMaps
* @param moreOptions
* @returns {string|*}
*/
const resolveThirdParty = (text, list, {
source,
subRootDir,
workingDir,
regexRequiredPath,
nonHybridModuleMap,
importMaps,
moreOptions
}) =>
{
try
{
const outputDir = moreOptions.outputDir;
let moduleName = regexRequiredPath;
if (nonHybridModuleMap[moduleName])
{
regexRequiredPath = moduleName = nonHybridModuleMap[moduleName];
}
let requiredPath;
if (moreOptions.extras.target === TARGET.BROWSER || moreOptions.extras.target === TARGET.ESM)
{
requiredPath = getESMModuleEntryPath(moduleName, workingDir, moreOptions.extras.target);
if (!requiredPath)
{
console.warn({
lid : 1099,
color: "#FF0000"
}, ` The module [${moduleName}] for [target: ${moreOptions.extras.target}] was not found in your node_modules directory. `
+ "Skipping.");
return regexRequiredPath;
}
let isESM = isESMCompatible(requiredPath);
if (isESM)
{
// When the "require" is for Node (ESM)
// we return the original library name
if (moreOptions.extras.target === TARGET.ESM)
{
return regexRequiredPath;
}
// When the "require" is for browser,
// we need to solve the relative path to the browser script entry point
else if (moreOptions.extras.target === TARGET.BROWSER)
{
if (isBrowserCompatible(requiredPath))
{
// Calculate project path for this source
let {projectedPath} = getProjectedPathAll({outputDir, source, subRootDir});
// Extract absolute path for error checking
const absoluteProjectedPath = joinPath(workingDir, projectedPath);
const absoluteRequiredPath = joinPath(workingDir, outputDir, requiredPath);
// Render relative path
let relativePath = calculateRelativePath(absoluteProjectedPath, absoluteRequiredPath);
importMaps[moduleName] = requiredPath;
if (moreOptions.extras.useImportMaps)
{
return regexRequiredPath;
}
if (moreOptions.extras.prefixpath)
{
relativePath = joinPath(moreOptions.extras.prefixpath, relativePath);
relativePath = normalisePath(relativePath);
}
// When the target is the browser, any third party modules linked to the processed file
// need to be generated again.
// The reason being is that there is no centralized modules repositories like in Node
// (node_modules) in a browser environment, therefore no matter what we do, if our original
// file
// linked itself to a third party module, this third party module won't exist in the browser.
// It needs to be imported.
// Now, if we consider module bundlers, they tend to mitigate this issue as they have access
// to the whole project (depending on your set-up), so they can import any dependencies just
// once. Now, Google has introduced something called importmaps. Imagine if they ever extend
// the idea in making the system working like a whole centralized directory like in Node. We
// could import automatically without bundling any third party library. Let's take an example:
// import lodash from lodash-min With import map, the browser would know automatically where to
// download the library. - Increasing security because they can monitor any eventual defect -
// Increasing speed because it's easy to cache by the browser as no more hash id would be abuse
// like it's often the case after bundling - Increasing reactivity, as anything broken would be
// detected straight away - And much much more...
// ----------------------------------------------- Let's see what happen in the future. //
// -------------------------------------------- TODO: Move this comment elsewhere. God bless!
const entry = addFileToIndex({
source : requiredPath,
rootDir : workingDir,
outputDir,
workingDir,
notOnDisk: false,
referrer : source,
origin : ORIGIN_ADDING_TO_INDEX.RESOLVE_THIRD_PARTY,
moduleName,
moreOptions
});
const relativeTargetPath = joinPath(outputDir, entry.mjsTarget);
const absoluteTargetPath = joinPath(workingDir, relativeTargetPath);
relativePath = calculateRelativePath(absoluteProjectedPath, absoluteTargetPath);
return relativePath;
}
console.warn({
lid : 1101,
color: "yellow"
}, ` The file [${requiredPath}] is not browser compatible. The system will try to generate one`);
// If not, start conversion from the .cjs
requiredPath = getCJSModuleEntryPath(moduleName, workingDir);
}
if (!moreOptions.firstPass)
{
displayWarningOncePerModule(
{lid: 2238},
`The npm module '${moduleName}' is ESM compatible, but the target is set to ${moreOptions.extras.target}.` +
`(The system will try to generate a new one if possible)`);
}
}
}
if (!moreOptions.firstPass)
{
if (moreOptions.extras?.skipEsmResolution)
{
displayWarningOncePerModule(moduleName, `The npm module '${moduleName}' does not seem to be ESM compatible.`);
return moduleName;
}
displayWarningOncePerModule(moduleName, `The npm module '${moduleName}' does not seem to be ESM compatible. (The system will try to generate a new one)`);
}
// Need conversion from .cjs because module is incompatible with ESM
requiredPath = getCJSModuleEntryPath(moduleName, workingDir);
let projectedRequiredPath = resolveReqPath(source, requiredPath, {
outputDir,
subRootDir: moreOptions.subRootDir
});
projectedRequiredPath = changePathExtensionToESM(projectedRequiredPath, {esmExtension: moreOptions.extension});
importMaps[moduleName] = requiredPath;
const entry = addFileToIndex({
source : requiredPath,
rootDir : workingDir,
outputDir,
workingDir,
notOnDisk : moreOptions.extras.useImportMaps,
referrer : source,
origin : ORIGIN_ADDING_TO_INDEX.RESOLVE_THIRD_PARTY,
moduleName,
moreOptions,
subRootDir: moreOptions.subRootDir
});
// When importMaps is enabled, we return the original require
// Resolvers will be set in the HTML code
if (moreOptions.extras.useImportMaps)
{
regexRequiredPath = reviewEntryImportMaps(regexRequiredPath, projectedRequiredPath, moreOptions);
return regexRequiredPath;
}
let {projectedPath} = getProjectedPathAll({outputDir, source, subRootDir: moreOptions.subRootDir});
const relativePath = calculateRelativePath(projectedPath, entry.mjsTargetAbs);
return relativePath;
// return projectedRequiredPath;
}
catch (e)
{
console.error({lid: 3022}, "", e.message);
}
return regexRequiredPath;
};
const resolveReqPath = function (source, regexRequiredPath,
{outputDir, subRootDir})
{
try
{
let {projectedPath} = getProjectedPathAll({outputDir, source, subRootDir});
// const relativePath = calculateRelativePath(projectedPath, entry.mjsTargetAbs);
// Absolute path to source
// let sourcePathAbs = joinPath(moreOptions.outputDir, source);
// Absolute path to required
let mjsPathAbs = joinPath(outputDir, regexRequiredPath);
// Distance
regexRequiredPath = calculateRelativePath(projectedPath, mjsPathAbs);
return regexRequiredPath;
}
catch (e)
{
console.error({lid: 3024}, e.message);
}
return regexRequiredPath;
};
const isResolveAbsoluteMode = function (moreOptions)
{
return !!moreOptions?.extras?.resolveAbsolute?.length;
};
const getLookUpDirs = function (moreOptions)
{
return moreOptions?.extras?.resolveAbsolute;
};
const isExternalSource = function (moreOptions)
{
// Don't create a copy of the referenced file
return !!moreOptions.extras?.keepExternal;
};
/**
* Re-evaluate a require new path relative to the source is in
* @param text
* @param list
* @param source
* @param rootDir
* @param regexRequiredPath
* @param moreOptions
* @param workingDir
* @param outputDir
* @param subRootDir
* @param origin
* @returns {string}
*/
const resolveRelativeImport = (text, list, {
source,
rootDir,
regexRequiredPath,
moreOptions,
workingDir,
subRootDir,
origin = ""
}) =>
{
// Source path of projected original source (the .cjs)
try
{
let controlSourceAbs;
if (origin === ORIGIN_ADDING_TO_INDEX.RESOLVE_THIRD_PARTY)
{
controlSourceAbs = joinPath(workingDir, source);
rootDir = workingDir;
}
else if (origin === ORIGIN_ADDING_TO_INDEX.RESOLVE_RELATIVE_IMPORT)
{
controlSourceAbs = joinPath(rootDir, source);
}
else if (origin === ORIGIN_ADDING_TO_INDEX.RESOLVE_ABSOLUTE)
{
controlSourceAbs = joinPath(source);
}
else
{
controlSourceAbs = joinPath(rootDir, source);
}
if (!fs.existsSync(controlSourceAbs))
{
console.error({lid: 5581}, `Source not found: ${controlSourceAbs}`);
}
let requiredPath = concatenatePaths(source, regexRequiredPath);
addFileToIndex({
source : requiredPath,
rootDir,
referrer : source,
origin : origin || ORIGIN_ADDING_TO_INDEX.RESOLVE_RELATIVE_IMPORT,
workingDir,
outputDir: moreOptions.outputDir,
subRootDir,
moreOptions
});
}
catch (e)
{
console.error({lid: 3026}, "", e.message);
}
regexRequiredPath = changePathExtensionToESM(regexRequiredPath, {esmExtension: moreOptions.esmExtension});
return regexRequiredPath;
};
/**
* Parse the absolute given paths
* @param {string} text Source content (likely already modified in the pipeline)
* @param {CjsInfoType[]} list File list already parsed
* @param {string} source Source file that contains the absolute required path to translate
* @param rootDir
* @param {string} outputDir Folder for the target file
* @param {string} workingDir
* @param {string} regexRequiredPath Absolute required path
* @param {*} moreOptions
* @param subRootDir
* @returns {string}
*/
const resolveAbsoluteImport = (text, list, {
source,
rootDir,
outputDir,
workingDir,
regexRequiredPath,
moreOptions,
subRootDir
}) =>
{
// Source path of projected original source (the .cjs)
try
{
const lookupDirLists = getLookUpDirs(moreOptions);
let relativeRequiredPath, idRequiredPath;
relativeRequiredPath = regexRequiredPath;
let isAbsolutePath = true;
if (lookupDirLists)
{
const props = getRelativePathsAgainstSuggestedRoots({
regexRequiredPath,
source,
rootDir,
lookupDirLists,
outputDir
});
if (props)
{
relativeRequiredPath = props.relativeRequiredPath;
idRequiredPath = props.idRequiredPath;
isAbsolutePath = false;
}
}
// The required path from the source path above
const entry = addFileToIndex({
source : relativeRequiredPath,
rootDir,
outputDir,
workingDir,
referrer : source,
isAbsolutePath,
moreOptions,
origin : ORIGIN_ADDING_TO_INDEX.RESOLVE_ABSOLUTE,
externalSource: true,
subRootDir,
});
if (isExternalSource(moreOptions))
{
regexRequiredPath = idRequiredPath;
}
else
{
regexRequiredPath = resolveReqPath(source, entry.mjsTarget, {outputDir, subRootDir});
}
return regexRequiredPath;
}
catch (e)
{
console.error({lid: 3028}, "", e.message);
}
regexRequiredPath = changePathExtensionToESM(regexRequiredPath, {esmExtension: moreOptions.extension});
return regexRequiredPath;
};
/**
* Parse imported for ESM
* @param text
* @param list
* @param fileProp
* @returns {*}
*/
const reviewEsmImports = (text, list, {
source,
rootDir,
outputDir,
importMaps,
nonHybridModuleMap,
workingDir,
moreOptions,
origin
}) =>
{
// Locate third party
// const re = /\bfrom\s+["']([^.\/~@][^"']+)["'];?/gmu;
const re = /\bfrom\s+["']([^"']+?)["'];?/gmu;
const sourceExtractedComments = [];
text = stripCodeComments(text, sourceExtractedComments, commentMasks);
const sourceExtractedRegexes = [];
text = stripCodeRegexes(text, sourceExtractedRegexes);
const subRootDir = moreOptions.subRootDir || "";
text = text.replace(re, function (match, regexRequiredPath)
{
try
{
if (~nativeModules.indexOf(regexRequiredPath))
{
if (moreOptions.extras.target === TARGET.BROWSER)
{
console.info({lid: 1017}, ` ${regexRequiredPath} is a built-in NodeJs module.`);
}
return match;
}
if (regexRequiredPath.startsWith("./") || regexRequiredPath.startsWith(".."))
{
const solvedRelativeRequire = resolveRelativeImport(text, list, {
source,
rootDir,
outputDir,
workingDir,
moreOptions,
regexRequiredPath,
subRootDir,
origin
});
match = match.replace(regexRequiredPath, solvedRelativeRequire);
}
else if (regexRequiredPath.startsWith("/"))
{
const solvedAbsoluteRequire = resolveAbsoluteImport(text, list, {
source,
rootDir,
outputDir,
workingDir,
regexRequiredPath,
moreOptions,
subRootDir,
origin
});
match = match.replace(regexRequiredPath, solvedAbsoluteRequire);
}
else // Third party libraries
{
let solvedPath = resolveThirdParty(text, list, {
source,
rootDir,
outputDir,
workingDir,
importMaps,
nonHybridModuleMap,
regexRequiredPath,
moreOptions,
match,
subRootDir,
origin
});
match = match.replace(regexRequiredPath, solvedPath);
}
/* istanbul ignore next */
return match;
}
catch (e)
{
/* istanbul ignore next */
console.error({lid: 3030}, "", e.message);
}
});
text = putBackComments(text, sourceExtractedComments, commentMasks);
text = putBackRegexes(text, sourceExtractedRegexes);
return text;
};
/**
* Parse the string within "requires" to evaluate given paths
* @param text
* @param list
* @param fileProp
* @param workingDir
* @param esmExtension
* @returns {*}
*/
const parseImportWithRegex = (text, list, fileProp, {workingDir, esmExtension = ESM_EXTENSION} = {}) =>
{
const parsedFilePath = joinPath(workingDir, fileProp.source);
const parsedFileDir = path.dirname(parsedFilePath);
const re = /require\(["'`]([.\/][^)]+)["'`]\)/gmu;
return text.replace(re, function (match, group)
{
const target = joinPath(parsedFileDir, group);
const extension = path.extname(target);
const targets = [];
if (!extension)
{
targets.push(target + ".cjs");
targets.push(target + ".js");
}
else if (![".js", ".cjs"].includes(extension))
{
/* istanbul ignore next */
return match;
}
else
{
targets.push(target);
}
const index = list.findIndex(function ({source})
{
const possibleFilePath = joinPath(workingDir, source);
return (targets.includes(possibleFilePath));
});
if (index < 0)
{
return match;
}
// current file's absolute path
const sourcePath = resolvePath(fileProp.outputDir);
const {source, outputDir} = list[index];
const basename = path.parse(source).name;
// Absolute path in the "require"
const destinationPath = resolvePath(outputDir);
let relativePath = path.relative(sourcePath, destinationPath);
relativePath = joinPath(relativePath, basename + esmExtension);
relativePath = relativePath.replace(/\\/g, "/");
if (!([".", "/"].includes(relativePath.charAt(0))))
{
relativePath = "./" + relativePath;
}
return match.replace(group, relativePath);
});
};
/**
* Rename variables declared as exports
* @example
* let exports = ...
* @param converted
* @param ambiguousList
* @returns {*}
*/
const convertAmbiguous = (converted, ambiguousList) =>
{
const n = ambiguousList.length;
let extract = "";
for (let i = n - 1; i >= 0; --i)
{
let ambiguous = ambiguousList[i];
let block = ambiguous.block;
let start = block.start;
let end = block.end;
extract = converted.substring(start, end);
for (let ii = 0; ii < AMBIGUOUS.length; ++ii)
{
let ambiguousWord = AMBIGUOUS[ii];
let search = ambiguousWord.search;
let replace = ambiguousWord.replace;
extract = extract.replace(search, replace);
}
converted = converted.substring(0, start) + extract + converted.substring(end);
}
return converted;
};
/**
* Put back original naming for ambiguous declarations
* like
* let exports = ...
* @param converted
* @returns {*}
*/
const putBackAmbiguous = (converted) =>
{
const n = AMBIGUOUS.length;
for (let i = 0; i < n; ++i)
{
let ambiguousWord = AMBIGUOUS[i];
let search = ambiguousWord.replace;
let replace = ambiguousWord.original;
converted = converted.replaceAll(search, replace);
}
return converted;
};
const removeShebang = (converted) =>
{
const firstLine = converted.split("\n")[0];
if (/^(?:\/\/ *)?#!.+/.test(firstLine))
{
strSheBang = firstLine.trim();
converted = converted.substring(strSheBang.length).trim();
}
return converted;
};
const restoreShebang = (converted) =>
{
if (strSheBang)
{
converted = strSheBang + EOL + converted;
}
strSheBang = "";
return converted;
};
const removeDuplicateExports = function (converted)
{
try
{
let oldArray = converted.split("\n");
let newArray = [];
oldArray.reverse().forEach(line =>
{
if (/\bexports?\b\./.test(line))
{
if (newArray.includes(line))
{
return false;
}
}
newArray.unshift(line);
return true;
});
converted = newArray.join("\n");
}
catch (e)
{
console.error({lid: 6551}, e.message);
}
return converted;
};
/**
* Will not work if a variable is named "exports"
* @param converted
* @param source
* @returns {*}
*/
const convertModuleExportsToExport = (converted, source) =>
{
converted = hideKeyElementCode(converted, source);
// Convert exports = module.exports = ... to module.exports =
converted = converted.replace(/\bexports\b\s*=\s*module.exports\s*=/, "module.exports =");
// Convert module.exports = exports = ... to module.exports =
converted = converted.replace(/\bmodule\.exports\b\s*=\s*exports\s*=/, "module.exports =");
converted = converted.replace(
/\b(const|let|var|class|function\s*\*)\s+\b(\w+)\b([\s\S]*?)(\bmodule\b\.)?\bexports\b\.\2\s*=\s*\2.*/gm,
"export $1 $2 $3");
let converted0;
do
{
converted0 = converted;
// Convert module.exports.something ... function something
// to import something
converted = converted.replaceAll(
/\b(?:\bmodule\b\.)?\bexports\b\.([\w]+)\s*=\s*\1.*([\s\S]*)(\bfunction\s*\*?\1)/sgm,
"$2 export $3");
}
while (converted !== converted0);
// Convert module.exports to export default
converted = converted.replace(/(^\s*)(?:\bmodule\b\.)?\bexports\b\s*=/gm, "$1export default");
// Convert module.exports.default to export default
converted = converted.replace(/(^\s*)(?:\bmodule\b\.)?\bexports\b\.default\s*=/gm, "$1export default");
// Convert module.exports.something to export something
converted = converted.replace(/(^\s*)(?:\bmodule\b\.)?\bexports\b\.([\w]+)\s*=/gm, "$1export const $2 =");
const arr = converted.split("export default");
const defaultExportNumber = arr.length - 1;
if (defaultExportNumber > 1)
{
const twiceExported = arr[1].trim().split(/\W/)[0];
console.log({
lid : 1016,
color: "yellow"
}, `${defaultExportNumber} default exports detected => \`export default ${twiceExported}\` `);
console.log({lid: 1018, color: "yellow"}, "Assure that you have only one export (or none) of type " +
`"module.exports = ..."` +
" and use named export if possible => i.e. \"module.exports.myValue = ...\"");
}
converted = restoreKeyElementCode(converted);
return converted;
};
const convertJsonImportToVars = (converted, {
source,
outputDir,
}) =>
{
const matchData = converted.matchAll(/(?:const|let|var|class|function\s*\*?)\s+([^=]+)\s*=\s*require\s*\(\s*['"`]([^)]+.json)[^)]+\)/g);
const matches = [...matchData];
const found = Array.from(matches, m => m[0]);
const identifiers = Array.from(matches, m => m[1]);
const files = Array.from(matches, m => m[2]);
const n = identifiers.length;
for (let i = 0; i < n; ++i)
{
try {
const filepath = files[i];
let absPath = resolvePath(source);
// Path to Json file to convert and translate
let srcJsonPath = concatenatePaths(absPath, filepath);
if (!fs.existsSync(srcJsonPath))
{
console.log({lid: "9801"}, `The file ${srcJsonPath} could not be found. Skipping...`);
continue;
}
// Read and validate the json file
const jsonContent = fs.readFileSync(srcJsonPath, "utf-8");
const json = JSON.parse(jsonContent.toString());
// Convert the json file to ESM
const esmifyJson = convertJsonToESM(json);
const srcPath = matches[i][2];
const esmifyJsonPath = joinPath(outputDir, srcPath + ESM_EXTENSION);
writeFileContent(esmifyJsonPath, esmifyJson, {encoding: