ui5-tooling-modules
Version:
UI5 CLI extensions to load and convert node modules as UI5 AMD-like modules
1,272 lines (1,198 loc) • 77.8 kB
JavaScript
/* eslint-disable no-unused-vars */
const path = require("path");
const { readFileSync, statSync, readdirSync, existsSync, realpathSync } = require("fs");
const { readFile, stat, writeFile, mkdir } = require("fs").promises;
const rollup = require("rollup");
const instructions = require("./rollup-plugin-instructions");
const commonjs = require("@rollup/plugin-commonjs");
const json = require("@rollup/plugin-json");
const nodePolyfills = require("rollup-plugin-polyfill-node");
const nodePolyfillsOverride = require("./rollup-plugin-polyfill-node-override");
const fetchShim = require("./rollup-plugin-fetch-shim");
const skipAssets = require("./rollup-plugin-skip-assets");
const injectESModule = require("./rollup-plugin-inject-esmodule");
const logger = require("./rollup-plugin-logger");
const resolveModulePlugin = require("./rollup-plugin-resolve-module");
const dynamicImports = require("./rollup-plugin-dynamic-imports");
const replace = require("@rollup/plugin-replace");
const transformTopLevelThis = require("./rollup-plugin-transform-top-level-this");
const webcomponents = require("./rollup-plugin-webcomponents");
const importMeta = require("./rollup-plugin-import-meta");
const walk = require("ignore-walk");
const { minimatch } = require("minimatch");
const { XMLParser } = require("fast-xml-parser");
const parseJS = require("./utils/parseJS");
const { createHash } = require("crypto");
const sanitize = require("sanitize-filename");
// ============================================================================
// Cache invalidation helpers — sha256 layered fingerprint (Option A)
// See packages/ui5-tooling-modules/CACHE-INVALIDATION.md for the design.
// ============================================================================
const TOOL_LIB_DIR = __dirname;
const LOCKFILE_NAMES = ["pnpm-lock.yaml", "package-lock.json", "yarn.lock", "npm-shrinkwrap.json"];
const TOOL_SOURCE_EXTS = new Set([".js", ".mjs", ".cjs", ".hbs"]);
const TOOL_SKIP_DIRS = new Set(["node_modules", ".git", "dist", "coverage"]);
/**
* Recursively walks a directory, collecting files matching the given extensions.
* @param {string} dir directory to walk
* @param {string[]} [out] accumulator array (used by recursion)
* @returns {string[]} absolute file paths whose extension is in TOOL_SOURCE_EXTS
*/
function walkDirSync(dir, out = []) {
for (const dirent of readdirSync(dir, { withFileTypes: true })) {
if (dirent.isDirectory()) {
if (!TOOL_SKIP_DIRS.has(dirent.name)) {
walkDirSync(path.join(dir, dirent.name), out);
}
} else if (dirent.isFile() && TOOL_SOURCE_EXTS.has(path.extname(dirent.name))) {
out.push(path.join(dir, dirent.name));
}
}
return out;
}
/**
* Stable JSON stringification so config objects produce identical fingerprints
* regardless of key insertion order.
* @param {string|number|boolean|null|object|Array} obj value to serialize
* @returns {string} deterministic JSON string
*/
function stableStringify(obj) {
if (obj === null || typeof obj !== "object") return JSON.stringify(obj);
if (Array.isArray(obj)) return `[${obj.map(stableStringify).join(",")}]`;
const keys = Object.keys(obj).sort();
return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`).join(",")}}`;
}
let _toolFingerprint;
/**
* Fingerprint of the tooling itself: package version + content of all source
* files under lib/ (js/mjs/cjs/hbs). Memoized for the lifetime of the process,
* so the I/O is paid once.
* @returns {string} sha256 hex digest of the tool sources
*/
function getToolFingerprint() {
if (_toolFingerprint) return _toolFingerprint;
const pkg = require("../package.json");
const files = walkDirSync(TOOL_LIB_DIR).sort();
const h = createHash("sha256");
h.update(`${pkg.name}@${pkg.version}\n`);
for (const f of files) {
h.update(`${path.relative(TOOL_LIB_DIR, f)}\0`);
h.update(readFileSync(f));
h.update("\0");
}
return (_toolFingerprint = h.digest("hex"));
}
/**
* Fingerprint of every option that affects bundle output.
* @param {object} opts the configuration options used by getBundleInfo
* @returns {string} sha256 hex digest of the stable JSON of `opts`
*/
function getConfigFingerprint(opts) {
const relevant = {
pluginOptions: opts.pluginOptions ?? null,
generatedCode: opts.generatedCode ?? null,
minify: !!opts.minify,
inject: opts.inject ?? null,
sourcemap: !!opts.sourcemap,
keepDynamicImports: opts.keepDynamicImports ?? null,
skipTransform: opts.skipTransform ?? null,
dynamicEntriesPath: opts.dynamicEntriesPath ?? null,
};
return createHash("sha256").update(stableStringify(relevant)).digest("hex");
}
const _lockfileFingerprintCache = new Map();
/**
* Fingerprint of the project's package.json + lockfile, so dependency-version
* bumps invalidate the cache. Walks up from `cwd` to find the nearest lockfile.
* Memoized per (lockfile, package.json) mtime.
* @param {string} cwd current working directory to start the lockfile search from
* @returns {string} sha256 hex digest of the lockfile + package.json contents
*/
function getLockfileFingerprint(cwd) {
let lockfile;
let dir = cwd;
for (;;) {
for (const name of LOCKFILE_NAMES) {
const candidate = path.join(dir, name);
if (existsSync(candidate)) {
lockfile = candidate;
break;
}
}
if (lockfile) break;
const parent = path.dirname(dir);
if (parent === dir) break;
dir = parent;
}
const pkgJson = lockfile ? path.join(path.dirname(lockfile), "package.json") : path.join(cwd, "package.json");
const pkgJsonExists = existsSync(pkgJson);
const lockMtime = lockfile ? statSync(lockfile).mtimeMs : 0;
const pkgMtime = pkgJsonExists ? statSync(pkgJson).mtimeMs : 0;
const cacheKey = `${lockfile || ""}@${lockMtime}:${pkgJson}@${pkgMtime}`;
if (_lockfileFingerprintCache.has(cacheKey)) {
return _lockfileFingerprintCache.get(cacheKey);
}
const h = createHash("sha256");
if (lockfile) {
h.update(`${path.basename(lockfile)}\0`);
h.update(readFileSync(lockfile));
h.update("\0");
}
if (pkgJsonExists) {
h.update("package.json\0");
h.update(readFileSync(pkgJson));
h.update("\0");
}
const fp = h.digest("hex");
_lockfileFingerprintCache.set(cacheKey, fp);
return fp;
}
/**
* Fingerprint of a set of input file paths, using path + size + mtime. Used
* for both the cheap pre-bundle check (entry modules) and the full transitive
* graph validation step (recorded `relatedPaths` of a persisted BundleInfo).
* @param {string[]} paths absolute file paths to fingerprint
* @returns {string} sha256 hex digest of the path/size/mtime triples
*/
function getInputGraphFingerprint(paths) {
const h = createHash("sha256");
for (const p of [...new Set(paths)].sort()) {
try {
const st = statSync(p);
h.update(`${p}\0${st.size}\0${st.mtimeMs}\0`);
} catch {
h.update(`${p}\0MISSING\0`);
}
}
return h.digest("hex");
}
const { runInContext, createContext } = require("vm");
const { minVersion } = require("semver");
/**
* checks if the given version is a valid semver version
* @param {string} version the version to check
* @returns {boolean} true if the version is a valid semver version
*/
function isValidVersion(version) {
try {
return minVersion(version) !== null;
} catch (e) {
return false;
}
}
/**
* converts a wildcard pattern to a regular expression
* @param {string} pattern the pattern to convert
* @returns {RegExp} the regular expression
*/
function wildcardToRegex(pattern) {
// escape special characters
const escapedPattern = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&");
// replace the wildcard with a capture group
const regexPattern = escapedPattern.replace(/\*/g, "(.*)");
// return the regular expression
return new RegExp(`^${regexPattern}$`);
}
/**
* helper to check the existence of a resource (case-sensitive)
* @param {string} file file path
* @returns {boolean} true if the file exists
*/
function existsSyncWithCase(file) {
var dir = path.dirname(file);
if (dir === path.dirname(dir)) {
return true;
}
if (existsSync(dir)) {
var filenames = readdirSync(dir);
if (filenames.indexOf(path.basename(file)) === -1) {
return false;
}
return existsSyncWithCase(dir);
}
return false;
}
/**
* helper to check the existence of a "file" resource (case-sensitive)
* @param {string} file file path
* @returns {boolean} true if the file exists
*/
function existsSyncWithCaseAndIsFile(file) {
return existsSyncWithCase(file) && existsSync(file) && statSync(file).isFile();
}
/**
* helper to check the existence of a "file" resource
* @param {string} file file path
* @returns {boolean} true if the file exists
*/
function existsSyncAndIsFile(file) {
return existsSync(file) && statSync(file).isFile();
}
/**
* returns the module path with the proper file extension
* @param {string} modulePath module path
* @returns {string} the module path with the proper file extension
*/
function getModulePathWithExtension(modulePath) {
// check for the module path exists and to be a file
if (existsSyncWithCaseAndIsFile(modulePath)) {
return modulePath;
} else if (existsSyncWithCaseAndIsFile(`${modulePath}.js`)) {
return `${modulePath}.js`;
} else if (existsSyncWithCaseAndIsFile(`${modulePath}.cjs`)) {
return `${modulePath}.cjs`;
} else if (existsSyncWithCaseAndIsFile(`${modulePath}.mjs`)) {
return `${modulePath}.mjs`;
}
// reset the module path if it doesn't exist
return undefined;
}
/**
* detects the "node_modules" directories relative to the current working directory
* @param {string} cwd current working directory
* @returns {string[]} list of existing node_modules directories
*/
function detectNodeModulesPaths(cwd = process.cwd()) {
const nodeModules = [];
let dir = cwd;
while (dir !== path.dirname(dir)) {
const nm = path.join(dir, "node_modules");
if (existsSync(nm)) {
nodeModules.push(nm);
}
dir = path.dirname(dir);
}
return nodeModules;
}
/**
* resolves the module name by testing file extensions
* @param {string} moduleName the module name
* @param {object} options options for require.resolve
* @param {string[]} options.paths paths to lookup the module
* @returns {string} the resolved module path
*/
function resolve(moduleName, options) {
let modulePath,
errors = [];
// try to resolve the module path with the default extensions
for (const ext of ["", ".js", ".cjs", ".mjs"]) {
try {
modulePath = require.resolve(`${moduleName}${ext}`, options);
if (modulePath) {
break;
}
} catch (err) {
errors.push(err);
}
}
// if an error occurred and the module path is still undefined, we throw the error
if (modulePath === undefined && errors.length > 0) {
throw errors.shift(); // throw the first error only
}
// if the module is a built-in module, we ignore it
if (modulePath === moduleName) {
/* eslint-disable-next-line no-useless-assignment */
modulePath = undefined;
const err = new Error(`Found built-in module "${moduleName}". Ignoring and trigger manual resolution to find custom modules!`);
err.code = "ERR_BUILTIN_MODULE";
throw err;
}
return modulePath;
}
// regex to extract the NPM package name from a dependency
const npmPackageScopeRegEx = /^((?:(@[^/]+)\/)?([^/]+))(?:\/(.*))?$/;
/**
* find the dependency by its name
* @param {string} dep name of the dependency
* @param {string} cwd current working directory
* @param {string[]} depPaths list of dependency paths
* @returns {string} the module path
*/
function findDependency(dep, cwd = process.cwd(), depPaths = []) {
let modulePath;
try {
try {
modulePath = resolve(dep, { paths: [cwd, ...depPaths] });
} catch (err) {
// sometimes the package.json is not found, therefore we try to resolve the dependency
// without the package.json, just with the npm package name (and lookup the package.json manually)
if (dep.endsWith("/package.json") && err.code === "MODULE_NOT_FOUND") {
modulePath = resolve(dep.substring(0, dep.length - "/package.json".length), { paths: [cwd, ...depPaths] });
// if the module path doesn't end with package.json, we try to find the package.json manually
// this is needed for workspaces, where the package.json file is not exported in the "exports" field of the package.json file
if (!modulePath.endsWith("package.json")) {
modulePath = findPackageJson(modulePath);
}
} else {
throw err;
}
}
} catch (err) {
// if the module is not exported, we try to resolve it manually
const [, npmPackage, , , module] = npmPackageScopeRegEx.exec(dep) || [];
if (err.code === "ERR_PACKAGE_PATH_NOT_EXPORTED" || err.code === "ERR_BUILTIN_MODULE") {
// the node_modules path of the dependency are importan as require.resolve.paths
// returns the node_modules paths relative to the location of this module
const resolvePaths = [...detectNodeModulesPaths(cwd)];
depPaths?.forEach((depPath) => {
resolvePaths.push(...detectNodeModulesPaths(depPath));
});
resolvePaths.push(...(require.resolve.paths(npmPackage) || []));
// lookup the dependency in the node_modules directories
for (const resolvePath of resolvePaths) {
modulePath = path.join(resolvePath, npmPackage);
if (module) {
modulePath = path.join(modulePath, module);
}
if (!existsSyncAndIsFile(modulePath)) {
modulePath = undefined;
} else {
// resolve the symlink to the real path
// the check for symlink is not working
// therefore we always resolve the real path
modulePath = realpathSync(modulePath);
break;
}
}
} else {
// ignoring the error
//console.error(`Failed to find dependency ${dep} due to ${err.code}`, ex);
}
}
return modulePath;
}
/**
* finds the package.json file for the given module path
* @param {string} modulePath the path of the module
* @returns {string} the path of the package.json file
*/
function findPackageJson(modulePath) {
let pkgJsonFile;
// lookup the parent dirs recursive to find package.json
const nodeModules = `${path.sep}node_modules${path.sep}`;
if (modulePath.lastIndexOf(nodeModules) > -1) {
const localModuleRootPath = modulePath.substring(0, modulePath.lastIndexOf(nodeModules) + nodeModules.length);
const localModuleName = modulePath.substring(localModuleRootPath.length)?.replace(/\\/g, "/");
const [, npmPackage] = npmPackageScopeRegEx.exec(localModuleName) || [];
pkgJsonFile = path.join(localModuleRootPath, npmPackage, "package.json");
if (!existsSyncAndIsFile(pkgJsonFile)) {
pkgJsonFile = undefined;
}
} else {
let dir = path.dirname(modulePath);
while (dir !== path.dirname(dir)) {
const pkgJson = path.join(dir, "package.json");
if (existsSyncAndIsFile(pkgJson)) {
pkgJsonFile = pkgJson;
break;
}
dir = path.dirname(dir);
}
}
return pkgJsonFile;
}
/**
* bundle info object containing all entries
*/
class BundleInfoCache {
static #bundleInfoCache = {};
static #cachePath(key) {
return path.join(process.cwd(), ".ui5-tooling-modules", `${key}.bundleinfo.json`);
}
static get(key, { persist } = {}) {
if (this.#bundleInfoCache[key]) {
return this.#bundleInfoCache[key];
} else if (persist) {
const bundleInfoPath = this.#cachePath(key);
if (existsSync(bundleInfoPath)) {
const loaded = new BundleInfo().fromJSON(readFileSync(bundleInfoPath, { encoding: "utf8" }));
// Re-stat the recorded transitive module graph as a confirmation step.
// On fresh checkouts (e.g. CI / `git checkout`) entry-only mtime checks
// can produce a false positive; this validates the full graph.
const recordedFp = loaded.getInputFingerprint();
if (recordedFp) {
const currentFp = getInputGraphFingerprint(loaded.getRelatedPaths());
if (recordedFp !== currentFp) {
return undefined;
}
}
return (this.#bundleInfoCache[key] = loaded);
}
}
return undefined;
}
static store(key, bundleInfo, { persist } = {}) {
this.#bundleInfoCache[key] = bundleInfo;
if (persist) {
// Capture the transitive input graph fingerprint so a future persistent
// cache load can verify the recorded graph still matches disk state.
bundleInfo.setInputFingerprint(getInputGraphFingerprint(bundleInfo.getRelatedPaths()));
const bundleInfoPath = this.#cachePath(key);
mkdir(path.dirname(bundleInfoPath), { recursive: true })
.then(() => {
return writeFile(bundleInfoPath, JSON.stringify(bundleInfo, null, 2), { encoding: "utf8" });
})
.catch((err) => {
console.error(`Failed to store bundle info in ${bundleInfoPath} on disk! Using in-memory cache only!`, err);
});
}
}
}
/**
* bundle info object containing all entries
*/
class BundleInfo {
_entries = [];
_inputFingerprint = null;
fromJSON(s) {
const parsed = JSON.parse(s);
this._entries = parsed._entries;
this._inputFingerprint = parsed._inputFingerprint ?? null;
return this;
}
setInputFingerprint(fp) {
this._inputFingerprint = fp;
}
getInputFingerprint() {
return this._inputFingerprint;
}
getEntry(name) {
return this._entries.find((entry) => entry.name === name);
}
getEntries() {
return this._entries;
}
addModule(entryInfo) {
return this._entries.push(Object.assign(entryInfo, { type: "module" }));
}
getModules() {
return this._entries.filter((entry) => entry.type === "module");
}
addChunk(entryInfo) {
return this._entries.push(Object.assign(entryInfo, { type: "chunk" }));
}
getChunks() {
return this._entries.filter((entry) => entry.type === "chunk");
}
addResource(entryInfo) {
return this._entries.push(Object.assign(entryInfo, { type: "resource" }));
}
getResources() {
return this._entries.filter((entry) => entry.type === "resource");
}
addScript(entryInfo) {
return this._entries.push(Object.assign(entryInfo, { type: "script" }));
}
getScripts() {
return this._entries.filter((entry) => entry.type === "script");
}
getBundledResources() {
return this._entries.filter((entry) => /^(module|chunk|resource|script)$/.test(entry.type));
}
getRelatedPaths() {
return this._entries
.map((e) => e.relatedPaths)
.flat()
.filter((e) => typeof e === "string");
}
}
// the utiltiy module
module.exports = function (log, projectInfo) {
// performande metrics
const perfmetrics = {
resolveModulesTime: 0,
resolveModules: {},
};
// local cache of resolved module paths
const modulesCache = {};
// local cache of negative modules (avoid additional lookups)
const modulesNegativeCache = [];
// local cache for package.json files
const packageJsonCache = {};
// local cache for dependencies list
const findDependenciesCache = {};
/**
* load and cache the package.json file for the given path
* @param {string} pkgJsonFile the path of the package.json file
* @returns {object} the package.json content
*/
function getPackageJson(pkgJsonFile) {
if (!packageJsonCache[pkgJsonFile]) {
try {
const pkgJson = JSON.parse(readFileSync(pkgJsonFile, { encoding: "utf8" }));
packageJsonCache[pkgJsonFile] = pkgJson;
} catch (err) {
console.error(`Failed to read package.json file ${pkgJsonFile}!`, err);
throw err;
}
}
return packageJsonCache[pkgJsonFile];
}
/**
* find the dependencies of the current project and its transitive dependencies
* (excluding devDependencies and providedDependencies)
* @param {object} options options
* @param {string} [options.cwd] current working directory
* @param {string[]} [options.depPaths] list of dependency paths
* @param {boolean} [options.linkedOnly] find only the linked dependencies
* @param {string[]} [options.additionalDeps] list of additional dependencies (e.g. dev dependencies to include)
* @param {string[]} [knownDeps] list of known dependencies
* @returns {string[]} array of dependency root directories
*/
function findDependencies({ cwd = process.cwd(), depPaths = [], linkedOnly, additionalDeps = [] } = {}, knownDeps = []) {
const pkgJson = getPackageJson(path.join(cwd, "package.json"));
let dependencies = [...Object.keys(pkgJson.dependencies || {}), ...Object.keys(pkgJson.optionalDependencies || {})];
if (additionalDeps?.length > 0) {
dependencies = dependencies.concat(additionalDeps);
}
if (linkedOnly) {
dependencies = dependencies.filter((dep) => {
const version = pkgJson.dependencies?.[dep] || pkgJson.optionalDependencies?.[dep];
return !isValidVersion(version);
});
}
const depRoots = [];
const findDeps = [];
for (const dep of dependencies) {
const npmPackage = npmPackageScopeRegEx.exec(dep)?.[1];
if (knownDeps.indexOf(npmPackage) !== -1) {
continue;
}
knownDeps.push(npmPackage);
const depPath = findDependency(path.posix.join(npmPackage, "package.json"), cwd, depPaths);
let depRoot = depPath && path.dirname(depPath);
if (depRoot && depRoots.indexOf(depRoot) === -1) {
depRoots.push(depRoot);
// delay the dependency lookup to avoid finding transitive dependencies before local dependencies
findDeps.push({ cwd: depRoot, depPaths, linkedOnly, additionalDeps });
}
}
// lookup the transitive dependencies
for (const dep of findDeps) {
depRoots.push(...findDependencies(dep, knownDeps));
}
return depRoots;
}
/**
* Checks whether the file behind the given path is a module to skip for transformation or not
* @param {string} source the path of a JS module
* @returns {boolean} true, if the module should be skipped for transformation
*/
async function shouldSkipModule(source) {
let m = Date.now();
let isUI5Module = false;
let isSystemJSModule = false;
const content = await readFile(source, { encoding: "utf8" });
if (content) {
try {
const context = createContext({
sap: {
ui: {
require: () => {
isUI5Module = true;
},
define: () => {
isUI5Module = true;
},
},
},
System: {
register: () => {
isSystemJSModule = true;
},
},
});
runInContext(content, context);
} catch (err) {
// ESM, CJS, AMD, UMD, etc. modules should not be skipped
log.verbose(`Failed to detect module type for "${source}"!`, err.message);
}
}
log.verbose(`shouldSkipModule took ${Date.now() - m}ms`);
return isUI5Module || isSystemJSModule;
}
const that = {
/**
* scans the project resources
* @param {module:@ui5/fs/AbstractReader[]} reader resources reader
* @param {object} [config] configuration
* @param {boolean} [config.debug] debug mode
* @param {boolean} [config.providedDependencies] list of provided dependencies which should not be processed
* @param {object<string, string[]>} [config.includeAssets] map of assets (key: npm package name, value: local paths) to be included (embedded)
* @param {boolean} [config.legacyDependencyResolution] includes all dependencies (including devDependencies) for entry point resolution
* @param {object} [options] additional options
* @param {string} [options.cwd] current working directory
* @param {string[]} [options.depPaths] paths of the dependencies (in addition for cwd)
* @returns {object} unique dependencies, resources, namespaces, chunks, ...
*/
scan: async function (reader, config, { cwd = process.cwd(), depPaths = [] }) {
const { parse } = await import("@typescript-eslint/typescript-estree");
const { walk } = await import("estree-walker");
const providedDependencies = Array.isArray(config?.providedDependencies) ? config?.providedDependencies : [];
// find the root directories of the dependencies (excludes devDependencies)
// since all modules should be declared as dependencies in the package.json
let millis = Date.now();
const dependencyRoots = !config?.legacyDependencyResolution && findDependencies({ cwd, depPaths, additionalDeps: config.additionalDependencies });
log.verbose(`Dependency (${depPaths.length} deps) lookup took ${Date.now() - millis}ms`);
// find all sources to determine their dependencies
millis = Date.now();
let allSources = await reader.byGlob("/**/*.{js,jsx,ts,tsx}");
log.verbose(`Source (${allSources.length} files) lookup took ${Date.now() - millis}ms`);
// only keep the TS resources for which no parallel JS resource exist
// as we assume that the transpiling creates the parallel JS resource
millis = Date.now();
const allJsSources = new Set(allSources.filter((source) => path.extname(source.getPath()) === ".js").map((source) => source.getPath().replace(/\.js$/, "")));
if (allJsSources.size > 0) {
allSources = allSources.filter((source) => {
const sourcePath = source.getPath();
const ext = path.extname(sourcePath);
// exclude generated TS types for control wrappers
if (sourcePath.endsWith(".d.ts")) {
return false;
} else if (ext !== ".js" && allJsSources.has(sourcePath.replace(new RegExp(`\\${ext}$`), ""))) {
log.info(`Removing source ${sourcePath} (as a parallel JS resource was found)`);
return false;
}
return true;
});
}
log.verbose(`JS filter took ${Date.now() - millis}ms`);
// find all XML resources to determine their dependencies
millis = Date.now();
const allXmlResources = await reader.byGlob("/**/*.xml");
log.verbose(`XML (${allXmlResources.length} files) lookup took ${Date.now() - millis}ms`);
// collector for unique dependencies and resources
//const uniqueNPMPackages = new Set();
const uniqueDeps = new Set();
const uniqueModules = new Set();
const uniqueResources = new Set();
const uniqueNS = new Set();
// eslint-disable-next-line jsdoc/require-jsdoc
function isProvided(depOrRes) {
return providedDependencies.filter((e) => depOrRes === e || depOrRes.startsWith(`${e}/`)).length > 0;
}
/* TODO: keep functionality commented till needed!
function addUniqueNPMPackage(npmPackageName) {
if (!uniqueNPMPackages.has(npmPackageName) && npmPackageName && !npmPackageName.startsWith(".") && resolveModule(`${npmPackageName}/package.json`, { cwd, depPaths })) {
uniqueNPMPackages.add(npmPackageName);
}
}
*/
// eslint-disable-next-line jsdoc/require-jsdoc
function addUniqueDep(dep) {
if (uniqueDeps.has(dep) || isProvided(dep)) {
return false;
} else {
// add the dependency (by default we already filter the UI5 modules we know)
//if (!dep.startsWith("sap/")) {
uniqueDeps.add(dep);
//}
// also add the NPM package name
//const npmPackageName = /((?:@[^/]+\/)?(?:[^/]+)).*/.exec(dep)?.[1];
//addUniqueNPMPackage(npmPackageName);
return true;
}
}
// eslint-disable-next-line jsdoc/require-jsdoc
function moduleNameEqualsNpmPackageName(module, modulePath) {
// in some cases the module name is the same as the NPM package name and if the NPM package name has the
// suffix .js then this file extension is stripped from the module name and the module resolution doesn't
// work anymore - so we check if the module name is the same as the NPM package name (e.g. easytimer.js)
const reNpmPackageName = new RegExp(`.*${path.sep}node_modules${path.sep}([^${path.sep}]+)${path.sep}`);
return reNpmPackageName.exec(modulePath)?.[1] === module;
}
// eslint-disable-next-line jsdoc/require-jsdoc
function addUniqueModule(module, modulePath) {
if (!uniqueModules.has(module)) {
const moduleName = module.split("/").pop();
const moduleFileName = modulePath.split(path.sep).pop();
// identify modules which already provide their file extension in the module name
// => this avoids the duplication of the modules specified with and without file
// extension in the uniqueModules set (e.g. "myns/module" and "myns/module.js")
if (moduleName === moduleFileName && module.endsWith(".js") && !moduleNameEqualsNpmPackageName(module, modulePath)) {
//console.log(`Module name and file name are equal: ${module} => ${modulePath}`);
module = module.slice(0, -3); // remove the file extension
}
uniqueModules.add(module);
}
}
// eslint-disable-next-line jsdoc/require-jsdoc
function addUniqueResource(res) {
if (!uniqueResources.has(res) && !isProvided(res) && that.existsResource(res, { cwd, depPaths })) {
uniqueResources.add(res);
}
}
// eslint-disable-next-line jsdoc/require-jsdoc
function addUniqueNamespace(ns) {
if (!uniqueNS.has(ns) && !isProvided(ns)) {
uniqueNS.add(ns);
}
}
// eslint-disable-next-line jsdoc/require-jsdoc
function addDep(dep) {
if (addUniqueDep(dep)) {
// each dependency which can be resolved via the NPM package name
// should also be checked for its dependencies to finally handle them
// here if they also require to be transpiled by the task
try {
const depPath = that.resolveModule(dep, { cwd, depPaths });
const existsDep = depPath && existsSyncWithCase(depPath);
let allowedDep = existsDep && (!dependencyRoots || dependencyRoots.some((root) => depPath.startsWith(root)));
// determine local dependencies (e.g. for bundle definitions)
if (!allowedDep && dep.startsWith(`${projectInfo.pkgJson.name}/`)) {
allowedDep = true;
}
if (allowedDep) {
// add the dependency to the list of unique dependencies
addUniqueModule(dep, depPath);
// only analyze the dependencies of UI5 projects recursively
const depRoot = dependencyRoots?.find((root) => depPath.startsWith(root));
// check if the dependency is a UI5 project (has a ui5.yaml or a .ui5pkg marker file)
if (existsSync(path.join(depRoot, "ui5.yaml")) || existsSync(path.join(depRoot, ".ui5pkg"))) {
const depContent = readFileSync(depPath, { encoding: "utf8" });
// for UI5 projects we analyze the sap.ui.define|require|requireSync plus imports (due to TS)
findUniqueJSDeps(depContent, depPath);
}
} else if (existsDep) {
log.warn(`Skipping dependency "${dep}" as it is not part of the dependencies in package.json!`);
}
} catch (ex) {
/* noop */
}
}
}
// utility to lookup unique JS dependencies
// eslint-disable-next-line jsdoc/require-jsdoc
function findUniqueJSDeps(content, parentDepPath, ignoreImports) {
// use @typescript-eslint/typescript-estree to parse the UI5 modules
// and extract the UI5 module dependencies
// => can parse modern JS, TS, JSX, and TSX syntax
// => supports the ES6 import/export syntax
// => supports the UI5 sap.ui.require.toUrl and sap.ui.require/define/requireSync
try {
const program = parse(content, { jsx: path.extname(parentDepPath) !== ".ts", allowInvalidAST: true });
walk(program, {
enter(node, parent, prop, index) {
if (
/* sap.ui.require.toUrl */
node?.type === "CallExpression" &&
node?.callee?.property?.name == "toUrl" &&
node?.callee?.object?.property?.name == "require" &&
node?.callee?.object?.object?.property?.name == "ui" &&
node?.callee?.object?.object?.object?.name == "sap"
) {
const elDep = node.arguments[0];
if (elDep?.type === "Literal") {
addUniqueResource(elDep.value);
}
} else if (
/* sap.ui.(require|define) */
(node?.type === "CallExpression" &&
/require|define|requireSync/.test(node?.callee?.property?.name) &&
node?.callee?.object?.property?.name == "ui" &&
node?.callee?.object?.object?.name == "sap") ||
/* __ui5_require_async (babel-plugin-transform-modules-ui5) */
(node?.type === "CallExpression" && node?.callee?.name == "__ui5_require_async")
) {
let deps;
if (/requireSync/.test(node?.callee?.property?.name) || /__ui5_require_async/.test(node?.callee?.name)) {
const elDep = node.arguments[0];
if (elDep?.type === "Literal") {
deps = [elDep.value];
}
} else {
const depsArray = node.arguments.filter((arg) => arg.type === "ArrayExpression");
deps = depsArray?.[0]?.elements.filter((el) => el.type === "Literal").map((el) => el.value);
}
deps?.forEach((dep) => addDep(dep));
} else if (
!ignoreImports &&
/* ES6 dynamic import */
node?.type === "ImportExpression"
) {
addDep(node.source.value);
} else if (!ignoreImports && node?.type === "ImportDeclaration") {
/* ES6 import */
addDep(node.source.value);
}
},
});
} catch (err) {
config.debug && log.warn(`Failed to analyze resource "${parentDepPath}" (${err})!${config.debug === "verbose" ? `\n${err.stack}` : ""}`);
}
}
// utility to lookup unique XML dependencies
// eslint-disable-next-line jsdoc/require-jsdoc
function findUniqueXMLDeps(node, ns = {}, imports = {}) {
if (typeof node === "object") {
// attributes
Object.keys(node)
.filter((key) => key.startsWith("@_"))
.forEach((key) => {
const nsParts = /@_xmlns(?::(.*))?/.exec(key);
if (nsParts) {
// namespace (default namespace => "")
ns[nsParts[1] || ""] = node[key];
return;
}
const requireParts = /@_(?:(.*):)?require/.exec(key);
if (requireParts && ns[requireParts[1]] === "sap.ui.core") {
try {
const requires = parseJS(node[key]);
Object.values(requires).forEach((dep) => addDep(dep));
} catch (err) {
log.error(`Failed to parse the "${node[key]}" as JS object!`);
}
}
});
// nodes
Object.keys(node)
.filter((key) => !key.startsWith("@_"))
.forEach((key) => {
const children = Array.isArray(node[key]) ? node[key] : [node[key]];
children.forEach((child) => {
const nodeParts = /(?:([^:]*):)?(.*)/.exec(key);
if (nodeParts) {
// skip #text nodes
let module = nodeParts[2];
if (module !== "#text") {
// only add those dependencies whose namespace is known
// - in some cases the ns is defined directly at the element
// this is handled by the OR case here to avoid recursive parsing
let namespace = ns[nodeParts[1] || ""] || (nodeParts[1] && child[`@_xmlns:${nodeParts[1]}`]);
if (typeof namespace === "string") {
namespace = namespace.replace(/\./g, "/");
addUniqueNamespace(namespace);
const importPath = imports[nodeParts[1]];
const dep = `${importPath || namespace}/${module}`;
addDep(dep);
}
if (typeof child === "object") {
findUniqueXMLDeps(child, ns, imports);
}
}
}
});
});
}
}
// lookup all resources for their dependencies via the above utility
millis = Date.now();
if (allXmlResources.length > 0) {
const parser = new XMLParser({
attributeNamePrefix: "@_",
ignoreAttributes: false,
ignoreNameSpace: false,
});
await Promise.all(
allXmlResources.map(async (resource) => {
log.verbose(`Processing XML resource: ${resource.getPath()}`);
const content = await resource.getString();
const xmldom = parser.parse(content);
findUniqueXMLDeps(xmldom);
return resource;
}),
);
}
log.verbose(`XML scan took ${Date.now() - millis}ms`);
// lookup all sources for their dependencies via the above utility
millis = Date.now();
await Promise.all(
allSources.map(async (resource) => {
log.verbose(`Processing source: ${resource.getPath()}`);
const content = await resource.getString();
findUniqueJSDeps(content, resource.getPath());
return resource;
}),
);
log.verbose(`JS scan took ${Date.now() - millis}ms`);
// lookup the assets to be included which are configured in the ui5.yaml
if (config.includeAssets) {
if (typeof config.includeAssets === "object") {
Object.keys(config.includeAssets).forEach((npmPackageName) => {
const ignore = config.includeAssets[npmPackageName];
if (!ignore || Array.isArray(ignore)) {
log.verbose(`Including assets for dependency: ${npmPackageName}`);
try {
const assets = that.listResources(npmPackageName, { cwd, depPaths, ignore });
if (log.isLevelEnabled("verbose")) {
assets.forEach((asset) => log.verbose(` - ${asset}`));
}
assets.forEach((asset) => uniqueResources.add(asset));
} catch (ex) {
log.error(`The npm package ${npmPackageName} defined in "includeAssets" not found! Skipping assets...`);
}
} else {
log.error(`The option "includeAssets" must be type of map with the key being a npm package name and optionally values being a list of glob patterns!`);
}
});
} else {
log.error(`The option "includeAssets" must be type of map with the key being a npm package name!`);
}
}
return {
uniqueDeps,
uniqueModules,
uniqueResources,
uniqueNS,
sourceFiles: [].concat(allSources, allXmlResources).map((res) => {
return res.getSourceMetadata().fsPath;
}),
};
},
/**
* Resolves the bare module name from node_modules utilizing Node.js resolution algorithm
* @param {string} moduleName name of the module (e.g. "chart.js/auto")
* @param {object} options configuration options
* @param {string} options.cwd current working directory
* @param {string[]} options.depPaths paths of the dependencies (in addition for cwd)
* @param {string[]} [options.additionalDeps] list of additional dependencies (e.g. dev dependencies to include)
* @returns {string} the path of the module in the filesystem
*/
// ignore module paths starting with a segment from the ignore list (TODO: maybe a better check?)
resolveModule: function resolveModule(moduleName, { cwd = process.cwd(), depPaths = [], additionalDeps = [] } = {}) {
// create a cache key for the module and check the cache
const cacheKey = createHash("sha256")
.update(`${[moduleName, cwd, ...depPaths].sort().join(",")}`)
.digest("hex");
if (modulesCache[cacheKey]) {
return modulesCache[cacheKey];
}
// if a module is listed in the negative cache, we ignore it!
if (modulesNegativeCache.indexOf(moduleName) !== -1 || /^\./.test(moduleName)) {
return undefined;
}
// retrieve or create a resolved module
const millis = Date.now();
log.verbose(`Resolving ${moduleName}...`);
// package.json of app
const pkg = getPackageJson(path.join(cwd, "package.json"));
// create the extended dependencies path (incl. direct dependencies for module lookup)
// hint: we include the direct dependencies to resolve the module path in the context
// of the app (which is required e.g. when linking dependencies to the project)
if (!findDependenciesCache[cwd]) {
findDependenciesCache[cwd] = findDependencies({ cwd, depPaths, additionalDeps, linkedOnly: true });
}
const extendedDepPaths = [...depPaths, ...findDependenciesCache[cwd]];
// no module found => resolve it
let modulePath;
// resolve the module path
if (moduleName?.startsWith(`${pkg.name}/`)) {
// special handling for app-local resources!
modulePath = path.join(cwd, moduleName.substring(`${pkg.name}/`.length));
// check for the module path exists and to be a file
modulePath = getModulePathWithExtension(modulePath);
} else {
// Implementation of the Node.js Package Entry Points mechanism
// https://nodejs.org/api/packages.html#package-entry-points
let pkgJsonFile, relativeModulePath;
// when resolving absolute files we lookup the package.json in the parent dirs
// for later resolution of the module path (e.g. the mapping in browsers field)
if (path.isAbsolute(moduleName)) {
// lookup the parent dirs recursive to find package.json
pkgJsonFile = findPackageJson(moduleName);
// the module path is the absolute path (but with file extension)
modulePath = getModulePathWithExtension(moduleName);
// if the absolute path is not a file we try to lookup the index module
if (modulePath === undefined) {
modulePath = getModulePathWithExtension(`${moduleName}${path.sep}index`);
}
// determine the relative module path to the package.json as the module resolution
// below resolves and maps the module path relative to the package.json only
if (modulePath) {
relativeModulePath = path.relative(path.dirname(pkgJsonFile), modulePath)?.replace(/\\/g, "/");
}
} else {
// lookup the package.json with the npm package name
const [, npmPackage, , , relModulePath] = npmPackageScopeRegEx.exec(moduleName) || [];
pkgJsonFile = findDependency(`${npmPackage}/package.json`, cwd, extendedDepPaths);
// short track to derive the relative module path relative to the package.json
if (pkgJsonFile && relModulePath) {
relativeModulePath = relModulePath;
}
}
// if a package.json file was found we try to resolve the module path
// from the mappings in the package.json file (e.g. browser field)
if (pkgJsonFile) {
// determine the root path of the package.json file and load the package.json
const rootPath = path.dirname(pkgJsonFile);
const pkgJson = getPackageJson(pkgJsonFile);
// all modules must be resolved from the package.json fields if available
// Hints about package.json exports resolution:
// - https://nodejs.org/api/packages.html#packages_subpath_exports
// - https://github.com/rollup/plugins/blob/master/packages/node-resolve/src/package/resolvePackageExports.ts
// - https://webpack.js.org/guides/package-exports/
// conditions means that exports is an object without any entry starting with a dot
// mappins means that exports is an object having no conditions
// conditions and mappings are mutually exclusive
const isConditions = (exports) => {
return exports && !Object.keys(exports).some((key) => key.startsWith("."));
};
const isMappings = (exports) => {
return exports && !isConditions(exports);
};
// the conditions and mappings to resolve the module path
const communityConditions = ["browser", "production" /*, "development"*/]; // could be injected from outside
const mainModuleConditions = ["esnext", "module", "main"];
const resolveConditions = [...communityConditions, ...mainModuleConditions, "import", "default", "require"]; // require is just the fallback!
// for the browser field, we need to resolve the conditions in a different order (axios showed this)
// the require field should be used before the default field to ensure the correct resolution
// as e.g. axios provides the browser package in the require field
const resolveConditionsBrowser = [...communityConditions, ...mainModuleConditions, "import", "require", "default"]; // default is just the fallback!
// helper to resolve the target of a mapping:
// - if the target is a string it is returned (and if needed the wildcard is replaced)
// - if the target is an array the first resolved target is returned
// - if the target is an object the first resolved target of the conditions is returned
const resolveTarget = (target, wildcardMatch, conditionsToResolve = resolveConditions) => {
if (typeof target === "string") {
if (!wildcardMatch) {
return target;
} else {
// replace the wildcard with the matched wildcard:
// determine the value of the wildcard in the exports field
return target.replace(/\*/g, wildcardMatch);
}
} else if (Array.isArray(target)) {
// resolve the target from the array of targets
for (const entry of target) {
const resolved = resolveTarget(entry, wildcardMatch);
if (resolved) {
return resolved;
}
}
} else if (target && typeof target === "object") {
// resolve the target of the conditions
for (const condition of conditionsToResolve) {
if (condition in target) {
let resolved = resolveTarget(target[condition], wildcardMatch, condition === "browser" ? resolveConditionsBrowser : resolveConditions);
if (resolved) {
return resolved;
}
}
}
// if no condition matches, check for a wildcard match
// which is typically a path mapping in the exports field
if (wildcardMatch in target) {
return target[wildcardMatch];
}
}
};
// helper to resolve the given subpath from the exports field of the package.json
const resolveExports = (exports, subPath) => {
if (subPath.indexOf("*") === -1 && exports[subPath]) {
// no wildcard in the subPath and the subPath is a direct mapping
let resolved = exports[subPath];
if (typeof resolved === "object") {
resolved = resolveTarget(resolved);
}
return resolved;
} else {
// sort the export keys by the position of the wildcard
const exportKeys = Object.keys(exports)
.filter((v) => v.indexOf("*") !== -1)
.sort((a, b) => {
return b.indexOf("*") - a.indexOf("*");
});
// helper to lookup the first match for the subPath
const findMatch = (exportKeys, subPath) => {
for (const key of exportKeys) {
const re = wildcardToRegex(key);
const match = re.exec(subPath);
if (match) {
return { target: exports[key], wildcardMatch: match[1] };
}
}
};
// lookup the first match for the subPath
const match = findMatch(exportKeys, subPath);
if (match) {
let resolved = resolveTarget(match.target, match.wildcardMatch);
// check if another mapping is required (e.g. ./* to ./dist/* to ./dist/prod/*)
const resolvedMatch = findMatch(exportKeys, resolved);
if (resolvedMatch?.target !== match.target) {
resolved = resolveExports(exports, resolved);
}
return resolved;
}
}
return subPath;
};
// lookup the module path from the package.json browser or exports field
const { browser, exports } = pkgJson;
let subPath = relativeModulePath ? `./${relativeModulePath}` : ".";
let mainExport;
if (subPath === ".") {
// if the module is the main module of the package
// => first we resolve the exports property
// => then the default exports fields
// detected with the issue #1196 in which it shows that
// sinon wrongly configured the browser field which
// doesn't match the exports > browser field and therefore
// the generated module doesn't properly export itself
if (typeof exports === "string" || Array.isArray(exports) || isConditions(exports)) {
// if exports is a string, an array or an object with conditions
// the main module is defined by the exports field
mainExport = exports;
} else if (isMappings(exports)) {
// if exports is an object with mappings it is defined in the "." exports field
mainExport = exports["."];
} else if (typeof browser === "string") {
// if a browser field is a string value in the package.json
// the main module is defined in the browser field
// (an object value means it is a mapping used below!)
mainExport = browser;
} else {
// lookup the entry modules in the package.json root (esnext, module, main, ...)
mainExport = pkgJson;
}
const resolved = resolveTarget(mainExport);
modulePath = resolved ? path.join(rootPath, resolved) : undefined;
} else if (isMappings(exports)) {
// if the module is a sub module of the package
const resolved = resolveExports(exports, subPath);
modulePath = resolved ? path.join(rootPath, resolved) : undefined;
} else if (isMappings(browser)) {
// if the module is a sub module of the package and the package has a browser field
const resolved = resolveExports(browser, subPath);
modulePath = resolved ? path.join(rootPath, resolved) : undefined;
}
// check for the module path exists and is a file
// only then the module path is valid and can be returned
if (modulePath !== undefined && !existsSyncAndIsFile(modulePath)) {
modulePath = undefined;
}
}
// use the findDependency utility to resolve the module name
if (modulePath === undefined && !path.isAbsolute(moduleName)) {
//console.log("##################", "findDependency", moduleName);
modulePath = findDependency(moduleName, cwd, extendedDepPaths);
}
}
if (modulePath === undefined) {
modulesNegativeCache.push(moduleName);
log.verbose(` => not found!`);
} else if (!path.isAbsolute(modulePath) && modulePath === moduleName) {
modulePath = undefined;
modulesNegativeCache.push(moduleName);
log.verbose(` => found but ignored (identified as native node module)!`);
} else {
log.verbose(` => found at ${modulePath}`);
}
if (modulePath) {
modulesCache[cacheKey] = modulePath;
}
const took = Date.now() - millis;
//console.log(`resolveModule "${moduleName}" took ${took}ms`);
perfmetrics.resolveModulesTime += took;
if (!perfmetrics.resolveModules[moduleName]) {
perfmetrics.resolveModules[moduleName] = took;
} else {
console.error(`Duplicate module resolution for "${moduleName}"!`);
}
return modulePath;
},
/**
* creates a bundle for the given module(s) using rollup
* @param {string|string[]} moduleNames name of the module (e.g. "chart.js/auto") or an array of module names
* @param {object} config configuration options
* @param {string} config.cwd current working directory