@jsenv/util
Version:
Set of functions often needed when using Node.js.
1,674 lines (1,419 loc) • 70.9 kB
JavaScript
'use strict';
Object.defineProperty(exports, '__esModule', { value: true });
var urlMeta = require('@jsenv/url-meta');
var url = require('url');
var fs = require('fs');
var crypto = require('crypto');
var cancellation = require('@jsenv/cancellation');
var path = require('path');
var util = require('util');
const ensureUrlTrailingSlash = url => {
return url.endsWith("/") ? url : `${url}/`;
};
const isFileSystemPath = value => {
if (typeof value !== "string") {
throw new TypeError(`isFileSystemPath first arg must be a string, got ${value}`);
}
if (value[0] === "/") return true;
return startsWithWindowsDriveLetter(value);
};
const startsWithWindowsDriveLetter = string => {
const firstChar = string[0];
if (!/[a-zA-Z]/.test(firstChar)) return false;
const secondChar = string[1];
if (secondChar !== ":") return false;
return true;
};
const fileSystemPathToUrl = value => {
if (!isFileSystemPath(value)) {
throw new Error(`received an invalid value for fileSystemPath: ${value}`);
}
return String(url.pathToFileURL(value));
};
const assertAndNormalizeDirectoryUrl = value => {
let urlString;
if (value instanceof URL) {
urlString = value.href;
} else if (typeof value === "string") {
if (isFileSystemPath(value)) {
urlString = fileSystemPathToUrl(value);
} else {
try {
urlString = String(new URL(value));
} catch (e) {
throw new TypeError(`directoryUrl must be a valid url, received ${value}`);
}
}
} else {
throw new TypeError(`directoryUrl must be a string or an url, received ${value}`);
}
if (!urlString.startsWith("file://")) {
throw new Error(`directoryUrl must starts with file://, received ${value}`);
}
return ensureUrlTrailingSlash(urlString);
};
const assertAndNormalizeFileUrl = (value, baseUrl) => {
let urlString;
if (value instanceof URL) {
urlString = value.href;
} else if (typeof value === "string") {
if (isFileSystemPath(value)) {
urlString = fileSystemPathToUrl(value);
} else {
try {
urlString = String(new URL(value, baseUrl));
} catch (e) {
throw new TypeError(`fileUrl must be a valid url, received ${value}`);
}
}
} else {
throw new TypeError(`fileUrl must be a string or an url, received ${value}`);
}
if (!urlString.startsWith("file://")) {
throw new Error(`fileUrl must starts with file://, received ${value}`);
}
return urlString;
};
const statsToType = stats => {
if (stats.isFile()) return "file";
if (stats.isDirectory()) return "directory";
if (stats.isSymbolicLink()) return "symbolic-link";
if (stats.isFIFO()) return "fifo";
if (stats.isSocket()) return "socket";
if (stats.isCharacterDevice()) return "character-device";
if (stats.isBlockDevice()) return "block-device";
return undefined;
};
const urlToFileSystemPath = fileUrl => {
if (fileUrl[fileUrl.length - 1] === "/") {
// remove trailing / so that nodejs path becomes predictable otherwise it logs
// the trailing slash on linux but does not on windows
fileUrl = fileUrl.slice(0, -1);
}
const fileSystemPath = url.fileURLToPath(fileUrl);
return fileSystemPath;
};
// https://github.com/coderaiser/cloudcmd/issues/63#issuecomment-195478143
// https://nodejs.org/api/fs.html#fs_file_modes
// https://github.com/TooTallNate/stat-mode
// cannot get from fs.constants because they are not available on windows
const S_IRUSR = 256;
/* 0000400 read permission, owner */
const S_IWUSR = 128;
/* 0000200 write permission, owner */
const S_IXUSR = 64;
/* 0000100 execute/search permission, owner */
const S_IRGRP = 32;
/* 0000040 read permission, group */
const S_IWGRP = 16;
/* 0000020 write permission, group */
const S_IXGRP = 8;
/* 0000010 execute/search permission, group */
const S_IROTH = 4;
/* 0000004 read permission, others */
const S_IWOTH = 2;
/* 0000002 write permission, others */
const S_IXOTH = 1;
/* 0000001 execute/search permission, others */
/*
here we could warn that on windows only 0o444 or 0o666 will work
0o444 (readonly)
{
owner: {read: true, write: false, execute: false},
group: {read: true, write: false, execute: false},
others: {read: true, write: false, execute: false},
}
0o666 (read and write)
{
owner: {read: true, write: true, execute: false},
group: {read: true, write: true, execute: false},
others: {read: true, write: true, execute: false},
}
*/
const binaryFlagsToPermissions = binaryFlags => {
const owner = {
read: Boolean(binaryFlags & S_IRUSR),
write: Boolean(binaryFlags & S_IWUSR),
execute: Boolean(binaryFlags & S_IXUSR)
};
const group = {
read: Boolean(binaryFlags & S_IRGRP),
write: Boolean(binaryFlags & S_IWGRP),
execute: Boolean(binaryFlags & S_IXGRP)
};
const others = {
read: Boolean(binaryFlags & S_IROTH),
write: Boolean(binaryFlags & S_IWOTH),
execute: Boolean(binaryFlags & S_IXOTH)
};
return {
owner,
group,
others
};
};
const permissionsToBinaryFlags = ({
owner,
group,
others
}) => {
let binaryFlags = 0;
if (owner.read) binaryFlags |= S_IRUSR;
if (owner.write) binaryFlags |= S_IWUSR;
if (owner.execute) binaryFlags |= S_IXUSR;
if (group.read) binaryFlags |= S_IRGRP;
if (group.write) binaryFlags |= S_IWGRP;
if (group.execute) binaryFlags |= S_IXGRP;
if (others.read) binaryFlags |= S_IROTH;
if (others.write) binaryFlags |= S_IWOTH;
if (others.execute) binaryFlags |= S_IXOTH;
return binaryFlags;
};
const writeFileSystemNodePermissions = async (source, permissions) => {
const sourceUrl = assertAndNormalizeFileUrl(source);
const sourcePath = urlToFileSystemPath(sourceUrl);
let binaryFlags;
if (typeof permissions === "object") {
permissions = {
owner: {
read: getPermissionOrComputeDefault("read", "owner", permissions),
write: getPermissionOrComputeDefault("write", "owner", permissions),
execute: getPermissionOrComputeDefault("execute", "owner", permissions)
},
group: {
read: getPermissionOrComputeDefault("read", "group", permissions),
write: getPermissionOrComputeDefault("write", "group", permissions),
execute: getPermissionOrComputeDefault("execute", "group", permissions)
},
others: {
read: getPermissionOrComputeDefault("read", "others", permissions),
write: getPermissionOrComputeDefault("write", "others", permissions),
execute: getPermissionOrComputeDefault("execute", "others", permissions)
}
};
binaryFlags = permissionsToBinaryFlags(permissions);
} else {
binaryFlags = permissions;
}
return chmodNaive(sourcePath, binaryFlags);
};
const chmodNaive = (fileSystemPath, binaryFlags) => {
return new Promise((resolve, reject) => {
fs.chmod(fileSystemPath, binaryFlags, error => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
};
const actionLevels = {
read: 0,
write: 1,
execute: 2
};
const subjectLevels = {
others: 0,
group: 1,
owner: 2
};
const getPermissionOrComputeDefault = (action, subject, permissions) => {
if (subject in permissions) {
const subjectPermissions = permissions[subject];
if (action in subjectPermissions) {
return subjectPermissions[action];
}
const actionLevel = actionLevels[action];
const actionFallback = Object.keys(actionLevels).find(actionFallbackCandidate => actionLevels[actionFallbackCandidate] > actionLevel && actionFallbackCandidate in subjectPermissions);
if (actionFallback) {
return subjectPermissions[actionFallback];
}
}
const subjectLevel = subjectLevels[subject]; // do we have a subject with a stronger level (group or owner)
// where we could read the action permission ?
const subjectFallback = Object.keys(subjectLevels).find(subjectFallbackCandidate => subjectLevels[subjectFallbackCandidate] > subjectLevel && subjectFallbackCandidate in permissions);
if (subjectFallback) {
const subjectPermissions = permissions[subjectFallback];
return action in subjectPermissions ? subjectPermissions[action] : getPermissionOrComputeDefault(action, subjectFallback, permissions);
}
return false;
};
const isWindows$3 = process.platform === "win32";
const readFileSystemNodeStat = async (source, {
nullIfNotFound = false,
followLink = true
} = {}) => {
if (source.endsWith("/")) source = source.slice(0, -1);
const sourceUrl = assertAndNormalizeFileUrl(source);
const sourcePath = urlToFileSystemPath(sourceUrl);
const handleNotFoundOption = nullIfNotFound ? {
handleNotFoundError: () => null
} : {};
return readStat(sourcePath, {
followLink,
...handleNotFoundOption,
...(isWindows$3 ? {
// Windows can EPERM on stat
handlePermissionDeniedError: async error => {
console.error(`trying to fix windows EPERM after stats on ${sourcePath}`);
try {
// unfortunately it means we mutate the permissions
// without being able to restore them to the previous value
// (because reading current permission would also throw)
await writeFileSystemNodePermissions(sourceUrl, 0o666);
const stats = await readStat(sourcePath, {
followLink,
...handleNotFoundOption,
// could not fix the permission error, give up and throw original error
handlePermissionDeniedError: () => {
console.error(`still got EPERM after stats on ${sourcePath}`);
throw error;
}
});
return stats;
} catch (e) {
console.error(`error while trying to fix windows EPERM after stats on ${sourcePath}: ${e.stack}`);
throw error;
}
}
} : {})
});
};
const readStat = (sourcePath, {
followLink,
handleNotFoundError = null,
handlePermissionDeniedError = null
} = {}) => {
const nodeMethod = followLink ? fs.stat : fs.lstat;
return new Promise((resolve, reject) => {
nodeMethod(sourcePath, (error, statsObject) => {
if (error) {
if (handleNotFoundError && error.code === "ENOENT") {
resolve(handleNotFoundError(error));
} else if (handlePermissionDeniedError && (error.code === "EPERM" || error.code === "EACCES")) {
resolve(handlePermissionDeniedError(error));
} else {
reject(error);
}
} else {
resolve(statsObject);
}
});
});
};
const assertDirectoryPresence = async source => {
const sourceUrl = assertAndNormalizeFileUrl(source);
const sourcePath = urlToFileSystemPath(sourceUrl);
const sourceStats = await readFileSystemNodeStat(sourceUrl, {
nullIfNotFound: true
});
if (!sourceStats) {
throw new Error(`directory not found at ${sourcePath}`);
}
if (!sourceStats.isDirectory()) {
throw new Error(`directory expected at ${sourcePath} and found ${statsToType(sourceStats)} instead`);
}
};
const assertFilePresence = async source => {
const sourceUrl = assertAndNormalizeFileUrl(source);
const sourcePath = urlToFileSystemPath(sourceUrl);
const sourceStats = await readFileSystemNodeStat(sourceUrl, {
nullIfNotFound: true
});
if (!sourceStats) {
throw new Error(`file not found at ${sourcePath}`);
}
if (!sourceStats.isFile()) {
throw new Error(`file expected at ${sourcePath} and found ${statsToType(sourceStats)} instead`);
}
};
const ETAG_FOR_EMPTY_CONTENT = '"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk"';
const bufferToEtag = buffer => {
if (!Buffer.isBuffer(buffer)) {
throw new TypeError(`buffer expected, got ${buffer}`);
}
if (buffer.length === 0) {
return ETAG_FOR_EMPTY_CONTENT;
}
const hash = crypto.createHash("sha1");
hash.update(buffer, "utf8");
const hashBase64String = hash.digest("base64");
const hashBase64StringSubset = hashBase64String.slice(0, 27);
const length = buffer.length;
return `"${length.toString(16)}-${hashBase64StringSubset}"`;
};
const readDirectory = async (url, {
emfileMaxWait = 1000
} = {}) => {
const directoryUrl = assertAndNormalizeDirectoryUrl(url);
const directoryPath = urlToFileSystemPath(directoryUrl);
const startMs = Date.now();
let attemptCount = 0;
const attempt = () => {
return readdirNaive(directoryPath, {
handleTooManyFilesOpenedError: async error => {
attemptCount++;
const nowMs = Date.now();
const timeSpentWaiting = nowMs - startMs;
if (timeSpentWaiting > emfileMaxWait) {
throw error;
}
return new Promise(resolve => {
setTimeout(() => {
resolve(attempt());
}, attemptCount);
});
}
});
};
return attempt();
};
const readdirNaive = (directoryPath, {
handleTooManyFilesOpenedError = null
} = {}) => {
return new Promise((resolve, reject) => {
fs.readdir(directoryPath, (error, names) => {
if (error) {
// https://nodejs.org/dist/latest-v13.x/docs/api/errors.html#errors_common_system_errors
if (handleTooManyFilesOpenedError && (error.code === "EMFILE" || error.code === "ENFILE")) {
resolve(handleTooManyFilesOpenedError(error));
} else {
reject(error);
}
} else {
resolve(names);
}
});
});
};
const getCommonPathname = (pathname, otherPathname) => {
const firstDifferentCharacterIndex = findFirstDifferentCharacterIndex(pathname, otherPathname); // pathname and otherpathname are exactly the same
if (firstDifferentCharacterIndex === -1) {
return pathname;
}
const commonString = pathname.slice(0, firstDifferentCharacterIndex + 1); // the first different char is at firstDifferentCharacterIndex
if (pathname.charAt(firstDifferentCharacterIndex) === "/") {
return commonString;
}
if (otherPathname.charAt(firstDifferentCharacterIndex) === "/") {
return commonString;
}
const firstDifferentSlashIndex = commonString.lastIndexOf("/");
return pathname.slice(0, firstDifferentSlashIndex + 1);
};
const findFirstDifferentCharacterIndex = (string, otherString) => {
const maxCommonLength = Math.min(string.length, otherString.length);
let i = 0;
while (i < maxCommonLength) {
const char = string.charAt(i);
const otherChar = otherString.charAt(i);
if (char !== otherChar) {
return i;
}
i++;
}
if (string.length === otherString.length) {
return -1;
} // they differ at maxCommonLength
return maxCommonLength;
};
const pathnameToParentPathname = pathname => {
const slashLastIndex = pathname.lastIndexOf("/");
if (slashLastIndex === -1) {
return "/";
}
return pathname.slice(0, slashLastIndex + 1);
};
const urlToRelativeUrl = (urlArg, baseUrlArg) => {
const url = new URL(urlArg);
const baseUrl = new URL(baseUrlArg);
if (url.protocol !== baseUrl.protocol) {
return urlArg;
}
if (url.username !== baseUrl.username || url.password !== baseUrl.password) {
return urlArg.slice(url.protocol.length);
}
if (url.host !== baseUrl.host) {
return urlArg.slice(url.protocol.length);
}
const {
pathname,
hash,
search
} = url;
if (pathname === "/") {
return baseUrl.pathname.slice(1);
}
const {
pathname: basePathname
} = baseUrl;
const commonPathname = getCommonPathname(pathname, basePathname);
if (!commonPathname) {
return urlArg;
}
const specificPathname = pathname.slice(commonPathname.length);
const baseSpecificPathname = basePathname.slice(commonPathname.length);
if (baseSpecificPathname.includes("/")) {
const baseSpecificParentPathname = pathnameToParentPathname(baseSpecificPathname);
const relativeDirectoriesNotation = baseSpecificParentPathname.replace(/.*?\//g, "../");
return `${relativeDirectoriesNotation}${specificPathname}${search}${hash}`;
}
return `${specificPathname}${search}${hash}`;
};
const comparePathnames = (leftPathame, rightPathname) => {
const leftPartArray = leftPathame.split("/");
const rightPartArray = rightPathname.split("/");
const leftLength = leftPartArray.length;
const rightLength = rightPartArray.length;
const maxLength = Math.max(leftLength, rightLength);
let i = 0;
while (i < maxLength) {
const leftPartExists = (i in leftPartArray);
const rightPartExists = (i in rightPartArray); // longer comes first
if (!leftPartExists) {
return +1;
}
if (!rightPartExists) {
return -1;
}
const leftPartIsLast = i === leftPartArray.length - 1;
const rightPartIsLast = i === rightPartArray.length - 1; // folder comes first
if (leftPartIsLast && !rightPartIsLast) {
return +1;
}
if (!leftPartIsLast && rightPartIsLast) {
return -1;
}
const leftPart = leftPartArray[i];
const rightPart = rightPartArray[i];
i++; // local comparison comes first
const comparison = leftPart.localeCompare(rightPart);
if (comparison !== 0) {
return comparison;
}
}
if (leftLength < rightLength) {
return +1;
}
if (leftLength > rightLength) {
return -1;
}
return 0;
};
const collectDirectoryMatchReport = async ({
cancellationToken = cancellation.createCancellationToken(),
directoryUrl,
structuredMetaMap,
predicate
}) => {
const matchingArray = [];
const ignoredArray = [];
const rootDirectoryUrl = assertAndNormalizeDirectoryUrl(directoryUrl);
if (typeof predicate !== "function") {
throw new TypeError(`predicate must be a function, got ${predicate}`);
}
const structuredMetaMapNormalized = urlMeta.normalizeStructuredMetaMap(structuredMetaMap, rootDirectoryUrl);
const visitDirectory = async directoryUrl => {
const directoryItems = await cancellation.createOperation({
cancellationToken,
start: () => readDirectory(directoryUrl)
});
await Promise.all(directoryItems.map(async directoryItem => {
const directoryChildNodeUrl = `${directoryUrl}${directoryItem}`;
const relativeUrl = urlToRelativeUrl(directoryChildNodeUrl, rootDirectoryUrl);
const directoryChildNodeStats = await cancellation.createOperation({
cancellationToken,
start: () => readFileSystemNodeStat(directoryChildNodeUrl, {
// we ignore symlink because recursively traversed
// so symlinked file will be discovered.
// Moreover if they lead outside of directoryPath it can become a problem
// like infinite recursion of whatever.
// that we could handle using an object of pathname already seen but it will be useless
// because directoryPath is recursively traversed
followLink: false
})
});
if (directoryChildNodeStats.isDirectory()) {
const subDirectoryUrl = `${directoryChildNodeUrl}/`;
if (!urlMeta.urlCanContainsMetaMatching({
url: subDirectoryUrl,
structuredMetaMap: structuredMetaMapNormalized,
predicate
})) {
ignoredArray.push({
relativeUrl: ensureUrlTrailingSlash(relativeUrl),
fileStats: directoryChildNodeStats
});
return;
}
await visitDirectory(subDirectoryUrl);
return;
}
if (directoryChildNodeStats.isFile()) {
const meta = urlMeta.urlToMeta({
url: directoryChildNodeUrl,
structuredMetaMap: structuredMetaMapNormalized
});
if (!predicate(meta)) {
ignoredArray.push({
relativeUrl,
meta,
fileStats: directoryChildNodeStats
});
return;
}
matchingArray.push({
relativeUrl,
meta,
fileStats: directoryChildNodeStats
});
return;
}
}));
};
await visitDirectory(rootDirectoryUrl);
return {
matchingArray: sortByRelativeUrl(matchingArray),
ignoredArray: sortByRelativeUrl(ignoredArray)
};
};
const sortByRelativeUrl = array => array.sort((left, right) => {
return comparePathnames(left.relativeUrl, right.relativeUrl);
});
const collectFiles = async ({
cancellationToken = cancellation.createCancellationToken(),
directoryUrl,
structuredMetaMap,
predicate,
matchingFileOperation = () => null
}) => {
const rootDirectoryUrl = assertAndNormalizeDirectoryUrl(directoryUrl);
if (typeof predicate !== "function") {
throw new TypeError(`predicate must be a function, got ${predicate}`);
}
if (typeof matchingFileOperation !== "function") {
throw new TypeError(`matchingFileOperation must be a function, got ${matchingFileOperation}`);
}
const structuredMetaMapNormalized = urlMeta.normalizeStructuredMetaMap(structuredMetaMap, rootDirectoryUrl);
const matchingFileResultArray = [];
const visitDirectory = async directoryUrl => {
const directoryItems = await cancellation.createOperation({
cancellationToken,
start: () => readDirectory(directoryUrl)
});
await Promise.all(directoryItems.map(async directoryItem => {
const directoryChildNodeUrl = `${directoryUrl}${directoryItem}`;
const directoryChildNodeStats = await cancellation.createOperation({
cancellationToken,
start: () => readFileSystemNodeStat(directoryChildNodeUrl, {
// we ignore symlink because recursively traversed
// so symlinked file will be discovered.
// Moreover if they lead outside of directoryPath it can become a problem
// like infinite recursion of whatever.
// that we could handle using an object of pathname already seen but it will be useless
// because directoryPath is recursively traversed
followLink: false
})
});
if (directoryChildNodeStats.isDirectory()) {
const subDirectoryUrl = `${directoryChildNodeUrl}/`;
if (!urlMeta.urlCanContainsMetaMatching({
url: subDirectoryUrl,
structuredMetaMap: structuredMetaMapNormalized,
predicate
})) {
return;
}
await visitDirectory(subDirectoryUrl);
return;
}
if (directoryChildNodeStats.isFile()) {
const meta = urlMeta.urlToMeta({
url: directoryChildNodeUrl,
structuredMetaMap: structuredMetaMapNormalized
});
if (!predicate(meta)) return;
const relativeUrl = urlToRelativeUrl(directoryChildNodeUrl, rootDirectoryUrl);
const operationResult = await cancellation.createOperation({
cancellationToken,
start: () => matchingFileOperation({
cancellationToken,
relativeUrl,
meta,
fileStats: directoryChildNodeStats
})
});
matchingFileResultArray.push({
relativeUrl,
meta,
fileStats: directoryChildNodeStats,
operationResult
});
return;
}
}));
};
await visitDirectory(rootDirectoryUrl); // When we operate on thoose files later it feels more natural
// to perform operation in the same order they appear in the filesystem.
// It also allow to get a predictable return value.
// For that reason we sort matchingFileResultArray
matchingFileResultArray.sort((leftFile, rightFile) => {
return comparePathnames(leftFile.relativeUrl, rightFile.relativeUrl);
});
return matchingFileResultArray;
};
const {
mkdir
} = fs.promises;
const writeDirectory = async (destination, {
recursive = true,
allowUseless = false
} = {}) => {
const destinationUrl = assertAndNormalizeDirectoryUrl(destination);
const destinationPath = urlToFileSystemPath(destinationUrl);
const destinationStats = await readFileSystemNodeStat(destinationUrl, {
nullIfNotFound: true,
followLink: false
});
if (destinationStats) {
if (destinationStats.isDirectory()) {
if (allowUseless) {
return;
}
throw new Error(`directory already exists at ${destinationPath}`);
}
const destinationType = statsToType(destinationStats);
throw new Error(`cannot write directory at ${destinationPath} because there is a ${destinationType}`);
}
try {
await mkdir(destinationPath, {
recursive
});
} catch (error) {
if (allowUseless && error.code === "EEXIST") {
return;
}
throw error;
}
};
const resolveUrl = (specifier, baseUrl) => {
if (typeof baseUrl === "undefined") {
throw new TypeError(`baseUrl missing to resolve ${specifier}`);
}
return String(new URL(specifier, baseUrl));
};
const removeFileSystemNode = async (source, {
allowUseless = false,
recursive = false,
maxRetries = 3,
retryDelay = 100,
onlyContent = false
} = {}) => {
const sourceUrl = assertAndNormalizeFileUrl(source);
const sourceStats = await readFileSystemNodeStat(sourceUrl, {
nullIfNotFound: true,
followLink: false
});
if (!sourceStats) {
if (allowUseless) {
return;
}
throw new Error(`nothing to remove at ${urlToFileSystemPath(sourceUrl)}`);
} // https://nodejs.org/dist/latest-v13.x/docs/api/fs.html#fs_class_fs_stats
// FIFO and socket are ignored, not sure what they are exactly and what to do with them
// other libraries ignore them, let's do the same.
if (sourceStats.isFile() || sourceStats.isSymbolicLink() || sourceStats.isCharacterDevice() || sourceStats.isBlockDevice()) {
await removeNonDirectory(sourceUrl.endsWith("/") ? sourceUrl.slice(0, -1) : sourceUrl, {
maxRetries,
retryDelay
});
} else if (sourceStats.isDirectory()) {
await removeDirectory(ensureUrlTrailingSlash(sourceUrl), {
recursive,
maxRetries,
retryDelay,
onlyContent
});
}
};
const removeNonDirectory = (sourceUrl, {
maxRetries,
retryDelay
}) => {
const sourcePath = urlToFileSystemPath(sourceUrl);
let retryCount = 0;
const attempt = () => {
return unlinkNaive(sourcePath, { ...(retryCount >= maxRetries ? {} : {
handleTemporaryError: async () => {
retryCount++;
return new Promise(resolve => {
setTimeout(() => {
resolve(attempt());
}, retryCount * retryDelay);
});
}
})
});
};
return attempt();
};
const unlinkNaive = (sourcePath, {
handleTemporaryError = null
} = {}) => {
return new Promise((resolve, reject) => {
fs.unlink(sourcePath, error => {
if (error) {
if (error.code === "ENOENT") {
resolve();
} else if (handleTemporaryError && (error.code === "EBUSY" || error.code === "EMFILE" || error.code === "ENFILE" || error.code === "ENOENT")) {
resolve(handleTemporaryError(error));
} else {
reject(error);
}
} else {
resolve();
}
});
});
};
const removeDirectory = async (rootDirectoryUrl, {
maxRetries,
retryDelay,
recursive,
onlyContent
}) => {
const visit = async sourceUrl => {
const sourceStats = await readFileSystemNodeStat(sourceUrl, {
nullIfNotFound: true,
followLink: false
}); // file/directory not found
if (sourceStats === null) {
return;
}
if (sourceStats.isFile() || sourceStats.isCharacterDevice() || sourceStats.isBlockDevice()) {
await visitFile(sourceUrl);
} else if (sourceStats.isSymbolicLink()) {
await visitSymbolicLink(sourceUrl);
} else if (sourceStats.isDirectory()) {
await visitDirectory(`${sourceUrl}/`);
}
};
const visitDirectory = async directoryUrl => {
const directoryPath = urlToFileSystemPath(directoryUrl);
const optionsFromRecursive = recursive ? {
handleNotEmptyError: async () => {
await removeDirectoryContent(directoryUrl);
await visitDirectory(directoryUrl);
}
} : {};
await removeDirectoryNaive(directoryPath, { ...optionsFromRecursive,
// Workaround for https://github.com/joyent/node/issues/4337
...(process.platform === "win32" ? {
handlePermissionError: async error => {
console.error(`trying to fix windows EPERM after readir on ${directoryPath}`);
let openOrCloseError;
try {
const fd = fs.openSync(directoryPath);
fs.closeSync(fd);
} catch (e) {
openOrCloseError = e;
}
if (openOrCloseError) {
if (openOrCloseError.code === "ENOENT") {
return;
}
console.error(`error while trying to fix windows EPERM after readir on ${directoryPath}: ${openOrCloseError.stack}`);
throw error;
}
await removeDirectoryNaive(directoryPath, { ...optionsFromRecursive
});
}
} : {})
});
};
const removeDirectoryContent = async directoryUrl => {
const names = await readDirectory(directoryUrl);
await Promise.all(names.map(async name => {
const url = resolveUrl(name, directoryUrl);
await visit(url);
}));
};
const visitFile = async fileUrl => {
await removeNonDirectory(fileUrl, {
maxRetries,
retryDelay
});
};
const visitSymbolicLink = async symbolicLinkUrl => {
await removeNonDirectory(symbolicLinkUrl, {
maxRetries,
retryDelay
});
};
if (onlyContent) {
await removeDirectoryContent(rootDirectoryUrl);
} else {
await visitDirectory(rootDirectoryUrl);
}
};
const removeDirectoryNaive = (directoryPath, {
handleNotEmptyError = null,
handlePermissionError = null
} = {}) => {
return new Promise((resolve, reject) => {
fs.rmdir(directoryPath, (error, lstatObject) => {
if (error) {
if (handlePermissionError && error.code === "EPERM") {
resolve(handlePermissionError(error));
} else if (error.code === "ENOENT") {
resolve();
} else if (handleNotEmptyError && ( // linux os
error.code === "ENOTEMPTY" || // SunOS
error.code === "EEXIST")) {
resolve(handleNotEmptyError(error));
} else {
reject(error);
}
} else {
resolve(lstatObject);
}
});
});
};
const ensureEmptyDirectory = async source => {
const stats = await readFileSystemNodeStat(source, {
nullIfNotFound: true,
followLink: false
});
if (stats === null) {
// if there is nothing, create a directory
return writeDirectory(source, {
allowUseless: true
});
}
if (stats.isDirectory()) {
// if there is a directory remove its content and done
return removeFileSystemNode(source, {
allowUseless: true,
recursive: true,
onlyContent: true
});
}
const sourceType = statsToType(stats);
const sourcePath = urlToFileSystemPath(assertAndNormalizeFileUrl(source));
throw new Error(`ensureEmptyDirectory expect directory at ${sourcePath}, found ${sourceType} instead`);
};
const ensureParentDirectories = async destination => {
const destinationUrl = assertAndNormalizeFileUrl(destination);
const destinationPath = urlToFileSystemPath(destinationUrl);
const destinationParentPath = path.dirname(destinationPath);
return writeDirectory(destinationParentPath, {
recursive: true,
allowUseless: true
});
};
const isWindows$2 = process.platform === "win32";
const baseUrlFallback = fileSystemPathToUrl(process.cwd());
/**
* Some url might be resolved or remapped to url without the windows drive letter.
* For instance
* new URL('/foo.js', 'file:///C:/dir/file.js')
* resolves to
* 'file:///foo.js'
*
* But on windows it becomes a problem because we need the drive letter otherwise
* url cannot be converted to a filesystem path.
*
* ensureWindowsDriveLetter ensure a resolved url still contains the drive letter.
*/
const ensureWindowsDriveLetter = (url, baseUrl) => {
try {
url = String(new URL(url));
} catch (e) {
throw new Error(`absolute url expected but got ${url}`);
}
if (!isWindows$2) {
return url;
}
try {
baseUrl = String(new URL(baseUrl));
} catch (e) {
throw new Error(`absolute baseUrl expected but got ${baseUrl} to ensure windows drive letter on ${url}`);
}
if (!url.startsWith("file://")) {
return url;
}
const afterProtocol = url.slice("file://".length); // we still have the windows drive letter
if (extractDriveLetter(afterProtocol)) {
return url;
} // drive letter was lost, restore it
const baseUrlOrFallback = baseUrl.startsWith("file://") ? baseUrl : baseUrlFallback;
const driveLetter = extractDriveLetter(baseUrlOrFallback.slice("file://".length));
if (!driveLetter) {
throw new Error(`drive letter expected on baseUrl but got ${baseUrl} to ensure windows drive letter on ${url}`);
}
return `file:///${driveLetter}:${afterProtocol}`;
};
const extractDriveLetter = ressource => {
// we still have the windows drive letter
if (/[a-zA-Z]/.test(ressource[1]) && ressource[2] === ":") {
return ressource[1];
}
return null;
};
const urlTargetsSameFileSystemPath = (leftUrl, rightUrl) => {
if (leftUrl.endsWith("/")) leftUrl = leftUrl.slice(0, -1);
if (rightUrl.endsWith("/")) rightUrl = rightUrl.slice(0, -1);
return leftUrl === rightUrl;
};
const writeFileSystemNodeModificationTime = (source, mtime) => {
const sourceUrl = assertAndNormalizeFileUrl(source);
const sourcePath = urlToFileSystemPath(sourceUrl);
const mtimeValue = typeof mtime === "number" ? new Date(Math.floor(mtime)) : mtime; // reading atime mutates its value so there is no use case I can think of
// where we want to modify it
const atimeValue = mtimeValue;
return new Promise((resolve, reject) => {
fs.utimes(sourcePath, atimeValue, mtimeValue, error => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
};
const replaceBackSlashesWithSlashes = string => string.replace(/\\/g, "/");
const readSymbolicLink = url => {
const symbolicLinkUrl = assertAndNormalizeFileUrl(url);
const symbolicLinkPath = urlToFileSystemPath(symbolicLinkUrl);
return new Promise((resolve, reject) => {
fs.readlink(symbolicLinkPath, (error, resolvedPath) => {
if (error) {
reject(error);
} else {
resolve(isFileSystemPath(resolvedPath) ? fileSystemPathToUrl(resolvedPath) : replaceBackSlashesWithSlashes(resolvedPath));
}
});
});
};
const {
symlink
} = fs.promises;
const isWindows$1 = process.platform === "win32";
const writeSymbolicLink = async (destination, target, {
type
} = {}) => {
const destinationUrl = assertAndNormalizeFileUrl(destination);
let targetValue;
if (typeof target === "string") {
// absolute filesystem path
if (isFileSystemPath(target)) {
targetValue = target;
} // relative url
else if (target.startsWith("./") || target.startsWith("../")) {
targetValue = target;
} // absolute url
else {
const targetUrl = String(new URL(target, destinationUrl));
targetValue = urlToFileSystemPath(targetUrl);
}
} else if (target instanceof URL) {
targetValue = urlToFileSystemPath(target);
} else {
throw new TypeError(`symbolic link target must be a string or an url, received ${target}`);
}
if (isWindows$1 && typeof type === "undefined") {
// without this if you write a symbolic link without specifying the type on windows
// you later get EPERM when doing stat on the symlink
const targetUrl = resolveUrl(targetValue, destinationUrl);
const targetStats = await readFileSystemNodeStat(targetUrl, {
nullIfNotFound: true
});
type = targetStats && targetStats.isDirectory() ? "dir" : "file";
}
const symbolicLinkPath = urlToFileSystemPath(destinationUrl);
try {
await symlink(targetValue, symbolicLinkPath, type);
} catch (error) {
if (error.code === "ENOENT") {
await ensureParentDirectories(destinationUrl);
await symlink(targetValue, symbolicLinkPath, type);
return;
}
throw error;
}
};
const urlIsInsideOf = (urlValue, otherUrlValue) => {
const url = new URL(urlValue);
const otherUrl = new URL(otherUrlValue);
if (url.origin !== otherUrl.origin) {
return false;
}
const urlPathname = url.pathname;
const otherUrlPathname = otherUrl.pathname;
if (urlPathname === otherUrlPathname) {
return false;
}
return urlPathname.startsWith(otherUrlPathname);
};
/* eslint-disable import/max-dependencies */
const copyFileSystemNode = async (source, destination, {
overwrite = false,
preserveStat = true,
preserveMtime = preserveStat,
preservePermissions = preserveStat,
allowUseless = false,
followLink = true
} = {}) => {
const sourceUrl = assertAndNormalizeFileUrl(source);
let destinationUrl = assertAndNormalizeFileUrl(destination);
const sourcePath = urlToFileSystemPath(sourceUrl);
const sourceStats = await readFileSystemNodeStat(sourceUrl, {
nullIfNotFound: true,
followLink: false
});
if (!sourceStats) {
throw new Error(`nothing to copy at ${sourcePath}`);
}
let destinationStats = await readFileSystemNodeStat(destinationUrl, {
nullIfNotFound: true,
// we force false here but in fact we will follow the destination link
// to know where we will actually move and detect useless move overrite etc..
followLink: false
});
if (followLink && destinationStats && destinationStats.isSymbolicLink()) {
const target = await readSymbolicLink(destinationUrl);
destinationUrl = resolveUrl(target, destinationUrl);
destinationStats = await readFileSystemNodeStat(destinationUrl, {
nullIfNotFound: true
});
}
const destinationPath = urlToFileSystemPath(destinationUrl);
if (urlTargetsSameFileSystemPath(sourceUrl, destinationUrl)) {
if (allowUseless) {
return;
}
throw new Error(`cannot copy ${sourcePath} because destination and source are the same`);
}
if (destinationStats) {
const sourceType = statsToType(sourceStats);
const destinationType = statsToType(destinationStats);
if (sourceType !== destinationType) {
throw new Error(`cannot copy ${sourceType} from ${sourcePath} to ${destinationPath} because destination exists and is not a ${sourceType} (it's a ${destinationType})`);
}
if (!overwrite) {
throw new Error(`cannot copy ${sourceType} from ${sourcePath} to ${destinationPath} because destination exists and overwrite option is disabled`);
} // remove file, link, directory...
await removeFileSystemNode(destinationUrl, {
recursive: true,
allowUseless: true
});
} else {
await ensureParentDirectories(destinationUrl);
}
if (sourceStats.isDirectory()) {
destinationUrl = ensureUrlTrailingSlash(destinationUrl);
}
const visit = async (url, stats) => {
if (stats.isFile() || stats.isCharacterDevice() || stats.isBlockDevice()) {
await visitFile(url, stats);
} else if (stats.isSymbolicLink()) {
await visitSymbolicLink(url);
} else if (stats.isDirectory()) {
await visitDirectory(ensureUrlTrailingSlash(url), stats);
}
};
const visitFile = async (fileUrl, fileStats) => {
const fileRelativeUrl = urlToRelativeUrl(fileUrl, sourceUrl);
const fileCopyUrl = resolveUrl(fileRelativeUrl, destinationUrl);
await copyFileContentNaive(urlToFileSystemPath(fileUrl), urlToFileSystemPath(fileCopyUrl));
await copyStats(fileCopyUrl, fileStats);
};
const visitSymbolicLink = async symbolicLinkUrl => {
const symbolicLinkRelativeUrl = urlToRelativeUrl(symbolicLinkUrl, sourceUrl);
const symbolicLinkTarget = await readSymbolicLink(symbolicLinkUrl);
const symbolicLinkTargetUrl = resolveUrl(symbolicLinkTarget, symbolicLinkUrl);
const linkIsRelative = symbolicLinkTarget.startsWith("./") || symbolicLinkTarget.startsWith("../");
let symbolicLinkCopyTarget;
if (symbolicLinkTargetUrl === sourceUrl) {
symbolicLinkCopyTarget = linkIsRelative ? symbolicLinkTarget : destinationUrl;
} else if (urlIsInsideOf(symbolicLinkTargetUrl, sourceUrl)) {
// symbolic link targets something inside the directory we want to copy
// reflects it inside the copied directory structure
const linkCopyTargetRelative = urlToRelativeUrl(symbolicLinkTargetUrl, sourceUrl);
symbolicLinkCopyTarget = linkIsRelative ? `./${linkCopyTargetRelative}` : resolveUrl(linkCopyTargetRelative, destinationUrl);
} else {
// symbolic link targets something outside the directory we want to copy
symbolicLinkCopyTarget = symbolicLinkTarget;
} // we must guess ourself the type of the symlink
// because the destination might not exists because not yet copied
// https://nodejs.org/dist/latest-v13.x/docs/api/fs.html#fs_fs_symlink_target_path_type_callback
const targetStats = await readFileSystemNodeStat(symbolicLinkTargetUrl, {
nullIfNotFound: true,
followLink: false
});
const linkType = targetStats && targetStats.isDirectory() ? "dir" : "file";
const symbolicLinkCopyUrl = resolveUrl(symbolicLinkRelativeUrl, destinationUrl);
await writeSymbolicLink(symbolicLinkCopyUrl, symbolicLinkCopyTarget, {
type: linkType
});
};
const copyStats = async (destinationUrl, stats) => {
if (preservePermissions || preserveMtime) {
const {
mode,
mtimeMs
} = stats;
if (preservePermissions) {
await writeFileSystemNodePermissions(destinationUrl, binaryFlagsToPermissions(mode));
}
if (preserveMtime) {
await writeFileSystemNodeModificationTime(destinationUrl, mtimeMs);
}
}
};
const visitDirectory = async (directoryUrl, directoryStats) => {
const directoryRelativeUrl = urlToRelativeUrl(directoryUrl, sourceUrl);
const directoryCopyUrl = resolveUrl(directoryRelativeUrl, destinationUrl);
await writeDirectory(directoryCopyUrl);
await copyDirectoryContent(directoryUrl);
await copyStats(directoryCopyUrl, directoryStats);
};
const copyDirectoryContent = async directoryUrl => {
const names = await readDirectory(directoryUrl);
await Promise.all(names.map(async name => {
const fileSystemNodeUrl = resolveUrl(name, directoryUrl);
const stats = await readFileSystemNodeStat(fileSystemNodeUrl, {
followLink: false
});
await visit(fileSystemNodeUrl, stats);
}));
};
await visit(sourceUrl, sourceStats);
};
const copyFileContentNaive = (filePath, fileDestinationPath) => {
return new Promise((resolve, reject) => {
fs.copyFile(filePath, fileDestinationPath, error => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
};
const {
stat
} = fs.promises;
const readFileSystemNodePermissions = async source => {
const sourceUrl = assertAndNormalizeFileUrl(source);
const sourcePath = urlToFileSystemPath(sourceUrl);
const {
mode
} = await stat(sourcePath);
return binaryFlagsToPermissions(mode);
};
const grantPermissionsOnFileSystemNode = async (source, {
read = false,
write = false,
execute = false
}) => {
const sourceUrl = assertAndNormalizeFileUrl(source);
const filePermissions = await readFileSystemNodePermissions(sourceUrl);
await writeFileSystemNodePermissions(sourceUrl, {
owner: {
read,
write,
execute
},
group: {
read,
write,
execute
},
others: {
read,
write,
execute
}
});
return async () => {
await writeFileSystemNodePermissions(sourceUrl, filePermissions);
};
};
const memoize = compute => {
let memoized = false;
let memoizedValue;
const fnWithMemoization = (...args) => {
if (memoized) {
return memoizedValue;
} // if compute is recursive wait for it to be fully done before storing the value
// so set memoized boolean after the call
memoizedValue = compute(...args);
memoized = true;
return memoizedValue;
};
fnWithMemoization.forget = () => {
const value = memoizedValue;
memoized = false;
memoizedValue = undefined;
return value;
};
return fnWithMemoization;
};
/* eslint-disable import/max-dependencies */
const moveFileSystemNode = async (source, destination, {
overwrite = false,
allowUseless = false,
followLink = true
} = {}) => {
const sourceUrl = assertAndNormalizeFileUrl(source);
let destinationUrl = assertAndNormalizeFileUrl(destination);
const sourcePath = urlToFileSystemPath(sourceUrl);
const sourceStats = await readFileSystemNodeStat(sourceUrl, {
nullIfNotFound: true,
followLink: false
});
if (!sourceStats) {
throw new Error(`nothing to move from ${sourcePath}`);
}
let destinationStats = await readFileSystemNodeStat(destinationUrl, {
nullIfNotFound: true,
// we force false here but in fact we will follow the destination link
// to know where we will actually move and detect useless move overrite etc..
followLink: false
});
if (followLink && destinationStats && destinationStats.isSymbolicLink()) {
const target = await readSymbolicLink(destinationUrl);
destinationUrl = resolveUrl(target, destinationUrl);
destinationStats = await readFileSystemNodeStat(destinationUrl, {
nullIfNotFound: true
});
}
const destinationPath = urlToFileSystemPath(destinationUrl);
if (urlTargetsSameFileSystemPath(sourceUrl, destinationUrl)) {
if (allowUseless) {
return;
}
throw new Error(`no move needed for ${sourcePath} because destination and source are the same`);
}
if (destinationStats) {
const sourceType = statsToType(sourceStats);
const destinationType = statsToType(destinationStats);
if (sourceType !== destinationType) {
throw new Error(`cannot move ${sourceType} from ${sourcePath} to ${destinationPath} because destination exists and is not a ${sourceType} (it's a ${destinationType})`);
}
if (!overwrite) {
throw new Error(`cannot move ${sourceType} from ${sourcePath} to ${destinationPath} because destination exists and overwrite option is disabled`);
} // remove file, link, directory...
await removeFileSystemNode(destinationUrl, {
recursive: true
});
} else {
await ensureParentDirectories(destinationUrl);
}
await moveNaive(sourcePath, destinationPath, {
handleCrossDeviceError: async () => {
await copyFileSystemNode(sourceUrl, destinationUrl, {
preserveStat: true
});
await removeFileSystemNode(sourceUrl, {
recursive: true
});
}
});
};
const moveNaive = (sourcePath, destinationPath, {
handleCrossDeviceError = null
} = {}) => {
return new Promise((resolve, reject) => {
fs.rename(sourcePath, destinationPath, error => {
if (error) {
if (handleCrossDeviceError && error.code === "EXDEV") {
resolve(handleCrossDeviceError(error));
} else {
reject(error);
}
} else {
resolve();
}
});
});
};
const moveDirectoryContent = async (source, destination, {
overwrite,
followLink = true
} = {}) => {
const sourceUrl = assertAndNormalizeDirectoryUrl(source);
let destinationUrl = assertAndNormalizeDirectoryUrl(destination);
const sourcePath = urlToFileSystemPath(sourceUrl);
const sourceStats = await readFileSystemNodeStat(sourceUrl, {
nullIfNotFound: true,
followLink: false
});
if (!sourceStats) {
throw new Error(`no directory to move content from at ${sourcePath}`);
}
if (!sourceStats.isDirectory()) {
const sourceType = statsToType(sourceStats);
throw new Error(`found a ${sourceType} instead of a directory at ${sourcePath}`);
}
let destinationStats = await readFileSystemNodeStat(destinationUrl, {
nullIfNotFound: true,
// we force false here but in fact we will follow the destination link
// to know where we will actually move and detect useless move overrite etc..
followLink: false
});
if (followLink && destinationStats && destinationStats.isSymbolicLink()) {
const target = await readSymbolicLink(destinationUrl);
destinationUrl = resolveUrl(target, destinationUrl);
destinationStats = await readFileSystemNodeStat(destinationUrl, {
nullIfNotFound: true
});
}
const destinationPath = urlToFileSystemPath(destinationUrl);
if (destinationStats === null) {
throw new Error(`no directory to move content into at ${destinationPath}`);
}
if (!destinationStats.isDirectory()) {
const destinationType = statsToType(destinationStats);
throw new Error(`destination leads to a ${destinationType} instead of a directory at ${destinationPath}`);
}
if (urlTargetsSameFileSystemPath(sourceUrl, destinationUrl)) {
throw new Error(`cannot move directory content, source and destination are the same (${sourcePath})`);
}
const directoryEntries = await readDirectory(sourceUrl);
await Promise.all(directoryEntries.map(async directoryEntry => {
const from = resolveUrl(directoryEntry, sourceUrl);
const to = resolveUrl(directoryEntry, destinationUrl);
await moveFileSystemNode(from, to, {
overwrite,
followLink
});
}));
};
const readFilePromisified = util.promisify(fs.readFile);
const readFile = async (value, {
as = "string"
} = {}) => {
const fileUrl = assertAndNormalizeFileUrl(value);
const filePath = urlToFileSystemPath(fileUrl);
const buffer = await readFilePromisified(filePath);
if (as === "buffer") {
return buffer;
}
if (as === "string") {
return buffer.toString();
}
if (as === "json") {
return JSON.parse(buffer.toString());
}
throw new Error(`as must be one of buffer,string,json, received ${as}.`);
};
const readFileSystemNodeModificationTime = async source => {
const stats = await readFileSystemNodeStat(source);
return Math.floor(stats.mtimeMs);
};
const fileSystemNodeToTypeOrNull = url => {
const path = urlToF