@pika/pack
Version:
package building, reimagined.
688 lines (687 loc) • 24.3 kB
JavaScript
import { promisify } from 'util';
import _rimraf from 'rimraf';
import _mkdirp from 'mkdirp';
import _glob from 'glob';
export const unlink = promisify(_rimraf);
export const glob = promisify(_glob);
export const mkdirp = promisify(_mkdirp);
//
// import {ReadStream} from 'fs';
// import Reporter from '../reporters/base-reporter.js';
// import {CopyFileAction} from './fs-normalized.js';
import * as os from 'os';
import * as path from 'path';
// import BlockingQueue from './blocking-queue.js';
// import * as promise from './promise.js';
import map from './map.js';
// import {copyFile, fileDatesEqual, unlink} from './fs-normalized.js';
// export const constants =
// typeof fs.constants !== 'undefined'
// ? fs.constants
// : {
// R_OK: fs.R_OK,
// W_OK: fs.W_OK,
// X_OK: fs.X_OK,
// };
// export const lockQueue = new BlockingQueue('fs lock');
import * as fs from 'fs';
import * as util from 'util';
export const open = util.promisify(fs.open);
export const writeFile = util.promisify(fs.writeFile);
export const readlink = util.promisify(fs.readlink);
export const realpath = util.promisify(fs.realpath);
export const readdir = util.promisify(fs.readdir);
export const rename = util.promisify(fs.rename);
export const access = util.promisify(fs.access);
export const stat = util.promisify(fs.stat);
export const exists = util.promisify(fs.exists);
export const lstat = util.promisify(fs.lstat);
export const chmod = util.promisify(fs.chmod);
export const link = util.promisify(fs.link);
export const copyFile = util.promisify(fs.copyFile);
const readFileBuffer = util.promisify(fs.readFile);
export const readFile = (path) => {
return util.promisify(fs.readFile)(path, { encoding: 'utf-8' });
};
// export {unlink};
// // fs.copyFile uses the native file copying instructions on the system, performing much better
// // than any JS-based solution and consumes fewer resources. Repeated testing to fine tune the
// // concurrency level revealed 128 as the sweet spot on a quad-core, 16 CPU Intel system with SSD.
// const CONCURRENT_QUEUE_ITEMS = fs.copyFile ? 128 : 4;
// const fsSymlink: (target: string, path: string, type?: 'dir' | 'file' | 'junction') => Promise<void> = promisify(
// fs.symlink,
// );
// import invariant from 'invariant';
import stripBOM from 'strip-bom';
// const noop = () => {};
// export type CopyQueueItem = {
// src: string,
// dest: string,
// type?: string,
// onFresh?: () => void,
// onDone?: () => void,
// };
// type CopyQueue = Array<CopyQueueItem>;
// type LinkFileAction = {
// src: string,
// dest: string,
// removeDest: boolean,
// };
// type CopySymlinkAction = {
// dest: string,
// linkname: string,
// };
// type CopyActions = {
// file: Array<CopyFileAction>,
// symlink: Array<CopySymlinkAction>,
// link: Array<LinkFileAction>,
// };
// type CopyOptions = {
// onProgress: (dest: string) => void,
// onStart: (num: number) => void,
// possibleExtraneous: Set<string>,
// ignoreBasenames: Array<string>,
// artifactFiles: Array<string>,
// };
// type FailedFolderQuery = {
// error: Error,
// folder: string,
// };
// type FolderQueryResult = {
// skipped: Array<FailedFolderQuery>,
// folder?: string,
// };
// async function buildActionsForCopy(
// queue: CopyQueue,
// events: CopyOptions,
// possibleExtraneous: Set<string>,
// reporter: Reporter,
// ): Promise<CopyActions> {
// const artifactFiles: Set<string> = new Set(events.artifactFiles || []);
// const files: Set<string> = new Set();
// // initialise events
// for (const item of queue) {
// const onDone = item.onDone;
// item.onDone = () => {
// events.onProgress(item.dest);
// if (onDone) {
// onDone();
// }
// };
// }
// events.onStart(queue.length);
// // start building actions
// const actions: CopyActions = {
// file: [],
// symlink: [],
// link: [],
// };
// // custom concurrency logic as we're always executing stacks of CONCURRENT_QUEUE_ITEMS queue items
// // at a time due to the requirement to push items onto the queue
// while (queue.length) {
// const items = queue.splice(0, CONCURRENT_QUEUE_ITEMS);
// await Promise.all(items.map(build));
// }
// // simulate the existence of some files to prevent considering them extraneous
// for (const file of artifactFiles) {
// if (possibleExtraneous.has(file)) {
// reporter.verbose(reporter.lang('verboseFilePhantomExtraneous', file));
// possibleExtraneous.delete(file);
// }
// }
// for (const loc of possibleExtraneous) {
// if (files.has(loc.toLowerCase())) {
// possibleExtraneous.delete(loc);
// }
// }
// return actions;
// //
// async function build(data: CopyQueueItem): Promise<void> {
// const {src, dest, type} = data;
// const onFresh = data.onFresh || noop;
// const onDone = data.onDone || noop;
// // TODO https://github.com/yarnpkg/yarn/issues/3751
// // related to bundled dependencies handling
// if (files.has(dest.toLowerCase())) {
// reporter.verbose(`The case-insensitive file ${dest} shouldn't be copied twice in one bulk copy`);
// } else {
// files.add(dest.toLowerCase());
// }
// if (type === 'symlink') {
// await mkdirp(path.dirname(dest));
// onFresh();
// actions.symlink.push({
// dest,
// linkname: src,
// });
// onDone();
// return;
// }
// if (events.ignoreBasenames.indexOf(path.basename(src)) >= 0) {
// // ignored file
// return;
// }
// const srcStat = await lstat(src);
// let srcFiles;
// if (srcStat.isDirectory()) {
// srcFiles = await readdir(src);
// }
// let destStat;
// try {
// // try accessing the destination
// destStat = await lstat(dest);
// } catch (e) {
// // proceed if destination doesn't exist, otherwise error
// if (e.code !== 'ENOENT') {
// throw e;
// }
// }
// // if destination exists
// if (destStat) {
// const bothSymlinks = srcStat.isSymbolicLink() && destStat.isSymbolicLink();
// const bothFolders = srcStat.isDirectory() && destStat.isDirectory();
// const bothFiles = srcStat.isFile() && destStat.isFile();
// // EINVAL access errors sometimes happen which shouldn't because node shouldn't be giving
// // us modes that aren't valid. investigate this, it's generally safe to proceed.
// /* if (srcStat.mode !== destStat.mode) {
// try {
// await access(dest, srcStat.mode);
// } catch (err) {}
// } */
// if (bothFiles && artifactFiles.has(dest)) {
// // this file gets changed during build, likely by a custom install script. Don't bother checking it.
// onDone();
// reporter.verbose(reporter.lang('verboseFileSkipArtifact', src));
// return;
// }
// if (bothFiles && srcStat.size === destStat.size && fileDatesEqual(srcStat.mtime, destStat.mtime)) {
// // we can safely assume this is the same file
// onDone();
// reporter.verbose(reporter.lang('verboseFileSkip', src, dest, srcStat.size, +srcStat.mtime));
// return;
// }
// if (bothSymlinks) {
// const srcReallink = await readlink(src);
// if (srcReallink === (await readlink(dest))) {
// // if both symlinks are the same then we can continue on
// onDone();
// reporter.verbose(reporter.lang('verboseFileSkipSymlink', src, dest, srcReallink));
// return;
// }
// }
// if (bothFolders) {
// // mark files that aren't in this folder as possibly extraneous
// const destFiles = await readdir(dest);
// invariant(srcFiles, 'src files not initialised');
// for (const file of destFiles) {
// if (srcFiles.indexOf(file) < 0) {
// const loc = path.join(dest, file);
// possibleExtraneous.add(loc);
// if ((await lstat(loc)).isDirectory()) {
// for (const file of await readdir(loc)) {
// possibleExtraneous.add(path.join(loc, file));
// }
// }
// }
// }
// }
// }
// if (destStat && destStat.isSymbolicLink()) {
// await unlink(dest);
// destStat = null;
// }
// if (srcStat.isSymbolicLink()) {
// onFresh();
// const linkname = await readlink(src);
// actions.symlink.push({
// dest,
// linkname,
// });
// onDone();
// } else if (srcStat.isDirectory()) {
// if (!destStat) {
// reporter.verbose(reporter.lang('verboseFileFolder', dest));
// await mkdirp(dest);
// }
// const destParts = dest.split(path.sep);
// while (destParts.length) {
// files.add(destParts.join(path.sep).toLowerCase());
// destParts.pop();
// }
// // push all files to queue
// invariant(srcFiles, 'src files not initialised');
// let remaining = srcFiles.length;
// if (!remaining) {
// onDone();
// }
// for (const file of srcFiles) {
// queue.push({
// dest: path.join(dest, file),
// onFresh,
// onDone: () => {
// if (--remaining === 0) {
// onDone();
// }
// },
// src: path.join(src, file),
// });
// }
// } else if (srcStat.isFile()) {
// onFresh();
// actions.file.push({
// src,
// dest,
// atime: srcStat.atime,
// mtime: srcStat.mtime,
// mode: srcStat.mode,
// });
// onDone();
// } else {
// throw new Error(`unsure how to copy this: ${src}`);
// }
// }
// }
// async function buildActionsForHardlink(
// queue: CopyQueue,
// events: CopyOptions,
// possibleExtraneous: Set<string>,
// reporter: Reporter,
// ): Promise<CopyActions> {
// const artifactFiles: Set<string> = new Set(events.artifactFiles || []);
// const files: Set<string> = new Set();
// // initialise events
// for (const item of queue) {
// const onDone = item.onDone || noop;
// item.onDone = () => {
// events.onProgress(item.dest);
// onDone();
// };
// }
// events.onStart(queue.length);
// // start building actions
// const actions: CopyActions = {
// file: [],
// symlink: [],
// link: [],
// };
// // custom concurrency logic as we're always executing stacks of CONCURRENT_QUEUE_ITEMS queue items
// // at a time due to the requirement to push items onto the queue
// while (queue.length) {
// const items = queue.splice(0, CONCURRENT_QUEUE_ITEMS);
// await Promise.all(items.map(build));
// }
// // simulate the existence of some files to prevent considering them extraneous
// for (const file of artifactFiles) {
// if (possibleExtraneous.has(file)) {
// reporter.verbose(reporter.lang('verboseFilePhantomExtraneous', file));
// possibleExtraneous.delete(file);
// }
// }
// for (const loc of possibleExtraneous) {
// if (files.has(loc.toLowerCase())) {
// possibleExtraneous.delete(loc);
// }
// }
// return actions;
// //
// async function build(data: CopyQueueItem): Promise<void> {
// const {src, dest} = data;
// const onFresh = data.onFresh || noop;
// const onDone = data.onDone || noop;
// if (files.has(dest.toLowerCase())) {
// // Fixes issue https://github.com/yarnpkg/yarn/issues/2734
// // When bulk hardlinking we have A -> B structure that we want to hardlink to A1 -> B1,
// // package-linker passes that modules A1 and B1 need to be hardlinked,
// // the recursive linking algorithm of A1 ends up scheduling files in B1 to be linked twice which will case
// // an exception.
// onDone();
// return;
// }
// files.add(dest.toLowerCase());
// if (events.ignoreBasenames.indexOf(path.basename(src)) >= 0) {
// // ignored file
// return;
// }
// const srcStat = await lstat(src);
// let srcFiles;
// if (srcStat.isDirectory()) {
// srcFiles = await readdir(src);
// }
// const destExists = await exists(dest);
// if (destExists) {
// const destStat = await lstat(dest);
// const bothSymlinks = srcStat.isSymbolicLink() && destStat.isSymbolicLink();
// const bothFolders = srcStat.isDirectory() && destStat.isDirectory();
// const bothFiles = srcStat.isFile() && destStat.isFile();
// if (srcStat.mode !== destStat.mode) {
// try {
// await access(dest, srcStat.mode);
// } catch (err) {
// // EINVAL access errors sometimes happen which shouldn't because node shouldn't be giving
// // us modes that aren't valid. investigate this, it's generally safe to proceed.
// reporter.verbose(err);
// }
// }
// if (bothFiles && artifactFiles.has(dest)) {
// // this file gets changed during build, likely by a custom install script. Don't bother checking it.
// onDone();
// reporter.verbose(reporter.lang('verboseFileSkipArtifact', src));
// return;
// }
// // correct hardlink
// if (bothFiles && srcStat.ino !== null && srcStat.ino === destStat.ino) {
// onDone();
// reporter.verbose(reporter.lang('verboseFileSkip', src, dest, srcStat.ino));
// return;
// }
// if (bothSymlinks) {
// const srcReallink = await readlink(src);
// if (srcReallink === (await readlink(dest))) {
// // if both symlinks are the same then we can continue on
// onDone();
// reporter.verbose(reporter.lang('verboseFileSkipSymlink', src, dest, srcReallink));
// return;
// }
// }
// if (bothFolders) {
// // mark files that aren't in this folder as possibly extraneous
// const destFiles = await readdir(dest);
// invariant(srcFiles, 'src files not initialised');
// for (const file of destFiles) {
// if (srcFiles.indexOf(file) < 0) {
// const loc = path.join(dest, file);
// possibleExtraneous.add(loc);
// if ((await lstat(loc)).isDirectory()) {
// for (const file of await readdir(loc)) {
// possibleExtraneous.add(path.join(loc, file));
// }
// }
// }
// }
// }
// }
// if (srcStat.isSymbolicLink()) {
// onFresh();
// const linkname = await readlink(src);
// actions.symlink.push({
// dest,
// linkname,
// });
// onDone();
// } else if (srcStat.isDirectory()) {
// reporter.verbose(reporter.lang('verboseFileFolder', dest));
// await mkdirp(dest);
// const destParts = dest.split(path.sep);
// while (destParts.length) {
// files.add(destParts.join(path.sep).toLowerCase());
// destParts.pop();
// }
// // push all files to queue
// invariant(srcFiles, 'src files not initialised');
// let remaining = srcFiles.length;
// if (!remaining) {
// onDone();
// }
// for (const file of srcFiles) {
// queue.push({
// onFresh,
// src: path.join(src, file),
// dest: path.join(dest, file),
// onDone: () => {
// if (--remaining === 0) {
// onDone();
// }
// },
// });
// }
// } else if (srcStat.isFile()) {
// onFresh();
// actions.link.push({
// src,
// dest,
// removeDest: destExists,
// });
// onDone();
// } else {
// throw new Error(`unsure how to copy this: ${src}`);
// }
// }
// }
// export function copy(src: string, dest: string, reporter: Reporter): Promise<void> {
// return copyBulk([{src, dest}], reporter);
// }
// export async function copyBulk(
// queue: CopyQueue,
// reporter: Reporter,
// _events?: {
// onProgress?: (dest: string) => void,
// onStart?: (num: number) => void,
// possibleExtraneous: Set<string>,
// ignoreBasenames?: Array<string>,
// artifactFiles?: Array<string>,
// },
// ): Promise<void> {
// const events: CopyOptions = {
// onStart: (_events && _events.onStart) || noop,
// onProgress: (_events && _events.onProgress) || noop,
// possibleExtraneous: _events ? _events.possibleExtraneous : new Set(),
// ignoreBasenames: (_events && _events.ignoreBasenames) || [],
// artifactFiles: (_events && _events.artifactFiles) || [],
// };
// const actions: CopyActions = await buildActionsForCopy(queue, events, events.possibleExtraneous, reporter);
// events.onStart(actions.file.length + actions.symlink.length + actions.link.length);
// const fileActions: Array<CopyFileAction> = actions.file;
// const currentlyWriting: Map<string, Promise<void>> = new Map();
// await promise.queue(
// fileActions,
// async (data: CopyFileAction): Promise<void> => {
// let writePromise;
// while ((writePromise = currentlyWriting.get(data.dest))) {
// await writePromise;
// }
// reporter.verbose(reporter.lang('verboseFileCopy', data.src, data.dest));
// const copier = copyFile(data, () => currentlyWriting.delete(data.dest));
// currentlyWriting.set(data.dest, copier);
// events.onProgress(data.dest);
// return copier;
// },
// CONCURRENT_QUEUE_ITEMS,
// );
// // we need to copy symlinks last as they could reference files we were copying
// const symlinkActions: Array<CopySymlinkAction> = actions.symlink;
// await promise.queue(symlinkActions, (data): Promise<void> => {
// const linkname = path.resolve(path.dirname(data.dest), data.linkname);
// reporter.verbose(reporter.lang('verboseFileSymlink', data.dest, linkname));
// return symlink(linkname, data.dest);
// });
// }
// export async function hardlinkBulk(
// queue: CopyQueue,
// reporter: Reporter,
// _events?: {
// onProgress?: (dest: string) => void,
// onStart?: (num: number) => void,
// possibleExtraneous: Set<string>,
// artifactFiles?: Array<string>,
// },
// ): Promise<void> {
// const events: CopyOptions = {
// onStart: (_events && _events.onStart) || noop,
// onProgress: (_events && _events.onProgress) || noop,
// possibleExtraneous: _events ? _events.possibleExtraneous : new Set(),
// artifactFiles: (_events && _events.artifactFiles) || [],
// ignoreBasenames: [],
// };
// const actions: CopyActions = await buildActionsForHardlink(queue, events, events.possibleExtraneous, reporter);
// events.onStart(actions.file.length + actions.symlink.length + actions.link.length);
// const fileActions: Array<LinkFileAction> = actions.link;
// await promise.queue(
// fileActions,
// async (data): Promise<void> => {
// reporter.verbose(reporter.lang('verboseFileLink', data.src, data.dest));
// if (data.removeDest) {
// await unlink(data.dest);
// }
// await link(data.src, data.dest);
// },
// CONCURRENT_QUEUE_ITEMS,
// );
// // we need to copy symlinks last as they could reference files we were copying
// const symlinkActions: Array<CopySymlinkAction> = actions.symlink;
// await promise.queue(symlinkActions, (data): Promise<void> => {
// const linkname = path.resolve(path.dirname(data.dest), data.linkname);
// reporter.verbose(reporter.lang('verboseFileSymlink', data.dest, linkname));
// return symlink(linkname, data.dest);
// });
// }
// function _readFile(loc: string, encoding: string): Promise<any> {
// return new Promise((resolve, reject) => {
// fs.readFile(loc, encoding, function(err, content) {
// if (err) {
// reject(err);
// } else {
// resolve(content);
// }
// });
// });
// }
// export function readFile(loc: string): Promise<string> {
// return _readFile(loc, 'utf8').then(normalizeOS);
// }
// export function readFileRaw(loc: string): Promise<Buffer> {
// return _readFile(loc, 'binary');
// }
// export async function readFileAny(files: Array<string>): Promise<string | null> {
// for (const file of files) {
// if (await exists(file)) {
// return readFile(file);
// }
// }
// return null;
// }
export async function readJson(loc) {
return (await readJsonAndFile(loc)).object;
}
export async function readJsonAndFile(loc) {
const file = await readFile(loc);
try {
return {
object: map(JSON.parse(stripBOM(file))),
content: file,
};
}
catch (err) {
err.message = `${loc}: ${err.message}`;
throw err;
}
}
export async function walk(dir, relativeDir, ignoreBasenames = new Set()) {
let files = [];
let filenames = await readdir(dir);
if (ignoreBasenames.size) {
filenames = filenames.filter(name => !ignoreBasenames.has(name));
}
for (const name of filenames) {
const relative = relativeDir ? path.join(relativeDir, name) : name;
const loc = path.join(dir, name);
const stat = await lstat(loc);
files.push({
relative,
basename: name,
absolute: loc,
mtime: +stat.mtime,
});
if (stat.isDirectory()) {
files = files.concat(await walk(loc, relative, ignoreBasenames));
}
}
return files;
}
// export async function getFileSizeOnDisk(loc: string): Promise<number> {
// const stat = await lstat(loc);
// const {size, blksize: blockSize} = stat;
// return Math.ceil(size / blockSize) * blockSize;
// }
// export function normalizeOS(body: string): string {
// return body.replace(/\r\n/g, '\n');
// }
const cr = '\r'.charCodeAt(0);
const lf = '\n'.charCodeAt(0);
async function getEolFromFile(path) {
if (!(await exists(path))) {
return undefined;
}
const buffer = await readFileBuffer(path);
for (let i = 0; i < buffer.length; ++i) {
if (buffer[i] === cr) {
return '\r\n';
}
if (buffer[i] === lf) {
return '\n';
}
}
return undefined;
}
export async function writeFilePreservingEol(path, data) {
const eol = (await getEolFromFile(path)) || os.EOL;
if (eol !== '\n') {
data = data.replace(/\n/g, eol);
}
await writeFile(path, data);
}
// export async function hardlinksWork(dir: string): Promise<boolean> {
// const filename = 'test-file' + Math.random();
// const file = path.join(dir, filename);
// const fileLink = path.join(dir, filename + '-link');
// try {
// await writeFile(file, 'test');
// await link(file, fileLink);
// } catch (err) {
// return false;
// } finally {
// await unlink(file);
// await unlink(fileLink);
// }
// return true;
// }
// // not a strict polyfill for Node's fs.mkdtemp
// export async function makeTempDir(prefix?: string): Promise<string> {
// const dir = path.join(os.tmpdir(), `pika-${prefix || ''}-${Date.now()}-${Math.random()}`);
// await unlink(dir);
// await mkdirp(dir);
// return dir;
// }
// export async function readFirstAvailableStream(paths: Iterable<string>): Promise<?ReadStream> {
// for (const path of paths) {
// try {
// const fd = await open(path, 'r');
// return fs.createReadStream(path, {fd});
// } catch (err) {
// // Try the next one
// }
// }
// return null;
// }
// export async function getFirstSuitableFolder(
// paths: Iterable<string>,
// mode: number = constants.W_OK | constants.X_OK, // eslint-disable-line no-bitwise
// ): Promise<FolderQueryResult> {
// const result: FolderQueryResult = {
// skipped: [],
// folder: null,
// };
// for (const folder of paths) {
// try {
// await mkdirp(folder);
// await access(folder, mode);
// result.folder = folder;
// return result;
// } catch (error) {
// result.skipped.push({
// error,
// folder,
// });
// }
// }
// return result;
// }