readdirp/index.js

Recursive version of fs.readdir with small RAM & CPU footprint

import { lstat, readdir, realpath, stat } from 'node:fs/promises';
import { join as pjoin, resolve as presolve, sep as psep } from 'node:path';
import { Readable } from 'node:stream';
export const EntryTypes = {
    FILE_TYPE: 'files',
    DIR_TYPE: 'directories',
    FILE_DIR_TYPE: 'files_directories',
    EVERYTHING_TYPE: 'all',
};
const defaultOptions = {
    root: '.',
    fileFilter: (_entryInfo) => true,
    directoryFilter: (_entryInfo) => true,
    type: EntryTypes.FILE_TYPE,
    lstat: false,
    depth: 2147483648,
    alwaysStat: false,
    // Throughput is flat from 16 to 65536 (traversal is I/O-bound), but
    // batches of 1024+ entries survive young-gen GC and bloat RSS ~20-60%.
    highWaterMark: 256,
};
Object.freeze(defaultOptions);
const RECURSIVE_ERROR_CODE = 'READDIRP_RECURSIVE_ERROR';
const NORMAL_FLOW_ERRORS = new Set(['ENOENT', 'EPERM', 'EACCES', 'ELOOP', RECURSIVE_ERROR_CODE]);
const ALL_TYPES = [
    EntryTypes.DIR_TYPE,
    EntryTypes.EVERYTHING_TYPE,
    EntryTypes.FILE_DIR_TYPE,
    EntryTypes.FILE_TYPE,
];
const DIR_TYPES = new Set([
    EntryTypes.DIR_TYPE,
    EntryTypes.EVERYTHING_TYPE,
    EntryTypes.FILE_DIR_TYPE,
]);
const FILE_TYPES = new Set([
    EntryTypes.EVERYTHING_TYPE,
    EntryTypes.FILE_DIR_TYPE,
    EntryTypes.FILE_TYPE,
]);
const isNormalFlowError = (error) => NORMAL_FLOW_ERRORS.has(error.code);
const wantBigintFsStats = process.platform === 'win32';
const emptyFn = (_entryInfo) => true;
const normalizeFilter = (filter) => {
    if (filter === undefined)
        return emptyFn;
    if (typeof filter === 'function')
        return filter;
    if (typeof filter === 'string') {
        const fl = filter.trim();
        return (entry) => entry.basename === fl;
    }
    if (Array.isArray(filter)) {
        const trItems = filter.map((item) => item.trim());
        return (entry) => trItems.some((f) => entry.basename === f);
    }
    return emptyFn;
};
export class ReaddirpStream extends Readable {
    /**
     * Directories discovered but not yet emitted from. Listings are read
     * lazily (on pop, plus one prefetch) instead of eagerly on discovery:
     * keeping whole listings for every queued dir balloons RAM on wide trees.
     */
    parents;
    reading;
    parent;
    _stat;
    _maxDepth;
    _wantsDir;
    _wantsFile;
    _wantsEverything;
    _root;
    _isDirent;
    _statsProp;
    _rdOptions;
    _fileFilter;
    _directoryFilter;
    _relStart;
    constructor(options = {}) {
        super({
            objectMode: true,
            autoDestroy: true,
            highWaterMark: options.highWaterMark ?? defaultOptions.highWaterMark,
        });
        const opts = { ...defaultOptions, ...options };
        // Use ?? so an explicit `undefined` in user options doesn't shadow defaults.
        const root = opts.root ?? defaultOptions.root;
        const type = opts.type ?? defaultOptions.type;
        this._fileFilter = normalizeFilter(opts.fileFilter);
        this._directoryFilter = normalizeFilter(opts.directoryFilter);
        const statMethod = opts.lstat ? lstat : stat;
        // Use bigint stats if it's windows and stat() supports options (node 10+).
        if (wantBigintFsStats) {
            this._stat = (path) => statMethod(path, { bigint: true });
        }
        else {
            this._stat = statMethod;
        }
        this._maxDepth =
            opts.depth != null && Number.isSafeInteger(opts.depth) ? opts.depth : defaultOptions.depth;
        this._wantsDir = DIR_TYPES.has(type);
        this._wantsFile = FILE_TYPES.has(type);
        this._wantsEverything = type === EntryTypes.EVERYTHING_TYPE;
        this._root = presolve(root);
        // Every fullPath is `_root + sep + relative path` (see _formatEntry), so
        // the relative path is a slice starting past the root and its trailing
        // separator (which resolved paths lack, except fs roots like '/', 'C:\').
        this._relStart = this._root.endsWith(psep) ? this._root.length : this._root.length + 1;
        this._isDirent = !opts.alwaysStat;
        this._statsProp = this._isDirent ? 'dirent' : 'stats';
        this._rdOptions = { encoding: 'utf8', withFileTypes: this._isDirent };
        // Launch stream with one parent, the root dir, whose readdir starts
        // right away. Explore the resolved root so all parent paths stay
        // absolute even if process.cwd() changes mid-iteration.
        const rootDir = { path: this._root, depth: 1 };
        rootDir.pending = this._exploreDir(this._root, 1);
        this.parents = [rootDir];
        this.reading = false;
        this.parent = undefined;
    }
    async _read(batch) {
        if (this.reading)
            return;
        this.reading = true;
        try {
            while (!this.destroyed && batch > 0) {
                const par = this.parent;
                const fil = par && par.files;
                if (fil && fil.length > 0) {
                    const { path, depth } = par;
                    const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path));
                    // In dirent mode _formatEntry is synchronous: skip Promise.all and
                    // its per-entry microtask overhead.
                    const awaited = this._isDirent
                        ? slice
                        : await Promise.all(slice);
                    for (const entry of awaited) {
                        if (!entry)
                            continue;
                        if (this.destroyed)
                            return;
                        // Only symlinks require async work; plain files / dirs resolve synchronously.
                        let entryType = this._getEntryType(entry);
                        if (typeof entryType !== 'string')
                            entryType = await entryType;
                        if (entryType === 'directory' && this._directoryFilter(entry)) {
                            if (depth <= this._maxDepth) {
                                // Lazy: don't readdir until this dir is popped. Keeping whole
                                // listings for every queued dir would balloon RAM on wide trees.
                                this.parents.push({ path: entry.fullPath, depth: depth + 1 });
                            }
                            if (this._wantsDir) {
                                this.push(entry);
                                batch--;
                            }
                        }
                        else if ((entryType === 'file' || this._includeAsFile(entry)) &&
                            this._fileFilter(entry)) {
                            if (this._wantsFile) {
                                this.push(entry);
                                batch--;
                            }
                        }
                    }
                }
                else {
                    const parent = this.parents.pop();
                    if (!parent) {
                        this.push(null);
                        break;
                    }
                    const dir = parent.pending ?? this._exploreDir(parent.path, parent.depth);
                    // Prefetch the next dir so its readdir overlaps with processing
                    // this one's entries. Only the stack top is prefetched, keeping at
                    // most a handful of listings (~tree depth) in RAM at once.
                    const next = this.parents[this.parents.length - 1];
                    if (next && !next.pending) {
                        next.pending = this._exploreDir(next.path, next.depth);
                    }
                    this.parent = await dir;
                    if (this.destroyed)
                        return;
                }
            }
        }
        catch (error) {
            this.destroy(error);
        }
        finally {
            this.reading = false;
        }
    }
    // NOTE: native `readdir(path, { recursive: true })` was evaluated as a
    // replacement for this per-directory traversal and rejected:
    // - Not faster: node implements it in JS, walking directories sequentially
    //   just like this loop, but with extra path bookkeeping. Benchmarks
    //   (node 24): ~10% slower on wide trees, ~40% slower on small ones,
    //   parity on deep ones.
    // - Much more RAM: it buffers the entire subtree listing in one array,
    //   instead of one directory at a time, defeating streaming.
    // - Semantics diverge: it can't limit depth, can't skip directories a
    //   directoryFilter rejects, doesn't follow symlinked dirs, and fails
    //   wholesale (all entries lost) if anything in the subtree is unreadable,
    //   instead of emitting a 'warn' and continuing.
    async _exploreDir(path, depth) {
        let files;
        try {
            files = await readdir(path, this._rdOptions);
        }
        catch (error) {
            this._onError(error);
        }
        return { files, depth, path };
    }
    // Synchronous in dirent mode; returns a promise only when stats are needed.
    _formatEntry(dirent, path) {
        const basename = this._isDirent ? dirent.name : dirent;
        // `path` is always an absolute, normalized parent dir (see _exploreDir
        // seeding in the constructor), so a plain join is enough — resolve()
        // would re-read cwd on every entry.
        const fullPath = pjoin(path, basename);
        // Slice instead of path.relative(): equivalent here (fullPath is always
        // under _root) and avoids several intermediate allocations per entry.
        const entry = { path: fullPath.slice(this._relStart), fullPath, basename };
        if (this._isDirent) {
            entry.dirent = dirent;
            return entry;
        }
        return this._stat(fullPath).then((stats) => {
            entry.stats = stats;
            return entry;
        }, (err) => {
            this._onError(err);
            return undefined;
        });
    }
    _onError(err) {
        if (isNormalFlowError(err) && !this.destroyed) {
            this.emit('warn', err);
        }
        else {
            this.destroy(err);
        }
    }
    // Synchronous for regular files and directories; returns a promise only for
    // symlinks, which need realpath() to be classified.
    _getEntryType(entry) {
        // entry may be undefined, because a warning or an error were emitted
        // and the statsProp is undefined
        if (!entry || !(this._statsProp in entry)) {
            return '';
        }
        const stats = entry[this._statsProp];
        if (stats.isFile())
            return 'file';
        if (stats.isDirectory())
            return 'directory';
        if (stats.isSymbolicLink())
            return this._getSymlinkEntryType(entry);
        return '';
    }
    async _getSymlinkEntryType(entry) {
        const full = entry.fullPath;
        try {
            const entryRealPath = await realpath(full);
            const entryRealPathStats = await lstat(entryRealPath);
            if (entryRealPathStats.isFile()) {
                return 'file';
            }
            if (entryRealPathStats.isDirectory()) {
                const len = entryRealPath.length;
                if (full.startsWith(entryRealPath) && full[len] === psep) {
                    const recursiveError = new Error(`Circular symlink detected: "${full}" points to "${entryRealPath}"`);
                    // @ts-ignore
                    recursiveError.code = RECURSIVE_ERROR_CODE;
                    this._onError(recursiveError);
                    return '';
                }
                return 'directory';
            }
        }
        catch (error) {
            this._onError(error);
        }
        return '';
    }
    _includeAsFile(entry) {
        const stats = entry && entry[this._statsProp];
        return stats && this._wantsEverything && !stats.isDirectory();
    }
}
/**
 * Streaming version: Reads all files and directories in given root recursively.
 * Consumes ~constant small amount of RAM.
 * @param root Root directory
 * @param options Options to specify root (start directory), filters and recursion depth
 */
export function readdirp(root, options = {}) {
    // @ts-ignore
    let type = options.entryType || options.type;
    if (type === 'both')
        type = EntryTypes.FILE_DIR_TYPE; // backwards-compatibility
    if (!root) {
        throw new Error('readdirp: root argument is required. Usage: readdirp(root, options)');
    }
    else if (typeof root !== 'string') {
        throw new TypeError('readdirp: root argument must be a string. Usage: readdirp(root, options)');
    }
    else if (type && !ALL_TYPES.includes(type)) {
        throw new Error(`readdirp: Invalid type passed. Use one of ${ALL_TYPES.join(', ')}`);
    }
    // Copy options instead of mutating the caller's object.
    const opts = { ...options, root };
    if (type)
        opts.type = type;
    return new ReaddirpStream(opts);
}
/**
 * Promise version: Reads all files and directories in given root recursively.
 * Compared to streaming version, will consume a lot of RAM e.g. when 1 million files are listed.
 * @returns array of paths and their entry infos
 */
export function readdirpPromise(root, options = {}) {
    return new Promise((resolve, reject) => {
        const files = [];
        readdirp(root, options)
            .on('data', (entry) => files.push(entry))
            .on('end', () => resolve(files))
            .on('error', (error) => reject(error));
    });
}
export default readdirp;