repomix/lib/core/file/fileSearch.js

A tool to pack repository contents to single file for AI consumption

import fs from 'node:fs/promises';
import path from 'node:path';
import { globby } from 'globby';
import { defaultIgnoreList } from '../../config/defaultIgnore.js';
import { mapWithConcurrency } from '../../shared/asyncMap.js';
import { RepomixError } from '../../shared/errorHandle.js';
import { logger } from '../../shared/logger.js';
import { sortPaths } from './filePathSort.js';
import { checkDirectoryPermissions, PermissionError } from './permissionCheck.js';
const EMPTY_DIR_CHECK_CONCURRENCY = 20;
const findEmptyDirectories = async (rootDir, directories) => {
    const results = await mapWithConcurrency(directories, EMPTY_DIR_CHECK_CONCURRENCY, async (dir) => {
        const fullPath = path.join(rootDir, dir);
        try {
            const entries = await fs.readdir(fullPath);
            const hasVisibleContents = entries.some((entry) => !entry.startsWith('.'));
            return hasVisibleContents ? null : dir;
        }
        catch (error) {
            logger.debug(`Error checking directory ${dir}:`, error);
            return null;
        }
    });
    return results.filter((dir) => dir !== null);
};
const isGitWorktreeRef = async (gitPath) => {
    try {
        const stats = await fs.stat(gitPath);
        if (!stats.isFile()) {
            return false;
        }
        const content = await fs.readFile(gitPath, 'utf8');
        return content.startsWith('gitdir:');
    }
    catch {
        return false;
    }
};
export const escapeGlobPattern = (pattern) => {
    const escapedBackslashes = pattern.replace(/\\/g, '\\\\');
    return escapedBackslashes.replace(/[()[\]]/g, '\\$&');
};
export const normalizeGlobPattern = (pattern) => {
    if (pattern.endsWith('/') && !pattern.endsWith('**/')) {
        return pattern.slice(0, -1);
    }
    if (pattern.startsWith('**/') && !pattern.includes('/**')) {
        return `${pattern}/**`;
    }
    return pattern;
};
export const searchFiles = async (rootDir, config, explicitFiles) => {
    let pathStats;
    try {
        pathStats = await fs.stat(rootDir);
    }
    catch (error) {
        if (error instanceof Error && 'code' in error) {
            const errorCode = error.code;
            if (errorCode === 'ENOENT') {
                throw new RepomixError(`Target path does not exist: ${rootDir}`);
            }
            if (errorCode === 'EPERM' || errorCode === 'EACCES') {
                throw new PermissionError(`Permission denied while accessing path. Please check folder access permissions for your terminal app. path: ${rootDir}`, rootDir, errorCode);
            }
            throw new RepomixError(`Failed to access path: ${rootDir}. Error code: ${errorCode}. ${error.message}`);
        }
        const repomixError = new RepomixError(`Failed to access path: ${rootDir}. Reason: ${error instanceof Error ? error.message : JSON.stringify(error)}`);
        repomixError.cause = error;
        throw repomixError;
    }
    if (!pathStats.isDirectory()) {
        throw new RepomixError(`Target path is not a directory: ${rootDir}. Please specify a directory path, not a file path.`);
    }
    const permissionCheck = await checkDirectoryPermissions(rootDir);
    if (permissionCheck.details?.read !== true) {
        if (permissionCheck.error instanceof PermissionError) {
            throw permissionCheck.error;
        }
        throw new RepomixError(`Target directory is not readable or does not exist. Please check folder access permissions for your terminal app.\npath: ${rootDir}`);
    }
    try {
        const { adjustedIgnorePatterns, ignoreFilePatterns } = await prepareIgnoreContext(rootDir, config);
        logger.trace('Ignore patterns:', adjustedIgnorePatterns);
        logger.trace('Ignore file patterns:', ignoreFilePatterns);
        let includePatterns = config.include.map((pattern) => escapeGlobPattern(pattern));
        if (explicitFiles) {
            if (explicitFiles.length === 0) {
                logger.warn('[stdin mode] No files received from stdin. Will search all files matching include patterns.');
            }
            else {
                logger.debug(`[stdin mode] Processing ${explicitFiles.length} explicit files`);
                logger.trace('[stdin mode] Explicit files (absolute):', explicitFiles);
                const relativePaths = explicitFiles.map((filePath) => {
                    const relativePath = path.relative(rootDir, filePath);
                    return escapeGlobPattern(relativePath);
                });
                logger.trace('[stdin mode] Explicit files (relative, escaped):', relativePaths);
                logger.trace('[stdin mode] Include patterns before merge:', includePatterns);
                includePatterns = [...includePatterns, ...relativePaths];
                logger.debug(`[stdin mode] Total include patterns after merge: ${includePatterns.length}`);
            }
        }
        if (includePatterns.length === 0) {
            includePatterns = ['**/*'];
        }
        logger.trace('Include patterns with explicit files:', includePatterns);
        logger.trace('Ignore patterns:', adjustedIgnorePatterns);
        logger.trace('Ignore file patterns (for globby):', ignoreFilePatterns);
        const handleGlobbyError = (error) => {
            const code = error?.code;
            if (code === 'EPERM' || code === 'EACCES') {
                throw new PermissionError(`Permission denied while scanning directory. Please check folder access permissions for your terminal app. path: ${rootDir}`, rootDir);
            }
            throw error;
        };
        logger.debug('[globby] Starting file search...');
        const globbyStartTime = Date.now();
        let filePaths;
        let emptyDirPaths = [];
        if (config.output.includeEmptyDirectories) {
            const entries = await globby(includePatterns, {
                ...createBaseGlobbyOptions(rootDir, config, adjustedIgnorePatterns, ignoreFilePatterns),
                onlyFiles: false,
                objectMode: true,
            }).catch(handleGlobbyError);
            const files = [];
            const directories = [];
            for (const entry of entries) {
                if (entry.dirent.isFile()) {
                    files.push(entry.path);
                }
                else if (entry.dirent.isDirectory()) {
                    directories.push(entry.path);
                }
            }
            filePaths = files;
            const globbyElapsedTime = Date.now() - globbyStartTime;
            logger.debug(`[globby] Completed in ${globbyElapsedTime}ms, found ${filePaths.length} files and ${directories.length} directories`);
            const filterStartTime = Date.now();
            emptyDirPaths = await findEmptyDirectories(rootDir, directories);
            const filterTime = Date.now() - filterStartTime;
            logger.debug(`[empty dirs] Filtered to ${emptyDirPaths.length} empty directories in ${filterTime}ms`);
        }
        else {
            filePaths = await globby(includePatterns, {
                ...createBaseGlobbyOptions(rootDir, config, adjustedIgnorePatterns, ignoreFilePatterns),
                onlyFiles: true,
            }).catch(handleGlobbyError);
            const globbyElapsedTime = Date.now() - globbyStartTime;
            logger.debug(`[globby] Completed in ${globbyElapsedTime}ms, found ${filePaths.length} files`);
        }
        logger.debug(`[result] Total files: ${filePaths.length}, empty directories: ${emptyDirPaths.length}`);
        logger.trace(`Filtered ${filePaths.length} files`);
        return {
            filePaths: sortPaths(filePaths),
            emptyDirPaths: sortPaths(emptyDirPaths),
        };
    }
    catch (error) {
        if (error instanceof PermissionError) {
            throw error;
        }
        if (error instanceof Error) {
            logger.error('Error filtering files:', error.message);
            throw new Error(`Failed to filter files in directory ${rootDir}. Reason: ${error.message}`);
        }
        logger.error('An unexpected error occurred:', error);
        throw new Error('An unexpected error occurred while filtering files.');
    }
};
export const parseIgnoreContent = (content) => {
    if (!content)
        return [];
    return content.split('\n').reduce((acc, line) => {
        const trimmedLine = line.trim();
        if (trimmedLine && !trimmedLine.startsWith('#')) {
            acc.push(trimmedLine);
        }
        return acc;
    }, []);
};
const prepareIgnoreContext = async (rootDir, config) => {
    const [ignorePatterns, ignoreFilePatterns] = await Promise.all([
        getIgnorePatterns(rootDir, config),
        getIgnoreFilePatterns(config),
    ]);
    const normalizedIgnorePatterns = ignorePatterns.map(normalizeGlobPattern);
    const gitPath = path.join(rootDir, '.git');
    const isWorktree = await isGitWorktreeRef(gitPath);
    const adjustedIgnorePatterns = [...normalizedIgnorePatterns];
    if (isWorktree) {
        const gitIndex = adjustedIgnorePatterns.indexOf('.git/**');
        if (gitIndex !== -1) {
            adjustedIgnorePatterns.splice(gitIndex, 1);
            adjustedIgnorePatterns.push('.git');
        }
    }
    return { adjustedIgnorePatterns, ignoreFilePatterns };
};
const createBaseGlobbyOptions = (rootDir, config, ignorePatterns, ignoreFilePatterns) => ({
    cwd: rootDir,
    ignore: ignorePatterns,
    gitignore: config.ignore.useGitignore,
    ignoreFiles: ignoreFilePatterns,
    absolute: false,
    dot: true,
    followSymbolicLinks: false,
});
export const getIgnoreFilePatterns = async (config) => {
    const ignoreFilePatterns = [];
    if (config.ignore.useDotIgnore) {
        ignoreFilePatterns.push('**/.ignore');
    }
    ignoreFilePatterns.push('**/.repomixignore');
    return ignoreFilePatterns;
};
export const getIgnorePatterns = async (rootDir, config) => {
    const ignorePatterns = new Set();
    if (config.ignore.useDefaultPatterns) {
        logger.trace('Adding default ignore patterns');
        for (const pattern of defaultIgnoreList) {
            ignorePatterns.add(pattern);
        }
    }
    if (config.output.filePath) {
        const absoluteOutputPath = path.resolve(config.cwd, config.output.filePath);
        const relativeToTargetPath = path.relative(rootDir, absoluteOutputPath);
        logger.trace('Adding output file to ignore patterns:', relativeToTargetPath);
        ignorePatterns.add(relativeToTargetPath);
    }
    if (config.ignore.customPatterns) {
        logger.trace('Adding custom ignore patterns:', config.ignore.customPatterns);
        for (const pattern of config.ignore.customPatterns) {
            ignorePatterns.add(pattern);
        }
    }
    if (config.ignore.useGitignore) {
        const excludeFilePath = path.join(rootDir, '.git', 'info', 'exclude');
        try {
            const excludeFileContent = await fs.readFile(excludeFilePath, 'utf8');
            const excludePatterns = parseIgnoreContent(excludeFileContent);
            for (const pattern of excludePatterns) {
                ignorePatterns.add(pattern);
            }
        }
        catch (error) {
            logger.trace('Could not read .git/info/exclude file:', error instanceof Error ? error.message : String(error));
        }
    }
    return Array.from(ignorePatterns);
};
export const listDirectories = async (rootDir, config) => {
    const { adjustedIgnorePatterns, ignoreFilePatterns } = await prepareIgnoreContext(rootDir, config);
    const directories = await globby(['**/*'], {
        ...createBaseGlobbyOptions(rootDir, config, adjustedIgnorePatterns, ignoreFilePatterns),
        onlyDirectories: true,
    });
    return sortPaths(directories);
};
export const listFiles = async (rootDir, config) => {
    const { adjustedIgnorePatterns, ignoreFilePatterns } = await prepareIgnoreContext(rootDir, config);
    const files = await globby(['**/*'], {
        ...createBaseGlobbyOptions(rootDir, config, adjustedIgnorePatterns, ignoreFilePatterns),
        onlyFiles: true,
    });
    return sortPaths(files);
};