repomix/lib/core/file/fileProcess.js

A tool to pack repository contents to single file for AI consumption

import pc from 'picocolors';
import { logger } from '../../shared/logger.js';
import { initTaskRunner } from '../../shared/processConcurrency.js';
import { getFileManipulator } from './fileManipulate.js';
import { truncateBase64Content } from './truncateBase64.js';
export const applyLightweightTransforms = (files, config, progressCallback, deps) => {
    const totalFiles = files.length;
    const results = Array.from({ length: totalFiles });
    for (let i = 0; i < totalFiles; i++) {
        const file = files[i];
        let content = file.content;
        if (config.output.truncateBase64) {
            content = truncateBase64Content(content);
        }
        if (config.output.removeEmptyLines) {
            const manipulator = deps.getFileManipulator(file.path);
            if (manipulator) {
                content = manipulator.removeEmptyLines(content);
            }
        }
        content = content.trim();
        if (config.output.showLineNumbers && !config.output.compress) {
            const lines = content.split('\n');
            const padding = lines.length.toString().length;
            const numberedLines = lines.map((line, idx) => `${(idx + 1).toString().padStart(padding)}: ${line}`);
            content = numberedLines.join('\n');
        }
        results[i] = { path: file.path, content };
        if ((i + 1) % 50 === 0 || i === totalFiles - 1) {
            progressCallback(`Processing file... (${i + 1}/${totalFiles}) ${pc.dim(file.path)}`);
        }
    }
    return results;
};
export const processFiles = async (rawFiles, config, progressCallback, deps = {
    initTaskRunner,
    getFileManipulator,
}) => {
    const startTime = process.hrtime.bigint();
    let files;
    const useWorkers = config.output.compress || config.output.removeComments;
    if (useWorkers) {
        logger.trace(`Starting file processing for ${rawFiles.length} files using worker pool`);
        const taskRunner = deps.initTaskRunner({
            numOfTasks: rawFiles.length,
            workerType: 'fileProcess',
            runtime: 'worker_threads',
        });
        const tasks = rawFiles.map((rawFile) => ({
            rawFile,
            config,
        }));
        try {
            let completedTasks = 0;
            const totalTasks = tasks.length;
            files = await Promise.all(tasks.map((task) => taskRunner.run(task).then((result) => {
                completedTasks++;
                progressCallback(`Processing file... (${completedTasks}/${totalTasks}) ${pc.dim(task.rawFile.path)}`);
                logger.trace(`Processing file... (${completedTasks}/${totalTasks}) ${task.rawFile.path}`);
                return result;
            })));
        }
        catch (error) {
            logger.error('Error during file processing:', error);
            throw error;
        }
        finally {
            await taskRunner.cleanup();
        }
        files = applyLightweightTransforms(files, config, () => { }, deps);
    }
    else {
        logger.trace(`Starting file processing for ${rawFiles.length} files in main thread (lightweight mode)`);
        const inputFiles = rawFiles.map((rawFile) => ({ path: rawFile.path, content: rawFile.content }));
        files = applyLightweightTransforms(inputFiles, config, progressCallback, deps);
    }
    const endTime = process.hrtime.bigint();
    const duration = Number(endTime - startTime) / 1e6;
    logger.trace(`File processing completed in ${duration.toFixed(2)}ms`);
    return files;
};