rxcc/lib/core/file/workers/fileCollectWorker.js

A tool to pack repository contents to single file for AI consumption

var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
import * as fs from 'node:fs/promises';
import path from 'node:path';
import iconv from 'iconv-lite';
import { isBinary } from 'istextorbinary';
import jschardet from 'jschardet';
import pc from 'picocolors';
import { logger } from '../../../shared/logger.js';
// Maximum file size to process (50MB)
// This prevents out-of-memory errors when processing very large files
export const MAX_FILE_SIZE = 50 * 1024 * 1024;
export default (_a) => __awaiter(void 0, [_a], void 0, function* ({ filePath, rootDir }) {
    const fullPath = path.resolve(rootDir, filePath);
    const content = yield readRawFile(fullPath);
    if (content) {
        return {
            path: filePath,
            content,
        };
    }
    return null;
});
const readRawFile = (filePath) => __awaiter(void 0, void 0, void 0, function* () {
    try {
        const stats = yield fs.stat(filePath);
        if (stats.size > MAX_FILE_SIZE) {
            const sizeMB = (stats.size / 1024 / 1024).toFixed(1);
            logger.log('');
            logger.log('⚠️ Large File Warning:');
            logger.log('──────────────────────');
            logger.log(`File exceeds size limit: ${sizeMB}MB > ${MAX_FILE_SIZE / 1024 / 1024}MB (${filePath})`);
            logger.log(pc.dim('Add this file to .repomixignore if you want to exclude it permanently'));
            logger.log('');
            return null;
        }
        if (isBinary(filePath)) {
            logger.debug(`Skipping binary file: ${filePath}`);
            return null;
        }
        logger.trace(`Reading file: ${filePath}`);
        const buffer = yield fs.readFile(filePath);
        if (isBinary(null, buffer)) {
            logger.debug(`Skipping binary file (content check): ${filePath}`);
            return null;
        }
        const encoding = jschardet.detect(buffer).encoding || 'utf-8';
        const content = iconv.decode(buffer, encoding);
        return content;
    }
    catch (error) {
        logger.warn(`Failed to read file: ${filePath}`, error);
        return null;
    }
});
//# sourceMappingURL=fileCollectWorker.js.map