repomix/lib/core/file/fileRead.js

A tool to pack repository contents to single file for AI consumption

import * as fs from 'node:fs/promises';
import isBinaryPath from 'is-binary-path';
import { isBinaryFile } from 'isbinaryfile';
import { logger } from '../../shared/logger.js';
let _encodingDepsPromise;
const getEncodingDeps = () => {
    _encodingDepsPromise ??= Promise.all([import('jschardet'), import('iconv-lite')]).then(([jschardet, iconv]) => ({
        jschardet,
        iconv,
    }));
    return _encodingDepsPromise;
};
export const readRawFile = async (filePath, maxFileSize) => {
    try {
        if (isBinaryPath(filePath)) {
            logger.debug(`Skipping binary file: ${filePath}`);
            return { content: null, skippedReason: 'binary-extension' };
        }
        logger.trace(`Reading file: ${filePath}`);
        const buffer = await fs.readFile(filePath);
        if (buffer.length > maxFileSize) {
            const sizeKB = (buffer.length / 1024).toFixed(1);
            const maxSizeKB = (maxFileSize / 1024).toFixed(1);
            logger.trace(`File exceeds size limit: ${sizeKB}KB > ${maxSizeKB}KB (${filePath})`);
            return { content: null, skippedReason: 'size-limit' };
        }
        if (await isBinaryFile(buffer)) {
            logger.debug(`Skipping binary file (content check): ${filePath}`);
            return { content: null, skippedReason: 'binary-content' };
        }
        try {
            let content = new TextDecoder('utf-8', { fatal: true }).decode(buffer);
            if (content.charCodeAt(0) === 0xfeff) {
                content = content.slice(1);
            }
            return { content };
        }
        catch {
        }
        const encodingDeps = await getEncodingDeps();
        const { encoding: detectedEncoding } = encodingDeps.jschardet.detect(buffer) ?? {};
        const encoding = detectedEncoding && encodingDeps.iconv.encodingExists(detectedEncoding) ? detectedEncoding : 'utf-8';
        const content = encodingDeps.iconv.decode(buffer, encoding, { stripBOM: true });
        if (content.includes('\uFFFD')) {
            logger.debug(`Skipping file due to encoding errors (detected: ${encoding}): ${filePath}`);
            return { content: null, skippedReason: 'encoding-error' };
        }
        return { content };
    }
    catch (error) {
        logger.warn(`Failed to read file: ${filePath}`, error);
        return { content: null, skippedReason: 'encoding-error' };
    }
};