repomix/lib/core/metrics/calculateFileMetrics.js

A tool to pack repository contents to single file for AI consumption

import pc from 'picocolors';
import { logger } from '../../shared/logger.js';
import { runBatchTokenCount } from './metricsWorkerRunner.js';
const METRICS_BATCH_SIZE = 10;
export const calculateFileMetrics = async (processedFiles, targetFilePaths, tokenCounterEncoding, progressCallback, deps) => {
    const targetFileSet = new Set(targetFilePaths);
    const filesToProcess = processedFiles.filter((file) => targetFileSet.has(file.path));
    if (filesToProcess.length === 0) {
        return [];
    }
    try {
        const startTime = process.hrtime.bigint();
        logger.trace(`Starting file metrics calculation for ${filesToProcess.length} files using worker pool`);
        const batches = [];
        for (let i = 0; i < filesToProcess.length; i += METRICS_BATCH_SIZE) {
            batches.push(filesToProcess.slice(i, i + METRICS_BATCH_SIZE));
        }
        logger.trace(`Split ${filesToProcess.length} files into ${batches.length} batches for token counting`);
        let completedItems = 0;
        const batchResults = await Promise.all(batches.map(async (batch) => {
            const tokenCounts = await runBatchTokenCount(deps.taskRunner, {
                items: batch.map((file) => ({ content: file.content, path: file.path })),
                encoding: tokenCounterEncoding,
            });
            const results = batch.map((file, index) => ({
                path: file.path,
                charCount: file.content.length,
                tokenCount: tokenCounts[index],
            }));
            completedItems += batch.length;
            const lastFile = batch[batch.length - 1];
            progressCallback(`Calculating metrics... (${completedItems}/${filesToProcess.length}) ${pc.dim(lastFile.path)}`);
            logger.trace(`Calculating metrics... (${completedItems}/${filesToProcess.length}) ${lastFile.path}`);
            return results;
        }));
        const allResults = batchResults.flat();
        const endTime = process.hrtime.bigint();
        const duration = Number(endTime - startTime) / 1e6;
        logger.trace(`File metrics calculation completed in ${duration.toFixed(2)}ms`);
        return allResults;
    }
    catch (error) {
        logger.error('Error during file metrics calculation:', error);
        throw error;
    }
};