repomix/lib/core/metrics/calculateOutputMetrics.js

A tool to pack repository contents to single file for AI consumption

import { logger } from '../../shared/logger.js';
import { runTokenCount } from './metricsWorkerRunner.js';
const TARGET_CHARS_PER_CHUNK = 200_000;
const MIN_CONTENT_LENGTH_FOR_PARALLEL = 1_000_000;
export const calculateOutputMetrics = async (content, encoding, path, deps) => {
    const shouldRunInParallel = content.length > MIN_CONTENT_LENGTH_FOR_PARALLEL;
    try {
        logger.trace(`Starting output token count for ${path || 'output'}`);
        const startTime = process.hrtime.bigint();
        let result;
        if (shouldRunInParallel) {
            const chunks = [];
            for (let i = 0; i < content.length; i += TARGET_CHARS_PER_CHUNK) {
                chunks.push(content.slice(i, i + TARGET_CHARS_PER_CHUNK));
            }
            const chunkResults = await Promise.all(chunks.map(async (chunk, index) => {
                return runTokenCount(deps.taskRunner, {
                    content: chunk,
                    encoding,
                    path: path ? `${path}-chunk-${index}` : undefined,
                });
            }));
            result = chunkResults.reduce((sum, count) => sum + count, 0);
        }
        else {
            result = await runTokenCount(deps.taskRunner, {
                content,
                encoding,
                path,
            });
        }
        const endTime = process.hrtime.bigint();
        const duration = Number(endTime - startTime) / 1e6;
        logger.trace(`Output token count completed in ${duration.toFixed(2)}ms`);
        return result;
    }
    catch (error) {
        logger.error('Error during token count:', error);
        throw error;
    }
};