repomix/lib/shared/processConcurrency.js

A tool to pack repository contents to single file for AI consumption

var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
import os from 'node:os';
import { Tinypool } from 'tinypool';
import { logger } from './logger.js';
// Worker initialization is expensive, so we prefer fewer threads unless there are many files
const TASKS_PER_THREAD = 100;
export const getProcessConcurrency = () => {
    return typeof os.availableParallelism === 'function' ? os.availableParallelism() : os.cpus().length;
};
export const getWorkerThreadCount = (numOfTasks) => {
    const processConcurrency = getProcessConcurrency();
    const minThreads = 1;
    // Limit max threads based on number of tasks
    const maxThreads = Math.max(minThreads, Math.min(processConcurrency, Math.ceil(numOfTasks / TASKS_PER_THREAD)));
    return {
        minThreads,
        maxThreads,
    };
};
export const createWorkerPool = (numOfTasks, workerPath) => {
    const { minThreads, maxThreads } = getWorkerThreadCount(numOfTasks);
    logger.trace(`Initializing worker pool with min=${minThreads}, max=${maxThreads} threads. Worker path: ${workerPath}`);
    const startTime = process.hrtime.bigint();
    const pool = new Tinypool({
        filename: workerPath,
        // Use child_process for better memory management
        runtime: 'child_process',
        minThreads,
        maxThreads,
        idleTimeout: 5000,
        workerData: {
            logLevel: logger.getLogLevel(),
        },
    });
    const endTime = process.hrtime.bigint();
    const initTime = Number(endTime - startTime) / 1e6; // Convert to milliseconds
    logger.debug(`Tinypool initialization took ${initTime.toFixed(2)}ms`);
    return pool;
};
export const cleanupWorkerPool = (pool) => __awaiter(void 0, void 0, void 0, function* () {
    var _a;
    try {
        logger.debug('Cleaning up worker pool...');
        // Check if running in Bun runtime
        const isBun = (_a = process.versions) === null || _a === void 0 ? void 0 : _a.bun;
        if (isBun) {
            // If running in Bun, we cannot use Tinypool's destroy method
            logger.debug('Running in Bun environment, skipping Tinypool destroy method');
        }
        else {
            // Standard Node.js cleanup
            yield pool.destroy();
        }
        logger.debug('Worker pool cleaned up successfully');
    }
    catch (error) {
        logger.debug('Error during worker pool cleanup:', error);
    }
});
export const initTaskRunner = (numOfTasks, workerPath) => {
    const pool = createWorkerPool(numOfTasks, workerPath);
    return {
        run: (task) => pool.run(task),
        cleanup: () => cleanupWorkerPool(pool),
    };
};
//# sourceMappingURL=processConcurrency.js.map