diffjam/lib/src/WorkerPool.js

cli for diffjam.com

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.WorkerPool = void 0;
/*
  Manages a pool of workers, and distributes the file paths to process to each.
  The workers, defined by src/workerProcess.ts, each run in a separate process.
  Each communicates back to the master process with matches of the policies.
  The Runner, defined by src/Runner.ts, assigns a `resultsMap` with keys for each
  policy and an `onResults` callback that the WorkerPool calls when all work is done.
*/
var node_os_1 = require("node:os");
var node_cluster_1 = __importDefault(require("node:cluster"));
var node_assert_1 = require("node:assert");
var WorkerPool = /** @class */ (function () {
    function WorkerPool(configFilePath, cwd) {
        this.configFilePath = configFilePath;
        this.cwd = cwd;
        this.closed = false;
        this.queued = [];
        this.workers = [];
        this.filesChecked = [];
        this.resultsMap = {};
        this.onResults = function () { };
        var numCpus = (0, node_os_1.cpus)().length;
        while (this.workers.length < numCpus - 1) {
            this.createWorker();
        }
    }
    WorkerPool.prototype.createWorker = function () {
        var _this = this;
        var inProgress = new Set();
        var worker = node_cluster_1.default.fork({
            configFilePath: this.configFilePath,
            cwd: this.cwd,
        });
        var workerEntry = { worker: worker, inProgress: inProgress };
        worker.on("message", function (msg) {
            // console.log(msg.type, (msg as any).filePath, this.workers);
            if (msg.type === "match") {
                _this.resultsMap[msg.policyName].matches.push(msg.match);
            }
            else if (msg.type === "processedFile") {
                if (!inProgress.has(msg.filePath)) {
                    throw new Error('file not in progress: ' + msg.filePath);
                }
                inProgress.delete(msg.filePath);
                var queuedFilePath = _this.queued.shift();
                if (queuedFilePath) {
                    _this.checkFile(workerEntry, queuedFilePath);
                }
                else if (_this.isWorkDone()) {
                    _this.onDone();
                }
            }
        });
        worker.on("error", function (error) {
            throw new Error("FROM WORKER: " + error.message);
        });
        this.workers.push(workerEntry);
    };
    WorkerPool.prototype.isWorkDone = function () {
        return (this.closed
            && !this.queued.length
            && this.workers.every(function (_a) {
                var inProgress = _a.inProgress;
                return !inProgress.size;
            }));
    };
    WorkerPool.prototype.workerWithLeastInProgress = function () {
        var leastSoFar = this.workers[0];
        for (var _i = 0, _a = this.workers; _i < _a.length; _i++) {
            var worker = _a[_i];
            // Exit early if we find a worker with no files in progress as this is the minimum
            if (!worker.inProgress) {
                return worker;
            }
            if (worker.inProgress.size < leastSoFar.inProgress.size) {
                leastSoFar = worker;
            }
        }
        return leastSoFar;
    };
    WorkerPool.prototype.checkFile = function (worker, filePath) {
        this.filesChecked.push(filePath);
        worker.inProgress.add(filePath);
        worker.worker.send({ type: "processFile", filePath: filePath });
    };
    WorkerPool.prototype.processFile = function (filePath) {
        var worker = this.workerWithLeastInProgress();
        if (worker.inProgress.size >= 3) {
            this.queued.push(filePath);
        }
        else {
            this.checkFile(worker, filePath);
        }
    };
    WorkerPool.prototype.killAllWorkers = function () {
        var _this = this;
        return new Promise(function (resolve) {
            _this.workers.forEach(function (_a) {
                var worker = _a.worker;
                return worker.kill();
            });
            _this.workers.forEach(function (_a) {
                var worker = _a.worker;
                return worker.once("exit", function () {
                    if (_this.workers.every(function (_a) {
                        var worker = _a.worker;
                        return worker.isDead();
                    })) {
                        resolve();
                    }
                });
            });
        });
    };
    WorkerPool.prototype.onDone = function () {
        this.killAllWorkers();
        this.onResults();
    };
    WorkerPool.prototype.onFilesDone = function () {
        (0, node_assert_1.equal)(this.closed, false, "onFilesDone called twice");
        this.closed = true;
        if (this.isWorkDone()) {
            this.onDone();
        }
    };
    return WorkerPool;
}());
exports.WorkerPool = WorkerPool;