peanar/dist/app.js

A job queue for Node.js based on RabbitMQ

"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.EAppState = void 0;
const uuid = __importStar(require("uuid"));
const debug_1 = __importDefault(require("debug"));
const debug = (0, debug_1.default)('peanar:app');
const exceptions_1 = require("./exceptions");
const broker_1 = __importDefault(require("./amqplib_compat/broker"));
const worker_1 = __importDefault(require("./worker"));
const job_1 = __importDefault(require("./job"));
const stream_1 = require("stream");
const registry_1 = __importDefault(require("./registry"));
const transact_1 = __importDefault(require("./transact"));
var EAppState;
(function (EAppState) {
    EAppState["RUNNING"] = "RUNNING";
    EAppState["CLOSING"] = "CLOSING";
    EAppState["CLOSED"] = "CLOSED";
})(EAppState || (exports.EAppState = EAppState = {}));
class PeanarApp {
    registry = new registry_1.default;
    log;
    broker;
    jobClass;
    consumers = new Map;
    workers = new Map;
    jobs = new Map;
    transactions = new Set();
    state = EAppState.RUNNING;
    constructor(options = {}) {
        this.broker = new broker_1.default({
            connection: options.connection,
            poolSize: options.poolSize || 5,
            prefetch: options.prefetch || 1
        });
        this.jobClass = options.jobClass || job_1.default;
        this.log = options.logger || console.log.bind(console);
    }
    async _shutdown(timeout) {
        // Immediately stop receiving new messages
        await Promise.all([...this.consumers.values()].flat().map(c => c.cancel()));
        debug('shutdown(): consumers cancelled');
        // Anything that is subject to the timeout should go in here
        await Promise.all([
            // Wait a few seconds for running jobs to finish their work
            ...[...this.workers.values()].flat().map(w => w.shutdown(timeout)),
            // Also wait for pending transactions to be finalized
            ...[...this.transactions].map(t => t.waitUntil(timeout).catch(() => {
                this.log(`Transaction ${t.queueName} timed out while shutting down Peanar.`);
            })),
        ]);
        debug('shutdown(): workers shut down');
        debug('shutdown(): transactions concluded');
        // Close the channel pool and then the amqp connection
        await this.broker.shutdown();
        debug('shutdown(): broker shut down');
    }
    async shutdown(timeout) {
        debug('shutdown() called');
        this.state = EAppState.CLOSING;
        await this._shutdown(timeout);
        this.state = EAppState.CLOSED;
    }
    _registerWorker(queue, worker) {
        const workers = this.workers.get(queue) || [];
        workers.push(worker);
        this.workers.set(queue, workers);
    }
    _registerConsumer(queue, consumer) {
        const consumers = this.consumers.get(queue) || [];
        consumers.push(consumer);
        this.consumers.set(queue, consumers);
    }
    call(name, args) {
        const enqueue = this.jobs.get(name);
        if (typeof enqueue !== 'function') {
            throw new exceptions_1.PeanarInternalError(`Job ${name} is not registered with the app.`);
        }
        return enqueue.apply(this, args);
    }
    async _publish(routing_key, exchange, def, req) {
        if (this.state !== EAppState.RUNNING) {
            throw new exceptions_1.PeanarInternalError('PeanarApp::_publish() called while app is not in running state.');
        }
        const properties = {
            correlationId: req.correlationId,
            replyTo: def.replyTo
        };
        if (typeof def.expires === 'number') {
            properties.expiration = def.expires.toString();
        }
        if (typeof def.max_priority === 'number') {
            if (typeof req.priority === 'number' && req.priority <= def.max_priority) {
                properties.priority = req.priority;
            }
            else if (typeof def.default_priority === 'number') {
                properties.priority = def.default_priority;
            }
        }
        await this.broker.publish({
            routing_key,
            exchange,
            properties,
            body: {
                id: req.id,
                name: req.name,
                args: req.args
            }
        });
        return req.id;
    }
    async enqueueJobRequest(def, req) {
        debug(`Peanar: enqueueJob(${def.queue}:${def.name}})`);
        return this._publish(def.routingKey, def.exchange, def, req);
    }
    async _enqueueJobResponse(job, result) {
        debug('Peanar: _enqueueJobResponse()');
        if (!job.def.replyTo)
            throw new exceptions_1.PeanarInternalError('PeanarApp::_enqueueJobResponse() called with no replyTo defined');
        await this.broker.publish({
            routing_key: job.def.replyTo,
            exchange: '',
            properties: {
                correlationId: job.correlationId || job.id
            },
            body: this._prepareJobResponse(job, result)
        });
    }
    _prepareJobRequest(name, args) {
        return {
            id: uuid.v4(),
            name,
            args,
            attempt: 1
        };
    }
    _prepareJobResponse(job, result) {
        const res = {
            id: job.id,
            name: job.name,
            status: result.status,
        };
        if (result.status === 'SUCCESS') {
            res.result = result.result;
        }
        else {
            res.error = result.error;
        }
        return res;
    }
    _createTransactor(def) {
        return () => {
            const t = new transact_1.default(def, this);
            this.transactions.add(t);
            t.once('conclude', () => this.transactions.delete(t));
            return t;
        };
    }
    _createDelayedEnqueuer(def) {
        return (...args) => {
            return this._publish(`${def.queue}.delayed`, '', def, this._prepareJobRequest(def.name, args));
        };
    }
    _createEnqueuerWithPriority(def) {
        return (priority, ...args) => {
            debug(`Peanar: job.enqueueJob('${def.name}').withPriority(${priority})`);
            const req = this._prepareJobRequest(def.name, args);
            req.priority = priority;
            return this.enqueueJobRequest(def, req);
        };
    }
    _createEnqueuer(def) {
        const self = this;
        function enqueueJob(...args) {
            debug(`Peanar: job.enqueueJob('${def.name}')`);
            return self.enqueueJobRequest(def, self._prepareJobRequest(def.name, args));
        }
        enqueueJob.rpc = async (...args) => { };
        enqueueJob.delayed = this._createDelayedEnqueuer(def);
        enqueueJob.transaction = this._createTransactor(def);
        enqueueJob.withPriority = this._createEnqueuerWithPriority(def);
        return enqueueJob;
    }
    job(def) {
        if (def.max_priority && def.default_priority) {
            if (def.default_priority > def.max_priority) {
                throw new Error('max_priority should be greater than or equeal to the default priority.');
            }
        }
        else if (def.default_priority) {
            def.max_priority = def.default_priority;
        }
        const job_def = this.registry.registerJob(def);
        debug(`Peanar: job('${def.queue}', '${job_def.name}')`);
        const enqueueJob = this._createEnqueuer(job_def);
        this.jobs.set(job_def.name, enqueueJob);
        return enqueueJob;
    }
    async declareAmqResources() {
        await this.broker.queues(this.registry.queues);
        await this.broker.exchanges(this.registry.exchanges);
        await this.broker.bindings(this.registry.bindings);
    }
    pauseQueue(queue) {
        const consumers = this.consumers.get(queue);
        if (!consumers)
            return;
        for (const c of consumers)
            c.pause();
    }
    resumeQueue(queue) {
        const consumers = this.consumers.get(queue);
        if (!consumers)
            return;
        for (const c of consumers)
            c.resume();
    }
    async _startWorker(queue, channel, consumer, options) {
        const worker = new worker_1.default(this, channel, queue, {
            logger: options?.logger
        });
        consumer.on('channelChanged', ch => {
            worker.channel = ch;
        });
        this._registerConsumer(queue, consumer);
        this._registerWorker(queue, worker);
        return consumer
            .pipe(worker)
            .pipe(options?.outputWritable ?? new stream_1.Writable({
            objectMode: true,
            write: (result, _encoding, cb) => {
                if (result.status === 'FAILURE') {
                    this.log(result.error);
                }
                if (result.job.def.replyTo) {
                    this._enqueueJobResponse(result.job, result).then(_ => cb(), ex => cb(ex));
                }
                else {
                    return cb();
                }
            }
        }));
    }
    async worker(options) {
        const { queues, concurrency, prefetch = 1 } = options;
        const worker_queues = (Array.isArray(queues) && queues.length > 0)
            ? queues
            : this.registry.workerQueues;
        const queues_to_start = [...worker_queues].flatMap(q => Array(concurrency).fill(q));
        return Promise.all(this.broker.consumeOver(queues_to_start).map(async (p) => {
            const { queue, channel, consumer } = await p;
            return this._startWorker(queue, channel, consumer, options);
        }));
    }
}
exports.default = PeanarApp;