wakaq/dist/childWorker copy.js

Background task queue for Node backed by Redis, a super minimal Celery

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.WakaQChildWorker = void 0;
const exceptions_js_1 = require("./exceptions.js");
const logger_js_1 = require("./logger.js");
const serializer_js_1 = require("./serializer.js");
class WakaQChildWorker {
    constructor(wakaq) {
        this._stopProcessing = false;
        this._numTasksProcessed = 0;
        this.wakaq = wakaq;
        this.logger = (0, logger_js_1.setupLogging)(this.wakaq, true);
        this.wakaq.logger = this.logger;
    }
    async start() {
        const _this = this;
        process.on('SIGINT', this._ignoreSignal);
        process.on('SIGTERM', () => _this._stopFromSigTerm());
        process.on('SIGQUIT', () => _this._onSoftTimeout());
        process.on('message', this._onMessageFromParent);
        try {
            this.logger.debug(`started worker process ${process.pid}`);
            if (this.wakaq.afterWorkerStartedCallback)
                await this.wakaq.afterWorkerStartedCallback();
            this._numTasksProcessed = 0;
            while (!this._stopProcessing) {
                this._sendPingToParent();
                const { queueBrokerKey, payload } = (await this._blockingDequeue()) || {};
                if (queueBrokerKey !== undefined && payload !== undefined) {
                    const task = this.wakaq.tasks.get(payload.name);
                    if (!task && payload.name)
                        this.logger.error(`Task not found: ${payload.name}`);
                    if (task) {
                        const queue = this.wakaq.queuesByKey.get(queueBrokerKey);
                        this.wakaq.currentTask = task;
                        const retry = payload.retry ?? 0;
                        try {
                            await this._executeTask(task, payload.args, queue);
                        }
                        catch (error) {
                            if (error instanceof exceptions_js_1.SoftTimeout) {
                                const maxRetries = task.maxRetries ?? queue?.maxRetries ?? this.wakaq.maxRetries;
                                if (retry + 1 > maxRetries) {
                                    this.logger.error(error);
                                }
                                else {
                                    this.logger.warning(error);
                                    this.wakaq.enqueueAtEnd(task.name, payload.args, queue, retry);
                                }
                            }
                            else {
                                this.logger.error(error);
                            }
                        }
                        finally {
                            this.wakaq.currentTask = undefined;
                        }
                    }
                    else {
                        this._sendPingToParent();
                    }
                }
                else {
                    this._sendPingToParent();
                }
                if (this.wakaq.maxTasksPerWorker && this._numTasksProcessed >= this.wakaq.maxTasksPerWorker) {
                    this.logger.info(`restarting worker after ${this._numTasksProcessed} tasks`);
                    this._stopProcessing = true;
                }
            }
        }
        catch (error) {
            if (error instanceof exceptions_js_1.SoftTimeout) {
                if (this.wakaq.currentTask !== null)
                    throw error;
            }
            else {
                console.log('got error');
                console.log(error);
                this.logger.error(error);
            }
        }
        finally {
            this.wakaq.dispose();
            process.off('message', this._onMessageFromParent);
        }
    }
    _ignoreSignal() {
        console.log('Got SIGINT');
    }
    _stopFromSigTerm() {
        console.log('Got SIGTERM');
        this._stop();
    }
    _stop() {
        console.log('Child exiting');
        this._stopProcessing = true;
    }
    _onSoftTimeout() {
        console.log('Got SIGQUIT');
        this._stopProcessing = true;
        throw new exceptions_js_1.SoftTimeout('SoftTimeout');
    }
    async _blockingDequeue() {
        if (this.wakaq.brokerKeys.length === 0) {
            this.wakaq.sleep(this.wakaq.waitTimeout);
            return undefined;
        }
        const data = await this.wakaq.broker.blpop(this.wakaq.brokerKeys, this.wakaq.waitTimeout.seconds);
        if (!data)
            return undefined;
        return { queueBrokerKey: data[0], payload: (0, serializer_js_1.deserialize)(data[1]) };
    }
    _sendPingToParent(taskName = '', queueName = '') {
        const msg = {
            type: 'wakaq-ping',
            taskName,
            queueName,
        };
        if (process.send) {
            process.send(msg, undefined, undefined, (e) => {
                if (e)
                    this.logger.warn(e);
            });
        }
    }
    async _executeTask(task, args, queue) {
        this._sendPingToParent(task.name, queue?.name);
        this.logger.debug(`running with args ${args}`);
        if (this.wakaq.beforeTaskStartedCallback)
            this.wakaq.beforeTaskStartedCallback(task);
        try {
            await task.fn(...args);
        }
        finally {
            this._sendPingToParent();
            this._numTasksProcessed += 1;
            if (this.wakaq.afterTaskFinishedCallback)
                this.wakaq.afterTaskFinishedCallback(task);
        }
    }
    async _onMessageFromParent(message) {
        console.log('_onMessageFromParent');
        console.log(message);
        const payload = (0, serializer_js_1.deserialize)(message);
        const task = this.wakaq.tasks.get(payload.name);
        if (!task) {
            this.logger.error(`Task not found: ${payload.name}`);
            return;
        }
        let retry = 0;
        this.wakaq.currentTask = task;
        try {
            while (true) {
                try {
                    await this._executeTask(task, payload.payload);
                    break;
                }
                catch (error) {
                    if (error instanceof exceptions_js_1.SoftTimeout) {
                        retry += 1;
                        const maxRetries = task.maxRetries ?? this.wakaq.maxRetries;
                        if (retry > maxRetries) {
                            this.logger.error(error);
                            break;
                        }
                        else {
                            this.logger.warning(error);
                        }
                    }
                    else {
                        this.logger.error(error);
                        break;
                    }
                }
            }
        }
        finally {
            this.wakaq.currentTask = undefined;
        }
    }
}
exports.WakaQChildWorker = WakaQChildWorker;