@penkov/tasks_queue/dist/tasks-queue.worker.js

A lightweight PostgreSQL-backed task queue system with scheduling, retries, backoff strategies, and priority handling. Designed for efficiency and observability in modern Node.js applications.

import { mutable, option, Try } from "scats";
import log4js from "log4js";
import { MetricsService } from "application-metrics";
import { TasksPipeline } from "./tasks-pipeline.js";
import { TaskFailed } from "./tasks-model.js";
import { TimeUtils } from "./time-utils.js";
const logger = log4js.getLogger("TasksQueueWorker");
export class TasksQueueWorker {
    tasksQueueDao;
    workers = new mutable.HashMap();
    pipeline;
    constructor(tasksQueueDao, concurrency = 4, loopInterval = TimeUtils.minute) {
        this.tasksQueueDao = tasksQueueDao;
        this.pipeline = new TasksPipeline(concurrency, () => this.tasksQueueDao.nextPending(this.workers.keySet), () => this.tasksQueueDao.peekNextStartAfter(this.workers.keySet), (t) => this.processNextTask(t), loopInterval);
    }
    start() {
        this.pipeline.start();
    }
    async stop() {
        return this.pipeline.stop();
    }
    registerWorker(queueName, worker) {
        if (this.workers.containsKey(queueName)) {
            logger.warn(`Replacing existing worker for queue: ${queueName}`);
        }
        this.workers.put(queueName, worker);
    }
    tasksScheduled(queueName) {
        if (this.workers.containsKey(queueName)) {
            this.pipeline.triggerLoop();
        }
    }
    async processNextTask(task) {
        MetricsService.counter("tasks_queue_started").inc();
        await this.workers.get(task.queue).match({
            some: async (worker) => {
                try {
                    Try(() => worker.starting(task.id, task.payload)).tapFailure((e) => logger.warn(`Failed to invoke 'starting' callback for task (id=${task.id}) in queue=${task.queue}`, e));
                    await worker.process(task.payload, {
                        maxAttempts: task.maxAttempts,
                        currentAttempt: task.currentAttempt,
                    });
                    if (option(task.repeatType).isEmpty) {
                        await this.tasksQueueDao.finish(task.id);
                    }
                    else {
                        await this.tasksQueueDao.rescheduleIfPeriodic(task.id);
                    }
                    MetricsService.counter("tasks_queue_processed").inc();
                    (await Try.promise(() => worker.completed(task.id, task.payload))).tapFailure((e) => logger.warn(`Failed to invoke 'completed' callback for task (id=${task.id}) in queue=${task.queue}`, e));
                }
                catch (e) {
                    const finalStatus = await this.tasksQueueDao.fail(task.id, e["message"] || e, e instanceof TaskFailed ? e.payload : task.payload);
                    (await Try.promise(() => worker.failed(task.id, task.payload, finalStatus, e))).tapFailure((e) => logger.warn(`Failed to invoke 'failed' callback for task (id=${task.id}) in queue=${task.queue}`, e));
                    MetricsService.counter("tasks_queue_failed").inc();
                    logger.warn(`Failed to process task (id=${task.id}) in queue=${task.queue}`, e);
                }
            },
            none: async () => {
                MetricsService.counter("tasks_queue_skipped_no_worker").inc();
                logger.info(`Failed to process task (id=${task.id}) in queue=${task.queue}: no suitable worker found`);
            },
        });
    }
}