@penkov/tasks_queue/dist/tasks-pipeline.js

A lightweight PostgreSQL-backed task queue system with scheduling, retries, backoff strategies, and priority handling. Designed for efficiency and observability in modern Node.js applications.

import { option } from "scats";
import log4js from "log4js";
import { TimeUtils } from "./time-utils.js";
const defaultLoopInterval = TimeUtils.minute;
var PipelineNextOperationType;
(function (PipelineNextOperationType) {
    PipelineNextOperationType[PipelineNextOperationType["PollNext"] = 0] = "PollNext";
    PipelineNextOperationType[PipelineNextOperationType["Sleep"] = 1] = "Sleep";
})(PipelineNextOperationType || (PipelineNextOperationType = {}));
const PipelineNextOperationFactory = {
    pollNext: {
        type: PipelineNextOperationType.PollNext,
    },
    sleep: (delayMs = defaultLoopInterval) => ({
        type: PipelineNextOperationType.Sleep,
        delayMs,
    }),
};
const logger = log4js.getLogger("TasksPipeline");
const noop = () => {
};
export class TasksPipeline {
    maxConcurrentTasks;
    pollNextTask;
    peekNextStartAfter;
    processTask;
    loopInterval;
    tasksInProcess = 0;
    loopRunning = false;
    nextLoopTime = 0;
    periodicTaskFetcher = null;
    stopRequested = false;
    tasksCountListener = noop;
    loopTimer;
    constructor(maxConcurrentTasks, pollNextTask, peekNextStartAfter, processTask, loopInterval = defaultLoopInterval) {
        this.maxConcurrentTasks = maxConcurrentTasks;
        this.pollNextTask = pollNextTask;
        this.peekNextStartAfter = peekNextStartAfter;
        this.processTask = processTask;
        this.loopInterval = loopInterval;
        this.loopTimer = async () => {
            this.periodicTaskFetcher = null;
            let sleepInterval = this.loopInterval;
            try {
                sleepInterval = await this.loop();
            }
            finally {
                if (!this.periodicTaskFetcher) {
                    this.nextLoopTime = Date.now() + sleepInterval;
                    this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), Math.min(this.loopInterval, sleepInterval));
                }
            }
        };
    }
    start() {
        this.stopRequested = false;
        option(this.periodicTaskFetcher).foreach((t) => clearTimeout(t));
        this.nextLoopTime = Date.now() + 1;
        this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), 1);
    }
    async stop() {
        this.stopRequested = true;
        option(this.periodicTaskFetcher).foreach((t) => clearTimeout(t));
        if (this.tasksInProcess > 0) {
            await new Promise((resolve) => {
                this.tasksCountListener = (n) => {
                    if (n <= 0) {
                        this.tasksCountListener = noop;
                        resolve();
                    }
                };
            });
        }
    }
    triggerLoop() {
        if (!this.loopRunning && !this.stopRequested) {
            setTimeout(() => this.loop(), 1);
        }
    }
    async loop() {
        if (this.loopRunning || this.stopRequested) {
            return this.loopInterval;
        }
        this.loopRunning = true;
        let sleepInterval = this.loopInterval;
        try {
            let nextOp = PipelineNextOperationFactory.pollNext;
            while (nextOp.type !== PipelineNextOperationType.Sleep) {
                if (this.tasksInProcess < this.maxConcurrentTasks &&
                    !this.stopRequested) {
                    nextOp = await this.fetchNextTask();
                }
                else {
                    nextOp = PipelineNextOperationFactory.sleep(this.loopInterval);
                }
            }
            sleepInterval = Math.min(nextOp.delayMs, this.loopInterval);
        }
        finally {
            this.loopRunning = false;
        }
        const nextTriggerTime = Date.now() + sleepInterval;
        if (this.periodicTaskFetcher && this.nextLoopTime > nextTriggerTime) {
            logger.trace(`Resetting loop timer to closer time: from ${new Date(this.nextLoopTime)} to new ${new Date(nextTriggerTime)}`);
            clearTimeout(this.periodicTaskFetcher);
            this.nextLoopTime = nextTriggerTime;
            this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), Math.min(this.loopInterval, sleepInterval));
        }
        return sleepInterval;
    }
    async fetchNextTask() {
        const task = await this.pollNextTask();
        return task.match({
            some: async (t) => {
                this.tasksInProcess++;
                setImmediate(() => this.processTaskInLoop(t));
                return this.tasksInProcess < this.maxConcurrentTasks
                    ? PipelineNextOperationFactory.pollNext
                    : PipelineNextOperationFactory.sleep(this.loopInterval);
            },
            none: async () => {
                const nextTimeOpt = await this.peekNextStartAfter();
                const delayMs = nextTimeOpt
                    .map((startAfter) => {
                    const delay = startAfter.getTime() - Date.now();
                    return Math.max(0, delay);
                })
                    .getOrElseValue(this.loopInterval);
                return PipelineNextOperationFactory.sleep(delayMs);
            },
        });
    }
    async processTaskInLoop(t) {
        try {
            logger.debug(`Starting task (id=${t.id}) in queue ${t.queue} (active ${this.tasksInProcess} of ${this.maxConcurrentTasks})`);
            await this.processTask(t);
        }
        catch (error) {
            logger.warn(`Failed to process task ${t.id} in queue ${t.queue} `, error);
        }
        finally {
            this.taskIsDone();
            logger.debug(`Finished working with task ${t.id} in queue ${t.queue} (active ${this.tasksInProcess} of ${this.maxConcurrentTasks})`);
            setImmediate(() => this.loop());
        }
    }
    taskIsDone() {
        this.tasksInProcess--;
        this.tasksCountListener(this.tasksInProcess);
    }
}