@penkov/tasks_queue/dist/tasks-pipeline.js

A lightweight PostgreSQL-backed task queue system with scheduling, retries, backoff strategies, and priority handling. Designed for efficiency and observability in modern Node.js applications.

import { option } from "scats";
import log4js from "log4js";
import { TimeUtils } from "./time-utils.js";
import { SystemClock } from "./clock.js";
import { MetricsService } from "application-metrics";
import { poolMetricName } from "./metrics-utils.js";
const defaultLoopInterval = TimeUtils.minute;
var PipelineNextOperationType;
(function (PipelineNextOperationType) {
    PipelineNextOperationType[PipelineNextOperationType["PollNext"] = 0] = "PollNext";
    PipelineNextOperationType[PipelineNextOperationType["Sleep"] = 1] = "Sleep";
})(PipelineNextOperationType || (PipelineNextOperationType = {}));
const PipelineNextOperationFactory = {
    pollNext: {
        type: PipelineNextOperationType.PollNext,
    },
    sleep: (delayMs = defaultLoopInterval) => ({
        type: PipelineNextOperationType.Sleep,
        delayMs,
    }),
};
const logger = log4js.getLogger("TasksPipeline");
const noop = () => {
};
export class TasksPipeline {
    maxConcurrentTasks;
    pollNextTask;
    peekNextStartAfter;
    processTask;
    loopInterval;
    clock;
    poolName;
    tasksInProcess = 0;
    loopRunning = false;
    nextLoopTime = 0;
    periodicTaskFetcher = null;
    stopRequested = false;
    tasksCountListener = noop;
    loopTimer;
    loopStartedAt = 0;
    lastPollStartedAt = 0;
    lastPollFinishedAt = 0;
    lastSuccessfulPollAt = 0;
    lastPollDurationMs = 0;
    constructor(maxConcurrentTasks, pollNextTask, peekNextStartAfter, processTask, loopInterval = defaultLoopInterval, clock = new SystemClock(), poolName = "default") {
        this.maxConcurrentTasks = maxConcurrentTasks;
        this.pollNextTask = pollNextTask;
        this.peekNextStartAfter = peekNextStartAfter;
        this.processTask = processTask;
        this.loopInterval = loopInterval;
        this.clock = clock;
        this.poolName = poolName;
        this.registerMetrics();
        this.loopTimer = async () => {
            this.periodicTaskFetcher = null;
            let sleepInterval = this.loopInterval;
            try {
                sleepInterval = await this.loop();
            }
            finally {
                if (!this.periodicTaskFetcher) {
                    this.nextLoopTime = this.nowMs() + sleepInterval;
                    this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), Math.min(this.loopInterval, sleepInterval));
                }
            }
        };
    }
    start() {
        this.stopRequested = false;
        option(this.periodicTaskFetcher).foreach((t) => clearTimeout(t));
        this.nextLoopTime = this.nowMs() + 1;
        this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), 1);
    }
    async runOnce() {
        await this.loop();
    }
    getState() {
        return {
            maxConcurrentTasks: this.maxConcurrentTasks,
            tasksInProcess: this.tasksInProcess,
            loopRunning: this.loopRunning,
            loopStartedAt: this.loopStartedAt,
            lastPollStartedAt: this.lastPollStartedAt,
            lastPollFinishedAt: this.lastPollFinishedAt,
            lastSuccessfulPollAt: this.lastSuccessfulPollAt,
            lastPollDurationMs: this.lastPollDurationMs,
            nextLoopTime: this.nextLoopTime,
        };
    }
    async stop() {
        this.stopRequested = true;
        option(this.periodicTaskFetcher).foreach((t) => clearTimeout(t));
        if (this.tasksInProcess > 0) {
            await new Promise((resolve) => {
                this.tasksCountListener = (n) => {
                    if (n <= 0) {
                        this.tasksCountListener = noop;
                        resolve();
                    }
                };
            });
        }
    }
    triggerLoop() {
        if (!this.loopRunning && !this.stopRequested) {
            setTimeout(() => this.loop(), 1);
        }
    }
    async loop() {
        if (this.loopRunning || this.stopRequested) {
            return this.loopInterval;
        }
        this.loopRunning = true;
        this.loopStartedAt = this.nowMs();
        this.lastPollStartedAt = this.loopStartedAt;
        MetricsService.counter(poolMetricName(this.poolName, "loop_poll_started_total")).inc();
        let sleepInterval = this.loopInterval;
        let completedSuccessfully = false;
        try {
            let nextOp = PipelineNextOperationFactory.pollNext;
            while (nextOp.type !== PipelineNextOperationType.Sleep) {
                if (this.tasksInProcess < this.maxConcurrentTasks &&
                    !this.stopRequested) {
                    nextOp = await this.fetchNextTask();
                }
                else {
                    nextOp = PipelineNextOperationFactory.sleep(this.loopInterval);
                }
            }
            sleepInterval = Math.min(nextOp.delayMs, this.loopInterval);
            completedSuccessfully = true;
        }
        finally {
            const finishedAt = this.nowMs();
            this.lastPollFinishedAt = finishedAt;
            this.lastPollDurationMs = finishedAt - this.loopStartedAt;
            if (completedSuccessfully) {
                this.lastSuccessfulPollAt = finishedAt;
            }
            this.loopRunning = false;
            this.loopStartedAt = 0;
            MetricsService.counter(poolMetricName(this.poolName, "loop_poll_finished_total")).inc();
        }
        const nextTriggerTime = this.nowMs() + sleepInterval;
        if (this.periodicTaskFetcher && this.nextLoopTime > nextTriggerTime) {
            logger.trace(`Resetting loop timer to closer time: from ${new Date(this.nextLoopTime)} to new ${new Date(nextTriggerTime)}`);
            clearTimeout(this.periodicTaskFetcher);
            this.nextLoopTime = nextTriggerTime;
            this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), Math.min(this.loopInterval, sleepInterval));
        }
        return sleepInterval;
    }
    async fetchNextTask() {
        const task = await this.pollNextTask();
        return task.match({
            some: async (t) => {
                if (this.stopRequested) {
                    return PipelineNextOperationFactory.sleep(this.loopInterval);
                }
                this.tasksInProcess++;
                setImmediate(() => this.processTaskInLoop(t));
                return this.tasksInProcess < this.maxConcurrentTasks
                    ? PipelineNextOperationFactory.pollNext
                    : PipelineNextOperationFactory.sleep(this.loopInterval);
            },
            none: async () => {
                if (this.stopRequested) {
                    return PipelineNextOperationFactory.sleep(this.loopInterval);
                }
                const nextTimeOpt = await this.peekNextStartAfter();
                const delayMs = nextTimeOpt
                    .map((startAfter) => {
                    const delay = startAfter.getTime() - this.nowMs();
                    return Math.max(0, delay);
                })
                    .getOrElseValue(this.loopInterval);
                return PipelineNextOperationFactory.sleep(delayMs);
            },
        });
    }
    async processTaskInLoop(t) {
        if (this.stopRequested) {
            return;
        }
        try {
            logger.debug(`Starting task (id=${t.id}) in queue ${t.queue} (active ${this.tasksInProcess} of ${this.maxConcurrentTasks})`);
            await this.processTask(t);
        }
        catch (error) {
            logger.warn(`Failed to process task ${t.id} in queue ${t.queue} `, error);
        }
        finally {
            this.taskIsDone();
            logger.debug(`Finished working with task ${t.id} in queue ${t.queue} (active ${this.tasksInProcess} of ${this.maxConcurrentTasks})`);
            setImmediate(() => this.loop());
        }
    }
    taskIsDone() {
        this.tasksInProcess--;
        this.tasksCountListener(this.tasksInProcess);
    }
    nowMs() {
        return this.clock.now().getTime();
    }
    registerMetrics() {
        MetricsService.gauge(poolMetricName(this.poolName, "slots_total"), () => this.maxConcurrentTasks);
        MetricsService.gauge(poolMetricName(this.poolName, "slots_busy"), () => this.tasksInProcess);
        MetricsService.gauge(poolMetricName(this.poolName, "loop_running"), () => this.loopRunning ? 1 : 0);
        MetricsService.gauge(poolMetricName(this.poolName, "loop_running_ms"), () => (this.loopRunning ? this.nowMs() - this.loopStartedAt : 0));
        MetricsService.gauge(poolMetricName(this.poolName, "loop_last_poll_duration_ms"), () => this.lastPollDurationMs);
        MetricsService.gauge(poolMetricName(this.poolName, "loop_last_poll_started_at"), () => this.lastPollStartedAt);
        MetricsService.gauge(poolMetricName(this.poolName, "loop_last_poll_finished_at"), () => this.lastPollFinishedAt);
        MetricsService.gauge(poolMetricName(this.poolName, "loop_last_successful_poll_at"), () => this.lastSuccessfulPollAt);
        MetricsService.gauge(poolMetricName(this.poolName, "loop_next_at"), () => this.nextLoopTime);
    }
}