@penkov/tasks_queue/dist/index.cjs

A lightweight PostgreSQL-backed task queue system with scheduling, retries, backoff strategies, and priority handling. Designed for efficiency and observability in modern Node.js applications.

'use strict';

var scats = require('scats');
var log4js = require('log4js');
var applicationMetrics = require('application-metrics');
var tslib = require('tslib');
var common = require('@nestjs/common');
var core = require('@nestjs/core');
var swagger = require('@nestjs/swagger');

exports.TaskStatus = void 0;
(function (TaskStatus) {
    TaskStatus["pending"] = "pending";
    TaskStatus["in_progress"] = "in_progress";
    TaskStatus["finished"] = "finished";
    TaskStatus["error"] = "error";
})(exports.TaskStatus || (exports.TaskStatus = {}));
exports.BackoffType = void 0;
(function (BackoffType) {
    BackoffType["constant"] = "constant";
    BackoffType["linear"] = "linear";
    BackoffType["exponential"] = "exponential";
})(exports.BackoffType || (exports.BackoffType = {}));
exports.TaskPeriodType = void 0;
(function (TaskPeriodType) {
    TaskPeriodType["fixed_rate"] = "fixed_rate";
    TaskPeriodType["fixed_delay"] = "fixed_delay";
})(exports.TaskPeriodType || (exports.TaskPeriodType = {}));
exports.MissedRunStrategy = void 0;
(function (MissedRunStrategy) {
    MissedRunStrategy["catch_up"] = "catch_up";
    MissedRunStrategy["skip_missed"] = "skip_missed";
})(exports.MissedRunStrategy || (exports.MissedRunStrategy = {}));
class TaskFailed extends Error {
    payload;
    constructor(message, payload) {
        super(message);
        this.payload = payload;
    }
}

class TimeUtils {
    static second = 1000;
    static minute = TimeUtils.second * 60;
    static hour = TimeUtils.minute * 60;
    static day = TimeUtils.hour * 24;
}

const defaultLoopInterval = TimeUtils.minute;
var PipelineNextOperationType;
(function (PipelineNextOperationType) {
    PipelineNextOperationType[PipelineNextOperationType["PollNext"] = 0] = "PollNext";
    PipelineNextOperationType[PipelineNextOperationType["Sleep"] = 1] = "Sleep";
})(PipelineNextOperationType || (PipelineNextOperationType = {}));
const PipelineNextOperationFactory = {
    pollNext: {
        type: PipelineNextOperationType.PollNext,
    },
    sleep: (delayMs = defaultLoopInterval) => ({
        type: PipelineNextOperationType.Sleep,
        delayMs,
    }),
};
const logger$4 = log4js.getLogger("TasksPipeline");
const noop = () => {
};
class TasksPipeline {
    maxConcurrentTasks;
    pollNextTask;
    peekNextStartAfter;
    processTask;
    loopInterval;
    tasksInProcess = 0;
    loopRunning = false;
    nextLoopTime = 0;
    periodicTaskFetcher = null;
    stopRequested = false;
    tasksCountListener = noop;
    loopTimer;
    constructor(maxConcurrentTasks, pollNextTask, peekNextStartAfter, processTask, loopInterval = defaultLoopInterval) {
        this.maxConcurrentTasks = maxConcurrentTasks;
        this.pollNextTask = pollNextTask;
        this.peekNextStartAfter = peekNextStartAfter;
        this.processTask = processTask;
        this.loopInterval = loopInterval;
        this.loopTimer = async () => {
            this.periodicTaskFetcher = null;
            let sleepInterval = this.loopInterval;
            try {
                sleepInterval = await this.loop();
            }
            finally {
                if (!this.periodicTaskFetcher) {
                    this.nextLoopTime = Date.now() + sleepInterval;
                    this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), Math.min(this.loopInterval, sleepInterval));
                }
            }
        };
    }
    start() {
        this.stopRequested = false;
        scats.option(this.periodicTaskFetcher).foreach((t) => clearTimeout(t));
        this.nextLoopTime = Date.now() + 1;
        this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), 1);
    }
    async stop() {
        this.stopRequested = true;
        scats.option(this.periodicTaskFetcher).foreach((t) => clearTimeout(t));
        if (this.tasksInProcess > 0) {
            await new Promise((resolve) => {
                this.tasksCountListener = (n) => {
                    if (n <= 0) {
                        this.tasksCountListener = noop;
                        resolve();
                    }
                };
            });
        }
    }
    triggerLoop() {
        if (!this.loopRunning && !this.stopRequested) {
            setTimeout(() => this.loop(), 1);
        }
    }
    async loop() {
        if (this.loopRunning || this.stopRequested) {
            return this.loopInterval;
        }
        this.loopRunning = true;
        let sleepInterval = this.loopInterval;
        try {
            let nextOp = PipelineNextOperationFactory.pollNext;
            while (nextOp.type !== PipelineNextOperationType.Sleep) {
                if (this.tasksInProcess < this.maxConcurrentTasks &&
                    !this.stopRequested) {
                    nextOp = await this.fetchNextTask();
                }
                else {
                    nextOp = PipelineNextOperationFactory.sleep(this.loopInterval);
                }
            }
            sleepInterval = Math.min(nextOp.delayMs, this.loopInterval);
        }
        finally {
            this.loopRunning = false;
        }
        const nextTriggerTime = Date.now() + sleepInterval;
        if (this.periodicTaskFetcher && this.nextLoopTime > nextTriggerTime) {
            logger$4.trace(`Resetting loop timer to closer time: from ${new Date(this.nextLoopTime)} to new ${new Date(nextTriggerTime)}`);
            clearTimeout(this.periodicTaskFetcher);
            this.nextLoopTime = nextTriggerTime;
            this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), Math.min(this.loopInterval, sleepInterval));
        }
        return sleepInterval;
    }
    async fetchNextTask() {
        const task = await this.pollNextTask();
        return task.match({
            some: async (t) => {
                this.tasksInProcess++;
                setImmediate(() => this.processTaskInLoop(t));
                return this.tasksInProcess < this.maxConcurrentTasks
                    ? PipelineNextOperationFactory.pollNext
                    : PipelineNextOperationFactory.sleep(this.loopInterval);
            },
            none: async () => {
                const nextTimeOpt = await this.peekNextStartAfter();
                const delayMs = nextTimeOpt
                    .map((startAfter) => {
                    const delay = startAfter.getTime() - Date.now();
                    return Math.max(0, delay);
                })
                    .getOrElseValue(this.loopInterval);
                return PipelineNextOperationFactory.sleep(delayMs);
            },
        });
    }
    async processTaskInLoop(t) {
        try {
            logger$4.debug(`Starting task (id=${t.id}) in queue ${t.queue} (active ${this.tasksInProcess} of ${this.maxConcurrentTasks})`);
            await this.processTask(t);
        }
        catch (error) {
            logger$4.warn(`Failed to process task ${t.id} in queue ${t.queue} `, error);
        }
        finally {
            this.taskIsDone();
            logger$4.debug(`Finished working with task ${t.id} in queue ${t.queue} (active ${this.tasksInProcess} of ${this.maxConcurrentTasks})`);
            setImmediate(() => this.loop());
        }
    }
    taskIsDone() {
        this.tasksInProcess--;
        this.tasksCountListener(this.tasksInProcess);
    }
}

const logger$3 = log4js.getLogger("TasksQueueWorker");
class TasksQueueWorker {
    tasksQueueDao;
    workers = new scats.mutable.HashMap();
    pipeline;
    constructor(tasksQueueDao, concurrency = 4, loopInterval = TimeUtils.minute) {
        this.tasksQueueDao = tasksQueueDao;
        this.pipeline = new TasksPipeline(concurrency, () => this.tasksQueueDao.nextPending(this.workers.keySet), () => this.tasksQueueDao.peekNextStartAfter(this.workers.keySet), (t) => this.processNextTask(t), loopInterval);
    }
    start() {
        this.pipeline.start();
    }
    async stop() {
        return this.pipeline.stop();
    }
    registerWorker(queueName, worker) {
        if (this.workers.containsKey(queueName)) {
            logger$3.warn(`Replacing existing worker for queue: ${queueName}`);
        }
        this.workers.put(queueName, worker);
    }
    tasksScheduled(queueName) {
        if (this.workers.containsKey(queueName)) {
            this.pipeline.triggerLoop();
        }
    }
    async processNextTask(task) {
        applicationMetrics.MetricsService.counter("tasks_queue_started").inc();
        await this.workers.get(task.queue).match({
            some: async (worker) => {
                try {
                    scats.Try(() => worker.starting(task.id, task.payload)).tapFailure((e) => logger$3.warn(`Failed to invoke 'starting' callback for task (id=${task.id}) in queue=${task.queue}`, e));
                    await worker.process(task.payload, {
                        maxAttempts: task.maxAttempts,
                        currentAttempt: task.currentAttempt,
                    });
                    if (scats.option(task.repeatType).isEmpty) {
                        await this.tasksQueueDao.finish(task.id);
                    }
                    else {
                        await this.tasksQueueDao.rescheduleIfPeriodic(task.id);
                    }
                    applicationMetrics.MetricsService.counter("tasks_queue_processed").inc();
                    (await scats.Try.promise(() => worker.completed(task.id, task.payload))).tapFailure((e) => logger$3.warn(`Failed to invoke 'completed' callback for task (id=${task.id}) in queue=${task.queue}`, e));
                }
                catch (e) {
                    const finalStatus = await this.tasksQueueDao.fail(task.id, e["message"] || e, e instanceof TaskFailed ? e.payload : task.payload);
                    (await scats.Try.promise(() => worker.failed(task.id, task.payload, finalStatus, e))).tapFailure((e) => logger$3.warn(`Failed to invoke 'failed' callback for task (id=${task.id}) in queue=${task.queue}`, e));
                    applicationMetrics.MetricsService.counter("tasks_queue_failed").inc();
                    logger$3.warn(`Failed to process task (id=${task.id}) in queue=${task.queue}`, e);
                }
            },
            none: async () => {
                applicationMetrics.MetricsService.counter("tasks_queue_skipped_no_worker").inc();
                logger$3.info(`Failed to process task (id=${task.id}) in queue=${task.queue}: no suitable worker found`);
            },
        });
    }
}

const logger$2 = log4js.getLogger("TasksAuxiliaryWorker");
class TasksAuxiliaryWorker {
    tasksQueueDao;
    manageTasksQueueService;
    workerTimer = null;
    metricsTimer = null;
    queuesCounts = scats.HashMap.empty;
    constructor(tasksQueueDao, manageTasksQueueService) {
        this.tasksQueueDao = tasksQueueDao;
        this.manageTasksQueueService = manageTasksQueueService;
    }
    start() {
        const runWorker = () => {
            this.runAuxiliaryJobs();
        };
        this.workerTimer = setInterval(() => {
            try {
                runWorker();
            }
            catch (e) {
                logger$2.warn("Failed to process stalled tasks", e);
            }
        }, TimeUtils.second * 30);
        try {
            runWorker();
        }
        catch (e) {
            logger$2.warn("Failed to process stalled tasks", e);
        }
        const runMetrics = () => {
            this.fetchMetrics();
        };
        this.metricsTimer = setInterval(() => {
            try {
                runMetrics();
            }
            catch (e) {
                logger$2.warn("Failed to sync metrics", e);
            }
        }, TimeUtils.minute * 2);
        try {
            runMetrics();
        }
        catch (e) {
            logger$2.warn("Failed to sync metrics", e);
        }
    }
    runAuxiliaryJobs() {
        try {
            this.tasksQueueDao
                .failStalled()
                .then((res) => {
                if (res.nonEmpty) {
                    logger$2.info(`Marked stalled as failed: ${res.mkString(", ")}`);
                }
            })
                .catch((e) => {
                logger$2.warn("Failed to process stalled tasks", e);
            });
            this.tasksQueueDao.resetFailed().catch((e) => {
                logger$2.warn("Failed to reset failed tasks", e);
            });
            this.tasksQueueDao.clearFinished().catch((e) => {
                logger$2.warn("Failed to clear finished tasks", e);
            });
        }
        catch (e) {
            logger$2.warn("Failed to process stalled tasks", e);
        }
    }
    fetchMetrics() {
        this.manageTasksQueueService
            .tasksCount()
            .then((tasksCounts) => {
            this.queuesCounts = tasksCounts.groupBy((c) => c.queueName);
            tasksCounts.foreach((c) => {
                applicationMetrics.MetricsService.gauge(`tasks_queue_${c.queueName}_${c.status}`.replace(/[^a-zA-Z0-9_:]/g, "_"), () => {
                    return this.queuesCounts
                        .get(c.queueName)
                        .getOrElseValue(scats.Nil)
                        .find((x) => c.status === x.status)
                        .map((c) => c.count)
                        .getOrElseValue(0);
                });
            });
        })
            .catch((e) => {
            logger$2.warn("Failed to sync metrics", e);
        });
    }
    async stop() {
        scats.option(this.workerTimer).foreach((t) => clearTimeout(t));
        scats.option(this.metricsTimer).foreach((t) => clearTimeout(t));
    }
}

const logger$1 = log4js.getLogger("TasksQueueService");
class TasksQueueService {
    tasksQueueDao;
    worker;
    auxiliaryWorker;
    constructor(tasksQueueDao, manageTasksQueueService, config) {
        this.tasksQueueDao = tasksQueueDao;
        this.worker = new TasksQueueWorker(this.tasksQueueDao, config.concurrency, config.loopInterval);
        if (config.runAuxiliaryWorker) {
            this.auxiliaryWorker = scats.some(new TasksAuxiliaryWorker(tasksQueueDao, manageTasksQueueService));
        }
        else {
            this.auxiliaryWorker = scats.none;
        }
    }
    async schedule(task) {
        await this.tasksQueueDao.schedule(task);
        this.taskScheduled(task.queue);
    }
    async scheduleAtFixedRate(task) {
        await this.tasksQueueDao.schedulePeriodic(task, exports.TaskPeriodType.fixed_rate);
        this.taskScheduled(task.queue);
    }
    async scheduleAtFixedDelay(task) {
        await this.tasksQueueDao.schedulePeriodic(task, exports.TaskPeriodType.fixed_delay);
        this.taskScheduled(task.queue);
    }
    taskScheduled(queueName) {
        this.worker.tasksScheduled(queueName);
    }
    registerWorker(queueName, worker) {
        this.worker.registerWorker(queueName, worker);
    }
    start() {
        try {
            this.worker.start();
            this.auxiliaryWorker.foreach((w) => w.start());
        }
        catch (e) {
            logger$1.warn("Failed to process stalled tasks", e);
        }
    }
    async stop() {
        await this.auxiliaryWorker.mapPromise((w) => w.stop());
        await this.worker.stop();
    }
}

const DEFAULT_POOL = "default";
const logger = log4js.getLogger("TasksPoolsService");
class TasksPoolsService {
    dao;
    pools;
    queuesPool = new scats.mutable.HashMap();
    auxiliaryWorker;
    constructor(dao, manageTasksQueueService, runAuxiliaryWorker, pools = [
        {
            name: DEFAULT_POOL,
            concurrency: 1,
            loopInterval: 60000,
        },
    ]) {
        this.dao = dao;
        const poolsCollection = scats.Collection.from(pools);
        const poolNames = poolsCollection.map((p) => p.name).toSet;
        if (poolsCollection.size !== poolNames.size) {
            throw new Error("Duplicate pool names detected");
        }
        this.pools = poolsCollection.toMap((p) => [
            p.name,
            new TasksQueueService(dao, manageTasksQueueService, {
                concurrency: p.concurrency,
                runAuxiliaryWorker: false,
                loopInterval: p.loopInterval,
            }),
        ]);
        this.auxiliaryWorker = runAuxiliaryWorker
            ? scats.some(new TasksAuxiliaryWorker(dao, manageTasksQueueService))
            : scats.none;
    }
    start() {
        logger.info(`Starting TasksPoolsService with ${this.pools.size} pools`);
        this.auxiliaryWorker.foreach((w) => w.start());
        this.pools.values.foreach((p) => p.start());
    }
    async stop(timeoutMs = 30000) {
        logger.info("Stopping TasksPoolsService");
        try {
            await Promise.race([
                Promise.all([
                    this.auxiliaryWorker.mapPromise((w) => w.stop()),
                    this.pools.values.mapPromise((p) => p.stop()),
                ]),
                new Promise((_, reject) => setTimeout(() => reject(new Error("Stop timeout")), timeoutMs)),
            ]);
            logger.info("TasksPoolsService stopped successfully");
        }
        catch (e) {
            logger.error("Failed to stop TasksPoolsService gracefully", e);
            throw e;
        }
    }
    registerWorker(queueName, worker, poolName = DEFAULT_POOL) {
        if (this.queuesPool.containsKey(queueName)) {
            throw new Error(`Queue '${queueName}' is already registered in pool '${this.queuesPool.get(queueName).getOrElseValue("unknown")}'`);
        }
        this.pools.get(poolName).match({
            some: (pool) => {
                pool.registerWorker(queueName, worker);
                this.queuesPool.put(queueName, poolName);
                logger.info(`Registered worker for queue '${queueName}' in pool '${poolName}'`);
            },
            none: () => {
                throw new Error(`Pool '${poolName}' not registered`);
            },
        });
    }
    async schedule(task) {
        const taskId = await this.dao.schedule(task);
        this.taskScheduled(task.queue, taskId);
    }
    async scheduleAtFixedRate(task) {
        const taskId = await this.dao.schedulePeriodic(task, exports.TaskPeriodType.fixed_rate);
        this.taskScheduled(task.queue, taskId);
    }
    async scheduleAtFixedDelay(task) {
        const taskId = await this.dao.schedulePeriodic(task, exports.TaskPeriodType.fixed_delay);
        this.taskScheduled(task.queue, taskId);
    }
    taskScheduled(queue, taskId) {
        this.queuesPool.get(queue).match({
            some: (poolName) => {
                this.pools.get(poolName).foreach((pool) => {
                    pool.taskScheduled(queue);
                });
            },
            none: () => {
                logger.info(`No worker registered for a queue '${queue}'. ` +
                    `Task (id=${taskId.getOrElseValue(-1)}) will remain in pending state`);
            },
        });
    }
}

class TasksQueueDao {
    pool;
    constructor(pool) {
        this.pool = pool;
    }
    async withClient(cb) {
        const client = await this.pool.connect();
        let res;
        try {
            res = await cb(client);
        }
        finally {
            client.release();
        }
        return res;
    }
    async schedule(task) {
        const now = new Date();
        return await this.withClient(async (cl) => {
            const res = await cl.query(`insert into tasks_queue (queue, created, status, priority, payload, timeout, max_attempts,
                                          start_after, initial_start, backoff, backoff_type)
                 values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
                 returning id`, [
                task.queue,
                now,
                exports.TaskStatus.pending,
                scats.option(task.priority).getOrElseValue(0),
                scats.option(task.payload).orNull,
                scats.option(task.timeout).getOrElseValue(TimeUtils.hour),
                scats.option(task.retries).getOrElseValue(1),
                scats.option(task.startAfter).orNull,
                scats.option(task.startAfter).getOrElseValue(now),
                scats.option(task.backoff).getOrElseValue(TimeUtils.minute),
                scats.option(task.backoffType).getOrElseValue(exports.BackoffType.linear),
            ]);
            return scats.Collection.from(res.rows).headOption.map((r) => r.id);
        });
    }
    async schedulePeriodic(task, periodType) {
        const now = new Date();
        return await this.withClient(async (cl) => {
            const res = await cl.query(`insert into tasks_queue (queue, created, status, priority, payload, timeout, max_attempts,
                                          start_after, initial_start, name,
                                          repeat_interval, repeat_type, backoff, backoff_type, missed_runs_strategy)
                 values ($1, $2, $3, $4, $5, $6, $7, $8, $8, $9, $10, $11, $12, $13, $14)
                 on conflict (name) do nothing
                 returning id`, [
                task.queue,
                now,
                exports.TaskStatus.pending,
                scats.option(task.priority).getOrElseValue(0),
                scats.option(task.payload).orNull,
                scats.option(task.timeout).getOrElseValue(TimeUtils.hour),
                scats.option(task.retries).getOrElseValue(1),
                scats.option(task.startAfter).getOrElseValue(now),
                task.name,
                task.period,
                periodType,
                scats.option(task.backoff).getOrElseValue(TimeUtils.minute),
                scats.option(task.backoffType).getOrElseValue(exports.BackoffType.linear),
                scats.option(task.missedRunStrategy).getOrElseValue(exports.MissedRunStrategy.skip_missed),
            ]);
            return scats.Collection.from(res.rows).headOption.map((r) => r.id);
        });
    }
    async nextPending(queueNames) {
        if (queueNames.isEmpty) {
            return scats.none;
        }
        const now = new Date();
        const placeholders = queueNames.zipWithIndex
            .map(([_, idx]) => `$${idx + 4}`)
            .mkString(",");
        const paramsStatic = [
            exports.TaskStatus.in_progress,
            now,
            exports.TaskStatus.pending,
        ];
        const params = paramsStatic.concat(queueNames.toArray);
        return await this.withClient(async (cl) => {
            const query = `
                WITH selected AS (SELECT id, payload, queue
                                  FROM tasks_queue
                                  WHERE status = $3
                                    AND queue IN (${placeholders})
                                    AND max_attempts > attempt
                                    AND (start_after IS NULL or start_after <= $2)
                                  ORDER BY priority DESC, id ASC
                                      FOR UPDATE SKIP LOCKED
                                  LIMIT 1)
                UPDATE tasks_queue
                SET status   = $1,
                    started  = $2,
                    finished = null,
                    attempt  = attempt + 1
                FROM selected
                WHERE tasks_queue.id = selected.id
                RETURNING 
                    tasks_queue.id, 
                    tasks_queue.payload, 
                    tasks_queue.queue,
                    tasks_queue.repeat_type,
                    tasks_queue.attempt,
                    tasks_queue.max_attempts
            `;
            const res = await cl.query(query, params);
            return scats.Collection.from(res.rows).headOption.map((r) => {
                return {
                    id: r["id"],
                    payload: scats.option(r["payload"]).orUndefined,
                    queue: r["queue"],
                    repeatType: scats.option(r["repeat_type"]).orUndefined,
                    currentAttempt: r["attempt"],
                    maxAttempts: r["max_attempts"],
                };
            });
        });
    }
    async peekNextStartAfter(queueNames) {
        if (queueNames.isEmpty) {
            return scats.none;
        }
        const now = new Date();
        const placeholders = queueNames.zipWithIndex
            .map(([_, idx]) => `$${idx + 4}`)
            .mkString(",");
        return this.withClient(async (cl) => {
            const paramsStatic = [exports.TaskStatus.pending, exports.TaskStatus.error, now];
            const params = paramsStatic.concat(queueNames.toArray);
            const res = await cl.query(`SELECT MIN(start_after) AS min_start
                 FROM tasks_queue
                 WHERE status IN ($1, $2)
                   AND queue IN (${placeholders})
                   AND start_after > $3`, params);
            return scats.Collection.from(res.rows)
                .headOption.flatMap((r) => scats.option(r["min_start"]))
                .map((ts) => new Date(ts));
        });
    }
    async finish(taskId) {
        const now = new Date();
        await this.withClient(async (cl) => {
            await cl.query(`update tasks_queue
                 set status=$1,
                     finished=$2,
                     error=null
                 where id = $3
                   and status = $4`, [exports.TaskStatus.finished, now, taskId, exports.TaskStatus.in_progress]);
        });
    }
    async rescheduleIfPeriodic(taskId) {
        await this.withClient(async (cl) => {
            const now = new Date();
            await cl.query(`
                    UPDATE tasks_queue
                    SET status      = $1,
                        start_after = CASE
                                          WHEN repeat_type = 'fixed_rate' AND missed_runs_strategy = 'catch_up' THEN
                                              start_after + (repeat_interval * interval '1 millisecond')
                                          WHEN repeat_type = 'fixed_rate' AND missed_runs_strategy = 'skip_missed' THEN
                                              initial_start +
                                              (CEIL(EXTRACT(EPOCH FROM ($2 - initial_start)) * 1000 / repeat_interval) *
                                               repeat_interval * interval '1 millisecond')
                                          ELSE
                                              $2 + (repeat_interval * interval '1 millisecond')
                            END,
                        finished    = $2,
                        error       = NULL,
                        attempt     = 0
                    WHERE id = $3
                      AND status = $4
                      AND repeat_interval IS NOT NULL
                      AND missed_runs_strategy IS NOT NULL
                      AND repeat_type IN ('${exports.TaskPeriodType.fixed_rate}', '${exports.TaskPeriodType.fixed_delay}');
                `, [exports.TaskStatus.pending, now, taskId, exports.TaskStatus.in_progress]);
        });
    }
    async fail(taskId, error, nextPayload) {
        const now = new Date();
        return await this.withClient(async (cl) => {
            const res = await cl.query(`
            UPDATE tasks_queue
            SET finished    = $1,
                error       = $2,
                status      = CASE
                                  WHEN attempt < max_attempts THEN $3 -- pending
                                  ELSE $4 -- error
                    END,
                start_after = CASE
                                  WHEN attempt < max_attempts THEN
                                      $1::timestamp + (
                                                          CASE backoff_type
                                                              WHEN 'constant' THEN backoff
                                                              WHEN 'linear' THEN (backoff * attempt)
                                                              WHEN 'exponential'
                                                                  THEN (backoff * POWER(2, (attempt - 1)))
                                                              ELSE backoff
                                                              END
                                                          ) * interval '1 millisecond'
                                  ELSE NULL
                    END,
                payload     = $7
            WHERE id = $5
              AND status = $6
            returning status
        `, [
                now,
                error,
                exports.TaskStatus.pending,
                exports.TaskStatus.error,
                taskId,
                exports.TaskStatus.in_progress,
                scats.option(nextPayload).orNull,
            ]);
            return scats.Collection.from(res.rows)
                .headOption.map((r) => exports.TaskStatus[r.status])
                .getOrElseValue(exports.TaskStatus.error);
        });
    }
    async failStalled() {
        const now = new Date();
        return await this.withClient(async (cl) => {
            const res = await cl.query(`update tasks_queue
                 set status=$1,
                     finished=$2,
                     error=$3
                 where status = $4
                   and timeout is not null
                   and started + timeout * interval '1  ms' < $5
                 returning id`, [exports.TaskStatus.error, now, "Timeout", exports.TaskStatus.in_progress, now]);
            return scats.Collection.from(res.rows).map((r) => r.id);
        });
    }
    async resetFailed() {
        await this.withClient(async (cl) => {
            await cl.query(`update tasks_queue
                 set status=$1,
                     finished=null
                 where status = $2
                   and timeout is not null
                   and max_attempts > COALESCE(attempt, 0)`, [exports.TaskStatus.pending, exports.TaskStatus.error]);
        });
    }
    async clearFinished(timeout = TimeUtils.day) {
        const expired = new Date(Date.now() - timeout);
        await this.withClient(async (cl) => {
            await cl.query(`delete
                 from tasks_queue
                 where status = $1
                   and finished < $2`, [exports.TaskStatus.finished, expired]);
        });
    }
    async statusCount(queue, status) {
        return await this.withClient(async (cl) => {
            const res = await cl.query(`select count(id) as cnt
                 from tasks_queue
                 where queue = $1
                   and status = $2`, [queue, status]);
            return scats.Collection.from(res.rows)
                .headOption.map((r) => r["cnt"])
                .getOrElseValue(0);
        });
    }
}
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", [Object]),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "schedule", null);
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", [Object, String]),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "schedulePeriodic", null);
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", [scats.HashSet]),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "nextPending", null);
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", [scats.HashSet]),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "peekNextStartAfter", null);
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", [Number]),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "finish", null);
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", [Number]),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "rescheduleIfPeriodic", null);
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", [Number, String, Object]),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "fail", null);
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", []),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "failStalled", null);
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", []),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "resetFailed", null);
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", [Number]),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "clearFinished", null);
tslib.__decorate([
    applicationMetrics.Metric(),
    tslib.__metadata("design:type", Function),
    tslib.__metadata("design:paramtypes", [String, String]),
    tslib.__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "statusCount", null);

const TASKS_QUEUE_OPTIONS = Symbol("TASKS_QUEUE_OPTIONS");

class TaskDto {
    id;
    queue;
    created;
    initialStart;
    started;
    finished;
    status;
    missedRunStrategy;
    priority;
    error;
    backoff;
    backoffType;
    timeout;
    name;
    startAfter;
    repeatInterval;
    repeatType;
    maxAttempts;
    attempt;
    payload;
    constructor(id, queue, created, initialStart, started, finished, status, missedRunStrategy, priority, error, backoff, backoffType, timeout, name, startAfter, repeatInterval, repeatType, maxAttempts, attempt, payload) {
        this.id = id;
        this.queue = queue;
        this.created = created;
        this.initialStart = initialStart;
        this.started = started;
        this.finished = finished;
        this.status = status;
        this.missedRunStrategy = missedRunStrategy;
        this.priority = priority;
        this.error = error;
        this.backoff = backoff;
        this.backoffType = backoffType;
        this.timeout = timeout;
        this.name = name;
        this.startAfter = startAfter;
        this.repeatInterval = repeatInterval;
        this.repeatType = repeatType;
        this.maxAttempts = maxAttempts;
        this.attempt = attempt;
        this.payload = payload;
    }
}
class TasksResult {
    items;
    total;
    constructor(items, total) {
        this.items = items;
        this.total = total;
    }
}
class TaskView {
    id;
    queue;
    created;
    initialStart;
    started;
    finished;
    status;
    missedRunStrategy;
    priority;
    error;
    backoff;
    backoffType;
    timeout;
    name;
    startAfter;
    repeatInterval;
    repeatType;
    maxAttempts;
    attempt;
    payload;
    static fromDto(dto) {
        const res = new TaskView();
        res.id = dto.id;
        res.queue = dto.queue;
        res.created = dto.created.getTime();
        res.initialStart = dto.initialStart.getTime();
        res.started = dto.started.map((d) => d.getTime()).orUndefined;
        res.finished = dto.finished.map((d) => d.getTime()).orUndefined;
        res.status = dto.status;
        res.missedRunStrategy = dto.missedRunStrategy;
        res.priority = dto.priority;
        res.error = dto.error.orUndefined;
        res.backoff = dto.backoff;
        res.backoffType = dto.backoffType;
        res.timeout = dto.timeout;
        res.name = dto.name.orUndefined;
        res.startAfter = dto.startAfter.map((d) => d.getTime()).orUndefined;
        res.repeatInterval = dto.repeatInterval.orUndefined;
        res.repeatType = dto.repeatType.orUndefined;
        res.maxAttempts = dto.maxAttempts;
        res.attempt = dto.attempt;
        res.payload = dto.payload;
        return res;
    }
}
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "id", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", String)
], TaskView.prototype, "queue", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "created", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "initialStart", void 0);
tslib.__decorate([
    swagger.ApiProperty({ required: false }),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "started", void 0);
tslib.__decorate([
    swagger.ApiProperty({ required: false }),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "finished", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", String)
], TaskView.prototype, "status", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", String)
], TaskView.prototype, "missedRunStrategy", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "priority", void 0);
tslib.__decorate([
    swagger.ApiProperty({ required: false }),
    tslib.__metadata("design:type", String)
], TaskView.prototype, "error", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "backoff", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", String)
], TaskView.prototype, "backoffType", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "timeout", void 0);
tslib.__decorate([
    swagger.ApiProperty({ required: false }),
    tslib.__metadata("design:type", String)
], TaskView.prototype, "name", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "startAfter", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "repeatInterval", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", String)
], TaskView.prototype, "repeatType", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "maxAttempts", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TaskView.prototype, "attempt", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Object)
], TaskView.prototype, "payload", void 0);
class TasksResultView {
    items;
    total;
}
tslib.__decorate([
    swagger.ApiProperty({ type: TaskView, isArray: true }),
    tslib.__metadata("design:type", Array)
], TasksResultView.prototype, "items", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], TasksResultView.prototype, "total", void 0);
class QueueStat {
    queueName;
    p50;
    p75;
    p95;
    p99;
    p999;
    constructor(queueName, p50, p75, p95, p99, p999) {
        this.queueName = queueName;
        this.p50 = p50;
        this.p75 = p75;
        this.p95 = p95;
        this.p99 = p99;
        this.p999 = p999;
    }
}
class TasksCount {
    queueName;
    status;
    count;
    constructor(queueName, status, count) {
        this.queueName = queueName;
        this.status = status;
        this.count = count;
    }
}
class QueueStatView {
    queueName;
    p50;
    p75;
    p95;
    p99;
    p999;
    static fromDto(o) {
        const res = new QueueStatView();
        res.queueName = o.queueName;
        res.p50 = o.p50;
        res.p75 = o.p75;
        res.p95 = o.p95;
        res.p99 = o.p99;
        res.p999 = o.p999;
        return res;
    }
}
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", String)
], QueueStatView.prototype, "queueName", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], QueueStatView.prototype, "p50", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], QueueStatView.prototype, "p75", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], QueueStatView.prototype, "p95", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], QueueStatView.prototype, "p99", void 0);
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", Number)
], QueueStatView.prototype, "p999", void 0);
class TasksCountView {
    queueName;
    status;
    count;
    static fromDto(o) {
        const res = new TasksCountView();
        res.queueName = o.queueName;
        res.status = o.status;
        res.count = o.count;
        return res;
    }
}
tslib.__decorate([
    swagger.ApiProperty(),
    tslib.__metadata("design:type", String)
], TasksCountView.prototype, "queueName", void 0);
tslib.__decorate([
    swagger.ApiProperty({ enum: exports.TaskStatus, enumName: "TaskStatus" }),
    tslib.__metadata("design:type", String)
], TasksCountView.prototype, "status", void 0);
tslib.__decorate([
    swagger.ApiProperty({ type: "integer" }),
    tslib.__metadata("design:type", Number)
], TasksCountView.prototype, "count", void 0);
class QueuesStat {
    waitTime;
    workTime;
    tasksCount;
}
tslib.__decorate([
    swagger.ApiProperty({ type: QueueStatView, isArray: true }),
    tslib.__metadata("design:type", Array)
], QueuesStat.prototype, "waitTime", void 0);
tslib.__decorate([
    swagger.ApiProperty({ type: QueueStatView, isArray: true }),
    tslib.__metadata("design:type", Array)
], QueuesStat.prototype, "workTime", void 0);
tslib.__decorate([
    swagger.ApiProperty({ type: TasksCountView, isArray: true }),
    tslib.__metadata("design:type", Array)
], QueuesStat.prototype, "tasksCount", void 0);

class ManageTasksQueueService {
    pool;
    constructor(pool) {
        this.pool = pool;
    }
    async findByStatus(params) {
        const parts = new scats.mutable.ArrayBuffer();
        scats.option(params.status).foreach((status) => {
            parts.append(`status='${status}'`);
        });
        const where = parts.nonEmpty ? `where ${parts.toArray.join(" and ")}` : "";
        const res = await this.pool.query(`select *
             from tasks_queue ${where}
             order by created desc
             limit $1 offset $2`, [params.limit, params.offset]);
        const total = await this.pool.query(`select count(*) as total
                                             from tasks_queue ${where}`);
        const items = scats.Collection.from(res.rows).map((row) => {
            return new TaskDto(row["id"], row["queue"], row["created"], row["initial_start"], scats.option(row["started"]), scats.option(row["finished"]), row["status"], row["missed_run_strategy"], row["priority"], scats.option(row["error"]), row["backoff"], row["backoff_type"], row["timeout"], scats.option(row["name"]), scats.option(row["start_after"]), scats.option(row["repeat_interval"]), scats.option(row["repeat_type"]), row["max_attempts"], row["attempt"], row["payload"]);
        });
        return new TasksResult(items, total.rows[0]["total"]);
    }
    async failedCount() {
        const res = await this.pool.query(`select count(*) as total
             from tasks_queue
             where status = '${exports.TaskStatus.error}'`);
        return res.rows[0]["total"];
    }
    clearFailed() {
        return this.pool.query(`delete
                                from tasks_queue
                                where status = '${exports.TaskStatus.error}'`);
    }
    async waitTimeByQueue() {
        const res = await this.pool.query(`
            SELECT queue,
                   percentile_disc(0.50) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (started - created)))  AS p50,
                   percentile_disc(0.75) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (started - created)))  AS p75,
                   percentile_disc(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (started - created)))  AS p95,
                   percentile_disc(0.99) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (started - created)))  AS p99,
                   percentile_disc(0.999) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (started - created))) AS p999
            FROM tasks_queue
            WHERE started IS NOT NULL
              and attempt = 1
            GROUP BY queue
            ORDER BY queue
        `);
        return scats.Collection.from(res.rows).map((row) => new QueueStat(row["queue"], Number(row["p50"]), Number(row["p75"]), Number(row["p95"]), Number(row["p99"]), Number(row["p999"])));
    }
    async restartFailedTask(taskId) {
        await this.pool.query(`
            update tasks_queue
            set status='${exports.TaskStatus.pending}',
                attempt=0
            where id = $1
              and status = '${exports.TaskStatus.error}'
        `, [taskId]);
    }
    async restartAllFailedInQueue(queue) {
        await this.pool.query(`
            update tasks_queue
            set status='${exports.TaskStatus.pending}',
                attempt=0
            where queue = $1
              and status = '${exports.TaskStatus.error}'
        `, [queue]);
    }
    async tasksCount() {
        const res = await this.pool.query(`
            SELECT queue,
                   status,
                   COUNT(*) AS task_count
            FROM tasks_queue
            GROUP BY queue, status
            ORDER BY queue, status
        `);
        return scats.Collection.from(res.rows).map((row) => new TasksCount(row["queue"], exports.TaskStatus[row["status"]], Number(row["task_count"])));
    }
    async workTimeByQueue() {
        const res = await this.pool.query(`
            SELECT queue,
                   percentile_disc(0.50) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished - started)))  AS p50,
                   percentile_disc(0.75) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished - started)))  AS p75,
                   percentile_disc(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished - started)))  AS p95,
                   percentile_disc(0.99) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished - started)))  AS p99,
                   percentile_disc(0.999) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished - started))) AS p999
            FROM tasks_queue
            WHERE started IS NOT NULL
              AND finished IS NOT NULL
            GROUP BY queue
            ORDER BY queue
        `);
        return scats.Collection.from(res.rows).map((row) => new QueueStat(row["queue"], Number(row["p50"]), Number(row["p75"]), Number(row["p95"]), Number(row["p99"]), Number(row["p999"])));
    }
}

var TasksQueueModule_1;
exports.TasksQueueModule = TasksQueueModule_1 = class TasksQueueModule {
    moduleRef;
    constructor(moduleRef) {
        this.moduleRef = moduleRef;
    }
    static forRootAsync(options) {
        const asyncProviders = this.createAsyncProviders(options);
        return {
            module: TasksQueueModule_1,
            imports: options.imports,
            providers: [
                ...asyncProviders,
                {
                    provide: TasksQueueDao,
                    inject: [TASKS_QUEUE_OPTIONS],
                    useFactory: (opts) => new TasksQueueDao(opts.db),
                },
                {
                    provide: ManageTasksQueueService,
                    inject: [TASKS_QUEUE_OPTIONS],
                    useFactory: (opts) => new ManageTasksQueueService(opts.db),
                },
                {
                    provide: TasksPoolsService,
                    inject: [TasksQueueDao, ManageTasksQueueService, TASKS_QUEUE_OPTIONS],
                    useFactory: (dao, manageService, opts) => new TasksPoolsService(dao, manageService, scats.option(opts.runAuxiliaryWorker).forall(scats.identity), opts.pools),
                },
            ],
            exports: [TasksPoolsService, ManageTasksQueueService],
        };
    }
    static createAsyncProviders(options) {
        if (options.useExisting || options.useFactory) {
            return [this.createAsyncOptionsProvider(options)];
        }
        const useClass = options.useClass;
        return [
            this.createAsyncOptionsProvider(options),
            {
                provide: useClass,
                useClass,
            },
        ];
    }
    static createAsyncOptionsProvider(options) {
        if (options.useFactory) {
            return {
                provide: TASKS_QUEUE_OPTIONS,
                useFactory: options.useFactory,
                inject: options.inject || [],
            };
        }
        const inject = [
            (options.useClass ||
                options.useExisting),
        ];
        return {
            provide: TASKS_QUEUE_OPTIONS,
            useFactory: async (optionsFactory) => await optionsFactory.createTelegrafOptions(),
            inject,
        };
    }
    onApplicationBootstrap() {
        const poolsService = this.moduleRef.get(TasksPoolsService);
        poolsService.start();
    }
    async onApplicationShutdown() {
        const poolsService = this.moduleRef.get(TasksPoolsService);
        await poolsService.stop(TimeUtils.minute);
    }
};
exports.