@penkov/tasks_queue
Version:
A lightweight PostgreSQL-backed task queue system with scheduling, retries, backoff strategies, and priority handling. Designed for efficiency and observability in modern Node.js applications.
336 lines (335 loc) • 14.2 kB
JavaScript
import { __decorate, __metadata } from "tslib";
import { Collection, HashSet, none, option } from "scats";
import { BackoffType, MissedRunStrategy, TaskPeriodType, TaskStatus, } from "./tasks-model.js";
import { Metric } from "application-metrics";
import { TimeUtils } from "./time-utils.js";
export class TasksQueueDao {
pool;
constructor(pool) {
this.pool = pool;
}
async withClient(cb) {
const client = await this.pool.connect();
let res;
try {
res = await cb(client);
}
finally {
client.release();
}
return res;
}
async schedule(task) {
const now = new Date();
return await this.withClient(async (cl) => {
const res = await cl.query(`insert into tasks_queue (queue, created, status, priority, payload, timeout, max_attempts,
start_after, initial_start, backoff, backoff_type)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
returning id`, [
task.queue,
now,
TaskStatus.pending,
option(task.priority).getOrElseValue(0),
option(task.payload).orNull,
option(task.timeout).getOrElseValue(TimeUtils.hour),
option(task.retries).getOrElseValue(1),
option(task.startAfter).orNull,
option(task.startAfter).getOrElseValue(now),
option(task.backoff).getOrElseValue(TimeUtils.minute),
option(task.backoffType).getOrElseValue(BackoffType.linear),
]);
return Collection.from(res.rows).headOption.map((r) => r.id);
});
}
async schedulePeriodic(task, periodType) {
const now = new Date();
return await this.withClient(async (cl) => {
const res = await cl.query(`insert into tasks_queue (queue, created, status, priority, payload, timeout, max_attempts,
start_after, initial_start, name,
repeat_interval, repeat_type, backoff, backoff_type, missed_runs_strategy)
values ($1, $2, $3, $4, $5, $6, $7, $8, $8, $9, $10, $11, $12, $13, $14)
on conflict (name) do nothing
returning id`, [
task.queue,
now,
TaskStatus.pending,
option(task.priority).getOrElseValue(0),
option(task.payload).orNull,
option(task.timeout).getOrElseValue(TimeUtils.hour),
option(task.retries).getOrElseValue(1),
option(task.startAfter).getOrElseValue(now),
task.name,
task.period,
periodType,
option(task.backoff).getOrElseValue(TimeUtils.minute),
option(task.backoffType).getOrElseValue(BackoffType.linear),
option(task.missedRunStrategy).getOrElseValue(MissedRunStrategy.skip_missed),
]);
return Collection.from(res.rows).headOption.map((r) => r.id);
});
}
async nextPending(queueNames) {
if (queueNames.isEmpty) {
return none;
}
const now = new Date();
const placeholders = queueNames.zipWithIndex
.map(([_, idx]) => `$${idx + 4}`)
.mkString(",");
const paramsStatic = [
TaskStatus.in_progress,
now,
TaskStatus.pending,
];
const params = paramsStatic.concat(queueNames.toArray);
return await this.withClient(async (cl) => {
const query = `
WITH selected AS (SELECT id, payload, queue
FROM tasks_queue
WHERE status = $3
AND queue IN (${placeholders})
AND max_attempts > attempt
AND (start_after IS NULL or start_after <= $2)
ORDER BY priority DESC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT 1)
UPDATE tasks_queue
SET status = $1,
started = $2,
finished = null,
attempt = attempt + 1
FROM selected
WHERE tasks_queue.id = selected.id
RETURNING
tasks_queue.id,
tasks_queue.payload,
tasks_queue.queue,
tasks_queue.repeat_type,
tasks_queue.attempt,
tasks_queue.max_attempts
`;
const res = await cl.query(query, params);
return Collection.from(res.rows).headOption.map((r) => {
return {
id: r["id"],
payload: option(r["payload"]).orUndefined,
queue: r["queue"],
repeatType: option(r["repeat_type"]).orUndefined,
currentAttempt: r["attempt"],
maxAttempts: r["max_attempts"],
};
});
});
}
async peekNextStartAfter(queueNames) {
if (queueNames.isEmpty) {
return none;
}
const now = new Date();
const placeholders = queueNames.zipWithIndex
.map(([_, idx]) => `$${idx + 4}`)
.mkString(",");
return this.withClient(async (cl) => {
const paramsStatic = [TaskStatus.pending, TaskStatus.error, now];
const params = paramsStatic.concat(queueNames.toArray);
const res = await cl.query(`SELECT MIN(start_after) AS min_start
FROM tasks_queue
WHERE status IN ($1, $2)
AND queue IN (${placeholders})
AND start_after > $3`, params);
return Collection.from(res.rows)
.headOption.flatMap((r) => option(r["min_start"]))
.map((ts) => new Date(ts));
});
}
async finish(taskId) {
const now = new Date();
await this.withClient(async (cl) => {
await cl.query(`update tasks_queue
set status=$1,
finished=$2,
error=null
where id = $3
and status = $4`, [TaskStatus.finished, now, taskId, TaskStatus.in_progress]);
});
}
async rescheduleIfPeriodic(taskId) {
await this.withClient(async (cl) => {
const now = new Date();
await cl.query(`
UPDATE tasks_queue
SET status = $1,
start_after = CASE
WHEN repeat_type = 'fixed_rate' AND missed_runs_strategy = 'catch_up' THEN
start_after + (repeat_interval * interval '1 millisecond')
WHEN repeat_type = 'fixed_rate' AND missed_runs_strategy = 'skip_missed' THEN
initial_start +
(CEIL(EXTRACT(EPOCH FROM ($2 - initial_start)) * 1000 / repeat_interval) *
repeat_interval * interval '1 millisecond')
ELSE
$2 + (repeat_interval * interval '1 millisecond')
END,
finished = $2,
error = NULL,
attempt = 0
WHERE id = $3
AND status = $4
AND repeat_interval IS NOT NULL
AND missed_runs_strategy IS NOT NULL
AND repeat_type IN ('${TaskPeriodType.fixed_rate}', '${TaskPeriodType.fixed_delay}');
`, [TaskStatus.pending, now, taskId, TaskStatus.in_progress]);
});
}
async fail(taskId, error, nextPayload) {
const now = new Date();
return await this.withClient(async (cl) => {
const res = await cl.query(`
UPDATE tasks_queue
SET finished = $1,
error = $2,
status = CASE
WHEN attempt < max_attempts THEN $3 -- pending
ELSE $4 -- error
END,
start_after = CASE
WHEN attempt < max_attempts THEN
$1::timestamp + (
CASE backoff_type
WHEN 'constant' THEN backoff
WHEN 'linear' THEN (backoff * attempt)
WHEN 'exponential'
THEN (backoff * POWER(2, (attempt - 1)))
ELSE backoff
END
) * interval '1 millisecond'
ELSE NULL
END,
payload = $7
WHERE id = $5
AND status = $6
returning status
`, [
now,
error,
TaskStatus.pending,
TaskStatus.error,
taskId,
TaskStatus.in_progress,
option(nextPayload).orNull,
]);
return Collection.from(res.rows)
.headOption.map((r) => TaskStatus[r.status])
.getOrElseValue(TaskStatus.error);
});
}
async failStalled() {
const now = new Date();
return await this.withClient(async (cl) => {
const res = await cl.query(`update tasks_queue
set status=$1,
finished=$2,
error=$3
where status = $4
and timeout is not null
and started + timeout * interval '1 ms' < $5
returning id`, [TaskStatus.error, now, "Timeout", TaskStatus.in_progress, now]);
return Collection.from(res.rows).map((r) => r.id);
});
}
async resetFailed() {
await this.withClient(async (cl) => {
await cl.query(`update tasks_queue
set status=$1,
finished=null
where status = $2
and timeout is not null
and max_attempts > COALESCE(attempt, 0)`, [TaskStatus.pending, TaskStatus.error]);
});
}
async clearFinished(timeout = TimeUtils.day) {
const expired = new Date(Date.now() - timeout);
await this.withClient(async (cl) => {
await cl.query(`delete
from tasks_queue
where status = $1
and finished < $2`, [TaskStatus.finished, expired]);
});
}
async statusCount(queue, status) {
return await this.withClient(async (cl) => {
const res = await cl.query(`select count(id) as cnt
from tasks_queue
where queue = $1
and status = $2`, [queue, status]);
return Collection.from(res.rows)
.headOption.map((r) => r["cnt"])
.getOrElseValue(0);
});
}
}
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Object]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "schedule", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Object, String]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "schedulePeriodic", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [HashSet]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "nextPending", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [HashSet]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "peekNextStartAfter", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "finish", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "rescheduleIfPeriodic", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number, String, Object]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "fail", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "failStalled", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "resetFailed", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "clearFinished", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [String, String]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "statusCount", null);