@penkov/tasks_queue
Version:
A lightweight PostgreSQL-backed task queue system with scheduling, retries, backoff strategies, and priority handling. Designed for efficiency and observability in modern Node.js applications.
745 lines (742 loc) • 32.1 kB
JavaScript
import { __decorate, __metadata } from "tslib";
import { Collection, HashSet, none, option, some } from "scats";
import { BackoffType, MissedRunStrategy, taskHeartbeatThrottleMs, TaskPeriodType, TaskStatus, } from "./tasks-model.js";
import { Metric } from "application-metrics";
import { TimeUtils } from "./time-utils.js";
import { CronExpressionUtils } from "./cron-expression-utils.js";
import { PeriodicScheduleUtils, } from "./periodic-schedule-utils.js";
import { MultiStepPayload } from "./multi-step-payload.js";
import { ActiveChildState } from "./active-child-state.js";
export class TasksQueueDao {
pool;
constructor(pool) {
this.pool = pool;
}
async insertOneTimeTask(cl, task, parentId, now = new Date()) {
const res = await cl.query(`insert into tasks_queue (parent_id, queue, created, status, priority, payload, timeout, max_attempts,
start_after, initial_start, backoff, backoff_type)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
returning id`, [
option(parentId).orNull,
task.queue,
now,
TaskStatus.pending,
option(task.priority).getOrElseValue(0),
option(task.payload).orNull,
option(task.timeout).getOrElseValue(TimeUtils.hour),
option(task.retries).getOrElseValue(1),
option(task.startAfter).orNull,
option(task.startAfter).getOrElseValue(now),
option(task.backoff).getOrElseValue(TimeUtils.minute),
option(task.backoffType).getOrElseValue(BackoffType.linear),
]);
return Collection.from(res.rows).headOption.map((r) => r.id);
}
async withClient(cb) {
const client = await this.pool.connect();
let res;
try {
res = await cb(client);
}
finally {
client.release();
}
return res;
}
async schedule(task, now = new Date()) {
return await this.withClient(async (cl) => {
return this.insertOneTimeTask(cl, task, undefined, now);
});
}
async blockParentAndScheduleChild(parentTaskId, childTask, parentPayload, expectedStarted, now = new Date()) {
return await this.withClient(async (cl) => {
await cl.query("BEGIN");
try {
const parentRes = await cl.query(`update tasks_queue
set status = $1,
attempt = greatest(attempt - 1, 0),
started = null,
last_heartbeat = null,
finished = null,
error = null
where id = $2
and status = $3
and started = $4
and repeat_type is null
returning id`, [
TaskStatus.blocked,
parentTaskId,
TaskStatus.in_progress,
expectedStarted,
]);
if (option(parentRes.rowCount).getOrElseValue(0) !== 1) {
await cl.query("ROLLBACK");
return none;
}
const childTaskId = await this.insertOneTimeTask(cl, childTask, parentTaskId, now);
await childTaskId.mapPromise(async (id) => {
await cl.query(`update tasks_queue
set payload = $1
where id = $2
and status = $3`, [
MultiStepPayload.fromJson(parentPayload).copy({
activeChild: option(new ActiveChildState(id, childTask.allowFailure === true)),
}).toJson,
parentTaskId,
TaskStatus.blocked,
]);
return id;
});
await cl.query("COMMIT");
return childTaskId;
}
catch (e) {
await cl.query("ROLLBACK");
throw e;
}
});
}
async schedulePeriodic(task, periodType, now = new Date()) {
const repeatInterval = PeriodicScheduleUtils.resolveRepeatInterval(task, periodType);
const cronExpression = PeriodicScheduleUtils.resolveCronExpression(task, periodType);
option(cronExpression).foreach((expr) => CronExpressionUtils.validate(expr));
if (task.name.length > 20) {
throw new Error("Periodic task 'name' must be 20 characters or less");
}
const replaceExisting = option(task.replaceExisting).getOrElseValue(false);
const hasExplicitStartAfter = task.startAfter !== undefined;
return await this.withClient(async (cl) => {
const params = Collection.of(task.queue, now, TaskStatus.pending, option(task.priority).getOrElseValue(0), option(task.payload).orNull, option(task.timeout).getOrElseValue(TimeUtils.hour), option(task.retries).getOrElseValue(1), option(task.startAfter).getOrElseValue(now), task.name, repeatInterval, cronExpression, periodType, option(task.backoff).getOrElseValue(TimeUtils.minute), option(task.backoffType).getOrElseValue(BackoffType.linear), option(task.missedRunStrategy).getOrElseValue(MissedRunStrategy.skip_missed)).toArray;
const res = await cl.query(replaceExisting
? `insert into tasks_queue (queue, created, status, priority, payload, timeout, max_attempts,
start_after, initial_start, name,
repeat_interval, cron_expression, repeat_type, backoff, backoff_type, missed_runs_strategy)
values ($1, $2, $3, $4, $5, $6, $7, $8, $8, $9, $10, $11, $12, $13, $14, $15)
on conflict (name) do update
set queue = excluded.queue,
status = excluded.status,
priority = excluded.priority,
payload = excluded.payload,
timeout = excluded.timeout,
max_attempts = excluded.max_attempts,
start_after = case when $16 then excluded.start_after else tasks_queue.start_after end,
initial_start = case when $16 then excluded.initial_start else tasks_queue.initial_start end,
repeat_interval = excluded.repeat_interval,
cron_expression = excluded.cron_expression,
repeat_type = excluded.repeat_type,
backoff = excluded.backoff,
backoff_type = excluded.backoff_type,
missed_runs_strategy = excluded.missed_runs_strategy,
started = null,
last_heartbeat = null,
finished = null,
error = null,
attempt = 0,
parent_id = null,
result = null
where tasks_queue.status = $3
returning id`
: `insert into tasks_queue (queue, created, status, priority, payload, timeout, max_attempts,
start_after, initial_start, name,
repeat_interval, cron_expression, repeat_type, backoff, backoff_type, missed_runs_strategy)
values ($1, $2, $3, $4, $5, $6, $7, $8, $8, $9, $10, $11, $12, $13, $14, $15)
on conflict (name) do nothing
returning id`, replaceExisting ? params.concat(hasExplicitStartAfter) : params);
return Collection.from(res.rows).headOption.map((r) => r.id);
});
}
async nextPending(queueNames, now = new Date()) {
if (queueNames.isEmpty) {
return none;
}
const placeholders = queueNames.zipWithIndex
.map(([_, idx]) => `$${idx + 4}`)
.mkString(",");
const params = Collection.of(TaskStatus.in_progress, now, TaskStatus.pending).appendedAll(queueNames.toArray).toArray;
return await this.withClient(async (cl) => {
const query = `
WITH selected AS (SELECT id, parent_id, payload, queue
FROM tasks_queue
WHERE status = $3
AND queue IN (${placeholders})
AND max_attempts > attempt
AND (start_after IS NULL or start_after <= $2)
ORDER BY priority DESC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT 1)
UPDATE tasks_queue
SET status = $1,
started = $2,
last_heartbeat = null,
result = null,
finished = null,
attempt = attempt + 1
FROM selected
WHERE tasks_queue.id = selected.id
RETURNING
tasks_queue.id,
tasks_queue.started,
tasks_queue.parent_id,
tasks_queue.payload,
tasks_queue.queue,
tasks_queue.repeat_type,
tasks_queue.timeout,
tasks_queue.attempt,
tasks_queue.max_attempts
`;
const res = await cl.query(query, params);
return Collection.from(res.rows).headOption.map((r) => {
return {
id: r["id"],
started: new Date(r["started"]),
parentId: option(r["parent_id"]).map(Number).orUndefined,
payload: option(r["payload"]).orUndefined,
queue: r["queue"],
repeatType: option(r["repeat_type"]).orUndefined,
timeout: Number(r["timeout"]),
currentAttempt: r["attempt"],
maxAttempts: r["max_attempts"],
};
});
});
}
async findTaskState(taskId) {
return await this.withClient(async (cl) => {
const res = await cl.query(`select id, parent_id, status, payload, result, error
from tasks_queue
where id = $1`, [taskId]);
return Collection.from(res.rows).headOption.map((r) => ({
id: Number(r["id"]),
parentId: option(r["parent_id"]).map(Number).orUndefined,
status: TaskStatus[r["status"]],
payload: option(r["payload"]).orUndefined,
result: option(r["result"]),
error: option(r["error"]).map(String).orUndefined,
}));
});
}
async ping(taskId, expectedStarted, now = new Date()) {
return await this.withClient(async (cl) => {
const state = await cl.query(`select status, started, last_heartbeat, timeout
from tasks_queue
where id = $1`, [taskId]);
const row = Collection.from(state.rows).headOption;
if (row.isEmpty) {
return false;
}
const current = row.get;
if (current["status"] !== TaskStatus.in_progress ||
new Date(current["started"]).getTime() !== expectedStarted.getTime()) {
return false;
}
const lastHeartbeat = option(current["last_heartbeat"]).map((value) => new Date(value));
const cutoff = new Date(now.getTime() - taskHeartbeatThrottleMs(Number(current["timeout"])));
if (lastHeartbeat
.map((value) => value.getTime() >= cutoff.getTime())
.getOrElseValue(false)) {
return true;
}
const res = await cl.query(`update tasks_queue
set last_heartbeat = $1
where id = $2
and status = $3
and started = $4`, [now, taskId, TaskStatus.in_progress, expectedStarted]);
return option(res.rowCount).getOrElseValue(0) === 1;
});
}
async failIfStalled(taskId, expectedStarted, now = new Date()) {
return await this.withClient(async (cl) => {
await cl.query("BEGIN");
try {
const taskRes = await cl.query(`select id,
parent_id,
status,
started,
last_heartbeat,
timeout,
attempt,
max_attempts,
backoff,
backoff_type
from tasks_queue
where id = $1
for update`, [taskId]);
const taskRow = Collection.from(taskRes.rows).headOption;
if (taskRow.isEmpty) {
await cl.query("COMMIT");
return none;
}
const row = taskRow.get;
const currentStatus = TaskStatus[row["status"]];
if (currentStatus !== TaskStatus.in_progress) {
await cl.query("COMMIT");
return some(currentStatus);
}
if (new Date(row["started"]).getTime() !== expectedStarted.getTime()) {
await cl.query("COMMIT");
return none;
}
const started = option(row["started"]).map((v) => new Date(v));
const lastHeartbeat = option(row["last_heartbeat"]).map((v) => new Date(v));
const timeout = option(row["timeout"]).map(Number);
const lastActivity = lastHeartbeat
.getOrElseValue(started.get)
.getTime();
const isStalled = timeout
.map((ms) => lastActivity + ms < now.getTime())
.getOrElseValue(false);
if (!isStalled) {
await cl.query("COMMIT");
return none;
}
const status = Number(row["attempt"]) < Number(row["max_attempts"])
? TaskStatus.pending
: TaskStatus.error;
const backoff = Number(row["backoff"]);
const backoffType = String(row["backoff_type"]);
let retryDelay = backoff * Math.pow(2, Number(row["attempt"]) - 1);
if (backoffType === "constant") {
retryDelay = backoff;
}
else if (backoffType === "linear") {
retryDelay = backoff * Number(row["attempt"]);
}
await cl.query(`update tasks_queue
set finished = $1,
error = $2,
status = $3,
start_after = $4
where id = $5
and status = $6`, [
now,
"Timeout",
status,
status === TaskStatus.pending
? new Date(now.getTime() + retryDelay)
: null,
taskId,
TaskStatus.in_progress,
]);
if (status === TaskStatus.error) {
await cl.query(`update tasks_queue
set status = $1,
start_after = $2,
finished = null,
error = null
where id = $3
and status = $4`, [
TaskStatus.pending,
now,
option(row["parent_id"]).orNull,
TaskStatus.blocked,
]);
}
await cl.query("COMMIT");
return some(status);
}
catch (e) {
await cl.query("ROLLBACK");
throw e;
}
});
}
async peekNextStartAfter(queueNames, now = new Date()) {
if (queueNames.isEmpty) {
return none;
}
const placeholders = queueNames.zipWithIndex
.map(([_, idx]) => `$${idx + 4}`)
.mkString(",");
return this.withClient(async (cl) => {
const params = Collection.of(TaskStatus.pending, TaskStatus.error, now).appendedAll(queueNames.toArray).toArray;
const res = await cl.query(`SELECT MIN(start_after) AS min_start
FROM tasks_queue
WHERE status IN ($1, $2)
AND queue IN (${placeholders})
AND start_after > $3`, params);
return Collection.from(res.rows)
.headOption.flatMap((r) => option(r["min_start"]))
.map((ts) => new Date(ts));
});
}
async finish(taskId, nextPayload, result, expectedStarted, now = new Date()) {
return await this.withClient(async (cl) => {
const res = await cl.query(`update tasks_queue
set status=$1,
finished=$2,
error=null,
payload=$5,
result=$6
where id = $3
and status = $4
and started = $7`, [
TaskStatus.finished,
now,
taskId,
TaskStatus.in_progress,
option(nextPayload).orNull,
option(result).orNull,
option(expectedStarted).orNull,
]);
return option(res.rowCount).getOrElseValue(0) === 1;
});
}
async wakeParentOnChildTerminal(childTaskId, now = new Date()) {
return await this.withClient(async (cl) => {
const res = await cl.query(`update tasks_queue p
set status = $1,
start_after = $2,
finished = null,
error = null
where p.id = (
select parent_id
from tasks_queue
where id = $3
)
and p.status = $4
returning p.id, p.queue`, [TaskStatus.pending, now, childTaskId, TaskStatus.blocked]);
return Collection.from(res.rows).headOption.map((r) => ({
id: Number(r["id"]),
queue: String(r["queue"]),
}));
});
}
async rescheduleIfPeriodic(taskId, nextPayload, result, expectedStarted, now = new Date()) {
return await this.withClient(async (cl) => {
await cl.query("BEGIN");
try {
const periodicTask = await this.findPeriodicForReschedule(cl, taskId, option(expectedStarted).orNull);
const updated = await periodicTask.mapPromise(async (task) => {
const nextStartAfter = PeriodicScheduleUtils.calculateNextStartAfter(task, now);
await cl.query(`UPDATE tasks_queue
SET status = $1,
start_after = $2,
finished = $3,
error = NULL,
attempt = 0,
payload = $6,
result = $7
WHERE id = $4
AND status = $5
AND started = $8`, [
TaskStatus.pending,
nextStartAfter,
now,
taskId,
TaskStatus.in_progress,
option(nextPayload).orNull,
option(result).orNull,
option(expectedStarted).orNull,
]);
return true;
});
await cl.query("COMMIT");
return updated.getOrElseValue(false);
}
catch (e) {
await cl.query("ROLLBACK");
throw e;
}
});
}
async findPeriodicForReschedule(cl, taskId, expectedStarted) {
const res = await cl.query(`SELECT repeat_type,
repeat_interval,
cron_expression,
start_after,
initial_start,
missed_runs_strategy
FROM tasks_queue
WHERE id = $1
AND status = $2
AND started = $6
AND missed_runs_strategy IS NOT NULL
AND repeat_type IN ($3, $4, $5)
AND (
(repeat_type IN ($3, $4) AND repeat_interval IS NOT NULL)
OR
(repeat_type = $5 AND cron_expression IS NOT NULL)
)
FOR UPDATE`, [
taskId,
TaskStatus.in_progress,
TaskPeriodType.fixed_rate,
TaskPeriodType.fixed_delay,
TaskPeriodType.cron,
expectedStarted,
]);
return Collection.from(res.rows).headOption.map((r) => ({
repeatType: TaskPeriodType[r["repeat_type"]],
repeatInterval: option(r["repeat_interval"]).map(Number),
cronExpression: option(r["cron_expression"]).map(String),
startAfter: new Date(r["start_after"]),
initialStart: new Date(r["initial_start"]),
missedRunsStrategy: MissedRunStrategy[r["missed_runs_strategy"]],
}));
}
async fail(taskId, error, nextPayload, result, expectedStarted, now = new Date()) {
return await this.withClient(async (cl) => {
const res = await cl.query(`
UPDATE tasks_queue
SET finished = $1,
error = $2,
status = CASE
WHEN attempt < max_attempts THEN $3 -- pending
ELSE $4 -- error
END,
start_after = CASE
WHEN attempt < max_attempts THEN
$1::timestamp + (
CASE backoff_type
WHEN 'constant' THEN backoff
WHEN 'linear' THEN (backoff * attempt)
WHEN 'exponential'
THEN (backoff * POWER(2, (attempt - 1)))
ELSE backoff
END
) * interval '1 millisecond'
ELSE NULL
END,
payload = $7,
result = CASE
WHEN attempt < max_attempts THEN NULL
ELSE $8::jsonb
END
WHERE id = $5
AND status = $6
AND started = $9
returning status
`, [
now,
error,
TaskStatus.pending,
TaskStatus.error,
taskId,
TaskStatus.in_progress,
option(nextPayload).orNull,
option(result).orNull,
option(expectedStarted).orNull,
]);
return Collection.from(res.rows)
.headOption.map((r) => TaskStatus[r.status])
.map((status) => status);
});
}
async failStalled(now = new Date()) {
return await this.withClient(async (cl) => {
await cl.query("BEGIN");
try {
const res = await cl.query(`with updated as (
update tasks_queue child
set finished = $1,
error = $2,
status = case
when attempt < max_attempts then $3
else $4
end,
start_after = case
when attempt < max_attempts then
$1::timestamp + (
case backoff_type
when 'constant' then backoff
when 'linear' then (backoff * attempt)
when 'exponential' then (backoff * power(2, (attempt - 1)))
else backoff
end
) * interval '1 millisecond'
else null
end
where child.status = $5
and child.timeout is not null
and greatest(
child.started,
coalesce(child.last_heartbeat, child.started)
) + child.timeout * interval '1 millisecond' < $6
returning child.id, child.parent_id, child.status
),
woken as (
update tasks_queue parent
set status = $3,
start_after = $1,
finished = null,
error = null
where parent.id in (
select parent_id
from updated
where parent_id is not null
and status = $4
)
and parent.status = $7
returning parent.id
)
select id
from updated`, [
now,
"Timeout",
TaskStatus.pending,
TaskStatus.error,
TaskStatus.in_progress,
now,
TaskStatus.blocked,
]);
await cl.query("COMMIT");
return Collection.from(res.rows).map((r) => r.id);
}
catch (e) {
await cl.query("ROLLBACK");
throw e;
}
});
}
async resetFailed() {
await this.withClient(async (cl) => {
await cl.query(`update tasks_queue
set status=$1,
finished=null
where status = $2
and timeout is not null
and max_attempts > COALESCE(attempt, 0)`, [TaskStatus.pending, TaskStatus.error]);
});
}
async clearFinished(timeout = TimeUtils.day, now = new Date()) {
const expired = new Date(now.getTime() - timeout);
await this.withClient(async (cl) => {
await cl.query(`with recursive ancestors as (
select child.id as task_id, parent.id, parent.parent_id, parent.status
from tasks_queue child
join tasks_queue parent on parent.id = child.parent_id
where child.status = $1
and child.finished < $2
union all
select ancestors.task_id, parent.id, parent.parent_id, parent.status
from ancestors
join tasks_queue parent on parent.id = ancestors.parent_id
)
delete
from tasks_queue task
where task.status = $1
and task.finished < $2
and not exists (
select 1
from ancestors
where ancestors.task_id = task.id
and ancestors.status <> $1
)`, [TaskStatus.finished, expired]);
});
}
async statusCount(queue, status) {
return await this.withClient(async (cl) => {
const res = await cl.query(`select count(id) as cnt
from tasks_queue
where queue = $1
and status = $2`, [queue, status]);
return Collection.from(res.rows)
.headOption.map((r) => r["cnt"])
.getOrElseValue(0);
});
}
}
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Object, Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "schedule", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number, Object, Object, Date,
Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "blockParentAndScheduleChild", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Object, String, Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "schedulePeriodic", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [HashSet,
Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "nextPending", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "findTaskState", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number, Date,
Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "ping", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number, Date,
Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "failIfStalled", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [HashSet,
Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "peekNextStartAfter", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number, Object, Object, Date,
Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "finish", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number, Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "wakeParentOnChildTerminal", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number, Object, Object, Date,
Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "rescheduleIfPeriodic", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number, String, Object, Object, Date,
Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "fail", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "failStalled", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", []),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "resetFailed", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Number, Date]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "clearFinished", null);
__decorate([
Metric(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [String, String]),
__metadata("design:returntype", Promise)
], TasksQueueDao.prototype, "statusCount", null);