@penkov/tasks_queue
Version:
A lightweight PostgreSQL-backed task queue system with scheduling, retries, backoff strategies, and priority handling. Designed for efficiency and observability in modern Node.js applications.
249 lines (244 loc) • 10.8 kB
JavaScript
import { Collection, mutable, option } from "scats";
import { TaskPeriodType, TaskStatus } from "./tasks-model.js";
import { QueueStat, TaskDto, TaskDateSortField, TaskSortDirection, TasksCount, TasksResult, } from "./manage.model.js";
import { CronExpressionUtils } from "./cron-expression-utils.js";
export class ManageTasksQueueService {
pool;
constructor(pool) {
this.pool = pool;
}
mapTaskRow(row) {
return new TaskDto(row["id"], option(row["parent_id"]).map(Number), row["queue"], row["created"], row["initial_start"], option(row["started"]), option(row["last_heartbeat"]), option(row["finished"]), row["status"], row["missed_runs_strategy"], row["priority"], option(row["error"]), row["backoff"], row["backoff_type"], row["timeout"], option(row["name"]), option(row["start_after"]), option(row["repeat_interval"]), option(row["cron_expression"]), option(row["repeat_type"]), row["max_attempts"], row["attempt"], row["payload"], row["result"]);
}
async findById(taskId) {
const res = await this.pool.query(`select *
from tasks_queue
where id = $1`, [taskId]);
return Collection.from(res.rows).headOption.map((row) => this.mapTaskRow(row));
}
async findChildrenTree(rootTaskId) {
const res = await this.pool.query(`with recursive task_children as (
select *
from tasks_queue
where parent_id = $1
union all
select child.*
from tasks_queue child
inner join task_children parent on parent.id = child.parent_id
)
select *
from task_children
order by id asc`, [rootTaskId]);
return Collection.from(res.rows).map((row) => this.mapTaskRow(row));
}
async findByParameters(params) {
const parts = new mutable.ArrayBuffer();
const queryParams = new mutable.ArrayBuffer();
option(params.status).foreach((status) => {
queryParams.append(status);
parts.append(`status = $${queryParams.size}`);
});
option(params.queue).foreach((queue) => {
queryParams.append(queue);
parts.append(`queue = $${queryParams.size}`);
});
option(params.isPeriodic).foreach((isPeriodic) => {
parts.append(isPeriodic ? "name is not null" : "name is null");
});
const where = parts.nonEmpty ? `where ${parts.mkString(" and ")}` : "";
const sortBy = option(params.sortBy).getOrElseValue(TaskDateSortField.created);
const sortDirection = option(params.sortDirection).getOrElseValue(TaskSortDirection.desc);
if (!Object.values(TaskDateSortField).includes(sortBy)) {
throw new Error(`Unsupported task date sort field: ${sortBy}`);
}
if (sortDirection !== TaskSortDirection.asc &&
sortDirection !== TaskSortDirection.desc) {
throw new Error(`Unsupported task sort direction: ${sortDirection}`);
}
const paginationParams = queryParams.appendedAll([
params.limit,
params.offset,
]);
const res = await this.pool.query(`select *
from tasks_queue ${where}
order by ${sortBy} ${sortDirection} nulls last
limit $${queryParams.size + 1} offset $${queryParams.size + 2}`, paginationParams.toArray);
const total = await this.pool.query(`select count(*) as total
from tasks_queue ${where}`, queryParams.toArray);
const items = Collection.from(res.rows).map((row) => this.mapTaskRow(row));
return new TasksResult(items, total.rows[0]["total"]);
}
async failedCount() {
const res = await this.pool.query(`select count(*) as total
from tasks_queue
where status = '${TaskStatus.error}'`);
return res.rows[0]["total"];
}
clearFailed() {
return this.pool.query(`delete
from tasks_queue
where status = '${TaskStatus.error}'`);
}
async deleteTask(taskId) {
const res = await this.pool.query(`
with recursive ancestors as (
select parent.id, parent.parent_id, parent.status
from tasks_queue child
join tasks_queue parent on parent.id = child.parent_id
where child.id = $1
union all
select parent.id, parent.parent_id, parent.status
from ancestors
join tasks_queue parent on parent.id = ancestors.parent_id
)
delete
from tasks_queue
where id = $1
and status in ($2, $3, $4)
and not exists (
select 1
from ancestors
where status <> $5
)
`, [
taskId,
TaskStatus.pending,
TaskStatus.error,
TaskStatus.finished,
TaskStatus.finished,
]);
return option(res.rowCount).getOrElseValue(0) > 0;
}
async updatePendingTask(taskId, details) {
if (details.timeout <= 0) {
throw new Error("Task timeout must be greater than 0");
}
if (details.retries <= 0) {
throw new Error("Task retries must be greater than 0");
}
if (details.backoff < 0) {
throw new Error("Task backoff must be greater than or equal to 0");
}
const res = await this.pool.query(`
update tasks_queue
set start_after = $2,
priority = $3,
timeout = $4,
payload = $5,
max_attempts = $6,
backoff = $7,
backoff_type = $8
where id = $1
and status = '${TaskStatus.pending}'
`, [
taskId,
details.startAfter,
details.priority,
details.timeout,
details.payload,
details.retries,
details.backoff,
details.backoffType,
]);
return option(res.rowCount).getOrElseValue(0) > 0;
}
async updatePendingPeriodicSchedule(taskId, details) {
switch (details.repeatType) {
case TaskPeriodType.fixed_rate:
case TaskPeriodType.fixed_delay:
if (details.period <= 0) {
throw new Error("Periodic task period must be greater than 0");
}
break;
case TaskPeriodType.cron:
CronExpressionUtils.validate(details.cronExpression);
break;
}
const repeatInterval = details.repeatType === TaskPeriodType.cron ? null : details.period;
const cronExpression = details.repeatType === TaskPeriodType.cron
? details.cronExpression
: null;
const res = await this.pool.query(`
update tasks_queue
set start_after = $2,
initial_start = $3,
repeat_type = $4,
repeat_interval = $5,
cron_expression = $6,
missed_runs_strategy = $7
where id = $1
and status = '${TaskStatus.pending}'
and repeat_type is not null
`, [
taskId,
details.startAfter,
details.initialStart,
details.repeatType,
repeatInterval,
cronExpression,
details.missedRunStrategy,
]);
return option(res.rowCount).getOrElseValue(0) > 0;
}
async waitTimeByQueue() {
const res = await this.pool.query(`
SELECT queue,
percentile_disc(0.50) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (started - created))) AS p50,
percentile_disc(0.75) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (started - created))) AS p75,
percentile_disc(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (started - created))) AS p95,
percentile_disc(0.99) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (started - created))) AS p99,
percentile_disc(0.999) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (started - created))) AS p999
FROM tasks_queue
WHERE started IS NOT NULL
and attempt = 1
GROUP BY queue
ORDER BY queue
`);
return Collection.from(res.rows).map((row) => new QueueStat(row["queue"], Number(row["p50"]), Number(row["p75"]), Number(row["p95"]), Number(row["p99"]), Number(row["p999"])));
}
async restartFailedTask(taskId) {
await this.pool.query(`
update tasks_queue
set status='${TaskStatus.pending}',
attempt=0
where id = $1
and status = '${TaskStatus.error}'
`, [taskId]);
}
async restartAllFailedInQueue(queue) {
await this.pool.query(`
update tasks_queue
set status='${TaskStatus.pending}',
attempt=0
where queue = $1
and status = '${TaskStatus.error}'
`, [queue]);
}
async tasksCount() {
const res = await this.pool.query(`
SELECT queue,
status,
COUNT(*) AS task_count
FROM tasks_queue
GROUP BY queue, status
ORDER BY queue, status
`);
return Collection.from(res.rows).map((row) => new TasksCount(row["queue"], TaskStatus[row["status"]], Number(row["task_count"])));
}
async workTimeByQueue() {
const res = await this.pool.query(`
SELECT queue,
percentile_disc(0.50) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished - started))) AS p50,
percentile_disc(0.75) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished - started))) AS p75,
percentile_disc(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished - started))) AS p95,
percentile_disc(0.99) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished - started))) AS p99,
percentile_disc(0.999) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM (finished - started))) AS p999
FROM tasks_queue
WHERE started IS NOT NULL
AND finished IS NOT NULL
GROUP BY queue
ORDER BY queue
`);
return Collection.from(res.rows).map((row) => new QueueStat(row["queue"], Number(row["p50"]), Number(row["p75"]), Number(row["p95"]), Number(row["p99"]), Number(row["p999"])));
}
}