@penkov/tasks_queue
Version:
A lightweight PostgreSQL-backed task queue system with scheduling, retries, backoff strategies, and priority handling. Designed for efficiency and observability in modern Node.js applications.
215 lines (214 loc) • 10.1 kB
JavaScript
import { Collection, mutable, none, option, some, Try } from "scats";
import log4js from "log4js";
import { MetricsService } from "application-metrics";
import { TasksPipeline } from "./tasks-pipeline.js";
import { TaskFailed, taskHeartbeatThrottleMs, TaskStatus, TaskTimedOutError, } from "./tasks-model.js";
import { TimeUtils } from "./time-utils.js";
import { SystemClock } from "./clock.js";
import { poolMetricName, queueMetricName } from "./metrics-utils.js";
const logger = log4js.getLogger("TasksQueueWorker");
class RuntimeTaskContext {
tasksQueueDao;
taskId;
startedAt;
currentAttempt;
maxAttempts;
timeout;
clock;
childTask = none;
_nextPayload = none;
_submittedResult = none;
attemptStartedAt;
lastHeartbeatAt = none;
resolvedChildTask = none;
constructor(tasksQueueDao, taskId, startedAt, currentAttempt, maxAttempts, timeout, clock) {
this.tasksQueueDao = tasksQueueDao;
this.taskId = taskId;
this.startedAt = startedAt;
this.currentAttempt = currentAttempt;
this.maxAttempts = maxAttempts;
this.timeout = timeout;
this.clock = clock;
this.attemptStartedAt = new Date(startedAt);
}
async ensureNotStalled() {
const now = this.clock.now();
if (now.getTime() -
this.lastHeartbeatAt.getOrElseValue(this.attemptStartedAt.getTime()) <=
this.timeout) {
return;
}
const finalStatus = (await this.tasksQueueDao.failIfStalled(this.taskId, this.attemptStartedAt, now)).getOrElseValue(TaskStatus.error);
throw new TaskTimedOutError(this.taskId, finalStatus);
}
async ping() {
await this.ensureNotStalled();
const now = this.clock.now().getTime();
if (this.lastHeartbeatAt
.map((lastHeartbeatAt) => now - lastHeartbeatAt < taskHeartbeatThrottleMs(this.timeout))
.getOrElseValue(false)) {
return;
}
const persisted = await this.tasksQueueDao.ping(this.taskId, this.attemptStartedAt, this.clock.now());
if (!persisted) {
throw new Error(`Task ${this.taskId} execution is no longer active`);
}
this.lastHeartbeatAt = some(now);
}
spawnChild(task) {
if (this.childTask.nonEmpty) {
throw new Error(`Task ${this.taskId} attempted to spawn more than one child task`);
}
this.childTask = some(task);
}
setPayload(payload) {
this._nextPayload = some(payload);
}
submitResult(result) {
this._submittedResult = some(result);
}
async findTask(taskId) {
return this.tasksQueueDao.findTaskState(taskId);
}
get spawnedChild() {
return this.childTask;
}
payloadToPersist(currentPayload) {
return this._nextPayload.getOrElseValue(option(currentPayload).getOrElseValue({}));
}
get nextPayload() {
return this._nextPayload;
}
get submittedResult() {
return this._submittedResult;
}
}
export class TasksQueueWorker {
tasksQueueDao;
clock;
poolName;
workers = new mutable.HashMap();
pipeline;
started = false;
queueNotifier;
constructor(tasksQueueDao, concurrency = 4, loopInterval = TimeUtils.minute, clock = new SystemClock(), queueNotifier, poolName = "default") {
this.tasksQueueDao = tasksQueueDao;
this.clock = clock;
this.poolName = poolName;
this.queueNotifier = option(queueNotifier).getOrElse(() => (queueName) => this.tasksScheduled(queueName));
this.pipeline = new TasksPipeline(concurrency, () => this.pollNextTask(), () => this.tasksQueueDao.peekNextStartAfter(this.workers.keySet, this.clock.now()), (t) => this.processNextTask(t), loopInterval, this.clock, this.poolName);
}
start() {
this.started = true;
this.pipeline.start();
}
async stop() {
this.started = false;
return await this.pipeline.stop();
}
async runOnce() {
const task = await this.pollNextTask();
await task.mapPromise(async (scheduledTask) => {
await this.processNextTask(scheduledTask);
return scheduledTask;
});
}
registerWorker(queueName, worker) {
if (this.workers.containsKey(queueName)) {
logger.warn(`Replacing existing worker for queue: ${queueName}`);
}
this.workers.put(queueName, worker);
this.updatePoolQueuesLabel();
}
tasksScheduled(queueName) {
if (this.started && this.workers.containsKey(queueName)) {
this.pipeline.triggerLoop();
}
}
async wakeBlockedParent(childTaskId) {
const parent = await this.tasksQueueDao.wakeParentOnChildTerminal(childTaskId, this.clock.now());
parent.foreach((p) => this.queueNotifier(p.queue));
}
async pollNextTask() {
this.updatePoolQueuesLabel();
this.workers.keySet.foreach((queueName) => {
MetricsService.counter(queueMetricName(queueName, "poll_total")).inc();
});
const task = await this.tasksQueueDao.nextPending(this.workers.keySet, this.clock.now());
task.foreach((scheduledTask) => {
MetricsService.counter(queueMetricName(scheduledTask.queue, "fetched_total")).inc();
});
return task;
}
updatePoolQueuesLabel() {
MetricsService.label(poolMetricName(this.poolName, "queues"), Collection.from(this.workers.keySet.toArray)
.sort((a, b) => a.localeCompare(b))
.mkString(","));
}
async persistSuccessfulOutcome(task, context) {
await context.ensureNotStalled();
return await context.spawnedChild.match({
some: async (spawnedChild) => {
option(task.repeatType).foreach(() => {
throw new Error(`Periodic task ${task.id} cannot spawn child tasks`);
});
const childTaskId = await this.tasksQueueDao.blockParentAndScheduleChild(task.id, spawnedChild, context.payloadToPersist(task.payload), task.started, this.clock.now());
childTaskId.foreach(() => this.queueNotifier(spawnedChild.queue));
return childTaskId.isDefined;
},
none: async () => {
return await option(task.repeatType).match({
some: async () => {
return await this.tasksQueueDao.rescheduleIfPeriodic(task.id, context.payloadToPersist(task.payload), context.submittedResult.orUndefined, task.started, this.clock.now());
},
none: async () => {
const finished = await this.tasksQueueDao.finish(task.id, context.payloadToPersist(task.payload), context.submittedResult.orUndefined, task.started, this.clock.now());
if (finished) {
await this.wakeBlockedParent(task.id);
}
return finished;
},
});
},
});
}
async processNextTask(task) {
MetricsService.counter("tasks_queue_started").inc();
await this.workers.get(task.queue).match({
some: async (worker) => {
MetricsService.counter(queueMetricName(task.queue, "task_started_total")).inc();
const context = new RuntimeTaskContext(this.tasksQueueDao, task.id, task.started, task.currentAttempt, task.maxAttempts, task.timeout, this.clock);
try {
Try(() => worker.starting(task.id, task.payload)).tapFailure((e) => logger.warn(`Failed to invoke 'starting' callback for task (id=${task.id}) in queue=${task.queue}`, e));
await worker.process(task.payload, context);
const persisted = await this.persistSuccessfulOutcome(task, context);
if (!persisted) {
logger.info(`Skipping completion for task (id=${task.id}) in queue=${task.queue}: execution ownership was lost`);
return;
}
MetricsService.counter("tasks_queue_processed").inc();
MetricsService.counter(queueMetricName(task.queue, "task_finished_total")).inc();
(await Try.promise(() => worker.completed(task.id, task.payload))).tapFailure((e) => logger.warn(`Failed to invoke 'completed' callback for task (id=${task.id}) in queue=${task.queue}`, e));
}
catch (e) {
const finalStatus = e instanceof TaskTimedOutError
? e.finalStatus
: (await this.tasksQueueDao.fail(task.id, e["message"] || e, e instanceof TaskFailed ? e.payload : task.payload, context.submittedResult.orUndefined, task.started, this.clock.now())).getOrElseValue(TaskStatus.error);
if (finalStatus === TaskStatus.error &&
!(e instanceof TaskTimedOutError)) {
await this.wakeBlockedParent(task.id);
}
MetricsService.counter(queueMetricName(task.queue, `task_failed_${finalStatus}_total`)).inc();
(await Try.promise(() => worker.failed(task.id, task.payload, finalStatus, e))).tapFailure((e) => logger.warn(`Failed to invoke 'failed' callback for task (id=${task.id}) in queue=${task.queue}`, e));
MetricsService.counter("tasks_queue_failed").inc();
logger.warn(`Failed to process task (id=${task.id}) in queue=${task.queue}`, e);
}
},
none: async () => {
MetricsService.counter("tasks_queue_skipped_no_worker").inc();
MetricsService.counter(queueMetricName(task.queue, "task_skipped_no_worker_total")).inc();
logger.info(`Failed to process task (id=${task.id}) in queue=${task.queue}: no suitable worker found`);
},
});
}
}