@penkov/tasks_queue
Version:
A lightweight PostgreSQL-backed task queue system with scheduling, retries, backoff strategies, and priority handling. Designed for efficiency and observability in modern Node.js applications.
1,133 lines (1,117 loc) • 123 kB
JavaScript
'use strict';
var scats = require('scats');
var cronParser = require('cron-parser');
var log4js = require('log4js');
var applicationMetrics = require('application-metrics');
var tslib = require('tslib');
var common = require('@nestjs/common');
var core = require('@nestjs/core');
var swagger = require('@nestjs/swagger');
require('reflect-metadata');
exports.TaskStatus = void 0;
(function (TaskStatus) {
TaskStatus["pending"] = "pending";
TaskStatus["in_progress"] = "in_progress";
TaskStatus["blocked"] = "blocked";
TaskStatus["finished"] = "finished";
TaskStatus["error"] = "error";
})(exports.TaskStatus || (exports.TaskStatus = {}));
exports.BackoffType = void 0;
(function (BackoffType) {
BackoffType["constant"] = "constant";
BackoffType["linear"] = "linear";
BackoffType["exponential"] = "exponential";
})(exports.BackoffType || (exports.BackoffType = {}));
exports.TaskPeriodType = void 0;
(function (TaskPeriodType) {
TaskPeriodType["fixed_rate"] = "fixed_rate";
TaskPeriodType["fixed_delay"] = "fixed_delay";
TaskPeriodType["cron"] = "cron";
})(exports.TaskPeriodType || (exports.TaskPeriodType = {}));
exports.MissedRunStrategy = void 0;
(function (MissedRunStrategy) {
MissedRunStrategy["catch_up"] = "catch_up";
MissedRunStrategy["skip_missed"] = "skip_missed";
})(exports.MissedRunStrategy || (exports.MissedRunStrategy = {}));
const TASK_HEARTBEAT_THROTTLE_MS = 60_000;
function taskHeartbeatThrottleMs(timeout) {
if (!Number.isFinite(timeout) || timeout <= 0) {
return TASK_HEARTBEAT_THROTTLE_MS;
}
return Math.min(TASK_HEARTBEAT_THROTTLE_MS, Math.max(1, timeout / 2));
}
class TaskFailed extends Error {
payload;
constructor(message, payload) {
super(message);
this.payload = payload;
}
}
class TaskTimedOutError extends Error {
taskId;
finalStatus;
constructor(taskId, finalStatus) {
super(`Task ${taskId} timed out before liveness was refreshed`);
this.taskId = taskId;
this.finalStatus = finalStatus;
}
}
exports.CronExpression = void 0;
(function (CronExpression) {
CronExpression["EVERY_SECOND"] = "* * * * * *";
CronExpression["EVERY_5_SECONDS"] = "*/5 * * * * *";
CronExpression["EVERY_10_SECONDS"] = "*/10 * * * * *";
CronExpression["EVERY_30_SECONDS"] = "*/30 * * * * *";
CronExpression["EVERY_MINUTE"] = "*/1 * * * *";
CronExpression["EVERY_5_MINUTES"] = "0 */5 * * * *";
CronExpression["EVERY_10_MINUTES"] = "0 */10 * * * *";
CronExpression["EVERY_30_MINUTES"] = "0 */30 * * * *";
CronExpression["EVERY_HOUR"] = "0 0-23/1 * * *";
CronExpression["EVERY_2_HOURS"] = "0 0-23/2 * * *";
CronExpression["EVERY_3_HOURS"] = "0 0-23/3 * * *";
CronExpression["EVERY_4_HOURS"] = "0 0-23/4 * * *";
CronExpression["EVERY_5_HOURS"] = "0 0-23/5 * * *";
CronExpression["EVERY_6_HOURS"] = "0 0-23/6 * * *";
CronExpression["EVERY_7_HOURS"] = "0 0-23/7 * * *";
CronExpression["EVERY_8_HOURS"] = "0 0-23/8 * * *";
CronExpression["EVERY_9_HOURS"] = "0 0-23/9 * * *";
CronExpression["EVERY_10_HOURS"] = "0 0-23/10 * * *";
CronExpression["EVERY_11_HOURS"] = "0 0-23/11 * * *";
CronExpression["EVERY_12_HOURS"] = "0 0-23/12 * * *";
CronExpression["EVERY_DAY_AT_1AM"] = "0 01 * * *";
CronExpression["EVERY_DAY_AT_2AM"] = "0 02 * * *";
CronExpression["EVERY_DAY_AT_3AM"] = "0 03 * * *";
CronExpression["EVERY_DAY_AT_4AM"] = "0 04 * * *";
CronExpression["EVERY_DAY_AT_5AM"] = "0 05 * * *";
CronExpression["EVERY_DAY_AT_6AM"] = "0 06 * * *";
CronExpression["EVERY_DAY_AT_7AM"] = "0 07 * * *";
CronExpression["EVERY_DAY_AT_8AM"] = "0 08 * * *";
CronExpression["EVERY_DAY_AT_9AM"] = "0 09 * * *";
CronExpression["EVERY_DAY_AT_10AM"] = "0 10 * * *";
CronExpression["EVERY_DAY_AT_11AM"] = "0 11 * * *";
CronExpression["EVERY_DAY_AT_NOON"] = "0 12 * * *";
CronExpression["EVERY_DAY_AT_1PM"] = "0 13 * * *";
CronExpression["EVERY_DAY_AT_2PM"] = "0 14 * * *";
CronExpression["EVERY_DAY_AT_3PM"] = "0 15 * * *";
CronExpression["EVERY_DAY_AT_4PM"] = "0 16 * * *";
CronExpression["EVERY_DAY_AT_5PM"] = "0 17 * * *";
CronExpression["EVERY_DAY_AT_6PM"] = "0 18 * * *";
CronExpression["EVERY_DAY_AT_7PM"] = "0 19 * * *";
CronExpression["EVERY_DAY_AT_8PM"] = "0 20 * * *";
CronExpression["EVERY_DAY_AT_9PM"] = "0 21 * * *";
CronExpression["EVERY_DAY_AT_10PM"] = "0 22 * * *";
CronExpression["EVERY_DAY_AT_11PM"] = "0 23 * * *";
CronExpression["EVERY_DAY_AT_MIDNIGHT"] = "0 0 * * *";
CronExpression["EVERY_WEEK"] = "0 0 * * 0";
CronExpression["EVERY_WEEKDAY"] = "0 0 * * 1-5";
CronExpression["EVERY_WEEKEND"] = "0 0 * * 6,0";
CronExpression["EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT"] = "0 0 1 * *";
CronExpression["EVERY_1ST_DAY_OF_MONTH_AT_NOON"] = "0 12 1 * *";
CronExpression["EVERY_2ND_HOUR"] = "0 */2 * * *";
CronExpression["EVERY_2ND_HOUR_FROM_1AM_THROUGH_11PM"] = "0 1-23/2 * * *";
CronExpression["EVERY_2ND_MONTH"] = "0 0 1 */2 *";
CronExpression["EVERY_QUARTER"] = "0 0 1 */3 *";
CronExpression["EVERY_6_MONTHS"] = "0 0 1 */6 *";
CronExpression["EVERY_YEAR"] = "0 0 1 1 *";
CronExpression["EVERY_30_MINUTES_BETWEEN_9AM_AND_5PM"] = "0 */30 9-17 * * *";
CronExpression["EVERY_30_MINUTES_BETWEEN_9AM_AND_6PM"] = "0 */30 9-18 * * *";
CronExpression["EVERY_30_MINUTES_BETWEEN_10AM_AND_7PM"] = "0 */30 10-19 * * *";
CronExpression["MONDAY_TO_FRIDAY_AT_1AM"] = "0 0 01 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_2AM"] = "0 0 02 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_3AM"] = "0 0 03 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_4AM"] = "0 0 04 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_5AM"] = "0 0 05 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_6AM"] = "0 0 06 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_7AM"] = "0 0 07 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_8AM"] = "0 0 08 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_9AM"] = "0 0 09 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_09_30AM"] = "0 30 09 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_10AM"] = "0 0 10 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_11AM"] = "0 0 11 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_11_30AM"] = "0 30 11 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_12PM"] = "0 0 12 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_1PM"] = "0 0 13 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_2PM"] = "0 0 14 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_3PM"] = "0 0 15 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_4PM"] = "0 0 16 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_5PM"] = "0 0 17 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_6PM"] = "0 0 18 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_7PM"] = "0 0 19 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_8PM"] = "0 0 20 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_9PM"] = "0 0 21 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_10PM"] = "0 0 22 * * 1-5";
CronExpression["MONDAY_TO_FRIDAY_AT_11PM"] = "0 0 23 * * 1-5";
})(exports.CronExpression || (exports.CronExpression = {}));
class CronExpressionUtils {
static DEFAULT_TIMEZONE = "UTC";
static validate(expression, timezone = CronExpressionUtils.DEFAULT_TIMEZONE) {
this.nextExecutionDate(expression, new Date(), timezone);
}
static nextExecutionDate(expression, baseDate, timezone = CronExpressionUtils.DEFAULT_TIMEZONE) {
const normalizedExpression = expression.trim();
if (normalizedExpression.length === 0) {
throw new Error("Cron expression must not be blank");
}
try {
const parsed = cronParser.CronExpressionParser.parse(normalizedExpression, {
currentDate: baseDate,
tz: scats.option(timezone).filter((s) => s.trim().length > 0).orUndefined,
});
return parsed.next().toDate();
}
catch (e) {
throw new Error(`Invalid cron expression '${normalizedExpression}': ${e.message}`);
}
}
}
class TimeUtils {
static second = 1000;
static minute = TimeUtils.second * 60;
static hour = TimeUtils.minute * 60;
static day = TimeUtils.hour * 24;
}
class SystemClock {
now() {
return new Date();
}
}
const sanitizeMetricPart = (value) => value.replace(/[^a-zA-Z0-9_:]/g, "_");
const poolMetricName = (poolName, suffix) => `tasks_queue_pool_${sanitizeMetricPart(poolName)}_${suffix}`;
const queueMetricName = (queueName, suffix) => `tasks_queue_queue_${sanitizeMetricPart(queueName)}_${suffix}`;
const defaultLoopInterval = TimeUtils.minute;
var PipelineNextOperationType;
(function (PipelineNextOperationType) {
PipelineNextOperationType[PipelineNextOperationType["PollNext"] = 0] = "PollNext";
PipelineNextOperationType[PipelineNextOperationType["Sleep"] = 1] = "Sleep";
})(PipelineNextOperationType || (PipelineNextOperationType = {}));
const PipelineNextOperationFactory = {
pollNext: {
type: PipelineNextOperationType.PollNext,
},
sleep: (delayMs = defaultLoopInterval) => ({
type: PipelineNextOperationType.Sleep,
delayMs,
}),
};
const logger$4 = log4js.getLogger("TasksPipeline");
const noop = () => {
};
class TasksPipeline {
maxConcurrentTasks;
pollNextTask;
peekNextStartAfter;
processTask;
loopInterval;
clock;
poolName;
tasksInProcess = 0;
loopRunning = false;
nextLoopTime = 0;
periodicTaskFetcher = null;
stopRequested = false;
tasksCountListener = noop;
loopTimer;
loopStartedAt = 0;
lastPollStartedAt = 0;
lastPollFinishedAt = 0;
lastSuccessfulPollAt = 0;
lastPollDurationMs = 0;
constructor(maxConcurrentTasks, pollNextTask, peekNextStartAfter, processTask, loopInterval = defaultLoopInterval, clock = new SystemClock(), poolName = "default") {
this.maxConcurrentTasks = maxConcurrentTasks;
this.pollNextTask = pollNextTask;
this.peekNextStartAfter = peekNextStartAfter;
this.processTask = processTask;
this.loopInterval = loopInterval;
this.clock = clock;
this.poolName = poolName;
this.registerMetrics();
this.loopTimer = async () => {
this.periodicTaskFetcher = null;
let sleepInterval = this.loopInterval;
try {
sleepInterval = await this.loop();
}
finally {
if (!this.periodicTaskFetcher) {
this.nextLoopTime = this.nowMs() + sleepInterval;
this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), Math.min(this.loopInterval, sleepInterval));
}
}
};
}
start() {
this.stopRequested = false;
scats.option(this.periodicTaskFetcher).foreach((t) => clearTimeout(t));
this.nextLoopTime = this.nowMs() + 1;
this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), 1);
}
async runOnce() {
await this.loop();
}
getState() {
return {
maxConcurrentTasks: this.maxConcurrentTasks,
tasksInProcess: this.tasksInProcess,
loopRunning: this.loopRunning,
loopStartedAt: this.loopStartedAt,
lastPollStartedAt: this.lastPollStartedAt,
lastPollFinishedAt: this.lastPollFinishedAt,
lastSuccessfulPollAt: this.lastSuccessfulPollAt,
lastPollDurationMs: this.lastPollDurationMs,
nextLoopTime: this.nextLoopTime,
};
}
async stop() {
this.stopRequested = true;
scats.option(this.periodicTaskFetcher).foreach((t) => clearTimeout(t));
if (this.tasksInProcess > 0) {
await new Promise((resolve) => {
this.tasksCountListener = (n) => {
if (n <= 0) {
this.tasksCountListener = noop;
resolve();
}
};
});
}
}
triggerLoop() {
if (!this.loopRunning && !this.stopRequested) {
setTimeout(() => this.loop(), 1);
}
}
async loop() {
if (this.loopRunning || this.stopRequested) {
return this.loopInterval;
}
this.loopRunning = true;
this.loopStartedAt = this.nowMs();
this.lastPollStartedAt = this.loopStartedAt;
applicationMetrics.MetricsService.counter(poolMetricName(this.poolName, "loop_poll_started_total")).inc();
let sleepInterval = this.loopInterval;
let completedSuccessfully = false;
try {
let nextOp = PipelineNextOperationFactory.pollNext;
while (nextOp.type !== PipelineNextOperationType.Sleep) {
if (this.tasksInProcess < this.maxConcurrentTasks &&
!this.stopRequested) {
nextOp = await this.fetchNextTask();
}
else {
nextOp = PipelineNextOperationFactory.sleep(this.loopInterval);
}
}
sleepInterval = Math.min(nextOp.delayMs, this.loopInterval);
completedSuccessfully = true;
}
finally {
const finishedAt = this.nowMs();
this.lastPollFinishedAt = finishedAt;
this.lastPollDurationMs = finishedAt - this.loopStartedAt;
if (completedSuccessfully) {
this.lastSuccessfulPollAt = finishedAt;
}
this.loopRunning = false;
this.loopStartedAt = 0;
applicationMetrics.MetricsService.counter(poolMetricName(this.poolName, "loop_poll_finished_total")).inc();
}
const nextTriggerTime = this.nowMs() + sleepInterval;
if (this.periodicTaskFetcher && this.nextLoopTime > nextTriggerTime) {
logger$4.trace(`Resetting loop timer to closer time: from ${new Date(this.nextLoopTime)} to new ${new Date(nextTriggerTime)}`);
clearTimeout(this.periodicTaskFetcher);
this.nextLoopTime = nextTriggerTime;
this.periodicTaskFetcher = setTimeout(() => this.loopTimer(), Math.min(this.loopInterval, sleepInterval));
}
return sleepInterval;
}
async fetchNextTask() {
const task = await this.pollNextTask();
return task.match({
some: async (t) => {
if (this.stopRequested) {
return PipelineNextOperationFactory.sleep(this.loopInterval);
}
this.tasksInProcess++;
setImmediate(() => this.processTaskInLoop(t));
return this.tasksInProcess < this.maxConcurrentTasks
? PipelineNextOperationFactory.pollNext
: PipelineNextOperationFactory.sleep(this.loopInterval);
},
none: async () => {
if (this.stopRequested) {
return PipelineNextOperationFactory.sleep(this.loopInterval);
}
const nextTimeOpt = await this.peekNextStartAfter();
const delayMs = nextTimeOpt
.map((startAfter) => {
const delay = startAfter.getTime() - this.nowMs();
return Math.max(0, delay);
})
.getOrElseValue(this.loopInterval);
return PipelineNextOperationFactory.sleep(delayMs);
},
});
}
async processTaskInLoop(t) {
if (this.stopRequested) {
return;
}
try {
logger$4.debug(`Starting task (id=${t.id}) in queue ${t.queue} (active ${this.tasksInProcess} of ${this.maxConcurrentTasks})`);
await this.processTask(t);
}
catch (error) {
logger$4.warn(`Failed to process task ${t.id} in queue ${t.queue} `, error);
}
finally {
this.taskIsDone();
logger$4.debug(`Finished working with task ${t.id} in queue ${t.queue} (active ${this.tasksInProcess} of ${this.maxConcurrentTasks})`);
setImmediate(() => this.loop());
}
}
taskIsDone() {
this.tasksInProcess--;
this.tasksCountListener(this.tasksInProcess);
}
nowMs() {
return this.clock.now().getTime();
}
registerMetrics() {
applicationMetrics.MetricsService.gauge(poolMetricName(this.poolName, "slots_total"), () => this.maxConcurrentTasks);
applicationMetrics.MetricsService.gauge(poolMetricName(this.poolName, "slots_busy"), () => this.tasksInProcess);
applicationMetrics.MetricsService.gauge(poolMetricName(this.poolName, "loop_running"), () => this.loopRunning ? 1 : 0);
applicationMetrics.MetricsService.gauge(poolMetricName(this.poolName, "loop_running_ms"), () => (this.loopRunning ? this.nowMs() - this.loopStartedAt : 0));
applicationMetrics.MetricsService.gauge(poolMetricName(this.poolName, "loop_last_poll_duration_ms"), () => this.lastPollDurationMs);
applicationMetrics.MetricsService.gauge(poolMetricName(this.poolName, "loop_last_poll_started_at"), () => this.lastPollStartedAt);
applicationMetrics.MetricsService.gauge(poolMetricName(this.poolName, "loop_last_poll_finished_at"), () => this.lastPollFinishedAt);
applicationMetrics.MetricsService.gauge(poolMetricName(this.poolName, "loop_last_successful_poll_at"), () => this.lastSuccessfulPollAt);
applicationMetrics.MetricsService.gauge(poolMetricName(this.poolName, "loop_next_at"), () => this.nextLoopTime);
}
}
const logger$3 = log4js.getLogger("TasksQueueWorker");
class RuntimeTaskContext {
tasksQueueDao;
taskId;
startedAt;
currentAttempt;
maxAttempts;
timeout;
clock;
childTask = scats.none;
_nextPayload = scats.none;
_submittedResult = scats.none;
attemptStartedAt;
lastHeartbeatAt = scats.none;
resolvedChildTask = scats.none;
constructor(tasksQueueDao, taskId, startedAt, currentAttempt, maxAttempts, timeout, clock) {
this.tasksQueueDao = tasksQueueDao;
this.taskId = taskId;
this.startedAt = startedAt;
this.currentAttempt = currentAttempt;
this.maxAttempts = maxAttempts;
this.timeout = timeout;
this.clock = clock;
this.attemptStartedAt = new Date(startedAt);
}
async ensureNotStalled() {
const now = this.clock.now();
if (now.getTime() -
this.lastHeartbeatAt.getOrElseValue(this.attemptStartedAt.getTime()) <=
this.timeout) {
return;
}
const finalStatus = (await this.tasksQueueDao.failIfStalled(this.taskId, this.attemptStartedAt, now)).getOrElseValue(exports.TaskStatus.error);
throw new TaskTimedOutError(this.taskId, finalStatus);
}
async ping() {
await this.ensureNotStalled();
const now = this.clock.now().getTime();
if (this.lastHeartbeatAt
.map((lastHeartbeatAt) => now - lastHeartbeatAt < taskHeartbeatThrottleMs(this.timeout))
.getOrElseValue(false)) {
return;
}
const persisted = await this.tasksQueueDao.ping(this.taskId, this.attemptStartedAt, this.clock.now());
if (!persisted) {
throw new Error(`Task ${this.taskId} execution is no longer active`);
}
this.lastHeartbeatAt = scats.some(now);
}
spawnChild(task) {
if (this.childTask.nonEmpty) {
throw new Error(`Task ${this.taskId} attempted to spawn more than one child task`);
}
this.childTask = scats.some(task);
}
setPayload(payload) {
this._nextPayload = scats.some(payload);
}
submitResult(result) {
this._submittedResult = scats.some(result);
}
async findTask(taskId) {
return this.tasksQueueDao.findTaskState(taskId);
}
get spawnedChild() {
return this.childTask;
}
payloadToPersist(currentPayload) {
return this._nextPayload.getOrElseValue(scats.option(currentPayload).getOrElseValue({}));
}
get nextPayload() {
return this._nextPayload;
}
get submittedResult() {
return this._submittedResult;
}
}
class TasksQueueWorker {
tasksQueueDao;
clock;
poolName;
workers = new scats.mutable.HashMap();
pipeline;
started = false;
queueNotifier;
constructor(tasksQueueDao, concurrency = 4, loopInterval = TimeUtils.minute, clock = new SystemClock(), queueNotifier, poolName = "default") {
this.tasksQueueDao = tasksQueueDao;
this.clock = clock;
this.poolName = poolName;
this.queueNotifier = scats.option(queueNotifier).getOrElse(() => (queueName) => this.tasksScheduled(queueName));
this.pipeline = new TasksPipeline(concurrency, () => this.pollNextTask(), () => this.tasksQueueDao.peekNextStartAfter(this.workers.keySet, this.clock.now()), (t) => this.processNextTask(t), loopInterval, this.clock, this.poolName);
}
start() {
this.started = true;
this.pipeline.start();
}
async stop() {
this.started = false;
return await this.pipeline.stop();
}
async runOnce() {
const task = await this.pollNextTask();
await task.mapPromise(async (scheduledTask) => {
await this.processNextTask(scheduledTask);
return scheduledTask;
});
}
registerWorker(queueName, worker) {
if (this.workers.containsKey(queueName)) {
logger$3.warn(`Replacing existing worker for queue: ${queueName}`);
}
this.workers.put(queueName, worker);
this.updatePoolQueuesLabel();
}
tasksScheduled(queueName) {
if (this.started && this.workers.containsKey(queueName)) {
this.pipeline.triggerLoop();
}
}
async wakeBlockedParent(childTaskId) {
const parent = await this.tasksQueueDao.wakeParentOnChildTerminal(childTaskId, this.clock.now());
parent.foreach((p) => this.queueNotifier(p.queue));
}
async pollNextTask() {
this.updatePoolQueuesLabel();
this.workers.keySet.foreach((queueName) => {
applicationMetrics.MetricsService.counter(queueMetricName(queueName, "poll_total")).inc();
});
const task = await this.tasksQueueDao.nextPending(this.workers.keySet, this.clock.now());
task.foreach((scheduledTask) => {
applicationMetrics.MetricsService.counter(queueMetricName(scheduledTask.queue, "fetched_total")).inc();
});
return task;
}
updatePoolQueuesLabel() {
applicationMetrics.MetricsService.label(poolMetricName(this.poolName, "queues"), scats.Collection.from(this.workers.keySet.toArray)
.sort((a, b) => a.localeCompare(b))
.mkString(","));
}
async persistSuccessfulOutcome(task, context) {
await context.ensureNotStalled();
return await context.spawnedChild.match({
some: async (spawnedChild) => {
scats.option(task.repeatType).foreach(() => {
throw new Error(`Periodic task ${task.id} cannot spawn child tasks`);
});
const childTaskId = await this.tasksQueueDao.blockParentAndScheduleChild(task.id, spawnedChild, context.payloadToPersist(task.payload), task.started, this.clock.now());
childTaskId.foreach(() => this.queueNotifier(spawnedChild.queue));
return childTaskId.isDefined;
},
none: async () => {
return await scats.option(task.repeatType).match({
some: async () => {
return await this.tasksQueueDao.rescheduleIfPeriodic(task.id, context.payloadToPersist(task.payload), context.submittedResult.orUndefined, task.started, this.clock.now());
},
none: async () => {
const finished = await this.tasksQueueDao.finish(task.id, context.payloadToPersist(task.payload), context.submittedResult.orUndefined, task.started, this.clock.now());
if (finished) {
await this.wakeBlockedParent(task.id);
}
return finished;
},
});
},
});
}
async processNextTask(task) {
applicationMetrics.MetricsService.counter("tasks_queue_started").inc();
await this.workers.get(task.queue).match({
some: async (worker) => {
applicationMetrics.MetricsService.counter(queueMetricName(task.queue, "task_started_total")).inc();
const context = new RuntimeTaskContext(this.tasksQueueDao, task.id, task.started, task.currentAttempt, task.maxAttempts, task.timeout, this.clock);
try {
scats.Try(() => worker.starting(task.id, task.payload)).tapFailure((e) => logger$3.warn(`Failed to invoke 'starting' callback for task (id=${task.id}) in queue=${task.queue}`, e));
await worker.process(task.payload, context);
const persisted = await this.persistSuccessfulOutcome(task, context);
if (!persisted) {
logger$3.info(`Skipping completion for task (id=${task.id}) in queue=${task.queue}: execution ownership was lost`);
return;
}
applicationMetrics.MetricsService.counter("tasks_queue_processed").inc();
applicationMetrics.MetricsService.counter(queueMetricName(task.queue, "task_finished_total")).inc();
(await scats.Try.promise(() => worker.completed(task.id, task.payload))).tapFailure((e) => logger$3.warn(`Failed to invoke 'completed' callback for task (id=${task.id}) in queue=${task.queue}`, e));
}
catch (e) {
const finalStatus = e instanceof TaskTimedOutError
? e.finalStatus
: (await this.tasksQueueDao.fail(task.id, e["message"] || e, e instanceof TaskFailed ? e.payload : task.payload, context.submittedResult.orUndefined, task.started, this.clock.now())).getOrElseValue(exports.TaskStatus.error);
if (finalStatus === exports.TaskStatus.error &&
!(e instanceof TaskTimedOutError)) {
await this.wakeBlockedParent(task.id);
}
applicationMetrics.MetricsService.counter(queueMetricName(task.queue, `task_failed_${finalStatus}_total`)).inc();
(await scats.Try.promise(() => worker.failed(task.id, task.payload, finalStatus, e))).tapFailure((e) => logger$3.warn(`Failed to invoke 'failed' callback for task (id=${task.id}) in queue=${task.queue}`, e));
applicationMetrics.MetricsService.counter("tasks_queue_failed").inc();
logger$3.warn(`Failed to process task (id=${task.id}) in queue=${task.queue}`, e);
}
},
none: async () => {
applicationMetrics.MetricsService.counter("tasks_queue_skipped_no_worker").inc();
applicationMetrics.MetricsService.counter(queueMetricName(task.queue, "task_skipped_no_worker_total")).inc();
logger$3.info(`Failed to process task (id=${task.id}) in queue=${task.queue}: no suitable worker found`);
},
});
}
}
const logger$2 = log4js.getLogger("TasksAuxiliaryWorker");
class TasksAuxiliaryWorker {
tasksQueueDao;
manageTasksQueueService;
clock;
workerTimer = null;
metricsTimer = null;
queuesCounts = scats.HashMap.empty;
constructor(tasksQueueDao, manageTasksQueueService, clock = new SystemClock()) {
this.tasksQueueDao = tasksQueueDao;
this.manageTasksQueueService = manageTasksQueueService;
this.clock = clock;
}
start() {
this.workerTimer = setInterval(() => {
void this.runMaintenanceOnce();
}, TimeUtils.second * 30);
void this.runMaintenanceOnce();
this.metricsTimer = setInterval(() => {
void this.syncMetricsOnce();
}, TimeUtils.minute * 2);
void this.syncMetricsOnce();
}
async runMaintenanceOnce() {
try {
const failStalledPromise = this.tasksQueueDao
.failStalled(this.clock.now())
.then((res) => {
if (res.nonEmpty) {
logger$2.info(`Processed stalled tasks: ${res.mkString(", ")}`);
}
})
.catch((e) => {
logger$2.warn("Failed to process stalled tasks", e);
});
const resetFailedPromise = this.tasksQueueDao.resetFailed().catch((e) => {
logger$2.warn("Failed to reset failed tasks", e);
});
const clearFinishedPromise = this.tasksQueueDao
.clearFinished(undefined, this.clock.now())
.catch((e) => {
logger$2.warn("Failed to clear finished tasks", e);
});
await Promise.all([
failStalledPromise,
resetFailedPromise,
clearFinishedPromise,
]);
}
catch (e) {
logger$2.warn("Failed to process stalled tasks", e);
}
}
async syncMetricsOnce() {
try {
const tasksCounts = await this.manageTasksQueueService.tasksCount();
this.queuesCounts = tasksCounts.groupBy((c) => c.queueName);
tasksCounts.foreach((c) => {
applicationMetrics.MetricsService.gauge(`tasks_queue_${c.queueName}_${c.status}`.replace(/[^a-zA-Z0-9_:]/g, "_"), () => {
return this.queuesCounts
.get(c.queueName)
.getOrElseValue(scats.Nil)
.find((x) => c.status === x.status)
.map((c) => c.count)
.getOrElseValue(0);
});
});
}
catch (e) {
logger$2.warn("Failed to sync metrics", e);
}
}
stop() {
scats.option(this.workerTimer).foreach((t) => clearInterval(t));
scats.option(this.metricsTimer).foreach((t) => clearInterval(t));
}
}
const logger$1 = log4js.getLogger("TasksQueueService");
class TasksQueueService {
tasksQueueDao;
clock;
poolName;
worker;
auxiliaryWorker;
constructor(tasksQueueDao, manageTasksQueueService, config, clock = new SystemClock(), poolName = "default") {
this.tasksQueueDao = tasksQueueDao;
this.clock = clock;
this.poolName = poolName;
this.worker = new TasksQueueWorker(this.tasksQueueDao, config.concurrency, config.loopInterval, this.clock, config.queueNotifier, this.poolName);
if (config.runAuxiliaryWorker) {
this.auxiliaryWorker = scats.some(new TasksAuxiliaryWorker(tasksQueueDao, manageTasksQueueService, this.clock));
}
else {
this.auxiliaryWorker = scats.none;
}
}
async schedule(task) {
const taskId = await this.tasksQueueDao.schedule(task, this.clock.now());
this.taskScheduled(task.queue);
return taskId;
}
async scheduleAtFixedRate(task) {
const taskId = await this.tasksQueueDao.schedulePeriodic(task, exports.TaskPeriodType.fixed_rate, this.clock.now());
this.taskScheduled(task.queue);
return taskId;
}
async scheduleAtFixedDelay(task) {
const taskId = await this.tasksQueueDao.schedulePeriodic(task, exports.TaskPeriodType.fixed_delay, this.clock.now());
this.taskScheduled(task.queue);
return taskId;
}
async scheduleAtCron(task) {
const taskId = await this.tasksQueueDao.schedulePeriodic(task, exports.TaskPeriodType.cron, this.clock.now());
this.taskScheduled(task.queue);
return taskId;
}
taskScheduled(queueName) {
this.worker.tasksScheduled(queueName);
}
registerWorker(queueName, worker) {
this.worker.registerWorker(queueName, worker);
}
async runOnce() {
await this.worker.runOnce();
}
start() {
try {
this.worker.start();
this.auxiliaryWorker.foreach((w) => w.start());
}
catch (e) {
logger$1.warn("Failed to process stalled tasks", e);
}
}
async stop() {
this.auxiliaryWorker.foreach((w) => w.stop());
await this.worker.stop();
}
}
const DEFAULT_POOL = "default";
const logger = log4js.getLogger("TasksPoolsService");
class TasksPoolsService {
dao;
clock;
pools;
queuesPool = new scats.mutable.HashMap();
auxiliaryWorker;
constructor(dao, manageTasksQueueService, runAuxiliaryWorker, pools = [
{
name: DEFAULT_POOL,
concurrency: 1,
loopInterval: 60000,
},
], clock = new SystemClock()) {
this.dao = dao;
this.clock = clock;
const poolsCollection = scats.Collection.from(pools);
const poolNames = poolsCollection.map((p) => p.name).toSet;
if (poolsCollection.size !== poolNames.size) {
throw new Error("Duplicate pool names detected");
}
this.pools = poolsCollection.toMap((p) => [
p.name,
new TasksQueueService(dao, manageTasksQueueService, {
concurrency: p.concurrency,
runAuxiliaryWorker: false,
loopInterval: p.loopInterval,
queueNotifier: (queueName) => this.notifyQueue(queueName),
}, this.clock, p.name),
]);
this.auxiliaryWorker = runAuxiliaryWorker
? scats.some(new TasksAuxiliaryWorker(dao, manageTasksQueueService, this.clock))
: scats.none;
}
start() {
logger.info(`Starting TasksPoolsService with ${this.pools.size} pools`);
this.auxiliaryWorker.foreach((w) => w.start());
this.pools.values.foreach((p) => p.start());
}
async stop(timeoutMs = 30000) {
logger.info("Stopping TasksPoolsService");
let stopTimer;
try {
this.auxiliaryWorker.foreach((w) => w.stop());
await Promise.race([
this.pools.values.mapPromise((p) => p.stop()),
new Promise((_, reject) => {
stopTimer = setTimeout(() => reject(new Error("Stop timeout")), timeoutMs);
}),
]);
logger.info("TasksPoolsService stopped successfully");
}
catch (e) {
logger.error("Failed to stop TasksPoolsService gracefully", e);
throw e;
}
finally {
scats.option(stopTimer).foreach((timer) => clearTimeout(timer));
}
}
registerWorker(queueName, worker, poolName = DEFAULT_POOL) {
if (this.queuesPool.containsKey(queueName)) {
throw new Error(`Queue '${queueName}' is already registered in pool '${this.queuesPool.get(queueName).getOrElseValue("unknown")}'`);
}
this.pools.get(poolName).match({
some: (pool) => {
pool.registerWorker(queueName, worker);
this.queuesPool.put(queueName, poolName);
logger.info(`Registered worker for queue '${queueName}' in pool '${poolName}'`);
},
none: () => {
throw new Error(`Pool '${poolName}' not registered`);
},
});
}
async schedule(task) {
const taskId = await this.dao.schedule(task, this.clock.now());
this.taskScheduled(task.queue, taskId);
return taskId;
}
async scheduleAtFixedRate(task) {
const taskId = await this.dao.schedulePeriodic(task, exports.TaskPeriodType.fixed_rate, this.clock.now());
this.taskScheduled(task.queue, taskId);
return taskId;
}
async scheduleAtFixedDelay(task) {
const taskId = await this.dao.schedulePeriodic(task, exports.TaskPeriodType.fixed_delay, this.clock.now());
this.taskScheduled(task.queue, taskId);
return taskId;
}
async scheduleAtCron(task) {
const taskId = await this.dao.schedulePeriodic(task, exports.TaskPeriodType.cron, this.clock.now());
this.taskScheduled(task.queue, taskId);
return taskId;
}
taskScheduled(queue, taskId) {
this.notifyQueue(queue, taskId);
}
notifyQueue(queue, taskId = scats.none) {
this.queuesPool.get(queue).match({
some: (poolName) => {
this.pools.get(poolName).foreach((pool) => {
pool.taskScheduled(queue);
});
},
none: () => {
logger.info(`No worker registered for a queue '${queue}'. ` +
`Task (id=${taskId.getOrElseValue(-1)}) will remain in pending state`);
},
});
}
}
class PeriodicScheduleUtils {
static resolveRepeatInterval(task, periodType) {
if (periodType === exports.TaskPeriodType.cron) {
return null;
}
const periodicTask = task;
if (periodicTask.period === undefined || periodicTask.period === null) {
throw new Error(`Periodic task with type='${periodType}' must define 'period' in milliseconds`);
}
return periodicTask.period;
}
static resolveCronExpression(task, periodType) {
if (periodType !== exports.TaskPeriodType.cron) {
return null;
}
const cronTask = task;
const cronExpression = scats.option(cronTask.cronExpression)
.map((value) => value.trim())
.filter((value) => value.length > 0);
if (cronExpression.isEmpty) {
throw new Error("Periodic task with type='cron' must define non-empty 'cronExpression'");
}
return cronExpression.get;
}
static calculateNextStartAfter(task, now) {
return task.repeatType === exports.TaskPeriodType.cron
? this.calculateNextCronStartAfter(task, now)
: this.calculateNextIntervalStartAfter(task, now);
}
static calculateNextCronStartAfter(task, now) {
const cronExpression = task.cronExpression.match({
some: (value) => value,
none: () => {
throw new Error("Cron periodic task must define 'cron_expression'");
},
});
const baseDate = task.missedRunsStrategy === exports.MissedRunStrategy.catch_up
? task.startAfter
: now;
return CronExpressionUtils.nextExecutionDate(cronExpression, baseDate);
}
static calculateNextIntervalStartAfter(task, now) {
const repeatInterval = task.repeatInterval.match({
some: (value) => value,
none: () => {
throw new Error(`Periodic task with type='${task.repeatType}' must define 'repeat_interval'`);
},
});
switch (task.repeatType) {
case exports.TaskPeriodType.fixed_rate:
return task.missedRunsStrategy === exports.MissedRunStrategy.catch_up
? new Date(task.startAfter.getTime() + repeatInterval)
: this.calculateFixedRateSkipMissedNextStartAfter(task.initialStart, now, repeatInterval);
case exports.TaskPeriodType.fixed_delay:
return new Date(now.getTime() + repeatInterval);
default:
throw new Error(`Unsupported periodic task type: ${task.repeatType}`);
}
}
static calculateFixedRateSkipMissedNextStartAfter(initialStart, now, repeatInterval) {
const elapsed = now.getTime() - initialStart.getTime();
const periods = Math.max(1, Math.floor(elapsed / repeatInterval) + 1);
return new Date(initialStart.getTime() + periods * repeatInterval);
}
}
class ActiveChildState {
taskId;
allowFailure;
constructor(taskId, allowFailure = false) {
this.taskId = taskId;
this.allowFailure = allowFailure;
}
copy(o) {
return new ActiveChildState(scats.option(o.taskId).map(Number).getOrElseValue(this.taskId), scats.option(o.allowFailure).map(Boolean).getOrElseValue(this.allowFailure));
}
get toJson() {
const res = {
taskId: this.taskId,
};
scats.option(this.allowFailure)
.filter((x) => x)
.foreach((allowFailure) => {
res["allowFailure"] = allowFailure;
});
return res;
}
static fromJson(j) {
return scats.option(j)
.map((x) => x)
.flatMap((x) => scats.option(x["taskId"]).map((taskId) => new ActiveChildState(Number(taskId), scats.option(x["allowFailure"]).map(Boolean).getOrElseValue(false))));
}
}
class MultiStepPayload {
activeChild;
workflowPayload;
userPayload;
static forUserPayload(userPayload) {
return new MultiStepPayload(scats.none, {}, userPayload);
}
constructor(activeChild, workflowPayload, userPayload) {
this.activeChild = activeChild;
this.workflowPayload = workflowPayload;
this.userPayload = userPayload;
}
copy(o) {
return new MultiStepPayload(scats.option(o.activeChild).getOrElseValue(this.activeChild), scats.option(o.workflowPayload).getOrElseValue(this.workflowPayload), scats.option(o.userPayload).getOrElseValue(this.userPayload));
}
get toJson() {
const res = {
workflowPayload: this.workflowPayload,
userPayload: this.userPayload,
};
this.activeChild.foreach((activeChild) => {
res["activeChild"] = activeChild.toJson;
});
return res;
}
static fromJson(j) {
const o = scats.option(j).map((x) => x);
return new MultiStepPayload(o.flatMap((x) => ActiveChildState.fromJson(x["activeChild"]).match({
some: (activeChild) => scats.option(activeChild),
none: () => scats.option(x["activeChildId"]).map((taskId) => new ActiveChildState(Number(taskId))),
})), o
.flatMap((x) => scats.option(x["workflowPayload"]))
.map((x) => x)
.getOrElseValue({}), o
.flatMap((x) => scats.option(x["userPayload"]))
.getOrElseValue({}));
}
}
class TasksQueueDao {
pool;
constructor(pool) {
this.pool = pool;
}
async insertOneTimeTask(cl, task, parentId, now = new Date()) {
const res = await cl.query(`insert into tasks_queue (parent_id, queue, created, status, priority, payload, timeout, max_attempts,
start_after, initial_start, backoff, backoff_type)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
returning id`, [
scats.option(parentId).orNull,
task.queue,
now,
exports.TaskStatus.pending,
scats.option(task.priority).getOrElseValue(0),
scats.option(task.payload).orNull,
scats.option(task.timeout).getOrElseValue(TimeUtils.hour),
scats.option(task.retries).getOrElseValue(1),
scats.option(task.startAfter).orNull,
scats.option(task.startAfter).getOrElseValue(now),
scats.option(task.backoff).getOrElseValue(TimeUtils.minute),
scats.option(task.backoffType).getOrElseValue(exports.BackoffType.linear),
]);
return scats.Collection.from(res.rows).headOption.map((r) => r.id);
}
async withClient(cb) {
const client = await this.pool.connect();
let res;
try {
res = await cb(client);
}
finally {
client.release();
}
return res;
}
async schedule(task, now = new Date()) {
return await this.withClient(async (cl) => {
return this.insertOneTimeTask(cl, task, undefined, now);
});
}
async blockParentAndScheduleChild(parentTaskId, childTask, parentPayload, expectedStarted, now = new Date()) {
return await this.withClient(async (cl) => {
await cl.query("BEGIN");
try {
const parentRes = await cl.query(`update tasks_queue
set status = $1,
attempt = greatest(attempt - 1, 0),
started = null,
last_heartbeat = null,
finished = null,
error = null
where id = $2
and status = $3
and started = $4
and repeat_type is null
returning id`, [
exports.TaskStatus.blocked,
parentTaskId,
exports.TaskStatus.in_progress,
expectedStarted,
]);
if (scats.option(parentRes.rowCount).getOrElseValue(0) !== 1) {
await cl.query("ROLLBACK");
return scats.none;
}
const childTaskId = await this.insertOneTimeTask(cl, childTask, parentTaskId, now);
await childTaskId.mapPromise(async (id) => {
await cl.query(`update tasks_queue
set payload = $1
where id = $2
and status = $3`, [
MultiStepPayload.fromJson(parentPayload).copy({
activeChild: scats.option(new ActiveChildState(id, childTask.allowFailure === true)),
}).toJson,
parentTaskId,
exports.TaskStatus.blocked,
]);
return id;
});
await cl.query("COMMIT");
return childTaskId;
}
catch (e) {
await cl.query("ROLLBACK");
throw e;
}
});
}
async schedulePeriodic(task, periodType, now = new Date()) {
const repeatInterval = PeriodicScheduleUtils.resolveRepeatInterval(task, periodType);
const cronExpression = PeriodicScheduleUtils.resolveCronExpression(task, periodType);
scats.option(cronExpression).foreach((expr) => CronExpressionUtils.validate(expr));
if (task.name.length > 20) {
throw new Error("Periodic task 'name' must be 20 characters or less");
}
const replaceExisting = scats.option(task.replaceExisting).getOrElseValue(false);
const hasExplicitStartAfter = task.startAfter !== undefined;
return await this.withClient(async (cl) => {
const params = scats.Collection.of(task.queue, now, exports.TaskStatus.pending, scats.option(task.priority).getOrElseValue(0), scats.option(task.payload).orNull, scats.option(task.timeout).getOrElseValue(TimeUtils.hour), scats.option(task.retries).getOrElseValue(1), scats.option(task.startAfter).getOrElseValue(now), task.name, repeatInterval, cronExpression, periodType, scats.option(task.backoff).getOrElseValue(TimeUtils.minute), scats.option(task.backoffType).getOrElseValue(exports.BackoffType.linear), scats.option(task.missedRunStrategy).getOrElseValue(exports.MissedRunStrategy.skip_missed)).toArray;
const res = await cl.query(replaceExisting
? `insert into tasks_queue (queue, created, status, priority, payload, timeout, max_attempts,
start_after, initial_start, name,
repeat_interval, cron_expression, repeat_type, backoff, backoff_type, missed_runs_strategy)
values ($1, $2, $3, $4, $5, $6, $7, $8, $8, $9, $10, $11, $12, $13, $14, $15)
on conflict (name) do update
set queue = excluded.queue,
status = excluded.status,
priority = excluded.priority,
payload = excluded.payload,
timeout = excluded.timeout,
max_attempts = excluded.max_attempts,
start_after = case when $16 then excluded.start_after else tasks_queue.start_after end,
initial_start = case when $16 then excluded.initial_start else tasks_queue.initial_start end,
repeat_interval = excluded.repeat_interval,
cron_expression = excluded.cron_expression,
repeat_type = excluded.repeat_type,
backoff = excluded.backoff,
backoff_type = excluded.backoff_type,
missed_runs_strategy = excluded.missed_runs_strategy,
started = null,
last_heartbeat = null,
finished = null,
error = null,
attempt = 0,
parent_id = null,
result = null
where tasks_queue.status = $3
returning id`
: `insert into tasks_queue (queue, created, status, priority, payload, timeout, max_attempts,
start_after, initial_start, name,
repeat_interval, cron_expression, repeat_type, backoff, backoff_type, missed_runs_strategy)
values ($1, $2, $3, $4, $5, $6, $7, $8, $8, $9, $10, $11, $12, $13, $14, $15)
on conflict (name) do nothing
returning id`, replaceExisting ? params.concat(hasExplicitStartAfter) : params);
return scats.Collection.from(res.rows).headOption.map((r) => r.id);
});
}
async nextPending(queueNam