wakaq
Version:
Background task queue for Node backed by Redis, a super minimal Celery
248 lines (247 loc) • 11.2 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.WakaQ = void 0;
const ioredis_1 = require("ioredis");
const os = __importStar(require("os"));
const ts_duration_1 = require("ts-duration");
const constants_js_1 = require("./constants.js");
const exceptions_js_1 = require("./exceptions.js");
const logger_js_1 = require("./logger.js");
const serializer_js_1 = require("./serializer.js");
const task_js_1 = require("./task.js");
class WakaQ {
constructor(params) {
this.tasks = new Map([]);
this.queuesByName = new Map([]);
this.queuesByKey = new Map([]);
this.broadcastKey = 'wakaq-broadcast';
const queues = params?.queues ?? [];
const schedules = params?.schedules ?? [];
const host = params?.host ?? 'localhost';
const port = params?.port ?? 6379;
const db = params?.db ?? 0;
const tls = params?.tls;
const concurrency = params?.concurrency ?? 1;
const excludeQueues = params?.excludeQueues ?? [];
const maxRetries = params?.maxRetries ?? 0;
const maxMemPercent = params?.maxMemPercent ?? 0;
const maxTasksPerWorker = params?.maxTasksPerWorker ?? 0;
this.connectTimeout = params?.connectTimeout ?? 15000;
this.commandTimeout = params?.commandTimeout ?? 15000;
this.keepAlive = params?.keepAlive ?? 0;
this.noDelay = params?.noDelay ?? true;
this.singleProcess = params?.singleProcess ?? false;
const { username, password, workerLogLevel, schedulerLogLevel, afterWorkerStartedCallback, beforeTaskStartedCallback, afterTaskFinishedCallback, } = params ?? {};
const lowestPriority = Math.max(...queues.map((q) => {
return q.priority;
}));
queues.forEach((q) => q.setDefaultPriority(lowestPriority));
queues.sort((a, b) => a.priority - b.priority);
this.queues = queues;
queues.forEach((q) => {
this.queuesByName.set(q.name, q);
this.queuesByKey.set(q.brokerKey, q);
});
this.excludeQueues = this._validateQueueNames(excludeQueues);
this.maxRetries = maxRetries;
this.brokerKeys = queues.filter((q) => !this.excludeQueues.includes(q.name)).map((q) => q.brokerKey);
this.schedules = schedules;
this.concurrency = this._formatConcurrency(concurrency);
this.softTimeout = this._asDuration(params?.softTimeout, 0);
this.hardTimeout = this._asDuration(params?.hardTimeout, 0);
this.waitTimeout = this._asDuration(params?.waitTimeout, 1);
if (this.waitTimeout.seconds < 1)
throw new exceptions_js_1.WakaQError(`Wait timeout (${this.waitTimeout.seconds}) can not be less than 1 second.`);
if (!this.singleProcess) {
if (this.softTimeout.seconds && this.softTimeout.seconds <= this.waitTimeout.seconds)
throw new exceptions_js_1.WakaQError(`Soft timeout (${this.softTimeout.seconds}) can not be less than or equal to wait timeout (${this.waitTimeout.seconds}).`);
if (this.hardTimeout.seconds && this.hardTimeout.seconds <= this.waitTimeout.seconds)
throw new exceptions_js_1.WakaQError(`Hard timeout (${this.hardTimeout.seconds}) can not be less than or equal to wait timeout (${this.waitTimeout.seconds}).`);
if (this.softTimeout.seconds && this.hardTimeout.seconds && this.hardTimeout.seconds <= this.softTimeout.seconds)
throw new exceptions_js_1.WakaQError(`Hard timeout (${this.hardTimeout.seconds}) can not be less than or equal to soft timeout (${this.softTimeout.seconds}).`);
}
if ((maxMemPercent && maxMemPercent < 1) || maxMemPercent > 99)
throw new exceptions_js_1.WakaQError(`Max memory percent must be between 1 and 99: ${maxMemPercent}`);
this.maxMemPercent = maxMemPercent;
this.maxTasksPerWorker = maxTasksPerWorker > 0 ? maxTasksPerWorker : 0;
this.workerLogFile = params?.workerLogFile;
this.schedulerLogFile = params?.schedulerLogFile;
this.workerLogLevel = workerLogLevel ?? logger_js_1.Level.INFO;
this.schedulerLogLevel = schedulerLogLevel ?? logger_js_1.Level.INFO;
this.afterWorkerStartedCallback = afterWorkerStartedCallback;
this.beforeTaskStartedCallback = beforeTaskStartedCallback;
this.afterTaskFinishedCallback = afterTaskFinishedCallback;
this.broker = new ioredis_1.Redis({
host: host,
port: port,
username: username,
password: password,
db: db,
tls: tls,
lazyConnect: true,
connectTimeout: this.connectTimeout,
commandTimeout: this.commandTimeout,
keepAlive: this.keepAlive,
noDelay: this.noDelay,
});
this.broker.on('error', (err) => {
this.logger?.error(err);
});
}
async connect() {
await this.broker.connect();
this.broker.defineCommand('getetatasks', {
numberOfKeys: 1,
lua: constants_js_1.ZRANGEPOP,
});
return this;
}
disconnect() {
this.broker.disconnect();
this._pubsub?.disconnect();
}
task(fn, params) {
const task = new task_js_1.Task(this, fn, params?.name, params?.queue, params?.softTimeout, params?.hardTimeout, params?.maxRetries);
if (this.tasks.has(task.name)) {
this.logger?.error(`Duplicate task name: ${task.name}`);
console.log(`Duplicate task name: ${task.name}`);
throw new exceptions_js_1.WakaQError(`Duplicate task name: ${task.name}`);
}
this.tasks.set(task.name, task);
return task;
}
afterWorkerStarted(callback) {
this.afterWorkerStartedCallback = callback;
return callback;
}
beforeTaskStarted(callback) {
this.beforeTaskStartedCallback = callback;
return callback;
}
afterTaskFinished(callback) {
this.afterTaskFinishedCallback = callback;
return callback;
}
_validateQueueNames(queueNames) {
queueNames.forEach((queueName) => {
if (!this.queuesByName.has(queueName))
throw new exceptions_js_1.WakaQError(`Invalid queue: ${queueName}`);
});
return queueNames;
}
_asDuration(obj, def) {
if (obj instanceof ts_duration_1.Duration)
return obj;
if (typeof obj === 'object' && typeof obj.seconds === 'number')
return obj;
if (typeof obj === 'number')
return ts_duration_1.Duration.second(obj);
return ts_duration_1.Duration.second(def ?? 0);
}
async enqueueAtFront(taskName, args, queue) {
queue = this._queueOrDefault(queue);
const payload = (0, serializer_js_1.serialize)({ name: taskName, args: args });
await this.broker.lpush(queue.brokerKey, payload);
}
async enqueueWithEta(taskName, args, eta, queue) {
queue = this._queueOrDefault(queue);
const payload = (0, serializer_js_1.serialize)({ name: taskName, args: args });
const timestamp = Math.round((eta instanceof ts_duration_1.Duration ? Date.now() + eta.milliseconds : eta.getTime()) / 1000);
await this.broker.zadd(queue.brokerEtaKey, 'NX', String(timestamp), payload);
}
async enqueueAtEnd(taskName, args, queue, retry = 0) {
queue = this._queueOrDefault(queue);
const payload = (0, serializer_js_1.serialize)({ name: taskName, args: args, retry: retry });
await this.broker.rpush(queue.brokerKey, payload);
}
async broadcast(taskName, args) {
const payload = (0, serializer_js_1.serialize)({ name: taskName, args: args });
const pubsub = await this.pubsub();
return await pubsub.publish(this.broadcastKey, payload);
}
async sleep(duration) {
return new Promise((resolve) => {
setTimeout(resolve, duration.milliseconds);
});
}
async pubsub() {
if (!this._pubsub) {
this._pubsub = this.broker.duplicate();
await this._pubsub.connect();
}
return this._pubsub;
}
get defaultQueue() {
if (this.queues.length === 0)
throw new exceptions_js_1.WakaQError('Missing queues.');
return this.queues[this.queues.length - 1];
}
async blockingDequeue() {
if (this.brokerKeys.length === 0) {
this.sleep(this.waitTimeout);
return {};
}
const data = await this.broker.blpop(this.brokerKeys, this.waitTimeout.seconds);
if (!data)
return {};
return { queueBrokerKey: data[0], payload: (0, serializer_js_1.deserialize)(data[1]) };
}
_queueOrDefault(queue) {
if (typeof queue === 'string')
queue = this.queuesByName.get(queue);
if (queue)
return queue;
return this.defaultQueue;
}
_formatConcurrency(concurrency, isRecursive = false) {
if (!concurrency)
return 0;
if (typeof concurrency === 'number') {
if (!isRecursive && concurrency < 1)
throw new exceptions_js_1.WakaQError(`Concurrency must be greater than zero: ${concurrency}`);
return Math.round(concurrency);
}
const parsed = this._parseConcurrency(concurrency);
if (Number.isNaN(parsed))
throw new exceptions_js_1.WakaQError(`Error parsing concurrency: ${concurrency}`);
if (!isRecursive && !parsed)
return 1;
if (!isRecursive && parsed < 1)
throw new exceptions_js_1.WakaQError(`Concurrency must be greater than zero: ${parsed}`);
return parsed;
}
_parseConcurrency(concurrency) {
const parts = concurrency.split('*');
if (parts.length > 1) {
return parts.map((part) => this._formatConcurrency(part, true)).reduce((a, n) => a * n, 1);
}
else {
const cores = String(os.cpus().length);
return Number.parseInt(concurrency.replace('cores', cores).trim());
}
}
}
exports.WakaQ = WakaQ;