queue-manager-pro
Version:
A flexible, TypeScript-first queue/task manager with pluggable backends ,dynamic persistence storage and event hooks.
218 lines • 9.17 kB
JavaScript
import { HandlerRegistry } from './handlerRegistry.js';
import { FileQueueRepository } from '../repositories/file.repository.js';
import { MemoryQueueRepository } from '../repositories/memory.repository.js';
import { EventEmitter } from 'events';
import {} from '../types/index.js';
import { warnings } from '../util/warnings.js';
import { QueueWorker } from './queueWorker.js';
import { RedisQueueRepository } from '../repositories/redis.repository.js';
import { randomUUID } from 'crypto';
import { PostgresQueueRepository } from '../repositories/postgres.repository.js';
import { InvalidHandlerParamsError, MaxRetriesLimitError, UnknownBackendTypeError } from '../util/errors.js';
import { CustomQueueRepository } from '../repositories/custom.repository.js';
const singletonRegistry = new HandlerRegistry();
const MAX_PROCESSING_TIME = 10 * 60 * 1000; // 10 minutes in milliseconds
const MAX_RETRIES = 3; // Default max retries for tasks
const MAX_RETRIES_LIMIT = 10; // max retries limit
const DEFAULT_DELAY = 10000; // Default delay between task checks in milliseconds
export class QueueManager {
emitter = new EventEmitter();
// If you want to use multiple instances, set `singleton` to false in `getInstance`
static instance;
// Delay between task checks in milliseconds
delay;
// Register and manage task handlers.
registry;
// Repository is the main interface for interacting with the persistent storage backend.
// It is used to load, save, enqueue, and dequeue tasks.
repository;
MAX_RETRIES;
MAX_PROCESSING_TIME;
backend;
worker;
crashOnWorkerError = false;
logger;
on(event, listener) {
this.emitter.on(event, listener);
return this;
}
emit = (event, ...args) => {
return this.emitter.emit(event, ...args);
};
constructor({ delay = DEFAULT_DELAY, singleton = true, maxRetries = MAX_RETRIES, maxProcessingTime = MAX_PROCESSING_TIME, logger, backend, repository, crashOnWorkerError, }) {
this.worker = new QueueWorker(this, logger);
this.repository = repository;
this.delay = delay;
this.registry = singleton ? singletonRegistry : new HandlerRegistry();
this.MAX_RETRIES = maxRetries;
this.MAX_PROCESSING_TIME = maxProcessingTime;
this.logger = logger; // Optional logger, can be used for logging events
this.backend = backend;
this.crashOnWorkerError = crashOnWorkerError ?? false;
if (backend.type === 'custom') {
this.log('warn', warnings.customRepository);
}
}
static getInstance(args) {
const maxRetries = args.maxRetries || MAX_RETRIES;
const maxProcessingTime = args.maxProcessingTime || MAX_PROCESSING_TIME;
if (maxRetries > MAX_RETRIES_LIMIT) {
throw new MaxRetriesLimitError(maxRetries);
}
const repository = this.getBackendRepository(args.backend, maxRetries, maxProcessingTime);
repository.logger = args.logger;
const isSingleton = args.singleton !== false; // default to singleton
const delay = args.delay ?? DEFAULT_DELAY;
if (isSingleton) {
if (!QueueManager.instance) {
QueueManager.instance = new QueueManager({
repository,
backend: args.backend,
delay,
singleton: true,
maxRetries,
maxProcessingTime,
logger: args.logger,
crashOnWorkerError: args.crashOnWorkerError,
});
}
else if (QueueManager.instance.repository.id !== repository.id) {
// Optional: warn if repository is different from the original
QueueManager.instance.log('warn', 'Different repository detected for singleton instance');
}
return QueueManager.instance;
}
return new QueueManager({
repository,
delay,
singleton: false,
backend: args.backend,
maxRetries,
maxProcessingTime,
logger: args.logger,
crashOnWorkerError: args.crashOnWorkerError,
});
}
static getBackendRepository(backend, maxRetries, maxProcessingTime) {
switch (backend.type) {
case 'file':
return new FileQueueRepository(backend.filePath, maxRetries, maxProcessingTime);
case 'memory':
return new MemoryQueueRepository(maxRetries, maxProcessingTime);
case 'postgres':
return new PostgresQueueRepository(backend.pg, maxRetries, maxProcessingTime, backend.options);
case 'redis':
return new RedisQueueRepository(backend.redisClient, maxRetries, maxProcessingTime, backend.options);
case 'custom':
return new CustomQueueRepository(backend.repository);
default:
throw new UnknownBackendTypeError();
}
}
async addTaskToQueue(handler, payload, options) {
if (options?.maxRetries && options?.maxRetries > MAX_RETRIES_LIMIT) {
throw new MaxRetriesLimitError(options.maxRetries);
}
const validationResult = this.validateHandlerParams(handler, payload);
if (!validationResult.isValid) {
if (options?.skipOnPayloadError) {
this.log('warn', `skipOnPayloadError set to true, but this task might fail due to invalid payload: ${validationResult.message}`);
}
else {
throw new InvalidHandlerParamsError(validationResult.message ?? undefined);
}
}
const handlerEntry = this.registry.get(handler);
const task = {
id: randomUUID(),
payload,
handler: handler,
status: 'pending',
log: '',
createdAt: new Date(),
updatedAt: new Date(),
maxRetries: options?.maxRetries ?? handlerEntry?.options?.maxRetries ?? this.MAX_RETRIES,
maxProcessingTime: options?.maxProcessingTime ?? handlerEntry?.options?.maxProcessingTime ?? this.MAX_PROCESSING_TIME,
retryCount: 0,
priority: options?.priority ?? 0,
};
await this.repository.enqueue(task);
this.emit('taskAdded', task);
return task;
}
async removeTask(id, hardDelete) {
const task = await this.repository.deleteTask(id.toString(), hardDelete);
if (task) {
this.emit('taskRemoved', task);
}
return task;
}
async updateTask(id, obj) {
try {
return await this.repository.updateTask(id, obj);
}
catch (error) {
this.log('error', 'Failed to update task:', error);
throw error;
}
}
async getAllTasks() {
try {
return await this.repository.loadTasks();
}
catch (error) {
this.log('error', 'Failed to load tasks:', error);
throw error;
}
}
async getTaskById(id) {
const tasks = await this.repository.loadTasks();
return tasks.find(task => task.id === id);
}
async startWorker(concurrency = 1) {
await this.worker.startWorker(concurrency);
}
async stopWorker() {
await this.worker.stopWorker();
}
/**
* Register a handler for a specific task type.
* This method registers a handler function for a specific task type,
* allowing the queue manager to execute the handler when a task of that type is dequeued.
* @param name The name of the handler
* @param handler The function to execute for this handler
* @param options Optional parameters for max retries and processing time
*/
register(name, handler, options) {
if (!options || (!options.paramSchema && !options.useAutoSchema)) {
const warningMessage = warnings.handlerRegistryWarning.replace(/\$1/g, name);
this.log('warn', warningMessage);
}
this.registry.register(name, handler, options);
}
/**
* Get the registered handler for a specific task type.
* This method retrieves the handler function and its parameters for a specific task type.
* - useful for validating payloads before adding tasks to the queue.
* @param name The name of the handler
* @returns An object containing the handler function and its parameters
*/
validateHandlerParams(name, payload) {
return this.registry.validateParams(name, payload);
}
log(level, ...args) {
if (this.logger?.[level]) {
this.logger[level](...args);
}
}
async runMigration() {
if (this.repository instanceof PostgresQueueRepository) {
await this.repository.postgresMigrateTasksTable();
}
else {
throw new Error(`runMigration is not available for your select backend type ${this.backend.type}`);
}
}
}
export default QueueManager;
//# sourceMappingURL=QueueManager.js.map