nodejs-cloud-taskmq
Version:
Node.js TypeScript library for integrating Google Cloud Tasks with MongoDB/Redis/Memory/Custom for a BullMQ-like queue system. Compatible with NestJS but framework-agnostic.
309 lines (308 loc) • 13 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ConsumerService = void 0;
const events_1 = require("events");
const storage_adapter_interface_1 = require("../interfaces/storage-adapter.interface");
const cloud_task_model_1 = require("../models/cloud-task.model");
const processor_decorator_1 = require("../decorators/processor.decorator");
const process_decorator_1 = require("../decorators/process.decorator");
const events_decorator_1 = require("../decorators/events.decorator");
require("reflect-metadata");
/**
* Consumer service for processing tasks
*/
class ConsumerService extends events_1.EventEmitter {
constructor(config, storageAdapter) {
super();
this.config = config;
this.storageAdapter = storageAdapter;
this.processors = new Map();
this.activeProcessors = new Map(); // queueName -> Set of taskIds
}
/**
* Initialize the consumer service
*/
async initialize() {
// Consumer is initialized when processors are registered
}
/**
* Register a processor instance
*/
registerProcessor(instance) {
const queueName = Reflect.getMetadata(processor_decorator_1.PROCESSOR_QUEUE_KEY, instance.constructor);
if (!queueName) {
throw new Error('Processor must be decorated with @Processor');
}
const options = Reflect.getMetadata(processor_decorator_1.PROCESSOR_METADATA_KEY, instance.constructor) || {};
const processHandlers = Reflect.getMetadata(process_decorator_1.PROCESS_METADATA_KEY, instance) || [];
const eventHandlers = Reflect.getMetadata(events_decorator_1.EVENT_HANDLERS_KEY, instance) || [];
if (processHandlers.length === 0) {
throw new Error(`Processor for queue "${queueName}" must have at least one @Process decorated method`);
}
const registration = {
instance,
queueName,
options,
processHandlers,
eventHandlers,
};
if (!this.processors.has(queueName)) {
this.processors.set(queueName, []);
this.activeProcessors.set(queueName, new Set());
}
this.processors.get(queueName).push(registration);
console.log(`Registered processor for queue "${queueName}" with ${processHandlers.length} process handlers`);
}
/**
* Process a task received from Cloud Tasks
*/
async processTask(payload) {
const { taskId, queueName } = payload;
// Get task from storage
let task = await this.storageAdapter.getTask(taskId);
if (!task) {
throw new Error(`Task ${taskId} not found in storage`);
}
// Create CloudTask instance
const cloudTask = new cloud_task_model_1.CloudTask(task);
// Check if task is already being processed
let activeProcessors = this.activeProcessors.get(queueName);
if (!activeProcessors) {
activeProcessors = new Set();
this.activeProcessors.set(queueName, activeProcessors);
}
if (activeProcessors.has(taskId)) {
throw new Error(`Task ${taskId} is already being processed`);
}
// Get processors for this queue
const processors = this.processors.get(queueName);
if (!processors || processors.length === 0) {
throw new Error(`No processors registered for queue "${queueName}"`);
}
// Mark task as active
cloudTask.markAsActive();
await this.storageAdapter.updateTaskStatus(taskId, storage_adapter_interface_1.TaskStatus.ACTIVE, {
updatedAt: cloudTask.updatedAt,
});
// Add to active processors
activeProcessors.add(taskId);
try {
// Emit active event
await this.emitTaskEvent('active', processors, cloudTask);
// Emit taskActive event on main instance
this.emit('taskActive', {
taskId: cloudTask.id,
queueName: cloudTask.queueName,
data: cloudTask.data,
timestamp: new Date(),
});
// Process the task
const result = await this.executeTaskProcessing(processors, cloudTask);
// Mark as completed
cloudTask.markAsCompleted(result);
await this.storageAdapter.updateTaskStatus(taskId, storage_adapter_interface_1.TaskStatus.COMPLETED, {
result,
completedAt: cloudTask.completedAt,
updatedAt: cloudTask.updatedAt,
});
// Emit completed event
const completedEvent = {
taskId: cloudTask.id,
queueName: cloudTask.queueName,
data: cloudTask.data,
result,
duration: cloudTask.getDuration() || 0,
timestamp: new Date(),
};
await this.emitTaskEvent('completed', processors, cloudTask, result);
this.emit('taskCompleted', completedEvent);
// Handle chain progression
if (cloudTask.isInChain() && !cloudTask.isLastInChain()) {
await this.processNextInChain(cloudTask);
}
// Clean up if configured
if (cloudTask.shouldRemoveOnComplete()) {
await this.storageAdapter.deleteTask(taskId);
}
// Remove uniqueness key if configured
if (cloudTask.uniquenessKey && cloudTask.shouldRemoveOnComplete()) {
await this.storageAdapter.removeUniquenessKey(cloudTask.uniquenessKey);
}
return result;
}
catch (error) {
// Handle task failure
cloudTask.incrementAttempts();
const isLastAttempt = cloudTask.hasExceededMaxAttempts();
if (isLastAttempt) {
cloudTask.markAsFailed(error instanceof Error ? error : new Error(String(error)));
await this.storageAdapter.updateTaskStatus(taskId, storage_adapter_interface_1.TaskStatus.FAILED, {
error: cloudTask.error,
failedAt: cloudTask.failedAt,
attempts: cloudTask.attempts,
updatedAt: cloudTask.updatedAt,
});
// Clean up if configured
if (cloudTask.shouldRemoveOnFail()) {
await this.storageAdapter.deleteTask(taskId);
}
// Remove uniqueness key if configured
if (cloudTask.uniquenessKey && cloudTask.shouldRemoveOnFail()) {
await this.storageAdapter.removeUniquenessKey(cloudTask.uniquenessKey);
}
}
else {
// Update attempts but keep as idle for retry
await this.storageAdapter.updateTaskStatus(taskId, storage_adapter_interface_1.TaskStatus.IDLE, {
attempts: cloudTask.attempts,
updatedAt: cloudTask.updatedAt,
});
}
// Emit failed event only on final attempt
if (isLastAttempt) {
const failedEvent = {
taskId: cloudTask.id,
queueName: cloudTask.queueName,
data: cloudTask.data,
error: {
message: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined,
},
attempts: cloudTask.attempts,
maxAttempts: cloudTask.maxAttempts,
isFinalAttempt: isLastAttempt,
timestamp: new Date(),
};
await this.emitTaskEvent('failed', processors, cloudTask, error);
this.emit('taskFailed', failedEvent);
}
if (isLastAttempt) {
throw error; // Re-throw for final attempt
}
else {
throw error; // Cloud Tasks will retry
}
}
finally {
// Remove from active processors
activeProcessors.delete(taskId);
}
}
/**
* Update task progress
*/
async updateTaskProgress(taskId, progress) {
const task = await this.storageAdapter.getTask(taskId);
if (!task) {
throw new Error(`Task ${taskId} not found`);
}
const cloudTask = new cloud_task_model_1.CloudTask(task);
cloudTask.updateProgress(progress);
await this.storageAdapter.updateTaskStatus(taskId, task.status, {
progress: cloudTask.progress,
updatedAt: cloudTask.updatedAt,
});
// Emit progress event
const progressEvent = {
taskId: cloudTask.id,
queueName: cloudTask.queueName,
data: cloudTask.data,
progress,
timestamp: new Date(),
};
const processors = this.processors.get(cloudTask.queueName);
if (processors) {
await this.emitTaskEvent('progress', processors, cloudTask, progress);
}
this.emit('taskProgress', progressEvent);
}
/**
* Get registered processors
*/
getProcessors() {
return new Map(this.processors);
}
/**
* Execute task processing using registered processors
*/
async executeTaskProcessing(processors, cloudTask) {
// Find the correct processor and handler based on taskName
const taskName = cloudTask.options?.taskName;
// Create a task wrapper that delegates updateProgress to emit events
const taskWrapper = {
...cloudTask,
updateProgress: async (progress) => {
await this.updateTaskProgress(cloudTask.id, progress);
// Also update the local CloudTask instance
cloudTask.updateProgress(progress);
}
};
for (const processor of processors) {
// Look for a handler that matches the task name
const processHandler = processor.processHandlers.find(handler => handler.name === taskName);
if (processHandler) {
const boundMethod = processHandler.handler.bind(processor.instance);
return await boundMethod(taskWrapper);
}
}
// If no specific handler found, try to use the first handler without a name (default handler)
for (const processor of processors) {
const defaultHandler = processor.processHandlers.find(handler => !handler.name);
if (defaultHandler) {
const boundMethod = defaultHandler.handler.bind(processor.instance);
return await boundMethod(taskWrapper);
}
}
// If still no handler found, use the first available handler as fallback
const processor = processors[0];
const processHandler = processor.processHandlers[0];
if (!processHandler) {
throw new Error(`No process handlers available for queue "${cloudTask.queueName}"`);
}
const boundMethod = processHandler.handler.bind(processor.instance);
return await boundMethod(taskWrapper);
}
/**
* Emit task events to registered event handlers
*/
async emitTaskEvent(eventType, processors, cloudTask, additionalData) {
for (const processor of processors) {
const eventHandlers = processor.eventHandlers.filter(h => h.event === eventType);
for (const handler of eventHandlers) {
try {
const boundMethod = handler.handler.bind(processor.instance);
await boundMethod(cloudTask, additionalData);
}
catch (error) {
console.error(`Error in ${eventType} event handler:`, error);
}
}
}
}
/**
* Process next task in chain
*/
async processNextInChain(cloudTask) {
if (!cloudTask.chain)
return;
const nextIndex = cloudTask.getNextChainIndex();
if (nextIndex === null)
return;
// Check if next task exists
const nextTaskId = `${cloudTask.chain.id}-${nextIndex}`;
const nextTask = await this.storageAdapter.getTask(nextTaskId);
if (nextTask && nextTask.status === storage_adapter_interface_1.TaskStatus.IDLE) {
// Trigger next task processing (this would normally be done by Cloud Tasks)
console.log(`Chain ${cloudTask.chain.id}: Triggering next task ${nextIndex}`);
}
}
/**
* Close the consumer service
*/
async close() {
this.processors.clear();
this.activeProcessors.clear();
this.removeAllListeners();
}
}
exports.ConsumerService = ConsumerService;