queuex-sdk
Version:
A TypeScript-based queue management SDK with Redis support
323 lines (322 loc) • 12.5 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.QueueManager = void 0;
const redis_1 = require("../storage/redis");
const job_1 = require("../models/job");
const queue_1 = require("../models/queue");
const uuid_1 = require("uuid");
const cron_parser_1 = require("../utils/cron-parser");
/**
* Manages queues and job scheduling with dependency resolution, cron support, and job chaining.
*
* Features:
* - Queue management and job scheduling
* - Dependency resolution between jobs
* - Cron-based job scheduling
* - Job chaining with context passing
* - Concurrent job processing
*
* Example:
* ```typescript
* // Create a queue manager
* const queueManager = new QueueManager(redisUrl, emitEvent);
*
* // Create a queue
* await queueManager.createQueue('myQueue', { maxConcurrency: 5 });
*
* // Enqueue a job with chaining
* await queueManager.enqueue('myQueue', initialData, {
* chain: [
* {
* data: { step: 1 },
* options: { priority: 'high' }
* },
* {
* data: { step: 2 },
* options: { retries: 5 }
* }
* ]
* });
* ```
*/
class QueueManager {
constructor(redisConnectionString, emitEvent) {
this.queues = new Map();
this.dependencyGraph = new Map();
this.schedulerInterval = null;
this.storage = new redis_1.RedisStorage(redisConnectionString);
this.emitEvent = emitEvent;
this.startScheduler().catch((err) => console.error('Scheduler startup failed:', err));
this.processOverdueJobs().catch((err) => console.error('Overdue job processing failed:', err));
}
/**
* Creates a new queue with the specified options.
* @param name - Name of the queue
* @param options - Queue configuration options
* @throws Error if queue already exists
*/
async createQueue(name, options = {}) {
if (this.queues.has(name)) {
throw new Error(`Queue ${name} already exists`);
}
const queue = {
name,
options: {
maxConcurrency: 1,
...options,
},
};
this.queues.set(name, queue);
}
/**
* Enqueues a job with optional chaining and dependencies.
*
* Job Chaining:
* - The chain option allows defining subsequent jobs that will execute after the current job
* - Each job in the chain receives the result of the previous job as context
* - Chain jobs inherit the queue of the parent job
*
* Dependencies:
* - Jobs can depend on other jobs using dependsOn
* - A job will only start when all its dependencies are completed
*
* @param queueName - Name of the queue to enqueue the job in
* @param data - Job data payload
* @param options - Job configuration including chain and dependencies
* @returns The created job
* @throws Error if queue doesn't exist or if dependency jobs don't exist
*/
async enqueue(queueName, data, options = {}) {
var _a;
const queue = this.queues.get(queueName);
if (!queue)
throw new Error(`Queue ${queueName} not found`);
const job = {
id: (0, uuid_1.v4)(),
queue: queueName,
data,
state: this.getInitialState(options),
options: {
priority: 'medium',
retries: 3,
...options,
chain: undefined, // Remove chain from options to prevent it from being passed to child jobs
},
attempts: 0,
createdAt: Date.now(),
scheduledAt: this.getScheduledAt(options),
logs: [],
dependencies: options.dependsOn || [],
dependents: [],
expiresAt: options.ttl ? Date.now() + options.ttl : undefined,
};
// Validate TTL and delay combination
if (job.expiresAt && job.scheduledAt && job.scheduledAt > job.expiresAt) {
throw new Error('Job TTL cannot be less than the scheduled delay');
}
this.dependencyGraph.set(job.id, job);
if (job.dependencies) {
for (const depId of job.dependencies) {
const depJob = this.dependencyGraph.get(depId);
if (depJob) {
depJob.dependents = depJob.dependents || [];
depJob.dependents.push(job.id);
this.dependencyGraph.set(depId, depJob);
}
else {
throw new Error(`Dependency job ${depId} not found`);
}
}
}
try {
await this.storage.enqueueJob(job, queue.options.strategy || queue_1.QueueStrategy.FIFO);
if (options.cron || options.delay)
this.emitEvent('jobDelayed', job);
else if ((_a = options.dependsOn) === null || _a === void 0 ? void 0 : _a.length)
this.emitEvent('jobPending', job);
}
catch (err) {
throw new Error(`Failed to enqueue job ${job.id}: ${err instanceof Error ? err.message : String(err)}`);
}
return job;
}
/**
* Gets the next available job from the queue.
* Handles dependency resolution and job state transitions.
*
* @param queueName - Name of the queue to get job from
* @returns The next available job or null if no jobs are available
*/
async getJob(queueName) {
var _a;
try {
const queue = this.queues.get(queueName);
if (!queue)
throw new Error(`Queue ${queueName} not found`);
const job = await this.storage.getNextJob(queueName, queue.options.strategy || queue_1.QueueStrategy.FIFO);
if (!job)
return null;
if ((_a = job.dependencies) === null || _a === void 0 ? void 0 : _a.length) {
const allDepsCompleted = job.dependencies.every((depId) => {
const depJob = this.dependencyGraph.get(depId);
return (depJob === null || depJob === void 0 ? void 0 : depJob.state) === job_1.JobState.COMPLETED;
});
if (!allDepsCompleted) {
job.state = job_1.JobState.PENDING;
await this.storage.enqueueJob(job, queue.options.strategy || queue_1.QueueStrategy.FIFO);
this.emitEvent('jobPending', job);
return null;
}
}
return job;
}
catch (err) {
throw new Error(`Failed to get job from queue ${queueName}: ${err instanceof Error ? err.message : String(err)}`);
}
}
/**
* Retrieves a job by its ID from the dependency graph.
* @param jobId - ID of the job to retrieve
* @returns The job if found, undefined otherwise
*/
getJobById(jobId) {
return this.dependencyGraph.get(jobId);
}
/**
* Unschedules a delayed job and marks it as failed.
* @param jobId - ID of the job to unschedule
* @throws Error if job doesn't exist or is not in DELAYED state
*/
async unscheduleJob(jobId) {
const job = this.dependencyGraph.get(jobId);
if (!job)
throw new Error(`Job ${jobId} not found`);
if (job.state !== job_1.JobState.DELAYED)
throw new Error(`Job ${jobId} is not in DELAYED state`);
try {
await this.storage.removeScheduledJob(job.queue, jobId);
this.dependencyGraph.delete(jobId);
this.emitEvent('jobFailed', {
...job,
state: job_1.JobState.FAILED,
logs: [...job.logs, 'Job unscheduled'],
});
}
catch (err) {
throw new Error(`Failed to unschedule job ${jobId}: ${err instanceof Error ? err.message : String(err)}`);
}
}
/**
* Updates the delay of a delayed or pending job.
* @param jobId - ID of the job to update
* @param newDelay - New delay in milliseconds
* @returns The updated job
* @throws Error if job doesn't exist or is not in DELAYED/PENDING state
*/
async updateJobDelay(jobId, newDelay) {
const job = this.dependencyGraph.get(jobId);
if (!job)
throw new Error(`Job ${jobId} not found`);
if (![job_1.JobState.DELAYED, job_1.JobState.PENDING].includes(job.state)) {
throw new Error(`Cannot update delay for job ${jobId} in state ${job.state}`);
}
try {
job.options.delay = newDelay;
delete job.options.cron;
job.scheduledAt = Date.now() + newDelay;
job.state = job_1.JobState.DELAYED;
job.logs.push(`Delay updated to ${newDelay}ms at ${Date.now()}`);
await this.storage.enqueueJob(job);
this.dependencyGraph.set(jobId, job);
this.emitEvent('jobDelayed', job);
return job;
}
catch (err) {
throw new Error(`Failed to update delay for job ${jobId}: ${err instanceof Error ? err.message : String(err)}`);
}
}
/**
* Determines the initial state of a job based on its options.
* @param options - Job options
* @returns The initial job state
*/
getInitialState(options) {
var _a;
if ((_a = options.dependsOn) === null || _a === void 0 ? void 0 : _a.length)
return job_1.JobState.PENDING;
if (options.delay || options.cron)
return job_1.JobState.DELAYED;
return job_1.JobState.WAITING;
}
/**
* Calculates the scheduled execution time for a job based on its options.
* @param options - Job options
* @returns The scheduled timestamp or undefined if not scheduled
* @throws Error if cron expression is invalid
*/
getScheduledAt(options) {
if (options.delay)
return Date.now() + options.delay;
if (options.cron) {
try {
const parser = new cron_parser_1.CronParser(options.cron);
return parser.next().getTime();
}
catch (err) {
throw new Error(`Invalid cron expression: ${options.cron} - ${err instanceof Error ? err.message : String(err)}`);
}
}
return undefined;
}
/**
* Processes overdue jobs from all queues.
* Handles both delayed and cron jobs.
*/
async processOverdueJobs() {
for (const queue of this.queues.values()) {
const now = Date.now();
const scheduledJobs = await this.storage.getScheduledJobs(queue.name, now);
for (const jobData of scheduledJobs) {
const job = JSON.parse(jobData);
if (job.state === job_1.JobState.DELAYED && job.scheduledAt <= now) {
job.state = job_1.JobState.WAITING;
await this.storage.enqueueJob(job);
await this.storage.removeScheduledJob(queue.name, job.id);
// Reschedule cron jobs
if (job.options.cron) {
const nextJob = {
...job,
id: (0, uuid_1.v4)(),
state: job_1.JobState.DELAYED,
scheduledAt: this.getScheduledAt(job.options),
attempts: 0,
logs: [],
};
await this.storage.enqueueJob(nextJob);
this.dependencyGraph.set(nextJob.id, nextJob);
this.emitEvent('jobDelayed', nextJob);
}
this.emitEvent('jobReady', job);
this.dependencyGraph.set(job.id, job);
}
}
}
}
async startScheduler() {
this.schedulerInterval = setInterval(async () => {
try {
await this.processOverdueJobs();
}
catch (err) {
console.error('Scheduler iteration failed:', err);
}
}, 1000);
}
stopScheduler() {
if (this.schedulerInterval) {
clearInterval(this.schedulerInterval);
this.schedulerInterval = null;
}
}
}
exports.QueueManager = QueueManager;