@vorsteh-queue/core
Version:
Core queue engine for Vorsteh Queue with TypeScript support, job scheduling, and event system
634 lines (629 loc) • 18.1 kB
JavaScript
import { BaseQueueAdapter, MemoryQueueAdapter, serializeError } from "./memory-BjE6rfBi.js";
import { TZDate } from "@date-fns/tz";
import { Cron } from "croner";
//#region src/utils/helpers.ts
/**
* Source: https://github.com/mgcrea/prisma-queue/blob/master/src/utils/time.ts
* License: MIT
* Copyright (c) 2022 Olivier Louvignes <olivier@mgcrea.io>
*/
const waitFor = async (ms) => new Promise((resolve) => {
setTimeout(resolve, ms);
});
const calculateDelay = (attempts) => Math.min(1e3 * Math.pow(2, Math.max(1, attempts)) + Math.random() * 100, Math.pow(2, 31) - 1);
//#endregion
//#region src/utils/scheduler.ts
/**
* Parse a cron expression in a timezone and return the next execution time as UTC.
* This is the core of our UTC-first approach - all timezone calculations happen here,
* and the result is always a UTC timestamp for storage and processing.
*
* @param expression Cron expression (e.g., "0 9 * * *")
* @param timezone IANA timezone (e.g. `America/New_York`)
* @param baseDate Base date for calculation (defaults to current time)
* @returns Next execution time as UTC Date
*
* @example
* ```typescript
* // 9 AM daily in New York timezone -> returns UTC timestamp
* const nextRun = parseCron("0 9 * * *", "America/New_York")
* // Result is always UTC, ready for database storage
* ```
*/
const parseCron = (expression, timezone = "UTC", baseDate = /* @__PURE__ */ new Date()) => {
try {
const cron = new Cron(expression, { timezone });
const nextRun = cron.nextRun(baseDate);
if (!nextRun) throw new Error(`No next run found for cron: ${expression}`);
if (timezone !== "UTC") {
const tzDate = new TZDate(nextRun, timezone);
return new Date(tzDate.getTime());
}
return new TZDate(nextRun.getTime(), "UTC");
} catch (error) {
throw new Error(`Invalid cron expression: ${expression}`);
}
};
/**
* Calculate the next run time for a job - always returns UTC.
* For cron jobs, timezone conversion happens here and result is UTC.
* For intervals, timezone is irrelevant (just add milliseconds).
*
* @param options Configuration object
* @returns Next execution time as UTC Date
*
* @example
* ```typescript
* // Cron: timezone conversion happens here, result is UTC
* const nextRun = calculateNextRun({
* cron: "0 9 * * *",
* timezone: "America/New_York",
* lastRun: new Date()
* })
*
* // Interval: timezone irrelevant, just add milliseconds
* const nextRun = calculateNextRun({
* repeatEvery: 3600000,
* lastRun: new Date()
* })
* ```
*/
const calculateNextRun = (options) => {
const { cron, repeatEvery, timezone = "UTC", lastRun } = options;
if (cron) return parseCron(cron, timezone, lastRun);
if (repeatEvery) return new Date(lastRun.getTime() + repeatEvery);
throw new Error("Either cron or repeatEvery must be provided");
};
/**
* Convert a local date to UTC by interpreting it in a specific timezone.
* Used when users provide local times that need timezone context.
*
* @param date Input date to convert
* @param timezone IANA timezone to interpret the date in
* @returns UTC Date
*
* @example
* ```typescript
* // User says "run at 9 AM" in New York - convert to UTC for storage
* const utcDate = toUtcDate(new Date("2024-01-15T09:00:00"), "America/New_York")
* ```
*/
const toUtcDate = (date, timezone = "UTC") => {
if (timezone === "UTC") return date;
const tzDate = new TZDate(date, timezone);
return new Date(tzDate.getTime());
};
/**
* Create a UTC Date object.
*
* @param input Input date to convert
* @returns UTC Date
*
* @example
* ```typescript
* // Create a UTC Date object
* const utcDate = asUtc(new Date("2024-01-15T09:00:00"))
* ```
*/
function asUtc(input) {
return new TZDate(input, "UTC");
}
//#endregion
//#region src/core/job-wrapper.ts
/**
* @internal
*/
function createJobWrapper(job, queue) {
return {
...job,
updateProgress: async (value) => {
await queue.updateJobProgress(job.id, value);
}
};
}
//#endregion
//#region src/core/queue.ts
/**
* Main queue class for managing job processing.
*
* @example
* ```typescript
* const queue = new Queue(adapter, { name: "my-queue" })
* queue.register("send-email", async (payload) => { ... })
* await queue.add("send-email", { to: "user@example.com" })
* queue.start()
* ```
*/
var Queue = class {
adapter;
config;
handlers = /* @__PURE__ */ new Map();
listeners = /* @__PURE__ */ new Map();
isRunning = false;
isPaused = false;
activeJobs = 0;
stopped = false;
constructor(adapter, config) {
this.adapter = adapter;
this.config = {
concurrency: 1,
defaultJobOptions: {
priority: 2,
maxAttempts: 3,
timeout: 3e4
},
retryDelay: 1e3,
maxRetryDelay: 3e4,
removeOnComplete: 100,
removeOnFail: 50,
pollInterval: 100,
jobInterval: 10,
...config
};
if ("setQueueName" in adapter && typeof adapter.setQueueName === "function") adapter.setQueueName(this.config.name);
}
async connect() {
await this.adapter.connect();
}
async disconnect() {
await this.stop();
await this.adapter.disconnect();
}
/**
* Register a job handler for a specific job type.
*
* @param name The job type name
* @param handler Function to process jobs of this type
*
* @example
* ```typescript
* queue.register("send-email", async (payload: { to: string }) => {
* // Send email logic
* return { sent: true }
* })
* ```
*/
register(name, handler) {
this.handlers.set(name, handler);
}
/**
* Add a new job to the queue.
*
* @param name The job type name (must be registered)
* @param payload Job data to process
* @param options Job configuration options
* @returns Promise resolving to the created job
*
* @example
* ```typescript
* // Basic job
* await queue.add("send-email", { to: "user@example.com" })
*
* // High priority job with delay
* await queue.add("urgent-task", { data: "important" }, {
* priority: 1,
* delay: 5000
* })
*
* // Recurring job
* await queue.add("cleanup", {}, {
* cron: "0 2 * * *" // Daily at 2 AM
* })
* ```
*/
async add(name, payload, options = {}) {
const jobOptions = {
...this.config.defaultJobOptions,
...options
};
const timezone = jobOptions.timezone ?? "UTC";
const now = /* @__PURE__ */ new Date();
let processAt;
let status = "pending";
if (jobOptions.runAt) {
processAt = toUtcDate(jobOptions.runAt, timezone);
status = processAt > now ? "delayed" : "pending";
} else if (jobOptions.delay) {
processAt = asUtc(new Date(now.getTime() + jobOptions.delay));
status = "delayed";
} else if (jobOptions.cron) {
processAt = parseCron(jobOptions.cron, timezone, now);
status = "delayed";
} else processAt = asUtc(now);
const job = await this.adapter.addJob({
name,
payload,
status,
priority: jobOptions.priority ?? 2,
attempts: 0,
maxAttempts: jobOptions.maxAttempts ?? 3,
processAt,
cron: jobOptions.cron,
repeatEvery: jobOptions.repeat?.every,
repeatLimit: jobOptions.repeat?.limit,
repeatCount: 0,
timeout: jobOptions.timeout
});
this.emit("job:added", job);
return job;
}
/**
* Alias for the `add()` method. Add a new job to the queue.
*
* @param name The job type name (must be registered)
* @param payload Job data to process
* @param options Job configuration options
* @returns Promise resolving to the created job
*
* @example
* ```typescript
* // Basic job
* await queue.enqueue("send-email", { to: "user@example.com" })
*
* // Priority job
* await queue.enqueue("urgent-task", { data: "important" }, { priority: 1 })
* ```
*/
async enqueue(name, payload, options = {}) {
return this.add(name, payload, options);
}
/**
* Start processing jobs from the queue.
* Jobs will be processed according to priority and concurrency settings.
*/
start() {
if (this.isRunning) return;
this.isRunning = true;
this.isPaused = false;
this.stopped = false;
this.poll();
}
/**
* Stop the queue and wait for active jobs to complete.
* Provides graceful shutdown functionality.
*/
async stop() {
this.isRunning = false;
this.stopped = true;
while (this.activeJobs > 0) await waitFor(100);
this.emit("queue:stopped", void 0);
}
/**
* Pause job processing. Jobs can still be added but won't be processed.
*/
pause() {
this.isPaused = true;
this.emit("queue:paused", void 0);
}
/**
* Resume job processing after being paused.
*/
resume() {
this.isPaused = false;
this.emit("queue:resumed", void 0);
}
/**
* Get current queue statistics.
*
* @returns Promise<QueueStats> resolving to queue statistics
*
* @example
* ```typescript
* const stats = await queue.getStats()
* console.log(`Pending: ${stats.pending}, Processing: ${stats.processing}`)
* ```
*/
async getStats() {
return this.adapter.getQueueStats();
}
/**
* Get current queue configuration.
*
* @returns Queue configuration object
*
* @example
* ```typescript
* const config = queue.getConfig()
* console.log(`Queue: ${config.name}, Concurrency: ${config.concurrency}`)
* ```
*/
getConfig() {
return { ...this.config };
}
/**
* Clear jobs from the queue.
*
* @param status Optional job status filter to clear only jobs with specific status
* @returns Promise<number> resolving to number of jobs cleared
*
* @example
* ```typescript
* // Clear all jobs
* const cleared = await queue.clear()
*
* // Clear only failed jobs
* const clearedFailed = await queue.clear("failed")
* ```
*/
async clear(status) {
return this.adapter.clearJobs(status);
}
/**
* Update the progress of a specific job.
*
* @param id The job ID to update progress for
* @param progress Progress percentage (0-100)
* @returns Promise that resolves when progress is updated
*
* @example
* ```typescript
* // Update job progress to 50%
* await queue.updateJobProgress(jobId, 50)
*
* // Update job progress to completion
* await queue.updateJobProgress(jobId, 100)
* ```
*/
async updateJobProgress(id, progress) {
await this.adapter.updateJobProgress(id, progress);
}
/**
* Manually dequeue and mark the next job as completed without processing.
* This is primarily used for testing or manual job management.
*
* @returns Promise<BaseJob | null> resolving to the dequeued job or null if no jobs available
*
* @example
* ```typescript
* // Manually dequeue next job
* const job = await queue.dequeue()
* if (job) {
* console.log(`Dequeued job: ${job.name}`)
* }
* ```
*/
async dequeue() {
const job = await this.adapter.getNextJob();
if (job) await this.adapter.updateJobStatus(job.id, "completed");
return job;
}
/**
* Listen to queue events.
*
* @param event Event name to listen for
* @param listener Event handler function
*
* @example
* ```typescript
* queue.on("job:completed", (job) => {
* console.log(`Job ${job.name} completed successfully`)
* })
*
* queue.on("job:failed", (job) => {
* console.error(`Job ${job.name} failed: ${job.error}`)
* })
* ```
*/
on(event, listener) {
if (!this.listeners.has(event)) this.listeners.set(event, []);
this.listeners.get(event)?.push(listener);
}
/**
* Internal method to emit queue events to registered listeners.
*
* @param event Event name to emit
* @param data Event data to pass to listeners
* @private
*
* @example
* ```typescript
* this.emit("job:completed", completedJobData)
* this.emit("queue:error", errorData)
* ```
*/
emit(event, data) {
const eventListeners = this.listeners.get(event) ?? [];
for (const listener of eventListeners) listener(data);
}
/**
* Internal polling method that continuously checks for and processes jobs.
* Handles concurrency limits and processing intervals.
*
* @returns Promise that resolves when polling is stopped
* @private
*/
async poll() {
const { concurrency, pollInterval, jobInterval } = this.config;
while (!this.stopped) {
if (this.isPaused || this.activeJobs >= concurrency) {
await waitFor(pollInterval);
continue;
}
let queueSize = await this.adapter.size();
if (queueSize === 0) {
await waitFor(pollInterval);
continue;
}
while (queueSize > 0) {
while (queueSize > 0 && this.activeJobs < concurrency) {
this.activeJobs++;
setImmediate(() => {
this.dequeueAndProcess().then((processed) => {
if (!processed) queueSize = 0;
else queueSize--;
}).catch((error) => {
this.emit("queue:error", error);
}).finally(() => {
this.activeJobs--;
});
});
await waitFor(jobInterval);
}
await waitFor(jobInterval * 2);
}
}
}
/**
* Internal method to dequeue and process a single job.
*
* @returns Promise resolving to boolean indicating if job was processed
* @private
*/
async dequeueAndProcess() {
const job = await this.adapter.getNextJob();
if (!job) return false;
await this.processJob(job);
return true;
}
/**
* Internal method to process a job using its registered handler.
* Handles job timeouts, completion, cleanup and recurring job scheduling.
*
* @param job The job to process
* @returns Promise that resolves when job processing is complete
* @private
*/
async processJob(job) {
const handler = this.handlers.get(job.name);
if (!handler) {
await this.failJob(job, serializeError(/* @__PURE__ */ new Error(`No handler registered for job: ${job.name}`)));
return;
}
await this.adapter.updateJobStatus(job.id, "processing");
this.emit("job:processing", {
...job,
status: "processing"
});
try {
const timeout = job.timeout ?? this.config.defaultJobOptions.timeout ?? 3e4;
const result = timeout === false ? await handler(createJobWrapper(job, this)) : await Promise.race([handler(createJobWrapper(job, this)), new Promise((_, reject) => setTimeout(() => reject(/* @__PURE__ */ new Error("Job timeout")), timeout))]);
await this.adapter.updateJobStatus(job.id, "completed", void 0, result);
this.emit("job:completed", {
...job,
status: "completed",
completedAt: /* @__PURE__ */ new Date(),
result
});
await this.cleanupCompletedJob();
this.handleRecurringJob(job).catch(() => {});
} catch (error) {
await this.handleJobError(job, error);
}
}
/**
* Internal method to handle job errors.
* Determines whether to retry or fail the job based on attempt count.
*
* @param job The failed job
* @param error The error that occurred
* @returns Promise that resolves when error handling is complete
* @private
*/
async handleJobError(job, error) {
const serializedError = serializeError(error);
if (job.attempts + 1 < job.maxAttempts) await this.retryJob(job, serializedError);
else await this.failJob(job, serializedError);
}
/**
* Internal method to retry a failed job.
* Increments attempt count and schedules next attempt with backoff delay.
*
* @param job The job to retry
* @param _error The error from the failed attempt
* @returns Promise that resolves when job is scheduled for retry
* @private
*/
async retryJob(job, _error) {
await this.adapter.incrementJobAttempts(job.id);
const delay = Math.min(calculateDelay(job.attempts), this.config.maxRetryDelay);
const processAt = new Date(Date.now() + delay);
await this.adapter.updateJobStatus(job.id, "pending");
this.emit("job:retried", {
...job,
attempts: job.attempts + 1,
processAt
});
}
/**
* Internal method to mark a job as permanently failed.
* Updates job status and triggers cleanup of failed jobs.
*
* @param job The job to fail
* @param error The error that caused the failure
* @returns Promise that resolves when job is marked as failed
* @private
*/
async failJob(job, error) {
await this.adapter.updateJobStatus(job.id, "failed", error);
this.emit("job:failed", {
...job,
status: "failed",
error,
failedAt: /* @__PURE__ */ new Date()
});
await this.cleanupFailedJob();
}
/**
* Internal method to clean up completed jobs based on queue configuration.
* Can remove jobs immediately, keep all jobs, or maintain a fixed number of completed jobs.
*
* @returns Promise that resolves when cleanup is complete
* @private
*/
async cleanupCompletedJob() {
const { removeOnComplete } = this.config;
if (removeOnComplete === true) await this.adapter.clearJobs("completed");
else if (removeOnComplete === false) return;
else if (typeof removeOnComplete === "number" && removeOnComplete > 0) await this.adapter.cleanupJobs("completed", removeOnComplete);
}
/**
* Internal method to clean up failed jobs based on queue configuration.
* Can remove jobs immediately, keep all jobs, or maintain a fixed number of failed jobs.
*
* @returns Promise that resolves when cleanup is complete
* @private
*/
async cleanupFailedJob() {
const { removeOnFail } = this.config;
if (removeOnFail === true) await this.adapter.clearJobs("failed");
else if (removeOnFail === false) return;
else if (typeof removeOnFail === "number" && removeOnFail > 0) await this.adapter.cleanupJobs("failed", removeOnFail);
}
/**
* Internal method to handle recurring job scheduling.
* Creates the next occurrence of recurring jobs based on cron or repeat interval.
*
* @param job The completed job that may need to be rescheduled
* @param originalTimezone Optional timezone for cron scheduling
* @returns Promise that resolves when next job is scheduled
* @private
*/
async handleRecurringJob(job, originalTimezone) {
if (!job.cron && !job.repeatEvery) return;
const nextCount = (job.repeatCount ?? 0) + 1;
if (job.repeatLimit && nextCount >= job.repeatLimit) return;
try {
const nextRun = calculateNextRun({
cron: job.cron,
repeatEvery: job.repeatEvery,
timezone: originalTimezone ?? "UTC",
lastRun: /* @__PURE__ */ new Date()
});
await this.adapter.addJob({
name: job.name,
payload: job.payload,
status: "delayed",
priority: job.priority,
attempts: 0,
maxAttempts: job.maxAttempts,
processAt: nextRun,
cron: job.cron,
repeatEvery: job.repeatEvery,
repeatLimit: job.repeatLimit,
repeatCount: nextCount
});
} catch {}
}
};
//#endregion
export { BaseQueueAdapter, MemoryQueueAdapter, Queue, asUtc, serializeError };