@backgroundjs/core
Version:
An extendible background job queue for js/ts applications
607 lines • 24.1 kB
JavaScript
import { generateId } from "../utils/id-generator.js";
import { QueueEvent } from "../utils/queue-event.js";
export class JobQueue extends EventTarget {
/**
* Job handlers registered with this queue
*/
handlers = new Map();
storage;
/**
* Set of job IDs that are currently being processed
*/
activeJobs = new Set();
/**
* Buffer that is filled based on prefetchBatchSize
*/
jobBuffer = [];
preFetchBatchSize;
/**
* Number of jobs that can be processed concurrently
*/
concurrency;
/**
* Interval in milliseconds at which to check for new jobs
*/
processing = false;
processingInterval = 1000; // 1 second
intervalId = null; // For Universal JS
name;
maxRetries = 3;
logging = false;
lastPollingInterval = 0;
pollingErrorCount = 0;
// Intelligent polling properties
intelligentPolling = false;
minInterval = 100; // Minimum polling interval (ms)
maxInterval = 5000; // Maximum polling interval (ms)
emptyPollsCount = 0;
maxEmptyPolls = 5; // Number of empty polls before increasing interval
loadFactor = 0.5; // Target load factor (0.0 to 1.0)
maxConcurrency = 10; // Maximum number of jobs that can be processed concurrently
// Stopping properties
isStopping = false;
isUpdatingInterval = false;
isStopped = false;
constructor(storage, options = {}) {
super();
this.storage = storage;
this.concurrency = options.concurrency || 1;
this.name = options.name || "default";
this.processingInterval = options.processingInterval || 1000;
this.maxRetries = options.maxRetries || 3;
this.logging = options.logging || false;
this.lastPollingInterval = this.processingInterval;
this.pollingErrorCount = 0;
this.preFetchBatchSize = options.preFetchBatchSize;
// Intelligent polling configuration
this.intelligentPolling = options.intelligentPolling || false;
if (this.intelligentPolling) {
this.minInterval = options.minInterval || 100;
this.maxInterval = options.maxInterval || 5000;
this.maxEmptyPolls = options.maxEmptyPolls || 5;
this.loadFactor = options.loadFactor || 0.5;
this.maxConcurrency = options.maxConcurrency || 10;
}
}
// Register a job handler
register(name, handler) {
this.handlers.set(name, handler);
}
// Add a job to the queue
async add(name, data, options) {
if (this.isStopped) {
throw new Error("Queue is stopped");
}
const priority = options?.priority || 1;
const job = {
id: generateId(),
name,
data,
status: "pending",
createdAt: new Date(),
priority,
timeout: options?.timeout || 10000,
};
if (this.logging) {
console.log(`[${this.name}] Scheduled job ${job.id} to run at ${job.createdAt}`);
}
await this.storage.saveJob(job);
this.dispatchEvent(new QueueEvent("scheduled", { job, status: "pending" }));
return job;
}
// Schedule a job to run at a specific time
async schedule(name, data, scheduledAt, options) {
if (this.isStopped) {
throw new Error("Queue is stopped");
}
if (scheduledAt < new Date()) {
throw new Error("Scheduled time must be in the future");
}
const job = {
id: generateId(),
name,
data,
status: "pending",
createdAt: new Date(),
scheduledAt,
timeout: options?.timeout || 10000,
};
if (this.logging) {
console.log(`[${this.name}] Scheduled job ${job.id} to run at ${scheduledAt}`);
}
await this.storage.saveJob(job);
this.dispatchEvent(new QueueEvent("scheduled", { job, status: "pending" }));
return job;
}
// Schedule a job to run after a delay (in milliseconds)
async scheduleIn(name, data, delayMs) {
const scheduledAt = new Date(Date.now() + delayMs);
return this.schedule(name, data, scheduledAt);
}
// Get a job by ID
async getJob(id) {
return this.storage.getJob(id);
}
// Get the name of the queue
getName() {
return this.name;
}
// Start processing jobs
start() {
if (this.processing || this.isStopping)
return;
if (this.logging) {
console.log(`[${this.name}] Starting job queue`);
}
this.processing = true;
this.intervalId = setInterval(() => this.processNextBatch(), this.processingInterval);
}
async rollbackActiveJobs() {
if (this.logging) {
console.log(`[${this.name}] Rolling back ${this.activeJobs.size} active jobs`);
}
const activeJobIds = [...this.activeJobs];
for (const jobId of activeJobIds) {
try {
const job = await this.storage.getJob(jobId);
if (job) {
job.status = "pending";
await this.storage.updateJob(job);
if (this.logging) {
console.log(`[${this.name}] Rolled back job ${jobId}`);
}
}
else {
if (this.logging) {
console.log(`[${this.name}] Could not find job ${jobId} to roll back`);
}
}
}
catch (error) {
if (this.logging) {
console.error(`[${this.name}] Error rolling back job ${jobId}:`, error);
}
}
}
}
// Stop processing jobs
async stop() {
if (!this.processing)
return;
this.isStopping = true;
if (this.logging) {
console.log(`[${this.name}] Stopping job queue`);
}
if (this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = undefined;
}
if (this.logging) {
console.log(`[${this.name}] Job queue stopped`);
}
for (const job of this.jobBuffer) {
job.status = "pending";
job.startedAt = undefined;
await this.storage.updateJob(job);
}
this.jobBuffer.length = 0;
this.processing = false;
this.isStopping = false;
this.isStopped = true;
}
// Set processing interval
setProcessingInterval(ms) {
this.processingInterval = ms;
if (this.processing && this.intervalId) {
clearInterval(this.intervalId);
this.intervalId = setInterval(() => this.processNextBatch(), this.processingInterval);
}
}
// Set concurrency level
setConcurrency(level) {
if (level < 1) {
throw new Error("Concurrency level must be at least 1");
}
this.concurrency = level;
}
// Process the next batch of pending jobs
/**
* Process jobs with prefetching
* Override the parent's protected method
*/
async processNextBatch() {
try {
if (this.isStopping && this.logging) {
console.log(`[${this.name}] Stopping job queue ... skipping`);
return;
}
if (this.activeJobs.size >= this.concurrency || this.isStopping) {
return;
}
if (this.preFetchBatchSize) {
await this.refillJobBuffer();
}
const availableSlots = this.concurrency - this.activeJobs.size;
let jobsProcessed = 0;
if (this.preFetchBatchSize) {
// Process jobs from buffer
for (let i = 0; i < availableSlots && this.jobBuffer.length > 0; i++) {
const job = this.jobBuffer.shift();
if (this.logging) {
console.log(`[${this.name}] Processing prefetched job:`, job.id);
}
if (!this.handlers.has(job.name)) {
if (this.logging) {
console.log(`[${this.name}] Job with no handler found: ${job.id}`);
console.log(`[${this.name}] Resetting Job status...`);
}
job.status = "pending";
job.startedAt = undefined;
this.storage.updateJob(job)
.then(() => {
if (this.logging) {
console.log(`[${this.name}] Job status reset: ${job.id}`);
}
})
.catch((error) => {
if (this.logging) {
console.error("Error resetting job status", error);
}
});
this.activeJobs.delete(job.id);
continue;
}
this.activeJobs.add(job.id);
this.processJob(job)
.catch((error) => {
if (this.logging) {
console.error("Error processing job", error);
}
})
.finally(() => {
this.activeJobs.delete(job.id);
});
jobsProcessed++;
}
}
else {
for (let i = 0; i < availableSlots; i++) {
const job = await this.storage.acquireNextJob();
if (!job) {
break;
}
if (!this.handlers.has(job.name)) {
if (this.logging) {
console.log(`[${this.name}] Job with no handler found: ${job.id}`);
console.log(`[${this.name}] Resetting Job status...`);
}
job.status = "pending";
job.startedAt = undefined;
this.storage.updateJob(job)
.then(() => {
if (this.logging) {
console.log(`[${this.name}] Job status reset: ${job.id}`);
}
})
.catch((error) => {
if (this.logging) {
console.error("Error resetting job status", error);
}
});
this.activeJobs.delete(job.id);
continue;
}
if (this.logging) {
console.log(`[${this.name}] Processing job:`, job);
console.log(`[${this.name}] Available handlers:`, Array.from(this.handlers.keys()));
console.log(`[${this.name}] Has handler for ${job.name}:`, this.handlers.has(job.name));
}
this.activeJobs.add(job.id);
this.processJob(job)
.catch((error) => {
if (this.logging) {
console.error("Error processing job", error);
}
})
.finally(() => {
this.activeJobs.delete(job.id);
});
jobsProcessed++;
}
}
this.updatePollingInterval(jobsProcessed > 0);
}
catch (error) {
if (this.logging) {
console.error(`[${this.name}] Error in processNextBatch:`, error);
}
}
}
/**
* Refill the job buffer when it's running low
*/
async refillJobBuffer() {
const bufferThreshold = Math.max(1, Math.floor((this.preFetchBatchSize ?? 1) / 3));
if (this.jobBuffer.length <= bufferThreshold) {
const neededJobs = (this.preFetchBatchSize ?? 1) - this.jobBuffer.length;
if (this.logging) {
console.log(`[${this.name}] Refilling job buffer, need ${neededJobs} jobs`);
}
const handlerNames = Array.from(this.handlers.keys());
const newJobs = await this.storage.acquireNextJobs(neededJobs, handlerNames);
this.jobBuffer.push(...newJobs);
this.dispatchEvent(new QueueEvent("buffer-refill-success", {}));
if (this.logging && newJobs.length > 0) {
console.log(`[${this.name}] Prefetched ${newJobs.length} jobs, buffer size: ${this.jobBuffer.length}`);
}
}
}
// Update polling interval based on processing results
updatePollingInterval(hadJobs) {
if (this.isStopped) {
if (this.logging) {
console.log(`[${this.name}] Queue is stopped, skipping`);
}
return;
}
if (this.isUpdatingInterval)
return;
try {
this.isUpdatingInterval = true;
if (!this.intelligentPolling) {
return; // Skip intelligent polling if disabled
}
if (hadJobs) {
// Jobs were found and processed
this.emptyPollsCount = 0;
// Calculate current load factor
const currentLoad = this.activeJobs.size / this.concurrency;
// Adjust interval based on load
if (currentLoad > this.loadFactor) {
// System is busy, poll more frequently
this.processingInterval = Math.max(this.minInterval, this.lastPollingInterval * 0.8);
if (this.concurrency < this.maxConcurrency) {
this.concurrency = Math.min(this.maxConcurrency, Math.ceil(this.concurrency * 1.2));
}
}
else {
// System is underutilized, poll less frequently
this.processingInterval = Math.min(this.maxInterval, this.lastPollingInterval * 1.2);
this.concurrency = Math.max(1, Math.floor(this.concurrency * 0.8));
}
}
else {
// No jobs were found
this.emptyPollsCount++;
if (this.emptyPollsCount >= this.maxEmptyPolls) {
// Gradually increase interval when queue is empty
this.processingInterval = Math.min(this.maxInterval, this.lastPollingInterval * 1.5);
this.concurrency = Math.max(1, Math.floor(this.concurrency * 0.8));
this.emptyPollsCount = 0;
}
}
// Update the interval if queue is running
if (this.processing &&
this.intervalId &&
this.processingInterval !== this.lastPollingInterval) {
clearInterval(this.intervalId);
this.lastPollingInterval = this.processingInterval;
this.intervalId = setInterval(() => this.processNextBatch(), this.processingInterval);
this.dispatchEvent(new QueueEvent("polling-interval-updated", {
message: `Polling interval adjusted to: ${this.processingInterval}ms. Concurrency: ${this.concurrency}`,
}));
if (this.logging) {
console.log(`[${this.name}] Polling interval adjusted to: ${this.processingInterval}ms. Concurrency: ${this.concurrency}`);
}
}
}
catch (error) {
if (this.logging) {
console.error(`[${this.name}] Error in updatePollingInterval:`, error);
this.pollingErrorCount++;
if (this.pollingErrorCount >= 5) {
this.intelligentPolling = false;
console.log(`[${this.name}] Intelligent polling disabled due to errors`);
}
this.dispatchEvent(new QueueEvent("polling-interval-error", {
message: `Polling interval error: ${error}`,
}));
}
}
finally {
this.isUpdatingInterval = false;
}
}
// Process a single job
async processJob(job) {
try {
if (this.isStopping) {
console.log(`[${this.name}] Queue is stopping, skipping job ${job.id}`);
return;
}
if (job.status !== "processing") {
// Mark job as processing
job.status = "processing";
job.startedAt = new Date();
await this.storage.updateJob(job);
}
// Get the handler
const handler = this.handlers.get(job.name);
if (!handler) {
throw new Error(`Handler for job "${job.name}" not found`);
}
let timeoutId;
let result;
try {
const timeoutPromise = new Promise((_, reject) => {
const timeoutMs = job.timeout || 10000;
timeoutId = setTimeout(() => {
reject(new Error(`Job timeout exceeded (${timeoutMs}ms)`));
}, timeoutMs);
});
result = await Promise.race([
handler(job.data),
timeoutPromise
]);
}
finally {
if (timeoutId) {
clearTimeout(timeoutId);
}
}
if (this.isStopping) {
console.log(`[${this.name}] Queue is stopping, skipping job ${job.id}`);
return;
}
await this.storage.completeJob(job.id, result);
this.dispatchEvent(new QueueEvent("completed", { job, status: "completed" }));
if (this.logging) {
console.log(`[${this.name}] Completed job ${job.id}`);
}
if (job.repeat && !this.isStopping) {
await this.scheduleNextRepeat(job);
}
}
catch (error) {
if (this.logging) {
console.error(`[${this.name}] Error processing job`);
}
this.dispatchEvent(new QueueEvent("failed", { job, status: "failed" }));
throw error;
}
}
// Schedule the next occurrence of a repeatable job
async scheduleNextRepeat(job) {
try {
if (!job.repeat)
return;
// Check if we've reached the repeat limit
if (job.repeat.limit) {
const executionCount = (job.retryCount || 0) + 1;
if (executionCount >= job.repeat.limit) {
return; // Don't schedule another repeat
}
}
// Check if we've passed the end date
if (job.repeat.endDate && new Date() > job.repeat.endDate) {
return; // Don't schedule another repeat
}
const nextExecutionTime = this.calculateNextExecutionTime(job);
if (!nextExecutionTime) {
return;
}
// Create a new job with the same parameters
const newJob = {
id: generateId(),
name: job.name,
data: job.data,
status: "pending",
createdAt: new Date(),
scheduledAt: nextExecutionTime,
priority: job.priority,
retryCount: (job.retryCount || 0) + 1,
repeat: job.repeat,
timeout: job.timeout,
};
if (this.logging) {
console.log(`[${this.name}] Scheduled repeatable job ${newJob.id} to run at ${nextExecutionTime}`);
}
await this.storage.saveJob(newJob);
this.dispatchEvent(new QueueEvent("scheduled", { job: newJob, status: "pending" }));
}
catch (error) {
if (this.logging) {
console.error(`[${this.name}] Error in scheduleNextRepeat:`, error);
}
this.dispatchEvent(new QueueEvent("scheduled-repeat-error", {
message: `Scheduled repeat error: ${error}`,
}));
}
}
// Calculate the next execution time based on the repeat configuration
calculateNextExecutionTime(job) {
try {
if (!job.repeat) {
throw new Error("Job does not have repeat configuration");
}
const now = new Date();
const { every, unit } = job.repeat;
if (every === undefined || unit === undefined) {
throw new Error("Invalid repeat configuration: missing every or unit");
}
let nextTime = new Date(now);
switch (unit) {
case "seconds":
nextTime.setSeconds(nextTime.getSeconds() + every);
break;
case "minutes":
nextTime.setMinutes(nextTime.getMinutes() + every);
break;
case "hours":
nextTime.setHours(nextTime.getHours() + every);
break;
case "days":
nextTime.setDate(nextTime.getDate() + every);
break;
case "weeks":
nextTime.setDate(nextTime.getDate() + every * 7);
break;
case "months":
nextTime.setMonth(nextTime.getMonth() + every);
break;
default:
throw new Error(`Unsupported repeat unit: ${unit}`);
}
return nextTime;
}
catch (error) {
if (this.logging) {
console.error(`[${this.name}] Error in calculateNextExecutionTime:`, error);
}
this.dispatchEvent(new QueueEvent("calculate-next-execution-time-error", {
message: `Calculate next execution time error: ${error}`,
}));
}
}
// Add a repeatable job to the queue
async addRepeatable(name, data, options) {
if (this.isStopped) {
throw new Error("Queue is stopped");
}
// Validate options
if (options.every <= 0) {
if (this.logging) {
console.log(`[${this.name}] Repeat interval must be greater than 0`);
}
throw new Error("Repeat interval must be greater than 0");
}
const priority = options.priority || 0;
const job = {
id: generateId(),
name,
data,
status: "pending",
createdAt: new Date(),
priority,
repeat: {
every: options.every,
unit: options.unit,
startDate: options.startDate || undefined,
endDate: options.endDate || undefined,
limit: options.limit || undefined,
},
timeout: options.timeout || undefined,
};
if (this.logging) {
console.log(`[${this.name}] Scheduled repeatable job ${job.id} to run at ${job.createdAt}`);
}
// Schedule the job to start at the specified time or now
if (options.startDate && options.startDate > new Date()) {
job.scheduledAt = options.startDate;
}
await this.storage.saveJob(job);
this.dispatchEvent(new QueueEvent("scheduled", { job, status: "pending" }));
return job;
}
}
//# sourceMappingURL=job-queue.js.map