knock-mq
Version:
Production-grade message queue implementation.
188 lines • 6.61 kB
JavaScript
import { EventEmitter } from 'events';
// === CIRCUIT BREAKER ===
class CircuitBreaker {
threshold;
timeout;
logger;
failures = 0;
lastFailure = 0;
constructor(threshold, timeout, logger) {
this.threshold = threshold;
this.timeout = timeout;
this.logger = logger;
}
async execute(fn) {
if (this.isOpen()) {
throw new Error('Circuit breaker is open');
}
try {
const result = await fn();
this.failures = 0;
return result;
}
catch (err) {
this.failures++;
this.lastFailure = Date.now();
this.logger.warn('Circuit breaker failure', { failures: this.failures });
throw err;
}
}
isOpen() {
return this.failures >= this.threshold &&
Date.now() - this.lastFailure < this.timeout;
}
}
export class Queue extends EventEmitter {
config;
processing = new Map();
stats = {
totalProcessed: 0,
totalErrors: 0,
processingTimes: [],
seen: new Set() // for deduplication
};
circuitBreaker;
paused = false;
constructor(config) {
super();
this.config = config;
this.circuitBreaker = new CircuitBreaker(5, 30000, config.logger);
setInterval(() => this.cleanupStuckItems(), 60_000);
}
pause() {
this.paused = true;
}
resume() {
this.paused = false;
}
async enqueue(data, opts = {}) {
const item = {
id: opts.id || crypto.randomUUID(),
priority: opts.priority || 'normal',
data,
metadata: {
attempts: 0,
status: 'queued',
queuedAt: new Date()
}
};
await this.config.storage.enqueue(item);
return item.id;
}
async processNext(_processor) {
if (this.paused)
return;
const stuckCutoff = new Date(Date.now() - this.config.timeoutMs * 2);
const stuck = await this.config.storage.findStuckProcessing(stuckCutoff);
for (const item of stuck) {
await this.config.storage.updateStatus(item.id, 'failed');
}
const capacity = await this.hasCapacity();
if (!capacity)
return;
// You'd normally fetch eligible items from DB here — in a real impl, `dequeue()` logic would be in storage
// For now, assume processor is attached externally via `start()` loop
}
async processItem(item, processor) {
if (this.processing.has(item.id))
return;
const start = Date.now();
this.processing.set(item.id, {
item,
startedAt: new Date(),
timeout: setTimeout(() => this.handleTimeout(item.id), this.config.timeoutMs)
});
try {
item.metadata.status = 'processing';
item.metadata.lastAttemptAt = new Date();
await this.config.storage.updateStatus(item.id, 'processing');
const result = await this.circuitBreaker.execute(() => processor(item));
if (result.success) {
await this.config.storage.complete(item.id, true);
item.metadata.status = 'delivered';
this.emit('itemDelivered', item);
}
else {
await this.handleFailure(item, processor);
}
if (!this.stats.seen.has(item.id)) {
this.stats.totalProcessed++;
this.stats.seen.add(item.id);
}
this.stats.processingTimes.push(Date.now() - start);
}
catch (err) {
await this.handleError(item, err, processor);
}
finally {
this.clearProcessing(item.id);
}
}
async handleFailure(item, _processor) {
item.metadata.attempts++;
item.metadata.status = 'failed';
if (item.metadata.attempts >= this.config.maxRetries) {
await this.config.storage.moveToDeadLetter(item.id, 'max_retries');
item.metadata.status = 'undeliverable';
this.emit('itemUndeliverable', item);
return;
}
const backoff = this.calculateBackoff(item.metadata.attempts);
item.metadata.nextAttemptAt = new Date(Date.now() + backoff);
item.metadata.status = 'queued';
await this.config.storage.updateStatus(item.id, 'queued');
this.emit('itemRetryScheduled', item);
}
async handleError(item, err, _processor) {
item.metadata.error = err.message;
this.stats.totalErrors++;
this.config.monitoring?.trackError(err, {
itemId: item.id,
attempt: item.metadata.attempts
});
await this.handleFailure(item, _processor);
}
async handleTimeout(id) {
const info = this.processing.get(id);
if (!info)
return;
await this.config.storage.updateStatus(id, 'failed');
this.emit('itemTimeout', info.item);
this.clearProcessing(id);
}
clearProcessing(id) {
const info = this.processing.get(id);
if (info?.timeout)
clearTimeout(info.timeout);
this.processing.delete(id);
}
calculateBackoff(attempt) {
return Math.min(this.config.backoffBaseMs * Math.pow(2, attempt - 1), 30_000);
}
async hasCapacity() {
const current = await this.config.storage.getProcessingCounts();
const { priorityConfig = { high: 5, normal: 3, low: 2 } } = this.config;
return ((current.high || 0) < priorityConfig.high ||
(current.normal || 0) < priorityConfig.normal ||
(current.low || 0) < priorityConfig.low);
}
async cleanupStuckItems() {
const stuckCutoff = new Date(Date.now() - this.config.timeoutMs * 2);
const stuck = await this.config.storage.findStuckProcessing(stuckCutoff);
for (const item of stuck) {
await this.config.storage.updateStatus(item.id, 'failed');
this.emit('itemStuck', item);
}
}
getStats() {
return {
totalProcessed: this.stats.totalProcessed,
totalErrors: this.stats.totalErrors,
averageProcessingTime: this.stats.processingTimes.length > 0
? this.stats.processingTimes.reduce((a, b) => a + b, 0) / this.stats.processingTimes.length
: 0,
currentlyProcessing: this.processing.size
};
}
}
//# sourceMappingURL=queue.js.map