@logistically/events
Version:
A production-ready event-driven architecture library for NestJS with Redis Streams, comprehensive batching, reliable consumption, and enterprise-grade features.
259 lines (258 loc) • 9.78 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.BatchedEventPublisher = void 0;
const index_1 = require("./index");
const event_types_1 = require("../event-types");
const strategy_factory_1 = require("./strategies/strategy-factory");
class BatchQueue {
constructor(config, strategy, onFlush, originServiceName, eventNamespace) {
this.config = config;
this.strategy = strategy;
this.onFlush = onFlush;
this.originServiceName = originServiceName;
this.eventNamespace = eventNamespace;
this.messages = [];
this.isFlushing = false;
this.concurrentBatches = 0;
}
async addMessage(eventType, body) {
const envelope = (0, event_types_1.createEventEnvelope)(eventType, this.originServiceName, body, this.eventNamespace);
const message = {
eventType,
body,
originalId: envelope.header.id,
timestamp: envelope.header.timestamp,
envelope,
};
// Check if this message can be batched with existing messages
if (this.messages.length > 0 && !this.strategy.canBatchTogether(this.messages[0], message)) {
// Flush current batch before adding new message
await this.flush();
}
this.messages.push(message);
// Set timeout for max wait
if (!this.flushTimeout) {
this.flushTimeout = setTimeout(() => this.flush(), this.config.maxWaitMs);
}
// Flush if max size reached
if (this.messages.length >= this.config.maxSize) {
await this.flush();
}
}
clearFlushTimeout() {
if (this.flushTimeout) {
clearTimeout(this.flushTimeout);
this.flushTimeout = undefined;
}
}
async flush() {
if (this.isFlushing || this.messages.length === 0) {
return;
}
// Clear timeout first to prevent race conditions
this.clearFlushTimeout();
this.isFlushing = true;
try {
// Check concurrent batch limit
if (this.concurrentBatches >= this.config.maxConcurrentBatches) {
throw new Error('Too many concurrent batches');
}
this.concurrentBatches++;
const messagesToSend = [...this.messages];
this.messages = [];
await this.onFlush(messagesToSend);
}
finally {
this.isFlushing = false;
this.concurrentBatches--;
}
}
async forceFlush() {
if (this.messages.length > 0) {
await this.flush();
}
}
getMessageCount() {
return this.messages.length;
}
// Cleanup method for memory management
cleanup() {
this.clearFlushTimeout();
this.messages = [];
this.isFlushing = false;
this.concurrentBatches = 0;
}
}
class BatchedEventPublisher {
constructor(transports, options) {
this.queues = new Map();
this.failedMessages = [];
this.activeBatches = 0;
// Memory management constants
this.MAX_FAILED_MESSAGES = 1000;
this.MAX_QUEUES = 100;
this.QUEUE_CLEANUP_INTERVAL = 60000; // 1 minute
this.lastQueueCleanup = Date.now();
this.basePublisher = new index_1.EventPublisher(transports, options);
this.batchConfig = options.batchConfig || {
maxSize: 100,
maxWaitMs: 1000,
maxConcurrentBatches: 5,
batchingTypeStrategy: 'exact',
};
const transportType = options.transportType || 'console';
const strategyOptions = {
typePrefix: options.typePrefix || 'default',
batchingTypeStrategy: this.batchConfig.batchingTypeStrategy,
};
this.strategy = strategy_factory_1.BatchingStrategyFactory.createStrategy(transportType, strategyOptions);
}
// Expose validator for tests
get validator() {
return this.basePublisher['validator'];
}
async addMessage(eventType, body) {
// Validate the message before adding to queue
const validationResult = this.basePublisher['validator'].validate(eventType, body);
if (!validationResult.valid) {
throw new Error(`Invalid event body for type ${eventType}: ${validationResult.error}`);
}
const batchingKey = this.strategy.getBatchingKey(eventType);
// Check queue limit to prevent memory leaks
if (!this.queues.has(batchingKey) && this.queues.size >= this.MAX_QUEUES) {
this.cleanupEmptyQueues();
if (this.queues.size >= this.MAX_QUEUES) {
throw new Error(`Too many unique event types (${this.MAX_QUEUES}). Consider using more specific batching strategies.`);
}
}
if (!this.queues.has(batchingKey)) {
this.queues.set(batchingKey, new BatchQueue(this.batchConfig, this.strategy, (messages) => this.sendBatch(messages), this.basePublisher['options'].originServiceName, this.basePublisher['eventNamespace']));
}
const queue = this.queues.get(batchingKey);
await queue.addMessage(eventType, body);
}
async publishBatch(eventType, bodies) {
const promises = bodies.map(body => this.addMessage(eventType, body));
await Promise.all(promises);
await this.flush();
}
async flush() {
const promises = Array.from(this.queues.values()).map(queue => queue.forceFlush());
await Promise.all(promises);
}
getFailedMessages() {
return [...this.failedMessages];
}
clearFailedMessages() {
this.failedMessages = [];
}
// Memory management methods
addToFailedMessages(messages) {
this.failedMessages.push(...messages);
// Keep only the most recent failed messages to prevent memory leaks
if (this.failedMessages.length > this.MAX_FAILED_MESSAGES) {
this.failedMessages = this.failedMessages.slice(-this.MAX_FAILED_MESSAGES);
}
}
cleanupEmptyQueues() {
const now = Date.now();
// Only cleanup periodically to avoid performance impact
if (now - this.lastQueueCleanup < this.QUEUE_CLEANUP_INTERVAL) {
return;
}
this.lastQueueCleanup = now;
for (const [key, queue] of this.queues.entries()) {
if (queue.getMessageCount() === 0) {
queue.cleanup();
this.queues.delete(key);
}
}
}
// Public cleanup method for graceful shutdown
cleanup() {
// Cleanup all queues
for (const queue of this.queues.values()) {
queue.cleanup();
}
this.queues.clear();
// Clear failed messages
this.failedMessages = [];
// Reset counters
this.activeBatches = 0;
}
// Close method for proper resource cleanup
async close() {
// Flush all pending batches
await this.flush();
// Cleanup all resources
this.cleanup();
// Close the base publisher
if (this.basePublisher && typeof this.basePublisher.close === 'function') {
await this.basePublisher.close();
}
}
// Memory monitoring methods
getMemoryStats() {
let totalMessages = 0;
for (const queue of this.queues.values()) {
totalMessages += queue.getMessageCount();
}
return {
queueCount: this.queues.size,
totalMessages,
failedMessageCount: this.failedMessages.length,
activeBatches: this.activeBatches,
};
}
// Force cleanup of old failed messages
cleanupOldFailedMessages(maxAgeMs = 3600000) {
const now = Date.now();
this.failedMessages = this.failedMessages.filter(message => {
const messageTime = new Date(message.timestamp).getTime();
return now - messageTime < maxAgeMs;
});
}
async sendBatch(messages) {
// Check concurrent batch limit
if (this.activeBatches >= this.batchConfig.maxConcurrentBatches) {
throw new Error('Too many concurrent batches');
}
this.activeBatches++;
try {
const batchEnvelope = this.strategy.createBatchEnvelope(messages);
const transport = this.getTransportForBatch();
await this.strategy.sendBatch(transport, batchEnvelope);
}
catch (error) {
// Retry once as individual messages using their original envelopes
try {
await this.strategy.sendIndividualMessages(this.basePublisher, messages);
}
catch (individualError) {
// If individual messages also fail, add them to failed messages
this.addToFailedMessages(messages);
// Don't re-throw the error, just log it and continue
}
}
finally {
this.activeBatches--;
}
}
getEventTypePrefix(eventType) {
const parts = eventType.split('.');
return parts.length > 1 ? `${parts[0]}.` : 'default.';
}
getTransportForBatch() {
// Try to get transport for 'batch' event type, fallback to first available transport
try {
const { transport } = this.basePublisher['getTransportForEvent']('batch');
return transport;
}
catch (error) {
// If no specific route for 'batch', use the first available transport
const transports = this.basePublisher['transports'];
return Object.values(transports)[0];
}
}
}
exports.BatchedEventPublisher = BatchedEventPublisher;