broker-lib
Version:
Multi-Broker Message Bus with Multi-Topic Support
595 lines • 23.4 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __exportStar = (this && this.__exportStar) || function(m, exports) {
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.BrokerManager = exports.SubscriptionManager = void 0;
exports.createBrokerConfigFromEnv = createBrokerConfigFromEnv;
exports.createSubscriptionManagerFromEnv = createSubscriptionManagerFromEnv;
exports.createBrokerManagerFromEnv = createBrokerManagerFromEnv;
exports.createBrokerConfigFromAppEnv = createBrokerConfigFromAppEnv;
exports.createSubscriptionManagerFromAppEnv = createSubscriptionManagerFromAppEnv;
exports.createBrokerManagerFromAppEnv = createBrokerManagerFromAppEnv;
const kafka_1 = require("./brokers/kafka");
const mqtt_1 = require("./brokers/mqtt");
const gcpPubSub_1 = require("./brokers/gcpPubSub");
const events_1 = require("events");
// Factory function to create broker configuration from environment variables
function createBrokerConfigFromEnv(env, defaultClientId = 'broker-lib-app') {
const brokerType = (env.BROKER_TYPE || 'KAFKA');
const clientId = env.BROKER_CLIENT_ID || defaultClientId;
switch (brokerType) {
case 'MQTT':
return {
brokerType: 'MQTT',
mqtt: {
url: env.MQTT_URL || 'mqtt://localhost:1883',
clientId,
clean: true,
reconnectPeriod: 1000,
connectTimeout: 30000,
},
};
case 'KAFKA':
return {
brokerType: 'KAFKA',
kafka: {
clientId,
brokers: (env.KAFKA_BROKERS || 'localhost:9092').split(',').map(b => b.trim()),
groupId: env.KAFKA_GROUP_ID || `${clientId}-group`,
},
};
case 'GCP_PUBSUB':
const gcpConfig = {
projectId: env.PROJECT_ID || 'your-project-id',
};
if (env.GCP_KEY_FILENAME) {
gcpConfig.keyFilename = env.GCP_KEY_FILENAME;
}
return {
brokerType: 'GCP_PUBSUB',
gcp: gcpConfig,
};
default:
// Default to Kafka
return {
brokerType: 'KAFKA',
kafka: {
clientId,
brokers: (env.KAFKA_BROKERS || 'localhost:9092').split(',').map(b => b.trim()),
groupId: env.KAFKA_GROUP_ID || `${clientId}-group`,
},
};
}
}
// Factory function to create SubscriptionManager from environment variables
function createSubscriptionManagerFromEnv(env, defaultClientId = 'broker-lib-app', logger = console) {
const config = createBrokerConfigFromEnv(env, defaultClientId);
return new SubscriptionManager(config, logger);
}
// Factory function to create BrokerManager from environment variables
function createBrokerManagerFromEnv(env, defaultClientId = 'broker-lib-app') {
const config = createBrokerConfigFromEnv(env, defaultClientId);
return new BrokerManager(config);
}
// Auto-configure functions that read from application's environment
function createBrokerConfigFromAppEnv(defaultClientId = 'broker-lib-app') {
return createBrokerConfigFromEnv(process.env, defaultClientId);
}
function createSubscriptionManagerFromAppEnv(defaultClientId = 'broker-lib-app', logger = console) {
return createSubscriptionManagerFromEnv(process.env, defaultClientId, logger);
}
function createBrokerManagerFromAppEnv(defaultClientId = 'broker-lib-app') {
return createBrokerManagerFromEnv(process.env, defaultClientId);
}
// New SubscriptionManager class for simplified subscription interface
class SubscriptionManager extends events_1.EventEmitter {
constructor(brokerConfig, logger = console) {
super();
this.topicHandlers = new Map();
this.brokerManager = new BrokerManager(brokerConfig);
this.logger = logger;
// Forward broker events
this.brokerManager.on('connect', () => this.emit('connect'));
this.brokerManager.on('disconnect', () => this.emit('disconnect'));
this.brokerManager.on('error', (error) => this.emit('error', error));
this.brokerManager.on('connecting', () => this.emit('connecting'));
this.brokerManager.on('reconnect', () => this.emit('reconnect'));
this.brokerManager.on('reconnect_failed', (error) => this.emit('reconnect_failed', error));
}
async connect() {
await this.brokerManager.connect();
}
async disconnect() {
await this.brokerManager.disconnect();
}
// Single topic subscription (existing method)
async subscribe(options, callback) {
try {
await this.ensureConnection();
// Store the handler for this topic
this.topicHandlers.set(options.topic, callback);
// Use the subscribe method with options
const subscribeOptions = {
qos: options.qos ?? 1, // Default QoS for MQTT compatibility
autoAck: options.autoAck ?? true, // Default auto-ack for GCP PubSub compatibility
};
if (options.fromBeginning !== undefined) {
subscribeOptions.fromBeginning = options.fromBeginning;
}
// Create the message handler that routes to the appropriate callback
const messageHandler = (topic, message) => {
this.logger.log(`Received message on topic ${topic}: ${message.toString()}`);
const handler = this.topicHandlers.get(topic);
if (handler) {
try {
const parsedMessage = JSON.parse(message.toString());
handler(parsedMessage);
}
catch (parseError) {
this.logger.warn('Failed to parse message as JSON, passing raw message');
handler(message.toString());
}
}
else {
this.logger.warn(`No handler found for topic: ${topic}`);
}
};
await this.brokerManager.subscribe([options.topic], messageHandler, subscribeOptions);
this.logger.log(`Subscribed to topic: ${options.topic}`);
}
catch (error) {
this.logger.error(`Failed to subscribe to topic: ${options.topic}`, error);
// Handle Kafka-specific subscription error
if (error instanceof Error && error.message.includes('Cannot subscribe to topic while consumer is running')) {
this.logger.warn(`Consumer is already running. Topic ${options.topic} may already be subscribed.`);
// For Kafka, we can't subscribe to additional topics after consumer is running
// The topic should already be subscribed if it was added in a previous call
return;
}
throw error;
}
}
// New method for multiple topic subscriptions with different handlers
async subscribeMultiple(mappings) {
try {
await this.ensureConnection();
// Store all handlers
for (const mapping of mappings) {
this.topicHandlers.set(mapping.topic, mapping.handler);
}
// Subscribe to all topics
const topics = mappings.map(m => m.topic);
const defaultOptions = {
qos: 1,
autoAck: true,
};
// Create the message handler that routes to the appropriate callback
const messageHandler = (topic, message) => {
this.logger.log(`Received message on topic ${topic}: ${message.toString()}`);
const handler = this.topicHandlers.get(topic);
if (handler) {
try {
const parsedMessage = JSON.parse(message.toString());
handler(parsedMessage);
}
catch (parseError) {
this.logger.warn('Failed to parse message as JSON, passing raw message');
handler(message.toString());
}
}
else {
this.logger.warn(`No handler found for topic: ${topic}`);
}
};
await this.brokerManager.subscribe(topics, messageHandler, defaultOptions);
this.logger.log(`Subscribed to ${topics.length} topics: ${topics.join(', ')}`);
}
catch (error) {
this.logger.error(`Failed to subscribe to topics: ${mappings.map(m => m.topic).join(', ')}`, error);
// Handle Kafka-specific subscription error
if (error instanceof Error && error.message.includes('Cannot subscribe to topic while consumer is running')) {
this.logger.warn(`Consumer is already running. Some topics may already be subscribed.`);
return;
}
throw error;
}
}
// Alternative method for subscribing with a single handler for all topics
async subscribeToTopics(topics, callback, options) {
const mappings = topics.map(topic => {
const mapping = {
topic,
handler: callback,
};
if (options) {
mapping.options = options;
}
return mapping;
});
await this.subscribeMultiple(mappings);
}
async publish(topic, message, options) {
try {
await this.ensureConnection();
const messageString = typeof message === 'string' ? message : JSON.stringify(message);
await this.brokerManager.publish(topic, messageString, options);
this.logger.log(`Published message to topic: ${topic}`);
}
catch (error) {
this.logger.error(`Failed to publish message to topic: ${topic}`, error);
throw error;
}
}
async ensureConnection() {
if (!this.brokerManager.isConnected()) {
await this.brokerManager.connect();
}
}
// Get all subscribed topics
getSubscribedTopics() {
return Array.from(this.topicHandlers.keys());
}
// Get handler for a specific topic
getHandler(topic) {
return this.topicHandlers.get(topic);
}
// Remove handler for a specific topic
removeHandler(topic) {
return this.topicHandlers.delete(topic);
}
// Clear all handlers
clearHandlers() {
this.topicHandlers.clear();
}
// Expose broker manager methods for advanced usage
getBrokerManager() {
return this.brokerManager;
}
isConnected() {
return this.brokerManager.isConnected();
}
getConnectionState() {
return this.brokerManager.getConnectionState();
}
getReconnectAttempts() {
return this.brokerManager.getReconnectAttempts();
}
setReconnectionConfig(config) {
this.brokerManager.setReconnectionConfig(config);
}
getBrokerType() {
return this.brokerManager.getBrokerType();
}
// Polling support for GCP Pub/Sub (when real-time subscription is not available)
async startPolling(topics, intervalMs = 5000) {
if (this.getBrokerType() !== 'GCP_PUBSUB') {
throw new Error('Polling is only supported for GCP Pub/Sub');
}
this.logger.log(`Starting polling for topics: ${topics.join(', ')} with interval: ${intervalMs}ms`);
const pollInterval = setInterval(async () => {
try {
for (const topic of topics) {
const handler = this.topicHandlers.get(topic);
if (handler) {
// Simulate message polling - in real implementation, this would call GCP Pub/Sub API
this.logger.log(`Polling topic: ${topic}`);
// For demonstration, we'll simulate receiving a message
// In a real implementation, you would:
// 1. Call GCP Pub/Sub API to pull messages
// 2. Process each message
// 3. Acknowledge messages
const mockMessage = {
topic,
timestamp: new Date().toISOString(),
data: `Polled message from ${topic}`,
messageId: `msg_${Date.now()}`
};
handler(mockMessage);
}
}
}
catch (error) {
this.logger.error('Error during polling:', error);
}
}, intervalMs);
// Store the interval ID for cleanup
this.pollInterval = pollInterval;
}
async stopPolling() {
if (this.pollInterval) {
clearInterval(this.pollInterval);
this.pollInterval = null;
this.logger.log('Stopped polling');
}
}
// Check if polling is active
isPolling() {
return !!this.pollInterval;
}
}
exports.SubscriptionManager = SubscriptionManager;
class BrokerManager extends events_1.EventEmitter {
constructor(config) {
super();
this.connectionState = 'disconnected';
this.reconnectAttempts = 0;
this.isReconnecting = false;
this.pendingOperations = [];
this.config = config;
this.broker = this.createBroker(config);
// Default reconnection configuration
this.reconnectionConfig = {
enabled: true,
maxAttempts: 10,
initialDelay: 1000,
maxDelay: 30000,
backoffMultiplier: 2,
};
// Set up event listeners for the broker
if (this.broker instanceof events_1.EventEmitter) {
this.broker.on('connect', () => {
this.connectionState = 'connected';
this.reconnectAttempts = 0;
this.isReconnecting = false;
this.emit('connect');
this.processPendingOperations();
});
this.broker.on('disconnect', () => {
this.connectionState = 'disconnected';
this.emit('disconnect');
this.handleDisconnection();
});
this.broker.on('error', (error) => {
this.connectionState = 'error';
this.emit('error', error);
this.handleDisconnection();
});
this.broker.on('reconnect', () => {
this.connectionState = 'connected';
this.reconnectAttempts = 0;
this.isReconnecting = false;
this.emit('reconnect');
this.processPendingOperations();
});
this.broker.on('reconnect_failed', (error) => {
this.connectionState = 'error';
this.emit('reconnect_failed', error);
this.handleReconnectFailure(error);
});
}
}
createBroker(config) {
switch (config.brokerType) {
case 'KAFKA':
if (!config.kafka) {
throw new Error('Kafka configuration is required for KAFKA broker type');
}
return new kafka_1.KafkaBroker(config.kafka);
case 'MQTT':
if (!config.mqtt) {
throw new Error('MQTT configuration is required for MQTT broker type');
}
return new mqtt_1.MqttBroker(config.mqtt);
case 'GCP_PUBSUB':
if (!config.gcp) {
throw new Error('GCP PubSub configuration is required for GCP_PUBSUB broker type');
}
return new gcpPubSub_1.GCPPubSubBroker(config.gcp);
default:
throw new Error(`Invalid broker type: ${config.brokerType}`);
}
}
handleDisconnection() {
if (!this.reconnectionConfig.enabled || this.isReconnecting) {
return;
}
if (this.reconnectAttempts < this.reconnectionConfig.maxAttempts) {
this.scheduleReconnect();
}
else {
this.emit('reconnect_failed', new Error('Max reconnection attempts reached'));
}
}
handleReconnectFailure(error) {
if (this.reconnectAttempts < this.reconnectionConfig.maxAttempts) {
this.scheduleReconnect();
}
else {
this.emit('reconnect_failed', error);
}
}
scheduleReconnect() {
if (this.reconnectTimeout) {
clearTimeout(this.reconnectTimeout);
}
const delay = Math.min(this.reconnectionConfig.initialDelay * Math.pow(this.reconnectionConfig.backoffMultiplier, this.reconnectAttempts), this.reconnectionConfig.maxDelay);
this.reconnectTimeout = setTimeout(() => {
this.attemptReconnect();
}, delay);
}
async attemptReconnect() {
if (this.isReconnecting) {
return;
}
this.isReconnecting = true;
this.reconnectAttempts++;
this.connectionState = 'connecting';
this.emit('connecting');
try {
await this.broker.connect();
this.connectionState = 'connected';
this.reconnectAttempts = 0;
this.isReconnecting = false;
this.emit('connect');
this.processPendingOperations();
}
catch (error) {
this.connectionState = 'error';
this.isReconnecting = false;
this.emit('error', error);
if (this.reconnectAttempts < this.reconnectionConfig.maxAttempts) {
this.scheduleReconnect();
}
else {
this.emit('reconnect_failed', error);
}
}
}
async processPendingOperations() {
const operations = [...this.pendingOperations];
this.pendingOperations = [];
for (const operation of operations) {
try {
await operation();
}
catch (error) {
console.error('Failed to process pending operation:', error);
}
}
}
async executeWithReconnection(operation) {
try {
if (!this.isConnected()) {
// Queue the operation and attempt to reconnect
return new Promise((resolve, reject) => {
this.pendingOperations.push(async () => {
try {
const result = await operation();
resolve(result);
}
catch (error) {
reject(error);
}
});
if (!this.isReconnecting) {
this.attemptReconnect();
}
});
}
return await operation();
}
catch (error) {
// If the operation fails due to connection issues, queue it for retry
if (this.isConnectionError(error)) {
return new Promise((resolve, reject) => {
this.pendingOperations.push(async () => {
try {
const result = await operation();
resolve(result);
}
catch (error) {
reject(error);
}
});
if (!this.isReconnecting) {
this.attemptReconnect();
}
});
}
throw error;
}
}
isConnectionError(error) {
const errorMessage = error?.message || '';
const connectionErrors = [
'ECONNRESET',
'ECONNREFUSED',
'ENOTFOUND',
'ETIMEDOUT',
'connection lost',
'connection closed',
'network error',
'socket error'
];
return connectionErrors.some(connError => errorMessage.toLowerCase().includes(connError.toLowerCase()));
}
async connect() {
try {
this.connectionState = 'connecting';
this.emit('connecting');
await this.broker.connect();
this.connectionState = 'connected';
this.emit('connect');
}
catch (error) {
this.connectionState = 'error';
this.emit('error', error);
throw error;
}
}
async publish(topic, message, options) {
return this.executeWithReconnection(async () => {
await this.broker.publish(topic, message, options);
});
}
async subscribe(topics, handler, options) {
return this.executeWithReconnection(async () => {
if (handler) {
this.messageHandler = handler;
}
await this.broker.subscribe(topics, this.messageHandler, options);
});
}
async disconnect() {
try {
if (this.reconnectTimeout) {
clearTimeout(this.reconnectTimeout);
}
await this.broker.disconnect();
this.connectionState = 'disconnected';
this.reconnectAttempts = 0;
this.isReconnecting = false;
this.pendingOperations = [];
this.emit('disconnect');
}
catch (error) {
this.emit('error', error);
throw error;
}
}
async reconnect() {
try {
await this.disconnect();
await this.connect();
}
catch (error) {
this.emit('error', error);
throw error;
}
}
setReconnectionConfig(config) {
this.reconnectionConfig = { ...this.reconnectionConfig, ...config };
}
isConnected() {
return this.broker.isConnected();
}
getConnectionState() {
return this.connectionState;
}
getReconnectAttempts() {
return this.reconnectAttempts;
}
getBrokerType() {
return this.config.brokerType;
}
setMessageHandler(handler) {
this.messageHandler = handler;
}
}
exports.BrokerManager = BrokerManager;
// Re-export types for convenience
__exportStar(require("./types"), exports);
__exportStar(require("./config"), exports);
//# sourceMappingURL=index.js.map