kafka-producer-js/src/producer.js

A configurable Kafka producer package for Node.js applications with NFL event examples and AWS MSK integration

/**
 * Kafka Producer Module
 * Provides a configurable Kafka producer with connection management
 */

const { Kafka, logLevel } = require('kafkajs');
const KafkaConfig = require('./config');

class KafkaProducer {
  constructor(config) {
    this.kafkaConfig = new KafkaConfig(config);
    this.kafka = null;
    this.producer = null;
    this.isConnected = false;
  }

  /**
   * Initializes the Kafka client and producer
   */
  async initialize() {
    try {
      const config = this.kafkaConfig.getConfig();
      
      // Create Kafka client
      this.kafka = new Kafka({
        clientId: config.clientId,
        brokers: config.brokers,
        connectionTimeout: config.connectionTimeout,
        authenticationTimeout: config.authenticationTimeout,
        reauthenticationThreshold: config.reauthenticationThreshold,
        requestTimeout: config.requestTimeout,
        enforceRequestTimeout: config.enforceRequestTimeout,
        retry: config.retry,
        logLevel: this.getLogLevel(config.logLevel),
        // Add SASL configuration if provided
        ...(config.sasl && { sasl: config.sasl }),
        // Add SSL configuration if provided
        ...(config.ssl && { ssl: config.ssl })
      });

      // Create producer
      const producerConfig = this.kafkaConfig.getProducerConfig();
      this.producer = this.kafka.producer(producerConfig);

      console.log('Kafka producer initialized successfully');
    } catch (error) {
      console.error('Failed to initialize Kafka producer:', error.message);
      throw error;
    }
  }

  /**
   * Connects to Kafka cluster
   */
  async connect() {
    try {
      if (!this.producer) {
        throw new Error('Producer not initialized. Call initialize() first.');
      }

      await this.producer.connect();
      this.isConnected = true;
      console.log('Connected to Kafka cluster');
    } catch (error) {
      console.error('Failed to connect to Kafka:', error.message);
      throw error;
    }
  }

  /**
   * Sends a single message to a topic
   * @param {string} topic - The topic to send the message to
   * @param {Object} message - The message object
   * @param {string} message.key - Optional message key
   * @param {string|Object} message.value - The message value
   * @param {Object} message.headers - Optional message headers
   * @param {number} message.partition - Optional specific partition
   * @param {number} message.timestamp - Optional timestamp
   * @returns {Promise<Object>} Send result
   */
  async send(topic, message) {
    try {
      if (!this.isConnected) {
        throw new Error('Producer not connected. Call connect() first.');
      }

      // Prepare the message
      const kafkaMessage = {
        key: message.key || null,
        value: typeof message.value === 'string' ? message.value : JSON.stringify(message.value),
        headers: message.headers || {},
        ...(message.partition !== undefined && { partition: message.partition }),
        ...(message.timestamp && { timestamp: message.timestamp })
      };

      const result = await this.producer.send({
        topic,
        messages: [kafkaMessage]
      });

      console.log(`Message sent successfully to topic: ${topic}`, result);
      return result;
    } catch (error) {
      console.error(`Failed to send message to topic ${topic}:`, error.message);
      throw error;
    }
  }

  /**
   * Sends multiple messages to a topic
   * @param {string} topic - The topic to send messages to
   * @param {Array<Object>} messages - Array of message objects
   * @returns {Promise<Object>} Send result
   */
  async sendBatch(topic, messages) {
    try {
      if (!this.isConnected) {
        throw new Error('Producer not connected. Call connect() first.');
      }

      if (!Array.isArray(messages) || messages.length === 0) {
        throw new Error('Messages must be a non-empty array');
      }

      // Prepare messages
      const kafkaMessages = messages.map(message => ({
        key: message.key || null,
        value: typeof message.value === 'string' ? message.value : JSON.stringify(message.value),
        headers: message.headers || {},
        ...(message.partition !== undefined && { partition: message.partition }),
        ...(message.timestamp && { timestamp: message.timestamp })
      }));

      const result = await this.producer.send({
        topic,
        messages: kafkaMessages
      });

      console.log(`Batch of ${messages.length} messages sent successfully to topic: ${topic}`, result);
      return result;
    } catch (error) {
      console.error(`Failed to send batch messages to topic ${topic}:`, error.message);
      throw error;
    }
  }

  /**
   * Sends messages to multiple topics
   * @param {Array<Object>} topicMessages - Array of topic-message objects
   * @param {string} topicMessages[].topic - Topic name
   * @param {Array<Object>} topicMessages[].messages - Messages for the topic
   * @returns {Promise<Object>} Send result
   */
  async sendToMultipleTopics(topicMessages) {
    try {
      if (!this.isConnected) {
        throw new Error('Producer not connected. Call connect() first.');
      }

      if (!Array.isArray(topicMessages) || topicMessages.length === 0) {
        throw new Error('topicMessages must be a non-empty array');
      }

      const batch = topicMessages.map(({ topic, messages }) => {
        if (!topic || !messages || !Array.isArray(messages)) {
          throw new Error('Each topic message must have topic and messages array');
        }

        const kafkaMessages = messages.map(message => ({
          key: message.key || null,
          value: typeof message.value === 'string' ? message.value : JSON.stringify(message.value),
          headers: message.headers || {},
          ...(message.partition !== undefined && { partition: message.partition }),
          ...(message.timestamp && { timestamp: message.timestamp })
        }));

        return {
          topic,
          messages: kafkaMessages
        };
      });

      const result = await this.producer.sendBatch({
        topicMessages: batch
      });

      console.log('Messages sent successfully to multiple topics', result);
      return result;
    } catch (error) {
      console.error('Failed to send messages to multiple topics:', error.message);
      throw error;
    }
  }

  /**
   * Disconnects from Kafka cluster
   */
  async disconnect() {
    try {
      if (this.producer && this.isConnected) {
        await this.producer.disconnect();
        this.isConnected = false;
        console.log('Disconnected from Kafka cluster');
      }
    } catch (error) {
      console.error('Error during disconnect:', error.message);
      throw error;
    }
  }

  /**
   * Gets the connection status
   * @returns {boolean} Connection status
   */
  isProducerConnected() {
    return this.isConnected;
  }

  /**
   * Converts string log level to kafkajs log level
   * @param {string} level - Log level string
   * @returns {number} kafkajs log level
   */
  getLogLevel(level) {
    const levels = {
      error: logLevel.ERROR,
      warn: logLevel.WARN,
      info: logLevel.INFO,
      debug: logLevel.DEBUG
    };
    return levels[level.toLowerCase()] || logLevel.WARN;
  }
}

module.exports = KafkaProducer;