kafka-pub-sub/BatchProduceEvent.js

Enterprise-grade Kafka publish/subscribe library for Node.js — producer pool, batch sending, DLQ, SSL/SASL, multi-broker, and real-world industry examples.

const { Partitioners, CompressionTypes } = require('kafkajs');
const kafka = require('./config/kafka');
const config = require('./config/config');
const validateTopic = require('./validation/validateTopic');
const validateData = require('./validation/validateData');

const COMPRESSION_MAP = {
  none: CompressionTypes.None,
  gzip: CompressionTypes.GZIP,
  snappy: CompressionTypes.Snappy,
  lz4: CompressionTypes.LZ4,
  zstd: CompressionTypes.ZSTD,
};

// Singleton batch producer (separate from the single-message producer)
let batchProducer = null;
let batchProducerConnected = false;

const getBatchProducer = async () => {
  if (!batchProducer) {
    batchProducer = kafka.producer({
      createPartitioner: Partitioners.DefaultPartitioner,
      idempotent: config.kafka_idempotent,
      allowAutoTopicCreation: config.kafka_auto_create_topics,
    });

    const shutdown = async () => {
      if (batchProducerConnected) {
        await batchProducer.disconnect();
        batchProducerConnected = false;
      }
      process.exit(0);
    };
    process.once('SIGTERM', shutdown);
    process.once('SIGINT', shutdown);
  }

  if (!batchProducerConnected) {
    await batchProducer.connect();
    batchProducerConnected = true;
  }

  return batchProducer;
};

/**
 * @typedef {Object} BatchMessage
 * @property {string}  topic   - Kafka topic name.
 * @property {string}  event   - Event name used as the message key.
 * @property {Object}  data    - Payload object.
 * @property {Object}  [headers={}] - Optional Kafka message headers.
 * @property {string}  [partitionKey] - Custom partition key.
 */

/**
 * Publishes multiple messages, optionally across different topics, in a single
 * broker round-trip using KafkaJS `producer.sendBatch`.
 *
 * Significantly more efficient than calling ProduceEvent in a loop for
 * high-throughput scenarios (IoT telemetry, bulk order imports, audit log bursts).
 *
 * @param {BatchMessage[]} messages - Array of message descriptors.
 * @param {Object} [options={}]
 * @param {string} [options.compression] - Compression codec override.
 * @param {string} [options.correlationId] - Adds a `correlation-id` header on all messages.
 * @return {Promise<Array>} Broker acknowledgement array from KafkaJS.
 * @throws {Error} Validation or broker errors.
 *
 * @example
 * await BatchProduceEvent([
 *   { topic: 'order.placed',    event: 'ORDER_PLACED',    data: { orderId: '1' } },
 *   { topic: 'inventory.check', event: 'INVENTORY_CHECK', data: { sku: 'SKU-99' } },
 *   { topic: 'order.placed',    event: 'ORDER_PLACED',    data: { orderId: '2' } },
 * ], { compression: 'gzip', correlationId: 'batch-run-001' });
 */
const BatchProduceEvent = async (messages, options = {}) => {
  if (!Array.isArray(messages) || messages.length === 0) {
    throw new Error('Invalid batch: messages must be a non-empty array');
  }

  const { compression, correlationId } = options;
  const compressionType = COMPRESSION_MAP[compression || config.kafka_compression_type] || CompressionTypes.None;
  const now = new Date().toISOString();

  // Group messages by topic for sendBatch
  const topicMap = new Map();

  messages.forEach((msg, index) => {
    if (!msg || typeof msg !== 'object') {
      throw new Error(`Invalid batch message at index ${index}: must be an object`);
    }

    validateTopic(msg.topic);
    validateData(msg.data || {});

    const headers = {
      ...(msg.headers || {}),
      ...(correlationId ? { 'correlation-id': correlationId } : {}),
      'produced-at': now,
    };

    const kafkaMessage = {
      key: msg.partitionKey || `key-${msg.event || 'batch'}`,
      value: JSON.stringify({
        event: msg.event,
        data: msg.data || {},
        timestamp: Date.now(),
      }),
      headers,
    };

    if (!topicMap.has(msg.topic)) {
      topicMap.set(msg.topic, []);
    }
    topicMap.get(msg.topic).push(kafkaMessage);
  });

  const topicMessages = Array.from(topicMap.entries()).map(([topic, msgs]) => ({
    topic,
    messages: msgs,
  }));

  const producer = await getBatchProducer();

  const response = await producer.sendBatch({
    topicMessages,
    compression: compressionType,
  });

  return response;
};

/**
 * Disconnects the batch producer. Call during graceful shutdown.
 */
BatchProduceEvent.disconnect = async () => {
  if (batchProducerConnected && batchProducer) {
    await batchProducer.disconnect();
    batchProducerConnected = false;
    batchProducer = null;
  }
};

module.exports = BatchProduceEvent;