kafka-pub-sub/ConsumeEvent.js

Enterprise-grade Kafka publish/subscribe library for Node.js — producer pool, batch sending, DLQ, SSL/SASL, multi-broker, and real-world industry examples.

const { CompressionTypes, Partitioners } = require('kafkajs');
const kafka = require('./config/kafka');
const config = require('./config/config');
const validateTopic = require('./validation/validateTopic');

// ── Internal helpers ──────────────────────────────────────────────────────────

const sleep = (ms) =>
  new Promise((resolve) => {
    setTimeout(resolve, ms);
  });

/**
 * Decode all header values from Buffer to string.
 */
const decodeHeaders = (raw) =>
  Object.keys(raw).reduce((acc, key) => {
    acc[key] = Buffer.isBuffer(raw[key]) ? raw[key].toString() : raw[key];
    return acc;
  }, {});

/**
 * Parse a raw KafkaJS message into a clean data object.
 */
const parseMessage = (topic, partition, message) => ({
  topic,
  partition,
  offset: message.offset,
  timestamp: message.timestamp,
  key: message.key ? message.key.toString() : null,
  value: message.value ? JSON.parse(message.value.toString()) : null,
  headers: message.headers ? decodeHeaders(message.headers) : {},
});

// ── DLQ producer (module-level singleton shared across all consumers) ─────────
//
// The DLQ producer is intentionally shared so we keep a single broker connection
// regardless of how many consumers are running. It is only disconnected on
// process exit — never inside an individual consumer's stop() — so that stopping
// one consumer does not break DLQ routing for the others.

let dlqProducer = null;
let dlqConnected = false;

const getDlqProducer = async () => {
  if (!dlqProducer) {
    dlqProducer = kafka.producer({
      createPartitioner: Partitioners.DefaultPartitioner,
      allowAutoTopicCreation: true,
    });
  }
  if (!dlqConnected) {
    await dlqProducer.connect();
    dlqConnected = true;
  }
  return dlqProducer;
};

/**
 * Cleanly disconnect the shared DLQ producer.
 * Called once at process exit, not inside individual consumer stop() functions.
 */
const disconnectDlqProducer = async () => {
  if (dlqConnected && dlqProducer) {
    await dlqProducer.disconnect();
    dlqConnected = false;
    dlqProducer = null;
  }
};

const sendToDLQ = async (originalTopic, rawMessage, error) => {
  try {
    const producer = await getDlqProducer();
    await producer.send({
      topic: `${originalTopic}.dlq`,
      compression: CompressionTypes.None,
      messages: [
        {
          key: rawMessage.key,
          value: rawMessage.value,
          headers: {
            ...(rawMessage.headers || {}),
            'dlq-original-topic': originalTopic,
            'dlq-failed-at': new Date().toISOString(),
            'dlq-error': error.message || String(error),
          },
        },
      ],
    });
  } catch (dlqError) {
    // DLQ failure must not crash the consumer — log and continue
    // eslint-disable-next-line no-console
    console.error('[kafka-pub-sub] Failed to send message to DLQ:', dlqError.message);
  }
};

// Disconnect the DLQ producer when the process exits
process.once('SIGTERM', disconnectDlqProducer);
process.once('SIGINT', disconnectDlqProducer);

// ── ConsumeEvent ─────────────────────────────────────────────────────────────

/**
 * Subscribes to a Kafka topic and invokes a handler for every incoming message.
 *
 * Key improvements over v1:
 *  - Callback-based: handler is called for every message (not just the first).
 *  - Automatic retry with exponential back-off on handler failure.
 *  - Dead-letter queue (DLQ): messages that exhaust retries are routed to
 *    `<topic>.dlq` so they are never silently dropped.
 *  - Returns a `stop()` function for graceful, orderly shutdown.
 *  - Heartbeat is sent after each message to prevent consumer group rebalancing.
 *  - Multiple consumers can run in parallel safely — each has its own connection,
 *    but they share a single DLQ producer for efficiency.
 *
 * @param {string}   topic   - Kafka topic name to subscribe to.
 * @param {Function} handler - `async (message) => void`. Receives a parsed message object.
 * @param {Object}   [options={}]
 * @param {boolean}  [options.fromBeginning=false]  - Consume from earliest offset.
 * @param {number}   [options.retry=3]              - Max handler retries before DLQ.
 * @param {boolean}  [options.dlq=true]             - Route exhausted messages to DLQ.
 * @param {number}   [options.sessionTimeout]       - Override consumer session timeout (ms).
 * @param {number}   [options.heartbeatInterval]    - Override heartbeat interval (ms).
 * @param {string}   [options.groupId]              - Override consumer group ID.
 * @return {Promise<Function>} Resolves to a `stop()` async function.
 *
 * @example — basic usage
 * const stop = await ConsumeEvent('order.placed', async (msg) => {
 *   console.log('New order:', msg.value.data);
 * });
 * // Later, during shutdown:
 * await stop();
 *
 * @example — multiple consumers in parallel (safe)
 * const [stopA, stopB] = await Promise.all([
 *   ConsumeEvent('order.placed',       handlerA, { groupId: 'svc-a' }),
 *   ConsumeEvent('payment.transaction', handlerB, { groupId: 'svc-b' }),
 * ]);
 * // Stopping one does NOT affect the other's DLQ routing
 * await stopA();
 */
const ConsumeEvent = async (topic, handler, options = {}) => {
  if (typeof handler !== 'function') {
    throw new Error('ConsumeEvent requires a handler function as the second argument');
  }

  validateTopic(topic);

  const {
    fromBeginning = false,
    retry = 3,
    dlq = true,
    sessionTimeout = config.kafka_session_timeout,
    heartbeatInterval = config.kafka_heartbeat_interval,
    groupId = config.kafka_group_id,
  } = options;

  const consumer = kafka.consumer({
    groupId,
    sessionTimeout,
    heartbeatInterval,
    maxWaitTimeInMs: config.kafka_max_wait_time,
  });

  await consumer.connect();
  await consumer.subscribe({ topics: [topic], fromBeginning });

  // eslint-disable-next-line no-shadow
  await consumer.run({
    // eslint-disable-next-line no-shadow
    eachMessage: async ({ topic: msgTopic, partition, message, heartbeat }) => {
      const parsed = parseMessage(msgTopic, partition, message);

      let attempt = 0;
      let lastError;

      while (attempt < retry) {
        try {
          // eslint-disable-next-line no-await-in-loop
          await handler(parsed);
          lastError = null;
          break;
        } catch (err) {
          attempt += 1;
          lastError = err;

          if (attempt < retry) {
            // Exponential back-off: 100ms, 200ms, 400ms, …
            // eslint-disable-next-line no-await-in-loop
            await sleep(100 * 2 ** (attempt - 1));
          }
        }
      }

      if (lastError) {
        if (dlq) {
          await sendToDLQ(msgTopic, message, lastError);
        } else {
          throw lastError; // surface to KafkaJS — it will pause and retry at broker level
        }
      }

      await heartbeat();
    },
  });

  /**
   * Gracefully disconnect THIS consumer only.
   * The shared DLQ producer stays connected until process exit.
   * @return {Promise<void>}
   */
  const stop = async () => {
    await consumer.disconnect();
  };

  // Auto-stop on process signals
  process.once('SIGTERM', stop);
  process.once('SIGINT', stop);

  return stop;
};

module.exports = ConsumeEvent;