kafka-pub-sub/ProduceEvent.js

Enterprise-grade Kafka publish/subscribe library for Node.js — producer pool, batch sending, DLQ, SSL/SASL, multi-broker, and real-world industry examples.

const { Partitioners, CompressionTypes } = require('kafkajs');
const kafka = require('./config/kafka');
const config = require('./config/config');
const validateTopic = require('./validation/validateTopic');
const validateEvent = require('./validation/validateEvent');
const validateData = require('./validation/validateData');
const validateHeaders = require('./validation/validateHeaders');

// ── Compression codec map ────────────────────────────────────────────────────
const COMPRESSION_MAP = {
  none: CompressionTypes.None,
  gzip: CompressionTypes.GZIP,
  snappy: CompressionTypes.Snappy,
  lz4: CompressionTypes.LZ4,
  zstd: CompressionTypes.ZSTD,
};

// ── Singleton producer pool ──────────────────────────────────────────────────
// Keeps one persistent connection rather than connect/disconnect per message.
class ProducerPool {
  constructor() {
    this._producer = null;
    this._connected = false;
  }

  async get() {
    if (!this._producer) {
      this._producer = kafka.producer({
        createPartitioner: Partitioners.DefaultPartitioner,
        idempotent: config.kafka_idempotent,
        transactionalId: config.kafka_transactional_id || undefined,
        allowAutoTopicCreation: config.kafka_auto_create_topics,
      });

      // Graceful shutdown — disconnect once before process exits
      const shutdown = async () => {
        await this.close();
        process.exit(0);
      };
      process.once('SIGTERM', shutdown);
      process.once('SIGINT', shutdown);
    }

    if (!this._connected) {
      await this._producer.connect();
      this._connected = true;
    }

    return this._producer;
  }

  async close() {
    if (this._connected && this._producer) {
      await this._producer.disconnect();
      this._connected = false;
      this._producer = null;
    }
  }
}

const pool = new ProducerPool();

/**
 * Publishes a single event to a Kafka topic.
 *
 * The producer connection is kept alive across calls (singleton pool).
 * Disconnect by calling `ProduceEvent.disconnect()` during graceful shutdown.
 *
 * @param {string} topic   - Kafka topic name (1–249 chars, [a-zA-Z0-9._-]).
 * @param {string} event   - Event name used as the message key prefix (1–100 chars).
 * @param {Object} [data={}]    - Payload object serialised to JSON.
 * @param {Object} [headers={}] - Kafka message headers (string values).
 * @param {Object} [options={}] - Per-call overrides.
 * @param {string} [options.partitionKey]   - Custom partition key (defaults to `key-<event>`).
 * @param {string} [options.compression]    - Override compression for this message ('none'|'gzip'|'snappy'|'lz4'|'zstd').
 * @param {string} [options.correlationId]  - Adds a `correlation-id` header for distributed tracing.
 * @return {Promise<Array>} Broker acknowledgement array from KafkaJS.
 * @throws {Error} Validation or broker errors.
 *
 * @example
 * const result = await ProduceEvent(
 *   'order.placed',
 *   'ORDER_PLACED',
 *   { orderId: 'ORD-001', total: 149.99 },
 *   { 'x-source': 'checkout-service' },
 *   { correlationId: 'req-abc-123' }
 * );
 */
const ProduceEvent = async (topic, event, data = {}, headers = {}, options = {}) => {
  validateTopic(topic);
  validateEvent(event);
  validateData(data);
  validateHeaders(headers);

  const { partitionKey, compression, correlationId } = options;

  const resolvedHeaders = {
    ...headers,
    ...(correlationId ? { 'correlation-id': correlationId } : {}),
    'produced-at': new Date().toISOString(),
  };

  const compressionType = COMPRESSION_MAP[compression || config.kafka_compression_type] || CompressionTypes.None;

  const producer = await pool.get();

  const response = await producer.send({
    topic,
    compression: compressionType,
    messages: [
      {
        key: partitionKey || `key-${event}`,
        value: JSON.stringify({ event, data, timestamp: Date.now() }),
        headers: resolvedHeaders,
      },
    ],
  });

  return response;
};

/**
 * Disconnects the shared producer. Call during graceful shutdown
 * if you manage the lifecycle yourself.
 */
ProduceEvent.disconnect = () => pool.close();

module.exports = ProduceEvent;