kafka-pub-sub/config/config.js

Enterprise-grade Kafka publish/subscribe library for Node.js — producer pool, batch sending, DLQ, SSL/SASL, multi-broker, and real-world industry examples.

const Joi = require('joi');

// ── Read process.env directly ─────────────────────────────────────────────────
// This is a library. We read process.env as-is.
// It is the consuming application's responsibility to populate env vars
// (via dotenv, platform secrets, Docker/Kubernetes env, etc.).

const envVarsSchema = Joi.object()
  .keys({
    // ── Core (required) ───────────────────────────────────────────────────────
    KAFKA_CLIENT_ID: Joi.string().required().description('Kafka client identifier'),
    KAFKA_BROKER_URLS: Joi.string().required().description('Comma-separated broker list, e.g. b1:9092,b2:9092'),
    KAFKA_GROUP_ID: Joi.string().required().description('Consumer group ID'),

    // ── SSL ───────────────────────────────────────────────────────────────────
    KAFKA_SSL_ENABLED: Joi.boolean().default(false).description('Enable TLS/SSL'),
    KAFKA_SSL_CA_PATH: Joi.string().optional().description('Path to CA certificate file'),
    KAFKA_SSL_KEY_PATH: Joi.string().optional().description('Path to client key file'),
    KAFKA_SSL_CERT_PATH: Joi.string().optional().description('Path to client certificate file'),
    KAFKA_SSL_REJECT_UNAUTHORIZED: Joi.boolean().default(true).description('Reject unauthorised TLS certs'),

    // ── SASL ──────────────────────────────────────────────────────────────────
    KAFKA_SASL_MECHANISM: Joi.string()
      .valid('plain', 'scram-sha-256', 'scram-sha-512', 'oauthbearer')
      .optional()
      .description('SASL mechanism'),
    KAFKA_SASL_USERNAME: Joi.string().optional().description('SASL username'),
    KAFKA_SASL_PASSWORD: Joi.string().optional().description('SASL password'),

    // ── Producer ──────────────────────────────────────────────────────────────
    KAFKA_COMPRESSION_TYPE: Joi.string()
      .valid('none', 'gzip', 'snappy', 'lz4', 'zstd')
      .default('none')
      .description('Message compression codec'),
    KAFKA_IDEMPOTENT: Joi.boolean().default(false).description('Enable idempotent producer (exactly-once)'),
    KAFKA_TRANSACTIONAL_ID: Joi.string().optional().description('Transactional producer ID'),
    KAFKA_AUTO_CREATE_TOPICS: Joi.boolean().default(true).description('Auto-create topics on produce'),

    // ── Retry ─────────────────────────────────────────────────────────────────
    KAFKA_RETRY_INITIAL_TIME: Joi.number().integer().default(300).description('Initial retry delay in ms'),
    KAFKA_RETRY_COUNT: Joi.number().integer().default(8).description('Max connection retries'),

    // ── Consumer ──────────────────────────────────────────────────────────────
    KAFKA_SESSION_TIMEOUT: Joi.number().integer().default(30000).description('Consumer session timeout in ms'),
    KAFKA_HEARTBEAT_INTERVAL: Joi.number().integer().default(3000).description('Consumer heartbeat interval in ms'),
    KAFKA_MAX_WAIT_TIME: Joi.number().integer().default(5000).description('Max wait time per fetch in ms'),

    // ── Logging ───────────────────────────────────────────────────────────────
    KAFKA_LOG_LEVEL: Joi.string()
      .valid('error', 'warn', 'info', 'debug')
      .default('error')
      .description('Internal KafkaJS log level'),
  })
  .unknown();

const { value: envVars, error } = envVarsSchema.prefs({ errors: { label: 'key' } }).validate(process.env);

if (error) {
  throw new Error(`kafka-pub-sub config error: ${error.message}`);
}

module.exports = {
  // Core
  kafka_client_id: envVars.KAFKA_CLIENT_ID,
  kafka_broker_urls: envVars.KAFKA_BROKER_URLS.split(',').map((b) => b.trim()),
  kafka_group_id: envVars.KAFKA_GROUP_ID,

  // SSL
  kafka_ssl_enabled: envVars.KAFKA_SSL_ENABLED,
  kafka_ssl_ca_path: envVars.KAFKA_SSL_CA_PATH,
  kafka_ssl_key_path: envVars.KAFKA_SSL_KEY_PATH,
  kafka_ssl_cert_path: envVars.KAFKA_SSL_CERT_PATH,
  kafka_ssl_reject_unauthorized: envVars.KAFKA_SSL_REJECT_UNAUTHORIZED,

  // SASL
  kafka_sasl_mechanism: envVars.KAFKA_SASL_MECHANISM,
  kafka_sasl_username: envVars.KAFKA_SASL_USERNAME,
  kafka_sasl_password: envVars.KAFKA_SASL_PASSWORD,

  // Producer
  kafka_compression_type: envVars.KAFKA_COMPRESSION_TYPE,
  kafka_idempotent: envVars.KAFKA_IDEMPOTENT,
  kafka_transactional_id: envVars.KAFKA_TRANSACTIONAL_ID,
  kafka_auto_create_topics: envVars.KAFKA_AUTO_CREATE_TOPICS,

  // Retry
  kafka_retry_initial_time: envVars.KAFKA_RETRY_INITIAL_TIME,
  kafka_retry_count: envVars.KAFKA_RETRY_COUNT,

  // Consumer
  kafka_session_timeout: envVars.KAFKA_SESSION_TIMEOUT,
  kafka_heartbeat_interval: envVars.KAFKA_HEARTBEAT_INTERVAL,
  kafka_max_wait_time: envVars.KAFKA_MAX_WAIT_TIME,

  // Logging
  kafka_log_level: envVars.KAFKA_LOG_LEVEL,
};