kafka-pub-sub/examples/financial/paymentConsumer.js

Enterprise-grade Kafka publish/subscribe library for Node.js — producer pool, batch sending, DLQ, SSL/SASL, multi-broker, and real-world industry examples.

/**
 * Financial Services — Payment Consumer
 *
 * Demonstrates:
 *  - Consuming payment transactions with retry + DLQ
 *  - Consuming fraud alerts separately with dedicated consumer group
 *  - Reading from DLQ for manual review / replay
 *  - Consumer lag monitoring via KafkaAdmin
 *
 * DLQ behaviour:
 *   If the handler throws after all retries, the message is automatically
 *   routed to `payment.transaction.dlq`. A separate consumer listens there
 *   for manual reconciliation or automated replay.
 *
 * Run:
 *   node examples/financial/paymentConsumer.js
 */

require('dotenv').config({ path: require('path').resolve(__dirname, '../../.env') });

const { ConsumeEvent, KafkaAdmin } = require('../../index');

// eslint-disable-next-line no-console
const log = (service, ...args) =>
  console.log(`[${new Date().toISOString()}] [${service}]`, ...args);

// ── Payment processor ─────────────────────────────────────────────────────────
async function startPaymentProcessor() {
  return ConsumeEvent(
    'payment.transaction',
    async (msg) => {
      const tx = msg.value.data || msg.value;

      log('PAYMENT-PROC', `Processing TX ${tx.transactionId} | ${tx.currency} ${tx.amount}`);

      // Simulate occasional processing failure (for DLQ demo)
      if (tx.riskScore > 0.95) {
        throw new Error(`Blocked high-risk transaction ${tx.transactionId} (score=${tx.riskScore?.toFixed(3)})`);
      }

      // In production: write to ledger DB, call clearing APIs, etc.
      await new Promise((r) => setTimeout(r, 20));

      log('PAYMENT-PROC', `✓ TX ${tx.transactionId} processed successfully`);
    },
    {
      groupId: 'payment-processor',
      fromBeginning: false,
      retry: 3,
      dlq: true, // failed messages go to payment.transaction.dlq
    }
  );
}

// ── Fraud alert handler ───────────────────────────────────────────────────────
async function startFraudAlertHandler() {
  return ConsumeEvent(
    'payment.fraud-alert',
    async (msg) => {
      const alert = msg.value.data || msg.value;
      log(
        'FRAUD-HANDLER',
        `ALERT: TX ${alert.transactionId} flagged | score=${alert.riskScore?.toFixed(3)} | manual=${alert.requiresManualReview}`
      );
      // In production: create case in fraud review system, notify compliance team
    },
    {
      groupId: 'fraud-review',
      fromBeginning: false,
      retry: 5,
      dlq: false, // fraud alerts must never be silently dropped
    }
  );
}

// ── DLQ consumer — dead-letter review ────────────────────────────────────────
async function startDlqReviewer() {
  return ConsumeEvent(
    'payment.transaction.dlq',
    async (msg) => {
      const originalTopic = msg.headers['dlq-original-topic'] || 'unknown';
      const failedAt = msg.headers['dlq-failed-at'] || 'unknown';
      const error = msg.headers['dlq-error'] || 'unknown';

      log('DLQ-REVIEWER', `Dead-lettered message from [${originalTopic}]`);
      log('DLQ-REVIEWER', `  Failed at: ${failedAt}`);
      log('DLQ-REVIEWER', `  Reason:    ${error}`);
      log('DLQ-REVIEWER', `  Payload:   ${JSON.stringify(msg.value).slice(0, 120)}…`);

      // In production: store to reconciliation table, alert on-call, attempt replay
    },
    {
      groupId: 'dlq-reviewer',
      fromBeginning: true, // always read from beginning to catch all failed messages
    }
  );
}

// ── Settlement consumer ───────────────────────────────────────────────────────
async function startSettlementConsumer() {
  return ConsumeEvent(
    'payment.settled',
    async (msg) => {
      const s = msg.value.data || msg.value;
      log('SETTLEMENT', `Confirmed settlement ${s.transactionId} | ref=${s.clearingHouseRef}`);
    },
    { groupId: 'settlement-recorder', fromBeginning: false }
  );
}

// ── Consumer lag monitor ──────────────────────────────────────────────────────
async function logConsumerLag() {
  const admin = KafkaAdmin();
  try {
    const lag = await admin.getConsumerGroupOffsets('payment-processor', 'payment.transaction');
    log('LAG-MONITOR', 'payment-processor lag:', JSON.stringify(lag));
  } catch (err) {
    log('LAG-MONITOR', 'Could not fetch lag (topic may not exist yet):', err.message);
  } finally {
    await admin.disconnect();
  }
}

// ── Main ──────────────────────────────────────────────────────────────────────
async function main() {
  log('SYSTEM', 'Starting financial consumers...');

  const stops = await Promise.all([
    startPaymentProcessor(),
    startFraudAlertHandler(),
    startDlqReviewer(),
    startSettlementConsumer(),
  ]);

  log('SYSTEM', '✓ All consumers running (Ctrl+C to stop)');

  // Log consumer lag every 30 seconds
  const lagInterval = setInterval(logConsumerLag, 30000);

  const shutdown = async () => {
    clearInterval(lagInterval);
    await Promise.all(stops.map((s) => s()));
    process.exit(0);
  };
  process.on('SIGTERM', shutdown);
  process.on('SIGINT', shutdown);
}

main().catch((err) => {
  // eslint-disable-next-line no-console
  console.error('Financial consumer startup failed:', err);
  process.exit(1);
});