kafka-pub-sub/examples/audit/auditConsumer.js

Enterprise-grade Kafka publish/subscribe library for Node.js — producer pool, batch sending, DLQ, SSL/SASL, multi-broker, and real-world industry examples.

/**
 * Compliance & Audit Logging — Consumer
 *
 * A compliance audit consumer that:
 *  - Persists every event to a write-once data store (simulated)
 *  - Detects security anomalies in real time
 *  - Tracks GDPR data-subject access requests (DSAR)
 *  - Never uses DLQ (audit events must not be silently discarded)
 *
 * In production this would write to:
 *  - AWS S3 / GCS (long-term immutable storage)
 *  - Elasticsearch (searchable audit index)
 *  - PostgreSQL with append-only table (WORM-equivalent)
 *
 * Run:
 *   node examples/audit/auditConsumer.js
 */

require('dotenv').config({ path: require('path').resolve(__dirname, '../../.env') });

const { ConsumeEvent } = require('../../index');

// eslint-disable-next-line no-console
const log = (service, ...args) =>
  console.log(`[${new Date().toISOString()}] [${service}]`, ...args);

// Simulated in-memory audit store (use S3/Postgres/Elasticsearch in production)
const auditStore = [];
const DSAR_REQUESTS = new Set(['CUST-7890']); // data subjects with active access requests

const persistToStore = async (event) => {
  auditStore.push({ ...event, _persistedAt: new Date().toISOString() });
  // In production: INSERT INTO audit_log ... (append-only, no DELETE/UPDATE)
};

// ── User action consumer ──────────────────────────────────────────────────────
async function startUserActionConsumer() {
  return ConsumeEvent(
    'audit.user-action',
    async (msg) => {
      const { actor, action, resource, outcome, occurredAt } = msg.value.data || msg.value;

      await persistToStore({ topic: 'audit.user-action', actor, action, resource, outcome, occurredAt });

      log('USER-AUDIT', `${actor.id} → ${action} on ${resource.type}:${resource.id} → ${outcome}`);
    },
    {
      groupId: 'audit-consumer-user-actions',
      fromBeginning: true,
      retry: 10, // audit consumers get many retries
      dlq: false, // audit events must NEVER be dropped — surface error instead
    }
  );
}

// ── Data access consumer (GDPR) ───────────────────────────────────────────────
async function startDataAccessConsumer() {
  return ConsumeEvent(
    'audit.data-access',
    async (msg) => {
      const payload = msg.value.data || msg.value;
      const { actor, action, resource, outcome, occurredAt, metadata } = payload;

      await persistToStore({ topic: 'audit.data-access', ...payload });

      log(
        'DATA-AUDIT',
        `${actor.id} (${actor.type}) ${action} ${resource.type}:${resource.id || 'query'} → ${outcome} | basis=${metadata?.legalBasis || 'n/a'}`
      );

      // GDPR DSAR detection: if the accessed resource matches a data subject
      // with an active access request, flag it for DSAR report generation
      const subjectId = resource.id || resource.targetCustomerId;
      if (DSAR_REQUESTS.has(subjectId)) {
        log(
          'DSAR-TRACKER',
          `⚑ Access to DSAR subject ${subjectId} recorded — event will be included in subject access report`
        );
        // In production: tag record in DSAR processing queue
      }
    },
    {
      groupId: 'audit-consumer-data-access',
      fromBeginning: true,
      retry: 10,
      dlq: false,
    }
  );
}

// ── Security event consumer ───────────────────────────────────────────────────
async function startSecurityConsumer() {
  return ConsumeEvent(
    'audit.security',
    async (msg) => {
      const payload = msg.value.data || msg.value;
      const { actor, action, resource, outcome, metadata } = payload;

      await persistToStore({ topic: 'audit.security', ...payload });

      if (outcome === 'failure') {
        log(
          'SECURITY-ALERT',
          `⚠ SECURITY EVENT: ${actor.id} attempted ${action} on ${resource.type} → ${outcome}`
        );
        if (metadata?.alertSent) {
          log('SECURITY-ALERT', '  → On-call team already notified');
        }
        // In production: push to PagerDuty / Splunk SIEM / Chronicle
      } else {
        log('SECURITY-AUDIT', `${actor.id} → ${action} → ${outcome}`);
      }
    },
    {
      groupId: 'audit-consumer-security',
      fromBeginning: true,
      retry: 10,
      dlq: false,
    }
  );
}

// ── System event consumer (SOC 2) ────────────────────────────────────────────
async function startSystemEventConsumer() {
  return ConsumeEvent(
    'audit.system-event',
    async (msg) => {
      const payload = msg.value.data || msg.value;
      const { actor, action, resource, outcome, metadata } = payload;

      await persistToStore({ topic: 'audit.system-event', ...payload });

      log(
        'SYSTEM-AUDIT',
        `${actor.id} → ${action} ${resource.type}:${resource.id} → ${outcome} | pipeline=${metadata?.deployPipelineId || 'n/a'}`
      );
    },
    {
      groupId: 'audit-consumer-system',
      fromBeginning: true,
      retry: 10,
      dlq: false,
    }
  );
}

// ── Stats reporter ────────────────────────────────────────────────────────────
function startStatsReporter() {
  return setInterval(() => {
    log('STATS', `Audit records persisted this session: ${auditStore.length}`);
  }, 15000);
}

// ── Main ──────────────────────────────────────────────────────────────────────
async function main() {
  log('SYSTEM', 'Starting compliance audit consumers...');

  const [stopUser, stopData, stopSecurity, stopSystem] = await Promise.all([
    startUserActionConsumer(),
    startDataAccessConsumer(),
    startSecurityConsumer(),
    startSystemEventConsumer(),
  ]);

  const statsTimer = startStatsReporter();

  log('SYSTEM', '✓ All audit consumers running (Ctrl+C to stop)');

  const shutdown = async () => {
    clearInterval(statsTimer);
    log('STATS', `Total audit records persisted: ${auditStore.length}`);
    await Promise.all([stopUser(), stopData(), stopSecurity(), stopSystem()]);
    process.exit(0);
  };

  process.on('SIGTERM', shutdown);
  process.on('SIGINT', shutdown);
}

main().catch((err) => {
  // eslint-disable-next-line no-console
  console.error('Audit consumer startup failed:', err);
  process.exit(1);
});