kafka-pub-sub
Version:
Enterprise-grade Kafka publish/subscribe library for Node.js — producer pool, batch sending, DLQ, SSL/SASL, multi-broker, and real-world industry examples.
207 lines (186 loc) • 6.58 kB
JavaScript
/**
* Compliance & Audit Logging — Producer
*
* Demonstrates an immutable audit trail suitable for:
* - GDPR (user data access logging)
* - SOC 2 (system event recording)
* - PCI-DSS (cardholder data environment activity)
* - HIPAA (PHI access auditing)
*
* Every event is:
* - Timestamped at the source (not the broker)
* - Stamped with actor, resource, action, and outcome
* - Correlated across services via correlationId + sessionId
* - Compressed (gzip) to minimise broker storage cost
*
* Topics follow a consistent schema: `audit.<domain>`
* Retention policy (set via KafkaAdmin): 7 years for financial, 1 year for general.
*
* Run:
* node examples/audit/auditProducer.js
*/
require('dotenv').config({ path: require('path').resolve(__dirname, '../../.env') });
const { BatchProduceEvent, KafkaAdmin } = require('../../index');
const AUDIT_TOPICS = {
USER_ACTION: 'audit.user-action',
DATA_ACCESS: 'audit.data-access',
SYSTEM_EVENT: 'audit.system-event',
SECURITY: 'audit.security',
};
// eslint-disable-next-line no-console
const log = (...args) => console.log(`[${new Date().toISOString()}]`, ...args);
/**
* Provision audit topics with long retention before producing.
* This would run once during service deployment, not on every startup.
*/
async function provisionAuditTopics() {
const admin = KafkaAdmin();
log('Provisioning audit topics with 7-year retention...');
await admin.createTopics(
[
{
topic: AUDIT_TOPICS.USER_ACTION,
numPartitions: 12,
replicationFactor: 3,
configEntries: {
'retention.ms': String(7 * 365 * 24 * 60 * 60 * 1000), // 7 years
'cleanup.policy': 'delete',
'compression.type': 'gzip',
},
},
{
topic: AUDIT_TOPICS.DATA_ACCESS,
numPartitions: 12,
replicationFactor: 3,
configEntries: {
'retention.ms': String(7 * 365 * 24 * 60 * 60 * 1000),
'cleanup.policy': 'delete',
'compression.type': 'gzip',
},
},
{
topic: AUDIT_TOPICS.SYSTEM_EVENT,
numPartitions: 6,
replicationFactor: 3,
configEntries: {
'retention.ms': String(365 * 24 * 60 * 60 * 1000), // 1 year
'cleanup.policy': 'delete',
},
},
{
topic: AUDIT_TOPICS.SECURITY,
numPartitions: 6,
replicationFactor: 3,
configEntries: {
'retention.ms': String(7 * 365 * 24 * 60 * 60 * 1000),
'cleanup.policy': 'delete',
},
},
],
{ waitForLeaders: true }
);
await admin.disconnect();
log('✓ Audit topics provisioned');
}
/**
* Builds a compliant audit event envelope.
*/
const buildAuditEvent = ({ topic, event, actor, action, resource, outcome, metadata = {} }) => ({
topic,
event,
data: {
// Who
actor: { id: actor.id, type: actor.type, ip: actor.ip, userAgent: actor.userAgent },
// What
action,
resource,
outcome, // 'success' | 'failure' | 'partial'
// When (source timestamp — broker timestamp is also recorded by Kafka)
occurredAt: new Date().toISOString(),
// Context
metadata,
// Compliance
schemaVersion: '2.0',
},
headers: {
'x-source': metadata.serviceName || 'unknown-service',
'content-type': 'application/json',
},
partitionKey: actor.id, // partition by actor for ordered per-user audit trails
});
async function publishAuditBurst() {
const sessionId = `SES-${Math.random().toString(36).slice(2, 10).toUpperCase()}`;
const correlationId = `CORR-${Date.now()}`;
const events = [
// User logs in
buildAuditEvent({
topic: AUDIT_TOPICS.USER_ACTION,
event: 'USER_LOGIN',
actor: { id: 'USR-1042', type: 'human', ip: '203.0.113.45', userAgent: 'Mozilla/5.0' },
action: 'AUTHENTICATE',
resource: { type: 'session', id: sessionId },
outcome: 'success',
metadata: { serviceName: 'auth-service', mfaUsed: true },
}),
// User accesses PII data (GDPR-relevant)
buildAuditEvent({
topic: AUDIT_TOPICS.DATA_ACCESS,
event: 'PII_ACCESS',
actor: { id: 'USR-1042', type: 'human', ip: '203.0.113.45', userAgent: 'Mozilla/5.0' },
action: 'READ',
resource: { type: 'customer-profile', id: 'CUST-7890', fields: ['name', 'email', 'dob'] },
outcome: 'success',
metadata: { serviceName: 'crm-service', legalBasis: 'legitimate_interest', sessionId },
}),
// Service-to-service data access (API call)
buildAuditEvent({
topic: AUDIT_TOPICS.DATA_ACCESS,
event: 'API_DATA_ACCESS',
actor: { id: 'SVC-reporting', type: 'service', ip: '10.0.1.5', userAgent: 'reporting-service/2.1' },
action: 'EXPORT',
resource: { type: 'payment-records', query: 'date_range=2024-Q4', recordCount: 15432 },
outcome: 'success',
metadata: { serviceName: 'reporting-service', approvedBy: 'USR-admin-001', ticketId: 'JIRA-4521' },
}),
// Failed admin action (security-relevant)
buildAuditEvent({
topic: AUDIT_TOPICS.SECURITY,
event: 'PRIVILEGE_ESCALATION_ATTEMPT',
actor: { id: 'USR-0099', type: 'human', ip: '198.51.100.22', userAgent: 'curl/7.88' },
action: 'ASSIGN_ROLE',
resource: { type: 'user-role', targetUserId: 'USR-0099', role: 'super-admin' },
outcome: 'failure',
metadata: { serviceName: 'iam-service', reason: 'Insufficient privileges', alertSent: true },
}),
// System config change (SOC 2)
buildAuditEvent({
topic: AUDIT_TOPICS.SYSTEM_EVENT,
event: 'CONFIG_CHANGED',
actor: { id: 'SVC-deploy', type: 'service', ip: '10.0.0.1', userAgent: 'deployer/1.0' },
action: 'UPDATE',
resource: { type: 'feature-flag', id: 'new-checkout-flow', previousValue: false, newValue: true },
outcome: 'success',
metadata: { serviceName: 'config-service', deployPipelineId: 'PIPE-88321', rollbackAvailable: true },
}),
];
await BatchProduceEvent(events, {
compression: 'gzip',
correlationId,
});
log(`✓ Audit burst of ${events.length} events published (correlationId: ${correlationId})`);
}
async function main() {
try {
await provisionAuditTopics();
} catch (err) {
log('Topic provisioning skipped (topics may already exist):', err.message);
}
await publishAuditBurst();
await BatchProduceEvent.disconnect();
log('Done.');
}
main().catch((err) => {
// eslint-disable-next-line no-console
console.error('Audit producer failed:', err);
process.exit(1);
});