kafka-consumer-host
Version:
A NestJS Kafka consumer host module for scalable message consumption.
123 lines (122 loc) • 5.25 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
var KafkaService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.KafkaService = void 0;
const common_1 = require("@nestjs/common");
const kafkajs_1 = require("kafkajs");
const handler_registry_1 = require("./handlers/handler.registry");
const kafka_constants_1 = require("./constants/kafka.constants");
let KafkaService = KafkaService_1 = class KafkaService {
constructor(config) {
this.config = config;
this.logger = new common_1.Logger(KafkaService_1.name);
const { bootstrapServer, clientId, ssl, sasl, ...others } = this.config;
this.kafka = new kafkajs_1.Kafka({
clientId: clientId,
brokers: Array.isArray(bootstrapServer)
? bootstrapServer
: [bootstrapServer],
ssl: ssl,
sasl: sasl,
...others,
});
this.consumer = this.kafka.consumer({
groupId: this.config.groupId,
});
}
async handleCreateTopic() {
var topicsAndPartions = this.config.topicsPartition?.map((topicPartition) => {
const topic = topicPartition.topic;
let numberOfPartitions = topicPartition.partition;
numberOfPartitions = Math.max(1, numberOfPartitions);
this.logger.log(`🚀 Creating topic: ${topic} with partitions:${numberOfPartitions}`);
return {
topic: topic,
numPartitions: numberOfPartitions,
replicationFactor: 1,
};
}) ?? [];
this.logger.log(`🚀 Creating topics: ${JSON.stringify(topicsAndPartions)}`);
// Create topics if they do not exist
const admin = this.kafka.admin();
await admin.connect();
var createRes = await admin.createTopics({
topics: topicsAndPartions.map((tp) => ({
numPartitions: tp.numPartitions,
topic: tp.topic,
replicationFactor: tp.replicationFactor,
})),
});
await admin.disconnect();
this.logger.log(`🚀 Topics created: ${JSON.stringify(topicsAndPartions)}`, createRes);
}
async onApplicationBootstrap() {
await this.consumer.connect();
try {
await this.handleCreateTopic();
}
catch (error) {
this.logger.error("An error occurred while creating topics", error);
}
const fromBeginning = this.config.offsetReset === "earliest";
for (const topic of handler_registry_1.handlerRegistry.keys()) {
this.logger.log(`🚀 Subscribing to topic: ${topic}`);
await this.consumer.subscribe({
topic,
fromBeginning,
});
}
await this.consumer.run({
eachMessage: async ({ topic, partition, message }) => {
const value = message.value?.toString();
let payload;
if (value) {
try {
payload = JSON.parse(value);
}
catch {
payload = value;
}
}
else {
payload = null;
}
const metadata = {
topic,
partition,
offset: message.offset,
timestamp: message.timestamp,
key: message.key?.toString(),
headers: message.headers,
};
this.logger.log(`📥 Message received on topic: ${topic}`);
this.logger.debug(`Payload: ${JSON.stringify(payload)}`);
const handler = handler_registry_1.handlerRegistry.get(topic);
if (handler) {
await handler(payload, metadata);
}
else {
this.logger.warn(`⚠️ No handler registered for topic: ${topic}`);
}
},
});
}
};
exports.KafkaService = KafkaService;
exports.KafkaService = KafkaService = KafkaService_1 = __decorate([
(0, common_1.Injectable)(),
__param(0, (0, common_1.Inject)(kafka_constants_1.KAFKA_CONSUMER_OPTIONS)),
__metadata("design:paramtypes", [Object])
], KafkaService);