kafka-nestjs
Version:
A structured Kafka integration for NestJS inspired by BullMQ, offering decorators, modular configuration, and a scalable consumer/producer architecture.
198 lines • 8.59 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
var KafkaExplorer_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.KafkaExplorer = void 0;
const core_1 = require("@nestjs/core");
const kafkajs_1 = require("kafkajs");
const common_1 = require("@nestjs/common");
const kafka_constants_1 = require("../constants/kafka.constants");
const kafka_metadata_accessor_1 = require("./kafka-metadata.accessor");
let KafkaExplorer = KafkaExplorer_1 = class KafkaExplorer {
constructor(discoveryService, metadataScanner, metadataAccessor, options) {
this.discoveryService = discoveryService;
this.metadataScanner = metadataScanner;
this.metadataAccessor = metadataAccessor;
this.options = options;
this.logger = new common_1.Logger(KafkaExplorer_1.name);
this.consumers = [];
this.initialized = false;
this.kafka = new kafkajs_1.Kafka(this.options);
}
async ensureTopicExists(topic) {
try {
const admin = this.kafka.admin();
await admin.connect();
const topics = await admin.listTopics();
if (!topics.includes(topic)) {
await admin.createTopics({
topics: [{
topic,
numPartitions: 1,
replicationFactor: 1,
}],
});
}
await admin.disconnect();
}
catch (error) {
this.logger.error({
message: `Failed to ensure topic exists`,
info: {
topic,
error: Object.assign({ message: error.message, stack: error.stack, name: error.name }, error),
},
});
throw error;
}
}
async onModuleInit() {
await this.explore();
}
async onModuleDestroy() {
await this.destroy();
}
async explore() {
if (this.initialized) {
return;
}
const topicPromises = [];
const consumerPromises = [];
const providers = this.discoveryService.getProviders()
.filter((wrapper) => {
var _a;
return this.metadataAccessor.isProcessor(!wrapper.metatype || wrapper.inject
? (_a = wrapper.instance) === null || _a === void 0 ? void 0 : _a.constructor
: wrapper.metatype);
});
console.log('providers', providers.length);
providers.forEach((wrapper) => {
const { instance } = wrapper;
const prototype = Object.getPrototypeOf(instance);
const methods = this.metadataScanner.getAllMethodNames(prototype)
.filter(methodName => typeof instance[methodName] === 'function' && methodName !== 'constructor')
.map(methodName => prototype[methodName]);
methods.forEach((method) => {
const kafkaOptions = this.metadataAccessor.getConsumerOptionsMetadata(method);
if (kafkaOptions) {
try {
for (const topic of kafkaOptions.subscribe.topics) {
if (typeof topic === 'string') {
topicPromises.push(this.ensureTopicExists(topic));
}
}
consumerPromises.push(this.bindConsumer(kafkaOptions, method.bind(instance)));
}
catch (error) {
if (error instanceof kafkajs_1.KafkaJSProtocolError) {
this.logger.error({
message: `Kafka protocol error for method`,
info: {
methodName: method.name,
error: Object.assign({ stack: error.stack }, error),
},
});
}
else {
this.logger.error({
message: `Error binding consumer for method`,
info: {
methodName: method.name,
error: Object.assign({ message: error.message, stack: error.stack, name: error.name }, error),
},
});
}
}
}
});
});
await Promise.all(topicPromises);
await Promise.all(consumerPromises);
this.initialized = true;
}
async bindConsumer(options, handler) {
const consumer = this.kafka.consumer(options.consumerConfig);
this.consumers.push(consumer);
try {
await consumer.connect();
await consumer.subscribe(options.subscribe);
await consumer.run({
eachMessage: async (payload) => {
var _a;
const value = (_a = payload.message.value) === null || _a === void 0 ? void 0 : _a.toString();
try {
const parsed = value ? JSON.parse(value) : null;
await handler.call(handler, parsed, payload);
}
catch (error) {
this.logger.error({
message: `Kafka message handler error for topic`,
info: {
error: Object.assign({ message: error.message, stack: error.stack, name: error.name }, error),
options,
},
}, this.bindConsumer.name);
}
},
});
this.logger.log({
message: `Kafka consumer registered for topic`,
info: {
options,
},
}, this.bindConsumer.name);
}
catch (error) {
this.logger.error({
message: `Failed to bind consumer`,
info: {
error: Object.assign({ message: error.message, stack: error.stack, name: error.name }, error),
options,
},
}, this.bindConsumer.name);
throw error;
}
}
async destroy() {
await Promise.all(this.consumers.map(async (consumer) => {
try {
await consumer.disconnect();
this.logger.log({
message: 'Kafka consumer disconnected',
info: {
consumer,
},
}, this.destroy.name);
}
catch (error) {
this.logger.error({
message: 'Error disconnecting Kafka consumer',
info: {
error: Object.assign({ message: error.message, stack: error.stack, name: error.name }, error),
consumer,
},
}, this.destroy.name);
}
}));
}
};
exports.KafkaExplorer = KafkaExplorer;
exports.KafkaExplorer = KafkaExplorer = KafkaExplorer_1 = __decorate([
(0, common_1.Injectable)(),
__param(3, (0, common_1.Inject)(kafka_constants_1.KAFKA_MODULE_OPTIONS)),
__metadata("design:paramtypes", [core_1.DiscoveryService,
core_1.MetadataScanner,
kafka_metadata_accessor_1.KafkaMetadataAccessor, Object])
], KafkaExplorer);
//# sourceMappingURL=kafka.explorer.js.map