@alexknips/nestjs-kafka-events
Version:
Lightweight, tested, straight-forward wrapper around KafkaJS and Confluent's Schema Registry.
216 lines • 10.1 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.KafkaService = void 0;
const common_1 = require("@nestjs/common");
const kafkajs_1 = require("kafkajs");
const deserializer_1 = require("./deserializer");
const retry_helper_1 = require("./helpers/retry.helper");
const topic_subject_helper_1 = require("./helpers/topic-subject.helper");
const kafka_event_functions_service_1 = require("./kafka-event-functions.service");
const loggers_1 = require("./loggers");
const providers_1 = require("./providers");
const serializer_1 = require("./serializer");
let KafkaService = class KafkaService {
constructor(kafkaModuleConfigurationProvider, kafkaEventFunctionsService, kafkaLogger, kafkaSerializer, kafkaDeserializer) {
this.kafkaModuleConfigurationProvider = kafkaModuleConfigurationProvider;
this.kafkaEventFunctionsService = kafkaEventFunctionsService;
this.kafkaLogger = kafkaLogger;
this.kafkaSerializer = kafkaSerializer;
this.kafkaDeserializer = kafkaDeserializer;
this.topicOffsets = new Map();
this.config = kafkaModuleConfigurationProvider.get();
this.kafka = new kafkajs_1.Kafka(Object.assign(Object.assign({}, this.config.client), { logCreator: this.kafkaLogger.getKafkaJSLoggingAdapter.bind(this.kafkaLogger) }));
if (this.config.consumer) {
this.consumer = this.kafka.consumer(this.config.consumer);
}
if (this.config.producer) {
this.producer = this.kafka.producer(this.config.producer);
}
this.admin = this.kafka.admin();
}
async connectToKafka() {
if (this.producer) {
await this.producer.connect();
}
if (this.consumer) {
await this.consumer.connect();
}
await this.admin.connect();
}
async connectToSchemaRegistry() {
var _a, _b;
const eventEmitterTopics = (_a = this.kafkaEventFunctionsService.getEventEmitterTopics()) !== null && _a !== void 0 ? _a : [];
if (!!this.producer !== eventEmitterTopics.length > 0) {
this.kafkaLogger.error(`When having producer config set, event emitters need to be registered too (and vice versa)`);
}
if (this.producer) {
await this.kafkaSerializer.initialize(this.config.schemaRegistry, eventEmitterTopics);
}
if (this.consumer) {
const sTopics = (_b = this.kafkaEventFunctionsService.getEventHandlerTopics()) !== null && _b !== void 0 ? _b : [];
let subjectProbe = undefined;
if (sTopics.length > 0) {
subjectProbe = (0, topic_subject_helper_1.getSchemaRegistryValueSubjectByTopic)(sTopics[Math.floor(Math.random() * sTopics.length)]);
}
await this.kafkaDeserializer.initialize(this.config.schemaRegistry, subjectProbe);
}
}
async disconnect() {
if (this.producer) {
await this.producer.disconnect();
}
if (this.consumer) {
await this.consumer.stop();
await this.consumer.disconnect();
}
await this.admin.disconnect();
}
async onApplicationBootstrap() {
await this.connectToKafka();
await (0, retry_helper_1.retry)(async () => await this.connectToSchemaRegistry(), 5, 2);
await this.fetchTopicOffsets();
await this.subscribeToTopics();
await this.bindEventHandlers();
}
async onApplicationShutdown() {
await this.disconnect();
}
async fetchTopicOffsets() {
var e_1, _a;
try {
for (var _b = __asyncValues(this.kafkaEventFunctionsService.getEventHandlerTopics()), _c; _c = await _b.next(), !_c.done;) {
const topic = _c.value;
try {
const topicOffsets = await this.admin.fetchTopicOffsets(topic);
this.topicOffsets.set(topic, topicOffsets);
}
catch (reject) {
this.kafkaLogger.error(`Error while fetching topic offset for ${topic}`, reject);
throw reject;
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (_c && !_c.done && (_a = _b.return)) await _a.call(_b);
}
finally { if (e_1) throw e_1.error; }
}
}
async subscribeToTopics() {
var e_2, _a;
if (!this.consumer) {
return;
}
try {
for (var _b = __asyncValues(this.kafkaEventFunctionsService.getEventHandlerTopics()), _c; _c = await _b.next(), !_c.done;) {
const topic = _c.value;
try {
await this.consumer.subscribe({ topic, fromBeginning: false });
}
catch (reject) {
this.kafkaLogger.error(`Error while subscribing to topic ${topic}`, reject);
throw reject;
}
}
}
catch (e_2_1) { e_2 = { error: e_2_1 }; }
finally {
try {
if (_c && !_c.done && (_a = _b.return)) await _a.call(_b);
}
finally { if (e_2) throw e_2.error; }
}
}
bindEventHandlers() {
var _a, _b;
if (!this.consumer) {
return;
}
this.consumer.run(Object.assign(Object.assign({}, ((_b = (_a = this.config) === null || _a === void 0 ? void 0 : _a.consumerRun) !== null && _b !== void 0 ? _b : {})), { eachMessage: async ({ topic, message }) => {
var _a, _b, _c, _d;
try {
await (0, retry_helper_1.retry)(async () => {
const event = await this.kafkaDeserializer.deserialize(message);
await this.kafkaEventFunctionsService.callEventHandler(topic, event);
}, 5, (_d = (_c = (_b = (_a = this.config) === null || _a === void 0 ? void 0 : _a.consumer) === null || _b === void 0 ? void 0 : _b.retry) === null || _c === void 0 ? void 0 : _c.retries) !== null && _d !== void 0 ? _d : 2, (reason) => {
this.kafkaLogger.warn(`Retrying to process message on topic ${topic}.`, reason);
});
}
catch (reject) {
this.kafkaLogger.error(`Error while processing message on topic ${topic}.`, reject);
}
} }));
}
async emit(payload, failSilent = true) {
var e_3, _a;
if (!this.producer) {
this.kafkaLogger.error(`Cannot emit event(s) on topic(s) ${Array.isArray(payload)
? payload.map((p) => p.topic).join(', ')
: payload.topic}`);
return;
}
try {
if (Array.isArray(payload)) {
const topicMessages = [];
try {
for (var payload_1 = __asyncValues(payload), payload_1_1; payload_1_1 = await payload_1.next(), !payload_1_1.done;) {
const p = payload_1_1.value;
topicMessages.push({
messages: [await this.kafkaSerializer.serialize(p)],
topic: p.topic,
});
}
}
catch (e_3_1) { e_3 = { error: e_3_1 }; }
finally {
try {
if (payload_1_1 && !payload_1_1.done && (_a = payload_1.return)) await _a.call(payload_1);
}
finally { if (e_3) throw e_3.error; }
}
await this.producer.sendBatch({
topicMessages,
});
return;
}
await this.producer.send({
topic: payload.topic,
messages: [await this.kafkaSerializer.serialize(payload)],
});
}
catch (reject) {
this.kafkaLogger.error(`Error while emitting event(s): ${JSON.stringify(payload)}`, reject);
if (!failSilent) {
throw reject;
}
}
}
};
KafkaService = __decorate([
(0, common_1.Injectable)(),
__metadata("design:paramtypes", [providers_1.KafkaModuleConfigurationProvider,
kafka_event_functions_service_1.KafkaEventFunctionsService,
loggers_1.KafkaLogger,
serializer_1.KafkaSerializer,
deserializer_1.KafkaDeserializer])
], KafkaService);
exports.KafkaService = KafkaService;
//# sourceMappingURL=kafka.service.js.map