kafka-retry
Version:
Handle kafka non-blocking retries and dead letter topics for nestjs microservice
207 lines • 9.73 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.KafkaStrategy = void 0;
const microservices_1 = require("@nestjs/microservices");
const rxjs_1 = require("rxjs");
const constants_1 = require("./constants");
const kafka_admin_1 = require("./kafka-admin");
const retry_metadata_global_1 = require("./retry-metadata.global");
const utils_1 = require("./utils");
const version_1 = require("./version");
class KafkaStrategy extends microservices_1.ServerKafka {
constructor(options) {
super(options);
this.options = options;
}
async listen(callback) {
this.client = this.createClient();
await this.start(callback);
}
async start(callback) {
const consumerOptions = Object.assign(this.options.consumer || {}, {
groupId: this.groupId,
});
this.consumer = this.client.consumer(consumerOptions);
this.producer = this.client.producer(this.options.producer);
await this.consumer.connect();
await this.producer.connect();
await this.bindRetryEvents(this.consumer);
await this.bindEvents(this.consumer);
callback();
}
async bindEvents(consumer) {
var _a;
const registeredPatterns = [...this.messageHandlers.keys()];
const consumerSubscribeOptions = this.options.subscribe || {};
const subscribeToPattern = async (pattern) => consumer.subscribe(Object.assign({ topic: pattern }, consumerSubscribeOptions));
await Promise.all(registeredPatterns.map(subscribeToPattern));
const autoCommit = ((_a = this.options.run) === null || _a === void 0 ? void 0 : _a.autoCommit) || false;
const consumerRunOptions = Object.assign(this.options.run || {}, {
autoCommit,
eachMessage: async (payload) => {
const { rawMessage, isRetry } = this.parseRawMessage(payload);
const { topic, partition } = rawMessage;
const offset = parseInt(rawMessage.offset) + 1;
if (isRetry) {
const remainingTime = this.getRemainingTimeToProcess(rawMessage);
if (remainingTime > 0) {
this.consumer.pause([{ topic, partitions: [partition] }]);
setTimeout(() => {
this.consumer.resume([{ topic, partitions: [partition] }]);
}, remainingTime);
setTimeout(async () => {
await this.handleMessage(rawMessage);
this.consumer.commitOffsets([
{ topic: topic, partition, offset: `${offset}` },
]);
}, remainingTime);
return;
}
}
await this.handleMessage(rawMessage);
this.consumer.commitOffsets([
{ topic: topic, partition, offset: `${offset}` },
]);
},
});
await consumer.run(consumerRunOptions);
}
getRemainingTimeToProcess(rawMessage) {
const { timestamp, headers } = rawMessage;
const delay = headers.delay;
return parseInt(timestamp) + parseInt(delay) - +new Date();
}
parseRawMessage(payload) {
const rawMessage = this.parser.parse(Object.assign(payload.message, {
topic: payload.topic,
partition: payload.partition,
}));
const isRetry = rawMessage.headers.tried &&
rawMessage.headers.delay &&
!rawMessage.headers.isCompleted;
return { isRetry, rawMessage };
}
async handleMessage(rawMessage) {
const { topic, partition, headers } = rawMessage;
console.log(`**** Handle message - Topic: ${topic} - Partition: ${partition} ***** ${new Date().toLocaleString()} `);
console.log(rawMessage.value);
const correlationId = headers[microservices_1.KafkaHeaders.CORRELATION_ID];
const replyTopic = headers[microservices_1.KafkaHeaders.REPLY_TOPIC];
const replyPartition = headers[microservices_1.KafkaHeaders.REPLY_PARTITION];
const packet = await this.deserializer.deserialize(rawMessage, {
channel: topic,
});
const kafkaContext = new microservices_1.KafkaContext([rawMessage, partition, topic]);
if (!correlationId || !replyTopic) {
return this.handleEvent(packet.pattern, packet, kafkaContext);
}
const publish = this.getPublisher(replyTopic, replyPartition, correlationId);
const handler = this.getHandlerByPattern(packet.pattern);
if (!handler) {
return publish({
id: correlationId,
err: constants_1.NO_MESSAGE_HANDLER,
});
}
const response$ = this.transformToObservable(await handler(packet.data, kafkaContext));
response$ && this.send(response$, publish);
}
async handleEvent(pattern, packet, context) {
const posRetryChar = pattern.indexOf(constants_1.TopicSuffixingStrategy.RETRY_SUFFIX);
if (posRetryChar !== -1) {
pattern = pattern.substring(0, posRetryChar);
}
const handler = this.getHandlerByPattern(pattern);
if (!handler) {
return this.logger.error(`${constants_1.NO_EVENT_HANDLER} Event pattern: ${JSON.stringify(pattern)}.`);
}
try {
const resultOrStream = await handler(packet.data, context);
if ((0, rxjs_1.isObservable)(resultOrStream)) {
resultOrStream.subscribe({
error: (error) => {
console.log(error);
const headers = packet.data.headers;
const payload = packet.data.value;
if (!(headers === null || headers === void 0 ? void 0 : headers.isCompleted)) {
this.handleRetry(pattern, headers, payload);
}
},
});
const connectableSource = (0, rxjs_1.connectable)(resultOrStream, {
connector: () => new rxjs_1.Subject(),
resetOnDisconnect: false,
});
connectableSource.connect();
}
}
catch (error) {
console.log('handle error', error);
}
}
handleRetry(pattern, headers, payload) {
var _a, _b, _c, _d, _e, _f;
const handlerPattern = this.getHandlerByPattern(pattern);
let retry = ((_a = handlerPattern === null || handlerPattern === void 0 ? void 0 : handlerPattern.extras) === null || _a === void 0 ? void 0 : _a.retry) || null;
if (!(0, version_1.isGTEV8_3_1)()) {
retry = (_b = (0, retry_metadata_global_1.getRetryMetadataByKey)(pattern)['retry']) !== null && _b !== void 0 ? _b : null;
}
if (!retry || (retry === null || retry === void 0 ? void 0 : retry.attempts) === 0)
return;
const initialDelay = ((_c = retry.backoff) === null || _c === void 0 ? void 0 : _c.delay) || constants_1.KAFKA_DEFAULT_DELAY;
let multiplier = 1;
if (parseInt(headers === null || headers === void 0 ? void 0 : headers.tried) >= 1) {
multiplier = ((_d = retry === null || retry === void 0 ? void 0 : retry.backoff) === null || _d === void 0 ? void 0 : _d.multiplier) || constants_1.KAFKA_DEFAULT_MULTIPLIER;
}
let nextTopic;
if ((headers === null || headers === void 0 ? void 0 : headers.tried) >= retry.attempts) {
headers.isCompleted = '1';
nextTopic = (0, utils_1.getDeadTopicName)(pattern);
}
else {
const tried = (_e = headers === null || headers === void 0 ? void 0 : headers.tried) !== null && _e !== void 0 ? _e : '0';
const delay = (_f = headers === null || headers === void 0 ? void 0 : headers.delay) !== null && _f !== void 0 ? _f : `${initialDelay}`;
headers = {
tried: `${parseInt(tried) + 1}`,
delay: `${parseInt(delay) * multiplier}`,
};
nextTopic = (0, utils_1.getRetryTopicName)(pattern, headers.tried);
}
this.producer.send({
topic: nextTopic,
messages: [
{
value: JSON.stringify(payload),
headers,
},
],
});
}
async bindRetryEvents(consumer) {
const kafkaAdmin = new kafka_admin_1.KafkaAdmin(this.client.admin());
const handler = this.getHandlers();
for (const [key, value] of handler.entries()) {
if (!(0, version_1.isGTEV8_3_1)()) {
value['extras'] = (0, retry_metadata_global_1.getRetryMetadataByKey)(key);
}
if (value.extras.retry && value.extras.retry.attempts > 0) {
const retryTopics = await kafkaAdmin.createRetryTopics(key, value.extras.retry);
const subscribeToPattern = async (pattern) => {
return consumer.subscribe({
topic: pattern,
});
};
await Promise.all(retryTopics.map(subscribeToPattern));
}
}
}
async close() {
this.consumer && (await this.consumer.disconnect());
this.producer && (await this.producer.disconnect());
this.consumer = null;
this.producer = null;
this.client = null;
}
}
exports.KafkaStrategy = KafkaStrategy;
//# sourceMappingURL=kafka-strategy.js.map