@joktec/kafka
Version:
JokTec - Kafka Service
150 lines • 7.68 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.KafkaService = void 0;
const core_1 = require("@joktec/core");
const utils_1 = require("@joktec/utils");
const kafkajs_1 = require("kafkajs");
const lodash_1 = require("lodash");
const kafka_config_1 = require("./kafka.config");
const kafka_metric_1 = require("./kafka.metric");
const RETRY_OPTS = 'kafka.retry';
let KafkaService = class KafkaService extends core_1.AbstractClientService {
constructor(kafkaMetricService) {
super('kafka', kafka_config_1.KafkaConfig);
this.kafkaMetricService = kafkaMetricService;
this.props = {};
}
async init(config) {
config.log(this.logService);
this.props[config.conId] = { producers: {}, consumers: {} };
return new kafkajs_1.Kafka({ ...config });
}
async start(client, conId = core_1.DEFAULT_CON_ID) {
}
async stop(client, conId = core_1.DEFAULT_CON_ID) {
const props = this.props[conId];
for (const producerKey of Object.keys(props.producers)) {
await props.producers[producerKey].disconnect();
this.logService.info('`%s` [%s] disconnect', conId, producerKey);
}
for (const consumerKey of Object.keys(props.consumers)) {
await props.consumers[consumerKey].disconnect();
this.logService.info('`%s` [%s] disconnect', conId, consumerKey);
}
}
async initProducer(key, cfg = {}, conId = core_1.DEFAULT_CON_ID) {
let producer = this.props[conId].producers[key];
if (!producer) {
this.props[conId].producers[key] = producer = this.getClient(conId).producer({
createPartitioner: kafkajs_1.Partitioners.LegacyPartitioner,
...cfg,
});
this.logService.info(cfg, '`%s` [%s] created', conId, key);
await producer.connect();
this.logService.info('`%s` [%s] connected', conId, key);
}
return producer;
}
async initConsumer(groupId, cfg = {}, conId = core_1.DEFAULT_CON_ID) {
let consumer = this.props[conId].consumers[groupId];
if (!consumer) {
this.props[conId].consumers[groupId] = consumer = this.getClient(conId).consumer({
...cfg,
groupId,
});
this.logService.info(cfg, '`%s` [%s] created', conId, groupId);
await consumer.connect();
this.logService.info('`%s` [%s] connected', conId, groupId);
}
return consumer;
}
async consume(topics, groupId, callback, options = {}, conId = core_1.DEFAULT_CON_ID) {
const consumer = await this.initConsumer(groupId, options.config, conId);
await consumer.subscribe({ topics, fromBeginning: (0, utils_1.toBool)(options.fromBeginning, false) });
topics.map(t => this.logService.info('`%s` [%s] of topic `%s` ready to consume', conId, groupId, t));
const onMessageFn = async (payload) => {
const key = `${payload.topic}-${payload.partition}`;
const content = payload.message.value?.toString();
try {
this.logService.debug('`%s` [%s] kafka consumed message: %s', conId, groupId, content);
await callback(payload);
this.kafkaMetricService.receive('consume', kafka_metric_1.KafkaMetricStatus.SUCCESS, key, conId);
}
catch (error) {
this.logService.error(error, '`%s` [%s] kafka handle message fail', conId, groupId);
this.kafkaMetricService.receive('consume', kafka_metric_1.KafkaMetricStatus.ERROR, key, conId);
}
};
await consumer.run({ ...options.runConfig, eachMessage: onMessageFn });
}
async consumeBatch(topics, groupId, callback, options = {}, conId = core_1.DEFAULT_CON_ID) {
const consumer = await this.initConsumer(groupId, options.config, conId);
await consumer.subscribe({ topics, fromBeginning: (0, utils_1.toBool)(options.fromBeginning, false) });
topics.map(t => this.logService.info('`%s` [%s] of topic `%s` ready to batch consume', conId, groupId, t));
const onMessageFn = async (payload) => {
const key = `${payload.batch.topic}-${payload.batch.partition}`;
const totalContent = payload.batch.messages.length;
try {
this.logService.debug('`%s` [%s] kafka consumed %s message(s)', conId, groupId, totalContent);
await callback(payload);
this.kafkaMetricService.receive('consumeBatch', kafka_metric_1.KafkaMetricStatus.SUCCESS, key, conId);
}
catch (error) {
const msg = '`%s` [%s] kafka handle %s message(s) fail';
this.logService.error(error, msg, conId, groupId, totalContent);
this.kafkaMetricService.receive('consumeBatch', kafka_metric_1.KafkaMetricStatus.ERROR, key, conId);
}
};
await consumer.run({ ...options.runConfig, eachBatch: onMessageFn });
}
async send(topic, messages, options = {}, conId = core_1.DEFAULT_CON_ID) {
const key = options.record?.producerKey ?? topic;
const producer = await this.initProducer(key, options.config, conId);
await producer.send({
...options.record,
topic,
messages: messages.map(msg => ((0, lodash_1.isBuffer)(msg) || (0, lodash_1.isString)(msg) ? { value: msg } : msg)),
});
}
async sendBatch(batch, options = {}, conId = core_1.DEFAULT_CON_ID) {
const producer = await this.initProducer(batch.producerKey, options.config, conId);
await producer.sendBatch(batch);
}
};
exports.KafkaService = KafkaService;
__decorate([
(0, core_1.Retry)(RETRY_OPTS),
__metadata("design:type", Function),
__metadata("design:paramtypes", [kafka_config_1.KafkaConfig]),
__metadata("design:returntype", Promise)
], KafkaService.prototype, "init", null);
__decorate([
(0, kafka_metric_1.KafkaSendMetric)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [String, Array, Object, String]),
__metadata("design:returntype", Promise)
], KafkaService.prototype, "send", null);
__decorate([
(0, kafka_metric_1.KafkaSendMetric)(),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Object, Object, String]),
__metadata("design:returntype", Promise)
], KafkaService.prototype, "sendBatch", null);
exports.KafkaService = KafkaService = __decorate([
(0, core_1.Injectable)(),
__param(0, (0, core_1.Inject)()),
__metadata("design:paramtypes", [kafka_metric_1.KafkaMetricService])
], KafkaService);
//# sourceMappingURL=kafka.service.js.map