@vpriem/kafka-broker
Version:
Easily compose and manage your kafka resources in one place
195 lines • 7.21 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Subscription = void 0;
const events_1 = __importDefault(require("events"));
const kafkajs_1 = require("kafkajs");
const decodeMessage_1 = require("./decodeMessage");
const BrokerError_1 = require("./BrokerError");
class Subscription extends events_1.default {
consumer;
publisher;
config;
registry;
handlers = [];
aliasToTopic = {};
topicToHandlers = {};
isRunning;
constructor(consumer, publisher, config, registry) {
super({ captureRejections: true });
this.consumer = consumer;
this.publisher = publisher;
this.config = config;
this.registry = registry;
this.consumer.on('consumer.crash', ({ payload: { error } }) => {
if (error instanceof kafkajs_1.KafkaJSNonRetriableError) {
this.emit('error', error);
}
});
if (this.config.handler) {
this.handlers.push(this.config.handler);
}
this.aliasToTopic = Object.fromEntries(this.config.topics.map(({ topic, alias }) => [
alias || topic.toString(),
topic.toString(),
]));
this.topicToHandlers = Object.fromEntries(this.config.topics.map(({ topic, handler }) => [
topic.toString(),
handler ? [handler] : [],
]));
}
addHandler(handler, topicOrAlias) {
if (topicOrAlias) {
const topic = this.aliasToTopic[topicOrAlias] || topicOrAlias;
if (!this.topicToHandlers[topic]) {
throw new BrokerError_1.BrokerError(`Unknown topic or alias "${topicOrAlias}"`);
}
this.topicToHandlers[topic].push(handler);
}
else {
this.handlers.push(handler);
}
return this;
}
removeHandler(handler, topicOrAlias) {
if (topicOrAlias) {
const topic = this.aliasToTopic[topicOrAlias] || topicOrAlias;
if (!this.topicToHandlers[topic]) {
throw new BrokerError_1.BrokerError(`Unknown topic or alias "${topicOrAlias}"`);
}
this.topicToHandlers[topic] = this.topicToHandlers[topic].filter((h) => h !== handler);
}
else {
this.handlers = this.handlers.filter((h) => h !== handler);
}
return this;
}
on(event, listener) {
if (typeof event === 'string' && event.startsWith('message')) {
return this.addHandler(listener, event.split('.')[1]);
}
return super.on(event, listener);
}
off(event, listener) {
if (typeof event === 'string' && event.startsWith('message')) {
return this.removeHandler(listener, event.split('.')[1]);
}
return super.off(event, listener);
}
async consumeMessage(payload) {
const { contentType, deadLetter } = this.config;
const publish = this.publisher.publish.bind(this.publisher);
const { message, topic } = payload;
try {
const value = await (0, decodeMessage_1.decodeMessage)(message, this.registry, contentType);
const handlers = this.topicToHandlers[topic]
? [...this.handlers, ...this.topicToHandlers[topic]]
: this.handlers;
await Promise.all(handlers.map((handler) => handler(value, payload, publish)));
}
catch (error) {
if (deadLetter) {
/**
* Cannot emit error without triggering KafkaJs retry mechanism :/
*/
// this.emit('error', error);
await publish(deadLetter, {
value: {
...payload,
error: error.message,
},
});
return;
}
throw error;
}
}
async eachBatchByPartitionKey({ batch, isRunning, isStale, heartbeat, pause, resolveOffset, commitOffsetsIfNecessary, }) {
const { topic, partition } = batch;
const messagesByKey = {};
batch.messages.forEach((message) => {
const key = message.key?.toString() || 'no-key';
if (!messagesByKey[key]) {
messagesByKey[key] = [];
}
messagesByKey[key].push(message);
});
await Promise.all(Object.values(messagesByKey).map(async (messages) => {
// eslint-disable-next-line no-restricted-syntax
for (const message of messages) {
if (!isRunning() || isStale())
break;
// eslint-disable-next-line no-await-in-loop
await this.consumeMessage({
topic,
partition,
message,
heartbeat,
pause,
});
resolveOffset(message.offset);
// eslint-disable-next-line no-await-in-loop
await heartbeat();
// eslint-disable-next-line no-await-in-loop
await commitOffsetsIfNecessary();
}
}));
}
async eachBatch({ batch, isRunning, isStale, heartbeat, pause, resolveOffset, commitOffsetsIfNecessary, }) {
const { topic, partition } = batch;
await Promise.all(batch.messages.map(async (message) => {
if (!isRunning() || isStale())
return;
await this.consumeMessage({
topic,
partition,
message,
heartbeat,
pause,
});
resolveOffset(message.offset);
await heartbeat();
await commitOffsetsIfNecessary();
}));
}
async subscribe() {
await this.consumer.connect();
await Promise.all(this.config.topics.map(({ alias, handler, ...topicConfig }) => this.consumer.subscribe(topicConfig)));
}
async subscribeAndRun() {
const { runConfig, parallelism } = this.config;
await this.subscribe();
if (parallelism === 'by-partition-key') {
await this.consumer.run({
...runConfig,
eachBatch: this.eachBatchByPartitionKey.bind(this),
});
}
else if (parallelism === 'all-at-once') {
await this.consumer.run({
...runConfig,
eachBatch: this.eachBatch.bind(this),
});
}
else {
await this.consumer.run({
...runConfig,
eachMessage: this.consumeMessage.bind(this),
});
}
return this;
}
async run() {
if (typeof this.isRunning === 'undefined') {
this.isRunning = this.subscribeAndRun();
}
return this.isRunning;
}
async disconnect() {
return this.consumer.disconnect();
}
}
exports.Subscription = Subscription;
//# sourceMappingURL=Subscription.js.map