@awesomeniko/kafka-trail
Version:
A Node.js library for managing message queue with Kafka
390 lines • 16.4 kB
JavaScript
import { clearInterval } from "node:timers";
import pino from "pino";
import { ArgumentIsRequired, NoHandlersError, ProducerInitRequiredForDLQError, ProducerNotInitializedError } from "../custom-errors/kafka-errors.js";
import { KTKafkaConsumer } from "../kafka/kafka-consumer.js";
import { KTKafkaProducer } from "../kafka/kafka-producer.js";
import { DLQKTTopic } from "../kafka/topic.js";
import { KafkaMessageKey, KafkaTopicName } from "../libs/branded-types/kafka/index.js";
import { createHandlerTraceAttributes } from "../libs/helpers/observability.js";
class KTMessageQueue {
#registeredHandlers = new Map();
#ktProducer;
#ktConsumer;
#ctx;
#addPayloadToTrace = false;
#otel;
constructor(params) {
let ctx = params?.ctx();
if (!ctx) {
ctx = {};
}
if (!ctx.logger) {
ctx.logger = pino();
}
this.#ctx = ctx;
this.#addPayloadToTrace = params?.tracingSettings?.addPayloadToTrace ?? false;
this.#otel = params?.tracingSettings?.otel;
}
getConsumer() {
return this.#ktConsumer;
}
getProducer() {
return this.#ktProducer;
}
getAdmin() {
return this.#ktProducer?.getAdmin();
}
#requireConsumer() {
if (!this.#ktConsumer) {
throw new Error("Consumer is not initialized");
}
return this.#ktConsumer;
}
#requireProducer() {
if (!this.#ktProducer) {
throw new ProducerNotInitializedError();
}
return this.#ktProducer;
}
#extractErrorMessage(err) {
if (err instanceof Error) {
this.#ctx.logger.error(err);
return err.message;
}
return '';
}
async #withSpan(name, options, run) {
if (!this.#otel) {
return run();
}
const span = this.#otel.trace
.getTracer("kafka-trail", "1.0.0")
.startSpan(name, options);
return this.#otel.context.with(this.#otel.trace.setSpan(this.#otel.context.active(), span), async () => run(span));
}
async #publishToDlq(params) {
const Topic = DLQKTTopic(params.handler.topic.topicSettings);
const Payload = Topic({
originalOffset: params.originalOffset,
originalTopic: params.originalTopic,
originalPartition: params.originalPartition,
key: params.key,
value: params.value,
errorMessage: params.errorMessage,
failedAt: Date.now(),
}, {
messageKey: KafkaMessageKey.NULL,
meta: {},
});
await this.publishSingleMessage(Payload);
}
async #runHandlerWithTracing(params) {
const attributes = createHandlerTraceAttributes({
topicName: params.topicName,
partition: params.partition,
lastOffset: params.lastOffset,
batchedValues: params.batchedValues,
payloadContentLength: params.payloadContentLength,
opts: {
addPayloadToTrace: this.#addPayloadToTrace,
},
});
await this.#withSpan(`kafka-trail: handler ${params.topicName}`, {
kind: this.#otel?.SpanKind.CONSUMER ?? 0,
attributes,
}, async (handlerSpan) => {
try {
await params.handler.run(params.batchedValues, this.#ctx, this, params.kafkaTopicParams);
}
catch (err) {
const errorMessage = this.#extractErrorMessage(err);
if (params.handler.topic.topicSettings.createDLQ) {
await this.#publishToDlq({
handler: params.handler,
originalOffset: params.lastOffset,
originalTopic: params.topicName,
originalPartition: params.partition,
key: params.failedKey,
value: params.batchedValues,
errorMessage,
});
}
else {
throw err;
}
}
finally {
handlerSpan?.end();
}
});
}
#getRawPayloadContentLength(value) {
if (!value) {
return 0;
}
if (Buffer.isBuffer(value)) {
return value.byteLength;
}
return Buffer.byteLength(value, "utf8");
}
async initProducer(params) {
const { kafkaSettings: { brokerUrls } } = params;
if (!brokerUrls || !brokerUrls.length) {
throw new ArgumentIsRequired('brokerUrls');
}
this.#ktProducer = new KTKafkaProducer({ ...params, logger: this.#ctx.logger });
await this.#ktProducer.init();
}
async initConsumer(params) {
const { kafkaSettings: { brokerUrls }, } = params;
if (!brokerUrls || !brokerUrls.length) {
throw new ArgumentIsRequired("brokerUrls");
}
const registeredHandlers = [...this.#registeredHandlers.values()];
if (registeredHandlers.length === 0) {
throw new NoHandlersError('subscribe to consumer');
}
const hasDlqHandlers = registeredHandlers.some((handler) => handler.topic.topicSettings.createDLQ);
if (hasDlqHandlers && !this.#ktProducer) {
throw new ProducerInitRequiredForDLQError();
}
this.#ktConsumer = new KTKafkaConsumer({ ...params, logger: this.#ctx.logger });
await this.#ktConsumer.init();
if (params.kafkaSettings.batchConsuming) {
await this.#subscribeAll();
}
else {
await this.#subscribeAllEachMessages();
}
}
async destroyAll() {
await Promise.all([
this.destroyProducer(),
this.destroyConsumer(),
]);
}
async destroyProducer() {
if (this.#ktProducer) {
await this.#ktProducer.destroy();
}
}
async destroyConsumer() {
if (this.#ktConsumer) {
await this.#ktConsumer.destroy();
}
}
async #subscribeAllEachMessages() {
const topicNames = [...this.#registeredHandlers.values()].map(item => item.topic.topicSettings.topic);
const consumer = this.#requireConsumer();
await consumer.subscribeTopic(topicNames);
await consumer.consumer.run({
partitionsConsumedConcurrently: 1,
eachMessage: async (eachMessagePayload) => {
await this.#withSpan(`kafka-trail: eachMessage`, {
kind: this.#otel?.SpanKind.CONSUMER ?? 0,
attributes: {
'messaging.system': 'kafka',
'messaging.destination': topicNames,
},
}, async (eachMessageSpan) => {
try {
const { topic, message, partition } = eachMessagePayload;
const topicName = KafkaTopicName.fromString(topic);
const handler = this.#registeredHandlers.get(topicName);
if (handler) {
const batchedValues = [];
let lastOffset = undefined;
let payloadContentLength = 0;
if (message.value) {
payloadContentLength = this.#getRawPayloadContentLength(message.value);
const decodedMessage = handler.topic.decode(message.value);
batchedValues.push(decodedMessage);
lastOffset = message.offset;
}
await this.#runHandlerWithTracing({
handler,
topicName,
partition,
lastOffset,
batchedValues,
payloadContentLength,
kafkaTopicParams: {
partition,
lastOffset,
heartBeat: () => eachMessagePayload.heartbeat(),
},
failedKey: KafkaMessageKey.fromString(message.key?.toString()),
});
}
}
finally {
eachMessageSpan?.end();
}
});
},
});
}
async #subscribeAll() {
const topicNames = [...this.#registeredHandlers.values()].map(item => item.topic.topicSettings.topic);
const consumer = this.#requireConsumer();
await consumer.subscribeTopic(topicNames);
await consumer.consumer.run({
eachBatchAutoResolve: false,
partitionsConsumedConcurrently: 1,
eachBatch: async (eachBatchPayload) => {
await this.#withSpan(`kafka-trail: eachBatch`, {
kind: this.#otel?.SpanKind.CONSUMER ?? 0,
attributes: {
'messaging.system': 'kafka',
'messaging.destination': topicNames,
},
}, async (eachBatchSpan) => {
try {
const { batch: { topic, messages, partition } } = eachBatchPayload;
const topicName = KafkaTopicName.fromString(topic);
const handler = this.#registeredHandlers.get(topicName);
if (handler) {
const heartbeatIntervalMs = consumer.heartBeatInterval - Math.floor(consumer.heartBeatInterval * consumer.heartbeatEarlyFactor);
const heartBeatInterval = setInterval(() => {
void this.#withSpan(`kafka-trail: manual-heartbeat`, {
kind: this.#otel?.SpanKind.CONSUMER ?? 0,
attributes: {
'messaging.system': 'kafka',
'messaging.destination': topicNames,
},
}, async (heartbeatSpan) => {
try {
await eachBatchPayload.heartbeat();
}
catch (err) {
this.#ctx.logger.error(err);
}
finally {
heartbeatSpan?.end();
}
});
}, heartbeatIntervalMs);
try {
const batchedValues = [];
let lastOffset = undefined;
let payloadContentLength = 0;
for (const message of messages) {
if (batchedValues.length < handler.topic.topicSettings.batchMessageSizeToConsume) {
if (message.value) {
payloadContentLength += this.#getRawPayloadContentLength(message.value);
const decodedMessage = handler.topic.decode(message.value);
batchedValues.push(decodedMessage);
lastOffset = message.offset;
}
}
else {
break;
}
}
await this.#runHandlerWithTracing({
handler,
topicName,
partition,
lastOffset,
batchedValues,
payloadContentLength,
kafkaTopicParams: {
partition,
lastOffset,
heartBeat: () => eachBatchPayload.heartbeat(),
resolveOffset: (offset) => eachBatchPayload.resolveOffset(offset),
},
failedKey: KafkaMessageKey.fromString(JSON.stringify(messages.map(m => m.key?.toString()))),
});
if (lastOffset) {
eachBatchPayload.resolveOffset(lastOffset);
}
}
finally {
clearInterval(heartBeatInterval);
}
}
await eachBatchPayload.heartbeat();
}
finally {
eachBatchSpan?.end();
}
});
},
});
}
async initTopics(topicEvents) {
const producer = this.#requireProducer();
for (const topicEvent of topicEvents) {
if (!topicEvent) {
throw new Error("Attemt to create topic that doesn't exists (null, instead of KTTopicEvent)");
}
await producer.createTopic(topicEvent.topicSettings);
}
}
getRegisteredHandler(topic) {
return this.#registeredHandlers.get(topic);
}
registerHandlers(mqHandlers) {
for (const handler of mqHandlers) {
if (!this.#registeredHandlers.has(handler.topic.topicSettings.topic)) {
this.#registeredHandlers.set(handler.topic.topicSettings.topic, handler);
}
else {
this.#ctx.logger.warn(`Attempting to register an already registered handler ${handler.topic.topicSettings.topic}`);
}
}
}
publishSingleMessage(topic) {
const producer = this.#ktProducer;
if (!producer) {
return Promise.reject(new ProducerNotInitializedError());
}
return this.#withSpan(`kafka-trail: publishSingleMessage ${topic.topicName}`, {
kind: this.#otel?.SpanKind.PRODUCER ?? 0,
}, async (span) => {
try {
const res = await producer.sendSingleMessage({
topicName: topic.topicName,
value: topic.message,
messageKey: topic.messageKey,
headers: topic.meta ?? {},
});
span?.end();
return res;
}
catch (error) {
span?.recordException(error);
span?.setStatus({ code: this.#otel?.SpanStatusCode.ERROR ?? 2, message: String(error) });
span?.end();
throw error;
}
});
}
publishBatchMessages(topic) {
const producer = this.#ktProducer;
if (!producer) {
return Promise.reject(new ProducerNotInitializedError());
}
return this.#withSpan(`kafka-trail: publishBatchMessages ${topic.topicName}`, {
kind: this.#otel?.SpanKind.PRODUCER ?? 0,
attributes: {
messageSize: topic.messages.length,
},
}, async (span) => {
try {
const res = await producer.sendBatchMessages(topic);
span?.end();
return res;
}
catch (error) {
span?.recordException(error);
span?.setStatus({ code: this.#otel?.SpanStatusCode.ERROR ?? 2, message: String(error) });
span?.end();
throw error;
}
});
}
}
export { KTMessageQueue };
//# sourceMappingURL=index.js.map