@awesomeniko/kafka-trail/dist/message-queue/index.js

A Node.js library for managing message queue with Kafka

import { context, SpanKind, trace } from "@opentelemetry/api";
import { pino } from "pino";
import { ArgumentIsRequired, NoHandlersError } from "../custom-errors/kafka-errors.js";
import { KTKafkaConsumer } from "../kafka/kafka-consumer.js";
import { KTKafkaProducer } from "../kafka/kafka-producer.js";
import { KafkaTopicName } from "../libs/branded-types/kafka/index.js";
class KTMessageQueue {
    #registeredHandlers = new Map();
    #ktProducer;
    #ktConsumer;
    #logger = console;
    #ctx;
    constructor(params) {
        let ctx = params?.ctx();
        if (!ctx) {
            ctx = {};
        }
        if (!ctx.logger) {
            ctx.logger = pino();
        }
        this.#ctx = ctx;
    }
    getConsumer() {
        return this.#ktConsumer;
    }
    getProducer() {
        return this.#ktProducer;
    }
    async initProducer(params) {
        const { kafkaSettings: { brokerUrls } } = params;
        if (!brokerUrls || !brokerUrls.length) {
            throw new ArgumentIsRequired('brokerUrls');
        }
        this.#ktProducer = new KTKafkaProducer({ ...params, logger: this.#ctx.logger });
        await this.#ktProducer.init();
    }
    async initConsumer(params) {
        const registeredHandlers = [...this.#registeredHandlers.values()];
        if (registeredHandlers.length === 0) {
            throw new NoHandlersError('subscribe to consumer');
        }
        this.#ktConsumer = new KTKafkaConsumer({ ...params, logger: this.#ctx.logger });
        await this.#ktConsumer.init();
        await this.#subscribeAll();
    }
    async destroyAll() {
        await Promise.all([
            this.destroyProducer(),
            this.destroyConsumer(),
        ]);
    }
    async destroyProducer() {
        if (this.#ktProducer) {
            await this.#ktProducer.destroy();
        }
    }
    async destroyConsumer() {
        if (this.#ktConsumer) {
            await this.#ktConsumer.destroy();
        }
    }
    async #subscribeAll() {
        const topicNames = [...this.#registeredHandlers.values()].map(item => item.topic.topicSettings.topic);
        await this.#ktConsumer.subscribeTopic(topicNames);
        await this.#ktConsumer.consumer.run({
            eachBatchAutoResolve: false,
            partitionsConsumedConcurrently: 1,
            eachBatch: async (eachBatchPayload) => {
                const tracer = trace.getTracer(`kafka-trail`, '1.0.0');
                const span = tracer.startSpan(`kafka-trail: eachBatch`, {
                    kind: SpanKind.CONSUMER,
                    attributes: {
                        'messaging.system': 'kafka',
                        'messaging.destination': topicNames,
                    },
                });
                await context.with(trace.setSpan(context.active(), span), async () => {
                    const { batch: { topic, messages, partition } } = eachBatchPayload;
                    const topicName = KafkaTopicName.fromString(topic);
                    const handler = this.#registeredHandlers.get(topicName);
                    if (handler) {
                        const batchedValues = [];
                        let lastOffset = undefined;
                        for (const message of messages) {
                            if (batchedValues.length < handler.topic.topicSettings.batchMessageSizeToConsume) {
                                if (message.value) {
                                    const decodedMessage = handler.topic.decode(message.value);
                                    batchedValues.push(decodedMessage);
                                    lastOffset = message.offset;
                                }
                            }
                            else {
                                break;
                            }
                        }
                        await handler.run(batchedValues, this.#ctx, this, {
                            partition,
                            lastOffset,
                        });
                        if (lastOffset) {
                            eachBatchPayload.resolveOffset(lastOffset);
                        }
                    }
                    await eachBatchPayload.heartbeat();
                    span.end();
                });
            },
        });
    }
    async initTopics(topicEvents) {
        if (!this.#ktProducer) {
            throw new Error("Producer field is required");
        }
        for (const topicEvent of topicEvents) {
            await this.#ktProducer.createTopic(topicEvent.topicSettings.topic, topicEvent.topicSettings.numPartitions, topicEvent.topicSettings.configEntries);
        }
    }
    getRegisteredHandler(topic) {
        return this.#registeredHandlers.get(topic);
    }
    registerHandlers(mqHandlers) {
        for (const handler of mqHandlers) {
            if (!this.#registeredHandlers.has(handler.topic.topicSettings.topic)) {
                this.#registeredHandlers.set(handler.topic.topicSettings.topic, handler);
            }
            else {
                this.#logger.warn(`Attempting to register an already registered handler ${handler.topic.topicSettings.topic}`);
            }
        }
    }
    publishSingleMessage(topic) {
        return this.#ktProducer.sendSingleMessage({
            topicName: topic.topicName,
            message: topic.message,
            messageKey: topic.messageKey,
        }, topic.meta);
    }
}
export { KTMessageQueue };
//# sourceMappingURL=index.js.map