@vpriem/kafka-broker/dist/BatchProducer.js

Easily compose and manage your kafka resources in one place

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.BatchProducer = exports.isProducerBatch = void 0;
const events_1 = __importDefault(require("events"));
const isProducerBatch = (record) => typeof record.topicMessages !== 'undefined';
exports.isProducerBatch = isProducerBatch;
const toSingleTopicMessage = (topic, messages) => messages.flatMap((message) => ({ topic, message }));
const noop = () => {
    /** */
};
class BatchProducer extends events_1.default {
    producer;
    topicMessages = [];
    timer;
    hrTime = process.hrtime();
    isSending = false;
    batchSize;
    batchRequest = Promise.resolve();
    lingerMs;
    config;
    constructor(producer, batchConfig) {
        super({ captureRejections: true });
        this.producer = producer;
        const { size, lingerMs, ...config } = batchConfig;
        this.batchSize = size;
        this.lingerMs = lingerMs;
        this.config = config;
        this.producer.on('producer.connect', 
        /* istanbul ignore next */ () => this.startAutoSendIfNecessary());
        this.producer.on('producer.disconnect', () => this.stopAutoSendIfNecessary());
    }
    get length() {
        return this.topicMessages.length;
    }
    elapsedMs() {
        const [s, ns] = process.hrtime(this.hrTime);
        return s * 1000 + ns / 1000000;
    }
    async sendIfNecessary() {
        if (this.isSending)
            return;
        if (this.topicMessages.length >= this.batchSize) {
            await this.sendAllBatch();
        }
        else if (this.elapsedMs() >= this.lingerMs) {
            await this.sendOneBatch();
        }
    }
    startAutoSendIfNecessary() {
        if (this.timer)
            return;
        this.timer = setInterval(() => {
            this.sendIfNecessary()
                .then(noop)
                .catch(
            /* istanbul ignore next */ (error) => this.emit('error', error));
        }, this.lingerMs);
    }
    stopAutoSendIfNecessary() {
        clearInterval(this.timer);
    }
    push(record) {
        if ((0, exports.isProducerBatch)(record)) {
            this.topicMessages.push(...(record.topicMessages ?? []).flatMap(({ topic, messages }) => toSingleTopicMessage(topic, messages)));
        }
        else {
            this.topicMessages.push(...toSingleTopicMessage(record.topic, record.messages));
        }
        this.sendIfNecessary()
            .then(noop)
            .catch(
        /* istanbul ignore next */ (error) => this.emit('error', error));
        this.startAutoSendIfNecessary();
    }
    createBatch(topicMessages) {
        const messagesByTopic = {};
        topicMessages.forEach(({ topic, message }) => {
            if (!messagesByTopic[topic]) {
                messagesByTopic[topic] = {
                    topic,
                    messages: [],
                };
            }
            messagesByTopic[topic].messages.push(message);
        });
        return {
            ...this.config,
            topicMessages: Object.values(messagesByTopic),
        };
    }
    async sendBatch() {
        await this.batchRequest;
        const messages = this.topicMessages.splice(0, this.batchSize);
        if (!messages.length)
            return;
        this.hrTime = process.hrtime();
        const batch = this.createBatch(messages);
        this.emit('batch.start', batch);
        this.batchRequest = this.producer.sendBatch(batch).catch(
        /* istanbul ignore next */ (error) => {
            this.emit('error', error);
        });
        await this.batchRequest;
        this.batchRequest = Promise.resolve();
    }
    async sendOneBatch() {
        this.isSending = true;
        await this.sendBatch();
        this.isSending = false;
    }
    async sendAllBatch() {
        this.isSending = true;
        while (this.topicMessages.length >= this.batchSize) {
            // eslint-disable-next-line no-await-in-loop
            await this.sendBatch();
        }
        this.isSending = false;
    }
    async flush() {
        this.isSending = true;
        while (this.topicMessages.length) {
            // eslint-disable-next-line no-await-in-loop
            await this.sendBatch();
        }
        this.isSending = false;
    }
}
exports.BatchProducer = BatchProducer;
//# sourceMappingURL=BatchProducer.js.map