kafka-ts/dist/producer/producer.js

**KafkaTS** is a Apache Kafka client library for Node.js. It provides both a low-level API for communicating directly with the Apache Kafka cluster and high-level APIs for publishing and subscribing to Kafka topics.

"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Producer = void 0;
const api_1 = require("../api");
const group_by_leader_id_1 = require("../distributors/group-by-leader-id");
const partitioner_1 = require("../distributors/partitioner");
const metadata_1 = require("../metadata");
const error_1 = require("../utils/error");
const logger_1 = require("../utils/logger");
const promise_chain_1 = require("../utils/promise-chain");
const shared_1 = require("../utils/shared");
const tracer_1 = require("../utils/tracer");
const producer_buffer_1 = require("./producer-buffer");
const producer_state_1 = require("./producer-state");
const trace = (0, tracer_1.createTracer)('Producer');
class Producer {
    cluster;
    options;
    metadata;
    state;
    partition;
    chain = new promise_chain_1.PromiseChain();
    bufferByNodeId = {};
    constructor(cluster, options) {
        this.cluster = cluster;
        this.options = {
            ...options,
            allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
            partitioner: options.partitioner ?? partitioner_1.defaultPartitioner,
            maxBatchSize: options.maxBatchSize ?? 500,
        };
        this.metadata = new metadata_1.Metadata({ cluster });
        this.state = new producer_state_1.ProducerState({ cluster });
        this.partition = this.options.partitioner({ metadata: this.metadata });
    }
    async send(messages) {
        await this.ensureProducerInitialized();
        const topics = [...new Set(messages.map((message) => message.topic))];
        await this.fetchMetadataForTopics(topics);
        const partitionedMessages = messages.map((message) => {
            message.partition = this.partition(message);
            return message;
        });
        const messagesByLeaderId = (0, group_by_leader_id_1.groupByLeaderId)(partitionedMessages, this.metadata.getTopicPartitionLeaderIds());
        await Promise.all(Object.entries(messagesByLeaderId).map(async ([leaderId, messages]) => {
            const nodeId = parseInt(leaderId);
            const buffer = (this.bufferByNodeId[nodeId] ??= new producer_buffer_1.ProducerBuffer({
                nodeId,
                maxBatchSize: this.options.maxBatchSize,
                cluster: this.cluster,
                state: this.state,
            }));
            try {
                await buffer.enqueue(messages);
            }
            catch (error) {
                await this.handleError(error);
                return this.send(messages);
            }
        }));
    }
    async close() {
        await this.cluster.disconnect();
    }
    ensureProducerInitialized = (0, shared_1.shared)(async () => {
        await this.cluster.ensureConnected();
        if (!this.state.producerId) {
            await this.state.initProducerId();
        }
    });
    fetchMetadataForTopics = (0, shared_1.shared)(async (topics) => {
        const { allowTopicAutoCreation } = this.options;
        await this.chain.run(topics.map((topic) => `metadata:${topic}`), () => this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation }));
    });
    async handleError(error) {
        await (0, api_1.handleApiError)(error).catch(async (error) => {
            if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER) {
                logger_1.log.debug('Refreshing metadata', { reason: error.message });
                const topics = Object.keys(this.metadata.getTopicPartitions());
                await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation: false });
                return;
            }
            if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OUT_OF_ORDER_SEQUENCE_NUMBER) {
                logger_1.log.debug('Out of order sequence number. Reinitializing producer ID');
                await this.state.initProducerId();
                return;
            }
            throw error;
        });
    }
}
exports.Producer = Producer;
__decorate([
    trace(() => ({ root: true })),
    __metadata("design:type", Function),
    __metadata("design:paramtypes", [Array]),
    __metadata("design:returntype", Promise)
], Producer.prototype, "send", null);