kafka-ts/dist/consumer/consumer.js

**KafkaTS** is a Apache Kafka client library for Node.js. It provides both a low-level API for communicating directly with the Apache Kafka cluster and high-level APIs for publishing and subscribing to Kafka topics.

"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Consumer = void 0;
const events_1 = __importDefault(require("events"));
const api_1 = require("../api");
const group_by_leader_id_1 = require("../distributors/group-by-leader-id");
const group_partitions_by_topic_1 = require("../distributors/group-partitions-by-topic");
const delay_1 = require("../utils/delay");
const error_1 = require("../utils/error");
const logger_1 = require("../utils/logger");
const retrier_1 = require("../utils/retrier");
const retry_1 = require("../utils/retry");
const tracer_1 = require("../utils/tracer");
const consumer_group_1 = require("./consumer-group");
const consumer_metadata_1 = require("./consumer-metadata");
const fetch_manager_1 = require("./fetch-manager");
const offset_manager_1 = require("./offset-manager");
const trace = (0, tracer_1.createTracer)('Consumer');
class Consumer extends events_1.default {
    cluster;
    options;
    metadata;
    consumerGroup;
    offsetManager;
    fetchManager;
    stopHook;
    constructor(cluster, options) {
        super();
        this.cluster = cluster;
        this.options = {
            ...options,
            groupId: options.groupId ?? null,
            groupInstanceId: options.groupInstanceId ?? null,
            rackId: options.rackId ?? '',
            sessionTimeoutMs: options.sessionTimeoutMs ?? 30_000,
            rebalanceTimeoutMs: options.rebalanceTimeoutMs ?? 60_000,
            maxWaitMs: options.maxWaitMs ?? 5000,
            minBytes: options.minBytes ?? 1,
            maxBytes: options.maxBytes ?? 1_048_576,
            partitionMaxBytes: options.partitionMaxBytes ?? 1_048_576,
            isolationLevel: options.isolationLevel ?? 1 /* IsolationLevel.READ_COMMITTED */,
            allowTopicAutoCreation: options.allowTopicAutoCreation ?? false,
            fromBeginning: options.fromBeginning ?? false,
            fromTimestamp: options.fromTimestamp ?? (options.fromBeginning ? -2n : -1n),
            retrier: options.retrier ?? retrier_1.defaultRetrier,
        };
        this.metadata = new consumer_metadata_1.ConsumerMetadata({ cluster: this.cluster });
        this.offsetManager = new offset_manager_1.OffsetManager({
            cluster: this.cluster,
            metadata: this.metadata,
            isolationLevel: this.options.isolationLevel,
        });
        this.consumerGroup = this.options.groupId
            ? new consumer_group_1.ConsumerGroup({
                cluster: this.cluster,
                topics: this.options.topics,
                groupId: this.options.groupId,
                groupInstanceId: this.options.groupInstanceId,
                sessionTimeoutMs: this.options.sessionTimeoutMs,
                rebalanceTimeoutMs: this.options.rebalanceTimeoutMs,
                metadata: this.metadata,
                offsetManager: this.offsetManager,
                consumer: this,
            })
            : undefined;
        this.setMaxListeners(Infinity);
    }
    async start() {
        this.stopHook = undefined;
        try {
            await this.cluster.connect();
            await this.fetchMetadata();
            this.metadata.setAssignment(this.metadata.getTopicPartitions());
            await this.fetchOffsets();
            await this.consumerGroup?.init();
        }
        catch (error) {
            logger_1.log.error('Failed to start consumer', error);
            logger_1.log.debug(`Restarting consumer in 1 second...`);
            await (0, delay_1.delay)(1000);
            if (this.stopHook)
                return this.stopHook();
            return this.close(true).then(() => this.start());
        }
        this.startFetchManager();
    }
    async close(force = false) {
        if (!force) {
            await new Promise(async (resolve) => {
                this.stopHook = resolve;
                await this.fetchManager?.stop();
            });
        }
        await this.consumerGroup
            ?.leaveGroup()
            .catch((error) => logger_1.log.debug('Failed to leave group', { reason: error.message }));
        await this.cluster.disconnect().catch(() => { });
    }
    async startFetchManager() {
        const { groupId } = this.options;
        while (!this.stopHook) {
            try {
                await this.consumerGroup?.join();
                // TODO: If leader is not available, find another read replica
                const topicPartitions = Object.entries(this.metadata.getAssignment()).flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })));
                const topicPartitionsByLeaderId = (0, group_by_leader_id_1.groupByLeaderId)(topicPartitions, this.metadata.getTopicPartitionLeaderIds());
                const nodeAssignments = Object.entries(topicPartitionsByLeaderId).map(([leaderId, topicPartitions]) => ({
                    nodeId: parseInt(leaderId),
                    assignment: (0, group_partitions_by_topic_1.groupPartitionsByTopic)(topicPartitions),
                }));
                this.fetchManager = new fetch_manager_1.FetchManager({
                    fetch: this.fetch.bind(this),
                    process: this.process.bind(this),
                    nodeAssignments,
                });
                await this.fetchManager.start();
                if (!nodeAssignments.length) {
                    await this.waitForReassignment();
                }
            }
            catch (error) {
                await this.fetchManager?.stop();
                if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.REBALANCE_IN_PROGRESS) {
                    logger_1.log.debug('Rebalance in progress...', { apiName: error.apiName, groupId });
                    continue;
                }
                if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.FENCED_INSTANCE_ID) {
                    logger_1.log.debug('New consumer with the same groupInstanceId joined. Exiting the consumer...');
                    this.close();
                    break;
                }
                if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_COORDINATOR) {
                    logger_1.log.debug('Not coordinator. Searching for new coordinator...');
                    await this.consumerGroup?.findCoordinator();
                    continue;
                }
                if (error instanceof error_1.ConnectionError) {
                    logger_1.log.debug(`${error.message}. Restarting consumer...`, { stack: error.stack });
                    this.close().then(() => this.start());
                    break;
                }
                logger_1.log.error(error.message, error);
                logger_1.log.debug(`Restarting consumer in 1 second...`);
                await (0, delay_1.delay)(1000);
                this.close().then(() => this.start());
                break;
            }
        }
        this.stopHook?.();
    }
    async waitForReassignment() {
        const { groupId } = this.options;
        logger_1.log.debug('No partitions assigned. Waiting for reassignment...', { groupId });
        while (!this.stopHook) {
            await (0, delay_1.delay)(1000);
            this.consumerGroup?.handleLastHeartbeat();
        }
    }
    async process(response) {
        const { options } = this;
        const { retrier } = options;
        this.consumerGroup?.handleLastHeartbeat();
        const topicPartitions = {};
        const messages = response.responses.flatMap((response) => {
            const topic = 'topicName' in response ? response.topicName : this.metadata.getTopicNameById(response.topicId);
            topicPartitions[topic] ??= new Set();
            return response.partitions.flatMap(({ partitionIndex, records }) => {
                topicPartitions[topic].add(partitionIndex);
                return records.flatMap(({ baseTimestamp, baseOffset, records }) => records.flatMap((message) => ({
                    topic,
                    partition: partitionIndex,
                    key: message.key ?? null,
                    value: message.value ?? null,
                    headers: Object.fromEntries(message.headers.map(({ key, value }) => [key, value])),
                    timestamp: baseTimestamp + BigInt(message.timestampDelta),
                    offset: baseOffset + BigInt(message.offsetDelta),
                })));
            });
        });
        if (!messages.length) {
            return;
        }
        const commitOffset = () => this.consumerGroup?.offsetCommit(topicPartitions).then(() => this.offsetManager.flush(topicPartitions));
        const resolveOffset = (message) => this.offsetManager.resolve(message.topic, message.partition, message.offset + 1n);
        const abortController = new AbortController();
        const onRebalance = () => {
            abortController.abort();
            commitOffset()?.catch();
        };
        this.once('rebalanceInProgress', onRebalance);
        try {
            await retrier(() => options.onBatch(messages.filter((message) => !this.offsetManager.isResolved(message)), { resolveOffset, abortSignal: abortController.signal }));
        }
        catch (error) {
            await commitOffset()?.catch();
            throw error;
        }
        finally {
            this.off('rebalanceInProgress', onRebalance);
        }
        if (!abortController.signal.aborted) {
            response.responses.forEach((response) => {
                response.partitions.forEach(({ partitionIndex, records }) => {
                    records.forEach(({ baseOffset, lastOffsetDelta }) => {
                        const topic = 'topicName' in response
                            ? response.topicName
                            : this.metadata.getTopicNameById(response.topicId);
                        this.offsetManager.resolve(topic, partitionIndex, baseOffset + BigInt(lastOffsetDelta) + 1n);
                    });
                });
            });
        }
        await commitOffset();
    }
    async fetch(nodeId, assignment) {
        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
            const { rackId, maxWaitMs, minBytes, maxBytes, partitionMaxBytes, isolationLevel } = this.options;
            this.consumerGroup?.handleLastHeartbeat();
            return this.cluster.sendRequestToNode(nodeId)(api_1.API.FETCH, {
                maxWaitMs,
                minBytes,
                maxBytes,
                isolationLevel,
                sessionId: 0,
                sessionEpoch: -1,
                topics: Object.entries(assignment).map(([topicName, partitions]) => ({
                    topicId: this.metadata.getTopicIdByName(topicName),
                    topicName,
                    partitions: partitions.map((partition) => ({
                        partition,
                        currentLeaderEpoch: -1,
                        fetchOffset: this.offsetManager.getCurrentOffset(topicName, partition),
                        lastFetchedEpoch: -1,
                        logStartOffset: -1n,
                        partitionMaxBytes,
                    })),
                })),
                forgottenTopicsData: [],
                rackId,
            });
        });
    }
    async fetchMetadata() {
        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
            const { topics, allowTopicAutoCreation } = this.options;
            await this.metadata.fetchMetadata({ topics, allowTopicAutoCreation });
        });
    }
    async fetchOffsets() {
        return (0, retry_1.withRetry)(this.handleError.bind(this))(async () => {
            const { fromTimestamp } = this.options;
            await this.offsetManager.fetchOffsets({ fromTimestamp });
        });
    }
    async handleError(error) {
        await (0, api_1.handleApiError)(error).catch(async (error) => {
            if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.NOT_LEADER_OR_FOLLOWER) {
                logger_1.log.debug('Refreshing metadata', { reason: error.message });
                await this.fetchMetadata();
                return;
            }
            if (error instanceof error_1.KafkaTSApiError && error.errorCode === api_1.API_ERROR.OFFSET_OUT_OF_RANGE) {
                logger_1.log.warn('Offset out of range. Resetting offsets.');
                await this.fetchOffsets();
                return;
            }
            throw error;
        });
    }
}
exports.Consumer = Consumer;
__decorate([
    trace(),
    __metadata("design:type", Function),
    __metadata("design:paramtypes", []),
    __metadata("design:returntype", Promise)
], Consumer.prototype, "start", null);
__decorate([
    trace(),
    __metadata("design:type", Function),
    __metadata("design:paramtypes", [Object]),
    __metadata("design:returntype", Promise)
], Consumer.prototype, "close", null);
__decorate([
    trace(),
    __metadata("design:type", Function),
    __metadata("design:paramtypes", [Object]),
    __metadata("design:returntype", Promise)
], Consumer.prototype, "process", null);