@nestjs/microservices/client/client-kafka.js

Nest - modern, fast, powerful node.js web framework (@microservices)

import { throwError as _throw, connectable, defer, Subject, } from 'rxjs';
import { mergeMap } from 'rxjs/operators';
import { KAFKA_DEFAULT_BROKER, KAFKA_DEFAULT_CLIENT, KAFKA_DEFAULT_GROUP, } from '../constants.js';
import { KafkaResponseDeserializer } from '../deserializers/kafka-response.deserializer.js';
import { KafkaHeaders } from '../enums/index.js';
import { InvalidKafkaClientTopicException } from '../errors/invalid-kafka-client-topic.exception.js';
import { InvalidMessageException } from '../errors/invalid-message.exception.js';
import { KafkaLogger, KafkaParser, KafkaReplyPartitionAssigner, } from '../helpers/index.js';
import { KafkaRequestSerializer, } from '../serializers/kafka-request.serializer.js';
import { ClientProxy } from './client-proxy.js';
import { Logger } from '@nestjs/common';
import { loadPackage, isNil, isUndefined } from '@nestjs/common/internal';
/**
 * @publicApi
 */
export class ClientKafka extends ClientProxy {
    options;
    logger = new Logger(ClientKafka.name);
    client = null;
    parser = null;
    initialized = null;
    responsePatterns = [];
    consumerAssignments = {};
    brokers;
    clientId;
    groupId;
    producerOnlyMode;
    _consumer = null;
    _producer = null;
    get consumer() {
        if (!this._consumer) {
            throw new Error('No consumer initialized. Please, call the "connect" method first.');
        }
        return this._consumer;
    }
    get producer() {
        if (!this._producer) {
            throw new Error('No producer initialized. Please, call the "connect" method first.');
        }
        return this._producer;
    }
    constructor(options) {
        super();
        this.options = options;
        const clientOptions = this.getOptionsProp(this.options, 'client', {});
        const consumerOptions = this.getOptionsProp(this.options, 'consumer', {});
        const postfixId = this.getOptionsProp(this.options, 'postfixId', '-client');
        this.producerOnlyMode = this.getOptionsProp(this.options, 'producerOnlyMode', false);
        this.brokers = clientOptions.brokers || [KAFKA_DEFAULT_BROKER];
        // Append a unique id to the clientId and groupId
        // so they don't collide with a microservices client
        this.clientId =
            (clientOptions.clientId || KAFKA_DEFAULT_CLIENT) + postfixId;
        this.groupId = (consumerOptions.groupId || KAFKA_DEFAULT_GROUP) + postfixId;
        this.parser = new KafkaParser((options && options.parser) || undefined);
        this.initializeSerializer(options);
        this.initializeDeserializer(options);
    }
    subscribeToResponseOf(pattern) {
        const request = this.normalizePattern(pattern);
        this.responsePatterns.push(this.getResponsePatternName(request));
    }
    async close() {
        this.handleClose();
        try {
            // Both clients have to be disconnected even if one of them fails, so a
            // consumer does not stay in its group when the producer cannot be closed.
            const results = await Promise.allSettled([
                this._producer?.disconnect(),
                this._consumer?.disconnect(),
            ]);
            const [failure, ...otherFailures] = results.filter((result) => result.status === 'rejected');
            otherFailures.forEach(({ reason }) => this.logger.error(reason));
            if (failure) {
                throw failure.reason;
            }
        }
        finally {
            // The state must be reset even when a disconnect throws, otherwise a
            // later `connect()` reuses the cached `initialized` promise and hands
            // back the closed producer.
            this._producer = null;
            this._consumer = null;
            this.initialized = null;
            this.client = null;
        }
    }
    handleClose() {
        if (this.routingMap.size > 0) {
            const err = new Error('Connection closed');
            const callbacks = [...this.routingMap.values()];
            this.routingMap.clear();
            for (const callback of callbacks) {
                try {
                    callback({ err });
                }
                catch (callbackErr) {
                    // A failing callback must not keep the remaining requests pending
                    // nor prevent the connection from being closed.
                    this.logger.error(callbackErr);
                }
            }
        }
    }
    async connect() {
        if (this.initialized) {
            return this.initialized.then(() => this._producer);
        }
        const attempt = {};
        const initialized = this.initializeClientAndConnections(attempt).catch(async (err) => {
            // A rejected attempt must not be cached: the next `connect()` call
            // has to try again once the broker is reachable, instead of failing
            // forever with the error of the first attempt.
            await this.discardPartialConnection(attempt, initialized);
            throw err;
        });
        this.initialized = initialized;
        return initialized.then(() => this._producer);
    }
    /**
     * Tears down whatever a failed connection attempt managed to create, so a
     * consumer that joined its group before the producer failed does not stay
     * behind when the next attempt creates a new one. The client's state is only
     * reset if no newer attempt has replaced it in the meantime (e.g., `close()`
     * followed by `connect()` while the failed attempt was still pending).
     */
    async discardPartialConnection(attempt, initialized) {
        if (this.initialized === initialized) {
            this._consumer = null;
            this._producer = null;
            this.client = null;
            this.initialized = null;
        }
        await Promise.allSettled([
            attempt.consumer?.disconnect(),
            attempt.producer?.disconnect(),
        ]);
    }
    async initializeClientAndConnections(attempt) {
        this.client = await this.createClient();
        if (!this.producerOnlyMode) {
            const partitionAssigners = [
                (config) => new KafkaReplyPartitionAssigner(this, config),
            ];
            const consumerOptions = {
                partitionAssigners,
                ...(this.options.consumer || {}),
                groupId: this.groupId,
            };
            this._consumer = attempt.consumer =
                this.client.consumer(consumerOptions);
            this.registerConsumerEventListeners();
            // Set member assignments on join and rebalance
            this._consumer.on(this._consumer.events.GROUP_JOIN, this.setConsumerAssignments.bind(this));
            await this._consumer.connect();
            await this.bindTopics();
        }
        this._producer = attempt.producer = this.client.producer(this.options.producer || {});
        this.registerProducerEventListeners();
        await this._producer.connect();
    }
    async bindTopics() {
        if (!this._consumer) {
            throw Error('No consumer initialized');
        }
        const consumerSubscribeOptions = this.options.subscribe || {};
        if (this.responsePatterns.length > 0) {
            await this._consumer.subscribe({
                ...consumerSubscribeOptions,
                topics: this.responsePatterns,
            });
        }
        await this._consumer.run({
            ...(this.options.run || {}),
            eachMessage: this.createResponseCallback(),
        });
    }
    async createClient() {
        const kafkaPackage = await loadPackage('kafkajs', ClientKafka.name, () => import('kafkajs'));
        const kafkaConfig = {
            logCreator: KafkaLogger.bind(null, this.logger),
            ...this.options.client,
            brokers: this.brokers,
            clientId: this.clientId,
        };
        return new kafkaPackage.Kafka(kafkaConfig);
    }
    createResponseCallback() {
        return async (payload) => {
            const rawMessage = this.parser.parse(Object.assign(payload.message, {
                topic: payload.topic,
                partition: payload.partition,
            }));
            const correlationId = rawMessage.headers[KafkaHeaders.CORRELATION_ID];
            // kafkajs decodes an empty header value as null
            if (isNil(correlationId)) {
                return;
            }
            let incomingResponse;
            try {
                incomingResponse = await this.deserializer.deserialize(rawMessage);
            }
            catch (error) {
                // The correlation id is on the headers, so the pending request can be
                // failed here instead of staying pending forever.
                const callback = this.routingMap.get(correlationId.toString());
                return callback?.({ err: error, isDisposed: true });
            }
            const { err, response, isDisposed, id } = incomingResponse;
            const callback = this.routingMap.get(id);
            if (!callback) {
                return;
            }
            if (err || isDisposed) {
                return callback({
                    err,
                    response,
                    isDisposed,
                });
            }
            callback({
                err,
                response,
            });
        };
    }
    getConsumerAssignments() {
        return this.consumerAssignments;
    }
    emitBatch(pattern, data) {
        if (isNil(pattern) || isNil(data)) {
            return _throw(() => new InvalidMessageException());
        }
        const source = defer(async () => this.connect()).pipe(mergeMap(() => this.dispatchBatchEvent({ pattern, data })));
        const connectableSource = connectable(source, {
            connector: () => new Subject(),
            resetOnDisconnect: false,
        });
        connectableSource.connect();
        return connectableSource;
    }
    commitOffsets(topicPartitions) {
        if (this._consumer) {
            return this._consumer.commitOffsets(topicPartitions);
        }
        else {
            throw new Error('No consumer initialized');
        }
    }
    unwrap() {
        if (!this.client) {
            throw new Error('Not initialized. Please call the "connect" method first.');
        }
        return this.client;
    }
    on(event, callback) {
        throw new Error('Method is not supported for Kafka client');
    }
    registerConsumerEventListeners() {
        if (!this._consumer) {
            return;
        }
        this._consumer.on(this._consumer.events.CONNECT, () => this._status$.next("connected" /* KafkaStatus.CONNECTED */));
        this._consumer.on(this._consumer.events.DISCONNECT, () => this._status$.next("disconnected" /* KafkaStatus.DISCONNECTED */));
        this._consumer.on(this._consumer.events.REBALANCING, () => this._status$.next("rebalancing" /* KafkaStatus.REBALANCING */));
        this._consumer.on(this._consumer.events.STOP, () => this._status$.next("stopped" /* KafkaStatus.STOPPED */));
        this._consumer.on(this._consumer.events.CRASH, () => this._status$.next("crashed" /* KafkaStatus.CRASHED */));
    }
    registerProducerEventListeners() {
        if (!this._producer) {
            return;
        }
        this._producer.on(this._producer.events.CONNECT, () => this._status$.next("connected" /* KafkaStatus.CONNECTED */));
        this._producer.on(this._producer.events.DISCONNECT, () => this._status$.next("disconnected" /* KafkaStatus.DISCONNECTED */));
    }
    async dispatchBatchEvent(packets) {
        if (packets.data.messages.length === 0) {
            return;
        }
        const pattern = this.normalizePattern(packets.pattern);
        const outgoingEvents = await Promise.all(packets.data.messages.map(message => {
            return this.serializer.serialize(message, { pattern });
        }));
        const message = {
            topic: pattern,
            messages: outgoingEvents,
            ...(this.options.send || {}),
        };
        return this.producer.send(message);
    }
    async dispatchEvent(packet) {
        const pattern = this.normalizePattern(packet.pattern);
        const outgoingEvent = await this.serializer.serialize(packet.data, {
            pattern,
        });
        const message = {
            topic: pattern,
            messages: [outgoingEvent],
            ...(this.options.send || {}),
        };
        return this._producer.send(message);
    }
    getReplyTopicPartition(topic) {
        const minimumPartition = this.consumerAssignments[topic];
        if (isUndefined(minimumPartition)) {
            throw new InvalidKafkaClientTopicException(topic);
        }
        // Get the minimum partition
        return minimumPartition.toString();
    }
    publish(partialPacket, callback) {
        const packet = this.assignPacketId(partialPacket);
        this.routingMap.set(packet.id, callback);
        const cleanup = () => this.routingMap.delete(packet.id);
        const errorCallback = (err) => {
            cleanup();
            callback({ err });
        };
        try {
            const pattern = this.normalizePattern(partialPacket.pattern);
            const replyTopic = this.getResponsePatternName(pattern);
            const replyPartition = this.getReplyTopicPartition(replyTopic);
            Promise.resolve(this.serializer.serialize(packet.data, { pattern }))
                .then((serializedPacket) => {
                serializedPacket.headers[KafkaHeaders.CORRELATION_ID] = packet.id;
                serializedPacket.headers[KafkaHeaders.REPLY_TOPIC] = replyTopic;
                serializedPacket.headers[KafkaHeaders.REPLY_PARTITION] =
                    replyPartition;
                const message = {
                    topic: pattern,
                    messages: [serializedPacket],
                    ...(this.options.send || {}),
                };
                return this._producer.send(message);
            })
                .catch(err => errorCallback(err));
            return cleanup;
        }
        catch (err) {
            errorCallback(err);
            return () => null;
        }
    }
    getResponsePatternName(pattern) {
        return `${pattern}.reply`;
    }
    setConsumerAssignments(data) {
        const consumerAssignments = {};
        // Only need to set the minimum
        for (const [topic, memberPartitions] of Object.entries(data.payload.memberAssignment)) {
            if (memberPartitions.length) {
                consumerAssignments[topic] = Math.min(...memberPartitions);
            }
        }
        this.consumerAssignments = consumerAssignments;
    }
    initializeSerializer(options) {
        this.serializer =
            (options && options.serializer) || new KafkaRequestSerializer();
    }
    initializeDeserializer(options) {
        this.deserializer =
            (options && options.deserializer) || new KafkaResponseDeserializer();
    }
}