@nestjs/microservices/client/client-kafka.js

Nest - modern, fast, powerful node.js web framework (@microservices)

import { throwError as _throw, connectable, defer, Subject, } from 'rxjs';
import { mergeMap } from 'rxjs/operators';
import { KAFKA_DEFAULT_BROKER, KAFKA_DEFAULT_CLIENT, KAFKA_DEFAULT_GROUP, } from '../constants.js';
import { KafkaResponseDeserializer } from '../deserializers/kafka-response.deserializer.js';
import { KafkaHeaders } from '../enums/index.js';
import { InvalidKafkaClientTopicException } from '../errors/invalid-kafka-client-topic.exception.js';
import { InvalidMessageException } from '../errors/invalid-message.exception.js';
import { KafkaLogger, KafkaParser, KafkaReplyPartitionAssigner, } from '../helpers/index.js';
import { KafkaRequestSerializer, } from '../serializers/kafka-request.serializer.js';
import { ClientProxy } from './client-proxy.js';
import { Logger } from '@nestjs/common';
import { loadPackage, isNil, isUndefined } from '@nestjs/common/internal';
/**
 * @publicApi
 */
export class ClientKafka extends ClientProxy {
    options;
    logger = new Logger(ClientKafka.name);
    client = null;
    parser = null;
    initialized = null;
    responsePatterns = [];
    consumerAssignments = {};
    brokers;
    clientId;
    groupId;
    producerOnlyMode;
    _consumer = null;
    _producer = null;
    get consumer() {
        if (!this._consumer) {
            throw new Error('No consumer initialized. Please, call the "connect" method first.');
        }
        return this._consumer;
    }
    get producer() {
        if (!this._producer) {
            throw new Error('No producer initialized. Please, call the "connect" method first.');
        }
        return this._producer;
    }
    constructor(options) {
        super();
        this.options = options;
        const clientOptions = this.getOptionsProp(this.options, 'client', {});
        const consumerOptions = this.getOptionsProp(this.options, 'consumer', {});
        const postfixId = this.getOptionsProp(this.options, 'postfixId', '-client');
        this.producerOnlyMode = this.getOptionsProp(this.options, 'producerOnlyMode', false);
        this.brokers = clientOptions.brokers || [KAFKA_DEFAULT_BROKER];
        // Append a unique id to the clientId and groupId
        // so they don't collide with a microservices client
        this.clientId =
            (clientOptions.clientId || KAFKA_DEFAULT_CLIENT) + postfixId;
        this.groupId = (consumerOptions.groupId || KAFKA_DEFAULT_GROUP) + postfixId;
        this.parser = new KafkaParser((options && options.parser) || undefined);
        this.initializeSerializer(options);
        this.initializeDeserializer(options);
    }
    subscribeToResponseOf(pattern) {
        const request = this.normalizePattern(pattern);
        this.responsePatterns.push(this.getResponsePatternName(request));
    }
    async close() {
        this._producer && (await this._producer.disconnect());
        this._consumer && (await this._consumer.disconnect());
        this._producer = null;
        this._consumer = null;
        this.initialized = null;
        this.client = null;
    }
    async connect() {
        if (this.initialized) {
            return this.initialized.then(() => this._producer);
        }
        this.initialized = this.initializeClientAndConnections();
        return this.initialized.then(() => this._producer);
    }
    async initializeClientAndConnections() {
        this.client = await this.createClient();
        if (!this.producerOnlyMode) {
            const partitionAssigners = [
                (config) => new KafkaReplyPartitionAssigner(this, config),
            ];
            const consumerOptions = {
                partitionAssigners,
                ...(this.options.consumer || {}),
                groupId: this.groupId,
            };
            this._consumer = this.client.consumer(consumerOptions);
            this.registerConsumerEventListeners();
            // Set member assignments on join and rebalance
            this._consumer.on(this._consumer.events.GROUP_JOIN, this.setConsumerAssignments.bind(this));
            await this._consumer.connect();
            await this.bindTopics();
        }
        this._producer = this.client.producer(this.options.producer || {});
        this.registerProducerEventListeners();
        await this._producer.connect();
    }
    async bindTopics() {
        if (!this._consumer) {
            throw Error('No consumer initialized');
        }
        const consumerSubscribeOptions = this.options.subscribe || {};
        if (this.responsePatterns.length > 0) {
            await this._consumer.subscribe({
                ...consumerSubscribeOptions,
                topics: this.responsePatterns,
            });
        }
        await this._consumer.run({
            ...(this.options.run || {}),
            eachMessage: this.createResponseCallback(),
        });
    }
    async createClient() {
        const kafkaPackage = await loadPackage('kafkajs', ClientKafka.name, () => import('kafkajs'));
        const kafkaConfig = {
            logCreator: KafkaLogger.bind(null, this.logger),
            ...this.options.client,
            brokers: this.brokers,
            clientId: this.clientId,
        };
        return new kafkaPackage.Kafka(kafkaConfig);
    }
    createResponseCallback() {
        return async (payload) => {
            const rawMessage = this.parser.parse(Object.assign(payload.message, {
                topic: payload.topic,
                partition: payload.partition,
            }));
            if (isUndefined(rawMessage.headers[KafkaHeaders.CORRELATION_ID])) {
                return;
            }
            const { err, response, isDisposed, id } = await this.deserializer.deserialize(rawMessage);
            const callback = this.routingMap.get(id);
            if (!callback) {
                return;
            }
            if (err || isDisposed) {
                return callback({
                    err,
                    response,
                    isDisposed,
                });
            }
            callback({
                err,
                response,
            });
        };
    }
    getConsumerAssignments() {
        return this.consumerAssignments;
    }
    emitBatch(pattern, data) {
        if (isNil(pattern) || isNil(data)) {
            return _throw(() => new InvalidMessageException());
        }
        const source = defer(async () => this.connect()).pipe(mergeMap(() => this.dispatchBatchEvent({ pattern, data })));
        const connectableSource = connectable(source, {
            connector: () => new Subject(),
            resetOnDisconnect: false,
        });
        connectableSource.connect();
        return connectableSource;
    }
    commitOffsets(topicPartitions) {
        if (this._consumer) {
            return this._consumer.commitOffsets(topicPartitions);
        }
        else {
            throw new Error('No consumer initialized');
        }
    }
    unwrap() {
        if (!this.client) {
            throw new Error('Not initialized. Please call the "connect" method first.');
        }
        return this.client;
    }
    on(event, callback) {
        throw new Error('Method is not supported for Kafka client');
    }
    registerConsumerEventListeners() {
        if (!this._consumer) {
            return;
        }
        this._consumer.on(this._consumer.events.CONNECT, () => this._status$.next("connected" /* KafkaStatus.CONNECTED */));
        this._consumer.on(this._consumer.events.DISCONNECT, () => this._status$.next("disconnected" /* KafkaStatus.DISCONNECTED */));
        this._consumer.on(this._consumer.events.REBALANCING, () => this._status$.next("rebalancing" /* KafkaStatus.REBALANCING */));
        this._consumer.on(this._consumer.events.STOP, () => this._status$.next("stopped" /* KafkaStatus.STOPPED */));
        this._consumer.on(this._consumer.events.CRASH, () => this._status$.next("crashed" /* KafkaStatus.CRASHED */));
    }
    registerProducerEventListeners() {
        if (!this._producer) {
            return;
        }
        this._producer.on(this._producer.events.CONNECT, () => this._status$.next("connected" /* KafkaStatus.CONNECTED */));
        this._producer.on(this._producer.events.DISCONNECT, () => this._status$.next("disconnected" /* KafkaStatus.DISCONNECTED */));
    }
    async dispatchBatchEvent(packets) {
        if (packets.data.messages.length === 0) {
            return;
        }
        const pattern = this.normalizePattern(packets.pattern);
        const outgoingEvents = await Promise.all(packets.data.messages.map(message => {
            return this.serializer.serialize(message, { pattern });
        }));
        const message = {
            topic: pattern,
            messages: outgoingEvents,
            ...(this.options.send || {}),
        };
        return this.producer.send(message);
    }
    async dispatchEvent(packet) {
        const pattern = this.normalizePattern(packet.pattern);
        const outgoingEvent = await this.serializer.serialize(packet.data, {
            pattern,
        });
        const message = {
            topic: pattern,
            messages: [outgoingEvent],
            ...(this.options.send || {}),
        };
        return this._producer.send(message);
    }
    getReplyTopicPartition(topic) {
        const minimumPartition = this.consumerAssignments[topic];
        if (isUndefined(minimumPartition)) {
            throw new InvalidKafkaClientTopicException(topic);
        }
        // Get the minimum partition
        return minimumPartition.toString();
    }
    publish(partialPacket, callback) {
        const packet = this.assignPacketId(partialPacket);
        this.routingMap.set(packet.id, callback);
        const cleanup = () => this.routingMap.delete(packet.id);
        const errorCallback = (err) => {
            cleanup();
            callback({ err });
        };
        try {
            const pattern = this.normalizePattern(partialPacket.pattern);
            const replyTopic = this.getResponsePatternName(pattern);
            const replyPartition = this.getReplyTopicPartition(replyTopic);
            Promise.resolve(this.serializer.serialize(packet.data, { pattern }))
                .then((serializedPacket) => {
                serializedPacket.headers[KafkaHeaders.CORRELATION_ID] = packet.id;
                serializedPacket.headers[KafkaHeaders.REPLY_TOPIC] = replyTopic;
                serializedPacket.headers[KafkaHeaders.REPLY_PARTITION] =
                    replyPartition;
                const message = {
                    topic: pattern,
                    messages: [serializedPacket],
                    ...(this.options.send || {}),
                };
                return this._producer.send(message);
            })
                .catch(err => errorCallback(err));
            return cleanup;
        }
        catch (err) {
            errorCallback(err);
            return () => null;
        }
    }
    getResponsePatternName(pattern) {
        return `${pattern}.reply`;
    }
    setConsumerAssignments(data) {
        const consumerAssignments = {};
        // Only need to set the minimum
        for (const [topic, memberPartitions] of Object.entries(data.payload.memberAssignment)) {
            if (memberPartitions.length) {
                consumerAssignments[topic] = Math.min(...memberPartitions);
            }
        }
        this.consumerAssignments = consumerAssignments;
    }
    initializeSerializer(options) {
        this.serializer =
            (options && options.serializer) || new KafkaRequestSerializer();
    }
    initializeDeserializer(options) {
        this.deserializer =
            (options && options.deserializer) || new KafkaResponseDeserializer();
    }
}