@nestjs/microservices/server/server-kafka.js

Nest - modern, fast, powerful node.js web framework (@microservices)

import { isObservable, lastValueFrom, ReplaySubject } from 'rxjs';
import { KAFKA_DEFAULT_BROKER, KAFKA_DEFAULT_CLIENT, KAFKA_DEFAULT_GROUP, NO_EVENT_HANDLER, NO_MESSAGE_HANDLER, } from '../constants.js';
import { KafkaContext } from '../ctx-host/index.js';
import { KafkaRequestDeserializer } from '../deserializers/kafka-request.deserializer.js';
import { KafkaHeaders, Transport } from '../enums/index.js';
import { KafkaRetriableException } from '../exceptions/index.js';
import { KafkaLogger, KafkaParser } from '../helpers/index.js';
import { KafkaRequestSerializer } from '../serializers/kafka-request.serializer.js';
import { Server } from './server.js';
import { Logger } from '@nestjs/common';
import { isNil } from '@nestjs/common/internal';
/**
 * @publicApi
 */
export class ServerKafka extends Server {
    options;
    transportId = Transport.KAFKA;
    logger = new Logger(ServerKafka.name);
    client = null;
    consumer = null;
    producer = null;
    parser = null;
    brokers;
    clientId;
    groupId;
    constructor(options) {
        super();
        this.options = options;
        const clientOptions = this.getOptionsProp(this.options, 'client', {});
        const consumerOptions = this.getOptionsProp(this.options, 'consumer', {});
        const postfixId = this.getOptionsProp(this.options, 'postfixId', '-server');
        this.brokers = clientOptions.brokers || [KAFKA_DEFAULT_BROKER];
        // Append a unique id to the clientId and groupId
        // so they don't collide with a microservices client
        this.clientId =
            (clientOptions.clientId || KAFKA_DEFAULT_CLIENT) + postfixId;
        this.groupId = (consumerOptions.groupId || KAFKA_DEFAULT_GROUP) + postfixId;
        this.parser = new KafkaParser((options && options.parser) || undefined);
        this.initializeSerializer(options);
        this.initializeDeserializer(options);
    }
    addHandler(pattern, callback, isEventHandler = false, extras = {}) {
        if (!(pattern instanceof RegExp)) {
            return super.addHandler(pattern, callback, isEventHandler, extras);
        }
        const messageHandlers = this.messageHandlers;
        callback.isEventHandler = isEventHandler;
        callback.extras = extras;
        if (messageHandlers.has(pattern) && isEventHandler) {
            const headRef = messageHandlers.get(pattern);
            const getTail = (handler) => handler?.next ? getTail(handler.next) : handler;
            const tailRef = getTail(headRef);
            tailRef.next = callback;
        }
        else {
            messageHandlers.set(pattern, callback);
        }
    }
    async listen(callback) {
        try {
            this.client = await this.createClient();
            await this.start(callback);
        }
        catch (err) {
            callback(err);
        }
    }
    async close() {
        this.consumer && (await this.consumer.disconnect());
        this.producer && (await this.producer.disconnect());
        this.consumer = null;
        this.producer = null;
        this.client = null;
    }
    async start(callback) {
        const consumerOptions = {
            ...(this.options.consumer || {}),
            groupId: this.groupId,
        };
        this.consumer = this.client.consumer(consumerOptions);
        this.producer = this.client.producer(this.options.producer);
        this.registerConsumerEventListeners();
        this.registerProducerEventListeners();
        await this.consumer.connect();
        await this.producer.connect();
        await this.bindEvents(this.consumer);
        callback();
    }
    registerConsumerEventListeners() {
        if (!this.consumer) {
            return;
        }
        this.consumer.on(this.consumer.events.CONNECT, () => this._status$.next("connected" /* KafkaStatus.CONNECTED */));
        this.consumer.on(this.consumer.events.DISCONNECT, () => this._status$.next("disconnected" /* KafkaStatus.DISCONNECTED */));
        this.consumer.on(this.consumer.events.REBALANCING, () => this._status$.next("rebalancing" /* KafkaStatus.REBALANCING */));
        this.consumer.on(this.consumer.events.STOP, () => this._status$.next("stopped" /* KafkaStatus.STOPPED */));
        this.consumer.on(this.consumer.events.CRASH, () => this._status$.next("crashed" /* KafkaStatus.CRASHED */));
    }
    registerProducerEventListeners() {
        if (!this.producer) {
            return;
        }
        this.producer.on(this.producer.events.CONNECT, () => this._status$.next("connected" /* KafkaStatus.CONNECTED */));
        this.producer.on(this.producer.events.DISCONNECT, () => this._status$.next("disconnected" /* KafkaStatus.DISCONNECTED */));
    }
    async createClient() {
        const kafkaPackage = await this.loadPackage('kafkajs', ServerKafka.name, () => import('kafkajs'));
        return new kafkaPackage.Kafka({
            logCreator: KafkaLogger.bind(null, this.logger),
            ...this.options.client,
            clientId: this.clientId,
            brokers: this.brokers,
        });
    }
    async bindEvents(consumer) {
        const registeredPatterns = [...this.messageHandlers.keys()];
        const consumerSubscribeOptions = this.options.subscribe || {};
        if (registeredPatterns.length > 0) {
            await this.consumer.subscribe({
                ...consumerSubscribeOptions,
                topics: registeredPatterns,
            });
        }
        const consumerRunOptions = {
            ...(this.options.run || {}),
            eachMessage: this.getMessageHandler(),
        };
        await consumer.run(consumerRunOptions);
    }
    getHandlerByPattern(pattern) {
        const handler = super.getHandlerByPattern(pattern);
        if (handler) {
            return handler;
        }
        const route = this.getRouteFromPattern(pattern);
        const messageHandlers = this.messageHandlers;
        for (const [registeredPattern, registeredHandler] of messageHandlers) {
            if (registeredPattern instanceof RegExp &&
                this.isPatternMatch(registeredPattern, route)) {
                return registeredHandler;
            }
        }
        return null;
    }
    isPatternMatch(pattern, route) {
        pattern.lastIndex = 0;
        const isMatch = pattern.test(route);
        pattern.lastIndex = 0;
        return isMatch;
    }
    getMessageHandler() {
        return async (payload) => this.handleMessage(payload);
    }
    getPublisher(replyTopic, replyPartition, correlationId, context) {
        return (data) => this.sendMessage(data, replyTopic, replyPartition, correlationId, context);
    }
    async handleMessage(payload) {
        const channel = payload.topic;
        const rawMessage = this.parser.parse(Object.assign(payload.message, {
            topic: payload.topic,
            partition: payload.partition,
        }));
        const headers = rawMessage.headers;
        const correlationId = headers[KafkaHeaders.CORRELATION_ID];
        const replyTopic = headers[KafkaHeaders.REPLY_TOPIC];
        const replyPartition = headers[KafkaHeaders.REPLY_PARTITION];
        const packet = await this.deserializer.deserialize(rawMessage, { channel });
        const kafkaContext = new KafkaContext([
            rawMessage,
            payload.partition,
            payload.topic,
            this.consumer,
            payload.heartbeat,
            this.producer,
        ]);
        const handler = this.getHandlerByPattern(packet.pattern);
        // if the correlation id or reply topic is not set
        // then this is an event (events could still have correlation id)
        if (handler?.isEventHandler || !correlationId || !replyTopic) {
            return this.handleEvent(packet.pattern, packet, kafkaContext);
        }
        const publish = this.getPublisher(replyTopic, replyPartition, correlationId, kafkaContext);
        if (!handler) {
            return publish({
                id: correlationId,
                err: NO_MESSAGE_HANDLER,
            });
        }
        return this.onProcessingStartHook(this.transportId, kafkaContext, async () => {
            const response$ = this.transformToObservable(handler(packet.data, kafkaContext));
            const replayStream$ = new ReplaySubject();
            await this.combineStreamsAndThrowIfRetriable(response$, replayStream$);
            this.send(replayStream$, publish);
        });
    }
    unwrap() {
        if (!this.client) {
            throw new Error('Not initialized. Please call the "listen"/"startAllMicroservices" method before accessing the server.');
        }
        return [this.client, this.consumer, this.producer];
    }
    on(event, callback) {
        throw new Error('Method is not supported for Kafka server');
    }
    combineStreamsAndThrowIfRetriable(response$, replayStream$) {
        return new Promise((resolve, reject) => {
            let isPromiseResolved = false;
            response$.subscribe({
                next: val => {
                    replayStream$.next(val);
                    if (!isPromiseResolved) {
                        isPromiseResolved = true;
                        resolve();
                    }
                },
                error: err => {
                    if (err instanceof KafkaRetriableException && !isPromiseResolved) {
                        isPromiseResolved = true;
                        reject(err);
                    }
                    else {
                        resolve();
                    }
                    replayStream$.error(err);
                },
                complete: () => replayStream$.complete(),
            });
        });
    }
    async sendMessage(message, replyTopic, replyPartition, correlationId, context) {
        const outgoingMessage = await this.serializer.serialize(message.response);
        this.assignReplyPartition(replyPartition, outgoingMessage);
        this.assignCorrelationIdHeader(correlationId, outgoingMessage);
        this.assignErrorHeader(message, outgoingMessage);
        this.assignIsDisposedHeader(message, outgoingMessage);
        const replyMessage = {
            topic: replyTopic,
            messages: [outgoingMessage],
            ...(this.options.send || {}),
        };
        return this.producer.send(replyMessage).finally(() => {
            this.onProcessingEndHook?.(this.transportId, context);
        });
    }
    assignIsDisposedHeader(outgoingResponse, outgoingMessage) {
        if (!outgoingResponse.isDisposed) {
            return;
        }
        outgoingMessage.headers[KafkaHeaders.NEST_IS_DISPOSED] = Buffer.alloc(1);
    }
    assignErrorHeader(outgoingResponse, outgoingMessage) {
        if (!outgoingResponse.err) {
            return;
        }
        const stringifiedError = typeof outgoingResponse.err === 'object'
            ? JSON.stringify(outgoingResponse.err)
            : outgoingResponse.err;
        outgoingMessage.headers[KafkaHeaders.NEST_ERR] =
            Buffer.from(stringifiedError);
    }
    assignCorrelationIdHeader(correlationId, outgoingMessage) {
        outgoingMessage.headers[KafkaHeaders.CORRELATION_ID] =
            Buffer.from(correlationId);
    }
    assignReplyPartition(replyPartition, outgoingMessage) {
        if (isNil(replyPartition)) {
            return;
        }
        outgoingMessage.partition = parseFloat(replyPartition);
    }
    async handleEvent(pattern, packet, context) {
        const handler = this.getHandlerByPattern(pattern);
        if (!handler) {
            return this.logger.error(NO_EVENT_HANDLER `${pattern}`);
        }
        return this.onProcessingStartHook(this.transportId, context, async () => {
            const resultOrStream = await handler(packet.data, context);
            if (isObservable(resultOrStream)) {
                await lastValueFrom(resultOrStream);
                this.onProcessingEndHook?.(this.transportId, context);
            }
        });
    }
    initializeSerializer(options) {
        this.serializer =
            (options && options.serializer) || new KafkaRequestSerializer();
    }
    initializeDeserializer(options) {
        this.deserializer = options?.deserializer ?? new KafkaRequestDeserializer();
    }
}