@message-queue-toolkit/kafka/dist/AbstractKafkaConsumer.js

Kafka adapter for message-queue-toolkit

import { randomUUID } from 'node:crypto';
import { setTimeout } from 'node:timers/promises';
import { InternalError, stringValueSerializer, } from '@lokalise/node-core';
import { Consumer, stringDeserializer, } from '@platformatic/kafka';
import { AbstractKafkaService } from "./AbstractKafkaService.js";
import { KafkaHandlerContainer } from "./handler-container/KafkaHandlerContainer.js";
import { safeJsonDeserializer } from './utils/safeJsonDeserializer.js';
/*
TODO: Proper retry mechanism + DLQ -> https://lokalise.atlassian.net/browse/EDEXP-498
In the meantime, we will retry in memory up to 3 times
 */
const MAX_IN_MEMORY_RETRIES = 3;
export class AbstractKafkaConsumer extends AbstractKafkaService {
    consumer;
    consumerStream;
    transactionObservabilityManager;
    handlerContainer;
    executionContext;
    constructor(dependencies, options, executionContext) {
        super(dependencies, options);
        this.transactionObservabilityManager = dependencies.transactionObservabilityManager;
        this.handlerContainer = new KafkaHandlerContainer(options.handlers, options.messageTypeField);
        this.executionContext = executionContext;
        this.consumer = new Consumer({
            ...this.options.kafka,
            ...this.options,
            autocommit: false, // Handling commits manually
            deserializers: {
                key: stringDeserializer,
                value: safeJsonDeserializer,
                headerKey: stringDeserializer,
                headerValue: stringDeserializer,
            },
        });
    }
    async init() {
        if (this.consumerStream)
            return Promise.resolve();
        const topics = this.handlerContainer.topics;
        if (topics.length === 0)
            throw new Error('At least one topic must be defined');
        try {
            const { handlers, ...consumeOptions } = this.options; // Handlers cannot be passed to consume method
            this.consumerStream = await this.consumer.consume({ ...consumeOptions, topics });
        }
        catch (error) {
            throw new InternalError({
                message: 'Consumer init failed',
                errorCode: 'KAFKA_CONSUMER_INIT_ERROR',
                cause: error,
            });
        }
        this.consumerStream.on('data', (message) => this.consume(message));
        this.consumerStream.on('error', (error) => this.handlerError(error));
    }
    async close() {
        if (!this.consumerStream)
            return Promise.resolve();
        await new Promise((done) => this.consumerStream?.close(done));
        this.consumerStream = undefined;
        await this.consumer.close();
    }
    async consume(message) {
        // message.value can be undefined if the message is not JSON-serializable
        if (!message.value)
            return message.commit();
        const handler = this.handlerContainer.resolveHandler(message.topic, message.value);
        // if there is no handler for the message, we ignore it (simulating subscription)
        if (!handler)
            return message.commit();
        /* v8 ignore next */
        const transactionId = this.resolveMessageId(message.value) ?? randomUUID();
        this.transactionObservabilityManager?.start(this.buildTransactionName(message), transactionId);
        const parseResult = handler.schema.safeParse(message.value);
        if (!parseResult.success) {
            this.handlerError(parseResult.error, {
                topic: message.topic,
                message: stringValueSerializer(message.value),
            });
            this.handleMessageProcessed({
                topic: message.topic,
                message: message.value,
                processingResult: { status: 'error', errorReason: 'invalidMessage' },
            });
            return message.commit();
        }
        const validatedMessage = parseResult.data;
        const requestContext = this.getRequestContext(message);
        let retries = 0;
        let consumed = false;
        do {
            // exponential backoff -> 2^(retry-1)
            if (retries > 0)
                await setTimeout(Math.pow(2, retries - 1));
            consumed = await this.tryToConsume({ ...message, value: validatedMessage }, handler.handler, requestContext);
            if (consumed)
                break;
            retries++;
        } while (retries < MAX_IN_MEMORY_RETRIES);
        if (consumed) {
            this.handleMessageProcessed({
                topic: message.topic,
                message: validatedMessage,
                processingResult: { status: 'consumed' },
            });
        }
        else {
            this.handleMessageProcessed({
                topic: message.topic,
                message: validatedMessage,
                processingResult: { status: 'error', errorReason: 'handlerError' },
            });
        }
        this.transactionObservabilityManager?.stop(transactionId);
        return message.commit();
    }
    async tryToConsume(message, handler, requestContext) {
        try {
            await handler(message, this.executionContext, requestContext);
            return true;
        }
        catch (error) {
            this.handlerError(error, {
                topic: message.topic,
                message: stringValueSerializer(message.value),
            });
        }
        return false;
    }
    buildTransactionName(message) {
        const messageType = this.resolveMessageType(message.value);
        let name = `kafka:${message.topic}`;
        if (messageType?.trim().length)
            name += `:${messageType.trim()}`;
        return name;
    }
    getRequestContext(message) {
        let reqId = message.headers.get(this.resolveHeaderRequestIdField());
        if (!reqId || reqId.trim().length === 0)
            reqId = randomUUID();
        return {
            reqId,
            logger: this.logger.child({
                'x-request-id': reqId,
                topic: message.topic,
                messageKey: message.key,
            }),
        };
    }
}
//# sourceMappingURL=AbstractKafkaConsumer.js.map