@message-queue-toolkit/kafka/dist/utils/KafkaMessageBatchStream.d.ts

Kafka adapter for message-queue-toolkit

import { Duplex } from 'node:stream';
type CallbackFunction = (error?: Error | null) => void;
type MessageWithTopicAndPartition = {
    topic: string;
    partition: number;
};
/**
 * Options for configuring the KafkaMessageBatchStream behavior.
 */
export type KafkaMessageBatchOptions = {
    /** Maximum number of messages to accumulate across all partitions before flushing */
    batchSize: number;
    /** Time in milliseconds to wait before flushing incomplete batches */
    timeoutMilliseconds: number;
    /**
     * Maximum number of topic-partition batches to buffer on the readable side before signaling backpressure.
     * Each unit represents one array of messages belonging to the same topic-partition, produced per flush.
     * A single flush may push multiple such arrays (one per distinct topic-partition in the accumulated batch).
     * Defaults to Node.js object-mode default (16). Lower values trigger backpressure sooner,
     * reducing downstream memory pressure at the cost of more frequent flow-control cycles.
     */
    readableHighWaterMark?: number;
};
/**
 * Interface extending Duplex to provide strong typing for the 'data' event.
 * The stream emits arrays of messages grouped by topic-partition.
 */
export interface KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Duplex {
    on(event: string | symbol, listener: (...args: any[]) => void): this;
    /** Listen for batches of messages from the same topic-partition */
    on(event: 'data', listener: (chunk: TMessage[]) => void): this;
    push(chunk: TMessage[] | null): boolean;
}
/**
 * A Duplex stream that batches Kafka messages based on size and timeout constraints.
 *
 * Key features:
 * - Accumulates messages across all partitions up to `batchSize` for true memory control
 * - Groups messages by topic-partition when flushing
 * - Implements backpressure: pauses input when downstream consumers are overwhelmed
 * - Auto-flushes on timeout to prevent messages from waiting indefinitely
 *
 * @example
 * ```typescript
 * const batchStream = new KafkaMessageBatchStream({ batchSize: 100, timeoutMilliseconds: 1000 })
 * batchStream.on('data', (batch) => {
 *   console.log(`Received ${batch.length} messages from ${batch[0].topic}:${batch[0].partition}`)
 * })
 * ```
 */
export declare class KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Duplex {
    private readonly batchSize;
    private readonly timeout;
    private messages;
    private existingTimeout;
    private pendingCallback;
    private isBackPressured;
    constructor(options: KafkaMessageBatchOptions);
    /**
     * Called when the downstream consumer is ready to receive more data.
     * This is the backpressure release mechanism: we resume the writable side
     * by calling the pending callback that was held during backpressure.
     */
    _read(): void;
    /**
     * Writes a message to the stream.
     * Messages accumulate until batchSize is reached or timeout expires.
     * Implements backpressure by holding the callback when downstream cannot consume.
     */
    _write(message: TMessage, _encoding: BufferEncoding, callback: CallbackFunction): void;
    _final(callback: CallbackFunction): void;
    private flushMessages;
}
export {};