roach-storm/dist/lib/kafka/Consumer.js

Apache Kafka to Google Pub/Sub Gateway, API controlled

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const Debug = require("debug");
const debug = Debug("roach:consumer");
const sinek_1 = require("sinek");
class Consumer {
    constructor(config, roachStorm) {
        this.consumedLately = 0;
        this.config = config;
        this.roachStorm = roachStorm;
        this.consumer = null;
        this.intv = setInterval(() => {
            if (this.consumedLately > 0) {
                debug("Consumed", this.consumedLately, "messages lately");
                this.consumedLately = 0;
            }
        }, 45000);
    }
    async processMessagesWithRetry(messages, attempts = 0) {
        try {
            debug("Processing messages", messages.length, "with attempt", attempts);
            attempts++;
            await this.roachStorm.messageHandler.handleSortedMessageBatch(messages);
            return true;
        }
        catch (error) {
            debug("Failed to process kafka message, attempt", attempts, "with error", error.message);
            return (new Promise((resolve) => setTimeout(resolve, attempts * 1000)))
                .then(() => {
                return this.processMessagesWithRetry(messages, attempts);
            });
        }
    }
    async start() {
        debug("Connecting..");
        this.consumer = new sinek_1.NConsumer([], this.config.consumer);
        await this.consumer.connect();
        this.consumer.on("message", (message) => {
            this.roachStorm.metrics.inc(`kafka_msg_in`, 1, { topic: message.topic });
            this.consumedLately++;
        });
        this.consumer.consume(async (messages, callback) => {
            await this.processMessagesWithRetry(messages);
            callback();
        }, false, false, this.config.batchOptions);
        this.consumer.enableAnalytics({
            analyticsInterval: 1000 * 60 * 4,
            lagFetchInterval: 1000 * 60 * 8,
        });
        debug("Connected.");
    }
    getAnalytics() {
        return this.consumer ? this.consumer.getAnalytics() : null;
    }
    getLagStatus() {
        return this.consumer ? this.consumer.getLagStatus(false) : null;
    }
    adjustSubscriptions(topics) {
        if (this.consumer) {
            debug("Adjusting topic subscription", topics.length);
            this.consumer.adjustSubscription(topics);
        }
    }
    getKafkaClient() {
        return this.consumer;
    }
    getKafkaStats() {
        return this.consumer ? this.consumer.getStats() : {};
    }
    getTopicMetadata() {
        return this.consumer ? this.consumer.getMetadata(2500) : Promise.resolve({});
    }
    async close() {
        debug("Closing..");
        if (this.intv) {
            clearInterval(this.intv);
        }
        if (this.consumer) {
            this.consumer.haltAnalytics();
            await this.consumer.close(true);
            this.consumer = null;
        }
    }
}
exports.default = Consumer;