zamza/dist/lib/kafka/Consumer.js

Apache Kafka discovery, indexing, searches, storage, hooks and HTTP gateway

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const Debug = require("debug");
const debug = Debug("zamza:consumer");
const sinek_1 = require("sinek");
class Consumer {
    constructor(config, zamza) {
        this.consumedLately = 0;
        this.config = config;
        this.zamza = zamza;
        this.consumer = null;
        this.intv = setInterval(() => {
            if (this.consumedLately > 0) {
                debug("Consumed", this.consumedLately, "messages lately");
                this.consumedLately = 0;
            }
        }, 45000);
    }
    async processMessageWithRetry(message, attempts = 0) {
        try {
            attempts++;
            await this.zamza.messageHandler.handleMessage(message);
            return true;
        }
        catch (error) {
            debug("Failed to process kafka message, attempt", attempts, "with error", error.message);
            // reset marshalling, in case error was due to bson insert error
            this.zamza.messageHandler.resetMarshallStateForTopic(message.topic);
            return (new Promise((resolve) => setTimeout(resolve, attempts * 1000)))
                .then(() => {
                return this.processMessageWithRetry(message, attempts);
            });
        }
    }
    async start() {
        debug("Connecting..");
        this.consumer = new sinek_1.NConsumer([], this.config.consumer);
        await this.consumer.connect();
        this.consumer.consume(async (message, callback) => {
            this.consumedLately++;
            await this.processMessageWithRetry(message);
            callback(null);
        }, false, false, this.config.batchOptions);
        this.consumer.enableAnalytics({
            analyticsInterval: 1000 * 60 * 4,
            lagFetchInterval: 1000 * 60 * 8,
        });
        debug("Connected.");
    }
    getAnalytics() {
        return this.consumer ? this.consumer.getAnalytics() : null;
    }
    getLagStatus() {
        return this.consumer ? this.consumer.getLagStatus(false) : null;
    }
    adjustSubscriptions(topics) {
        if (this.consumer) {
            debug("Adjusting topic subscription", topics.length);
            this.consumer.adjustSubscription(topics);
        }
    }
    getKafkaClient() {
        return this.consumer;
    }
    getKafkaStats() {
        return this.consumer ? this.consumer.getStats() : {};
    }
    getTopicMetadata() {
        return this.consumer ? this.consumer.getMetadata(2500) : Promise.resolve({});
    }
    async close() {
        debug("Closing..");
        if (this.intv) {
            clearInterval(this.intv);
        }
        if (this.consumer) {
            this.consumer.haltAnalytics();
            await this.consumer.close(true);
            this.consumer = null;
        }
    }
}
exports.default = Consumer;