zamza/dist/lib/db/MetadataFetcher.js

Apache Kafka discovery, indexing, searches, storage, hooks and HTTP gateway

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const Debug = require("debug");
const debug = Debug("zamza:metadatafetcher");
const Bluebird = require("bluebird");
const models_1 = require("./models");
class MetadataFetcher {
    constructor(mongoWrapper, metrics) {
        this.topicConfigModel = mongoWrapper.getTopicConfig();
        this.keyIndexModel = mongoWrapper.getKeyIndex();
        this.lockModel = mongoWrapper.getLock();
        this.topicMetadataModel = mongoWrapper.getTopicMetadata();
        this.sharedStateModel = mongoWrapper.getSharedState();
        this.metrics = metrics;
        this.intv = null;
    }
    async start(intervalMs = 12 * 62000) {
        this.close();
        this.intv = setInterval(() => {
            this.onInterval()
                .catch((error) => {
                debug("Error on metadata job", error.message);
            });
        }, intervalMs);
        // no instant initial poll
        setTimeout(() => {
            this.onInterval().catch((error) => {
                debug("Initial metadata job failed", error.message);
            });
        }, 12000);
    }
    close() {
        if (this.intv) {
            clearInterval(this.intv);
        }
    }
    async onInterval() {
        this.metrics.inc("job_metadata_ran");
        // if this is a new setup, reset to false by default
        let shouldRun = await this.sharedStateModel.get(models_1.STATE_KEYS.ENABLE_METADATA_JOB);
        if (!shouldRun) {
            shouldRun = await this.sharedStateModel.set(models_1.STATE_KEYS.ENABLE_METADATA_JOB, "false");
        }
        if (shouldRun !== "true") {
            debug("Wont run metadata job, as its shared state value is not 'true': ", shouldRun);
            return;
        }
        const startTime = Date.now();
        const topicConfigs = await this.topicConfigModel.listAsTopics();
        await Bluebird.map(topicConfigs, async (topic) => {
            const lockName = `metadata:${topic}`;
            if (!await this.lockModel.getLock(lockName, 3 * 60000)) {
                return; // did not get a lock for this topic
            }
            const innerStartTime = Date.now();
            debug("Got lock for topic", topic, "fetching and storing metadata..");
            // got lock for topic, fetch and store metadata
            const topicMetadata = await this.keyIndexModel.getMetadataForTopic(topic);
            await this.topicMetadataModel.upsert(topicMetadata);
            await this.lockModel.removeLock(lockName);
            const innerDuration = Date.now() - innerStartTime;
            debug("Fetched and stored", topic, "metadata in", innerDuration, "ms");
        }, { concurrency: 1 });
        const duration = Date.now() - startTime;
        this.metrics.set("job_metadata_ms", duration);
        this.metrics.inc("job_metadata_ran_success");
    }
}
exports.default = MetadataFetcher;