zamza/dist/lib/HookDealer.js

Apache Kafka discovery, indexing, searches, storage, hooks and HTTP gateway

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const Debug = require("debug");
const debug = Debug("zamza:hookdealer");
const Bluebird = require("bluebird");
const HookClient_1 = require("./HookClient");
const MessageHandler_1 = require("./MessageHandler");
const DEFAULT_TIMEOUT = 1500;
const DEFAUT_RETRIES = 0;
const DEFAULT_RETRY_TIMEOUT = 1000;
const DEFAULT_SUBSCRIPTION_CONCURRENCY = 4;
const DEFAULT_REPLAY_CONCURRENCY = 4;
class HookDealer {
    constructor(zamza) {
        this.initialHooksLoaded = false;
        this.oldTopicSubscriptionLength = 0;
        this.metrics = zamza.metrics;
        zamza.config.hooks = zamza.config.hooks ? zamza.config.hooks : {};
        this.timeout = zamza.config.hooks.timeout ? zamza.config.hooks.timeout : DEFAULT_TIMEOUT;
        this.retries = typeof zamza.config.hooks.retries === "number" ? zamza.config.hooks.retries : DEFAUT_RETRIES;
        this.retryTimeout = zamza.config.hooks.retryTimeoutMs ?
            zamza.config.hooks.retryTimeoutMs : DEFAULT_RETRY_TIMEOUT;
        this.subscriptionConcurrency = zamza.config.hooks.subscriptionConcurrency ?
            zamza.config.hooks.subscriptionConcurrency : DEFAULT_SUBSCRIPTION_CONCURRENCY;
        this.replayConcurrency = zamza.config.hooks.replayConcurrency ?
            zamza.config.hooks.replayConcurrency : DEFAULT_REPLAY_CONCURRENCY;
        try {
            this.validateConfigForConvenience(zamza.config.hooks.ignoreConfigValidation);
        }
        catch (error) {
            debug("(HookDealer) Validation error: " + error.message);
            process.exit(1);
        }
        this.mongoPoller = zamza.mongoPoller;
        this.hookModel = zamza.mongoWrapper.getHook();
        this.hookClient = new HookClient_1.default(zamza);
        this.retryProducer = zamza.retryProducer;
        this.producer = zamza.producer;
        this.topicSubscriptionMap = {};
    }
    validateConfigForConvenience(ignoreConfigValidation) {
        if (ignoreConfigValidation) {
            return;
        }
        if (typeof this.timeout !== "number" || this.timeout < 50 || this.timeout > 45000) {
            throw new Error("Configuring the hook timeout below 50 ms or above 45 seconds is not a good idea.");
        }
        if (typeof this.retries !== "number" || this.retries < 0 || this.retries > 25) {
            throw new Error("Configuring the retries below 0 or above 25 is not a good idea.");
        }
        if (typeof this.retryTimeout !== "number" || this.retryTimeout < 0 || this.retryTimeout > 15000) {
            throw new Error("Configuring the retryTimeout below 0 ms or above 15 seconds is not a good idea.");
        }
        if (typeof this.subscriptionConcurrency !== "number" || this.subscriptionConcurrency < 1 ||
            this.subscriptionConcurrency > 150) {
            throw new Error("Configuring the subscriptionConcurrency below 1 or above 150 is not a good idea.");
        }
        if (typeof this.replayConcurrency !== "number" || this.replayConcurrency < 1 ||
            this.replayConcurrency > 150) {
            throw new Error("Configuring the replayConcurrency below 1 or above 150 is not a good idea.");
        }
    }
    findConfigForTopic(topic) {
        const topicConfigs = this.mongoPoller.getCollected().topicConfigs;
        for (let i = topicConfigs.length - 1; i >= 0; i--) {
            if (topicConfigs[i].topic === topic) {
                return topicConfigs[i];
            }
        }
        return null;
    }
    processHookUpdate(hooks) {
        if (!hooks) {
            return;
        }
        if (!this.initialHooksLoaded) {
            debug("Initial hooks loaded", hooks.length);
            this.initialHooksLoaded = true;
        }
        let endpoints = 0;
        // transform the hooks into a structure that is more performant to process
        const topicSubscriptionMap = {};
        hooks.forEach((hook) => {
            if (hook.disabled || !hook.subscriptions) {
                return; // skip
            }
            hook.subscriptions.forEach((subscription) => {
                if (subscription.disabled) {
                    return; // skip
                }
                if (topicSubscriptionMap[subscription.topic]) {
                    topicSubscriptionMap[subscription.topic] = [];
                }
                const hookClone = JSON.parse(JSON.stringify(hook));
                delete hookClone.subscriptions;
                hookClone.ignoreReplay = subscription.ignoreReplay;
                topicSubscriptionMap[subscription.topic].push(hookClone);
                endpoints++;
            });
        });
        if (endpoints !== this.oldTopicSubscriptionLength) {
            debug("Topic Subscription Map has changed from", this.oldTopicSubscriptionLength, "to", endpoints);
            this.oldTopicSubscriptionLength = endpoints;
            this.metrics.set("configured_active_subscriptions", endpoints);
        }
        this.topicSubscriptionMap = topicSubscriptionMap;
    }
    async handleSubscription(message, mappedHook, replayPayload, retryPayload) {
        const body = {
            message,
            context: null,
        };
        if (replayPayload) {
            delete replayPayload.message;
            body.context = {
                type: "replay",
                data: replayPayload,
            };
        }
        else if (retryPayload) {
            delete retryPayload.message;
            body.context = {
                type: "retry",
                data: replayPayload,
            };
        }
        const options = {
            method: "POST",
            url: mappedHook.endpoint,
            headers: {
                "content-type": "application/json",
            },
            timeout: this.timeout,
            body: JSON.stringify(body),
        };
        if (mappedHook.authorizationHeader && mappedHook.authorizationValue) {
            options.headers[mappedHook.authorizationHeader] = mappedHook.authorizationValue;
        }
        const { status, body: responseBody } = await this.hookClient.call(options);
        // simple delivery hook
        if (status === 200) {
            return;
        }
        // hook delivered, however a message should be produced in return
        if (status === 205) {
            try {
                const { topic, partition, key, value, } = JSON.parse(responseBody);
                await this.producer.produceMessage(topic, partition, key, value);
            }
            catch (error) {
                debug("Failed to produce message on 205 hook response: "
                    + error.message + ", " + mappedHook.endpoint);
            }
            return;
        }
        throw new Error(`Expected status code did not match 200 or 205 ${status}.`);
    }
    async handleMessage(message) {
        // called after message handler has handled mongo storage
        const subscriptions = this.topicSubscriptionMap[message.topic];
        if (!subscriptions) {
            return false;
        }
        this.metrics.inc("hook_processed_messages");
        await Bluebird.map(subscriptions, (subscription) => {
            return this.handleSubscription(message, subscription).then(() => {
                this.metrics.inc("hook_delivered");
            }).catch((_) => {
                this.metrics.inc("hook_failed");
                if (this.retries > 0) {
                    const retryMessage = {
                        message,
                        hookId: subscription._id,
                        retryCount: 0,
                        fromReplay: false,
                    };
                    this.metrics.inc("hook_produce_retry");
                    setTimeout(() => {
                        this.retryProducer.produceMessage(MessageHandler_1.INTERNAL_TOPICS.RETRY_TOPIC, undefined, undefined, JSON.stringify(retryMessage));
                    }, this.retryTimeout);
                }
            });
        }, { concurrency: this.subscriptionConcurrency });
        this.metrics.inc("hook_processed_messages_success");
        return true;
    }
    async handleRetryMessage(message) {
        // called before message handler has handled mongo storage
        this.metrics.inc("hook_processed_retry_messages");
        if (!message.value) {
            return false;
        }
        let parsedMessage = null;
        try {
            if (typeof message.value === "string") {
                parsedMessage = JSON.parse(Buffer.isBuffer(message.value) ?
                    message.value.toString("utf8") : message.value);
            }
            else {
                parsedMessage = message.value;
            }
            if (!parsedMessage) {
                throw new Error("Parsed Message empty.");
            }
            if (!parsedMessage.hookId) {
                throw new Error("HookID missing on retry message payload.");
            }
        }
        catch (error) {
            debug("Failed to parse retry message payload: " + error.message);
            return false;
        }
        // no topic config present anymore, skip hooks for replays
        if (!this.findConfigForTopic(parsedMessage.message.topic)) {
            this.metrics.inc("hook_processed_retry_messages_success");
            return false;
        }
        // subscription has been deleted OR is disabled currently
        // let error fall through for retry
        const subscription = await this.hookModel.get(parsedMessage.hookId);
        if (!subscription || subscription.disabled) {
            return false;
        }
        return this.handleSubscription(parsedMessage.message, subscription, undefined, parsedMessage).then(() => {
            this.metrics.inc("hook_retry_delivered");
            this.metrics.inc("hook_processed_retry_messages_success");
            return true;
        }).catch((_) => {
            this.metrics.inc("hook_retry_failed");
            if (this.retries > parsedMessage.retryCount) {
                const retryMessage = {
                    message,
                    hookId: subscription._id,
                    retryCount: parsedMessage.retryCount + 1,
                };
                this.metrics.inc("hook_produce_retry");
                setTimeout(() => {
                    this.retryProducer.produceMessage(MessageHandler_1.INTERNAL_TOPICS.RETRY_TOPIC, undefined, undefined, JSON.stringify(retryMessage));
                }, this.retryTimeout);
            }
            else {
                this.metrics.inc("hook_retry_reached");
            }
            this.metrics.inc("hook_processed_retry_messages_success");
            return true;
        });
    }
    async handleReplayMessage(message) {
        // called before message handler has handled mongo storage
        this.metrics.inc("hook_processed_replay_messages");
        if (!message.value) {
            return false;
        }
        let parsedMessage = null;
        try {
            if (typeof message.value === "string") {
                parsedMessage = JSON.parse(Buffer.isBuffer(message.value) ?
                    message.value.toString("utf8") : message.value);
            }
            else {
                parsedMessage = message.value;
            }
            if (!parsedMessage) {
                throw new Error("Parsed Message empty.");
            }
        }
        catch (error) {
            debug("Failed to parse retry message payload: " + error.message);
            return false;
        }
        // no topic config present anymore, skip hooks for replays
        if (!this.findConfigForTopic(parsedMessage.message.topic)) {
            this.metrics.inc("hook_processed_retry_messages_success");
            return false;
        }
        let subscriptions = this.topicSubscriptionMap[parsedMessage.message.topic];
        if (!subscriptions) {
            return false;
        }
        // filter out subscriptions that dont participate in replays
        subscriptions = subscriptions.filter((subscription) => !subscription.ignoreReplay);
        await Bluebird.map(subscriptions, (subscription) => {
            return this.handleSubscription(parsedMessage.message, subscription, parsedMessage).then(() => {
                this.metrics.inc("hook_replay_delivered");
            }).catch((_) => {
                this.metrics.inc("hook_replay_failed");
                if (this.retries > 0) {
                    const retryMessage = {
                        message,
                        hookId: subscription._id,
                        retryCount: 0,
                        fromReplay: true,
                    };
                    this.metrics.inc("hook_replay_produce_retry");
                    setTimeout(() => {
                        this.retryProducer.produceMessage(MessageHandler_1.INTERNAL_TOPICS.RETRY_TOPIC, undefined, undefined, JSON.stringify(retryMessage));
                    }, this.retryTimeout);
                }
            });
        }, { concurrency: this.replayConcurrency });
        this.metrics.inc("hook_processed_replay_messages_success");
        return true;
    }
    close() {
        if (this.hookClient) {
            return this.hookClient.close();
        }
        return null;
    }
}
exports.default = HookDealer;