zamza
Version:
Apache Kafka discovery, indexing, searches, storage, hooks and HTTP gateway
186 lines (185 loc) • 8 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const Debug = require("debug");
const debug = Debug("zamza:zamza");
const MongoWrapper_1 = require("./db/MongoWrapper");
const MongoPoller_1 = require("./db/MongoPoller");
const Discovery_1 = require("./kafka/Discovery");
const HttpServer_1 = require("./api/HttpServer");
const MessageHandler_1 = require("./MessageHandler");
const Consumer_1 = require("./kafka/Consumer");
const Producer_1 = require("./kafka/Producer");
const Metrics_1 = require("./Metrics");
const MetadataFetcher_1 = require("./db/MetadataFetcher");
const ReplayConsumer_1 = require("./kafka/ReplayConsumer");
const RetryConsumer_1 = require("./kafka/RetryConsumer");
const ReplayProducer_1 = require("./kafka/ReplayProducer");
const RetryProducer_1 = require("./kafka/RetryProducer");
const HookDealer_1 = require("./HookDealer");
const ReplayHandler_1 = require("./ReplayHandler");
const GRACE_EXIT_MS = 1250;
class Zamza {
constructor(config) {
this.alive = true;
this.ready = false;
if (!config || typeof config !== "object") {
throw new Error("Config must be an object: {kafka,discovery,mongo,http,jobs}");
}
if (!config.hooks) {
debug("No hook configuration found.");
config.hooks = {};
}
this.config = config;
this.metrics = new Metrics_1.Metrics("zamza");
this.discovery = new Discovery_1.default(this.config.discovery, this.metrics);
this.mongoWrapper = new MongoWrapper_1.default(this.config.mongo, this);
this.mongoPoller = new MongoPoller_1.default(this.mongoWrapper, this.metrics);
this.producer = new Producer_1.default(this.config.kafka, this);
this.httpServer = new HttpServer_1.default(this.config.http, this);
this.consumer = new Consumer_1.default(this.config.kafka, this);
this.replayConsumer = new ReplayConsumer_1.default(Object.assign(this.config.kafka, {
consumer: this.cloneKafkaConsumerConfig("zamza-internal-replay-consumer-group-1", this.config.kafka),
}), this);
this.retryConsumer = new RetryConsumer_1.default(Object.assign(this.config.kafka, {
consumer: this.cloneKafkaConsumerConfig("zamza-internal-retry-consumer-group-1", this.config.kafka),
}), this);
this.replayProducer = new ReplayProducer_1.default(Object.assign(this.config.kafka, {
producer: this.cloneKafkaProducerConfig("zamza-internal-replay-producer-1", this.config.kafka),
}), this);
this.retryProducer = new RetryProducer_1.default(Object.assign(this.config.kafka, {
producer: this.cloneKafkaProducerConfig("zamza-internal-retry-producer-1", this.config.kafka),
}), this);
this.hookDealer = new HookDealer_1.default(this);
this.replayHandler = new ReplayHandler_1.ReplayHandler(this);
this.messageHandler = new MessageHandler_1.default(this);
this.metadataFetcher = new MetadataFetcher_1.default(this.mongoWrapper, this.metrics);
}
cloneKafkaProducerConfig(clientId, config) {
const cloneConfig = JSON.parse(JSON.stringify(config.consumer));
cloneConfig.noptions = Object.assign(cloneConfig.noptions, {
"client.id": clientId,
});
return cloneConfig;
}
shutdownOnErrorIfNotProduction() {
if (!Zamza.isProduction()) {
debug("Shutting down (because of error) in", GRACE_EXIT_MS, "ms");
this.close();
setTimeout(() => {
process.exit(1);
}, GRACE_EXIT_MS);
}
}
shutdownGracefully() {
debug("\nShutting down gracefully in", GRACE_EXIT_MS, "ms");
this.close();
debug("Bye..");
setTimeout(() => {
process.exit(0);
}, GRACE_EXIT_MS);
}
init() {
process.on("SIGINT", this.shutdownGracefully.bind(this));
process.on("SIGUSR1", this.shutdownGracefully.bind(this));
process.on("SIGUSR2", this.shutdownGracefully.bind(this));
/*
process.on("warning", (warning: Error) => {
debug("Warning:", warning.message);
}); */
process.on("uncaughtException", (error) => {
debug("Unhandled Exception: ", error.message, error.stack);
this.shutdownOnErrorIfNotProduction();
});
process.on("unhandledRejection", (reason, promise) => {
debug("Unhandled Rejection: ", reason);
this.shutdownOnErrorIfNotProduction();
});
if (Zamza.isProduction()) {
debug("Running production.");
}
else {
debug("Running NOT in production.");
}
}
async run() {
this.init();
debug("Starting..");
this.metrics.registerDefault();
// its okay to start these first, as consumer not subscribe to anything until the
// poller told him about the configured topics
// NOTE: this is necessary, because consumer requires connection before adjusting subscriptions
await this.mongoWrapper.start();
await this.producer.start();
await this.consumer.start();
await this.replayProducer.start();
await this.retryProducer.start();
await this.replayConsumer.start();
await this.retryConsumer.start();
this.mongoPoller.on("error", (error) => {
debug("MongoDB polling error: " + error.message, error.stack);
});
this.mongoPoller.on("topic-config-changed", (topics) => {
// no need to adjust subscriptions
if (this.messageHandler.hooksOnly) {
return;
}
debug("Topic Configuration changed, adjusting subscription of consumer accordingly..", topics.length);
this.consumer.adjustSubscriptions(topics);
});
this.mongoPoller.on("hooks-changed", (hooks) => {
this.hookDealer.processHookUpdate(hooks);
});
await this.discovery.start(this.consumer.getKafkaClient());
await this.mongoPoller.start(this.config.jobs ? this.config.jobs.topicConfigPollingMs : undefined);
await this.metadataFetcher.start(this.config.jobs ? this.config.jobs.metadataFetcherMs : undefined);
await this.httpServer.start();
this.replayConsumer.adjustSubscriptions([MessageHandler_1.INTERNAL_TOPICS.REPLAY_TOPIC]);
this.retryConsumer.adjustSubscriptions([MessageHandler_1.INTERNAL_TOPICS.RETRY_TOPIC]);
this.setReadyState(true);
debug("Running..");
}
async close() {
debug("Closing..");
this.setAliveState(false);
this.setReadyState(false);
await this.replayHandler.close();
this.mongoPoller.close();
this.metadataFetcher.close();
this.discovery.close();
this.httpServer.close();
await this.consumer.close();
await this.replayConsumer.close();
await this.retryConsumer.close();
await this.producer.close();
await this.replayProducer.close();
await this.retryProducer.close();
this.mongoWrapper.close();
this.metrics.close();
this.hookDealer.close();
}
cloneKafkaConsumerConfig(groupId, config) {
const cloneConfig = JSON.parse(JSON.stringify(config.consumer));
cloneConfig.noptions = Object.assign(cloneConfig.noptions, {
"group.id": groupId,
});
return cloneConfig;
}
static isProduction() {
return process.env.NODE_ENV === "production";
}
setAliveState(state) {
debug("Setting alive state from", this.alive, "to", state);
this.alive = state;
}
isAlive() {
return this.alive;
}
setReadyState(state) {
debug("Setting ready state from", this.ready, "to", state);
this.ready = state;
}
isReady() {
return this.ready;
}
}
exports.default = Zamza;