xkite-core
Version:
Core library for xkite application
325 lines (324 loc) • 22.8 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const js_yaml_1 = __importDefault(require("js-yaml"));
const fs_extra_1 = __importDefault(require("fs-extra"));
const path_1 = __importDefault(require("path"));
const yml = __importStar(require("./constants/yml.js"));
const { SPRING, SPARK, JUPYTER, KSQL_SCHEMA, KSQL_CLI, KSQL, POSTGRES, GRAFANA, PROMETHEUS, ZOOKEEPER, KAFKA_BROKER, JMX, PROMCONFIG, downloadDir, network, _ports_, KAFKA_CONNECT_SRC, KAFKA_CONNECT_SINK, } = yml;
const ymlGenerator = () => {
const YAML = {
services: {},
volumes: {
jupyterhub_data: {
driver: 'local',
},
dashboards: {
driver: 'local',
driver_opts: {
o: 'bind',
type: 'none',
device: `${path_1.default.join(downloadDir, 'grafana/dashboards')}`,
},
},
provisioning: {
driver: 'local',
driver_opts: {
o: 'bind',
type: 'none',
device: `${path_1.default.join(downloadDir, 'grafana/provisioning')}`,
},
},
},
};
const dependencies = [];
let setup = {
kafkaSetup: {
clientId: 'xkite',
brokers: [],
ssl: false,
},
};
return (config) => {
var _a, _b, _c;
console.log('creating Kite Config yml...');
const { kafka, db, sink, grafana, prometheus } = config;
try {
const dBSetup = createDB(db);
if (dBSetup !== undefined)
setup = Object.assign(Object.assign({}, setup), { dBSetup });
createSink(config);
if (prometheus !== undefined) {
const extPromPort = (_a = prometheus.port) !== null && _a !== void 0 ? _a : _ports_.prometheus.external;
YAML.services.prometheus = Object.assign(Object.assign({}, PROMETHEUS), { ports: [`${extPromPort}:${_ports_.prometheus.internal}`] });
setup = Object.assign(Object.assign({}, setup), { prometheus: { port: extPromPort } });
fs_extra_1.default.ensureDirSync(path_1.default.resolve(downloadDir, 'prometheus'));
}
if (grafana !== undefined) {
const extGrafPort = (_b = grafana === null || grafana === void 0 ? void 0 : grafana.port) !== null && _b !== void 0 ? _b : _ports_.grafana.external;
YAML.services.grafana = Object.assign(Object.assign({}, GRAFANA), { ports: [`${extGrafPort}:${_ports_.grafana.internal}`] });
setup = Object.assign(Object.assign({}, setup), { grafana: { port: extGrafPort } });
}
const servers = createZooKeepers(kafka);
createBrokers(config, servers);
fs_extra_1.default.writeFileSync(path_1.default.resolve(downloadDir, 'docker-compose.yml'), js_yaml_1.default.dump(YAML, { noRefs: true }));
if (prometheus !== undefined) {
PROMCONFIG.global.scrape_interval = `${prometheus.scrape_interval}s`;
PROMCONFIG.global.scrape_timeout =
prometheus.scrape_interval !== undefined
? prometheus.scrape_interval - 1 <= 0
? '1s'
: `${prometheus.scrape_interval - 1}s`
: '10s';
PROMCONFIG.global.evaluation_interval = `${prometheus.evaluation_interval}s`;
fs_extra_1.default.writeFileSync(path_1.default.resolve(downloadDir, 'prometheus/prometheus.yml'), js_yaml_1.default.dump(PROMCONFIG, { noRefs: true }));
PROMCONFIG.scrape_configs[0].static_configs[0].targets = [];
}
}
catch (error) {
console.error(error);
}
finally {
for (const service in YAML.services) {
if ('docker' in setup) {
if (((_c = setup.docker) === null || _c === void 0 ? void 0 : _c.services) !== undefined) {
setup = Object.assign(Object.assign({}, setup), { docker: {
services: [...setup.docker.services, service],
} });
}
}
else {
setup = Object.assign(Object.assign({}, setup), { docker: { services: [service] } });
}
}
return setup;
}
};
function createDB(db) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
if ((db === null || db === void 0 ? void 0 : db.name) === 'postgresql') {
YAML.services.postgresql = Object.assign(Object.assign({}, POSTGRES), { ports: [
`${(_a = db.port) !== null && _a !== void 0 ? _a : _ports_.postgresql.external}:${_ports_.postgresql.internal}`,
], environment: Object.assign(Object.assign({}, POSTGRES.environment), { POSTGRES_USER: (_c = (_b = db.postgresql) === null || _b === void 0 ? void 0 : _b.username) !== null && _c !== void 0 ? _c : 'admin', POSTGRES_PASSWORD: (_e = (_d = db.postgresql) === null || _d === void 0 ? void 0 : _d.password) !== null && _e !== void 0 ? _e : 'admin', POSTGRES_DB: (_g = (_f = db.postgresql) === null || _f === void 0 ? void 0 : _f.dbname) !== null && _g !== void 0 ? _g : 'xkiteDB' }) });
YAML.volumes = Object.assign(Object.assign({}, YAML.volumes), { postgresql: {
driver: 'local',
} });
YAML.services.kafka_connect_src = KAFKA_CONNECT_SRC;
}
else if ((db === null || db === void 0 ? void 0 : db.name) === 'ksql') {
YAML.services.ksql = Object.assign(Object.assign({}, KSQL), { ports: [`${(_h = db.port) !== null && _h !== void 0 ? _h : _ports_.ksql.external}:${_ports_.ksql.internal}`], environment: Object.assign(Object.assign({}, KSQL.environment), { KSQL_LISTENERS: `http://0.0.0.0:${(_j = db.port) !== null && _j !== void 0 ? _j : _ports_.ksql.external}`, KSQL_KSQL_SCHEMA_REGISTRY_URL: `http://schema-registry:${(_l = (_k = db.ksql) === null || _k === void 0 ? void 0 : _k.schema_port) !== null && _l !== void 0 ? _l : _ports_.ksql_schema.internal}` }) });
YAML.services.ksql_schema = Object.assign(Object.assign({}, KSQL_SCHEMA), { ports: [
`${(_o = (_m = db.ksql) === null || _m === void 0 ? void 0 : _m.schema_port) !== null && _o !== void 0 ? _o : _ports_.ksql_schema.external}:${_ports_.ksql_schema.internal}`,
] });
YAML.services.ksql_cli = Object.assign(Object.assign({}, KSQL_CLI), { depends_on: [YAML.services.ksql.container_name] });
YAML.services.kafka_connect_src = KAFKA_CONNECT_SRC;
}
return db;
}
function createSink(config) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
if (config.sink !== undefined) {
YAML.services.kafka_connect_sink = Object.assign(Object.assign({}, KAFKA_CONNECT_SINK), { ports: [
`${(_c = (_b = (_a = config.sink) === null || _a === void 0 ? void 0 : _a.kafkaconnect) === null || _b === void 0 ? void 0 : _b.port) !== null && _c !== void 0 ? _c : _ports_.kafkaconnect_sink.external}:${_ports_.kafkaconnect_sink.internal}`,
] });
if (((_d = config.sink) === null || _d === void 0 ? void 0 : _d.name) === 'jupyter') {
YAML.services.jupyter = Object.assign(Object.assign({}, JUPYTER), { ports: [
`${(_f = (_e = config.sink) === null || _e === void 0 ? void 0 : _e.port) !== null && _f !== void 0 ? _f : _ports_.jupyter.external}:${_ports_.jupyter.internal}`,
] });
setup = Object.assign(Object.assign({}, setup), { jupyter: { port: (_h = (_g = config.sink) === null || _g === void 0 ? void 0 : _g.port) !== null && _h !== void 0 ? _h : _ports_.jupyter.external } });
}
else if (((_j = config.sink) === null || _j === void 0 ? void 0 : _j.name) === 'spark') {
YAML.services.spark = Object.assign(Object.assign({}, SPARK), { ports: [
`${(_l = (_k = config.sink) === null || _k === void 0 ? void 0 : _k.port) !== null && _l !== void 0 ? _l : _ports_.spark.webui.external}:${_ports_.spark.webui.internal}`,
`${(_o = (_m = config.sink) === null || _m === void 0 ? void 0 : _m.rpc_port) !== null && _o !== void 0 ? _o : _ports_.spark.rpc.external}:${_ports_.spark.rpc.internal}`,
] });
setup = Object.assign(Object.assign({}, setup), { spark: {
port: [
(_q = (_p = config.sink) === null || _p === void 0 ? void 0 : _p.port) !== null && _q !== void 0 ? _q : _ports_.spark.webui.external,
(_s = (_r = config.sink) === null || _r === void 0 ? void 0 : _r.rpc_port) !== null && _s !== void 0 ? _s : _ports_.spark.rpc.external,
],
} });
}
}
}
function createZooKeepers(kafka) {
var _a;
console.log('creating zookeepers...');
const numOfZKs = kafka.zookeepers.size > 1 ? kafka.zookeepers.size : 1;
const name = (x) => `zookeeper${x}`;
const getZKServerPorts = () => {
let zkClients = '';
let zkPeers = '';
const getPeerPort = (x) => {
if (kafka.zookeepers.ports !== undefined &&
kafka.zookeepers.ports.peer !== undefined)
return `${kafka.zookeepers.ports.peer.external}:${kafka.zookeepers.ports.peer.internal};`;
return `${x + 1}${_ports_.zookeeper.peer.external}:${x + 1}${_ports_.zookeeper.peer.internal};`;
};
const getClientPort = (x) => {
var _a, _b;
if (kafka.zookeepers.ports !== undefined &&
((_a = kafka.zookeepers.ports) === null || _a === void 0 ? void 0 : _a.client) !== undefined)
return ((_b = kafka.zookeepers.ports) === null || _b === void 0 ? void 0 : _b.client[x]) + ',';
return `${x + 1}${_ports_.zookeeper.client.external},`;
};
for (let i = 0; i < numOfZKs; i++) {
zkClients += `${name(i + 1)}:${getClientPort(i)}`;
zkPeers += `${name(i + 1)}:${getPeerPort(i)}`;
}
zkClients = zkClients.slice(0, -1);
zkPeers = zkPeers.slice(0, -1);
return { zkClients, zkPeers };
};
const servers = getZKServerPorts();
setup = Object.assign(Object.assign({}, setup), { zookeeper: { ports: [] } });
for (let i = 0; i < numOfZKs; i++) {
const n = i + 1;
const name = `zookeeper${n}`;
let cport = 10000 * n + _ports_.zookeeper.client.external;
if (((_a = kafka.zookeepers.ports) === null || _a === void 0 ? void 0 : _a.client) !== undefined &&
kafka.zookeepers.ports.client[i] !== undefined) {
cport = kafka.zookeepers.ports.client[i];
}
YAML.services[name] = Object.assign(Object.assign({}, ZOOKEEPER), { environment: Object.assign(Object.assign({}, ZOOKEEPER.environment), { ZOOKEEPER_SERVER_ID: n, ZOOKEEPER_CLIENT_PORT: cport, ZOOKEEPER_SERVERS: servers.zkPeers }), ports: [`${cport}:${_ports_.zookeeper.client.internal}`], container_name: name });
if (setup.zookeeper !== undefined)
setup = Object.assign(Object.assign({}, setup), { zookeeper: { ports: [...setup.zookeeper.ports, cport] } });
if (YAML.services.ksql_schema !== undefined) {
YAML.services.ksql_schema.environment.SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL += `${name}:${_ports_.zookeeper.client.internal},`;
}
dependencies.push(name);
}
return servers;
}
function createBrokers(config, servers) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
const { kafka } = config;
console.log('creating brokers...');
let jmxExporterConfig;
if (kafka.jmx !== undefined) {
fs_extra_1.default.ensureDirSync(path_1.default.resolve(downloadDir, 'jmx'));
jmxExporterConfig = js_yaml_1.default.load(fs_extra_1.default.readFileSync(path_1.default.resolve(downloadDir, 'jmx/exporter/template.yml'), 'utf8'));
setup = Object.assign(Object.assign({}, setup), { jmx: { ports: [] } });
}
const springBSServers = [];
const springDeps = [];
for (let i = 0; i < kafka.brokers.size; i++) {
const n = i + 1;
const brokerName = `kafka${n}`;
let extPort = _ports_.kafka.broker.external + i;
if (kafka.brokers.ports !== undefined &&
kafka.brokers.ports.brokers !== undefined &&
kafka.brokers.ports.brokers[i] !== undefined)
extPort = kafka.brokers.ports.brokers[i];
let metricsPort = _ports_.kafka.metrics;
if (kafka.brokers.ports !== undefined &&
kafka.brokers.ports.metrics !== undefined)
metricsPort = kafka.brokers.ports.metrics;
springBSServers.push(`${brokerName}:${_ports_.kafka.spring}`);
let jmxHostPort = _ports_.kafka.jmx + i;
if (kafka.brokers.ports !== undefined &&
kafka.brokers.ports.jmx !== undefined &&
kafka.brokers.ports.jmx[i] !== undefined)
jmxHostPort = kafka.brokers.ports.jmx[i];
let brokerID = 101 + i;
if (kafka.brokers.id !== undefined && kafka.brokers.id[i] !== undefined)
brokerID = kafka.brokers.id[i];
YAML.services[brokerName] = Object.assign(Object.assign({}, KAFKA_BROKER), { ports: [`${extPort}:${_ports_.kafka.broker.internal}`], container_name: brokerName, depends_on: dependencies, environment: Object.assign(Object.assign({}, KAFKA_BROKER.environment), { KAFKA_BROKER_ID: brokerID, KAFKA_JMX_PORT: jmxHostPort, KAFKA_LISTENERS: `METRICS://:${metricsPort},PLAINTEXT://:${extPort},INTERNAL://:${_ports_.kafka.spring},KSQL://${brokerName}:${_ports_.kafka.ksql},CONNECT_SRC://${brokerName}:${_ports_.kafka.connect_src},CONNECT_SINK://${brokerName}:${_ports_.kafka.connect_sink}`, KAFKA_ADVERTISED_LISTENERS: `METRICS://${brokerName}:${metricsPort},PLAINTEXT://${network}:${extPort},INTERNAL://${brokerName}:${_ports_.kafka.spring},KSQL://${brokerName}:${_ports_.kafka.ksql},CONNECT_SRC://${brokerName}:${_ports_.kafka.connect_src},CONNECT_SINK://${brokerName}:${_ports_.kafka.connect_sink}`, KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: ((_a = kafka.brokers.replicas) !== null && _a !== void 0 ? _a : 1) > kafka.brokers.size
? kafka.brokers.size
: (_b = kafka.brokers.replicas) !== null && _b !== void 0 ? _b : 1, KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: ((_c = kafka.brokers.replicas) !== null && _c !== void 0 ? _c : 1) > kafka.brokers.size
? kafka.brokers.size
: (_d = kafka.brokers.replicas) !== null && _d !== void 0 ? _d : 1, CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: `${brokerName}:${metricsPort}`, KAFKA_ZOOKEEPER_CONNECT: servers.zkClients, CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: servers.zkClients }) });
setup = Object.assign(Object.assign({}, setup), { kafkaSetup: Object.assign(Object.assign({}, setup.kafkaSetup), { brokers: [...setup.kafkaSetup.brokers, `${network}:${extPort}`] }) });
springDeps.push(brokerName);
if (kafka.jmx !== undefined) {
let jmxPort = _ports_.jmx.external + n;
const jmxName = `jmx-kafka${n}`;
if (kafka.jmx !== undefined && ((_e = kafka.jmx) === null || _e === void 0 ? void 0 : _e.ports) !== undefined) {
jmxPort = (_g = (_f = kafka.jmx) === null || _f === void 0 ? void 0 : _f.ports[i]) !== null && _g !== void 0 ? _g : jmxPort;
}
YAML.services[jmxName] = Object.assign(Object.assign({}, JMX), { command: [`${_ports_.jmx.internal}`, '/etc/myconfig.yml'], ports: [`${jmxPort}:${_ports_.jmx.internal}`], environment: Object.assign(Object.assign({}, JMX.environment), { SERVICE_PORT: _ports_.jmx.internal }), container_name: jmxName, volumes: [
`${path_1.default.join(downloadDir, `/jmx/jmxConfigKafka${n}.yml`)}:/etc/myconfig.yml`,
], depends_on: [`kafka${n}`] });
let jmxArr = [jmxPort];
if (((_h = setup.jmx) === null || _h === void 0 ? void 0 : _h.ports) !== undefined)
jmxArr = [...(_j = setup.jmx) === null || _j === void 0 ? void 0 : _j.ports, jmxPort];
setup = Object.assign(Object.assign({}, setup), { jmx: { ports: jmxArr } });
jmxExporterConfig.hostPort = `kafka${n}:${jmxHostPort}`;
fs_extra_1.default.writeFileSync(path_1.default.resolve(downloadDir, `jmx/jmxConfigKafka${n}.yml`), js_yaml_1.default.dump(jmxExporterConfig, { noRefs: true }));
if (config.prometheus !== undefined) {
PROMCONFIG.scrape_configs[0].static_configs[0].targets.push(`${jmxName}:${_ports_.jmx.internal}`);
}
}
if (YAML.services.ksql !== undefined) {
if (YAML.services.ksql_schema !== undefined) {
YAML.services.ksql_schema.environment = Object.assign(Object.assign({}, YAML.services.ksql_schema.environment), { SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: YAML.services.ksql_schema.environment
.SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS +
`PLAINTEXT://${brokerName}:9092,` });
}
YAML.services.ksql = Object.assign(Object.assign({}, YAML.services.ksql), { environment: Object.assign(Object.assign({}, YAML.services.ksql.environment), { KSQL_BOOTSTRAP_SERVERS: YAML.services.ksql.environment.KSQL_BOOTSTRAP_SERVERS +
`${brokerName}:${_ports_.kafka.ksql},` }) });
if (YAML.services.ksql_cli !== undefined) {
const deps = (_k = YAML.services.ksql_cli.depends_on) !== null && _k !== void 0 ? _k : [
YAML.services.ksql.container_name,
];
deps.push(`${brokerName}`);
YAML.services.ksql_cli = Object.assign(Object.assign({}, YAML.services.ksql_cli), { depends_on: deps });
}
}
if (YAML.services.kafka_connect_src !== undefined) {
const kcenv = YAML.services.kafka_connect_src.environment;
const kcdeps = YAML.services.kafka_connect_src.depends_on === undefined ||
((_l = YAML.services.kafka_connect_src.depends_on) === null || _l === void 0 ? void 0 : _l.length) === 0
? [brokerName]
: [...YAML.services.kafka_connect_src.depends_on, brokerName];
YAML.services.kafka_connect_src = Object.assign(Object.assign({}, YAML.services.kafka_connect_src), { environment: Object.assign(Object.assign({}, kcenv), { CONNECT_BOOTSTRAP_SERVERS: kcenv.CONNECT_BOOTSTRAP_SERVERS !== ''
? `${kcenv.CONNECT_BOOTSTRAP_SERVERS},${brokerName}:${_ports_.kafka.connect_src}`
: `${brokerName}:${_ports_.kafka.connect_src}` }), depends_on: kcdeps });
}
if (YAML.services.kafka_connect_sink !== undefined) {
const kcenv = YAML.services.kafka_connect_sink.environment;
const kcdeps = YAML.services.kafka_connect_sink.depends_on === undefined ||
((_m = YAML.services.kafka_connect_sink.depends_on) === null || _m === void 0 ? void 0 : _m.length) === 0
? [brokerName]
: [...YAML.services.kafka_connect_sink.depends_on, brokerName];
YAML.services.kafka_connect_sink = Object.assign(Object.assign({}, YAML.services.kafka_connect_sink), { environment: Object.assign(Object.assign({}, kcenv), { CONNECT_BOOTSTRAP_SERVERS: kcenv.CONNECT_BOOTSTRAP_SERVERS !== ''
? `${kcenv.CONNECT_BOOTSTRAP_SERVERS},${brokerName}:${_ports_.kafka.connect_sink}`
: `${brokerName}:${_ports_.kafka.connect_sink}` }), depends_on: kcdeps });
}
}
if (kafka.spring !== undefined) {
const springPort = (_p = (_o = kafka.spring) === null || _o === void 0 ? void 0 : _o.port) !== null && _p !== void 0 ? _p : _ports_.spring.external;
YAML.services.spring = Object.assign(Object.assign({}, SPRING), { ports: [`${springPort}:${_ports_.spring.external}`], environment: Object.assign(Object.assign({}, SPRING.environment), { 'SPRING_KAFKA_BOOTSTRAP-SERVERS': springBSServers.join(','), 'SPRING_KAFKA_CONSUMER_BOOTSTRAP-SERVERS': springBSServers.join(','), 'SPRING_KAFKA_PRODUCER_BOOTSTRAP-SERVERS': springBSServers.join(',') }), depends_on: springDeps });
setup = Object.assign(Object.assign({}, setup), { spring: { port: springPort } });
}
if (YAML.services.ksql_schema !== undefined) {
YAML.services.ksql_schema.depends_on = [...dependencies, ...springDeps];
}
}
};
exports.default = ymlGenerator;