xkite-core
Version:
Core library for xkite application
434 lines (433 loc) • 20.1 kB
JavaScript
import yaml from 'js-yaml';
import fs from 'fs-extra';
import path from 'path';
import * as yml from './constants/yml.js';
const { SPRING, SPARK, JUPYTER, KSQL_SCHEMA, KSQL_CLI, KSQL, POSTGRES, GRAFANA, PROMETHEUS, ZOOKEEPER, KAFKA_BROKER, JMX, PROMCONFIG, downloadDir, network, _ports_, KAFKA_CONNECT_SRC, KAFKA_CONNECT_SINK, } = yml;
const ymlGenerator = () => {
const YAML = {
services: {},
volumes: {
jupyterhub_data: {
driver: 'local',
},
dashboards: {
driver: 'local',
driver_opts: {
o: 'bind',
type: 'none',
device: `${path.join(downloadDir, 'grafana/dashboards')}`,
},
},
provisioning: {
driver: 'local',
driver_opts: {
o: 'bind',
type: 'none',
device: `${path.join(downloadDir, 'grafana/provisioning')}`,
},
},
},
};
const dependencies = [];
let setup = {
kafkaSetup: {
clientId: 'xkite',
brokers: [],
ssl: false,
},
};
return (config) => {
console.log('creating Kite Config yml...');
const { kafka, db, sink, grafana, prometheus } = config;
try {
const dBSetup = createDB(db);
if (dBSetup !== undefined)
setup = { ...setup, dBSetup };
createSink(config);
if (prometheus !== undefined) {
const extPromPort = prometheus.port ?? _ports_.prometheus.external;
YAML.services.prometheus = {
...PROMETHEUS,
ports: [`${extPromPort}:${_ports_.prometheus.internal}`],
};
setup = {
...setup,
prometheus: { port: extPromPort },
};
fs.ensureDirSync(path.resolve(downloadDir, 'prometheus'));
}
if (grafana !== undefined) {
const extGrafPort = grafana?.port ?? _ports_.grafana.external;
YAML.services.grafana = {
...GRAFANA,
ports: [`${extGrafPort}:${_ports_.grafana.internal}`],
};
setup = { ...setup, grafana: { port: extGrafPort } };
}
const servers = createZooKeepers(kafka);
createBrokers(config, servers);
fs.writeFileSync(path.resolve(downloadDir, 'docker-compose.yml'), yaml.dump(YAML, { noRefs: true }));
if (prometheus !== undefined) {
PROMCONFIG.global.scrape_interval = `${prometheus.scrape_interval}s`;
PROMCONFIG.global.scrape_timeout =
prometheus.scrape_interval !== undefined
? prometheus.scrape_interval - 1 <= 0
? '1s'
: `${prometheus.scrape_interval - 1}s`
: '10s';
PROMCONFIG.global.evaluation_interval = `${prometheus.evaluation_interval}s`;
fs.writeFileSync(path.resolve(downloadDir, 'prometheus/prometheus.yml'), yaml.dump(PROMCONFIG, { noRefs: true }));
PROMCONFIG.scrape_configs[0].static_configs[0].targets = [];
}
}
catch (error) {
console.error(error);
}
finally {
for (const service in YAML.services) {
if ('docker' in setup) {
if (setup.docker?.services !== undefined) {
setup = {
...setup,
docker: {
services: [...setup.docker.services, service],
},
};
}
}
else {
setup = { ...setup, docker: { services: [service] } };
}
}
return setup;
}
};
function createDB(db) {
if (db?.name === 'postgresql') {
YAML.services.postgresql = {
...POSTGRES,
ports: [
`${db.port ?? _ports_.postgresql.external}:${_ports_.postgresql.internal}`,
],
environment: {
...POSTGRES.environment,
POSTGRES_USER: db.postgresql?.username ?? 'admin',
POSTGRES_PASSWORD: db.postgresql?.password ?? 'admin',
POSTGRES_DB: db.postgresql?.dbname ?? 'xkiteDB',
},
};
YAML.volumes = {
...YAML.volumes,
postgresql: {
driver: 'local',
},
};
YAML.services.kafka_connect_src = KAFKA_CONNECT_SRC;
}
else if (db?.name === 'ksql') {
YAML.services.ksql = {
...KSQL,
ports: [`${db.port ?? _ports_.ksql.external}:${_ports_.ksql.internal}`],
environment: {
...KSQL.environment,
KSQL_LISTENERS: `http://0.0.0.0:${db.port ?? _ports_.ksql.external}`,
KSQL_KSQL_SCHEMA_REGISTRY_URL: `http://schema-registry:${db.ksql?.schema_port ?? _ports_.ksql_schema.internal}`,
},
};
YAML.services.ksql_schema = {
...KSQL_SCHEMA,
ports: [
`${db.ksql?.schema_port ?? _ports_.ksql_schema.external}:${_ports_.ksql_schema.internal}`,
],
};
YAML.services.ksql_cli = {
...KSQL_CLI,
depends_on: [YAML.services.ksql.container_name],
};
YAML.services.kafka_connect_src = KAFKA_CONNECT_SRC;
}
return db;
}
function createSink(config) {
if (config.sink !== undefined) {
YAML.services.kafka_connect_sink = {
...KAFKA_CONNECT_SINK,
ports: [
`${config.sink?.kafkaconnect?.port ??
_ports_.kafkaconnect_sink.external}:${_ports_.kafkaconnect_sink.internal}`,
],
};
if (config.sink?.name === 'jupyter') {
YAML.services.jupyter = {
...JUPYTER,
ports: [
`${config.sink?.port ?? _ports_.jupyter.external}:${_ports_.jupyter.internal}`,
],
};
setup = {
...setup,
jupyter: { port: config.sink?.port ?? _ports_.jupyter.external },
};
}
else if (config.sink?.name === 'spark') {
YAML.services.spark = {
...SPARK,
ports: [
`${config.sink?.port ?? _ports_.spark.webui.external}:${_ports_.spark.webui.internal}`,
`${config.sink?.rpc_port ?? _ports_.spark.rpc.external}:${_ports_.spark.rpc.internal}`,
],
};
setup = {
...setup,
spark: {
port: [
config.sink?.port ?? _ports_.spark.webui.external,
config.sink?.rpc_port ?? _ports_.spark.rpc.external,
],
},
};
}
}
}
function createZooKeepers(kafka) {
console.log('creating zookeepers...');
const numOfZKs = kafka.zookeepers.size > 1 ? kafka.zookeepers.size : 1;
const name = (x) => `zookeeper${x}`;
const getZKServerPorts = () => {
let zkClients = '';
let zkPeers = '';
const getPeerPort = (x) => {
if (kafka.zookeepers.ports !== undefined &&
kafka.zookeepers.ports.peer !== undefined)
return `${kafka.zookeepers.ports.peer.external}:${kafka.zookeepers.ports.peer.internal};`;
return `${x + 1}${_ports_.zookeeper.peer.external}:${x + 1}${_ports_.zookeeper.peer.internal};`;
};
const getClientPort = (x) => {
if (kafka.zookeepers.ports !== undefined &&
kafka.zookeepers.ports?.client !== undefined)
return kafka.zookeepers.ports?.client[x] + ',';
return `${x + 1}${_ports_.zookeeper.client.external},`;
};
for (let i = 0; i < numOfZKs; i++) {
zkClients += `${name(i + 1)}:${getClientPort(i)}`;
zkPeers += `${name(i + 1)}:${getPeerPort(i)}`;
}
zkClients = zkClients.slice(0, -1);
zkPeers = zkPeers.slice(0, -1);
return { zkClients, zkPeers };
};
const servers = getZKServerPorts();
setup = { ...setup, zookeeper: { ports: [] } };
for (let i = 0; i < numOfZKs; i++) {
const n = i + 1;
const name = `zookeeper${n}`;
let cport = 10000 * n + _ports_.zookeeper.client.external;
if (kafka.zookeepers.ports?.client !== undefined &&
kafka.zookeepers.ports.client[i] !== undefined) {
cport = kafka.zookeepers.ports.client[i];
}
YAML.services[name] = {
...ZOOKEEPER,
environment: {
...ZOOKEEPER.environment,
ZOOKEEPER_SERVER_ID: n,
ZOOKEEPER_CLIENT_PORT: cport,
ZOOKEEPER_SERVERS: servers.zkPeers,
},
ports: [`${cport}:${_ports_.zookeeper.client.internal}`],
container_name: name,
};
if (setup.zookeeper !== undefined)
setup = {
...setup,
zookeeper: { ports: [...setup.zookeeper.ports, cport] },
};
if (YAML.services.ksql_schema !== undefined) {
YAML.services.ksql_schema.environment.SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL += `${name}:${_ports_.zookeeper.client.internal},`;
}
dependencies.push(name);
}
return servers;
}
function createBrokers(config, servers) {
const { kafka } = config;
console.log('creating brokers...');
let jmxExporterConfig;
if (kafka.jmx !== undefined) {
fs.ensureDirSync(path.resolve(downloadDir, 'jmx'));
jmxExporterConfig = yaml.load(fs.readFileSync(path.resolve(downloadDir, 'jmx/exporter/template.yml'), 'utf8'));
setup = { ...setup, jmx: { ports: [] } };
}
const springBSServers = [];
const springDeps = [];
for (let i = 0; i < kafka.brokers.size; i++) {
const n = i + 1;
const brokerName = `kafka${n}`;
let extPort = _ports_.kafka.broker.external + i;
if (kafka.brokers.ports !== undefined &&
kafka.brokers.ports.brokers !== undefined &&
kafka.brokers.ports.brokers[i] !== undefined)
extPort = kafka.brokers.ports.brokers[i];
let metricsPort = _ports_.kafka.metrics;
if (kafka.brokers.ports !== undefined &&
kafka.brokers.ports.metrics !== undefined)
metricsPort = kafka.brokers.ports.metrics;
springBSServers.push(`${brokerName}:${_ports_.kafka.spring}`);
let jmxHostPort = _ports_.kafka.jmx + i;
if (kafka.brokers.ports !== undefined &&
kafka.brokers.ports.jmx !== undefined &&
kafka.brokers.ports.jmx[i] !== undefined)
jmxHostPort = kafka.brokers.ports.jmx[i];
let brokerID = 101 + i;
if (kafka.brokers.id !== undefined && kafka.brokers.id[i] !== undefined)
brokerID = kafka.brokers.id[i];
YAML.services[brokerName] = {
...KAFKA_BROKER,
ports: [`${extPort}:${_ports_.kafka.broker.internal}`],
container_name: brokerName,
depends_on: dependencies,
environment: {
...KAFKA_BROKER.environment,
KAFKA_BROKER_ID: brokerID,
KAFKA_JMX_PORT: jmxHostPort,
KAFKA_LISTENERS: `METRICS://:${metricsPort},PLAINTEXT://:${extPort},INTERNAL://:${_ports_.kafka.spring},KSQL://${brokerName}:${_ports_.kafka.ksql},CONNECT_SRC://${brokerName}:${_ports_.kafka.connect_src},CONNECT_SINK://${brokerName}:${_ports_.kafka.connect_sink}`,
KAFKA_ADVERTISED_LISTENERS: `METRICS://${brokerName}:${metricsPort},PLAINTEXT://${network}:${extPort},INTERNAL://${brokerName}:${_ports_.kafka.spring},KSQL://${brokerName}:${_ports_.kafka.ksql},CONNECT_SRC://${brokerName}:${_ports_.kafka.connect_src},CONNECT_SINK://${brokerName}:${_ports_.kafka.connect_sink}`,
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: (kafka.brokers.replicas ?? 1) > kafka.brokers.size
? kafka.brokers.size
: kafka.brokers.replicas ?? 1,
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: (kafka.brokers.replicas ?? 1) > kafka.brokers.size
? kafka.brokers.size
: kafka.brokers.replicas ?? 1,
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: `${brokerName}:${metricsPort}`,
KAFKA_ZOOKEEPER_CONNECT: servers.zkClients,
CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: servers.zkClients,
},
};
setup = {
...setup,
kafkaSetup: {
...setup.kafkaSetup,
brokers: [...setup.kafkaSetup.brokers, `${network}:${extPort}`],
},
};
springDeps.push(brokerName);
if (kafka.jmx !== undefined) {
let jmxPort = _ports_.jmx.external + n;
const jmxName = `jmx-kafka${n}`;
if (kafka.jmx !== undefined && kafka.jmx?.ports !== undefined) {
jmxPort = kafka.jmx?.ports[i] ?? jmxPort;
}
YAML.services[jmxName] = {
...JMX,
command: [`${_ports_.jmx.internal}`, '/etc/myconfig.yml'],
ports: [`${jmxPort}:${_ports_.jmx.internal}`],
environment: {
...JMX.environment,
SERVICE_PORT: _ports_.jmx.internal,
},
container_name: jmxName,
volumes: [
`${path.join(downloadDir, `/jmx/jmxConfigKafka${n}.yml`)}:/etc/myconfig.yml`,
],
depends_on: [`kafka${n}`],
};
let jmxArr = [jmxPort];
if (setup.jmx?.ports !== undefined)
jmxArr = [...setup.jmx?.ports, jmxPort];
setup = {
...setup,
jmx: { ports: jmxArr },
};
jmxExporterConfig.hostPort = `kafka${n}:${jmxHostPort}`;
fs.writeFileSync(path.resolve(downloadDir, `jmx/jmxConfigKafka${n}.yml`), yaml.dump(jmxExporterConfig, { noRefs: true }));
if (config.prometheus !== undefined) {
PROMCONFIG.scrape_configs[0].static_configs[0].targets.push(`${jmxName}:${_ports_.jmx.internal}`);
}
}
if (YAML.services.ksql !== undefined) {
if (YAML.services.ksql_schema !== undefined) {
YAML.services.ksql_schema.environment = {
...YAML.services.ksql_schema.environment,
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: YAML.services.ksql_schema.environment
.SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS +
`PLAINTEXT://${brokerName}:9092,`,
};
}
YAML.services.ksql = {
...YAML.services.ksql,
environment: {
...YAML.services.ksql.environment,
KSQL_BOOTSTRAP_SERVERS: YAML.services.ksql.environment.KSQL_BOOTSTRAP_SERVERS +
`${brokerName}:${_ports_.kafka.ksql},`,
},
};
if (YAML.services.ksql_cli !== undefined) {
const deps = YAML.services.ksql_cli.depends_on ?? [
YAML.services.ksql.container_name,
];
deps.push(`${brokerName}`);
YAML.services.ksql_cli = {
...YAML.services.ksql_cli,
depends_on: deps,
};
}
}
if (YAML.services.kafka_connect_src !== undefined) {
const kcenv = YAML.services.kafka_connect_src.environment;
const kcdeps = YAML.services.kafka_connect_src.depends_on === undefined ||
YAML.services.kafka_connect_src.depends_on?.length === 0
? [brokerName]
: [...YAML.services.kafka_connect_src.depends_on, brokerName];
YAML.services.kafka_connect_src = {
...YAML.services.kafka_connect_src,
environment: {
...kcenv,
CONNECT_BOOTSTRAP_SERVERS: kcenv.CONNECT_BOOTSTRAP_SERVERS !== ''
? `${kcenv.CONNECT_BOOTSTRAP_SERVERS},${brokerName}:${_ports_.kafka.connect_src}`
: `${brokerName}:${_ports_.kafka.connect_src}`,
},
depends_on: kcdeps,
};
}
if (YAML.services.kafka_connect_sink !== undefined) {
const kcenv = YAML.services.kafka_connect_sink.environment;
const kcdeps = YAML.services.kafka_connect_sink.depends_on === undefined ||
YAML.services.kafka_connect_sink.depends_on?.length === 0
? [brokerName]
: [...YAML.services.kafka_connect_sink.depends_on, brokerName];
YAML.services.kafka_connect_sink = {
...YAML.services.kafka_connect_sink,
environment: {
...kcenv,
CONNECT_BOOTSTRAP_SERVERS: kcenv.CONNECT_BOOTSTRAP_SERVERS !== ''
? `${kcenv.CONNECT_BOOTSTRAP_SERVERS},${brokerName}:${_ports_.kafka.connect_sink}`
: `${brokerName}:${_ports_.kafka.connect_sink}`,
},
depends_on: kcdeps,
};
}
}
if (kafka.spring !== undefined) {
const springPort = kafka.spring?.port ?? _ports_.spring.external;
YAML.services.spring = {
...SPRING,
ports: [`${springPort}:${_ports_.spring.external}`],
environment: {
...SPRING.environment,
'SPRING_KAFKA_BOOTSTRAP-SERVERS': springBSServers.join(','),
'SPRING_KAFKA_CONSUMER_BOOTSTRAP-SERVERS': springBSServers.join(','),
'SPRING_KAFKA_PRODUCER_BOOTSTRAP-SERVERS': springBSServers.join(','),
},
depends_on: springDeps,
};
setup = {
...setup,
spring: { port: springPort },
};
}
if (YAML.services.ksql_schema !== undefined) {
YAML.services.ksql_schema.depends_on = [...dependencies, ...springDeps];
}
}
};
export default ymlGenerator;