xkite-core
Version:
Core library for xkite application
609 lines (608 loc) • 22 kB
JavaScript
import path from 'path';
import fs from 'fs-extra';
import compose from 'docker-compose';
import ymlGenerator from './yml.js';
const fetch = require('node-fetch');
const zipper = require('zip-a-folder');
import * as consts from './constants/kite.js';
const { defaultCfg } = consts;
import * as yml from './constants/yml.js';
const { _ports_, downloadDir } = yml;
import getAvailablePorts from './getPorts.js';
const configPath = path.join(downloadDir, 'docker-compose.yml');
const zipPath = path.join(downloadDir, '../pipeline.zip');
import store from './state/store.js';
import * as slice from './state/slice.js';
const { setPackageBuild, setConfig, setServer, setSetup, setState, setServerState, setServiceState, setConfigFile, } = slice;
function KiteCreator() {
let selectedPorts = new Set([
_ports_.kafka.metrics,
_ports_.kafka.spring,
_ports_.kafka.ksql,
_ports_.kafka.connect_src,
_ports_.kafka.connect_sink,
]);
async function configServer(server) {
store.dispatch(setServer(server));
store.dispatch(setState('Init'));
store.dispatch(setServerState('Disconnected'));
try {
let res = await fetch(`${server}/api/kite/getConfig`);
store.dispatch(setConfig(await res.json()));
res = await fetch(`${server}/api/kite/getSetup`);
store.dispatch(setConfig(await res.json()));
res = await fetch(`${server}/api/kite/getConfigFile`);
store.dispatch(setConfig(await res.json()));
store.dispatch(setServerState('Connected'));
}
catch (err) {
console.error(`error fetching from ${server}/api/:\n${err}`);
}
}
async function checkPorts(args) {
try {
const retPorts = [];
for (const port of args) {
let notFound = true;
let candidatePort = port;
while (notFound) {
while (selectedPorts.has(candidatePort)) {
candidatePort++;
}
const avport = await getAvailablePorts(port, 1);
if (!selectedPorts.has(avport[0])) {
selectedPorts.add(avport[0]);
retPorts.push(avport[0]);
notFound = false;
}
}
}
return retPorts;
}
catch (error) {
console.error('Error occurred while checking available ports!', error);
}
}
async function checkPort(port) {
try {
const avPort = await getAvailablePorts(port, 1);
return avPort[0];
}
catch (error) {
console.error('Error occurred while checking available ports!', error);
}
}
async function checkConfigPorts(config) {
try {
selectedPorts = new Set([
_ports_.kafka.metrics,
_ports_.kafka.spring,
_ports_.kafka.ksql,
_ports_.kafka.connect_src,
_ports_.kafka.connect_sink,
]);
let cfg = Object.assign({}, config);
let kafka = {
...cfg.kafka,
brokers: {
...cfg.kafka.brokers,
ports: {
...cfg.kafka.brokers.ports,
brokers: await checkPorts(cfg.kafka.brokers?.ports?.brokers ??
new Array(cfg.kafka.brokers.size).fill(_ports_.kafka.broker.external)),
},
},
zookeepers: {
...cfg.kafka.zookeepers,
ports: {
...cfg.kafka.zookeepers.ports,
client: await checkPorts(cfg.kafka.zookeepers?.ports?.client ??
new Array(cfg.kafka.zookeepers.size).fill(_ports_.zookeeper.client.external)),
},
},
};
if (cfg.kafka.jmx !== undefined) {
const jPorts = [];
for (let i = 0; i < cfg.kafka.brokers.size; i++) {
if (cfg.kafka.jmx?.ports !== undefined &&
i < cfg.kafka.jmx?.ports.length)
jPorts.push(cfg.kafka.jmx.ports[i]);
else
jPorts.push(_ports_.jmx.external + i);
}
kafka = {
...kafka,
jmx: {
...cfg.kafka.jmx,
ports: await checkPorts(jPorts),
},
};
}
if (cfg.kafka.spring !== undefined) {
kafka = {
...kafka,
spring: {
...kafka.spring,
port: await checkPort(cfg.kafka.spring.port ?? _ports_.spring.external),
},
};
}
cfg = { ...cfg, kafka: { ...cfg.kafka, ...kafka } };
if (cfg.db !== undefined) {
let db = Object.assign({}, cfg.db);
if (db.kafkaconnect !== undefined) {
db = {
...db,
kafkaconnect: {
...db.kafkaconnect,
port: await checkPort(db.kafkaconnect?.port ?? _ports_.kafkaconnect_src.external),
},
};
}
else {
db = {
...db,
kafkaconnect: {
port: await checkPort(_ports_.kafkaconnect_src.external),
},
};
}
db = {
...db,
port: await checkPort(db.port ??
(db.name === 'ksql'
? _ports_.ksql.external
: _ports_.postgresql.external)),
};
if (db.ksql !== undefined) {
db = {
...db,
ksql: {
...db.ksql,
schema_port: await checkPort(db.ksql.schema_port ?? _ports_.ksql_schema.external),
},
};
}
cfg = { ...cfg, db: { ...cfg.db, ...db } };
}
if (cfg.sink !== undefined) {
let sink = Object.assign({}, cfg.sink);
if (sink.kafkaconnect !== undefined) {
sink = {
...sink,
kafkaconnect: {
...sink.kafkaconnect,
port: await checkPort(sink.kafkaconnect.port ?? _ports_.kafkaconnect_sink.external),
},
};
}
else {
sink = {
...sink,
kafkaconnect: {
port: await checkPort(_ports_.kafkaconnect_sink.external),
},
};
}
if (sink.name === 'jupyter') {
sink = {
...sink,
port: await checkPort(sink?.port ?? _ports_.jupyter.external),
};
}
else {
sink = {
...sink,
port: await checkPort(sink?.port ?? _ports_.spark.webui.external),
rpc_port: await checkPort(sink?.rpc_port ?? _ports_.spark.rpc.external),
};
}
cfg = { ...cfg, sink: { ...cfg.sink, ...sink } };
}
if (cfg.grafana !== undefined) {
const grafana = {
...cfg.grafana,
port: await checkPort(cfg.grafana.port ?? _ports_.grafana.external),
};
cfg = { ...cfg, grafana: { ...cfg.grafana, ...grafana } };
}
if (cfg.prometheus !== undefined) {
const prometheus = {
...cfg.prometheus,
port: await checkPort(cfg.prometheus.port ?? _ports_.prometheus.external),
};
cfg = { ...cfg, prometheus: { ...cfg.prometheus, ...prometheus } };
}
return cfg;
}
catch (error) {
console.error('Error occurred while checking config ports!', error);
return config;
}
}
async function configLocal(config) {
const cfg = await checkConfigPorts(config);
store.dispatch(setState('Init'));
store.dispatch(setServerState('Disconnected'));
try {
const generate = ymlGenerator();
store.dispatch(setConfig(cfg));
store.dispatch(setSetup(generate(cfg)));
store.dispatch(setPackageBuild(zipPath));
const header = {
'Content-Type': 'text/yml',
'Content-Length': fs.statSync(configPath).size,
};
const fileStream = fs.readFileSync(configPath, 'utf-8');
store.dispatch(setConfigFile({ header, fileStream }));
store.dispatch(setState('Configured'));
console.log('yaml configuration complete...');
}
catch (err) {
console.error(`KITE failed to initialize: ${err}\nConfiguration ${cfg}`);
}
}
async function deployServer() {
try {
const { server } = store.getState();
await fetch(`${server}/api/kite/deploy`);
store.dispatch(setState('Running'));
}
catch (err) {
console.error(`Kite deployment failed:\n${JSON.stringify(err)}`);
}
}
async function deployLocal() {
try {
console.log('deploying docker containers...');
await compose.upAll({
cwd: downloadDir,
log: true,
});
store.dispatch(setState('Running'));
console.log('docker deployment successful');
}
catch (err) {
console.error(`Kite deployment failed:\n${JSON.stringify(err)}`);
}
}
async function pauseServer(service) {
try {
const { server } = store.getState();
await fetch(`${server}/api/kite/pause`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
},
body: JSON.stringify({ service }),
});
}
catch (err) {
console.error(`Could not pause docker instances on server:\n${err}`);
}
}
async function pauseLocal(service) {
if (service === undefined)
return;
for (const name of service) {
try {
await compose.pauseOne(name, {
cwd: downloadDir,
log: true,
});
}
catch (err) {
console.error(`Could not pause docker instances on local:\n${err}`);
}
}
}
async function unpauseServer(service) {
try {
const { server } = store.getState();
await fetch(`${server}/api/kite/unpause`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
},
body: JSON.stringify({ service }),
});
}
catch (err) {
console.error(`Could not unpause docker instances on server:\n${err}`);
}
}
async function unpauseLocal(service) {
if (service === undefined)
return;
for (const name of service) {
try {
await compose.unpauseOne(name, {
cwd: downloadDir,
log: true,
});
}
catch (err) {
console.error(`Could not unpause docker instances on local:\n${err}`);
}
}
}
async function shutdownServer() {
try {
const { server } = store.getState();
await fetch(`${server}/api/kite/shutdown`, {
method: 'DELETE',
headers: {
Accept: 'application/json',
},
});
}
catch (err) {
console.error(`Could not shutdown docker instances on server:\n${err}`);
}
}
async function shutdownLocal() {
try {
await compose.down({
cwd: downloadDir,
log: true,
commandOptions: ['--remove-orphans', '--volumes'],
});
}
catch (err) {
console.error(`Could not shutdown docker instances on local:\n${err}`);
}
}
async function disconnectServer() {
try {
const { server } = store.getState();
await fetch(`${server}/api/kite/disconnect`, {
method: 'POST',
headers: {
Accept: 'application/json',
},
});
}
catch (err) {
console.error(`Could not disconnect docker instances on server:\n${err}`);
}
}
async function disconnectLocal() {
try {
await compose.down({
cwd: downloadDir,
log: true,
commandOptions: ['--remove-orphans', '--volumes'],
});
}
catch (err) {
console.error(`Could not disconnect docker instances on local:\n${err}`);
}
}
async function getPackageBuildServer() {
try {
const { server } = store.getState();
const res = await fetch(`${server}/api/kite/getPackageBuild`, {
method: 'GET',
headers: {
Accept: 'application/zip',
},
});
return { fileStream: await res.arrayBuffer() };
}
catch (err) {
console.error(`Could not retrieve package.zip from server:\n${err}`);
throw err;
}
}
async function getPackageBuildLocal() {
fs.removeSync(zipPath);
await zipper.zip(downloadDir, zipPath);
return new Promise((res, rej) => {
try {
const header = {
'Content-Type': 'application/zip',
'Content-Length': fs.statSync(zipPath).size,
};
const fileStream = fs.readFileSync(zipPath);
res({ header, fileStream });
}
catch (err) {
rej(err);
}
});
}
async function getKiteStateServer() {
const { server } = store.getState();
const res = await fetch(`${server}/api/kite/getKiteState`, {
method: 'GET',
headers: {
Accept: 'application/text',
},
});
return await res.text();
}
async function getConfigFileServer() {
const { server } = store.getState();
const res = await fetch(`${server}/api/kite/getConfigFile`, {
method: 'GET',
headers: {
Accept: 'application/json',
},
});
return await res.json();
}
async function getConfigServer() {
const { server } = store.getState();
const res = await fetch(`${server}/api/kite/getConfig`, {
method: 'GET',
headers: {
Accept: 'application/json',
},
});
return (await res.json());
}
async function getDBSetupServer() {
const { dBSetup } = await getSetupServer();
return dBSetup;
}
async function getKafkaSetupServer() {
const { kafkaSetup } = await getSetupServer();
return kafkaSetup;
}
async function getSetupServer() {
const { server } = store.getState();
const res = await fetch(`${server}/api/kite/getSetup`, {
method: 'GET',
headers: {
Accept: 'application/json',
},
});
return (await res.json());
}
return {
defaultCfg: defaultCfg,
configure: async function (arg) {
console.log('Configuring docker instances...');
if (arg === undefined) {
await configLocal(defaultCfg);
}
else {
switch (typeof arg) {
case 'string':
await configServer(arg);
break;
default:
await configLocal(arg);
break;
}
}
},
deploy: async function (arg) {
const { serverState } = store.getState();
console.log('Deploying docker instances...');
if (serverState === 'Connected') {
await deployServer();
}
else {
await deployLocal();
}
},
getSetup: function () {
const { serverState } = store.getState();
if (serverState === 'Connected')
return getSetupServer();
else
return store.getState().setup;
},
getKafkaSetup: function () {
const { serverState } = store.getState();
if (serverState === 'Connected')
return getKafkaSetupServer();
else
return store.getState().kafkaSetup;
},
getDBSetup: function () {
const { serverState } = store.getState();
if (serverState === 'Connected')
return getDBSetupServer();
else
return store.getState().dBSetup;
},
getConfig: function () {
const { serverState } = store.getState();
if (serverState === 'Connected')
return getConfigServer();
else
return store.getState().config;
},
getConfigFile: function () {
const { serverState } = store.getState();
if (serverState === 'Connected')
return getConfigFileServer();
else
return store.getState().configFile;
},
getKiteState: function () {
const { serverState } = store.getState();
if (serverState === 'Connected')
return getKiteStateServer();
else
return store.getState().state;
},
getKiteServerState: function () {
return store.getState().serverState;
},
getPackageBuild: function () {
const { serverState } = store.getState();
console.log('Getting package build zip...');
if (serverState === 'Connected') {
return getPackageBuildServer();
}
else {
return getPackageBuildLocal();
}
},
disconnect: async function () {
const { serverState } = store.getState();
console.log('Disconnecting from docker instances...');
if (serverState === 'Connected') {
store.dispatch(setServerState('Disconnected'));
disconnectServer();
}
else {
disconnectLocal();
}
store.dispatch(setState('Shutdown'));
},
shutdown: async function () {
const { serverState } = store.getState();
console.log('Shutting down docker instances and removing volumes...');
if (serverState === 'Connected') {
store.dispatch(setServerState('Disconnected'));
await shutdownServer();
}
else {
await shutdownLocal();
}
store.dispatch(setState('Shutdown'));
},
pause: async function (service) {
const { serverState, services } = store.getState();
if (service === undefined || service.length === 0 || service[0] === '') {
console.log('Pausing all docker instances...');
service = services;
}
else {
console.log('Pausing docker instance(s): ' + JSON.stringify(service));
}
if (serverState === 'Connected') {
await pauseServer(service);
}
else {
await pauseLocal(service);
}
store.dispatch(setServiceState({ type: 'pause', service }));
},
unpause: async function (service) {
const { serverState, services } = store.getState();
if (service === undefined || service.length === 0 || service[0] === '') {
service = services;
console.log('Unpausing all docker instances...');
}
else {
console.log('Unpausing docker instance(s): ' + JSON.stringify(service));
}
if (serverState === 'Connected') {
await unpauseServer(service);
}
else {
await unpauseLocal(service);
}
store.dispatch(setServiceState({ type: 'unpause', service }));
},
};
}
export const Kite = KiteCreator();
export const default_ports = _ports_;