xkite-core
Version:
Core library for xkite application
635 lines (634 loc) • 29 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.default_ports = exports.Kite = void 0;
const path_1 = __importDefault(require("path"));
const fs_extra_1 = __importDefault(require("fs-extra"));
const docker_compose_1 = __importDefault(require("docker-compose"));
const yml_js_1 = __importDefault(require("./yml.js"));
const fetch = require('node-fetch');
const zipper = require('zip-a-folder');
const consts = __importStar(require("./constants/kite.js"));
const { defaultCfg } = consts;
const yml = __importStar(require("./constants/yml.js"));
const { _ports_, downloadDir } = yml;
const getPorts_js_1 = __importDefault(require("./getPorts.js"));
const configPath = path_1.default.join(downloadDir, 'docker-compose.yml');
const zipPath = path_1.default.join(downloadDir, '../pipeline.zip');
const store_js_1 = __importDefault(require("./state/store.js"));
const slice = __importStar(require("./state/slice.js"));
const { setPackageBuild, setConfig, setServer, setSetup, setState, setServerState, setServiceState, setConfigFile, } = slice;
function KiteCreator() {
let selectedPorts = new Set([
_ports_.kafka.metrics,
_ports_.kafka.spring,
_ports_.kafka.ksql,
_ports_.kafka.connect_src,
_ports_.kafka.connect_sink,
]);
function configServer(server) {
return __awaiter(this, void 0, void 0, function* () {
store_js_1.default.dispatch(setServer(server));
store_js_1.default.dispatch(setState('Init'));
store_js_1.default.dispatch(setServerState('Disconnected'));
try {
let res = yield fetch(`${server}/api/kite/getConfig`);
store_js_1.default.dispatch(setConfig(yield res.json()));
res = yield fetch(`${server}/api/kite/getSetup`);
store_js_1.default.dispatch(setConfig(yield res.json()));
res = yield fetch(`${server}/api/kite/getConfigFile`);
store_js_1.default.dispatch(setConfig(yield res.json()));
store_js_1.default.dispatch(setServerState('Connected'));
}
catch (err) {
console.error(`error fetching from ${server}/api/:\n${err}`);
}
});
}
function checkPorts(args) {
return __awaiter(this, void 0, void 0, function* () {
try {
const retPorts = [];
for (const port of args) {
let notFound = true;
let candidatePort = port;
while (notFound) {
while (selectedPorts.has(candidatePort)) {
candidatePort++;
}
const avport = yield (0, getPorts_js_1.default)(port, 1);
if (!selectedPorts.has(avport[0])) {
selectedPorts.add(avport[0]);
retPorts.push(avport[0]);
notFound = false;
}
}
}
return retPorts;
}
catch (error) {
console.error('Error occurred while checking available ports!', error);
}
});
}
function checkPort(port) {
return __awaiter(this, void 0, void 0, function* () {
try {
const avPort = yield (0, getPorts_js_1.default)(port, 1);
return avPort[0];
}
catch (error) {
console.error('Error occurred while checking available ports!', error);
}
});
}
function checkConfigPorts(config) {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u;
return __awaiter(this, void 0, void 0, function* () {
try {
selectedPorts = new Set([
_ports_.kafka.metrics,
_ports_.kafka.spring,
_ports_.kafka.ksql,
_ports_.kafka.connect_src,
_ports_.kafka.connect_sink,
]);
let cfg = Object.assign({}, config);
let kafka = Object.assign(Object.assign({}, cfg.kafka), { brokers: Object.assign(Object.assign({}, cfg.kafka.brokers), { ports: Object.assign(Object.assign({}, cfg.kafka.brokers.ports), { brokers: yield checkPorts((_c = (_b = (_a = cfg.kafka.brokers) === null || _a === void 0 ? void 0 : _a.ports) === null || _b === void 0 ? void 0 : _b.brokers) !== null && _c !== void 0 ? _c : new Array(cfg.kafka.brokers.size).fill(_ports_.kafka.broker.external)) }) }), zookeepers: Object.assign(Object.assign({}, cfg.kafka.zookeepers), { ports: Object.assign(Object.assign({}, cfg.kafka.zookeepers.ports), { client: yield checkPorts((_f = (_e = (_d = cfg.kafka.zookeepers) === null || _d === void 0 ? void 0 : _d.ports) === null || _e === void 0 ? void 0 : _e.client) !== null && _f !== void 0 ? _f : new Array(cfg.kafka.zookeepers.size).fill(_ports_.zookeeper.client.external)) }) }) });
if (cfg.kafka.jmx !== undefined) {
const jPorts = [];
for (let i = 0; i < cfg.kafka.brokers.size; i++) {
if (((_g = cfg.kafka.jmx) === null || _g === void 0 ? void 0 : _g.ports) !== undefined &&
i < ((_h = cfg.kafka.jmx) === null || _h === void 0 ? void 0 : _h.ports.length))
jPorts.push(cfg.kafka.jmx.ports[i]);
else
jPorts.push(_ports_.jmx.external + i);
}
kafka = Object.assign(Object.assign({}, kafka), { jmx: Object.assign(Object.assign({}, cfg.kafka.jmx), { ports: yield checkPorts(jPorts) }) });
}
if (cfg.kafka.spring !== undefined) {
kafka = Object.assign(Object.assign({}, kafka), { spring: Object.assign(Object.assign({}, kafka.spring), { port: yield checkPort((_j = cfg.kafka.spring.port) !== null && _j !== void 0 ? _j : _ports_.spring.external) }) });
}
cfg = Object.assign(Object.assign({}, cfg), { kafka: Object.assign(Object.assign({}, cfg.kafka), kafka) });
if (cfg.db !== undefined) {
let db = Object.assign({}, cfg.db);
if (db.kafkaconnect !== undefined) {
db = Object.assign(Object.assign({}, db), { kafkaconnect: Object.assign(Object.assign({}, db.kafkaconnect), { port: yield checkPort((_l = (_k = db.kafkaconnect) === null || _k === void 0 ? void 0 : _k.port) !== null && _l !== void 0 ? _l : _ports_.kafkaconnect_src.external) }) });
}
else {
db = Object.assign(Object.assign({}, db), { kafkaconnect: {
port: yield checkPort(_ports_.kafkaconnect_src.external),
} });
}
db = Object.assign(Object.assign({}, db), { port: yield checkPort((_m = db.port) !== null && _m !== void 0 ? _m : (db.name === 'ksql'
? _ports_.ksql.external
: _ports_.postgresql.external)) });
if (db.ksql !== undefined) {
db = Object.assign(Object.assign({}, db), { ksql: Object.assign(Object.assign({}, db.ksql), { schema_port: yield checkPort((_o = db.ksql.schema_port) !== null && _o !== void 0 ? _o : _ports_.ksql_schema.external) }) });
}
cfg = Object.assign(Object.assign({}, cfg), { db: Object.assign(Object.assign({}, cfg.db), db) });
}
if (cfg.sink !== undefined) {
let sink = Object.assign({}, cfg.sink);
if (sink.kafkaconnect !== undefined) {
sink = Object.assign(Object.assign({}, sink), { kafkaconnect: Object.assign(Object.assign({}, sink.kafkaconnect), { port: yield checkPort((_p = sink.kafkaconnect.port) !== null && _p !== void 0 ? _p : _ports_.kafkaconnect_sink.external) }) });
}
else {
sink = Object.assign(Object.assign({}, sink), { kafkaconnect: {
port: yield checkPort(_ports_.kafkaconnect_sink.external),
} });
}
if (sink.name === 'jupyter') {
sink = Object.assign(Object.assign({}, sink), { port: yield checkPort((_q = sink === null || sink === void 0 ? void 0 : sink.port) !== null && _q !== void 0 ? _q : _ports_.jupyter.external) });
}
else {
sink = Object.assign(Object.assign({}, sink), { port: yield checkPort((_r = sink === null || sink === void 0 ? void 0 : sink.port) !== null && _r !== void 0 ? _r : _ports_.spark.webui.external), rpc_port: yield checkPort((_s = sink === null || sink === void 0 ? void 0 : sink.rpc_port) !== null && _s !== void 0 ? _s : _ports_.spark.rpc.external) });
}
cfg = Object.assign(Object.assign({}, cfg), { sink: Object.assign(Object.assign({}, cfg.sink), sink) });
}
if (cfg.grafana !== undefined) {
const grafana = Object.assign(Object.assign({}, cfg.grafana), { port: yield checkPort((_t = cfg.grafana.port) !== null && _t !== void 0 ? _t : _ports_.grafana.external) });
cfg = Object.assign(Object.assign({}, cfg), { grafana: Object.assign(Object.assign({}, cfg.grafana), grafana) });
}
if (cfg.prometheus !== undefined) {
const prometheus = Object.assign(Object.assign({}, cfg.prometheus), { port: yield checkPort((_u = cfg.prometheus.port) !== null && _u !== void 0 ? _u : _ports_.prometheus.external) });
cfg = Object.assign(Object.assign({}, cfg), { prometheus: Object.assign(Object.assign({}, cfg.prometheus), prometheus) });
}
return cfg;
}
catch (error) {
console.error('Error occurred while checking config ports!', error);
return config;
}
});
}
function configLocal(config) {
return __awaiter(this, void 0, void 0, function* () {
const cfg = yield checkConfigPorts(config);
store_js_1.default.dispatch(setState('Init'));
store_js_1.default.dispatch(setServerState('Disconnected'));
try {
const generate = (0, yml_js_1.default)();
store_js_1.default.dispatch(setConfig(cfg));
store_js_1.default.dispatch(setSetup(generate(cfg)));
store_js_1.default.dispatch(setPackageBuild(zipPath));
const header = {
'Content-Type': 'text/yml',
'Content-Length': fs_extra_1.default.statSync(configPath).size,
};
const fileStream = fs_extra_1.default.readFileSync(configPath, 'utf-8');
store_js_1.default.dispatch(setConfigFile({ header, fileStream }));
store_js_1.default.dispatch(setState('Configured'));
console.log('yaml configuration complete...');
}
catch (err) {
console.error(`KITE failed to initialize: ${err}\nConfiguration ${cfg}`);
}
});
}
function deployServer() {
return __awaiter(this, void 0, void 0, function* () {
try {
const { server } = store_js_1.default.getState();
yield fetch(`${server}/api/kite/deploy`);
store_js_1.default.dispatch(setState('Running'));
}
catch (err) {
console.error(`Kite deployment failed:\n${JSON.stringify(err)}`);
}
});
}
function deployLocal() {
return __awaiter(this, void 0, void 0, function* () {
try {
console.log('deploying docker containers...');
yield docker_compose_1.default.upAll({
cwd: downloadDir,
log: true,
});
store_js_1.default.dispatch(setState('Running'));
console.log('docker deployment successful');
}
catch (err) {
console.error(`Kite deployment failed:\n${JSON.stringify(err)}`);
}
});
}
function pauseServer(service) {
return __awaiter(this, void 0, void 0, function* () {
try {
const { server } = store_js_1.default.getState();
yield fetch(`${server}/api/kite/pause`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
},
body: JSON.stringify({ service }),
});
}
catch (err) {
console.error(`Could not pause docker instances on server:\n${err}`);
}
});
}
function pauseLocal(service) {
return __awaiter(this, void 0, void 0, function* () {
if (service === undefined)
return;
for (const name of service) {
try {
yield docker_compose_1.default.pauseOne(name, {
cwd: downloadDir,
log: true,
});
}
catch (err) {
console.error(`Could not pause docker instances on local:\n${err}`);
}
}
});
}
function unpauseServer(service) {
return __awaiter(this, void 0, void 0, function* () {
try {
const { server } = store_js_1.default.getState();
yield fetch(`${server}/api/kite/unpause`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
},
body: JSON.stringify({ service }),
});
}
catch (err) {
console.error(`Could not unpause docker instances on server:\n${err}`);
}
});
}
function unpauseLocal(service) {
return __awaiter(this, void 0, void 0, function* () {
if (service === undefined)
return;
for (const name of service) {
try {
yield docker_compose_1.default.unpauseOne(name, {
cwd: downloadDir,
log: true,
});
}
catch (err) {
console.error(`Could not unpause docker instances on local:\n${err}`);
}
}
});
}
function shutdownServer() {
return __awaiter(this, void 0, void 0, function* () {
try {
const { server } = store_js_1.default.getState();
yield fetch(`${server}/api/kite/shutdown`, {
method: 'DELETE',
headers: {
Accept: 'application/json',
},
});
}
catch (err) {
console.error(`Could not shutdown docker instances on server:\n${err}`);
}
});
}
function shutdownLocal() {
return __awaiter(this, void 0, void 0, function* () {
try {
yield docker_compose_1.default.down({
cwd: downloadDir,
log: true,
commandOptions: ['--remove-orphans', '--volumes'],
});
}
catch (err) {
console.error(`Could not shutdown docker instances on local:\n${err}`);
}
});
}
function disconnectServer() {
return __awaiter(this, void 0, void 0, function* () {
try {
const { server } = store_js_1.default.getState();
yield fetch(`${server}/api/kite/disconnect`, {
method: 'POST',
headers: {
Accept: 'application/json',
},
});
}
catch (err) {
console.error(`Could not disconnect docker instances on server:\n${err}`);
}
});
}
function disconnectLocal() {
return __awaiter(this, void 0, void 0, function* () {
try {
yield docker_compose_1.default.down({
cwd: downloadDir,
log: true,
commandOptions: ['--remove-orphans', '--volumes'],
});
}
catch (err) {
console.error(`Could not disconnect docker instances on local:\n${err}`);
}
});
}
function getPackageBuildServer() {
return __awaiter(this, void 0, void 0, function* () {
try {
const { server } = store_js_1.default.getState();
const res = yield fetch(`${server}/api/kite/getPackageBuild`, {
method: 'GET',
headers: {
Accept: 'application/zip',
},
});
return { fileStream: yield res.arrayBuffer() };
}
catch (err) {
console.error(`Could not retrieve package.zip from server:\n${err}`);
throw err;
}
});
}
function getPackageBuildLocal() {
return __awaiter(this, void 0, void 0, function* () {
fs_extra_1.default.removeSync(zipPath);
yield zipper.zip(downloadDir, zipPath);
return new Promise((res, rej) => {
try {
const header = {
'Content-Type': 'application/zip',
'Content-Length': fs_extra_1.default.statSync(zipPath).size,
};
const fileStream = fs_extra_1.default.readFileSync(zipPath);
res({ header, fileStream });
}
catch (err) {
rej(err);
}
});
});
}
function getKiteStateServer() {
return __awaiter(this, void 0, void 0, function* () {
const { server } = store_js_1.default.getState();
const res = yield fetch(`${server}/api/kite/getKiteState`, {
method: 'GET',
headers: {
Accept: 'application/text',
},
});
return yield res.text();
});
}
function getConfigFileServer() {
return __awaiter(this, void 0, void 0, function* () {
const { server } = store_js_1.default.getState();
const res = yield fetch(`${server}/api/kite/getConfigFile`, {
method: 'GET',
headers: {
Accept: 'application/json',
},
});
return yield res.json();
});
}
function getConfigServer() {
return __awaiter(this, void 0, void 0, function* () {
const { server } = store_js_1.default.getState();
const res = yield fetch(`${server}/api/kite/getConfig`, {
method: 'GET',
headers: {
Accept: 'application/json',
},
});
return (yield res.json());
});
}
function getDBSetupServer() {
return __awaiter(this, void 0, void 0, function* () {
const { dBSetup } = yield getSetupServer();
return dBSetup;
});
}
function getKafkaSetupServer() {
return __awaiter(this, void 0, void 0, function* () {
const { kafkaSetup } = yield getSetupServer();
return kafkaSetup;
});
}
function getSetupServer() {
return __awaiter(this, void 0, void 0, function* () {
const { server } = store_js_1.default.getState();
const res = yield fetch(`${server}/api/kite/getSetup`, {
method: 'GET',
headers: {
Accept: 'application/json',
},
});
return (yield res.json());
});
}
return {
defaultCfg: defaultCfg,
configure: function (arg) {
return __awaiter(this, void 0, void 0, function* () {
console.log('Configuring docker instances...');
if (arg === undefined) {
yield configLocal(defaultCfg);
}
else {
switch (typeof arg) {
case 'string':
yield configServer(arg);
break;
default:
yield configLocal(arg);
break;
}
}
});
},
deploy: function (arg) {
return __awaiter(this, void 0, void 0, function* () {
const { serverState } = store_js_1.default.getState();
console.log('Deploying docker instances...');
if (serverState === 'Connected') {
yield deployServer();
}
else {
yield deployLocal();
}
});
},
getSetup: function () {
const { serverState } = store_js_1.default.getState();
if (serverState === 'Connected')
return getSetupServer();
else
return store_js_1.default.getState().setup;
},
getKafkaSetup: function () {
const { serverState } = store_js_1.default.getState();
if (serverState === 'Connected')
return getKafkaSetupServer();
else
return store_js_1.default.getState().kafkaSetup;
},
getDBSetup: function () {
const { serverState } = store_js_1.default.getState();
if (serverState === 'Connected')
return getDBSetupServer();
else
return store_js_1.default.getState().dBSetup;
},
getConfig: function () {
const { serverState } = store_js_1.default.getState();
if (serverState === 'Connected')
return getConfigServer();
else
return store_js_1.default.getState().config;
},
getConfigFile: function () {
const { serverState } = store_js_1.default.getState();
if (serverState === 'Connected')
return getConfigFileServer();
else
return store_js_1.default.getState().configFile;
},
getKiteState: function () {
const { serverState } = store_js_1.default.getState();
if (serverState === 'Connected')
return getKiteStateServer();
else
return store_js_1.default.getState().state;
},
getKiteServerState: function () {
return store_js_1.default.getState().serverState;
},
getPackageBuild: function () {
const { serverState } = store_js_1.default.getState();
console.log('Getting package build zip...');
if (serverState === 'Connected') {
return getPackageBuildServer();
}
else {
return getPackageBuildLocal();
}
},
disconnect: function () {
return __awaiter(this, void 0, void 0, function* () {
const { serverState } = store_js_1.default.getState();
console.log('Disconnecting from docker instances...');
if (serverState === 'Connected') {
store_js_1.default.dispatch(setServerState('Disconnected'));
disconnectServer();
}
else {
disconnectLocal();
}
store_js_1.default.dispatch(setState('Shutdown'));
});
},
shutdown: function () {
return __awaiter(this, void 0, void 0, function* () {
const { serverState } = store_js_1.default.getState();
console.log('Shutting down docker instances and removing volumes...');
if (serverState === 'Connected') {
store_js_1.default.dispatch(setServerState('Disconnected'));
yield shutdownServer();
}
else {
yield shutdownLocal();
}
store_js_1.default.dispatch(setState('Shutdown'));
});
},
pause: function (service) {
return __awaiter(this, void 0, void 0, function* () {
const { serverState, services } = store_js_1.default.getState();
if (service === undefined || service.length === 0 || service[0] === '') {
console.log('Pausing all docker instances...');
service = services;
}
else {
console.log('Pausing docker instance(s): ' + JSON.stringify(service));
}
if (serverState === 'Connected') {
yield pauseServer(service);
}
else {
yield pauseLocal(service);
}
store_js_1.default.dispatch(setServiceState({ type: 'pause', service }));
});
},
unpause: function (service) {
return __awaiter(this, void 0, void 0, function* () {
const { serverState, services } = store_js_1.default.getState();
if (service === undefined || service.length === 0 || service[0] === '') {
service = services;
console.log('Unpausing all docker instances...');
}
else {
console.log('Unpausing docker instance(s): ' + JSON.stringify(service));
}
if (serverState === 'Connected') {
yield unpauseServer(service);
}
else {
yield unpauseLocal(service);
}
store_js_1.default.dispatch(setServiceState({ type: 'unpause', service }));
});
},
};
}
exports.Kite = KiteCreator();
exports.default_ports = _ports_;