zcatalyst-cli
Version:
Command Line Tool for CATALYST
400 lines (399 loc) • 21.7 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.checkIfRuntimeServerRunning = void 0;
const better_queue_1 = __importDefault(require("better-queue"));
const express_1 = __importDefault(require("express"));
const path_1 = require("path");
const url_1 = require("url");
const error_1 = __importDefault(require("../../error"));
const authenticator_1 = __importDefault(require("../../express_middlewares/authenticator"));
const cookie_parser_1 = __importDefault(require("../../express_middlewares/cookie-parser"));
const logger_1 = __importDefault(require("../../express_middlewares/logger"));
const project_1 = __importDefault(require("../../express_middlewares/project"));
const fn_utils_1 = require("../../fn-utils");
const runtime_store_1 = __importDefault(require("../../runtime-store"));
const constants_1 = require("../../util_modules/constants");
const runtime_1 = __importDefault(require("../../util_modules/constants/lib/runtime"));
const fs_1 = require("../../util_modules/fs");
const js_1 = require("../../util_modules/js");
const userConfig_1 = __importDefault(require("../../userConfig"));
const logger_2 = require("../../util_modules/logger");
const project_2 = require("../../util_modules/project");
const shell_1 = require("../../util_modules/shell");
const request_1 = __importDefault(require("request"));
const ensure_java_userconfig_1 = require("../../fn-utils/lib/ensure-java-userconfig");
const option_1 = require("../../util_modules/option");
const server_1 = require("../../util_modules/server");
const logUrl = (name, pthName, httpPort, masterPort) => {
(0, logger_2.labeled)(`functions(${name})`, 'URL => http://localhost:' + (masterPort === -1 ? httpPort : masterPort) + pthName).MESSAGE();
};
const checkIfRuntimeServerRunning = (port, itr = 0) => __awaiter(void 0, void 0, void 0, function* () {
if (itr > 50) {
throw new error_1.default('Unable to spin up python runtime server. Max retry reached', {
exit: 2
});
}
return new Promise((res) => {
request_1.default
.get(`http://127.0.0.1:${port}/ruok`, {
headers: {
'x-zoho-catalyst-internal': 'true'
}
})
.on('response', (_response) => __awaiter(void 0, void 0, void 0, function* () {
(0, logger_2.debug)(`ruok response: ${_response.statusCode}: ${_response.statusMessage}`);
if (_response.statusCode === 200) {
res(true);
}
else if (_response.statusCode === 503) {
yield js_1.JS.sleep(100);
yield (0, exports.checkIfRuntimeServerRunning)(port, ++itr).then((x) => res(x));
}
else {
yield js_1.JS.sleep(500);
throw new error_1.default('Unable to spin up python runtime server. StatusCode: ' +
_response.statusCode, {
exit: 2
});
}
}))
.on('error', (error) => __awaiter(void 0, void 0, void 0, function* () {
const incomingError = error_1.default.getErrorInstance(error).original;
if (incomingError.code === 'ECONNREFUSED' || incomingError.code === 'ECONNRESET') {
yield js_1.JS.sleep(100);
yield (0, exports.checkIfRuntimeServerRunning)(port, ++itr).then((x) => res(x));
}
else {
throw new error_1.default('Unable to spin up python runtime server. ErrorCode: ' + incomingError.code, {
exit: 2
});
}
}));
});
});
exports.checkIfRuntimeServerRunning = checkIfRuntimeServerRunning;
class HttpFunctions {
constructor({ repl, primary = true } = {}) {
const projectRoot = runtime_store_1.default.get('project.root');
this.repl = repl;
this.app = (0, express_1.default)();
this.app.locals.primary = primary;
this.nodeInvoker = {
[constants_1.FN_TYPE.basic]: (0, path_1.normalize)((0, path_1.join)(__dirname, './invoker', constants_1.FN_TYPE.basic, 'node.mjs'))
};
this.javaInvoker = {
[constants_1.FN_TYPE.basic]: (0, path_1.join)(projectRoot, constants_1.FOLDERNAME.build, '.catalyst')
};
this.requestFile = (0, path_1.join)(projectRoot, constants_1.FOLDERNAME.build, '.catalyst', 'user_req_body');
this.responseFile = (0, path_1.join)(projectRoot, constants_1.FOLDERNAME.build, '.catalyst', 'user_res_body');
this.metaFile = (0, path_1.join)(projectRoot, constants_1.FOLDERNAME.build, '.catalyst', 'user_meta.json');
this.runningSlaves = [];
this.targets = [];
this.q = new better_queue_1.default((opts, cb) => {
var _a, _b;
fs_1.SYNC.ensureFile(this.responseFile, true);
fs_1.SYNC.ensureFile(this.requestFile, true);
fs_1.SYNC.ensureFile(this.metaFile, true);
const fileStream = fs_1.SYNC.getWriteStream(this.requestFile);
opts.req.pipe(fileStream);
fs_1.SYNC.writeFile(this.metaFile, JSON.stringify({
request: {
'content-type': opts.req.headers['content-type'],
'content-length': opts.req.headers['content-length']
}
}));
const slave = (0, shell_1.spawn)(opts.command, opts.options, {
cwd: opts.pth,
stdio: 'pipe',
env: Object.assign({ X_ZOHO_CATALYST_IS_LOCAL: 'true', X_ZOHO_CATALYST_FUNCTION_LOADED: 'true', X_ZOHO_CATALYST_ACCOUNTS_URL: constants_1.ORIGIN.auth, X_ZOHO_CATALYST_CONSOLE_URL: constants_1.ORIGIN.admin, X_ZOHO_STRATUS_RESOURCE_SUFFIX: constants_1.ORIGIN.stratusSuffix, CATALYST_PORTAL_DOMAIN: constants_1.ORIGIN.iamPortal, CATALYST_PROJECT_TIMEZONE: (0, project_2.getProjectTimezone)(Intl.DateTimeFormat().resolvedOptions().timeZone) }, opts.envVars)
}).RAW();
(_a = slave.stdout) === null || _a === void 0 ? void 0 : _a.on('data', (data) => {
(0, logger_2.info)(Buffer.isBuffer(data) ? data.toString() : data);
});
(_b = slave.stderr) === null || _b === void 0 ? void 0 : _b.on('data', (data) => {
const errorStr = data.toString().trim();
(0, logger_2.info)(errorStr);
});
slave.on('uncaughtException', (err, origin) => {
const errorString = `Caught exception: ${err}\n` + `Exception origin: ${origin}`;
(0, logger_2.error)(errorString);
cb(errorString, null);
});
slave.on('exit', (code) => {
if (!fs_1.SYNC.fileExists(this.responseFile)) {
(0, logger_2.debug)(`${this.responseFile} not present`);
cb(new Error('File does not exists: ' + this.responseFile));
return;
}
const response = fs_1.SYNC.getReadStream(this.responseFile);
const metaFileString = fs_1.SYNC.readFile(this.metaFile);
let meta = {};
try {
meta =
JSON.parse(metaFileString === undefined ? '{}' : metaFileString)
.response || {};
}
catch (err) {
meta = {};
}
switch (code) {
case 0:
cb(null, { response, meta });
break;
default:
cb(new Error(JSON.stringify(code === null ? 0 : code)), null);
break;
}
});
this.runningSlaves.push(slave);
}, { batchSize: 1, concurrent: 1, maxTimeout: Infinity }).on('task_failed', (targetName, err) => {
(0, logger_2.debug)('Unable to serve the function: ' + targetName);
(0, logger_2.debug)(err);
});
}
_spinUpServer() {
return __awaiter(this, void 0, void 0, function* () {
const debugPort = parseInt(runtime_store_1.default.get('context.port.debug.' + constants_1.FN_TYPE.basic, '-1'), 10);
const httpPort = parseInt(runtime_store_1.default.get('context.port.http.' + constants_1.FN_TYPE.basic), 10);
if (this.repl !== undefined) {
this.app.use((_req, _res, next) => {
(0, shell_1.clearLine)(process.stdout);
next();
});
}
if (this.app.locals.primary) {
this.app.use((0, logger_1.default)());
this.app.use(cookie_parser_1.default);
this.app.use((0, project_1.default)({
id: (0, project_2.getProjectId)(),
domain_prefix: (0, project_2.getDomainPrefix)(),
domain: constants_1.ORIGIN.app.replace('https://', ''),
key: (0, project_2.getDomainKey)(),
env_name: (0, project_2.getEnvName)()
}));
this.app.use(authenticator_1.default);
}
const reqHandler = (fnTarget) => (request, response, next) => {
var _a, _b;
const urlParts = new url_1.URL(request.originalUrl, request.protocol + '://' + request.get('host')).searchParams;
const query = JSON.stringify(Object.fromEntries(urlParts.entries()));
const slaveOptions = [];
let slave = null;
const projectRoot = runtime_store_1.default.get('project.root');
if (fnTarget.type === undefined) {
throw new error_1.default('Function target type is not defined', {
exit: 2
});
}
const slaveFnTarget = { index: fnTarget.index, name: fnTarget.name };
if ((_a = fnTarget.stack) === null || _a === void 0 ? void 0 : _a.startsWith(runtime_1.default.language.node.value)) {
if (debugPort !== -1) {
slaveOptions.push('--inspect-brk=' + debugPort);
}
slaveOptions.push(this.nodeInvoker[fnTarget.type]);
slaveOptions.push(JSON.stringify(slaveFnTarget));
slaveOptions.push(query);
slaveOptions.push(JSON.stringify({
'x-zc-projectid': request.headers['x-zc-projectid'],
'x-zc-project-domain': request.headers['x-zc-project-domain'],
'x-zc-project-key': request.headers['x-zc-project-key'],
'x-zc-environment': request.headers['x-zc-environment']
}));
slaveOptions.push(JSON.stringify({
'x-zc-user-cred-type': request.headers['x-zc-user-cred-type'],
'x-zc-user-cred-token': request.headers['x-zc-user-cred-token'],
'x-zc-admin-cred-type': request.headers['x-zc-admin-cred-type'],
'x-zc-admin-cred-token': request.headers['x-zc-admin-cred-token'],
'x-zc-cookie': request.headers['x-zc-cookie'],
'x-zc-user-type': request.headers['x-zc-user-type']
}));
slaveOptions.push(JSON.stringify((0, path_1.join)(projectRoot, constants_1.FOLDERNAME.build)));
slave = this.q.push({
command: 'node',
options: slaveOptions,
pth: (0, path_1.join)(projectRoot, constants_1.FOLDERNAME.build, constants_1.FOLDERNAME.functions, fnTarget.name),
req: request,
id: fnTarget.name,
envVars: fnTarget.env_var
});
}
else if ((_b = fnTarget.stack) === null || _b === void 0 ? void 0 : _b.startsWith(runtime_1.default.language.java.value)) {
const invoker = fnTarget.type === constants_1.FN_TYPE.basic
? this.javaInvoker[fnTarget.type] + `/${fnTarget.stack}/JavabioInvoker`
: this.javaInvoker[fnTarget.type];
const javaInvokerDir = (0, path_1.parse)(invoker).dir;
slaveOptions.push('-cp');
slaveOptions.push(javaInvokerDir +
fn_utils_1.fnUtils.java.classPathSep +
(0, path_1.join)(javaInvokerDir, 'lib', '*'));
if (debugPort !== -1) {
slaveOptions.push('-Xdebug');
slaveOptions.push('-Xrunjdwp:transport=dt_socket,address=' +
debugPort +
',server=y,suspend=y');
}
slaveOptions.push((0, path_1.basename)(invoker));
slaveOptions.push(javaInvokerDir);
slaveOptions.push(JSON.stringify(slaveFnTarget));
slaveOptions.push(query);
slaveOptions.push(JSON.stringify({
'x-zc-projectid': request.headers['x-zc-projectid'],
'x-zc-project-domain': request.headers['x-zc-project-domain'],
'x-zc-project-key': request.headers['x-zc-project-key'],
'x-zc-environment': request.headers['x-zc-environment']
}));
slaveOptions.push(JSON.stringify({
'x-zc-user-cred-type': request.headers['x-zc-user-cred-type'],
'x-zc-user-cred-token': request.headers['x-zc-user-cred-token'],
'x-zc-admin-cred-type': request.headers['x-zc-admin-cred-type'],
'x-zc-admin-cred-token': request.headers['x-zc-admin-cred-token'],
'x-zc-cookie': request.headers['x-zc-cookie'],
'x-zc-user-type': request.headers['x-zc-user-type']
}));
const configKey = `${fnTarget.stack}.bin`;
const userConfigCmd = userConfig_1.default.get(configKey);
const spawnCommand = (0, ensure_java_userconfig_1.getJavaSpawnCommand)(userConfigCmd, 'java', fnTarget.stack);
slave = this.q.push({
command: spawnCommand,
options: slaveOptions,
target: fnTarget,
pth: (0, path_1.join)(runtime_store_1.default.get('project.root'), constants_1.FOLDERNAME.build, constants_1.FOLDERNAME.functions, fnTarget.name),
req: request,
id: fnTarget.name,
envVars: fnTarget.env_var
});
}
if (slave === null) {
throw new error_1.default('Slave is null event after pushing opts', {
exit: 2
});
}
slave
.on('finish', (result) => {
if (result.meta['Content-Length']) {
response.set('Content-Length', result.meta['Content-Length']);
}
if (result.meta['Content-Type']) {
response.set('Content-Type', result.meta['Content-Type']);
}
response.writeHead(result.meta.statusCode || 200);
result.response.pipe(response);
if (this.repl !== undefined) {
this.repl.showPrompt();
}
next();
})
.on('failed', (err) => {
if (err.message !== '0') {
response.status(500).send('Error : ' + err);
if (this.repl !== undefined) {
this.repl.showPrompt();
}
next();
}
});
};
(0, shell_1.clearLine)(process.stdout);
const masterPort = runtime_store_1.default.get(`context.port.http.master`, -1);
const fnTargets = runtime_store_1.default.get('context.functions.targets', []);
fnTargets
.filter((t) => {
var _a;
return t.url !== undefined &&
t.type === constants_1.FN_TYPE.basic &&
!((_a = t.stack) === null || _a === void 0 ? void 0 : _a.startsWith(runtime_1.default.language.python.value));
})
.map((t) => {
this.targets.push(t);
return js_1.JS.omit(t, ['zip_stream', 'watcher', 'localFn']);
})
.forEach((t) => {
var _a;
if ((_a = t.stack) === null || _a === void 0 ? void 0 : _a.startsWith(runtime_1.default.language.java.value)) {
const invoker = t.type === constants_1.FN_TYPE.basic
? this.javaInvoker[t.type] +
`/${t.stack}/JavabioInvoker`
: this.javaInvoker[t.type];
fn_utils_1.fnUtils.java.ensureJavaInvoker(invoker, (0, path_1.normalize)((0, path_1.join)(__dirname, './invoker', t.type, 'java', 'Java' + t.type + 'Invoker.java')), t);
}
const pthName = new url_1.URL(t.url).pathname;
if (pthName !== null) {
this.app.use(pthName, reqHandler(t));
}
if (t.url_with_id !== undefined) {
const pathWithId = new url_1.URL(t.url_with_id).pathname;
if (pathWithId !== null) {
this.app.use(pathWithId, reqHandler(t));
}
}
});
return new Promise((res) => {
const server = this.app.listen(httpPort, '127.0.0.1', () => {
this.targets.forEach((target) => {
var _a, _b;
(_a = target.watcher) === null || _a === void 0 ? void 0 : _a.on('preparing', () => {
if (this.repl && !this.repl.paused) {
this.repl.pause();
}
});
(_b = target.watcher) === null || _b === void 0 ? void 0 : _b.on('compiled', () => {
(0, logger_2.labeled)(`functions[${target.name}]`, 'ready!').MESSAGE();
logUrl(target.name, new url_1.URL(target.url).pathname, httpPort, masterPort);
if (this.repl && this.repl.paused) {
this.repl.resume();
this.repl.showPrompt();
}
setTimeout(() => {
var _a;
(_a = target.watcher) === null || _a === void 0 ? void 0 : _a.emit('next');
}, 1000);
});
if ((0, option_1.getCurrentCommand)() === 'functions:shell') {
logUrl(target.name, new url_1.URL(target.url).pathname, httpPort, masterPort);
}
});
if (this.repl !== undefined) {
this.repl.showPrompt();
}
res(server);
});
});
});
}
start() {
return __awaiter(this, void 0, void 0, function* () {
const server = yield this._spinUpServer();
const connDestroyer = new server_1.ConnectionDestroyer(server);
this.connDestroyer = connDestroyer;
return server;
});
}
stop() {
return __awaiter(this, void 0, void 0, function* () {
return new Promise((res) => {
this.runningSlaves.forEach((slave) => {
slave.kill('SIGINT');
});
this.q.destroy(() => __awaiter(this, void 0, void 0, function* () {
var _a;
yield Promise.all(this.targets.map((target) => __awaiter(this, void 0, void 0, function* () { var _b; return (_b = target.watcher) === null || _b === void 0 ? void 0 : _b.close(); })));
yield ((_a = this.connDestroyer) === null || _a === void 0 ? void 0 : _a.destroy());
res();
}));
});
});
}
}
exports.default = HttpFunctions;