exthos
Version:
stream processing in nodejs using the power of golang
832 lines • 54.5 kB
JavaScript
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
};
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
if (kind === "m") throw new TypeError("Private method is not writable");
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
};
var _Engine_instances, _a, _Engine_engineConfigFilePath, _Engine_engineConfig, _Engine_engineExtraConfig, _Engine_engineProcess, _Engine_abortController, _Engine_mgmtEventsFreqMs, _Engine_keepAliveInterval, _Engine_debug, _Engine_debugLog, _Engine_traceLog, _Engine_eventLog, _Engine_eventNameToEventLog, _Engine_isActive, _Engine_engineConstrStartStopMutex, _Engine_engineStreamAddUpdateRemoveMutex, _Engine_engineUpdateConfigOptionsMutex, _Engine_constructorDone, _Engine_scheme, _Engine_tempLocalServer, _Engine_streamsMap, _Engine_benthosEXEFullPath, _Engine_setEngineConfig, _Engine_setEngineExtraConfig, _Engine_createAxiosInstance, _Engine_startMgmtEvents, _Engine_writeToEngineConfigFilePath, _Engine_defaultEngineEventHandler;
import * as constants from "../constants.js";
import * as path from "path";
import { tmpdir, EOL } from "os";
import * as fs from "fs";
import { randomUUID } from "crypto";
import { Deferred, sleep, getISOStringLocalTz, getCaller, formatErrorForEvent, standardizeAxiosErrors, } from "../utils/utils.js";
import { execaCommand, execaCommandSync } from "execa";
import * as net from "net";
import axios from "axios";
import axiosRetry from "axios-retry";
import debug from "debug";
import * as stream from "stream";
import { promises as streamPromises } from "stream";
import { EngineProcessAPI } from "./engineProcessAPI.js";
import { once } from "events";
import { Mutex } from "async-mutex";
import { clearInterval } from "timers";
import EventEmitter2 from "eventemitter2";
import merge from "lodash.merge";
import config from "../config/config.js";
var engineEventsEnums;
(function (engineEventsEnums) {
engineEventsEnums["engine.**"] = "engine.**";
engineEventsEnums["engine.*.*"] = "engine.*.*";
engineEventsEnums["engine.active"] = "engine.active";
engineEventsEnums["engine.inactive"] = "engine.inactive";
engineEventsEnums["engine.warn"] = "engine.warn";
engineEventsEnums["engine.error"] = "engine.error";
engineEventsEnums["engine.fatal"] = "engine.fatal";
engineEventsEnums["engine.stream.add"] = "engine.stream.add";
engineEventsEnums["engine.stream.update"] = "engine.stream.update";
engineEventsEnums["engine.stream.remove"] = "engine.stream.remove";
engineEventsEnums["engine.stream.error"] = "engine.stream.error";
engineEventsEnums["engineProcess.stream.fatal"] = "engineProcess.stream.fatal";
engineEventsEnums["engineProcess.stream.error"] = "engineProcess.stream.error";
engineEventsEnums["engineProcess.stream.warn"] = "engineProcess.stream.warn";
engineEventsEnums["engineProcess.stream.info"] = "engineProcess.stream.info";
engineEventsEnums["engineProcess.stream.debug"] = "engineProcess.stream.debug";
engineEventsEnums["engineProcess.stream.trace"] = "engineProcess.stream.trace";
})(engineEventsEnums || (engineEventsEnums = {}));
class Engine extends EngineProcessAPI {
constructor(engineConfig = {}, engineExtraConfig = {}) {
super();
_Engine_instances.add(this);
_Engine_engineConfigFilePath.set(this, path.join(tmpdir(), "exthos_engine_conf_" + randomUUID() + ".json"));
_Engine_engineConfig.set(this, void 0);
_Engine_engineExtraConfig.set(this, void 0);
_Engine_engineProcess.set(this, void 0);
_Engine_abortController.set(this, new AbortController());
_Engine_mgmtEventsFreqMs.set(this, 2000);
this.waitForActiveEventMs = 5000;
_Engine_keepAliveInterval.set(this, void 0);
_Engine_debug.set(this, debug("exthos"));
_Engine_debugLog.set(this, __classPrivateFieldGet(this, _Engine_debug, "f").extend("engine").extend("debugLog"));
_Engine_traceLog.set(this, __classPrivateFieldGet(this, _Engine_debug, "f").extend("engine").extend("traceLog"));
_Engine_eventLog.set(this, __classPrivateFieldGet(this, _Engine_debug, "f").extend("eventLog"));
_Engine_eventNameToEventLog.set(this, {});
_Engine_isActive.set(this, false);
_Engine_engineConstrStartStopMutex.set(this, new Mutex());
_Engine_engineStreamAddUpdateRemoveMutex.set(this, new Mutex());
_Engine_engineUpdateConfigOptionsMutex.set(this, new Mutex());
_Engine_constructorDone.set(this, new Deferred());
_Engine_scheme.set(this, "http");
_Engine_tempLocalServer.set(this, void 0);
_Engine_streamsMap.set(this, {});
_Engine_benthosEXEFullPath.set(this, "/tmp");
this.engineEvents = engineEventsEnums;
this.emit = (event, eventObj, ...values) => {
let self = this;
try {
if (!__classPrivateFieldGet(self, _Engine_eventNameToEventLog, "f")[event]) {
__classPrivateFieldGet(self, _Engine_eventNameToEventLog, "f")[event] = __classPrivateFieldGet(self, _Engine_eventLog, "f").extend(event);
}
__classPrivateFieldGet(self, _Engine_eventNameToEventLog, "f")[event](JSON.stringify(eventObj));
return super.emit(event, eventObj, ...values);
}
catch (e) {
self.emit(self.engineEvents["engine.error"], {
msg: "unable to emit events",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
return false;
}
};
let self = this;
let caller = getCaller();
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `engine constructor called from: ${caller}`);
__classPrivateFieldGet(self, _Engine_engineConstrStartStopMutex, "f").runExclusive(() => {
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `engine constructor mutex acquired from: ${caller}`);
try {
__classPrivateFieldGet(self, _Engine_instances, "m", _Engine_setEngineConfig).call(self, engineConfig).then((_) => {
__classPrivateFieldGet(self, _Engine_instances, "m", _Engine_setEngineExtraConfig).call(self, engineExtraConfig).then((_) => {
__classPrivateFieldGet(self, _Engine_instances, "m", _Engine_createAxiosInstance).call(self).then((_) => {
__classPrivateFieldGet(self, _Engine_constructorDone, "f").resolve();
});
});
});
}
catch (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "engine constructor failed",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
}
self.on(self.engineEvents["engine.active"], () => {
__classPrivateFieldSet(self, _Engine_isActive, true, "f");
});
self.on(self.engineEvents["engine.inactive"], () => {
__classPrivateFieldSet(self, _Engine_isActive, false, "f");
});
self.on(self.engineEvents["engine.fatal"], (eventObj) => {
self.stop(eventObj.msg);
});
});
}
get numStreams() {
try {
return Object.keys(__classPrivateFieldGet(this, _Engine_streamsMap, "f")).length;
}
catch (error) {
return 0;
}
}
async updateEngineConfigs(receivedEngineConfig = {}, receivedEngineExtraConfig = {}) {
let self = this;
let caller = getCaller();
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `updateEngineConfigs called from: ${caller}`);
try {
await __classPrivateFieldGet(self, _Engine_constructorDone, "f").promise;
return await __classPrivateFieldGet(self, _Engine_engineUpdateConfigOptionsMutex, "f").runExclusive(async () => {
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `updateEngineConfigs mutex acquired from: ${caller}`);
await __classPrivateFieldGet(self, _Engine_instances, "m", _Engine_setEngineConfig).call(self, receivedEngineConfig);
await __classPrivateFieldGet(self, _Engine_instances, "m", _Engine_setEngineExtraConfig).call(self, receivedEngineExtraConfig);
});
}
catch (e) {
self.emit(self.engineEvents["engine.error"], {
msg: "unable to update engine config and options",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
}
}
async start() {
let self = this;
let caller = getCaller();
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `start called from: ${caller}`);
try {
return await __classPrivateFieldGet(self, _Engine_engineConstrStartStopMutex, "f").runExclusive(async () => {
var _b, _c;
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `start mutex acquired from: ${caller}`);
if (__classPrivateFieldGet(this, _Engine_isActive, "f")) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "engine isActive=true, ignoring the call to start()");
return self;
}
if (__classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").isLocal) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "isLocal=true");
let benthosTag = "v" + __classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").benthosVersion;
let benthosVersion = __classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").benthosVersion;
let benthosOS = __classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").benthosOS;
let benthosArch = __classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").benthosArch;
let benthosFileName = config.engineExtraConfig.benthosFileName ||
`benthos_${benthosVersion}_${benthosOS}_${benthosArch}`;
let benthosDir = config.engineExtraConfig.benthosDir;
__classPrivateFieldSet(self, _Engine_benthosEXEFullPath, path.join(benthosDir, benthosFileName), "f");
let benthosArchiveFullPath = path.join(benthosDir, benthosFileName + ".tar.gz");
if (!config.engineExtraConfig.benthosFileName) {
try {
await fs.promises.stat(__classPrivateFieldGet(self, _Engine_benthosEXEFullPath, "f"));
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `${__classPrivateFieldGet(self, _Engine_benthosEXEFullPath, "f")} exists and will be using it.`);
}
catch (e) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `${__classPrivateFieldGet(self, _Engine_benthosEXEFullPath, "f")} doesn't exist`);
try {
let benthosURL = `https://github.com/benthosdev/benthos/releases/download/${benthosTag}/${benthosFileName + ".tar.gz"}`;
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `downloading archive from: ${benthosURL}`);
let resp;
try {
resp = await axios.get(benthosURL, {
responseType: "stream",
});
}
catch (e) {
throw standardizeAxiosErrors(e);
}
await streamPromises.pipeline(resp.data, fs.createWriteStream(benthosArchiveFullPath));
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `extracting archive ${benthosArchiveFullPath}`);
execaCommandSync(`tar xzvf ${benthosArchiveFullPath} -C ${benthosDir} benthos`);
execaCommandSync(`mv ${path.join(benthosDir, "benthos")} ${path.join(benthosDir, benthosFileName)}`);
await fs.promises.chmod(__classPrivateFieldGet(self, _Engine_benthosEXEFullPath, "f"), "0777");
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `benthos installation completed`);
}
catch (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "benthos cannot be installed",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
return self;
}
}
}
let versionRequiredMinArray = constants.minBenthosVSupported.split(".");
let versionRequiredMin = {
major: parseInt(versionRequiredMinArray[0], 10),
minor: parseInt(versionRequiredMinArray[1], 10),
patch: parseInt(versionRequiredMinArray[2], 10),
};
try {
let versionPresentArray = execaCommandSync(`${__classPrivateFieldGet(self, _Engine_benthosEXEFullPath, "f")} -v`)
.stdout.split(EOL)[0]
.split(": ")[1]
.split(".");
let versionPresent = {
major: parseInt(versionPresentArray[0], 10),
minor: parseInt(versionPresentArray[1], 10),
patch: parseInt(versionPresentArray[2], 10),
};
if (!(versionPresent.major === versionRequiredMin.major &&
(versionPresent.minor > versionRequiredMin.minor ||
(versionPresent.minor === versionRequiredMin.minor &&
versionPresent.patch >= versionRequiredMin.patch)))) {
throw new Error(`benthos version ${versionPresentArray.join(".")} is not supported by exthos. exthos supports MINOR versions greater than equal to ${versionRequiredMinArray.join(".")}`);
}
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `using benthos version: ${versionPresentArray.join(".")}`);
}
catch (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: `benthos cannot be used`,
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
return self;
}
try {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "creating tempLocalServer to establish ADDR, PORT availability");
__classPrivateFieldSet(self, _Engine_tempLocalServer, net.createServer(), "f");
let host = __classPrivateFieldGet(self, _Engine_engineConfig, "f").http.address.split(":")[0];
let port = __classPrivateFieldGet(self, _Engine_engineConfig, "f").http.address.split(":")[1];
let isListeningDeferred = new Deferred();
__classPrivateFieldGet(self, _Engine_tempLocalServer, "f").on("listening", () => {
isListeningDeferred.resolve("deferred_listening");
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "tempLocalServer is listening");
});
let isErrorDeferred = new Deferred();
__classPrivateFieldGet(self, _Engine_tempLocalServer, "f").on("error", (e) => {
if (!e) {
isErrorDeferred.resolve();
}
else {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "tempLocalServer errored out", e);
isErrorDeferred.reject(e);
}
});
let raceProm = Promise.race([
isListeningDeferred.promise,
isErrorDeferred.promise,
new Promise((_, rj) => {
setTimeout(() => {
rj("tempLocalServer timedout");
}, 2000);
}),
]);
__classPrivateFieldGet(self, _Engine_tempLocalServer, "f").listen(parseInt(port, 10), host, () => { });
await raceProm;
let isCloseErrorDeferred = new Deferred();
__classPrivateFieldGet(self, _Engine_tempLocalServer, "f").close((e) => {
if (!e) {
__classPrivateFieldGet(self, _Engine_tempLocalServer, "f").unref();
isCloseErrorDeferred.resolve();
}
else {
self.emit(self.engineEvents["engine.fatal"], {
msg: "unable to close tempLocalServer",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
isCloseErrorDeferred.reject(e);
}
});
await isCloseErrorDeferred.promise;
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "tempLocalServer closed, i.e. listening=", __classPrivateFieldGet(self, _Engine_tempLocalServer, "f").listening);
}
catch (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "unable to start tempLocalServer",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
return self;
}
__classPrivateFieldGet(Engine, _a, "m", _Engine_writeToEngineConfigFilePath).call(self);
__classPrivateFieldSet(self, _Engine_engineProcess, execaCommand(`${__classPrivateFieldGet(self, _Engine_benthosEXEFullPath, "f")} -w -c ${__classPrivateFieldGet(self, _Engine_engineConfigFilePath, "f")} streams`, {
signal: __classPrivateFieldGet(self, _Engine_abortController, "f").signal,
buffer: false,
detached: true,
}), "f");
process.stdin.pipe(__classPrivateFieldGet(self, _Engine_engineProcess, "f").stdin);
process.on("SIGINT", () => {
self.stop("SIGINT was received");
});
let loggerWritable = new stream.Writable({
write: function (chunk, _, next) {
setTimeout(() => {
try {
chunk
.toString()
.trim()
.split("\n")
.forEach((str) => {
if (str !== "null") {
let j = {
level: "",
msg: "",
time: getISOStringLocalTz(),
};
try {
j = JSON.parse(str);
j.stream = __classPrivateFieldGet(self, _Engine_streamsMap, "f")[j.stream]
? __classPrivateFieldGet(self, _Engine_streamsMap, "f")[j.stream]
: j.stream;
str = JSON.stringify(j);
}
catch (error) {
}
switch (j.level) {
case "off":
case "none":
break;
case "fatal":
j.error = { message: j.msg };
self.emit(self.engineEvents["engineProcess.stream.fatal"], j);
break;
case "error":
j.error = { message: j.msg };
self.emit(self.engineEvents["engineProcess.stream.error"], j);
break;
case "warn":
case "warning":
j.error = { message: j.msg };
self.emit(self.engineEvents["engineProcess.stream.warn"], j);
break;
case "info":
self.emit(self.engineEvents["engineProcess.stream.info"], j);
break;
case "debug":
self.emit(self.engineEvents["engineProcess.stream.debug"], j);
break;
case "trace":
self.emit(self.engineEvents["engineProcess.stream.trace"], j);
break;
default:
console.log(str);
break;
}
}
});
}
catch (e) {
self.emit(self.engineEvents["engine.error"], {
msg: "unable to write engineProcess events into loggerWritable",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
}
}, 0);
next();
},
});
(_b = __classPrivateFieldGet(self, _Engine_engineProcess, "f").stdout) === null || _b === void 0 ? void 0 : _b.pipe(loggerWritable);
(_c = __classPrivateFieldGet(self, _Engine_engineProcess, "f").stderr) === null || _c === void 0 ? void 0 : _c.pipe(loggerWritable);
__classPrivateFieldGet(self, _Engine_engineProcess, "f").catch((e) => {
if (e.killed && e.isCanceled) {
self.emit(self.engineEvents["engine.inactive"], {
msg: "aborted successfully",
time: getISOStringLocalTz(),
});
}
else if (e.all) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "engineProcess exited unexpectedly (1)",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
}
else {
self.emit(self.engineEvents["engine.fatal"], {
msg: "engineProcess exited unexpectedly (2)",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
}
});
await once(__classPrivateFieldGet(self, _Engine_engineProcess, "f"), "spawn");
try {
await self._apiGetPing({ "axios-retry": { retries: 3 } });
self.emit(self.engineEvents["engine.active"], {
msg: "engineProcess=isLocal. first ping pass & marked active",
time: getISOStringLocalTz(),
});
}
catch (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "engineProcess=isLocal. first ping failed",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
return self;
}
}
else {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "isLocal=false");
try {
await self._apiGetPing({ "axios-retry": { retries: 3 } });
self.emit(self.engineEvents["engine.active"], {
msg: "engineProcess<>isLocal. first ping pass & marked active",
time: getISOStringLocalTz(),
});
}
catch (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "engineProcess<>isLocal. first ping failed",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
return self;
}
}
if (self.numStreams > 0) {
await self.add(...Object.values(__classPrivateFieldGet(self, _Engine_streamsMap, "f")));
}
__classPrivateFieldGet(self, _Engine_instances, "m", _Engine_startMgmtEvents).call(self);
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "waiting for event=engine.active before finish of start");
await EventEmitter2.once(self, self.engineEvents["engine.active"]);
__classPrivateFieldSet(self, _Engine_keepAliveInterval, setInterval(() => { }, 1 << 30), "f");
return self;
});
}
catch (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "start failed",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
return self;
}
}
async stop(reason, force) {
if (reason === undefined) {
reason = "";
}
if (force === undefined) {
force = false;
}
let self = this;
let caller = getCaller();
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `stop called from: ${caller}`);
try {
return await __classPrivateFieldGet(self, _Engine_engineConstrStartStopMutex, "f").runExclusive(async () => {
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `stop mutex acquired from: ${caller}`);
if (!__classPrivateFieldGet(self, _Engine_isActive, "f")) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `waiting for event=engine.active for ${self.waitForActiveEventMs} seconds before stopping`);
try {
await self.waitFor(self.engineEvents["engine.active"], self.waitForActiveEventMs);
}
catch (e) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "engine isActive=false, skipping stopping");
clearInterval(__classPrivateFieldGet(self, _Engine_keepAliveInterval, "f"));
return self;
}
}
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "removing all streams before stopping");
await self.remove();
if (__classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").isLocal) {
await fs.promises.unlink(__classPrivateFieldGet(self, _Engine_engineConfigFilePath, "f"));
if (force) {
__classPrivateFieldGet(self, _Engine_abortController, "f").abort();
}
else {
__classPrivateFieldGet(self, _Engine_engineProcess, "f").kill("SIGTERM", {
forceKillAfterTimeout: parseInt(__classPrivateFieldGet(self, _Engine_engineConfig, "f").shutdown_timeout, 10) + 1,
});
}
}
clearInterval(__classPrivateFieldGet(self, _Engine_keepAliveInterval, "f"));
self.emit(self.engineEvents["engine.inactive"], {
msg: `stopped successfully` + (reason ? ". reason:" + reason : ""),
time: getISOStringLocalTz(),
});
return self;
});
}
catch (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "stop failed. performing process.exit",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
process.exit(1);
}
}
async add(...streams) {
let self = this;
let caller = getCaller();
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `add called from: ${caller}`);
try {
return await __classPrivateFieldGet(self, _Engine_engineStreamAddUpdateRemoveMutex, "f").runExclusive(async () => {
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `add mutex acquired from: ${caller}`);
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `add called for streams: ${streams.map((s) => s.streamID)}`);
if (!__classPrivateFieldGet(self, _Engine_isActive, "f")) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `waiting for event=engine.active for ${self.waitForActiveEventMs} seconds before adding`);
try {
await self.waitFor(self.engineEvents["engine.active"], self.waitForActiveEventMs);
}
catch (e) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "engine isActive=false, skipping adding/_apiPostStream");
return self;
}
}
for (let stream of streams) {
try {
if (stream.hasInport) {
stream.createInport();
}
if (stream.hasOutport) {
stream.createOutport();
}
await stream.beforeAdd();
await self._apiPostStream(stream);
__classPrivateFieldGet(self, _Engine_streamsMap, "f")[stream.streamID] = stream;
this.emit(self.engineEvents["engine.stream.add"], {
msg: `stream added to engine`,
stream,
time: getISOStringLocalTz(),
});
}
catch (e) {
self.emit(self.engineEvents["engine.stream.error"], {
msg: `stream add to engine failed`,
error: formatErrorForEvent(e),
stream,
time: getISOStringLocalTz(),
});
}
}
return self;
});
}
catch (e) {
self.emit(self.engineEvents["engine.error"], {
msg: `engine unable to add any streams`,
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
return self;
}
}
async update(...streams) {
let self = this;
let caller = getCaller();
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `update called from: ${caller}`);
try {
return await __classPrivateFieldGet(self, _Engine_engineStreamAddUpdateRemoveMutex, "f").runExclusive(async () => {
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `update mutex acquired from: ${caller}`);
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `update called for streams: ${streams.map((s) => s.streamID)}`);
if (!__classPrivateFieldGet(self, _Engine_isActive, "f")) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `waiting for event=engine.active for ${self.waitForActiveEventMs} seconds before updating`);
try {
await self.waitFor(self.engineEvents["engine.active"], self.waitForActiveEventMs);
}
catch (e) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "engine isActive=false, skipping update/_apiPutStream");
return self;
}
}
for (let stream of streams) {
try {
await self._apiPutStream(stream);
__classPrivateFieldGet(self, _Engine_streamsMap, "f")[stream.streamID] = stream;
this.emit(self.engineEvents["engine.stream.update"], {
msg: `stream updated to engine`,
stream,
time: getISOStringLocalTz(),
});
}
catch (e) {
self.emit(self.engineEvents["engine.stream.error"], {
msg: "stream update to engine failed",
error: formatErrorForEvent(e),
stream,
time: getISOStringLocalTz(),
});
}
}
return self;
});
}
catch (e) {
self.emit(self.engineEvents["engine.error"], {
msg: `engine unable to update any streams`,
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
return self;
}
}
async remove(...streamsWWOReason) {
let reason = "";
let streams;
if (typeof streamsWWOReason[0] === "string") {
reason = streamsWWOReason[0];
streams = streamsWWOReason.slice(1);
}
else {
streams = streamsWWOReason;
}
let self = this;
let caller = getCaller();
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `remove called from: ${caller}`);
try {
return await __classPrivateFieldGet(self, _Engine_engineStreamAddUpdateRemoveMutex, "f").runExclusive(async () => {
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `remove mutex acquired from: ${caller}`);
if (streams.length === 0) {
streams = Object.values(__classPrivateFieldGet(this, _Engine_streamsMap, "f"));
}
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `remove called for streams: ${streams.map((s) => s.streamID)}`);
if (!__classPrivateFieldGet(self, _Engine_isActive, "f")) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `waiting for event=engine.active for ${self.waitForActiveEventMs} seconds before removing`);
try {
await self.waitFor(self.engineEvents["engine.active"], self.waitForActiveEventMs);
}
catch (e) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "engine isActive=false, skipping removing/_apiDeleteStream");
return self;
}
}
if (streams.length === 0) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "no stream to remove");
}
for (let stream of streams) {
try {
if (!__classPrivateFieldGet(self, _Engine_streamsMap, "f")[stream.streamID]) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `stream [ID=${stream.streamID}] not present in engine streamMap. possibly already removed`);
continue;
}
await stream.afterRemove();
await self._apiDeleteStream(stream);
if (stream.hasOutport) {
stream.outport.close();
}
if (stream.hasInport) {
stream.inport.close();
}
delete __classPrivateFieldGet(self, _Engine_streamsMap, "f")[stream.streamID];
self.emit(self.engineEvents["engine.stream.remove"], {
msg: `stream removed from engine ${reason ? "reason:" + reason : ""}`,
stream,
time: getISOStringLocalTz(),
});
}
catch (e) {
self.emit(self.engineEvents["engine.stream.error"], {
msg: "stream remove from engine failed",
error: formatErrorForEvent(e),
stream,
time: getISOStringLocalTz(),
});
}
}
return self;
});
}
catch (e) {
self.emit(self.engineEvents["engine.error"], {
msg: `engine unable to remove any streams`,
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
return self;
}
}
useDefaultEventHandler(addnEventHandlers = {}) {
let self = this;
self.onAny(__classPrivateFieldGet(Engine, _a, "f", _Engine_defaultEngineEventHandler).bind(self));
Object.keys(addnEventHandlers).forEach((eventName) => {
self.on(eventName, addnEventHandlers[eventName]);
});
}
}
_a = Engine, _Engine_engineConfigFilePath = new WeakMap(), _Engine_engineConfig = new WeakMap(), _Engine_engineExtraConfig = new WeakMap(), _Engine_engineProcess = new WeakMap(), _Engine_abortController = new WeakMap(), _Engine_mgmtEventsFreqMs = new WeakMap(), _Engine_keepAliveInterval = new WeakMap(), _Engine_debug = new WeakMap(), _Engine_debugLog = new WeakMap(), _Engine_traceLog = new WeakMap(), _Engine_eventLog = new WeakMap(), _Engine_eventNameToEventLog = new WeakMap(), _Engine_isActive = new WeakMap(), _Engine_engineConstrStartStopMutex = new WeakMap(), _Engine_engineStreamAddUpdateRemoveMutex = new WeakMap(), _Engine_engineUpdateConfigOptionsMutex = new WeakMap(), _Engine_constructorDone = new WeakMap(), _Engine_scheme = new WeakMap(), _Engine_tempLocalServer = new WeakMap(), _Engine_streamsMap = new WeakMap(), _Engine_benthosEXEFullPath = new WeakMap(), _Engine_instances = new WeakSet(), _Engine_setEngineConfig = async function _Engine_setEngineConfig(receivedEngineConfig) {
let self = this;
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `#setEngineConfig called from: ${getCaller()}`);
try {
if (__classPrivateFieldGet(self, _Engine_isActive, "f")) {
throw new Error("cannot set engineConfig on an active engine");
}
if (__classPrivateFieldGet(this, _Engine_engineConfig, "f") === undefined) {
__classPrivateFieldSet(this, _Engine_engineConfig, merge({}, config.engineConfig, receivedEngineConfig), "f");
}
else {
__classPrivateFieldSet(this, _Engine_engineConfig, merge({}, config.engineConfig, __classPrivateFieldGet(this, _Engine_engineConfig, "f"), receivedEngineConfig), "f");
}
if (receivedEngineConfig.metrics) {
__classPrivateFieldGet(this, _Engine_engineConfig, "f").metrics = receivedEngineConfig.metrics;
}
if (receivedEngineConfig.tracer) {
__classPrivateFieldGet(this, _Engine_engineConfig, "f").tracer = receivedEngineConfig.tracer;
}
__classPrivateFieldGet(this, _Engine_debugLog, "f").call(this, "received engineConfig:\n", JSON.stringify(receivedEngineConfig, null, 0));
__classPrivateFieldGet(this, _Engine_debugLog, "f").call(this, "sanitized engineConfig created:\n", JSON.stringify(__classPrivateFieldGet(this, _Engine_engineConfig, "f"), null, 0));
__classPrivateFieldSet(this, _Engine_scheme, __classPrivateFieldGet(this, _Engine_engineConfig, "f").http.cert_file && __classPrivateFieldGet(this, _Engine_engineConfig, "f").http.key_file
? "https"
: "http", "f");
}
catch (e) {
self.emit(self.engineEvents["engine.warn"], {
msg: "unable to update engine config. using defaultEngineConfig",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
}
}, _Engine_setEngineExtraConfig = async function _Engine_setEngineExtraConfig(receivedEngineExtraConfig) {
let self = this;
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `#setEngineExtraConfig called from: ${getCaller()}`);
try {
if (__classPrivateFieldGet(self, _Engine_isActive, "f")) {
throw new Error("cannot set engineExtraConfig on an active engine");
}
if (__classPrivateFieldGet(this, _Engine_engineExtraConfig, "f") === undefined) {
__classPrivateFieldSet(this, _Engine_engineExtraConfig, merge({}, config.engineExtraConfig, receivedEngineExtraConfig), "f");
}
else {
__classPrivateFieldSet(this, _Engine_engineExtraConfig, merge({}, config.engineExtraConfig, __classPrivateFieldGet(this, _Engine_engineExtraConfig, "f"), receivedEngineExtraConfig), "f");
}
__classPrivateFieldGet(this, _Engine_debugLog, "f").call(this, "received engineExtraConfig:\n", JSON.stringify(receivedEngineExtraConfig, null, 0));
__classPrivateFieldGet(this, _Engine_debugLog, "f").call(this, "sanitized engineExtraConfig created:\n", JSON.stringify(__classPrivateFieldGet(this, _Engine_engineExtraConfig, "f"), null, 0));
if (__classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").handleProcessUncaughtException) {
process.on("uncaughtException", function (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "uncaughtException was received, the engine will attempt to stop gracefully now",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
});
}
if (__classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").handleProcessUnhandledRejection) {
process.on("unhandledRejection", function (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "unhandledRejection was received, the engine will attempt to stop gracefully now",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
});
}
if (__classPrivateFieldGet(this, _Engine_engineExtraConfig, "f").debugNamespace) {
let prevNamespaces = debug.disable();
debug.enable([prevNamespaces, __classPrivateFieldGet(this, _Engine_engineExtraConfig, "f").debugNamespace].join(","));
}
}
catch (e) {
self.emit(self.engineEvents["engine.warn"], {
msg: "unable to update engine options. keeping defaults",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
}
}, _Engine_createAxiosInstance = async function _Engine_createAxiosInstance() {
let self = this;
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `createAxiosInstance constructor called from: ${getCaller()}`);
try {
this._axiosInstance = axios.create({
baseURL: `${__classPrivateFieldGet(this, _Engine_scheme, "f")}://${__classPrivateFieldGet(this, _Engine_engineConfig, "f").http.address}`,
});
axiosRetry(this._axiosInstance, {
retries: 3,
retryDelay: axiosRetry.exponentialDelay,
onRetry: (retryCount, err, requestConfig) => {
__classPrivateFieldGet(this, _Engine_debugLog, "f").call(this, `retrying (do not panic): ${requestConfig.url}`, JSON.stringify({ retryCount: retryCount, error: err.toJSON() }));
},
});
}
catch (e) {
self.emit(self.engineEvents["engine.fatal"], {
msg: "unable to create axios instance",
error: formatErrorForEvent(e),
time: getISOStringLocalTz(),
});
}
}, _Engine_startMgmtEvents = async function _Engine_startMgmtEvents() {
let self = this;
let caller = getCaller();
__classPrivateFieldGet(self, _Engine_traceLog, "f").call(self, `_startMgmtEvents called from: ${caller}`);
let shutDownTimer;
do {
try {
if (!__classPrivateFieldGet(self, _Engine_isActive, "f")) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, "engine is not active. existing mgmt event loop");
if (shutDownTimer) {
clearTimeout(shutDownTimer);
shutDownTimer.unref();
}
break;
}
if (!__classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").keepAlive) {
if (self.numStreams === 0 &&
!(shutDownTimer !== undefined && shutDownTimer.hasRef())) {
__classPrivateFieldGet(self, _Engine_debugLog, "f").call(self, `engine.stop will be called if no streams exist for the next ${__classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").shutdownAfterInactivityForMs}ms`);
shutDownTimer = setTimeout(() => {
if (self.numStreams === 0) {
self.stop(`no streams for the last ${__classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").shutdownAfterInactivityForMs}ms`);
}
}, __classPrivateFieldGet(self, _Engine_engineExtraConfig, "f").shutdownAfterInactivityForMs);
}
else if (self.numStreams > 0 &&
shutDownTimer !== undefined &&
shutDownTimer.hasRef()) {
clearTimeout(shutDownTimer);
shutDownTimer.u