exthos
Version:
stream processing in nodejs using the power of golang
242 lines (241 loc) • 10.9 kB
JavaScript
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
};
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
if (kind === "m") throw new TypeError("Private method is not writable");
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
};
var _a, _Stream_streamConfig, _Stream_debugLog, _Stream_inport, _Stream_outport, _Stream_JSFilesToWrite, _Stream_sanitizeStreamConfig, _Stream_wrapJSCode;
import { defaultInputValues } from "../defaults/defaultInputValues.js";
import { defaultOutputValues } from "../defaults/defaultOutputValues.js";
import * as path from "path";
import { tmpdir } from "os";
import * as fs from "fs";
import { randomUUID } from "crypto";
import * as utils from "../utils/utils.js";
import { defaultProcessorValues } from "../defaults/defaultProcessorValues.js";
import debug from "debug";
import * as nanomsg from "nanomsg";
import merge from "lodash.merge";
class Stream {
constructor(streamConfig) {
_Stream_streamConfig.set(this, void 0);
this.hasInport = false;
this.hasOutport = false;
_Stream_debugLog.set(this, debug("exthos").extend("stream:debugLog"));
_Stream_inport.set(this, void 0);
_Stream_outport.set(this, void 0);
_Stream_JSFilesToWrite.set(this, {});
this.streamID = randomUUID();
this.beforeAdd = function () {
let self = this;
var executed = false;
return async function () {
if (!executed) {
executed = true;
let proms = [];
Object.keys(__classPrivateFieldGet(self, _Stream_JSFilesToWrite, "f")).forEach((jsFile) => {
let unWrapedCode = __classPrivateFieldGet(Stream, _a, "m", _Stream_wrapJSCode).call(Stream, __classPrivateFieldGet(self, _Stream_JSFilesToWrite, "f")[jsFile]);
__classPrivateFieldGet(self, _Stream_debugLog, "f").call(self, "writing javascript to file:", jsFile);
proms.push(fs.promises.writeFile(jsFile, unWrapedCode));
});
return await Promise.all(proms);
}
return await Promise.all([]);
};
}.apply(this);
this.afterRemove = function () {
let self = this;
var executed = false;
return async function () {
if (!executed) {
executed = true;
let proms = [];
Object.keys(__classPrivateFieldGet(self, _Stream_JSFilesToWrite, "f")).forEach((jsFile) => {
__classPrivateFieldGet(self, _Stream_debugLog, "f").call(self, "removing javascript to file:", jsFile);
proms.push(fs.promises.unlink(jsFile));
});
return await Promise.all(proms);
}
return await Promise.all([]);
};
}.apply(this);
__classPrivateFieldGet(this, _Stream_debugLog, "f").call(this, "received streamConfig:\n", JSON.stringify(streamConfig, null, 0));
__classPrivateFieldSet(this, _Stream_streamConfig, __classPrivateFieldGet(Stream, _a, "m", _Stream_sanitizeStreamConfig).call(this, streamConfig), "f");
__classPrivateFieldGet(this, _Stream_debugLog, "f").call(this, "sanitized streamConfig:\n", JSON.stringify(__classPrivateFieldGet(this, _Stream_streamConfig, "f"), null, 0));
}
get streamConfig() {
return __classPrivateFieldGet(this, _Stream_streamConfig, "f");
}
set streamConfig(s) {
__classPrivateFieldSet(this, _Stream_streamConfig, __classPrivateFieldGet(Stream, _a, "m", _Stream_sanitizeStreamConfig).call(this, s), "f");
}
get inport() {
return __classPrivateFieldGet(this, _Stream_inport, "f");
}
get outport() {
return __classPrivateFieldGet(this, _Stream_outport, "f");
}
createInport() {
let self = this;
if (!__classPrivateFieldGet(this, _Stream_inport, "f")) {
__classPrivateFieldSet(this, _Stream_inport, nanomsg.socket("push"), "f");
__classPrivateFieldGet(this, _Stream_inport, "f").bind(`ipc:///tmp/${self.streamID}.inport.sock`);
}
}
createOutport() {
let self = this;
if (!__classPrivateFieldGet(self, _Stream_outport, "f")) {
__classPrivateFieldSet(this, _Stream_outport, nanomsg.socket("pull"), "f");
__classPrivateFieldGet(this, _Stream_outport, "f").bind(`ipc:///tmp/${self.streamID}.outport.sock`);
}
}
}
_a = Stream, _Stream_streamConfig = new WeakMap(), _Stream_debugLog = new WeakMap(), _Stream_inport = new WeakMap(), _Stream_outport = new WeakMap(), _Stream_JSFilesToWrite = new WeakMap(), _Stream_sanitizeStreamConfig = function _Stream_sanitizeStreamConfig(receivedStreamConfig) {
let self = this;
let streamConfig = merge({}, receivedStreamConfig);
utils.replaceValueForKey(streamConfig, {
javascript: (existingValue) => {
let jsFile = path.join(tmpdir(), "exthos_jsFile_" + randomUUID() + ".js");
__classPrivateFieldGet(self, _Stream_JSFilesToWrite, "f")[jsFile] = existingValue;
return {
request_map: `root = {}
root.content = this.catch(content())
root.meta = meta()
`,
processors: [
{
subprocess: {
name: "node",
args: [jsFile],
},
},
],
result_map: `
root = if (this.exists("content") && this.exists("meta")).catch(false) {
this.content
} else {
deleted()
}
meta = if (this.exists("content") && this.exists("meta")).catch(false) {
this.meta
} else {
meta()
}`,
};
},
inport: (_) => {
return {
urls: [`ipc:///tmp/${self.streamID}.inport.sock`],
bind: false,
};
},
outport: (_) => {
return {
urls: [`ipc:///tmp/${self.streamID}.outport.sock`],
};
},
});
utils.replaceKeys(streamConfig, {
javascript: () => {
return "branch";
},
inport: () => {
self.hasInport = true;
return "nanomsg";
},
outport: () => {
self.hasOutport = true;
return "nanomsg";
},
direct: () => {
return "inproc";
},
});
utils.replaceValueForKey(streamConfig, {
label: (existingValue) => {
let newValue = existingValue.toLowerCase();
newValue = newValue.replace(/[^a-z0-9_]/g, "_");
newValue = newValue.replace(/^_*/g, "");
return newValue;
},
input: (existingValue) => {
let componentType = Object.keys(existingValue).filter((x) => x !== "label")[0];
return merge({}, {
label: "",
[componentType]: defaultInputValues[componentType],
}, existingValue);
},
output: (existingValue) => {
let componentType = Object.keys(existingValue).filter((x) => x !== "label")[0];
return merge({}, {
label: "",
[componentType]: defaultOutputValues[componentType],
}, existingValue);
},
processors: (existingValues) => {
let toReturn = [];
existingValues.forEach((existingValue) => {
let componentType = Object.keys(existingValue).filter((x) => x !== "label")[0];
toReturn.push(merge({}, {
label: "",
[componentType]: defaultProcessorValues[componentType],
}, existingValue));
});
return toReturn;
},
inputs: (existingValues) => {
let toReturn = [];
existingValues.forEach((existingValue) => {
let componentType = Object.keys(existingValue).filter((x) => x !== "label")[0];
toReturn.push(merge({}, {
label: "",
[componentType]: defaultInputValues[componentType],
}, existingValue));
});
return toReturn;
},
outputs: (existingValues) => {
let toReturn = [];
existingValues.forEach((existingValue) => {
let componentType = Object.keys(existingValue).filter((x) => x !== "label")[0];
toReturn.push(merge({}, {
label: "",
[componentType]: defaultOutputValues[componentType],
}, existingValue));
});
return toReturn;
},
});
return streamConfig;
}, _Stream_wrapJSCode = function _Stream_wrapJSCode(jscode) {
return `//js code autocreated by exthos
try {
process.stdin.setEncoding('utf8');
process.stdout.setEncoding('utf8');
var lineReader = require('readline').createInterface({
input: process.stdin
});
lineReader.on('line', function (msg) {
try{
msg = JSON.parse(msg.toString())
;(()=>{
let console = null
let process = null
${jscode}
})();
console.log(JSON.stringify(msg))
} catch(e) {
console.error(e)
}
});
} catch (e) {
console.error(e.message)
}`;
};
export { Stream };
//# sourceMappingURL=stream.js.map