UNPKG

total5

Version:
2,027 lines (1,679 loc) 83.3 kB
// Total.js FlowStream module // The MIT License // Copyright 2021-2025 (c) Peter Širka <petersirka@gmail.com> 'use strict'; if (!global.F) require('./index'); const W = F.Worker; const Fork = F.Child.fork; const VERSION = 32; const NOTIFYPATH = '/notify/'; var isFLOWSTREAMWORKER = false; var Parent = W.parentPort; var CALLBACKS = {}; var FLOWS = {}; var PROXIES = {}; var TMS = {}; var RPC = {}; var CALLBACKID = 1; var ASFILES = true; var isrunning = false; /* var instance = MODULE('flowstream').init({ components: {}, design: {}, variables: {}, variables2: {} }, true/false); Module exports: module.init(meta [isworker]); module.socket(meta, socket, check(client) => true); module.input([flowstreamid], [id], data); module.trigger(flowstreamid, id, data); module.refresh([flowstreamid], [type]); module.rpc(name, callback); module.exec(id, opt); Methods: instance.trigger(id, data); instance.destroy(); instance.input([flowstreamid], [fromid], [toid], data); instance.add(id, body, [callback]); instance.rem(id, [callback]); instance.components(callback); instance.refresh([type]); instance.io(callback); instance.ioread(flowstreamid, id, callback); instance.reconfigure(id, config); instance.variables(variables); instance.variables2(variables); instance.pause(is); instance.socket(socket); instance.exec(opt, callback); instance.cmd(path, data); instance.httprequest(opt, callback); instance.eval(msg, callback); Delegates: instance.onsave(data); instance.ondone(); instance.onerror(err, type, instanceid, componentid); instance.output(fid, data, tfsid, tid); instance.onhttproute(url, remove); instance.ondestroy(); Extended Flow instances by: instance.save(); instance.toinput(data, [flowstreamid], [id]); instance.output(data, [flowstreamid], [id]); instance.reconfigure(config); instance.newflowstream(meta, isworker); instance.input = function(data) {} */ function Instance(instance, id) { var self = this; self.httproutes = {}; self.version = VERSION; self.id = id; self.flow = instance; // this.onoutput = null; } Instance.prototype = { get stats() { return this.worker ? this.worker.stats : this.flow.stats; }, get worker() { return this.flow; }, get stream() { return this.flow; } }; Instance.prototype.postMessage = function(msg) { this.flow.postMessage && this.flow.postMessage(msg); }; Instance.prototype.httprequest = function(opt, callback) { // opt.route {String} a URL address // opt.params {Object} // opt.query {Object} // opt.body {Object} // opt.headers {Object} // opt.files {Array Object} // opt.url {String} // opt.callback {Function(err, meta)} if (opt.callback) { callback = opt.callback; opt.callback = undefined; } var self = this; if (self.flow.isworkerthread) { var callbackid = callback ? (CALLBACKID++) : -1; if (callbackid !== -1) CALLBACKS[callbackid] = { id: self.flow.id, callback: callback }; self.flow.postMessage2({ TYPE: 'stream/httprequest', data: opt, callbackid: callbackid }); } else httprequest(self.flow, opt, callback); return self; }; // Can't be used in the FlowStream component Instance.prototype.httprouting = function() { var instance = this; instance.onhttproute = function(url, remove) { // GET / #upload #5000 #10000 // - #flag means a flag name // - first #number is timeout // - second #number is max. limit for the payload var flags = []; var limit = 0; var timeout = 0; var id = url; url = url.replace(/#[a-z0-9]+/g, function(text) { text = text.substring(1); if ((/^\d+$/).test(text)) { if (timeout) limit = +text; else timeout = +text; } else flags.push(text); return ''; }).trim(); var route; if (remove) { route = instance.httproutes[id]; if (route) { route.remove(); delete instance.httproutes[id]; } return; } if (instance.httproutes[id]) instance.httproutes[id].remove(); if (timeout) flags.push(timeout); route = ROUTE(url, function() { var self = this; var opt = {}; opt.route = id; opt.params = self.params; opt.query = self.query; opt.body = self.body; opt.files = self.files; opt.headers = self.headers; opt.url = self.url; opt.ip = self.ip; opt.cookies = {}; var cookie = self.headers.cookie; if (cookie) { var arr = cookie.split(';'); for (var i = 0; i < arr.length; i++) { var line = arr[i].trim(); var index = line.indexOf('='); if (index !== -1) { try { opt.cookies[line.substring(0, index)] = decodeURIComponent(line.substring(index + 1)); } catch {} } } } instance.httprequest(opt, function(meta) { if (meta.status) self.status = meta.status; if (meta.headers) { for (var key in meta.headers) self.response.headers[key] = meta.headers[key]; } if (meta.cookies && meta.cookies instanceof Array) { for (var item of meta.cookies) { var name = item.name || item.id; var value = item.value; var expiration = item.expiration || item.expires || item.expire; if (name && value && expiration) self.cookie(name, value, expiration, item.options || item.config); } } var data = meta.body || meta.data || meta.payload; switch (meta.type) { case 'error': self.invalid(meta.body); break; case 'text': case 'plain': self.text(data); break; case 'html': self.html(data); break; case 'xml': self.binary(Buffer.from(data, 'utf8'), 'text/xml'); break; case 'json': self.json(data); break; case 'empty': self.empty(); break; default: if (meta.filename) { var stream = F.Fs.createReadStream(meta.filename); self.stream(stream, meta.type, meta.download); meta.remove && F.cleanup(stream, () => F.Fs.unlink(meta.filename, NOOP)); } else { if (typeof(data) === 'string') self.binary(Buffer.from(data, 'base64'), meta.type); else self.json(data); } break; } }, flags, limit); }); instance.httproutes[id] = route; }; return instance; }; Instance.prototype.cmd = function(path, data) { var self = this; if (self.flow.isworkerthread) { self.flow.postMessage2({ TYPE: 'stream/cmd', path: path, data: data }); } else { var fn = path.indexOf('.') === - 1 ? global[path] : F.TUtils.get(global, path); if (typeof(fn) === 'function') fn(data); } return self; }; Instance.prototype.send = function(id, data, callback) { var self = this; if (self.flow.isworkerthread) { var callbackid = callback ? (CALLBACKID++) : -1; if (callbackid !== -1) CALLBACKS[callbackid] = { id: self.flow.id, callback: callback }; self.flow.postMessage2({ TYPE: 'stream/send', id: id, data: data, callbackid: callbackid }); } else send(self.flow, id, data, callback); }; Instance.prototype.exec = function(opt, callback) { // opt.id = instance_ID // opt.callback = function(err, msg) // opt.uid = String/Number; --> returned back // opt.ref = String/Number; --> returned back // opt.repo = {}; --> returned back // opt.data = {}; --> returned back // opt.vars = {}; // opt.timeout = Number; if (callback && !opt.callback) opt.callback = callback; var self = this; if (self.flow.isworkerthread) { var callbackid = opt.callback ? (CALLBACKID++) : -1; if (callbackid !== -1) CALLBACKS[callbackid] = { id: self.flow.id, callback: opt.callback }; self.flow.postMessage2({ TYPE: 'stream/exec', id: opt.id, uid: opt.uid, ref: opt.ref, vars: opt.vars, repo: opt.repo, data: opt.data, timeout: opt.timeout, callbackid: callbackid }); } else exec(self.flow, opt); return self; }; function execfn(self, name, id, data) { var flow = self.flow; if (flow.isworkerthread) flow.postMessage2({ TYPE: 'stream/' + name, id: id, data: data }); else { if (!flow.paused) { if (id[0] === '@') { id = id.substring(1); for (let key in flow.meta.flow) { let com = flow.meta.flow[key]; if (com.component === id && com[name]) com[name](data); } } else if (id[0] === '#') { id = id.substring(1); for (let key in flow.meta.flow) { let com = flow.meta.flow[key]; if (com.module.name === id && com[name]) com[name](data); } } else { let com = flow.meta.flow[id]; if (com && com[name]) com[name](data); } } } } // Performs trigger Instance.prototype.trigger = function(id, data) { execfn(this, 'trigger', id, data); return this; }; // Notifies instance Instance.prototype.notify = function(id, data) { execfn(this, 'notify', id, data); return this; }; // Performs pause Instance.prototype.pause = function(is) { var self = this; var flow = self.flow; if (flow.isworkerthread) flow.postMessage2({ TYPE: 'stream/pause', is: is }); else flow.pause(is == null ? !flow.paused : is); return self; }; // Asssigns UI websocket the the FlowStream Instance.prototype.socket = function(socket) { var self = this; exports.socket(self.flow, socket); return self; }; Instance.prototype.eval = function(msg, callback) { var self = this; if (self.flow.isworkerthread) { var callbackid = callback ? (CALLBACKID++) : -1; if (callback) CALLBACKS[callbackid] = { id: self.flow.id, callback: callback }; self.flow.postMessage2({ TYPE: 'ui/message', data: msg, callbackid: callbackid }); } else self.flow.proxy.message(msg, -1, callback); return self; }; Instance.prototype.restart = function() { var self = this; self.flow.$socket && self.flow.$socket.destroy(); self.flow.$client && self.flow.$client.destroy(); if (self.flow.terminate) self.flow.terminate(); else self.flow.kill(9); }; Instance.prototype.remove = function() { F.TFlow.remove(this.id); }; // Destroys the Flow Instance.prototype.kill = Instance.prototype.destroy = function() { var self = this; setTimeout(() => exports.refresh(self.id, 'destroy'), 500); self.flow.$destroyed = true; self.flow.$terminated = true; self.flow.$socket && self.flow.$socket.close(); if (self.flow.isworkerthread) { self.postMessage({ TYPE: 'stream/destroy' }); setTimeout(self => self.flow.terminate ? self.flow.terminate() : self.flow.kill(9), 1000, self); if (PROXIES[self.id]) { PROXIES[self.id].remove(); delete PROXIES[self.id]; } } else { if (self.flow.sockets) { for (var key in self.flow.sockets) self.flow.sockets[key].destroy(); } self.flow.destroy(); } self.flow.$socket && self.flow.$socket.destroy(); self.flow.$client && self.flow.$client.destroy(); for (var key in CALLBACKS) { if (CALLBACKS[key].id === self.id) delete CALLBACKS[key]; } if (self.httproutes) { for (var key in self.httproutes) self.httproutes[key].remove(); } self.ondestroy && self.ondestroy(); delete FLOWS[self.id]; }; // Sends data to the speficic input // "@id" sends to all component with "id" // "id" sends to instance with "id" Instance.prototype.input = function(flowstreamid, fromid, toid, data, reference) { var self = this; var flow = self.flow; if (flow.isworkerthread) { flow.postMessage2({ TYPE: 'stream/input', flowstreamid: flowstreamid, fromid: fromid, id: toid, data: data, reference: reference }); return self; } if (toid) { if (toid[0] === '@') { var tmpid = toid.substring(1); for (let key in flow.meta.flow) { let tmp = flow.meta.flow[key]; if (tmp.input && tmp.component === tmpid) tmp.input(flowstreamid, fromid, data, reference); } } else { let tmp = flow.meta.flow[toid]; if (tmp) { tmp.input && tmp.input(flowstreamid, fromid, data, reference); } else { for (let key in flow.meta.flow) { let tmp = flow.meta.flow[key]; if (tmp.input && tmp.config.name === toid) tmp.input(flowstreamid, fromid, data, reference); } } } } else { // Send to all inputs for (let key in flow.meta.flow) { var f = flow.meta.flow[key]; var c = flow.meta.components[f.component]; if (f.input && c.type === 'input2') f.input(flowstreamid, fromid, data, reference); } } return self; }; // Adds a new component Instance.prototype.add = function(id, body, callback) { var self = this; if (self.flow.isworkerthread) { var callbackid = callback ? (CALLBACKID++) : -1; if (callback) CALLBACKS[callbackid] = { id: self.flow.id, callback: callback }; self.flow.postMessage2({ TYPE: 'stream/add', id: id, data: body, callbackid: callbackid }); } else { self.flow.add(id, body, function(err) { callback && callback(err); self.flow.redraw(); self.flow.save(); }, ASFILES); } return self; }; // Removes specific component Instance.prototype.rem = function(id, callback) { var self = this; if (self.flow.isworkerthread) { var callbackid = callback ? (CALLBACKID++) : -1; if (callback) CALLBACKS[callbackid] = { id: self.flow.id, callback: callback }; self.flow.postMessage2({ TYPE: 'stream/rem', id: id, callbackid: callbackid }); } else self.flow.unregister(id, callback); return self; }; // Reads all components Instance.prototype.components = function(callback) { var self = this; if (self.flow.isworkerthread) { var callbackid = CALLBACKID++; CALLBACKS[callbackid] = { id: self.flow.id, callback: callback }; self.flow.postMessage2({ TYPE: 'stream/components', callbackid: callbackid }); } else callback(null, self.flow.components(true)); return self; }; function readmeta(meta) { var obj = {}; obj.id = meta.id; obj.name = meta.name; obj.version = meta.version; obj.icon = meta.icon; obj.color = meta.color; obj.reference = meta.reference; obj.group = meta.group; obj.author = meta.author; return obj; } function readinstance(flow, id) { var tmp = flow.meta.flow[id]; if (tmp) { var com = flow.meta.components[tmp.component]; if (com) { if ((com.type === 'output' || com.type === 'input' || com.type === 'config')) return { id: id, componentid: tmp.component, component: com.name, name: tmp.config.name || com.name, schema: com.schemaid ? com.schemaid[1] : undefined, icon: com.icon, color: com.color, type: com.type, readme: tmp.config.readme, outputs: tmp.outputs, inputs: tmp.inputs }; } else flow.clean(); } } // Reads all inputs, outputs, publish, subscribe instances Instance.prototype.io = function(id, callback) { var self = this; if (self.flow.isworkerthread) { var callbackid = CALLBACKID++; CALLBACKS[callbackid] = { id: self.flow.id, callback: callback }; self.flow.postMessage2({ TYPE: 'stream/io', id: id, callbackid: callbackid }); return self; } var flow = self.flow; if (id) { var obj = null; if (flow.meta.flow[id]) callback(null, readinstance(flow, id)); else callback(); return; } var arr = []; for (var key in flow.meta.flow) { var obj = readinstance(flow, key); obj && arr.push(obj); } callback(null, arr); }; // Reconfigures a component Instance.prototype.reconfigure = function(id, config) { var self = this; if (self.flow.isworkerthread) self.flow.postMessage2({ TYPE: 'stream/reconfigure', id: id, data: config }); else self.flow.reconfigure(id, config); return self; }; Instance.prototype.reload = function(data) { var self = this; var flow = self.flow; if (PROXIES[data.id]) { PROXIES[data.id].remove(); delete PROXIES[data.id]; } if (flow.isworkerthread) { if (data.proxypath) { if (!data.unixsocket) { data.unixsocket = flow.$schema.unixsocket || makeunixsocket(data.id); flow.$schema.unixsocket = data.unixsocket; } PROXIES[data.id] = F.proxy(data.proxypath, data.unixsocket); } for (let key in data) flow.$schema[key] = data[key]; self.proxypath = data.proxypath; flow.postMessage2({ TYPE: 'stream/rewrite', data: data }); } else { for (let key in data) flow.$schema[key] = data[key]; flow.variables = data.variables; if (data.variables2) flow.variables2 = data.variables2; flow.rewrite(data, () => flow.proxy.refreshmeta()); } return self; }; Instance.prototype.refresh = function(id, type, data, restart) { var self = this; var flow = self.flow; if (flow.isworkerthread) { for (var key in data) flow.$schema[key] = data[key]; if (restart) { if (flow.terminate) flow.terminate(); else flow.kill(9); } else flow.postMessage2({ TYPE: 'stream/refresh', id: id, type: type, data: data }); } else { if (type === 'meta' && data) { for (var key in data) flow.$schema[key] = data[key]; flow.proxy.refreshmeta(); } for (var key in flow.meta.flow) { var instance = flow.meta.flow[key]; instance.flowstream && instance.flowstream(id, type); } } }; // Updates variables Instance.prototype.variables = function(variables) { var self = this; var flow = self.flow; if (flow.isworkerthread) { flow.$schema.variables = variables; flow.postMessage2({ TYPE: 'stream/variables', data: variables }); } else { flow.variables = variables; for (var key in flow.meta.flow) { var instance = flow.meta.flow[key]; instance.variables && instance.variables(flow.variables); instance.vary && instance.vary('variables'); } flow.proxy.online && flow.proxy.send({ TYPE: 'flow/variables', data: variables }); flow.save(); } return self; }; // Updates global variables Instance.prototype.variables2 = function(variables) { var self = this; var flow = self.flow; if (flow.isworkerthread) { flow.$schema.variables2 = variables; flow.postMessage2({ TYPE: 'stream/variables2', data: variables }); } else { flow.variables2 = variables; for (var key in flow.meta.flow) { var instance = flow.meta.flow[key]; instance.variables2 && instance.variables2(flow.variables2); instance.vary && instance.vary('variables2'); } flow.save(); } return self; }; Instance.prototype.export = function(callback) { var self = this; var flow = self.flow; if (flow.isworkerthread) { var callbackid = callback ? (CALLBACKID++) : -1; CALLBACKS[callbackid] = { id: self.flow.id, callback: callback }; self.flow.postMessage2({ TYPE: 'stream/export', callbackid: callbackid }); } else callback(null, self.flow.export2()); return self; }; // Initializes FlowStream exports.init = function(meta, isworker, callback, nested) { return isworker ? init_worker(meta, isworker, callback) : init_current(meta, callback, nested); }; exports.exec = function(id, opt) { var fs = FLOWS[id]; if (fs) fs.exec(id, opt); else if (opt.callback) opt.callback(404); }; exports.eval = function(id, opt) { var fs = FLOWS[id]; if (fs) fs.eval(id, opt); else if (opt.callback) opt.callback(404); }; exports.input = function(ffsid, fid, tfsid, tid, data, reference) { if (tfsid) { var fs = FLOWS[tfsid]; fs && fs.$instance.input(ffsid, fid, tid, data, reference); } else { for (var key in FLOWS) { var flow = FLOWS[key]; flow.$instance.input(ffsid, fid, tid, data, reference); } } }; exports.trigger = function(flowstreamid, id, data) { var fs = FLOWS[flowstreamid]; fs && fs.trigger(id, data); }; exports.refresh = function(id, type) { for (var key in FLOWS) { var flow = FLOWS[key]; flow.$instance.refresh(id, type); } }; exports.rpc = function(name, callback) { RPC[name] = callback; }; exports.version = VERSION; function send(self, id, data, callback) { var index = id.lastIndexOf('/'); var input = ''; if (index !== -1) { input = id.substring(index + 1); id = id.substring(0, index); } var instances = self.meta.flow; var instance = null; if (id[0] === '@') { id = id.substring(1); for (let key in instances) { if (instances[key].component === id) { instance = instances[key]; break; } } } else if (id[0] === '#') { id = id.substring(1); for (let key in instances) { if (instances[key].module.name === id) { instance = instances[key]; break; } } } else { if (instances[id]) instance = instances[id]; } if (!instance) { if (callback) { if (Parent) { let opt = {}; opt.callbackid = callback; opt.data = { error: 404 }; Parent.postMessage(opt); } else callback(404); } return; } var msg = instance.newmessage(data); msg.input = input; callback && msg.on('end', function(msg) { let output = {}; output.error = msg.error; output.repo = msg.repo; output.data = msg.data; output.count = msg.count; output.cloned = msg.cloned; output.duration = Date.now() - msg.duration; output.meta = { id: instance.id, component: instance.component }; if (Parent) { let opt = {}; opt.TYPE = 'stream/send'; opt.callbackid = callback; opt.data = output; Parent.postMessage(opt); } else callback(output.error, output); }); instance.message(msg); } function exec(self, opt) { var target = []; var instances = self.meta.flow; var id; if (opt.id[0] === '@') { id = opt.id.substring(1); for (let key in instances) { if (instances[key].component === id) target.push(instances[key]); } } else if (opt.id[0] === '#') { id = opt.id.substring(1); for (let key in instances) { if (instances[key].module.name === id) target.push(instances[key]); } } else { if (instances[opt.id]) target.push(instances[opt.id]); } target.wait(function(instance, next) { var msg = instance && instance.message ? instance.newmessage() : null; if (msg) { if (opt.vars) msg.vars = opt.vars; if (opt.repo) msg.repo = opt.repo; msg.data = opt.data == null ? {} : opt.data; if (opt.callbackid !== -1) { msg.on('end', function(msg) { var output = {}; output.uid = opt.uid; output.ref = opt.ref; output.error = msg.error; output.repo = msg.repo; output.data = msg.data; output.count = msg.count; output.cloned = msg.cloned; output.duration = Date.now() - msg.duration; output.meta = { id: instance.id, component: instance.component }; if (Parent) { if (opt.callbackid !== -1) { opt.repo = undefined; opt.vars = undefined; opt.data = output; Parent.postMessage(opt); } } else if (opt.callback) opt.callback(output.error, output); }); } if (opt.timeout) msg.totaltimeout(opt.timeout); instance.message(msg); } else if (opt.callback) { opt.callback(404); } else if (Parent && opt.callbackid !== -1) { opt.repo = undefined; opt.vars = undefined; opt.data = { error: 404 }; Parent.postMessage(opt); } setImmediate(next); }, function() { if (!target.length) { if (opt.callback) { opt.callback(404); } else if (Parent && opt.callbackid !== -1) { opt.repo = undefined; opt.vars = undefined; opt.data = { error: 404 }; Parent.postMessage(opt); } } }); } function rpc(name, data, callback) { var fn = RPC[name]; if (fn) fn(data, callback); else callback('Invalid remote procedure name'); } function httprequest(self, opt, callback) { if (self.httproutes[opt.route]) { self.httproutes[opt.route].callback(opt, function(data) { // data.status {Number} // data.headers {Object} // data.body {Buffer} if (Parent) Parent.postMessage({ TYPE: 'stream/httpresponse', data: data, callbackid: callback }); else callback(data); }); } else { if (Parent) Parent.postMessage({ TYPE: 'stream/httpresponse', data: { type: 'error', body: 404 }, callbackid: callback }); else callback({ type: 'error', body: 404 }); } } function killprocess() { // console.error('Main process doesn\'t respond'); process.exit(1); } function init_current(meta, callback, nested) { initrunning(); if (!meta.directory) meta.directory = F.path.root('flowstream'); // Due to C/C++ modules if (W.workerData || meta.sandbox) F.config.$node_modules = F.path.join(meta.directory, meta.id, 'node_modules'); ASFILES = meta.asfiles === true; var flow = MAKEFLOWSTREAM(meta); FLOWS[meta.id] = flow; if (isFLOWSTREAMWORKER) { if (meta.unixsocket && meta.proxypath) { if (!F.isWindows) F.Fs.unlink(meta.unixsocket, NOOP); F.http({ load: 'none', unixsocket: meta.unixsocket, clear: false, config: { $stats: false, $sourcemap: false }}); } else { F.config.$sourcemap = false; F.config.$stats = false; F.load('none', null, false); } } flow.name = meta.name || meta.id; flow.env = meta.env; flow.origin = meta.origin; flow.proxypath = meta.proxypath || ''; flow.proxy.online = false; flow.proxy.ping = 0; if (meta.import) { let tmp = meta.import.split(/,|;/).trim(); for (let m of tmp) { let mod = require(F.path.root(m)); mod.install && mod.install(flow); mod.init && mod.init(flow); } } if (meta.initscript) { try { new Function('instance', meta.initscript)(flow); } catch (e) { flow.error(e, 'initscript'); } } flow.$instance = new Instance(flow, meta.id); flow.$instance.output = function(fid, data, tfsid, tid, reference) { exports.input(meta.id, fid, tfsid, tid, data, reference); }; if (!nested && Parent) { Parent.on('message', function(msg) { var id; switch (msg.TYPE) { case 'ping': flow.proxy.ping && clearTimeout(flow.proxy.ping); flow.proxy.ping = setTimeout(killprocess, 10000); break; case 'stream/destroy': flow.destroy(); break; case 'stream/export': msg.data = flow.export2(); Parent.postMessage(msg); break; case 'stream/reconfigure': flow.reconfigure(msg.id, msg.data); break; case 'stream/httprequest': httprequest(flow, msg.data, msg.callbackid); break; case 'stream/cmd': var fn = msg.path.indexOf('.') === - 1 ? global[msg.path] : F.TUtils.get(global, msg.path); if (fn && typeof(fn) === 'function') fn(msg.data); break; case 'stream/send': send(flow, msg.id, msg.data, msg.callbackid); break; case 'stream/exec': exec(flow, msg); break; case 'stream/eval': if (msg.callbackid) { flow.proxy.message(msg, function(response) { msg.data = response; Parent.postMessage(msg); }); } else flow.proxy.message(msg); break; case 'stream/notify': case 'stream/trigger': id = msg.id; var type = msg.TYPE.substring(7); if (!flow.paused) { if (id[0] === '@') { id = id.substring(1); for (var key in flow.meta.flow) { var com = flow.meta.flow[key]; if (com.component === id && com[type]) com[type](msg.data); } } else if (id[0] === '#') { id = id.substring(1); for (var key in flow.meta.flow) { var com = flow.meta.flow[key]; if (com.module.name === id && com[type]) com[type](msg.data); } } else { var com = flow.meta.flow[id]; if (com && com[type]) com[type](msg.data); } } break; case 'stream/pause': flow.pause(msg.is == null ? !flow.paused : msg.is); flow.save(); break; case 'stream/rewrite': for (var key in msg.data) flow.$schema[key] = msg.data[key]; flow.rewrite(msg.data, function() { // @err {Error} flow.proxy.refreshmeta(); if (flow.proxy.online) { flow.proxy.send({ TYPE: 'flow/components', data: flow.components(true) }); flow.proxy.send({ TYPE: 'flow/design', data: flow.export() }); flow.proxy.send({ TYPE: 'flow/variables', data: flow.variables }); } }); break; case 'stream/refresh': if (msg.type === 'meta' && msg.data) { for (var key in msg.data) flow.$schema[key] = msg.data[key]; flow.proxy.refreshmeta(); } for (var key in flow.meta.flow) { var instance = flow.meta.flow[key]; instance.flowstream && instance.flowstream(msg.id, msg.type); } break; case 'stream/io2': case 'stream/rpcresponse': var cb = CALLBACKS[msg.callbackid]; if (cb) { delete CALLBACKS[msg.callbackid]; cb.callback(msg.error, msg.data); } break; case 'stream/components': msg.data = flow.components(true); Parent.postMessage(msg); break; case 'stream/io': if (msg.id) { msg.data = readinstance(flow, msg.id); } else { var arr = []; for (var key in flow.meta.flow) { let tmp = readinstance(flow, key); if (tmp) arr.push(tmp); } msg.data = arr; } Parent.postMessage(msg); break; case 'stream/input': if (msg.id) { if (msg.id[0] === '@') { id = msg.id.substring(1); for (let key in flow.meta.flow) { let tmp = flow.meta.flow[key]; if (tmp.input && tmp.component === id) tmp.input(msg.flowstreamid, msg.fromid, msg.data, msg.reference); } } else { let tmp = flow.meta.flow[msg.id]; if (tmp) { tmp.input && tmp.input(msg.flowstreamid, msg.fromid, msg.data, msg.reference); } else { for (let key in flow.meta.flow) { let tmp = flow.meta.flow[key]; if (tmp.input && tmp.config.name === msg.id) tmp.input(msg.flowstreamid, msg.fromid, msg.data, msg.reference); } } } } else { for (let key in flow.meta.flow) { var f = flow.meta.flow[key]; var c = flow.meta.components[f.component]; if (f.input && c.type === 'input2') f.input(msg.flowstreamid, msg.fromid, msg.data, msg.reference); } } break; case 'stream/add': flow.add(msg.id, msg.data, function(err) { msg.error = err ? err.toString() : null; if (msg.callbackid !== -1) Parent.postMessage(msg); flow.redraw(); flow.save(); }, ASFILES); break; case 'stream/rem': flow.unregister(msg.id, function(err) { msg.error = err ? err.toString() : null; if (msg.callbackid !== -1) Parent.postMessage(msg); flow.save(); }); break; case 'stream/variables': flow.variables = msg.data; for (var key in flow.meta.flow) { var instance = flow.meta.flow[key]; instance.variables && instance.variables(flow.variables); } flow.proxy.online && flow.proxy.send({ TYPE: 'flow/variables', data: msg.data }); flow.save(); break; case 'stream/variables2': flow.variables2 = msg.data; for (var key in flow.meta.flow) { var instance = flow.meta.flow[key]; instance.variables2 && instance.variables2(flow.variables2); } flow.save(); break; case 'ui/newclient': flow.proxy.online = true; flow.proxy.newclient(msg.clientid); break; case 'ui/online': flow.proxy.online = msg.online; break; case 'ui/message': if (msg.callbackid) { flow.proxy.message(msg.data, msg.clientid, function(data) { msg.TYPE = 'stream/eval'; msg.data = data; Parent.postMessage(msg); }); } else flow.proxy.message(msg.data, msg.clientid); break; } }); flow.proxy.remove = function() { Parent.postMessage({ TYPE: 'stream/remove' }); }; flow.proxy.kill = function() { Parent.postMessage({ TYPE: 'stream/kill' }); }; flow.proxy.send = function(msg, type, clientid) { Parent.postMessage({ TYPE: 'ui/send', data: msg, type: type, clientid: clientid }); }; flow.proxy.save = function(data) { if (!flow.$schema || !flow.$schema.readonly) Parent.postMessage({ TYPE: 'stream/save', data: data }); }; flow.proxy.httproute = function(url, callback, instance) { if (!flow.$schema || !flow.$schema.readonly) { if (callback) flow.httproutes[url] = { id: instance.id, component: instance.component, callback: callback }; else delete flow.httproutes[url]; Parent.postMessage({ TYPE: 'stream/httproute', data: { url: url, remove: callback == null }}); } }; flow.proxy.done = function(err) { Parent.postMessage({ TYPE: 'stream/done', error: err }); }; flow.proxy.error = function(err, source, instance) { var instanceid = ''; var componentid = ''; if (instance) { if (typeof(instance) === 'string') { if (source === 'add' || source === 'register') { componentid = instance; F.error(err, 'FlowStream | register component | ' + instance); } else if (meta.design[instance]) { instanceid = instance; componentid = meta.design[instance].component; } } else if (source === 'instance_message') { if (instance.instance) { instanceid = instance.instance.id; componentid = instance.instance.component; } else F.error(err, 'FlowStream | message | ' + instance); } else if (source === 'instance_close') { instanceid = instance.id; componentid = instance.component; } else if (source === 'instance_make') { instanceid = instance.id; componentid = instance.component; } else { instanceid = instance.id; componentid = instance.module.id; } } Parent.postMessage({ TYPE: 'stream/error', error: err.toString(), stack: err.stack, source: source, id: instanceid, component: componentid }); }; flow.proxy.refresh = function(type) { Parent.postMessage({ TYPE: 'stream/refresh', type: type }); }; flow.proxy.output = function(id, data, flowstreamid, instanceid, reference) { Parent.postMessage({ TYPE: 'stream/output', id: id, data: data, flowstreamid: flowstreamid, instanceid: instanceid, reference: reference }); }; flow.proxy.input = function(fromid, tfsid, toid, data, reference) { Parent.postMessage({ TYPE: 'stream/toinput', fromflowstreamid: flow.id, fromid: fromid, toflowstreamid: tfsid, toid: toid, data: data, reference: reference }); }; flow.proxy.restart = function() { Parent.postMessage({ TYPE: 'stream/restart' }); }; flow.proxy.io = function(flowstreamid, id, callback) { if (typeof(flowstreamid) === 'function') { callback = flowstreamid; id = null; flowstreamid = null; } else if (typeof(id) === 'function') { callback = id; id = null; } var callbackid = callback ? (CALLBACKID++) : -1; if (callback) CALLBACKS[callbackid] = { id: flow.id, callback: callback }; Parent.postMessage({ TYPE: 'stream/io2', flowstreamid: flowstreamid, id: id, callbackid: callbackid }); }; } else { flow.proxy.io = function(flowstreamid, id, callback) { exports.io(flowstreamid, id, callback); }; flow.proxy.restart = function() { // nothing }; flow.proxy.remove = function() { flow.$instance.remove(); }; flow.proxy.kill = function() { flow.$instance.kill(); }; flow.proxy.send = NOOP; flow.proxy.save = function(data) { if (!flow.$schema || !flow.$schema.readonly) flow.$instance.onsave && flow.$instance.onsave(data); }; flow.proxy.httproute = function(url, callback, instanceid) { if (!flow.$schema || !flow.$schema.readonly) { if (callback) flow.httproutes[url] = { id: instanceid, callback: callback }; else delete flow.httproutes[url]; flow.$instance.onhttproute && flow.$instance.onhttproute(url, callback == null); } }; flow.proxy.refresh = function(type) { exports.refresh(flow.id, type); }; flow.proxy.done = function(err) { flow.$instance.ondone && setImmediate(flow.$instance.ondone, err); }; flow.proxy.input = function(fromid, tfsid, toid, data, reference) { exports.input(flow.id, fromid, tfsid, toid, data, reference); }; flow.proxy.error = function(err, source, instance) { let instanceid = ''; let componentid = ''; if (instance) { if (source === 'instance_message') { instanceid = instance.instance.id; componentid = instance.instance.component; } else if (source === 'instance_close') { instanceid = instance.id; componentid = instance.component; } else if (source === 'instance_make') { instanceid = instance.id; componentid = instance.component; } else if (source === 'register') { instanceid = ''; componentid = instance; } else if (source === 'add') { componentid = instance; } else { instanceid = instance.id; componentid = instance.module ? instance.module.id : null; } } let tmp = { TYPE: 'flow/error', error: err.toString(), source: source, id: instanceid, component: componentid, ts: new Date() }; flow.$socket && flow.$socket.send(tmp); flow.$client && flow.$client.send(tmp); flow.$instance.onerror && flow.$instance.onerror(err, source, instanceid, componentid); }; flow.proxy.output = function(id, data, flowstreamid, instanceid, reference) { flow.$instance.output && flow.$instance.output(id, data, flowstreamid, instanceid, reference); }; } callback && callback(null, flow.$instance); return flow.$instance; } function init_worker(meta, type, callback) { var forkargs = [F.directory, '--fork']; if (F.config.$insecure) forkargs.push('--insecure'); if (meta.memory) forkargs.push('--max-old-space-size=' + meta.memory); var worker = type === 'worker' ? (new W.Worker(__filename, { workerData: meta })) : Fork(__filename, forkargs, { serialization: 'json', detached: false }); var ischild = false; meta.unixsocket = makeunixsocket(meta.id); if (PROXIES[meta.id]) { PROXIES[meta.id].remove(); delete PROXIES[meta.id]; } if (meta.proxypath) PROXIES[meta.id] = F.proxy(meta.proxypath, meta.unixsocket); if (!worker.postMessage) { worker.postMessage = worker.send; ischild = true; } worker.postMessage2 = function(a, b) { if (!worker.$terminated) worker.postMessage(a, b); }; worker.$instance = new Instance(worker, meta.id); worker.$instance.isworkerthread = true; worker.isworkerthread = true; worker.$schema = meta; worker.$instance.output = function(id, data, flowstreamid, instanceid, reference) { exports.input(meta.id, id, flowstreamid, instanceid, data, reference); }; FLOWS[meta.id] = worker; var restart = function(code) { worker.$terminated = true; setTimeout(function(worker, code) { worker.$socket && setTimeout(socket => socket && socket.destroy(), 2000, worker.$socket); worker.$client && setTimeout(client => client && client.destroy(), 2000, worker.$client); if (!worker.$destroyed) { console.log('FlowStream auto-restart: ' + worker.$schema.name + ' (exit code: ' + ((code || '0') + '') + ')'); init_worker(worker.$schema, type, callback); worker.$instance = null; worker.$schema = null; worker.$destroyed = true; } }, 1000, worker, code); }; worker.on('exit', restart); worker.on('message', function(msg) { var tmp; Flow.$events.message && Flow.emit('message', worker.$instance.id, msg); switch (msg.TYPE) { case 'stream/stats': worker.stats = msg.data; if (Flow.$events.stats) Flow.emit('stats', meta.id, msg.data); break; case 'stream/restart': if (worker.terminate) worker.terminate(); else worker.kill(9); break; case 'stream/kill': if (!worker.$terminated) worker.$instance.destroy(msg.code || 9); break; case 'stream/remove': if (!worker.$terminated) worker.$instance.remove(); break; case 'stream/send': tmp = CALLBACKS[msg.callbackid]; if (tmp) { delete CALLBACKS[msg.callbackid]; tmp.callback(msg.data.error, msg.data); } break; case 'stream/exec': tmp = CALLBACKS[msg.callbackid]; if (tmp) { delete CALLBACKS[msg.callbackid]; tmp.callback(msg.data.error, msg.data, msg.meta); } break; case 'stream/httpresponse': tmp = CALLBACKS[msg.callbackid]; if (tmp) { delete CALLBACKS[msg.callbackid]; tmp.callback(msg.data, msg.meta); } break; case 'stream/rpc': rpc(msg.name, msg.data, (err, response) => worker.postMessage2({ TYPE: 'stream/rpcresponse', error: err, data: response, callbackid: msg.callbackid })); break; case 'stream/export': case 'stream/components': var cb = CALLBACKS[msg.callbackid]; if (cb) { delete CALLBACKS[msg.callbackid]; cb.callback(null, msg.data); } break; case 'stream/toinput': exports.input(msg.fromflowstreamid, msg.fromid, msg.toflowstreamid, msg.toid, msg.data, msg.reference); break; case 'stream/refresh': exports.refresh(meta.id, msg.type); break; case 'stream/error': tmp = { TYPE: 'flow/error', error: msg.error, stack: msg.stack, source: msg.source, id: msg.id, component: msg.component, ts: new Date() }; worker.$socket && worker.$socket.send(tmp); worker.$client && worker.$client.send(tmp); worker.$instance.onerror && worker.$instance.onerror(msg.error, msg.source, msg.id, msg.component, msg.stack); break; case 'stream/save': worker.$schema.name = msg.data.name; worker.$schema.components = msg.data.components; worker.$schema.design = msg.data.design; worker.$schema.variables = msg.data.variables; worker.$schema.origin = msg.data.origin; worker.$schema.sources = msg.data.sources; worker.$instance.onsave && worker.$instance.onsave(msg.data); break; case 'stream/httproute': worker.$instance.onhttproute && worker.$instance.onhttproute(msg.data.url, msg.data.remove); break; case 'stream/done': worker.$instance.ondone && worker.$instance.ondone(msg.error); break; case 'stream/io2': exports.io(msg.flowstreamid, msg.id, function(err, data) { msg.data = data; msg.error = err; worker.postMessage(msg); }); break; case 'stream/output': if (worker.$instance.onoutput) { tmp = meta.design[msg.id]; tmp && worker.$instance.onoutput({ id: msg.id, name: tmp.config.name, data: msg.data, reference: msg.reference }); } worker.$instance.output && worker.$instance.output(msg.id, msg.data, msg.flowstreamid, msg.instanceid, msg.reference); break; case 'stream/add': case 'stream/rem': tmp = CALLBACKS[msg.callbackid]; if (tmp) { delete CALLBACKS[msg.callbackid]; tmp.callback(msg.error); } break; case 'stream/io': case 'stream/eval': tmp = CALLBACKS[msg.callbackid]; if (tmp) { delete CALLBACKS[msg.callbackid]; tmp.callback(msg.error, msg.data); } break; case 'ui/send': worker.$client && worker.$client.send(msg.data); switch (msg.type) { case 1: worker.$socket && worker.$socket.send(msg.data, client => client.id === msg.clientid); break; case 2: worker.$socket && worker.$socket.send(msg.data, client => client.id !== msg.clientid); break; default: worker.$socket && worker.$socket.send(msg.data); break; } break; } }); ischild && worker.send({ TYPE: 'init', data: meta }); callback && callback(null, worker.$instance); return worker.$instance; } exports.io = function(flowstreamid, id, callback) { if (typeof(flowstreamid) === 'function') { callback = flowstreamid; id = null; flowstreamid = null; } else if (typeof(id) === 'function') { callback = id; id = null; } var flow; if (id) { flow = FLOWS[flowstreamid]; if (flow) { flow.$instance.io(id, function(err, data) { if (data) { var tmp = readmeta(flow.$schema); tmp.item = data; data = tmp; } callback(err, data); }); } else callback(); return; } if (flowstreamid) { flow = FLOWS[flowstreamid]; if (flow) { flow.$instance.io(null, function(err, data) { var f = flow.$schema || EMPTYOBJECT; var meta = readmeta(f); meta.items = data; callback(null, meta); }); } else callback(); return; } var arr = []; Object.keys(FLOWS).wait(function(key, next) { var flow = FLOWS[key]; if (flow) { flow.$instance.io(null, function(err, data) { var f = flow.$schema || EMPTYOBJECT; var meta = readmeta(f); meta.items = data; arr.push(meta); next(); }); } else next(); }, function() { callback(null, arr); }); }; exports.socket = function(flow, socket, verify, check) { if (typeof(flow) === 'string') flow = FLOWS[flow]; if (!flow) { setTimeout(() => socket.destroy(), 100); return; } flow.$socket = socket; var newclient = function(client) { client.isflowstreamready = true; if (flow.isworkerthread) { flow.postMessage2({ TYPE: 'ui/newclient', clientid: client.id }); } else { flow.proxy.online = true; flow.proxy.newclient(client.id); } }; socket.on('open', function(client) { if (verify) verify(client, () => newclient(client)); else newclient(client); }); socket.autodestroy(function() { delete flow.$socket; if (flow.isworkerthread) flow.postMessage2({ TYPE: 'ui/online', online: false }); else flow.proxy.online = false; }); socket.on('close', function(client) { if (client.isflowstreamready) { var is = socket.online > 0; if (flow.isworkerthread) flow.postMessage2({ TYPE: 'ui/online', online: is }); else flow.proxy.online = is; } }); socket.on('message', function(client, msg) { if (client.isflowstreamready) { // It can check permissions if (check) { let err = check(client, msg); if (err !== true) { if (msg.callbackid) client.send({ callbackid: msg.callbackid, error: typeof(err) === 'string' ? err : '401' }); return; } } if (flow.isworkerthread) flow.postMessage2({ TYPE: 'ui/message', clientid: client.id, data: msg }); else flow.proxy.message(msg, client.id); } }); if (flow.isworkerthread) return; flow.proxy.send = function(msg, type, clientid) { // 0: all // 1: client // 2: with except client switch (type) { case 1: clientid && socket.send(msg, conn => conn.id === clientid); break; case 2: socket.send(msg, conn => conn.id !== clientid); break; default: socket.send(msg); break; } }; }; exports.client = function(flow, socket) { if (typeof(flow) === 'string') flow = FLOWS[flow]; var clientid = flow.id; flow.$client = socket; socket.on('close', function() { if (flow.isworkerthread) flow.postMessage2({ TYPE: 'ui/online', online: false }); else flow.proxy.online = false; }); socket.on('message', function(msg) { if (msg.TYPE === 'flow') { if (flow.isworkerthread) { flow.postMessage2({ TYPE: 'ui/newclient', clientid: clientid }); } else { flow.proxy.online = true; flow.proxy.newclient(clientid); } } else { if (flow.isworkerthread) flow.postMessage2({ TYPE: 'ui/message', clientid: clientid, data: msg }); else flow.proxy.message(msg, clientid); } }); if (flow.isworkerthread) return; flow.proxy.send = msg => socket.send(msg); }; function MAKEFLOWSTREAM(meta) { var flow = F.TFlowStream.create(meta.id, function(err, type, instance) { this.proxy.error(err, type, instance); }); var saveid = null; flow.cloning = meta.cloning != false; flow.export_instance2 = function(id) { var com = flow.meta.flow[id]; if (com) { if (id === 'paused' || id === 'groups' || id === 'tabs') return CLONE(com); var tmp = {}; tmp.id = id; tmp.config = CLONE(com.config); tmp.x = com.x; tmp.y = com.y; tmp.offset = com.offset; tmp.size = com.size; tmp.meta = com.meta; tmp.schemaid = com.schemaid; tmp.note = com.note; tmp.schema = com.schema; tmp.component = com.component; tmp.connections = CLONE(com.connections); tmp.tab = com.tab; if (com.outputs) tmp.outputs = com.outputs; if (com.inputs) tmp.inputs = com.inputs; var c = flow.meta.components[com.component]; if (c) { tmp.template = { type: c.type, icon: c.icon, color: c.color, group: c.group, name: c.name, inputs: c.inputs, outputs: c.outputs }; return tmp; } } }; function stringifyskip(key, value) { return key === '$$ID' || key === '$$REQUIRED' ? undefined : value; } flow.export2 = function() { var variables = flow.variables; var design = {}; var components = {}; var sources = flow.sources ? JSON.parse(JSON.stringify(flow.sources, stringifyskip)) : {}; for (let key in flow.meta.components) { let com = flow.meta.components[key]; components[key] = com.ui.raw; } for (let key in flow.meta.flow) { design[key] = flow.export_instance2(key); delete design[key].template; } var data = {}; var blacklist = { unixsocket: 1, components: 1, variables2: 1, sources: 1, design: 1, size: 1, directory: 1 }; for (let key in flow.$schema) { if (!blacklist[key]) data[key] = flow.$schema[key]; } data.paused = flow.paused; data.components = components; data.design = design; data.variables = variables; data.sources = sources; return data; }; var saveforce = function() { saveid && clearTimeout(saveid); saveid = null; if (!flow.$destroyed) flow.proxy.save(flow.export2()); }; var save = function() { // reloads TMS for (var key in flow.sockets) flow.sockets[key].synchronize(); if (flow.$schema && flow.$schema.readonly) return; clearTimeout(saveid); saveid = setTimeout(saveforce, 5000); }; flow.save = function() { save(); }; flow.remove = function() { flow.proxy.remove(); }; flow.kill = function(code) { flow.proxy.kill(code); }; flow.restart = function() { flow.proxy.restart(); }; var timeoutrefresh = null; var refresh_components_force = function() { timeoutrefresh = null; if (!flow.$destroyed && flow.proxy.online) { flow.proxy.send({ TYPE: 'flow/components', data: flow.components(true) }); let instances = flow.export(); flow.proxy.send({ TYPE: 'flow/design', data: instances }); } }; var refresh_components = function() { timeoutrefresh && clearTimeout(timeoutrefresh); timeoutrefresh = setTimeout(refresh_components_force, 700); }; flow.rpc = function(name, data, callback) { if (Parent) { va