pipeproc
Version:
Multi-process log processing for nodejs
263 lines (262 loc) • 9.64 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const debug_1 = __importDefault(require("debug"));
const proc_1 = require("./proc");
const messaging_1 = require("./messaging");
const messages_1 = require("../common/messages");
const async_1 = require("async");
const transaction_1 = require("./transaction");
const d = debug_1.default("pipeproc:node");
function systemProc(db, activeProcs, activeSystemProcs, activeWorkers, options, callback) {
d("starting system proc creation...");
let procsCreated = [];
let systemProcsCreated = [];
async_1.series([
function (cb) {
if (!options.inlineProcessor && !options.externalProcessor) {
return callback(new Error("no_valid_processor_provided"));
}
validateProcs(options.name, options.from, activeProcs, function (err, procs) {
if (err) {
callback(err);
}
else if (procs) {
if (Array.isArray(procs)) {
if (procs.length === options.from.length) {
callback(null, procs);
}
else {
options.from = options.from.filter(function (topic) {
return procs.filter(function (p) {
return p.name !== `${options.name}__${topic}`;
}).length === 0;
});
cb();
}
}
else {
callback(null, procs);
}
}
else {
cb();
}
});
},
function (cb) {
createSystemProcs(db, options).commitUpdate(function (err, procs, systemProcs) {
if (err)
return callback(err);
if (Array.isArray(procs) && procs.length > 0 &&
Array.isArray(systemProcs) && systemProcs.length > 0) {
procsCreated = procs;
systemProcsCreated = systemProcs;
systemProcsCreated.forEach(sp => activeSystemProcs.push(sp));
procsCreated.forEach(p => activeProcs.push(p));
cb();
}
else if (procs && systemProcs && !Array.isArray(procs) && !Array.isArray(systemProcs)) {
procsCreated.push(procs);
systemProcsCreated.push(systemProcs);
activeSystemProcs.push(systemProcs);
activeProcs.push(procs);
cb();
}
else {
callback(new Error("proc_creation_failed"));
}
});
},
function (cb) {
async_1.eachOfSeries(activeWorkers, function (worker, i, next) {
d("sending systemProc to worker:", i);
const msg = messages_1.prepareRegisterSystemProcsMessage(procsCreated, systemProcsCreated);
const listener = function (e) {
if (e.msgKey === msg.msgKey) {
worker.process.removeListener("message", listener);
if (e.type === "register_system_proc_ok") {
next();
}
else {
d("Worker Error:", e.errStatus);
next(new Error(e.errStatus || "register_system_proc_uknown_error"));
}
}
};
worker.process.on("message", listener);
messaging_1.sendMessageToWorker(msg, worker);
}, cb);
}
], function () {
callback(null, (procsCreated.length === 1 ? procsCreated[0] : procsCreated));
});
}
exports.systemProc = systemProc;
function createSystemProcs(db, options) {
const tx = transaction_1.transaction(db);
d("creating...", options.from);
if (Array.isArray(options.from)) {
options.from.forEach(function (topic) {
const procName = `${options.name}__${topic}`;
d("creating", procName);
const systemProcPrefix = `~~system~~#systemProc#${topic}#${procName}#`;
tx.add(proc_1.createProc(db, {
name: procName,
topic: topic,
offset: options.offset,
count: options.count,
maxReclaims: options.maxReclaims,
reclaimTimeout: options.reclaimTimeout,
onMaxReclaimsReached: options.onMaxReclaimsReached
}));
tx.add({
key: `${systemProcPrefix}name`,
value: procName
});
if (options.inlineProcessor) {
tx.add({
key: `${systemProcPrefix}inlineProcessor`,
value: options.inlineProcessor
});
}
else if (options.externalProcessor) {
tx.add({
key: `${systemProcPrefix}externalProcessor`,
value: options.externalProcessor
});
}
let to = [];
if (Array.isArray(options.to)) {
to = options.to;
}
else {
to.push(options.to);
}
to.forEach(function (toTopic, i) {
tx.add({
key: `${systemProcPrefix}to#${i}`,
value: toTopic
});
});
tx.done(function () {
const result = {
name: procName,
topic: topic,
to: options.to
};
if (options.inlineProcessor) {
result.inlineProcessor = options.inlineProcessor;
}
else if (options.externalProcessor) {
result.externalProcessor = options.externalProcessor;
}
return result;
}, "systemProcs");
});
}
else {
const procName = options.name;
d("creating", procName);
const systemProcPrefix = `~~system~~#systemProc#${options.from}#${procName}#`;
tx.add(proc_1.createProc(db, {
name: procName,
topic: options.from,
offset: options.offset,
count: options.count,
maxReclaims: options.maxReclaims,
reclaimTimeout: options.reclaimTimeout,
onMaxReclaimsReached: options.onMaxReclaimsReached
}));
tx.add({
key: `${systemProcPrefix}name`,
value: procName
});
if (options.inlineProcessor) {
tx.add({
key: `${systemProcPrefix}inlineProcessor`,
value: options.inlineProcessor
});
}
else if (options.externalProcessor) {
tx.add({
key: `${systemProcPrefix}externalProcessor`,
value: options.externalProcessor
});
}
let to = [];
if (Array.isArray(options.to)) {
to = options.to;
}
else {
to.push(options.to);
}
to.forEach(function (toTopic, i) {
tx.add({
key: `${systemProcPrefix}to#${i}`,
value: toTopic
});
});
tx.done(function () {
const result = {
name: procName,
topic: options.from,
to: options.to
};
if (options.inlineProcessor) {
result.inlineProcessor = options.inlineProcessor;
}
else if (options.externalProcessor) {
result.externalProcessor = options.externalProcessor;
}
return result;
}, "systemProcs");
}
return tx;
}
function validateProcs(name, from, activeProcs, callback) {
d("validating...");
if (Array.isArray(from)) {
let valid = true;
const procs = [];
from.forEach(function (topic) {
const procName = `${name}__${topic}`;
const myProc = activeProcs.find(p => p.name === procName);
if (myProc && myProc.topic !== topic) {
d(procName, "NOT_UNIQUE");
valid = false;
}
else if (myProc && myProc.topic === topic) {
d(procName, "OK_EXISTS");
procs.push(myProc);
}
else {
d(procName, "OK_NOT_EXISTS");
}
});
if (valid) {
callback(null, procs);
}
else {
callback(new Error("proc_name_not_unique"));
}
}
else {
const myProc = activeProcs.find(p => p.name === name);
if (myProc && myProc.topic !== from) {
d(name, "NOT_UNIQUE");
callback(new Error("proc_name_not_unique"));
}
else if (myProc && myProc.topic === from) {
d(name, "OK_EXISTS");
callback(null, myProc);
}
else {
d(name, "OK_NOT_EXISTS");
callback();
}
}
}
exports.validateProcs = validateProcs;