pipeproc/lib/node/pipeProc.js

Multi-process log processing for nodejs

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const debug_1 = __importDefault(require("debug"));
const path_1 = require("path");
const leveldown_1 = __importDefault(require("leveldown"));
const memdown_1 = __importDefault(require("memdown"));
const messages_1 = require("../common/messages");
const commitLog_1 = require("./commitLog");
const restoreState_1 = require("./restoreState");
const shutdown_1 = require("./shutdown");
const getRange_1 = require("./getRange");
const proc_1 = require("./proc");
const systemProc_1 = require("./systemProc");
const workerManager_1 = require("./workerManager");
const ack_1 = require("./ack");
const ackCommitLog_1 = require("./ackCommitLog");
const destroyProc_1 = require("./destroyProc");
const messaging_1 = require("./messaging");
const writeBuffer_1 = require("./writeBuffer");
const resumeDisableProc_1 = require("./resumeDisableProc");
const reclaimProc_1 = require("./reclaimProc");
const collect_1 = require("./gc/collect");
const waitForProcs_1 = require("./waitForProcs");
const tones_1 = require("./tones");
const d = debug_1.default("pipeproc:node");
let db;
let connectionAddress;
let serverSocket;
let clientTLS;
const activeTopics = {};
const systemState = { active: false };
const messageRegistry = {};
const writeBuffer = [];
let stopWriteBuffer;
const activeProcs = [];
const activeSystemProcs = [];
const activeWorkers = [];
let gcInterval;
messaging_1.registerMessage(messageRegistry, {
    messageType: "system_init",
    replySuccess: "system_ready",
    replyError: "system_ready_error",
    writeOp: true,
    listener: function (data, callback) {
        if (systemState.active) {
            return callback("system_already_active");
        }
        d("starting up...");
        if (data.options.memory) {
            d("using in-memory adapter");
            db = memdown_1.default();
        }
        else {
            d("using disk adapter");
            if (data.options.location) {
                let location;
                if (path_1.isAbsolute(data.options.location)) {
                    location = data.options.location;
                }
                else {
                    location = path_1.resolve(data.options.location);
                }
                d("data location:", location);
                db = leveldown_1.default(location);
            }
            else {
                db = leveldown_1.default("./pipeproc");
            }
        }
        restoreState_1.restoreState(db, activeTopics, systemState, activeProcs, activeSystemProcs, data.options.memory, function (err) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                workerManager_1.spawnWorkers(data.options.workers, activeWorkers, activeProcs, activeSystemProcs, connectionAddress, clientTLS, data.options.workerConcurrency, data.options.workerRestartAfter, function (spawnErr) {
                    if (err) {
                        callback((spawnErr && spawnErr.message) || "uknown_error");
                    }
                    else {
                        if (data.options.gc) {
                            d("gc is enabled");
                            const MIN_PRUNE_TIME = (data.options.gc &&
                                data.options.gc.minPruneTime) || 30000;
                            const GC_INTERVAL = (data.options.gc &&
                                data.options.gc.interval) || 30000;
                            let gcRunning = false;
                            gcInterval = setInterval(function () {
                                if (gcRunning)
                                    return;
                                d("gc will start running...");
                                gcRunning = true;
                                collect_1.collect(db, activeTopics, activeProcs, { minPruneTime: MIN_PRUNE_TIME }, function (gcErr) {
                                    if (gcErr) {
                                        d("gc error:", gcErr);
                                    }
                                    gcRunning = false;
                                    d("gc ended");
                                });
                            }, GC_INTERVAL);
                        }
                        else {
                            d("gc is disabled");
                        }
                        callback();
                    }
                });
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "ping",
    replySuccess: "pong",
    replyError: "ping_error",
    writeOp: false,
    listener: function (_data, callback) {
        callback();
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "wait_for_procs",
    replySuccess: "procs_completed",
    replyError: "procs_completed_error",
    writeOp: false,
    listener: function (data, callback) {
        d("waiting for procs...");
        waitForProcs_1.waitForProcs(activeTopics, activeProcs, data.procs, function () {
            callback();
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "commit",
    replySuccess: "commit_completed",
    replyError: "commit_error",
    writeOp: true,
    listener: function (data, callback) {
        commitLog_1.commitLog(db, activeTopics, data.commitLog, function (err, id) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                if (id && Array.isArray(id)) {
                    callback(null, {
                        id: id.map(tones_1.convertToClientId)
                    });
                }
                else if (id) {
                    callback(null, { id: tones_1.convertToClientId(id) });
                }
                else {
                    callback("uncommited");
                }
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "get_range",
    replySuccess: "range_reply",
    replyError: "range_error",
    writeOp: false,
    listener: function (data, callback) {
        getRange_1.getRange(db, activeTopics, data.topic, tones_1.convertRangeParams(data.options.start), tones_1.convertRangeParams(data.options.end), data.options.limit, data.options.exclusive, data.options.reverse, function (err, results) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                if (results) {
                    callback(null, {
                        results: results.map(function (re) {
                            return {
                                id: tones_1.convertToClientId(re.id),
                                data: re.data
                            };
                        })
                    });
                }
                else {
                    callback();
                }
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "proc",
    replySuccess: "proc_ok",
    replyError: "proc_error",
    writeOp: true,
    listener: function (data, callback) {
        proc_1.proc(db, activeProcs, activeTopics, data.options, function (err, log) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                if (log) {
                    if (Array.isArray(log)) {
                        callback(null, log.map(function (l) {
                            return {
                                id: tones_1.convertToClientId(l.id),
                                data: l.data
                            };
                        }));
                    }
                    else {
                        callback(null, {
                            id: tones_1.convertToClientId(log.id),
                            data: log.data
                        });
                    }
                }
                else {
                    callback();
                }
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "available_proc",
    replySuccess: "available_proc_ok",
    replyError: "available_proc_error",
    writeOp: true,
    listener: function (data, callback) {
        proc_1.getAvailableProc(db, activeProcs, activeTopics, data.procList, function (err, result) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                if (result) {
                    let log;
                    if (result.log) {
                        if (Array.isArray(result.log)) {
                            log = result.log.map(function (l) {
                                return {
                                    id: tones_1.convertToClientId(l.id),
                                    data: l.data
                                };
                            });
                        }
                        else {
                            log = {
                                id: tones_1.convertToClientId(result.log.id),
                                data: result.log.data
                            };
                        }
                    }
                    callback(null, {
                        procName: result.procName,
                        log: log
                    });
                }
                else {
                    callback();
                }
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "system_proc",
    replySuccess: "system_proc_ok",
    replyError: "system_proc_error",
    writeOp: true,
    listener: function (data, callback) {
        systemProc_1.systemProc(db, activeProcs, activeSystemProcs, activeWorkers, data.options, function (err, theProc) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                if (Array.isArray(theProc)) {
                    callback(null, {
                        proc: theProc.map(function (p) {
                            const myProc = JSON.parse(JSON.stringify(p));
                            myProc.lastAckedRange = tones_1.convertToClientId(myProc.lastAckedRange);
                            myProc.lastClaimedRange = tones_1.convertToClientId(myProc.lastClaimedRange);
                            myProc.previousClaimedRange = tones_1.convertToClientId(myProc.previousClaimedRange);
                            return myProc;
                        })
                    });
                }
                else if (theProc) {
                    const myProc = JSON.parse(JSON.stringify(theProc));
                    myProc.lastAckedRange = tones_1.convertToClientId(myProc.lastAckedRange);
                    myProc.lastClaimedRange = tones_1.convertToClientId(myProc.lastClaimedRange);
                    myProc.previousClaimedRange = tones_1.convertToClientId(myProc.previousClaimedRange);
                    callback(null, { proc: myProc });
                }
                else {
                    callback();
                }
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "ack",
    replySuccess: "ack_ok",
    replyError: "ack_error",
    writeOp: true,
    listener: function (data, callback) {
        ack_1.ack(db, activeProcs, data.procName, function (err, ackedLogId) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                if (ackedLogId) {
                    callback(null, { id: tones_1.convertToClientId(ackedLogId) });
                }
                else {
                    callback("invalid_ack");
                }
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "ack_commit",
    replySuccess: "ack_commit_completed",
    replyError: "ack_commit_error",
    writeOp: true,
    listener: function (data, callback) {
        ackCommitLog_1.ackCommitLog(db, activeTopics, activeProcs, data.procName, data.commitLog, function (err, status) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                let ackedLogId = status[0];
                let commitedId = status[1];
                if (status && ackedLogId && commitedId) {
                    ackedLogId = tones_1.convertToClientId(ackedLogId);
                    if (Array.isArray(commitedId)) {
                        commitedId = commitedId.map(tones_1.convertToClientId);
                    }
                    else {
                        commitedId = tones_1.convertToClientId(commitedId);
                    }
                    callback(null, { ackedLogId: ackedLogId, id: commitedId });
                }
                else {
                    callback("invalid_ack_or_commit");
                }
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "inspect_proc",
    replySuccess: "inspect_proc_reply",
    replyError: "inspect_proc_error",
    writeOp: false,
    listener: function (data, callback) {
        const theProc = activeProcs.find(p => p.name === data.procName);
        d("proc inspection request:", data.procName);
        if (theProc) {
            const myProc = JSON.parse(JSON.stringify(theProc));
            myProc.lastAckedRange = tones_1.convertToClientId(myProc.lastAckedRange);
            myProc.lastClaimedRange = tones_1.convertToClientId(myProc.lastClaimedRange);
            myProc.previousClaimedRange = tones_1.convertToClientId(myProc.previousClaimedRange);
            callback(null, { proc: myProc });
        }
        else {
            callback("invalid_proc");
        }
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "destroy_proc",
    replySuccess: "destroy_proc_ok",
    replyError: "destroy_proc_error",
    writeOp: true,
    listener: function (data, callback) {
        destroyProc_1.destroyProc(db, activeProcs, data.procName, function (err, theProc) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                if (theProc) {
                    const myProc = JSON.parse(JSON.stringify(theProc));
                    myProc.lastAckedRange = tones_1.convertToClientId(myProc.lastAckedRange);
                    myProc.lastClaimedRange = tones_1.convertToClientId(myProc.lastClaimedRange);
                    myProc.previousClaimedRange = tones_1.convertToClientId(myProc.previousClaimedRange);
                    callback(null, { proc: myProc });
                }
                else {
                    callback("invalid_proc");
                }
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "disable_proc",
    replySuccess: "disable_proc_ok",
    replyError: "disable_proc_error",
    writeOp: true,
    listener: function (data, callback) {
        resumeDisableProc_1.disableProc(db, activeProcs, data.procName, function (err, theProc) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                if (theProc) {
                    const myProc = JSON.parse(JSON.stringify(theProc));
                    myProc.lastAckedRange = tones_1.convertToClientId(myProc.lastAckedRange);
                    myProc.lastClaimedRange = tones_1.convertToClientId(myProc.lastClaimedRange);
                    myProc.previousClaimedRange = tones_1.convertToClientId(myProc.previousClaimedRange);
                    callback(null, { proc: myProc });
                }
                else {
                    callback("invalid_proc");
                }
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "resume_proc",
    replySuccess: "resume_proc_ok",
    replyError: "resume_proc_error",
    writeOp: true,
    listener: function (data, callback) {
        resumeDisableProc_1.resumeProc(db, activeProcs, data.procName, function (err, theProc) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                if (theProc) {
                    const myProc = JSON.parse(JSON.stringify(theProc));
                    myProc.lastAckedRange = tones_1.convertToClientId(myProc.lastAckedRange);
                    myProc.lastClaimedRange = tones_1.convertToClientId(myProc.lastClaimedRange);
                    myProc.previousClaimedRange = tones_1.convertToClientId(myProc.previousClaimedRange);
                    callback(null, { proc: myProc });
                }
                else {
                    callback("invalid_proc");
                }
            }
        });
    }
});
messaging_1.registerMessage(messageRegistry, {
    messageType: "reclaim_proc",
    replySuccess: "reclaim_proc_ok",
    replyError: "reclaim_proc_error",
    writeOp: true,
    listener: function (data, callback) {
        reclaimProc_1.reclaimProc(db, activeProcs, data.procName, function (err, lastClaimedRange) {
            if (err) {
                callback((err && err.message) || "uknown_error");
            }
            else {
                callback(null, {
                    lastClaimedRange: lastClaimedRange ? tones_1.convertToClientId(lastClaimedRange) : ""
                });
            }
        });
    }
});
const initIPCListener = function (e) {
    if (e.type === "init_ipc") {
        process.removeListener("message", initIPCListener);
        connectionAddress = e.data.address;
        clientTLS = e.data.tls && e.data.tls.client;
        messaging_1.initializeMessages(writeBuffer, messageRegistry, connectionAddress, e.data.tls && e.data.tls.server, function (err, socket) {
            if (err) {
                if (process && typeof process.send === "function") {
                    process.send(messages_1.prepareMessage({
                        type: "ipc_established",
                        msgKey: e.msgKey,
                        errStatus: err.message
                    }));
                }
            }
            else {
                serverSocket = socket;
                if (process && typeof process.send === "function") {
                    process.send(messages_1.prepareMessage({ type: "ipc_established", msgKey: e.msgKey }));
                }
            }
        });
    }
};
const shutdownListener = function (e) {
    if (e.type === "system_shutdown") {
        d("shutting down...");
        stopWriteBuffer(function () {
            if (serverSocket) {
                serverSocket.close();
            }
            if (gcInterval) {
                d("stopping gc");
                clearInterval(gcInterval);
            }
            process.removeListener("message", shutdownListener);
            shutdown_1.runShutdownHooks(db, systemState, activeWorkers, function (err) {
                if (err) {
                    if (process && typeof process.send === "function") {
                        process.send(messages_1.prepareMessage({
                            type: "system_closed_error",
                            msgKey: e.msgKey,
                            errStatus: (err && err.message) || "uknown_error"
                        }));
                    }
                }
                else {
                    if (process && typeof process.send === "function") {
                        process.send(messages_1.prepareMessage({
                            type: "system_closed",
                            msgKey: e.msgKey
                        }));
                    }
                }
            });
        });
    }
};
process.on("message", initIPCListener);
process.on("message", shutdownListener);
stopWriteBuffer = writeBuffer_1.startWriteBuffer(writeBuffer);