process-rerun
Version:
The purpose of this library is - build simple and flexible interface for parallel command execution with rerun (on fail) possibility
147 lines • 7.75 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.buildExecRunner = buildExecRunner;
/* eslint-disable sonarjs/cognitive-complexity */
/** biome-ignore-all lint/suspicious/noConsole: <explanation> */
const sat_utils_1 = require("sat-utils");
const logger_execution_1 = require("./logger.execution");
const exec_1 = require("./exec");
const logger_1 = require("./logger");
const error_1 = require("./error");
function buildExecRunner(notRetriable, runOpts) {
const { currentExecutionVariable, longestProcessTime, processResultAnalyzer, execOpts = { maxBuffer: 1000 * 1024 }, pollTime, killGraceTime = 5000, successExitCode = 0, logProcessResult = logger_execution_1.internalLogProcessResult, onExitCloseProcess, onErrorProcess, onCache, } = runOpts;
if (!(0, sat_utils_1.isNumber)(successExitCode)) {
throw new error_1.ProcessRerunError('Type', 'successExitCode should be a number');
}
return (cmd, index) => new Promise(resolve => {
const onErrorCloseHandler = null;
const executionHolder = { stackTrace: '' };
/**
* @now this variable will be used for process kill if time more than @longestProcessTime
*/
const startTime = +Date.now();
/**
* kill escalation state:
* 0 - nothing sent yet
* 1 - SIGTERM sent (waiting for graceful exit)
* 2 - SIGKILL sent (waiting for forced exit)
* 3 - gave up, marked manually as killed (process may still be alive / unkillable)
*/
let killStage = 0;
let lastSignalAt = 0;
if (currentExecutionVariable) {
cmd = cmd.includes(currentExecutionVariable)
? cmd.replace(new RegExp(`${currentExecutionVariable}=\\d+`, 'ig'), `${currentExecutionVariable}=${index}`)
: `${currentExecutionVariable}=${index} ${cmd}`;
}
const execProc = (0, exec_1.execute)({ cmd, logProcessResult, executionHolder, execOpts });
/**
* Send a signal to the whole detached process group (negative pid) so that
* children spawned by the shell are reached too. Falls back to the direct
* process handle on win32 or when the group is gone.
*/
const signalProcess = (procWhatShouldBeKilled, signal) => {
try {
if (process.platform !== 'win32' && procWhatShouldBeKilled.pid) {
process.kill(-procWhatShouldBeKilled.pid, signal);
}
else {
procWhatShouldBeKilled.kill(signal);
}
}
catch (e) {
// ESRCH - process group already exited between the alive-check and the kill call
if (e.code !== 'ESRCH') {
console.error(e);
}
}
};
const killTooLongExecution = (procWhatShouldBeKilled) => {
// process already exited - nothing to do
if (procWhatShouldBeKilled.exitCode !== null || procWhatShouldBeKilled.signalCode !== null) {
return;
}
const executionTime = +Date.now() - startTime;
if (executionTime <= longestProcessTime) {
return;
}
const now = +Date.now();
// stage 0 -> 1: first timeout breach, ask politely with SIGTERM
if (killStage === 0) {
logger_1.logger.info(`Process killed due to long time execution: ${(0, sat_utils_1.millisecondsToMinutes)(executionTime)}\n${cmd}`);
signalProcess(procWhatShouldBeKilled, 'SIGTERM');
killStage = 1;
lastSignalAt = now;
return;
}
// stage 1 -> 2: SIGTERM ignored for the grace period, force with SIGKILL
if (killStage === 1 && now - lastSignalAt > killGraceTime) {
logger_1.logger.warn(`Process did not exit on SIGTERM, sending SIGKILL\n${cmd}`);
signalProcess(procWhatShouldBeKilled, 'SIGKILL');
killStage = 2;
lastSignalAt = now;
return;
}
// stage 2 -> 3: even SIGKILL did not reap it (uninterruptible sleep / re-parented
// session that escaped the group). Give up and mark as killed so the runner can
// proceed - the OS process may still be alive and now orphaned.
if (killStage === 2 && now - lastSignalAt > killGraceTime) {
procWhatShouldBeKilled.emit('exit', 100, 'PRO_RERUN_KILL');
procWhatShouldBeKilled.emit('close', 100, 'PRO_RERUN_KILL');
killStage = 3;
logger_1.logger.error(`Process survived SIGKILL and was marked manually as killed - it may still be running`);
}
};
const watcher = setInterval(() => killTooLongExecution(execProc), pollTime);
if (onExitCloseProcess) {
execProc.on('exit', (code, signal) => onExitCloseProcess(execProc, code, signal));
}
if (onErrorProcess) {
execProc.on('error', error => onErrorProcess(execProc, error));
}
execProc.on('close', async (code, signal) => {
if (onExitCloseProcess) {
onExitCloseProcess(execProc, code, signal);
}
// clear watcher interval
clearInterval(watcher);
// if the process published shared cache entries on its stdout - hand each to the
// executor; empty the buffer so a re-emitted `close` (manual kill path) can't
// deliver the same entries twice
if (Array.isArray(executionHolder.cache) && executionHolder.cache.length && (0, sat_utils_1.isFunction)(onCache)) {
executionHolder.cache.forEach(entry => onCache(entry));
executionHolder.cache = [];
}
if (onErrorCloseHandler && ((0, sat_utils_1.isFunction)(onErrorCloseHandler) || (0, sat_utils_1.isAsyncFunction)(onErrorCloseHandler))) {
await onErrorCloseHandler();
}
// if process code 0 - exit as a success result
if (code === successExitCode) {
return resolve(null);
}
// processResultAnalyzer - check that stack contains or not contains some specific data
if ((0, sat_utils_1.isFunction)(processResultAnalyzer)) {
const countInNotRetriableBeforeAnalyzation = notRetriable.length;
const analyzationResult = processResultAnalyzer(cmd, executionHolder.stackTrace, notRetriable);
if (!(0, sat_utils_1.isString)(analyzationResult) && !(0, sat_utils_1.isBoolean)(analyzationResult)) {
logger_1.logger.warn('processResultAnalyzer should return boolean or string');
}
// if analyzationResult is string - we want to re-execute command
if ((0, sat_utils_1.isString)(analyzationResult)) {
return resolve(analyzationResult);
}
// if analyzationResult is true - we make assumption that process was finished successfully
if ((0, sat_utils_1.isBoolean)(analyzationResult) && analyzationResult) {
return resolve(null);
}
if (countInNotRetriableBeforeAnalyzation === notRetriable.length ||
((0, sat_utils_1.isBoolean)(analyzationResult) && !analyzationResult)) {
notRetriable.push(cmd);
}
return resolve(null);
}
return resolve(cmd);
});
});
}
//# sourceMappingURL=exec.proc.js.map