domotz-remote-pawn/src/external_processes/stallWatcher.js

Domotz Agent

/** This file is part of Domotz Agent.
 * Copyright (C) 2016  Domotz Ltd
 *
 * Domotz Agent is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Domotz Agent is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Domotz Agent.  If not, see <http://www.gnu.org/licenses/>.

 * Created by Iacopo Papalini <iacopo@domotz.com> on 07/12/16.
 *
 * This module spawns a process that checks periodically the responsiveness
 * of the Domotz Agent and kills it if is not responding over HTTP over socket
 * requests in a sensible time.
 *
 * To be sure that only one instance of watcher is running we need:
 *
 *              gracePeriod > pollPeriod
 */

var MODULE = 'DOMOTZ_STALL_WATCHER';

module.exports.init = function (resourceLocator) {
    var env = process.env;
    var os = resourceLocator.os;
    var path = resourceLocator.path;
    var fs = resourceLocator.fs;
    var childProcess = resourceLocator.utils.childProcess;
    var watcherConf = resourceLocator.settingsManager.getStallWatcherSettings();
    var pollPeriod = parseInt(env.DOMOTZ_POLL_PERIOD || 60);
    var retryPeriod = parseInt(env.DOMOTZ_RETRY_PERIOD || 10);
    var retryNumber = parseInt(env.DOMOTZ_RETRY_NUMBER || 11);
    var gracePeriod = parseInt(env.DOMOTZ_GRACE_PERIOD || 120) * 1000;
    var flushLogFailure = 6;
    var myConsole = resourceLocator.log.decorateLogs();

    var fileName = path.join(os.tmpdir(), MODULE.toLocaleLowerCase() + '.sh');
    var pidFileName = path.join(os.tmpdir(), MODULE.toLocaleLowerCase() + '.pid');
    var logFileName = path.join(env.DOMOTZ_LOG_DIR, MODULE.toLocaleLowerCase() + '.log');

    var terminateStallWatcher = function () {
        if (fs.existsSync(pidFileName)) {
            var pid = parseInt(fs.readFileSync(pidFileName, 'utf8').trim(), 10);
            if (pid && !isNaN(pid)) {
                try {
                    process.kill(pid, 'SIGTERM');
                    myConsole.info('Stall watcher with PID %s killed', pid);
                } catch (e) {
                    myConsole.error('Error killing stall watcher with PID %s: %s', pid, e.message);
                }
            } else {
                myConsole.warn('Invalid PID in file %s', pidFileName);
            }
            fs.unlinkSync(pidFileName);
        }
    };

    var contents =
        '#!/bin/sh\n' +
        'LOG_FILE=' +
        logFileName +
        '\n' +
        'period=' +
        pollPeriod +
        '\n' +
        'parent_pid=' +
        process.pid +
        '\n' +
        'failure=0\n' +
        'echo `date` "$0 start with pid $$ to monitor listener with pid $parent_pid." >> $LOG_FILE\n' +
        'while true\n' +
        'do\n' +
        '\tsleep ${period}\n' +
        '\tcurrent_listener_pid=$(cat ' +
        env.LISTENER_PID_FILE +
        ')\n' +
        '\tif [ $current_listener_pid -ne $parent_pid ]; then\n' +
        '\t\techo `date` "Parent listener is died [pid: $parent_pid]. Current instance is running with pid $current_listener_pid. Exit..." >> $LOG_FILE\n' +
        '\t\texit 0\n' +
        '\tfi\n' +
        '\tdomotz_curl --fail --silent  -X POST -A domotz-zabbix-proxy -d \'{"action": "listener_status_check"}\' --unix-socket ' +
        env.DOMOTZ_HTTP_ON_UNIX_SOCKET_FILE +
        ' --max-time ' +
        watcherConf.timeout +
        ' http://localhost\n' +
        '\tif [ $? -ne 0 ]; then\n' +
        '\t\tfailure=$((failure+1))\n' +
        '\t\tif [ $failure -eq ' +
        retryNumber +
        ' ]; then\n' +
        '\t\t\techo `date` "Killing not responding listener process; pid: $parent_pid" >> $LOG_FILE\n' +
        '\t\t\tkill -9 $parent_pid\n' +
        '\t\t\texit 1\n' +
        '\t\telse\n' +
        '\t\t\techo `date` "Not responding listener process; pid: $parent_pid - tentative $failure" >> $LOG_FILE\n' +
        '\t\t\tif [ $failure -eq ' +
        flushLogFailure +
        ' ]; then\n' +
        '\t\t\t\tkill -1 $parent_pid\n' +
        '\t\t\tfi\n' +
        '\t\t\tperiod=' +
        retryPeriod +
        '\n' +
        '\t\tfi\n' +
        '\telse\n' +
        '\t\tperiod=' +
        pollPeriod +
        '\n' +
        '\t\tfailure=0\n' +
        '\tfi\n' +
        'done\n';
    return {
        start: function () {
            'use strict';
            if (process.platform === 'win32') {
                myConsole.info('Stall watcher disabled on windows');
                return;
            }
            if (!watcherConf.enabled) {
                myConsole.info('Stall watcher disabled by settings');
                return;
            }
            terminateStallWatcher();
            myConsole.info(
                'Start process: [%s]. Grace Time: %d s. ' + 'Poll Period: %d s. Retry Period: %d s. #Retry: %d',
                fileName,
                gracePeriod / 1000,
                pollPeriod,
                retryPeriod,
                retryNumber
            );
            setTimeout(function () {
                fs.writeFileSync(fileName, contents, { mode: '0700' });
                var watcher = childProcess.spawn(fileName, [], {
                    env: process.env,
                });
                if (watcher.pid) {
                    myConsole.info('Stall watcher started with PID %s', watcher.pid);
                }
                fs.writeFileSync(pidFileName, watcher.pid.toString(), { mode: '0644' });

                watcher.on('error', function (err) {
                    myConsole.error('Unable to start stall watcher %s', err.toString());
                });
                watcher.on('exit', function (code) {
                    if (code !== 0) {
                        myConsole.error('Exit with code %s for stall watcher.', code);
                    } else {
                        myConsole.warn('Graceful exit for stall watcher.');
                    }
                    process.exit(0);
                });
            }, gracePeriod);
        },
        kill: function () {
            terminateStallWatcher();
        },
    };
};