domotz-remote-pawn/src/discovery/ip_network/big_network.js

Domotz Agent

/** This file is part of Domotz Agent.
 * Copyright (C) 2021  Domotz Ltd
 *
 * Domotz Agent is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Domotz Agent is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Domotz Agent.  If not, see <http://www.gnu.org/licenses/>.
 **/
const MIN_NETMASK_SIZE = 16;
const SPAWN_HARD_TIMEOUT = 5.4e6; // 90 minutes - used for killing the nmap process if it does not finish in 1.5h
const UP_REGEXP = /Host: ([0-9\\.]+) .+Status: Up/;

var baseConsole = null;

function NetworkScanner(iface, resourceLocator) {
    var async = resourceLocator.async;
    var minNetmask;
    const settingsManager = resourceLocator.settingsManager;
    const lodash = resourceLocator.lodash;
    if (!settingsManager.isBigNetworksEnabled()) {
        minNetmask = settingsManager.getDiscoverySettings().max_ip_netmask; // 22
    } else {
        minNetmask = Math.max(parseInt(settingsManager.bigNetworksBitmask(), 10), MIN_NETMASK_SIZE);
    }

    if (baseConsole === null) {
        baseConsole = resourceLocator.log.decorateLogs();
    }
    const ipWithNetmask = iface.ip + '/' + Math.max(parseInt(iface.netmask, 10), minNetmask);

    const myConsole = resourceLocator.log.decorateLogs(ipWithNetmask, baseConsole);
    var childProcessModule = resourceLocator.utils.childProcess;
    var processKiller = resourceLocator.utils.stalledProcessesKiller;
    var startTime = null;
    var stdErr = null;
    var iteration = 0;
    var discoveredHosts = [];

    var processes = {};
    var processesKillTimers = {};
    var bigNetworksPerformanceParams = resourceLocator.settingsManager.getBigNetworksPerformanceParams();
    var limit = bigNetworksPerformanceParams.parallel_executions;
    var maxSubnetworkMask = bigNetworksPerformanceParams.max_subnetwork_mask;

    function reportStatus() {
        if (isScanning()) {
            myConsole.info(
                'Scan in progress with pid %s - iteration number: %s, hosts discovered so far: %s, time elapsed: %ss',
                process.pid,
                iteration,
                discoveredHosts.length,
                (new Date() - startTime) / 1000
            );
        } else {
            myConsole.warn('Scan is stopped');
        }
    }

    function reportIterationEnd() {
        var elapsed = (new Date() - startTime) / 1000;
        myConsole.info('Iteration %s finished - %s hosts discovered in %s seconds', iteration, discoveredHosts.length, elapsed);
    }

    function isScanning() {
        return lodash.some(processes);
    }

    function parseOutput(output) {
        var lines = output.toString().trim().split('\n');
        lines.forEach(function (line) {
            myConsole.debug('Processing nmap line `%s`', line);
            var matches = line.match(UP_REGEXP);
            if (matches) {
                var ip = matches[1];
                myConsole.info('Adding or refreshing ip %s', ip);
                resourceLocator.discovery.host.addIpToFullList(ip);
                discoveredHosts.push(ip);
            }
        });
    }

    function registerProcess(process_, ipWithNetmask, idx) {
        if (processes[idx]) {
            throw new Error('Process for scanning network ' + ipWithNetmask + ' already existing: ' + process_.pid);
        }
        processes[idx] = process_;
        processKiller.addLongRunningProcess(process_.pid, 'BigScan for ' + ipWithNetmask);
    }

    function scanSubnet(cmd, idx, callback) {
        myConsole.debug('Starting command #%s/%s %s', idx, maxSubnetworkMask, cmd);
        var callbackCalled = false;

        function unregisterProcess() {
            processKiller.removeLongRunningProcess(childProcess.pid);
            delete processes[idx];
        }

        var tokens = cmd.trim().split(' ');
        if (processes[idx]) {
            myConsole.error('Process with id %s is still running, waiting for automatic kill after timeout');
            if (!callbackCalled) {
                callbackCalled = true;
                callback();
            }
            return;
        }

        var childProcess = childProcessModule.spawn(tokens[0], tokens.slice(1));
        registerProcess(childProcess, ipWithNetmask, idx);

        childProcess.stdout.on('data', parseOutput);
        childProcess.stderr.on('data', parseErrorOutput);
        childProcess.on('error', function (err) {
            myConsole.warn('Process with pid:%s exited with unexpected error', childProcess.pid, err);
            if (!callbackCalled) {
                callback();
                callbackCalled = true;
            }
        });

        function onProcessClose(code) {
            var duration = (new Date() - startTime) / 1000;
            myConsole.info('Command `%s` (pid: %s) exited with code %s after %ss', cmd, childProcess.pid, code, duration);
            if (code !== 0) {
                myConsole.warn('StdErr: %s', stdErr);
            }
            unregisterProcess();
            clearTimeout(processesKillTimers[idx]);
            delete processesKillTimers[idx];

            if (!callbackCalled) {
                callback();
                callbackCalled = true;
            }
        }

        childProcess.on('close', onProcessClose);

        processesKillTimers[idx] = setTimeout(function killProcess() {
            if (childProcess) {
                myConsole.warn('Killing process %s after %ss - nmap might have hanged', childProcess.pid, SPAWN_HARD_TIMEOUT / 1000);
                childProcess.kill();
            }
        }, SPAWN_HARD_TIMEOUT);
    }

    function execute() {
        if (resourceLocator.emergencyNetworkRestore.isEmergencyInterfaceEnabled()) {
            myConsole.info('Big Network Discovery skipped for emergency interface');
            return;
        }

        var functions = [];
        const commandsList = resourceLocator.networkTools.nmapCommands.getScanBigNetworksNmapCmd(ipWithNetmask);

        commandsList.forEach(function (cmd, idx) {
            functions.push(scanSubnet.bind(null, cmd, idx));
        });

        myConsole.info('Starting %s commands, max parallel instances %s', commandsList.length, limit);

        async.parallelLimit(functions, limit, function () {
            reportIterationEnd();

            myConsole.info('Waiting %ss before next iteration', bigNetworksPerformanceParams.cool_down_before_restart / 1000);
            setTimeout(function () {
                myConsole.info('Restarting BigScan for network: %s', ipWithNetmask);
                startScanProcess();
            }, bigNetworksPerformanceParams.cool_down_before_restart);
        });
    }

    function parseErrorOutput(output) {
        stdErr += output.toString();
    }

    function startScanProcess() {
        if (!settingsManager.isBigNetworksEnabled()) {
            myConsole.info('BigNetworks not enabled for this agent');
            return;
        }
        if (settingsManager.isBroadcastDiscoveryDisabled()) {
            myConsole.info('Broadcast Discovery disabled, skip');
            return;
        }

        stdErr = '';
        iteration++;
        discoveredHosts = [];

        try {
            startTime = new Date();
            execute();
        } catch (err) {
            myConsole.warn('Process for network %s exited with unexpected error: %s', ipWithNetmask, String(err));
        }
    }

    setInterval(reportStatus, 30000);
    return {
        startScanProcess: startScanProcess,
        isScanning: isScanning,
        network: function () {
            return ipWithNetmask;
        },
        // Testing
        parseOutput_: parseOutput,
        execute_: execute,
        scanSubnet_: scanSubnet,
    };
}

module.exports.BigNetworkScannerFactory = function (resourceLocator) {
    var myConsole = resourceLocator.log.decorateLogs();
    const settingsManager = resourceLocator.settingsManager;
    var maxNetmask = settingsManager.getDiscoverySettings().max_ip_netmask - 1;
    var scanners = {};

    function isBigNetwork(iface) {
        var netmask = iface.netmask;
        if (!netmask) {
            return false;
        }
        if (netmask > maxNetmask) {
            return false;
        }
        if (!settingsManager.isBigNetworksEnabled()) {
            myConsole.info('BigNetworks scan disabled from settings for interface %s/%s', iface.ip, iface.netmask);
            return false;
        }
        return true;
    }

    function getScanner(iface, ifaceName) {
        if (scanners[ifaceName] === undefined) {
            scanners[ifaceName] = NetworkScanner(iface, resourceLocator);
        }
        return scanners[ifaceName];
    }

    return {
        keepScanning: function (iface, ifaceName) {
            var scanner = getScanner(iface, ifaceName);
            if (scanner.isScanning()) {
                return;
            }
            var ip = iface.ip;
            var netmask = iface.netmask;
            var ipWithNetmask = ip + '/' + netmask;
            if (!isBigNetwork(iface)) {
                myConsole.warn('Network %s is not between %s and %s', ipWithNetmask, MIN_NETMASK_SIZE, maxNetmask);
                return;
            }

            myConsole.info('Performing BigNetwork scan on network %s', ipWithNetmask);
            scanner.startScanProcess();
        },
        isScanning: function (iface, ifaceName) {
            return getScanner(iface, ifaceName).isScanning();
        },
        isBigNetwork: isBigNetwork,
    };
};

module.exports._NetworkScanner = NetworkScanner; // for testing