domotz-remote-pawn/src/discovery/snmp/snmp_overlay_discovery.js

Domotz Agent

/** This file is part of Domotz Agent.
 * Copyright (C) 2021  Domotz Ltd
 *
 * Domotz Agent is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Domotz Agent is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Domotz Agent.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * Created by Fausto Lelli <fausto@domotz.com>
 */

const DISCOVERY_SEMAPHORE = 'SNMP_OVERLAY_SEMAPHORE';
const snmpCommonDiscovery = require('./snmp_common');

var snmpOverlayDiscoveryFactory = function (resourceLocator, snmpGetterFactory, snmpSubtreeFactory) {
    'use strict';

    var mutex = resourceLocator.utils.mutex;
    var lodash = resourceLocator.lodash;

    var snmpOverlayDb = resourceLocator.discovery.snmp_overlay_db;
    var snmpOverlayDeviceDb = resourceLocator.discovery.snmp_overlay_device_db;
    var myConsole = resourceLocator.log.decorateLogs('SNMP_OVERLAY_DISCOVERY');
    var interfacesBinding = resourceLocator.interfacesBindingStorage;
    var snmpCommon = snmpCommonDiscovery.snmpCommonOptions();
    var engineRequest = resourceLocator.engineRequest;
    var commandBuilder = require('./command_builder').factory(resourceLocator);
    var async = resourceLocator.async;
    var snmpCheckStatus = resourceLocator.discovery.snmp_check_status_discovery;
    var ipConflict = resourceLocator.networkTools.ipConflictDetection;
    var settingsManager = resourceLocator.settingsManager;

    var lockDiscovery = function () {
        return mutex.lock(DISCOVERY_SEMAPHORE, 1);
    };

    var unlockDiscovery = function () {
        mutex.unlock(DISCOVERY_SEMAPHORE, 1);
    };

    function getTimestamp() {
        return Math.floor(new Date().getTime() / 1000);
    }

    var deferredSnmpSubtree = function (ip, mac, options, overlayId, oids, cb) {
        var snmpCommand = commandBuilder.buildSubtreeCommand(oids, ip);
        var commandOptions = snmpCommon.getSnmpOptions(options);
        snmpSubtreeFactory(snmpCommand, resourceLocator).execute(function (result) {
            if (result.error) {
                return cb(null, null);
            }
            if (myConsole.isEnabledFor('debug')) {
                myConsole.debug('Received result for SUBTREE (mac %s) (ip %s) (result %s)', mac, ip, JSON.stringify(result));
            }
            cb(null, {
                device_hw_address: mac,
                overlay_id: overlayId,
                oid_column: result,
                timestamp: getTimestamp(),
            });
        }, commandOptions);
    };

    var deferredSnmpBulkGet = function (ip, mac, options, overlayId, oids, cb) {
        var snmpCommand = commandBuilder.buildBulkGetCommand(oids, ip);
        var commandOptions = snmpCommon.getSnmpOptions(options);
        var callbackCalled = false;
        snmpGetterFactory(snmpCommand, resourceLocator).execute(function (result) {
            if (result.error) {
                if (!callbackCalled) {
                    callbackCalled = true;
                    cb(null, null);
                }
                return;
            }
            if (myConsole.isEnabledFor('debug')) {
                myConsole.debug('Received result for GET (mac %s) (ip %s) (result %s)', mac, ip, JSON.stringify(result));
            }
            if (!callbackCalled) {
                callbackCalled = true;
                cb(null, {
                    device_hw_address: mac,
                    overlay_id: overlayId,
                    oid_scalar: result,
                    timestamp: getTimestamp(),
                });
            }
        }, commandOptions);
    };

    var renderDiscoveryData = function (rawResultList) {
        if (myConsole.isEnabledFor('debug')) {
            myConsole.debug('Rendering discover raw list : |%s|', JSON.stringify(rawResultList));
        }
        sendDiscoveryDataToCloud(rawResultList);
        return rawResultList;
    };

    var doDiscovery = function (payload, callback) {
        var overlayDeviceDb = [];
        if (payload) {
            if (myConsole.isEnabledFor('debug')) {
                myConsole.debug('PAYLOAD %s', JSON.stringify(payload));
            }
            overlayDeviceDb = payload.device_overlay;
            if (myConsole.isEnabledFor('debug')) {
                myConsole.debug('Device Overlay from payload %s', JSON.stringify(overlayDeviceDb));
            }
        } else {
            overlayDeviceDb = snmpOverlayDeviceDb.getOverlayDeviceDB();
        }
        if (myConsole.isEnabledFor('debug')) {
            myConsole.debug('Overlay Device DB: %s', JSON.stringify(overlayDeviceDb));
        }
        if (overlayDeviceDb === undefined || overlayDeviceDb === null || overlayDeviceDb.length === 0) {
            myConsole.info('Overlay device DB empty, null or undefined. Skip discovery');
            return;
        }

        var maxParallel = settingsManager.getSetting('configuration.settings.snmp_overlay_discovery.parallel_limit', 5);

        var communitiesProvider = resourceLocator.snmpCommunitiesProvider;

        communitiesProvider
            .get()
            .then(function (communities) {
                var functions = [];
                var deviceIp;

                var freshBindings = interfacesBinding.listFresh(true);
                overlayDeviceDb.forEach(function (deviceOverlay) {
                    var overlayId = deviceOverlay.overlay_id;
                    var deviceMac = deviceOverlay.device_hw_address;

                    if (!snmpCheckStatus.isSNMPEnabledForMacAddress(deviceMac)) {
                        return;
                    }

                    if (deviceOverlay.device_ip_address) {
                        deviceIp = deviceOverlay.device_ip_address;
                    } else {
                        var deviceBinding = lodash.find(freshBindings, function (binding) {
                            return binding.mac === deviceMac;
                        });
                        if (!deviceBinding || !deviceBinding.ip) {
                            myConsole.info('Cannot perform discovery for device: %s, IP binding is missing', deviceMac);
                            return;
                        }
                        deviceIp = deviceBinding.ip;
                    }
                    if (ipConflict.checkIfIpInConflict(deviceIp)) {
                        myConsole.warn('Skip SNMP overlay discovery for ip: %s because there is an IP conflict', deviceIp);
                        return;
                    }

                    myConsole.info('Performing discovery: (mac %s) (ip %s) (overlay_id %s)', deviceMac, deviceIp, overlayId);
                    var overlay;
                    if (payload) {
                        payload.overlays.forEach(function (item) {
                            if (item.id === overlayId) {
                                overlay = item;
                                if (myConsole.isEnabledFor('debug')) {
                                    myConsole.debug('Overlay from payload %s', JSON.stringify(overlay));
                                }
                            }
                        });
                    } else {
                        overlay = snmpOverlayDb.getOverlay(overlayId);
                    }
                    if (overlay === undefined) {
                        myConsole.info('Missing overlay info for overlay_id: %s', overlayId);
                        return;
                    }
                    if (overlay.oid_column !== undefined && Object.keys(overlay.oid_column).length > 0) {
                        myConsole.debug('Check SUBTREE (mac %s) (overlay_id %s) (oids %s)', deviceMac, overlayId, JSON.stringify(overlay.oid_column));
                        functions.push(deferredSnmpSubtree.bind(null, deviceIp, deviceMac, communities[deviceMac], overlayId, overlay.oid_column));
                    }
                    if (overlay.oid_scalar !== undefined && Object.keys(overlay.oid_scalar).length > 0) {
                        myConsole.debug('Check GET (mac %s) (overlay_id %s) (oids %s)', deviceMac, overlayId, JSON.stringify(overlay.oid_scalar));
                        functions.push(deferredSnmpBulkGet.bind(null, deviceIp, deviceMac, communities[deviceMac], overlayId, overlay.oid_scalar));
                    }
                });

                var startTime = new Date();

                async.parallelLimit(functions, maxParallel, function (error, results) {
                    var elapsed = (new Date() - startTime) / 1000;
                    myConsole.info('discovery completed in %ss. Parallelism is %s', elapsed, maxParallel);
                    if (error) {
                        myConsole.info('Error performing parallelLimit overlay discovery %', error.toString());
                        if (callback) {
                            callback();
                        }
                        return;
                    }

                    var filteredResults = lodash.filter(results, function (v) {
                        return v !== null;
                    });

                    myConsole.info('Uploading result to cloud ...');
                    if (callback) {
                        myConsole.info('replying result through callback ...');
                        callback(filteredResults);
                    } else {
                        myConsole.info('Uploading result to cloud ...');
                        renderDiscoveryData(filteredResults);
                    }
                });
            })
            .catch(function () {
                myConsole.error('Error retrieving communities');
            });
    };

    var discoverOverlay = function (payload, callback) {
        if (!lockDiscovery()) {
            myConsole.warn('WARNING: SNMP Overlay Discovery already in progress, ignoring the call');
            return;
        }

        try {
            doDiscovery(payload, callback);
        } catch (e) {
            myConsole.error(e.toString());
        } finally {
            unlockDiscovery();
            myConsole.info('Discovery unlocked');
        }
    };

    /**
     * Split discoveryData array elements in array of arrays, each element having a byte size less or equal to
     * snmp_overlay_discovery.max_chunk_size setting
     * @param discoveryData data gathered from devices
     * @return an array of chunks
     */
    var _splitDiscoveryData = function (discoveryData) {
        var MAX_SIZE = settingsManager.getSetting('configuration.settings.snmp_overlay_discovery.max_chunk_size', 524288);
        var chunks = [];
        var currentChunk = [];
        var currentSize = 0;
        lodash.forEach(discoveryData, function (item) {
            var itemSize = Buffer.byteLength(JSON.stringify(item), 'utf8');

            if (currentSize + itemSize > MAX_SIZE) {
                chunks.push(currentChunk);
                currentChunk = [];
                currentSize = 0;
            }

            currentChunk.push(item);
            currentSize += itemSize;
        });

        if (!lodash.isEmpty(currentChunk)) {
            chunks.push(currentChunk);
        }

        return chunks;
    };

    /**
     * Send to engine all data gathered from devices with multiple HTTP post, each of one having a maximum payload size
     * determined by _splitDiscoveryData function
     * @param discoveryData data gathered from devices
     */
    var sendDiscoveryDataToCloud = function (discoveryData) {
        myConsole.info('Overlay discovery FINISHED, Results number to upload %s', lodash.size(discoveryData));
        if (myConsole.isEnabledFor('debug')) {
            myConsole.debug('Overlay discovery FINISHED, Results to upload %s', JSON.stringify(discoveryData));
        }

        try {
            var discoveryDataChunks = _splitDiscoveryData(discoveryData);
            var resource = '/overlay/sample';
            lodash.forEach(discoveryDataChunks, function (chunk) {
                engineRequest.sendWithPromise(resource, 'POST', { data: chunk }, null, null, { doNotFollow204: true }).catch(function (e) {
                    myConsole.error('Error sending Overlay data to cloud: %s', JSON.stringify(e));
                });
            });
        } catch (e) {
            myConsole.error('Error sending data %s', JSON.stringify(e));
        }
    };

    return {
        discoverOverlay: discoverOverlay,
        sendDiscoveryDataToEngine: sendDiscoveryDataToCloud,
        isDiscoveryInProgress: function () {
            return mutex.isLocked(DISCOVERY_SEMAPHORE);
        },
    };
};
module.exports.factory = snmpOverlayDiscoveryFactory;