iobroker.js-controller
Version:
Updated by reinstall.js on 2018-06-11T15:19:56.688Z
1,202 lines (1,123 loc) • 249 kB
JavaScript
/**
* application.controller
*
* Controls Adapter-Processes
*
* Copyright 2013-2022 bluefox <dogafox@gmail.com>,
* 2013-2014 hobbyquaker <hq@ccu.io>
* MIT License
*
*/
'use strict';
const schedule = require('node-schedule');
const os = require('os');
const fs = require('fs-extra');
const path = require('path');
const cp = require('child_process');
const semver = require('semver');
const restart = require('./lib/restart');
const ioPackage = require('./io-package.json');
const { tools: dbTools } = require('@iobroker/js-controller-common-db');
const version = ioPackage.common.version;
const pidUsage = require('pidusage');
const deepClone = require('deep-clone');
const { isDeepStrictEqual, inspect } = require('util');
const { tools, EXIT_CODES, logger: toolsLogger } = require('@iobroker/js-controller-common');
const { PluginHandler } = require('@iobroker/plugin-base');
const NotificationHandler = require('./lib/notificationHandler');
const controllerVersions = {};
let pluginHandler;
let notificationHandler;
const exec = cp.exec;
const spawn = cp.spawn;
let zipFiles;
let upload; // will be used only once by upload of adapter
/* Use require('loadavg-windows') to enjoy os.loadavg() on Windows OS.
Currently, Node.js on Windows platform does not implement os.loadavg() functionality - it returns [0,0,0]
Expect first results after 1 min from application start (before 1 min runtime it will return [0,0,0])
Requiring it on other operating systems has NO influence.*/
if (os.platform() === 'win32') {
require('loadavg-windows');
}
let title = `${tools.appName}.js-controller`;
let Objects;
let States;
let decache;
let logger;
let isDaemon = false;
let callbackId = 1;
let callbacks = {};
const hostname = tools.getHostName();
let hostObjectPrefix = `system.host.${hostname}`;
let hostLogPrefix = `host.${hostname}`;
const compactGroupObjectPrefix = '.compactgroup';
const logList = [];
let detectIpsCount = 0;
let objectsDisconnectTimeout = null;
let statesDisconnectTimeout = null;
let connected = null; // not false, because want to detect first connection
let lastDiskSizeCheck = 0;
let restartTimeout = null;
let connectTimeout = null;
let reportInterval = null;
let primaryHostInterval = null;
let isPrimary = false;
const PRIMARY_HOST_LOCK_TIME = 60000;
const procs = {};
const hostAdapter = {};
const subscribe = {};
const stopTimeouts = {};
let states = null;
let objects = null;
let storeTimer = null;
let mhTimer = null;
let isStopping = null;
let allInstancesStopped = true;
let stopTimeout = 10000;
let uncaughtExceptionCount = 0;
let installQueue = [];
let started = false;
let inputCount = 0;
let outputCount = 0;
let eventLoopLags = [];
let mhService = null; // multihost service
const uptimeStart = Date.now();
let compactGroupController = false;
let compactGroup = null;
const compactProcs = {};
const scheduledInstances = {};
const VENDOR_BOOTSTRAP_FILE = '/opt/iobroker/iob-vendor-secret.json';
const VENDOR_FILE = '/etc/iob-vendor.json';
let updateIPsTimer = null;
let lastDiagSend = null;
const uploadTasks = [];
const config = getConfig();
function getErrorText(code) {
const texts = Object.keys(EXIT_CODES);
for (let i = 0; i < texts.length; i++) {
if (EXIT_CODES[texts[i]] === code) {
return texts[i];
}
}
return code;
}
/**
* Get the config directly from fs - never cached
*
* @returns {null|object}
*/
function getConfig() {
const configFile = tools.getConfigFileName();
if (!fs.existsSync(configFile)) {
if (process.argv.indexOf('start') !== -1) {
isDaemon = true;
logger = toolsLogger('info', [tools.appName], true);
} else {
logger = toolsLogger('info', [tools.appName]);
}
logger.error(`${hostLogPrefix} conf/${tools.appName}.json missing - call node ${tools.appName}.js setup`);
process.exit(EXIT_CODES.MISSING_CONFIG_JSON);
return null;
} else {
const _config = fs.readJSONSync(configFile);
if (!_config.states) {
_config.states = { type: 'jsonl' };
}
if (!_config.objects) {
_config.objects = { type: 'jsonl' };
}
if (!_config.system) {
_config.system = {};
}
return _config;
}
}
function _startMultihost(_config, secret) {
const MHService = require('./lib/multihostServer.js');
const cpus = os.cpus();
mhService = new MHService(
hostname,
logger,
_config,
{
node: process.version,
arch: os.arch(),
model: cpus && cpus[0] && cpus[0].model ? cpus[0].model : 'unknown',
cpus: cpus ? cpus.length : 1,
mem: os.totalmem(),
ostype: os.type()
},
tools.findIPs(),
secret
);
}
/**
* Starts or stops the multihost discovery server, depending on the config and temp information
*
* @param {object} __config - the iobroker config object
* @returns {boolean|void}
*/
async function startMultihost(__config) {
if (compactGroupController) {
return;
}
if (mhTimer) {
clearTimeout(mhTimer);
mhTimer = null;
}
const _config = __config || getConfig();
if (_config.multihostService && _config.multihostService.enabled) {
if (mhService) {
try {
mhService.close(() => {
mhService = null;
setImmediate(() => startMultihost(_config));
});
return;
} catch (e) {
logger.warn(`${hostLogPrefix} Cannot stop multihost discovery server: ${e.message}`);
}
}
if (!_config.objects.host || dbTools.isLocalObjectsDbServer(_config.objects.type, _config.objects.host, true)) {
logger.warn(
`${hostLogPrefix} Multihost Master on this system is not possible, because IP address for objects is ${_config.objects.host}. Please allow remote connections to the server by adjusting the IP.`
);
return false;
} else if (
!_config.states.host ||
dbTools.isLocalObjectsDbServer(_config.states.type, _config.states.host, true)
) {
logger.warn(
`${hostLogPrefix} Multihost Master on this system is not possible, because IP address for states is ${_config.states.host}. Please allow remote connections to the server by adjusting the IP.`
);
return false;
}
if (_config.multihostService.secure) {
if (typeof _config.multihostService.password === 'string' && _config.multihostService.password.length) {
let obj, errText;
try {
obj = await objects.getObjectAsync('system.config');
} catch (e) {
// will log error below
errText = e.message;
}
if (obj && obj.native && obj.native.secret) {
if (!_config.multihostService.password.startsWith(`$/aes-192-cbc:`)) {
// if old encryption was used, we need to decrypt in old fashion
tools.decryptPhrase(obj.native.secret, _config.multihostService.password, secret =>
_startMultihost(_config, secret)
);
} else {
try {
// it can throw in edge cases #1474, we need further investigation
const secret = tools.decrypt(obj.native.secret, _config.multihostService.password);
_startMultihost(_config, secret);
} catch (e) {
logger.error(
`${hostLogPrefix} Cannot decrypt password for multihost discovery server: ${e.message}`
);
}
}
} else {
logger.error(
`${hostLogPrefix} Cannot start multihost discovery server: no system.config found (err: ${errText})`
);
}
} else {
logger.error(
`${hostLogPrefix} Cannot start multihost discovery server: secure mode was configured, but no secret was set. Please check the configuration!`
);
}
} else {
_startMultihost(_config, false);
}
if (!_config.multihostService.persist) {
mhTimer = setTimeout(async () => {
if (mhService) {
try {
mhService.close();
mhService = null;
logger.info(
`${hostLogPrefix} Multihost discovery server stopped after 15 minutes, because only temporarily activated`
);
_config.multihostService.persist = false;
_config.multihostService.enabled = false;
const configFile = tools.getConfigFileName();
await fs.writeFile(configFile, JSON.stringify(_config, null, 2));
} catch (e) {
logger.warn(`${hostLogPrefix} Cannot stop multihost discovery: ${e.message}`);
}
}
mhTimer = null;
}, 15 * 60000);
}
return true;
} else if (mhService) {
try {
mhService.close();
mhService = null;
} catch (e) {
logger.warn(`${hostLogPrefix} Cannot stop multihost discovery: ${e.message}`);
}
return false;
}
}
/**
* Starts cyclic update of IP interfaces.
* At start every 30 seconds and after 5 minutes, every hour.
* Because DHCP could change the IPs.
*/
function startUpdateIPs() {
if (!updateIPsTimer) {
updateIPsTimer = setInterval(() => {
if (Date.now() - uptimeStart > 5 * 60000) {
// 5 minutes at start check every 30 seconds because of DHCP
clearInterval(updateIPsTimer);
updateIPsTimer = setInterval(() => setIPs(), 3600000); // update IPs every hour
}
setIPs();
}, 30000);
}
}
// subscribe or unsubscribe loggers
function logRedirect(isActive, id, reason) {
console.log(`================================== > LOG REDIRECT ${id} => ${isActive} [${reason}]`);
if (isActive) {
if (!logList.includes(id)) {
logList.push(id);
}
} else {
const pos = logList.indexOf(id);
if (pos !== -1) {
logList.splice(pos, 1);
}
}
}
function handleDisconnect() {
if (!connected || restartTimeout || isStopping) {
return;
}
if (statesDisconnectTimeout) {
clearTimeout(statesDisconnectTimeout);
statesDisconnectTimeout = null;
}
if (objectsDisconnectTimeout) {
clearTimeout(objectsDisconnectTimeout);
objectsDisconnectTimeout = null;
}
connected = false;
logger.warn(`${hostLogPrefix} Slave controller detected disconnection. Stop all instances.`);
if (compactGroupController) {
stop(true);
} else {
stop(true, () => {
restartTimeout = setTimeout(() => {
processMessage({ command: 'cmdExec', message: { data: '_restart' } });
setTimeout(() => process.exit(EXIT_CODES.JS_CONTROLLER_STOPPED), 1000);
}, 10000);
});
}
}
function createStates(onConnect) {
states = new States({
namespace: hostLogPrefix,
connection: config.states,
logger: logger,
hostname: hostname,
change: (id, state) => {
inputCount++;
if (!id) {
return logger.error(`${hostLogPrefix} change event with no ID: ${JSON.stringify(state)}`);
}
// If some log transporter activated or deactivated
if (id.endsWith('.logging')) {
logRedirect(state ? state.val : false, id.substring(0, id.length - '.logging'.length), id);
} else if (!compactGroupController && id === `messagebox.${hostObjectPrefix}`) {
// If this is messagebox, only the main controller is handling the host messages
const obj = state;
if (obj) {
// If callback stored for this request
if (
obj.callback &&
obj.callback.ack &&
obj.callback.id &&
callbacks &&
callbacks[`_${obj.callback.id}`]
) {
// Call callback function
if (callbacks[`_${obj.callback.id}`].cb) {
callbacks[`_${obj.callback.id}`].cb(obj.message);
delete callbacks[`_${obj.callback.id}`];
}
// delete too old callbacks IDs
const now = Date.now();
for (const _id of Object.keys(callbacks)) {
if (now - callbacks[_id].time > 3600000) {
delete callbacks[_id];
}
}
} else {
processMessage(obj);
}
}
} else if (!compactGroupController && id.match(/^system.adapter.[^.]+\.\d+\.alive$/)) {
// If this system.adapter.NAME.0.alive, only main controller is handling this
if (state && !state.ack) {
const enabled = state.val;
objects.getObject(id.substring(0, id.length - 6 /*'.alive'.length*/), (err, obj) => {
if (err) {
logger.error(`${hostLogPrefix} Cannot read object: ${err}`);
}
if (obj && obj.common) {
// IF adapter enabled => disable it
if ((obj.common.enabled && !enabled) || (!obj.common.enabled && enabled)) {
obj.common.enabled = !!enabled;
logger.info(
`${hostLogPrefix} instance "${obj._id}" ${
obj.common.enabled ? 'enabled' : 'disabled'
} via .alive`
);
obj.from = hostObjectPrefix;
obj.ts = Date.now();
objects.setObject(
obj._id,
obj,
err => err && logger.error(`${hostLogPrefix} Cannot set object: ${err}`)
);
}
}
});
}
} else if (subscribe[id]) {
for (let i = 0; i < subscribe[id].length; i++) {
// wake up adapter
if (procs[subscribe[id][i]]) {
console.log(`Wake up ${id} ${JSON.stringify(state)}`);
startInstance(subscribe[id][i], true);
} else {
logger.warn(`${hostLogPrefix} controller Adapter subscribed on ${id} does not exist!`);
}
}
} else if (id === `${hostObjectPrefix}.logLevel`) {
if (!config || !config.log || !state || state.ack) {
return;
}
let currentLevel = config.log.level;
if (
state.val &&
state.val !== currentLevel &&
['silly', 'debug', 'info', 'warn', 'error'].includes(state.val)
) {
config.log.level = state.val;
for (const transport of Object.keys(logger.transports)) {
if (
logger.transports[transport].level === currentLevel &&
!logger.transports[transport]._defaultConfigLoglevel
) {
logger.transports[transport].level = state.val;
}
}
logger.info(`${hostLogPrefix} Loglevel changed from "${currentLevel}" to "${state.val}"`);
currentLevel = state.val;
} else if (state.val && state.val !== currentLevel) {
logger.info(`${hostLogPrefix} Got invalid loglevel "${state.val}", ignoring`);
}
states.setState(`${hostObjectPrefix}.logLevel`, {
val: currentLevel,
ack: true,
from: hostObjectPrefix
});
} else if (id.startsWith(`${hostObjectPrefix}.plugins.`) && id.endsWith('.enabled')) {
if (!config || !config.log || !state || state.ack) {
return;
}
const pluginStatesIndex = `${hostObjectPrefix}.plugins.`.length;
let nameEndIndex = id.indexOf('.', pluginStatesIndex + 1);
if (nameEndIndex === -1) {
nameEndIndex = undefined;
}
const pluginName = id.substring(pluginStatesIndex, nameEndIndex);
if (!pluginHandler.pluginExists(pluginName)) {
return;
}
if (pluginHandler.isPluginActive(pluginName) !== state.val) {
if (state.val) {
if (!pluginHandler.isPluginInstanciated(pluginName)) {
pluginHandler.instanciatePlugin(
pluginName,
pluginHandler.getPluginConfig(pluginName),
__dirname
);
pluginHandler.setDatabaseForPlugin(pluginName, objects, states);
pluginHandler.initPlugin(pluginName, ioPackage);
}
} else {
if (!pluginHandler.destroy(pluginName)) {
logger.info(
`${hostLogPrefix} Plugin ${pluginName} could not be disabled. Please restart ioBroker to disable it.`
);
}
}
}
}
},
connected: () => {
if (statesDisconnectTimeout) {
clearTimeout(statesDisconnectTimeout);
statesDisconnectTimeout = null;
}
// logs and cleanups are only handled by the main controller process
if (!compactGroupController) {
states.clearAllLogs && states.clearAllLogs();
deleteAllZipPackages();
}
initMessageQueue();
startAliveInterval();
initializeController();
onConnect && onConnect();
},
disconnected: (/*error*/) => {
if (restartTimeout) {
return;
}
statesDisconnectTimeout && clearTimeout(statesDisconnectTimeout);
statesDisconnectTimeout = setTimeout(() => {
statesDisconnectTimeout = null;
handleDisconnect();
}, (config.states.connectTimeout || 2000) + (!compactGroupController ? 500 : 0));
}
});
return true;
}
async function initializeController() {
if (!states || !objects || connected) {
return;
}
logger.info(`${hostLogPrefix} connected to Objects and States`);
// initialize notificationHandler
const notificationSettings = {
states: states,
objects: objects,
log: logger,
logPrefix: hostLogPrefix,
host: hostname
};
notificationHandler = new NotificationHandler(notificationSettings);
if (ioPackage.notifications) {
try {
await notificationHandler.addConfig(ioPackage.notifications);
logger.info(`${hostLogPrefix} added notifications configuration of host`);
// load setup of all adapters to class, to remember messages even of non-running hosts
await notificationHandler.getSetupOfAllAdaptersFromHost();
} catch (e) {
logger.error(`${hostLogPrefix} Could not add notifications config of this host: ${e.message}`);
}
}
if (connected === null) {
connected = true;
if (!isStopping) {
pluginHandler.setDatabaseForPlugins(objects, states);
pluginHandler.initPlugins(ioPackage, () => {
states.subscribe(`${hostObjectPrefix}.plugins.*`);
// Do not start if we still stopping the instances
checkHost(() => {
startMultihost(config);
setMeta();
started = true;
getInstances();
});
});
}
} else {
connected = true;
started = true;
// Do not start if we still stopping the instances
if (!isStopping) {
getInstances();
}
}
}
// create "objects" object
function createObjects(onConnect) {
objects = new Objects({
namespace: hostLogPrefix,
connection: config.objects,
controller: true,
logger: logger,
hostname: hostname,
connected: async () => {
// stop disconnect timeout
if (objectsDisconnectTimeout) {
clearTimeout(objectsDisconnectTimeout);
objectsDisconnectTimeout = null;
}
// subscribe to primary host expiration
try {
await objects.subscribePrimaryHost();
} catch (e) {
logger.error(`${hostLogPrefix} Cannot subscribe to primary host expiration: ${e.message}`);
}
if (!primaryHostInterval && !compactGroupController) {
primaryHostInterval = setInterval(checkPrimaryHost, PRIMARY_HOST_LOCK_TIME / 2);
}
// first execution now
checkPrimaryHost();
initializeController();
onConnect && onConnect();
},
disconnected: (/*error*/) => {
if (restartTimeout) {
return;
}
// on reconnection this will be determiend anew
isPrimary = false;
objectsDisconnectTimeout && clearTimeout(objectsDisconnectTimeout);
objectsDisconnectTimeout = setTimeout(() => {
objectsDisconnectTimeout = null;
handleDisconnect();
}, (config.objects.connectTimeout || 2000) + (!compactGroupController ? 500 : 0));
// give main controller a bit longer, so that adapter and compact processes can exit before
},
change: async (id, obj) => {
if (!started || !id.match(/^system\.adapter\.[a-zA-Z0-9-_]+\.[0-9]+$/)) {
return;
}
try {
logger.debug(`${hostLogPrefix} object change ${id} (from: ${obj ? obj.from : null})`);
// known adapter
if (procs[id]) {
// if adapter deleted
if (!obj) {
// deleted: also remove from instance list of compactGroup
if (
!compactGroupController &&
procs[id].config.common.compactGroup &&
compactProcs[procs[id].config.common.compactGroup] &&
compactProcs[procs[id].config.common.compactGroup].instances &&
compactProcs[procs[id].config.common.compactGroup].instances.includes(id)
) {
compactProcs[procs[id].config.common.compactGroup].instances.splice(
compactProcs[procs[id].config.common.compactGroup].instances.indexOf(id),
1
);
}
// instance removed -> remove all notifications
await notificationHandler.clearNotifications(null, null, id);
procs[id].config.common.enabled = false;
procs[id].config.common.host = null;
procs[id].config.deleted = true;
delete hostAdapter[id];
logger.info(`${hostLogPrefix} object deleted ${id}`);
} else {
if (procs[id].config.common.enabled && !obj.common.enabled) {
logger.info(`${hostLogPrefix} "${id}" disabled`);
}
if (!procs[id].config.common.enabled && obj.common.enabled) {
logger.info(`${hostLogPrefix} "${id}" enabled`);
procs[id].downloadRetry = 0;
}
// Check if compactgroup or compact mode changed
if (
!compactGroupController &&
procs[id].config.common.compactGroup &&
(procs[id].config.common.compactGroup !== obj.common.compactGroup ||
procs[id].config.common.runAsCompactMode !== obj.common.runAsCompactMode) &&
compactProcs[procs[id].config.common.compactGroup] &&
compactProcs[procs[id].config.common.compactGroup].instances &&
compactProcs[procs[id].config.common.compactGroup].instances.includes(id)
) {
compactProcs[procs[id].config.common.compactGroup].instances.splice(
compactProcs[procs[id].config.common.compactGroup].instances.indexOf(id),
1
);
}
procs[id].config = obj;
hostAdapter[id] = hostAdapter[id] || {};
hostAdapter[id].config = obj;
}
if (
procs[id].process ||
procs[id].config.common.mode === 'schedule' ||
procs[id].config.common.mode === 'subscribe'
) {
procs[id].restartExpected = true;
stopInstance(id, async () => {
if (!procs[id]) {
return;
}
const _ipArr = tools.findIPs();
if (checkAndAddInstance(procs[id].config, _ipArr)) {
if (
procs[id].config.common.enabled &&
(procs[id].config.common.mode !== 'extension' ||
!procs[id].config.native.webInstance)
) {
if (procs[id].restartTimer) {
clearTimeout(procs[id].restartTimer);
}
const restartTimeout = (procs[id].config.common.stopTimeout || 500) + 2500;
procs[id].restartTimer = setTimeout(_id => startInstance(_id), restartTimeout, id);
}
} else {
// moved: also remove from instance list of compactGroup
if (
!compactGroupController &&
procs[id].config.common.compactGroup &&
compactProcs[procs[id].config.common.compactGroup] &&
compactProcs[procs[id].config.common.compactGroup].instances &&
compactProcs[procs[id].config.common.compactGroup].instances.includes(id)
) {
compactProcs[procs[id].config.common.compactGroup].instances.splice(
compactProcs[procs[id].config.common.compactGroup].instances.indexOf(id),
1
);
}
if (procs[id].restartTimer) {
clearTimeout(procs[id].restartTimer);
delete procs[id].restartTimer;
}
// instance moved -> remove all notifications, new host has to take care
await notificationHandler.clearNotifications(null, null, id);
delete procs[id];
delete hostAdapter[id];
}
});
} else if (installQueue.find(obj => obj.id === id)) {
// ignore object changes when still in install queue
logger.debug(
`${hostLogPrefix} ignore object change because the adapter is still in installation/rebuild queue`
);
} else {
const _ipArr = tools.findIPs();
if (procs[id].config && checkAndAddInstance(procs[id].config, _ipArr)) {
if (
procs[id].config.common.enabled &&
(procs[id].config.common.mode !== 'extension' || !procs[id].config.native.webInstance)
) {
startInstance(id);
}
} else {
// moved: also remove from instance list of compactGroup
if (
!compactGroupController &&
procs[id].config.common.compactGroup &&
compactProcs[procs[id].config.common.compactGroup] &&
compactProcs[procs[id].config.common.compactGroup].instances &&
compactProcs[procs[id].config.common.compactGroup].instances.includes(id)
) {
compactProcs[procs[id].config.common.compactGroup].instances.splice(
compactProcs[procs[id].config.common.compactGroup].instances.indexOf(id),
1
);
}
if (procs[id].restartTimer) {
clearTimeout(procs[id].restartTimer);
delete procs[id].restartTimer;
}
delete procs[id];
delete hostAdapter[id];
}
}
} else if (obj && obj.type === 'host' && obj.common && obj.common.installedVersion) {
// host object changed
// TODO: remove this shim if 4.0 is old enough
if (controllerVersions[id]) {
if (
semver.lt(controllerVersions[id], '4.0.0') &&
semver.gte(obj.common.installedVersion, '4.0.0')
) {
// old version lower 4 new version greater 4, restart needed
logger.info(`${hostLogPrefix} Multihost controller upgrade detected, restarting ...`);
restart();
} else if (
semver.gte(controllerVersions[id], '4.0.0') &&
semver.lt(obj.common.installedVersion, '4.0.0')
) {
// controller was above 4 and now below 4
logger.info(`${hostLogPrefix} Multihost controller downgrade detected, restarting ...`);
restart();
}
} else {
// we don't know this host yet, so it is new to the mh system
let restartRequired = true;
// if we are a already a multihost make the version check else restart in all cases
if (Object.keys(controllerVersions).length > 1) {
if (semver.lt(obj.common.installedVersion, '4.0.0')) {
for (const controllerVersion of controllerVersions) {
if (semver.lt(controllerVersion, '4.0.0')) {
// there was another host < 4 so no restart required
restartRequired = false;
break;
}
}
} else {
// version is greater equal 4
for (const controllerVersion of controllerVersions) {
if (semver.gte(controllerVersion, '4.0.0')) {
// there was already another host greater equal 4 -> no restart needed
restartRequired = false;
break;
}
}
}
} else {
// change from single to multihost - deactivate sets asap but also restart
await objects.deactivateSets();
}
if (restartRequired) {
logger.info(`${hostLogPrefix} New multihost participant detected, restarting ...`);
restart();
}
}
controllerVersions[id] = obj.common.installedVersion;
} else if (!obj && /^system\.host\.[^.]+$/.test(id)) {
const delVersion = controllerVersions[id];
delete controllerVersions[id];
// host object deleted
if (delVersion && semver.lt(delVersion, '4.0.0')) {
// check if the only below 4 host has been deleted, then we need restart
for (const version of Object.entries(controllerVersions)) {
if (semver.lt(version, '4.0.0')) {
// another version below 4, so still need old protocol
return;
}
}
logger.info(`${hostLogPrefix} Multihost controller deletion detected, restarting ...`);
restart();
}
} else if (obj && obj.common) {
const _ipArr = tools.findIPs();
// new adapter
if (
checkAndAddInstance(obj, _ipArr) &&
procs[id].config.common.enabled &&
(procs[id].config.common.mode !== 'extension' || !procs[id].config.native.webInstance)
) {
// We should give is a slight delay to allow an pot. former existing process on other host to exit
const restartTimeout = (procs[id].config.common.stopTimeout || 500) + 2500;
procs[id].restartTimer = setTimeout(_id => startInstance(_id), restartTimeout, id);
}
}
} catch (err) {
if (
!compactGroupController ||
(obj && obj.common && obj.common.runAsCompactMode && obj.common.compactGroup === compactGroup)
) {
logger.error(`${hostLogPrefix} cannot process: ${id}: ${err} / ${err.stack}`);
}
}
},
primaryHostLost: () => {
if (!isStopping) {
isPrimary = false;
logger.info('The primary host is no longer active. Checking responsibilities.');
checkPrimaryHost();
}
}
});
return true;
}
function startAliveInterval() {
config.system = config.system || {};
config.system.statisticsInterval = parseInt(config.system.statisticsInterval, 10) || 15000;
config.system.checkDiskInterval =
config.system.checkDiskInterval !== 0 ? parseInt(config.system.checkDiskInterval, 10) || 300000 : 0;
if (!compactGroupController) {
// Provide info to see for each host if compact is enabled or not and be able to use in Admin or such
states.setState(`${hostObjectPrefix}.compactModeEnabled`, {
ack: true,
from: hostObjectPrefix,
val: config.system.compact || false
});
}
reportInterval = setInterval(reportStatus, config.system.statisticsInterval);
reportStatus();
tools.measureEventLoopLag(1000, lag => eventLoopLags.push(lag));
}
/**
* Ensures that we take over primary host if no other is doing the job
*
* @return {Promise<void>}
*/
async function checkPrimaryHost() {
// we cannot interact with db now because currently reconnecting
if (objectsDisconnectTimeout || compactGroupController) {
return;
}
// let our host value live PRIMARY_HOST_LOCK_TIME seconds, while it should be renewed lock time / 2
try {
if (!isPrimary) {
isPrimary = !!(await objects.setPrimaryHost(PRIMARY_HOST_LOCK_TIME));
} else {
const lockExtended = !!(await objects.extendPrimaryHostLock(PRIMARY_HOST_LOCK_TIME));
if (!lockExtended) {
// if we are host, lock extension should always work, fallback to acquire lock
isPrimary = !!(await objects.setPrimaryHost(PRIMARY_HOST_LOCK_TIME));
}
}
} catch (e) {
logger.error(`${hostLogPrefix} Could not execute primary host determination: ${e.message}`);
}
}
function reportStatus() {
if (!states) {
return;
}
const id = hostObjectPrefix;
outputCount += 10;
states.setState(`${id}.alive`, {
val: true,
ack: true,
expire: Math.floor(config.system.statisticsInterval / 1000) + 10,
from: id
});
// provide infos about current process
// pidUsage([pid,pid,...], function (err, stats) {
// => {
// cpu: 10.0, // percentage (from 0 to 100*vcore)
// memory: 357306368, // bytes
// ppid: 312, // PPID
// pid: 727, // PID
// ctime: 867000, // ms user + system time
// elapsed: 6650000, // ms since the start of the process
// timestamp: 864000000 // ms since epoch
// }
try {
pidUsage(process.pid, (err, stats) => {
// controller.s might be stopped, but this is still running
if (!err && states && states.setState && stats) {
states.setState(`${id}.cpu`, {
ack: true,
from: id,
val: Math.round(100 * parseFloat(stats.cpu)) / 100
});
states.setState(`${id}.cputime`, { ack: true, from: id, val: stats.ctime / 1000 });
outputCount += 2;
}
});
} catch (e) {
logger.error(`${hostLogPrefix} Cannot read pidUsage data : ${e.message}`);
}
try {
const mem = process.memoryUsage();
states.setState(`${id}.memRss`, {
val: Math.round(mem.rss / 10485.76 /* 1MB / 100 */) / 100,
ack: true,
from: id
});
states.setState(`${id}.memHeapTotal`, {
val: Math.round(mem.heapTotal / 10485.76 /* 1MB / 100 */) / 100,
ack: true,
from: id
});
states.setState(`${id}.memHeapUsed`, {
val: Math.round(mem.heapUsed / 10485.76 /* 1MB / 100 */) / 100,
ack: true,
from: id
});
} catch (e) {
logger.error(`${hostLogPrefix} Cannot read memoryUsage data: ${e.message}`);
}
// provide machine infos
states.setState(`${id}.load`, { val: Math.round(os.loadavg()[0] * 100) / 100, ack: true, from: id }); //require('loadavg-windows')
states.setState(`${id}.uptime`, { val: Math.round(process.uptime()), ack: true, from: id });
states.setState(`${id}.mem`, { val: Math.round((1000 * os.freemem()) / os.totalmem()) / 10, ack: true, from: id });
states.setState(`${id}.freemem`, { val: Math.round(os.freemem() / 1048576 /* 1MB */), ack: true, from: id });
if (fs.existsSync('/proc/meminfo')) {
try {
const text = fs.readFileSync('/proc/meminfo', 'utf8');
const m = text && text.match(/MemAvailable:\s*(\d+)/);
if (m && m[1]) {
states.setState(`${id}.memAvailable`, {
val: Math.round(parseInt(m[1], 10) * 0.001024),
ack: true,
from: id
});
outputCount++;
}
} catch (e) {
logger.error(`${hostLogPrefix} Cannot read /proc/meminfo: ${e.message}`);
}
}
if (config.system.checkDiskInterval && Date.now() - lastDiskSizeCheck >= config.system.checkDiskInterval) {
lastDiskSizeCheck = Date.now();
tools.getDiskInfo(os.platform(), (err, info) => {
if (err) {
logger.error(`${hostLogPrefix} Cannot read disk size: ${err}`);
}
try {
if (info) {
states.setState(`${id}.diskSize`, {
val: Math.round((info['Disk size'] || 0) / (1024 * 1024)),
ack: true,
from: id
});
states.setState(`${id}.diskFree`, {
val: Math.round((info['Disk free'] || 0) / (1024 * 1024)),
ack: true,
from: id
});
outputCount += 2;
}
} catch (e) {
logger.error(`${hostLogPrefix} Cannot read disk information: ${e.message}`);
}
});
}
// some statistics
states.setState(`${id}.inputCount`, { val: inputCount, ack: true, from: id });
states.setState(`${id}.outputCount`, { val: outputCount, ack: true, from: id });
if (eventLoopLags.length) {
const eventLoopLag = Math.ceil(eventLoopLags.reduce((a, b) => a + b) / eventLoopLags.length);
states.setState(`${id}.eventLoopLag`, { val: eventLoopLag, ack: true, from: id }); // average of measured values
eventLoopLags = [];
}
states.setState(`${id}.compactgroupProcesses`, { val: Object.keys(compactProcs).length, ack: true, from: id });
let realProcesses = 0;
let compactProcesses = 0;
Object.keys(procs).forEach(proc => {
if (procs[proc].process) {
if (procs[proc].startedInCompactMode) {
compactProcesses++;
} else {
realProcesses++;
}
}
});
states.setState(`${id}.instancesAsProcess`, { val: realProcesses, ack: true, from: id });
states.setState(`${id}.instancesAsCompact`, { val: compactProcesses, ack: true, from: id });
inputCount = 0;
outputCount = 0;
if (!isStopping && compactGroupController && started && compactProcesses === 0 && realProcesses === 0) {
logger.info(`${hostLogPrefix} Compact group controller ${compactGroup} does not own any processes, stop`);
stop(false);
}
}
function changeHost(objs, oldHostname, newHostname, callback) {
if (!objs || !objs.length) {
typeof callback === 'function' && callback();
} else {
const row = objs.shift();
if (row && row.value && row.value.common && row.value.common.host === oldHostname) {
const obj = row.value;
obj.common.host = newHostname;
logger.info(
`${hostLogPrefix} Reassign instance ${obj._id.substring(
'system.adapter.'.length
)} from ${oldHostname} to ${newHostname}`
);
obj.from = `system.host.${tools.getHostName()}`;
obj.ts = Date.now();
objects.setObject(obj._id, obj, (/* err */) =>
setImmediate(() => changeHost(objs, oldHostname, newHostname, callback)));
} else {
setImmediate(() => changeHost(objs, oldHostname, newHostname, callback));
}
}
}
function cleanAutoSubscribe(instance, autoInstance, callback) {
inputCount++;
states.getState(autoInstance + '.subscribes', (err, state) => {
if (!state || !state.val) {
return typeof callback === 'function' && setImmediate(() => callback());
}
let subs;
try {
subs = JSON.parse(state.val);
} catch {
logger.error(`${hostLogPrefix} Cannot parse subscribes: ${state.val}`);
return typeof callback === 'function' && setImmediate(() => callback());
}
let modified = false;
// look for all subscribes from this instance
for (const pattern of Object.keys(subs)) {
for (const id of Object.keys(subs[pattern])) {
if (id === instance) {
modified = true;
delete subs[pattern][id];
}
}
// check if array is now empty
if (!Object.keys(subs[pattern]).length) {
modified = true;
delete subs[pattern];
}
}
if (modified) {
outputCount++;
states.setState(`${autoInstance}.subscribes`, subs, () => typeof callback === 'function' && callback());
} else if (typeof callback === 'function') {
setImmediate(() => callback());
}
});
}
function cleanAutoSubscribes(instance, callback) {
// instance = 'system.adapter.name.0'
instance = instance.substring(15); // get name.0
// read all instances
objects.getObjectView(
'system',
'instance',
{ startkey: 'system.adapter.', endkey: 'system.adapter.\u9999' },
(err, res) => {
let count = 0;
if (res && res.rows) {
for (let c = res.rows.length - 1; c >= 0; c--) {
// remove this instance from autoSubscribe
if (res.rows[c].value && res.rows[c].value.common && res.rows[c].value.common.subscribable) {
count++;
cleanAutoSubscribe(instance, res.rows[c].id, () => !--count && callback && callback());
}
}
}
!count && callback && callback();
}
);
}
function delObjects(objs, callback) {
if (!objs || !objs.length) {
typeof callback === 'function' && callback();
} else {
const row = objs.shift();
if (row && row.id) {
logger.info(`${hostLogPrefix} Delete state "${row.id}"`);
if (row.value && row.value.type === 'state') {
states.delState(row.id, (/* err */) =>
objects.delObject(row.id, (/* err */) => setImmediate(() => delObjects(objs, callback))));
} else {
objects.delObject(row.id, (/* err */) => setImmediate(() => delObjects(objs, callback)));
}
} else {
setImmediate(() => delObjects(objs, callback));
}
}
}
/**
* try to check host in objects
* <p>
* This function tries to find all hosts in the objects and if
* only one host found and it is not actual host, change the
* host name to new one.
* <p>
*
* @return none
*/
function checkHost(callback) {
const objectData = objects.getStatus();
// only file master host controller needs to check/fix the host assignments from the instances
// for redis it is currently not possible to detect a single host system with a changed hostname for sure!
if (compactGroupController || !objectData.server) {
return callback && callback();
}
objects.getObjectView(
'system',
'host',
{
startkey: 'system.host.',
endkey: 'system.host.\u9999'
},
(_err, doc) => {
if (!_err && doc && doc.rows && doc.rows.length === 1 && doc.rows[0].value.common.name !== hostname) {
const oldHostname = doc.rows[0].value.common.name;
const oldId = doc.rows[0].value._id;
// find out all instances and rewrite i