iobroker.js-controller
Version:
Updated by reinstall.js on 2018-06-11T15:19:56.688Z
1,206 lines • 230 kB
JavaScript
/// <reference types="@iobroker/types-dev" />
import schedule from 'node-schedule';
import os from 'node:os';
import fs from 'fs-extra';
import path from 'node:path';
import cp, { spawn, exec } from 'node:child_process';
import semver from 'semver';
import restart from './lib/restart.js';
import pidUsage from 'pidusage';
import deepClone from 'deep-clone';
import { isDeepStrictEqual, inspect } from 'node:util';
import { MHServer } from './lib/multihostServer.js';
import { tools, EXIT_CODES, logger as toolsLogger, isLocalObjectsDbServer, isLocalStatesDbServer, NotificationHandler, getObjectsConstructor, getStatesConstructor, zipFiles, getInstancesOrderedByStartPrio, isInstalledFromNpm, } from '@iobroker/js-controller-common';
import { SYSTEM_ADAPTER_PREFIX, SYSTEM_CONFIG_ID, SYSTEM_HOST_PREFIX, SYSTEM_REPOSITORIES_ID, } from '@iobroker/js-controller-common-db/constants';
import { PluginHandler } from '@iobroker/plugin-base';
import { BlocklistManager } from './lib/blocklistManager.js';
import { Upload, PacketManager } from '@iobroker/js-controller-cli';
import decache from 'decache';
import cronParser from 'cron-parser';
import { DEFAULT_DISK_WARNING_LEVEL, getCronExpression, getDiskWarningLevel } from './lib/utils.js';
import { AdapterAutoUpgradeManager } from './lib/adapterAutoUpgradeManager.js';
import { getHostObject, getDefaultNodeArgs, isAdapterEsmModule, } from '@iobroker/js-controller-common-db/tools';
import { AdapterUpgradeManager } from './lib/adapterUpgradeManager.js';
import { setTimeout as wait } from 'node:timers/promises';
import { getHostObjects } from './lib/objects.js';
import * as url from 'node:url';
import { createRequire } from 'node:module';
// eslint-disable-next-line unicorn/prefer-module
const thisDir = url.fileURLToPath(new URL('.', import.meta.url || `file://${__filename}`));
// eslint-disable-next-line unicorn/prefer-module
const require = createRequire(import.meta.url || `file://${__filename}`);
const VIS_ADAPTERS = ['vis', 'vis-2'];
const ioPackage = fs.readJSONSync(path.join(tools.getControllerDir(), 'io-package.json'));
const version = ioPackage.common.version;
/** controller versions of multihost environments */
const controllerVersions = {};
let pluginHandler;
let notificationHandler;
let blocklistManager;
let autoUpgradeManager;
/** array of instances which have requested repo update */
let requestedRepoUpdates = [];
let upload; // will be used only once by upload of adapter
/* Use require('loadavg-windows') to enjoy os.loadavg() on Windows OS.
Currently, Node.js on a Windows platform does not implement os.loadavg() functionality - it returns [0,0,0]
Expect first results after 1 min from application start (before 1 min runtime it will return [0,0,0])
Requiring it on other operating systems has NO influence.*/
if (os.platform() === 'win32') {
require('loadavg-windows');
}
tools.ensureDNSOrder();
let Objects;
let States;
let logger;
let isDaemon = false;
let callbackId = 1;
const callbacks = {};
const hostname = tools.getHostName();
const controllerDir = tools.getControllerDir();
let hostObjectPrefix = `system.host.${hostname}`;
let hostLogPrefix = `host.${hostname}`;
const compactGroupObjectPrefix = '.compactgroup';
const logList = [];
let detectIpsCount = 0;
let objectsDisconnectTimeout = null;
let statesDisconnectTimeout = null;
let connected = null; // not false, because want to detect first connection
let lastDiskSizeCheck = 0;
let restartTimeout = null;
let connectTimeout = null;
let reportInterval = null;
let primaryHostInterval = null;
let isPrimary = false;
/** If system reboot is required */
let isRebootRequired = false;
const PRIMARY_HOST_LOCK_TIME = 60_000;
const VENDOR_BOOTSTRAP_FILE = '/opt/iobroker/iob-vendor-secret.json';
const VENDOR_FILE = '/etc/iob-vendor.json';
const procs = {};
const subscribe = {};
const stopTimeouts = {};
let states = null;
let objects = null;
let storeTimer = null;
let mhTimer = null;
let isStopping = null;
let allInstancesStopped = true;
let stopTimeout = 10_000;
let uncaughtExceptionCount = 0;
let installQueue = [];
let started = false;
let inputCount = 0;
let outputCount = 0;
let eventLoopLags = [];
let mhService = null; // multihost service
const uptimeStart = Date.now();
let compactGroupController = false;
let compactGroup = null;
const compactProcs = {};
const scheduledInstances = {};
/** If less than this disk space free in %, generate a warning */
let diskWarningLevel = DEFAULT_DISK_WARNING_LEVEL;
let updateIPsTimer = null;
let lastDiagSend = null;
const config = getConfig();
/**
* Get the error text from an exit code
*
* @param code exit code
*/
function getErrorText(code) {
return EXIT_CODES[code];
}
/**
* Get the config directly from fs - never cached
*/
function getConfig() {
const configFile = tools.getConfigFileName();
if (!fs.existsSync(configFile)) {
if (process.argv.indexOf('start') !== -1) {
isDaemon = true;
logger = toolsLogger('info', [tools.appName], true);
}
else {
logger = toolsLogger('info', [tools.appName]);
}
logger.error(`${hostLogPrefix} conf/${tools.appName.toLowerCase()}.json missing - call node ${tools.appName.toLowerCase()}.js setup`);
process.exit(EXIT_CODES.MISSING_CONFIG_JSON);
}
else {
// TODO: adjust return type as soon as #2120 merged and we have the type
const _config = fs.readJSONSync(configFile);
if (!_config.states) {
_config.states = { type: 'jsonl' };
}
if (!_config.objects) {
_config.objects = { type: 'jsonl' };
}
if (!_config.system) {
_config.system = {};
}
return _config;
}
}
/**
* Starts the multihost discovery server
*
* @param _config Configuration fron iobroker.json
* @param secret MultiHost communication password
*/
function _startMultihost(_config, secret) {
const cpus = os.cpus();
mhService = new MHServer(hostname, logger, _config, {
node: process.version,
arch: os.arch(),
model: cpus && cpus[0] && cpus[0].model ? cpus[0].model : 'unknown',
cpus: cpus ? cpus.length : 1,
mem: os.totalmem(),
ostype: os.type(),
}, secret);
}
/**
* Starts or stops the multihost discovery server, depending on the config and temp information
*
* @param __config - the iobroker config object
*/
async function startMultihost(__config) {
if (compactGroupController) {
return;
}
if (mhTimer) {
clearTimeout(mhTimer);
mhTimer = null;
}
const _config = __config || getConfig();
if (_config.multihostService?.enabled) {
if (mhService) {
try {
mhService.close(() => {
mhService = null;
setImmediate(() => startMultihost(_config));
});
return;
}
catch (e) {
logger.warn(`${hostLogPrefix} Cannot stop multihost discovery server: ${e.message}`);
}
}
const hasLocalObjectsServer = await isLocalObjectsDbServer(_config.objects.type, _config.objects.host, true);
const hasLocalStatesServer = await isLocalStatesDbServer(_config.states.type, _config.states.host, true);
if (!_config.objects.host || hasLocalObjectsServer) {
logger.warn(`${hostLogPrefix} Multihost Master on this system is not possible, because IP address for objects is ${_config.objects.host}. Please allow remote connections to the server by adjusting the IP.`);
return false;
}
else if (!_config.states.host || hasLocalStatesServer) {
logger.warn(`${hostLogPrefix} Multihost Master on this system is not possible, because IP address for states is ${_config.states.host}. Please allow remote connections to the server by adjusting the IP.`);
return false;
}
if (_config.multihostService.secure) {
if (typeof _config.multihostService.password === 'string' && _config.multihostService.password.length) {
let obj;
let errText;
try {
obj = await objects.getObject(SYSTEM_CONFIG_ID);
}
catch (e) {
// will log error below
errText = e.message;
}
if (obj?.native?.secret) {
if (!_config.multihostService.password.startsWith(`$/aes-192-cbc:`)) {
// if old encryption was used, we need to decrypt in old fashion
tools.decryptPhrase(obj.native.secret, _config.multihostService.password, secret => _startMultihost(_config, secret));
}
else {
try {
// it can throw in edge cases #1474, we need further investigation
const secret = tools.decrypt(obj.native.secret, _config.multihostService.password);
_startMultihost(_config, secret);
}
catch (e) {
logger.error(`${hostLogPrefix} Cannot decrypt password for multihost discovery server: ${e.message}`);
}
}
}
else {
logger.error(`${hostLogPrefix} Cannot start multihost discovery server: no system.config found (err: ${errText})`);
}
}
else {
logger.error(`${hostLogPrefix} Cannot start multihost discovery server: secure mode was configured, but no secret was set. Please check the configuration!`);
}
}
else {
_startMultihost(_config, false);
}
if (!_config.multihostService.persist) {
mhTimer = setTimeout(async () => {
if (mhService) {
try {
mhService.close();
mhService = null;
logger.info(`${hostLogPrefix} Multihost discovery server stopped after 15 minutes, because only temporarily activated`);
_config.multihostService.persist = false;
_config.multihostService.enabled = false;
const configFile = tools.getConfigFileName();
await fs.writeFile(configFile, JSON.stringify(_config, null, 2));
}
catch (e) {
logger.warn(`${hostLogPrefix} Cannot stop multihost discovery: ${e.message}`);
}
}
mhTimer = null;
}, 15 * 60000);
}
return true;
}
else if (mhService) {
try {
mhService.close();
mhService = null;
}
catch (e) {
logger.warn(`${hostLogPrefix} Cannot stop multihost discovery: ${e.message}`);
}
return false;
}
}
/**
* Starts cyclic update of IP interfaces.
* At start every 30 seconds and after 5 minutes, every hour.
* Because DHCP could change the IPs.
*/
function startUpdateIPs() {
if (!updateIPsTimer) {
updateIPsTimer = setInterval(() => {
if (Date.now() - uptimeStart > 5 * 60_000) {
// 5 minutes at start check every 30 seconds because of DHCP
clearInterval(updateIPsTimer);
updateIPsTimer = setInterval(() => setIPs(), 3_600_000); // update IPs every hour
}
setIPs();
}, 30_000);
}
}
// subscribe or unsubscribe loggers
/**
*
* @param isActive
* @param id
* @param reason
*/
function logRedirect(isActive, id, reason) {
console.log(`================================== > LOG REDIRECT ${id} => ${isActive} [${reason}]`);
if (isActive) {
if (!logList.includes(id)) {
logList.push(id);
}
}
else {
const pos = logList.indexOf(id);
if (pos !== -1) {
logList.splice(pos, 1);
}
}
}
function handleDisconnect() {
if (!connected || restartTimeout || isStopping) {
return;
}
if (statesDisconnectTimeout) {
clearTimeout(statesDisconnectTimeout);
statesDisconnectTimeout = null;
}
if (objectsDisconnectTimeout) {
clearTimeout(objectsDisconnectTimeout);
objectsDisconnectTimeout = null;
}
connected = false;
logger.warn(`${hostLogPrefix} Slave controller detected disconnection. Stop all instances.`);
if (compactGroupController) {
stop(true);
}
else {
stop(true, () => {
restartTimeout = setTimeout(() => {
processMessage({ command: 'cmdExec', message: { data: '_restart' }, from: hostObjectPrefix });
setTimeout(() => process.exit(EXIT_CODES.JS_CONTROLLER_STOPPED), 1_000);
}, 10_000);
});
}
}
/**
*
* @param onConnect
*/
function createStates(onConnect) {
states = new States({
namespace: hostLogPrefix,
connection: config.states,
logger: logger,
hostname: hostname,
change: async (id, stateOrMessage) => {
if (!states || !objects) {
logger.error(`${hostLogPrefix} Could not handle state change of "${id}", because not connected`);
return;
}
inputCount++;
if (!id) {
return logger.error(`${hostLogPrefix} change event with no ID: ${JSON.stringify(stateOrMessage)}`);
}
// If some log transporter activated or deactivated
if (id.startsWith(SYSTEM_ADAPTER_PREFIX) && id.endsWith('.logging')) {
const state = stateOrMessage;
logRedirect(state ? state.val : false, id.substring(0, id.length - '.logging'.length), id);
}
else if (!compactGroupController && id === `messagebox.${hostObjectPrefix}`) {
// If this is messagebox, only the main controller is handling the host messages
const obj = stateOrMessage;
if (obj) {
// If callback stored for this request
if (obj.callback && obj.callback.ack && obj.callback.id && callbacks[`_${obj.callback.id}`]) {
callbacks[`_${obj.callback.id}`].cb(obj.message);
delete callbacks[`_${obj.callback.id}`];
// delete too old callbacks IDs
const now = Date.now();
for (const _id of Object.keys(callbacks)) {
if (now - callbacks[_id].time > 3_600_000) {
delete callbacks[_id];
}
}
}
else {
processMessage(obj);
}
}
}
else if (!compactGroupController && id.match(/^system.adapter.[^.]+\.\d+\.alive$/)) {
const state = stateOrMessage;
// If this system.adapter.NAME.0.alive, only main controller is handling this
if (state && !state.ack) {
const enabled = state.val;
let obj;
try {
obj = await objects.getObject(id.substring(0, id.length - 6 /*'.alive'.length*/));
}
catch (e) {
logger.error(`${hostLogPrefix} Cannot read object: ${e.message}`);
}
if (obj?.common) {
// IF adapter enabled => disable it
if ((obj.common.enabled && !enabled) || (!obj.common.enabled && enabled)) {
obj.common.enabled = !!enabled;
logger.info(`${hostLogPrefix} instance "${obj._id}" ${obj.common.enabled ? 'enabled' : 'disabled'} via .alive`);
obj.from = hostObjectPrefix;
obj.ts = Date.now();
try {
await objects.setObject(obj._id, obj);
}
catch (e) {
logger.error(`${hostLogPrefix} Cannot set object: ${e.message}`);
}
}
}
}
}
else if (subscribe[id]) {
const state = stateOrMessage;
for (const sub of subscribe[id]) {
// wake up adapter
if (procs[sub]) {
console.log(`Wake up ${id} ${JSON.stringify(state)}`);
startInstance(sub, true);
}
else {
logger.warn(`${hostLogPrefix} controller Adapter subscribed on ${id} does not exist!`);
}
}
}
else if (id === `${hostObjectPrefix}.logLevel`) {
const state = stateOrMessage;
if (!config || !config.log || !state || state.ack) {
return;
}
let currentLevel = config.log.level;
if (typeof state.val === 'string' &&
state.val !== currentLevel &&
['silly', 'debug', 'info', 'warn', 'error'].includes(state.val)) {
config.log.level = state.val;
for (const transport in logger.transports) {
if (logger.transports[transport].level === currentLevel &&
// @ts-expect-error it's our custom property
!logger.transports[transport]._defaultConfigLoglevel) {
logger.transports[transport].level = state.val;
}
}
logger.info(`${hostLogPrefix} Loglevel changed from "${currentLevel}" to "${state.val}"`);
currentLevel = state.val;
}
else if (state.val && state.val !== currentLevel) {
logger.info(`${hostLogPrefix} Got invalid loglevel "${state.val}", ignoring`);
}
await states.setState(`${hostObjectPrefix}.logLevel`, {
val: currentLevel,
ack: true,
from: hostObjectPrefix,
});
}
else if (id.startsWith(`${hostObjectPrefix}.plugins.`) && id.endsWith('.enabled')) {
const state = stateOrMessage;
if (!config || !config.log || !state || state.ack) {
return;
}
const pluginStatesIndex = `${hostObjectPrefix}.plugins.`.length;
let nameEndIndex = id.indexOf('.', pluginStatesIndex + 1);
if (nameEndIndex === -1) {
nameEndIndex = undefined;
}
const pluginName = id.substring(pluginStatesIndex, nameEndIndex);
if (!pluginHandler.pluginExists(pluginName)) {
return;
}
if (pluginHandler.isPluginActive(pluginName) !== state.val) {
if (state.val) {
if (!pluginHandler.isPluginInstantiated(pluginName)) {
pluginHandler.instantiatePlugin(pluginName, pluginHandler.getPluginConfig(pluginName), controllerDir);
pluginHandler.setDatabaseForPlugin(pluginName, objects, states);
pluginHandler.initPlugin(pluginName, ioPackage);
}
}
else {
if (!pluginHandler.destroy(pluginName)) {
logger.info(`${hostLogPrefix} Plugin ${pluginName} could not be disabled. Please restart ioBroker to disable it.`);
}
}
}
}
else if (id === `${hostObjectPrefix}.diskWarning` &&
stateOrMessage &&
'ack' in stateOrMessage &&
!stateOrMessage.ack) {
const warningLevel = getDiskWarningLevel(stateOrMessage);
diskWarningLevel = warningLevel;
await states.setState(id, { val: warningLevel, ack: true });
}
},
connected: () => {
if (statesDisconnectTimeout) {
clearTimeout(statesDisconnectTimeout);
statesDisconnectTimeout = null;
}
initMessageQueue();
startAliveInterval();
initializeController();
onConnect && onConnect();
},
disconnected: () => {
if (restartTimeout) {
return;
}
statesDisconnectTimeout && clearTimeout(statesDisconnectTimeout);
statesDisconnectTimeout = setTimeout(() => {
statesDisconnectTimeout = null;
handleDisconnect();
}, (config.states.connectTimeout || 2000) + (!compactGroupController ? 500 : 0));
},
});
}
async function initializeController() {
if (!states || !objects || connected) {
return;
}
logger.info(`${hostLogPrefix} connected to Objects and States`);
// initialize notificationHandler
const notificationSettings = {
states: states,
objects: objects,
log: logger,
logPrefix: hostLogPrefix,
host: hostname,
};
notificationHandler = new NotificationHandler(notificationSettings);
if (ioPackage.notifications) {
try {
await notificationHandler.addConfig(ioPackage.notifications);
logger.info(`${hostLogPrefix} added notifications configuration of host`);
// load setup of all adapters to class, to remember messages even of non-running hosts
await notificationHandler.getSetupOfAllAdaptersFromHost();
}
catch (e) {
logger.error(`${hostLogPrefix} Could not add notifications config of this host: ${e.message}`);
}
}
autoUpgradeManager = new AdapterAutoUpgradeManager({ objects, states, logger, logPrefix: hostLogPrefix });
blocklistManager = new BlocklistManager({ objects });
checkSystemLocaleSupported();
if (connected === null) {
connected = true;
if (!isStopping) {
pluginHandler.setDatabaseForPlugins(objects, states);
await pluginHandler.initPlugins(ioPackage);
states.subscribe(`${hostObjectPrefix}.plugins.*`);
// Do not start if we're still stopping the instances
await checkHost();
startMultihost(config);
setMeta();
started = true;
getInstances();
}
}
else {
connected = true;
started = true;
// Do not start if we're still stopping the instances
if (!isStopping) {
getInstances();
}
}
}
// create "objects" object
/**
*
* @param onConnect
*/
function createObjects(onConnect) {
objects = new Objects({
namespace: hostLogPrefix,
connection: config.objects,
controller: true,
logger: logger,
hostname: hostname,
connected: async () => {
// stop disconnect timeout
if (objectsDisconnectTimeout) {
clearTimeout(objectsDisconnectTimeout);
objectsDisconnectTimeout = null;
}
// subscribe to primary host expiration
try {
await objects.subscribePrimaryHost();
}
catch (e) {
logger.error(`${hostLogPrefix} Cannot subscribe to primary host expiration: ${e.message}`);
}
if (!primaryHostInterval && !compactGroupController) {
primaryHostInterval = setInterval(checkPrimaryHost, PRIMARY_HOST_LOCK_TIME / 2);
}
// first execution now
checkPrimaryHost();
initializeController();
onConnect && onConnect();
},
disconnected: ( /*error*/) => {
if (restartTimeout) {
return;
}
// on reconnection this will be determined anew
isPrimary = false;
objectsDisconnectTimeout && clearTimeout(objectsDisconnectTimeout);
objectsDisconnectTimeout = setTimeout(() => {
objectsDisconnectTimeout = null;
handleDisconnect();
}, (config.objects.connectTimeout || 2000) + (!compactGroupController ? 500 : 0));
// give the main controller a bit longer, so that adapter and compact processes can exit before
},
change: async (_id, _obj) => {
if (!started || !_id.match(/^system\.adapter\.[a-zA-Z0-9-_]+\.[0-9]+$/)) {
return;
}
const obj = _obj;
const id = _id;
try {
logger.debug(`${hostLogPrefix} object change ${id} (from: ${obj ? obj.from : null})`);
// known adapter
const proc = procs[id];
if (proc) {
// if adapter deleted
if (!obj) {
// deleted: also remove from an instance list of compactGroup
if (!compactGroupController &&
proc.config.common.compactGroup &&
compactProcs[proc.config.common.compactGroup]?.instances?.includes(id)) {
compactProcs[proc.config.common.compactGroup].instances.splice(compactProcs[proc.config.common.compactGroup].instances.indexOf(id), 1);
}
// instance removed -> remove all notifications
await notificationHandler.clearNotifications(null, null, id);
proc.config.common.enabled = false;
// @ts-expect-error check if we can handle it different
proc.config.common.host = null;
// @ts-expect-error it is only used in checkAndAddInstance, find a way without modifying the InstanceObject
proc.config.deleted = true;
logger.info(`${hostLogPrefix} object deleted ${id}`);
}
else {
if (proc.config.common.enabled && !obj.common.enabled) {
logger.info(`${hostLogPrefix} "${id}" disabled`);
}
if (!proc.config.common.enabled && obj.common.enabled) {
logger.info(`${hostLogPrefix} "${id}" enabled`);
proc.downloadRetry = 0;
}
// Check if compactgroup or compact mode changed
if (!compactGroupController &&
proc.config.common.compactGroup &&
(proc.config.common.compactGroup !== obj.common.compactGroup ||
proc.config.common.runAsCompactMode !== obj.common.runAsCompactMode) &&
compactProcs[proc.config.common.compactGroup]?.instances?.includes(id)) {
compactProcs[proc.config.common.compactGroup].instances.splice(compactProcs[proc.config.common.compactGroup].instances.indexOf(id), 1);
}
proc.config = obj;
}
if (proc.process || proc.config.common.mode === 'schedule') {
proc.restartExpected = true;
await stopInstance(id, false);
if (!procs[id]) {
return;
}
const _ipArr = tools.findIPs();
if (checkAndAddInstance(proc.config, _ipArr)) {
if (proc.config.common.enabled &&
(proc.config.common.mode !== 'extension' || !proc.config.native.webInstance)) {
if (proc.restartTimer) {
clearTimeout(proc.restartTimer);
}
const restartTimeout = (proc.config.common.stopTimeout || 500) + 2_500;
proc.restartTimer = setTimeout(_id => startInstance(_id), restartTimeout, id);
}
}
else {
// moved: also remove from an instance list of compactGroup
if (!compactGroupController &&
proc.config.common.compactGroup &&
compactProcs[proc.config.common.compactGroup]?.instances?.includes(id)) {
compactProcs[proc.config.common.compactGroup].instances.splice(compactProcs[proc.config.common.compactGroup].instances.indexOf(id), 1);
}
if (proc.restartTimer) {
clearTimeout(proc.restartTimer);
delete proc.restartTimer;
}
// instance moved -> remove all notifications, new host has to take care
await notificationHandler.clearNotifications(null, null, id);
delete procs[id];
}
}
else if (installQueue.find(obj => obj.id === id)) {
// ignore object changes when still in the installation queue
logger.debug(`${hostLogPrefix} ignore object change because the adapter is still in installation/rebuild queue`);
}
else {
const _ipArr = tools.findIPs();
if (proc.config && checkAndAddInstance(proc.config, _ipArr)) {
if (proc.config.common.enabled &&
(proc.config.common.mode !== 'extension' || !proc.config.native.webInstance)) {
startInstance(id);
}
}
else {
// moved: also remove from an instance list of compactGroup
if (!compactGroupController &&
proc.config.common.compactGroup &&
compactProcs[proc.config.common.compactGroup]?.instances?.includes(id)) {
compactProcs[proc.config.common.compactGroup].instances.splice(compactProcs[proc.config.common.compactGroup].instances.indexOf(id), 1);
}
if (proc.restartTimer) {
clearTimeout(proc.restartTimer);
delete proc.restartTimer;
}
delete procs[id];
}
}
}
else if (obj?.common) {
const _ipArr = tools.findIPs();
// new adapter
if (!checkAndAddInstance(obj, _ipArr)) {
return;
}
const proc = procs[id];
if (proc.config.common.enabled &&
(proc.config.common.mode !== 'extension' || !proc.config.native.webInstance)) {
// We should give a slight delay to allow a potentially former existing process on another host to exit
const restartTimeout = (proc.config.common.stopTimeout || 500) + 2_500;
proc.restartTimer = setTimeout(_id => startInstance(_id), restartTimeout, id);
}
}
}
catch (err) {
if (!compactGroupController ||
(obj?.common?.runAsCompactMode && obj.common.compactGroup === compactGroup)) {
logger.error(`${hostLogPrefix} cannot process: ${id}: ${err} / ${err.stack}`);
}
}
},
primaryHostLost: () => {
if (!isStopping) {
isPrimary = false;
logger.info('The primary host is no longer active. Checking responsibilities.');
checkPrimaryHost();
}
},
});
}
function startAliveInterval() {
config.system = config.system || {};
config.system.statisticsInterval = Math.round(config.system.statisticsInterval) || 15_000;
config.system.checkDiskInterval =
config.system.checkDiskInterval !== 0 ? Math.round(config.system.checkDiskInterval) || 300_000 : 0;
if (!compactGroupController) {
// Provide info to see for each host if compact is enabled or not and be able to use in Admin or such
states.setState(`${hostObjectPrefix}.compactModeEnabled`, {
ack: true,
from: hostObjectPrefix,
val: config.system.compact || false,
});
}
reportInterval = setInterval(reportStatus, config.system.statisticsInterval);
reportStatus();
tools.measureEventLoopLag(1_000, lag => eventLoopLags.push(lag));
}
/**
* Check if the current redis Locale is supported, else register notification
*/
async function checkSystemLocaleSupported() {
if (!objects) {
throw new Error('Objects database not connected');
}
const isSupported = await objects.isSystemLocaleSupported();
if (!isSupported) {
await notificationHandler.addMessage({
category: 'system',
scope: 'databaseErrors',
message: 'Your redis server is using an unsupported locale. This can lead to unexpected behavior of your ioBroker installation as well as data loss. ' +
'Please configure your Redis Server according to https://forum.iobroker.net/topic/52976/wichtiger-hinweis-f%C3%BCr-redis-installationen?_=1678099836122',
instance: `system.host.${hostname}`,
});
}
}
/**
* Ensures that we take over primary host if no other is doing the job
*/
async function checkPrimaryHost() {
// we cannot interact with db now because currently reconnecting
if (objectsDisconnectTimeout || compactGroupController) {
return;
}
// let our host value live PRIMARY_HOST_LOCK_TIME seconds, while it should be renewed lock time / 2
try {
if (!isPrimary) {
isPrimary = !!(await objects.setPrimaryHost(PRIMARY_HOST_LOCK_TIME));
}
else {
const lockExtended = !!(await objects.extendPrimaryHostLock(PRIMARY_HOST_LOCK_TIME));
if (!lockExtended) {
// if we are host, a lock extension should always work, fallback to acquire lock
isPrimary = !!(await objects.setPrimaryHost(PRIMARY_HOST_LOCK_TIME));
}
}
}
catch (e) {
logger.error(`${hostLogPrefix} Could not execute primary host determination: ${e.message}`);
}
}
async function reportStatus() {
if (!states) {
return;
}
const id = hostObjectPrefix;
outputCount += 10;
states.setState(`${id}.alive`, {
val: true,
ack: true,
expire: Math.floor(config.system.statisticsInterval / 1_000) + 10,
from: id,
});
// provide infos about current process
// pidUsage([pid,pid,...], function (err, stats) {
// => {
// cpu: 10.0, // percentage (from 0 to 100*vcore)
// memory: 357306368, // bytes
// ppid: 312, // PPID
// pid: 727, // PID
// ctime: 867000, // ms user + system time
// elapsed: 6650000, // ms since the start of the process
// timestamp: 864000000 // ms since epoch
// }
try {
pidUsage(process.pid, (err, stats) => {
// controller.s might be stopped, but this is still running
if (!err && states && states.setState && stats) {
states.setState(`${id}.cpu`, {
ack: true,
from: id,
val: Math.round(100 * stats.cpu) / 100,
});
states.setState(`${id}.cputime`, { ack: true, from: id, val: stats.ctime / 1_000 });
outputCount += 2;
}
});
}
catch (e) {
logger.error(`${hostLogPrefix} Cannot read pidUsage data : ${e.message}`);
}
try {
const mem = process.memoryUsage();
states.setState(`${id}.memRss`, {
val: Math.round(mem.rss / 10485.76 /* 1MB / 100 */) / 100,
ack: true,
from: id,
});
states.setState(`${id}.memHeapTotal`, {
val: Math.round(mem.heapTotal / 10485.76 /* 1MB / 100 */) / 100,
ack: true,
from: id,
});
states.setState(`${id}.memHeapUsed`, {
val: Math.round(mem.heapUsed / 10485.76 /* 1MB / 100 */) / 100,
ack: true,
from: id,
});
}
catch (e) {
logger.error(`${hostLogPrefix} Cannot read memoryUsage data: ${e.message}`);
}
// provide machine infos
states.setState(`${id}.load`, { val: Math.round(os.loadavg()[0] * 100) / 100, ack: true, from: id });
states.setState(`${id}.uptime`, { val: Math.round(process.uptime()), ack: true, from: id });
states.setState(`${id}.mem`, { val: Math.round(100 - (os.freemem() / os.totalmem()) * 100), ack: true, from: id });
states.setState(`${id}.freemem`, { val: Math.round(os.freemem() / 1_048_576 /* 1MB */), ack: true, from: id });
if (fs.existsSync('/proc/meminfo')) {
try {
const text = fs.readFileSync('/proc/meminfo', 'utf8');
const m = text && text.match(/MemAvailable:\s*(\d+)/);
if (m && m[1]) {
states.setState(`${id}.memAvailable`, {
val: Math.round(parseInt(m[1], 10) * 0.001024),
ack: true,
from: id,
});
outputCount++;
}
}
catch (e) {
logger.error(`${hostLogPrefix} Cannot read /proc/meminfo: ${e.message}`);
}
}
if (config.system.checkDiskInterval && Date.now() - lastDiskSizeCheck >= config.system.checkDiskInterval) {
lastDiskSizeCheck = Date.now();
let info = null;
try {
info = await tools.getDiskInfo();
}
catch (e) {
logger.error(`${hostLogPrefix} Cannot read disk size: ${e.message}`);
}
try {
if (info) {
const diskSize = Math.round((info['Disk size'] || 0) / (1024 * 1024));
const diskFree = Math.round((info['Disk free'] || 0) / (1024 * 1024));
const percentageFree = (diskFree / diskSize) * 100;
const isDiskWarningActive = percentageFree < diskWarningLevel;
if (isDiskWarningActive) {
await notificationHandler.addMessage({
scope: 'system',
category: 'diskSpaceIssues',
message: `Your system has only ${percentageFree.toFixed(2)} % of disk space left.`,
instance: `system.host.${hostname}`,
});
}
states.setState(`${id}.diskSize`, {
val: diskSize,
ack: true,
from: id,
});
states.setState(`${id}.diskFree`, {
val: diskFree,
ack: true,
from: id,
});
outputCount += 2;
}
}
catch (e) {
logger.error(`${hostLogPrefix} Cannot read disk information: ${e.message}`);
}
}
// some statistics
states.setState(`${id}.inputCount`, { val: inputCount, ack: true, from: id });
states.setState(`${id}.outputCount`, { val: outputCount, ack: true, from: id });
if (eventLoopLags.length) {
const eventLoopLag = Math.ceil(eventLoopLags.reduce((a, b) => a + b) / eventLoopLags.length);
states.setState(`${id}.eventLoopLag`, { val: eventLoopLag, ack: true, from: id }); // average of measured values
eventLoopLags = [];
}
states.setState(`${id}.compactgroupProcesses`, { val: Object.keys(compactProcs).length, ack: true, from: id });
let realProcesses = 0;
let compactProcesses = 0;
Object.values(procs).forEach(proc => {
if (proc.process) {
if (proc.startedInCompactMode) {
compactProcesses++;
}
else {
realProcesses++;
}
}
});
states.setState(`${id}.instancesAsProcess`, { val: realProcesses, ack: true, from: id });
states.setState(`${id}.instancesAsCompact`, { val: compactProcesses, ack: true, from: id });
inputCount = 0;
outputCount = 0;
if (!isStopping && compactGroupController && started && compactProcesses === 0 && realProcesses === 0) {
logger.info(`${hostLogPrefix} Compact group controller ${compactGroup} does not own any processes, stop`);
stop(false);
}
}
/**
*
* @param objs
* @param oldHostname
* @param newHostname
*/
async function changeHost(objs, oldHostname, newHostname) {
for (const row of objs) {
if (row?.value?.common.host === oldHostname) {
const obj = row.value;
obj.common.host = newHostname;
logger.info(`${hostLogPrefix} Reassign instance ${obj._id.substring(SYSTEM_ADAPTER_PREFIX.length)} from ${oldHostname} to ${newHostname}`);
obj.from = `system.host.${tools.getHostName()}`;
obj.ts = Date.now();
try {
await objects.setObject(obj._id, obj);
}
catch (e) {
logger.error(`Error changing host of ${obj._id}: ${e.message}`);
}
}
}
}
/**
* Clean a single auto subscribe
*
* @param instance instance id without `system.adapter.` prefix
* @param autoInstance instance id
* @param callback
*/
function cleanAutoSubscribe(instance, autoInstance, callback) {
inputCount++;
states.getState(`${autoInstance}.subscribes`, async (err, state) => {
if (!state || !state.val) {
return setImmediate(() => callback());
}
let subs;
try {
subs = JSON.parse(state.val);
}
catch {
logger.error(`${hostLogPrefix} Cannot parse subscribes: ${state.val}`);
return setImmediate(() => callback());
}
let modified = false;
// look for all subscribes from this instance
for (const pattern of Object.keys(subs)) {
for (const id of Object.keys(subs[pattern])) {
if (id === instance) {
modified = true;
delete subs[pattern][id];
}
}
// check if the array is now empty
if (!Object.keys(subs[pattern]).length) {
modified = true;
delete subs[pattern];
}
}
if (modified) {
outputCount++;
await states.setState(`${autoInstance}.subscribes`, subs);
}
setImmediate(() => callback());
});
}
/**
*
* @param instanceID
* @param callback
*/
function cleanAutoSubscribes(instanceID, callback) {
const instance = instanceID.substring(15); // get name.0
// read all instances
objects.getObjectView('system', 'instance', { startkey: SYSTEM_ADAPTER_PREFIX, endkey: `${SYSTEM_ADAPTER_PREFIX}\u9999` }, (err, res) => {
let count = 0;
if (res) {
for (const row of res.rows) {
// remove this instance from autoSubscribe
if (row.value?.common.subscribable) {
count++;
cleanAutoSubscribe(instance, row.id, () => !--count && callback && callback());
}
}
}
!count && callback && callback();
});
}
/**
*
* @param objs
*/
async function delObjects(objs) {
for (const row of objs) {
if (row?.id) {
logger.info(`${hostLogPrefix} Delete state "${row.id}"`);
try {
if (row.value && row.value.type === 'state') {
await states.delState(row.id);
await objects.delObject(row.id);
}
else {
await objects.delObject(row.id);
}
}
catch {
// ignore
}
}
}
}
/**
* try to check host in objects
* <p>
* This function tries to find all hosts in the objects and if
* only one host found and it is not actual host, change the
* host name to new one.
* <p>
*
*/
async function checkHost() {
const objectData = objects.getStatus();
// only file master host controller needs to check/fix the host assignments from the instances
// for redis it is currently not possible to detect a single host system with a changed hostname for sure!
if (compactGroupController || !objectData.server) {
return;
}
let hostDoc;
try {
hostDoc = await objects.getObjectViewAsync('system', 'host', {
startkey: 'system.host.',
endkey: 'system.host.\u9999',
});
}
catch {
// ignore
}
if (hostDoc?.rows.length === 1 && hostDoc?.rows[0].value.common.name !== hostname) {
const oldHostname = hostDoc.rows[0].value.common.name;
const oldId = hostDoc.rows[0].value._id;
let instanceDoc;
try {
// find out all instances and rewrite it to actual hostname
instanceDoc = await objects.getObjectViewAsync('system', 'instance', {
startkey: SYSTEM_ADAPTER_PREFIX,
endkey: `${SYSTEM_ADAPTER_PREFIX}\u9999`,
});
}
catch (e) {
if (e.message.startsWith('Cannot find ')) {
return;
}
}
if (!instanceDoc?.rows || instanceDoc.rows.length === 0) {
logger.info(`${hostLogPrefix} no instances found`);
// no instances found
return;
}
// reassign all instances
await changeHost(instanceDoc.rows, oldHostname, hostname);
logger.info(`${hostLogPrefix} Delete host ${oldId}`);
try {
// delete host object
await objects.delObjectAsync(oldId);
}
catch {
// ignore
}
try {
// delete all hosts states
const newHostDoc = await objects.getObjectViewAsync('system', 'state', {
startkey: `system.host.${oldHostname}.`,
endkey: `system.host.${oldHostname}.\u9999`,
include_docs: true,
});
await delObjects(newHostDoc.rows);
return;
}
catch {
// ignore
}
}
}
/**
* Collects the dialog information, e.g., used by Admin "System Settings"
*
* @param type - type of required information
*/
async function collectDiagInfo(type) {
if (type !== 'extended' && type !== 'normal' && type !== 'no-city') {
return null;
}
let systemConfig;
let err;
try {
systemConfig = await objects.getObject(SYSTEM_CONFIG_ID);
}
catch (e) {
err = e;
}
if (err || !systemConfig?.common) {
logger.warn(`System config object is corrupt, please run "${tools.appNameLowerCase} setup first". Error: ${err.message}`);
systemConfig = systemConfig || { common: {} };
systemConfig.common = systemConfig.common || {};
}
let obj;
try {
obj = await objects.getObjectAsync('system.meta.uuid');
}
catch {
// ignore obj is undefined
}
// create uuid
if (!obj) {
obj = { native: { uuid: 'not found' } };
}
let doc;
err = null;
try {
doc = await objects.getObjectViewAsync('system', 'host', {
startkey: 'system.host.',
endkey: 'system.host.\u9999',
});
}
catch (e) {
err = e;
}
const { noCompactInstances, noInstances } = await _getNumberOfInstances();
// we need to show city and country at the beginning, so include it now and delete it later if not allowed.
const diag = {
uuid: obj.native.uuid,
language: systemConfig.common.language,
country: '',
city: '',
hosts: [],
node: process.version,
arch: os.arch(),
docker: tools.isDocker(),
adapters: {},
statesType: config.states.type, // redis or file
objectsType: config.objects.type, // redis or file
noInstances,
compactMode: config.system.compact,
noCompactInstances,
};
if (type === 'extended' || type === 'no-city') {
const cpus = os.cpus();
diag.country = 'country' in systemConfig.common ? systemConfig.common.country : 'unknown';
diag.model = cpus && cpus[0] && cpus[0].model ? cpus[0].model : 'unknown';
diag.cpus = cpus ? cpus.length : 1;
diag.mem = os.totalmem();