meshcentral
Version:
Web based remote computer management server
794 lines (734 loc) • 301 kB
JavaScript
/**
* @description MeshCentral database module
* @author Ylian Saint-Hilaire
* @copyright Intel Corporation 2018-2022
* @license Apache-2.0
* @version v0.0.2
*/
/*xjslint node: true */
/*xjslint plusplus: true */
/*xjslint maxlen: 256 */
/*jshint node: true */
/*jshint strict: false */
/*jshint esversion: 6 */
"use strict";
//
// Construct Meshcentral database object
//
// The default database is NeDB
// https://github.com/louischatriot/nedb
//
// Alternativety, MongoDB can be used
// https://www.mongodb.com/
// Just run with --mongodb [connectionstring], where the connection string is documented here: https://docs.mongodb.com/manual/reference/connection-string/
// The default collection is "meshcentral", but you can override it using --mongodbcol [collection]
//
module.exports.CreateDB = function (parent, func) {
var obj = {};
var Datastore = null;
var expireEventsSeconds = (60 * 60 * 24 * 20); // By default, expire events after 20 days (1728000). (Seconds * Minutes * Hours * Days)
var expirePowerEventsSeconds = (60 * 60 * 24 * 10); // By default, expire power events after 10 days (864000). (Seconds * Minutes * Hours * Days)
var expireServerStatsSeconds = (60 * 60 * 24 * 30); // By default, expire server stats after 30 days (2592000). (Seconds * Minutes * Hours * Days)
const common = require('./common.js');
const path = require('path');
const fs = require('fs');
const DB_NEDB = 1, DB_MONGOJS = 2, DB_MONGODB = 3,DB_MARIADB = 4, DB_MYSQL = 5, DB_POSTGRESQL = 6, DB_ACEBASE = 7, DB_SQLITE = 8;
const DB_LIST = ['None', 'NeDB', 'MongoJS', 'MongoDB', 'MariaDB', 'MySQL', 'PostgreSQL', 'AceBase', 'SQLite']; //for the info command
let databaseName = 'meshcentral';
let datapathParentPath = path.dirname(parent.datapath);
let datapathFoldername = path.basename(parent.datapath);
const SQLITE_AUTOVACUUM = ['none', 'full', 'incremental'];
const SQLITE_SYNCHRONOUS = ['off', 'normal', 'full', 'extra'];
obj.sqliteConfig = {
maintenance: '',
startupVacuum: false,
autoVacuum: 'full',
incrementalVacuum: 100,
journalMode: 'delete',
journalSize: 4096000,
synchronous: 'full',
};
obj.performingBackup = false;
const BACKUPFAIL_ZIPCREATE = 0x0001;
const BACKUPFAIL_ZIPMODULE = 0x0010;
const BACKUPFAIL_DBDUMP = 0x0100;
obj.backupStatus = 0x0;
obj.newAutoBackupFile = null;
obj.newDBDumpFile = null;
obj.identifier = null;
obj.dbKey = null;
obj.dbRecordsEncryptKey = null;
obj.dbRecordsDecryptKey = null;
obj.changeStream = false;
obj.pluginsActive = ((parent.config) && (parent.config.settings) && (parent.config.settings.plugins != null) && (parent.config.settings.plugins != false) && ((typeof parent.config.settings.plugins != 'object') || (parent.config.settings.plugins.enabled != false)));
obj.dbCounters = {
fileSet: 0,
fileRemove: 0,
powerSet: 0,
eventsSet: 0
}
// MongoDB bulk operations state
if (parent.config.settings.mongodbbulkoperations) {
// Added counters
obj.dbCounters.fileSetPending = 0;
obj.dbCounters.fileSetBulk = 0;
obj.dbCounters.fileRemovePending = 0;
obj.dbCounters.fileRemoveBulk = 0;
obj.dbCounters.powerSetPending = 0;
obj.dbCounters.powerSetBulk = 0;
obj.dbCounters.eventsSetPending = 0;
obj.dbCounters.eventsSetBulk = 0;
/// Added bulk accumulators
obj.filePendingGet = null;
obj.filePendingGets = null;
obj.filePendingRemove = null;
obj.filePendingRemoves = null;
obj.filePendingSet = false;
obj.filePendingSets = null;
obj.filePendingCb = null;
obj.filePendingCbs = null;
obj.powerFilePendingSet = false;
obj.powerFilePendingSets = null;
obj.powerFilePendingCb = null;
obj.powerFilePendingCbs = null;
obj.eventsFilePendingSet = false;
obj.eventsFilePendingSets = null;
obj.eventsFilePendingCb = null;
obj.eventsFilePendingCbs = null;
}
obj.SetupDatabase = function (func) {
// Check if the database unique identifier is present
// This is used to check that in server peering mode, everyone is using the same database.
obj.Get('DatabaseIdentifier', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (Get DatabaseIdentifier): ' + err); }
if ((err == null) && (docs.length == 1) && (docs[0].value != null)) {
obj.identifier = docs[0].value;
} else {
obj.identifier = Buffer.from(require('crypto').randomBytes(48), 'binary').toString('hex');
obj.Set({ _id: 'DatabaseIdentifier', value: obj.identifier });
}
});
// Load database schema version and check if we need to update
obj.Get('SchemaVersion', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (Get SchemaVersion): ' + err); }
var ver = 0;
if ((err == null) && (docs.length == 1)) { ver = docs[0].value; }
if (ver == 1) { console.log('This is an unsupported beta 1 database, delete it to create a new one.'); process.exit(0); }
// TODO: Any schema upgrades here...
obj.Set({ _id: 'SchemaVersion', value: 2 });
func(ver);
});
};
// Perform database maintenance
obj.maintenance = function () {
parent.debug('db', 'Entering database maintenance');
if (obj.databaseType == DB_NEDB) { // NeDB will not remove expired records unless we try to access them. This will force the removal.
obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
obj.serverstatsfile.remove({ time: { '$lt': new Date(Date.now() - (expireServerStatsSeconds * 1000)) } }, { multi: true }); // Force delete older events
} else if ((obj.databaseType == DB_MARIADB) || (obj.databaseType == DB_MYSQL)) { // MariaDB or MySQL
sqlDbQuery('DELETE FROM events WHERE time < ?', [new Date(Date.now() - (expireEventsSeconds * 1000))], function (doc, err) { }); // Delete events older than expireEventsSeconds
sqlDbQuery('DELETE FROM power WHERE time < ?', [new Date(Date.now() - (expirePowerEventsSeconds * 1000))], function (doc, err) { }); // Delete events older than expirePowerSeconds
sqlDbQuery('DELETE FROM serverstats WHERE expire < ?', [new Date()], function (doc, err) { }); // Delete events where expiration date is in the past
sqlDbQuery('DELETE FROM smbios WHERE expire < ?', [new Date()], function (doc, err) { }); // Delete events where expiration date is in the past
} else if (obj.databaseType == DB_ACEBASE) { // AceBase
//console.log('Performing AceBase maintenance');
obj.file.query('events').filter('time', '<', new Date(Date.now() - (expireEventsSeconds * 1000))).remove().then(function () {
obj.file.query('stats').filter('time', '<', new Date(Date.now() - (expireServerStatsSeconds * 1000))).remove().then(function () {
obj.file.query('power').filter('time', '<', new Date(Date.now() - (expirePowerEventsSeconds * 1000))).remove().then(function () {
//console.log('AceBase maintenance done');
});
});
});
} else if (obj.databaseType == DB_SQLITE) { // SQLite3
//sqlite does not return rows affected for INSERT, UPDATE or DELETE statements, see https://www.sqlite.org/pragma.html#pragma_count_changes
obj.file.serialize(function () {
obj.file.run('DELETE FROM events WHERE time < ?', [new Date(Date.now() - (expireEventsSeconds * 1000))]);
obj.file.run('DELETE FROM power WHERE time < ?', [new Date(Date.now() - (expirePowerEventsSeconds * 1000))]);
obj.file.run('DELETE FROM serverstats WHERE expire < ?', [new Date()]);
obj.file.run('DELETE FROM smbios WHERE expire < ?', [new Date()]);
obj.file.exec(obj.sqliteConfig.maintenance, function (err) {
if (err) {console.log('Maintenance error: ' + err.message)};
if (parent.config.settings.debug) {
sqliteGetPragmas(['freelist_count', 'page_size', 'page_count', 'cache_size' ], function (pragma, pragmaValue) {
parent.debug('db', 'SQLite Maintenance: ' + pragma + '=' + pragmaValue);
});
};
});
});
}
obj.removeInactiveDevices();
}
// Remove inactive devices
obj.removeInactiveDevices = function (showall, cb) {
// Get a list of domains and what their inactive device removal setting is
var removeInactiveDevicesPerDomain = {}, minRemoveInactiveDevicesPerDomain = {}, minRemoveInactiveDevice = 9999;
for (var i in parent.config.domains) {
if (typeof parent.config.domains[i].autoremoveinactivedevices == 'number') {
var v = parent.config.domains[i].autoremoveinactivedevices;
if ((v >= 1) && (v <= 2000)) {
if (v < minRemoveInactiveDevice) { minRemoveInactiveDevice = v; }
removeInactiveDevicesPerDomain[i] = v;
minRemoveInactiveDevicesPerDomain[i] = v;
}
}
}
// Check if any device groups have a inactive device removal setting
for (var i in parent.webserver.meshes) {
if (typeof parent.webserver.meshes[i].expireDevs == 'number') {
var v = parent.webserver.meshes[i].expireDevs;
if ((v >= 1) && (v <= 2000)) {
if (v < minRemoveInactiveDevice) { minRemoveInactiveDevice = v; }
if ((minRemoveInactiveDevicesPerDomain[parent.webserver.meshes[i].domain] == null) || (minRemoveInactiveDevicesPerDomain[parent.webserver.meshes[i].domain] > v)) {
minRemoveInactiveDevicesPerDomain[parent.webserver.meshes[i].domain] = v;
}
} else {
delete parent.webserver.meshes[i].expireDevs;
}
}
}
// If there are no such settings for any domain, we can exit now.
if (minRemoveInactiveDevice == 9999) { if (cb) { cb("No device removal policy set, nothing to do."); } return; }
const now = Date.now();
// For each domain with a inactive device removal setting, get a list of last device connections
for (var domainid in minRemoveInactiveDevicesPerDomain) {
obj.GetAllTypeNoTypeField('lastconnect', domainid, function (err, docs) {
if ((err != null) || (docs == null)) return;
for (var j in docs) {
const days = Math.floor((now - docs[j].time) / 86400000); // Calculate the number of inactive days
var expireDays = -1;
if (removeInactiveDevicesPerDomain[docs[j].domain]) { expireDays = removeInactiveDevicesPerDomain[docs[j].domain]; }
const mesh = parent.webserver.meshes[docs[j].meshid];
if (mesh && (typeof mesh.expireDevs == 'number')) { expireDays = mesh.expireDevs; }
var remove = false;
if (expireDays > 0) {
if (expireDays < days) { remove = true; }
if (cb) { if (showall || remove) { cb(docs[j]._id.substring(2) + ', ' + days + ' days, expire ' + expireDays + ' days' + (remove ? ', removing' : '')); } }
if (remove) {
// Check if this device is connected right now
const nodeid = docs[j]._id.substring(2);
const conn = parent.GetConnectivityState(nodeid);
if (conn == null) {
// Remove the device
obj.Get(nodeid, function (err, docs) {
if (err != null) return;
if ((docs == null) || (docs.length != 1)) { obj.Remove('lc' + nodeid); return; } // Remove last connect time
const node = docs[0];
// Delete this node including network interface information, events and timeline
obj.Remove(node._id); // Remove node with that id
obj.Remove('if' + node._id); // Remove interface information
obj.Remove('nt' + node._id); // Remove notes
obj.Remove('lc' + node._id); // Remove last connect time
obj.Remove('si' + node._id); // Remove system information
obj.Remove('al' + node._id); // Remove error log last time
if (obj.RemoveSMBIOS) { obj.RemoveSMBIOS(node._id); } // Remove SMBios data
obj.RemoveAllNodeEvents(node._id); // Remove all events for this node
obj.removeAllPowerEventsForNode(node._id); // Remove all power events for this node
if (typeof node.pmt == 'string') { obj.Remove('pmt_' + node.pmt); } // Remove Push Messaging Token
obj.Get('ra' + node._id, function (err, nodes) {
if ((nodes != null) && (nodes.length == 1)) { obj.Remove('da' + nodes[0].daid); } // Remove diagnostic agent to real agent link
obj.Remove('ra' + node._id); // Remove real agent to diagnostic agent link
});
// Remove any user node links
if (node.links != null) {
for (var i in node.links) {
if (i.startsWith('user/')) {
var cuser = parent.webserver.users[i];
if ((cuser != null) && (cuser.links != null) && (cuser.links[node._id] != null)) {
// Remove the user link & save the user
delete cuser.links[node._id];
if (Object.keys(cuser.links).length == 0) { delete cuser.links; }
obj.SetUser(cuser);
// Notify user change
var targets = ['*', 'server-users', cuser._id];
var event = { etype: 'user', userid: cuser._id, username: cuser.name, action: 'accountchange', msgid: 86, msgArgs: [cuser.name], msg: 'Removed user device rights for ' + cuser.name, domain: node.domain, account: parent.webserver.CloneSafeUser(cuser) };
if (obj.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to change the user. Another event will come.
parent.DispatchEvent(targets, obj, event);
}
}
}
}
// Event node deletion
var meshname = '(unknown)';
if ((parent.webserver.meshes[node.meshid] != null) && (parent.webserver.meshes[node.meshid].name != null)) { meshname = parent.webserver.meshes[node.meshid].name; }
var event = { etype: 'node', action: 'removenode', nodeid: node._id, msgid: 87, msgArgs: [node.name, meshname], msg: 'Removed device ' + node.name + ' from device group ' + meshname, domain: node.domain };
// TODO: We can't use the changeStream for node delete because we will not know the meshid the device was in.
//if (obj.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to remove the node. Another event will come.
parent.DispatchEvent(parent.webserver.CreateNodeDispatchTargets(node.meshid, node._id), obj, event);
});
}
}
}
}
});
}
}
// Remove all reference to a domain from the database
obj.removeDomain = function (domainName, func) {
var pendingCalls;
// Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now.
if (obj.databaseType == DB_ACEBASE) {
// AceBase
pendingCalls = 3;
obj.file.query('meshcentral').filter('domain', '==', domainName).remove().then(function () { if (--pendingCalls == 0) { func(); } });
obj.file.query('events').filter('domain', '==', domainName).remove().then(function () { if (--pendingCalls == 0) { func(); } });
obj.file.query('power').filter('domain', '==', domainName).remove().then(function () { if (--pendingCalls == 0) { func(); } });
} else if ((obj.databaseType == DB_MARIADB) || (obj.databaseType == DB_MYSQL) || (obj.databaseType == DB_POSTGRESQL)) {
// MariaDB, MySQL or PostgreSQL
pendingCalls = 2;
sqlDbQuery('DELETE FROM main WHERE domain = $1', [domainName], function () { if (--pendingCalls == 0) { func(); } });
sqlDbQuery('DELETE FROM events WHERE domain = $1', [domainName], function () { if (--pendingCalls == 0) { func(); } });
} else if (obj.databaseType == DB_MONGODB) {
// MongoDB
pendingCalls = 3;
obj.file.deleteMany({ domain: domainName }, { multi: true }, function () { if (--pendingCalls == 0) { func(); } });
obj.eventsfile.deleteMany({ domain: domainName }, { multi: true }, function () { if (--pendingCalls == 0) { func(); } });
obj.powerfile.deleteMany({ domain: domainName }, { multi: true }, function () { if (--pendingCalls == 0) { func(); } });
} else {
// NeDB or MongoJS
pendingCalls = 3;
obj.file.remove({ domain: domainName }, { multi: true }, function () { if (--pendingCalls == 0) { func(); } });
obj.eventsfile.remove({ domain: domainName }, { multi: true }, function () { if (--pendingCalls == 0) { func(); } });
obj.powerfile.remove({ domain: domainName }, { multi: true }, function () { if (--pendingCalls == 0) { func(); } });
}
}
obj.cleanup = function (func) {
// TODO: Remove all mesh links to invalid users
// TODO: Remove all meshes that dont have any links
// Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now.
if ((obj.databaseType == DB_MARIADB) || (obj.databaseType == DB_MYSQL) || (obj.databaseType == DB_POSTGRESQL)) {
// MariaDB, MySQL or PostgreSQL
obj.RemoveAllOfType('event', function () { });
obj.RemoveAllOfType('power', function () { });
obj.RemoveAllOfType('smbios', function () { });
} else if (obj.databaseType == DB_MONGODB) {
// MongoDB
obj.file.deleteMany({ type: 'event' }, { multi: true });
obj.file.deleteMany({ type: 'power' }, { multi: true });
obj.file.deleteMany({ type: 'smbios' }, { multi: true });
} else if ((obj.databaseType == DB_NEDB) || (obj.databaseType == DB_MONGOJS)) {
// NeDB or MongoJS
obj.file.remove({ type: 'event' }, { multi: true });
obj.file.remove({ type: 'power' }, { multi: true });
obj.file.remove({ type: 'smbios' }, { multi: true });
}
// List of valid identifiers
var validIdentifiers = {}
// Load all user groups
obj.GetAllType('ugrp', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (GetAll user): ' + err); }
if ((err == null) && (docs.length > 0)) {
for (var i in docs) {
// Add this as a valid user identifier
validIdentifiers[docs[i]._id] = 1;
}
}
// Fix all of the creating & login to ticks by seconds, not milliseconds.
obj.GetAllType('user', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (GetAll user): ' + err); }
if ((err == null) && (docs.length > 0)) {
for (var i in docs) {
var fixed = false;
// Add this as a valid user identifier
validIdentifiers[docs[i]._id] = 1;
// Fix email address capitalization
if (docs[i].email && (docs[i].email != docs[i].email.toLowerCase())) {
docs[i].email = docs[i].email.toLowerCase(); fixed = true;
}
// Fix account creation
if (docs[i].creation) {
if (docs[i].creation > 1300000000000) { docs[i].creation = Math.floor(docs[i].creation / 1000); fixed = true; }
if ((docs[i].creation % 1) != 0) { docs[i].creation = Math.floor(docs[i].creation); fixed = true; }
}
// Fix last account login
if (docs[i].login) {
if (docs[i].login > 1300000000000) { docs[i].login = Math.floor(docs[i].login / 1000); fixed = true; }
if ((docs[i].login % 1) != 0) { docs[i].login = Math.floor(docs[i].login); fixed = true; }
}
// Fix last password change
if (docs[i].passchange) {
if (docs[i].passchange > 1300000000000) { docs[i].passchange = Math.floor(docs[i].passchange / 1000); fixed = true; }
if ((docs[i].passchange % 1) != 0) { docs[i].passchange = Math.floor(docs[i].passchange); fixed = true; }
}
// Fix subscriptions
if (docs[i].subscriptions != null) { delete docs[i].subscriptions; fixed = true; }
// Save the user if needed
if (fixed) { obj.Set(docs[i]); }
}
// Remove all objects that have a "meshid" that no longer points to a valid mesh.
// Fix any incorrectly escaped user identifiers
obj.GetAllType('mesh', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (GetAll mesh): ' + err); }
var meshlist = [];
if ((err == null) && (docs.length > 0)) {
for (var i in docs) {
var meshChange = false;
docs[i] = common.unEscapeLinksFieldName(docs[i]);
meshlist.push(docs[i]._id);
// Make sure all mesh types are number type, if not, fix it.
if (typeof docs[i].mtype == 'string') { docs[i].mtype = parseInt(docs[i].mtype); meshChange = true; }
// If the device group is deleted, remove any invite codes
if (docs[i].deleted && docs[i].invite) { delete docs[i].invite; meshChange = true; }
// Take a look at the links
if (docs[i].links != null) {
for (var j in docs[i].links) {
if (validIdentifiers[j] == null) {
// This identifier is not known, let see if we can fix it.
var xid = j, xid2 = common.unEscapeFieldName(xid);
while ((xid != xid2) && (validIdentifiers[xid2] == null)) { xid = xid2; xid2 = common.unEscapeFieldName(xid2); }
if (validIdentifiers[xid2] == 1) {
//console.log('Fixing id: ' + j + ' to ' + xid2);
docs[i].links[xid2] = docs[i].links[j];
delete docs[i].links[j];
meshChange = true;
} else {
// TODO: here, we may want to clean up links to users and user groups that do not exist anymore.
//console.log('Unknown id: ' + j);
}
}
}
}
// Save the updated device group if needed
if (meshChange) { obj.Set(docs[i]); }
}
}
if (obj.databaseType == DB_SQLITE) {
// SQLite
} else if (obj.databaseType == DB_ACEBASE) {
// AceBase
} else if (obj.databaseType == DB_POSTGRESQL) {
// Postgres
sqlDbQuery('DELETE FROM Main WHERE ((extra != NULL) AND (extra LIKE (\'mesh/%\')) AND (extra != ANY ($1)))', [meshlist], function (err, response) { });
} else if ((obj.databaseType == DB_MARIADB) || (obj.databaseType == DB_MYSQL)) {
// MariaDB
sqlDbQuery('DELETE FROM Main WHERE (extra LIKE ("mesh/%") AND (extra NOT IN ?)', [meshlist], function (err, response) { });
} else if (obj.databaseType == DB_MONGODB) {
// MongoDB
obj.file.deleteMany({ meshid: { $exists: true, $nin: meshlist } }, { multi: true });
} else {
// NeDB or MongoJS
obj.file.remove({ meshid: { $exists: true, $nin: meshlist } }, { multi: true });
}
// We are done
validIdentifiers = null;
if (func) { func(); }
});
}
});
});
};
// Get encryption key
obj.getEncryptDataKey = function (password, salt, iterations) {
if (typeof password != 'string') return null;
let key;
try {
key = parent.crypto.pbkdf2Sync(password, salt, iterations, 32, 'sha384');
} catch (ex) {
// If this previous call fails, it's probably because older pbkdf2 did not specify the hashing function, just use the default.
key = parent.crypto.pbkdf2Sync(password, salt, iterations, 32);
}
return key
}
// Encrypt data
obj.encryptData = function (password, plaintext) {
let encryptionVersion = 0x01;
let iterations = 100000
const iv = parent.crypto.randomBytes(16);
var key = obj.getEncryptDataKey(password, iv, iterations);
if (key == null) return null;
const aes = parent.crypto.createCipheriv('aes-256-gcm', key, iv);
var ciphertext = aes.update(plaintext);
let versionbuf = Buffer.allocUnsafe(2);
versionbuf.writeUInt16BE(encryptionVersion);
let iterbuf = Buffer.allocUnsafe(4);
iterbuf.writeUInt32BE(iterations);
let encryptedBuf = aes.final();
ciphertext = Buffer.concat([versionbuf, iterbuf, aes.getAuthTag(), iv, ciphertext, encryptedBuf]);
return ciphertext.toString('base64');
}
// Decrypt data
obj.decryptData = function (password, ciphertext) {
// Adding an encryption version lets us avoid try catching in the future
let ciphertextBytes = Buffer.from(ciphertext, 'base64');
let encryptionVersion = ciphertextBytes.readUInt16BE(0);
try {
switch (encryptionVersion) {
case 0x01:
let iterations = ciphertextBytes.readUInt32BE(2);
let authTag = ciphertextBytes.slice(6, 22);
const iv = ciphertextBytes.slice(22, 38);
const data = ciphertextBytes.slice(38);
let key = obj.getEncryptDataKey(password, iv, iterations);
if (key == null) return null;
const aes = parent.crypto.createDecipheriv('aes-256-gcm', key, iv);
aes.setAuthTag(authTag);
let plaintextBytes = Buffer.from(aes.update(data));
plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
return plaintextBytes;
default:
return obj.oldDecryptData(password, ciphertextBytes);
}
} catch (ex) { return obj.oldDecryptData(password, ciphertextBytes); }
}
// Encrypt data
// The older encryption system uses CBC without integraty checking.
// This method is kept only for testing
obj.oldEncryptData = function (password, plaintext) {
let key = parent.crypto.createHash('sha384').update(password).digest('raw').slice(0, 32);
if (key == null) return null;
const iv = parent.crypto.randomBytes(16);
const aes = parent.crypto.createCipheriv('aes-256-cbc', key, iv);
var ciphertext = aes.update(plaintext);
ciphertext = Buffer.concat([iv, ciphertext, aes.final()]);
return ciphertext.toString('base64');
}
// Decrypt data
// The older encryption system uses CBC without integraty checking.
// This method is kept only to convert the old encryption to the new one.
obj.oldDecryptData = function (password, ciphertextBytes) {
if (typeof password != 'string') return null;
try {
const iv = ciphertextBytes.slice(0, 16);
const data = ciphertextBytes.slice(16);
let key = parent.crypto.createHash('sha384').update(password).digest('raw').slice(0, 32);
const aes = parent.crypto.createDecipheriv('aes-256-cbc', key, iv);
let plaintextBytes = Buffer.from(aes.update(data));
plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
return plaintextBytes;
} catch (ex) { return null; }
}
// Get the number of records in the database for various types, this is the slow NeDB way.
// WARNING: This is a terrible query for database performance. Only do this when needed. This query will look at almost every document in the database.
obj.getStats = function (func) {
if (obj.databaseType == DB_ACEBASE) {
// AceBase
// TODO
} else if (obj.databaseType == DB_POSTGRESQL) {
// PostgreSQL
// TODO
} else if (obj.databaseType == DB_MYSQL) {
// MySQL
// TODO
} else if (obj.databaseType == DB_MARIADB) {
// MariaDB
// TODO
} else if (obj.databaseType == DB_MONGODB) {
// MongoDB
obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }]).toArray(function (err, docs) {
var counters = {}, totalCount = 0;
if (err == null) { for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } }
func(counters);
});
} else if (obj.databaseType == DB_MONGOJS) {
// MongoJS
obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }], function (err, docs) {
var counters = {}, totalCount = 0;
if (err == null) { for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } }
func(counters);
});
} else if (obj.databaseType == DB_NEDB) {
// NeDB version
obj.file.count({ type: 'node' }, function (err, nodeCount) {
obj.file.count({ type: 'mesh' }, function (err, meshCount) {
obj.file.count({ type: 'user' }, function (err, userCount) {
obj.file.count({ type: 'sysinfo' }, function (err, sysinfoCount) {
obj.file.count({ type: 'note' }, function (err, noteCount) {
obj.file.count({ type: 'iploc' }, function (err, iplocCount) {
obj.file.count({ type: 'ifinfo' }, function (err, ifinfoCount) {
obj.file.count({ type: 'cfile' }, function (err, cfileCount) {
obj.file.count({ type: 'lastconnect' }, function (err, lastconnectCount) {
obj.file.count({}, function (err, totalCount) {
func({ node: nodeCount, mesh: meshCount, user: userCount, sysinfo: sysinfoCount, iploc: iplocCount, note: noteCount, ifinfo: ifinfoCount, cfile: cfileCount, lastconnect: lastconnectCount, total: totalCount });
});
});
});
});
});
});
});
});
});
});
}
}
// This is used to rate limit a number of operation per day. Returns a startValue each new days, but you can substract it and save the value in the db.
obj.getValueOfTheDay = function (id, startValue, func) { obj.Get(id, function (err, docs) { var date = new Date(), t = date.toLocaleDateString(); if ((err == null) && (docs.length == 1)) { var r = docs[0]; if (r.day == t) { func({ _id: id, value: r.value, day: t }); return; } } func({ _id: id, value: startValue, day: t }); }); };
obj.escapeBase64 = function escapeBase64(val) { return (val.replace(/\+/g, '@').replace(/\//g, '$')); }
// Encrypt an database object
obj.performRecordEncryptionRecode = function (func) {
var count = 0;
obj.GetAllType('user', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (performRecordEncryptionRecode): ' + err); }
if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
obj.GetAllType('node', function (err, docs) {
if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
obj.GetAllType('mesh', function (err, docs) {
if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
if (obj.databaseType == DB_NEDB) { // If we are using NeDB, compact the database.
obj.file.compactDatafile();
obj.file.on('compaction.done', function () { func(count); }); // It's important to wait for compaction to finish before exit, otherwise NeDB may corrupt.
} else {
func(count); // For all other databases, normal exit.
}
});
});
});
}
// Encrypt an database object
function performTypedRecordDecrypt(data) {
if ((data == null) || (obj.dbRecordsDecryptKey == null) || (typeof data != 'object')) return data;
for (var i in data) {
if ((data[i] == null) || (typeof data[i] != 'object')) continue;
data[i] = performPartialRecordDecrypt(data[i]);
if ((data[i].intelamt != null) && (typeof data[i].intelamt == 'object') && (data[i].intelamt._CRYPT)) { data[i].intelamt = performPartialRecordDecrypt(data[i].intelamt); }
if ((data[i].amt != null) && (typeof data[i].amt == 'object') && (data[i].amt._CRYPT)) { data[i].amt = performPartialRecordDecrypt(data[i].amt); }
if ((data[i].kvm != null) && (typeof data[i].kvm == 'object') && (data[i].kvm._CRYPT)) { data[i].kvm = performPartialRecordDecrypt(data[i].kvm); }
}
return data;
}
// Encrypt an database object
function performTypedRecordEncrypt(data) {
if (obj.dbRecordsEncryptKey == null) return data;
if (data.type == 'user') { return performPartialRecordEncrypt(Clone(data), ['otpkeys', 'otphkeys', 'otpsecret', 'salt', 'hash', 'oldpasswords']); }
else if ((data.type == 'node') && (data.ssh || data.rdp || data.intelamt)) {
var xdata = Clone(data);
if (data.ssh || data.rdp) { xdata = performPartialRecordEncrypt(xdata, ['ssh', 'rdp']); }
if (data.intelamt) { xdata.intelamt = performPartialRecordEncrypt(xdata.intelamt, ['pass', 'mpspass']); }
return xdata;
}
else if ((data.type == 'mesh') && (data.amt || data.kvm)) {
var xdata = Clone(data);
if (data.amt) { xdata.amt = performPartialRecordEncrypt(xdata.amt, ['password']); }
if (data.kvm) { xdata.kvm = performPartialRecordEncrypt(xdata.kvm, ['pass']); }
return xdata;
}
return data;
}
// Encrypt an object and return a buffer.
function performPartialRecordEncrypt(plainobj, encryptNames) {
if (typeof plainobj != 'object') return plainobj;
var enc = {}, enclen = 0;
for (var i in encryptNames) { if (plainobj[encryptNames[i]] != null) { enclen++; enc[encryptNames[i]] = plainobj[encryptNames[i]]; delete plainobj[encryptNames[i]]; } }
if (enclen > 0) { plainobj._CRYPT = performRecordEncrypt(enc); } else { delete plainobj._CRYPT; }
return plainobj;
}
// Encrypt an object and return a buffer.
function performPartialRecordDecrypt(plainobj) {
if ((typeof plainobj != 'object') || (plainobj._CRYPT == null)) return plainobj;
var enc = performRecordDecrypt(plainobj._CRYPT);
if (enc != null) { for (var i in enc) { plainobj[i] = enc[i]; } }
delete plainobj._CRYPT;
return plainobj;
}
// Encrypt an object and return a base64.
function performRecordEncrypt(plainobj) {
if (obj.dbRecordsEncryptKey == null) return null;
const iv = parent.crypto.randomBytes(12);
const aes = parent.crypto.createCipheriv('aes-256-gcm', obj.dbRecordsEncryptKey, iv);
var ciphertext = aes.update(JSON.stringify(plainobj));
var cipherfinal = aes.final();
ciphertext = Buffer.concat([iv, aes.getAuthTag(), ciphertext, cipherfinal]);
return ciphertext.toString('base64');
}
// Takes a base64 and return an object.
function performRecordDecrypt(ciphertext) {
if (obj.dbRecordsDecryptKey == null) return null;
const ciphertextBytes = Buffer.from(ciphertext, 'base64');
const iv = ciphertextBytes.slice(0, 12);
const data = ciphertextBytes.slice(28);
const aes = parent.crypto.createDecipheriv('aes-256-gcm', obj.dbRecordsDecryptKey, iv);
aes.setAuthTag(ciphertextBytes.slice(12, 28));
var plaintextBytes, r;
try {
plaintextBytes = Buffer.from(aes.update(data));
plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
r = JSON.parse(plaintextBytes.toString());
} catch (e) { throw "Incorrect DbRecordsDecryptKey/DbRecordsEncryptKey or invalid database _CRYPT data: " + e; }
return r;
}
// Clone an object (TODO: Make this more efficient)
function Clone(v) { return JSON.parse(JSON.stringify(v)); }
// Read expiration time from configuration file
if (typeof parent.args.dbexpire == 'object') {
if (typeof parent.args.dbexpire.events == 'number') { expireEventsSeconds = parent.args.dbexpire.events; }
if (typeof parent.args.dbexpire.powerevents == 'number') { expirePowerEventsSeconds = parent.args.dbexpire.powerevents; }
if (typeof parent.args.dbexpire.statsevents == 'number') { expireServerStatsSeconds = parent.args.dbexpire.statsevents; }
}
// If a DB record encryption key is provided, perform database record encryption
if ((typeof parent.args.dbrecordsencryptkey == 'string') && (parent.args.dbrecordsencryptkey.length != 0)) {
// Hash the database password into a AES256 key and setup encryption and decryption.
obj.dbRecordsEncryptKey = obj.dbRecordsDecryptKey = parent.crypto.createHash('sha384').update(parent.args.dbrecordsencryptkey).digest('raw').slice(0, 32);
}
// If a DB record decryption key is provided, perform database record decryption
if ((typeof parent.args.dbrecordsdecryptkey == 'string') && (parent.args.dbrecordsdecryptkey.length != 0)) {
// Hash the database password into a AES256 key and setup encryption and decryption.
obj.dbRecordsDecryptKey = parent.crypto.createHash('sha384').update(parent.args.dbrecordsdecryptkey).digest('raw').slice(0, 32);
}
function createTablesIfNotExist(dbname) {
var useDatabase = 'USE ' + dbname;
sqlDbQuery(useDatabase, null, function (err, docs) {
if (err != null) {
console.log("Unable to connect to database: " + err);
process.exit();
}
if (err == null) {
parent.debug('db', 'Checking tables...');
sqlDbBatchExec([
'CREATE TABLE IF NOT EXISTS main (id VARCHAR(256) NOT NULL, type CHAR(32), domain CHAR(64), extra CHAR(255), extraex CHAR(255), doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
'CREATE TABLE IF NOT EXISTS events (id INT NOT NULL AUTO_INCREMENT, time DATETIME, domain CHAR(64), action CHAR(255), nodeid CHAR(255), userid CHAR(255), doc JSON, PRIMARY KEY(id), CHECK(json_valid(doc)))',
'CREATE TABLE IF NOT EXISTS eventids (fkid INT NOT NULL, target CHAR(255), CONSTRAINT fk_eventid FOREIGN KEY (fkid) REFERENCES events (id) ON DELETE CASCADE ON UPDATE RESTRICT)',
'CREATE TABLE IF NOT EXISTS serverstats (time DATETIME, expire DATETIME, doc JSON, PRIMARY KEY(time), CHECK (json_valid(doc)))',
'CREATE TABLE IF NOT EXISTS power (id INT NOT NULL AUTO_INCREMENT, time DATETIME, nodeid CHAR(255), doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
'CREATE TABLE IF NOT EXISTS smbios (id CHAR(255), time DATETIME, expire DATETIME, doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
'CREATE TABLE IF NOT EXISTS plugin (id INT NOT NULL AUTO_INCREMENT, doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))'
], function (err) {
parent.debug('db', 'Checking indexes...');
sqlDbExec('CREATE INDEX ndxtypedomainextra ON main (type, domain, extra)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxextra ON main (extra)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxextraex ON main (extraex)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventstime ON events(time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventsusername ON events(domain, userid, time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventsdomainnodeidtime ON events(domain, nodeid, time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventids ON eventids(target)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxserverstattime ON serverstats (time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxserverstatexpire ON serverstats (expire)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxpowernodeidtime ON power (nodeid, time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxsmbiostime ON smbios (time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxsmbiosexpire ON smbios (expire)', null, function (err, response) { });
setupFunctions(func);
});
}
});
}
if (parent.args.sqlite3) {
// SQLite3 database setup
obj.databaseType = DB_SQLITE;
const sqlite3 = require('sqlite3');
let configParams = parent.config.settings.sqlite3;
if (typeof configParams == 'string') {databaseName = configParams} else {databaseName = configParams.name ? configParams.name : 'meshcentral';};
obj.sqliteConfig.startupVacuum = configParams.startupvacuum ? configParams.startupvacuum : false;
obj.sqliteConfig.autoVacuum = configParams.autovacuum ? configParams.autovacuum.toLowerCase() : 'incremental';
obj.sqliteConfig.incrementalVacuum = configParams.incrementalvacuum ? configParams.incrementalvacuum : 100;
obj.sqliteConfig.journalMode = configParams.journalmode ? configParams.journalmode.toLowerCase() : 'delete';
//allowed modes, 'none' excluded because not usefull for this app, maybe also remove 'memory'?
if (!(['delete', 'truncate', 'persist', 'memory', 'wal'].includes(obj.sqliteConfig.journalMode))) { obj.sqliteConfig.journalMode = 'delete'};
obj.sqliteConfig.journalSize = configParams.journalsize ? configParams.journalsize : 409600;
//wal can use the more performant 'normal' mode, see https://www.sqlite.org/pragma.html#pragma_synchronous
obj.sqliteConfig.synchronous = (obj.sqliteConfig.journalMode == 'wal') ? 'normal' : 'full';
if (obj.sqliteConfig.journalMode == 'wal') {obj.sqliteConfig.maintenance += 'PRAGMA wal_checkpoint(PASSIVE);'};
if (obj.sqliteConfig.autoVacuum == 'incremental') {obj.sqliteConfig.maintenance += 'PRAGMA incremental_vacuum(' + obj.sqliteConfig.incrementalVacuum + ');'};
obj.sqliteConfig.maintenance += 'PRAGMA optimize;';
parent.debug('db', 'SQlite config options: ' + JSON.stringify(obj.sqliteConfig, null, 4));
if (obj.sqliteConfig.journalMode == 'memory') { console.log('[WARNING] journal_mode=memory: this can lead to database corruption if there is a crash during a transaction. See https://www.sqlite.org/pragma.html#pragma_journal_mode') };
//.cached not usefull
obj.file = new sqlite3.Database(path.join(parent.datapath, databaseName + '.sqlite'), sqlite3.OPEN_READWRITE, function (err) {
if (err && (err.code == 'SQLITE_CANTOPEN')) {
// Database needs to be created
obj.file = new sqlite3.Database(path.join(parent.datapath, databaseName + '.sqlite'), function (err) {
if (err) { console.log("SQLite Error: " + err); process.exit(1); }
obj.file.exec(`
CREATE TABLE main (id VARCHAR(256) PRIMARY KEY NOT NULL, type CHAR(32), domain CHAR(64), extra CHAR(255), extraex CHAR(255), doc JSON);
CREATE TABLE events(id INTEGER PRIMARY KEY, time TIMESTAMP, domain CHAR(64), action CHAR(255), nodeid CHAR(255), userid CHAR(255), doc JSON);
CREATE TABLE eventids(fkid INT NOT NULL, target CHAR(255), CONSTRAINT fk_eventid FOREIGN KEY (fkid) REFERENCES events (id) ON DELETE CASCADE ON UPDATE RESTRICT);
CREATE TABLE serverstats (time TIMESTAMP PRIMARY KEY, expire TIMESTAMP, doc JSON);
CREATE TABLE power (id INTEGER PRIMARY KEY, time TIMESTAMP,