meshcentral
Version:
Web based remote computer management server
785 lines (722 loc) • 191 kB
JavaScript
/**
* @description MeshCentral database module
* @author Ylian Saint-Hilaire
* @copyright Intel Corporation 2018-2021
* @license Apache-2.0
* @version v0.0.2
*/
/*xjslint node: true */
/*xjslint plusplus: true */
/*xjslint maxlen: 256 */
/*jshint node: true */
/*jshint strict: false */
/*jshint esversion: 6 */
"use strict";
//
// Construct Meshcentral database object
//
// The default database is NeDB
// https://github.com/louischatriot/nedb
//
// Alternativety, MongoDB can be used
// https://www.mongodb.com/
// Just run with --mongodb [connectionstring], where the connection string is documented here: https://docs.mongodb.com/manual/reference/connection-string/
// The default collection is "meshcentral", but you can override it using --mongodbcol [collection]
//
module.exports.CreateDB = function (parent, func) {
var obj = {};
var Datastore = null;
var expireEventsSeconds = (60 * 60 * 24 * 20); // By default, expire events after 20 days (1728000). (Seconds * Minutes * Hours * Days)
var expirePowerEventsSeconds = (60 * 60 * 24 * 10); // By default, expire power events after 10 days (864000). (Seconds * Minutes * Hours * Days)
var expireServerStatsSeconds = (60 * 60 * 24 * 30); // By default, expire power events after 30 days (2592000). (Seconds * Minutes * Hours * Days)
const common = require('./common.js');
obj.identifier = null;
obj.dbKey = null;
obj.dbRecordsEncryptKey = null;
obj.dbRecordsDecryptKey = null;
obj.changeStream = false;
obj.pluginsActive = ((parent.config) && (parent.config.settings) && (parent.config.settings.plugins != null) && (parent.config.settings.plugins != false) && ((typeof parent.config.settings.plugins != 'object') || (parent.config.settings.plugins.enabled != false)));
obj.dbCounters = {
fileSet: 0,
fileRemove: 0,
powerSet: 0,
eventsSet: 0
}
// MongoDB bulk operations state
if (parent.config.settings.mongodbbulkoperations) {
// Added counters
obj.dbCounters.fileSetPending = 0;
obj.dbCounters.fileSetBulk = 0;
obj.dbCounters.fileRemovePending = 0;
obj.dbCounters.fileRemoveBulk = 0;
obj.dbCounters.powerSetPending = 0;
obj.dbCounters.powerSetBulk = 0;
obj.dbCounters.eventsSetPending = 0;
obj.dbCounters.eventsSetBulk = 0;
/// Added bulk accumulators
obj.filePendingGet = null;
obj.filePendingGets = null;
obj.filePendingRemove = null;
obj.filePendingRemoves = null;
obj.filePendingSet = false;
obj.filePendingSets = null;
obj.filePendingCb = null;
obj.filePendingCbs = null;
obj.powerFilePendingSet = false;
obj.powerFilePendingSets = null;
obj.powerFilePendingCb = null;
obj.powerFilePendingCbs = null;
obj.eventsFilePendingSet = false;
obj.eventsFilePendingSets = null;
obj.eventsFilePendingCb = null;
obj.eventsFilePendingCbs = null;
}
obj.SetupDatabase = function (func) {
// Check if the database unique identifier is present
// This is used to check that in server peering mode, everyone is using the same database.
obj.Get('DatabaseIdentifier', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (Get DatabaseIdentifier): ' + err); }
if ((err == null) && (docs.length == 1) && (docs[0].value != null)) {
obj.identifier = docs[0].value;
} else {
obj.identifier = Buffer.from(require('crypto').randomBytes(48), 'binary').toString('hex');
obj.Set({ _id: 'DatabaseIdentifier', value: obj.identifier });
}
});
// Load database schema version and check if we need to update
obj.Get('SchemaVersion', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (Get SchemaVersion): ' + err); }
var ver = 0;
if ((err == null) && (docs.length == 1)) { ver = docs[0].value; }
if (ver == 1) { console.log('This is an unsupported beta 1 database, delete it to create a new one.'); process.exit(0); }
// TODO: Any schema upgrades here...
obj.Set({ _id: 'SchemaVersion', value: 2 });
func(ver);
});
};
// Perform database maintenance
obj.maintenance = function () {
if (obj.databaseType == 1) { // NeDB will not remove expired records unless we try to access them. This will force the removal.
obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
obj.serverstatsfile.remove({ time: { '$lt': new Date(Date.now() - (expireServerStatsSeconds * 1000)) } }, { multi: true }); // Force delete older events
}
obj.removeInactiveDevices();
}
// Remove inactive devices
obj.removeInactiveDevices = function (showall, cb) {
// Get a list of domains and what their inactive device removal setting is
var removeInactiveDevicesPerDomain = {}, minRemoveInactiveDevicesPerDomain = {}, minRemoveInactiveDevice = 9999;
for (var i in parent.config.domains) {
if (typeof parent.config.domains[i].autoremoveinactivedevices == 'number') {
var v = parent.config.domains[i].autoremoveinactivedevices;
if ((v > 1) && (v <= 2000)) {
if (v < minRemoveInactiveDevice) { minRemoveInactiveDevice = v; }
removeInactiveDevicesPerDomain[i] = v;
minRemoveInactiveDevicesPerDomain[i] = v;
}
}
}
// Check if any device groups have a inactive device removal setting
for (var i in parent.webserver.meshes) {
if (typeof parent.webserver.meshes[i].expireDevs == 'number') {
var v = parent.webserver.meshes[i].expireDevs;
if ((v > 1) && (v <= 2000)) {
if (v < minRemoveInactiveDevice) { minRemoveInactiveDevice = v; }
if ((minRemoveInactiveDevicesPerDomain[parent.webserver.meshes[i].domain] == null) || (minRemoveInactiveDevicesPerDomain[parent.webserver.meshes[i].domain] > v)) {
minRemoveInactiveDevicesPerDomain[parent.webserver.meshes[i].domain] = v;
}
} else {
delete parent.webserver.meshes[i].expireDevs;
}
}
}
// If there are no such settings for any domain, we can exit now.
if (minRemoveInactiveDevice == 9999) { if (cb) { cb("No device removal policy set, nothing to do."); } return; }
const now = Date.now();
// For each domain with a inactive device removal setting, get a list of last device connections
for (var domainid in minRemoveInactiveDevicesPerDomain) {
obj.GetAllTypeNoTypeField('lastconnect', domainid, function (err, docs) {
if ((err != null) || (docs == null)) return;
for (var j in docs) {
const days = Math.floor((now - docs[j].time) / 86400000); // Calculate the number of inactive days
var expireDays = -1;
if (removeInactiveDevicesPerDomain[docs[j].domain]) { expireDays = removeInactiveDevicesPerDomain[docs[j].domain]; }
const mesh = parent.webserver.meshes[docs[j].meshid];
if (mesh && (typeof mesh.expireDevs == 'number')) { expireDays = mesh.expireDevs; }
var remove = false;
if (expireDays > 0) {
if (expireDays < days) { remove = true; }
if (cb) { if (showall || remove) { cb(docs[j]._id.substring(2) + ', ' + days + ' days, expire ' + expireDays + ' days' + (remove ? ', removing' : '')); } }
if (remove) {
// Check if this device is connected right now
const nodeid = docs[j]._id.substring(2);
const conn = parent.GetConnectivityState(nodeid);
if (conn == null) {
// Remove the device
obj.Get(nodeid, function (err, docs) {
if (err != null) return;
if ((docs == null) || (docs.length != 1)) { obj.Remove('lc' + nodeid); return; } // Remove last connect time
const node = docs[0];
// Delete this node including network interface information, events and timeline
obj.Remove(node._id); // Remove node with that id
obj.Remove('if' + node._id); // Remove interface information
obj.Remove('nt' + node._id); // Remove notes
obj.Remove('lc' + node._id); // Remove last connect time
obj.Remove('si' + node._id); // Remove system information
obj.Remove('al' + node._id); // Remove error log last time
if (obj.RemoveSMBIOS) { obj.RemoveSMBIOS(node._id); } // Remove SMBios data
obj.RemoveAllNodeEvents(node._id); // Remove all events for this node
obj.removeAllPowerEventsForNode(node._id); // Remove all power events for this node
if (typeof node.pmt == 'string') { obj.Remove('pmt_' + node.pmt); } // Remove Push Messaging Token
obj.Get('ra' + node._id, function (err, nodes) {
if ((nodes != null) && (nodes.length == 1)) { obj.Remove('da' + nodes[0].daid); } // Remove diagnostic agent to real agent link
obj.Remove('ra' + node._id); // Remove real agent to diagnostic agent link
});
// Remove any user node links
if (node.links != null) {
for (var i in node.links) {
if (i.startsWith('user/')) {
var cuser = parent.webserver.users[i];
if ((cuser != null) && (cuser.links != null) && (cuser.links[node._id] != null)) {
// Remove the user link & save the user
delete cuser.links[node._id];
if (Object.keys(cuser.links).length == 0) { delete cuser.links; }
obj.SetUser(cuser);
// Notify user change
var targets = ['*', 'server-users', cuser._id];
var event = { etype: 'user', userid: cuser._id, username: cuser.name, action: 'accountchange', msgid: 86, msgArgs: [cuser.name], msg: 'Removed user device rights for ' + cuser.name, domain: node.domain, account: parent.webserver.CloneSafeUser(cuser) };
if (obj.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to change the user. Another event will come.
parent.DispatchEvent(targets, obj, event);
}
}
}
}
// Event node deletion
var meshname = '(unknown)';
if ((parent.webserver.meshes[node.meshid] != null) && (parent.webserver.meshes[node.meshid].name != null)) { meshname = parent.webserver.meshes[node.meshid].name; }
var event = { etype: 'node', action: 'removenode', nodeid: node._id, msgid: 87, msgArgs: [node.name, meshname], msg: 'Removed device ' + node.name + ' from device group ' + meshname, domain: node.domain };
// TODO: We can't use the changeStream for node delete because we will not know the meshid the device was in.
//if (obj.changeStream) { event.noact = 1; } // If DB change stream is active, don't use this event to remove the node. Another event will come.
parent.DispatchEvent(parent.webserver.CreateNodeDispatchTargets(node.meshid, node._id), obj, event);
});
}
}
}
}
});
}
}
obj.cleanup = function (func) {
// TODO: Remove all mesh links to invalid users
// TODO: Remove all meshes that dont have any links
// Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now.
if ((obj.databaseType == 4) || (obj.databaseType == 5) || (obj.databaseType == 6)) {
// MariaDB or MySQL
obj.RemoveAllOfType('event', function () { });
obj.RemoveAllOfType('power', function () { });
obj.RemoveAllOfType('smbios', function () { });
} else if (obj.databaseType == 3) {
// MongoDB
obj.file.deleteMany({ type: 'event' }, { multi: true });
obj.file.deleteMany({ type: 'power' }, { multi: true });
obj.file.deleteMany({ type: 'smbios' }, { multi: true });
} else {
// NeDB or MongoJS
obj.file.remove({ type: 'event' }, { multi: true });
obj.file.remove({ type: 'power' }, { multi: true });
obj.file.remove({ type: 'smbios' }, { multi: true });
}
// List of valid identifiers
var validIdentifiers = {}
// Load all user groups
obj.GetAllType('ugrp', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (GetAll user): ' + err); }
if ((err == null) && (docs.length > 0)) {
for (var i in docs) {
// Add this as a valid user identifier
validIdentifiers[docs[i]._id] = 1;
}
}
// Fix all of the creating & login to ticks by seconds, not milliseconds.
obj.GetAllType('user', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (GetAll user): ' + err); }
if ((err == null) && (docs.length > 0)) {
for (var i in docs) {
var fixed = false;
// Add this as a valid user identifier
validIdentifiers[docs[i]._id] = 1;
// Fix email address capitalization
if (docs[i].email && (docs[i].email != docs[i].email.toLowerCase())) {
docs[i].email = docs[i].email.toLowerCase(); fixed = true;
}
// Fix account creation
if (docs[i].creation) {
if (docs[i].creation > 1300000000000) { docs[i].creation = Math.floor(docs[i].creation / 1000); fixed = true; }
if ((docs[i].creation % 1) != 0) { docs[i].creation = Math.floor(docs[i].creation); fixed = true; }
}
// Fix last account login
if (docs[i].login) {
if (docs[i].login > 1300000000000) { docs[i].login = Math.floor(docs[i].login / 1000); fixed = true; }
if ((docs[i].login % 1) != 0) { docs[i].login = Math.floor(docs[i].login); fixed = true; }
}
// Fix last password change
if (docs[i].passchange) {
if (docs[i].passchange > 1300000000000) { docs[i].passchange = Math.floor(docs[i].passchange / 1000); fixed = true; }
if ((docs[i].passchange % 1) != 0) { docs[i].passchange = Math.floor(docs[i].passchange); fixed = true; }
}
// Fix subscriptions
if (docs[i].subscriptions != null) { delete docs[i].subscriptions; fixed = true; }
// Save the user if needed
if (fixed) { obj.Set(docs[i]); }
}
// Remove all objects that have a "meshid" that no longer points to a valid mesh.
// Fix any incorrectly escaped user identifiers
obj.GetAllType('mesh', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (GetAll mesh): ' + err); }
var meshlist = [];
if ((err == null) && (docs.length > 0)) {
for (var i in docs) {
var meshChange = false;
docs[i] = common.unEscapeLinksFieldName(docs[i]);
meshlist.push(docs[i]._id);
// Make sure all mesh types are number type, if not, fix it.
if (typeof docs[i].mtype == 'string') { docs[i].mtype = parseInt(docs[i].mtype); meshChange = true; }
// If the device group is deleted, remove any invite codes
if (docs[i].deleted && docs[i].invite) { delete docs[i].invite; meshChange = true; }
// Take a look at the links
if (docs[i].links != null) {
for (var j in docs[i].links) {
if (validIdentifiers[j] == null) {
// This identifier is not known, let see if we can fix it.
var xid = j, xid2 = common.unEscapeFieldName(xid);
while ((xid != xid2) && (validIdentifiers[xid2] == null)) { xid = xid2; xid2 = common.unEscapeFieldName(xid2); }
if (validIdentifiers[xid2] == 1) {
//console.log('Fixing id: ' + j + ' to ' + xid2);
docs[i].links[xid2] = docs[i].links[j];
delete docs[i].links[j];
meshChange = true;
} else {
// TODO: here, we may want to clean up links to users and user groups that do not exist anymore.
//console.log('Unknown id: ' + j);
}
}
}
}
// Save the updated device group if needed
if (meshChange) { obj.Set(docs[i]); }
}
}
if (obj.databaseType == 6) {
// Postgres
sqlDbQuery('DELETE FROM Main WHERE ((extra != NULL) AND (extra LIKE (\'mesh/%\')) AND (extra != ANY ($1)))', [meshlist], function (err, response) { });
} else if ((obj.databaseType == 4) || (obj.databaseType == 5)) {
// MariaDB
sqlDbQuery('DELETE FROM Main WHERE (extra LIKE ("mesh/%") AND (extra NOT IN ?)', [meshlist], function (err, response) { });
} else if (obj.databaseType == 3) {
// MongoDB
obj.file.deleteMany({ meshid: { $exists: true, $nin: meshlist } }, { multi: true });
} else {
// NeDB or MongoJS
obj.file.remove({ meshid: { $exists: true, $nin: meshlist } }, { multi: true });
}
// We are done
validIdentifiers = null;
if (func) { func(); }
});
}
});
});
};
// Get encryption key
obj.getEncryptDataKey = function (password) {
if (typeof password != 'string') return null;
return parent.crypto.createHash('sha384').update(password).digest("raw").slice(0, 32);
}
// Encrypt data
obj.encryptData = function (password, plaintext) {
var key = obj.getEncryptDataKey(password);
if (key == null) return null;
const iv = parent.crypto.randomBytes(16);
const aes = parent.crypto.createCipheriv('aes-256-cbc', key, iv);
var ciphertext = aes.update(plaintext);
ciphertext = Buffer.concat([iv, ciphertext, aes.final()]);
return ciphertext.toString('base64');
}
// Decrypt data
obj.decryptData = function (password, ciphertext) {
try {
var key = obj.getEncryptDataKey(password);
if (key == null) return null;
const ciphertextBytes = Buffer.from(ciphertext, 'base64');
const iv = ciphertextBytes.slice(0, 16);
const data = ciphertextBytes.slice(16);
const aes = parent.crypto.createDecipheriv('aes-256-cbc', key, iv);
var plaintextBytes = Buffer.from(aes.update(data));
plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
return plaintextBytes;
} catch (ex) { return null; }
}
// Get the number of records in the database for various types, this is the slow NeDB way.
// WARNING: This is a terrible query for database performance. Only do this when needed. This query will look at almost every document in the database.
obj.getStats = function (func) {
if (obj.databaseType == 3) {
// MongoDB
obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }]).toArray(function (err, docs) {
var counters = {}, totalCount = 0;
if (err == null) { for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } }
func(counters);
});
} else if (obj.databaseType == 2) {
// MongoJS
obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }], function (err, docs) {
var counters = {}, totalCount = 0;
if (err == null) { for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } }
func(counters);
});
} else if (obj.databaseType == 1) {
// NeDB version
obj.file.count({ type: 'node' }, function (err, nodeCount) {
obj.file.count({ type: 'mesh' }, function (err, meshCount) {
obj.file.count({ type: 'user' }, function (err, userCount) {
obj.file.count({ type: 'sysinfo' }, function (err, sysinfoCount) {
obj.file.count({ type: 'note' }, function (err, noteCount) {
obj.file.count({ type: 'iploc' }, function (err, iplocCount) {
obj.file.count({ type: 'ifinfo' }, function (err, ifinfoCount) {
obj.file.count({ type: 'cfile' }, function (err, cfileCount) {
obj.file.count({ type: 'lastconnect' }, function (err, lastconnectCount) {
obj.file.count({}, function (err, totalCount) {
func({ node: nodeCount, mesh: meshCount, user: userCount, sysinfo: sysinfoCount, iploc: iplocCount, note: noteCount, ifinfo: ifinfoCount, cfile: cfileCount, lastconnect: lastconnectCount, total: totalCount });
});
});
});
});
});
});
});
});
});
});
}
}
// This is used to rate limit a number of operation per day. Returns a startValue each new days, but you can substract it and save the value in the db.
obj.getValueOfTheDay = function (id, startValue, func) { obj.Get(id, function (err, docs) { var date = new Date(), t = date.toLocaleDateString(); if ((err == null) && (docs.length == 1)) { var r = docs[0]; if (r.day == t) { func({ _id: id, value: r.value, day: t }); return; } } func({ _id: id, value: startValue, day: t }); }); };
obj.escapeBase64 = function escapeBase64(val) { return (val.replace(/\+/g, '@').replace(/\//g, '$')); }
// Encrypt an database object
obj.performRecordEncryptionRecode = function (func) {
var count = 0;
obj.GetAllType('user', function (err, docs) {
if (err != null) { parent.debug('db', 'ERROR (performRecordEncryptionRecode): ' + err); }
if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
obj.GetAllType('node', function (err, docs) {
if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
obj.GetAllType('mesh', function (err, docs) {
if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
if (obj.databaseType == 1) { // If we are using NeDB, compact the database.
obj.file.persistence.compactDatafile();
obj.file.on('compaction.done', function () { func(count); }); // It's important to wait for compaction to finish before exit, otherwise NeDB may corrupt.
} else {
func(count); // For all other databases, normal exit.
}
});
});
});
}
// Encrypt an database object
function performTypedRecordDecrypt(data) {
if ((data == null) || (obj.dbRecordsDecryptKey == null) || (typeof data != 'object')) return data;
for (var i in data) {
if (data[i] == null) continue;
if (data[i].type == 'user') {
data[i] = performPartialRecordDecrypt(data[i]);
} else if ((data[i].type == 'node') && (data[i].intelamt != null)) {
data[i].intelamt = performPartialRecordDecrypt(data[i].intelamt);
} else if ((data[i].type == 'mesh') && (data[i].amt != null)) {
data[i].amt = performPartialRecordDecrypt(data[i].amt);
}
}
return data;
}
// Encrypt an database object
function performTypedRecordEncrypt(data) {
if (obj.dbRecordsEncryptKey == null) return data;
if (data.type == 'user') { return performPartialRecordEncrypt(Clone(data), ['otpkeys', 'otphkeys', 'otpsecret', 'salt', 'hash', 'oldpasswords']); }
else if ((data.type == 'node') && (data.intelamt != null)) { var xdata = Clone(data); xdata.intelamt = performPartialRecordEncrypt(xdata.intelamt, ['user', 'pass', 'mpspass']); return xdata; }
else if ((data.type == 'mesh') && (data.amt != null)) { var xdata = Clone(data); xdata.amt = performPartialRecordEncrypt(xdata.amt, ['password']); return xdata; }
return data;
}
// Encrypt an object and return a buffer.
function performPartialRecordEncrypt(plainobj, encryptNames) {
if (typeof plainobj != 'object') return plainobj;
var enc = {}, enclen = 0;
for (var i in encryptNames) { if (plainobj[encryptNames[i]] != null) { enclen++; enc[encryptNames[i]] = plainobj[encryptNames[i]]; delete plainobj[encryptNames[i]]; } }
if (enclen > 0) { plainobj._CRYPT = performRecordEncrypt(enc); } else { delete plainobj._CRYPT; }
return plainobj;
}
// Encrypt an object and return a buffer.
function performPartialRecordDecrypt(plainobj) {
if ((typeof plainobj != 'object') || (plainobj._CRYPT == null)) return plainobj;
var enc = performRecordDecrypt(plainobj._CRYPT);
if (enc != null) { for (var i in enc) { plainobj[i] = enc[i]; } }
delete plainobj._CRYPT;
return plainobj;
}
// Encrypt an object and return a base64.
function performRecordEncrypt(plainobj) {
if (obj.dbRecordsEncryptKey == null) return null;
const iv = parent.crypto.randomBytes(12);
const aes = parent.crypto.createCipheriv('aes-256-gcm', obj.dbRecordsEncryptKey, iv);
var ciphertext = aes.update(JSON.stringify(plainobj));
var cipherfinal = aes.final();
ciphertext = Buffer.concat([iv, aes.getAuthTag(), ciphertext, cipherfinal]);
return ciphertext.toString('base64');
}
// Takes a base64 and return an object.
function performRecordDecrypt(ciphertext) {
if (obj.dbRecordsDecryptKey == null) return null;
const ciphertextBytes = Buffer.from(ciphertext, 'base64');
const iv = ciphertextBytes.slice(0, 12);
const data = ciphertextBytes.slice(28);
const aes = parent.crypto.createDecipheriv('aes-256-gcm', obj.dbRecordsDecryptKey, iv);
aes.setAuthTag(ciphertextBytes.slice(12, 28));
var plaintextBytes, r;
try {
plaintextBytes = Buffer.from(aes.update(data));
plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
r = JSON.parse(plaintextBytes.toString());
} catch (e) { throw "Incorrect DbRecordsDecryptKey/DbRecordsEncryptKey or invalid database _CRYPT data: " + e; }
return r;
}
// Clone an object (TODO: Make this more efficient)
function Clone(v) { return JSON.parse(JSON.stringify(v)); }
// Read expiration time from configuration file
if (typeof parent.args.dbexpire == 'object') {
if (typeof parent.args.dbexpire.events == 'number') { expireEventsSeconds = parent.args.dbexpire.events; }
if (typeof parent.args.dbexpire.powerevents == 'number') { expirePowerEventsSeconds = parent.args.dbexpire.powerevents; }
if (typeof parent.args.dbexpire.statsevents == 'number') { expireServerStatsSeconds = parent.args.dbexpire.statsevents; }
}
// If a DB record encryption key is provided, perform database record encryption
if ((typeof parent.args.dbrecordsencryptkey == 'string') && (parent.args.dbrecordsencryptkey.length != 0)) {
// Hash the database password into a AES256 key and setup encryption and decryption.
obj.dbRecordsEncryptKey = obj.dbRecordsDecryptKey = parent.crypto.createHash('sha384').update(parent.args.dbrecordsencryptkey).digest('raw').slice(0, 32);
}
// If a DB record decryption key is provided, perform database record decryption
if ((typeof parent.args.dbrecordsdecryptkey == 'string') && (parent.args.dbrecordsdecryptkey.length != 0)) {
// Hash the database password into a AES256 key and setup encryption and decryption.
obj.dbRecordsDecryptKey = parent.crypto.createHash('sha384').update(parent.args.dbrecordsdecryptkey).digest('raw').slice(0, 32);
}
function createTablesIfNotExist(dbname) {
var useDatabase = 'USE ' + dbname;
sqlDbQuery(useDatabase, null, function (err, docs) {
if (err != null) {
console.log("Unable to connect to database: " + err);
process.exit();
}
if (err == null) {
parent.debug('db', 'Checking tables...');
sqlDbBatchExec([
'CREATE TABLE IF NOT EXISTS main (id VARCHAR(256) NOT NULL, type CHAR(32), domain CHAR(64), extra CHAR(255), extraex CHAR(255), doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
'CREATE TABLE IF NOT EXISTS events(id INT NOT NULL AUTO_INCREMENT, time DATETIME, domain CHAR(64), action CHAR(255), nodeid CHAR(255), userid CHAR(255), doc JSON, PRIMARY KEY(id), CHECK(json_valid(doc)))',
'CREATE TABLE IF NOT EXISTS eventids(fkid INT NOT NULL, target CHAR(255), CONSTRAINT fk_eventid FOREIGN KEY (fkid) REFERENCES events (id) ON DELETE CASCADE ON UPDATE RESTRICT)',
'CREATE TABLE IF NOT EXISTS serverstats (time DATETIME, expire DATETIME, doc JSON, PRIMARY KEY(time), CHECK (json_valid(doc)))',
'CREATE TABLE IF NOT EXISTS power (id INT NOT NULL AUTO_INCREMENT, time DATETIME, nodeid CHAR(255), doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
'CREATE TABLE IF NOT EXISTS smbios (id CHAR(255), time DATETIME, expire DATETIME, doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
'CREATE TABLE IF NOT EXISTS plugin (id INT NOT NULL AUTO_INCREMENT, doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))'
], function (err) {
parent.debug('db', 'Checking indexes...');
sqlDbExec('CREATE INDEX ndxtypedomainextra ON main (type, domain, extra)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxextra ON main (extra)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxextraex ON main (extraex)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventstime ON events(time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventsusername ON events(domain, userid, time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventsdomainnodeidtime ON events(domain, nodeid, time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventids ON eventids(target)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxserverstattime ON serverstats (time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxserverstatexpire ON serverstats (expire)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxpowernodeidtime ON power (nodeid, time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxsmbiostime ON smbios (time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxsmbiosexpire ON smbios (expire)', null, function (err, response) { });
setupFunctions(func);
});
}
});
}
if (parent.args.mariadb || parent.args.mysql) {
var connectinArgs = (parent.args.mariadb) ? parent.args.mariadb : parent.args.mysql;
var dbname = (connectinArgs.database != null) ? connectinArgs.database : 'meshcentral';
// Including the db name in the connection obj will cause a connection failure if it does not exist
var connectionObject = Clone(connectinArgs);
delete connectionObject.database;
try {
if (connectinArgs.ssl) {
if (connectinArgs.ssl.dontcheckserveridentity == true) { connectionObject.ssl.checkServerIdentity = function(name, cert) { return undefined; } };
if (connectinArgs.ssl.cacertpath) { connectionObject.ssl.ca = [require('fs').readFileSync(connectinArgs.ssl.cacertpath, 'utf8')]; }
if (connectinArgs.ssl.clientcertpath) { connectionObject.ssl.cert = [require('fs').readFileSync(connectinArgs.ssl.clientcertpath, 'utf8')]; }
if (connectinArgs.ssl.clientkeypath) { connectionObject.ssl.key = [require('fs').readFileSync(connectinArgs.ssl.clientkeypath, 'utf8')]; }
}
} catch (ex) {
console.log('Error loading SQL Connector certificate: ' + ex);
process.exit();
}
if (parent.args.mariadb) {
// Use MariaDB
obj.databaseType = 4;
var tempDatastore = require('mariadb').createPool(connectionObject);
tempDatastore.getConnection().then(function (conn) {
conn.query('CREATE DATABASE IF NOT EXISTS ' + dbname).then(function (result) {
conn.release();
}).catch(function (ex) { console.log('Auto-create database failed: ' + ex); });
}).catch(function (ex) { console.log('Auto-create database failed: ' + ex); });
setTimeout(function () { tempDatastore.end(); }, 2000);
connectionObject.database = dbname;
Datastore = require('mariadb').createPool(connectionObject);
createTablesIfNotExist(dbname);
} else if (parent.args.mysql) {
// Use MySQL
obj.databaseType = 5;
var tempDatastore = require('mysql').createConnection(connectionObject);
tempDatastore.query('CREATE DATABASE IF NOT EXISTS ' + dbname, function (error) {
if (error != null) {
console.log('Auto-create database failed: ' + error);
}
connectionObject.database = dbname;
Datastore = require('mysql').createConnection(connectionObject);
createTablesIfNotExist(dbname);
});
setTimeout(function () { tempDatastore.end(); }, 2000);
}
} else if (parent.args.postgres) {
// Postgres SQL
var connectinArgs = parent.args.postgres;
var dbname = (connectinArgs.database != null) ? connectinArgs.database : 'meshcentral';
obj.databaseType = 6;
const pgtools = require('pgtools');
pgtools.createdb(connectinArgs, dbname, function (err, res) {
const { Pool, Client } = require('pg');
connectinArgs.database = dbname;
Datastore = new Client(connectinArgs);
Datastore.connect()
parent.debug('db', 'Checking tables...');
sqlDbBatchExec([
'CREATE TABLE IF NOT EXISTS main (id VARCHAR(256) PRIMARY KEY NOT NULL, type CHAR(32), domain CHAR(64), extra CHAR(255), extraex CHAR(255), doc JSON)',
'CREATE TABLE IF NOT EXISTS events(id SERIAL PRIMARY KEY, time TIMESTAMP, domain CHAR(64), action CHAR(255), nodeid CHAR(255), userid CHAR(255), doc JSON)',
'CREATE TABLE IF NOT EXISTS eventids(fkid INT NOT NULL, target CHAR(255), CONSTRAINT fk_eventid FOREIGN KEY (fkid) REFERENCES events (id) ON DELETE CASCADE ON UPDATE RESTRICT)',
'CREATE TABLE IF NOT EXISTS serverstats (time TIMESTAMP PRIMARY KEY, expire TIMESTAMP, doc JSON)',
'CREATE TABLE IF NOT EXISTS power (id SERIAL PRIMARY KEY, time TIMESTAMP, nodeid CHAR(255), doc JSON)',
'CREATE TABLE IF NOT EXISTS smbios (id CHAR(255) PRIMARY KEY, time TIMESTAMP, expire TIMESTAMP, doc JSON)',
'CREATE TABLE IF NOT EXISTS plugin (id SERIAL PRIMARY KEY, doc JSON)'
], function (results) {
parent.debug('db', 'Checking indexes...');
sqlDbExec('CREATE INDEX ndxtypedomainextra ON main (type, domain, extra)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxextra ON main (extra)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxextraex ON main (extraex)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventstime ON events(time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventsusername ON events(domain, userid, time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventsdomainnodeidtime ON events(domain, nodeid, time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxeventids ON eventids(target)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxserverstattime ON serverstats (time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxserverstatexpire ON serverstats (expire)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxpowernodeidtime ON power (nodeid, time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxsmbiostime ON smbios (time)', null, function (err, response) { });
sqlDbExec('CREATE INDEX ndxsmbiosexpire ON smbios (expire)', null, function (err, response) { });
setupFunctions(func);
});
});
} else if (parent.args.mongodb) {
// Use MongoDB
obj.databaseType = 3;
require('mongodb').MongoClient.connect(parent.args.mongodb, { useNewUrlParser: true, useUnifiedTopology: true }, function (err, client) {
if (err != null) { console.log("Unable to connect to database: " + err); process.exit(); return; }
Datastore = client;
parent.debug('db', 'Connected to MongoDB database...');
// Get the database name and setup the database client
var dbname = 'meshcentral';
if (parent.args.mongodbname) { dbname = parent.args.mongodbname; }
const dbcollectionname = (parent.args.mongodbcol) ? (parent.args.mongodbcol) : 'meshcentral';
const db = client.db(dbname);
// Check the database version
db.admin().serverInfo(function (err, info) {
if ((err != null) || (info == null) || (info.versionArray == null) || (Array.isArray(info.versionArray) == false) || (info.versionArray.length < 2) || (typeof info.versionArray[0] != 'number') || (typeof info.versionArray[1] != 'number')) {
console.log('WARNING: Unable to check MongoDB version.');
} else {
if ((info.versionArray[0] < 3) || ((info.versionArray[0] == 3) && (info.versionArray[1] < 6))) {
// We are running with mongoDB older than 3.6, this is not good.
parent.addServerWarning("Current version of MongoDB (" + info.version + ") is too old, please upgrade to MongoDB 3.6 or better.");
}
}
});
// Setup MongoDB main collection and indexes
obj.file = db.collection(dbcollectionname);
obj.file.indexes(function (err, indexes) {
// Check if we need to reset indexes
var indexesByName = {}, indexCount = 0;
for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
if ((indexCount != 4) || (indexesByName['TypeDomainMesh1'] == null) || (indexesByName['Email1'] == null) || (indexesByName['Mesh1'] == null)) {
console.log('Resetting main indexes...');
obj.file.dropIndexes(function (err) {
obj.file.createIndex({ type: 1, domain: 1, meshid: 1 }, { sparse: 1, name: 'TypeDomainMesh1' }); // Speeds up GetAllTypeNoTypeField() and GetAllTypeNoTypeFieldMeshFiltered()
obj.file.createIndex({ email: 1 }, { sparse: 1, name: 'Email1' }); // Speeds up GetUserWithEmail() and GetUserWithVerifiedEmail()
obj.file.createIndex({ meshid: 1 }, { sparse: 1, name: 'Mesh1' }); // Speeds up RemoveMesh()
});
}
});
// Setup the changeStream on the MongoDB main collection if possible
if (parent.args.mongodbchangestream == true) {
obj.dbCounters.changeStream = { change: 0, update: 0, insert: 0, delete: 0 };
if (typeof obj.file.watch != 'function') {
console.log('WARNING: watch() is not a function, MongoDB ChangeStream not supported.');
} else {
obj.fileChangeStream = obj.file.watch([{ $match: { $or: [{ 'fullDocument.type': { $in: ['node', 'mesh', 'user', 'ugrp'] } }, { 'operationType': 'delete' }] } }], { fullDocument: 'updateLookup' });
obj.fileChangeStream.on('change', function (change) {
obj.dbCounters.changeStream.change++;
if ((change.operationType == 'update') || (change.operationType == 'replace')) {
obj.dbCounters.changeStream.update++;
switch (change.fullDocument.type) {
case 'node': { dbNodeChange(change, false); break; } // A node has changed
case 'mesh': { dbMeshChange(change, false); break; } // A device group has changed
case 'user': { dbUserChange(change, false); break; } // A user account has changed
case 'ugrp': { dbUGrpChange(change, false); break; } // A user account has changed
}
} else if (change.operationType == 'insert') {
obj.dbCounters.changeStream.insert++;
switch (change.fullDocument.type) {
case 'node': { dbNodeChange(change, true); break; } // A node has added
case 'mesh': { dbMeshChange(change, true); break; } // A device group has created
case 'user': { dbUserChange(change, true); break; } // A user account has created
case 'ugrp': { dbUGrpChange(change, true); break; } // A user account has created
}
} else if (change.operationType == 'delete') {
obj.dbCounters.changeStream.delete++;
if ((change.documentKey == null) || (change.documentKey._id == null)) return;
var splitId = change.documentKey._id.split('/');
switch (splitId[0]) {
case 'node': {
//Not Good: Problem here is that we don't know what meshid the node belonged to before the delete.
//parent.DispatchEvent(['*', node.meshid], obj, { etype: 'node', action: 'removenode', nodeid: change.documentKey._id, domain: splitId[1] });
break;
}
case 'mesh': {
parent.DispatchEvent(['*', change.documentKey._id], obj, { etype: 'mesh', action: 'deletemesh', meshid: change.documentKey._id, domain: splitId[1] });
break;
}
case 'user': {
//Not Good: This is not a perfect user removal because we don't know what groups the user was in.
//parent.DispatchEvent(['*', 'server-users'], obj, { etype: 'user', action: 'accountremove', userid: change.documentKey._id, domain: splitId[1], username: splitId[2] });
break;
}
case 'ugrp': {
parent.DispatchEvent(['*', change.documentKey._id], obj, { etype: 'ugrp', action: 'deleteusergroup', ugrpid: change.documentKey._id, domain: splitId[1] });
break;
}
}
}
});
obj.changeStream = true;
}
}
// Setup MongoDB events collection and indexes
obj.eventsfile = db.collection('events'); // Collection containing all events
obj.eventsfile.indexes(function (err, indexes) {
// Check if we need to reset indexes
var indexesByName = {}, indexCount = 0;
for