meshcentral/db.js

Web based remote computer management and file server

/** 
* @description MeshCentral database module
* @author Ylian Saint-Hilaire
* @copyright Intel Corporation 2018-2020
* @license Apache-2.0
* @version v0.0.2
*/

/*xjslint node: true */
/*xjslint plusplus: true */
/*xjslint maxlen: 256 */
/*jshint node: true */
/*jshint strict: false */
/*jshint esversion: 6 */
"use strict";

//
// Construct Meshcentral database object
//
// The default database is NeDB
// https://github.com/louischatriot/nedb
//
// Alternativety, MongoDB can be used
// https://www.mongodb.com/
// Just run with --mongodb [connectionstring], where the connection string is documented here: https://docs.mongodb.com/manual/reference/connection-string/
// The default collection is "meshcentral", but you can override it using --mongodbcol [collection]
//
module.exports.CreateDB = function (parent, func) {
    var obj = {};
    var Datastore = null;
    var expireEventsSeconds = (60 * 60 * 24 * 20);              // By default, expire events after 20 days (1728000). (Seconds * Minutes * Hours * Days)
    var expirePowerEventsSeconds = (60 * 60 * 24 * 10);         // By default, expire power events after 10 days (864000). (Seconds * Minutes * Hours * Days)
    var expireServerStatsSeconds = (60 * 60 * 24 * 30);         // By default, expire power events after 30 days (2592000). (Seconds * Minutes * Hours * Days)
    const common = require('./common.js');
    obj.identifier = null;
    obj.dbKey = null;
    obj.dbRecordsEncryptKey = null;
    obj.dbRecordsDecryptKey = null;
    obj.changeStream = false;
    obj.pluginsActive = ((parent.config) && (parent.config.settings) && (parent.config.settings.plugins != null) && (parent.config.settings.plugins != false) && ((typeof parent.config.settings.plugins != 'object') || (parent.config.settings.plugins.enabled != false)));

    obj.SetupDatabase = function (func) {
        // Check if the database unique identifier is present
        // This is used to check that in server peering mode, everyone is using the same database.
        obj.Get('DatabaseIdentifier', function (err, docs) {
            if (err != null) { parent.debug('db', 'ERROR (Get DatabaseIdentifier): ' + err); }
            if ((err == null) && (docs.length == 1) && (docs[0].value != null)) {
                obj.identifier = docs[0].value;
            } else {
                obj.identifier = Buffer.from(require('crypto').randomBytes(48), 'binary').toString('hex');
                obj.Set({ _id: 'DatabaseIdentifier', value: obj.identifier });
            }
        });

        // Load database schema version and check if we need to update
        obj.Get('SchemaVersion', function (err, docs) {
            if (err != null) { parent.debug('db', 'ERROR (Get SchemaVersion): ' + err); }
            var ver = 0;
            if ((err == null) && (docs.length == 1)) { ver = docs[0].value; }
            if (ver == 1) { console.log('This is an unsupported beta 1 database, delete it to create a new one.'); process.exit(0); }

            // TODO: Any schema upgrades here...
            obj.Set({ _id: 'SchemaVersion', value: 2 });

            func(ver);
        });
    };

    // Perform database maintenance
    obj.maintenance = function () {
        if (obj.databaseType == 1) { // NeDB will not remove expired records unless we try to access them. This will force the removal.
            obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
            obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events
            obj.serverstatsfile.remove({ time: { '$lt': new Date(Date.now() - (expireServerStatsSeconds * 1000)) } }, { multi: true }); // Force delete older events
        }
    }

    obj.cleanup = function (func) {
        // TODO: Remove all mesh links to invalid users
        // TODO: Remove all meshes that dont have any links

        // Remove all events, power events and SMBIOS data from the main collection. They are all in seperate collections now.
        if ((obj.databaseType == 4) || (obj.databaseType == 5)) {
            // MariaDB or MySQL
            obj.RemoveAllOfType('event', function () { });
            obj.RemoveAllOfType('power', function () { });
            obj.RemoveAllOfType('smbios', function () { });
        } else if (obj.databaseType == 3) {
            // MongoDB
            obj.file.deleteMany({ type: 'event' }, { multi: true });
            obj.file.deleteMany({ type: 'power' }, { multi: true });
            obj.file.deleteMany({ type: 'smbios' }, { multi: true });
        } else {
            // NeDB or MongoJS
            obj.file.remove({ type: 'event' }, { multi: true });
            obj.file.remove({ type: 'power' }, { multi: true });
            obj.file.remove({ type: 'smbios' }, { multi: true });
        }

        // Remove all objects that have a "meshid" that no longer points to a valid mesh.
        obj.GetAllType('mesh', function (err, docs) {
            if (err != null) { parent.debug('db', 'ERROR (GetAll mesh): ' + err); }
            var meshlist = [];
            if ((err == null) && (docs.length > 0)) { for (var i in docs) { meshlist.push(docs[i]._id); } }
            if ((obj.databaseType == 4) || (obj.databaseType == 5)) {
                // MariaDB
                sqlDbQuery('DELETE FROM MeshCentral.Main WHERE (extra LIKE ("mesh/%") AND (extra NOT IN ?)', [meshlist], func);
            } else if (obj.databaseType == 3) {
                // MongoDB
                obj.file.deleteMany({ meshid: { $exists: true, $nin: meshlist } }, { multi: true });
            } else {
                // NeDB or MongoJS
                obj.file.remove({ meshid: { $exists: true, $nin: meshlist } }, { multi: true });
            }

            // Fix all of the creating & login to ticks by seconds, not milliseconds.
            obj.GetAllType('user', function (err, docs) {
                if (err != null) { parent.debug('db', 'ERROR (GetAll user): ' + err); }
                if ((err == null) && (docs.length > 0)) {
                    for (var i in docs) {
                        var fixed = false;

                        // Fix email address capitalization
                        if (docs[i].email && (docs[i].email != docs[i].email.toLowerCase())) {
                            docs[i].email = docs[i].email.toLowerCase(); fixed = true;
                        }

                        // Fix account creation
                        if (docs[i].creation) {
                            if (docs[i].creation > 1300000000000) { docs[i].creation = Math.floor(docs[i].creation / 1000); fixed = true; }
                            if ((docs[i].creation % 1) != 0) { docs[i].creation = Math.floor(docs[i].creation); fixed = true; }
                        }

                        // Fix last account login
                        if (docs[i].login) {
                            if (docs[i].login > 1300000000000) { docs[i].login = Math.floor(docs[i].login / 1000); fixed = true; }
                            if ((docs[i].login % 1) != 0) { docs[i].login = Math.floor(docs[i].login); fixed = true; }
                        }

                        // Fix last password change
                        if (docs[i].passchange) {
                            if (docs[i].passchange > 1300000000000) { docs[i].passchange = Math.floor(docs[i].passchange / 1000); fixed = true; }
                            if ((docs[i].passchange % 1) != 0) { docs[i].passchange = Math.floor(docs[i].passchange); fixed = true; }
                        }

                        // Fix subscriptions
                        if (docs[i].subscriptions != null) { delete docs[i].subscriptions; fixed = true; }

                        // Save the user if needed
                        if (fixed) { obj.Set(docs[i]); }

                        // We are done
                        if (func) { func(); }
                    }
                }
            });
        });
    };

    // Get encryption key
    obj.getEncryptDataKey = function (password) {
        if (typeof password != 'string') return null;
        return parent.crypto.createHash('sha384').update(password).digest("raw").slice(0, 32);
    }

    // Encrypt data 
    obj.encryptData = function (password, plaintext) {
        var key = obj.getEncryptDataKey(password);
        if (key == null) return null;
        const iv = parent.crypto.randomBytes(16);
        const aes = parent.crypto.createCipheriv('aes-256-cbc', key, iv);
        var ciphertext = aes.update(plaintext);
        ciphertext = Buffer.concat([iv, ciphertext, aes.final()]);
        return ciphertext.toString('base64');
    }

    // Decrypt data 
    obj.decryptData = function (password, ciphertext) {
        try {
            var key = obj.getEncryptDataKey(password);
            if (key == null) return null;
            const ciphertextBytes = Buffer.from(ciphertext, 'base64');
            const iv = ciphertextBytes.slice(0, 16);
            const data = ciphertextBytes.slice(16);
            const aes = parent.crypto.createDecipheriv('aes-256-cbc', key, iv);
            var plaintextBytes = Buffer.from(aes.update(data));
            plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
            return plaintextBytes;
        } catch (ex) { return null; }
    }

    // Get the number of records in the database for various types, this is the slow NeDB way.
    // WARNING: This is a terrible query for database performance. Only do this when needed. This query will look at almost every document in the database.
    obj.getStats = function (func) {
        if (obj.databaseType == 3) {
            // MongoDB
            obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }]).toArray(function (err, docs) {
                var counters = {}, totalCount = 0;
                if (err == null) { for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } }
                func(counters);
            });
        } else if (obj.databaseType == 2) {
            // MongoJS
            obj.file.aggregate([{ "$group": { _id: "$type", count: { $sum: 1 } } }], function (err, docs) {
                var counters = {}, totalCount = 0;
                if (err == null) { for (var i in docs) { if (docs[i]._id != null) { counters[docs[i]._id] = docs[i].count; totalCount += docs[i].count; } } }
                func(counters);
            });
        } else if (obj.databaseType == 1) {
            // NeDB version
            obj.file.count({ type: 'node' }, function (err, nodeCount) {
                obj.file.count({ type: 'mesh' }, function (err, meshCount) {
                    obj.file.count({ type: 'user' }, function (err, userCount) {
                        obj.file.count({ type: 'sysinfo' }, function (err, sysinfoCount) {
                            obj.file.count({ type: 'note' }, function (err, noteCount) {
                                obj.file.count({ type: 'iploc' }, function (err, iplocCount) {
                                    obj.file.count({ type: 'ifinfo' }, function (err, ifinfoCount) {
                                        obj.file.count({ type: 'cfile' }, function (err, cfileCount) {
                                            obj.file.count({ type: 'lastconnect' }, function (err, lastconnectCount) {
                                                obj.file.count({}, function (err, totalCount) {
                                                    func({ node: nodeCount, mesh: meshCount, user: userCount, sysinfo: sysinfoCount, iploc: iplocCount, note: noteCount, ifinfo: ifinfoCount, cfile: cfileCount, lastconnect: lastconnectCount, total: totalCount });
                                                });
                                            });
                                        });
                                    });
                                });
                            });
                        });
                    });
                });
            });
        }
    }

    // This is used to rate limit a number of operation per day. Returns a startValue each new days, but you can substract it and save the value in the db.
    obj.getValueOfTheDay = function (id, startValue, func) { obj.Get(id, function (err, docs) { var date = new Date(), t = date.toLocaleDateString(); if ((err == null) && (docs.length == 1)) { var r = docs[0]; if (r.day == t) { func({ _id: id, value: r.value, day: t }); return; } } func({ _id: id, value: startValue, day: t }); }); };
    obj.escapeBase64 = function escapeBase64(val) { return (val.replace(/\+/g, '@').replace(/\//g, '$')); }

    // Encrypt an database object
    obj.performRecordEncryptionRecode = function (func) {
        var count = 0;
        obj.GetAllType('user', function (err, docs) {
            if (err != null) { parent.debug('db', 'ERROR (performRecordEncryptionRecode): ' + err); }
            if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
            obj.GetAllType('node', function (err, docs) {
                if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
                obj.GetAllType('mesh', function (err, docs) {
                    if (err == null) { for (var i in docs) { count++; obj.Set(docs[i]); } }
                    if (obj.databaseType == 1) { // If we are using NeDB, compact the database.
                        obj.file.persistence.compactDatafile();
                        obj.file.on('compaction.done', function () { func(count); }); // It's important to wait for compaction to finish before exit, otherwise NeDB may corrupt.
                    } else {
                        func(count); // For all other databases, normal exit.
                    }
                });
            });
        });
    }

    // Encrypt an database object
    function performTypedRecordDecrypt(data) {
        if ((data == null) || (obj.dbRecordsDecryptKey == null) || (typeof data != 'object')) return data;
        for (var i in data) {
            if (data[i].type == 'user') {
                data[i] = performPartialRecordDecrypt(data[i]);
            } else if ((data[i].type == 'node') && (data[i].intelamt != null)) {
                data[i].intelamt = performPartialRecordDecrypt(data[i].intelamt);
            } else if ((data[i].type == 'mesh') && (data[i].amt != null)) {
                data[i].amt = performPartialRecordDecrypt(data[i].amt);
            }
        }
        return data;
    }

    // Encrypt an database object
    function performTypedRecordEncrypt(data) {
        if (obj.dbRecordsEncryptKey == null) return data;
        if (data.type == 'user') { return performPartialRecordEncrypt(Clone(data), ['otpkeys', 'otphkeys', 'otpsecret', 'salt', 'hash', 'oldpasswords']); }
        else if ((data.type == 'node') && (data.intelamt != null)) { var xdata = Clone(data); xdata.intelamt = performPartialRecordEncrypt(xdata.intelamt, ['user', 'pass']); return xdata; }
        else if ((data.type == 'mesh') && (data.amt != null)) { var xdata = Clone(data); xdata.amt = performPartialRecordEncrypt(xdata.amt, ['password']); return xdata; }
        return data;
    }

    // Encrypt an object and return a buffer.
    function performPartialRecordEncrypt(plainobj, encryptNames) {
        if (typeof plainobj != 'object') return plainobj;
        var enc = {}, enclen = 0;
        for (var i in encryptNames) { if (plainobj[encryptNames[i]] != null) { enclen++; enc[encryptNames[i]] = plainobj[encryptNames[i]]; delete plainobj[encryptNames[i]]; } }
        if (enclen > 0) { plainobj._CRYPT = performRecordEncrypt(enc); } else { delete plainobj._CRYPT; }
        return plainobj;
    }

    // Encrypt an object and return a buffer.
    function performPartialRecordDecrypt(plainobj) {
        if ((typeof plainobj != 'object') || (plainobj._CRYPT == null)) return plainobj;
        var enc = performRecordDecrypt(plainobj._CRYPT);
        if (enc != null) { for (var i in enc) { plainobj[i] = enc[i]; } }
        delete plainobj._CRYPT;
        return plainobj;
    }

    // Encrypt an object and return a base64.
    function performRecordEncrypt(plainobj) {
        if (obj.dbRecordsEncryptKey == null) return null;
        const iv = parent.crypto.randomBytes(12);
        const aes = parent.crypto.createCipheriv('aes-256-gcm', obj.dbRecordsEncryptKey, iv);
        var ciphertext = aes.update(JSON.stringify(plainobj));
        var cipherfinal = aes.final();
        ciphertext = Buffer.concat([iv, aes.getAuthTag(), ciphertext, cipherfinal]);
        return ciphertext.toString('base64');
    }

    // Takes a base64 and return an object.
    function performRecordDecrypt(ciphertext) {
        if (obj.dbRecordsDecryptKey == null) return null;
        const ciphertextBytes = Buffer.from(ciphertext, 'base64');
        const iv = ciphertextBytes.slice(0, 12);
        const data = ciphertextBytes.slice(28);
        const aes = parent.crypto.createDecipheriv('aes-256-gcm', obj.dbRecordsDecryptKey, iv);
        aes.setAuthTag(ciphertextBytes.slice(12, 28));
        var plaintextBytes, r;
        try {
            plaintextBytes = Buffer.from(aes.update(data));
            plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
            r = JSON.parse(plaintextBytes.toString());
        } catch (e) { throw "Incorrect DbRecordsDecryptKey/DbRecordsEncryptKey or invalid database _CRYPT data: " + e; }
        return r;
    }

    // Clone an object (TODO: Make this more efficient)
    function Clone(v) { return JSON.parse(JSON.stringify(v)); }

    // Read expiration time from configuration file
    if (typeof parent.args.dbexpire == 'object') {
        if (typeof parent.args.dbexpire.events == 'number') { expireEventsSeconds = parent.args.dbexpire.events; }
        if (typeof parent.args.dbexpire.powerevents == 'number') { expirePowerEventsSeconds = parent.args.dbexpire.powerevents; }
        if (typeof parent.args.dbexpire.statsevents == 'number') { expireServerStatsSeconds = parent.args.dbexpire.statsevents; }
    }

    // If a DB record encryption key is provided, perform database record encryption
    if ((typeof parent.args.dbrecordsencryptkey == 'string') && (parent.args.dbrecordsencryptkey.length != 0)) {
        // Hash the database password into a AES256 key and setup encryption and decryption.
        obj.dbRecordsEncryptKey = obj.dbRecordsDecryptKey = parent.crypto.createHash('sha384').update(parent.args.dbrecordsencryptkey).digest('raw').slice(0, 32);
    }

    // If a DB record decryption key is provided, perform database record decryption
    if ((typeof parent.args.dbrecordsdecryptkey == 'string') && (parent.args.dbrecordsdecryptkey.length != 0)) {
        // Hash the database password into a AES256 key and setup encryption and decryption.
        obj.dbRecordsDecryptKey = parent.crypto.createHash('sha384').update(parent.args.dbrecordsdecryptkey).digest('raw').slice(0, 32);
    }

    if (parent.args.mariadb || parent.args.mysql) {
        if (parent.args.mariadb) {
            // Use MariaDB
            obj.databaseType = 4;
            Datastore = require('mariadb').createPool(parent.args.mariadb);
        } else if (parent.args.mysql) {
            // Use MySQL
            Datastore = require('mysql').createConnection(parent.args.mysql);
            obj.databaseType = 5;
        }
        //sqlDbQuery('DROP DATABASE MeshCentral', null, function (err, docs) { console.log('DROP'); }); return;
        sqlDbQuery('USE meshcentral', null, function (err, docs) {
            if (err != null) { parent.debug('db', 'ERROR: USE meshcentral: ' + err); }
            if (err == null) { setupFunctions(func); } else {
                parent.debug('db', 'Creating database...');
                sqlDbBatchExec([
                    'CREATE DATABASE meshcentral',
                    // Main table
                    'CREATE TABLE meshcentral.main (id VARCHAR(256) NOT NULL, type CHAR(32), domain CHAR(64), extra CHAR(255), extraex CHAR(255), doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
                    'CREATE INDEX ndxtypedomainextra ON meshcentral.main (type, domain, extra)',
                    'CREATE INDEX ndxextra ON meshcentral.main (extra)',
                    'CREATE INDEX ndxextraex ON meshcentral.main (extraex)',
                    // Events table
                    'CREATE TABLE meshcentral.events(id INT NOT NULL AUTO_INCREMENT, time DATETIME, domain CHAR(64), action CHAR(255), nodeid CHAR(255), userid CHAR(255), doc JSON, PRIMARY KEY(id), CHECK(json_valid(doc)))',
                    'CREATE INDEX ndxeventstime ON meshcentral.events(time)',
                    'CREATE INDEX ndxeventsusername ON meshcentral.events(domain, userid, time)',
                    'CREATE INDEX ndxeventsdomainnodeidtime ON meshcentral.events(domain, nodeid, time)',
                    // Events ID table
                    'CREATE TABLE meshcentral.eventids(fkid INT NOT NULL, target CHAR(255), CONSTRAINT fk_eventid FOREIGN KEY (fkid) REFERENCES events (id) ON DELETE CASCADE ON UPDATE RESTRICT)',
                    'CREATE INDEX ndxeventids ON meshcentral.eventids(target)',
                    // Server stats table
                    'CREATE TABLE meshcentral.serverstats (time DATETIME, expire DATETIME, doc JSON, PRIMARY KEY(time), CHECK (json_valid(doc)))',
                    'CREATE INDEX ndxserverstattime ON meshcentral.serverstats (time)',
                    'CREATE INDEX ndxserverstatexpire ON meshcentral.serverstats (expire)',
                    // Power events table
                    'CREATE TABLE meshcentral.power (id INT NOT NULL AUTO_INCREMENT, time DATETIME, nodeid CHAR(255), doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
                    'CREATE INDEX ndxpowernodeidtime ON meshcentral.power (nodeid, time)',
                    // SMBIOS table
                    'CREATE TABLE meshcentral.smbios (id CHAR(255), time DATETIME, expire DATETIME, doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))',
                    'CREATE INDEX ndxsmbiostime ON meshcentral.smbios (time)',
                    'CREATE INDEX ndxsmbiosexpire ON meshcentral.smbios (expire)',
                    // Plugins table
                    'CREATE TABLE meshcentral.plugin (id INT NOT NULL AUTO_INCREMENT, doc JSON, PRIMARY KEY(id), CHECK (json_valid(doc)))'
                ], function (err) {
                    if (err != null) { parent.debug('db', 'BatchSetupDb: ' + err); }
                    setupFunctions(func);
                });
            }
        });
    } else if (parent.args.mongodb) {
        // Use MongoDB
        obj.databaseType = 3;
        require('mongodb').MongoClient.connect(parent.args.mongodb, { useNewUrlParser: true, useUnifiedTopology: true }, function (err, client) {
            if (err != null) { console.log("Unable to connect to database: " + err); process.exit(); return; }
            Datastore = client;
            parent.debug('db', 'Connected to MongoDB database...');

            // Get the database name and setup the database client
            var dbname = 'meshcentral';
            if (parent.args.mongodbname) { dbname = parent.args.mongodbname; }
            const dbcollectionname = (parent.args.mongodbcol) ? (parent.args.mongodbcol) : 'meshcentral';
            const db = client.db(dbname);

            // Check the database version
            db.admin().serverInfo(function (err, info) {
                if ((err != null) || (info == null) || (info.versionArray == null) || (Array.isArray(info.versionArray) == false) || (info.versionArray.length < 2) || (typeof info.versionArray[0] != 'number') || (typeof info.versionArray[1] != 'number')) {
                    console.log('WARNING: Unable to check MongoDB version.');
                } else {
                    if ((info.versionArray[0] < 3) || ((info.versionArray[0] == 3) && (info.versionArray[1] < 6))) {
                        // We are running with mongoDB older than 3.6, this is not good.
                        parent.addServerWarning("Current version of MongoDB (" + info.version + ") is too old, please upgrade to MongoDB 3.6 or better.");
                    }
                }
            });

            // Setup MongoDB main collection and indexes
            obj.file = db.collection(dbcollectionname);
            obj.file.indexes(function (err, indexes) {
                // Check if we need to reset indexes
                var indexesByName = {}, indexCount = 0;
                for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
                if ((indexCount != 4) || (indexesByName['TypeDomainMesh1'] == null) || (indexesByName['Email1'] == null) || (indexesByName['Mesh1'] == null)) {
                    console.log('Resetting main indexes...');
                    obj.file.dropIndexes(function (err) {
                        obj.file.createIndex({ type: 1, domain: 1, meshid: 1 }, { sparse: 1, name: 'TypeDomainMesh1' });       // Speeds up GetAllTypeNoTypeField() and GetAllTypeNoTypeFieldMeshFiltered()
                        obj.file.createIndex({ email: 1 }, { sparse: 1, name: 'Email1' });                                     // Speeds up GetUserWithEmail() and GetUserWithVerifiedEmail()
                        obj.file.createIndex({ meshid: 1 }, { sparse: 1, name: 'Mesh1' });                                     // Speeds up RemoveMesh()
                    });
                }
            });

            // Setup the changeStream on the MongoDB main collection if possible
            if (parent.args.mongodbchangestream == true) {
                if (typeof obj.file.watch != 'function') {
                    console.log('WARNING: watch() is not a function, MongoDB ChangeStream not supported.');
                } else {
                    obj.fileChangeStream = obj.file.watch([{ $match: { $or: [{ 'fullDocument.type': { $in: ['node', 'mesh', 'user', 'ugrp'] } }, { 'operationType': 'delete' }] } }], { fullDocument: 'updateLookup' });
                    obj.fileChangeStream.on('change', function (change) {
                        if (change.operationType == 'update') {
                            switch (change.fullDocument.type) {
                                case 'node': { dbNodeChange(change, false); break; } // A node has changed
                                case 'mesh': { dbMeshChange(change, false); break; } // A device group has changed
                                case 'user': { dbUserChange(change, false); break; } // A user account has changed
                                case 'ugrp': { dbUGrpChange(change, false); break; } // A user account has changed
                            }
                        } else if (change.operationType == 'insert') {
                            switch (change.fullDocument.type) {
                                case 'node': { dbNodeChange(change, true); break; } // A node has added
                                case 'mesh': { dbMeshChange(change, true); break; } // A device group has created
                                case 'user': { dbUserChange(change, true); break; } // A user account has created
                                case 'ugrp': { dbUGrpChange(change, true); break; } // A user account has created
                            }
                        } else if (change.operationType == 'delete') {
                            var splitId = change.documentKey._id.split('/');
                            switch (splitId[0]) {
                                case 'node': {
                                    //Not Good: Problem here is that we don't know what meshid the node belonged to before the delete.
                                    //parent.DispatchEvent(['*', node.meshid], obj, { etype: 'node', action: 'removenode', nodeid: change.documentKey._id, domain: splitId[1] });
                                    break;
                                }
                                case 'mesh': {
                                    parent.DispatchEvent(['*', change.documentKey._id], obj, { etype: 'mesh', action: 'deletemesh', meshid: change.documentKey._id, domain: splitId[1] });
                                    break;
                                }
                                case 'user': {
                                    //Not Good: This is not a perfect user removal because we don't know what groups the user was in.
                                    //parent.DispatchEvent(['*', 'server-users'], obj, { etype: 'user', action: 'accountremove', userid: change.documentKey._id, domain: splitId[1], username: splitId[2] });
                                    break;
                                }
                                case 'ugrp': {
                                    parent.DispatchEvent(['*', change.documentKey._id], obj, { etype: 'ugrp', action: 'deleteusergroup', ugrpid: change.documentKey._id, domain: splitId[1] });
                                    break;
                                }
                            }
                        }
                    });
                    obj.changeStream = true;
                }
            }

            // Setup MongoDB events collection and indexes
            obj.eventsfile = db.collection('events'); // Collection containing all events
            obj.eventsfile.indexes(function (err, indexes) {
                // Check if we need to reset indexes
                var indexesByName = {}, indexCount = 0;
                for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
                if ((indexCount != 5) || (indexesByName['Username1'] == null) || (indexesByName['DomainNodeTime1'] == null) || (indexesByName['IdsAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) {
                    // Reset all indexes
                    console.log("Resetting events indexes...");
                    obj.eventsfile.dropIndexes(function (err) {
                        obj.eventsfile.createIndex({ username: 1 }, { sparse: 1, name: 'Username1' });
                        obj.eventsfile.createIndex({ domain: 1, nodeid: 1, time: -1 }, { sparse: 1, name: 'DomainNodeTime1' });
                        obj.eventsfile.createIndex({ ids: 1, time: -1 }, { sparse: 1, name: 'IdsAndTime1' });
                        obj.eventsfile.createIndex({ time: 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' });
                    });
                } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireEventsSeconds) {
                    // Reset the timeout index
                    console.log("Resetting events expire index...");
                    obj.eventsfile.dropIndex('ExpireTime1', function (err) {
                        obj.eventsfile.createIndex({ time: 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' });
                    });
                }
            });

            // Setup MongoDB power events collection and indexes
            obj.powerfile = db.collection('power');                                 // Collection containing all power events
            obj.powerfile.indexes(function (err, indexes) {
                // Check if we need to reset indexes
                var indexesByName = {}, indexCount = 0;
                for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
                if ((indexCount != 3) || (indexesByName['NodeIdAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) {
                    // Reset all indexes
                    console.log("Resetting power events indexes...");
                    obj.powerfile.dropIndexes(function (err) {
                        // Create all indexes
                        obj.powerfile.createIndex({ nodeid: 1, time: 1 }, { sparse: 1, name: 'NodeIdAndTime1' });
                        obj.powerfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' });
                    });
                } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expirePowerEventsSeconds) {
                    // Reset the timeout index
                    console.log("Resetting power events expire index...");
                    obj.powerfile.dropIndex('ExpireTime1', function (err) {
                        // Reset the expire power events index
                        obj.powerfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' });
                    });
                }
            });

            // Setup MongoDB smbios collection, no indexes needed
            obj.smbiosfile = db.collection('smbios');                               // Collection containing all smbios information

            // Setup MongoDB server stats collection
            obj.serverstatsfile = db.collection('serverstats');                     // Collection of server stats
            obj.serverstatsfile.indexes(function (err, indexes) {
                // Check if we need to reset indexes
                var indexesByName = {}, indexCount = 0;
                for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
                if ((indexCount != 3) || (indexesByName['ExpireTime1'] == null)) {
                    // Reset all indexes
                    console.log("Resetting server stats indexes...");
                    obj.serverstatsfile.dropIndexes(function (err) {
                        // Create all indexes
                        obj.serverstatsfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' });
                        obj.serverstatsfile.createIndex({ 'expire': 1 }, { expireAfterSeconds: 0, name: 'ExpireTime2' });  // Auto-expire events
                    });
                } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireServerStatsSeconds) {
                    // Reset the timeout index
                    console.log("Resetting server stats expire index...");
                    obj.serverstatsfile.dropIndex('ExpireTime1', function (err) {
                        // Reset the expire server stats index
                        obj.serverstatsfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' });
                    });
                }
            });

            // Setup plugin info collection
            if (obj.pluginsActive) { obj.pluginsfile = db.collection('plugins'); }

            setupFunctions(func); // Completed setup of MongoDB
        });
    } else if (parent.args.xmongodb) {
        // Use MongoJS, this is the old system.
        obj.databaseType = 2;
        Datastore = require('mongojs');
        var db = Datastore(parent.args.xmongodb);
        var dbcollection = 'meshcentral';
        if (parent.args.mongodbcol) { dbcollection = parent.args.mongodbcol; }

        // Setup MongoDB main collection and indexes
        obj.file = db.collection(dbcollection);
        obj.file.getIndexes(function (err, indexes) {
            // Check if we need to reset indexes
            var indexesByName = {}, indexCount = 0;
            for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
            if ((indexCount != 4) || (indexesByName['TypeDomainMesh1'] == null) || (indexesByName['Email1'] == null) || (indexesByName['Mesh1'] == null)) {
                console.log("Resetting main indexes...");
                obj.file.dropIndexes(function (err) {
                    obj.file.createIndex({ type: 1, domain: 1, meshid: 1 }, { sparse: 1, name: 'TypeDomainMesh1' });       // Speeds up GetAllTypeNoTypeField() and GetAllTypeNoTypeFieldMeshFiltered()
                    obj.file.createIndex({ email: 1 }, { sparse: 1, name: 'Email1' });                                     // Speeds up GetUserWithEmail() and GetUserWithVerifiedEmail()
                    obj.file.createIndex({ meshid: 1 }, { sparse: 1, name: 'Mesh1' });                                     // Speeds up RemoveMesh()
                });
            }
        });

        // Setup MongoDB events collection and indexes
        obj.eventsfile = db.collection('events');                               // Collection containing all events
        obj.eventsfile.getIndexes(function (err, indexes) {
            // Check if we need to reset indexes
            var indexesByName = {}, indexCount = 0;
            for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
            if ((indexCount != 5) || (indexesByName['Username1'] == null) || (indexesByName['DomainNodeTime1'] == null) || (indexesByName['IdsAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) {
                // Reset all indexes
                console.log("Resetting events indexes...");
                obj.eventsfile.dropIndexes(function (err) {
                    obj.eventsfile.createIndex({ username: 1 }, { sparse: 1, name: 'Username1' });
                    obj.eventsfile.createIndex({ domain: 1, nodeid: 1, time: -1 }, { sparse: 1, name: 'DomainNodeTime1' });
                    obj.eventsfile.createIndex({ ids: 1, time: -1 }, { sparse: 1, name: 'IdsAndTime1' });
                    obj.eventsfile.createIndex({ time: 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' });
                });
            } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireEventsSeconds) {
                // Reset the timeout index
                console.log("Resetting events expire index...");
                obj.eventsfile.dropIndex('ExpireTime1', function (err) {
                    obj.eventsfile.createIndex({ time: 1 }, { expireAfterSeconds: expireEventsSeconds, name: 'ExpireTime1' });
                });
            }
        });

        // Setup MongoDB power events collection and indexes
        obj.powerfile = db.collection('power');                                 // Collection containing all power events
        obj.powerfile.getIndexes(function (err, indexes) {
            // Check if we need to reset indexes
            var indexesByName = {}, indexCount = 0;
            for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
            if ((indexCount != 3) || (indexesByName['NodeIdAndTime1'] == null) || (indexesByName['ExpireTime1'] == null)) {
                // Reset all indexes
                console.log("Resetting power events indexes...");
                obj.powerfile.dropIndexes(function (err) {
                    // Create all indexes
                    obj.powerfile.createIndex({ nodeid: 1, time: 1 }, { sparse: 1, name: 'NodeIdAndTime1' });
                    obj.powerfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' });
                });
            } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expirePowerEventsSeconds) {
                // Reset the timeout index
                console.log("Resetting power events expire index...");
                obj.powerfile.dropIndex('ExpireTime1', function (err) {
                    // Reset the expire power events index
                    obj.powerfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expirePowerEventsSeconds, name: 'ExpireTime1' });
                });
            }
        });

        // Setup MongoDB smbios collection, no indexes needed
        obj.smbiosfile = db.collection('smbios');                               // Collection containing all smbios information

        // Setup MongoDB server stats collection
        obj.serverstatsfile = db.collection('serverstats');                     // Collection of server stats
        obj.serverstatsfile.getIndexes(function (err, indexes) {
            // Check if we need to reset indexes
            var indexesByName = {}, indexCount = 0;
            for (var i in indexes) { indexesByName[indexes[i].name] = indexes[i]; indexCount++; }
            if ((indexCount != 3) || (indexesByName['ExpireTime1'] == null)) {
                // Reset all indexes
                console.log("Resetting server stats indexes...");
                obj.serverstatsfile.dropIndexes(function (err) {
                    // Create all indexes
                    obj.serverstatsfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' });
                    obj.serverstatsfile.createIndex({ 'expire': 1 }, { expireAfterSeconds: 0, name: 'ExpireTime2' });  // Auto-expire events
                });
            } else if (indexesByName['ExpireTime1'].expireAfterSeconds != expireServerStatsSeconds) {
                // Reset the timeout index
                console.log("Resetting server stats expire index...");
                obj.serverstatsfile.dropIndex('ExpireTime1', function (err) {
                    // Reset the expire server stats index
                    obj.serverstatsfile.createIndex({ 'time': 1 }, { expireAfterSeconds: expireServerStatsSeconds, name: 'ExpireTime1' });
                });
            }
        });

        // Setup plugin info collection
        if (obj.pluginsActive) { obj.pluginsfile = db.collection('plugins'); }

        setupFunctions(func); // Completed setup of MongoJS
    } else {
        // Use NeDB (The default)
        obj.databaseType = 1;
        Datastore = require('nedb');
        var datastoreOptions = { filename: parent.getConfigFilePath('meshcentral.db'), autoload: true };

        // If a DB encryption key is provided, perform database encryption
        if ((typeof parent.args.dbencryptkey == 'string') && (parent.args.dbencryptkey.length != 0)) {
            // Hash the database password into a AES256 key and setup encryption and decryption.
            obj.dbKey = parent.crypto.createHash('sha384').update(parent.args.dbencryptkey).digest('raw').slice(0, 32);
            datastoreOptions.afterSerialization = function (plaintext) {
                const iv = parent.crypto.randomBytes(16);
                const aes = parent.crypto.createCipheriv('aes-256-cbc', obj.dbKey, iv);
                var ciphertext = aes.update(plaintext);
                ciphertext = Buffer.concat([iv, ciphertext, aes.final()]);
                return ciphertext.toString('base64');
            }
            datastoreOptions.beforeDeserialization = function (ciphertext) {
                const ciphertextBytes = Buffer.from(ciphertext, 'base64');
                const iv = ciphertextBytes.slice(0, 16);
                const data = ciphertextBytes.slice(16);
                const aes = parent.crypto.createDecipheriv('aes-256-cbc', obj.dbKey, iv);
                var plaintextBytes = Buffer.from(aes.update(data));
                plaintextBytes = Buffer.concat([plaintextBytes, aes.final()]);
                return plaintextBytes.toString();
            }
        }

        // Start NeDB main collection and setup indexes
        obj.file = new Datastore(datastoreOptions);
        obj.file.persistence.setAutocompactionInterval(86400000); // Compact once a day
        obj.file.ensureIndex({ fieldName: 'type' });
        obj.file.ensureIndex({ fieldName: 'domain' });
        obj.file.ensureIndex({ fieldName: 'meshid', sparse: true });
        obj.file.ensureIndex({ fieldName: 'nodeid', sparse: true });
        obj.file.ensureIndex({ fieldName: 'email', sparse: true });

        // Setup the events collection and setup indexes
        obj.eventsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-events.db'), autoload: true, corruptAlertThreshold: 1 });
        obj.eventsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
        obj.eventsfile.ensureIndex({ fieldName: 'ids' }); // TODO: Not sure if this is a good index, this is a array field.
        obj.eventsfile.ensureIndex({ fieldName: 'nodeid', sparse: true });
        obj.eventsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: expireEventsSeconds });
        obj.eventsfile.remove({ time: { '$lt': new Date(Date.now() - (expireEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events

        // Setup the power collection and setup indexes
        obj.powerfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-power.db'), autoload: true, corruptAlertThreshold: 1 });
        obj.powerfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
        obj.powerfile.ensureIndex({ fieldName: 'nodeid' });
        obj.powerfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: expirePowerEventsSeconds });
        obj.powerfile.remove({ time: { '$lt': new Date(Date.now() - (expirePowerEventsSeconds * 1000)) } }, { multi: true }); // Force delete older events

        // Setup the SMBIOS collection
        obj.smbiosfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-smbios.db'), autoload: true, corruptAlertThreshold: 1 });

        // Setup the server stats collection and setup indexes
        obj.serverstatsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-stats.db'), autoload: true, corruptAlertThreshold: 1 });
        obj.serverstatsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
        obj.serverstatsfile.ensureIndex({ fieldName: 'time', expireAfterSeconds: expireServerStatsSeconds });
        obj.serverstatsfile.ensureIndex({ fieldName: 'expire', expireAfterSeconds: 0 }); // Auto-expire events
        obj.serverstatsfile.remove({ time: { '$lt': new Date(Date.now() - (expireServerStatsSeconds * 1000)) } }, { multi: true }); // Force delete older events

        // Setup plugin info collection
        if (obj.pluginsActive) {
            obj.pluginsfile = new Datastore({ filename: parent.getConfigFilePath('meshcentral-plugins.db'), autoload: true });
            obj.pluginsfile.persistence.setAutocompactionInterval(86400000); // Compact once a day
        }

        setupFunctions(func); // Completed setup of NeDB
    }

    // Check the object names for a "."
    function checkObjectNames(r, tag) {
        if (typeof r != 'object') return;
        for (var i in r) {
            if (i.indexOf('.') >= 0) { throw ('BadDbName (' + tag + '): ' + JSON.stringify(r)); }
            checkObjectNames(r[i], tag);
        }
    }

    // Query the database
    function sqlDbQuery(query, args, func) {
        if (obj.databaseType == 4) { // MariaDB
            Datastore.getConnection()
                .then(function (conn) {
                    conn.query(query, args)
                        .then(function (rows) {
                            conn.release();
                            const docs = [];
                            for (var i in rows) { if (rows[i].doc) { docs.push(performTypedRecordDecrypt(JSON.parse(rows[i].doc))); } }
                            if (func) try { func(null, docs); } catch (ex) { console.log('SQLERR1', ex); }
                        })
                        .catch(function (err) { conn.release(); if (func) try { func(err); } catch (ex) { console.log('SQLERR2', ex); } });
                }).catch(function (err) { if (func) { try { func(err); } catch (ex) { console.log('SQLERR3', ex); } } });
        } else if (obj.databaseType == 5) { // MySQL
            Datastore.query(query, args, function (error, results, fields) {
                if (error != null) {
                    if (func) try { func(error); } catch (ex) { console.log('SQLERR4', ex); }
                } else {
                    var docs = [];
                    for (var i in results) { if (results[i].doc) { docs.push(JSON.parse(results[i].doc)); } }
                    //console.log(docs);
                    if (func) { try { func(null, docs); } catch (ex) { console.log('SQLERR5', ex); } }
                }
            });
        }
    }

    // Exec on the database
    function sqlDbExec(query, args, func) {
        if (obj.databaseType == 4) { // MariaDB
            Datastore.getConnection()
                .then(function (conn) {
                    conn.query(query, args)
                        .then(function (rows) {
                            conn.release();
                            if (func) try { func(null, rows[0]); } catch (ex) { console.log(ex); }
                        })
                        .catch(function (err) { conn.release(); if (func) try { func(err); } catch (ex) { console.log(ex); } });
                    }).catch(function (err) { if (func) { try { func(err); } catch (ex) { console.log(ex); } } });
        } else if (obj.databaseType == 5) { // MySQL
            Datastore.query(query, args, function (error, results, fields) {
                if (func) try { func(error, results[0]); } catch (ex) { console.log(ex); }
            });
        }
    }

    // E