crypto-bot/lib/utils/tools.js

multi-function tools for blockchain automation

const { logger } = require('./logger');
const fs = require('fs');
const fsExtra = require('fs-extra')
const path = require('path');
const { ethers } = require("ethers");

const  botType  = require('./../core/type');
const  botConfig  = require('./../core/filename');

module.exports = {
    create_UUID: function (){
        var dt = new Date().getTime();
        var uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
            var r = (dt + Math.random()*16)%16 | 0;
            dt = Math.floor(dt/16);
            return (c=='x' ? r :(r&0x3|0x8)).toString(16);
        });
        return uuid;
    }, 
    
    create_MongoUUID: function (){
        var dt = new Date().getTime();
        var uuid = 'xxxxxxxxxxxxcxxxyxxxxxxx'.replace(/[xy]/g, function(c) {
            var r = (dt + Math.random()*16)%16 | 0;
            dt = Math.floor(dt/16);
            return (c=='x' ? r :(r&0x3|0x8)).toString(16);
        });
        return uuid;
    },

    create_BACKUP_name: function(){
        var _bck = new Date().toISOString().replace('T', '_').replace(':', '_').replace(/:+/, '_').substr(0, 19);
        return _bck
    },

    DB_WALLET_BACKUP: function(src, dst){
        
        const _src = './data/wallet/database/walletDB.json'
        const _dst = './data/wallet/database/backup/' + this.create_BACKUP_name() + "_walletDB.json"; 
        try{
            fsExtra.copySync(_src, _dst)
            return true
        }
        catch(e){
            return false
        }
    },

    createBackup: function(Network, _type, _TypeObject){

        const dir = this.getDirPath(Network, _type, _TypeObject);

        const dest_dir = dir + "backup/"
        const filename = this.getFilename(Network, _type, _TypeObject);

        const _src = dir + filename
        const _dst = dest_dir + this.create_BACKUP_name() + "_" + filename; 
        console.log(_src)
        console.log(_dst)

        if (!fs.existsSync(_src)){
            logger.info(`no backup created,  ${_src} file does not exists.`)
            return true
        }

        try{
            fsExtra.copySync(_src, _dst)
            return true
        }
        catch(e){
            return false
        }
    },

        // Merge function
    mergeWalletDB: async function(){
        const _CurrentDB = await this.loadJSONWalletData(true)
        logger.info(`${_CurrentDB.length} record to be merged and validated.`)
        
        _CurrentDB.sort(function(a, b){return a.privkey - b.privkey});

        const _mergeList = new Array()
        const _duplicateList = new Array()
        let _duplicateCount = 0;
        let _noMnemonic = 0;

        for(let i = 0 ; i < _CurrentDB.length ; i++){
            const walletObject = _CurrentDB[i];


            if( _duplicateList.indexOf(walletObject.address) == -1 ){
                if(walletObject.mnemonic == undefined){
                    //logger.error("no mnemonic found.")
                    updWallet = new Object(this.checkNewKey(walletObject.privkey));
                    _noMnemonic++;
                    //console.log(updWallet)

                    const _walletObj = new Object({
                        address: updWallet.address, 
                        privkey: updWallet.privateKey,
                        mnemonic: updWallet.mnemonic
                    });
    
                    _duplicateList.push(walletObject.address);
                    _mergeList.push(_walletObj);
                }
                else {
                    const _walletObj = new Object({
                        address: walletObject.address, 
                        privkey: walletObject.privkey,
                        mnemonic: walletObject.mnemonic
                    });
    
                    _duplicateList.push(walletObject.address);
                    _mergeList.push(_walletObj);
                }
            } 
            else {
                _duplicateCount++;
            }
        }
        if(_duplicateCount > 0) logger.warn(`Found ${_duplicateCount} duplicate... will be removed.`)
        if(_noMnemonic > 0) logger.warn(`${_noMnemonic} record with missing mnemonic have been updated.`)
        
        let _overwrite = true;  // add option check here

        if(_overwrite){
            _backup = this.DB_WALLET_BACKUP();

            if(_backup){
                logger.info('Wallet DB has been backup. ')

            
                const _result = await this.writeFile(null, botType.WALLET_DB, null, _mergeList)
                if(_result) {
                    logger.info(`${_mergeList.length} record saved. `);
                    logger.info(`Temp folder will be cleared now... `);
                    await this.deleteFolderData(null, botType.WALLET_TEMP, null);
                    return true;
                }
                else {
                    logger.error(`error while saving merged record saved. `)
                    return false;
                }
            }
            else {
                logger.error(`Unable to create backup.`)
                return false;
            }
         }
        else{
            const _result = await this.writeFile(null, botType.WALLET_MERGE, null, _mergeList)
            if(_result) {
                logger.info(`${_mergeList.length} record saved. `)
                return true;
            }
            else {
                logger.error(`error while saving merged record saved. `)
                return false;
            }
        }
    },

    loadJSONWalletData: async function(_option){
      const FinalList = new Array();

      const _currentList = await this.loadMasterDBWalletData();
      const _curLen = _currentList.length;

      logger.info(` ${_curLen} Master DB wallet. `);
      for(let i = 0; i < _curLen ; i++){
          let _obj = _currentList[i];
          //console.log(_obj)
          FinalList.push(_obj)
      }

      let _target = undefined
      let _tempLen = 0;
      if( _option == undefined) _option = false;

      if( _option == true) {
          const _tempData = await this.loadTempDBWalletData();
          _tempLen = _tempData.length;
          for(let i = 0; i < _tempLen ; i++){
              let _obj = _tempData[i];
              FinalList.push(_obj)
          }
          logger.info(` ${_tempLen} Wallet from previous run. `);
      }
 
      if (_option) {
          _target = _tempLen + _curLen;

      } else{
          _target = _curLen;
      }
      const _count = FinalList.length;
      
      if( _count != _target){
          logger.error(`Mismatch loading master data... Count: ${_count} Target: ${_target} | Temp: ${_tempLen} | Current: ${_curLen} Option: ${_option}`)
      } else{
          logger.info(`Successfully loaded ${_count} wallet data.`)
      }

      //this.list = FinalList
      //this.status = 1        
      return FinalList;
    },

    loadMongoDBWalletData: async function(){


        const FinalList = new Array();
  
        const _currentList = await this.loadMasterDBWalletData();
        const _curLen = _currentList.length;
  
        logger.info(` ${_curLen} Master DB wallet. `);
        for(let i = 0; i < _curLen ; i++){
            let _obj = _currentList[i];
            //console.log(_obj)
            FinalList.push(_obj)
        }
  
        let _target = undefined
        let _tempLen = 0;
        if( _option == undefined) _option = false;
  
        if( _option == true) {
            const _tempData = await this.loadTempDBWalletData();
            _tempLen = _tempData.length;
            for(let i = 0; i < _tempLen ; i++){
                let _obj = _tempData[i];
                FinalList.push(_obj)
            }
            logger.info(` ${_tempLen} Wallet from previous run. `);
        }
   
        if (_option) {
            _target = _tempLen + _curLen;
  
        } else{
            _target = _curLen;
        }
        const _count = FinalList.length;
        
        if( _count != _target){
            logger.error(`Mismatch loading master data... Count: ${_count} Target: ${_target} | Temp: ${_tempLen} | Current: ${_curLen} Option: ${_option}`)
        } else{
            logger.info(`Successfully loaded ${_count} wallet data.`)
        }
  
        //this.list = FinalList
        //this.status = 1        
        return FinalList;
      },


        // return list of wallet data object.
    loadMasterDBWalletData: async function(){
        //console.log(botType)
        const _data = await this.readFile(null, botType.WALLET_DB, null)
        return _data
    },
    // return list of wallet data object.
    loadTempDBWalletData: async function(){
        const _data = await this.readFolderData(null, botType.WALLET_TEMP, null)
        return _data
    },

    initMongoDBWalletData: async function(data){
        const db = require("./config/models");

        console.log(db.url);
        
        db.mongoose
          .connect(db.url, {
            useNewUrlParser: true,
            useUnifiedTopology: true
          })
          .then(() => {
            console.log("Connected to the database!");
          })
          .catch(err => {
            console.log("Cannot connect to the database!", err);
            process.exit();
          });
        //************ */
        const _list = data
        const WalletData = new Array(); 
      
        const WalletsList = db.wallets;
      
        _list.forEach(result => {
          //const foldername = result.name;
          //const blockchain = getBlockchainInfo(foldername)
          const WalletInfo = result;
          WalletInfo._id = this.create_MongoUUID();
          
          //WalletInfo.info.id = foldername //+ tools.create_UUID();
          //WalletInfo.uuid = WalletInfo.info._id;
          
          WalletData.push(WalletInfo)
          //***** MONGODB INSERT */
          console.log(WalletInfo)
      
            
          });

          WalletsList.insertMany(WalletData, function(err, res) {
            if (err) throw err;
            console.log("Number of record inserted: " + res.length);
            db.mongoose.connection.close();
      
          });

    },

    readFile: async function(Network, _type, _TypeObject){
        logger.verbose(` Reading data from disk...`);

        const dir = this.getDirPath(Network, _type, _TypeObject);
        const filename = this.getFilename(Network, _type, _TypeObject);

        file = dir + filename
        logger.verbose(` ${file}`);
        if (!fs.existsSync(dir)){
            fs.mkdirSync(dir, { recursive: true });
            logger.verbose(` directory ${dir} has been created.`);
        }

        if (!fs.existsSync(file)){
            logger.warn(`file does not exists... ${file} ... file will be created.`)
            await this.writeFile(Network, _type, _TypeObject, [])            
            return [];
        }

        const _dataArray = JSON.parse(fs.readFileSync(file));
        logger.verbose(` ${_dataArray.length} data from disk loaded.`);
        return _dataArray;
    },

    writeFile: async function(Network, _type, _TypeObject, _DataObject){

        logger.verbose(` Saving data to disk... `);
        const dir = this.getDirPath(Network, _type, _TypeObject)
        const filename = this.getFilename(Network, _type, _TypeObject)

        const data = JSON.stringify(_DataObject, null, 4);

        file = dir + filename;
        if (!fs.existsSync(dir)){
            fs.mkdirSync(dir, { recursive: true });
            logger.verbose(` directory ${dir} has been created.`);
        }

        fs.writeFileSync(file, data, (err) => {
            if (err) {
                logger.error(` error writing file: ${file}`);
                throw err;
                return false
            }
            logger.info(` saved ${_DataObject.length} to file ${file}`);
        });
        return true;
    },

    getDirPath: function(Network, _type, _TypeObject){
        // build path.
        let dir = undefined;

        //console.log(_type)
        switch(_type) {
            // PAIRDATA_TYPE
            case botType.PAIRDATA_TYPE:
                dir =  botConfig.pairdata + Network.getNetwork() + "/" + Network.chainId + "/";
              break;
            // EXCHANGE_TYPE
            case botType.EXCHANGE_TYPE:
                //dir =  botConfig.exchange_temp + Network.network.toLowerCase() + "/" + Network.chainId + "/" + _TypeObject.factory + "/";
                dir =  botConfig.exchange_temp + Network.getNetwork() + "/" + Network.chainId + "/" + _TypeObject.factory + "/";
              break;
            // EXCHANGE_TEMP_TYPE
            case botType.EXCHANGE_TEMP_TYPE:
                dir =  botConfig.exchange_temp + Network.getNetwork() + "/" + Network.chainId + "/" + _TypeObject.factory + "/";
              break;
            // EXCHANGE_TYPE
            case botType.EXCHANGE_CONFIG_TYPE:
                //dir =  botConfig.exchange + Network.network.toLowerCase() + "/";
                dir =  botConfig.exchange + Network.getNetwork() + "/";
              break;
              // PAIRLIST_TYPE
            case botType.PAIRLIST_TYPE:
                dir =  botConfig.pairlist_temp + Network.getNetwork() + "/" + Network.chainId + "/";
              break;
            // WALLET_TEMP
            case botType.WALLET_TEMP:
                dir =   botConfig.wallet_temp;
              break;
            // WALLET_DB
            case botType.WALLET_DB:
                dir =  botConfig.wallet_db;
              break;
            // WALLET_MERGE
            case botType.WALLET_MERGE:
                dir =  botConfig.wallet_merge;
              break;
            // SNIPER_TYPE
            case botType.SNIPER_TYPE:
                dir =  botConfig.sniper_temp;
              break;
            // NETWORK_TYPE
            case botType.NETWORK_TYPE:
                dir =  botConfig.network;
              break;
              // SCENARIO_TYPE
            case botType.SCENARIO_TYPE:
                dir =  botConfig.scenario;
              break;
              // STATE_TYPE
            case botType.STATE_TYPE:
                dir =  botConfig.state;
              break;
            // ADV_STATE_TYPE
            case botType.ADV_STATE_TYPE:
                dir =  botConfig.state;
              break;
              // WALLET_DB
            case botType.CLEANER_TYPE:
                dir =  botConfig.cleaner_temp;
              break;
            // TOKEN_TYPE
            case botType.TOKEN_TYPE:
                dir =  botConfig.token + Network.getNetwork() + "/" + Network.chainId.toString() + "-" + Network.shortName.toLowerCase() + "/";
              break;            
            // TOKEN_TYPE
            case botType.TOKEN_TEMP_TYPE:
                dir =  botConfig.token_temp + Network.getNetwork() + "/" + Network.chainId.toString() + "-" + Network.shortName.toLowerCase() + "/";
              break;            
            // PAIRDATA_TEMP_TYPE
            case botType.PAIRDATA_TEMP_TYPE:
                dir =  botConfig.pairdata_temp + Network.getNetwork() + "/" + Network.chainId + "/" + _TypeObject.factory + "/";
              break;               
            // PAIRINDEX_TYPE
            case botType.PAIRINDEX_TYPE:
                dir =  botConfig.pairindex + Network.getNetwork() + "/" + Network.chainId + "/" ;
              break;               
            // default for (null, null, String)
            default:
                dir =  botConfig.default_temp  + _TypeObject + "/"
          }

        if (!fs.existsSync(dir)){
            fs.mkdirSync(dir, { recursive: true });
            logger.verbose(` directory ${dir} has been created.`);        
        } 
        return dir;
    },

    getFilename: function(Network, _type, _TypeObject){
        // build path.
        let _filename = undefined;
        switch(_type) {
            // PAIRDATA_TYPE
            case botType.PAIRDATA_TYPE:
                _filename =  _TypeObject.factory + ".json";
              break;
            // EXCHANGE_TYPE
            case botType.EXCHANGE_TYPE:
                _filename =  _TypeObject.factory + ".json";
              break;
            // NETWORK_TYPE
            case botType.NETWORK_TYPE:
                _filename =  "networks.json";
              break;
              // EXCHANGE_TEMP_TYPE
            case botType.EXCHANGE_TEMP_TYPE:
                _filename =  _TypeObject.factory + ".json";
              break;
            // EXCHANGE_CONFIG_TYPE
            case botType.EXCHANGE_CONFIG_TYPE:
                _filename =  Network.getShortname() + ".json";
              break;
              // PAIRLIST_TYPE
            case botType.PAIRLIST_TYPE:
                _filename =  _TypeObject.factory + ".json";
              break;
            // WALLET_TEMP
            case botType.WALLET_TEMP:
                _filename =  this.create_UUID() + ".json";
              break;
              // WALLET_DB
            case botType.WALLET_DB:
                _filename =  "walletDB.json";
              break;
            // WALLET_MERGE
            case botType.WALLET_MERGE:
                _filename =  this.create_BACKUP_name() + "_MergedWalletDB.json";
              break;
            // SNIPER_TYPE
            case botType.SNIPER_TYPE:
                _filename =  this.create_UUID() + ".json";
              break;
            //  CLEANER_TYPE
            case botType.CLEANER_TYPE:
                _filename =  this.create_UUID() + ".json";
              break;
            // TOKEN_TYPE
            case botType.TOKEN_TYPE:
                _filename =  Network.getShortname()  + ".json";
              break;                    
            // STATE_TYPE
            case botType.STATE_TYPE:
                _filename =  Network.getShortname() + "-" + Network.network.toLowerCase()  + ".json";
              break;                    
            // ADV_STATE_TYPE
            case botType.ADV_STATE_TYPE:
                _filename = "adv-" + Network.getShortname() + "-" + Network.network.toLowerCase()  + ".json";
              break;                    
            // SCENARIO_TYPE
            case botType.SCENARIO_TYPE:
                _filename =  "scenario.json";
              break;
              // TOKEN_TEMP_TYPE
            case botType.TOKEN_TEMP_TYPE:
                _filename =  Network.getShortname()  + ".json";
              break;                    
            // PAIRDATA_TEMP_TYPE
            case botType.PAIRDATA_TEMP_TYPE:
                _filename =  this.create_UUID() + ".json";
              break;               
            // PAIRINDEX_TYPE
            case botType.PAIRINDEX_TYPE:
                _filename =  _TypeObject.factory + ".json";
              break;               
            // default for (null, null, String)
              // default for (null, null, String)
            default:
                _filename =  this.create_UUID() + ".json";
          }

        return _filename;
    },

    readFolderData: async function(Network, _type, _TypeObject){

        // build path.
        const dir = this.getDirPath(Network, _type, _TypeObject);

        const DataObjectArray = new Array();
        
        logger.verbose(` Reading temp data from disk..`);
        if (!fs.existsSync(dir)){
            fs.mkdirSync(dir, { recursive: true });
            logger.verbose(` directory ${dir} has been created.`);
        }

        fs.readdirSync(dir).forEach(file => {
            let objectpath = dir + file;

            var filedetail = path.parse(objectpath);
            if(filedetail.ext == ".json"){
                const DataObject = JSON.parse(fs.readFileSync(objectpath));
                //console.log(TokenObject)
                for( let i = 0; i < DataObject.length ; i++){
                    DataObjectArray.push(DataObject[i]);
                }
       
                logger.verbose(` Loading ${DataObject.length} pair data from ${objectpath}`);
            } else{
                logger.error(` invalid file found: ${objectpath}`);
                return undefined;
            }
        });
        // fsExtra.emptyDirSync(dir)
        logger.verbose(` Done reading ${DataObjectArray.length} temp data from disk.`);
        return DataObjectArray;
    }, 

    deleteSpecificFolderData: async function(_folder){
        fsExtra.emptyDirSync(_folder);
    },

    deleteFolderData: async function(Network, _type, _TypeObject){
        const dir = this.getDirPath(Network, _type, _TypeObject);
        const _folder = dir
        fsExtra.emptyDirSync(_folder);
    },

    checkNewKey: function(_data){
        logger.verbose(` check new key data received: ${_data}`);
                // PRIVATE KEYS

        // PRIVATE KEYS
        const privateKey = _data;
        let _walletObject = undefined;
        let _valid = false;
        try{
            _walletObject = new Object(new ethers.Wallet(privateKey));
            _valid = true;
        }
        catch (e){
            _valid = false;
            //logger.error(`${e.reason} | ${e.value}`)
        }

        // MNEMONICS
        if( !_valid ){
            try{
                _walletObject = new Object(new ethers.Wallet.fromMnemonic(privateKey));
                _valid = true;
            }
            catch (e){
                //logger.error(`${e.reason} | ${e.value}`)
                _valid = false;
            }    
        }
        
        if(_valid){
            logger.verbose(`valid wallet found with data ${_walletObject.address}`);
        }
        return _walletObject;
    },

    isNewWallet: function(walletObject, _ValidWallet){

        if( !(walletObject == undefined) ){

            logger.verbose(` ${_ValidWallet.length} total wallet...`);
            
            let _newWallet = new Array();

            // WALLET TO SCAN
            const wallet = walletObject.address;
            let _newWalletFound = false;

            //logger.info("Wallet(" + (_index+1) + "\/" + _walletMnemonicCount + ") : " + wallet)
            //console.log(_ValidWallet)
            //console.log(walletObject)

            if (_ValidWallet.indexOf(wallet) == -1) {
                //console.log(_ValidWallet)
                let _walletObj = new Object({
                    address: walletObject.address, 
                    privkey: walletObject.privateKey,
                    mnemonic: walletObject.mnemonic
                })
                _ValidWallet.push(wallet);
                _newWallet.push(_walletObj);
                _newWalletFound = true;
            }
            else {
                logger.debug("Duplicate wallet found: " + wallet);
            }

            if( _newWalletFound ){
                const _result = this.writeFile(null, 3, null, _newWallet);
                return _result;
            }
            else{
                return false;
            }
        };
        return false;
    },
    latency_test: async function (roundNumber) {
        //const l1 = new Latenz();
        //const l2 = new Latenz();
        //const l3 = new Latenz();
        //l1.measure('nd-263-108-105.p2pify.com').then(result => {logger.info(`(${roundNumber}) Chainstack: ${result[2].key} ${result[2].time}ms`)});
        //l2.measure('proud-divine-butterfly.fantom.quiknode.pro').then(result => {logger.info(`(${roundNumber}) Quicknode:  ${result[2].key} ${result[2].time}ms`)});
        //l3.measure('speedy-nodes-nyc.moralis.io').then(result => {logger.info(`(${roundNumber}) SpeedyNode:  ${result[2].key} ${result[2].time}ms`)});

    },
}