UNPKG

underpost

Version:

Underpost Platform — end-to-end CI/CD and application-delivery toolchain CLI. Covers bare metal, Kubernetes, K3s, kubeadm, LXD, container/image orchestration, secrets, databases, cron jobs, monitoring, SSH, runners, PWA + Workbox delivery, and release orc

1,541 lines 64.1 kB
/**
 * UnderpostDB CLI module
 * @module src/cli/db.js
 * @namespace UnderpostDB
 * @description Manages database operations, backups, and cluster metadata for Kubernetes deployments.
 * Supports MariaDB and MongoDB with import/export capabilities, Git integration, and multi-pod operations.
 */

import {
  loadConfInstances,
  loadConfServerJson,
  loadReplicas,
  mergeFile,
  pathPortAssignmentFactory,
  splitFileFactory,
} from '../server/conf.js';
import { loggerFactory } from '../server/logger.js';
import { shellExec } from '../server/process.js';
import fs from 'fs-extra';
import { DataBaseProviderService } from '../db/DataBaseProvider.js';
import { loadCronDeployEnv } from '../server/cron.js';
import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
import Underpost from '../index.js';
import { timer } from '../client/components/core/CommonJs.js';
const logger = loggerFactory(import.meta);

/**
 * Constants for database operations
 * @constant {number} MAX_BACKUP_RETENTION - Maximum number of backups to retain
 * @memberof UnderpostDB
 */
const MAX_BACKUP_RETENTION = 5;

/**
 * @typedef {Object} DatabaseOptions
 * @memberof UnderpostDB
 * @property {boolean} [import=false] - Flag to import data from a backup
 * @property {boolean} [export=false] - Flag to export data to a backup
 * @property {string} [podName=''] - Comma-separated list of pod names or patterns
 * @property {string} [ns='default'] - Kubernetes namespace
 * @property {string} [collections=''] - Comma-separated list of collections to include
 * @property {string} [outPath=''] - Output path for backup files
 * @property {boolean} [drop=false] - Flag to drop the database before importing
 * @property {boolean} [preserveUUID=false] - Flag to preserve UUIDs during import
 * @property {boolean} [git=false] - Flag to enable Git integration
 * @property {string} [hosts=''] - Comma-separated list of hosts to include
 * @property {string} [paths=''] - Comma-separated list of paths to include
 * @property {boolean} [allPods=false] - Flag to target all matching pods
 * @property {boolean} [primaryPod=false] - Flag to automatically detect and use MongoDB primary pod
 * @property {boolean} [stats=false] - Flag to display collection/table statistics
 * @property {boolean} [forceClone=false] - Flag to force remove and re-clone cron backup repository
 */

/**
 * @typedef {Object} PodInfo
 * @memberof UnderpostDB
 * @property {string} NAME - Pod name
 * @property {string} NAMESPACE - Pod namespace
 * @property {string} NODE - Node where pod is running
 * @property {string} STATUS - Pod status
 * @property {string} [IP] - Pod IP address
 */

/**
 * @typedef {Object} DatabaseConfig
 * @memberof UnderpostDB
 * @property {string} provider - Database provider (mariadb, mongoose)
 * @property {string} name - Database name
 * @property {string} user - Database user
 * @property {string} password - Database password
 * @property {string} hostFolder - Host folder path
 * @property {string} host - Host identifier
 * @property {string} path - Path identifier
 * @property {number} [currentBackupTimestamp] - Timestamp of current backup
 */

/**
 * @class UnderpostDB
 * @description Manages database operations and backups for Kubernetes-based deployments.
 * Provides comprehensive database management including import/export, multi-pod targeting,
 * Git integration, and cluster metadata management.
 */
/**
 * UnderpostDB class for managing database operations and backups.
 * @class UnderpostDB
 * @memberof UnderpostDB
 */
class UnderpostDB {
  /**
   * Static API object containing all database operation methods.
   * @static
   * @memberof UnderpostDB
   */
  static API = {
    /**
     * Helper: Resolves the latest backup timestamp from an existing backup directory.
     * Scans the directory for numeric (epoch) sub-folders and returns the most recent one.
     * @method _getLatestBackupTimestamp
     * @memberof UnderpostDB
     * @param {string} backupDir - Path to the host-folder backup directory.
     * @return {string|null} The latest timestamp string, or null if none found.
     */
    _getLatestBackupTimestamp(backupDir) {
      if (!fs.existsSync(backupDir)) return null;
      const entries = fs.readdirSync(backupDir).filter((e) => /^\d+$/.test(e));
      if (entries.length === 0) return null;
      return entries.sort((a, b) => parseInt(b) - parseInt(a))[0];
    },

    /**
     * Helper: Performs MariaDB import operation.
     * @method _importMariaDB
     * @memberof UnderpostDB
     * @param {Object} params - Import parameters.
     * @param {PodInfo} params.pod - Target pod.
     * @param {string} params.namespace - Namespace.
     * @param {string} params.dbName - Database name.
     * @param {string} params.user - Database user.
     * @param {string} params.password - Database password.
     * @param {string} params.sqlPath - SQL file path.
     * @return {boolean} Success status.
     */
    _importMariaDB({ pod, namespace, dbName, user, password, sqlPath }) {
      try {
        const podName = pod.NAME;
        const containerSqlPath = `/${dbName}.sql`;

        logger.info('Importing MariaDB database', { podName, dbName });

        // Always ensure the database exists first — required for WP even when no backup is available
        Underpost.kubectl.run(
          `kubectl exec -n ${namespace} -i ${podName} -- mariadb -p${password} -e 'CREATE DATABASE IF NOT EXISTS ${dbName};'`,
          { context: `create database ${dbName}` },
        );

        // If no SQL file is available, the empty database is enough — return early
        if (!sqlPath || !fs.existsSync(sqlPath)) {
          logger.warn('No SQL backup file found — empty database ensured', { podName, dbName, sqlPath });
          return true;
        }

        // Remove existing SQL file in container
        Underpost.kubectl.exec({
          podName,
          namespace,
          command: `rm -rf ${containerSqlPath}`,
        });

        // Copy SQL file to pod
        if (
          !Underpost.kubectl.cpTo({
            sourcePath: sqlPath,
            podName,
            namespace,
            destPath: containerSqlPath,
          })
        ) {
          return false;
        }

        // Import SQL file
        const importCmd = `mariadb -u ${user} -p${password} ${dbName} < ${containerSqlPath}`;
        Underpost.kubectl.exec({ podName, namespace, command: importCmd });

        logger.info('Successfully imported MariaDB database', { podName, dbName });
        return true;
      } catch (error) {
        logger.error('MariaDB import failed', { podName: pod.NAME, dbName, error: error.message });
        return false;
      }
    },

    /**
     * Helper: Performs MariaDB export operation.
     * @method _exportMariaDB
     * @memberof UnderpostDB
     * @param {Object} params - Export parameters.
     * @param {PodInfo} params.pod - Source pod.
     * @param {string} params.namespace - Namespace.
     * @param {string} params.dbName - Database name.
     * @param {string} params.user - Database user.
     * @param {string} params.password - Database password.
     * @param {string} params.outputPath - Output file path.
     * @return {Promise<boolean>} A promise that resolves with success status.
     */
    async _exportMariaDB({ pod, namespace, dbName, user, password, outputPath }) {
      try {
        const podName = pod.NAME;
        const containerSqlPath = `/home/${dbName}.sql`;

        logger.info('Exporting MariaDB database', { podName, dbName });

        // Remove existing SQL file in container
        Underpost.kubectl.exec({
          podName,
          namespace,
          command: `rm -rf ${containerSqlPath}`,
        });

        // Dump database
        const dumpCmd = `mariadb-dump --user=${user} --password=${password} --lock-tables ${dbName} > ${containerSqlPath}`;
        Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });

        // Copy SQL file from pod
        if (
          !Underpost.kubectl.cpFrom({
            podName,
            namespace,
            sourcePath: containerSqlPath,
            destPath: outputPath,
          })
        ) {
          return false;
        }

        // Split file if it exists
        if (fs.existsSync(outputPath)) {
          await splitFileFactory(dbName, outputPath);
        }

        logger.info('Successfully exported MariaDB database', { podName, dbName, outputPath });
        return true;
      } catch (error) {
        logger.error('MariaDB export failed', { podName: pod.NAME, dbName, error: error.message });
        return false;
      }
    },

    /**
     * Helper: Performs MongoDB import operation.
     * @method _importMongoDB
     * @memberof UnderpostDB
     * @param {Object} params - Import parameters.
     * @param {PodInfo} params.pod - Target pod.
     * @param {string} params.namespace - Namespace.
     * @param {string} params.dbName - Database name.
     * @param {string} params.bsonPath - BSON directory path.
     * @param {boolean} params.drop - Whether to drop existing database.
     * @param {boolean} params.preserveUUID - Whether to preserve UUIDs.
      * @param {string} [params.user=''] - MongoDB username for authenticated restore.
      * @param {string} [params.password=''] - MongoDB password for authenticated restore.
      * @param {string} [params.authDatabase='admin'] - Auth database for restore command.
     * @return {boolean} Success status.
     */
    _importMongoDB({ pod, namespace, dbName, bsonPath, drop, preserveUUID, user = '', password = '', authDatabase = 'admin' }) {
      try {
        const podName = pod.NAME;
        const containerBsonPath = `/${dbName}`;

        logger.info('Importing MongoDB database', { podName, dbName });

        // If no BSON directory is available, skip — MongoDB creates the DB on first write
        if (!bsonPath || !fs.existsSync(bsonPath)) {
          logger.warn('No BSON backup directory found — database will be created on first write', {
            podName,
            dbName,
            bsonPath,
          });
          return true;
        }

        // Remove existing BSON directory in container
        Underpost.kubectl.exec({
          podName,
          namespace,
          command: `rm -rf ${containerBsonPath}`,
        });

        // Copy BSON directory to pod
        if (
          !Underpost.kubectl.cpTo({
            sourcePath: bsonPath,
            podName,
            namespace,
            destPath: containerBsonPath,
          })
        ) {
          return false;
        }

        // Restore database
        const authFlags = user && password
          ? ` --username ${JSON.stringify(user)} --password ${JSON.stringify(password)} --authenticationDatabase ${JSON.stringify(authDatabase)}`
          : '';
        const restoreCmd = `mongorestore -d ${dbName} ${containerBsonPath} --numParallelCollections=1 --numInsertionWorkersPerCollection=1 --batchSize=100 --stopOnError${drop ? ' --drop' : ''}${preserveUUID ? ' --preserveUUID' : ''}${authFlags}`;
        Underpost.kubectl.exec({ podName, namespace, command: restoreCmd });

        logger.info('Successfully imported MongoDB database', { podName, dbName });
        return true;
      } catch (error) {
        logger.error('MongoDB import failed', { podName: pod.NAME, dbName, error: error.message });
        return false;
      }
    },

    /**
     * Helper: Performs MongoDB export operation.
     * @method _exportMongoDB
     * @memberof UnderpostDB
     * @param {Object} params - Export parameters.
     * @param {PodInfo} params.pod - Source pod.
     * @param {string} params.namespace - Namespace.
     * @param {string} params.dbName - Database name.
     * @param {string} params.outputPath - Output directory path.
     * @param {string} [params.collections=''] - Comma-separated collection list.
      * @param {string} [params.user=''] - MongoDB username for authenticated dump.
      * @param {string} [params.password=''] - MongoDB password for authenticated dump.
      * @param {string} [params.authDatabase='admin'] - Auth database for dump command.
     * @return {boolean} Success status.
     */
    _exportMongoDB({ pod, namespace, dbName, outputPath, collections = '', user = '', password = '', authDatabase = 'admin' }) {
      try {
        const podName = pod.NAME;
        const containerBsonPath = `/${dbName}`;

        logger.info('Exporting MongoDB database', { podName, dbName, collections });

        // Remove existing BSON directory in container
        Underpost.kubectl.exec({
          podName,
          namespace,
          command: `rm -rf ${containerBsonPath}`,
        });

        // Dump database or specific collections
        const authFlags = user && password
          ? ` --username ${JSON.stringify(user)} --password ${JSON.stringify(password)} --authenticationDatabase ${JSON.stringify(authDatabase)}`
          : '';

        if (collections) {
          const collectionList = collections.split(',').map((c) => c.trim());
          for (const collection of collectionList) {
            const dumpCmd = `mongodump -d ${dbName} --collection ${collection} -o /${authFlags}`;
            Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
          }
        } else {
          const dumpCmd = `mongodump -d ${dbName} -o /${authFlags}`;
          Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
        }

        // Copy BSON directory from pod
        if (
          !Underpost.kubectl.cpFrom({
            podName,
            namespace,
            sourcePath: containerBsonPath,
            destPath: outputPath,
          })
        ) {
          return false;
        }

        logger.info('Successfully exported MongoDB database', { podName, dbName, outputPath });
        return true;
      } catch (error) {
        logger.error('MongoDB export failed', { podName: pod.NAME, dbName, error: error.message });
        return false;
      }
    },

    /**
     * Helper: Gets MongoDB collection statistics.
     * @method _getMongoStats
     * @memberof UnderpostDB
     * @param {Object} params - Parameters.
     * @param {string} params.podName - Pod name.
     * @param {string} params.namespace - Namespace.
     * @param {string} params.dbName - Database name.
      * @param {string} [params.user=''] - MongoDB username for authenticated stats query.
      * @param {string} [params.password=''] - MongoDB password for authenticated stats query.
      * @param {string} [params.authDatabase='admin'] - Auth database for stats query.
     * @return {Object|null} Collection statistics or null on error.
     */
    _getMongoStats({ podName, namespace, dbName, user = '', password = '', authDatabase = 'admin' }) {
      try {
        logger.info('Getting MongoDB collection statistics', { podName, dbName });

        // Use db.getSiblingDB() instead of 'use' command
        const script = `db.getSiblingDB('${dbName}').getCollectionNames().map(function(c) { return { collection: c, count: db.getSiblingDB('${dbName}')[c].countDocuments() }; })`;

        // Execute the script
        const authFlags = user && password
          ? ` --authenticationDatabase ${JSON.stringify(authDatabase)} -u ${JSON.stringify(user)} -p ${JSON.stringify(password)}`
          : '';
        const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mongosh --quiet${authFlags} --eval "${script}"`;
        const output = shellExec(command, { stdout: true, silent: true, silentOnError: true });

        if (!output || output.trim() === '') {
          logger.warn('No collections found or empty output');
          return null;
        }

        // Clean the output: remove newlines, handle EJSON format, replace single quotes with double quotes
        let cleanedOutput = output
          .trim()
          .replace(/\n/g, '')
          .replace(/\s+/g, ' ')
          .replace(/NumberLong\("(\d+)"\)/g, '$1')
          .replace(/NumberLong\((\d+)\)/g, '$1')
          .replace(/NumberInt\("(\d+)"\)/g, '$1')
          .replace(/NumberInt\((\d+)\)/g, '$1')
          .replace(/ISODate\("([^"]+)"\)/g, '"$1"')
          .replace(/'/g, '"')
          .replace(/(\w+):/g, '"$1":');

        try {
          const stats = JSON.parse(cleanedOutput);
          logger.info('MongoDB statistics retrieved', { dbName, collections: stats.length });
          return stats;
        } catch (parseError) {
          logger.error('Failed to parse MongoDB output', {
            podName,
            dbName,
            error: parseError.message,
            rawOutput: output.substring(0, 200),
            cleanedOutput: cleanedOutput.substring(0, 200),
          });
          return null;
        }
      } catch (error) {
        logger.error('Failed to get MongoDB statistics', { podName, dbName, error: error.message });
        return null;
      }
    },

    /**
     * Helper: Gets MariaDB table statistics.
     * @method _getMariaDBStats
     * @memberof UnderpostDB
     * @param {Object} params - Parameters.
     * @param {string} params.podName - Pod name.
     * @param {string} params.namespace - Namespace.
     * @param {string} params.dbName - Database name.
     * @param {string} params.user - Database user.
     * @param {string} params.password - Database password.
     * @return {Object|null} Table statistics or null on error.
     */
    _getMariaDBStats({ podName, namespace, dbName, user, password }) {
      try {
        logger.info('Getting MariaDB table statistics', { podName, dbName });

        const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mariadb -u ${user} -p${password} ${dbName} -e "SELECT TABLE_NAME as 'table', TABLE_ROWS as 'count' FROM information_schema.TABLES WHERE TABLE_SCHEMA = '${dbName}' ORDER BY TABLE_NAME;" --skip-column-names --batch`;
        const output = shellExec(command, { stdout: true, silent: true, disableLog: true, silentOnError: true });

        if (!output || output.trim() === '') {
          logger.warn('No tables found or empty output');
          return null;
        }

        // Parse the output (tab-separated values)
        const lines = output.trim().split('\n');
        const stats = lines.map((line) => {
          const [table, count] = line.split('\t');
          return { table, count: parseInt(count) || 0 };
        });

        logger.info('MariaDB statistics retrieved', { dbName, tables: stats.length });
        return stats;
      } catch (error) {
        logger.error('Failed to get MariaDB statistics', { podName, dbName, error: error.message });
        return null;
      }
    },

    /**
     * Helper: Displays database statistics in table format.
     * @method _displayStats
     * @memberof UnderpostDB
     * @param {Object} params - Parameters.
     * @param {string} params.provider - Database provider.
     * @param {string} params.dbName - Database name.
     * @param {Array<Object>} params.stats - Statistics array.
     * @return {void}
     */
    _displayStats({ provider, dbName, stats }) {
      if (!stats || stats.length === 0) {
        logger.warn('No statistics to display', { provider, dbName });
        return;
      }

      const title = provider === 'mongoose' ? 'Collections' : 'Tables';
      const itemKey = provider === 'mongoose' ? 'collection' : 'table';

      console.log('\n' + '='.repeat(70));
      console.log(`DATABASE: ${dbName} (${provider.toUpperCase()})`);
      console.log('='.repeat(70));
      console.log(`${title.padEnd(50)} ${'Documents/Rows'.padStart(18)}`);
      console.log('-'.repeat(70));

      let totalCount = 0;
      stats.forEach((item) => {
        const name = item[itemKey] || 'Unknown';
        const count = item.count || 0;
        totalCount += count;
        console.log(`${name.padEnd(50)} ${count.toString().padStart(18)}`);
      });

      console.log('-'.repeat(70));
      console.log(`${'TOTAL'.padEnd(50)} ${totalCount.toString().padStart(18)}`);
      console.log('='.repeat(70) + '\n');
    },



    /**
     * Main callback: Initiates database backup workflow.
     * Orchestrates the backup process for multiple deployments, handling
     * database connections, backup storage, and optional Git integration for version control.
     * Supports targeting multiple specific pods, nodes, and namespaces with advanced filtering.
     * @method callback
     * @memberof UnderpostDB
     * @param {string} [deployList='default'] - Comma-separated list of deployment IDs.
     * @param {Object} [options={}] - Backup options.
     * @param {boolean} [options.import=false] - Whether to perform import operation.
     * @param {boolean} [options.export=false] - Whether to perform export operation.
     * @param {string} [options.podName=''] - Comma-separated pod name patterns to target.
     * @param {string} [options.ns='default'] - Kubernetes namespace.
     * @param {string} [options.collections=''] - Comma-separated MongoDB collections for export.
     * @param {string} [options.outPath=''] - Output path for backups.
     * @param {boolean} [options.drop=false] - Whether to drop existing database on import.
     * @param {boolean} [options.preserveUUID=false] - Whether to preserve UUIDs on MongoDB import.
     * @param {boolean} [options.git=false] - Whether to use Git for backup versioning.
     * @param {string} [options.hosts=''] - Comma-separated list of hosts to filter databases.
     * @param {string} [options.paths=''] - Comma-separated list of paths to filter databases.
     * @param {boolean} [options.allPods=false] - Whether to target all pods in deployment.
     * @param {boolean} [options.primaryPod=false] - Whether to target MongoDB primary pod only.
     * @param {string} [options.primaryPodEnsure=''] - Pod name to ensure MongoDB primary pod is running.
     * @param {boolean} [options.stats=false] - Whether to display database statistics.
     * @param {number} [options.macroRollbackExport=1] - Number of commits to rollback in macro export.
     * @param {boolean} [options.forceClone=false] - Whether to force re-clone Git repository.
     * @param {boolean} [options.cleanFsCollection=false] - Clean orphaned File documents flag.
     * @param {boolean} [options.cleanFsDryRun=false] - Dry run mode flag (use with cleanFsCollection).
     * @param {boolean} [options.dev=false] - Development mode flag.
     * @param {boolean} [options.k3s=false] - k3s cluster flag.
     * @param {boolean} [options.kubeadm=false] - kubeadm cluster flag.
     * @param {boolean} [options.kind=false] - kind cluster flag.
     * @param {boolean} [options.repoBackup=false] - Backs up repositories (git commit+push) inside deployment pods via kubectl exec.
     * @return {Promise<void>} Resolves when operation is complete.
     */
    async callback(
      deployList = 'default',
      options = {
        import: false,
        export: false,
        podName: '',
        ns: 'default',
        collections: '',
        outPath: '',
        drop: false,
        preserveUUID: false,
        git: false,
        hosts: '',
        paths: '',
        allPods: false,
        primaryPod: false,
        primaryPodEnsure: '',
        stats: false,
        macroRollbackExport: 1,
        forceClone: false,
        cleanFsCollection: false,
        cleanFsDryRun: false,
        dev: false,
        k3s: false,
        kubeadm: false,
        kind: false,
        repoBackup: false,
      },
    ) {
      // Ensure engine-private is available (clone if inside a deployment
      // container where globalSecretClean has already removed it).
      const firstDeployId = deployList !== 'dd' ? deployList.split(',')[0].trim() : '';
      try {
        loadCronDeployEnv();
        const newBackupTimestamp = new Date().getTime();
        const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';

        if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');

        // Handle repository backup (git commit+push inside deployment pod)
        if (options.repoBackup) {
          const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
          for (const _deployId of deployList.split(',')) {
            const deployId = _deployId.trim();
            if (!deployId) continue;
            logger.info('Starting pod repository backup', { deployId, namespace });
            Underpost.repo.backupPodRepositories({
              deployId,
              namespace,
              env: options.dev ? 'development' : 'production',
            });
          }
          return;
        }

        // Handle clean-fs-collection operation
        if (options.cleanFsCollection || options.cleanFsDryRun) {
          logger.info('Starting File collection cleanup operation', { deployList });
          await Underpost.db.cleanFsCollection(deployList, {
            hosts: options.hosts,
            paths: options.paths,
            dryRun: options.cleanFsDryRun,
          });
          return;
        }

        logger.info('Starting database operation', {
          deployList,
          namespace,
          import: options.import,
          export: options.export,
        });

        if (options.primaryPodEnsure) {
          const primaryPodName = MongoBootstrap.getPrimaryPodName({
            namespace,
            podName: options.primaryPodEnsure,
            username: process.env.MONGODB_USERNAME || process.env.DB_USER || '',
            password: process.env.MONGODB_PASSWORD || process.env.DB_PASSWORD || '',
            authDatabase: process.env.MONGODB_AUTH_DB || 'admin',
          });
          if (!primaryPodName) {
            const baseCommand = options.dev ? 'node bin' : 'underpost';
            const baseClusterCommand = options.dev ? ' --dev' : '';
            let clusterFlag = options.k3s ? ' --k3s' : options.kubeadm ? ' --kubeadm' : '';
            shellExec(`${baseCommand} cluster${baseClusterCommand}${clusterFlag} --mongodb`);
          }
          return;
        }

        // Track processed repositories to avoid duplicate Git operations
        const processedRepos = new Set();
        // Track processed host+path combinations to avoid duplicates
        const processedHostPaths = new Set();

        for (const _deployId of deployList.split(',')) {
          const deployId = _deployId.trim();
          if (!deployId) continue;

          logger.info('Processing deployment', { deployId });

          /** @type {Object.<string, Object.<string, DatabaseConfig>>} */
          const dbs = {};
          const repoName = `engine-${deployId.includes('dd-') ? deployId.split('dd-')[1] : deployId}-cron-backups`;

          // Load server configuration
          const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
          if (!fs.existsSync(confServerPath)) {
            logger.error('Configuration file not found', { path: confServerPath });
            continue;
          }

          const confServer = loadConfServerJson(confServerPath, { resolve: true });

          // Build database configuration map
          for (const host of Object.keys(confServer)) {
            for (const path of Object.keys(confServer[host])) {
              const { db } = confServer[host][path];
              if (db) {
                const { provider, name, user, password } = db;
                if (!dbs[provider]) dbs[provider] = {};

                if (!(name in dbs[provider])) {
                  dbs[provider][name] = {
                    user,
                    password,
                    hostFolder: host + path.replaceAll('/', '-'),
                    host,
                    path,
                  };
                }
              }
            }
          }

          // Handle Git operations - execute only once per repository
          if (!processedRepos.has(repoName)) {
            logger.info('Processing Git operations for repository', { repoName, deployId });
            if (options.git === true) {
              Underpost.repo.manageBackupRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
              Underpost.repo.manageBackupRepo({ repoName, operation: 'pull' });
            }

            if (options.macroRollbackExport) {
              // Only clone if not already done by git option above
              if (options.git !== true) {
                Underpost.repo.manageBackupRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
                Underpost.repo.manageBackupRepo({ repoName, operation: 'pull' });
              }

              const nCommits = parseInt(options.macroRollbackExport);
              const repoPath = `../${repoName}`;
              const username = process.env.GITHUB_USERNAME;

              if (fs.existsSync(repoPath) && username) {
                logger.info('Executing macro rollback export', { repoName, nCommits });
                shellExec(`cd ${repoPath} && underpost cmt . reset ${nCommits}`);
                shellExec(`cd ${repoPath} && git reset`);
                shellExec(`cd ${repoPath} && git checkout .`);
                shellExec(`cd ${repoPath} && git clean -f -d`);
                shellExec(`cd ${repoPath} && underpost push . ${username}/${repoName} -f`);
              } else {
                if (!username) logger.error('GITHUB_USERNAME environment variable not set');
                logger.warn('Repository not found for macro rollback', { repoPath });
              }
            }

            processedRepos.add(repoName);
            logger.info('Repository marked as processed', { repoName });
          } else {
            logger.info('Skipping Git operations for already processed repository', { repoName, deployId });
          }

          // Process each database provider
          for (const provider of Object.keys(dbs)) {
            for (const dbName of Object.keys(dbs[provider])) {
              const { hostFolder, user, password, host, path } = dbs[provider][dbName];

              // Create unique identifier for host+path combination
              const hostPathKey = `${deployId}:${host}:${path}`;

              // Skip if this host+path combination was already processed
              if (processedHostPaths.has(hostPathKey)) {
                logger.info('Skipping already processed host/path', { dbName, host, path, deployId });
                continue;
              }

              // Filter by hosts and paths if specified
              if (
                (options.hosts &&
                  !options.hosts
                    .split(',')
                    .map((h) => h.trim())
                    .includes(host)) ||
                (options.paths &&
                  !options.paths
                    .split(',')
                    .map((p) => p.trim())
                    .includes(path))
              ) {
                logger.info('Skipping database due to host/path filter', { dbName, host, path });
                continue;
              }

              if (!hostFolder) {
                logger.warn('No hostFolder defined for database', { dbName, provider });
                continue;
              }

              logger.info('Processing database', { hostFolder, provider, dbName, deployId });

              const latestBackupTimestamp = Underpost.db._getLatestBackupTimestamp(`../${repoName}/${hostFolder}`);

              dbs[provider][dbName].currentBackupTimestamp = latestBackupTimestamp;

              const currentTimestamp = latestBackupTimestamp || newBackupTimestamp;
              const sqlContainerPath = `/home/${dbName}.sql`;
              const fromPartsPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}-parths.json`;
              const toSqlPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}.sql`;
              const toNewSqlPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}.sql`;
              const toBsonPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}`;
              const toNewBsonPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}`;

              // Merge split SQL files if needed for import
              if (options.import === true && fs.existsSync(fromPartsPath) && !fs.existsSync(toSqlPath)) {
                const names = JSON.parse(fs.readFileSync(fromPartsPath, 'utf8')).map((_path) => {
                  return `../${repoName}/${hostFolder}/${currentTimestamp}/${_path.split('/').pop()}`;
                });
                logger.info('Merging backup parts', { fromPartsPath, toSqlPath, parts: names.length });
                await mergeFile(names, toSqlPath);
              }

              // Get target pods based on provider and options
              let targetPods = [];
              const podCriteria = {
                podNames: options.podName,
                namespace,
                deployId: provider === 'mariadb' ? 'mariadb' : 'mongo',
              };

              targetPods = Underpost.kubectl.getFilteredPods(podCriteria);

              // Fallback to default if no custom pods specified
              if (targetPods.length === 0 && !options.podName) {
                const defaultPods = Underpost.kubectl.get(
                  provider === 'mariadb' ? 'mariadb' : 'mongo',
                  'pods',
                  namespace,
                );
                console.log('defaultPods', defaultPods);
                targetPods = defaultPods;
              }

              if (targetPods.length === 0) {
                logger.warn('No pods found matching criteria', { provider, criteria: podCriteria });
                continue;
              }

              // Handle primary pod detection for MongoDB
              let podsToProcess = [];
              if (provider === 'mongoose' && (options.import === true || !options.allPods)) {
                // For MongoDB, always use primary pod unless allPods is true
                if (!targetPods || targetPods.length === 0) {
                  logger.warn('No MongoDB pods available to check for primary');
                  podsToProcess = [];
                } else {
                  const firstPod = targetPods[0].NAME;
                  const primaryPodName = MongoBootstrap.getPrimaryPodName({
                    namespace,
                    podName: firstPod,
                    username: user,
                    password,
                    authDatabase: 'admin',
                  });

                  if (primaryPodName) {
                    const primaryPod = targetPods.find((p) => p.NAME === primaryPodName);
                    if (primaryPod) {
                      podsToProcess = [primaryPod];
                      logger.info('Using MongoDB primary pod', { primaryPod: primaryPodName });
                    } else {
                      logger.warn('Primary pod not in filtered list, using first pod', { primaryPodName });
                      podsToProcess = [targetPods[0]];
                    }
                  } else {
                    logger.warn('Could not detect primary pod, using first pod');
                    podsToProcess = [targetPods[0]];
                  }
                }
              } else {
                // For MariaDB or when allPods is true, limit to first pod unless allPods is true
                podsToProcess = options.allPods === true ? targetPods : [targetPods[0]];
              }

              logger.info(`Processing ${podsToProcess.length} pod(s) for ${provider}`, {
                dbName,
                pods: podsToProcess.map((p) => p.NAME),
              });

              let exportSucceeded = false;

              // Process each pod
              for (const pod of podsToProcess) {
                logger.info('Processing pod', { podName: pod.NAME, node: pod.NODE, status: pod.STATUS });

                switch (provider) {
                  case 'mariadb': {
                    if (options.stats === true) {
                      const stats = Underpost.db._getMariaDBStats({
                        podName: pod.NAME,
                        namespace,
                        dbName,
                        user,
                        password,
                      });
                      if (stats) {
                        Underpost.db._displayStats({ provider, dbName, stats });
                      }
                    }

                    if (options.import === true) {
                      Underpost.db._importMariaDB({
                        pod,
                        namespace,
                        dbName,
                        user,
                        password,
                        sqlPath: toSqlPath,
                      });
                    }

                    if (options.export === true) {
                      const outputPath = options.outPath || toNewSqlPath;
                      const success = await Underpost.db._exportMariaDB({
                        pod,
                        namespace,
                        dbName,
                        user,
                        password,
                        outputPath,
                      });
                      exportSucceeded = exportSucceeded || success;
                    }
                    break;
                  }

                  case 'mongoose': {
                    if (options.stats === true) {
                      const stats = Underpost.db._getMongoStats({
                        podName: pod.NAME,
                        namespace,
                        dbName,
                        user,
                        password,
                        authDatabase: 'admin',
                      });
                      if (stats) {
                        Underpost.db._displayStats({ provider, dbName, stats });
                      }
                    }

                    if (options.import === true) {
                      const bsonPath = options.outPath || toBsonPath;
                      Underpost.db._importMongoDB({
                        pod,
                        namespace,
                        dbName,
                        bsonPath,
                        drop: options.drop,
                        preserveUUID: options.preserveUUID,
                        user,
                        password,
                        authDatabase: 'admin',
                      });
                    }

                    if (options.export === true) {
                      const outputPath = options.outPath || toNewBsonPath;
                      const success = Underpost.db._exportMongoDB({
                        pod,
                        namespace,
                        dbName,
                        outputPath,
                        collections: options.collections,
                        user,
                        password,
                        authDatabase: 'admin',
                      });
                      exportSucceeded = exportSucceeded || success;
                    }
                    break;
                  }

                  default:
                    logger.warn('Unsupported database provider', { provider });
                    break;
                }
              }

              if (options.export === true && exportSucceeded === true) {
                Underpost.db._enforceBackupRetention(`../${repoName}/${hostFolder}`);
              }

              // Mark this host+path combination as processed
              processedHostPaths.add(hostPathKey);
            }
          }

          // Commit and push to Git if enabled - execute only once per repository
          if (options.export === true && options.git === true && !processedRepos.has(`${repoName}-committed`)) {
            const commitMessage = `${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(
              newBackupTimestamp,
            ).toLocaleTimeString()}`;
            Underpost.repo.manageBackupRepo({ repoName, operation: 'commit', message: commitMessage });
            Underpost.repo.manageBackupRepo({ repoName, operation: 'push' });
            processedRepos.add(`${repoName}-committed`);
          }
        }

        logger.info('Database operation completed successfully');
      } catch (error) {
        logger.error('Database operation failed', { error: error.message });
        throw error;
      }
    },
    /**
     * Helper: Removes old timestamp backup folders and keeps only the newest ones.
     * @method _enforceBackupRetention
     * @memberof UnderpostDB
     * @param {string} backupDir - Path to host-folder backup directory.
     * @param {number} [maxRetention=MAX_BACKUP_RETENTION] - Maximum folders to keep.
     * @return {number} Number of removed backup folders.
     */
    _enforceBackupRetention(backupDir, maxRetention = MAX_BACKUP_RETENTION) {
      try {
        if (!fs.existsSync(backupDir)) return 0;

        const timestamps = fs
          .readdirSync(backupDir)
          .filter((entry) => /^\d+$/.test(entry))
          .sort((a, b) => parseInt(b, 10) - parseInt(a, 10));

        if (timestamps.length <= maxRetention) return 0;

        const staleTimestamps = timestamps.slice(maxRetention);
        staleTimestamps.forEach((timestamp) => {
          fs.removeSync(`${backupDir}/${timestamp}`);
        });

        logger.info('Pruned old backup timestamp folders', {
          backupDir,
          kept: maxRetention,
          removed: staleTimestamps.length,
          removedTimestamps: staleTimestamps,
        });

        return staleTimestamps.length;
      } catch (error) {
        logger.error('Failed to enforce backup retention', { backupDir, maxRetention, error: error.message });
        return 0;
      }
    },

    /**
     * Creates cluster metadata for the specified deployment.
     * Loads database configuration and initializes cluster metadata including
     * instances and cron jobs. This method populates the database with deployment information.
     * @method clusterMetadataFactory
     * @memberof UnderpostDB
     * @param {string} [deployId=process.env.DEFAULT_DEPLOY_ID] - The deployment ID.
     * @param {string} [host=process.env.DEFAULT_DEPLOY_HOST] - The host identifier.
     * @param {string} [path=process.env.DEFAULT_DEPLOY_PATH] - The path identifier.
     * @param {object} [options] - Options.
     * @param {boolean} [options.dev=false] - Development mode flag.
     * @return {Promise<void>} Resolves when metadata creation is complete.
     * @throws {Error} If database configuration is invalid or connection fails.
     */
    async clusterMetadataFactory(
      deployId = process.env.DEFAULT_DEPLOY_ID,
      host = process.env.DEFAULT_DEPLOY_HOST,
      path = process.env.DEFAULT_DEPLOY_PATH,
      options = { dev: false },
    ) {
      try {
        loadCronDeployEnv();
        deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
        host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
        path = path ? path : process.env.DEFAULT_DEPLOY_PATH;

        logger.info('Creating cluster metadata', { deployId, host, path });

        const env = 'production';
        const deployListPath = './engine-private/deploy/dd.router';

        if (!fs.existsSync(deployListPath)) {
          logger.error('Deploy router file not found', { path: deployListPath });
          throw new Error(`Deploy router file not found: ${deployListPath}`);
        }

        const deployList = fs.readFileSync(deployListPath, 'utf8').split(',');

        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
        if (!fs.existsSync(confServerPath)) {
          logger.error('Server configuration not found', { path: confServerPath });
          throw new Error(`Server configuration not found: ${confServerPath}`);
        }

        const { db } = loadConfServerJson(confServerPath, { resolve: true })[host][path];

        const maxRetries = 5;
        const retryDelay = 3000;
        for (let attempt = 1; attempt <= maxRetries; attempt++) {
          try {
            await DataBaseProviderService.load({ apis: ['instance', 'cron'], host, path, db });
            break;
          } catch (err) {
            if (attempt === maxRetries) {
              logger.error('Failed to connect to database after retries', { attempts: maxRetries, error: err.message });
              throw err;
            }
            logger.warn('Database connection failed, retrying...', { attempt, maxRetries, error: err.message });
            await timer(retryDelay);
          }
        }

        try {
          /** @type {import('../api/instance/instance.model.js').InstanceModel} */
          const Instance = DataBaseProviderService.getModel('instance', { host, path });

          await Instance.deleteMany();
          logger.info('Cleared existing instance metadata');

          for (const _deployId of deployList) {
            const deployId = _deployId.trim();
            if (!deployId) continue;

            logger.info('Processing deployment for metadata', { deployId });

            const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
            if (!fs.existsSync(confServerPath)) {
              logger.warn('Configuration not found for deployment', { deployId, path: confServerPath });
              continue;
            }

            const confServer = loadReplicas(deployId, loadConfServerJson(confServerPath, { resolve: true }));
            const router = await Underpost.deploy.routerFactory(deployId, env);
            const pathPortAssignmentData = await pathPortAssignmentFactory(deployId, router, confServer);

            for (const host of Object.keys(confServer)) {
              for (const { path, port } of pathPortAssignmentData[host]) {
                if (!confServer[host][path]) continue;

                const { client, runtime, apis, peer } = confServer[host][path];

                // Save main instance
                {
                  const body = {
                    deployId,
                    host,
                    path,
                    port,
                    client,
                    runtime,
                    apis,
                  };

                  logger.info('Saving instance metadata', body);
                  await new Instance(body).save();
                }

                // Save peer instance if exists
                if (peer) {
                  const body = {
                    deployId,
                    host,
                    path: path === '/' ? '/peer' : `${path}/peer`,
                    port: port + 1,
                    runtime: 'nodejs',
                  };

                  logger.info('Saving peer instance metadata', body);
                  await new Instance(body).save();
                }
              }
            }

            // Process additional instances
            const confInstancesPath = `./engine-private/conf/${deployId}/conf.instances.json`;
            if (fs.existsSync(confInstancesPath)) {
              // Expands multiInstance variants so each deployed instance is
              // registered (handles both the bare-array and { instances } shapes).
              const confInstances = loadConfInstances(deployId);
              for (const instance of confInstances) {
                const { id, host, path, fromPort, metadata } = instance;
                const { runtime } = metadata;
                const body = {
                  deployId,
                  host,
                  path,
                  port: fromPort,
                  client: id,
                  runtime,
                };
                logger.info('Saving additional instance metadata', body);
                await new Instance(body).save();
              }
            }
          }
        } catch (error) {
          logger.error('Failed to create instance metadata', { error: error.message });
          throw error;
        }

        try {
          const cronDeployPath = './engine-private/deploy/dd.cron';
          if (!fs.existsSync(cronDeployPath)) {
            logger.warn('Cron deploy file not found', { path: cronDeployPath });
            return;
          }

          const cronDeployId = fs.readFileSync(cronDeployPath, 'utf8').trim();
          const confCronPath = `./engine-private/conf/${cronDeployId}/conf.cron.json`;

          if (!fs.existsSync(confCronPath)) {
            logger.warn('Cron configuration not found', { path: confCronPath });
            return;
          }

          const confCron = JSON.parse(fs.readFileSync(confCronPath, 'utf8'));

          await DataBaseProviderService.load({ apis: ['cron'], host, path, db });

          /** @type {import('../api/cron/cron.model.js').CronModel} */
          const Cron = DataBaseProviderService.getModel('cron', { host, path });

          await Cron.deleteMany();
          logger.info('Cleared existing cron metadata');

          for (const jobId of Object.keys(confCron.jobs)) {
            const body = {
              jobId,
              deployId: Underpost.cron.getRelatedDeployIdList(jobId),
              expression: confCron.jobs[jobId].expression,
              enabled: confCron.jobs[jobId].enabled,
            };
            logger.info('Saving cron metadata', body);
            await new Cron(body).save();
          }
        } catch (error) {
          logger.error('Failed to create cron metadata', { error: error.message });
        }

        await DataBaseProviderService.getProvider({ host, path }, 'mongoose').close();
        logger.info('Cluster metadata creation completed');
      } catch (error) {
        logger.error('Cluster metadata creation failed', { error: error.message });
        throw error;
      }
    },

    /**
     * Cleans orphaned File references from database collections.
     * Iterates over all deploy-ids and checks if File documents are actually referenced
     * by other collections. Removes File documents that are not referenced anywhere.
     * @method cleanFsCollection
     * @memberof UnderpostDB
     * @param {string} [deployList='dd'] - Comma-separated list of deployment IDs.
     * @param {Object} [options={}] - Clean operation options.
     * @param {string} [options.hosts=''] - Comma-separated list of hosts to filter.
     * @param {string} [options.paths=''] - Comma-separated list of paths to filter.
     * @param {boolean} [options.dryRun=false] - If true, only reports what would be deleted.
     * @param {boolean} [options.dev=false] - Development mode flag.
     * @return {Promise<void>} Resolves when clean operation is complete.
     */
    async cleanFsCollection(
      deployList = 'dd',
      options = {
        hosts: '',
        paths: '',
        dryRun: false,
        dev: false,
      },
    ) {
      const firstDeployId = deployList !== 'dd' ? deployList.split(',')[0].trim() : '';
      try {
        loadCronDeployEnv();
        if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');

        logger.info('Starting File collection cleanup', { deployList, options });

        // Load file.ref.json to know which models reference File
        const fileRefPath = './src/api/file/file.ref.json';
        if (!fs.existsSync(fileRefPath)) {
          logger.error('file.ref.json not found', { path: fileRefPath });
          return;
        }

        const fileRefData = JSON.parse(fs.readFileSync(fileRefPath, 'utf8'));
        logger.info('Loaded file reference configuration', { apis: fileRefData.length });

        // Filter hosts and paths if specified
        const filterHosts = options.hosts ? options.hosts.split(',').map((h) => h.trim()) : [];
        const filterPaths = options.paths ? options.paths.split(',').map((p) => p.trim()) : [];

        // Track all connections to close them at the end
        const connectionsToClose = [];

        for (const _deployId of deployList.split(',')) {
          const deployId = _deployId.trim();
          if (!deployId) continue;

          logger.info('Processing deployment for File cleanup', { deployId });

          // Load server configuration
          const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
          if (!fs.existsSync(confServerPath)) {
            logger.error('Configuration file not found', { path: confServerPath });
            continue;
          }

          const confServer = loadConfServerJson(confServerPath, { resolve: true });

          // Process each host+path combination
          for (const host of Object.keys(confServer)) {
            if (filterHosts.length > 0 && !filterHosts.includes(host)) continue;

            for (const path of Object.keys(confServer[host])) {
              if (filterPaths.length > 0 && !filterPaths.includes(path)) continue;

              const { db, apis } = confServer[host][path];
              if (!db || !apis) continue;

              // Check if 'file' api is in the apis list
              if (!apis.includes('file')) {
                logger.info('Skipping - no file api in configuration', { host, path });
                continue;
              }

              // logger.info('Processing host+path with file api', { host, path, db: db.name });

              try {
                // Connect to database with retry
                let dbProvider;
                for (let attempt = 1; attempt <= 3; attempt++) {
                  try {
                    dbProvider = await DataBaseProviderService.load({ apis, host, path, db });
                    break;
                  } catch (err) {
                    if (attempt === 3) throw err;
                    logger.warn('Database connection failed, retrying...', { attempt, host, path, error: err.message });
                    await timer(3000);
                  }
                }
                if (!dbProvider || !dbProvider.models) {
                  logger.error('Failed to load database provider', { host, path });
                  continue;
                }

                const { models } = dbProvider;

                // Track this connection for cleanup
                connectionsToClose.push({ host, path, dbProvider });

                // Check if File model exists
                if (!models.File) {
                  logger.warn('File model not loaded', { host, path });
                  continue;
                }

                // Get all File documents
                const allFiles = await models.File.find({}, '_id').lean();
                logger.info('Found File documents', { count: allFiles.length, host, path });

                if (allFiles.length === 0) continue;

                // Track which File IDs are referenced
                const referencedFileIds = new Set();

                // Check each API from file.ref.json
                for (const refConfig of fileRefData) {
                  const { api, model: modelFields } = refConfig;

                  // Check if this API is loaded in current context
                  const modelName = api
                    .split('-')
                    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
                    .join('');
                  const Model = models[modelName];

                  if (!Model) {
                    logger.debug('Model not loaded in current context', { api, modelName, host, path });
                    continue;
                  }

                  logger.info('Checking references in model', { api, modelName });

                  // Helper function to recursively check field references
                  const checkFieldReferences = async (fieldPath, fieldConfig) => {
                    for (const [fieldName, fieldValue] of Object.entries(fieldConfig)) {
                      const currentPath = fieldPath ? `${fieldPath}.${fieldName}` : fieldName;

                      if (fieldValue === true) {
                        // This is a File reference field
                        const query = {};
                        query[currentPath] = { $exists: true, $ne: null };

                        const docs = await Model.find(query, currentPath).lean();

                        for (const doc of docs) {
                          // Navigate to the nested field
                          const parts = currentPath.split('.');
                          let value = doc;
                          for (const part of parts) {
                            value = value?.[part];
                          }

                          if (value) {
                            if (Array.isArray(value)) {
                              value.forEach((id) => id && referencedFileIds.add(id.toString()));
                            } else {
                              referencedFileIds.add(value.toString());
                            }
                          }
                        }

                        logger.info('Found references', {
                          model: modelName,
                          field: currentPath,
                          count: docs.length,
                        });
                      } else if (typeof fieldValue === 'object') {
                        // Nested object, recurse
                        await checkFieldReferences(currentPath, fieldValue);
                      }
                    }
                  };

                  await checkFieldReferences('', modelFields);
                }

                logger.info('Total referenced File IDs', { count: referencedFileIds.size, host, path });

                // Find orphaned files
                const orphanedFiles = allFiles.filter((file) => !referencedFileIds.has(file._id.toString()));

                if (orphanedFiles.length === 0) {
                  logger.info('No orphaned files found', { host, path });
                } else {
                  logger.info('Found orphaned files', { count: orphanedFiles.length, host, path });

                  if (options.dryRun) {
                    logger.info('Dry run - would delete files', {
                      count: orphanedFiles.length,
                      ids: orphanedFiles.map((f) => f._id.toString()),
                    });
                  } else {
                    const orphanedIds = orphanedFiles.map((f) => f._id);
                    const deleteResult = await models.File.deleteMany({ _id: { $in: orphanedIds } });
                    logger.info('Deleted orphaned files', {
                      deletedCount: deleteResult.deletedCount,
                      host,
                      path,
                    });
                  }
                }
              } catch (error) {
                logger.error('Error processing host+path', {
                  host,
                  path,
                  error: error.message,
                });
              }
            }
          }
        }

        // Close all connections
        logger.info('Closing all database connections', { count: connectionsToClose.length });
        for (const { host, path, dbProvider } of connectionsToClose) {
          try {
            if (dbProvider && dbProvider.close) {
              await dbProvider.close();
              logger.info('Connection closed', { host, path });
            }
          } catch (error) {
            logger.error('Error closing connection', { host, path, error: error.message });
          }
        }

        logger.info('File collection cleanup completed');
      } catch (error) {
        logger.error('File collection cleanup failed', { error: error.message });
        throw error;
      }
    },

    /**
     * Handles backup of cluster metadata.
     * Orchestrates backup and restore operations for cluster metadata including
     * instances and cron jobs. Supports import/export and metadata generation.
     * @method clusterMetadataBackupCallback
     * @memberof UnderpostDB
     * @param {string} [deployId=process.env.DEFAULT_DEPLOY_ID] - The deployment ID.
     * @param {string} [host=process.env.DEFAULT_DEPLOY_HOST] - The host identifier.
     * @param {string} [path=process.env.DEFAULT_DEPLOY_PATH] - The path identifier.
     * @param {Object} [options={}] - Backup operation options.
     * @param {boolean} [options.generate=false] - Generate cluster metadata.
     * @param {boolean} [options.itc=false] - Execute in container context.
     * @param {boolean} [options.import=false] - Import metadata from backup.
     * @param {boolean} [options.export=false] - Export metadata to backup.
     * @param {boolean} [options.instances=false] - Process instances collection.
     * @param {boolean} [options.crons=false] - Process crons collection.
     * @param {boolean} [options.dev=false] - Development mode flag.
     * @return {Promise<void>} Resolves when backup operation is complete.
     */
    async clusterMetadataBackupCallback(
      deployId = process.env.DEFAULT_DEPLOY_ID,
      host = process.env.DEFAULT_DEPLOY_HOST,
      path = process.env.DEFAULT_DEPLOY_PATH,
      options = {
        generate: false,
        itc: false,
        import: false,
        export: false,
        instances: false,
        crons: false,
        dev: false,
      },
    ) {
      try {
        loadCronDeployEnv();
        deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
        host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
        path = path ? path : process.env.DEFAULT_DEPLOY_PATH;

        logger.info('Starting cluster metadata backup operation', {
          deployId,
          host,
          path,
          options,
        });

        if (options.generate === true) {
          logger.info('Generating cluster metadata');
          await Underpost.db.clusterMetadataFactory(deployId, host, path);
        }

        if (options.instances === true) {
          const outputPath = './engine-private/instances';
          if (!fs.existsSync(outputPath)) {
            fs.mkdirSync(outputPath, { recursive: true });
          }
          const collection = 'instances';

          if (options.export === true) {
            logger.info('Exporting instances collection', { outputPath });
            shellExec(
              `node bin db --export --primary-pod --collections ${collection} --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
            );
          }

          if (options.import === true) {
            logger.info('Importing instances collection', { outputPath });
            shellExec(
              `node bin db --import --primary-pod --drop --preserveUUID --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
            );
          }
        }

        if (options.crons === true) {
          const outputPath = './engine-private/crons';
          if (!fs.existsSync(outputPath)) {
            fs.mkdirSync(outputPath, { recursive: true });
          }
          const collection = 'crons';

          if (options.export === true) {
            logger.info('Exporting crons collection', { outputPath });
            shellExec(
              `node bin db --export --primary-pod --collections ${collection} --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
            );
          }

          if (options.import === true) {
            logger.info('Importing crons collection', { outputPath });
            shellExec(
              `node bin db --import --primary-pod --drop --preserveUUID --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
            );
          }
        }

        logger.info('Cluster metadata backup operation completed');
      } catch (error) {
        logger.error('Cluster metadata backup operation failed', { error: error.message });
        throw error;
      }
    },
  };
}

export default UnderpostDB;