@underpostnet/underpost
Version:
Underpost Platform — end-to-end CI/CD and application-delivery toolchain CLI. Covers bare metal, Kubernetes, K3s, kubeadm, LXD, container/image orchestration, secrets, databases, cron jobs, monitoring, SSH, runners, PWA + Workbox delivery, and release orc
1,197 lines (1,059 loc) • 63.8 kB
JavaScript
/**
* UnderpostDB CLI module
* @module src/cli/db.js
* @namespace UnderpostDB
* @description Manages database operations, backups, and cluster metadata for Kubernetes deployments.
* Supports MariaDB and MongoDB with import/export capabilities, Git integration, and multi-pod operations.
*/
import { mergeFile, splitFileFactory, loadConfServerJson, resolveConfSecrets } from '../server/conf.js';
import { loggerFactory } from '../server/logger.js';
import { shellExec } from '../server/process.js';
import fs from 'fs-extra';
import { DataBaseProviderService } from '../db/DataBaseProvider.js';
import { loadReplicas, pathPortAssignmentFactory, loadCronDeployEnv } from '../server/conf.js';
import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
import Underpost from '../index.js';
import { timer } from '../client/components/core/CommonJs.js';
const logger = loggerFactory(import.meta);
/**
* Constants for database operations
* @constant {number} MAX_BACKUP_RETENTION - Maximum number of backups to retain
* @memberof UnderpostDB
*/
const MAX_BACKUP_RETENTION = 5;
/**
* @typedef {Object} DatabaseOptions
* @memberof UnderpostDB
* @property {boolean} [import=false] - Flag to import data from a backup
* @property {boolean} [export=false] - Flag to export data to a backup
* @property {string} [podName=''] - Comma-separated list of pod names or patterns
* @property {string} [ns='default'] - Kubernetes namespace
* @property {string} [collections=''] - Comma-separated list of collections to include
* @property {string} [outPath=''] - Output path for backup files
* @property {boolean} [drop=false] - Flag to drop the database before importing
* @property {boolean} [preserveUUID=false] - Flag to preserve UUIDs during import
* @property {boolean} [git=false] - Flag to enable Git integration
* @property {string} [hosts=''] - Comma-separated list of hosts to include
* @property {string} [paths=''] - Comma-separated list of paths to include
* @property {boolean} [allPods=false] - Flag to target all matching pods
* @property {boolean} [primaryPod=false] - Flag to automatically detect and use MongoDB primary pod
* @property {boolean} [stats=false] - Flag to display collection/table statistics
* @property {boolean} [forceClone=false] - Flag to force remove and re-clone cron backup repository
*/
/**
* @typedef {Object} PodInfo
* @memberof UnderpostDB
* @property {string} NAME - Pod name
* @property {string} NAMESPACE - Pod namespace
* @property {string} NODE - Node where pod is running
* @property {string} STATUS - Pod status
* @property {string} [IP] - Pod IP address
*/
/**
* @typedef {Object} DatabaseConfig
* @memberof UnderpostDB
* @property {string} provider - Database provider (mariadb, mongoose)
* @property {string} name - Database name
* @property {string} user - Database user
* @property {string} password - Database password
* @property {string} hostFolder - Host folder path
* @property {string} host - Host identifier
* @property {string} path - Path identifier
* @property {number} [currentBackupTimestamp] - Timestamp of current backup
*/
/**
* @class UnderpostDB
* @description Manages database operations and backups for Kubernetes-based deployments.
* Provides comprehensive database management including import/export, multi-pod targeting,
* Git integration, and cluster metadata management.
*/
/**
* UnderpostDB class for managing database operations and backups.
* @class UnderpostDB
* @memberof UnderpostDB
*/
class UnderpostDB {
/**
* Static API object containing all database operation methods.
* @static
* @memberof UnderpostDB
*/
static API = {
/**
* Helper: Resolves the latest backup timestamp from an existing backup directory.
* Scans the directory for numeric (epoch) sub-folders and returns the most recent one.
* @method _getLatestBackupTimestamp
* @memberof UnderpostDB
* @param {string} backupDir - Path to the host-folder backup directory.
* @return {string|null} The latest timestamp string, or null if none found.
*/
_getLatestBackupTimestamp(backupDir) {
if (!fs.existsSync(backupDir)) return null;
const entries = fs.readdirSync(backupDir).filter((e) => /^\d+$/.test(e));
if (entries.length === 0) return null;
return entries.sort((a, b) => parseInt(b) - parseInt(a))[0];
},
/**
* Helper: Performs MariaDB import operation.
* @method _importMariaDB
* @memberof UnderpostDB
* @param {Object} params - Import parameters.
* @param {PodInfo} params.pod - Target pod.
* @param {string} params.namespace - Namespace.
* @param {string} params.dbName - Database name.
* @param {string} params.user - Database user.
* @param {string} params.password - Database password.
* @param {string} params.sqlPath - SQL file path.
* @return {boolean} Success status.
*/
_importMariaDB({ pod, namespace, dbName, user, password, sqlPath }) {
try {
const podName = pod.NAME;
const containerSqlPath = `/${dbName}.sql`;
logger.info('Importing MariaDB database', { podName, dbName });
// Always ensure the database exists first — required for WP even when no backup is available
Underpost.kubectl.run(
`kubectl exec -n ${namespace} -i ${podName} -- mariadb -p${password} -e 'CREATE DATABASE IF NOT EXISTS ${dbName};'`,
{ context: `create database ${dbName}` },
);
// If no SQL file is available, the empty database is enough — return early
if (!sqlPath || !fs.existsSync(sqlPath)) {
logger.warn('No SQL backup file found — empty database ensured', { podName, dbName, sqlPath });
return true;
}
// Remove existing SQL file in container
Underpost.kubectl.exec({
podName,
namespace,
command: `rm -rf ${containerSqlPath}`,
});
// Copy SQL file to pod
if (
!Underpost.kubectl.cpTo({
sourcePath: sqlPath,
podName,
namespace,
destPath: containerSqlPath,
})
) {
return false;
}
// Import SQL file
const importCmd = `mariadb -u ${user} -p${password} ${dbName} < ${containerSqlPath}`;
Underpost.kubectl.exec({ podName, namespace, command: importCmd });
logger.info('Successfully imported MariaDB database', { podName, dbName });
return true;
} catch (error) {
logger.error('MariaDB import failed', { podName: pod.NAME, dbName, error: error.message });
return false;
}
},
/**
* Helper: Performs MariaDB export operation.
* @method _exportMariaDB
* @memberof UnderpostDB
* @param {Object} params - Export parameters.
* @param {PodInfo} params.pod - Source pod.
* @param {string} params.namespace - Namespace.
* @param {string} params.dbName - Database name.
* @param {string} params.user - Database user.
* @param {string} params.password - Database password.
* @param {string} params.outputPath - Output file path.
* @return {Promise<boolean>} A promise that resolves with success status.
*/
async _exportMariaDB({ pod, namespace, dbName, user, password, outputPath }) {
try {
const podName = pod.NAME;
const containerSqlPath = `/home/${dbName}.sql`;
logger.info('Exporting MariaDB database', { podName, dbName });
// Remove existing SQL file in container
Underpost.kubectl.exec({
podName,
namespace,
command: `rm -rf ${containerSqlPath}`,
});
// Dump database
const dumpCmd = `mariadb-dump --user=${user} --password=${password} --lock-tables ${dbName} > ${containerSqlPath}`;
Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
// Copy SQL file from pod
if (
!Underpost.kubectl.cpFrom({
podName,
namespace,
sourcePath: containerSqlPath,
destPath: outputPath,
})
) {
return false;
}
// Split file if it exists
if (fs.existsSync(outputPath)) {
await splitFileFactory(dbName, outputPath);
}
logger.info('Successfully exported MariaDB database', { podName, dbName, outputPath });
return true;
} catch (error) {
logger.error('MariaDB export failed', { podName: pod.NAME, dbName, error: error.message });
return false;
}
},
/**
* Helper: Performs MongoDB import operation.
* @method _importMongoDB
* @memberof UnderpostDB
* @param {Object} params - Import parameters.
* @param {PodInfo} params.pod - Target pod.
* @param {string} params.namespace - Namespace.
* @param {string} params.dbName - Database name.
* @param {string} params.bsonPath - BSON directory path.
* @param {boolean} params.drop - Whether to drop existing database.
* @param {boolean} params.preserveUUID - Whether to preserve UUIDs.
* @param {string} [params.user=''] - MongoDB username for authenticated restore.
* @param {string} [params.password=''] - MongoDB password for authenticated restore.
* @param {string} [params.authDatabase='admin'] - Auth database for restore command.
* @return {boolean} Success status.
*/
_importMongoDB({ pod, namespace, dbName, bsonPath, drop, preserveUUID, user = '', password = '', authDatabase = 'admin' }) {
try {
const podName = pod.NAME;
const containerBsonPath = `/${dbName}`;
logger.info('Importing MongoDB database', { podName, dbName });
// If no BSON directory is available, skip — MongoDB creates the DB on first write
if (!bsonPath || !fs.existsSync(bsonPath)) {
logger.warn('No BSON backup directory found — database will be created on first write', {
podName,
dbName,
bsonPath,
});
return true;
}
// Remove existing BSON directory in container
Underpost.kubectl.exec({
podName,
namespace,
command: `rm -rf ${containerBsonPath}`,
});
// Copy BSON directory to pod
if (
!Underpost.kubectl.cpTo({
sourcePath: bsonPath,
podName,
namespace,
destPath: containerBsonPath,
})
) {
return false;
}
// Restore database
const authFlags = user && password
? ` --username ${JSON.stringify(user)} --password ${JSON.stringify(password)} --authenticationDatabase ${JSON.stringify(authDatabase)}`
: '';
const restoreCmd = `mongorestore -d ${dbName} ${containerBsonPath}${drop ? ' --drop' : ''}${preserveUUID ? ' --preserveUUID' : ''}${authFlags}`;
Underpost.kubectl.exec({ podName, namespace, command: restoreCmd });
logger.info('Successfully imported MongoDB database', { podName, dbName });
return true;
} catch (error) {
logger.error('MongoDB import failed', { podName: pod.NAME, dbName, error: error.message });
return false;
}
},
/**
* Helper: Performs MongoDB export operation.
* @method _exportMongoDB
* @memberof UnderpostDB
* @param {Object} params - Export parameters.
* @param {PodInfo} params.pod - Source pod.
* @param {string} params.namespace - Namespace.
* @param {string} params.dbName - Database name.
* @param {string} params.outputPath - Output directory path.
* @param {string} [params.collections=''] - Comma-separated collection list.
* @param {string} [params.user=''] - MongoDB username for authenticated dump.
* @param {string} [params.password=''] - MongoDB password for authenticated dump.
* @param {string} [params.authDatabase='admin'] - Auth database for dump command.
* @return {boolean} Success status.
*/
_exportMongoDB({ pod, namespace, dbName, outputPath, collections = '', user = '', password = '', authDatabase = 'admin' }) {
try {
const podName = pod.NAME;
const containerBsonPath = `/${dbName}`;
logger.info('Exporting MongoDB database', { podName, dbName, collections });
// Remove existing BSON directory in container
Underpost.kubectl.exec({
podName,
namespace,
command: `rm -rf ${containerBsonPath}`,
});
// Dump database or specific collections
const authFlags = user && password
? ` --username ${JSON.stringify(user)} --password ${JSON.stringify(password)} --authenticationDatabase ${JSON.stringify(authDatabase)}`
: '';
if (collections) {
const collectionList = collections.split(',').map((c) => c.trim());
for (const collection of collectionList) {
const dumpCmd = `mongodump -d ${dbName} --collection ${collection} -o /${authFlags}`;
Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
}
} else {
const dumpCmd = `mongodump -d ${dbName} -o /${authFlags}`;
Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
}
// Copy BSON directory from pod
if (
!Underpost.kubectl.cpFrom({
podName,
namespace,
sourcePath: containerBsonPath,
destPath: outputPath,
})
) {
return false;
}
logger.info('Successfully exported MongoDB database', { podName, dbName, outputPath });
return true;
} catch (error) {
logger.error('MongoDB export failed', { podName: pod.NAME, dbName, error: error.message });
return false;
}
},
/**
* Helper: Gets MongoDB collection statistics.
* @method _getMongoStats
* @memberof UnderpostDB
* @param {Object} params - Parameters.
* @param {string} params.podName - Pod name.
* @param {string} params.namespace - Namespace.
* @param {string} params.dbName - Database name.
* @param {string} [params.user=''] - MongoDB username for authenticated stats query.
* @param {string} [params.password=''] - MongoDB password for authenticated stats query.
* @param {string} [params.authDatabase='admin'] - Auth database for stats query.
* @return {Object|null} Collection statistics or null on error.
*/
_getMongoStats({ podName, namespace, dbName, user = '', password = '', authDatabase = 'admin' }) {
try {
logger.info('Getting MongoDB collection statistics', { podName, dbName });
// Use db.getSiblingDB() instead of 'use' command
const script = `db.getSiblingDB('${dbName}').getCollectionNames().map(function(c) { return { collection: c, count: db.getSiblingDB('${dbName}')[c].countDocuments() }; })`;
// Execute the script
const authFlags = user && password
? ` --authenticationDatabase ${JSON.stringify(authDatabase)} -u ${JSON.stringify(user)} -p ${JSON.stringify(password)}`
: '';
const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mongosh --quiet${authFlags} --eval "${script}"`;
const output = shellExec(command, { stdout: true, silent: true, silentOnError: true });
if (!output || output.trim() === '') {
logger.warn('No collections found or empty output');
return null;
}
// Clean the output: remove newlines, handle EJSON format, replace single quotes with double quotes
let cleanedOutput = output
.trim()
.replace(/\n/g, '')
.replace(/\s+/g, ' ')
.replace(/NumberLong\("(\d+)"\)/g, '$1')
.replace(/NumberLong\((\d+)\)/g, '$1')
.replace(/NumberInt\("(\d+)"\)/g, '$1')
.replace(/NumberInt\((\d+)\)/g, '$1')
.replace(/ISODate\("([^"]+)"\)/g, '"$1"')
.replace(/'/g, '"')
.replace(/(\w+):/g, '"$1":');
try {
const stats = JSON.parse(cleanedOutput);
logger.info('MongoDB statistics retrieved', { dbName, collections: stats.length });
return stats;
} catch (parseError) {
logger.error('Failed to parse MongoDB output', {
podName,
dbName,
error: parseError.message,
rawOutput: output.substring(0, 200),
cleanedOutput: cleanedOutput.substring(0, 200),
});
return null;
}
} catch (error) {
logger.error('Failed to get MongoDB statistics', { podName, dbName, error: error.message });
return null;
}
},
/**
* Helper: Gets MariaDB table statistics.
* @method _getMariaDBStats
* @memberof UnderpostDB
* @param {Object} params - Parameters.
* @param {string} params.podName - Pod name.
* @param {string} params.namespace - Namespace.
* @param {string} params.dbName - Database name.
* @param {string} params.user - Database user.
* @param {string} params.password - Database password.
* @return {Object|null} Table statistics or null on error.
*/
_getMariaDBStats({ podName, namespace, dbName, user, password }) {
try {
logger.info('Getting MariaDB table statistics', { podName, dbName });
const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mariadb -u ${user} -p${password} ${dbName} -e "SELECT TABLE_NAME as 'table', TABLE_ROWS as 'count' FROM information_schema.TABLES WHERE TABLE_SCHEMA = '${dbName}' ORDER BY TABLE_NAME;" --skip-column-names --batch`;
const output = shellExec(command, { stdout: true, silent: true, disableLog: true, silentOnError: true });
if (!output || output.trim() === '') {
logger.warn('No tables found or empty output');
return null;
}
// Parse the output (tab-separated values)
const lines = output.trim().split('\n');
const stats = lines.map((line) => {
const [table, count] = line.split('\t');
return { table, count: parseInt(count) || 0 };
});
logger.info('MariaDB statistics retrieved', { dbName, tables: stats.length });
return stats;
} catch (error) {
logger.error('Failed to get MariaDB statistics', { podName, dbName, error: error.message });
return null;
}
},
/**
* Helper: Displays database statistics in table format.
* @method _displayStats
* @memberof UnderpostDB
* @param {Object} params - Parameters.
* @param {string} params.provider - Database provider.
* @param {string} params.dbName - Database name.
* @param {Array<Object>} params.stats - Statistics array.
* @return {void}
*/
_displayStats({ provider, dbName, stats }) {
if (!stats || stats.length === 0) {
logger.warn('No statistics to display', { provider, dbName });
return;
}
const title = provider === 'mongoose' ? 'Collections' : 'Tables';
const itemKey = provider === 'mongoose' ? 'collection' : 'table';
console.log('\n' + '='.repeat(70));
console.log(`DATABASE: ${dbName} (${provider.toUpperCase()})`);
console.log('='.repeat(70));
console.log(`${title.padEnd(50)} ${'Documents/Rows'.padStart(18)}`);
console.log('-'.repeat(70));
let totalCount = 0;
stats.forEach((item) => {
const name = item[itemKey] || 'Unknown';
const count = item.count || 0;
totalCount += count;
console.log(`${name.padEnd(50)} ${count.toString().padStart(18)}`);
});
console.log('-'.repeat(70));
console.log(`${'TOTAL'.padEnd(50)} ${totalCount.toString().padStart(18)}`);
console.log('='.repeat(70) + '\n');
},
/**
* Main callback: Initiates database backup workflow.
* Orchestrates the backup process for multiple deployments, handling
* database connections, backup storage, and optional Git integration for version control.
* Supports targeting multiple specific pods, nodes, and namespaces with advanced filtering.
* @method callback
* @memberof UnderpostDB
* @param {string} [deployList='default'] - Comma-separated list of deployment IDs.
* @param {Object} [options={}] - Backup options.
* @param {boolean} [options.import=false] - Whether to perform import operation.
* @param {boolean} [options.export=false] - Whether to perform export operation.
* @param {string} [options.podName=''] - Comma-separated pod name patterns to target.
* @param {string} [options.ns='default'] - Kubernetes namespace.
* @param {string} [options.collections=''] - Comma-separated MongoDB collections for export.
* @param {string} [options.outPath=''] - Output path for backups.
* @param {boolean} [options.drop=false] - Whether to drop existing database on import.
* @param {boolean} [options.preserveUUID=false] - Whether to preserve UUIDs on MongoDB import.
* @param {boolean} [options.git=false] - Whether to use Git for backup versioning.
* @param {string} [options.hosts=''] - Comma-separated list of hosts to filter databases.
* @param {string} [options.paths=''] - Comma-separated list of paths to filter databases.
* @param {boolean} [options.allPods=false] - Whether to target all pods in deployment.
* @param {boolean} [options.primaryPod=false] - Whether to target MongoDB primary pod only.
* @param {string} [options.primaryPodEnsure=''] - Pod name to ensure MongoDB primary pod is running.
* @param {boolean} [options.stats=false] - Whether to display database statistics.
* @param {number} [options.macroRollbackExport=1] - Number of commits to rollback in macro export.
* @param {boolean} [options.forceClone=false] - Whether to force re-clone Git repository.
* @param {boolean} [options.cleanFsCollection=false] - Clean orphaned File documents flag.
* @param {boolean} [options.cleanFsDryRun=false] - Dry run mode flag (use with cleanFsCollection).
* @param {boolean} [options.dev=false] - Development mode flag.
* @param {boolean} [options.k3s=false] - k3s cluster flag.
* @param {boolean} [options.kubeadm=false] - kubeadm cluster flag.
* @param {boolean} [options.kind=false] - kind cluster flag.
* @param {boolean} [options.repoBackup=false] - Backs up repositories (git commit+push) inside deployment pods via kubectl exec.
* @return {Promise<void>} Resolves when operation is complete.
*/
async callback(
deployList = 'default',
options = {
import: false,
export: false,
podName: '',
ns: 'default',
collections: '',
outPath: '',
drop: false,
preserveUUID: false,
git: false,
hosts: '',
paths: '',
allPods: false,
primaryPod: false,
primaryPodEnsure: '',
stats: false,
macroRollbackExport: 1,
forceClone: false,
cleanFsCollection: false,
cleanFsDryRun: false,
dev: false,
k3s: false,
kubeadm: false,
kind: false,
repoBackup: false,
},
) {
// Ensure engine-private is available (clone if inside a deployment
// container where globalSecretClean has already removed it).
const firstDeployId = deployList !== 'dd' ? deployList.split(',')[0].trim() : '';
try {
loadCronDeployEnv();
const newBackupTimestamp = new Date().getTime();
const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
// Handle repository backup (git commit+push inside deployment pod)
if (options.repoBackup) {
const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
for (const _deployId of deployList.split(',')) {
const deployId = _deployId.trim();
if (!deployId) continue;
logger.info('Starting pod repository backup', { deployId, namespace });
Underpost.repo.backupPodRepositories({
deployId,
namespace,
env: options.dev ? 'development' : 'production',
});
}
return;
}
// Handle clean-fs-collection operation
if (options.cleanFsCollection || options.cleanFsDryRun) {
logger.info('Starting File collection cleanup operation', { deployList });
await Underpost.db.cleanFsCollection(deployList, {
hosts: options.hosts,
paths: options.paths,
dryRun: options.cleanFsDryRun,
});
return;
}
logger.info('Starting database operation', {
deployList,
namespace,
import: options.import,
export: options.export,
});
if (options.primaryPodEnsure) {
const primaryPodName = MongoBootstrap.getPrimaryPodName({
namespace,
podName: options.primaryPodEnsure,
username: process.env.MONGODB_USERNAME || process.env.DB_USER || '',
password: process.env.MONGODB_PASSWORD || process.env.DB_PASSWORD || '',
authDatabase: process.env.MONGODB_AUTH_DB || 'admin',
});
if (!primaryPodName) {
const baseCommand = options.dev ? 'node bin' : 'underpost';
const baseClusterCommand = options.dev ? ' --dev' : '';
let clusterFlag = options.k3s ? ' --k3s' : options.kubeadm ? ' --kubeadm' : '';
shellExec(`${baseCommand} cluster${baseClusterCommand}${clusterFlag} --mongodb`);
}
return;
}
// Track processed repositories to avoid duplicate Git operations
const processedRepos = new Set();
// Track processed host+path combinations to avoid duplicates
const processedHostPaths = new Set();
for (const _deployId of deployList.split(',')) {
const deployId = _deployId.trim();
if (!deployId) continue;
logger.info('Processing deployment', { deployId });
/** @type {Object.<string, Object.<string, DatabaseConfig>>} */
const dbs = {};
const repoName = `engine-${deployId.includes('dd-') ? deployId.split('dd-')[1] : deployId}-cron-backups`;
// Load server configuration
const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
if (!fs.existsSync(confServerPath)) {
logger.error('Configuration file not found', { path: confServerPath });
continue;
}
const confServer = loadConfServerJson(confServerPath, { resolve: true });
// Build database configuration map
for (const host of Object.keys(confServer)) {
for (const path of Object.keys(confServer[host])) {
const { db } = confServer[host][path];
if (db) {
const { provider, name, user, password } = db;
if (!dbs[provider]) dbs[provider] = {};
if (!(name in dbs[provider])) {
dbs[provider][name] = {
user,
password,
hostFolder: host + path.replaceAll('/', '-'),
host,
path,
};
}
}
}
}
// Handle Git operations - execute only once per repository
if (!processedRepos.has(repoName)) {
logger.info('Processing Git operations for repository', { repoName, deployId });
if (options.git === true) {
Underpost.repo.manageBackupRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
Underpost.repo.manageBackupRepo({ repoName, operation: 'pull' });
}
if (options.macroRollbackExport) {
// Only clone if not already done by git option above
if (options.git !== true) {
Underpost.repo.manageBackupRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
Underpost.repo.manageBackupRepo({ repoName, operation: 'pull' });
}
const nCommits = parseInt(options.macroRollbackExport);
const repoPath = `../${repoName}`;
const username = process.env.GITHUB_USERNAME;
if (fs.existsSync(repoPath) && username) {
logger.info('Executing macro rollback export', { repoName, nCommits });
shellExec(`cd ${repoPath} && underpost cmt . reset ${nCommits}`);
shellExec(`cd ${repoPath} && git reset`);
shellExec(`cd ${repoPath} && git checkout .`);
shellExec(`cd ${repoPath} && git clean -f -d`);
shellExec(`cd ${repoPath} && underpost push . ${username}/${repoName} -f`);
} else {
if (!username) logger.error('GITHUB_USERNAME environment variable not set');
logger.warn('Repository not found for macro rollback', { repoPath });
}
}
processedRepos.add(repoName);
logger.info('Repository marked as processed', { repoName });
} else {
logger.info('Skipping Git operations for already processed repository', { repoName, deployId });
}
// Process each database provider
for (const provider of Object.keys(dbs)) {
for (const dbName of Object.keys(dbs[provider])) {
const { hostFolder, user, password, host, path } = dbs[provider][dbName];
// Create unique identifier for host+path combination
const hostPathKey = `${deployId}:${host}:${path}`;
// Skip if this host+path combination was already processed
if (processedHostPaths.has(hostPathKey)) {
logger.info('Skipping already processed host/path', { dbName, host, path, deployId });
continue;
}
// Filter by hosts and paths if specified
if (
(options.hosts &&
!options.hosts
.split(',')
.map((h) => h.trim())
.includes(host)) ||
(options.paths &&
!options.paths
.split(',')
.map((p) => p.trim())
.includes(path))
) {
logger.info('Skipping database due to host/path filter', { dbName, host, path });
continue;
}
if (!hostFolder) {
logger.warn('No hostFolder defined for database', { dbName, provider });
continue;
}
logger.info('Processing database', { hostFolder, provider, dbName, deployId });
const latestBackupTimestamp = Underpost.db._getLatestBackupTimestamp(`../${repoName}/${hostFolder}`);
dbs[provider][dbName].currentBackupTimestamp = latestBackupTimestamp;
const currentTimestamp = latestBackupTimestamp || newBackupTimestamp;
const sqlContainerPath = `/home/${dbName}.sql`;
const fromPartsPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}-parths.json`;
const toSqlPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}.sql`;
const toNewSqlPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}.sql`;
const toBsonPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}`;
const toNewBsonPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}`;
// Merge split SQL files if needed for import
if (options.import === true && fs.existsSync(fromPartsPath) && !fs.existsSync(toSqlPath)) {
const names = JSON.parse(fs.readFileSync(fromPartsPath, 'utf8')).map((_path) => {
return `../${repoName}/${hostFolder}/${currentTimestamp}/${_path.split('/').pop()}`;
});
logger.info('Merging backup parts', { fromPartsPath, toSqlPath, parts: names.length });
await mergeFile(names, toSqlPath);
}
// Get target pods based on provider and options
let targetPods = [];
const podCriteria = {
podNames: options.podName,
namespace,
deployId: provider === 'mariadb' ? 'mariadb' : 'mongo',
};
targetPods = Underpost.kubectl.getFilteredPods(podCriteria);
// Fallback to default if no custom pods specified
if (targetPods.length === 0 && !options.podName) {
const defaultPods = Underpost.kubectl.get(
provider === 'mariadb' ? 'mariadb' : 'mongo',
'pods',
namespace,
);
console.log('defaultPods', defaultPods);
targetPods = defaultPods;
}
if (targetPods.length === 0) {
logger.warn('No pods found matching criteria', { provider, criteria: podCriteria });
continue;
}
// Handle primary pod detection for MongoDB
let podsToProcess = [];
if (provider === 'mongoose' && !options.allPods) {
// For MongoDB, always use primary pod unless allPods is true
if (!targetPods || targetPods.length === 0) {
logger.warn('No MongoDB pods available to check for primary');
podsToProcess = [];
} else {
const firstPod = targetPods[0].NAME;
const primaryPodName = MongoBootstrap.getPrimaryPodName({
namespace,
podName: firstPod,
username: user,
password,
authDatabase: 'admin',
});
if (primaryPodName) {
const primaryPod = targetPods.find((p) => p.NAME === primaryPodName);
if (primaryPod) {
podsToProcess = [primaryPod];
logger.info('Using MongoDB primary pod', { primaryPod: primaryPodName });
} else {
logger.warn('Primary pod not in filtered list, using first pod', { primaryPodName });
podsToProcess = [targetPods[0]];
}
} else {
logger.warn('Could not detect primary pod, using first pod');
podsToProcess = [targetPods[0]];
}
}
} else {
// For MariaDB or when allPods is true, limit to first pod unless allPods is true
podsToProcess = options.allPods === true ? targetPods : [targetPods[0]];
}
logger.info(`Processing ${podsToProcess.length} pod(s) for ${provider}`, {
dbName,
pods: podsToProcess.map((p) => p.NAME),
});
let exportSucceeded = false;
// Process each pod
for (const pod of podsToProcess) {
logger.info('Processing pod', { podName: pod.NAME, node: pod.NODE, status: pod.STATUS });
switch (provider) {
case 'mariadb': {
if (options.stats === true) {
const stats = Underpost.db._getMariaDBStats({
podName: pod.NAME,
namespace,
dbName,
user,
password,
});
if (stats) {
Underpost.db._displayStats({ provider, dbName, stats });
}
}
if (options.import === true) {
Underpost.db._importMariaDB({
pod,
namespace,
dbName,
user,
password,
sqlPath: toSqlPath,
});
}
if (options.export === true) {
const outputPath = options.outPath || toNewSqlPath;
const success = await Underpost.db._exportMariaDB({
pod,
namespace,
dbName,
user,
password,
outputPath,
});
exportSucceeded = exportSucceeded || success;
}
break;
}
case 'mongoose': {
if (options.stats === true) {
const stats = Underpost.db._getMongoStats({
podName: pod.NAME,
namespace,
dbName,
user,
password,
authDatabase: 'admin',
});
if (stats) {
Underpost.db._displayStats({ provider, dbName, stats });
}
}
if (options.import === true) {
const bsonPath = options.outPath || toBsonPath;
Underpost.db._importMongoDB({
pod,
namespace,
dbName,
bsonPath,
drop: options.drop,
preserveUUID: options.preserveUUID,
user,
password,
authDatabase: 'admin',
});
}
if (options.export === true) {
const outputPath = options.outPath || toNewBsonPath;
const success = Underpost.db._exportMongoDB({
pod,
namespace,
dbName,
outputPath,
collections: options.collections,
user,
password,
authDatabase: 'admin',
});
exportSucceeded = exportSucceeded || success;
}
break;
}
default:
logger.warn('Unsupported database provider', { provider });
break;
}
}
if (options.export === true && exportSucceeded === true) {
Underpost.db._enforceBackupRetention(`../${repoName}/${hostFolder}`);
}
// Mark this host+path combination as processed
processedHostPaths.add(hostPathKey);
}
}
// Commit and push to Git if enabled - execute only once per repository
if (options.export === true && options.git === true && !processedRepos.has(`${repoName}-committed`)) {
const commitMessage = `${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(
newBackupTimestamp,
).toLocaleTimeString()}`;
Underpost.repo.manageBackupRepo({ repoName, operation: 'commit', message: commitMessage });
Underpost.repo.manageBackupRepo({ repoName, operation: 'push' });
processedRepos.add(`${repoName}-committed`);
}
}
logger.info('Database operation completed successfully');
} catch (error) {
logger.error('Database operation failed', { error: error.message });
throw error;
}
},
/**
* Helper: Removes old timestamp backup folders and keeps only the newest ones.
* @method _enforceBackupRetention
* @memberof UnderpostDB
* @param {string} backupDir - Path to host-folder backup directory.
* @param {number} [maxRetention=MAX_BACKUP_RETENTION] - Maximum folders to keep.
* @return {number} Number of removed backup folders.
*/
_enforceBackupRetention(backupDir, maxRetention = MAX_BACKUP_RETENTION) {
try {
if (!fs.existsSync(backupDir)) return 0;
const timestamps = fs
.readdirSync(backupDir)
.filter((entry) => /^\d+$/.test(entry))
.sort((a, b) => parseInt(b, 10) - parseInt(a, 10));
if (timestamps.length <= maxRetention) return 0;
const staleTimestamps = timestamps.slice(maxRetention);
staleTimestamps.forEach((timestamp) => {
fs.removeSync(`${backupDir}/${timestamp}`);
});
logger.info('Pruned old backup timestamp folders', {
backupDir,
kept: maxRetention,
removed: staleTimestamps.length,
removedTimestamps: staleTimestamps,
});
return staleTimestamps.length;
} catch (error) {
logger.error('Failed to enforce backup retention', { backupDir, maxRetention, error: error.message });
return 0;
}
},
/**
* Creates cluster metadata for the specified deployment.
* Loads database configuration and initializes cluster metadata including
* instances and cron jobs. This method populates the database with deployment information.
* @method clusterMetadataFactory
* @memberof UnderpostDB
* @param {string} [deployId=process.env.DEFAULT_DEPLOY_ID] - The deployment ID.
* @param {string} [host=process.env.DEFAULT_DEPLOY_HOST] - The host identifier.
* @param {string} [path=process.env.DEFAULT_DEPLOY_PATH] - The path identifier.
* @param {object} [options] - Options.
* @param {boolean} [options.dev=false] - Development mode flag.
* @return {Promise<void>} Resolves when metadata creation is complete.
* @throws {Error} If database configuration is invalid or connection fails.
*/
async clusterMetadataFactory(
deployId = process.env.DEFAULT_DEPLOY_ID,
host = process.env.DEFAULT_DEPLOY_HOST,
path = process.env.DEFAULT_DEPLOY_PATH,
options = { dev: false },
) {
try {
loadCronDeployEnv();
deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
path = path ? path : process.env.DEFAULT_DEPLOY_PATH;
logger.info('Creating cluster metadata', { deployId, host, path });
const env = 'production';
const deployListPath = './engine-private/deploy/dd.router';
if (!fs.existsSync(deployListPath)) {
logger.error('Deploy router file not found', { path: deployListPath });
throw new Error(`Deploy router file not found: ${deployListPath}`);
}
const deployList = fs.readFileSync(deployListPath, 'utf8').split(',');
const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
if (!fs.existsSync(confServerPath)) {
logger.error('Server configuration not found', { path: confServerPath });
throw new Error(`Server configuration not found: ${confServerPath}`);
}
const { db } = loadConfServerJson(confServerPath, { resolve: true })[host][path];
const maxRetries = 5;
const retryDelay = 3000;
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
await DataBaseProviderService.load({ apis: ['instance', 'cron'], host, path, db });
break;
} catch (err) {
if (attempt === maxRetries) {
logger.error('Failed to connect to database after retries', { attempts: maxRetries, error: err.message });
throw err;
}
logger.warn('Database connection failed, retrying...', { attempt, maxRetries, error: err.message });
await timer(retryDelay);
}
}
try {
/** @type {import('../api/instance/instance.model.js').InstanceModel} */
const Instance = DataBaseProviderService.getModel('instance', { host, path });
await Instance.deleteMany();
logger.info('Cleared existing instance metadata');
for (const _deployId of deployList) {
const deployId = _deployId.trim();
if (!deployId) continue;
logger.info('Processing deployment for metadata', { deployId });
const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
if (!fs.existsSync(confServerPath)) {
logger.warn('Configuration not found for deployment', { deployId, path: confServerPath });
continue;
}
const confServer = loadReplicas(deployId, loadConfServerJson(confServerPath, { resolve: true }));
const router = await Underpost.deploy.routerFactory(deployId, env);
const pathPortAssignmentData = await pathPortAssignmentFactory(deployId, router, confServer);
for (const host of Object.keys(confServer)) {
for (const { path, port } of pathPortAssignmentData[host]) {
if (!confServer[host][path]) continue;
const { client, runtime, apis, peer } = confServer[host][path];
// Save main instance
{
const body = {
deployId,
host,
path,
port,
client,
runtime,
apis,
};
logger.info('Saving instance metadata', body);
await new Instance(body).save();
}
// Save peer instance if exists
if (peer) {
const body = {
deployId,
host,
path: path === '/' ? '/peer' : `${path}/peer`,
port: port + 1,
runtime: 'nodejs',
};
logger.info('Saving peer instance metadata', body);
await new Instance(body).save();
}
}
}
// Process additional instances
const confInstancesPath = `./engine-private/conf/${deployId}/conf.instances.json`;
if (fs.existsSync(confInstancesPath)) {
const confInstances = JSON.parse(fs.readFileSync(confInstancesPath, 'utf8'));
for (const instance of confInstances) {
const { id, host, path, fromPort, metadata } = instance;
const { runtime } = metadata;
const body = {
deployId,
host,
path,
port: fromPort,
client: id,
runtime,
};
logger.info('Saving additional instance metadata', body);
await new Instance(body).save();
}
}
}
} catch (error) {
logger.error('Failed to create instance metadata', { error: error.message });
throw error;
}
try {
const cronDeployPath = './engine-private/deploy/dd.cron';
if (!fs.existsSync(cronDeployPath)) {
logger.warn('Cron deploy file not found', { path: cronDeployPath });
return;
}
const cronDeployId = fs.readFileSync(cronDeployPath, 'utf8').trim();
const confCronPath = `./engine-private/conf/${cronDeployId}/conf.cron.json`;
if (!fs.existsSync(confCronPath)) {
logger.warn('Cron configuration not found', { path: confCronPath });
return;
}
const confCron = JSON.parse(fs.readFileSync(confCronPath, 'utf8'));
await DataBaseProviderService.load({ apis: ['cron'], host, path, db });
/** @type {import('../api/cron/cron.model.js').CronModel} */
const Cron = DataBaseProviderService.getModel('cron', { host, path });
await Cron.deleteMany();
logger.info('Cleared existing cron metadata');
for (const jobId of Object.keys(confCron.jobs)) {
const body = {
jobId,
deployId: Underpost.cron.getRelatedDeployIdList(jobId),
expression: confCron.jobs[jobId].expression,
enabled: confCron.jobs[jobId].enabled,
};
logger.info('Saving cron metadata', body);
await new Cron(body).save();
}
} catch (error) {
logger.error('Failed to create cron metadata', { error: error.message });
}
await DataBaseProviderService.getProvider({ host, path }, 'mongoose').close();
logger.info('Cluster metadata creation completed');
} catch (error) {
logger.error('Cluster metadata creation failed', { error: error.message });
throw error;
}
},
/**
* Cleans orphaned File references from database collections.
* Iterates over all deploy-ids and checks if File documents are actually referenced
* by other collections. Removes File documents that are not referenced anywhere.
* @method cleanFsCollection
* @memberof UnderpostDB
* @param {strin