@underpostnet/underpost/src/cli/run.js

Underpost Platform — end-to-end CI/CD and application-delivery toolchain CLI. Covers bare metal, Kubernetes, K3s, kubeadm, LXD, container/image orchestration, secrets, databases, cron jobs, monitoring, SSH, runners, PWA + Workbox delivery, and release orc

/**
 * @description The main entry point for the Underpost CLI applications.
 * @module src/cli/run.js
 * @namespace UnderpostRun
 */

import { daemonProcess, getTerminalPid, pbcopy, shellCd, shellExec } from '../server/process.js';
import crypto from 'crypto';
import {
  awaitDeployMonitor,
  buildKindPorts,
  Config,
  cronDeployIdResolve,
  etcHostFactory,
  getNpmRootPath,
  isDeployRunnerContext,
  loadConfServerJson,
  loadReplicas,
  writeEnv,
} from '../server/conf.js';
import { actionInitLog, loggerFactory } from '../server/logger.js';

import fs from 'fs-extra';
import net from 'net';
import { range, s4, setPad, timer } from '../client/components/core/CommonJs.js';

import os from 'os';
import Underpost from '../index.js';
import dotenv from 'dotenv';
import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';

const waitForPort = (port, host = '127.0.0.1', { maxAttempts = 30, interval = 2000 } = {}) =>
  new Promise((resolve, reject) => {
    let attempts = 0;
    const tryConnect = () => {
      attempts++;
      const socket = net.createConnection({ port, host }, () => {
        socket.destroy();
        resolve();
      });
      socket.on('error', () => {
        socket.destroy();
        if (attempts >= maxAttempts) return reject(new Error(`Port ${port} not ready after ${maxAttempts} attempts`));
        setTimeout(tryConnect, interval);
      });
    };
    tryConnect();
  });

const logger = loggerFactory(import.meta);

/**
 * @constant DEFAULT_OPTION
 * @description Default options for the UnderpostRun class.
 * @typedef {Object} UnderpostRunDefaultOptions
 * @type {Object}
 * @property {boolean} dev - Whether to run in development mode.
 * @property {string} podName - The name of the pod to run.
 * @property {string} nodeName - The name of the node to run.
 * @property {string} sshKeyPath - Private key path for node SSH operations, forwarded to volume shipping over SSH.
 * @property {number} port - Custom port to use.
 * @property {string} volumeHostPath - The host path for the volume.
 * @property {string} volumeMountPath - The mount path for the volume.
 * @property {string} imageName - The name of the image to run.
 * @property {string} containerName - The name of the container to run.
 * @property {string} namespace - The namespace to run in.
 * @property {string} timeoutResponse - The response timeout duration.
 * @property {string} timeoutIdle - The idle timeout duration.
 * @property {string} retryCount - The number of retries.
 * @property {string} retryPerTryTimeout - The timeout duration per retry.
 * @property {boolean} build - Whether to build the image.
 * @property {number} replicas - The number of replicas to run.
 * @property {boolean} force - Whether to force the operation.
 * @property {boolean} reset - Whether to reset the operation.
 * @property {boolean} tls - Whether to use TLS.
 * @property {string} cmd - The command to run in the container.
 * @property {string} tty - The TTY option for the container.
 * @property {string} stdin - The stdin option for the container.
 * @property {string} restartPolicy - The restart policy for the container.
 * @property {string} runtimeClassName - The runtime class name for the container.
 * @property {string} imagePullPolicy - The image pull policy for the container.
 * @property {string} apiVersion - The API version for the container.
 * @property {string} claimName - The claim name for the volume.
 * @property {string} kindType - The kind of resource to create.
 * @property {number} devProxyPortOffset - The port offset for the development proxy.
 * @property {boolean} hostNetwork - Whether to use host networking.
 * @property {string} requestsMemory - The memory request for the container.
 * @property {string} requestsCpu - The CPU request for the container.
 * @property {string} limitsMemory - The memory limit for the container.
 * @property {string} limitsCpu - The CPU limit for the container.
 * @property {string} resourceTemplateId - The resource template ID.
 * @property {boolean} expose - Whether to expose the service.
 * @property {boolean} etcHosts - Whether to modify /etc/hosts.
 * @property {string} confServerPath - The configuration server path.
 * @property {string} underpostRoot - The root path of the Underpost installation.
 * @property {string} cmdCronJobs - Pre-script commands to run before cron job execution.
 * @property {string} deployIdCronJobs - The deployment ID for cron jobs.
 * @property {string} timezone - The timezone to set.
 * @property {boolean} kubeadm - Whether to run in kubeadm mode.
 * @property {boolean} kind - Whether to run in kind mode.
 * @property {boolean} k3s - Whether to run in k3s mode.
 * @property {string} hosts - The hosts to use.
 * @property {string} deployId - The deployment ID.
 * @property {string} instanceId - The instance ID.
 * @property {string} user - The user to run as.
 * @property {string} group - The group to use.
 * @property {string} pid - The process ID.
 * @property {boolean} disablePrivateConfUpdate - Whether to disable private configuration updates.
 * @property {string} monitorStatus - The monitor status option.
 * @property {string} monitorStatusKindType - The monitor status kind type option.
 * @property {string} monitorStatusDeltaMs - The monitor status delta in milliseconds.
 * @property {string} monitorStatusMaxAttempts - The maximum number of attempts for monitor status.
 * @property {boolean} logs - Whether to enable logs.
 * @property {boolean} dryRun - Whether to perform a dry run.
 * @property {boolean} createJobNow - Whether to create the job immediately.
 * @property {number} fromNCommit - Number of commits back to use for message propagation (default: 1, last commit only).
 * @property {string|Array<{ip: string, hostnames: string[]}>} hostAliases - Adds entries to the Pod /etc/hosts via Kubernetes hostAliases.
 *   As a string (CLI): semicolon-separated entries of "ip=hostname1,hostname2" (e.g., "127.0.0.1=foo.local,bar.local;10.1.2.3=foo.remote").
 *   As an array (programmatic): objects with `ip` and `hostnames` fields (e.g., [{ ip: "127.0.0.1", hostnames: ["foo.local"] }]).
 * @property {boolean} gitClean - Whether to perform a `git clean` before running.
 * @property {boolean} copy - Whether to copy the command to the clipboard instead of executing it.
 * @property {boolean} skipFullBuild - Whether to skip the full client bundle build during deployment (supported by: sync, template-deploy).
 * @property {boolean} pullBundle - Whether to pull the bundle before running. Use together with --skip-full-build to skip the local build entirely (supported by: sync, template-deploy).
 * @property {boolean} remove - Whether to remove/teardown resources instead of creating them (e.g. delete-expose for k3s proxy devices in dev-cluster).
 * @property {boolean} test - Whether to enable test/generic-purpose mode (e.g. use self-signed TLS instead of cert-manager).
 * @memberof UnderpostRun
 */
const DEFAULT_OPTION = {
  dev: false,
  podName: '',
  nodeName: '',
  sshKeyPath: '',
  port: 0,
  volumeHostPath: '',
  volumeMountPath: '',
  imageName: '',
  containerName: '',
  namespace: 'default',
  timeoutResponse: '',
  timeoutIdle: '',
  retryCount: '',
  retryPerTryTimeout: '',
  build: false,
  replicas: 1,
  force: false,
  reset: false,
  tls: false,
  cmd: '',
  tty: '',
  stdin: '',
  restartPolicy: '',
  runtimeClassName: '',
  imagePullPolicy: '',
  apiVersion: '',
  claimName: '',
  kindType: '',
  devProxyPortOffset: 0,
  hostNetwork: false,
  requestsMemory: '',
  requestsCpu: '',
  limitsMemory: '',
  limitsCpu: '',
  resourceTemplateId: '',
  expose: false,
  etcHosts: false,
  confServerPath: '',
  underpostRoot: '',
  cmdCronJobs: '',
  deployIdCronJobs: '',
  timezone: '',
  kubeadm: false,
  kind: false,
  k3s: false,
  hosts: '',
  deployId: '',
  instanceId: '',
  user: '',
  group: '',
  pid: '',
  disablePrivateConfUpdate: false,
  monitorStatus: '',
  monitorStatusKindType: '',
  monitorStatusDeltaMs: '',
  monitorStatusMaxAttempts: '',
  logs: false,
  dryRun: false,
  createJobNow: false,
  fromNCommit: 0,
  hostAliases: '',
  gitClean: false,
  copy: false,
  skipFullBuild: false,
  pullBundle: false,
  remove: false,
  test: false,
};

/**
 * @class UnderpostRun
 * @description Manages the execution of various CLI commands and operations.
 * This class provides a set of static methods to perform different tasks
 * such as running tests, deploying applications, managing environment variables,
 * and more. It also includes a default option configuration and a collection of
 * runners for executing specific commands.
 * @memberof UnderpostRun
 */
class UnderpostRun {
  /**
   * @static
   * @description Collection of runners for executing specific commands.
   * @type {Object}
   * @memberof UnderpostRun
   */
  static RUNNERS = {
    /**
     * @method dev-cluster
     * @description Resets and deploys a full development cluster including MongoDB, Valkey, exposes services, and updates `/etc/hosts` for local access.
     * @param {string} path - The input value, identifier, or path for the operation.
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'dev-cluster': (path, options = DEFAULT_OPTION) => {
      const baseCommand = options.dev ? 'node bin' : 'underpost';
      const mongoHosts = ['mongodb-0.mongodb-service'];
      let primaryMongoHost = 'mongodb-0.mongodb-service';
      if (!options.expose) {
        shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''} --reset`);
        shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''}`);

        shellExec(
          `${baseCommand} cluster${options.dev ? ' --dev' : ''} --mongodb --service-host ${mongoHosts.join(
            ',',
          )} --pull-image`,
        );
        shellExec(`${baseCommand} cluster${options.dev ? ' --dev' : ''} --valkey --pull-image`);
      }
      if (options.k3s) {
        if (options.remove) {
          shellExec(`${baseCommand} lxd --delete-expose k3s-control:27017`);
          shellExec(`${baseCommand} lxd --delete-expose k3s-control:6379`);
        } else {
          shellExec(`${baseCommand} lxd --expose k3s-control:27017 --node-port 32017`);
          shellExec(`${baseCommand} lxd --expose k3s-control:6379 --node-port 32079`);
        }
        shellExec(`lxc config device show k3s-control`);
      } else {
        try {
          const primaryPodName =
            MongoBootstrap.getPrimaryPodName({
              namespace: options.namespace,
              podName: 'mongodb-0',
              disableAuth: options.dev,
            }) || 'mongodb-0';
          shellExec(
            `${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config mongo`,
            { async: true },
          );
          shellExec(
            `${baseCommand} deploy --expose --namespace ${options.namespace} --disable-update-underpost-config valkey`,
            { async: true },
          );
        } catch (error) {
          logger.warn('Failed to detect MongoDB primary pod, using default', {
            error: error.message,
            default: primaryMongoHost,
          });
        }
      }
      const hostListenResult = etcHostFactory([primaryMongoHost]);
      logger.info(hostListenResult.renderHosts);
    },

    /**
     * @method etc-hosts
     * @description Modifies the `/etc/hosts` file to add entries for local access to services,
     * based on the provided path input.
     * @param {string} path - The input value, identifier, or path for the operation (used to specify the entries to add to /etc/hosts).
     */
    'etc-hosts': (path = '', options = DEFAULT_OPTION) => {
      etcHostFactory(path.split(','));
    },

    /**
     * @method ipfs-expose
     * @description Exposes IPFS Cluster services on specified ports for local access.
     * @type {Function}
     * @memberof UnderpostRun
     */
    'ipfs-expose': (path, options = DEFAULT_OPTION) => {
      const ports = [5001, 9094, 8080];
      for (const port of ports)
        shellExec(`node bin deploy --expose ipfs-cluster --expose-port ${port} --disable-update-underpost-config`, {
          async: true,
        });
    },

    /**
     * @method metadata
     * @description Generates metadata for the specified path after exposing the development cluster.
     * @param {string} path - The input value, identifier, or path for the operation.
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    metadata: async (path, options = DEFAULT_OPTION) => {
      const ports = '6379,27017';
      shellExec(`node bin run kill '${ports}'`);
      shellExec(`node bin run dev-cluster --dev --expose --namespace ${options.namespace}`, { async: true });
      logger.info('Waiting for port-forward services to be ready...');
      try {
        await Promise.all([waitForPort(27017), waitForPort(6379)]);
        logger.info('Port-forward services are ready');
      } catch (err) {
        logger.error('Port-forward services failed to become ready', { error: err.message });
        shellExec(`node bin run kill '${ports}'`);
        throw err;
      }
      shellExec(`node bin metadata --generate ${path}`);
      shellExec(`node bin db --dev --clean-fs-collection dd`);
      shellExec(`node bin run kill '${ports}'`);
    },

    /**
     * @method svc-ls
     * @description Lists systemd services and installed packages, optionally filtering by the provided path.
     * @param {string} path - The input value, identifier, or path for the operation (used as the optional filter for services and packages).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'svc-ls': (path, options = DEFAULT_OPTION) => {
      const log = shellExec(`systemctl list-units --type=service${path ? ` | grep ${path}` : ''}`, {
        silent: true,
        stdout: true,
      });
      console.log(path ? log.replaceAll(path, path.red) : log);
      const log0 = shellExec(`sudo dnf list installed${path ? ` | grep ${path}` : ''}`, {
        silent: true,
        stdout: true,
      });
      console.log(path ? log0.replaceAll(path, path.red) : log0);
    },

    /**
     * @method svc-rm
     * @description Removes a systemd service by stopping it, disabling it, uninstalling the package, and deleting related files.
     * @param {string} path - The input value, identifier, or path for the operation (used as the service name).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'svc-rm': (path, options = DEFAULT_OPTION) => {
      shellExec(`sudo systemctl stop ${path}`);
      shellExec(`sudo systemctl disable --now ${path}`);
      shellExec(`sudo dnf remove -y ${path}*`);
      shellExec(`sudo rm -f /usr/lib/systemd/system/${path}.service`);
      shellExec(`sudo rm -f /etc/yum.repos.d/${path}*.repo`);
    },

    /**
     * @method ssh-deploy-info
     * @description Retrieves deployment status and pod information from a remote server via SSH.
     * @param {string} path - The input value, identifier, or path for the operation.
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'ssh-deploy-info': async (path = '', options = DEFAULT_OPTION) => {
      const env = options.dev ? 'development' : 'production';
      await Underpost.ssh.sshRemoteRunner(
        `node bin deploy ${path ? path : 'dd'} ${env} --status && kubectl get pods -A`,
        {
          deployId: options.deployId,
          user: options.user,
          dev: options.dev,
          remote: true,
          useSudo: true,
          cd: '/home/dd/engine',
        },
      );
    },

    /**
     * @method node-move
     * @description Abstract runner that relocates any schedulable Kubernetes workload
     * (Deployment, StatefulSet, DaemonSet, ReplicaSet, Job, CronJob, ReplicationController)
     * onto a target node by patching its pod-template `nodeSelector` and rolling it out.
     * Resource-kind agnostic: it resolves the kind dynamically and applies the right
     * patch path, so it works for `sts`, `deployment`, etc. without bespoke logic.
     *
     * Selection grammar via `path`:
     *   - `<kind>/<name>`  -> a single resource (e.g. `deployment/dd-core-production-blue`)
     *   - `<kind>`         -> every resource of that kind in the namespace (e.g. `statefulset`)
     *   - ``               -> all movable workloads (deployment, statefulset, daemonset) in the namespace
     *
     * Placement:
     *   - default: built-in `kubernetes.io/hostname=<node>` (no node mutation required)
     *   - `--labels k=v,...`: label the target node with those pairs and use them as the
     *     nodeSelector (matches the "label node + nodeSelector" pattern), enabling reusable
     *     workload pools instead of pinning to a single hostname.
     *
     * Flags: `--node-name <node>` (target), `--namespace <ns>`, `--dry-run` (preview only),
     *        `--remove` (clear the nodeSelector / unpin placement).
     *
     * Caveats: Services/ConfigMaps and bare Pods are not schedulable controllers and are
     * skipped (move the owning controller). StatefulSets bound to node-local PVs may stay
     * Pending after a move until their volume is available on the target node.
     * @param {string} path - Resource selector (`kind/name`, `kind`, or empty).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     * @returns {Array<{ref:string,kind:string,status:string,node?:string}>} Per-resource outcome.
     */
    'node-move': (path = '', options = DEFAULT_OPTION) => {
      const node = options.nodeName;
      const ns = options.namespace || 'default';
      const dryRun = options.dryRun === true;
      const remove = options.remove === true;

      if (!remove && !node) {
        throw new Error('node-move requires --node-name <target-node> (or --remove to clear placement)');
      }

      const normalizeKind = (k) =>
        ({
          deploy: 'deployment',
          deployments: 'deployment',
          deployment: 'deployment',
          sts: 'statefulset',
          statefulsets: 'statefulset',
          statefulset: 'statefulset',
          ds: 'daemonset',
          daemonsets: 'daemonset',
          daemonset: 'daemonset',
          rs: 'replicaset',
          replicasets: 'replicaset',
          replicaset: 'replicaset',
          rc: 'replicationcontroller',
          replicationcontroller: 'replicationcontroller',
          job: 'job',
          jobs: 'job',
          cj: 'cronjob',
          cronjob: 'cronjob',
          cronjobs: 'cronjob',
          po: 'pod',
          pod: 'pod',
          pods: 'pod',
          svc: 'service',
          service: 'service',
          services: 'service',
        })[k] || k;

      // Kinds that own a pod template we can patch; rolloutKinds additionally
      // support `kubectl rollout restart` to reschedule existing pods now.
      const templated = [
        'deployment',
        'statefulset',
        'daemonset',
        'replicaset',
        'job',
        'cronjob',
        'replicationcontroller',
      ];
      const rolloutKinds = ['deployment', 'statefulset', 'daemonset'];
      const templateSelectorPath = (kind) =>
        kind === 'cronjob'
          ? ['spec', 'jobTemplate', 'spec', 'template', 'spec', 'nodeSelector']
          : ['spec', 'template', 'spec', 'nodeSelector'];

      // Resolve the desired nodeSelector. Custom --labels enables reusable pools;
      // otherwise pin by the always-present hostname label.
      let selector = { 'kubernetes.io/hostname': node };
      if (!remove && options.labels) {
        selector = {};
        for (const pair of `${options.labels}`
          .split(',')
          .map((s) => s.trim())
          .filter(Boolean)) {
          const eq = pair.indexOf('=');
          if (eq < 0) continue;
          selector[pair.slice(0, eq).trim()] = pair.slice(eq + 1).trim();
        }
      }

      // Verify the target node exists, and apply custom labels to it if provided.
      if (!remove) {
        const found = shellExec(`kubectl get node ${node} -o name`, {
          silent: true,
          stdout: true,
          silentOnError: true,
        }).trim();
        if (!found) throw new Error(`Target node not found: ${node}`);
        if (options.labels) {
          const labelArgs = Object.entries(selector)
            .map(([k, v]) => `${k}=${v}`)
            .join(' ');
          const labelCmd = `kubectl label node ${node} ${labelArgs} --overwrite`;
          if (dryRun) logger.info(`[dry-run] ${labelCmd}`);
          else shellExec(labelCmd);
        }
      }

      const kubectlNames = (kind) =>
        (
          shellExec(`kubectl get ${kind} -n ${ns} -o name`, {
            silent: true,
            stdout: true,
            silentOnError: true,
          }).trim() || ''
        )
          .split('\n')
          .map((s) => s.trim())
          .filter(Boolean);

      // Build the list of "kind/name" targets from the selection grammar.
      let targets = [];
      if (!path) {
        for (const kind of ['deployment', 'statefulset', 'daemonset']) targets.push(...kubectlNames(kind));
      } else if (path.includes('/')) {
        targets = [path];
      } else {
        targets = kubectlNames(path);
      }

      if (targets.length === 0) {
        logger.warn('node-move: no matching resources found', { path, namespace: ns });
        return [];
      }

      // Merge-patch body that sets (or clears, when --remove) the nodeSelector.
      const buildPatch = (kind) => {
        const keys = templateSelectorPath(kind);
        let obj = remove ? null : selector;
        for (let i = keys.length - 1; i >= 0; i--) obj = { [keys[i]]: obj };
        return JSON.stringify(obj);
      };

      const results = [];
      for (const ref of targets) {
        const slash = ref.indexOf('/');
        const rawKind = slash >= 0 ? ref.slice(0, slash) : path && !path.includes('/') ? path : '';
        const name = slash >= 0 ? ref.slice(slash + 1) : ref;
        const kind = normalizeKind(`${rawKind}`.split('.')[0].toLowerCase());

        if (!templated.includes(kind)) {
          logger.warn(`node-move: ${kind}/${name} is not a schedulable controller; skipping`, {
            hint: 'move its owning controller (deployment/statefulset/daemonset) instead',
          });
          results.push({ ref, kind, status: 'skipped' });
          continue;
        }

        // Idempotency: skip the patch + rollout if the resource is already where
        // we want it. Compares the live pod-template nodeSelector against the
        // desired placement so a repeated run does not trigger an unnecessary
        // rollout restart.
        const basePath = kind === 'cronjob' ? 'spec.jobTemplate.spec.template.spec' : 'spec.template.spec';
        const jsonpath = (expr) =>
          shellExec(`kubectl get ${kind} ${name} -n ${ns} -o jsonpath='${expr}'`, {
            silent: true,
            stdout: true,
            silentOnError: true,
            disableLog: true,
          }).trim();

        if (remove) {
          const current = jsonpath(`{.${basePath}.nodeSelector}`);
          if (!current || current === 'map[]') {
            logger.info(`node-move: ${kind}/${name} already has no nodeSelector; nothing to clear`);
            results.push({ ref, kind, status: 'already-cleared' });
            continue;
          }
        } else {
          const alreadyOnNode = Object.entries(selector).every(([k, v]) => {
            const esc = k.replace(/\./g, '\\.');
            return jsonpath(`{.${basePath}.nodeSelector.${esc}}`) === v;
          });
          if (alreadyOnNode) {
            logger.info(`node-move: ${kind}/${name} already pinned to ${node}; skipping`, { namespace: ns });
            results.push({ ref, kind, status: 'already-on-node', node });
            continue;
          }
        }

        const patchCmd = `kubectl patch ${kind} ${name} -n ${ns} --type=merge -p '${buildPatch(kind)}'`;
        const restartCmd = `kubectl rollout restart ${kind} ${name} -n ${ns}`;
        if (dryRun) {
          logger.info(`[dry-run] ${patchCmd}`);
          if (rolloutKinds.includes(kind)) logger.info(`[dry-run] ${restartCmd}`);
          results.push({ ref, kind, status: 'dry-run', node: remove ? undefined : node });
          continue;
        }

        shellExec(patchCmd);
        if (rolloutKinds.includes(kind)) shellExec(restartCmd);
        logger.info(remove ? `Cleared node placement: ${kind}/${name}` : `Moved ${kind}/${name} -> ${node}`, {
          namespace: ns,
        });
        results.push({ ref, kind, status: remove ? 'cleared' : 'moved', node: remove ? undefined : node });
      }

      logger.info('node-move complete', { namespace: ns, node: remove ? null : node, count: results.length });
      return results;
    },

    /**
     * @method dev-hosts-expose
     * @description Deploys a specified service in development mode with `/etc/hosts` modification for local access.
     * @param {string} path - The input value, identifier, or path for the operation (used as the deployment ID to deploy).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'dev-hosts-expose': (path, options = DEFAULT_OPTION) => {
      shellExec(
        `node bin deploy ${path} development --disable-update-deployment --disable-update-proxy --kubeadm --etc-hosts`,
      );
    },

    /**
     * @method dev-hosts-restore
     * @description Restores the `/etc/hosts` file to its original state after modifications made during development deployments.
     * @param {string} path - The input value, identifier, or path for the operation.
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'dev-hosts-restore': (path, options = DEFAULT_OPTION) => {
      shellExec(`node bin deploy --restore-hosts`);
    },

    /**
     * @method cluster-build
     * @description Build configuration for cluster deployment.
     * @param {string} path - The input value, identifier, or path for the operation.
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'cluster-build': (path, options = DEFAULT_OPTION) => {
      const nodeOptions =
        (options.nodeName ? ` --node-name ${options.nodeName}` : '') +
        (options.sshKeyPath ? ` --ssh-key-path ${options.sshKeyPath}` : '');
      shellExec(`node bin run clean`);
      shellExec(`node bin run --dev sync-replica template-deploy${nodeOptions}`);
      shellExec(`node bin run sync-replica template-deploy${nodeOptions}`);
      shellExec(`node bin env clean`);
      for (const deployId of fs.readFileSync('./engine-private/deploy/dd.router', 'utf8').split(','))
        shellExec(`node bin new --default-conf --deploy-id ${deployId.trim()}`);
      if (path === 'cmt') {
        shellExec(`git add . && underpost cmt . build cluster-build`);
        shellExec(`cd engine-private && git add . && underpost cmt . build cluster-build`);
      }
    },
    /**
     * @method template-deploy
     * @description Pushes `engine-private`, dispatches CI workflow to build `pwa-microservices-template`,
     * and optionally triggers engine-<conf-id> CI with sync/init which in turn dispatches the CD workflow
     * after the build chain completes (template → ghpkg → engine-<conf-id> → CD).
     * @param {string} path - The deployment path identifier (e.g., 'sync-engine-core', 'init-engine-core', or empty for build-only).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'template-deploy': (path = '', options = DEFAULT_OPTION) => {
      const baseCommand = options.dev ? 'node bin' : 'underpost';
      shellExec(`npm run security:secrets`);
      const reportPath = './gitleaks-report.json';
      if (fs.existsSync(reportPath) && JSON.parse(fs.readFileSync(reportPath, 'utf8')).length > 0) {
        logger.error('Secrets detected in gitleaks-report.json, aborting template-deploy');
        return;
      }
      shellExec(`${baseCommand} run pull`);
      shellExec(`${baseCommand} run shared-dir`);

      // Capture the sanitized message from the last N commits (--from-n-commit, default 1) for
      // propagation to pwa-microservices-template and every engine-* repo.
      const fromN = parseInt(options.fromNCommit) > 0 ? parseInt(options.fromNCommit) : 1;
      const sanitizedMessage = shellExec(`node bin cmt --changelog-msg --from-n-commit ${fromN} --changelog-no-hash`, {
        silent: true,
        stdout: true,
      }).trim();

      shellExec(
        `${baseCommand} push ./engine-private ${options.force ? '-f ' : ''}${
          process.env.GITHUB_USERNAME
        }/engine-private`,
      );
      shellCd('/home/dd/engine');

      // Push engine repo so workflow YAML changes reach GitHub
      shellExec(`git reset`);
      shellExec(`${baseCommand} push . ${options.force ? '-f ' : ''}${process.env.GITHUB_USERNAME}/engine`);

      // Determine deploy conf and type from path (sync-engine-core, init-engine-core, etc.)
      let deployConfId = '';
      let deployType = '';
      if (path.startsWith('sync-')) {
        deployConfId = path.replace(/^sync-/, '');
        deployType = 'sync-and-deploy';
      } else if (path.startsWith('init-')) {
        deployConfId = path.replace(/^init-/, '');
        deployType = 'init';
      }

      // If --build is set and path is a sync-engine-* target, push the pre-built client bundle
      // to Cloudinary so the remote container can pull it instead of rebuilding from source.
      if (options.build && deployConfId && deployConfId.startsWith('engine-')) {
        const confName = deployConfId.replace(/^engine-/, '');
        const pushDeployId = options.deployId || `dd-${confName}`;
        logger.info(`[template-deploy] Running push-bundle for deployId: ${pushDeployId}`);
        shellExec(`${baseCommand} run push-bundle --deploy-id ${pushDeployId}`);
      }

      // Dispatch npmpkg CI workflow — this builds pwa-microservices-template first.
      // If deployConfId is set, npmpkg.ci.yml will dispatch the engine-<conf-id> CI
      // with sync=true after template build completes. The engine CI then dispatches
      // the CD workflow after the engine repo build finishes — ensuring correct sequence:
      // npmpkg.ci → engine-<id>.ci → engine-<id>.cd
      const repo = `${process.env.GITHUB_USERNAME}/engine`;
      const inputs = {};
      if (sanitizedMessage) inputs.message = sanitizedMessage;
      if (deployConfId) inputs.deploy_conf_id = deployConfId;
      if (deployType) inputs.deploy_type = deployType;

      Underpost.repo.dispatchWorkflow({
        repo,
        workflowFile: 'npmpkg.ci.yml',
        ref: 'master',
        inputs,
      });
    },

    /**
     * @method template-deploy-local
     * @description Similar to `template-deploy` but runs the workflow locally without dispatching GitHub Actions. It pulls the latest changes, pushes to GitHub, builds the template, and optionally triggers a local release with CI push.
     * @param {string} path - The deployment path identifier (e.g., 'sync-engine-core', 'init-engine-core', or empty for build-only).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'template-deploy-local': async (path, options = DEFAULT_OPTION) => {
      const baseCommand = options.dev ? 'node bin' : 'underpost';
      shellExec(`npm run security:secrets`);
      const reportPath = './gitleaks-report.json';
      if (fs.existsSync(reportPath) && JSON.parse(fs.readFileSync(reportPath, 'utf8')).length > 0) {
        logger.error('Secrets detected in gitleaks-report.json, aborting template-deploy');
        return;
      }
      shellExec(`${baseCommand} run pull`);
      shellExec(`${baseCommand} run shared-dir`);

      // Capture the sanitized message from the last N commits (--from-n-commit, default 1).
      const fromN = parseInt(options.fromNCommit) > 0 ? parseInt(options.fromNCommit) : 1;
      const sanitizedMessage = shellExec(`node bin cmt --changelog-msg --from-n-commit ${fromN} --changelog-no-hash`, {
        silent: true,
        stdout: true,
      }).trim();

      const { triggerCmd } = path
        ? await Underpost.release.ci(path, sanitizedMessage, options)
        : await Underpost.release.pwa(sanitizedMessage, options);
      pbcopy(triggerCmd + ' && cd /home/dd/engine');
    },
    /**
     * @method docker-image
     * @description Dispatches the Docker image CI workflow (`docker-image[.<runtime>].ci.yml`) via `workflow_dispatch`.
     * Repository resolution is delegated to `Underpost.repo.resolveInstanceRepo(path)`.
     * @param {string} path - Optional runtime / workflow suffix (e.g. `cyberia-server`, `cyberia-client`).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'docker-image': (path, options = DEFAULT_OPTION) => {
      const repo = Underpost.repo.resolveInstanceRepo(path, options.dev);
      Underpost.repo.dispatchWorkflow({
        repo,
        workflowFile: `docker-image${path ? `.${path}` : ''}${options.dev ? '.dev' : ''}.ci.yml`,
        inputs: {},
      });
    },
    /**
     * @method clean
     * @description Changes directory to the provided path (defaulting to `/home/dd/engine`) and runs `node bin/deploy clean-core-repo`.
     * @param {string} path - The input value, identifier, or path for the operation (used as the optional directory path).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    clean: (path = '', options = DEFAULT_OPTION) => {
      Underpost.repo.clean({ paths: path ? path.split(',') : ['/home/dd/engine', '/home/dd/engine/engine-private'] });
      if (options.dev) shellExec(`node bin run shared-dir ${path ? path : '/home/dd/engine'}`);
    },
    /**
     * @method pull
     * @description Clones or pulls updates for the `engine` and `engine-private` repositories into `/home/dd/engine` and `/home/dd/engine/engine-private`.
     * @param {string} path - The input value, identifier, or path for the operation.
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    pull: (path, options = DEFAULT_OPTION) => {
      // shellExec is fail-fast by default — any non-zero exit throws and
      // propagates up to the workflow step. No per-call flag required.
      if (!fs.existsSync(`/home/dd`) || !fs.existsSync(`/home/dd/engine`)) {
        fs.mkdirSync(`/home/dd`, { recursive: true });
        shellExec(`cd /home/dd && underpost clone ${process.env.GITHUB_USERNAME}/engine`, { silent: true });
      } else {
        shellExec(`underpost run clean`);
        shellExec(`cd /home/dd/engine && underpost pull . ${process.env.GITHUB_USERNAME}/engine`, { silent: true });
      }
      if (!fs.existsSync(`/home/dd/engine/engine-private`))
        shellExec(`cd /home/dd/engine && underpost clone ${process.env.GITHUB_USERNAME}/engine-private`, {
          silent: true,
        });
      else
        shellExec(
          `cd /home/dd/engine/engine-private && underpost pull . ${process.env.GITHUB_USERNAME}/engine-private`,
          { silent: true },
        );
    },
    /**
     * @method release-deploy
     * @description Executes deployment (`underpost run deploy`) for all deployment IDs listed in `./engine-private/deploy/dd.router`.
     * @param {string} path - The input value, identifier, or path for the operation.
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'release-deploy': (path, options = DEFAULT_OPTION) => {
      actionInitLog();
      shellExec(`underpost --version`);
      shellCd(`/home/dd/engine`);
      for (const _deployId of fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8').split(',')) {
        const deployId = _deployId.trim();
        shellExec(`underpost run deploy ${deployId}`, { async: true });
      }
    },
    /**
     * @method ssh-deploy
     * @description Dispatches the corresponding CD workflow for SSH-based deployment, replacing empty commits with workflow_dispatch.
     * @param {string} path - The deployment identifier (e.g., 'engine-core', 'sync-engine-core', 'init-engine-core').
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'ssh-deploy': (path, options = DEFAULT_OPTION) => {
      actionInitLog();

      let job = 'deploy';
      let confId = path;
      if (path.startsWith('sync-')) {
        job = 'sync-and-deploy';
        confId = path.replace(/^sync-/, '');
      } else if (path.startsWith('init-')) {
        job = 'init';
        confId = path.replace(/^init-/, '');
      }
      const repo = Underpost.repo.resolveInstanceRepo(confId, options.dev);
      Underpost.repo.dispatchWorkflow({
        repo,
        workflowFile: `${confId}.cd.yml`,
        ref: 'master',
        inputs: { job },
      });
    },
    /**
     * @method ide
     * @description Opens a Visual Studio Code (VS Code) session for the specified path using `node ${underpostRoot}/bin/zed ${path}`,
     * or installs Zed and sublime-text IDE if `path` is 'install'.
     * @param {string} path - The input value, identifier, or path for the operation (used as the path to the directory to open in the IDE).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    ide: (path = '', options = DEFAULT_OPTION) => {
      const underpostRoot = options.dev ? '.' : options.underpostRoot;
      const [projectPath, customIde] = path.split(',');
      if (projectPath === 'install') {
        if (customIde === 'zed') shellExec(`sudo curl -f https://zed.dev/install.sh | sh`);
        else if (customIde === 'subl') {
          shellExec(
            `sudo dnf config-manager --add-repo https://download.sublimetext.com/rpm/stable/x86_64/sublime-text.repo`,
          );
          shellExec(`sudo dnf install -y sublime-text`);
        } else {
          shellExec(`sudo rpm --import https://packages.microsoft.com/keys/microsoft.asc &&
echo -e "[code]\nname=Visual Studio Code\nbaseurl=https://packages.microsoft.com/yumrepos/vscode\nenabled=1\nautorefresh=1\ntype=rpm-md\ngpgcheck=1\ngpgkey=https://packages.microsoft.com/keys/microsoft.asc" | sudo tee /etc/yum.repos.d/vscode.repo > /dev/null`);
          shellExec(`sudo dnf install -y code`);
        }
        return;
      }
      if (customIde === 'zed') shellExec(`node ${underpostRoot}/bin/zed ${projectPath}`);
      else shellExec(`node ${underpostRoot}/bin/vs ${projectPath}`);
    },
    /**
     * @method crypto-policy
     * @description Sets the system's crypto policies to `DEFAULT:SHA1` using `update-crypto-policies` command.
     * @param {string} path - The input value, identifier, or path for the operation.
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'crypto-policy': (path, options = DEFAULT_OPTION) => {
      shellExec(`sudo update-crypto-policies --set DEFAULT:SHA1`);
    },
    /**
     * @method sync
     * @description Cleans up, and then runs a deployment synchronization command (`underpost deploy --kubeadm --build-manifest --sync...`) using parameters parsed from `path` (deployId, replicas, versions, image, node).
     *
     * Forwards `--image-pull-policy <policy>` to the underlying `deploy --build-manifest` invocation when `options.imagePullPolicy` is set,
     * which then plumbs through `buildManifest` and `deploymentYamlPartsFactory` to override the container `imagePullPolicy` in the generated
     * `deployment.yaml`. Useful when you want to force `Always` so the kubelet re-pulls a mutable tag on every rollout. Example:
     *   `node bin run sync dd-core --kubeadm --image-pull-policy Always`
     * @param {string} path - The input value, identifier, or path for the operation (used as a comma-separated string containing deploy parameters).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    sync: async (path, options = DEFAULT_OPTION) => {
      // Dev usage: node bin run --dev --build sync dd-default
      const env = options.dev ? 'development' : 'production';
      const baseCommand = 'node bin'; // options.dev ? 'node bin' : 'underpost';
      const baseClusterCommand = options.dev ? ' --dev' : '';
      const clusterFlag = options.k3s ? ' --k3s' : options.kind ? ' --kind' : ' --kubeadm';
      const defaultPath = [
        'dd-default',
        options.replicas,
        ``,
        ``,
        !options.kubeadm && !options.k3s ? 'kind-control-plane' : os.hostname(),
      ];
      let [deployId, replicas, versions, image, node] = path ? path.split(',') : defaultPath;
      deployId = deployId ? deployId : defaultPath[0];
      replicas = replicas ? replicas : defaultPath[1];
      versions = versions ? versions.replaceAll('+', ',') : defaultPath[2];
      image = image ? image : defaultPath[3];
      node = node ? node : options.nodeName ? options.nodeName : defaultPath[4];
      shellExec(`${baseCommand} cluster --ns-use ${options.namespace}`);

      if (image && !image.startsWith('localhost'))
        Underpost.image.pullDockerHubImage({
          dockerhubImage: image,
          kind: options.kind || (!options.nodeName && !options.kubeadm && !options.k3s),
          kubeadm: options.nodeName || options.kubeadm,
          k3s: options.k3s,
        });

      if (isDeployRunnerContext(path, options)) {
        if (!options.disablePrivateConfUpdate) {
          const { validVersion } = Underpost.repo.privateConfUpdate(deployId);
          if (!validVersion) throw new Error('Version mismatch');
        }
        if (options.timezone !== 'none') shellExec(`${baseCommand} run${baseClusterCommand} tz`);
        if (options.deployIdCronJobs !== 'none')
          shellExec(`node bin cron${baseClusterCommand}${clusterFlag} --setup-start --git --apply`);
      }

      const currentTraffic = isDeployRunnerContext(path, options)
        ? Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace })
        : '';
      let targetTraffic = currentTraffic ? (currentTraffic === 'blue' ? 'green' : 'blue') : 'green';
      if (targetTraffic) versions = versions ? versions : targetTraffic;

      const ignorePods =
        isDeployRunnerContext(path, options) && targetTraffic
          ? Underpost.kubectl.get(`${deployId}-${env}-${targetTraffic}`, 'pods', options.namespace).map((p) => p.NAME)
          : [];

      const timeoutFlags = Underpost.deploy.timeoutFlagsFactory(options);
      const cmdString = options.cmd
        ? ' --cmd ' + (options.cmd.find((c) => c.match('"')) ? '"' + options.cmd + '"' : "'" + options.cmd + "'")
        : '';
      const gitCleanFlag = options.gitClean ? ' --git-clean' : '';

      const skipFullBuildFlag = options.skipFullBuild ? ' --skip-full-build' : '';
      const pullBundleFlag = options.pullBundle ? ' --pull-bundle' : '';
      const imagePullPolicyFlag = options.imagePullPolicy ? ` --image-pull-policy ${options.imagePullPolicy}` : '';
      const sshKeyPathFlag = options.sshKeyPath ? ` --ssh-key-path ${options.sshKeyPath}` : '';

      shellExec(
        `${baseCommand} deploy${clusterFlag} --build-manifest --sync --info-router --replicas ${replicas} --node ${node}${
          image ? ` --image ${image}` : ''
        }${versions ? ` --versions ${versions}` : ''}${
          options.namespace ? ` --namespace ${options.namespace}` : ''
        }${timeoutFlags}${cmdString}${gitCleanFlag}${skipFullBuildFlag}${pullBundleFlag}${imagePullPolicyFlag}${sshKeyPathFlag} ${deployId} ${env}`,
      );

      if (isDeployRunnerContext(path, options)) {
        // Backup app/services repositories with repo-backup configured
        shellExec(
          `${baseCommand} db ${deployId} ${clusterFlag}${baseClusterCommand} --repo-backup --primary-pod --git --force-clone --preserveUUID ${options.namespace ? ` --ns ${options.namespace}` : ''}`,
        );
        shellExec(
          `${baseCommand} deploy${clusterFlag}${cmdString} --replicas ${replicas} --node ${node} --disable-update-proxy ${deployId} ${env} --versions ${versions}${
            options.namespace ? ` --namespace ${options.namespace}` : ''
          }${timeoutFlags}${gitCleanFlag}${imagePullPolicyFlag}${sshKeyPathFlag}`,
        );
        if (!targetTraffic)
          targetTraffic = Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace });
        await Underpost.monitor.monitorReadyRunner(deployId, env, targetTraffic, ignorePods, options.namespace);
        Underpost.deploy.switchTraffic(deployId, env, targetTraffic, replicas, options.namespace, options);
      } else
        logger.info('current traffic', Underpost.deploy.getCurrentTraffic(deployId, { namespace: options.namespace }));
    },

    /**
     * @method stop
     * @description Stops a deployment by deleting the corresponding Kubernetes deployment and service resources.
     * @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    stop: async (path = '', options = DEFAULT_OPTION) => {
      let currentTraffic = Underpost.deploy.getCurrentTraffic(options.deployId, {
        namespace: options.namespace,
        hostTest: options.hosts,
      });
      const env = options.dev ? 'development' : 'production';

      if (!path.match('current')) currentTraffic === 'blue' ? (currentTraffic = 'green') : (currentTraffic = 'blue');
      const [_deployId] = path.split(',');
      const deploymentId = `${_deployId ? _deployId : options.deployId}${
        options.instanceId ? `-${options.instanceId}` : ''
      }-${env}-${currentTraffic}`;

      shellExec(`kubectl delete deployment ${deploymentId} -n ${options.namespace}`);
      shellExec(`kubectl delete svc ${deploymentId}-service -n ${options.namespace}`);
    },

    /**
     * @method ssh-deploy-stop
     * @description Stops a remote deployment via SSH by executing the appropriate Underpost command on the remote server.
     * @param {string} path - The input value, identifier, or path for the operation (used to determine which traffic to stop).
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @memberof UnderpostRun
     */
    'ssh-deploy-stop': async (path, options = DEFAULT_OPTION) => {
      const baseCommand = options.dev ? 'node bin' : 'underpost';
      const baseClusterCommand = options.dev ? ' --dev' : '';

      const remoteCommand = [
        `${baseCommand} run${baseClusterCommand} stop${path ? ` ${path}` : ''}`,
        ` --deploy-id ${options.deployId}${options.instanceId ? ` --instance-id ${options.instanceId}` : ''}`,
        ` --namespace ${options.namespace}${options.hosts ? ` --hosts ${options.hosts}` : ''}`,
      ].join('');

      await Underpost.ssh.sshRemoteRunner(remoteCommand, {
        deployId: options.deployId,
        user: options.user,
        dev: options.dev,
        remote: true,
        useSudo: true,
        cd: '/home/dd/engine',
      });
    },

    /**
     * @method ssh-deploy-db-rollback
     * @description Performs a database rollback on remote deployment via SSH.
     * @param {string} path - Comma-separated deployId and optional number of commits to reset (format: "deployId,nCommits")
     * @param {UnderpostRunDefaultOptions} options - The default underpost runner options for customizing workflow
     * @param {string} options.deployId - The deployment identifier
     * @param {string} options.user - The SSH user for credential lookup
     * @param {boolean} options.dev - Development mode flag
     * @memberof UnderpostRun
     */
    'ssh-deploy-db-rollback': async (path = '', options = DEFAULT_OPTION) => {
      const baseCommand = options.dev ? 'node bin' : 'underpost';
      let [deployId, nCommitsReset] = path.split(',');
      if (!nCommitsReset) nCommitsReset = 1;

      const remoteCommand = `${baseCommand} db ${deployId} --git --kubeadm --primary-pod --force-clone --macro-rollback-export ${nCommitsReset}${options.namespace ? ` --ns ${options.namespace}` : ''}`;

      await Underpost.ssh.sshRemoteRunner(remoteCommand, {
        deployId: options.deployId,
        user: options.user,
        dev: options.dev,
        remote: true,
        useSudo: true,
        cd: '/home/dd/engine',
      });
    },

    /**
     * @method ssh-