@underpostnet/underpost/src/cli/fs.js

Underpost Platform — end-to-end CI/CD and application-delivery toolchain CLI. Covers bare metal, Kubernetes, K3s, kubeadm, LXD, container/image orchestration, secrets, databases, cron jobs, monitoring, SSH, runners, PWA + Workbox delivery, and release orc

/**
 * File storage module for managing file operations using Cloudinary.
 * @module src/cli/fs.js
 * @namespace UnderpostFileStorage
 */

import { v2 as cloudinary } from 'cloudinary';
import { loggerFactory } from '../server/logger.js';
import AdmZip from 'adm-zip';
import * as dir from 'path';
import fs from 'fs-extra';
import Downloader from '../server/downloader.js';
import { shellExec } from '../server/process.js';
import Underpost from '../index.js';

const logger = loggerFactory(import.meta);

/**
 * @class UnderpostFileStorage
 * @description Manages file storage operations using Cloudinary.
 * This class provides a set of static methods to upload, pull, and delete files
 * from Cloudinary, as well as manage a local storage configuration file.
 */
class UnderpostFileStorage {
  static API = {
    /**
     * @method cloudinaryConfig
     * @description Configures the Cloudinary client with environment variables.
     * @memberof UnderpostFileStorage
     */
    cloudinaryConfig() {
      // https://console.cloudinary.com/
      cloudinary.config({
        cloud_name: process.env.CLOUDINARY_CLOUD_NAME,
        api_key: process.env.CLOUDINARY_API_KEY,
        api_secret: process.env.CLOUDINARY_API_SECRET,
      });
    },
    /**
     * @method getStorageConf
     * @description Retrieves the storage configuration for a specific deployment.
     * @param {object} options - An object containing deployment-specific options.
     * @param {string} options.deployId - The identifier for the deployment.
     * @param {string} [options.storageFilePath] - The path to the storage configuration file.
     * @returns {object} An object containing the storage configuration and storage file path.
     * @memberof UnderpostFileStorage
     */
    getStorageConf(options) {
      let storage, storageConf;
      if (options.deployId && typeof options.deployId === 'string') {
        storageConf = options.storageFilePath ?? `./engine-private/conf/${options.deployId}/storage.json`;
        if (!fs.existsSync(storageConf)) fs.writeFileSync(storageConf, JSON.stringify({}), 'utf8');
        storage = JSON.parse(fs.readFileSync(storageConf, 'utf8'));
      }
      return { storage, storageConf };
    },
    /**
     * @method writeStorageConf
     * @description Writes the storage configuration to a file.
     * @param {object} storage - The storage configuration object.
     * @param {string} storageConf - The path to the storage configuration file.
     * @memberof UnderpostFileStorage
     */
    writeStorageConf(storage, storageConf) {
      if (storage) fs.writeFileSync(storageConf, JSON.stringify(storage, null, 4), 'utf8');
    },
    /**
     * @method gitTrack
     * @description Optional, non-fatal Git tracking layer. Any Git error is logged and swallowed
     * so it can never interrupt or roll back the canonical `storage.*.json` workflow.
     * @param {string} gitPath - The working directory to stage/commit.
     * @param {object} [options] - Tracking options.
     * @param {boolean} [options.init=false] - If true, initialize a local repo before staging.
     * @param {string} [options.message=''] - Explicit commit message; when omitted, `underpost cmt` is used.
     * @memberof UnderpostFileStorage
     */
    gitTrack(gitPath, options = { init: false, message: '' }) {
      try {
        if (options.init === true) Underpost.repo.initLocalRepo({ path: gitPath });
        shellExec(`cd ${gitPath} && git add .`, { silentOnError: true, silent: true, disableLog: true });
        if (options.message)
          shellExec(`cd ${gitPath} && git commit -m "${options.message}"`, {
            silentOnError: true,
            silent: true,
            disableLog: true,
          });
        else shellExec(`underpost cmt ${gitPath} feat`, { silentOnError: true, silent: true, disableLog: true });
      } catch (error) {
        logger.warn('git tracking skipped (non-fatal)', { gitPath, error: error?.message });
      }
    },
    /**
     * @method recursiveCallback
     * @description Recursively processes files and directories based on the provided options.
     * @param {string} path - The path to the directory to process.
     * @param {object} [options] - An object containing options for the recursive callback.
     * @param {boolean} [options.rm=false] - Flag to remove files and directories.
     * @param {boolean} [options.recursive=false] - Flag to process directories recursively.
     * @param {string} [options.deployId=''] - The identifier for the deployment.
     * @param {boolean} [options.force=false] - Flag to force file operations.
     * @param {boolean} [options.pull=false] - Flag to pull files from storage.
     * @param {boolean} [options.git=false] - Flag to use Git for file operations.
     * @param {boolean} [options.omitUnzip=false] - If true, do not extract zip and keep downloaded zip file.
     * @param {string} [options.storageFilePath=''] - The path to the storage configuration file.
     * @returns {Promise<void>} A promise that resolves when the recursive callback is complete.
     * @memberof UnderpostFileStorage
     */
    async recursiveCallback(
      path,
      options = {
        rm: false,
        recursive: false,
        deployId: '',
        force: false,
        pull: false,
        git: false,
        omitUnzip: false,
        storageFilePath: '',
      },
    ) {
      const { storage, storageConf } = Underpost.fs.getStorageConf(options);

      // ── Single-file handling: when path is a file (not a directory), use parent dir
      //    as the git working directory and process just that one file. ──────────────
      let isSingleFile = false;
      let parentDir = path;
      let singleFileName = '';
      if (fs.existsSync(path) && !fs.statSync(path).isDirectory()) {
        isSingleFile = true;
        parentDir = dir.dirname(path);
        singleFileName = path.split('/').pop();
      }

      // In recursive remove mode, delete every tracked storage key under the requested path,
      // even when local files/directories are already missing.
      if (options.rm === true) {
        const normalizedPath = typeof path === 'string' ? path.trim() : '';
        const basePath = normalizedPath.replace(/\/+$/, '');
        const hasPathFilter = basePath.length > 0;

        const associatedPaths = Object.keys(storage || {}).filter((storedPath) => {
          if (!hasPathFilter) return true;
          return storedPath === basePath || storedPath.startsWith(`${basePath}/`);
        });

        for (const associatedPath of associatedPaths) {
          await Underpost.fs.delete(associatedPath);
          if (storage) delete storage[associatedPath];
        }

        if (hasPathFilter && options.force === true && fs.existsSync(basePath)) fs.removeSync(basePath);

        // Storage is canonical: persist the removal before any (optional) git tracking runs.
        Underpost.fs.writeStorageConf(storage, storageConf);

        if (associatedPaths.length === 0)
          logger.warn('No associated tracked storage paths found', { path: hasPathFilter ? basePath : '*' });
        else
          logger.info('Removed associated tracked storage paths', {
            path: hasPathFilter ? basePath : '*',
            removed: associatedPaths.length,
          });

        if (options.git === true) {
          const gitPath = !hasPathFilter ? '.' : isSingleFile ? parentDir : basePath;
          Underpost.fs.gitTrack(gitPath);
        }

        return;
      }

      // For single files, run getDeleteFiles against the parent directory to avoid
      // trying to `cd` into a file.
      const gitContextPath = isSingleFile ? parentDir : path;
      // Detecting locally-deleted files is a best-effort enhancement backed by git; if the path is
      // not a repo (or git is unavailable) it must not block the canonical storage workflow.
      let deleteFiles = [];
      if (options.pull !== true) {
        try {
          deleteFiles = Underpost.repo.getDeleteFiles(gitContextPath);
        } catch (error) {
          logger.warn('delete detection skipped (git unavailable)', { path: gitContextPath, error: error?.message });
        }
      }

      // When processing a single file, only consider it for deletion
      for (const relativePath of deleteFiles) {
        const _path = isSingleFile ? (relativePath === singleFileName ? path : null) : path + '/' + relativePath;
        if (_path && _path in storage) {
          await Underpost.fs.delete(_path);
          delete storage[_path];
        }
      }
      if (options.pull === true) {
        let pullSkipCount = 0;
        for (const _path of Object.keys(storage)) {
          if (!fs.existsSync(_path) || options.force === true) {
            if (options.force === true && fs.existsSync(_path)) fs.removeSync(_path);
            await Underpost.fs.pull(_path, options);
          } else pullSkipCount++;
        }
        if (pullSkipCount > 0) logger.warn(`Pull skipped ${pullSkipCount} files that already exist`);
        // Only run git init/commit when the caller explicitly requests git tracking (--git flag).
        // For bundle pulls into ./build the git step is unwanted and would error on a non-repo path.
        if (options.git === true) Underpost.fs.gitTrack(gitContextPath, { init: true, message: 'Base pull state' });
      } else {
        let files;
        if (isSingleFile) {
          // Single file: treat the file itself as the sole item to process
          files = [singleFileName];
        } else {
          files =
            options.git === true
              ? Underpost.repo.getChangedFiles(gitContextPath)
              : await fs.readdir(path, { recursive: true });
        }
        for (const relativePath of files) {
          const _path = isSingleFile ? path : path + '/' + relativePath;
          if (fs.existsSync(_path) && fs.statSync(_path).isDirectory()) {
            if (options.pull === true && !fs.existsSync(_path)) fs.mkdirSync(_path, { recursive: true });
            continue;
          } else if (!(_path in storage) || options.force === true) {
            await Underpost.fs.upload(_path, options);
            if (storage) storage[_path] = {};
          } else logger.warn('File already exists', _path);
        }
      }
      // Storage is canonical and always persisted; git is an optional layer on top.
      Underpost.fs.writeStorageConf(storage, storageConf);
      if (options.git === true) Underpost.fs.gitTrack(gitContextPath);
    },
    /**
     * @method callback
     * @description Orchestrates file storage operations based on the provided options.
     * This method handles file uploads, deletions, and recursive processing of directories.
     * @param {string} path - The path to the file or directory to process.
     * @param {object} [options] - An object containing options for the callback.
     * @param {boolean} [options.rm=false] - Flag to remove files and directories.
     * @param {boolean} [options.recursive=false] - Flag to process directories recursively.
     * @param {string} [options.deployId=''] - The identifier for the deployment.
     * @param {boolean} [options.force=false] - Flag to force file operations.
     * @param {boolean} [options.pull=false] - Flag to pull files from storage.
     * @param {boolean} [options.git=false] - Flag to use Git for file operations.
     * @param {boolean} [options.omitUnzip=false] - If true, do not extract zip and keep downloaded zip file.
     * @returns {Promise<void>} A promise that resolves when the callback is complete.
     * @memberof UnderpostFileStorage
     */
    async callback(
      path,
      options = { rm: false, recursive: false, deployId: '', force: false, pull: false, git: false, omitUnzip: false },
    ) {
      // rm always routes through recursiveCallback so storage.*.json is updated regardless of
      // --recursive/--git. The bare `delete` primitive only removes the remote asset and would
      // otherwise leave the tracked storage key orphaned.
      if (options.recursive === true || options.git === true || options.rm === true)
        return await Underpost.fs.recursiveCallback(path, options);
      if (options.pull === true) return await Underpost.fs.pull(path, options);
      return await Underpost.fs.upload(path, options);
    },
    /**
     * @method upload
     * @description Uploads a file to Cloudinary.
     * @param {string} path - The path to the file to upload.
     * @param {object} [options] - An object containing options for the upload.
     * @param {string} [options.deployId=''] - The identifier for the deployment (used to locate the storage config file).
     * @param {boolean} [options.force=false] - Flag to force file operations (overwrites existing remote asset).
     * @param {string} [options.storageFilePath=''] - The path to the storage configuration file.
     * @returns {Promise<object>} A promise that resolves to the upload result.
     * @memberof UnderpostFileStorage
     */

    async upload(
      path,
      options = { rm: false, recursive: false, deployId: '', force: false, pull: false, storageFilePath: '' },
    ) {
      Underpost.fs.cloudinaryConfig();
      const { storage, storageConf } = Underpost.fs.getStorageConf(options);
      // path = Underpost.fs.file2Zip(path);
      const uploadResult = await cloudinary.uploader
        .upload(path, {
          public_id: path,
          resource_type: 'raw',
          overwrite: options.force === true ? true : false,
        })
        .catch((error) => {
          logger.error(error, { path, stack: error.stack });
        });
      logger.info('upload result', uploadResult);
      if (storage) storage[path] = {};
      Underpost.fs.writeStorageConf(storage, storageConf);
      return uploadResult;
    },
    /**
     * @method pull
     * @description Pulls a file from Cloudinary.
     * @param {string} path - The path to the file to pull.
     * @param {object} [options] - Pull options.
     * @param {boolean} [options.omitUnzip=false] - If true, do not extract zip and keep downloaded zip file.
     * @param {boolean} [options.force=false] - If true, re-download even if the local zip already exists.
     * @returns {Promise<void>} A promise that resolves when the file is pulled.
     * @memberof UnderpostFileStorage
     */
    async pull(path, options = { omitUnzip: false, force: false }) {
      Underpost.fs.cloudinaryConfig();
      const folder = dir.dirname(path);
      if (!fs.existsSync(folder)) fs.mkdirSync(folder, { recursive: true });
      const zipPath = `${path}.zip`;

      if (options.omitUnzip === true && options.force !== true && fs.existsSync(zipPath)) {
        logger.warn('pull skipped, zip already exists and omit-unzip is enabled', { path, zipPath });
        return;
      }

      const downloadResult = await cloudinary.utils.download_archive_url({
        public_ids: [path],
        resource_type: 'raw',
      });
      logger.info('download result', downloadResult);
      await Downloader.downloadFile(downloadResult, zipPath);

      if (options.omitUnzip === true) {
        logger.warn('omit unzip enabled, keeping downloaded zip file', { path, zipPath });
        return;
      }

      path = Underpost.fs.zip2File(zipPath);
      fs.removeSync(`${path}.zip`);
    },
    /**
     * @method delete
     * @description Deletes a file from Cloudinary by its public ID.
     * @param {string} path - The path (public ID) of the file to delete.
     * @returns {Promise<object>} A promise that resolves to the Cloudinary delete result.
     * @memberof UnderpostFileStorage
     */
    async delete(path) {
      Underpost.fs.cloudinaryConfig();
      const deleteResult = await cloudinary.api
        .delete_resources([path], { type: 'upload', resource_type: 'raw' })
        .catch((error) => {
          logger.error(error, { path, stack: error.stack });
        });
      logger.info('delete result', deleteResult);
      return deleteResult;
    },
    /**
     * @method file2Zip
     * @description Converts a file to a zip file.
     * @param {string} path - The path to the file to convert.
     * @returns {string} The path to the zip file.
     * @memberof UnderpostFileStorage
     */
    file2Zip(path) {
      const zip = new AdmZip();
      zip.addLocalFile(path, '/');
      path = path + '.zip';
      zip.writeZip(path);
      return path;
    },
    /**
     * @method zip2File
     * @description Converts a zip file to a file.
     * @param {string} path - The path to the zip file to convert.
     * @returns {string} The path to the file.
     * @memberof UnderpostFileStorage
     */
    zip2File(path) {
      const zip = new AdmZip(path);
      path = path.replaceAll('.zip', '');
      zip.extractEntryTo(
        /*entry name*/ path.split('/').pop(),
        /*target path*/ dir.dirname(path),
        /*maintainEntryPath*/ false,
        /*overwrite*/ true,
      );
      return path;
    },
  };
}

export default UnderpostFileStorage;