@hotglue/cli/lib/commands/etl/download.js

hotglue CLI tools

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.handler = exports.desc = exports.command = exports.builder = void 0;

var _path = _interopRequireDefault(require("path"));

var _promises = require("fs/promises");

var _fs = require("fs");

var _debug = _interopRequireDefault(require("../../helpers/debug"));

var _ora = _interopRequireDefault(require("ora"));

var _axios = _interopRequireDefault(require("axios/dist/node/axios.cjs"));

var _cliTable = _interopRequireDefault(require("cli-table"));

var _awsSdk = _interopRequireDefault(require("aws-sdk"));

var _progress = _interopRequireDefault(require("progress"));

var _descriptions = _interopRequireDefault(require("../../helpers/descriptions"));

var _print = _interopRequireWildcard(require("../../helpers/print"));

var _utils = require("../../helpers/utils");

var _api = require("../../helpers/api");

var _flow = require("../../helpers/flow");

function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function (nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }

function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

const debug = (0, _debug.default)('commands:etl:download');
const command = 'download';
exports.command = command;
const desc = 'Download ETL scripts';
exports.desc = desc;

const builder = async yargs => {
  debug('builder', command);
  return yargs.option('flow', _descriptions.default.options['flow'].config).demandOption('flow', _descriptions.default.options['flow'].demandText).option('tap', _descriptions.default.options['tap'].config).demandOption('tap', _descriptions.default.options['tap'].demandText).option('downloadTo', _descriptions.default.options['downloadTo'].config).demandOption('downloadTo', _descriptions.default.options['downloadTo'].demandText).option('tenant', { ..._descriptions.default.options['tenant'].config,
    default: 'default'
  }).option('overwrite', _descriptions.default.options['overwrite'].config).demandOption('overwrite', _descriptions.default.options['overwrite'].demandText);
};

exports.builder = builder;

const handler = async argv => {
  debug('handler', command, argv);
  const {
    hg,
    json,
    apikey,
    env,
    flow,
    tap,
    tenant,
    downloadTo,
    overwrite
  } = argv;
  const {
    clientApiBaseUri
  } = hg;
  let message;
  let spinner = (0, _ora.default)();

  const folderPrefix = _path.default.resolve(process.cwd(), downloadTo);

  try {
    // 1. get STS credentials
    message = (0, _print.themed)(`Verifying user and authorizing`);
    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
    const {
      accessKeyId,
      secretAccessKey,
      sessionToken
    } = await (0, _api.genCredentialsOnClientApi)({
      debug,
      baseUri: clientApiBaseUri,
      task: 'etl-download',
      env,
      tenant,
      flow,
      tap,
      apikey
    }); // created authenticated S3 instance

    const s3 = new _awsSdk.default.S3({
      accessKeyId,
      secretAccessKey,
      sessionToken
    });
    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary')); // 2. get list of downloadable files

    message = (0, _print.themed)(`Scanning for downloadable files`);
    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
    const keyPrefix = `${tenant}/flows/${flow}/${await (0, _flow.getEntityLabel)(s3, env, tenant, flow)}/${tap}/etl/`;
    const params = {
      Bucket: env,
      Prefix: keyPrefix
    };
    const {
      Contents: s3Files
    } = await s3.listObjectsV2(params).promise();
    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
    debug('s3-list-res', s3Files); // filter unwanted files from source

    const filesToDownload = s3Files ? (0, _utils.filterFiles)(s3Files.map(item => item.Key), {
      pattern: '!(.ipynb_checkpoints/*)'
    }) : [];

    if (!filesToDownload || filesToDownload.length === 0) {
      json ? (0, _print.printJSON)({
        status: 'success',
        downloadedFiles: []
      }) : spinner.warn((0, _print.themed)(`Warning: ${(0, _print.themed)('Nothing to download!')}`, 'secondary'));
      return;
    } // 3. download files


    message = (0, _print.themed)(`Downloading script files to ${(0, _print.themed)(folderPrefix, 'info')}`);
    !json && spinner.info((0, _print.themed)(`Info: ${message}.`, 'secondary'));
    const table = new _cliTable.default({
      head: ['File', 'Status']
    });

    for await (const file of filesToDownload) {
      const subPath = file.substring(keyPrefix.length); // path.basename(`${env}/${item.Key}`);

      const fullPath = _path.default.resolve(folderPrefix, subPath);

      debug('file', fullPath);

      if (file.endsWith('/')) {
        !(0, _fs.existsSync)(fullPath) && (await (0, _promises.mkdir)(fullPath, {
          recursive: true
        }));
        continue;
      } // skip if exists and overwrite is false


      if (!overwrite) {
        try {
          await (0, _promises.stat)(fullPath);
          debug('exists, skipping');
          table.push([subPath, 'Skipped']);
          continue;
        } catch (err) {
          if (err.code !== 'ENOENT') (0, _print.cl)(err);
        }
      }

      message = (0, _print.themed)(`Downloading file: ${(0, _print.themed)(subPath, 'info')}`);
      !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary')); // ensure destination folder exists

      await (0, _promises.mkdir)(_path.default.dirname(fullPath), {
        recursive: true
      }); // download s3 object into destination folder

      const fileStream = s3.getObject({
        Bucket: env,
        Key: file
      }).createReadStream();
      await (0, _promises.writeFile)(fullPath, await (0, _utils.streamToString)(fileStream));
      !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
      table.push([subPath, 'Downloaded']);
    }

    if (json) {
      (0, _print.printJSON)({
        status: 'success',
        downloadedFiles: filesToDownload
      });
    } else {
      // print results
      (0, _print.cl)(table.toString());
    }
  } catch (err) {
    if (json) {
      (0, _print.printJSON)({
        status: 'error',
        error: err
      });
    } else {
      spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
      (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)(err.message)}`, 'secondary'));
      debug(err);

      if (err && err.response && err.response.data) {
        debug('response', err.response.data);
      }
    }
  }
};

exports.handler = handler;