@hotglue/cli/lib/commands/tenants/custom-etl.js

hotglue CLI tools

"use strict";

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.handler = exports.desc = exports.command = exports.builder = void 0;

var _debug = _interopRequireDefault(require("../../helpers/debug"));

var _ora = _interopRequireDefault(require("ora"));

var _awsSdk = _interopRequireDefault(require("aws-sdk"));

var _cliTable = _interopRequireDefault(require("cli-table"));

var _descriptions = _interopRequireDefault(require("../../helpers/descriptions.js"));

var _print = require("../../helpers/print.js");

var _api = require("../../helpers/api.js");

var _utils = require("../../helpers/utils.js");

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

const debug = (0, _debug.default)('commands:tenants:custom-etl');
const command = 'custom-etl';
exports.command = command;
const desc = 'List tenants with custom ETL in a specific flow';
exports.desc = desc;

const builder = async yargs => {
  debug('builder', command);
  return yargs.option('tenant', _descriptions.default.options['tenant'].config).option('flow', _descriptions.default.options['flow'].config).demandOption('flow', _descriptions.default.options['flow'].demandText);
};

exports.builder = builder;

const processTenant = async ({
  debug,
  baseUri,
  apikey,
  env,
  flow,
  tenant,
  supportedSources
}) => {
  // Get all linked sources of the tuple (env, flow, tenant)
  const linkedSources = await (0, _api.getLinkedSources)({
    debug,
    baseUri,
    env,
    flow,
    tenant,
    apikey
  });
  let result = null;

  for (const {
    tap
  } of [...linkedSources, ...supportedSources]) {
    // Generate AWS credentials in order to read the S3 tap folder
    // Note(davi): I'm using "etl-download" because it's a "read ETL" operation
    const {
      accessKeyId,
      secretAccessKey,
      sessionToken
    } = await (0, _api.genCredentialsOnClientApi)({
      debug,
      baseUri,
      task: 'etl-download',
      env,
      tenant,
      flow,
      tap,
      apikey
    });
    const s3 = new _awsSdk.default.S3({
      accessKeyId,
      secretAccessKey,
      sessionToken
    });
    const keyPrefix = `${tenant}/flows/${flow}/taps/${tap}/etl/`;
    const params = {
      Bucket: env,
      Prefix: keyPrefix
    }; // List all files inside the S3 tap folder

    const {
      Contents: s3Files
    } = await s3.listObjectsV2(params).promise(); // Check if either "etl.py" or "etl.ipynb" exist

    const hasCustomEtl = s3Files.some(({
      Key
    }) => Key.endsWith('etl.py') || Key.endsWith('etl.ipynb'));
    if (hasCustomEtl) result = tenant; // If it's true, we can break already

    if (result !== null) break;
  }

  return result;
};

const handler = async argv => {
  debug('handler', command, argv);
  const {
    hg,
    json,
    apikey,
    env,
    flow
  } = argv;
  let message;
  let spinner = (0, _ora.default)();

  try {
    message = (0, _print.themed)(`Retrieving tenants for environment ${(0, _print.themed)(env, 'info')}`);
    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary'));
    const [allTenants, supportedSources] = await Promise.all([(0, _api.getTenants)({
      debug,
      baseUri: hg.clientApiBaseUri,
      apikey,
      env
    }), (0, _api.getSupportedSources)({
      debug,
      baseUri: hg.clientApiBaseUri,
      apikey,
      env,
      flow
    })]);
    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));
    let tenantsWithCustomEtl = [];
    message = (0, _print.themed)(`Querying for custom ETL scripts for flow ${(0, _print.themed)(flow, 'info')}`);
    !json && spinner.start((0, _print.themed)(`In progress: ${message}...`, 'secondary')); // Break the tenants array in to an array of arrays so we can make
    // parallel calls and make it faster

    for (const tenants of (0, _utils.buildChunks)(allTenants)) {
      const maybeTenantsWithCustomEtl = await Promise.all(tenants.map(tenant => processTenant({
        debug,
        baseUri: hg.clientApiBaseUri,
        apikey,
        env,
        flow,
        tenant,
        supportedSources
      }))); // Note(davi): we need the `.filter(Boolean)` because sometimes
      // the function above returns `null`

      tenantsWithCustomEtl = tenantsWithCustomEtl.concat(maybeTenantsWithCustomEtl.filter(Boolean));
    }

    !json && spinner.succeed((0, _print.themed)(`Finished: ${message}.`, 'secondary'));

    if (json) {
      (0, _print.printJSON)(tenantsWithCustomEtl);
    } else {
      // generate results table
      const table = new _cliTable.default({
        head: ['Tenant ID']
      });
      table.push(...tenantsWithCustomEtl.map(item => [item])); // print results

      console.log(table.toString());
    }
  } catch (err) {
    if (json) {
      (0, _print.printJSON)({
        status: 'error',
        error: err
      });
    } else {
      spinner.fail((0, _print.themed)(`Error: ${message}.`, 'secondary'));
      (0, _print.pr)((0, _print.themed)(`Message: ${(0, _print.themed)(err.message)}`, 'secondary'));
      debug(err);

      if (err && err.response && err.response.data) {
        debug('response', err.response.data);
      }
    }
  }
};

exports.handler = handler;