@csermet/multiprovider
Version:
cloud-graph provider plugin for AWS used to fetch AWS cloud data.
101 lines (100 loc) • 3.91 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const emr_1 = __importDefault(require("aws-sdk/clients/emr"));
const sdk_1 = __importDefault(require("@cloudgraph/sdk"));
const groupBy_1 = __importDefault(require("lodash/groupBy"));
const isEmpty_1 = __importDefault(require("lodash/isEmpty"));
const flatMap_1 = __importDefault(require("lodash/flatMap"));
const logger_1 = __importDefault(require("../../properties/logger"));
const utils_1 = require("../../utils");
const errorLog_1 = __importDefault(require("../../utils/errorLog"));
const data_1 = require("../emrCluster/data");
const services_1 = __importDefault(require("../../enums/services"));
const lt = { ...logger_1.default };
const { logger } = sdk_1.default;
const serviceName = 'EMR instance';
const errorLog = new errorLog_1.default(serviceName);
const endpoint = utils_1.initTestEndpoint(serviceName);
const listEmrClusterSteps = async (emr, clusterId) => new Promise(resolve => {
const clusterStepList = [];
const listStepsOpts = { ClusterId: clusterId };
const listSteps = (marker) => {
if (marker) {
listStepsOpts.Marker = marker;
}
emr.listSteps(listStepsOpts, (err, data) => {
if (err) {
errorLog.generateAwsErrorLog({
functionName: 'emr:listSteps',
err,
});
}
/**
* No EMR step data for this region
*/
if (isEmpty_1.default(data)) {
return resolve([]);
}
const { Steps = [], Marker: nextToken } = data;
clusterStepList.push(...Steps);
if (nextToken) {
logger.debug(lt.foundAnotherFiftySteps(clusterId));
listSteps(nextToken);
}
else {
resolve(clusterStepList);
}
});
};
listSteps();
});
exports.default = async ({ regions, config, rawData, }) => new Promise(async (resolve) => {
const emrSteps = [];
let emrClusters = [];
const existingData = flatMap_1.default(rawData.find(({ name }) => name === services_1.default.emrCluster)?.data) ||
[];
if (isEmpty_1.default(existingData)) {
/**
* Get all the EMR clusters for this region
*/
await Promise.all(regions.split(',').map(region => new Promise(async (resolveEmrClusters) => {
const emr = new emr_1.default({ ...config, region, endpoint });
const clusterList = await data_1.getEmrClusters(emr, region);
if (!isEmpty_1.default(clusterList)) {
emrClusters.push(...clusterList.map(cluster => ({
Id: cluster.Id,
region,
})));
}
resolveEmrClusters();
})));
}
else {
// Uses existing data
emrClusters = existingData;
}
logger.debug(lt.fetchedEmrClusters(emrClusters.length));
/**
* Get the list of steps for each EMR cluster
*/
let numOfSteps = 0;
const stepPromises = emrClusters.map((cluster) => new Promise(async (resolveSteps) => {
const emr = new emr_1.default({ ...config, region: cluster.region, endpoint });
const steps = await listEmrClusterSteps(emr, cluster.Id);
if (!isEmpty_1.default(steps)) {
numOfSteps += steps.length;
emrSteps.push(...steps.map(step => ({
region: cluster.region,
...step,
})));
}
resolveSteps();
}));
await Promise.all(stepPromises);
errorLog.reset();
logger.debug(lt.fetchedEmrClusterSteps(numOfSteps));
resolve(groupBy_1.default(emrSteps, 'region'));
});