@csermet/multiprovider
Version:
cloud-graph provider plugin for AWS used to fetch AWS cloud data.
448 lines (447 loc) • 18.9 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const groupBy_1 = __importDefault(require("lodash/groupBy"));
const isEmpty_1 = __importDefault(require("lodash/isEmpty"));
const camelCase_1 = __importDefault(require("lodash/camelCase"));
const ec2_1 = __importDefault(require("aws-sdk/clients/ec2"));
const cloudwatch_1 = __importDefault(require("aws-sdk/clients/cloudwatch"));
const sdk_1 = __importStar(require("@cloudgraph/sdk"));
const metrics_1 = __importStar(require("./metrics"));
const logger_1 = __importDefault(require("../../properties/logger"));
const utils_1 = require("../../utils");
const errorLog_1 = __importDefault(require("../../utils/errorLog"));
const format_1 = require("../../utils/format");
const data_1 = __importDefault(require("../iamInstanceProfile/data"));
const lt = { ...logger_1.default };
const { logger } = sdk_1.default;
const serviceName = 'EC2';
const errorLog = new errorLog_1.default(serviceName);
const endpoint = utils_1.initTestEndpoint(serviceName);
const cwGetMetricDataLimit = 500;
/**
* EC2
*/
exports.default = async ({ regions, config, rawData, }) => new Promise(async (resolve) => {
const ec2Instances = [];
/**
* Step 1) for all regions, list the EC2 Instances
*/
const listEc2Instances = async ({ ec2, region, nextToken: NextToken = '', resolveRegion, }) => {
let args = {};
if (NextToken) {
args = { ...args, NextToken };
}
return ec2.describeInstances(args, (err, data) => {
if (err) {
errorLog.generateAwsErrorLog({
functionName: 'ec2:describeInstances',
err,
});
}
/**
* No EC2 data for this region
*/
if (isEmpty_1.default(data)) {
return resolveRegion();
}
const { NextToken: nextToken, Reservations: res = [] } = data || {};
const instances = res
.map(({ Instances }) => Instances)
.reduce((current, acc) => [...acc, ...current], []);
logger.debug(lt.fetchedEc2Instances(instances.length));
/**
* No EC2 Instances Found
*/
if (isEmpty_1.default(instances)) {
return resolveRegion();
}
/**
* Check to see if there are more
*/
if (nextToken) {
listEc2Instances({ region, nextToken, ec2, resolveRegion });
}
ec2Instances.push(...instances.map(({ Tags, ...instance }) => ({
...instance,
region,
})));
/**
* If this is the last page of data then return the instances
*/
if (!nextToken) {
resolveRegion();
}
});
};
const regionPromises = regions.split(',').map(region => {
const ec2 = new ec2_1.default({ ...config, region, endpoint });
const regionPromise = new Promise(resolveRegion => listEc2Instances({ ec2, region, resolveRegion }));
return regionPromise;
});
await Promise.all(regionPromises);
/**
* Step 2) Get the Key Pair names for each instance's key pair
*/
const keyPairPromises = ec2Instances.map(({ region, KeyName }, ec2Idx) => {
const ec2 = new ec2_1.default({ ...config, region, endpoint });
const keyPairPromise = new Promise(resolveKeyPair => ec2.describeKeyPairs({ KeyNames: [KeyName] }, (err, data) => {
if (err) {
errorLog.generateAwsErrorLog({
functionName: 'ec2:describeKeyPairs',
err,
});
}
/**
* No Key Pair data for this instance
*/
if (isEmpty_1.default(data) || !KeyName) {
return resolveKeyPair();
}
const { KeyPairs: pairs } = data || {};
/**
* No Key Pairs Found
*/
if (isEmpty_1.default(pairs)) {
return resolveKeyPair();
}
const [keyPair] = pairs.map(({ KeyName: instanceKeyName, KeyPairId: id, KeyFingerprint: fingerprint, KeyType: type, Tags, }) => ({
id,
fingerprint,
type,
name: instanceKeyName,
tags: format_1.convertAwsTagsToTagMap(Tags),
}));
logger.debug(`${lt.foundKeyPair(keyPair.id)} for instance ${ec2Instances[ec2Idx]?.InstanceId}`);
ec2Instances[ec2Idx].KeyPair = keyPair;
resolveKeyPair();
}));
return keyPairPromise;
});
await Promise.all(keyPairPromises);
/**
* Step 3) check to see if disableApiTermination is on for each instance
*/
const deletionProtectionPromises = ec2Instances.map(({ region, InstanceId }, ec2Idx) => {
const ec2 = new ec2_1.default({ ...config, region, endpoint });
const deletionProtectionPromise = new Promise(resolveDeletionProtection => ec2.describeInstanceAttribute({ InstanceId, Attribute: 'disableApiTermination' }, (err, data) => {
if (err) {
errorLog.generateAwsErrorLog({
functionName: 'ec2:desrcibeInstanceAttributes',
err,
});
}
/**
* No data
*/
if (isEmpty_1.default(data)) {
return resolveDeletionProtection();
}
const { DisableApiTermination: { Value: val }, } = data || { DisableApiTermination: {} };
ec2Instances[ec2Idx].DisableApiTermination = val;
resolveDeletionProtection();
}));
return deletionProtectionPromise;
});
await Promise.all(deletionProtectionPromises);
/**
* Step 4) get all of the tags for all instances... For some reason, there
* Is an empty tags object returned from the initial describeInstances call...
*/
const allTags = {};
const listAllTagsforAllInstances = async ({ ec2, region, nextToken: NextToken = '', resolveTags, }) => {
let args = {
Filters: [{ Name: 'resource-type', Values: ['instance'] }],
};
if (NextToken) {
args = { ...args, NextToken };
}
return ec2.describeTags(args, (err, data) => {
if (err) {
errorLog.generateAwsErrorLog({
functionName: 'ec2:describeTags',
err,
});
}
/**
* No tag data for this region
*/
if (isEmpty_1.default(data)) {
return resolveTags();
}
const { NextToken: nextToken, Tags: tags = [] } = data || {};
logger.debug(lt.fetchedEc2InstanceTags(tags.length));
/**
* No Tags Found
*/
if (isEmpty_1.default(tags)) {
return resolveTags();
}
/**
* Check to see if there are more
*/
if (nextToken) {
listAllTagsforAllInstances({
region,
nextToken,
ec2,
resolveTags,
});
}
/**
* Add the tags to the allTags Cache
*/
tags.map(({ Key, Value, ResourceId }) => {
if (!allTags[ResourceId]) {
allTags[ResourceId] = {};
}
allTags[ResourceId] = !isEmpty_1.default(allTags[ResourceId])
? [...allTags[ResourceId], { Key, Value }]
: [{ Key, Value }];
});
/**
* If this is the last page of data then return the instances
*/
if (!nextToken) {
resolveTags();
}
});
};
const tagsPromises = regions.split(',').map(region => {
const ec2 = new ec2_1.default({ ...config, region, endpoint });
const tagPromise = new Promise(resolveTags => listAllTagsforAllInstances({
region,
ec2,
resolveTags,
}));
return tagPromise;
});
await Promise.all(tagsPromises);
/**
* Now add all of the tags to the instances
*/
ec2Instances.map(({ InstanceId }, ec2Idx) => {
ec2Instances[ec2Idx].Tags = (allTags[InstanceId] || [])
.map(({ Key, Value }) => ({ [Key]: Value }))
.reduce((acc, curr) => ({ ...acc, ...curr }), {});
});
/**
* Step 5) Get the platform details associated with the billing code of the AMI for all instances
*/
const describeImagesPromises = ec2Instances.map(({ region, ImageId }, ec2Idx) => {
const ec2 = new ec2_1.default({ ...config, region, endpoint });
const describeImagesPromise = new Promise(resolveImage => ec2.describeImages({ ImageIds: [ImageId] }, (err, data) => {
if (err) {
errorLog.generateAwsErrorLog({
functionName: 'ec2:describeImages',
err,
});
}
/**
* No data
*/
if (isEmpty_1.default(data)) {
return resolveImage();
}
const { Images: images } = data || {};
ec2Instances[ec2Idx].PlatformDetails = images[0]?.PlatformDetails;
resolveImage();
}));
return describeImagesPromise;
});
await Promise.all(describeImagesPromises);
const iamInstanceProfile = {};
const iamProfileAssociationsPromises = regions.split(',').map(region => {
const ec2 = new ec2_1.default({ ...config, region, endpoint });
return new Promise(resolveRegion => ec2.describeIamInstanceProfileAssociations({}, (err, data) => {
if (err) {
errorLog.generateAwsErrorLog({
functionName: 'ec2:describeIamInstanceProfileAssociations',
err,
});
}
const { IamInstanceProfileAssociations: iamProfileAssociations = [], } = data || {};
iamProfileAssociations.map(({ InstanceId, IamInstanceProfile: curr }) => {
if (!iamInstanceProfile[InstanceId]) {
iamInstanceProfile[InstanceId] = {};
}
iamInstanceProfile[InstanceId] = curr;
});
resolveRegion();
}));
});
const metricQueries = ec2Instances
.map(({ InstanceId }) => {
return metrics_1.default.map(metricName => ({
Id: `${sdk_1.generateUniqueId({ InstanceId, metricName })}_${metricName}`,
Label: `${metricName}:${InstanceId}`,
MetricStat: {
Metric: {
Dimensions: [{ Name: 'InstanceId', Value: InstanceId }],
Namespace: 'AWS/EC2',
MetricName: metricName,
},
Stat: metrics_1.metricStats[metricName],
Period: 60 * 25,
},
}));
})
.flat();
const getMeticsForTimePeriod = async ({ minsAgo, end = new Date(), }) => {
const metrics = [];
const startTime = new Date(Date.now() - 60000 * minsAgo);
await new Promise(async (resolveTimePeriod) => {
for (const region of regions.split(',')) {
const client = new cloudwatch_1.default({ ...config, region });
let loopBreak = false;
let token;
let metricsToFetch = [metricQueries];
if (metricQueries.length > cwGetMetricDataLimit) {
logger.debug('EC2:getMetricData has more than 500 requests, chunking the requests...');
metricsToFetch = metricQueries.reduce((resultArray, item, index) => {
const chunkIndex = Math.floor(index / cwGetMetricDataLimit);
if (!resultArray[chunkIndex]) {
// eslint-disable-next-line no-param-reassign
resultArray[chunkIndex] = []; // start a new chunk
}
resultArray[chunkIndex].push(item);
return resultArray;
}, []);
}
const promises = [];
metricsToFetch.forEach(async (metricSet) => {
const metricPromise = new Promise(async (resolveMetric) => {
while (!loopBreak) {
const params = {
MetricDataQueries: metricSet,
StartTime: startTime,
EndTime: end,
NextToken: token,
};
try {
const data = await client.getMetricData(params).promise();
if (data && data.MetricDataResults) {
for (const result of data.MetricDataResults) {
metrics.push({
metric: result.Label.split(':')[0],
label: result.Label.split(':')[1],
values: result.Values,
timestamps: result.Timestamps,
});
}
}
if (!data?.NextToken) {
loopBreak = true;
}
else {
token = data.NextToken;
}
}
catch (err) {
errorLog.generateAwsErrorLog({
functionName: 'ec2:getMetricData',
err,
});
loopBreak = true;
resolveMetric();
}
}
resolveMetric();
});
promises.push(metricPromise);
});
await Promise.all(promises);
resolveTimePeriod();
}
});
return metrics;
};
const timePeriod = {
last6Hours: { minsAgo: 360, metrics: [] },
last24Hours: { minsAgo: 1440, metrics: [] },
lastWeek: { minsAgo: 10080, metrics: [] },
lastMonth: { minsAgo: 43800, metrics: [] },
};
// First populate the time periods with the corresponding metrics
try {
for (const [key, periodObj] of Object.entries(timePeriod)) {
const metrics = await getMeticsForTimePeriod({
minsAgo: periodObj.minsAgo,
});
timePeriod[key].metrics = metrics;
}
// Now populate the ec2Instances with the metric data
ec2Instances.map(({ InstanceId }, idx) => {
ec2Instances[idx].cloudWatchMetricData = {};
for (const [key, { metrics }] of Object.entries(timePeriod)) {
ec2Instances[idx].cloudWatchMetricData[key] = {};
const instanceMetrics = metrics.filter(({ label }) => label === InstanceId);
const groupedMetrics = groupBy_1.default(instanceMetrics, 'metric');
Object.keys(groupedMetrics).map(metricKey => {
const camelKey = camelCase_1.default(metricKey);
const operator = metrics_1.metricStats[metricKey];
ec2Instances[idx].cloudWatchMetricData[key][`${camelKey}${operator}`] = 0;
const metric = groupedMetrics[metricKey];
const filteredMetric = metric.filter(({ values }) => values.length > 1);
let valuesSumOrAverage;
for (const { values } of filteredMetric) {
if (operator === 'Average') {
valuesSumOrAverage = values.reduce((prev, curr, _, self) => (curr + prev) / self.length);
}
else {
valuesSumOrAverage = values.reduce((prev, curr) => prev + curr);
}
}
ec2Instances[idx].cloudWatchMetricData[key][`${camelKey}${operator}`] = valuesSumOrAverage;
});
}
});
}
catch (e) {
logger.debug(e);
logger.error('There was an issue adding CW metric data to ec2 instances');
}
await Promise.all(iamProfileAssociationsPromises);
ec2Instances.map(({ InstanceId }, ec2Idx) => {
ec2Instances[ec2Idx].IamInstanceProfile =
iamInstanceProfile[InstanceId] || {};
});
// populate ec2Instances with the iamRoles Arn
const iamInstancesProfiles = Object.values(await data_1.default({
config,
rawData,
}))?.reduce((acc, val) => acc.concat(val), []) || [];
ec2Instances.map(({ IamInstanceProfile: instanceProfile }, ec2Idx) => {
const instance = iamInstancesProfiles.find(i => i.Arn === instanceProfile?.Arn);
if (instance) {
ec2Instances[ec2Idx].IamRolesArn =
instance?.Roles?.map(r => r.Arn) || [];
}
});
errorLog.reset();
/**
* Return the instances grouped by region
*/
resolve(groupBy_1.default(ec2Instances, 'region'));
});