@csermet/multiprovider
Version:
cloud-graph provider plugin for AWS used to fetch AWS cloud data.
311 lines (310 loc) • 13.5 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.formatAmmountAndUnit = exports.getRoundedAmount = void 0;
const sdk_1 = __importDefault(require("@cloudgraph/sdk"));
const costexplorer_1 = __importDefault(require("aws-sdk/clients/costexplorer"));
const isEmpty_1 = __importDefault(require("lodash/isEmpty"));
const head_1 = __importDefault(require("lodash/head"));
const get_1 = __importDefault(require("lodash/get"));
const regions_1 = require("../../enums/regions");
const logger_1 = __importDefault(require("../../properties/logger"));
const errorLog_1 = __importDefault(require("../../utils/errorLog"));
const utils_1 = require("../../utils");
const dateutils_1 = require("../../utils/dateutils");
const lt = { ...logger_1.default };
const { logger } = sdk_1.default;
const serviceName = 'Billing';
const errorLog = new errorLog_1.default(serviceName);
const endpoint = utils_1.initTestEndpoint(serviceName);
const getRoundedAmount = (amount) => Math.round((parseFloat(amount) + Number.EPSILON) * 100) / 100;
exports.getRoundedAmount = getRoundedAmount;
const formatAmmountAndUnit = ({ Amount: amount = 0, Unit: currency = 'USD', }) => new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(exports.getRoundedAmount(amount.toString()));
exports.formatAmmountAndUnit = formatAmmountAndUnit;
const listAvailabeServices = ({ costExplorer, resolveServices, }) => {
return costExplorer.getDimensionValues({
TimePeriod: { Start: dateutils_1.getDaysAgo(30), End: dateutils_1.getDaysAgo(0) },
Dimension: 'SERVICE',
}, (err, data) => {
/**
* Error fetching the services list
*/
if (err) {
errorLog.generateAwsErrorLog({
functionName: 'ce:getDimensionsValues',
err,
});
return resolveServices([]);
}
const { DimensionValues: dimensions = [] } = data || {};
/**
* The dimensions are for some reason empty when they should not be
*/
if (isEmpty_1.default(dimensions)) {
logger.debug(lt.unableToFindFinOpsServiceData);
return resolveServices([]);
}
resolveServices(dimensions.map(({ Value }) => Value));
});
};
/**
* AWS Billing
*/
exports.default = async ({ config, }) => {
const startDate = new Date();
const region = regions_1.regionMap.usEast1;
const results = {
totalCostLast30Days: {},
totalCostMonthToDate: {},
monthToDateDailyAverage: {},
last30DaysDailyAverage: {},
monthToDate: {},
last30Days: {},
individualData: {},
};
const resultPromises = [];
logger.debug(lt.fetchingAggregateFinOpsData);
try {
const listAggregateFinOpsData = ({ costExplorer, resolve, type, groupBy = true, timePeriod: TimePeriod, }) => {
const params = {
Metrics: ['BlendedCost'],
TimePeriod,
Granularity: 'MONTHLY',
};
if (groupBy) {
params.GroupBy = [{ Key: 'SERVICE', Type: 'DIMENSION' }];
}
logger.debug(lt.queryingAggregateFinOpsDataForRegion(region, type));
return costExplorer.getCostAndUsage(params, (err, data) => {
/**
* Error fetching the cost data
*/
if (err) {
errorLog.generateAwsErrorLog({
functionName: 'ce:GetCostAndUsageReport',
err,
});
return resolve();
}
const { ResultsByTime: resultsByTime = [] } = data || {};
/**
* The results are for some reason empty when they should not be
*/
if (isEmpty_1.default(resultsByTime)) {
logger.debug(lt.unableToFindFinOpsAggregateData);
return resolve();
}
if (groupBy) {
/**
* Map over the list of returned services and extract the pricing data for the last month, format of data is:
* { "Keys": [ "AWS CloudTrail"], "Metrics": { "BlendedCost": { "Amount": "0.1270885", "Unit": "USD" } } }
*/
const services = head_1.default(resultsByTime).Groups || [];
services.map(({ Keys, Metrics }) => {
const { Amount = '', Unit = '' } = get_1.default(Metrics, 'BlendedCost', {
Amount: '',
Unit: '',
}) || { Amount: '', Unit: '' };
const cost = exports.getRoundedAmount(Amount);
results[type][head_1.default(Keys)] = {
cost,
currency: Unit,
formattedCost: exports.formatAmmountAndUnit({ Amount: cost, Unit }),
};
});
}
else {
/**
* No service data, everything is just aggregated together so it looks like this:
* [ { Total: { BlendedCost: { Amount: '-0.2775129004', Unit: 'USD' } } } ... ]
*/
let currency;
const cost = resultsByTime.reduce((prev, { Total: { BlendedCost: blendedCost } = { BlendedCost: {} } }) => {
if (!currency) {
currency = blendedCost.Unit;
}
return prev + exports.getRoundedAmount(blendedCost.Amount);
}, 0);
results[type] = {
cost,
currency,
formattedCost: exports.formatAmmountAndUnit({
Amount: cost,
Unit: currency,
}),
};
}
resolve();
});
};
// Try to get individual entity data
const listIndividualFinOpsData = ({ costExplorer, resolve, services, timePeriod: TimePeriod, }) => {
logger.debug(lt.queryingIndividualFinOpsDataForRegion(region));
/**
* NOTES: This only currently with services that are compute (i.e. EC2) based such as instances
* NAT Gateways, and Dedicated Hosts. Everything else comes back as having "NoResourceId" like:
* { 'arn:aws:ec2:us-east-1:938345459433:dedicated-host/h-fssg93hq0400454': '$360.64',
* 'i-0043545435j4535': '$0.01', NoResourceId: '$2,857.64'... }
* More info here: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/CostExplorer.html#getCostAndUsageWithResources-property
*/
return costExplorer.getCostAndUsageWithResources({
Metrics: ['BlendedCost'],
TimePeriod,
Granularity: 'DAILY',
GroupBy: [{ Key: 'RESOURCE_ID', Type: 'DIMENSION' }],
Filter: {
Dimensions: {
Key: 'SERVICE',
Values: services,
},
},
}, (err, data) => {
/**
* Error fetching the cost data
*/
if (err) {
errorLog.generateAwsErrorLog({
functionName: 'ce:getCostAndUsageWithResources',
err,
});
return resolve();
}
const { ResultsByTime: resultsByTime = [] } = data || {};
/**
* The results are for some reason empty when they should not be
*/
if (isEmpty_1.default(resultsByTime)) {
logger.debug(lt.unableToFindFinOpsIndividualData);
return resolve();
}
const resources = head_1.default(resultsByTime).Groups || [];
resources.map(({ Keys, Metrics }) => {
const { Amount, Unit } = get_1.default(Metrics, 'BlendedCost', {
Amount: '',
Unit: '',
}) || { Amount: '', Unit: '' };
const cost = exports.getRoundedAmount(Amount);
results.individualData[head_1.default(Keys)] = {
cost,
currency: Unit,
formattedCost: exports.formatAmmountAndUnit({ Amount: cost, Unit }),
};
});
resolve();
});
};
/**
* Now we make 4 queries to the api in order to get aggregate pricing data sliced in various ways
* Note that this API is only availavbe in the us-east-1
*/
const costExplorer = new costexplorer_1.default({ ...config, region, endpoint });
const today = new Date().toLocaleDateString('en-ca'); // We use en-ca to ensure the correct date structure for CE
const startOfMonth = dateutils_1.getFirstDayOfMonth();
const commonArgs = {
costExplorer,
timePeriod: {
Start: dateutils_1.getDaysAgo(30),
End: today,
},
};
/**
* Breakdown by service types and spend for last 30 days
*/
const last30DaysData = new Promise(resolve => listAggregateFinOpsData({
...commonArgs,
resolve,
type: 'last30Days',
}));
resultPromises.push(last30DaysData);
/**
* Breakdown by service types and spend since the beginning of the month
*/
if (!(today === startOfMonth)) {
const monthToDateData = new Promise(resolve => listAggregateFinOpsData({
costExplorer,
resolve,
type: 'monthToDate',
timePeriod: {
Start: startOfMonth,
End: today,
},
}));
resultPromises.push(monthToDateData);
}
/**
* The single total cost of everything in the last 30 days
*/
const totalCostLast30Days = new Promise(resolve => listAggregateFinOpsData({
...commonArgs,
resolve,
type: 'totalCostLast30Days',
groupBy: false,
}));
resultPromises.push(totalCostLast30Days);
/**
* The single total cost of everything in the current month
*/
if (!(today === startOfMonth)) {
const totalCostMonthToDate = new Promise(resolve => listAggregateFinOpsData({
...commonArgs,
resolve,
type: 'totalCostMonthToDate',
groupBy: false,
timePeriod: {
Start: startOfMonth,
End: today,
},
}));
resultPromises.push(totalCostMonthToDate);
}
const individualDataPromise = new Promise(async (resolve) => {
return listIndividualFinOpsData({
costExplorer,
services: await new Promise(resolveServices => listAvailabeServices({ costExplorer, resolveServices })),
timePeriod: {
Start: dateutils_1.getDaysAgo(1),
End: new Date().toLocaleDateString('en-ca'), // We use en-ca to ensure correct date structure for CE
},
resolve,
});
});
resultPromises.push(individualDataPromise);
await Promise.all(resultPromises);
/**
* Create Daily Averages
*/
const createDailyAverage = ({ days, resultMonthlyData, resultAverageData, }) => Object.keys(resultMonthlyData).map(service => {
const { cost: aggregateCost, currency } = resultMonthlyData[service];
const cost = parseFloat((aggregateCost / days).toFixed(2));
results[resultAverageData][service] = {
cost,
currency,
formattedCost: exports.formatAmmountAndUnit({ Amount: cost, Unit: currency }),
};
});
if (!isEmpty_1.default(results.monthToDate)) {
createDailyAverage({
days: parseInt(dateutils_1.getCurrentDayOfMonth(), 10),
resultMonthlyData: results.monthToDate,
resultAverageData: 'monthToDateDailyAverage',
});
}
if (!isEmpty_1.default(results.last30Days)) {
createDailyAverage({
days: 30,
resultMonthlyData: results.last30Days,
resultAverageData: 'last30DaysDailyAverage',
});
}
logger.debug(lt.doneFetchingAggregateFinOpsData(dateutils_1.createDiffSecs(startDate)));
errorLog.reset();
return { [region]: [results] };
}
catch (e) {
logger.error(`There was an issue resolving data for ${serviceName}`);
logger.debug(e);
return {};
}
};