@cloud-carbon-footprint/gcp
Version:
The core logic to get cloud usage data and estimate energy and carbon emissions from Google Cloud Platform.
466 lines • 24 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.buildTagQuery = void 0;
const moment_1 = __importDefault(require("moment"));
const common_1 = require("@cloud-carbon-footprint/common");
const core_1 = require("@cloud-carbon-footprint/core");
const BillingExportTypes_1 = require("./BillingExportTypes");
const MachineTypes_1 = require("./MachineTypes");
const BillingExportRow_1 = __importDefault(require("./BillingExportRow"));
const domain_1 = require("../domain");
const ReplicationFactors_1 = require("./ReplicationFactors");
const GCPRegions_1 = require("./GCPRegions");
class BillingExportTable {
computeEstimator;
ssdStorageEstimator;
hddStorageEstimator;
networkingEstimator;
memoryEstimator;
unknownEstimator;
embodiedEmissionsEstimator;
bigQuery;
tableName;
billingExportTableLogger;
constructor(computeEstimator, ssdStorageEstimator, hddStorageEstimator, networkingEstimator, memoryEstimator, unknownEstimator, embodiedEmissionsEstimator, bigQuery) {
this.computeEstimator = computeEstimator;
this.ssdStorageEstimator = ssdStorageEstimator;
this.hddStorageEstimator = hddStorageEstimator;
this.networkingEstimator = networkingEstimator;
this.memoryEstimator = memoryEstimator;
this.unknownEstimator = unknownEstimator;
this.embodiedEmissionsEstimator = embodiedEmissionsEstimator;
this.bigQuery = bigQuery;
this.tableName = (0, common_1.configLoader)().GCP.BIG_QUERY_TABLE;
this.billingExportTableLogger = new common_1.Logger('BillingExportTable');
}
async getEstimates(start, end, grouping) {
const gcpConfig = (0, common_1.configLoader)().GCP;
const tagNames = gcpConfig.RESOURCE_TAG_NAMES;
const projects = gcpConfig.projects;
const usageRows = await this.getUsage(start, end, grouping, tagNames, projects);
const results = [];
const unknownRows = [];
this.billingExportTableLogger.info('Mapping over Usage Rows');
for (const usageRow of usageRows) {
usageRow.tags = this.rawTagsToTagCollection(usageRow);
const billingExportRow = new BillingExportRow_1.default(usageRow);
const emissionsFactors = await (0, common_1.getEmissionsFactors)(billingExportRow.region, billingExportRow.timestamp.toISOString(), (0, domain_1.getGCPEmissionsFactors)(), GCPRegions_1.GCP_MAPPED_REGIONS_TO_ELECTRICITY_MAPS_ZONES, this.billingExportTableLogger);
const footprintEstimate = this.getFootprintEstimateFromUsageRow(billingExportRow, unknownRows, emissionsFactors);
if (footprintEstimate)
(0, core_1.appendOrAccumulateEstimatesByDay)(results, billingExportRow, footprintEstimate, grouping, tagNames);
}
if (results.length > 0) {
unknownRows.map((rowData) => {
const footprintEstimate = this.getEstimateForUnknownUsage(rowData);
if (footprintEstimate)
(0, core_1.appendOrAccumulateEstimatesByDay)(results, rowData, footprintEstimate, grouping, tagNames);
});
}
return results;
}
async getEstimatesFromInputData(inputData) {
const results = [];
const unknownRows = [];
for (const inputDataRow of inputData) {
const usageRow = {
serviceName: inputDataRow.serviceName,
usageAmount: 3600,
usageType: inputDataRow.usageType,
usageUnit: inputDataRow.usageUnit,
cost: 1,
region: inputDataRow.region,
machineType: inputDataRow.machineType,
timestamp: new Date(''),
};
const billingExportRow = new BillingExportRow_1.default(usageRow);
if (billingExportRow.machineType) {
const { instancevCpu } = this.getDataFromMachineType(billingExportRow.machineType);
billingExportRow.vCpuHours = instancevCpu * billingExportRow.vCpuHours;
}
const dateTime = new Date().toISOString();
const emissionsFactors = await (0, common_1.getEmissionsFactors)(billingExportRow.region, dateTime, (0, domain_1.getGCPEmissionsFactors)(), GCPRegions_1.GCP_MAPPED_REGIONS_TO_ELECTRICITY_MAPS_ZONES, this.billingExportTableLogger);
const footprintEstimate = this.getFootprintEstimateFromUsageRow(billingExportRow, unknownRows, emissionsFactors);
if (footprintEstimate)
results.push({
serviceName: billingExportRow.serviceName,
region: billingExportRow.region,
usageType: billingExportRow.usageType,
machineType: billingExportRow.machineType,
kilowattHours: footprintEstimate.kilowattHours,
co2e: footprintEstimate.co2e,
});
}
if (results.length > 0) {
unknownRows.map((billingExportRow) => {
const footprintEstimate = this.getEstimateForUnknownUsage(billingExportRow);
if (footprintEstimate)
results.push({
serviceName: billingExportRow.serviceName,
region: billingExportRow.region,
usageType: billingExportRow.usageType,
machineType: billingExportRow.machineType,
kilowattHours: footprintEstimate.kilowattHours,
co2e: footprintEstimate.co2e,
});
});
}
return results;
}
getFootprintEstimateFromUsageRow(billingExportRow, unknownRows, emissionsFactors) {
if (this.isUnsupportedUsage(billingExportRow.usageType))
return;
if (this.isUnknownUsage(billingExportRow)) {
unknownRows.push(billingExportRow);
return;
}
return this.getEstimateByUsageUnit(billingExportRow, unknownRows, emissionsFactors);
}
getEstimateByUsageUnit(billingExportRow, unknownRows, emissionsFactors) {
const powerUsageEffectiveness = domain_1.GCP_CLOUD_CONSTANTS.getPUE(billingExportRow.region);
switch (billingExportRow.usageUnit) {
case 'seconds':
if (this.isComputeUsage(billingExportRow.usageType)) {
const computeFootprint = this.getComputeFootprintEstimate(billingExportRow, billingExportRow.timestamp, powerUsageEffectiveness, emissionsFactors);
const embodiedEmissions = this.getEmbodiedEmissions(billingExportRow, emissionsFactors);
if (embodiedEmissions.co2e) {
return {
timestamp: computeFootprint.timestamp,
kilowattHours: computeFootprint.kilowattHours +
embodiedEmissions.kilowattHours,
co2e: computeFootprint.co2e + embodiedEmissions.co2e,
usesAverageCPUConstant: computeFootprint.usesAverageCPUConstant,
};
}
return computeFootprint;
}
else {
unknownRows.push(billingExportRow);
}
break;
case 'byte-seconds':
if (this.isMemoryUsage(billingExportRow.usageType)) {
return this.getMemoryFootprintEstimate(billingExportRow, billingExportRow.timestamp, powerUsageEffectiveness, emissionsFactors);
}
else {
return this.getStorageFootprintEstimate(billingExportRow, billingExportRow.timestamp, powerUsageEffectiveness, emissionsFactors);
}
case 'bytes':
if (this.isNetworkingUsage(billingExportRow.usageType)) {
return this.getNetworkingFootprintEstimate(billingExportRow, billingExportRow.timestamp, powerUsageEffectiveness, emissionsFactors);
}
else {
unknownRows.push(billingExportRow);
}
break;
default:
this.billingExportTableLogger.warn(`Unsupported Usage unit: ${billingExportRow.usageUnit}`);
break;
}
}
getComputeFootprintEstimate(usageRow, timestamp, powerUsageEffectiveness, emissionsFactors) {
const isGPUComputeUsage = usageRow.usageType.includes('GPU');
const computeUsage = {
cpuUtilizationAverage: domain_1.GCP_CLOUD_CONSTANTS.AVG_CPU_UTILIZATION_2020,
vCpuHours: isGPUComputeUsage ? usageRow.gpuHours : usageRow.vCpuHours,
usesAverageCPUConstant: true,
timestamp,
};
let computeProcessors;
if (isGPUComputeUsage) {
computeProcessors = this.getGpuComputeProcessorsFromUsageType(usageRow.usageType);
}
else {
computeProcessors = this.getComputeProcessorsFromMachineType(usageRow.machineType);
}
const computeConstants = {
minWatts: domain_1.GCP_CLOUD_CONSTANTS.getMinWatts(computeProcessors),
maxWatts: domain_1.GCP_CLOUD_CONSTANTS.getMaxWatts(computeProcessors),
powerUsageEffectiveness: powerUsageEffectiveness,
replicationFactor: this.getReplicationFactor(usageRow),
};
const computeFootprint = this.computeEstimator.estimate([computeUsage], usageRow.region, emissionsFactors, computeConstants)[0];
if (computeFootprint)
(0, core_1.accumulateKilowattHours)(domain_1.GCP_CLOUD_CONSTANTS.KILOWATT_HOURS_BY_SERVICE_AND_USAGE_UNIT, usageRow, computeFootprint.kilowattHours, core_1.AccumulateKilowattHoursBy.USAGE_AMOUNT);
return computeFootprint;
}
getComputeProcessorsFromMachineType(machineType) {
const sharedCoreMatch = machineType &&
Object.values(MachineTypes_1.SHARED_CORE_PROCESSORS).find((core) => machineType.includes(core));
const includesPrefix = machineType?.substring(0, 2).toLowerCase();
const processor = sharedCoreMatch ? sharedCoreMatch : includesPrefix;
return (MachineTypes_1.INSTANCE_TYPE_COMPUTE_PROCESSOR_MAPPING[processor] || [
core_1.COMPUTE_PROCESSOR_TYPES.UNKNOWN,
]);
}
getGpuComputeProcessorsFromUsageType(usageType) {
const gpuComputeProcessors = MachineTypes_1.GPU_MACHINE_TYPES.filter((processor) => usageType.startsWith(processor));
return gpuComputeProcessors.length
? gpuComputeProcessors
: MachineTypes_1.GPU_MACHINE_TYPES;
}
getEmbodiedEmissions(usageRow, emissionsFactors) {
const { instancevCpu, scopeThreeEmissions, largestInstancevCpu } = this.getDataFromMachineType(usageRow.machineType);
const embodiedEmissionsUsage = {
instancevCpu,
largestInstancevCpu,
usageTimePeriod: usageRow.usageAmount / instancevCpu / 3600,
scopeThreeEmissions,
};
return this.embodiedEmissionsEstimator.estimate([embodiedEmissionsUsage], usageRow.region, emissionsFactors)[0];
}
getDataFromMachineType(machineType) {
if (!machineType) {
return {
instancevCpu: 0,
scopeThreeEmissions: 0,
largestInstancevCpu: 0,
};
}
const machineFamily = machineType?.split('-').slice(0, 2).join('-');
const [machineFamilySharedCore] = machineType?.split('-');
const instancevCpu = MachineTypes_1.MACHINE_FAMILY_TO_MACHINE_TYPE_MAPPING[machineFamily]?.[machineType]?.[0] ||
MachineTypes_1.MACHINE_FAMILY_SHARED_CORE_TO_MACHINE_TYPE_MAPPING[machineFamilySharedCore]?.[machineType]?.[0] ||
MachineTypes_1.N1_SHARED_CORE_MACHINE_FAMILY_TO_MACHINE_TYPE_MAPPING[machineType]?.[0];
const scopeThreeEmissions = MachineTypes_1.MACHINE_FAMILY_TO_MACHINE_TYPE_MAPPING[machineFamily]?.[machineType]?.[1] ||
MachineTypes_1.MACHINE_FAMILY_SHARED_CORE_TO_MACHINE_TYPE_MAPPING[machineFamilySharedCore]?.[machineType]?.[1] ||
MachineTypes_1.N1_SHARED_CORE_MACHINE_FAMILY_TO_MACHINE_TYPE_MAPPING[machineType]?.[1];
const familyMachineTypes = Object.values(MachineTypes_1.MACHINE_FAMILY_TO_MACHINE_TYPE_MAPPING[machineFamily] ||
MachineTypes_1.MACHINE_FAMILY_SHARED_CORE_TO_MACHINE_TYPE_MAPPING[machineFamilySharedCore] ||
MachineTypes_1.N1_SHARED_CORE_MACHINE_FAMILY_TO_MACHINE_TYPE_MAPPING);
const largestInstancevCpu = familyMachineTypes[familyMachineTypes.length - 1][0];
return {
instancevCpu,
scopeThreeEmissions,
largestInstancevCpu,
};
}
getStorageFootprintEstimate(usageRow, timestamp, powerUsageEffectiveness, emissionsFactors) {
const usageAmountTerabyteHours = (0, common_1.convertByteSecondsToTerabyteHours)(usageRow.usageAmount);
const storageUsage = {
timestamp,
terabyteHours: usageAmountTerabyteHours,
};
const storageConstants = {
powerUsageEffectiveness: powerUsageEffectiveness,
replicationFactor: this.getReplicationFactor(usageRow),
};
const storageEstimator = usageRow.usageType.includes('SSD')
? this.ssdStorageEstimator
: this.hddStorageEstimator;
const storageFootprint = storageEstimator.estimate([storageUsage], usageRow.region, emissionsFactors, storageConstants)[0];
if (storageFootprint) {
storageFootprint.usesAverageCPUConstant = false;
(0, core_1.accumulateKilowattHours)(domain_1.GCP_CLOUD_CONSTANTS.KILOWATT_HOURS_BY_SERVICE_AND_USAGE_UNIT, usageRow, storageFootprint.kilowattHours, core_1.AccumulateKilowattHoursBy.USAGE_AMOUNT);
}
return storageFootprint;
}
getMemoryFootprintEstimate(usageRow, timestamp, powerUsageEffectiveness, emissionsFactors) {
const memoryUsage = {
timestamp,
gigabyteHours: (0, common_1.convertByteSecondsToGigabyteHours)(usageRow.usageAmount),
};
const memoryConstants = {
powerUsageEffectiveness: powerUsageEffectiveness,
};
const memoryFootprint = this.memoryEstimator.estimate([memoryUsage], usageRow.region, emissionsFactors, memoryConstants)[0];
if (memoryFootprint) {
memoryFootprint.usesAverageCPUConstant = false;
(0, core_1.accumulateKilowattHours)(domain_1.GCP_CLOUD_CONSTANTS.KILOWATT_HOURS_BY_SERVICE_AND_USAGE_UNIT, usageRow, memoryFootprint.kilowattHours, core_1.AccumulateKilowattHoursBy.USAGE_AMOUNT);
}
return memoryFootprint;
}
getNetworkingFootprintEstimate(usageRow, timestamp, powerUsageEffectiveness, emissionsFactors) {
const networkingUsage = {
timestamp,
gigabytes: (0, common_1.convertBytesToGigabytes)(usageRow.usageAmount),
};
const networkingConstants = {
powerUsageEffectiveness: powerUsageEffectiveness,
};
const networkingFootprint = this.networkingEstimator.estimate([networkingUsage], usageRow.region, emissionsFactors, networkingConstants)[0];
if (networkingFootprint) {
networkingFootprint.usesAverageCPUConstant = false;
(0, core_1.accumulateKilowattHours)(domain_1.GCP_CLOUD_CONSTANTS.KILOWATT_HOURS_BY_SERVICE_AND_USAGE_UNIT, usageRow, networkingFootprint.kilowattHours, core_1.AccumulateKilowattHoursBy.USAGE_AMOUNT);
}
return networkingFootprint;
}
isUnknownUsage(usageRow) {
return ((0, common_1.containsAny)(BillingExportTypes_1.UNKNOWN_USAGE_TYPES, usageRow.usageType) ||
(0, common_1.containsAny)(BillingExportTypes_1.UNKNOWN_SERVICE_TYPES, usageRow.serviceName) ||
(0, common_1.containsAny)(BillingExportTypes_1.UNKNOWN_USAGE_UNITS, usageRow.usageUnit) ||
!usageRow.usageType);
}
isMemoryUsage(usageType) {
return ((0, common_1.containsAny)(BillingExportTypes_1.MEMORY_USAGE_TYPES, usageType) &&
!(0, common_1.containsAny)(BillingExportTypes_1.COMPUTE_STRING_FORMATS, usageType));
}
isUnsupportedUsage(usageType) {
return (0, common_1.containsAny)(BillingExportTypes_1.UNSUPPORTED_USAGE_TYPES, usageType);
}
isComputeUsage(usageType) {
return (0, common_1.containsAny)(BillingExportTypes_1.COMPUTE_STRING_FORMATS, usageType);
}
isNetworkingUsage(usageType) {
return (0, common_1.containsAny)(BillingExportTypes_1.NETWORKING_STRING_FORMATS, usageType);
}
getReplicationFactor(usageRow) {
return (ReplicationFactors_1.GCP_REPLICATION_FACTORS_FOR_SERVICES[usageRow.serviceName] &&
ReplicationFactors_1.GCP_REPLICATION_FACTORS_FOR_SERVICES[usageRow.serviceName](usageRow.usageType, usageRow.region));
}
getEstimateForUnknownUsage(rowData) {
const unknownUsage = {
timestamp: rowData.timestamp,
usageAmount: rowData.usageAmount,
usageUnit: rowData.usageUnit,
usageType: rowData.usageType,
replicationFactor: this.getReplicationFactor(rowData),
};
const unknownConstants = {
kilowattHoursByServiceAndUsageUnit: domain_1.GCP_CLOUD_CONSTANTS.KILOWATT_HOURS_BY_SERVICE_AND_USAGE_UNIT,
};
return this.unknownEstimator.estimate([unknownUsage], rowData.region, (0, domain_1.getGCPEmissionsFactors)(), unknownConstants)[0];
}
async getUsage(start, end, grouping, tagNames, projects) {
const startDate = new Date(moment_1.default.utc(start).startOf('day'));
const endDate = new Date(moment_1.default.utc(end).endOf('day'));
const [tags, projectLabels, labels] = this.tagNamesToQueryColumns(tagNames);
const [tagPropertySelections, tagPropertyJoins] = (0, exports.buildTagQuery)('tags', tags);
const [labelPropertySelections, labelPropertyJoins] = (0, exports.buildTagQuery)('labels', labels);
const [projectLabelPropertySelections, projectLabelPropertyJoins] = (0, exports.buildTagQuery)('projectLabels', projectLabels);
const projectFilter = this.buildProjectFilter(projects);
const query = `SELECT DATE_TRUNC(DATE(usage_start_time), ${BillingExportTypes_1.GCP_QUERY_GROUP_BY[grouping]}) as timestamp,
project.id as accountId,
project.name as accountName,
ifnull(location.region, location.location) as region,
service.description as serviceName,
sku.description as usageType,
usage.unit as usageUnit,
system_labels.value AS machineType,
SUM(usage.amount) AS usageAmount,
SUM(cost) AS cost ${tagPropertySelections} ${labelPropertySelections} ${projectLabelPropertySelections}
FROM
\`${this.tableName}\`
LEFT JOIN
UNNEST(system_labels) AS system_labels
ON system_labels.key = "compute.googleapis.com/machine_spec"
${tagPropertyJoins}
${labelPropertyJoins}
${projectLabelPropertyJoins}
WHERE
cost_type != 'rounding_error'
AND usage.unit IN ('byte-seconds'
, 'seconds'
, 'bytes'
, 'requests')
AND usage_start_time BETWEEN TIMESTAMP ('${moment_1.default
.utc(startDate)
.format('YYYY-MM-DDTHH:mm:ssZ')}')
AND TIMESTAMP ('${moment_1.default
.utc(endDate)
.format('YYYY-MM-DDTHH:mm:ssZ')}')
${projectFilter}
GROUP BY
timestamp,
accountId,
accountName,
region,
serviceName,
usageType,
usageUnit,
machineType`;
const job = await this.createQueryJob(query);
return await this.getQueryResults(job);
}
async getQueryResults(job) {
let rows;
try {
this.billingExportTableLogger.info('Getting Big Query Results');
[rows] = await job.getQueryResults();
}
catch (e) {
const { reason, domain, message } = e.errors[0];
throw new Error(`BigQuery get Query Results failed. Reason: ${reason}, Domain: ${domain}, Message: ${message}`);
}
return rows;
}
async createQueryJob(query) {
let job;
try {
;
[job] = await this.bigQuery.createQueryJob({ query });
}
catch (e) {
let errorMessage = e;
if (e.errors) {
const { reason, location, message } = e.errors[0];
errorMessage = `${reason}, Location: ${location}, Message: ${message}`;
}
throw new Error(`BigQuery create Query Job failed. Reason: ${errorMessage}`);
}
return job;
}
tagNamesToQueryColumns(tagNames) {
const tagColumns = {
tag: [],
project: [],
label: [],
};
if (!tagNames || !Array.isArray(tagNames)) {
this.billingExportTableLogger.warn('Configured list of tags is invalid. Tags must be a list of strings. Ignoring tags...');
return Object.values(tagColumns);
}
tagNames.forEach((tag) => {
const [prefix, key] = tag.split(':');
const column = tagColumns[prefix];
if (column) {
column.push(key);
}
else {
this.billingExportTableLogger.warn(`Unknown tag prefix: ${prefix}. Ignoring tag: ${tag}`);
}
});
return Object.values(tagColumns);
}
rawTagsToTagCollection(usageRow) {
const parsedTags = {};
const options = ['tags', 'projectLabels', 'labels'];
options.forEach((option) => {
const tags = usageRow[option];
if (tags) {
tags.split(', ').forEach((tag) => {
const [key, value] = tag.split(': ');
parsedTags[key] = value;
});
}
});
return parsedTags;
}
buildProjectFilter(projects) {
let projectFilter = '';
try {
projectFilter = (0, common_1.buildAccountFilter)(projects, 'project.id');
}
catch (e) {
this.billingExportTableLogger.warn('Configured list of Google Projects is invalid. Projects must be a list of of IDs or objects containing project IDs. Ignoring project filter...');
}
return projectFilter;
}
}
exports.default = BillingExportTable;
const buildTagQuery = (columnName, keys) => {
let propertySelections = '', propertyJoins = '';
if (keys.length > 0) {
propertySelections = `, STRING_AGG(DISTINCT CONCAT(${columnName}.key, ": ", ${columnName}.value), ", ") AS ${columnName}`;
propertyJoins = `\nLEFT JOIN\n UNNEST(${columnName === 'projectLabels' ? 'project.labels' : columnName}) AS ${columnName}\n`;
const keyJoins = keys
.map((tag) => `${columnName}.key = "${tag}"`)
.join(' OR ');
propertyJoins += `ON ${keyJoins}`;
}
return [propertySelections, propertyJoins];
};
exports.buildTagQuery = buildTagQuery;
//# sourceMappingURL=BillingExportTable.js.map