@cloud-carbon-footprint/aws/src/__tests__/EBS.test.ts

The core logic to get cloud usage data and estimate energy and carbon emissions from Amazon Web Services.

/*
 * © 2021 Thoughtworks, Inc.
 */

import { mockClient } from 'aws-sdk-client-mock'
import { Logger } from '@cloud-carbon-footprint/common'
import { StorageEstimator } from '@cloud-carbon-footprint/core'
import EBS from '../lib/EBS'
import { AWS_REGIONS } from '../lib'
import { ServiceWrapper } from '../lib'
import { buildCostExplorerGetUsageResponse } from './fixtures/builders'
import {
  AWS_EMISSIONS_FACTORS_METRIC_TON_PER_KWH,
  AWS_CLOUD_CONSTANTS,
} from '../domain'
import { S3Client } from '@aws-sdk/client-s3'
import {
  CostExplorerClient,
  GetCostAndUsageCommand,
} from '@aws-sdk/client-cost-explorer'
import { CloudWatchLogsClient } from '@aws-sdk/client-cloudwatch-logs'
import { CloudWatchClient } from '@aws-sdk/client-cloudwatch'

const costExplorerMock = mockClient(CostExplorerClient)

describe('Ebs', () => {
  const startDate = '2020-06-27'
  const endDate = '2020-06-30'
  const region = AWS_REGIONS.US_EAST_1
  const emissionsFactors = AWS_EMISSIONS_FACTORS_METRIC_TON_PER_KWH
  const constants = {
    powerUsageEffectiveness: AWS_CLOUD_CONSTANTS.getPUE(),
  }

  const getServiceWrapper = () =>
    new ServiceWrapper(
      new CloudWatchClient(),
      new CloudWatchLogsClient(),
      new CostExplorerClient(),
      new S3Client(),
    )

  afterEach(() => {
    costExplorerMock.reset()
    jest.restoreAllMocks()
  })

  it('gets EBS usage', async () => {
    costExplorerMock.on(GetCostAndUsageCommand).resolves(
      buildCostExplorerGetUsageResponse([
        {
          start: startDate,
          amount: 1.2120679,
          keys: ['EBS:VolumeUsage.gp2'],
        },
      ]),
    )

    const ebsService = new EBS(getServiceWrapper())
    const result = await ebsService.getUsage(
      new Date(startDate),
      new Date(endDate),
      region,
    )

    const calls = costExplorerMock.commandCalls(GetCostAndUsageCommand)

    expect(calls).toHaveLength(1)
    expect(calls[0].args[0].input).toEqual({
      Filter: {
        And: [
          {
            Dimensions: {
              Key: 'USAGE_TYPE_GROUP',
              Values: [
                'EC2: EBS - SSD(gp2)',
                'EC2: EBS - SSD(io1)',
                'EC2: EBS - HDD(sc1)',
                'EC2: EBS - HDD(st1)',
                'EC2: EBS - Magnetic',
              ],
            },
          },
          { Dimensions: { Key: 'REGION', Values: [region] } },
        ],
      },
      Granularity: 'DAILY',
      Metrics: ['UsageQuantity'],
      TimePeriod: {
        End: endDate,
        Start: startDate,
      },
      GroupBy: [
        {
          Key: 'USAGE_TYPE',
          Type: 'DIMENSION',
        },
      ],
    })

    expect(result).toEqual([
      {
        terabyteHours: 0.8726888880000001,
        timestamp: new Date(startDate),
        diskType: 'SSD',
      },
    ])
  })

  it('filters out results with no usage', async () => {
    // for valid date ranges, getCostAndUsage API will always return results for the date range, but with all zero usages
    costExplorerMock.on(GetCostAndUsageCommand).resolves(
      buildCostExplorerGetUsageResponse([
        { start: startDate, amount: 0, keys: ['EBS:VolumeUsage.gp2'] },
        {
          start: startDate,
          amount: 1.2120679,
          keys: ['EBS:VolumeUsage.gp2'],
        },
      ]),
    )

    const ebsService = new EBS(getServiceWrapper())

    const result = await ebsService.getUsage(
      new Date(startDate),
      new Date(startDate),
      region,
    )
    expect(result).toEqual([
      {
        terabyteHours: 0.8726888880000001,
        timestamp: new Date(startDate),
        diskType: 'SSD',
      },
    ])
  })

  it('should return empty array if no usage', async () => {
    costExplorerMock
      .on(GetCostAndUsageCommand)
      .resolves(buildCostExplorerGetUsageResponse([]))

    const ebsService = new EBS(getServiceWrapper())
    const result = await ebsService.getUsage(
      new Date(startDate),
      new Date(endDate),
      region,
    )
    expect(result).toEqual([])
  })

  it('filters out results with no Amount', async () => {
    costExplorerMock.on(GetCostAndUsageCommand).resolves(
      buildCostExplorerGetUsageResponse([
        {
          start: startDate,
          amount: undefined,
          keys: ['EBS:VolumeUsage.gp2'],
        },
        {
          start: startDate,
          amount: 1.2120679,
          keys: ['EBS:VolumeUsage.gp2'],
        },
      ]),
    )

    const ebsService = new EBS(getServiceWrapper())
    const result = await ebsService.getUsage(
      new Date(startDate),
      new Date(endDate),
      region,
    )
    expect(result).toEqual([
      {
        terabyteHours: 0.8726888880000001,
        timestamp: new Date(startDate),
        diskType: 'SSD',
      },
    ])
  })

  it('should calculate EBS HDD storage', async () => {
    costExplorerMock
      .on(GetCostAndUsageCommand)
      .resolves(
        buildCostExplorerGetUsageResponse([
          { start: startDate, amount: 1, keys: ['EBS:VolumeUsage.st1'] },
        ]),
      )

    const ebsService = new EBS(getServiceWrapper())
    const result = await ebsService.getUsage(
      new Date(startDate),
      new Date(endDate),
      region,
    )
    expect(result).toEqual([
      {
        terabyteHours: 0.72,
        timestamp: new Date(startDate),
        diskType: 'HDD',
      },
    ])
  })

  it('should get estimates for ebs st1 HDD storage', async () => {
    costExplorerMock
      .on(GetCostAndUsageCommand)
      .resolves(
        buildCostExplorerGetUsageResponse([
          { start: startDate, amount: 1, keys: ['EBS:VolumeUsage.st1'] },
        ]),
      )

    const ebsService = new EBS(getServiceWrapper())
    const hddStorageEstimator = new StorageEstimator(
      AWS_CLOUD_CONSTANTS.HDDCOEFFICIENT,
    )
    const result = await ebsService.getEstimates(
      new Date(startDate),
      new Date(endDate),
      region,
      emissionsFactors,
      constants,
    )
    expect(result).toEqual(
      hddStorageEstimator.estimate(
        [{ terabyteHours: 0.72, timestamp: new Date(startDate) }],
        region,
        emissionsFactors,
        constants,
      ),
    )
  })

  it('should get estimates for magnetic EBS HDD storage', async () => {
    costExplorerMock
      .on(GetCostAndUsageCommand)
      .resolves(
        buildCostExplorerGetUsageResponse([
          { start: startDate, amount: 1, keys: ['EBS:VolumeUsage'] },
        ]),
      )

    const ebsService = new EBS(getServiceWrapper())
    const hddStorageEstimator = new StorageEstimator(
      AWS_CLOUD_CONSTANTS.HDDCOEFFICIENT,
    )
    const result = await ebsService.getEstimates(
      new Date(startDate),
      new Date(endDate),
      region,
      emissionsFactors,
      constants,
    )
    expect(result).toEqual(
      hddStorageEstimator.estimate(
        [{ terabyteHours: 0.72, timestamp: new Date(startDate) }],
        region,
        emissionsFactors,
        constants,
      ),
    )
  })

  it('should get estimates for magnetic sc1 HDD storage', async () => {
    costExplorerMock
      .on(GetCostAndUsageCommand)
      .resolves(
        buildCostExplorerGetUsageResponse([
          { start: startDate, amount: 1, keys: ['EBS:VolumeUsage.sc1'] },
        ]),
      )

    const ebsService = new EBS(getServiceWrapper())
    const hddStorageEstimator = new StorageEstimator(
      AWS_CLOUD_CONSTANTS.HDDCOEFFICIENT,
    )
    const result = await ebsService.getEstimates(
      new Date(startDate),
      new Date(endDate),
      region,
      emissionsFactors,
      constants,
    )
    expect(result).toEqual(
      hddStorageEstimator.estimate(
        [{ terabyteHours: 0.72, timestamp: new Date(startDate) }],
        region,
        emissionsFactors,
        constants,
      ),
    )
  })

  it('should get estimates for EBS SSD storage', async () => {
    costExplorerMock
      .on(GetCostAndUsageCommand)
      .resolves(
        buildCostExplorerGetUsageResponse([
          { start: startDate, amount: 1, keys: ['EBS:VolumeUsage.piops'] },
        ]),
      )

    const ebsService = new EBS(getServiceWrapper())
    const sddStorageEstimator = new StorageEstimator(
      AWS_CLOUD_CONSTANTS.SSDCOEFFICIENT,
    )
    const result = await ebsService.getEstimates(
      new Date(startDate),
      new Date(endDate),
      region,
      emissionsFactors,
      constants,
    )
    expect(result).toEqual(
      sddStorageEstimator.estimate(
        [{ terabyteHours: 0.72, timestamp: new Date(startDate) }],
        region,
        emissionsFactors,
        constants,
      ),
    )
  })

  it('should filter unexpected cost explorer volume name', async () => {
    jest.spyOn(Logger.prototype, 'warn').mockImplementation()
    costExplorerMock
      .on(GetCostAndUsageCommand)
      .resolves(
        buildCostExplorerGetUsageResponse([
          { start: startDate, amount: 1, keys: ['EBS:anything'] },
        ]),
      )

    const ebsService = new EBS(getServiceWrapper())
    const result = await ebsService.getEstimates(
      new Date(startDate),
      new Date(endDate),
      region,
      emissionsFactors,
      constants,
    )
    expect(result).toEqual([])
  })

  it('should log warning if unexpected cost explorer volume name', async () => {
    jest.spyOn(Logger.prototype, 'warn').mockImplementation()
    costExplorerMock
      .on(GetCostAndUsageCommand)
      .resolves(
        buildCostExplorerGetUsageResponse([
          { start: startDate, amount: 1, keys: ['EBS:anything'] },
        ]),
      )

    const ebsService = new EBS(getServiceWrapper())
    await ebsService.getEstimates(
      new Date(startDate),
      new Date(endDate),
      region,
      emissionsFactors,
      constants,
    )
    expect(Logger.prototype.warn).toHaveBeenCalledWith(
      'Unexpected Cost explorer Dimension Name: EBS:anything',
    )
  })

  it('should get estimates for EBS SDD and HDD storage', async () => {
    costExplorerMock.on(GetCostAndUsageCommand).resolves(
      buildCostExplorerGetUsageResponse([
        { start: startDate, amount: 1, keys: ['EBS:VolumeUsage.st1'] },
        { start: startDate, amount: 1, keys: ['EBS:VolumeUsage.gp2'] },
      ]),
    )

    const ebsService = new EBS(getServiceWrapper())
    const hddStorageEstimator = new StorageEstimator(
      AWS_CLOUD_CONSTANTS.HDDCOEFFICIENT,
    )
    const sddStorageEstimator = new StorageEstimator(
      AWS_CLOUD_CONSTANTS.SSDCOEFFICIENT,
    )

    const result = await ebsService.getEstimates(
      new Date(startDate),
      new Date(endDate),
      region,
      emissionsFactors,
      constants,
    )

    const ssdEstimates = sddStorageEstimator.estimate(
      [{ terabyteHours: 0.72, timestamp: new Date(startDate) }],
      region,
      emissionsFactors,
      constants,
    )
    const hddEstimates = hddStorageEstimator.estimate(
      [{ terabyteHours: 0.72, timestamp: new Date(startDate) }],
      region,
      emissionsFactors,
      constants,
    )
    expect(result).toEqual([
      {
        timestamp: new Date(startDate),
        kilowattHours:
          ssdEstimates[0].kilowattHours + hddEstimates[0].kilowattHours,
        co2e: ssdEstimates[0].co2e + hddEstimates[0].co2e,
      },
    ])
  })
})