@cloud-copilot/iam-collect
Version:
Collect IAM information from AWS Accounts
229 lines • 11.8 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadData = downloadData;
const job_1 = require("@cloud-copilot/job");
const auth_js_1 = require("../aws/auth.js");
const coreAuth_js_1 = require("../aws/coreAuth.js");
const config_js_1 = require("../config/config.js");
const dataSource_js_1 = require("../config/dataSource.js");
const partitionDefaults_js_1 = require("../config/partitionDefaults.js");
const indexMap_js_1 = require("../indexing/indexMap.js");
const runIndexers_js_1 = require("../indexing/runIndexers.js");
const jobRunner_js_1 = require("../jobs/jobRunner.js");
const util_js_1 = require("../jobs/util.js");
const util_js_2 = require("../persistence/util.js");
const regions_js_1 = require("../regions.js");
const services_js_1 = require("../services.js");
const syncMap_js_1 = require("../syncs/syncMap.js");
const log_1 = require("@cloud-copilot/log");
/**
* Download data from AWS services.
*
* @param configs the configurations to use for the download
* @param accountIds the account IDs to download data for
* @param regions the regions to download data from
* @param services the services to download data from
* @param concurrency the maximum number of concurrent downloads
* @param skipIndex whether to skip indexing the downloaded data
* @param writeOnly only write the data returned from AWS, do not delete any existing data
*/
async function downloadData(configs, accountIds, regions, services, concurrency, skipIndex, writeOnly) {
if (concurrency === undefined || concurrency <= 0) {
concurrency = (0, util_js_1.defaultConcurrency)();
}
const deleteData = !writeOnly;
const defaultAuthConfig = (0, config_js_1.getDefaultAuthConfig)(configs);
const defaultCredentials = await (0, coreAuth_js_1.getNewInitialCredentials)(defaultAuthConfig, {
phase: 'initial default credentials'
});
if (accountIds.length === 0) {
/*
If no accounts were passed in:
1. Check the config for any configured accounts
2. If no configured accounts, use the default credentials to get the current account
*/
const configuredAccounts = (0, config_js_1.getConfiguredAccounts)(configs);
if (configuredAccounts.length > 0) {
accountIds = configuredAccounts;
}
else {
accountIds = [defaultCredentials.accountId];
}
}
// Create the client pool and initialize it
const dataSourceConfig = (0, config_js_1.getConfiguredDataSource)(configs);
const clientPool = await (0, dataSource_js_1.createClientPool)(dataSourceConfig);
await clientPool.init();
const defaultPartition = defaultCredentials.partition;
const storageConfig = (0, config_js_1.getStorageConfig)(configs);
if (!storageConfig) {
throw new Error('No storage configuration found. Cannot download data.');
}
const storage = (0, util_js_2.createStorageClient)(storageConfig, defaultPartition, deleteData);
const indexJobs = [];
log_1.log.debug('Starting download runner', { concurrency });
const downloadRunner = new jobRunner_js_1.JobRunner(concurrency);
const workerPool = new job_1.ConcurrentWorkerPool(concurrency, log_1.log);
for (const accountId of accountIds) {
log_1.log.info('Queuing downloads for account', { accountId });
const authForAccount = (0, config_js_1.getAccountAuthConfig)(accountId, configs);
const credentials = await getCredentialsForSync(clientPool, defaultPartition, accountId, authForAccount);
const accountPartition = credentials.partition;
if (accountPartition !== defaultPartition) {
//TODO: Consider updating the storage client to handle multiple partitions
log_1.log.error(`Account ${accountId} is in partition ${accountPartition}, but the default account is in partition ${defaultPartition}. This is not supported.`);
throw new Error('Cannot download data for multiple partitions in one run.');
}
const partitionConfig = (0, partitionDefaults_js_1.getPartitionDefaults)(accountPartition);
const accountConfigs = [partitionConfig, ...configs];
const accountRegions = await getAccountRegions(regions, accountId, configs, credentials, clientPool);
if (services.length === 0) {
services = services_js_1.allServices;
}
const syncOptions = {
workerPool,
writeOnly,
clientPool
};
const enabledServices = (0, config_js_1.servicesForAccount)(accountId, accountConfigs, services);
for (const service of enabledServices) {
log_1.log.info('Queuing downloads', { service, accountId });
const serviceRegions = (0, config_js_1.regionsForService)(service, accountId, accountConfigs, accountRegions);
//Global syncs for the service
const globalSyncs = (0, syncMap_js_1.getGlobalSyncsForService)(service);
const globalRegion = serviceRegions.at(0);
const globalConfig = (0, config_js_1.accountServiceRegionConfig)(service, accountId, globalRegion, accountConfigs);
for (const globalSync of globalSyncs) {
const customConfig = (0, config_js_1.customConfigForSync)(service, globalSync.name, accountId, globalRegion, accountConfigs);
if (!clientPool.isSyncSupported(service, globalSync.name, globalRegion)) {
log_1.log.info({
skippedSync: true,
service,
accountId,
sync: globalSync.name,
clientPool: clientPool.constructor.name
}, 'Skipping global sync, not supported by data source');
continue;
}
downloadRunner.enqueue({
properties: { service, accountId, sync: globalSync.name },
execute: async (context) => {
const logDetails = {
workerId: context.workerId,
...context.properties
};
const globalCredentials = await getCredentialsForSync(clientPool, defaultPartition, accountId, globalConfig.auth);
log_1.log.debug(logDetails, 'Executing global sync');
await globalSync.execute(accountId, globalRegion, globalCredentials, storage, globalConfig.endpoint, { ...syncOptions, customConfig });
log_1.log.trace(logDetails, 'Finished global sync');
}
});
}
const regionalSyncs = (0, syncMap_js_1.getRegionalSyncsForService)(service);
//Regional syncs for the service
for (const region of serviceRegions) {
if (regionalSyncs.length === 0) {
continue;
}
log_1.log.debug({ service, accountId, region }, 'Queuing regional syncs');
const asrConfig = (0, config_js_1.accountServiceRegionConfig)(service, accountId, region, accountConfigs);
for (const sync of regionalSyncs) {
const includeSync = (0, config_js_1.syncEnabledForRegion)(accountId, service, sync.name, accountConfigs, region);
if (!includeSync) {
log_1.log.debug({ service, accountId, region, syncName: sync.name }, 'Skipping regional sync');
continue;
}
if (!clientPool.isSyncSupported(service, sync.name, region)) {
log_1.log.info({
skippedSync: true,
service,
accountId,
sync: sync.name,
region,
clientPool: clientPool.constructor.name
}, 'Skipping regional sync, not supported by data source');
continue;
}
const customConfig = (0, config_js_1.customConfigForSync)(service, sync.name, accountId, region, accountConfigs);
downloadRunner.enqueue({
properties: { service, accountId, region, sync: sync.name },
execute: async (context) => {
const logDetails = {
workerId: context.workerId,
...context.properties
};
log_1.log.debug(logDetails, 'Executing regional sync');
const regionalCredentials = await getCredentialsForSync(clientPool, accountPartition, accountId, asrConfig.auth);
await sync.execute(accountId, region, regionalCredentials, storage, asrConfig.endpoint, { ...syncOptions, customConfig });
log_1.log.trace(logDetails, 'Finished regional sync');
}
});
}
}
const indexers = (0, indexMap_js_1.getIndexersForService)(service);
for (const indexer of indexers) {
indexJobs.push({
indexer,
partition: accountPartition,
accountId,
regions: serviceRegions
});
}
}
}
log_1.log.info('Waiting for downloads to complete');
await downloadRunner.finishAllWork();
log_1.log.info('Finished downloads', { jobs: downloadRunner.getResults().length });
const failedJobs = downloadRunner.getResults().filter((r) => r.status === 'rejected');
if (failedJobs.length > 0) {
log_1.log.error('Some downloads failed', { failedJobs: failedJobs.length });
for (const failedJob of failedJobs) {
log_1.log.error('Download failed', failedJob.reason, failedJob.properties);
}
throw new Error(`Failed to download some data. See logs for details.`);
}
if (skipIndex) {
log_1.log.info('Skipping indexing');
return;
}
await (0, runIndexers_js_1.runIndexJobs)(indexJobs, storageConfig, concurrency);
}
/**
* Get the regions to use for an account
*
* @param allRegions the list of all available regions
* @param accountId the ID of the account
* @param configs the iam-collect configs
* @param accountCredentials the credentials for the account
* @returns a list of regions to use for the account
*/
async function getAccountRegions(allRegions, accountId, configs, accountCredentials, clientPool) {
if (allRegions.length > 0) {
return allRegions;
}
const configuredRegions = (0, config_js_1.configuredRegionListForAccount)(configs, accountId);
if (configuredRegions) {
log_1.log.debug('Using configured regions', { regions: configuredRegions, accountId });
return configuredRegions;
}
else {
log_1.log.debug('No configured regions found, discovering regions for account', {
accountId
});
return (0, regions_js_1.getEnabledRegions)(accountCredentials, clientPool);
}
}
async function getCredentialsForSync(clientPool, currentPartition, accountId, authConfig) {
if (clientPool.requiresAwsCredentials()) {
return (0, auth_js_1.getCredentials)(accountId, authConfig);
}
return {
accountId: accountId,
partition: currentPartition,
cacheKey: 'no-credentials',
provider: async () => {
throw new Error('No credentials required for this data source');
}
};
}
//# sourceMappingURL=download.js.map