@cloud-copilot/iam-collect
Version:
Collect IAM information from AWS Accounts
135 lines • 7.16 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadData = downloadData;
const auth_js_1 = require("../aws/auth.js");
const coreAuth_js_1 = require("../aws/coreAuth.js");
const config_js_1 = require("../config/config.js");
const partitionDefaults_js_1 = require("../config/partitionDefaults.js");
const indexMap_js_1 = require("../indexing/indexMap.js");
const runIndexers_js_1 = require("../indexing/runIndexers.js");
const jobQueue_js_1 = require("../jobs/jobQueue.js");
const util_js_1 = require("../jobs/util.js");
const util_js_2 = require("../persistence/util.js");
const regions_js_1 = require("../regions.js");
const services_js_1 = require("../services.js");
const syncMap_js_1 = require("../syncs/syncMap.js");
const log_js_1 = require("../utils/log.js");
async function downloadData(configs, accountIds, regions, services, concurrency, skipIndex) {
if (concurrency === undefined || concurrency <= 0) {
concurrency = (0, util_js_1.defaultConcurrency)();
}
if (accountIds.length === 0) {
const configuredAccounts = (0, config_js_1.getConfiguredAccounts)(configs);
if (configuredAccounts.length > 0) {
accountIds = configuredAccounts;
}
else {
const defaultAuthConfig = (0, config_js_1.getDefaultAuthConfig)(configs);
const defaultCredentials = await (0, coreAuth_js_1.getNewInitialCredentials)(defaultAuthConfig, {
phase: 'discover account'
});
accountIds = [defaultCredentials.accountId];
}
}
const storageConfig = (0, config_js_1.getStorageConfig)(configs);
if (!storageConfig) {
throw new Error('No storage configuration found. Cannot download data.');
}
const jobs = [];
const indexJobs = [];
for (const accountId of accountIds) {
log_js_1.log.info('Queuing downloads for account', { accountId });
const authForAccount = (0, config_js_1.getAccountAuthConfig)(accountId, configs);
const credentials = await (0, auth_js_1.getCredentials)(accountId, authForAccount);
const partition = credentials.partition;
const partitionConfig = (0, partitionDefaults_js_1.getPartitionDefaults)(partition);
const accountConfigs = [partitionConfig, ...configs];
if (regions.length === 0) {
regions = await (0, regions_js_1.getEnabledRegions)(credentials);
}
const storage = (0, util_js_2.createStorageClient)(storageConfig, partition);
if (services.length === 0) {
services = services_js_1.allServices;
}
const syncOptions = {};
const enabledServices = (0, config_js_1.servicesForAccount)(accountId, accountConfigs, services);
for (const service of enabledServices) {
log_js_1.log.info('Queuing downloads', { service, accountId });
const serviceRegions = (0, config_js_1.regionsForService)(service, accountId, accountConfigs, regions);
//Global syncs for the service
const globalSyncs = (0, syncMap_js_1.getGlobalSyncsForService)(service);
const globalRegion = serviceRegions.at(0);
const globalConfig = (0, config_js_1.accountServiceRegionConfig)(service, accountId, globalRegion, accountConfigs);
for (const globalSync of globalSyncs) {
jobs.push({
properties: { service, accountId, sync: globalSync.name },
execute: async (context) => {
const logDetails = {
workerId: context.workerId,
...context.properties
};
const globalCredentials = await (0, auth_js_1.getCredentials)(accountId, globalConfig.auth);
log_js_1.log.debug(logDetails, 'Executing global sync');
await globalSync.execute(accountId, globalRegion, globalCredentials, storage, globalConfig.endpoint, syncOptions);
log_js_1.log.trace(logDetails, 'Finished global sync');
}
});
}
const regionalSyncs = (0, syncMap_js_1.getRegionalSyncsForService)(service);
//Regional syncs for the service
for (const region of serviceRegions) {
log_js_1.log.debug({ service, accountId, region }, 'Queuing regional syncs');
if (regionalSyncs.length === 0) {
continue;
}
const asrConfig = (0, config_js_1.accountServiceRegionConfig)(service, accountId, region, accountConfigs);
for (const sync of regionalSyncs) {
const includeSync = (0, config_js_1.syncEnabledForRegion)(accountId, service, sync.name, accountConfigs, region);
if (!includeSync) {
log_js_1.log.debug({ service, accountId, region, syncName: sync.name }, 'Skipping regional sync');
continue;
}
jobs.push({
properties: { service, accountId, region, sync: sync.name },
execute: async (context) => {
const logDetails = {
workerId: context.workerId,
...context.properties
};
log_js_1.log.debug(logDetails, 'Executing regional sync');
const regionalCredentials = await (0, auth_js_1.getCredentials)(accountId, asrConfig.auth);
await sync.execute(accountId, region, regionalCredentials, storage, asrConfig.endpoint, syncOptions);
log_js_1.log.debug(logDetails, 'Finished regional sync');
}
});
}
}
const indexers = (0, indexMap_js_1.getIndexersForService)(service);
for (const indexer of indexers) {
indexJobs.push({
indexer,
partition,
accountId,
regions: serviceRegions
});
}
}
}
log_js_1.log.debug('Starting downloads', { jobs: jobs.length, concurrency });
const results = await (0, jobQueue_js_1.runJobs)(jobs, concurrency);
const failedJobs = results.filter((r) => r.status === 'rejected');
if (failedJobs.length > 0) {
log_js_1.log.error('Some downloads failed', { failedJobs: failedJobs.length });
for (const failedJob of failedJobs) {
log_js_1.log.error('Download failed', failedJob.reason, failedJob.properties);
}
throw new Error(`Failed to download some data. See logs for details.`);
}
log_js_1.log.info('Finished downloads', { jobs: jobs.length });
if (skipIndex) {
log_js_1.log.info('Skipping indexing');
return;
}
await (0, runIndexers_js_1.runIndexJobs)(indexJobs, storageConfig, concurrency);
}
//# sourceMappingURL=download.js.map