@cloud-copilot/iam-collect
Version:
Collect IAM information from AWS Accounts
132 lines • 6.75 kB
JavaScript
import { getCredentials } from '../aws/auth.js';
import { getNewInitialCredentials } from '../aws/coreAuth.js';
import { accountServiceRegionConfig, getAccountAuthConfig, getConfiguredAccounts, getDefaultAuthConfig, getStorageConfig, regionsForService, servicesForAccount, syncEnabledForRegion } from '../config/config.js';
import { getPartitionDefaults } from '../config/partitionDefaults.js';
import { getIndexersForService } from '../indexing/indexMap.js';
import { runIndexJobs } from '../indexing/runIndexers.js';
import { runJobs } from '../jobs/jobQueue.js';
import { defaultConcurrency } from '../jobs/util.js';
import { createStorageClient } from '../persistence/util.js';
import { getEnabledRegions } from '../regions.js';
import { allServices } from '../services.js';
import { getGlobalSyncsForService, getRegionalSyncsForService } from '../syncs/syncMap.js';
import { log } from '../utils/log.js';
export async function downloadData(configs, accountIds, regions, services, concurrency, skipIndex) {
if (concurrency === undefined || concurrency <= 0) {
concurrency = defaultConcurrency();
}
if (accountIds.length === 0) {
const configuredAccounts = getConfiguredAccounts(configs);
if (configuredAccounts.length > 0) {
accountIds = configuredAccounts;
}
else {
const defaultAuthConfig = getDefaultAuthConfig(configs);
const defaultCredentials = await getNewInitialCredentials(defaultAuthConfig, {
phase: 'discover account'
});
accountIds = [defaultCredentials.accountId];
}
}
const storageConfig = getStorageConfig(configs);
if (!storageConfig) {
throw new Error('No storage configuration found. Cannot download data.');
}
const jobs = [];
const indexJobs = [];
for (const accountId of accountIds) {
log.info('Queuing downloads for account', { accountId });
const authForAccount = getAccountAuthConfig(accountId, configs);
const credentials = await getCredentials(accountId, authForAccount);
const partition = credentials.partition;
const partitionConfig = getPartitionDefaults(partition);
const accountConfigs = [partitionConfig, ...configs];
if (regions.length === 0) {
regions = await getEnabledRegions(credentials);
}
const storage = createStorageClient(storageConfig, partition);
if (services.length === 0) {
services = allServices;
}
const syncOptions = {};
const enabledServices = servicesForAccount(accountId, accountConfigs, services);
for (const service of enabledServices) {
log.info('Queuing downloads', { service, accountId });
const serviceRegions = regionsForService(service, accountId, accountConfigs, regions);
//Global syncs for the service
const globalSyncs = getGlobalSyncsForService(service);
const globalRegion = serviceRegions.at(0);
const globalConfig = accountServiceRegionConfig(service, accountId, globalRegion, accountConfigs);
for (const globalSync of globalSyncs) {
jobs.push({
properties: { service, accountId, sync: globalSync.name },
execute: async (context) => {
const logDetails = {
workerId: context.workerId,
...context.properties
};
const globalCredentials = await getCredentials(accountId, globalConfig.auth);
log.debug(logDetails, 'Executing global sync');
await globalSync.execute(accountId, globalRegion, globalCredentials, storage, globalConfig.endpoint, syncOptions);
log.trace(logDetails, 'Finished global sync');
}
});
}
const regionalSyncs = getRegionalSyncsForService(service);
//Regional syncs for the service
for (const region of serviceRegions) {
log.debug({ service, accountId, region }, 'Queuing regional syncs');
if (regionalSyncs.length === 0) {
continue;
}
const asrConfig = accountServiceRegionConfig(service, accountId, region, accountConfigs);
for (const sync of regionalSyncs) {
const includeSync = syncEnabledForRegion(accountId, service, sync.name, accountConfigs, region);
if (!includeSync) {
log.debug({ service, accountId, region, syncName: sync.name }, 'Skipping regional sync');
continue;
}
jobs.push({
properties: { service, accountId, region, sync: sync.name },
execute: async (context) => {
const logDetails = {
workerId: context.workerId,
...context.properties
};
log.debug(logDetails, 'Executing regional sync');
const regionalCredentials = await getCredentials(accountId, asrConfig.auth);
await sync.execute(accountId, region, regionalCredentials, storage, asrConfig.endpoint, syncOptions);
log.debug(logDetails, 'Finished regional sync');
}
});
}
}
const indexers = getIndexersForService(service);
for (const indexer of indexers) {
indexJobs.push({
indexer,
partition,
accountId,
regions: serviceRegions
});
}
}
}
log.debug('Starting downloads', { jobs: jobs.length, concurrency });
const results = await runJobs(jobs, concurrency);
const failedJobs = results.filter((r) => r.status === 'rejected');
if (failedJobs.length > 0) {
log.error('Some downloads failed', { failedJobs: failedJobs.length });
for (const failedJob of failedJobs) {
log.error('Download failed', failedJob.reason, failedJob.properties);
}
throw new Error(`Failed to download some data. See logs for details.`);
}
log.info('Finished downloads', { jobs: jobs.length });
if (skipIndex) {
log.info('Skipping indexing');
return;
}
await runIndexJobs(indexJobs, storageConfig, concurrency);
}
//# sourceMappingURL=download.js.map