@cloud-copilot/iam-collect
Version:
Collect IAM information from AWS Accounts
86 lines • 3.59 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.runIndexJobs = runIndexJobs;
const jobQueue_js_1 = require("../jobs/jobQueue.js");
const util_js_1 = require("../persistence/util.js");
const log_js_1 = require("../utils/log.js");
/**
* Run index jobs in parallel with a specified concurrency.
*
* @param indexJobs the index jobs to run
* @param storageConfig the storage configuration to use
* @param concurrency the number of jobs to run in parallel
*/
async function runIndexJobs(indexJobs, storageConfig, concurrency) {
const sortedJobs = new Map();
for (const job of indexJobs) {
const partition = job.partition;
const indexerName = job.indexer.name;
if (!sortedJobs.has(partition)) {
sortedJobs.set(partition, new Map());
}
const partitionJobs = sortedJobs.get(partition);
if (!partitionJobs.has(indexerName)) {
partitionJobs.set(indexerName, []);
}
partitionJobs.get(indexerName).push(job);
}
const jobs = [];
for (const [partition, partitionJobs] of sortedJobs.entries()) {
for (const [indexerName, indexerJobs] of partitionJobs.entries()) {
if (indexerJobs.length === 0) {
continue;
}
jobs.push({
properties: {
partition,
indexerName,
numIndexers: indexerJobs.length
},
execute: async (context) => {
log_js_1.log.debug('Running indexers', { workerId: context.workerId, ...context.properties });
await runIndexers(partition, storageConfig, indexerJobs);
log_js_1.log.trace('Finished indexers', { workerId: context.workerId, ...context.properties });
}
});
}
}
log_js_1.log.debug('Starting indexing', { jobs: jobs.length, concurrency });
const indexResults = await (0, jobQueue_js_1.runJobs)(jobs, concurrency);
const failedIndexes = indexResults.filter((r) => r.status === 'rejected');
if (failedIndexes.length > 0) {
log_js_1.log.error('Some indexers failed', { failedJobs: failedIndexes.length });
for (const failedJob of failedIndexes) {
log_js_1.log.error('Indexer failed', failedJob.reason, failedJob.properties);
}
throw new Error(`Failed to index some data. See logs for details.`);
}
log_js_1.log.info('Finished indexing', { jobs: jobs.length });
return indexResults;
}
/**
* Run a set of the same indexer jobs for a given partition.
* All jobs must use the same indexer
*
* @param partition the partition to run the indexer for
* @param storageConfig the storage configuration to use
* @param indexerJobs the indexer jobs to run
*/
async function runIndexers(partition, storageConfig, indexerJobs) {
let saved = false;
let saveAttempts = 0;
const indexer = indexerJobs[0].indexer;
const storage = (0, util_js_1.createStorageClient)(storageConfig, partition);
while (!saved && saveAttempts < 3) {
const cache = await indexer.getCache(storage);
for (const job of indexerJobs) {
await indexer.updateCache(cache.data, job.accountId, job.regions, storage);
}
saved = await indexer.saveCache(storage, cache.data, cache.lockId);
saveAttempts++;
}
if (!saved) {
throw new Error(`Failed to save indexer ${indexer.name} after ${saveAttempts} attempts`);
}
}
//# sourceMappingURL=runIndexers.js.map