@csermet/multiprovider
Version:
cloud-graph provider plugin for AWS used to fetch AWS cloud data.
408 lines (407 loc) • 14.5 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.publicBucketGrant = exports.awsBucketItemsLimit = void 0;
const sdk_1 = __importDefault(require("@cloudgraph/sdk"));
const groupBy_1 = __importDefault(require("lodash/groupBy"));
const isEmpty_1 = __importDefault(require("lodash/isEmpty"));
const aws_sdk_1 = __importDefault(require("aws-sdk"));
const s3_1 = __importDefault(require("aws-sdk/clients/s3"));
const translations_1 = __importDefault(require("../../properties/translations"));
const logger_1 = __importDefault(require("../../properties/logger"));
const utils_1 = require("../../utils");
const ids_1 = require("../../utils/ids");
const errorLog_1 = __importDefault(require("../../utils/errorLog"));
const format_1 = require("../../utils/format");
const lt = { ...logger_1.default };
const { logger } = sdk_1.default;
const serviceName = 'S3';
const errorLog = new errorLog_1.default(serviceName);
const endpoint = utils_1.initTestEndpoint(serviceName);
// S3 Properties
exports.awsBucketItemsLimit = 1000;
exports.publicBucketGrant = 'http://acs.amazonaws.com/groups/global/AllUsers';
const getAccountPublicAccessBlock = async ({ region, accountId, }) => new Promise(resolve => {
const s3Control = new aws_sdk_1.default.S3Control({
region: region,
});
s3Control.getPublicAccessBlock({
AccountId: accountId,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.PublicAccessBlockConfiguration);
}
resolve({});
});
});
const getBucketAcl = async (s3, name) => new Promise(resolve => {
s3.getBucketAcl({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data);
}
resolve({ Owner: {}, Grants: [] });
});
});
const getBucketAcceleration = async (s3, name) => new Promise(resolve => {
s3.getBucketAccelerateConfiguration({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.Status);
}
resolve(translations_1.default.disabled);
});
});
const getBucketCors = async (s3, name) => new Promise(resolve => {
s3.getBucketCors({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.CORSRules);
}
resolve([]);
});
});
const getBucketEncryption = async (s3, name) => new Promise(resolve => {
s3.getBucketEncryption({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.ServerSideEncryptionConfiguration);
}
resolve({});
});
});
const getBucketLifecycleConfiguration = async (s3, name) => new Promise(resolve => {
s3.getBucketLifecycleConfiguration({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.Rules);
}
resolve([]);
});
});
const checkIfBucketLocationMatchesCrawlRegion = async (s3, name, region) => new Promise(resolve => {
s3.getBucketLocation({ Bucket: name }, (err, data) => {
if (!isEmpty_1.default(data)) {
const foundRegion = data.LocationConstraint !== ''
? data.LocationConstraint
: 'us-east-1';
resolve(foundRegion === region);
}
resolve(false);
});
});
const getBucketLogging = async (s3, name) => new Promise(resolve => {
s3.getBucketLogging({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.LoggingEnabled);
}
resolve({});
});
});
// GetBucketPolicy
const getBucketPolicy = async (s3, name) => new Promise(resolve => {
s3.getBucketPolicy({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.Policy);
}
resolve('');
});
});
const getBucketPolicyStatus = async (s3, name) => new Promise(resolve => {
s3.getBucketPolicyStatus({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.PolicyStatus);
}
resolve({});
});
});
const getBucketPublicAccessBlock = async (s3, name) => new Promise(resolve => {
s3.getPublicAccessBlock({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.PublicAccessBlockConfiguration);
}
resolve({});
});
});
const getBucketReplication = async (s3, name) => new Promise(resolve => {
s3.getBucketReplication({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.ReplicationConfiguration);
}
resolve({});
});
});
const getBucketRequestPayment = async (s3, name) => new Promise(resolve => {
s3.getBucketRequestPayment({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.Payer);
}
resolve('');
});
});
const getBucketVersioning = async (s3, name) => new Promise(resolve => {
s3.getBucketVersioning({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data);
}
resolve({});
});
});
const getBucketTagging = async (s3, name) => new Promise(resolve => {
s3.getBucketTagging({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data.TagSet);
}
resolve([]);
});
});
const getBucketWebsite = async (s3, name) => new Promise(resolve => {
s3.getBucketWebsite({
Bucket: name,
}, (err, data) => {
if (!isEmpty_1.default(data)) {
resolve(data);
}
resolve({});
});
});
const getBucketNotificationConfiguration = async (s3, name) => new Promise(resolve => {
s3.getBucketNotificationConfiguration({
Bucket: name,
}, (err, data) => {
if (err) {
errorLog.generateAwsErrorLog({
functionName: 's3:getBucketNotificationConfiguration',
err,
});
}
if (!isEmpty_1.default(data)) {
resolve(data);
}
resolve({});
});
});
const getBucketAdditionalInfo = async (s3, name) => new Promise(async (resolve) => {
const promises = [
getBucketAcceleration(s3, name),
getBucketAcl(s3, name),
getBucketCors(s3, name),
getBucketEncryption(s3, name),
getBucketLifecycleConfiguration(s3, name),
getBucketLogging(s3, name),
getBucketPolicy(s3, name),
getBucketPolicyStatus(s3, name),
getBucketPublicAccessBlock(s3, name),
getBucketReplication(s3, name),
getBucketRequestPayment(s3, name),
getBucketTagging(s3, name),
getBucketVersioning(s3, name),
getBucketWebsite(s3, name),
getBucketNotificationConfiguration(s3, name),
];
const [AccelerationConfig, AclInfo, CorsInfo, EncryptionInfo, LifecycleConfig, LoggingInfo, Policy, PolicyStatus, PublicAccessBlockConfig, ReplicationConfig, ReqPaymentConfig, Tags, VersioningInfo, WebsiteInfo, NotificationConfig,] = (await Promise.allSettled(promises)).map(
/** We force the PromiseFulfilledResult interface
* because all promises that we input to Promise.allSettled
* are always resolved, that way we suppress the compiler error complaining
* that Promise.allSettled returns an Array<PromiseFulfilledResult | PromiseRejectedResult>
* and that the value property doesn't exist for the PromiseRejectedResult interface
* */
i => i.value);
const { Owner: BucketOwnerData, Grants } = AclInfo;
resolve({
AccelerationConfig,
AclInfo,
BucketOwnerData,
CorsInfo,
Grants,
EncryptionInfo,
LifecycleConfig,
LoggingInfo,
Policy,
PolicyStatus,
PublicAccessBlockConfig,
ReplicationConfig,
ReqPaymentConfig,
Tags: format_1.convertAwsTagsToTagMap(Tags),
VersioningInfo,
StaticWebsiteInfo: WebsiteInfo,
NotificationConfiguration: NotificationConfig,
});
});
const listBucketsForRegion = async (s3, resolveRegion) => new Promise(resolve => {
s3.listBuckets((err, data) => {
/**
* No Data for the region
*/
if (isEmpty_1.default(data)) {
return resolveRegion();
}
if (err) {
errorLog.generateAwsErrorLog({
functionName: 's3:listBuckets',
err,
});
}
const { Buckets: buckets = [], Owner: ownerId } = data;
/**
* No Buckets Found
*/
if (isEmpty_1.default(buckets)) {
return resolveRegion();
}
resolve({ buckets, ownerId });
});
});
const listBucketObjects = async (s3, name) => new Promise(resolve => {
const contents = [];
/**
* S3 Buckets can get quite large, so we limit the total number
* Of objects returned per bucket to 1000
*/
const opts = {
Bucket: name,
MaxKeys: exports.awsBucketItemsLimit,
};
const listAllObjects = (token) => {
if (token) {
opts.ContinuationToken = token;
}
s3.listObjectsV2(opts, (err, data) => {
const { Contents = [], IsTruncated, NextContinuationToken } = data || {};
if (err) {
errorLog.generateAwsErrorLog({
functionName: 's3:listObjectsV2',
err,
});
}
contents.push(...Contents);
if (IsTruncated && contents.length < exports.awsBucketItemsLimit) {
logger.debug(lt.foundAnotherThousand);
listAllObjects(NextContinuationToken);
}
else {
resolve(contents);
}
});
};
listAllObjects();
});
exports.default = async ({ regions, config, account, }) => new Promise(async (resolve) => {
const bucketData = [];
const regionPromises = [];
const bucketObjectListPromises = [];
const additionalInfoPromises = [];
regions.split(',').map((region) => {
// TODO: temp implementation to add account level public access block to bucket level
// need to find a better place/way to put the data
const regionPromise = new Promise(async (resolveRegion) => {
const { BlockPublicAcls, IgnorePublicAcls, BlockPublicPolicy, RestrictPublicBuckets, } = await getAccountPublicAccessBlock({
region,
accountId: account,
});
const s3 = new s3_1.default({
...config,
region,
endpoint,
s3ForcePathStyle: true,
});
const { buckets } = await listBucketsForRegion(s3, resolveRegion);
if (!isEmpty_1.default(buckets)) {
// Checks the location restraint (region) for each bucket
const areBucketsInSameRegionCheckPromise = await Promise.all(buckets.map((bucket) => checkIfBucketLocationMatchesCrawlRegion(s3, bucket.Name, region)));
buckets.map(async (bucket, idx) => {
const isBucketInSameRegion = areBucketsInSameRegionCheckPromise[idx];
if (isBucketInSameRegion) {
bucketData.push({
Id: ids_1.gets3BucketId(bucket.Name),
Name: bucket.Name,
region,
CreationDate: bucket.CreationDate,
Tags: {},
AccountLevelBlockPublicAcls: BlockPublicAcls,
AccountLevelIgnorePublicAcls: IgnorePublicAcls,
AccountLevelBlockPublicPolicy: BlockPublicPolicy,
AccountLevelRestrictPublicBuckets: RestrictPublicBuckets,
});
}
});
}
resolveRegion();
});
regionPromises.push(regionPromise);
});
await Promise.all(regionPromises);
logger.debug(lt.fetchedS3Buckets(bucketData.length));
bucketData.map(({ Name, region }, idx) => {
const bucketObjectListPromise = new Promise(async (resolveBucket) => {
const s3 = new s3_1.default({
...config,
region,
endpoint,
s3ForcePathStyle: true,
});
logger.debug(lt.gettingBucketBasicInfo(Name));
const bucketObjectList = await listBucketObjects(s3, Name);
bucketData[idx].Contents = [];
if (!isEmpty_1.default(bucketObjectList)) {
bucketData[idx].Contents = bucketObjectList;
}
resolveBucket();
});
bucketObjectListPromises.push(bucketObjectListPromise);
});
await Promise.all(bucketObjectListPromises);
bucketData.map(({ Name, region }, idx) => {
const s3 = new s3_1.default({
...config,
region,
endpoint,
s3ForcePathStyle: true,
});
const additionalInfoPromise = new Promise(async (resolvePublicStatus) => {
logger.debug(lt.gettingBucketAdditionalInfo(Name));
try {
const bucketAdditionalData = await getBucketAdditionalInfo(s3, Name);
if (!isEmpty_1.default(bucketAdditionalData)) {
const { Tags, ...additionalInfo } = bucketAdditionalData;
bucketData[idx].AdditionalInfo = {
...bucketData[idx].AdditionalInfo,
...additionalInfo,
};
bucketData[idx].Tags = Tags;
}
}
catch (error) {
logger.debug(lt.gettingBucketAdditionalInfoError(Name));
}
resolvePublicStatus();
});
additionalInfoPromises.push(additionalInfoPromise);
});
await Promise.all(additionalInfoPromises);
errorLog.reset();
resolve(groupBy_1.default(bucketData, 'region'));
});