@tradle/aws-s3-client
Version:
> TODO: description
453 lines • 18.2 kB
JavaScript
"use strict";
var __rest = (this && this.__rest) || function (s, e) {
var t = {};
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
t[p] = s[p];
if (s != null && typeof Object.getOwnPropertySymbols === "function")
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
t[p[i]] = s[p[i]];
}
return t;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createClient = exports.parseS3Url = exports.S3Client = exports.mapHeadersToS3PutOptions = void 0;
const url_1 = require("url");
const util_1 = require("util");
const zlib_1 = __importDefault(require("zlib"));
const omit_1 = __importDefault(require("lodash/omit"));
const util_2 = require("aws-sdk/lib/util");
const amazon_s3_uri_1 = __importDefault(require("amazon-s3-uri"));
const empty_aws_bucket_1 = __importDefault(require("empty-aws-bucket"));
const caseless_1 = __importDefault(require("caseless"));
const aws_common_utils_1 = require("@tradle/aws-common-utils");
const promise_utils_1 = require("@tradle/promise-utils");
const gzip = util_1.promisify(zlib_1.default.gzip.bind(zlib_1.default));
const gunzip = util_1.promisify(zlib_1.default.gunzip.bind(zlib_1.default));
const MAX_BUCKET_NAME_LENGTH = 63;
const PUBLIC_BUCKET_RULE_ID = 'MakeItPublic';
const LOCAL_S3_PATH_NAME_REGEX = /^\/?([^/]+)\/(.*)/;
const mapToS3PutOption = {
ContentType: 'ContentType',
'content-type': 'ContentType',
ContentEncoding: 'ContentEncoding',
'content-encoding': 'ContentEncoding'
};
const toS3PutOption = caseless_1.default(mapToS3PutOption);
exports.mapHeadersToS3PutOptions = (headers) => {
const putOpts = {};
for (const name in headers) {
const s3Option = toS3PutOption.get(name);
if (!s3Option) {
throw new aws_common_utils_1.Errors.InvalidInput(`unrecognized header: ${name}`);
}
putOpts[s3Option] = headers[name];
}
return putOpts;
};
class S3Client {
constructor({ client }) {
this.put = async ({ key, value, bucket, headers = {}, acl }) => {
const opts = Object.assign(Object.assign({}, exports.mapHeadersToS3PutOptions(headers)), { Bucket: bucket, Key: key, Body: aws_common_utils_1.toStringOrBuf(value) });
if (acl)
opts.ACL = acl;
return await this.s3.putObject(opts).promise();
};
this.gzipAndPut = async (opts) => {
const { value, headers = {} } = opts;
const compressed = await gzip(aws_common_utils_1.toStringOrBuf(value));
return await this.put(Object.assign(Object.assign({}, opts), { value: compressed, headers: Object.assign(Object.assign({}, headers), { ContentEncoding: 'gzip' }) }));
};
this.get = async ({ key, bucket, s3Opts }) => {
const params = Object.assign({ Bucket: bucket, Key: key }, s3Opts);
try {
const result = await this.s3.getObject(params).promise();
if (result.ContentEncoding === 'gzip') {
result.Body = await gunzip(result.Body);
delete result.ContentEncoding;
}
return result;
}
catch (err) {
if (err.code === 'NoSuchKey') {
throw new aws_common_utils_1.Errors.NotFound(`${bucket}/${key}`);
}
throw err;
}
};
this.getByUrl = async (url) => {
const { bucket, key } = exports.parseS3Url(url);
const props = { bucket, key };
if (key.endsWith('.json') || key.endsWith('.json.gz')) {
return this.getJSON(props);
}
return await this.get(props);
};
this.forEachItemInBucket = async ({ bucket, getBody, map, s3Opts }) => {
const params = Object.assign({ Bucket: bucket }, s3Opts);
while (true) {
const { Contents, ContinuationToken } = await this.s3.listObjectsV2(params).promise();
if (getBody) {
await promise_utils_1.batchProcess({
data: Contents,
batchSize: 20,
processOne: async (item, i) => {
const withBody = await this.get({ bucket, key: item.Key });
const result = map(Object.assign(Object.assign({}, item), withBody), i);
if (promise_utils_1.isPromise(result))
await result;
}
});
}
else {
await Promise.all(Contents.map(async (item, i) => {
const result = map(item, i);
if (promise_utils_1.isPromise(result))
await result;
}));
}
if (!ContinuationToken)
break;
params.ContinuationToken = ContinuationToken;
}
};
this.listBuckets = async () => {
const { Buckets } = await this.s3.listBuckets().promise();
return Buckets.map(({ Name }) => Name);
};
this.listObjects = async (opts) => {
return (await this.listBucket(Object.assign(Object.assign({}, opts), { getBody: true })));
};
this.listObjectsWithKeyPrefix = async (opts) => {
return (await this.listBucketWithPrefix(Object.assign(Object.assign({}, opts), { getBody: true })));
};
this.listBucket = async (opts) => {
const all = [];
await this.forEachItemInBucket(Object.assign(Object.assign({}, opts), { map: item => all.push(item) }));
return all;
};
this.clearBucket = async ({ bucket }) => {
await this.forEachItemInBucket({
bucket,
map: ({ Key }) => this.del({ bucket, key: Key })
});
};
this.getCacheable = (_a) => {
var { key, bucket, ttl, parse } = _a, defaultOpts = __rest(_a, ["key", "bucket", "ttl", "parse"]);
if (!key)
throw new Error('expected "key"');
if (!bucket)
throw new Error('expected "bucket"');
if (!ttl)
throw new Error('expected "ttl"');
let cached;
let type;
let etag;
let cachedTime = 0;
const invalidateCache = () => {
cached = undefined;
type = undefined;
etag = undefined;
cachedTime = 0;
};
const maybeGet = async (opts = {}) => {
const summary = { key, bucket, type };
if (!opts.force) {
const age = Date.now() - cachedTime;
if (etag && age < ttl) {
return cached;
}
}
opts = Object.assign(Object.assign({}, defaultOpts), omit_1.default(opts, ['force']));
if (etag) {
opts.IfNoneMatch = etag;
}
try {
cached = await this.get(Object.assign({ key, bucket }, opts));
}
catch (err) {
if (err.code === 'NotModified') {
return cached;
}
throw err;
}
if (cached.ETag !== etag) {
etag = cached.ETag;
}
if (parse) {
cached = parse(cached.Body);
}
cachedTime = Date.now();
return cached;
};
const putAndCache = async (_a) => {
var { value } = _a, opts = __rest(_a, ["value"]);
if (value == null)
throw new Error('expected "value"');
const result = await this.put(Object.assign(Object.assign({ bucket, key, value }, defaultOpts), opts));
cached = parse
? value
: Object.assign({ Body: JSON.stringify(value) }, result);
cachedTime = Date.now();
etag = result.ETag;
};
return {
get: maybeGet,
put: putAndCache,
invalidateCache
};
};
this.putJSON = this.put;
this.getJSON = ({ key, bucket }) => {
return this.get({ key, bucket }).then(({ Body }) => JSON.parse(Body.toString()));
};
this.head = async ({ key, bucket }) => {
try {
return await this.s3
.headObject({
Bucket: bucket,
Key: key
})
.promise();
}
catch (err) {
if (err.code === 'NoSuchKey' || err.code === 'NotFound') {
throw new aws_common_utils_1.Errors.NotFound(`${bucket}/${key}`);
}
throw err;
}
};
this.exists = async ({ key, bucket }) => {
try {
await this.head({ key, bucket });
return true;
}
catch (err) {
aws_common_utils_1.Errors.ignoreNotFound(err);
return false;
}
};
this.del = async ({ key, bucket }) => {
await this.s3
.deleteObject({
Bucket: bucket,
Key: key
})
.promise();
};
this.createPresignedUrl = ({ bucket, key }) => {
const url = this.s3.getSignedUrl('getObject', {
Bucket: bucket,
Key: key
});
return url;
};
this.createBucket = async ({ bucket }) => {
return await this.s3.createBucket({ Bucket: bucket }).promise();
};
this.destroyBucket = async ({ bucket }) => {
try {
await this.emptyBucket({ bucket });
await this.disableReplication({ bucket });
await this.deleteBucket({ bucket });
}
catch (err) {
aws_common_utils_1.Errors.ignore(err, { code: 'NoSuchBucket' });
}
};
this.disableReplication = async ({ bucket }) => {
await this.s3.deleteBucketReplication({ Bucket: bucket }).promise();
};
this.deleteBucket = async ({ bucket }) => {
try {
await this.s3.deleteBucket({ Bucket: bucket }).promise();
}
catch (err) {
aws_common_utils_1.Errors.ignore(err, { code: 'NoSuchBucket' });
}
};
this.getUrlForKey = ({ bucket, key, region }) => {
const { host } = this.s3.endpoint;
const encodedKey = util_2.uriEscapePath(key);
if (aws_common_utils_1.isLocalHost(host)) {
return `http://${host}/${bucket}${encodedKey}`;
}
if (region)
return `https://${bucket}.s3.${region}.amazonaws.com/${encodedKey}`;
else
return `https://${bucket}.s3.amazonaws.com/${encodedKey}`;
};
this.disableEncryption = async ({ bucket }) => {
await this.s3.deleteBucketEncryption({ Bucket: bucket }).promise();
};
this.enableEncryption = async ({ bucket, kmsKeyId }) => {
const params = toEncryptionParams({ bucket, kmsKeyId });
await this.s3.putBucketEncryption(params).promise();
};
this.getEncryption = async ({ bucket }) => {
return await this.s3.getBucketEncryption({ Bucket: bucket }).promise();
};
this.getLatest = (list) => {
let max = 0;
let latest;
for (const metadata of list) {
const date = new Date(metadata.LastModified).getTime();
if (date > max) {
latest = metadata;
max = date;
}
}
return latest;
};
this.makePublic = async ({ bucket }) => {
await this.s3
.putBucketPolicy({
Bucket: bucket,
Policy: `{
"Version": "2012-10-17",
"Statement": [{
"Sid": "${PUBLIC_BUCKET_RULE_ID}",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::${bucket}/*"
}]
}`
})
.promise();
};
this.isBucketPublic = async ({ bucket }) => {
let result;
try {
result = await this.getBucketPolicy({ bucket });
}
catch (err) {
aws_common_utils_1.Errors.ignoreNotFound(err);
return false;
}
return result.Statement.some(statement => isPublicBucketStatement({ statement, bucket }));
};
this.getBucketPolicy = async ({ bucket }) => {
try {
const { Policy } = await this.s3.getBucketPolicy({ Bucket: bucket }).promise();
return JSON.parse(Policy);
}
catch (err) {
aws_common_utils_1.Errors.ignoreNotFound(err);
throw new aws_common_utils_1.Errors.NotFound(err.message);
}
};
this.makeKeysPublic = async ({ bucket, keys }) => {
await this.setPolicyForKeys({ bucket, keys, policy: 'public-read' });
};
this.setPolicyForKeys = async ({ bucket, keys, policy }) => {
await Promise.all(keys.map(key => this.s3
.putObjectAcl({
Bucket: bucket,
Key: key,
ACL: policy
})
.promise()));
};
this.allowGuestToRead = async ({ bucket, keys }) => {
const isPublic = await this.isBucketPublic({ bucket });
if (!isPublic) {
await this.makeKeysPublic({ bucket, keys });
}
};
this.deleteVersions = async ({ bucket, versions }) => {
await this.s3
.deleteObjects({
Bucket: bucket,
Delete: {
Objects: versions.map(({ Key, VersionId }) => ({ Key, VersionId }))
}
})
.promise();
};
this.emptyBucket = async ({ bucket }) => {
const { s3 } = this;
return empty_aws_bucket_1.default({ s3, bucket });
};
this.listBucketWithPrefix = async (_a) => {
var { s3Opts, prefix } = _a, listOpts = __rest(_a, ["s3Opts", "prefix"]);
return await this.listBucket(Object.assign({ s3Opts: Object.assign(Object.assign({}, s3Opts), { Prefix: prefix }) }, listOpts));
};
this.copyFilesBetweenBuckets = async ({ source, target, keys, prefix, acl }) => {
if (!(prefix || keys))
throw new aws_common_utils_1.Errors.InvalidInput('expected "keys" or "prefix"');
if (!keys) {
const items = await this.listBucketWithPrefix({ bucket: source, prefix });
keys = items.map(i => i.Key);
}
const baseParams = {
Bucket: target,
CopySource: null,
Key: null
};
if (acl)
baseParams.ACL = acl;
await Promise.all(keys.map(async (key) => {
const params = Object.assign(Object.assign({}, baseParams), { CopySource: `${source}/${key}`, Key: key });
try {
await this.s3.copyObject(params).promise();
}
catch (err) {
aws_common_utils_1.Errors.ignoreNotFound(err);
throw new aws_common_utils_1.Errors.NotFound(`bucket: "${target}", key: "${key}"`);
}
}));
};
this.parseS3Url = exports.parseS3Url;
this.s3 = client;
}
}
exports.S3Client = S3Client;
exports.parseS3Url = (url) => {
try {
return amazon_s3_uri_1.default(url);
}
catch (err) {
if (!aws_common_utils_1.isLocalUrl(url)) {
throw new aws_common_utils_1.Errors.InvalidOption(`invalid s3 url: ${url}`);
}
}
const parsed = url_1.parse(url);
const { pathname = '' } = parsed;
const match = pathname.match(LOCAL_S3_PATH_NAME_REGEX);
if (!match)
return;
const [bucket, key] = match.slice(1);
if (bucket && key)
return { bucket, key, isPathStyle: true };
throw new aws_common_utils_1.Errors.InvalidInput(`invalid s3 url: ${url}`);
};
exports.createClient = (opts) => new S3Client(opts);
const toEncryptionParams = ({ bucket, kmsKeyId }) => {
const ApplyServerSideEncryptionByDefault = {
SSEAlgorithm: kmsKeyId ? 'aws:kms' : 'AES256'
};
if (kmsKeyId) {
ApplyServerSideEncryptionByDefault.KMSMasterKeyID = kmsKeyId;
}
return {
Bucket: bucket,
ServerSideEncryptionConfiguration: {
Rules: [
{
ApplyServerSideEncryptionByDefault
}
]
}
};
};
const isPublicBucketStatement = ({ statement, bucket }) => {
const { Principal, Resource, Action } = statement;
return (Principal === '*' &&
[].concat(Resource).some(r => r.includes(`:${bucket}/*`)) &&
[].concat(Action).some(a => a.toLowerCase() === 's3:GetObject'));
};
//# sourceMappingURL=client.js.map