lemon-core
Version:
Lemon Serverless Micro-Service Platform
462 lines • 20.7 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AWSS3Service = void 0;
/**
* `s3s-service.js`
* - common S3 services.
*
*
* @author Steve Jung <steve@lemoncloud.io>
* @date 2019-07-19 initial version
* @date 2019-11-26 cleanup and optimized for `lemon-core#v2`
* @date 2023-02-08 support of `listObject()`
*
* @copyright (C) lemoncloud.io 2019 - All Rights Reserved.
*/
/** ****************************************************************************************************************
* Common Headers
** ****************************************************************************************************************/
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const engine_1 = require("../../engine");
const NS = engine_1.$U.NS('S3', 'blue');
const client_s3_1 = require("@aws-sdk/client-s3");
// eslint-disable-next-line prettier/prettier
const client_s3_2 = require("@aws-sdk/client-s3");
const path_1 = __importDefault(require("path"));
const mime_types_1 = __importDefault(require("mime-types"));
const uuid_1 = require("uuid");
const test_helper_1 = require("../../common/test-helper");
const stream_1 = require("stream");
const tools_1 = require("../../tools");
/** ****************************************************************************************************************
* Public Instance Exported.
** ****************************************************************************************************************/
const region = () => engine_1.$engine.environ('REGION', 'ap-northeast-2');
/**
* use `target` as value or environment value.
* environ('abc') => string 'abc'
* environ('ABC') => use `env.ABC`
*/
const environ = (target, defEnvName, defEnvValue) => {
const isUpperStr = target && /^[A-Z][A-Z0-9_]+$/.test(target);
defEnvName = isUpperStr ? target : defEnvName;
const val = defEnvName ? engine_1.$engine.environ(defEnvName, defEnvValue) : defEnvValue;
target = isUpperStr ? '' : target;
return `${target || val}`;
};
/**
* get aws client for S3
*/
const instance = () => {
const cfg = (0, tools_1.awsConfig)(engine_1.$engine, region());
return new client_s3_1.S3Client(cfg); // SQS Instance. shared one???
};
/**
* main service implement.
*/
class AWSS3Service {
constructor() {
/**
* get name of this
*/
this.name = () => `S3`;
/**
* hello
*/
this.hello = () => `aws-s3-service:${this.bucket()}`;
/**
* get target endpoint by name.
*/
this.bucket = (target) => environ(target, AWSS3Service.ENV_S3_NAME, AWSS3Service.DEF_S3_BUCKET);
/**
* retrieve metadata without returning the object
*
* @param {string} key
* @return metadata object / null if not exists
*/
this.headObject = (key) => __awaiter(this, void 0, void 0, function* () {
var _a;
if (!key)
throw new Error(`@key (string) is required - headObject(${key !== null && key !== void 0 ? key : ''})`);
const Bucket = this.bucket();
const params = { Bucket, Key: key };
// call s3.headObject.
const s3 = instance();
try {
const data = yield s3.send(new client_s3_2.HeadObjectCommand(params));
(0, engine_1._log)(NS, '> data =', engine_1.$U.json(Object.assign(Object.assign({}, data), { Contents: undefined })));
// const sample = {
// AcceptRanges: 'bytes',
// ContentLength: 47,
// ContentType: 'application/json; charset=utf-8',
// ETag: '"51f209a54902230ac3395826d7fa1851"',
// Expiration: 'expiry-date="Mon, 10 Apr 2023 00:00:00 GMT", rule-id="delete-old-json"',
// LastModified: '2023-02-08T14:53:12.000Z',
// Metadata: { contenttype: 'application/json; charset=utf8', md5: '51f209a54902230ac3395826d7fa1851' },
// ServerSideEncryption: 'AES256',
// };
const result = {
ContentType: data.ContentType,
ContentLength: data.ContentLength,
Metadata: data.Metadata,
ETag: data.ETag,
LastModified: engine_1.$U.ts(data.LastModified),
};
return result;
}
catch (e) {
if (((_a = e === null || e === void 0 ? void 0 : e.$metadata) === null || _a === void 0 ? void 0 : _a.httpStatusCode) === 404 || e.name === 'NotFound')
return null;
(0, engine_1._err)(NS, '! err=', e);
throw e;
}
});
/**
* upload a file to S3 Bucket
*
* ```js
* const res = $s3.putObject(JSON.stringify({ message }), 'test.json');
* // response would be like
* {
* "Bucket": "lemon-hello-www",
* "ETag": "5e206.....8bd4c",
* "Key": "test.json",
* "Location": "https://lemon-hello-www.s3.ap-northeast-2.amazonaws.com/test.json",
* }
* ```
*
* @param {string|Buffer} content content body
* @param {string} key (optional) S3 key to put
* @param {Metadata} metadata (optional) metadata to store
* @param {object} tags (optional) tag set
*/
this.putObject = (content, key, metadata, tags) => __awaiter(this, void 0, void 0, function* () {
if (!content)
throw new Error(`@content (buffer) is required - putObject()`);
const paramBuilder = new S3PutObjectRequestBuilder(this.bucket(), content);
key && paramBuilder.setKey(key);
metadata && paramBuilder.setMetadata(metadata);
tags && paramBuilder.setTags(tags);
const params = paramBuilder.asParams();
(0, engine_1._log)(NS, `> params.ContentType =`, params.ContentType);
(0, engine_1._log)(NS, `> params.ContentLength =`, params.ContentLength);
(0, engine_1._log)(NS, `> params.Metadata =`, params.Metadata);
(0, engine_1._log)(NS, `> params.Tagging =`, params.Tagging);
// call s3.upload()
const s3 = instance();
try {
const data = yield s3.send(new client_s3_2.PutObjectCommand(params));
delete data.key; // NOTE: remove undeclared property 'key' returned from aws-sdk
const location = `https://${params.Bucket}.s3.${region()}.amazonaws.com/${params.Key}`;
(0, engine_1._log)(NS, `> data[${params.Bucket}].Location =`, engine_1.$U.json(location));
const result = {
Bucket: params.Bucket,
Location: location,
Key: params.Key,
ETag: data.ETag,
ContentType: params.ContentType,
ContentLength: params.ContentLength,
Metadata: params.Metadata,
};
return result;
}
catch (e) {
(0, engine_1._err)(NS, `! err[${params.Bucket}] =`, e);
throw e;
}
});
/**
* get a file from S3 Bucket
*
* @param {string} key
*/
this.getObject = (key) => __awaiter(this, void 0, void 0, function* () {
if (!key)
throw new Error(`@key (string) is required - getObject(${key !== null && key !== void 0 ? key : ''})`);
const Bucket = this.bucket();
const params = { Bucket, Key: key };
//* call s3.getObject.
const s3 = instance();
try {
const data = yield s3.send(new client_s3_2.GetObjectCommand(params));
(0, engine_1._log)(NS, '> data.type =', typeof data);
const { ContentType, ContentLength, Body, ETag, Metadata, TagCount } = data;
// convert stream to buffer for readable stream.
const buffer = Body && Body instanceof stream_1.Readable ? yield this.streamToBuffer(Body) : undefined;
const result = { ContentType, ContentLength, Body: buffer, ETag, Metadata };
if (TagCount)
result.TagCount = TagCount;
return result;
}
catch (e) {
(0, engine_1._err)(NS, '! err=', e);
throw e;
}
});
/**
* return decoded Object from bucket file.
*
* @param {string} key ex) 'hello-0001.json' , 'dist/hello-0001.json
*/
this.getDecodedObject = (key) => __awaiter(this, void 0, void 0, function* () {
if (!key)
throw new Error(`@key (string) is required - getDecodedObject(${key !== null && key !== void 0 ? key : ''})`);
const Bucket = this.bucket();
const params = { Bucket, Key: key };
//* call s3.getObject.
const s3 = instance();
try {
const data = yield s3.send(new client_s3_2.GetObjectCommand(params));
(0, engine_1._log)(NS, '> data.type =', typeof data);
const buffer = data.Body && data.Body instanceof stream_1.Readable ? yield this.streamToBuffer(data.Body) : undefined;
const content = buffer === null || buffer === void 0 ? void 0 : buffer.toString();
return JSON.parse(content);
}
catch (e) {
(0, engine_1._err)(NS, '! err=', e);
throw e;
}
});
/**
* get tag-set of object
*
* @param {string} key
*/
this.getObjectTagging = (key) => __awaiter(this, void 0, void 0, function* () {
var _b;
if (!key)
throw new Error(`@key (string) is required - getObjectTagging(${key !== null && key !== void 0 ? key : ''})`);
const Bucket = this.bucket();
const params = { Bucket, Key: key };
//* call s3.getObjectTagging.
const s3 = instance();
try {
const data = yield s3.send(new client_s3_2.GetObjectTaggingCommand(params));
(0, engine_1._log)(NS, `> data =`, engine_1.$U.json(data));
return (_b = data === null || data === void 0 ? void 0 : data.TagSet) === null || _b === void 0 ? void 0 : _b.reduce((tagSet, tag) => {
const { Key, Value } = tag;
tagSet[Key] = Value;
return tagSet;
}, {});
}
catch (e) {
(0, engine_1._err)(NS, '! err=', e);
throw e;
}
});
/**
* delete object from bucket
*
* @param {string} key
*/
this.deleteObject = (key) => __awaiter(this, void 0, void 0, function* () {
if (!key)
throw new Error(`@key (string) is required - deleteObject(${key !== null && key !== void 0 ? key : ''})`);
const Bucket = this.bucket();
const params = { Bucket, Key: key };
//* call s3.deleteObject.
const s3 = instance();
try {
const data = yield s3.send(new client_s3_2.DeleteObjectCommand(params));
(0, engine_1._log)(NS, '> data =', engine_1.$U.json(data));
}
catch (e) {
(0, engine_1._err)(NS, '! err=', e);
throw e;
}
});
/**
* list objects in bucket
*/
this.listObjects = (options) => __awaiter(this, void 0, void 0, function* () {
var _c, _d, _e, _f, _g, _h;
// if (!key) throw new Error('@key is required!');
const Prefix = (_c = options === null || options === void 0 ? void 0 : options.prefix) !== null && _c !== void 0 ? _c : '';
const Delimiter = (_d = options === null || options === void 0 ? void 0 : options.delimiter) !== null && _d !== void 0 ? _d : '/';
const MaxKeys = Math.min((_e = options === null || options === void 0 ? void 0 : options.limit) !== null && _e !== void 0 ? _e : 10, 1000);
const unlimited = (_f = options === null || options === void 0 ? void 0 : options.unlimited) !== null && _f !== void 0 ? _f : false;
const nextToken = options === null || options === void 0 ? void 0 : options.nextToken;
const throwable = (_g = options === null || options === void 0 ? void 0 : options.throwable) !== null && _g !== void 0 ? _g : true;
//* build the req-params.
const Bucket = this.bucket();
const params = {
Bucket,
Prefix,
Delimiter,
MaxKeys,
};
if (nextToken)
params.ContinuationToken = nextToken;
//* call s3.listObjectsV2.
const s3 = instance();
const result = {
Contents: null,
MaxKeys,
KeyCount: 0,
};
try {
const data = yield s3.send(new client_s3_2.ListObjectsV2Command(params));
//INFO! - minimize log output....
(0, engine_1._log)(NS, '> data =', engine_1.$U.json(Object.assign(Object.assign({}, data), { Contents: undefined })));
(0, engine_1._log)(NS, '> data[0] =', engine_1.$U.json((_h = data === null || data === void 0 ? void 0 : data.Contents) === null || _h === void 0 ? void 0 : _h[0]));
if (data) {
result.Contents = data.Contents;
result.MaxKeys = data.MaxKeys;
result.KeyCount = data.KeyCount;
result.IsTruncated = data.IsTruncated;
result.NextContinuationToken = data.NextContinuationToken;
}
//* list all keys.
if (unlimited) {
while (result.IsTruncated) {
//* fetch next list.
const res2 = yield s3.send(new client_s3_2.ListObjectsV2Command(Object.assign(Object.assign({}, params), { ContinuationToken: result.NextContinuationToken })));
//* update contents.
result.Contents = result.Contents.concat(0 ? res2.Contents.slice(1) : res2.Contents);
result.IsTruncated = res2.IsTruncated;
result.KeyCount += engine_1.$U.N(res2.KeyCount, 0);
result.NextContinuationToken = res2.NextContinuationToken;
}
}
}
catch (e) {
(0, engine_1._err)(NS, '! err=', e);
if (throwable)
throw e;
result.error = (0, test_helper_1.GETERR)(e);
}
// returns.
return result;
});
/**
* Convert a readable stream into a single Buffer.
* Required in AWS SDK v3, as S3.getObject() returns a stream instead of a Buffer.
*
* Used for JSON parsing in getDecodedObject() and getObject().
*/
this.streamToBuffer = (stream) => { var stream_2, stream_2_1; return __awaiter(this, void 0, void 0, function* () {
var e_1, _a;
const chunks = [];
try {
for (stream_2 = __asyncValues(stream); stream_2_1 = yield stream_2.next(), !stream_2_1.done;) {
const chunk = stream_2_1.value;
chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk);
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (stream_2_1 && !stream_2_1.done && (_a = stream_2.return)) yield _a.call(stream_2);
}
finally { if (e_1) throw e_1.error; }
}
return Buffer.concat(chunks);
}); };
}
}
exports.AWSS3Service = AWSS3Service;
/**
* environ name to use `bucket`
*/
AWSS3Service.ENV_S3_NAME = 'MY_S3_BUCKET';
/**
* default `bucket` name
*/
AWSS3Service.DEF_S3_BUCKET = 'lemon-hello-www';
/**
* class `S3PutObjectRequestBuilder`
* - util class to build S3.PutObjectRequest parameter
*/
class S3PutObjectRequestBuilder {
/**
* constructor
*/
constructor(bucket, content) {
/**
* guess content-type from filename
* @param filename
* @private
*/
this.getContentType = (filename) => {
const extname = path_1.default.extname(filename);
return mime_types_1.default.contentType(extname) || undefined;
};
const buffer = typeof content === 'string' ? Buffer.from(content) : content;
this.Body = buffer;
this.Bucket = bucket;
this.ContentLength = buffer.length;
this.Metadata = { md5: engine_1.$U.md5(buffer, 'hex') };
}
/**
* explicitly set key
* @param key S3 object key
*/
setKey(key) {
this.Key = key;
if (!this.Metadata['Content-Type']) {
this.setMetadata({ 'Content-Type': this.getContentType(key) });
}
return this;
}
/**
* add metadata
* @param metadata key-value dictionary (only string is allowed for values.)
*/
setMetadata(metadata) {
if (metadata['Content-Type']) {
this.ContentType = metadata['Content-Type'];
delete metadata['Content-Type'];
}
else if (metadata.origin) {
this.ContentType = this.getContentType(metadata.origin);
}
this.Metadata = Object.assign({ md5: this.Metadata.md5 }, metadata); // preserve 'md5' field
return this;
}
/**
* add tags
* @param tags key-value dictionary (only string is allowed for values.)
*/
setTags(tags) {
this.Tagging = new URLSearchParams(tags).toString();
return this;
}
/**
* return PutObjectRequest object
*/
asParams() {
const { Body, Bucket, ContentLength, Metadata, Tagging } = this;
let { ContentType, Key } = this;
// generate object key if not specified
// - generate UUID filename
// - get extension from content-type or use 'json'
if (!Key) {
const ext = (this.ContentType && mime_types_1.default.extension(this.ContentType)) || 'json';
Key = `${(0, uuid_1.v4)()}.${ext}`;
}
// generate content-type if not specified
if (!ContentType)
ContentType = this.getContentType(Key);
return { Bucket, Key, Body, ContentLength, ContentType, Metadata, Tagging };
}
}
//# sourceMappingURL=aws-s3-service.js.map