kura-s3/lib/s3/S3Accessor.js

The FileSystem API abstraction library, AWS S3 Plugin

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.S3Accessor = void 0;
const s3_1 = __importDefault(require("aws-sdk/clients/s3"));
const kura_1 = require("kura");
const S3FileSystem_1 = require("./S3FileSystem");
const S3Util_1 = require("./S3Util");
const EXPIRES = 60 * 60 * 24 * 7;
const isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined";
const isNode = typeof process !== "undefined" &&
    process.versions != null &&
    process.versions.node != null;
const isReactNative = typeof navigator !== "undefined" && navigator.product === "ReactNative";
const hasBuffer = typeof Buffer === "function";
class S3Accessor extends kura_1.AbstractAccessor {
    constructor(config, bucket, rootDir, s3Options) {
        super(s3Options);
        this.config = config;
        this.bucket = bucket;
        this.rootDir = rootDir;
        this.s3Options = s3Options;
        this.urlCache = {};
        if (!s3Options.expires) {
            s3Options.expires = EXPIRES;
        }
        if (!config.httpOptions) {
            config.httpOptions = {};
        }
        config.maxRetries = 0;
        if (config.httpOptions.timeout == null) {
            config.httpOptions.timeout = 1000;
            config.httpOptions.connectTimeout = 1000;
        }
        config.s3ForcePathStyle = true;
        config.signatureVersion = "v4";
        this.s3 = new s3_1.default(config);
        this.filesystem = new S3FileSystem_1.S3FileSystem(this);
        if (!this.rootDir.startsWith(kura_1.DIR_SEPARATOR)) {
            this.rootDir = kura_1.DIR_SEPARATOR + this.rootDir;
        }
        this.name = this.bucket + this.rootDir;
    }
    createIndexDir(dirPath) {
        let indexDir = kura_1.INDEX_DIR_PATH + dirPath;
        if (!indexDir.endsWith(kura_1.DIR_SEPARATOR)) {
            indexDir += kura_1.DIR_SEPARATOR;
        }
        return indexDir;
    }
    async doDelete(fullPath, isFile) {
        if (!isFile) {
            return;
        }
        const key = this.getKey(fullPath);
        const params = {
            Bucket: this.bucket,
            Key: key,
        };
        try {
            await this.s3.deleteObject(params).promise();
        }
        catch (err) {
            if (this.isNotFoundError(err)) {
                return;
            }
            throw new kura_1.InvalidModificationError(this.name, fullPath, err);
        }
    }
    async doGetObject(fullPath, isFile) {
        if (!isFile) {
            return { name: kura_1.getName(fullPath), fullPath, lastModified: Date.now() };
        }
        const key = this.getKey(fullPath);
        if (this.s3Options.getObjectUsingListObject) {
            const params = {
                Bucket: this.bucket,
                Prefix: key,
            };
            let data;
            try {
                data = await this.s3.listObjectsV2(params).promise();
            }
            catch (err) {
                this.handleNotFoundErrorS3(fullPath, err);
                throw new kura_1.NotReadableError(this.name, fullPath, err);
            }
            if (data.KeyCount === 0) {
                throw new kura_1.NotFoundError(this.name, fullPath);
            }
            for (const content of data.Contents) {
                if (content.Key === key) {
                    return {
                        name: kura_1.getName(fullPath),
                        fullPath: fullPath,
                        lastModified: content.LastModified.getTime(),
                        size: content.Size,
                    };
                }
            }
            throw new kura_1.NotFoundError(this.name, fullPath);
        }
        else {
            try {
                const data = await this.s3
                    .headObject({
                    Bucket: this.bucket,
                    Key: key,
                })
                    .promise();
                const name = key.split(kura_1.DIR_SEPARATOR).pop();
                return {
                    name,
                    fullPath: fullPath,
                    lastModified: data.LastModified.getTime(),
                    size: data.ContentLength,
                };
            }
            catch (err) {
                this.handleNotFoundErrorS3(fullPath, err);
                throw new kura_1.NotReadableError(this.name, fullPath, err);
            }
        }
    }
    async doGetObjects(dirPath) {
        const path = kura_1.normalizePath(this.rootDir + kura_1.DIR_SEPARATOR + dirPath);
        const prefix = S3Util_1.getPrefix(path);
        const params = {
            Bucket: this.bucket,
            Delimiter: kura_1.DIR_SEPARATOR,
            Prefix: prefix,
            ContinuationToken: null,
        };
        const objects = [];
        await this.doReadObjectsFromS3(params, dirPath, path, objects);
        return objects;
    }
    async doMakeDirectory(_fullPath) {
    }
    async doReadContent(fullPath) {
        if (this.s3Options.methodOfDoGetContent === "xhr") {
            return await this.doReadContentUsingXHR(fullPath, isBrowser ? "blob" : "arraybuffer");
        }
        else {
            return await this.doReadContentUsingGetObject(fullPath);
        }
    }
    async getURL(fullPath, method) {
        const keysToRemove = [];
        const now = Math.trunc(Date.now() / 1000);
        for (const [key, cache] of Object.entries(this.urlCache)) {
            if (cache.expirationTime <= now) {
                keysToRemove.push(key);
            }
        }
        for (const keyToRemove of keysToRemove) {
            delete this.urlCache[keyToRemove];
        }
        if (!method || method === "GET") {
            const key = fullPath + "|get";
            const cache = this.urlCache[key];
            if (cache) {
                return cache.url;
            }
            const url = await this.getSignedUrl(fullPath, "getObject");
            this.urlCache[key] = {
                expirationTime: now + this.s3Options.expires,
                url,
            };
            return url;
        }
        else if (method === "PUT") {
            const key = fullPath + "|put";
            const cache = this.urlCache[key];
            if (cache) {
                return cache.url;
            }
            const url = await this.getSignedUrl(fullPath, "putObject");
            this.urlCache[key] = {
                expirationTime: now + this.s3Options.expires,
                url,
            };
            return url;
        }
        else {
            return null;
        }
    }
    doWriteArrayBuffer(fullPath, buffer) {
        return this.doWriteContentToS3(fullPath, buffer);
    }
    async doWriteBase64(fullPath, base64) {
        return this.doWriteContentToS3(fullPath, base64);
    }
    doWriteBlob(fullPath, blob) {
        return this.doWriteContentToS3(fullPath, blob);
    }
    async doWriteBuffer(fullPath, buffer) {
        return this.doWriteContentToS3(fullPath, buffer);
    }
    initialize(options) {
        this.initializeIndexOptions(options);
        if (options.contentsCache == null) {
            options.contentsCache = false;
        }
        this.initializeContentsCacheOptions(options);
        this.debug("S3Accessor#initialize", JSON.stringify(options));
    }
    initializeIndexOptions(options) {
        if (!options.index) {
            return;
        }
        if (options.indexOptions == null) {
            options.indexOptions = {};
        }
        const indexOptions = options.indexOptions;
        if (indexOptions.noCache == null) {
            indexOptions.noCache = true;
        }
        if (indexOptions.logicalDelete == null) {
            indexOptions.logicalDelete = false;
        }
    }
    async doReadContentUsingGetObject(fullPath) {
        try {
            const key = this.getKey(fullPath);
            const req = {
                Bucket: this.bucket,
                Key: key,
            };
            if (this.s3Options.noCache) {
                req.ResponseCacheControl = "no-cache";
                req.ResponseExpires = new Date(0);
            }
            const data = await this.s3.getObject(req).promise();
            return this.fromBody(data.Body);
        }
        catch (err) {
            this.handleNotFoundErrorS3(fullPath, err);
            throw new kura_1.NotReadableError(this.name, fullPath, err);
        }
    }
    async doReadContentUsingXHR(fullPath, responseType) {
        const xhrOptions = {
            timeout: this.config.httpOptions.timeout,
        };
        if (this.s3Options.noCache) {
            xhrOptions.requestHeaders["Cache-Control"] = "no-cache";
        }
        const xhr = new kura_1.XHR(this.name, fullPath, xhrOptions);
        const url = await this.getSignedUrl(fullPath, "getObject");
        return xhr.get(url, responseType);
    }
    async doReadObjectsFromS3(params, dirPath, path, objects) {
        let data;
        try {
            data = await this.s3.listObjectsV2(params).promise();
        }
        catch (err) {
            this.handleNotFoundErrorS3(dirPath, err);
            throw new kura_1.NotReadableError(this.name, dirPath, err);
        }
        for (const content of data.CommonPrefixes) {
            const parts = content.Prefix.split(kura_1.DIR_SEPARATOR);
            const name = parts[parts.length - 2];
            const fullPath = kura_1.normalizePath(dirPath + kura_1.DIR_SEPARATOR + name);
            objects.push({
                name: name,
                fullPath: fullPath,
                lastModified: null,
                size: null,
            });
        }
        for (const content of data.Contents) {
            const parts = content.Key.split(kura_1.DIR_SEPARATOR);
            const name = parts[parts.length - 1];
            const fullPath = kura_1.normalizePath(dirPath + kura_1.DIR_SEPARATOR + name);
            objects.push({
                name: name,
                fullPath: fullPath,
                lastModified: content.LastModified.getTime(),
                size: content.Size,
            });
        }
        if (data.IsTruncated) {
            params.ContinuationToken = data.NextContinuationToken;
            await this.doReadObjectsFromS3(params, dirPath, path, objects);
        }
    }
    async doWriteContentToS3(fullPath, content) {
        const method = this.s3Options.methodOfDoPutContent;
        if (typeof content === "string") {
            if (hasBuffer) {
                content = await kura_1.toBuffer(content);
            }
            else {
                content = await kura_1.toBlob(content);
            }
        }
        if (method === "uploadPart") {
            content = await kura_1.toArrayBuffer(content);
            await this.doWriteContentUsingUploadPart(fullPath, content);
        }
        else if (method === "xhr") {
            await this.doWriteContentUsingXHR(fullPath, content);
        }
        else if (method === "upload") {
            await this.doWriteContentUsingUpload(fullPath, content);
        }
        else {
            await this.doWriteContentUsingPutObject(fullPath, content);
        }
    }
    async doWriteContentUsingPutObject(fullPath, content) {
        const body = await this.toBody(content);
        const key = this.getKey(fullPath);
        const contentLength = kura_1.isBlob(content) ? content.size : content.byteLength;
        try {
            await this.s3
                .putObject({
                Bucket: this.bucket,
                Key: key,
                Body: body,
                ContentLength: contentLength,
            })
                .promise();
        }
        catch (err) {
            throw new kura_1.InvalidModificationError(this.name, fullPath, err);
        }
    }
    async doWriteContentUsingUpload(fullPath, content) {
        const body = await this.toBody(content);
        const key = this.getKey(fullPath);
        const contentLength = kura_1.isBlob(content) ? content.size : content.byteLength;
        await this.s3
            .upload({
            Bucket: this.bucket,
            Key: key,
            Body: body,
            ContentLength: contentLength,
        })
            .promise();
    }
    async doWriteContentUsingUploadPart(fullPath, content) {
        const key = this.getKey(fullPath);
        const buffer = await kura_1.toArrayBuffer(content);
        const view = new Uint8Array(buffer);
        const allSize = view.byteLength;
        const partSize = 1024 * 1024;
        const multipartMap = {
            Parts: [],
        };
        const createReq = {
            Bucket: this.bucket,
            Key: key,
        };
        const multiPartUpload = await this.s3
            .createMultipartUpload(createReq)
            .promise();
        const uploadId = multiPartUpload.UploadId;
        let partNum = 0;
        const { ContentType, ...otherParams } = createReq;
        for (let rangeStart = 0; rangeStart < allSize; rangeStart += partSize) {
            partNum++;
            const end = Math.min(rangeStart + partSize, allSize);
            const chunk = view.slice(rangeStart, end);
            const partParams = {
                Body: chunk,
                PartNumber: partNum,
                UploadId: uploadId,
                ...otherParams,
            };
            const uploadPart = await this.s3.uploadPart(partParams).promise();
            multipartMap.Parts[partNum - 1] = {
                ETag: uploadPart.ETag,
                PartNumber: partNum,
            };
        }
        const completeReq = {
            ...otherParams,
            MultipartUpload: multipartMap,
            UploadId: uploadId,
        };
        await this.s3.completeMultipartUpload(completeReq).promise();
    }
    async doWriteContentUsingXHR(fullPath, content) {
        try {
            const url = await this.getSignedUrl(fullPath, "putObject");
            const xhr = new kura_1.XHR(this.name, fullPath, {
                timeout: this.config.httpOptions.timeout,
            });
            if (kura_1.isBlob(content) || ArrayBuffer.isView(content)) {
                await xhr.put(url, content);
            }
            else {
                const view = new Uint8Array(content);
                await xhr.put(url, view);
            }
        }
        catch (err) {
            throw new kura_1.InvalidModificationError(this.name, fullPath, err);
        }
    }
    async fromBody(body) {
        if (isReactNative) {
            return await kura_1.toArrayBuffer(body);
        }
        else {
            return body;
        }
    }
    getKey(fullPath) {
        const path = kura_1.normalizePath(this.rootDir + kura_1.DIR_SEPARATOR + fullPath);
        const key = S3Util_1.getKey(path);
        return key;
    }
    async getSignedUrl(fullPath, operation) {
        const key = this.getKey(fullPath);
        const url = await this.s3.getSignedUrlPromise(operation, {
            Bucket: this.bucket,
            Key: key,
            Expires: this.s3Options.expires,
        });
        return url;
    }
    handleNotFoundErrorS3(fullPath, err) {
        if (this.isNotFoundError(err)) {
            throw new kura_1.NotFoundError(this.name, fullPath, err);
        }
    }
    isNotFoundError(err) {
        if (!err) {
            return false;
        }
        const awsError = err;
        if (awsError.statusCode === 404) {
            return true;
        }
        return false;
    }
    async toBody(content) {
        if (isNode) {
            return kura_1.toBuffer(content);
        }
        if (isReactNative) {
            if (hasBuffer) {
                return kura_1.toBuffer(content);
            }
            else {
                return kura_1.toArrayBuffer(content);
            }
        }
        return kura_1.toBlob(content);
    }
}
exports.S3Accessor = S3Accessor;
//# sourceMappingURL=S3Accessor.js.map