@uizard/nx-fast-s3-cache/dist/runner-implementation/upload.js

Nx s3 cache using GNU tar, pigz and multipart s3 uploads/downloads

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.uploadToS3 = void 0;
const client_s3_1 = require("@aws-sdk/client-s3");
const lib_storage_1 = require("@aws-sdk/lib-storage");
const bluebird_1 = require("bluebird");
const fs_1 = __importDefault(require("fs"));
const oneMB = 1024 * 1024;
// Multipart uploads require a minimum size of 5 MB per part.
const chunkSize = oneMB * 7;
const readPartOfFile = ({ filePath, start, end, }) => new Promise((res, rej) => {
    const buffers = [];
    const stream = fs_1.default.createReadStream(filePath, { start, end });
    stream.on("data", (chunk) => buffers.push(chunk));
    stream.on("error", rej);
    stream.on("end", () => res(Buffer.concat(buffers)));
});
const uploadToS3 = async ({ s3Client, Bucket, Key, filePath, }) => {
    const { size: fileSize } = fs_1.default.statSync(filePath);
    let multipartUploadId = "";
    try {
        const numberOfChunks = Math.ceil(fileSize / chunkSize);
        if (numberOfChunks === 1) {
            // no reason to do a multipart upload
            const fileContent = await new Promise((res, rej) => fs_1.default.readFile(filePath, (err, data) => (err ? rej(err) : res(data))));
            const upload = new lib_storage_1.Upload({
                client: s3Client,
                params: {
                    Bucket,
                    Key,
                    // eslint-disable-next-line @typescript-eslint/naming-convention
                    Body: fileContent,
                },
            });
            return upload.done();
        }
        const multipartUpload = await s3Client.send(new client_s3_1.CreateMultipartUploadCommand({
            Bucket: Bucket,
            Key: Key,
        }));
        multipartUploadId = multipartUpload.UploadId;
        const uploadResults = await bluebird_1.Promise.map(Array.from({ length: numberOfChunks }, (_, i) => i), async (i) => {
            const start = i * chunkSize;
            const end = Math.min(start + chunkSize - 1, fileSize - 1);
            const result = await s3Client.send(new client_s3_1.UploadPartCommand({
                Bucket,
                Key,
                UploadId: multipartUploadId,
                Body: await readPartOfFile({ filePath, start, end }),
                PartNumber: i + 1,
            }));
            return result;
        }, { concurrency: 30 });
        return await s3Client.send(new client_s3_1.CompleteMultipartUploadCommand({
            Bucket,
            Key,
            UploadId: multipartUploadId,
            MultipartUpload: {
                Parts: uploadResults.map(({ ETag }, i) => ({
                    ETag,
                    PartNumber: i + 1,
                })),
            },
        }));
    }
    catch (err) {
        if (multipartUploadId) {
            const abortCommand = new client_s3_1.AbortMultipartUploadCommand({
                Bucket,
                Key,
                UploadId: multipartUploadId,
            });
            await s3Client.send(abortCommand);
        }
        throw err;
    }
};
exports.uploadToS3 = uploadToS3;