@uizard/nx-fast-s3-cache/dist/runner-implementation/index.js

Nx s3 cache using GNU tar, pigz and multipart s3 uploads/downloads

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const child_process_1 = require("child_process");
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const init_env_1 = require("../init-env");
const download_1 = require("./download");
const s3_client_1 = require("./s3-client");
const upload_1 = require("./upload");
const util_1 = require("./util");
const extractFolder = async (zipFilePath, destination) => {
    await fs_1.default.mkdirSync(destination, { recursive: true });
    await new Promise((res, rej) => {
        const args = ["xf", zipFilePath, "--strip", "1"];
        const spawnedProcess = (0, child_process_1.spawn)("/usr/bin/tar", args, { cwd: destination });
        spawnedProcess.stdout.on("data", (data) => {
            process.stdout.write(data.toString());
        });
        spawnedProcess.stderr.on("data", (data) => {
            process.stderr.write(data.toString());
        });
        spawnedProcess.on("exit", (code) => {
            if (code !== 0) {
                rej(`Non zero exit code: ${code}`);
            }
            else {
                res({ code });
            }
        });
    });
};
const archiveFolder = async ({ folder, destinationFile, }) => {
    await new Promise((res, rej) => {
        const args = [
            "--use-compress-program=pigz",
            "-cf",
            path_1.default.basename(destinationFile),
            folder,
        ];
        const spawnedProcess = (0, child_process_1.spawn)("/usr/bin/tar", args, {
            cwd: path_1.default.dirname(destinationFile),
        });
        spawnedProcess.stdout.on("data", (data) => {
            process.stdout.write(data.toString());
        });
        spawnedProcess.stderr.on("data", (data) => {
            process.stderr.write(data.toString());
        });
        spawnedProcess.on("exit", (code) => {
            if (code !== 0) {
                rej(`Non zero exit code: ${code}`);
            }
            else {
                res({ code });
            }
        });
    });
    return destinationFile;
};
const ENV_BUCKET = "NXCACHE_S3_BUCKET";
const ENV_PREFIX = "NXCACHE_S3_PREFIX";
const ENV_READ_ONLY = "NXCACHE_S3_READ_ONLY";
const ENV_WRITE_ONLY = "NXCACHE_S3_WRITE_ONLY";
exports.default = async (options) => {
    var _a, _b, _c;
    (0, init_env_1.initEnv)(options);
    const s3Client = (0, s3_client_1.buildS3Client)(options);
    const bucket = (_a = process.env[ENV_BUCKET]) !== null && _a !== void 0 ? _a : options.bucket;
    const prefix = (_c = (_b = process.env[ENV_PREFIX]) !== null && _b !== void 0 ? _b : options.prefix) !== null && _c !== void 0 ? _c : "";
    const readOnly = (0, util_1.isReadOnly)(options, ENV_READ_ONLY);
    const config = {
        name: "S3",
        writeOnly: (0, util_1.isWriteOnly)(options, ENV_WRITE_ONLY),
        fileExists: async (filename) => {
            try {
                const result = await s3Client.headObject((0, util_1.buildCommonCommandInput)({ bucket, prefix, filename }));
                return { result: !!result };
            }
            catch (error) {
                if (error.name === "403" ||
                    error.name === "NotFound") {
                    return { result: false };
                }
                else {
                    throw error;
                }
            }
        },
        retrieveFile: async (file, destination) => {
            const tmpFolder = fs_1.default.mkdtempSync("/tmp/nx-cache-");
            const tmpFile = path_1.default.join(tmpFolder, path_1.default.basename(file));
            const timings = { download: 0, extract: 0 };
            timings.download = Date.now();
            await (0, download_1.downloadFromS3)({
                ...(0, util_1.buildCommonCommandInput)({ bucket, prefix, filename: file }),
                s3Client,
                savePath: tmpFile,
            });
            timings.download = Date.now() - timings.download;
            timings.extract = Date.now();
            await extractFolder(tmpFile, destination);
            timings.extract = Date.now() - timings.extract;
            fs_1.default.rm(tmpFolder, { recursive: true }, () => { });
            return { path: destination, timings };
        },
        storeFile: async (filename, folderToArchive) => {
            if (readOnly) {
                throw new Error("ReadOnly storage, cannot store file");
            }
            const timings = { compress: 0, upload: 0 };
            timings.compress = Date.now();
            await archiveFolder({
                folder: folderToArchive,
                destinationFile: filename,
            });
            timings.compress = Date.now() - timings.compress;
            timings.upload = Date.now();
            await (0, upload_1.uploadToS3)({
                ...(0, util_1.buildCommonCommandInput)({
                    bucket,
                    prefix,
                    filename: path_1.default.basename(filename),
                }),
                s3Client,
                filePath: filename,
            });
            timings.upload = Date.now() - timings.upload;
            return { path: filename, timings };
        },
    };
    return config;
};