@uizard/nx-fast-s3-cache
Version:
Nx s3 cache using GNU tar, pigz and multipart s3 uploads/downloads
71 lines (70 loc) • 2.56 kB
JavaScript
;
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_Scenario_UsingLargeFiles_section.html
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.downloadFromS3 = void 0;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const client_s3_1 = require("@aws-sdk/client-s3");
const bluebird_1 = require("bluebird");
const fs_2 = require("fs");
const oneMB = 1024 * 1024;
const chunkSize = oneMB * 10;
/**
* massively parallel download ranges from s3
* speedy, optimized for large files
*/
const downloadFromS3 = async ({ s3Client, Bucket, Key, savePath, }) => {
const headObject = ({ bucket, key }) => {
const command = new client_s3_1.HeadObjectCommand({
Bucket: bucket,
Key: key,
});
return s3Client.send(command);
};
const getObjectRange = ({ bucket, key, start, end, }) => {
const command = new client_s3_1.GetObjectCommand({
Bucket: bucket,
Key: key,
Range: `bytes=${start}-${end}`,
});
return s3Client.send(command);
};
fs_1.default.mkdirSync(path_1.default.dirname(savePath), { recursive: true });
const writeStream = (0, fs_2.createWriteStream)(savePath).on("error", (err) => {
throw new Error(`Could not write to ${savePath}: ${err.message}`);
});
const { ContentLength: length } = await headObject({
bucket: Bucket,
key: Key,
});
if (!length) {
throw new Error(`Object ${Key} does not exist in bucket ${Bucket}`);
}
const numberOfChunks = Math.ceil(length / chunkSize);
let chunkIndexOrder = 0;
await bluebird_1.Promise.map(Array.from({ length: numberOfChunks }, (_, i) => i), async (i) => {
const start = i * chunkSize;
const end = Math.min(start + chunkSize - 1, length - 1);
const { Body } = await getObjectRange({
bucket: Bucket,
key: Key,
start,
end,
});
if (Body === undefined) {
return;
}
const byteArray = await Body.transformToByteArray();
while (chunkIndexOrder !== i) {
await bluebird_1.Promise.delay(100);
}
writeStream.write(byteArray);
chunkIndexOrder++;
},
// up to Gigabit
{ concurrency: 100 });
};
exports.downloadFromS3 = downloadFromS3;