s3-file-manager
Version:
A streamlined, high-level S3 client for Node.js with built-in retries and support for uploads, downloads, and file operations — works with any S3-compatible storage.
400 lines (399 loc) • 21.1 kB
JavaScript
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, PutObjectCommand, UploadPartCommand, } from "@aws-sdk/client-s3";
import { Readable } from "stream";
import { wait } from "../utils/wait.js";
import { backoffDelay } from "../utils/wait.js";
import { isStreamType } from "../utils/type-guards.js";
import { lookup as mimeLookup } from "mime-types";
import Bottleneck from "bottleneck";
import path from "path";
import fs from "fs";
/**
╔════════════════════════════════════════════════════════════════════════════════╗
║ 📤 UPLOAD MANAGER ║
║ Handles file uploads to S3, including direct uploads, multipart uploads, and ║
║ retry logic for reliability. ║
╚════════════════════════════════════════════════════════════════════════════════╝
*/
export class UploadManager {
ctx;
maxConcurrent;
limiter;
constructor(context, maxUploadConcurrency) {
this.ctx = context;
this.maxConcurrent = maxUploadConcurrency ?? 4;
this.limiter = new Bottleneck({ maxConcurrent: this.maxConcurrent });
}
// ════════════════════════════════════════════════════════════════
// 🔼 UPLOAD FILE
// Routes file upload requests to the appropriate upload function based on size
// ════════════════════════════════════════════════════════════════
async uploadFile(file, options = {}) {
const { content, sizeHintBytes, contentType } = file;
const { spanOptions = {}, prefix } = options;
const { name: spanName = "S3FileManager.uploadFile", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
filename: `${prefix ?? ""}${file.name}`,
}, } = spanOptions;
const result = await this.ctx.withSpan(spanName, spanAttributes, async () => {
let sizeInBytes;
let type;
if (typeof content === "string") {
sizeInBytes =
sizeHintBytes ?? Buffer.byteLength(content, "utf-8");
type = "string";
}
else if (Buffer.isBuffer(content)) {
sizeInBytes = sizeHintBytes ?? content.length;
type = "Buffer";
}
else if (content instanceof Uint8Array) {
sizeInBytes = sizeHintBytes ?? content.byteLength;
type = "Uint8Array";
}
else if (typeof Blob !== "undefined" &&
content instanceof Blob) {
sizeInBytes = sizeHintBytes ?? content.size;
type = "Blob";
}
else if (content instanceof Readable) {
sizeInBytes = sizeHintBytes ?? undefined;
type = "Readable";
}
else {
sizeInBytes = sizeHintBytes ?? undefined;
type = "ReadableStream";
}
this.ctx.verboseLog(`Uploading ${file.name} (${sizeInBytes} bytes) using ${sizeInBytes && sizeInBytes > this.ctx.maxUploadPartSize
? "multipart"
: "simple"} upload`);
const mimeType = contentType ??
(mimeLookup(file.name) || "application/octet-stream");
if (sizeInBytes === undefined ||
sizeInBytes > this.ctx.maxUploadPartSize) {
return await this.multipartUpload(file, type, mimeType, sizeInBytes, options.prefix);
}
else {
return await this.simpleUpload(file, mimeType, options.prefix);
}
});
return result;
}
// ════════════════════════════════════════════════════════════════
// 📦 UPLOAD MULTIPLE FILES
// Iterates through arrays of files and uploads them with predefined concurrency limits
// ════════════════════════════════════════════════════════════════
async uploadMultipleFiles(files, options = {}) {
// Limiter to prevent race conditions when pushing to skippedFiles array.
const mutex = new Bottleneck({ maxConcurrent: 1 });
const { spanOptions = {}, prefix } = options;
const { name: spanName = "S3FileManager.uploadMultipleFiles", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
}, } = spanOptions;
let skippedFiles = [];
let filePaths = [];
await this.ctx.withSpan(spanName, spanAttributes, async () => {
const results = await Promise.all(files.map(async (file) => {
try {
this.ctx.verboseLog(`Uploading file: ${file.name}`, "info");
const result = await this.uploadFile(file, { prefix });
this.ctx.verboseLog(`Successfully uploaded file: ${file.name}`, "info");
return result;
}
catch (error) {
await mutex.schedule(async () => {
skippedFiles.push(file.name);
});
this.ctx.verboseLog(String(error), "warn");
this.ctx.verboseLog("Skipping file...", "warn");
}
}));
filePaths = results.filter(Boolean);
});
if (skippedFiles.length > 0) {
this.ctx.logger.warn(`Upload multiple files finished, but the following ${skippedFiles.length} file(s) failed to upload: ${skippedFiles.join(", ")}`);
}
else {
this.ctx.logger.info("Upload multiple files complete. All files successfully uploaded.");
}
return { filePaths, skippedFiles };
}
// ════════════════════════════════════════════════════════════════
// 📥 UPLOAD FROM DISK
// Uploads a single local file, using streaming or buffering based on size
// ════════════════════════════════════════════════════════════════
async uploadFromDisk(localFilePath, options = {}) {
const { spanOptions = {}, prefix } = options;
this.ctx.verboseLog(`Reading file from disk: ${localFilePath}`, "info");
const stats = fs.statSync(localFilePath);
const sizeInBytes = stats.size;
this.ctx.verboseLog(sizeInBytes > this.ctx.maxUploadPartSize
? `Using stream for upload: ${localFilePath}`
: `Buffering file for upload: ${localFilePath}`, "info");
const fileName = path.basename(localFilePath);
const { name: spanName = "S3FileManager.uploadFile", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
sourceFile: `${localFilePath}`,
}, } = spanOptions;
const result = await this.ctx.withSpan(spanName, spanAttributes, async () => {
let content;
if (sizeInBytes > this.ctx.maxUploadPartSize) {
content = fs.createReadStream(localFilePath);
}
else {
content = fs.readFileSync(localFilePath);
}
const file = {
name: fileName,
content,
sizeHintBytes: sizeInBytes,
};
return await this.uploadFile(file, {
prefix,
spanOptions: {
name: "S3FileManager.uploadFromDisk > uploadFile",
attributes: {
bucket: this.ctx.bucketName,
sourceFile: `${localFilePath}`,
},
},
});
});
return result;
}
// ════════════════════════════════════════════════════════════════
// 📥 UPLOAD MULTIPLE FILES FROM DISK
// Uploads multiple local files, using streaming for all to minimize memory usage
// ════════════════════════════════════════════════════════════════
async uploadMultipleFromDisk(localFilePaths, options = {}) {
const { spanOptions = {}, prefix } = options;
const { name: spanName = "S3FileManager.uploadFile", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
numberOfFiles: `${localFilePaths.length}`,
}, } = spanOptions;
const result = await this.ctx.withSpan(spanName, spanAttributes, async () => {
const files = await Promise.all(localFilePaths.map(async (filePath) => {
this.ctx.verboseLog(`Preparing file for upload: ${filePath}`, "info");
const stats = await fs.promises.stat(filePath);
const sizeInBytes = stats.size;
const fileName = path.basename(filePath);
const content = fs.createReadStream(filePath);
const file = {
name: fileName,
content,
sizeHintBytes: sizeInBytes,
};
return file;
}));
this.ctx.verboseLog(`Prepared ${files.length} file(s) for upload`, "info");
return await this.uploadMultipleFiles(files, {
prefix,
spanOptions: {
name: "S3FileManager.uploadMultipleFromDisk > uploadFile",
attributes: {
bucket: this.ctx.bucketName,
numberOfFiles: `${localFilePaths.length}`,
},
},
});
});
return result;
}
// ════════════════════════════════════════════════════════════════
// 📤 SIMPLE UPLOAD
// Uploads a small file in a single PUT request
// ════════════════════════════════════════════════════════════════
async simpleUpload(file, mimeType, prefix) {
const key = `${prefix ?? ""}${file.name}`;
let attempt = 0;
const result = await this.ctx.withSpan("S3FileManager.uploadFile > simpleUpload", {
bucket: this.ctx.bucketName,
filename: `${prefix ?? ""}${file.name}`,
}, async () => {
while (true) {
try {
attempt++;
const command = new PutObjectCommand({
Bucket: this.ctx.bucketName,
Key: key,
Body: file.content,
ContentType: mimeType,
});
await this.limiter.schedule(() => this.ctx.s3.send(command));
this.ctx.verboseLog(`Successfully uploaded ${file.name}`);
return `${prefix ?? ""}${file.name}`;
}
catch (error) {
this.ctx.handleRetryErrorLogging(attempt, `to upload ${file.name}`, error);
// Wait before next attempt
await wait(backoffDelay(attempt));
}
}
});
return result;
}
// ════════════════════════════════════════════════════════════════
// 🧱 MULTIPART UPLOAD
// Splits large content into parts and uploads via multipart API
// ════════════════════════════════════════════════════════════════
async multipartUpload(file, type, mimeType, size, prefix) {
let fileChunks;
try {
if (isStreamType(type) && (!size || size > 200 * 1024 * 1024)) {
fileChunks = await this.streamToIterable(file.content, type);
this.ctx.verboseLog("Successfully prepared stream for multipart upload.");
}
else {
let buffer;
if (isStreamType(type)) {
buffer = await this.ctx.streamToBuffer(file.content, type);
}
else if (type === "string") {
buffer = Buffer.from(file.content);
}
else if (type === "Uint8Array") {
buffer = Buffer.from(file.content);
}
else {
buffer = file.content;
}
fileChunks = this.bufferToChunks(buffer);
this.ctx.verboseLog("Successfully constructed Buffer for multipart upload.");
}
}
catch (error) {
throw new Error(`Something went wrong while attempting to prepare content for multipart upload: ${this.ctx.errorString(error)}`, { cause: error });
}
const response = await this.ctx.withSpan("S3FileManager.uploadFile > multipartUpload", {
bucket: this.ctx.bucketName,
filename: `${prefix ?? ""}${file.name}`,
}, async () => {
let uploadId;
const filename = `${prefix ?? ""}${file.name}`;
let attempt = 0;
while (true) {
attempt++;
try {
const createResponse = await this.ctx.s3.send(new CreateMultipartUploadCommand({
Bucket: this.ctx.bucketName,
Key: filename,
ContentType: mimeType,
}));
uploadId = createResponse.UploadId;
if (!uploadId)
throw new Error("Failed to initiate multipart upload");
const parts = [];
let partNumber = 1;
for await (const chunk of fileChunks) {
const ETag = await this.uploadPartWithRetry(filename, uploadId, partNumber, chunk);
parts.push({
ETag,
PartNumber: partNumber,
});
partNumber++;
}
await this.ctx.s3.send(new CompleteMultipartUploadCommand({
Bucket: this.ctx.bucketName,
Key: filename,
UploadId: uploadId,
MultipartUpload: { Parts: parts },
}));
this.ctx.verboseLog(`File ${filename} successfully uploaded to S3 bucket`);
return `${prefix ?? ""}${file.name}`;
}
catch (error) {
await this.ctx.s3.send(new AbortMultipartUploadCommand({
Bucket: this.ctx.bucketName,
Key: filename,
UploadId: uploadId,
}));
this.ctx.handleRetryErrorLogging(attempt, `to upload ${filename}`, error);
// Wait before next attempt
await wait(backoffDelay(attempt));
}
}
});
return response;
}
// ════════════════════════════════════════════════════════════════
// 🔁 UPLOAD PART WITH RETRY
// Uploads individual chunks of a multipart upload with retry logic
// ════════════════════════════════════════════════════════════════
async uploadPartWithRetry(filename, uploadId, partNumber, chunk) {
let attempt = 0;
while (true) {
try {
attempt++;
this.ctx.verboseLog(`Uploading part ${partNumber} of ${filename}`);
const uploadPartResponse = await this.ctx.withSpan("S3FileManager.uploadFile > multipartUpload > uploadPartWithRetry", { filename, uploadId, partNumber }, async () => await this.limiter.schedule(() => this.ctx.s3.send(new UploadPartCommand({
Bucket: this.ctx.bucketName,
Key: filename,
PartNumber: partNumber,
UploadId: uploadId,
Body: chunk,
}))));
this.ctx.verboseLog(`Part ${partNumber} of ${filename} successfully uploaded.`);
return uploadPartResponse.ETag;
}
catch (error) {
this.ctx.handleRetryErrorLogging(attempt, `to upload part ${partNumber} of ${filename}`, error);
// Wait before next attempt
await wait(backoffDelay(attempt));
}
}
}
// ════════════════════════════════════════════════════════════════
// 🔄 STREAM TO ITERABLE
// Converts a readable stream into iterable buffer chunks
// ════════════════════════════════════════════════════════════════
async *streamToIterable(stream, type) {
let buffer = Buffer.alloc(0);
if (type === "Readable") {
for await (const chunk of stream) {
buffer = Buffer.concat([
buffer,
Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk),
]);
while (buffer.length >= this.ctx.maxUploadPartSize) {
yield buffer.subarray(0, this.ctx.maxUploadPartSize);
buffer = buffer.subarray(this.ctx.maxUploadPartSize);
}
}
if (buffer.length)
yield buffer;
}
else {
const preparedStream = type === "Blob"
? stream.stream()
: stream;
const reader = preparedStream.getReader();
while (true) {
const { done, value } = await reader.read();
if (done)
break;
buffer = Buffer.concat([buffer, Buffer.from(value)]);
while (buffer.length >= this.ctx.maxUploadPartSize) {
yield buffer.subarray(0, this.ctx.maxUploadPartSize);
buffer = buffer.subarray(this.ctx.maxUploadPartSize);
}
}
if (buffer.length)
yield buffer;
}
}
// ════════════════════════════════════════════════════════════════
// 📚 BUFFER TO CHUNKS
// Splits a Buffer into multipart-sized chunks
// ════════════════════════════════════════════════════════════════
bufferToChunks(buffer) {
const bufferChunks = [];
const totalFileSize = buffer.length;
const numberOfParts = Math.ceil(totalFileSize / this.ctx.maxUploadPartSize);
for (let part = 1; part <= numberOfParts; part++) {
const start = (part - 1) * this.ctx.maxUploadPartSize;
const end = Math.min(start + this.ctx.maxUploadPartSize, totalFileSize);
const chunk = buffer.subarray(start, end);
bufferChunks.push(chunk);
}
return bufferChunks;
}
}