s3-file-manager
Version:
A streamlined, high-level S3 client for Node.js with built-in retries and support for uploads, downloads, and file operations — works with any S3-compatible storage.
377 lines (376 loc) • 19.8 kB
JavaScript
import { GetObjectCommand, HeadObjectCommand, } from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import { backoffDelay, wait } from "../utils/wait.js";
import { fileTypeFromBuffer } from "s3-file-manager/file-type-wrapper";
import isUtf8 from "is-utf8";
import { createWriteStream } from "fs";
import { pipeline } from "stream/promises";
import { mkdir, writeFile } from "fs/promises";
import path from "path";
import { extension } from "mime-types";
import Bottleneck from "bottleneck";
const TEXT_MIME_PREFIXES = ["text/", "application/xml"];
const TEXT_EXTENSIONS = ["txt", "csv", "xml", "md", "html"];
/**
╔════════════════════════════════════════════════════════════════════════════════╗
║ 📥 DOWNLOAD MANAGER ║
║ Manages downloads from S3, supporting buffered and streamed file retrieval, ║
║ with support for metadata extraction and type detection. ║
╚════════════════════════════════════════════════════════════════════════════════╝
*/
export class DownloadManager {
ctx;
limiter;
constructor(context) {
this.ctx = context;
this.limiter = new Bottleneck({ maxConcurrent: 6 });
}
// ════════════════════════════════════════════════════════════════
// 🚿 STREAM FILE FROM S3
// Streams file data without loading it fully into memory
// ════════════════════════════════════════════════════════════════
async getStream(filePath, options = {}) {
const { spanOptions = {}, timeoutMS = 10000 } = options;
const { name: spanName = "S3FileManager.getStream", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
filePath: filePath,
}, } = spanOptions;
return await this.ctx.withSpan(spanName, spanAttributes, async () => {
let attempt = 0;
while (true) {
// Set up timeout function
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMS);
let response;
try {
attempt++;
const command = new GetObjectCommand({
Bucket: this.ctx.bucketName,
Key: filePath,
});
response = await this.ctx.s3.send(command, {
abortSignal: controller.signal,
});
clearTimeout(timeout);
if (!response.Body) {
throw new Error(`File ${filePath} not found in bucket ${this.ctx.bucketName}`);
}
this.ctx.verboseLog(`Streaming file: ${filePath}`);
return response.Body;
}
catch (error) {
clearTimeout(timeout);
// Close the stream if still open in case of error
if (response?.Body &&
"readableEnded" in response.Body &&
!response.Body.readableEnded) {
response.Body.destroy();
}
if (error.name === "AbortError") {
this.ctx.logger.warn(`Streaming ${filePath} timed out after ${timeoutMS}ms`);
}
this.ctx.handleRetryErrorLogging(attempt, `to stream ${filePath}`, error);
await wait(backoffDelay(attempt));
}
}
});
}
// ════════════════════════════════════════════════════════════════
// 📄 LOAD FILE CONTENTS
// Loads a file's contents into memory as Buffer, text, or object
// ════════════════════════════════════════════════════════════════
async downloadFile(filePath, options = {}) {
const { spanOptions = {} } = options;
const { name: spanName = "S3FileManager.downloadFile", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
filePath: filePath,
}, } = spanOptions;
const result = await this.ctx.withSpan(spanName, spanAttributes, async () => {
let attempt = 0;
while (true) {
try {
attempt++;
const stream = await this.getStream(filePath);
const fileBuffer = await this.ctx.streamToBuffer(stream, "Readable");
this.ctx.verboseLog(`Downloaded stream into buffer for ${filePath}`, "info");
const fileFormat = await this.getFileFormat({
filePath,
callerName: "S3FileManager.downloadFile",
});
const returnType = fileFormat.fileType;
this.ctx.verboseLog(`Parsed ${filePath} as ${returnType}`, "info");
switch (returnType) {
case "text":
return fileBuffer.toString("utf-8");
case "json":
return JSON.parse(fileBuffer.toString("utf-8"));
default:
return fileBuffer;
}
}
catch (error) {
this.ctx.handleRetryErrorLogging(attempt, `to load file: ${filePath}`, error);
await wait(backoffDelay(attempt));
}
}
});
return result;
}
// ════════════════════════════════════════════════════════════════
// 💾 DOWNLOAD TO DISK
// Downloads a file from S3 and writes it to the local file system
// ════════════════════════════════════════════════════════════════
async downloadToDisk(filePath, outDir, options = {}) {
const { spanOptions = {}, outputFilename } = options;
const { name: spanName = "S3FileManager.downloadToDisk", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
filePath: filePath,
outDir: outDir,
}, } = spanOptions;
// Normalize and correctly format outDir
const formattedOutDir = path.resolve(outDir) + path.sep;
await this.ctx.withSpan(spanName, spanAttributes, async () => {
const fileMetadata = await this.getFileMetadata(filePath, "S3FileManager.downloadToDisk");
const originalFileName = path.parse(filePath).name;
const stream = await this.getStream(filePath);
let fileBuffer;
if (fileMetadata.contentLength &&
fileMetadata.contentLength <= 200 * 1024 * 1024) {
fileBuffer = await this.ctx.streamToBuffer(stream, "Readable");
}
let destinationPath;
if (outputFilename) {
destinationPath = formattedOutDir + outputFilename;
}
else {
const fileFormat = await this.getFileFormat({
filePath,
callerName: "S3FileManager.downloadToDisk",
mimeType: fileMetadata.mimeType,
fileBuffer,
});
destinationPath = formattedOutDir + originalFileName;
if (fileFormat.extension) {
destinationPath += "." + fileFormat.extension;
}
}
await mkdir(formattedOutDir, { recursive: true });
if (fileBuffer) {
this.ctx.verboseLog(`Preparing to write file ${filePath} to ${formattedOutDir}`, "info");
await writeFile(destinationPath, fileBuffer);
}
else {
this.ctx.verboseLog(`Streaming large file ${filePath} directly to disk`, "info");
await pipeline(stream, createWriteStream(destinationPath));
}
this.ctx.verboseLog(`Successfully downloaded ${filePath}`);
});
}
// ════════════════════════════════════════════════════════════════
// 📦 BULK DOWNLOAD TO DISK
// Downloads all files with a given prefix to the local file system
// ════════════════════════════════════════════════════════════════
async downloadFolderToDisk(prefix, outDir, options = {}) {
const { spanOptions = {} } = options;
const { name: spanName = "S3FileManager.downloadFolderToDisk", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
prefix,
}, } = spanOptions;
const result = await this.ctx.withSpan(spanName, spanAttributes, async () => {
const filesToDownload = await this.ctx.listItems(prefix, {
spanOptions: {
name: "S3FileManager.downloadFolderToDisk > listItems",
attributes: { bucket: this.ctx.bucketName, prefix },
},
});
if (filesToDownload.length === 0) {
return {
success: true,
message: `No files found with prefix ${prefix}`,
downloadedFiles: 0,
failedToDownload: [],
};
}
// Construct final out directory (outPath) for all files (the input outDir plus the last folder from the prefix)
// If outDir = C:/myfolder and prefix = "sourcefolder/images"
// then outPath = C:/myfolder/images
const trimmedPrefix = prefix.replace(/\/+$/, ""); // Remove trailing slashes
const smallestFolder = path.basename(trimmedPrefix);
const outPath = path.join(outDir, smallestFolder);
this.ctx.verboseLog(`Downloading ${filesToDownload.length} files to ${outDir}`, "info");
const result = await Promise.all(filesToDownload.map(async (file) => {
// Construct out directory (adjustedOutDir) for specific files to preserve internal file structure
// by appending folders nested within the prefix to the out directory.
//
// If prefix = "sourcefolder" and outPath = C:/myfolder/images and file(key) = sourcefolder/images/animals/cats/cat.jpg
// then adjustedOutDir = C:/myfolder/images/animals/cats/
const relativeFolder = trimmedPrefix.length > 0
? path.dirname(file.slice(trimmedPrefix.length + 1))
: path.dirname(file);
const adjustedOutDir = path.join(outPath, relativeFolder);
this.ctx.verboseLog(`Starting download for ${file}`, "info");
try {
await this.limiter.schedule(() => this.downloadToDisk(file, adjustedOutDir, {
spanOptions: {
name: "S3FileManager.downloadFolderToDisk > downloadToDisk",
attributes: {
bucket: this.ctx.bucketName,
filePath: file,
outDir: adjustedOutDir,
},
},
}));
return null;
}
catch (error) {
this.ctx.verboseLog(`File ${file} failed to download: ${this.ctx.errorString(error)}`, "warn");
return file;
}
}));
const failedToDownload = result.filter(Boolean);
if (failedToDownload.length === 0) {
return {
success: true,
message: `All files ${prefix.length > 0
? `with prefix ${prefix}`
: "in root folder"} successfully downloaded`,
downloadedFiles: filesToDownload.length,
failedToDownload,
};
}
else if (failedToDownload.length === filesToDownload.length) {
return {
success: false,
message: `All files ${prefix.length > 0
? `with prefix ${prefix}`
: "in root folder"} failed to download. For details, enable verbose logging.`,
downloadedFiles: 0,
failedToDownload,
};
}
else {
return {
success: true,
message: `Some files ${prefix.length > 0
? `with prefix ${prefix}`
: "in root folder"} failed to download. For details, enable verbose logging.`,
downloadedFiles: filesToDownload.length - failedToDownload.length,
failedToDownload,
};
}
});
return result;
}
// ════════════════════════════════════════════════════════════════
// 🔗 GENERATE TEMPORARY SIGNED URL
// Generates a presigned URL for temporary access to an S3 file
// ════════════════════════════════════════════════════════════════
async getTemporaryDownloadUrl(filePath, options = {}) {
const { spanOptions = {}, expiresInSec = 60 * 60 } = options;
const { name: spanName = "S3FileManager.getTemporaryDownloadUrl", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
filePath: filePath,
expiresInSec,
}, } = spanOptions;
const command = new GetObjectCommand({
Bucket: this.ctx.bucketName,
Key: filePath,
});
const result = await this.ctx.withSpan(spanName, spanAttributes, async () => {
let attempt = 0;
while (true) {
try {
attempt++;
const signedUrl = await getSignedUrl(this.ctx.s3, command, {
expiresIn: expiresInSec,
});
this.ctx.verboseLog(`Generated temporary download URL for ${filePath}`, "info");
return signedUrl;
}
catch (error) {
this.ctx.handleRetryErrorLogging(attempt, `to generate temporary download link for ${filePath}`, error);
await wait(backoffDelay(attempt));
}
}
});
return result;
}
// ════════════════════════════════════════════════════════════════
// 🧾 GET FILE METADATA
// Retrieves file metadata such as MIME type and content length
// ════════════════════════════════════════════════════════════════
async getFileMetadata(filePath, callerName) {
const command = new HeadObjectCommand({
Bucket: this.ctx.bucketName,
Key: filePath,
});
const s3MetaData = await this.ctx.withSpan(`${callerName} > getMimeType`, { filePath }, async () => {
let attempt = 0;
while (true) {
attempt++;
try {
const response = await this.ctx.s3.send(command);
return response;
}
catch (error) {
if (error.name === "NotFound" ||
error.$metadata?.httpStatusCode === 404) {
throw new Error(`File ${filePath} not found`);
}
this.ctx.handleRetryErrorLogging(attempt, `to get MIME type of file ${filePath}`, error);
await wait(backoffDelay(attempt));
}
}
});
if (!s3MetaData.ContentType)
this.ctx.logger.warn(`Missing ContentType for ${filePath}`);
const fileMetadata = {
mimeType: s3MetaData.ContentType,
contentLength: s3MetaData.ContentLength,
};
return fileMetadata;
}
// ════════════════════════════════════════════════════════════════
// 🧪 DETERMINE FILE FORMAT
// Determines file content type and best-guess extension
// ════════════════════════════════════════════════════════════════
async getFileFormat({ filePath, callerName, fileBuffer, mimeType, }) {
const filePathLC = filePath.toLowerCase();
if (!mimeType) {
mimeType = (await this.getFileMetadata(filePath, `${callerName} > getFileFormat`)).mimeType;
}
const fileType = fileBuffer
? await fileTypeFromBuffer(fileBuffer)
: undefined;
// Determine file content type
let returnType = "buffer";
if (mimeType && mimeType !== "application/octet-stream") {
if (mimeType === "application/json") {
returnType = "json";
}
else if (TEXT_MIME_PREFIXES.some((prefix) => mimeType.startsWith(prefix))) {
returnType = "text";
}
}
else if (fileType || (fileBuffer && !isUtf8(fileBuffer))) {
returnType = "buffer";
}
else if (filePathLC.endsWith("json") &&
(!fileBuffer || isUtf8(fileBuffer))) {
returnType = "json";
}
else if (TEXT_EXTENSIONS.some((extension) => filePathLC.endsWith(extension))) {
returnType = "text";
}
else {
returnType = "buffer";
}
// Get file extension
let ext = extension(mimeType || "") ||
fileType?.ext ||
path.extname(filePath).slice(1);
if (ext === "")
this.ctx.logger.warn(`Unable to determine a file extension for file ${filePath}`);
return { fileType: returnType, extension: ext };
}
}