s3-file-manager
Version:
A streamlined, high-level S3 client for Node.js with built-in retries and support for uploads, downloads, and file operations — works with any S3-compatible storage.
384 lines (383 loc) • 20.6 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DownloadManager = void 0;
const client_s3_1 = require("@aws-sdk/client-s3");
const s3_request_presigner_1 = require("@aws-sdk/s3-request-presigner");
const wait_js_1 = require("../utils/wait.js");
const file_type_wrapper_1 = require("s3-file-manager/file-type-wrapper");
const is_utf8_1 = __importDefault(require("is-utf8"));
const fs_1 = require("fs");
const promises_1 = require("stream/promises");
const promises_2 = require("fs/promises");
const path_1 = __importDefault(require("path"));
const mime_types_1 = require("mime-types");
const bottleneck_1 = __importDefault(require("bottleneck"));
const TEXT_MIME_PREFIXES = ["text/", "application/xml"];
const TEXT_EXTENSIONS = ["txt", "csv", "xml", "md", "html"];
/**
╔════════════════════════════════════════════════════════════════════════════════╗
║ 📥 DOWNLOAD MANAGER ║
║ Manages downloads from S3, supporting buffered and streamed file retrieval, ║
║ with support for metadata extraction and type detection. ║
╚════════════════════════════════════════════════════════════════════════════════╝
*/
class DownloadManager {
ctx;
limiter;
constructor(context) {
this.ctx = context;
this.limiter = new bottleneck_1.default({ maxConcurrent: 6 });
}
// ════════════════════════════════════════════════════════════════
// 🚿 STREAM FILE FROM S3
// Streams file data without loading it fully into memory
// ════════════════════════════════════════════════════════════════
async getStream(filePath, options = {}) {
const { spanOptions = {}, timeoutMS = 10000 } = options;
const { name: spanName = "S3FileManager.getStream", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
filePath: filePath,
}, } = spanOptions;
return await this.ctx.withSpan(spanName, spanAttributes, async () => {
let attempt = 0;
while (true) {
// Set up timeout function
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMS);
let response;
try {
attempt++;
const command = new client_s3_1.GetObjectCommand({
Bucket: this.ctx.bucketName,
Key: filePath,
});
response = await this.ctx.s3.send(command, {
abortSignal: controller.signal,
});
clearTimeout(timeout);
if (!response.Body) {
throw new Error(`File ${filePath} not found in bucket ${this.ctx.bucketName}`);
}
this.ctx.verboseLog(`Streaming file: ${filePath}`);
return response.Body;
}
catch (error) {
clearTimeout(timeout);
// Close the stream if still open in case of error
if (response?.Body &&
"readableEnded" in response.Body &&
!response.Body.readableEnded) {
response.Body.destroy();
}
if (error.name === "AbortError") {
this.ctx.logger.warn(`Streaming ${filePath} timed out after ${timeoutMS}ms`);
}
this.ctx.handleRetryErrorLogging(attempt, `to stream ${filePath}`, error);
await (0, wait_js_1.wait)((0, wait_js_1.backoffDelay)(attempt));
}
}
});
}
// ════════════════════════════════════════════════════════════════
// 📄 LOAD FILE CONTENTS
// Loads a file's contents into memory as Buffer, text, or object
// ════════════════════════════════════════════════════════════════
async downloadFile(filePath, options = {}) {
const { spanOptions = {} } = options;
const { name: spanName = "S3FileManager.downloadFile", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
filePath: filePath,
}, } = spanOptions;
const result = await this.ctx.withSpan(spanName, spanAttributes, async () => {
let attempt = 0;
while (true) {
try {
attempt++;
const stream = await this.getStream(filePath);
const fileBuffer = await this.ctx.streamToBuffer(stream, "Readable");
this.ctx.verboseLog(`Downloaded stream into buffer for ${filePath}`, "info");
const fileFormat = await this.getFileFormat({
filePath,
callerName: "S3FileManager.downloadFile",
});
const returnType = fileFormat.fileType;
this.ctx.verboseLog(`Parsed ${filePath} as ${returnType}`, "info");
switch (returnType) {
case "text":
return fileBuffer.toString("utf-8");
case "json":
return JSON.parse(fileBuffer.toString("utf-8"));
default:
return fileBuffer;
}
}
catch (error) {
this.ctx.handleRetryErrorLogging(attempt, `to load file: ${filePath}`, error);
await (0, wait_js_1.wait)((0, wait_js_1.backoffDelay)(attempt));
}
}
});
return result;
}
// ════════════════════════════════════════════════════════════════
// 💾 DOWNLOAD TO DISK
// Downloads a file from S3 and writes it to the local file system
// ════════════════════════════════════════════════════════════════
async downloadToDisk(filePath, outDir, options = {}) {
const { spanOptions = {}, outputFilename } = options;
const { name: spanName = "S3FileManager.downloadToDisk", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
filePath: filePath,
outDir: outDir,
}, } = spanOptions;
// Normalize and correctly format outDir
const formattedOutDir = path_1.default.resolve(outDir) + path_1.default.sep;
await this.ctx.withSpan(spanName, spanAttributes, async () => {
const fileMetadata = await this.getFileMetadata(filePath, "S3FileManager.downloadToDisk");
const originalFileName = path_1.default.parse(filePath).name;
const stream = await this.getStream(filePath);
let fileBuffer;
if (fileMetadata.contentLength &&
fileMetadata.contentLength <= 200 * 1024 * 1024) {
fileBuffer = await this.ctx.streamToBuffer(stream, "Readable");
}
let destinationPath;
if (outputFilename) {
destinationPath = formattedOutDir + outputFilename;
}
else {
const fileFormat = await this.getFileFormat({
filePath,
callerName: "S3FileManager.downloadToDisk",
mimeType: fileMetadata.mimeType,
fileBuffer,
});
destinationPath = formattedOutDir + originalFileName;
if (fileFormat.extension) {
destinationPath += "." + fileFormat.extension;
}
}
await (0, promises_2.mkdir)(formattedOutDir, { recursive: true });
if (fileBuffer) {
this.ctx.verboseLog(`Preparing to write file ${filePath} to ${formattedOutDir}`, "info");
await (0, promises_2.writeFile)(destinationPath, fileBuffer);
}
else {
this.ctx.verboseLog(`Streaming large file ${filePath} directly to disk`, "info");
await (0, promises_1.pipeline)(stream, (0, fs_1.createWriteStream)(destinationPath));
}
this.ctx.verboseLog(`Successfully downloaded ${filePath}`);
});
}
// ════════════════════════════════════════════════════════════════
// 📦 BULK DOWNLOAD TO DISK
// Downloads all files with a given prefix to the local file system
// ════════════════════════════════════════════════════════════════
async downloadFolderToDisk(prefix, outDir, options = {}) {
const { spanOptions = {} } = options;
const { name: spanName = "S3FileManager.downloadFolderToDisk", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
prefix,
}, } = spanOptions;
const result = await this.ctx.withSpan(spanName, spanAttributes, async () => {
const filesToDownload = await this.ctx.listItems(prefix, {
spanOptions: {
name: "S3FileManager.downloadFolderToDisk > listItems",
attributes: { bucket: this.ctx.bucketName, prefix },
},
});
if (filesToDownload.length === 0) {
return {
success: true,
message: `No files found with prefix ${prefix}`,
downloadedFiles: 0,
failedToDownload: [],
};
}
// Construct final out directory (outPath) for all files (the input outDir plus the last folder from the prefix)
// If outDir = C:/myfolder and prefix = "sourcefolder/images"
// then outPath = C:/myfolder/images
const trimmedPrefix = prefix.replace(/\/+$/, ""); // Remove trailing slashes
const smallestFolder = path_1.default.basename(trimmedPrefix);
const outPath = path_1.default.join(outDir, smallestFolder);
this.ctx.verboseLog(`Downloading ${filesToDownload.length} files to ${outDir}`, "info");
const result = await Promise.all(filesToDownload.map(async (file) => {
// Construct out directory (adjustedOutDir) for specific files to preserve internal file structure
// by appending folders nested within the prefix to the out directory.
//
// If prefix = "sourcefolder" and outPath = C:/myfolder/images and file(key) = sourcefolder/images/animals/cats/cat.jpg
// then adjustedOutDir = C:/myfolder/images/animals/cats/
const relativeFolder = trimmedPrefix.length > 0
? path_1.default.dirname(file.slice(trimmedPrefix.length + 1))
: path_1.default.dirname(file);
const adjustedOutDir = path_1.default.join(outPath, relativeFolder);
this.ctx.verboseLog(`Starting download for ${file}`, "info");
try {
await this.limiter.schedule(() => this.downloadToDisk(file, adjustedOutDir, {
spanOptions: {
name: "S3FileManager.downloadFolderToDisk > downloadToDisk",
attributes: {
bucket: this.ctx.bucketName,
filePath: file,
outDir: adjustedOutDir,
},
},
}));
return null;
}
catch (error) {
this.ctx.verboseLog(`File ${file} failed to download: ${this.ctx.errorString(error)}`, "warn");
return file;
}
}));
const failedToDownload = result.filter(Boolean);
if (failedToDownload.length === 0) {
return {
success: true,
message: `All files ${prefix.length > 0
? `with prefix ${prefix}`
: "in root folder"} successfully downloaded`,
downloadedFiles: filesToDownload.length,
failedToDownload,
};
}
else if (failedToDownload.length === filesToDownload.length) {
return {
success: false,
message: `All files ${prefix.length > 0
? `with prefix ${prefix}`
: "in root folder"} failed to download. For details, enable verbose logging.`,
downloadedFiles: 0,
failedToDownload,
};
}
else {
return {
success: true,
message: `Some files ${prefix.length > 0
? `with prefix ${prefix}`
: "in root folder"} failed to download. For details, enable verbose logging.`,
downloadedFiles: filesToDownload.length - failedToDownload.length,
failedToDownload,
};
}
});
return result;
}
// ════════════════════════════════════════════════════════════════
// 🔗 GENERATE TEMPORARY SIGNED URL
// Generates a presigned URL for temporary access to an S3 file
// ════════════════════════════════════════════════════════════════
async getTemporaryDownloadUrl(filePath, options = {}) {
const { spanOptions = {}, expiresInSec = 60 * 60 } = options;
const { name: spanName = "S3FileManager.getTemporaryDownloadUrl", attributes: spanAttributes = {
bucket: this.ctx.bucketName,
filePath: filePath,
expiresInSec,
}, } = spanOptions;
const command = new client_s3_1.GetObjectCommand({
Bucket: this.ctx.bucketName,
Key: filePath,
});
const result = await this.ctx.withSpan(spanName, spanAttributes, async () => {
let attempt = 0;
while (true) {
try {
attempt++;
const signedUrl = await (0, s3_request_presigner_1.getSignedUrl)(this.ctx.s3, command, {
expiresIn: expiresInSec,
});
this.ctx.verboseLog(`Generated temporary download URL for ${filePath}`, "info");
return signedUrl;
}
catch (error) {
this.ctx.handleRetryErrorLogging(attempt, `to generate temporary download link for ${filePath}`, error);
await (0, wait_js_1.wait)((0, wait_js_1.backoffDelay)(attempt));
}
}
});
return result;
}
// ════════════════════════════════════════════════════════════════
// 🧾 GET FILE METADATA
// Retrieves file metadata such as MIME type and content length
// ════════════════════════════════════════════════════════════════
async getFileMetadata(filePath, callerName) {
const command = new client_s3_1.HeadObjectCommand({
Bucket: this.ctx.bucketName,
Key: filePath,
});
const s3MetaData = await this.ctx.withSpan(`${callerName} > getMimeType`, { filePath }, async () => {
let attempt = 0;
while (true) {
attempt++;
try {
const response = await this.ctx.s3.send(command);
return response;
}
catch (error) {
if (error.name === "NotFound" ||
error.$metadata?.httpStatusCode === 404) {
throw new Error(`File ${filePath} not found`);
}
this.ctx.handleRetryErrorLogging(attempt, `to get MIME type of file ${filePath}`, error);
await (0, wait_js_1.wait)((0, wait_js_1.backoffDelay)(attempt));
}
}
});
if (!s3MetaData.ContentType)
this.ctx.logger.warn(`Missing ContentType for ${filePath}`);
const fileMetadata = {
mimeType: s3MetaData.ContentType,
contentLength: s3MetaData.ContentLength,
};
return fileMetadata;
}
// ════════════════════════════════════════════════════════════════
// 🧪 DETERMINE FILE FORMAT
// Determines file content type and best-guess extension
// ════════════════════════════════════════════════════════════════
async getFileFormat({ filePath, callerName, fileBuffer, mimeType, }) {
const filePathLC = filePath.toLowerCase();
if (!mimeType) {
mimeType = (await this.getFileMetadata(filePath, `${callerName} > getFileFormat`)).mimeType;
}
const fileType = fileBuffer
? await (0, file_type_wrapper_1.fileTypeFromBuffer)(fileBuffer)
: undefined;
// Determine file content type
let returnType = "buffer";
if (mimeType && mimeType !== "application/octet-stream") {
if (mimeType === "application/json") {
returnType = "json";
}
else if (TEXT_MIME_PREFIXES.some((prefix) => mimeType.startsWith(prefix))) {
returnType = "text";
}
}
else if (fileType || (fileBuffer && !(0, is_utf8_1.default)(fileBuffer))) {
returnType = "buffer";
}
else if (filePathLC.endsWith("json") &&
(!fileBuffer || (0, is_utf8_1.default)(fileBuffer))) {
returnType = "json";
}
else if (TEXT_EXTENSIONS.some((extension) => filePathLC.endsWith(extension))) {
returnType = "text";
}
else {
returnType = "buffer";
}
// Get file extension
let ext = (0, mime_types_1.extension)(mimeType || "") ||
fileType?.ext ||
path_1.default.extname(filePath).slice(1);
if (ext === "")
this.ctx.logger.warn(`Unable to determine a file extension for file ${filePath}`);
return { fileType: returnType, extension: ext };
}
}
exports.DownloadManager = DownloadManager;