UNPKG

@qrvey/object-storage

Version:

![install size](https://packagephobia.com/badge?p=%40qrvey%2Fobject-storage) ![coverage](https://img.shields.io/badge/unit_test_coverage-8%25-brightgreen)

1,342 lines (1,316 loc) 64.2 kB
'use strict'; var stream = require('stream'); var storageBlob = require('@azure/storage-blob'); var clientS3 = require('@aws-sdk/client-s3'); var s3RequestPresigner = require('@aws-sdk/s3-request-presigner'); var libStorage = require('@aws-sdk/lib-storage'); var http = require('http'); var https = require('https'); function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; } var stream__default = /*#__PURE__*/_interopDefault(stream); var __defProp = Object.defineProperty; var __defProps = Object.defineProperties; var __getOwnPropDescs = Object.getOwnPropertyDescriptors; var __getOwnPropSymbols = Object.getOwnPropertySymbols; var __hasOwnProp = Object.prototype.hasOwnProperty; var __propIsEnum = Object.prototype.propertyIsEnumerable; var __knownSymbol = (name, symbol) => { if (symbol = Symbol[name]) return symbol; throw Error("Symbol." + name + " is not defined"); }; var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value; var __spreadValues = (a, b) => { for (var prop in b || (b = {})) if (__hasOwnProp.call(b, prop)) __defNormalProp(a, prop, b[prop]); if (__getOwnPropSymbols) for (var prop of __getOwnPropSymbols(b)) { if (__propIsEnum.call(b, prop)) __defNormalProp(a, prop, b[prop]); } return a; }; var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b)); var __objRest = (source, exclude) => { var target = {}; for (var prop in source) if (__hasOwnProp.call(source, prop) && exclude.indexOf(prop) < 0) target[prop] = source[prop]; if (source != null && __getOwnPropSymbols) for (var prop of __getOwnPropSymbols(source)) { if (exclude.indexOf(prop) < 0 && __propIsEnum.call(source, prop)) target[prop] = source[prop]; } return target; }; var __forAwait = (obj, it, method) => (it = obj[__knownSymbol("asyncIterator")]) ? it.call(obj) : (obj = obj[__knownSymbol("iterator")](), it = {}, method = (key, fn) => (fn = obj[key]) && (it[key] = (arg) => new Promise((yes, no, done) => (arg = fn.call(obj, arg), done = arg.done, Promise.resolve(arg.value).then((value) => yes({ value, done }), no)))), method("next"), method("return"), it); // src/shared/utils/errorHandler.ts var errorMessages = { AccessDenied: "Access denied", AccountProblem: "There is a problem with your account", AllAccessDisabled: "All access to this resource has been disabled", BucketAlreadyExists: "The requested bucket name is not available", BucketNotEmpty: "The bucket you tried to delete is not empty", EntityTooLarge: "The entity you are trying to upload is too large", ExpiredToken: "The provided token has expired", InternalError: "An internal server error has occurred", InvalidAccessKeyId: "The AWS access key ID you provided does not exist in our records", InvalidBucketName: "The specified bucket name is not valid", InvalidObjectState: "The operation is not valid for the current state of the object", InvalidToken: "The provided token is invalid", NoSuchBucket: "The specified bucket does not exist", NoSuchKey: "The specified key does not exist", PreconditionFailed: "The condition specified in the request is not met", RequestTimeout: "The request timed out", ServiceUnavailable: "The service is currently unavailable", SignatureDoesNotMatch: "The request signature we calculated does not match the signature you provided", SlowDown: "Please reduce your request rate", TemporaryRedirect: "Temporary redirect", ObjectNotFound: "ObjectNotFound - The specified key does not exist. (404)", DEFAULT: "An unknown error occurred" }; var ErrorHandler = class { static handleError(errorCode, errorMessage, errorObj) { const errorResponse = { code: errorCode, message: errorMessage || errorMessages[errorCode] || errorMessages["DEFAULT"], timestamp: (/* @__PURE__ */ new Date()).toISOString(), error: errorObj ? { name: errorObj.name, message: errorObj.message, stack: errorObj.stack || null } : null }; return errorResponse; } }; // src/services/storage/blob/blobHelpers.ts function BlobPropertiesResponseToObjectResponse(blobProperties) { return { lastModified: blobProperties.lastModified, contentLength: blobProperties.contentLength, contentType: blobProperties.contentType, eTag: blobProperties.etag, metadata: blobProperties.metadata, contentEncoding: blobProperties.contentEncoding, cacheControl: blobProperties.cacheControl, contentDisposition: blobProperties.contentDisposition, contentLanguage: blobProperties.contentLanguage }; } // src/services/storage/blob/blocIdStorage.ts var BlockIdStorage = class _BlockIdStorage { constructor() { this.blockIdMap = {}; } static getInstance() { if (!_BlockIdStorage.instance) { _BlockIdStorage.instance = new _BlockIdStorage(); } return _BlockIdStorage.instance; } addBlockId(blobName, blockId) { if (!this.blockIdMap[blobName]) { this.blockIdMap[blobName] = []; } this.blockIdMap[blobName].push(blockId); } getBlockIds(blobName) { return this.blockIdMap[blobName] || []; } clearBlockIds(blobName) { delete this.blockIdMap[blobName]; } }; // src/services/storage/blob/blobStorage.service.ts var BlobStorageService = class { /** * Retrieves the properties of a blob from the container by its name. * * @param {string} blobName - The name of the blob. * @return {Promise<GetObjectResponse>} A promise that resolves to the blob properties. */ constructor(containerName) { this.containerName = containerName; this.blobServiceClient = storageBlob.BlobServiceClient.fromConnectionString( process.env.AZURE_STORAGE_CONNECTION_STRING ); } /** * Creates a writable stream for uploading a file to the Blob storage. * * @param {string} blobName - The name of the blob to upload. * @return {CreateUploadWriteStreamResponse} An object containing the key of the uploaded blob, the writable stream, and a promise that resolves when the upload is complete. */ createUploadWriteStream(blobName) { const streamPassThrough = new stream__default.default.PassThrough(); const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const uploadPromise = blockBlobClient.uploadStream(streamPassThrough, void 0, void 0, { onProgress: (ev) => console.log(ev) }).then(() => { return { Bucket: this.containerName, Key: blobName }; }); return { key: blobName, stream: streamPassThrough, promise: uploadPromise }; } /** * Retrieves the properties of a blob from the container by its name. * * @param {string} blobName - The name of the blob. * @return {Promise<GetObjectResponse>} A promise that resolves to the blob properties. */ async getHeadObject(blobName) { try { const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const blobProperties = await blockBlobClient.getProperties(); return BlobPropertiesResponseToObjectResponse(blobProperties); } catch (error) { throw ErrorHandler.handleError("DEFAULT", "", error); } } /** * Retrieves an object from the blob storage service. * * @param {string} blobName - The name of the blob to retrieve. * @param {Object} [options] - The options that you can pass to the library. * @return {Promise<GetObjectResponse>} A promise that resolves to the object data, including the body, metadata, content type, and content length, or rejects with an error if there was an issue. */ async getObject(blobName, options) { var _a; try { let downloadBlockBlobResponse; const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const expression = (options == null ? void 0 : options.range) || ""; const matches = expression.match(/\d+/g); if (expression && (matches == null ? void 0 : matches.length)) { const [start, end] = matches.map(Number); downloadBlockBlobResponse = await blockBlobClient.download( start, end ); } else { downloadBlockBlobResponse = await blockBlobClient.download(0); } return { body: downloadBlockBlobResponse.readableStreamBody, metadata: downloadBlockBlobResponse.metadata, contentType: downloadBlockBlobResponse.contentType, contentLength: downloadBlockBlobResponse.contentLength }; } catch (error) { let errorKey = "DEFAULT"; if (((_a = error == null ? void 0 : error.response) == null ? void 0 : _a.status) === 404) { errorKey = "ObjectNotFound"; } throw ErrorHandler.handleError(errorKey, "", error); } } async getSignatureUrl(blobName, expiresInMinutes, permissions) { try { const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const blockBlobClient = containerClient.getBlockBlobClient(blobName); const startDate = /* @__PURE__ */ new Date(); const expiryDate = new Date(startDate); expiryDate.setMinutes(startDate.getMinutes() + expiresInMinutes); return blockBlobClient.generateSasUrl({ permissions: storageBlob.BlobSASPermissions.parse(permissions), startsOn: startDate, expiresOn: expiryDate }); } catch (error) { throw ErrorHandler.handleError("DEFAULT", "", error); } } async getUploadUrl(blobName, expiresInMinutes) { try { const sasUrl = await this.getSignatureUrl( blobName, expiresInMinutes, "w" ); return { key: blobName, signedUrl: sasUrl }; } catch (error) { throw ErrorHandler.handleError("DEFAULT", "", error); } } /** * Retrieves a signed URL for the blob with the specified name that expires after a certain period. * * @param {string} blobName - The name of the blob to generate the signed URL for. * @param {number} expiresInMinutes - The duration in minutes until the signed URL expires. * @return {Promise<string>} A Promise that resolves with the signed URL. */ async getDownloadUrl(blobName, expiresInMinutes) { try { const sasUrl = await this.getSignatureUrl( blobName, expiresInMinutes, "r" ); return sasUrl; } catch (error) { throw ErrorHandler.handleError("DEFAULT", "", error); } } /** * Uploads a file to the blob storage service. * * @param {string} blobName - The name of the blob to upload. * @param {FileContent} body - The content of the file to upload. * @param {{ [key: string]: string } | undefined} [metadata] - Optional metadata to associate with the blob. * @return {Promise<UploadResponse>} A promise that resolves to the response object containing the key of the uploaded blob, or rejects with an error if there was an issue. */ async upload(blobName, body, metadata = {}) { try { const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const blockBlobClient = containerClient.getBlockBlobClient(blobName); Buffer.isBuffer(body) ? await blockBlobClient.upload(body, body.length, { metadata }) : await blockBlobClient.uploadStream( body, void 0, void 0, { metadata } ); return { key: blobName }; } catch (error) { throw ErrorHandler.handleError("DEFAULT", "", error); } } /** * Deletes a blob from the blob storage service. * * @param {string} blobName - The name of the blob to be deleted. * @return {Promise<boolean>} A promise that resolves to true if the blob is successfully deleted, or rejects with an error if there was an issue. */ async delete(data) { try { const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); return this.deleteObject(containerClient, data); } catch (error) { throw ErrorHandler.handleError("DEFAULT", "", error); } } /** * Lists a chunk of blobs from the specified container client based on the provided options and continuation token. * * @param {ListRequestOptions} options - The options for listing the blobs. * @param {string | undefined} continuationToken - The continuation token for pagination. * @param {number} limit - The maximum number of blobs to list in a single page. * @param {ContainerClient} containerClient - The container client to list the blobs from. * @return {Promise<{ items: BlobItem[]; continuationToken?: string }>} - A promise that resolves to an object containing the list of blob items and an optional continuation token. */ async listChunk(options, continuationToken, limit, containerClient) { const iterator = containerClient.listBlobsFlat({ prefix: options.prefix }).byPage({ maxPageSize: limit, continuationToken }); const items = []; const response = (await iterator.next()).value; items.push(...response.segment.blobItems); return { items, continuationToken: response.continuationToken }; } /** * Retrieves a list of blob contents based on the provided options. * * @param {ListRequestOptions} options - The options for listing the blob contents. * @return {Promise<{ contents: BlobItem[]; nextContinuationToken?: string }>} A promise that resolves to the list of blob contents and the next continuation token. */ async listContents(options) { var _a; let responseContents = []; let continuationToken = options.pagination; const limit = (_a = options.limit) != null ? _a : 1e3; let continueListing = true; const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); while (continueListing) { const response = await this.listChunk( options, continuationToken, limit - responseContents.length, containerClient ); continuationToken = response.continuationToken; responseContents = responseContents.concat(response.items); if (responseContents.length >= limit || !continuationToken) { continueListing = false; } } return { contents: responseContents, nextContinuationToken: continuationToken }; } /** * Lists blobs in the Azure Blob Storage container with pagination. * @param options - The options for listing blobs. * @returns A promise that resolves to a paginated list of blob names. */ async list(options) { var _a; let items = []; const response = await this.listContents(options); items = response.contents.length ? response.contents.map((blob) => { return { key: blob.name, lastModified: blob.properties.lastModified, size: blob.properties.contentLength, eTag: blob.properties.etag }; }) : []; return { items, pagination: ((_a = response.nextContinuationToken) == null ? void 0 : _a.length) ? response.nextContinuationToken : null, count: items.length }; } /** * Lists all blobs in the Azure Blob Storage container. * @param options - The options for listing blobs. * @returns A promise that resolves to list of blob names. */ async listAll(options) { var _a, _b; let allItems = []; let pagination = void 0; do { const response = await this.list(__spreadProps(__spreadValues({}, options), { pagination })); if ((_a = response.items) == null ? void 0 : _a.length) allItems = [...allItems, ...response.items]; pagination = (_b = response.pagination) != null ? _b : void 0; } while (pagination); return { items: allItems, count: allItems.length }; } /** * Deletes a blob from the blob storage service. * * @param {string} blobName - The name of the blob to be deleted. * @return {Promise<boolean>} A promise that resolves to true if the blob is successfully deleted, or rejects with an error if there was an issue. */ async deleteObject(containerClient, blobName) { try { const blockBlobClient = containerClient.getBlockBlobClient(blobName); await blockBlobClient.delete(); return true; } catch (error) { throw ErrorHandler.handleError("DEFAULT", "", error); } } /** * Deletes multiple blobs from the blob storage service. * * @param {string[]} blobNames - An array of blob names to be deleted. * @return {Promise<{ key: string; deleted: boolean; error?: string }[]>} - A promise that resolves to an array of objects containing the key, deleted status, and an optional error message for each blob deleted. */ async deleteObjects(blobNames) { const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const deleteBlobPromises = blobNames.map(async (blobName) => { var _a; try { await this.deleteObject(containerClient, blobName); return { key: blobName, deleted: true }; } catch (error) { return { key: blobName, deleted: false, error: (_a = error == null ? void 0 : error.message) != null ? _a : "" }; } }); return Promise.all(deleteBlobPromises); } /** * Retrieves the properties of the container. * * @return {Promise<HeadBucketResponse>} A promise that resolves to the container properties. */ async getHeadBucket() { try { const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const properties = await containerClient.getProperties(); return properties; } catch (error) { throw new Error( "Error in Azure getHeadContainer. Container: " + this.containerName + " Error: " + error ); } } async listMultipartUploadsForBucket() { const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const blobList = containerClient.listBlobsFlat(); const uploads = []; try { for (var iter = __forAwait(blobList), more, temp, error; more = !(temp = await iter.next()).done; more = false) { const blob = temp.value; uploads.push({ uploadId: blob.name, key: blob.name, initiated: blob.properties.createdOn || /* @__PURE__ */ new Date() }); } } catch (temp) { error = [temp]; } finally { try { more && (temp = iter.return) && await temp.call(iter); } finally { if (error) throw error[0]; } } return { bucketName: this.containerName, uploads }; } /** * Retrieves the list of parts for a multipart upload. * * @param {string} blobName - The name of the blob (file) in Azure Blob Storage. * @return {Promise<ListPartsMultipartUploadResponse>} A promise that resolves to the list of parts for the multipart upload. */ async listMultipartUploadsForKey(blobName) { var _a; const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const blobClient = containerClient.getBlockBlobClient(blobName); const listResponse = await blobClient.getBlockList("all"); const parts = ((_a = listResponse == null ? void 0 : listResponse.uncommittedBlocks) == null ? void 0 : _a.map((block, index) => ({ PartNumber: index + 1, LastModified: /* @__PURE__ */ new Date(), // Azure Blob Storage doesn't provide the last modified date for individual parts ETag: block.name, Size: block.size }))) || []; return { Parts: parts }; } /** * Generates a unique upload ID for multipart uploads. * * @return {Promise<string>} A promise that resolves to a unique upload ID. */ async generateUploadIdMultipart() { const timestamp = Date.now(); const randomNum = Math.floor(Math.random() * 100); const blockIdBase = (timestamp - randomNum).toString(); return blockIdBase; } /** * Uploads a part of a file as a block to Azure Blob Storage and updates the metadata with the block ID. * * @param {string} blobName - The name of the blob (file) that you want to upload in parts to Azure Blob Storage. * @param {FileContent | any} file - The content of the file that you want to upload as a part of a multipart upload. * @param {number} partNumber - The number or index of the part being uploaded. * @param {string} [uploadId] - The ID of the multipart upload. If not provided, a new upload ID will be generated. * @return {Promise<string>} A Promise that resolves to a string representing the base64 encoded block ID of the uploaded part. */ async uploadMultipart(blobName, file, partNumber, uploadId) { const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const blobClient = containerClient.getBlockBlobClient(blobName); const blockIdBase = uploadId || await this.generateUploadIdMultipart(); const partId = partNumber.toString().padStart(6, "0"); const partIdBase64 = Buffer.from(partId).toString("base64"); await blobClient.stageBlock(partIdBase64, file, file.length); return blockIdBase; } /** * Completes a multipart upload by committing the blocks to create the blob. * * @param {string} blobName - The name of the blob for which the multipart upload is being completed. * @param {string} uploadId - The ID of the multipart upload to be completed. * @return {Promise<void>} A Promise that resolves when the multipart upload is successfully completed. * @throws {Error} If no block IDs are found in the metadata. */ async completeMultipartUpload(blobName, uploadId) { var _a; const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const blobClient = containerClient.getBlockBlobClient(blobName); const listMultipartUploads = await this.listMultipartUploadsForKey(blobName); const blockIds = ((_a = listMultipartUploads == null ? void 0 : listMultipartUploads.Parts) == null ? void 0 : _a.map((part) => part.ETag)) || []; if (blockIds.length === 0) { throw new Error("No block IDs found in metadata."); } await blobClient.commitBlockList(blockIds); } /** * Aborts a multipart upload by deleting the specified blob and clearing its block IDs metadata. * * @param {string} blobName - The name of the blob for which the multipart upload needs to be aborted. * @param {string} uploadId - The ID of the multipart upload to be aborted. * @return {Promise<void>} A promise that resolves when the multipart upload is successfully aborted. */ async abortMultipartUpload(blobName, uploadId) { const containerClient = this.blobServiceClient.getContainerClient( this.containerName ); const blobClient = containerClient.getBlockBlobClient(blobName); const blockIdStorage = BlockIdStorage.getInstance(); blockIdStorage.clearBlockIds(uploadId); await blobClient.delete(); } async getMultipartUploadPresignedUrl(blobName, uploadId, partNumber, expiresInMinutes = 2) { const partId = partNumber.toString().padStart(6, "0"); const partIdBase64 = Buffer.from(partId).toString("base64"); const sasUrl = await this.getSignatureUrl( blobName, expiresInMinutes, "w" ); const url = `${sasUrl}&comp=block&blockid=${partIdBase64}`; return url; } }; // src/services/storage/s3/s3Helpers.ts function listResponseContentsToListResponseItems(responseContents) { return responseContents.map((responseContent) => { return { key: responseContent.Key, lastModified: new Date(responseContent.LastModified), size: responseContent.Size, eTag: responseContent.ETag }; }); } function s3ObjectToObjectResponse(s3Object) { return { body: s3Object.Body, metadata: s3Object.Metadata, contentType: s3Object.ContentType, contentLength: s3Object.ContentLength, contentEncoding: s3Object.ContentEncoding, cacheControl: s3Object.CacheControl, contentDisposition: s3Object.ContentDisposition, contentLanguage: s3Object.ContentLanguage, eTag: s3Object.ETag, lastModified: s3Object.LastModified }; } // ../../node_modules/@smithy/types/dist-es/auth/auth.js var HttpAuthLocation; (function(HttpAuthLocation2) { HttpAuthLocation2["HEADER"] = "header"; HttpAuthLocation2["QUERY"] = "query"; })(HttpAuthLocation || (HttpAuthLocation = {})); // ../../node_modules/@smithy/types/dist-es/auth/HttpApiKeyAuth.js var HttpApiKeyAuthLocation; (function(HttpApiKeyAuthLocation2) { HttpApiKeyAuthLocation2["HEADER"] = "header"; HttpApiKeyAuthLocation2["QUERY"] = "query"; })(HttpApiKeyAuthLocation || (HttpApiKeyAuthLocation = {})); // ../../node_modules/@smithy/types/dist-es/endpoint.js var EndpointURLScheme; (function(EndpointURLScheme2) { EndpointURLScheme2["HTTP"] = "http"; EndpointURLScheme2["HTTPS"] = "https"; })(EndpointURLScheme || (EndpointURLScheme = {})); // ../../node_modules/@smithy/types/dist-es/extensions/checksum.js var AlgorithmId; (function(AlgorithmId2) { AlgorithmId2["MD5"] = "md5"; AlgorithmId2["CRC32"] = "crc32"; AlgorithmId2["CRC32C"] = "crc32c"; AlgorithmId2["SHA1"] = "sha1"; AlgorithmId2["SHA256"] = "sha256"; })(AlgorithmId || (AlgorithmId = {})); // ../../node_modules/@smithy/types/dist-es/http.js var FieldPosition; (function(FieldPosition2) { FieldPosition2[FieldPosition2["HEADER"] = 0] = "HEADER"; FieldPosition2[FieldPosition2["TRAILER"] = 1] = "TRAILER"; })(FieldPosition || (FieldPosition = {})); // ../../node_modules/@smithy/types/dist-es/profile.js var IniSectionType; (function(IniSectionType2) { IniSectionType2["PROFILE"] = "profile"; IniSectionType2["SSO_SESSION"] = "sso-session"; IniSectionType2["SERVICES"] = "services"; })(IniSectionType || (IniSectionType = {})); // ../../node_modules/@smithy/types/dist-es/transfer.js var RequestHandlerProtocol; (function(RequestHandlerProtocol2) { RequestHandlerProtocol2["HTTP_0_9"] = "http/0.9"; RequestHandlerProtocol2["HTTP_1_0"] = "http/1.0"; RequestHandlerProtocol2["TDS_8_0"] = "tds/8.0"; })(RequestHandlerProtocol || (RequestHandlerProtocol = {})); // ../../node_modules/@smithy/protocol-http/dist-es/httpResponse.js var HttpResponse = class { constructor(options) { this.statusCode = options.statusCode; this.reason = options.reason; this.headers = options.headers || {}; this.body = options.body; } static isInstance(response) { if (!response) return false; const resp = response; return typeof resp.statusCode === "number" && typeof resp.headers === "object"; } }; // ../../node_modules/@smithy/util-uri-escape/dist-es/escape-uri.js var escapeUri = (uri) => encodeURIComponent(uri).replace(/[!'()*]/g, hexEncode); var hexEncode = (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`; // ../../node_modules/@smithy/querystring-builder/dist-es/index.js function buildQueryString(query) { const parts = []; for (let key of Object.keys(query).sort()) { const value = query[key]; key = escapeUri(key); if (Array.isArray(value)) { for (let i = 0, iLen = value.length; i < iLen; i++) { parts.push(`${key}=${escapeUri(value[i])}`); } } else { let qsEntry = key; if (value || typeof value === "string") { qsEntry += `=${escapeUri(value)}`; } parts.push(qsEntry); } } return parts.join("&"); } // ../../node_modules/@smithy/node-http-handler/dist-es/constants.js var NODEJS_TIMEOUT_ERROR_CODES = ["ECONNRESET", "EPIPE", "ETIMEDOUT"]; // ../../node_modules/@smithy/node-http-handler/dist-es/get-transformed-headers.js var getTransformedHeaders = (headers) => { const transformedHeaders = {}; for (const name of Object.keys(headers)) { const headerValues = headers[name]; transformedHeaders[name] = Array.isArray(headerValues) ? headerValues.join(",") : headerValues; } return transformedHeaders; }; // ../../node_modules/@smithy/node-http-handler/dist-es/set-connection-timeout.js var setConnectionTimeout = (request, reject, timeoutInMs = 0) => { if (!timeoutInMs) { return; } const timeoutId = setTimeout(() => { request.destroy(); reject(Object.assign(new Error(`Socket timed out without establishing a connection within ${timeoutInMs} ms`), { name: "TimeoutError" })); }, timeoutInMs); request.on("socket", (socket) => { if (socket.connecting) { socket.on("connect", () => { clearTimeout(timeoutId); }); } else { clearTimeout(timeoutId); } }); }; // ../../node_modules/@smithy/node-http-handler/dist-es/set-socket-keep-alive.js var setSocketKeepAlive = (request, { keepAlive, keepAliveMsecs }) => { if (keepAlive !== true) { return; } request.on("socket", (socket) => { socket.setKeepAlive(keepAlive, keepAliveMsecs || 0); }); }; // ../../node_modules/@smithy/node-http-handler/dist-es/set-socket-timeout.js var setSocketTimeout = (request, reject, timeoutInMs = 0) => { request.setTimeout(timeoutInMs, () => { request.destroy(); reject(Object.assign(new Error(`Connection timed out after ${timeoutInMs} ms`), { name: "TimeoutError" })); }); }; var MIN_WAIT_TIME = 1e3; async function writeRequestBody(httpRequest, request, maxContinueTimeoutMs = MIN_WAIT_TIME) { var _a; const headers = (_a = request.headers) != null ? _a : {}; const expect = headers["Expect"] || headers["expect"]; let timeoutId = -1; let hasError = false; if (expect === "100-continue") { await Promise.race([ new Promise((resolve) => { timeoutId = Number(setTimeout(resolve, Math.max(MIN_WAIT_TIME, maxContinueTimeoutMs))); }), new Promise((resolve) => { httpRequest.on("continue", () => { clearTimeout(timeoutId); resolve(); }); httpRequest.on("error", () => { hasError = true; clearTimeout(timeoutId); resolve(); }); }) ]); } if (!hasError) { writeBody(httpRequest, request.body); } } function writeBody(httpRequest, body) { if (body instanceof stream.Readable) { body.pipe(httpRequest); return; } if (body) { if (Buffer.isBuffer(body) || typeof body === "string") { httpRequest.end(body); return; } const uint8 = body; if (typeof uint8 === "object" && uint8.buffer && typeof uint8.byteOffset === "number" && typeof uint8.byteLength === "number") { httpRequest.end(Buffer.from(uint8.buffer, uint8.byteOffset, uint8.byteLength)); return; } httpRequest.end(Buffer.from(body)); return; } httpRequest.end(); } // ../../node_modules/@smithy/node-http-handler/dist-es/node-http-handler.js var NodeHttpHandler = class _NodeHttpHandler { static create(instanceOrOptions) { if (typeof (instanceOrOptions == null ? void 0 : instanceOrOptions.handle) === "function") { return instanceOrOptions; } return new _NodeHttpHandler(instanceOrOptions); } static checkSocketUsage(agent, socketWarningTimestamp) { var _a, _b, _c, _d; const { sockets, requests, maxSockets } = agent; if (typeof maxSockets !== "number" || maxSockets === Infinity) { return socketWarningTimestamp; } const interval = 15e3; if (Date.now() - interval < socketWarningTimestamp) { return socketWarningTimestamp; } if (sockets && requests) { for (const origin in sockets) { const socketsInUse = (_b = (_a = sockets[origin]) == null ? void 0 : _a.length) != null ? _b : 0; const requestsEnqueued = (_d = (_c = requests[origin]) == null ? void 0 : _c.length) != null ? _d : 0; if (socketsInUse >= maxSockets && requestsEnqueued >= 2 * maxSockets) { console.warn("@smithy/node-http-handler:WARN", `socket usage at capacity=${socketsInUse} and ${requestsEnqueued} additional requests are enqueued.`, "See https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/node-configuring-maxsockets.html", "or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler config."); return Date.now(); } } } return socketWarningTimestamp; } constructor(options) { this.socketWarningTimestamp = 0; this.metadata = { handlerProtocol: "http/1.1" }; this.configProvider = new Promise((resolve, reject) => { if (typeof options === "function") { options().then((_options) => { resolve(this.resolveDefaultConfig(_options)); }).catch(reject); } else { resolve(this.resolveDefaultConfig(options)); } }); } resolveDefaultConfig(options) { const { requestTimeout, connectionTimeout, socketTimeout, httpAgent, httpsAgent } = options || {}; const keepAlive = true; const maxSockets = 50; return { connectionTimeout, requestTimeout: requestTimeout != null ? requestTimeout : socketTimeout, httpAgent: (() => { if (httpAgent instanceof http.Agent || typeof (httpAgent == null ? void 0 : httpAgent.destroy) === "function") { return httpAgent; } return new http.Agent(__spreadValues({ keepAlive, maxSockets }, httpAgent)); })(), httpsAgent: (() => { if (httpsAgent instanceof https.Agent || typeof (httpsAgent == null ? void 0 : httpsAgent.destroy) === "function") { return httpsAgent; } return new https.Agent(__spreadValues({ keepAlive, maxSockets }, httpsAgent)); })() }; } destroy() { var _a, _b, _c, _d; (_b = (_a = this.config) == null ? void 0 : _a.httpAgent) == null ? void 0 : _b.destroy(); (_d = (_c = this.config) == null ? void 0 : _c.httpsAgent) == null ? void 0 : _d.destroy(); } async handle(request, { abortSignal } = {}) { if (!this.config) { this.config = await this.configProvider; } let socketCheckTimeoutId; return new Promise((_resolve, _reject) => { var _a, _b, _c, _d, _e; let writeRequestBodyPromise = void 0; const resolve = async (arg) => { await writeRequestBodyPromise; clearTimeout(socketCheckTimeoutId); _resolve(arg); }; const reject = async (arg) => { await writeRequestBodyPromise; _reject(arg); }; if (!this.config) { throw new Error("Node HTTP request handler config is not resolved"); } if (abortSignal == null ? void 0 : abortSignal.aborted) { const abortError = new Error("Request aborted"); abortError.name = "AbortError"; reject(abortError); return; } const isSSL = request.protocol === "https:"; const agent = isSSL ? this.config.httpsAgent : this.config.httpAgent; socketCheckTimeoutId = setTimeout(() => { this.socketWarningTimestamp = _NodeHttpHandler.checkSocketUsage(agent, this.socketWarningTimestamp); }, (_c = this.config.socketAcquisitionWarningTimeout) != null ? _c : ((_a = this.config.requestTimeout) != null ? _a : 2e3) + ((_b = this.config.connectionTimeout) != null ? _b : 1e3)); const queryString = buildQueryString(request.query || {}); let auth = void 0; if (request.username != null || request.password != null) { const username = (_d = request.username) != null ? _d : ""; const password = (_e = request.password) != null ? _e : ""; auth = `${username}:${password}`; } let path = request.path; if (queryString) { path += `?${queryString}`; } if (request.fragment) { path += `#${request.fragment}`; } const nodeHttpsOptions = { headers: request.headers, host: request.hostname, method: request.method, path, port: request.port, agent, auth }; const requestFunc = isSSL ? https.request : http.request; const req = requestFunc(nodeHttpsOptions, (res) => { const httpResponse = new HttpResponse({ statusCode: res.statusCode || -1, reason: res.statusMessage, headers: getTransformedHeaders(res.headers), body: res }); resolve({ response: httpResponse }); }); req.on("error", (err) => { if (NODEJS_TIMEOUT_ERROR_CODES.includes(err.code)) { reject(Object.assign(err, { name: "TimeoutError" })); } else { reject(err); } }); setConnectionTimeout(req, reject, this.config.connectionTimeout); setSocketTimeout(req, reject, this.config.requestTimeout); if (abortSignal) { abortSignal.onabort = () => { req.abort(); const abortError = new Error("Request aborted"); abortError.name = "AbortError"; reject(abortError); }; } const httpAgent = nodeHttpsOptions.agent; if (typeof httpAgent === "object" && "keepAlive" in httpAgent) { setSocketKeepAlive(req, { keepAlive: httpAgent.keepAlive, keepAliveMsecs: httpAgent.keepAliveMsecs }); } writeRequestBodyPromise = writeRequestBody(req, request, this.config.requestTimeout).catch(_reject); }); } updateHttpClientConfig(key, value) { this.config = void 0; this.configProvider = this.configProvider.then((config) => { return __spreadProps(__spreadValues({}, config), { [key]: value }); }); } httpHandlerConfigs() { var _a; return (_a = this.config) != null ? _a : {}; } }; // src/services/storage/s3/s3Storage.service.ts var S3StorageService = class { constructor(bucketName, options) { this.httpHandler = new NodeHttpHandler({ socketTimeout: 6e5 }); this.s3Client = new clientS3.S3Client({ region: process.env.AWS_DEFAULT_REGION, requestHandler: this.httpHandler }); this.s3 = new clientS3.S3({ region: process.env.AWS_DEFAULT_REGION, requestHandler: this.httpHandler }); var _a, _b, _c, _d; this.bucketName = bucketName; if (((_a = options == null ? void 0 : options.credentials) == null ? void 0 : _a.accessKeyId) && ((_b = options == null ? void 0 : options.credentials) == null ? void 0 : _b.secretAccessKey)) { const s3Config = { region: (options == null ? void 0 : options.region) || process.env.AWS_REGION, credentials: { accessKeyId: (_c = options == null ? void 0 : options.credentials) == null ? void 0 : _c.accessKeyId, secretAccessKey: (_d = options == null ? void 0 : options.credentials) == null ? void 0 : _d.secretAccessKey } }; this.s3Client = new clientS3.S3Client(s3Config); } } /** * Retrieves a chunk of objects from the S3 bucket based on the provided options. * * @param {ListRequestOptions} options - The options for listing the objects. * @param {string | undefined} continuationToken - The continuation token for pagination. * @param {number} limit - The maximum number of objects to retrieve. * @return {Promise<ListObjectsV2CommandOutput>} - A promise that resolves to the response containing the list of objects and metadata. */ async listChunk(options, continuationToken, limit) { const command = new clientS3.ListObjectsV2Command({ Bucket: this.bucketName, ContinuationToken: continuationToken || options.pagination, Prefix: options.prefix, MaxKeys: limit }); return this.s3Client.send(command); } /** * Retrieves a list of contents from the S3 bucket based on the provided options. * * @param {ListRequestOptions} options - The options for listing the contents. * @return {Promise<{ contents: unknown[]; nextContinuationToken?: string }>} - A promise that resolves to an object containing the list of contents and an optional next continuation token. */ async listContents(options) { var _a, _b; let responseContents = []; let continuationToken = void 0; const limit = (_a = options.limit) != null ? _a : 1e3; let continueListing = true; while (continueListing) { const listChunkLimit = limit - responseContents.length; const response = await this.listChunk( options, continuationToken, listChunkLimit ); continuationToken = response.NextContinuationToken; responseContents = responseContents.concat((_b = response.Contents) != null ? _b : []); if (responseContents.length >= limit || !continuationToken) { continueListing = false; } } return { contents: responseContents, nextContinuationToken: continuationToken }; } /** * Lists objects in the S3 bucket with pagination. * @param options - The options for listing objects. * @returns A promise that resolves to a paginated list of object keys. */ async list(options) { var _a; let items = []; const response = await this.listContents(options); items = response.contents.length ? listResponseContentsToListResponseItems(response.contents) : []; return { items, pagination: (_a = response.nextContinuationToken) != null ? _a : null, count: items.length }; } /** * Lists all objects in the S3 bucket. * @param options - The options for listing objects. * @returns A promise that resolves to list of object keys. */ async listAll(options) { var _a; let allItems = []; let pagination = void 0; do { const response = await this.list(__spreadProps(__spreadValues({}, options), { pagination })); if ((_a = response.items) == null ? void 0 : _a.length) allItems = [...allItems, ...response.items]; pagination = response.pagination === null ? void 0 : response.pagination; } while (pagination); return { items: allItems, count: allItems.length }; } /** * Retrieves an object from the S3 bucket. * @param key - The key of the object to retrieve. * @returns A promise that resolves to the content of the file. */ async getHeadObject(key) { const command = new clientS3.HeadObjectCommand({ Bucket: this.bucketName, Key: key }); const response = await this.s3Client.send(command); return s3ObjectToObjectResponse(response); } /** * Retrieves an object from the S3 bucket. * @param key - The key of the object to retrieve. * @returns A promise that resolves to the content of the file. */ async getObject(key) { const command = new clientS3.GetObjectCommand({ Bucket: this.bucketName, Key: key }); const response = await this.s3Client.send(command); return s3ObjectToObjectResponse(response); } /** * Generates a signed URL for accessing an object in the S3 bucket. * @param key - The key of the object. * @param expiresInMinutes - The number of minutes the signed URL should be valid for. Defaults to 60 minutes. * @returns A promise that resolves to the signed URL. */ getDownloadUrl(key, expiresInMinutes = 60) { const expiresIn = expiresInMinutes * 60; const command = new clientS3.GetObjectCommand({ Bucket: this.bucketName, Key: key }); return s3RequestPresigner.getSignedUrl(this.s3Client, command, { expiresIn }); } /** * Retrieves a signed URL for uploading an object to the S3 bucket. * * @param {string} key - The key of the object to be uploaded. * @param {number} [expiresInMinutes=60] - The number of minutes the signed URL should be valid for. Defaults to 60 minutes. * @returns A promise that resolves to the signed URL. */ async getUploadUrl(key, expiresInMinutes = 60) { const expiresIn = expiresInMinutes * 60; const putObjectCommandParams = { Bucket: process.env.UPLOAD_BUCKET_NAME, Key: key, ACL: "private" }; const command = new clientS3.PutObjectCommand(putObjectCommandParams); const signedUrl = await s3RequestPresigner.getSignedUrl(this.s3Client, command, { expiresIn }); return { signedUrl, key: `https://${process.env.UPLOAD_BUCKET_NAME}.s3.${process.env.AWS_DEFAULT_REGION}.amazonaws.com/${key}` }; } /** * Uploads an object to the S3 bucket. * @param key - The key of the object to upload. * @param body - The content of the object to upload. * @param metadata - Optional metadata to associate with the object. * @returns A promise that resolves to the response containing the key of the uploaded object. */ async upload(key, body, metadata) { const command = new clientS3.PutObjectCommand({ Bucket: this.bucketName, Key: key, Body: body, Metadata: metadata }); await this.s3Client.send(command); return { key }; } /** * Creates a writable stream for uploading a file to the S3 bucket. * * @param {string} key - The key of the object to upload. * @return {CreateUploadWriteStreamResponse} An object containing the key of the uploaded object, the writable stream, and a promise that resolves when the upload is complete. */ createUploadWriteStream(key) { const streamPassThrough = new stream__default.default.PassThrough(); const params = { Bucket: this.bucketName, Key: key, Body: streamPassThrough }; const upload = new libStorage.Upload({ client: this.s3Client, params, queueSize: 5, partSize: 8 * 1024 * 1024 }); return { key, stream: streamPassThrough, promise: upload.done() }; } /** * Deletes an object from the S3 bucket. * @param key - The key of the object to delete. * @returns A promise that resolves to true if the object was deleted successfully. */ async delete(key) { const command = new clientS3.DeleteObjectCommand({ Bucket: this.bucketName, Key: key }); await this.s3Client.send(command); return true; } async getHeadBucket() { try { const command = new clientS3.HeadBucketCommand({ Bucket: this.bucketName }); return this.s3Client.send(command); } catch (error) { throw new Error( "Error in S3 getHeadContainer. bucketName: " + this.bucketName + " Error: " + error ); } } /** * Generates an upload ID for multipart upload in the S3 bucket. * * @param {string} key - The key of the object to upload. * @return {Promise<string>} A promise that resolves to the generated upload ID. * @throws {Error} If no upload ID was generated. */ async generateUploadIdMultipart(key) { const params = { Bucket: this.bucketName, Key: key }; const response = await this.s3.createMultipartUpload(params); if (!(response == null ? void 0 : response.UploadId)) throw new Error("No upload ID was generated"); return response.UploadId; } async listMultipartUploadsForBucket() { const params = { Bucket: this.bucketName }; const response = await this.s3.listMultipartUploads(params); return { bucketName: this.bucketName, uploads: response.Uploads ? response.Uploads.map((upload) => { var _a, _b; return { uploadId: upload.UploadId || "", key: upload.Key || "", initiator: ((_a = upload == null ? void 0 : upload.Initiator) == null ? void 0 : _a.ID) || "", owner: ((_b = upload == null ? void 0 : upload.Owner) == null ? void 0 : _b.ID) || "", storageClass: upload.StorageClass || "", initiated: upload.Initiated || "" }; }) : [] }; } /** * Retrieves the list of parts for a multipart upload. * * @param {string} key - The key of the object being uploaded. * @param {string} uploadId - The ID of the multipart upload. * @return {Promise<ListPartsMultipartUploadResponse>} A promise that resolves to the list of parts for the multipart upload. */ async listMultipartUploadsForKey(key, uploadId) { const params = { Bucket: this.bucketName, Key: key, UploadId: uploadId }; const response = await this.s3.listParts(params); const parts = response.Parts ? response.Parts.map((part) => ({ PartNumber: part.PartNumber, LastModified: part.LastModified, ETag: part.ETag, Size: part.Size })) : []; return { Parts: parts }; } async uploadMultipart(key, file, partNumber, uploadId) { const upId = uploadId || await this.generateUploadIdMultipart(key); const params = { Bucket: this.bucketName, Key: key, Body: file, PartNumber: partNumber, UploadId: upId }; await this.s3.uploadPart(params); return upId; } async completeMultipartUpload(key, uploadId) { const partsResponse = await this.listMultipartUploadsForKey( key, uploadId ); const partsList = (partsResponse == null ? void 0 : partsResponse.Parts) && partsResponse.Parts.map( // eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars (_a) => { var _b = _a, partList = __objRest(_b, ["Size", "LastModified"]); return partList; } ); const completeMultipartParams = { Bucket: this.bucketName, Key: key, UploadId: uploadId, MultipartUpload: { Parts: partsList } }; await this.s3.completeMultipartUpload(completeMultipartParams); } async ab