@qrvey/object-storage
Version:
 
1,342 lines (1,319 loc) • 64 kB
JavaScript
import stream, { Readable } from 'stream';
import { BlobServiceClient, BlobSASPermissions } from '@azure/storage-blob';
import { S3Client, S3, ListObjectsV2Command, HeadObjectCommand, GetObjectCommand, PutObjectCommand, DeleteObjectCommand, HeadBucketCommand, UploadPartCommand } from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { Upload } from '@aws-sdk/lib-storage';
import { Agent, request as request$1 } from 'http';
import { Agent as Agent$1, request } from 'https';
var __defProp = Object.defineProperty;
var __defProps = Object.defineProperties;
var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
var __getOwnPropSymbols = Object.getOwnPropertySymbols;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __propIsEnum = Object.prototype.propertyIsEnumerable;
var __knownSymbol = (name, symbol) => {
if (symbol = Symbol[name])
return symbol;
throw Error("Symbol." + name + " is not defined");
};
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __spreadValues = (a, b) => {
for (var prop in b || (b = {}))
if (__hasOwnProp.call(b, prop))
__defNormalProp(a, prop, b[prop]);
if (__getOwnPropSymbols)
for (var prop of __getOwnPropSymbols(b)) {
if (__propIsEnum.call(b, prop))
__defNormalProp(a, prop, b[prop]);
}
return a;
};
var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
var __objRest = (source, exclude) => {
var target = {};
for (var prop in source)
if (__hasOwnProp.call(source, prop) && exclude.indexOf(prop) < 0)
target[prop] = source[prop];
if (source != null && __getOwnPropSymbols)
for (var prop of __getOwnPropSymbols(source)) {
if (exclude.indexOf(prop) < 0 && __propIsEnum.call(source, prop))
target[prop] = source[prop];
}
return target;
};
var __forAwait = (obj, it, method) => (it = obj[__knownSymbol("asyncIterator")]) ? it.call(obj) : (obj = obj[__knownSymbol("iterator")](), it = {}, method = (key, fn) => (fn = obj[key]) && (it[key] = (arg) => new Promise((yes, no, done) => (arg = fn.call(obj, arg), done = arg.done, Promise.resolve(arg.value).then((value) => yes({ value, done }), no)))), method("next"), method("return"), it);
// src/shared/utils/errorHandler.ts
var errorMessages = {
AccessDenied: "Access denied",
AccountProblem: "There is a problem with your account",
AllAccessDisabled: "All access to this resource has been disabled",
BucketAlreadyExists: "The requested bucket name is not available",
BucketNotEmpty: "The bucket you tried to delete is not empty",
EntityTooLarge: "The entity you are trying to upload is too large",
ExpiredToken: "The provided token has expired",
InternalError: "An internal server error has occurred",
InvalidAccessKeyId: "The AWS access key ID you provided does not exist in our records",
InvalidBucketName: "The specified bucket name is not valid",
InvalidObjectState: "The operation is not valid for the current state of the object",
InvalidToken: "The provided token is invalid",
NoSuchBucket: "The specified bucket does not exist",
NoSuchKey: "The specified key does not exist",
PreconditionFailed: "The condition specified in the request is not met",
RequestTimeout: "The request timed out",
ServiceUnavailable: "The service is currently unavailable",
SignatureDoesNotMatch: "The request signature we calculated does not match the signature you provided",
SlowDown: "Please reduce your request rate",
TemporaryRedirect: "Temporary redirect",
ObjectNotFound: "ObjectNotFound - The specified key does not exist. (404)",
DEFAULT: "An unknown error occurred"
};
var ErrorHandler = class {
static handleError(errorCode, errorMessage, errorObj) {
const errorResponse = {
code: errorCode,
message: errorMessage || errorMessages[errorCode] || errorMessages["DEFAULT"],
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
error: errorObj ? {
name: errorObj.name,
message: errorObj.message,
stack: errorObj.stack || null
} : null
};
return errorResponse;
}
};
// src/services/storage/blob/blobHelpers.ts
function BlobPropertiesResponseToObjectResponse(blobProperties) {
return {
lastModified: blobProperties.lastModified,
contentLength: blobProperties.contentLength,
contentType: blobProperties.contentType,
eTag: blobProperties.etag,
metadata: blobProperties.metadata,
contentEncoding: blobProperties.contentEncoding,
cacheControl: blobProperties.cacheControl,
contentDisposition: blobProperties.contentDisposition,
contentLanguage: blobProperties.contentLanguage
};
}
// src/services/storage/blob/blocIdStorage.ts
var BlockIdStorage = class _BlockIdStorage {
constructor() {
this.blockIdMap = {};
}
static getInstance() {
if (!_BlockIdStorage.instance) {
_BlockIdStorage.instance = new _BlockIdStorage();
}
return _BlockIdStorage.instance;
}
addBlockId(blobName, blockId) {
if (!this.blockIdMap[blobName]) {
this.blockIdMap[blobName] = [];
}
this.blockIdMap[blobName].push(blockId);
}
getBlockIds(blobName) {
return this.blockIdMap[blobName] || [];
}
clearBlockIds(blobName) {
delete this.blockIdMap[blobName];
}
};
// src/services/storage/blob/blobStorage.service.ts
var BlobStorageService = class {
/**
* Retrieves the properties of a blob from the container by its name.
*
* @param {string} blobName - The name of the blob.
* @return {Promise<GetObjectResponse>} A promise that resolves to the blob properties.
*/
constructor(containerName) {
this.containerName = containerName;
this.blobServiceClient = BlobServiceClient.fromConnectionString(
process.env.AZURE_STORAGE_CONNECTION_STRING
);
}
/**
* Creates a writable stream for uploading a file to the Blob storage.
*
* @param {string} blobName - The name of the blob to upload.
* @return {CreateUploadWriteStreamResponse} An object containing the key of the uploaded blob, the writable stream, and a promise that resolves when the upload is complete.
*/
createUploadWriteStream(blobName) {
const streamPassThrough = new stream.PassThrough();
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const uploadPromise = blockBlobClient.uploadStream(streamPassThrough, void 0, void 0, {
onProgress: (ev) => console.log(ev)
}).then(() => {
return { Bucket: this.containerName, Key: blobName };
});
return {
key: blobName,
stream: streamPassThrough,
promise: uploadPromise
};
}
/**
* Retrieves the properties of a blob from the container by its name.
*
* @param {string} blobName - The name of the blob.
* @return {Promise<GetObjectResponse>} A promise that resolves to the blob properties.
*/
async getHeadObject(blobName) {
try {
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const blobProperties = await blockBlobClient.getProperties();
return BlobPropertiesResponseToObjectResponse(blobProperties);
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Retrieves an object from the blob storage service.
*
* @param {string} blobName - The name of the blob to retrieve.
* @param {Object} [options] - The options that you can pass to the library.
* @return {Promise<GetObjectResponse>} A promise that resolves to the object data, including the body, metadata, content type, and content length, or rejects with an error if there was an issue.
*/
async getObject(blobName, options) {
var _a;
try {
let downloadBlockBlobResponse;
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const expression = (options == null ? void 0 : options.range) || "";
const matches = expression.match(/\d+/g);
if (expression && (matches == null ? void 0 : matches.length)) {
const [start, end] = matches.map(Number);
downloadBlockBlobResponse = await blockBlobClient.download(
start,
end
);
} else {
downloadBlockBlobResponse = await blockBlobClient.download(0);
}
return {
body: downloadBlockBlobResponse.readableStreamBody,
metadata: downloadBlockBlobResponse.metadata,
contentType: downloadBlockBlobResponse.contentType,
contentLength: downloadBlockBlobResponse.contentLength
};
} catch (error) {
let errorKey = "DEFAULT";
if (((_a = error == null ? void 0 : error.response) == null ? void 0 : _a.status) === 404) {
errorKey = "ObjectNotFound";
}
throw ErrorHandler.handleError(errorKey, "", error);
}
}
async getSignatureUrl(blobName, expiresInMinutes, permissions) {
try {
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const startDate = /* @__PURE__ */ new Date();
const expiryDate = new Date(startDate);
expiryDate.setMinutes(startDate.getMinutes() + expiresInMinutes);
return blockBlobClient.generateSasUrl({
permissions: BlobSASPermissions.parse(permissions),
startsOn: startDate,
expiresOn: expiryDate
});
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
async getUploadUrl(blobName, expiresInMinutes) {
try {
const sasUrl = await this.getSignatureUrl(
blobName,
expiresInMinutes,
"w"
);
return { key: blobName, signedUrl: sasUrl };
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Retrieves a signed URL for the blob with the specified name that expires after a certain period.
*
* @param {string} blobName - The name of the blob to generate the signed URL for.
* @param {number} expiresInMinutes - The duration in minutes until the signed URL expires.
* @return {Promise<string>} A Promise that resolves with the signed URL.
*/
async getDownloadUrl(blobName, expiresInMinutes) {
try {
const sasUrl = await this.getSignatureUrl(
blobName,
expiresInMinutes,
"r"
);
return sasUrl;
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Uploads a file to the blob storage service.
*
* @param {string} blobName - The name of the blob to upload.
* @param {FileContent} body - The content of the file to upload.
* @param {{ [key: string]: string } | undefined} [metadata] - Optional metadata to associate with the blob.
* @return {Promise<UploadResponse>} A promise that resolves to the response object containing the key of the uploaded blob, or rejects with an error if there was an issue.
*/
async upload(blobName, body, metadata = {}) {
try {
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
Buffer.isBuffer(body) ? await blockBlobClient.upload(body, body.length, { metadata }) : await blockBlobClient.uploadStream(
body,
void 0,
void 0,
{ metadata }
);
return { key: blobName };
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Deletes a blob from the blob storage service.
*
* @param {string} blobName - The name of the blob to be deleted.
* @return {Promise<boolean>} A promise that resolves to true if the blob is successfully deleted, or rejects with an error if there was an issue.
*/
async delete(data) {
try {
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
return this.deleteObject(containerClient, data);
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Lists a chunk of blobs from the specified container client based on the provided options and continuation token.
*
* @param {ListRequestOptions} options - The options for listing the blobs.
* @param {string | undefined} continuationToken - The continuation token for pagination.
* @param {number} limit - The maximum number of blobs to list in a single page.
* @param {ContainerClient} containerClient - The container client to list the blobs from.
* @return {Promise<{ items: BlobItem[]; continuationToken?: string }>} - A promise that resolves to an object containing the list of blob items and an optional continuation token.
*/
async listChunk(options, continuationToken, limit, containerClient) {
const iterator = containerClient.listBlobsFlat({
prefix: options.prefix
}).byPage({
maxPageSize: limit,
continuationToken
});
const items = [];
const response = (await iterator.next()).value;
items.push(...response.segment.blobItems);
return {
items,
continuationToken: response.continuationToken
};
}
/**
* Retrieves a list of blob contents based on the provided options.
*
* @param {ListRequestOptions} options - The options for listing the blob contents.
* @return {Promise<{ contents: BlobItem[]; nextContinuationToken?: string }>} A promise that resolves to the list of blob contents and the next continuation token.
*/
async listContents(options) {
var _a;
let responseContents = [];
let continuationToken = options.pagination;
const limit = (_a = options.limit) != null ? _a : 1e3;
let continueListing = true;
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
while (continueListing) {
const response = await this.listChunk(
options,
continuationToken,
limit - responseContents.length,
containerClient
);
continuationToken = response.continuationToken;
responseContents = responseContents.concat(response.items);
if (responseContents.length >= limit || !continuationToken) {
continueListing = false;
}
}
return {
contents: responseContents,
nextContinuationToken: continuationToken
};
}
/**
* Lists blobs in the Azure Blob Storage container with pagination.
* @param options - The options for listing blobs.
* @returns A promise that resolves to a paginated list of blob names.
*/
async list(options) {
var _a;
let items = [];
const response = await this.listContents(options);
items = response.contents.length ? response.contents.map((blob) => {
return {
key: blob.name,
lastModified: blob.properties.lastModified,
size: blob.properties.contentLength,
eTag: blob.properties.etag
};
}) : [];
return {
items,
pagination: ((_a = response.nextContinuationToken) == null ? void 0 : _a.length) ? response.nextContinuationToken : null,
count: items.length
};
}
/**
* Lists all blobs in the Azure Blob Storage container.
* @param options - The options for listing blobs.
* @returns A promise that resolves to list of blob names.
*/
async listAll(options) {
var _a, _b;
let allItems = [];
let pagination = void 0;
do {
const response = await this.list(__spreadProps(__spreadValues({}, options), {
pagination
}));
if ((_a = response.items) == null ? void 0 : _a.length)
allItems = [...allItems, ...response.items];
pagination = (_b = response.pagination) != null ? _b : void 0;
} while (pagination);
return { items: allItems, count: allItems.length };
}
/**
* Deletes a blob from the blob storage service.
*
* @param {string} blobName - The name of the blob to be deleted.
* @return {Promise<boolean>} A promise that resolves to true if the blob is successfully deleted, or rejects with an error if there was an issue.
*/
async deleteObject(containerClient, blobName) {
try {
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
await blockBlobClient.delete();
return true;
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Deletes multiple blobs from the blob storage service.
*
* @param {string[]} blobNames - An array of blob names to be deleted.
* @return {Promise<{ key: string; deleted: boolean; error?: string }[]>} - A promise that resolves to an array of objects containing the key, deleted status, and an optional error message for each blob deleted.
*/
async deleteObjects(blobNames) {
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const deleteBlobPromises = blobNames.map(async (blobName) => {
var _a;
try {
await this.deleteObject(containerClient, blobName);
return { key: blobName, deleted: true };
} catch (error) {
return {
key: blobName,
deleted: false,
error: (_a = error == null ? void 0 : error.message) != null ? _a : ""
};
}
});
return Promise.all(deleteBlobPromises);
}
/**
* Retrieves the properties of the container.
*
* @return {Promise<HeadBucketResponse>} A promise that resolves to the container properties.
*/
async getHeadBucket() {
try {
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const properties = await containerClient.getProperties();
return properties;
} catch (error) {
throw new Error(
"Error in Azure getHeadContainer. Container: " + this.containerName + " Error: " + error
);
}
}
async listMultipartUploadsForBucket() {
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const blobList = containerClient.listBlobsFlat();
const uploads = [];
try {
for (var iter = __forAwait(blobList), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
const blob = temp.value;
uploads.push({
uploadId: blob.name,
key: blob.name,
initiated: blob.properties.createdOn || /* @__PURE__ */ new Date()
});
}
} catch (temp) {
error = [temp];
} finally {
try {
more && (temp = iter.return) && await temp.call(iter);
} finally {
if (error)
throw error[0];
}
}
return {
bucketName: this.containerName,
uploads
};
}
/**
* Retrieves the list of parts for a multipart upload.
*
* @param {string} blobName - The name of the blob (file) in Azure Blob Storage.
* @return {Promise<ListPartsMultipartUploadResponse>} A promise that resolves to the list of parts for the multipart upload.
*/
async listMultipartUploadsForKey(blobName) {
var _a;
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const blobClient = containerClient.getBlockBlobClient(blobName);
const listResponse = await blobClient.getBlockList("all");
const parts = ((_a = listResponse == null ? void 0 : listResponse.uncommittedBlocks) == null ? void 0 : _a.map((block, index) => ({
PartNumber: index + 1,
LastModified: /* @__PURE__ */ new Date(),
// Azure Blob Storage doesn't provide the last modified date for individual parts
ETag: block.name,
Size: block.size
}))) || [];
return {
Parts: parts
};
}
/**
* Generates a unique upload ID for multipart uploads.
*
* @return {Promise<string>} A promise that resolves to a unique upload ID.
*/
async generateUploadIdMultipart() {
const timestamp = Date.now();
const randomNum = Math.floor(Math.random() * 100);
const blockIdBase = (timestamp - randomNum).toString();
return blockIdBase;
}
/**
* Uploads a part of a file as a block to Azure Blob Storage and updates the metadata with the block ID.
*
* @param {string} blobName - The name of the blob (file) that you want to upload in parts to Azure Blob Storage.
* @param {FileContent | any} file - The content of the file that you want to upload as a part of a multipart upload.
* @param {number} partNumber - The number or index of the part being uploaded.
* @param {string} [uploadId] - The ID of the multipart upload. If not provided, a new upload ID will be generated.
* @return {Promise<string>} A Promise that resolves to a string representing the base64 encoded block ID of the uploaded part.
*/
async uploadMultipart(blobName, file, partNumber, uploadId) {
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const blobClient = containerClient.getBlockBlobClient(blobName);
const blockIdBase = uploadId || await this.generateUploadIdMultipart();
const partId = partNumber.toString().padStart(6, "0");
const partIdBase64 = Buffer.from(partId).toString("base64");
await blobClient.stageBlock(partIdBase64, file, file.length);
return blockIdBase;
}
/**
* Completes a multipart upload by committing the blocks to create the blob.
*
* @param {string} blobName - The name of the blob for which the multipart upload is being completed.
* @param {string} uploadId - The ID of the multipart upload to be completed.
* @return {Promise<void>} A Promise that resolves when the multipart upload is successfully completed.
* @throws {Error} If no block IDs are found in the metadata.
*/
async completeMultipartUpload(blobName, uploadId) {
var _a;
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const blobClient = containerClient.getBlockBlobClient(blobName);
const listMultipartUploads = await this.listMultipartUploadsForKey(blobName);
const blockIds = ((_a = listMultipartUploads == null ? void 0 : listMultipartUploads.Parts) == null ? void 0 : _a.map((part) => part.ETag)) || [];
if (blockIds.length === 0) {
throw new Error("No block IDs found in metadata.");
}
await blobClient.commitBlockList(blockIds);
}
/**
* Aborts a multipart upload by deleting the specified blob and clearing its block IDs metadata.
*
* @param {string} blobName - The name of the blob for which the multipart upload needs to be aborted.
* @param {string} uploadId - The ID of the multipart upload to be aborted.
* @return {Promise<void>} A promise that resolves when the multipart upload is successfully aborted.
*/
async abortMultipartUpload(blobName, uploadId) {
const containerClient = this.blobServiceClient.getContainerClient(
this.containerName
);
const blobClient = containerClient.getBlockBlobClient(blobName);
const blockIdStorage = BlockIdStorage.getInstance();
blockIdStorage.clearBlockIds(uploadId);
await blobClient.delete();
}
async getMultipartUploadPresignedUrl(blobName, uploadId, partNumber, expiresInMinutes = 2) {
const partId = partNumber.toString().padStart(6, "0");
const partIdBase64 = Buffer.from(partId).toString("base64");
const sasUrl = await this.getSignatureUrl(
blobName,
expiresInMinutes,
"w"
);
const url = `${sasUrl}&comp=block&blockid=${partIdBase64}`;
return url;
}
};
// src/services/storage/s3/s3Helpers.ts
function listResponseContentsToListResponseItems(responseContents) {
return responseContents.map((responseContent) => {
return {
key: responseContent.Key,
lastModified: new Date(responseContent.LastModified),
size: responseContent.Size,
eTag: responseContent.ETag
};
});
}
function s3ObjectToObjectResponse(s3Object) {
return {
body: s3Object.Body,
metadata: s3Object.Metadata,
contentType: s3Object.ContentType,
contentLength: s3Object.ContentLength,
contentEncoding: s3Object.ContentEncoding,
cacheControl: s3Object.CacheControl,
contentDisposition: s3Object.ContentDisposition,
contentLanguage: s3Object.ContentLanguage,
eTag: s3Object.ETag,
lastModified: s3Object.LastModified
};
}
// ../../node_modules/@smithy/types/dist-es/auth/auth.js
var HttpAuthLocation;
(function(HttpAuthLocation2) {
HttpAuthLocation2["HEADER"] = "header";
HttpAuthLocation2["QUERY"] = "query";
})(HttpAuthLocation || (HttpAuthLocation = {}));
// ../../node_modules/@smithy/types/dist-es/auth/HttpApiKeyAuth.js
var HttpApiKeyAuthLocation;
(function(HttpApiKeyAuthLocation2) {
HttpApiKeyAuthLocation2["HEADER"] = "header";
HttpApiKeyAuthLocation2["QUERY"] = "query";
})(HttpApiKeyAuthLocation || (HttpApiKeyAuthLocation = {}));
// ../../node_modules/@smithy/types/dist-es/endpoint.js
var EndpointURLScheme;
(function(EndpointURLScheme2) {
EndpointURLScheme2["HTTP"] = "http";
EndpointURLScheme2["HTTPS"] = "https";
})(EndpointURLScheme || (EndpointURLScheme = {}));
// ../../node_modules/@smithy/types/dist-es/extensions/checksum.js
var AlgorithmId;
(function(AlgorithmId2) {
AlgorithmId2["MD5"] = "md5";
AlgorithmId2["CRC32"] = "crc32";
AlgorithmId2["CRC32C"] = "crc32c";
AlgorithmId2["SHA1"] = "sha1";
AlgorithmId2["SHA256"] = "sha256";
})(AlgorithmId || (AlgorithmId = {}));
// ../../node_modules/@smithy/types/dist-es/http.js
var FieldPosition;
(function(FieldPosition2) {
FieldPosition2[FieldPosition2["HEADER"] = 0] = "HEADER";
FieldPosition2[FieldPosition2["TRAILER"] = 1] = "TRAILER";
})(FieldPosition || (FieldPosition = {}));
// ../../node_modules/@smithy/types/dist-es/profile.js
var IniSectionType;
(function(IniSectionType2) {
IniSectionType2["PROFILE"] = "profile";
IniSectionType2["SSO_SESSION"] = "sso-session";
IniSectionType2["SERVICES"] = "services";
})(IniSectionType || (IniSectionType = {}));
// ../../node_modules/@smithy/types/dist-es/transfer.js
var RequestHandlerProtocol;
(function(RequestHandlerProtocol2) {
RequestHandlerProtocol2["HTTP_0_9"] = "http/0.9";
RequestHandlerProtocol2["HTTP_1_0"] = "http/1.0";
RequestHandlerProtocol2["TDS_8_0"] = "tds/8.0";
})(RequestHandlerProtocol || (RequestHandlerProtocol = {}));
// ../../node_modules/@smithy/protocol-http/dist-es/httpResponse.js
var HttpResponse = class {
constructor(options) {
this.statusCode = options.statusCode;
this.reason = options.reason;
this.headers = options.headers || {};
this.body = options.body;
}
static isInstance(response) {
if (!response)
return false;
const resp = response;
return typeof resp.statusCode === "number" && typeof resp.headers === "object";
}
};
// ../../node_modules/@smithy/util-uri-escape/dist-es/escape-uri.js
var escapeUri = (uri) => encodeURIComponent(uri).replace(/[!'()*]/g, hexEncode);
var hexEncode = (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`;
// ../../node_modules/@smithy/querystring-builder/dist-es/index.js
function buildQueryString(query) {
const parts = [];
for (let key of Object.keys(query).sort()) {
const value = query[key];
key = escapeUri(key);
if (Array.isArray(value)) {
for (let i = 0, iLen = value.length; i < iLen; i++) {
parts.push(`${key}=${escapeUri(value[i])}`);
}
} else {
let qsEntry = key;
if (value || typeof value === "string") {
qsEntry += `=${escapeUri(value)}`;
}
parts.push(qsEntry);
}
}
return parts.join("&");
}
// ../../node_modules/@smithy/node-http-handler/dist-es/constants.js
var NODEJS_TIMEOUT_ERROR_CODES = ["ECONNRESET", "EPIPE", "ETIMEDOUT"];
// ../../node_modules/@smithy/node-http-handler/dist-es/get-transformed-headers.js
var getTransformedHeaders = (headers) => {
const transformedHeaders = {};
for (const name of Object.keys(headers)) {
const headerValues = headers[name];
transformedHeaders[name] = Array.isArray(headerValues) ? headerValues.join(",") : headerValues;
}
return transformedHeaders;
};
// ../../node_modules/@smithy/node-http-handler/dist-es/set-connection-timeout.js
var setConnectionTimeout = (request, reject, timeoutInMs = 0) => {
if (!timeoutInMs) {
return;
}
const timeoutId = setTimeout(() => {
request.destroy();
reject(Object.assign(new Error(`Socket timed out without establishing a connection within ${timeoutInMs} ms`), {
name: "TimeoutError"
}));
}, timeoutInMs);
request.on("socket", (socket) => {
if (socket.connecting) {
socket.on("connect", () => {
clearTimeout(timeoutId);
});
} else {
clearTimeout(timeoutId);
}
});
};
// ../../node_modules/@smithy/node-http-handler/dist-es/set-socket-keep-alive.js
var setSocketKeepAlive = (request, { keepAlive, keepAliveMsecs }) => {
if (keepAlive !== true) {
return;
}
request.on("socket", (socket) => {
socket.setKeepAlive(keepAlive, keepAliveMsecs || 0);
});
};
// ../../node_modules/@smithy/node-http-handler/dist-es/set-socket-timeout.js
var setSocketTimeout = (request, reject, timeoutInMs = 0) => {
request.setTimeout(timeoutInMs, () => {
request.destroy();
reject(Object.assign(new Error(`Connection timed out after ${timeoutInMs} ms`), { name: "TimeoutError" }));
});
};
var MIN_WAIT_TIME = 1e3;
async function writeRequestBody(httpRequest, request, maxContinueTimeoutMs = MIN_WAIT_TIME) {
var _a;
const headers = (_a = request.headers) != null ? _a : {};
const expect = headers["Expect"] || headers["expect"];
let timeoutId = -1;
let hasError = false;
if (expect === "100-continue") {
await Promise.race([
new Promise((resolve) => {
timeoutId = Number(setTimeout(resolve, Math.max(MIN_WAIT_TIME, maxContinueTimeoutMs)));
}),
new Promise((resolve) => {
httpRequest.on("continue", () => {
clearTimeout(timeoutId);
resolve();
});
httpRequest.on("error", () => {
hasError = true;
clearTimeout(timeoutId);
resolve();
});
})
]);
}
if (!hasError) {
writeBody(httpRequest, request.body);
}
}
function writeBody(httpRequest, body) {
if (body instanceof Readable) {
body.pipe(httpRequest);
return;
}
if (body) {
if (Buffer.isBuffer(body) || typeof body === "string") {
httpRequest.end(body);
return;
}
const uint8 = body;
if (typeof uint8 === "object" && uint8.buffer && typeof uint8.byteOffset === "number" && typeof uint8.byteLength === "number") {
httpRequest.end(Buffer.from(uint8.buffer, uint8.byteOffset, uint8.byteLength));
return;
}
httpRequest.end(Buffer.from(body));
return;
}
httpRequest.end();
}
// ../../node_modules/@smithy/node-http-handler/dist-es/node-http-handler.js
var NodeHttpHandler = class _NodeHttpHandler {
static create(instanceOrOptions) {
if (typeof (instanceOrOptions == null ? void 0 : instanceOrOptions.handle) === "function") {
return instanceOrOptions;
}
return new _NodeHttpHandler(instanceOrOptions);
}
static checkSocketUsage(agent, socketWarningTimestamp) {
var _a, _b, _c, _d;
const { sockets, requests, maxSockets } = agent;
if (typeof maxSockets !== "number" || maxSockets === Infinity) {
return socketWarningTimestamp;
}
const interval = 15e3;
if (Date.now() - interval < socketWarningTimestamp) {
return socketWarningTimestamp;
}
if (sockets && requests) {
for (const origin in sockets) {
const socketsInUse = (_b = (_a = sockets[origin]) == null ? void 0 : _a.length) != null ? _b : 0;
const requestsEnqueued = (_d = (_c = requests[origin]) == null ? void 0 : _c.length) != null ? _d : 0;
if (socketsInUse >= maxSockets && requestsEnqueued >= 2 * maxSockets) {
console.warn("@smithy/node-http-handler:WARN", `socket usage at capacity=${socketsInUse} and ${requestsEnqueued} additional requests are enqueued.`, "See https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/node-configuring-maxsockets.html", "or increase socketAcquisitionWarningTimeout=(millis) in the NodeHttpHandler config.");
return Date.now();
}
}
}
return socketWarningTimestamp;
}
constructor(options) {
this.socketWarningTimestamp = 0;
this.metadata = { handlerProtocol: "http/1.1" };
this.configProvider = new Promise((resolve, reject) => {
if (typeof options === "function") {
options().then((_options) => {
resolve(this.resolveDefaultConfig(_options));
}).catch(reject);
} else {
resolve(this.resolveDefaultConfig(options));
}
});
}
resolveDefaultConfig(options) {
const { requestTimeout, connectionTimeout, socketTimeout, httpAgent, httpsAgent } = options || {};
const keepAlive = true;
const maxSockets = 50;
return {
connectionTimeout,
requestTimeout: requestTimeout != null ? requestTimeout : socketTimeout,
httpAgent: (() => {
if (httpAgent instanceof Agent || typeof (httpAgent == null ? void 0 : httpAgent.destroy) === "function") {
return httpAgent;
}
return new Agent(__spreadValues({ keepAlive, maxSockets }, httpAgent));
})(),
httpsAgent: (() => {
if (httpsAgent instanceof Agent$1 || typeof (httpsAgent == null ? void 0 : httpsAgent.destroy) === "function") {
return httpsAgent;
}
return new Agent$1(__spreadValues({ keepAlive, maxSockets }, httpsAgent));
})()
};
}
destroy() {
var _a, _b, _c, _d;
(_b = (_a = this.config) == null ? void 0 : _a.httpAgent) == null ? void 0 : _b.destroy();
(_d = (_c = this.config) == null ? void 0 : _c.httpsAgent) == null ? void 0 : _d.destroy();
}
async handle(request$2, { abortSignal } = {}) {
if (!this.config) {
this.config = await this.configProvider;
}
let socketCheckTimeoutId;
return new Promise((_resolve, _reject) => {
var _a, _b, _c, _d, _e;
let writeRequestBodyPromise = void 0;
const resolve = async (arg) => {
await writeRequestBodyPromise;
clearTimeout(socketCheckTimeoutId);
_resolve(arg);
};
const reject = async (arg) => {
await writeRequestBodyPromise;
_reject(arg);
};
if (!this.config) {
throw new Error("Node HTTP request handler config is not resolved");
}
if (abortSignal == null ? void 0 : abortSignal.aborted) {
const abortError = new Error("Request aborted");
abortError.name = "AbortError";
reject(abortError);
return;
}
const isSSL = request$2.protocol === "https:";
const agent = isSSL ? this.config.httpsAgent : this.config.httpAgent;
socketCheckTimeoutId = setTimeout(() => {
this.socketWarningTimestamp = _NodeHttpHandler.checkSocketUsage(agent, this.socketWarningTimestamp);
}, (_c = this.config.socketAcquisitionWarningTimeout) != null ? _c : ((_a = this.config.requestTimeout) != null ? _a : 2e3) + ((_b = this.config.connectionTimeout) != null ? _b : 1e3));
const queryString = buildQueryString(request$2.query || {});
let auth = void 0;
if (request$2.username != null || request$2.password != null) {
const username = (_d = request$2.username) != null ? _d : "";
const password = (_e = request$2.password) != null ? _e : "";
auth = `${username}:${password}`;
}
let path = request$2.path;
if (queryString) {
path += `?${queryString}`;
}
if (request$2.fragment) {
path += `#${request$2.fragment}`;
}
const nodeHttpsOptions = {
headers: request$2.headers,
host: request$2.hostname,
method: request$2.method,
path,
port: request$2.port,
agent,
auth
};
const requestFunc = isSSL ? request : request$1;
const req = requestFunc(nodeHttpsOptions, (res) => {
const httpResponse = new HttpResponse({
statusCode: res.statusCode || -1,
reason: res.statusMessage,
headers: getTransformedHeaders(res.headers),
body: res
});
resolve({ response: httpResponse });
});
req.on("error", (err) => {
if (NODEJS_TIMEOUT_ERROR_CODES.includes(err.code)) {
reject(Object.assign(err, { name: "TimeoutError" }));
} else {
reject(err);
}
});
setConnectionTimeout(req, reject, this.config.connectionTimeout);
setSocketTimeout(req, reject, this.config.requestTimeout);
if (abortSignal) {
abortSignal.onabort = () => {
req.abort();
const abortError = new Error("Request aborted");
abortError.name = "AbortError";
reject(abortError);
};
}
const httpAgent = nodeHttpsOptions.agent;
if (typeof httpAgent === "object" && "keepAlive" in httpAgent) {
setSocketKeepAlive(req, {
keepAlive: httpAgent.keepAlive,
keepAliveMsecs: httpAgent.keepAliveMsecs
});
}
writeRequestBodyPromise = writeRequestBody(req, request$2, this.config.requestTimeout).catch(_reject);
});
}
updateHttpClientConfig(key, value) {
this.config = void 0;
this.configProvider = this.configProvider.then((config) => {
return __spreadProps(__spreadValues({}, config), {
[key]: value
});
});
}
httpHandlerConfigs() {
var _a;
return (_a = this.config) != null ? _a : {};
}
};
// src/services/storage/s3/s3Storage.service.ts
var S3StorageService = class {
constructor(bucketName, options) {
this.httpHandler = new NodeHttpHandler({
socketTimeout: 6e5
});
this.s3Client = new S3Client({
region: process.env.AWS_DEFAULT_REGION,
requestHandler: this.httpHandler
});
this.s3 = new S3({
region: process.env.AWS_DEFAULT_REGION,
requestHandler: this.httpHandler
});
var _a, _b, _c, _d;
this.bucketName = bucketName;
if (((_a = options == null ? void 0 : options.credentials) == null ? void 0 : _a.accessKeyId) && ((_b = options == null ? void 0 : options.credentials) == null ? void 0 : _b.secretAccessKey)) {
const s3Config = {
region: (options == null ? void 0 : options.region) || process.env.AWS_REGION,
credentials: {
accessKeyId: (_c = options == null ? void 0 : options.credentials) == null ? void 0 : _c.accessKeyId,
secretAccessKey: (_d = options == null ? void 0 : options.credentials) == null ? void 0 : _d.secretAccessKey
}
};
this.s3Client = new S3Client(s3Config);
}
}
/**
* Retrieves a chunk of objects from the S3 bucket based on the provided options.
*
* @param {ListRequestOptions} options - The options for listing the objects.
* @param {string | undefined} continuationToken - The continuation token for pagination.
* @param {number} limit - The maximum number of objects to retrieve.
* @return {Promise<ListObjectsV2CommandOutput>} - A promise that resolves to the response containing the list of objects and metadata.
*/
async listChunk(options, continuationToken, limit) {
const command = new ListObjectsV2Command({
Bucket: this.bucketName,
ContinuationToken: continuationToken || options.pagination,
Prefix: options.prefix,
MaxKeys: limit
});
return this.s3Client.send(command);
}
/**
* Retrieves a list of contents from the S3 bucket based on the provided options.
*
* @param {ListRequestOptions} options - The options for listing the contents.
* @return {Promise<{ contents: unknown[]; nextContinuationToken?: string }>} - A promise that resolves to an object containing the list of contents and an optional next continuation token.
*/
async listContents(options) {
var _a, _b;
let responseContents = [];
let continuationToken = void 0;
const limit = (_a = options.limit) != null ? _a : 1e3;
let continueListing = true;
while (continueListing) {
const listChunkLimit = limit - responseContents.length;
const response = await this.listChunk(
options,
continuationToken,
listChunkLimit
);
continuationToken = response.NextContinuationToken;
responseContents = responseContents.concat((_b = response.Contents) != null ? _b : []);
if (responseContents.length >= limit || !continuationToken) {
continueListing = false;
}
}
return {
contents: responseContents,
nextContinuationToken: continuationToken
};
}
/**
* Lists objects in the S3 bucket with pagination.
* @param options - The options for listing objects.
* @returns A promise that resolves to a paginated list of object keys.
*/
async list(options) {
var _a;
let items = [];
const response = await this.listContents(options);
items = response.contents.length ? listResponseContentsToListResponseItems(response.contents) : [];
return {
items,
pagination: (_a = response.nextContinuationToken) != null ? _a : null,
count: items.length
};
}
/**
* Lists all objects in the S3 bucket.
* @param options - The options for listing objects.
* @returns A promise that resolves to list of object keys.
*/
async listAll(options) {
var _a;
let allItems = [];
let pagination = void 0;
do {
const response = await this.list(__spreadProps(__spreadValues({}, options), {
pagination
}));
if ((_a = response.items) == null ? void 0 : _a.length)
allItems = [...allItems, ...response.items];
pagination = response.pagination === null ? void 0 : response.pagination;
} while (pagination);
return { items: allItems, count: allItems.length };
}
/**
* Retrieves an object from the S3 bucket.
* @param key - The key of the object to retrieve.
* @returns A promise that resolves to the content of the file.
*/
async getHeadObject(key) {
const command = new HeadObjectCommand({
Bucket: this.bucketName,
Key: key
});
const response = await this.s3Client.send(command);
return s3ObjectToObjectResponse(response);
}
/**
* Retrieves an object from the S3 bucket.
* @param key - The key of the object to retrieve.
* @returns A promise that resolves to the content of the file.
*/
async getObject(key) {
const command = new GetObjectCommand({
Bucket: this.bucketName,
Key: key
});
const response = await this.s3Client.send(command);
return s3ObjectToObjectResponse(response);
}
/**
* Generates a signed URL for accessing an object in the S3 bucket.
* @param key - The key of the object.
* @param expiresInMinutes - The number of minutes the signed URL should be valid for. Defaults to 60 minutes.
* @returns A promise that resolves to the signed URL.
*/
getDownloadUrl(key, expiresInMinutes = 60) {
const expiresIn = expiresInMinutes * 60;
const command = new GetObjectCommand({
Bucket: this.bucketName,
Key: key
});
return getSignedUrl(this.s3Client, command, { expiresIn });
}
/**
* Retrieves a signed URL for uploading an object to the S3 bucket.
*
* @param {string} key - The key of the object to be uploaded.
* @param {number} [expiresInMinutes=60] - The number of minutes the signed URL should be valid for. Defaults to 60 minutes.
* @returns A promise that resolves to the signed URL.
*/
async getUploadUrl(key, expiresInMinutes = 60) {
const expiresIn = expiresInMinutes * 60;
const putObjectCommandParams = {
Bucket: process.env.UPLOAD_BUCKET_NAME,
Key: key,
ACL: "private"
};
const command = new PutObjectCommand(putObjectCommandParams);
const signedUrl = await getSignedUrl(this.s3Client, command, {
expiresIn
});
return {
signedUrl,
key: `https://${process.env.UPLOAD_BUCKET_NAME}.s3.${process.env.AWS_DEFAULT_REGION}.amazonaws.com/${key}`
};
}
/**
* Uploads an object to the S3 bucket.
* @param key - The key of the object to upload.
* @param body - The content of the object to upload.
* @param metadata - Optional metadata to associate with the object.
* @returns A promise that resolves to the response containing the key of the uploaded object.
*/
async upload(key, body, metadata) {
const command = new PutObjectCommand({
Bucket: this.bucketName,
Key: key,
Body: body,
Metadata: metadata
});
await this.s3Client.send(command);
return { key };
}
/**
* Creates a writable stream for uploading a file to the S3 bucket.
*
* @param {string} key - The key of the object to upload.
* @return {CreateUploadWriteStreamResponse} An object containing the key of the uploaded object, the writable stream, and a promise that resolves when the upload is complete.
*/
createUploadWriteStream(key) {
const streamPassThrough = new stream.PassThrough();
const params = {
Bucket: this.bucketName,
Key: key,
Body: streamPassThrough
};
const upload = new Upload({
client: this.s3Client,
params,
queueSize: 5,
partSize: 8 * 1024 * 1024
});
return {
key,
stream: streamPassThrough,
promise: upload.done()
};
}
/**
* Deletes an object from the S3 bucket.
* @param key - The key of the object to delete.
* @returns A promise that resolves to true if the object was deleted successfully.
*/
async delete(key) {
const command = new DeleteObjectCommand({
Bucket: this.bucketName,
Key: key
});
await this.s3Client.send(command);
return true;
}
async getHeadBucket() {
try {
const command = new HeadBucketCommand({
Bucket: this.bucketName
});
return this.s3Client.send(command);
} catch (error) {
throw new Error(
"Error in S3 getHeadContainer. bucketName: " + this.bucketName + " Error: " + error
);
}
}
/**
* Generates an upload ID for multipart upload in the S3 bucket.
*
* @param {string} key - The key of the object to upload.
* @return {Promise<string>} A promise that resolves to the generated upload ID.
* @throws {Error} If no upload ID was generated.
*/
async generateUploadIdMultipart(key) {
const params = {
Bucket: this.bucketName,
Key: key
};
const response = await this.s3.createMultipartUpload(params);
if (!(response == null ? void 0 : response.UploadId))
throw new Error("No upload ID was generated");
return response.UploadId;
}
async listMultipartUploadsForBucket() {
const params = {
Bucket: this.bucketName
};
const response = await this.s3.listMultipartUploads(params);
return {
bucketName: this.bucketName,
uploads: response.Uploads ? response.Uploads.map((upload) => {
var _a, _b;
return {
uploadId: upload.UploadId || "",
key: upload.Key || "",
initiator: ((_a = upload == null ? void 0 : upload.Initiator) == null ? void 0 : _a.ID) || "",
owner: ((_b = upload == null ? void 0 : upload.Owner) == null ? void 0 : _b.ID) || "",
storageClass: upload.StorageClass || "",
initiated: upload.Initiated || ""
};
}) : []
};
}
/**
* Retrieves the list of parts for a multipart upload.
*
* @param {string} key - The key of the object being uploaded.
* @param {string} uploadId - The ID of the multipart upload.
* @return {Promise<ListPartsMultipartUploadResponse>} A promise that resolves to the list of parts for the multipart upload.
*/
async listMultipartUploadsForKey(key, uploadId) {
const params = {
Bucket: this.bucketName,
Key: key,
UploadId: uploadId
};
const response = await this.s3.listParts(params);
const parts = response.Parts ? response.Parts.map((part) => ({
PartNumber: part.PartNumber,
LastModified: part.LastModified,
ETag: part.ETag,
Size: part.Size
})) : [];
return {
Parts: parts
};
}
async uploadMultipart(key, file, partNumber, uploadId) {
const upId = uploadId || await this.generateUploadIdMultipart(key);
const params = {
Bucket: this.bucketName,
Key: key,
Body: file,
PartNumber: partNumber,
UploadId: upId
};
await this.s3.uploadPart(params);
return upId;
}
async completeMultipartUpload(key, uploadId) {
const partsResponse = await this.listMultipartUploadsForKey(
key,
uploadId
);
const partsList = (partsResponse == null ? void 0 : partsResponse.Parts) && partsResponse.Parts.map(
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
(_a) => {
var _b = _a, partList = __objRest(_b, ["Size", "LastModified"]);
return partList;
}
);
const completeMultipartParams = {
Bucket: this.bucketName,
Key: key,
UploadId: uploadId,
MultipartUpload: {
Parts: partsList
}
};
await this.s3.completeMultipartUpload(completeMultipartParams);
}
async abortMultipartUpload(key, uploadId) {
const params = {
Bucket: this.bucketName,
Key: key,
UploadId: uploadId
};
await this.