@qrvey/object-storage
Version:
 
1,198 lines (1,189 loc) • 51.9 kB
JavaScript
import stream from 'stream';
import { BlobServiceClient, BlobSASPermissions } from '@azure/storage-blob';
import { S3Client, S3, ListObjectsV2Command, HeadObjectCommand, GetObjectCommand, PutObjectCommand, DeleteObjectCommand, HeadBucketCommand, UploadPartCommand } from '@aws-sdk/client-s3';
import { defaultProvider } from '@aws-sdk/credential-provider-node';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { Upload } from '@aws-sdk/lib-storage';
import { NodeHttpHandler } from '@smithy/node-http-handler';
var __defProp = Object.defineProperty;
var __defProps = Object.defineProperties;
var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
var __getOwnPropSymbols = Object.getOwnPropertySymbols;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __propIsEnum = Object.prototype.propertyIsEnumerable;
var __knownSymbol = (name, symbol) => {
if (symbol = Symbol[name])
return symbol;
throw Error("Symbol." + name + " is not defined");
};
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __spreadValues = (a, b) => {
for (var prop in b || (b = {}))
if (__hasOwnProp.call(b, prop))
__defNormalProp(a, prop, b[prop]);
if (__getOwnPropSymbols)
for (var prop of __getOwnPropSymbols(b)) {
if (__propIsEnum.call(b, prop))
__defNormalProp(a, prop, b[prop]);
}
return a;
};
var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
var __objRest = (source, exclude) => {
var target = {};
for (var prop in source)
if (__hasOwnProp.call(source, prop) && exclude.indexOf(prop) < 0)
target[prop] = source[prop];
if (source != null && __getOwnPropSymbols)
for (var prop of __getOwnPropSymbols(source)) {
if (exclude.indexOf(prop) < 0 && __propIsEnum.call(source, prop))
target[prop] = source[prop];
}
return target;
};
var __publicField = (obj, key, value) => {
__defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
return value;
};
var __forAwait = (obj, it, method) => (it = obj[__knownSymbol("asyncIterator")]) ? it.call(obj) : (obj = obj[__knownSymbol("iterator")](), it = {}, method = (key, fn) => (fn = obj[key]) && (it[key] = (arg) => new Promise((yes, no, done) => (arg = fn.call(obj, arg), done = arg.done, Promise.resolve(arg.value).then((value) => yes({ value, done }), no)))), method("next"), method("return"), it);
// src/shared/utils/errorHandler.ts
var errorMessages = {
AccessDenied: "Access denied",
AccountProblem: "There is a problem with your account",
AllAccessDisabled: "All access to this resource has been disabled",
BucketAlreadyExists: "The requested bucket name is not available",
BucketNotEmpty: "The bucket you tried to delete is not empty",
EntityTooLarge: "The entity you are trying to upload is too large",
ExpiredToken: "The provided token has expired",
InternalError: "An internal server error has occurred",
InvalidAccessKeyId: "The AWS access key ID you provided does not exist in our records",
InvalidBucketName: "The specified bucket name is not valid",
InvalidObjectState: "The operation is not valid for the current state of the object",
InvalidToken: "The provided token is invalid",
NoSuchBucket: "The specified bucket does not exist",
NoSuchKey: "The specified key does not exist",
PreconditionFailed: "The condition specified in the request is not met",
RequestTimeout: "The request timed out",
ServiceUnavailable: "The service is currently unavailable",
SignatureDoesNotMatch: "The request signature we calculated does not match the signature you provided",
SlowDown: "Please reduce your request rate",
TemporaryRedirect: "Temporary redirect",
ObjectNotFound: "ObjectNotFound - The specified key does not exist. (404)",
DEFAULT: "An unknown error occurred"
};
var _ErrorHandler = class _ErrorHandler {
static handleError(errorCode, errorMessage, errorObj) {
const errorResponse = {
code: errorCode,
message: errorMessage || errorMessages[errorCode] || errorMessages["DEFAULT"],
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
error: errorObj ? {
name: errorObj.name,
message: errorObj.message,
stack: errorObj.stack || null
} : null
};
return errorResponse;
}
};
__name(_ErrorHandler, "ErrorHandler");
var ErrorHandler = _ErrorHandler;
// src/services/storage/blob/blobHelpers.ts
function BlobPropertiesResponseToObjectResponse(blobProperties) {
return {
lastModified: blobProperties.lastModified,
contentLength: blobProperties.contentLength,
contentType: blobProperties.contentType,
eTag: blobProperties.etag,
metadata: blobProperties.metadata,
contentEncoding: blobProperties.contentEncoding,
cacheControl: blobProperties.cacheControl,
contentDisposition: blobProperties.contentDisposition,
contentLanguage: blobProperties.contentLanguage
};
}
__name(BlobPropertiesResponseToObjectResponse, "BlobPropertiesResponseToObjectResponse");
// src/services/storage/blob/blocIdStorage.ts
var _BlockIdStorage = class _BlockIdStorage {
constructor() {
__publicField(this, "blockIdMap", {});
}
static getInstance() {
if (!_BlockIdStorage.instance) {
_BlockIdStorage.instance = new _BlockIdStorage();
}
return _BlockIdStorage.instance;
}
addBlockId(blobName, blockId) {
if (!this.blockIdMap[blobName]) {
this.blockIdMap[blobName] = [];
}
this.blockIdMap[blobName].push(blockId);
}
getBlockIds(blobName) {
return this.blockIdMap[blobName] || [];
}
clearBlockIds(blobName) {
delete this.blockIdMap[blobName];
}
};
__name(_BlockIdStorage, "BlockIdStorage");
__publicField(_BlockIdStorage, "instance");
var BlockIdStorage = _BlockIdStorage;
// src/services/storage/blob/blobStorage.service.ts
var _BlobStorageService = class _BlobStorageService {
/**
* Retrieves the properties of a blob from the container by its name.
*
* @param {string} blobName - The name of the blob.
* @return {Promise<GetObjectResponse>} A promise that resolves to the blob properties.
*/
constructor(containerName) {
__publicField(this, "blobServiceClient");
__publicField(this, "containerName");
this.containerName = containerName;
let connectionString;
if (process.env.DATA_LAKE_BUCKET && process.env.DATA_LAKE_BUCKET === this.containerName) {
connectionString = process.env.AZURE_DATALAKE_CONNECTION_STRING;
}
if (!connectionString) {
connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
}
this.blobServiceClient = BlobServiceClient.fromConnectionString(connectionString);
}
/**
* Creates a writable stream for uploading a file to the Blob storage.
*
* @param {string} blobName - The name of the blob to upload.
* @return {CreateUploadWriteStreamResponse} An object containing the key of the uploaded blob, the writable stream, and a promise that resolves when the upload is complete.
*/
createUploadWriteStream(blobName) {
const streamPassThrough = new stream.PassThrough();
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const uploadPromise = blockBlobClient.uploadStream(streamPassThrough, void 0, void 0, {
onProgress: (ev) => console.log(ev)
}).then(() => {
return {
Bucket: this.containerName,
Key: blobName
};
});
return {
key: blobName,
stream: streamPassThrough,
promise: uploadPromise
};
}
/**
* Retrieves the properties of a blob from the container by its name.
*
* @param {string} blobName - The name of the blob.
* @return {Promise<GetObjectResponse>} A promise that resolves to the blob properties.
*/
async getHeadObject(blobName) {
try {
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const blobProperties = await blockBlobClient.getProperties();
return BlobPropertiesResponseToObjectResponse(blobProperties);
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Retrieves an object from the blob storage service.
*
* @param {string} blobName - The name of the blob to retrieve.
* @param {Object} [options] - The options that you can pass to the library.
* @return {Promise<GetObjectResponse>} A promise that resolves to the object data, including the body, metadata, content type, and content length, or rejects with an error if there was an issue.
*/
async getObject(blobName, options) {
var _a;
try {
let downloadBlockBlobResponse;
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const expression = (options == null ? void 0 : options.range) || "";
const matches = expression.match(/\d+/g);
if (expression && (matches == null ? void 0 : matches.length)) {
const [start, end] = matches.map(Number);
downloadBlockBlobResponse = await blockBlobClient.download(start, end);
} else {
downloadBlockBlobResponse = await blockBlobClient.download(0);
}
return {
body: downloadBlockBlobResponse.readableStreamBody,
metadata: downloadBlockBlobResponse.metadata,
contentType: downloadBlockBlobResponse.contentType,
contentLength: downloadBlockBlobResponse.contentLength
};
} catch (error) {
let errorKey = "DEFAULT";
if (((_a = error == null ? void 0 : error.response) == null ? void 0 : _a.status) === 404) {
errorKey = "ObjectNotFound";
}
throw ErrorHandler.handleError(errorKey, "", error);
}
}
async getSignatureUrl(blobName, expiresInMinutes, permissions) {
try {
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const startDate = /* @__PURE__ */ new Date();
const expiryDate = new Date(startDate);
expiryDate.setMinutes(startDate.getMinutes() + expiresInMinutes);
return blockBlobClient.generateSasUrl({
permissions: BlobSASPermissions.parse(permissions),
startsOn: startDate,
expiresOn: expiryDate
});
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
async getUploadUrl(blobName, expiresInMinutes) {
try {
const sasUrl = await this.getSignatureUrl(blobName, expiresInMinutes, "w");
return {
key: blobName,
signedUrl: sasUrl
};
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Retrieves a signed URL for the blob with the specified name that expires after a certain period.
*
* @param {string} blobName - The name of the blob to generate the signed URL for.
* @param {number} expiresInMinutes - The duration in minutes until the signed URL expires.
* @return {Promise<string>} A Promise that resolves with the signed URL.
*/
async getDownloadUrl(blobName, expiresInMinutes) {
try {
const sasUrl = await this.getSignatureUrl(blobName, expiresInMinutes, "r");
return sasUrl;
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Uploads a file to the blob storage service.
*
* @param {string} blobName - The name of the blob to upload.
* @param {FileContent} body - The content of the file to upload.
* @param {{ [key: string]: string } | undefined} [metadata] - Optional metadata to associate with the blob.
* @return {Promise<UploadResponse>} A promise that resolves to the response object containing the key of the uploaded blob, or rejects with an error if there was an issue.
*/
async upload(blobName, body, metadata = {}) {
try {
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
Buffer.isBuffer(body) ? await blockBlobClient.upload(body, body.length, {
metadata
}) : await blockBlobClient.uploadStream(body, void 0, void 0, {
metadata
});
return {
key: blobName
};
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Deletes a blob from the blob storage service.
*
* @param {string} blobName - The name of the blob to be deleted.
* @return {Promise<boolean>} A promise that resolves to true if the blob is successfully deleted, or rejects with an error if there was an issue.
*/
async delete(data) {
try {
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
return this.deleteObject(containerClient, data);
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Lists a chunk of blobs from the specified container client based on the provided options and continuation token.
*
* @param {ListRequestOptions} options - The options for listing the blobs.
* @param {string | undefined} continuationToken - The continuation token for pagination.
* @param {number} limit - The maximum number of blobs to list in a single page.
* @param {ContainerClient} containerClient - The container client to list the blobs from.
* @return {Promise<{ items: BlobItem[]; continuationToken?: string }>} - A promise that resolves to an object containing the list of blob items and an optional continuation token.
*/
async listChunk(options, continuationToken, limit, containerClient) {
const iterator = containerClient.listBlobsFlat({
prefix: options.prefix
}).byPage({
maxPageSize: limit,
continuationToken
});
const items = [];
const response = (await iterator.next()).value;
items.push(...response.segment.blobItems);
return {
items,
continuationToken: response.continuationToken
};
}
/**
* Retrieves a list of blob contents based on the provided options.
*
* @param {ListRequestOptions} options - The options for listing the blob contents.
* @return {Promise<{ contents: BlobItem[]; nextContinuationToken?: string }>} A promise that resolves to the list of blob contents and the next continuation token.
*/
async listContents(options) {
var _a;
let responseContents = [];
let continuationToken = options.pagination;
const limit = (_a = options.limit) != null ? _a : 1e3;
let continueListing = true;
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
while (continueListing) {
const response = await this.listChunk(options, continuationToken, limit - responseContents.length, containerClient);
continuationToken = response.continuationToken;
responseContents = responseContents.concat(response.items);
if (responseContents.length >= limit || !continuationToken) {
continueListing = false;
}
}
return {
contents: responseContents,
nextContinuationToken: continuationToken
};
}
/**
* Lists blobs in the Azure Blob Storage container with pagination.
* @param options - The options for listing blobs.
* @returns A promise that resolves to a paginated list of blob names.
*/
async list(options) {
var _a;
let items = [];
const response = await this.listContents(options);
items = response.contents.length ? response.contents.map((blob) => {
return {
key: blob.name,
lastModified: blob.properties.lastModified,
size: blob.properties.contentLength,
eTag: blob.properties.etag
};
}) : [];
return {
items,
pagination: ((_a = response.nextContinuationToken) == null ? void 0 : _a.length) ? response.nextContinuationToken : null,
count: items.length
};
}
/**
* Lists all blobs in the Azure Blob Storage container.
* @param options - The options for listing blobs.
* @returns A promise that resolves to list of blob names.
*/
async listAll(options) {
var _a, _b;
let allItems = [];
let pagination = void 0;
do {
const response = await this.list(__spreadProps(__spreadValues({}, options), {
pagination
}));
if ((_a = response.items) == null ? void 0 : _a.length)
allItems = [
...allItems,
...response.items
];
pagination = (_b = response.pagination) != null ? _b : void 0;
} while (pagination);
return {
items: allItems,
count: allItems.length
};
}
/**
* Deletes a blob from the blob storage service.
*
* @param {string} blobName - The name of the blob to be deleted.
* @return {Promise<boolean>} A promise that resolves to true if the blob is successfully deleted, or rejects with an error if there was an issue.
*/
async deleteObject(containerClient, blobName) {
try {
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
await blockBlobClient.delete();
return true;
} catch (error) {
throw ErrorHandler.handleError("DEFAULT", "", error);
}
}
/**
* Deletes multiple blobs from the blob storage service.
*
* @param {string[]} blobNames - An array of blob names to be deleted.
* @return {Promise<{ key: string; deleted: boolean; error?: string }[]>} - A promise that resolves to an array of objects containing the key, deleted status, and an optional error message for each blob deleted.
*/
async deleteObjects(blobNames) {
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const deleteBlobPromises = blobNames.map(async (blobName) => {
var _a;
try {
await this.deleteObject(containerClient, blobName);
return {
key: blobName,
deleted: true
};
} catch (error) {
return {
key: blobName,
deleted: false,
error: (_a = error == null ? void 0 : error.message) != null ? _a : ""
};
}
});
return Promise.all(deleteBlobPromises);
}
/**
* Retrieves the properties of the container.
*
* @return {Promise<HeadBucketResponse>} A promise that resolves to the container properties.
*/
async getHeadBucket() {
try {
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const properties = await containerClient.getProperties();
return properties;
} catch (error) {
throw new Error("Error in Azure getHeadContainer. Container: " + this.containerName + " Error: " + error);
}
}
async listMultipartUploadsForBucket() {
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const blobList = containerClient.listBlobsFlat();
const uploads = [];
try {
for (var iter = __forAwait(blobList), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
const blob = temp.value;
uploads.push({
uploadId: blob.name,
key: blob.name,
initiated: blob.properties.createdOn || /* @__PURE__ */ new Date()
});
}
} catch (temp) {
error = [temp];
} finally {
try {
more && (temp = iter.return) && await temp.call(iter);
} finally {
if (error)
throw error[0];
}
}
return {
bucketName: this.containerName,
uploads
};
}
/**
* Retrieves the list of parts for a multipart upload.
*
* @param {string} blobName - The name of the blob (file) in Azure Blob Storage.
* @return {Promise<ListPartsMultipartUploadResponse>} A promise that resolves to the list of parts for the multipart upload.
*/
async listMultipartUploadsForKey(blobName) {
var _a;
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const blobClient = containerClient.getBlockBlobClient(blobName);
const listResponse = await blobClient.getBlockList("all");
const parts = ((_a = listResponse == null ? void 0 : listResponse.uncommittedBlocks) == null ? void 0 : _a.map((block, index) => ({
PartNumber: index + 1,
LastModified: /* @__PURE__ */ new Date(),
ETag: block.name,
Size: block.size
}))) || [];
return {
Parts: parts
};
}
/**
* Generates a unique upload ID for multipart uploads.
*
* @return {Promise<string>} A promise that resolves to a unique upload ID.
*/
async generateUploadIdMultipart() {
const timestamp = Date.now();
const randomNum = Math.floor(Math.random() * 100);
const blockIdBase = (timestamp - randomNum).toString();
return blockIdBase;
}
/**
* Uploads a part of a file as a block to Azure Blob Storage and updates the metadata with the block ID.
*
* @param {string} blobName - The name of the blob (file) that you want to upload in parts to Azure Blob Storage.
* @param {FileContent | any} file - The content of the file that you want to upload as a part of a multipart upload.
* @param {number} partNumber - The number or index of the part being uploaded.
* @param {string} [uploadId] - The ID of the multipart upload. If not provided, a new upload ID will be generated.
* @return {Promise<string>} A Promise that resolves to a string representing the base64 encoded block ID of the uploaded part.
*/
async uploadMultipart(blobName, file, partNumber, uploadId) {
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const blobClient = containerClient.getBlockBlobClient(blobName);
const blockIdBase = uploadId || await this.generateUploadIdMultipart();
const partId = partNumber.toString().padStart(6, "0");
const partIdBase64 = Buffer.from(partId).toString("base64");
await blobClient.stageBlock(partIdBase64, file, file.length);
return blockIdBase;
}
/**
* Completes a multipart upload by committing the blocks to create the blob.
*
* @param {string} blobName - The name of the blob for which the multipart upload is being completed.
* @param {string} uploadId - The ID of the multipart upload to be completed.
* @return {Promise<void>} A Promise that resolves when the multipart upload is successfully completed.
* @throws {Error} If no block IDs are found in the metadata.
*/
async completeMultipartUpload(blobName, uploadId) {
var _a;
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const blobClient = containerClient.getBlockBlobClient(blobName);
const listMultipartUploads = await this.listMultipartUploadsForKey(blobName);
const blockIds = ((_a = listMultipartUploads == null ? void 0 : listMultipartUploads.Parts) == null ? void 0 : _a.map((part) => part.ETag)) || [];
if (blockIds.length === 0) {
throw new Error("No block IDs found in metadata.");
}
await blobClient.commitBlockList(blockIds);
}
/**
* Aborts a multipart upload by deleting the specified blob and clearing its block IDs metadata.
*
* @param {string} blobName - The name of the blob for which the multipart upload needs to be aborted.
* @param {string} uploadId - The ID of the multipart upload to be aborted.
* @return {Promise<void>} A promise that resolves when the multipart upload is successfully aborted.
*/
async abortMultipartUpload(blobName, uploadId) {
const containerClient = this.blobServiceClient.getContainerClient(this.containerName);
const blobClient = containerClient.getBlockBlobClient(blobName);
const blockIdStorage = BlockIdStorage.getInstance();
blockIdStorage.clearBlockIds(uploadId);
await blobClient.delete();
}
async getMultipartUploadPresignedUrl(blobName, uploadId, partNumber, expiresInMinutes = 2) {
const partId = partNumber.toString().padStart(6, "0");
const partIdBase64 = Buffer.from(partId).toString("base64");
const sasUrl = await this.getSignatureUrl(blobName, expiresInMinutes, "w");
const url = `${sasUrl}&comp=block&blockid=${partIdBase64}`;
return url;
}
};
__name(_BlobStorageService, "BlobStorageService");
var BlobStorageService = _BlobStorageService;
// src/services/storage/s3/s3Helpers.ts
function listResponseContentsToListResponseItems(responseContents) {
return responseContents.map((responseContent) => {
return {
key: responseContent.Key,
lastModified: new Date(responseContent.LastModified),
size: responseContent.Size,
eTag: responseContent.ETag
};
});
}
__name(listResponseContentsToListResponseItems, "listResponseContentsToListResponseItems");
function s3ObjectToObjectResponse(s3Object) {
return {
body: s3Object.Body,
metadata: s3Object.Metadata,
contentType: s3Object.ContentType,
contentLength: s3Object.ContentLength,
contentEncoding: s3Object.ContentEncoding,
cacheControl: s3Object.CacheControl,
contentDisposition: s3Object.ContentDisposition,
contentLanguage: s3Object.ContentLanguage,
eTag: s3Object.ETag,
lastModified: s3Object.LastModified
};
}
__name(s3ObjectToObjectResponse, "s3ObjectToObjectResponse");
var sharedHttpHandler = new NodeHttpHandler({
socketTimeout: 6e5
});
var sharedCredentialsProvider = defaultProvider();
var defaultS3Client = new S3Client({
region: process.env.AWS_DEFAULT_REGION,
requestHandler: sharedHttpHandler,
credentials: sharedCredentialsProvider
});
var defaultS3 = new S3({
region: process.env.AWS_DEFAULT_REGION,
requestHandler: sharedHttpHandler,
credentials: sharedCredentialsProvider
});
var _S3StorageService = class _S3StorageService {
constructor(bucketName, options) {
__publicField(this, "s3Client");
__publicField(this, "s3");
__publicField(this, "bucketName");
var _a, _b;
this.bucketName = bucketName;
if (((_a = options == null ? void 0 : options.credentials) == null ? void 0 : _a.accessKeyId) && ((_b = options == null ? void 0 : options.credentials) == null ? void 0 : _b.secretAccessKey)) {
const s3Config = {
region: (options == null ? void 0 : options.region) || process.env.AWS_REGION,
requestHandler: sharedHttpHandler,
credentials: {
accessKeyId: options.credentials.accessKeyId,
secretAccessKey: options.credentials.secretAccessKey
}
};
this.s3Client = new S3Client(s3Config);
this.s3 = new S3(s3Config);
} else {
this.s3Client = defaultS3Client;
this.s3 = defaultS3;
}
}
/**
* Retrieves a chunk of objects from the S3 bucket based on the provided options.
*
* @param {ListRequestOptions} options - The options for listing the objects.
* @param {string | undefined} continuationToken - The continuation token for pagination.
* @param {number} limit - The maximum number of objects to retrieve.
* @return {Promise<ListObjectsV2CommandOutput>} - A promise that resolves to the response containing the list of objects and metadata.
*/
async listChunk(options, continuationToken, limit) {
const command = new ListObjectsV2Command({
Bucket: this.bucketName,
ContinuationToken: continuationToken || options.pagination,
Prefix: options.prefix,
MaxKeys: limit
});
return this.s3Client.send(command);
}
/**
* Retrieves a list of contents from the S3 bucket based on the provided options.
*
* @param {ListRequestOptions} options - The options for listing the contents.
* @return {Promise<{ contents: unknown[]; nextContinuationToken?: string }>} - A promise that resolves to an object containing the list of contents and an optional next continuation token.
*/
async listContents(options) {
var _a, _b;
let responseContents = [];
let continuationToken = void 0;
const limit = (_a = options.limit) != null ? _a : 1e3;
let continueListing = true;
while (continueListing) {
const listChunkLimit = limit - responseContents.length;
const response = await this.listChunk(options, continuationToken, listChunkLimit);
continuationToken = response.NextContinuationToken;
responseContents = responseContents.concat((_b = response.Contents) != null ? _b : []);
if (responseContents.length >= limit || !continuationToken) {
continueListing = false;
}
}
return {
contents: responseContents,
nextContinuationToken: continuationToken
};
}
/**
* Lists objects in the S3 bucket with pagination.
* @param options - The options for listing objects.
* @returns A promise that resolves to a paginated list of object keys.
*/
async list(options) {
var _a;
let items = [];
const response = await this.listContents(options);
items = response.contents.length ? listResponseContentsToListResponseItems(response.contents) : [];
return {
items,
pagination: (_a = response.nextContinuationToken) != null ? _a : null,
count: items.length
};
}
/**
* Lists all objects in the S3 bucket.
* @param options - The options for listing objects.
* @returns A promise that resolves to list of object keys.
*/
async listAll(options) {
var _a;
let allItems = [];
let pagination = void 0;
do {
const response = await this.list(__spreadProps(__spreadValues({}, options), {
pagination
}));
if ((_a = response.items) == null ? void 0 : _a.length)
allItems = [
...allItems,
...response.items
];
pagination = response.pagination === null ? void 0 : response.pagination;
} while (pagination);
return {
items: allItems,
count: allItems.length
};
}
/**
* Retrieves an object from the S3 bucket.
* @param key - The key of the object to retrieve.
* @returns A promise that resolves to the content of the file.
*/
async getHeadObject(key) {
const command = new HeadObjectCommand({
Bucket: this.bucketName,
Key: key
});
const response = await this.s3Client.send(command);
return s3ObjectToObjectResponse(response);
}
/**
* Retrieves an object from the S3 bucket.
* @param key - The key of the object to retrieve.
* @returns A promise that resolves to the content of the file.
*/
async getObject(key) {
const command = new GetObjectCommand({
Bucket: this.bucketName,
Key: key
});
const response = await this.s3Client.send(command);
return s3ObjectToObjectResponse(response);
}
/**
* Generates a signed URL for accessing an object in the S3 bucket.
* @param key - The key of the object.
* @param expiresInMinutes - The number of minutes the signed URL should be valid for. Defaults to 60 minutes.
* @returns A promise that resolves to the signed URL.
*/
getDownloadUrl(key, expiresInMinutes = 60) {
const expiresIn = expiresInMinutes * 60;
const command = new GetObjectCommand({
Bucket: this.bucketName,
Key: key
});
return getSignedUrl(this.s3Client, command, {
expiresIn
});
}
/**
* Retrieves a signed URL for uploading an object to the S3 bucket.
*
* @param {string} key - The key of the object to be uploaded.
* @param {number} [expiresInMinutes=60] - The number of minutes the signed URL should be valid for. Defaults to 60 minutes.
* @returns A promise that resolves to the signed URL.
*/
async getUploadUrl(key, expiresInMinutes = 60) {
const expiresIn = expiresInMinutes * 60;
const putObjectCommandParams = {
Bucket: process.env.UPLOAD_BUCKET_NAME,
Key: key,
ACL: "private"
};
const command = new PutObjectCommand(putObjectCommandParams);
const signedUrl = await getSignedUrl(this.s3Client, command, {
expiresIn
});
return {
signedUrl,
key: `https://${process.env.UPLOAD_BUCKET_NAME}.s3.${process.env.AWS_DEFAULT_REGION}.amazonaws.com/${key}`
};
}
/**
* Uploads an object to the S3 bucket.
* @param key - The key of the object to upload.
* @param body - The content of the object to upload.
* @param metadata - Optional metadata to associate with the object.
* @returns A promise that resolves to the response containing the key of the uploaded object.
*/
async upload(key, body, metadata) {
const command = new PutObjectCommand({
Bucket: this.bucketName,
Key: key,
Body: body,
Metadata: metadata
});
await this.s3Client.send(command);
return {
key
};
}
/**
* Creates a writable stream for uploading a file to the S3 bucket.
*
* @param {string} key - The key of the object to upload.
* @return {CreateUploadWriteStreamResponse} An object containing the key of the uploaded object, the writable stream, and a promise that resolves when the upload is complete.
*/
createUploadWriteStream(key) {
const streamPassThrough = new stream.PassThrough();
const params = {
Bucket: this.bucketName,
Key: key,
Body: streamPassThrough
};
const upload = new Upload({
client: this.s3Client,
params,
queueSize: 5,
partSize: 8 * 1024 * 1024
});
return {
key,
stream: streamPassThrough,
promise: upload.done()
};
}
/**
* Deletes an object from the S3 bucket.
* @param key - The key of the object to delete.
* @returns A promise that resolves to true if the object was deleted successfully.
*/
async delete(key) {
const command = new DeleteObjectCommand({
Bucket: this.bucketName,
Key: key
});
await this.s3Client.send(command);
return true;
}
async getHeadBucket() {
try {
const command = new HeadBucketCommand({
Bucket: this.bucketName
});
return this.s3Client.send(command);
} catch (error) {
throw new Error("Error in S3 getHeadContainer. bucketName: " + this.bucketName + " Error: " + error);
}
}
/**
* Generates an upload ID for multipart upload in the S3 bucket.
*
* @param {string} key - The key of the object to upload.
* @return {Promise<string>} A promise that resolves to the generated upload ID.
* @throws {Error} If no upload ID was generated.
*/
async generateUploadIdMultipart(key) {
const params = {
Bucket: this.bucketName,
Key: key
};
const response = await this.s3.createMultipartUpload(params);
if (!(response == null ? void 0 : response.UploadId))
throw new Error("No upload ID was generated");
return response.UploadId;
}
async listMultipartUploadsForBucket() {
const params = {
Bucket: this.bucketName
};
const response = await this.s3.listMultipartUploads(params);
return {
bucketName: this.bucketName,
uploads: response.Uploads ? response.Uploads.map((upload) => {
var _a, _b;
return {
uploadId: upload.UploadId || "",
key: upload.Key || "",
initiator: ((_a = upload == null ? void 0 : upload.Initiator) == null ? void 0 : _a.ID) || "",
owner: ((_b = upload == null ? void 0 : upload.Owner) == null ? void 0 : _b.ID) || "",
storageClass: upload.StorageClass || "",
initiated: upload.Initiated || ""
};
}) : []
};
}
/**
* Retrieves the list of parts for a multipart upload.
*
* @param {string} key - The key of the object being uploaded.
* @param {string} uploadId - The ID of the multipart upload.
* @return {Promise<ListPartsMultipartUploadResponse>} A promise that resolves to the list of parts for the multipart upload.
*/
async listMultipartUploadsForKey(key, uploadId) {
const params = {
Bucket: this.bucketName,
Key: key,
UploadId: uploadId
};
const response = await this.s3.listParts(params);
const parts = response.Parts ? response.Parts.map((part) => ({
PartNumber: part.PartNumber,
LastModified: part.LastModified,
ETag: part.ETag,
Size: part.Size
})) : [];
return {
Parts: parts
};
}
async uploadMultipart(key, file, partNumber, uploadId) {
const upId = uploadId || await this.generateUploadIdMultipart(key);
const params = {
Bucket: this.bucketName,
Key: key,
Body: file,
PartNumber: partNumber,
UploadId: upId
};
await this.s3.uploadPart(params);
return upId;
}
async completeMultipartUpload(key, uploadId) {
const partsResponse = await this.listMultipartUploadsForKey(key, uploadId);
const partsList = (partsResponse == null ? void 0 : partsResponse.Parts) && partsResponse.Parts.map(
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
(_a) => {
var _b = _a, partList = __objRest(_b, ["Size", "LastModified"]);
return partList;
}
);
const completeMultipartParams = {
Bucket: this.bucketName,
Key: key,
UploadId: uploadId,
MultipartUpload: {
Parts: partsList
}
};
await this.s3.completeMultipartUpload(completeMultipartParams);
}
async abortMultipartUpload(key, uploadId) {
const params = {
Bucket: this.bucketName,
Key: key,
UploadId: uploadId
};
await this.s3.abortMultipartUpload(params);
}
async getMultipartUploadPresignedUrl(key, uploadId, partNumber, expiresInMinutes = 60) {
const uploadPartParams = {
Bucket: this.bucketName,
Key: key,
UploadId: uploadId,
PartNumber: parseInt(partNumber, 10)
};
const expiresIn = expiresInMinutes * 60;
const presignedUrl = await getSignedUrl(this.s3Client, new UploadPartCommand(uploadPartParams), {
expiresIn
});
return presignedUrl;
}
};
__name(_S3StorageService, "S3StorageService");
var S3StorageService = _S3StorageService;
// src/shared/utils/constants.ts
var OBJECT_STORAGE_SERVICE_TYPES = {
AWS_S3: "aws_s3",
AZURE_BLOB_STORAGE: "azure_blob_storage"
};
// src/services/objectStorageFactory.service.ts
var _ObjectStorageFactory = class _ObjectStorageFactory {
static async instance(bucketName, options) {
var _a, _b;
const provider = ((_a = options == null ? void 0 : options.provider) == null ? void 0 : _a.toLowerCase()) || ((_b = process.env.OBJECT_STORAGE_SERVICE) == null ? void 0 : _b.toLowerCase());
switch (provider) {
case OBJECT_STORAGE_SERVICE_TYPES.AWS_S3:
return new S3StorageService(bucketName, options);
case OBJECT_STORAGE_SERVICE_TYPES.AZURE_BLOB_STORAGE:
return new BlobStorageService(bucketName);
default:
throw new Error(`Unsupported object storage provider: ${provider}`);
}
}
};
__name(_ObjectStorageFactory, "ObjectStorageFactory");
var ObjectStorageFactory = _ObjectStorageFactory;
// src/services/objectStorage.service.ts
var _ObjectStorageService = class _ObjectStorageService {
constructor(bucketName, options) {
_ObjectStorageService.bucketName = bucketName;
if (options)
_ObjectStorageService.objectStorageOptions = options;
}
static async getObjectStorageServiceInstance(bucketName = _ObjectStorageService.bucketName, options = _ObjectStorageService.objectStorageOptions) {
if (!bucketName)
throw new Error("Bucket name not provided or not valid value");
return ObjectStorageFactory.instance(bucketName, options);
}
/**
* Retrieves a list of objects from the object storage service.
*
* @param {ListRequestOptions} options - The options to apply to the list operation.
* @param {string} [bucketName] - The name of the bucket to list objects from. If not provided, the default bucket name will be used.
* @return {Promise<ListResponse>} A promise that resolves to the list of objects.
*/
static async list(options, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.list(options));
}
/**
* Retrieves a list of all objects from the object storage service.
*
* @param {ListRequestOptions} options - The options to apply to the list operation.
* @param {string} [bucketName] - The name of the bucket to list objects from. If not provided, the default bucket name will be used.
* @return {Promise<ListResponse>} A promise that resolves to the list of all objects.
*/
static async listAll(options, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.listAll(options));
}
/**
* Retrieves an object from the object storage service.
*
* @param {string} key - The key of the object to retrieve.
* @param {string} [bucketName] - The name of the bucket where the object is stored. If not provided, the default bucket name will be used.
* @return {Promise<GetObjectResponse>} A promise that resolves to the retrieved object.
*/
static async getObject(key, bucketName, options) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.getObject(key, options));
}
/**
* Retrieves an object info (without file content) from the object storage service.
* @param key - The key of the object to retrieve.
* @returns A promise that resolves to the info of the file.
*/
static async getHeadObject(key, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.getHeadObject(key));
}
/**
* Retrieves a signed URL for the specified object in the object storage service.
*
* @param {string} key - The key of the object for which to generate the signed URL.
* @param {number} expiresInMinutes - The number of minutes until the signed URL expires.
* @param {string} [bucketName] - The name of the bucket where the object is stored. If not provided, the default bucket name will be used.
* @return {Promise<string>} A promise that resolves to the generated signed URL.
*/
static async getDownloadUrl(key, expiresInMinutes, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.getDownloadUrl(key, expiresInMinutes));
}
/**
* Retrieves an upload URL for the specified object in the object storage service.
*
* @param {string} key - The key of the object for which to generate the upload URL.
* @param {number} expiresInMinutes - The number of minutes until the upload URL expires.
* @param {string} [bucketName] - The name of the bucket where the object will be stored. If not provided, the default bucket name will be used.
* @return {Promise<string>} A promise that resolves to the generated upload URL.
*/
static async getUploadUrl(key, expiresInMinutes, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.getUploadUrl(key, expiresInMinutes));
}
/**
* Uploads a file to the object storage service.
*
* @param {string} key - The key of the object to upload.
* @param {FileContent} body - The content of the file to upload.
* @param {Object} [metadata] - Optional metadata to associate with the object.
* @param {string} [bucketName] - The name of the bucket where the object will be stored. If not provided, the default bucket name will be used.
* @return {Promise<UploadResponse>} A promise that resolves to the response of the upload operation.
*/
static async upload(key, body, metadata, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.upload(key, body, metadata));
}
/**
* Creates an upload write stream for the specified key in the object storage service.
*
* @param {string} key - The key of the object to create the upload write stream for.
* @param {string} [bucketName] - The name of the bucket where the object will be stored. If not provided, the default bucket name will be used.
* @return {Promise<CreateUploadWriteStreamResponse>} A promise that resolves to the response of the upload operation.
*/
static async createUploadWriteStream(key, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.createUploadWriteStream(key));
}
/**
* Deletes an object or multiple objects from the object storage service.
*
* @param {string | string[]} key - The key or array of keys of the objects to delete.
* @param {string} [bucketName] - The name of the bucket where the objects are stored. If not provided, the default bucket name will be used.
* @return {Promise<{ key: string; deleted: boolean; error?: string }[] | boolean>} A promise that resolves to an array of objects containing the key, deleted status, and an optional error message for each object deleted, or a boolean value indicating the success of the deletion.
*/
static async delete(key, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then(async (instance) => {
if (Array.isArray(key)) {
const deleteBlobPromises = key.map(async (blobName) => {
var _a;
try {
await instance.delete(blobName);
return {
key: blobName,
deleted: true
};
} catch (error) {
return {
key: blobName,
deleted: false,
error: (_a = error == null ? void 0 : error.message) != null ? _a : ""
};
}
});
return Promise.all(deleteBlobPromises);
} else {
return instance.delete(key);
}
});
}
/**
* Retrieves the head bucket from the object storage service.
*
* @param {string} [bucketName] - The name of the bucket. If not provided, the default bucket name will be used.
* @return {Promise<HeadBucketResponse>} A promise that resolves to the head bucket response.
*/
static async getHeadBucket(bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.getHeadBucket());
}
/**
* Generates a unique upload ID for a multipart upload.
*
* @param {string} key - The key of the object to upload.
* @param {string} [bucketName] - The name of the bucket. If not provided, the default bucket name will be used.
* @return {Promise<string>} A promise that resolves to the generated upload ID.
*/
static async generateUploadIdMultipart(key, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.generateUploadIdMultipart(key));
}
/**
* Retrieves a list of multipart uploads for a specified key in the object storage service.
*
* @param {string} key - The key of the object to retrieve uploads for.
* @param {string} [bucketName] - The name of the bucket. If not provided, the default bucket name will be used.
* @param {string} [uploadId] - The upload ID to filter the results by.
* @return {Promise<ListPartsMultipartUploadResponse>} A promise that resolves to the list of multipart uploads for the specified key.
*/
static async listMultipartUploadsForKey(key, bucketName, uploadId) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.listMultipartUploadsForKey(key, uploadId));
}
/**
* Retrieves a list of all multipart uploads for a specified bucket in the object storage service.
*
* @param {string} [bucketName] - The name of the bucket. If not provided, the default bucket name will be used.
* @return {Promise<ListMultipartUploadsResponse>} A promise that resolves to the list of multipart uploads for the specified bucket.
*/
static async listMultipartUploadsForBucket(bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.listMultipartUploadsForBucket());
}
/**
* Uploads a multipart file to the specified bucket in the object storage service.
*
* @param {string} key - The key of the object to upload the multipart file to.
* @param {FileContent} file - The content of the file to upload.
* @param {number} partNumber - The number of the part being uploaded.
* @param {string} [uploadId] - The ID of the multipart upload.
* @param {string} [bucketName] - The name of the bucket to upload the file to. If not provided, the default bucket name will be used.
* @return {Promise<string>} A Promise that resolves to the base64 encoded block ID of the uploaded part.
*/
static async uploadMultipart(key, file, partNumber, uploadId, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => instance.uploadMultipart(key, file, partNumber, uploadId));
}
/**
* Completes a multipart upload for the specified object in the object storage service.
*
* @param {string} key - The key of the object to complete the multipart upload for.
* @param {string} uploadId - The ID of the multipart upload to complete.
* @param {string} [bucketName] - The name of the bucket to complete the multipart upload in. If not provided, the default bucket name will be used.
* @return {Promise<void>} A Promise that resolves when the multipart upload is completed.
*/
static async completeMultipartUpload(key, uploadId, bucketName) {
return _ObjectStorageService.getObjectStorageServiceInstance(bucketName).then((instance) => ins