flydrive
Version:
File storage library with unified API to manage files across multiple cloud storage providers like S3, GCS, R2 and so on
535 lines (531 loc) • 16.6 kB
JavaScript
import {
DriveDirectory,
DriveFile
} from "../../chunk-TZSKXVQT.js";
// drivers/s3/driver.ts
import mimeTypes from "mime-types";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import {
S3Client,
PutObjectCommand,
GetObjectCommand,
HeadObjectCommand,
CopyObjectCommand,
PutObjectAclCommand,
DeleteObjectCommand,
GetObjectAclCommand,
ListObjectsV2Command,
DeleteObjectsCommand
} from "@aws-sdk/client-s3";
// drivers/s3/debug.ts
import { debuglog } from "util";
var debug_default = debuglog("flydrive:s3");
// drivers/s3/driver.ts
import string from "@poppinss/utils/string";
var S3Driver = class {
constructor(options) {
this.options = options;
this.#client = "client" in options ? options.client : new S3Client(options);
if (options.supportsACL !== void 0) {
this.#supportsACL = options.supportsACL;
}
if (debug_default.enabled) {
debug_default("driver config %O", {
...options,
credentials: "REDACTED"
});
}
}
#client;
#supportsACL = true;
/**
* The URI that holds permission for public
*/
publicGrantUri = "http://acs.amazonaws.com/groups/global/AllUsers";
/**
* Creates the metadata for the file from the raw response
* returned by S3
*/
#createFileMetaData(apiFile) {
const metaData = {
contentType: apiFile.ContentType,
contentLength: apiFile.ContentLength,
etag: apiFile.ETag,
lastModified: new Date(apiFile.LastModified)
};
debug_default("file metadata %O", this.options.bucket, metaData);
return metaData;
}
/**
* Returns S3 options for the save operations.
*/
#getSaveOptions(key, options) {
const {
visibility,
// used locally
contentType,
// forwaded as metadata
cacheControl,
// forwaded as metadata
contentEncoding,
// forwaded as metadata
contentLength,
// forwaded as metadata
contentLanguage,
// forwaded as metadata
contentDisposition,
// forwaded as metadata
...rest
// forwarded as it is
} = options || {};
const s3Options = {
Bucket: this.options.bucket,
ServerSideEncryption: this.options.encryption,
...rest
};
if (this.#supportsACL) {
const isPublic = (visibility || this.options.visibility) === "public";
s3Options.ACL = isPublic ? "public-read" : "private";
}
if (contentType) {
s3Options.ContentType = contentType;
} else {
const detectedContentType = mimeTypes.lookup(key);
if (detectedContentType) {
debug_default(`setting "%s" file's content-type to "%s"`, key, detectedContentType);
s3Options.ContentType = detectedContentType;
}
}
if (cacheControl) {
s3Options.CacheControl = cacheControl;
}
if (contentEncoding) {
s3Options.ContentEncoding = contentEncoding;
}
if (contentLength) {
s3Options.ContentLength = contentLength;
}
if (contentLanguage) {
s3Options.ContentLanguage = contentLanguage;
}
if (contentDisposition) {
s3Options.ContentDisposition = contentDisposition;
}
debug_default("s3 write options %O", s3Options);
return s3Options;
}
/**
* Deletes files recursively with a batch of 1000 files at a time
*/
async #deleteFilesRecursively(prefix, paginationToken) {
const response = await this.#client.send(
this.createListObjectsV2Command(this.#client, {
Bucket: this.options.bucket,
ContinuationToken: paginationToken,
...prefix !== "/" ? { Prefix: prefix } : {}
})
);
if (!response.Contents || !response.Contents.length) {
return;
}
await this.#client.send(
this.createDeleteObjectsCommand(this.#client, {
Bucket: this.options.bucket,
Delete: {
Objects: Array.from(response.Contents).map((file) => {
return {
Key: file.Key
};
}),
Quiet: true
}
})
);
if (response.NextContinuationToken) {
debug_default("deleting next batch of files with token %s", response.NextContinuationToken);
await this.#deleteFilesRecursively(prefix, response.NextContinuationToken);
}
}
/**
* Creates S3 "PutObjectCommand". Feel free to override this method to
* manually create the command
*/
createPutObjectCommand(_, options) {
return new PutObjectCommand(options);
}
/**
* Creates S3 "GetObjectCommand". Feel free to override this method to
* manually create the command
*/
createGetObjectCommand(_, options) {
return new GetObjectCommand(options);
}
/**
* Creates S3 "HeadObjectCommand". Feel free to override this method to
* manually create the command
*/
createHeadObjectCommand(_, options) {
return new HeadObjectCommand(options);
}
/**
* Creates S3 "GetObjectAclCommand". Feel free to override this method to
* manually create the command
*/
createGetObjectAclCommand(_, options) {
return new GetObjectAclCommand(options);
}
/**
* Creates S3 "PutObjectAclCommand". Feel free to override this method to
* manually create the command
*/
createPutObjectAclCommand(_, options) {
return new PutObjectAclCommand(options);
}
/**
* Creates S3 "DeleteObjectCommand". Feel free to override this method to
* manually create the command
*/
createDeleteObjectCommand(_, options) {
return new DeleteObjectCommand(options);
}
/**
* Creates S3 "CopyObjectCommand". Feel free to override this method to
* manually create the command
*/
createCopyObjectCommand(_, options) {
return new CopyObjectCommand(options);
}
/**
* Creates S3 "ListObjectsV2Command". Feel free to override this method to
* manually create the command
*/
createListObjectsV2Command(_, options) {
return new ListObjectsV2Command(options);
}
/**
* Creates S3 "DeleteObjectsCommand". Feel free to override this method to
* manually create the command
*/
createDeleteObjectsCommand(_, options) {
return new DeleteObjectsCommand(options);
}
/**
* Returns a boolean indicating if the file exists
* or not.
*/
async exists(key) {
debug_default("checking if file exists %s:%s", this.options.bucket, key);
try {
const response = await this.#client.send(
this.createHeadObjectCommand(this.#client, {
Key: key,
Bucket: this.options.bucket
})
);
return response.$metadata.httpStatusCode === 200;
} catch (error) {
if (error.$metadata?.httpStatusCode === 404) {
return false;
}
throw error;
}
}
/**
* Returns the contents of a file as a UTF-8 string. An
* exception is thrown when object is missing.
*/
async get(key) {
debug_default("reading file contents %s:%s", this.options.bucket, key);
const response = await this.#client.send(
this.createGetObjectCommand(this.#client, {
Key: key,
Bucket: this.options.bucket
})
);
return response.Body.transformToString();
}
/**
* Returns the contents of the file as a Readable stream. An
* exception is thrown when the file is missing.
*/
async getStream(key) {
debug_default("reading file contents as a stream %s:%s", this.options.bucket, key);
const response = await this.#client.send(
this.createGetObjectCommand(this.#client, {
Key: key,
Bucket: this.options.bucket
})
);
return response.Body;
}
/**
* Returns the contents of the file as an Uint8Array. An
* exception is thrown when the file is missing.
*/
async getBytes(key) {
debug_default("reading file contents as array buffer %s:%s", this.options.bucket, key);
const response = await this.#client.send(
this.createGetObjectCommand(this.#client, {
Key: key,
Bucket: this.options.bucket
})
);
return response.Body.transformToByteArray();
}
/**
* Returns the file metadata.
*/
async getMetaData(key) {
debug_default("fetching file metadata %s:%s", this.options.bucket, key);
const response = await this.#client.send(
this.createHeadObjectCommand(this.#client, {
Key: key,
Bucket: this.options.bucket
})
);
return this.#createFileMetaData(response);
}
/**
* Returns the visibility of a file
*/
async getVisibility(key) {
if (!this.#supportsACL) {
return this.options.visibility;
}
debug_default("fetching file visibility %s:%s", this.options.bucket, key);
const response = await this.#client.send(
this.createGetObjectAclCommand(this.#client, {
Key: key,
Bucket: this.options.bucket
})
);
const isPublic = (response.Grants || []).find((grant) => {
return grant.Grantee?.URI === this.publicGrantUri && (grant.Permission === "READ" || grant.Permission === "FULL_CONTROL");
});
return isPublic ? "public" : "private";
}
/**
* Returns the public URL of the file. This method does not check
* if the file exists or not.
*/
async getUrl(key) {
const generateURL = this.options.urlBuilder?.generateURL;
if (generateURL) {
debug_default("using custom implementation for generating public URL %s:%s", this.options.bucket, key);
return generateURL(key, this.options.bucket, this.#client);
}
debug_default("generating file URL %s:%s", this.options.bucket, key);
if (this.options.cdnUrl) {
return new URL(key, this.options.cdnUrl).toString();
}
if (this.#client.config.endpoint) {
const endpoint = await this.#client.config.endpoint();
let baseUrl = `${endpoint.protocol}//${endpoint.hostname}`;
if (endpoint.port) {
baseUrl += `:${endpoint.port}`;
}
return new URL(`/${this.options.bucket}/${key}`, baseUrl).toString();
}
return new URL(`/${key}`, `https://${this.options.bucket}.s3.amazonaws.com`).toString();
}
/**
* Returns the signed/temporary URL of the file. By default, the signed URLs
* expire in 30mins, but a custom expiry can be defined using
* "options.expiresIn" property.
*/
async getSignedUrl(key, options) {
const { contentDisposition, contentType, expiresIn, ...rest } = Object.assign({}, options);
const expires = string.seconds.parse(expiresIn || "30mins");
const signedURLOptions = {
Key: key,
Bucket: this.options.bucket,
ResponseContentType: contentType,
ResponseContentDisposition: contentDisposition,
...rest
};
const generateSignedURL = this.options.urlBuilder?.generateSignedURL;
if (generateSignedURL) {
debug_default("using custom implementation for generating signed URL %s:%s", this.options.bucket, key);
return generateSignedURL(key, signedURLOptions, this.#client, expiresIn);
}
debug_default("generating signed URL %s:%s", this.options.bucket, key);
return getSignedUrl(this.#client, this.createGetObjectCommand(this.#client, signedURLOptions), {
expiresIn: expires
});
}
/**
* Returns a signed URL for uploading objects directly to S3.
*/
async getSignedUploadUrl(key, options) {
const { contentType, expiresIn, ...rest } = Object.assign({}, options);
const expires = string.seconds.parse(expiresIn || "30mins");
const signedURLOptions = {
Key: key,
Bucket: this.options.bucket,
ContentType: contentType,
...rest
};
const generateSignedUploadURL = this.options.urlBuilder?.generateSignedUploadURL;
if (generateSignedUploadURL) {
debug_default(
"using custom implementation for generating signed upload URL %s:%s",
this.options.bucket,
key
);
return generateSignedUploadURL(key, signedURLOptions, this.#client, expiresIn);
}
debug_default("generating signed upload URL %s:%s", this.options.bucket, key);
return getSignedUrl(this.#client, this.createPutObjectCommand(this.#client, signedURLOptions), {
expiresIn: expires
});
}
/**
* Updates the visibility of a file
*/
async setVisibility(key, visibility) {
if (!this.#supportsACL) {
return;
}
debug_default("updating file visibility %s:%s to %s", this.options.bucket, key, visibility);
await this.#client.send(
this.createPutObjectAclCommand(this.#client, {
Key: key,
Bucket: this.options.bucket,
ACL: visibility === "public" ? "public-read" : "private"
})
);
}
/**
* Writes a file to the bucket for the given key and contents.
*/
async put(key, contents, options) {
debug_default("creating/updating file %s:%s", this.options.bucket, key);
const command = this.createPutObjectCommand(this.#client, {
...this.#getSaveOptions(key, options),
Key: key,
Body: contents
});
await this.#client.send(command);
}
/**
* Writes a file to the bucket for the given key and stream
*/
putStream(key, contents, options) {
debug_default("creating/updating file %s:%s", this.options.bucket, key);
return new Promise((resolve, reject) => {
contents.once("error", reject);
try {
const command = this.createPutObjectCommand(this.#client, {
...this.#getSaveOptions(key, options),
Key: key,
Body: contents
});
return this.#client.send(command).then(() => resolve()).catch(reject);
} catch (error) {
reject(error);
}
});
}
/**
* Copies the source file to the destination. Both paths must
* be within the root location.
*/
async copy(source, destination, options) {
debug_default(
"copying file from %s:%s to %s:%s",
this.options.bucket,
source,
this.options.bucket,
destination
);
options = options || {};
if (!options.visibility && this.#supportsACL) {
options.visibility = await this.getVisibility(source);
}
await this.#client.send(
this.createCopyObjectCommand(this.#client, {
...this.#getSaveOptions(destination, options),
Key: destination,
CopySource: `/${this.options.bucket}/${source}`,
Bucket: this.options.bucket
})
);
}
/**
* Moves the source file to the destination. Both paths must
* be within the root location.
*/
async move(source, destination, options) {
debug_default(
"moving file from %s:%s to %s:%s",
this.options.bucket,
source,
this.options.bucket,
destination
);
await this.copy(source, destination, options);
await this.delete(source);
}
/**
* Deletes the object from the bucket
*/
async delete(key) {
debug_default("removing file %s:%s", this.options.bucket, key);
await this.#client.send(
this.createDeleteObjectCommand(this.#client, {
Key: key,
Bucket: this.options.bucket
})
);
}
/**
* Deletes the files and directories matching the provided
* prefix.
*/
async deleteAll(prefix) {
debug_default("removing all files matching prefix %s:%s", this.options.bucket, prefix);
await this.#deleteFilesRecursively(prefix);
}
/**
* Returns a list of files. The pagination token can be used to paginate
* through the files.
*/
async listAll(prefix, options) {
const self = this;
let { recursive, paginationToken, maxResults } = Object.assign({ recursive: false }, options);
if (prefix) {
prefix = !recursive ? `${prefix.replace(/\/$/, "")}/` : prefix;
}
debug_default("listing all files matching prefix %s:%s", this.options.bucket, prefix);
const response = await this.#client.send(
this.createListObjectsV2Command(this.#client, {
Bucket: this.options.bucket,
Delimiter: !recursive ? "/" : "",
ContinuationToken: paginationToken,
...prefix !== "/" ? { Prefix: prefix } : {},
...maxResults !== void 0 ? { MaxKeys: maxResults } : {}
})
);
function* filesGenerator() {
if (response.CommonPrefixes) {
for (const directory of response.CommonPrefixes) {
yield new DriveDirectory(directory.Prefix.replace(/\/$/, ""));
}
}
if (response.Contents) {
for (const file of response.Contents) {
yield new DriveFile(file.Key, self, self.#createFileMetaData(file));
}
}
}
return {
paginationToken: response.NextContinuationToken,
objects: {
[Symbol.iterator]: filesGenerator
}
};
}
};
export {
S3Driver
};