@webda/aws
Version:
Webda AWS Services implementation
355 lines • 12.2 kB
JavaScript
// Load the AWS SDK for Node.js
import { GetObjectCommand, PutObjectCommand, S3 } from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import { BinaryParameters, BinaryService, CloudBinary, WebdaError } from "@webda/core";
import bluebird from "bluebird";
import { AWSServiceParameters } from "./aws-mixin.js";
export class S3BinaryParameters extends AWSServiceParameters(BinaryParameters) {
constructor(params, service) {
super(params, service);
if (!this.bucket) {
throw new WebdaError.CodeError("S3BUCKET_PARAMETER_REQUIRED", "Need to define a bucket at least");
}
this.forcePathStyle ?? (this.forcePathStyle = false);
this.prefix = "";
}
}
/**
* S3Binary handles the storage of binary on a S3 bucket
*
* The structure used for now is
* /{hash}/data
* /{hash}/{targetStore}_{uuid}
* The challenge is stored on the metadata of the data object
*
* It takes parameters
* bucket: "bucketName"
* accessKeyId: ""
* secretAccessKey: ""
* region: ""
*
* See Binary the general interface
*
* We can register on S3 Event to get info when /data is pushed
*
* @WebdaModda
*/
export default class S3Binary extends CloudBinary {
/**
* Load the parameters
*
* @param params
*/
loadParameters(params) {
return new S3BinaryParameters(params, this);
}
/**
* @inheritdoc
*/
computeParameters() {
this._s3 = new S3(this.parameters);
}
/**
* @inheritdoc
*/
async putRedirectUrl(ctx) {
let body = await ctx.getRequestBody();
const { uuid, store, property } = ctx.getParameters();
let targetStore = this._verifyMapAndStore(ctx);
let object = await targetStore.get(uuid);
let base64String = Buffer.from(body.hash, "hex").toString("base64");
let params = {
Bucket: this.parameters.bucket,
Key: this._getKey(body.hash),
ContentType: "application/octet-stream",
ContentMD5: base64String
};
// List bucket
let data = await this._s3.listObjectsV2({ Bucket: this.parameters.bucket, Prefix: this._getKey(body.hash, "") });
let foundMap = false;
let foundData = false;
let challenge;
for (let i in data.Contents) {
if (data.Contents[i].Key.endsWith("data"))
foundData = true;
if (data.Contents[i].Key.endsWith(`${property}_${uuid}`))
foundMap = true;
if (data.Contents[i].Key.split("/").pop().startsWith("challenge_")) {
challenge = data.Contents[i].Key.split("/").pop().substring("challenge_".length);
}
}
const headers = { "Content-MD5": base64String, "Content-Type": "application/octet-stream" };
if (foundMap) {
if (foundData)
return;
return { url: await this.getSignedUrl(params.Key, "putObject", params), method: "PUT", headers };
}
if (foundData) {
if (challenge) {
// challenge and data prove it exists
if (challenge === body.challenge) {
await this.uploadSuccess(object, property, body);
return;
}
}
// Need to do something?
}
else {
await this.putMarker(body.hash, `challenge_${body.challenge}`, "challenge");
}
await this.uploadSuccess(object, property, body);
await this.putMarker(body.hash, `${property}_${uuid}`, store);
return { url: await this.getSignedUrl(params.Key, "putObject", params), method: "PUT", headers };
}
/**
* @inheritdoc
*/
putMarker(hash, suffix, storeName) {
let s3obj = new S3(this.parameters);
return s3obj.putObject({
Bucket: this.parameters.bucket,
Key: this._getKey(hash, suffix),
Metadata: { "x-amz-meta-store": storeName }
});
}
/**
* Return a signed url to an object
*
* @param key to the object
* @param action to perform
* @param params
* @returns
*/
async getSignedUrl(key, action = "getObject", params = {}) {
params.Bucket = params.Bucket || this.parameters.bucket;
params.Key = key;
let command;
if (action === "getObject") {
command = new GetObjectCommand(params);
}
else if (action === "putObject") {
command = new PutObjectCommand(params);
}
// Create new client as the implementation is an ugly middleware
return getSignedUrl(new S3(this.parameters), command, params);
}
/**
* @override
*/
async getSignedUrlFromMap(binaryMap, expire, _context) {
let params = {};
params.Expires = expire; // A get should not take more than 30s
params.ResponseContentDisposition = `attachment; filename=${binaryMap.name || binaryMap.originalname}`;
params.ResponseContentType = binaryMap.mimetype;
// Access-Control-Allow-Origin
return this.getSignedUrl(this._getKey(binaryMap.hash), "getObject", params);
}
/**
* @override
*/
async _get(info) {
return (await this._s3.getObject({ Bucket: this.parameters.bucket, Key: this._getKey(info.hash) })).Body;
}
/**
* Check if an object exists on S3
* @param key
* @param bucket
*/
async exists(Key, Bucket = this.parameters.bucket) {
try {
return await this._s3.headObject({ Bucket, Key });
}
catch (err) {
if (err.name === "NotFound") {
return null;
}
throw err;
}
}
/**
* @inheritdoc
*/
async getUsageCount(hash) {
// Not efficient if more than 1000 docs
let data = await this._s3.listObjects({ Bucket: this.parameters.bucket, Prefix: this._getKey(hash, "") });
data.Contents ?? (data.Contents = []);
return data.Contents.filter(k => !(k.Key.includes("data") || k.Key.includes("challenge"))).length;
}
/**
* @inheritdoc
*/
async _cleanHash(hash) {
let files = (await this._s3.listObjectsV2({ Bucket: this.parameters.bucket, Prefix: this._getKey(hash, "") }))
.Contents;
await bluebird.map(files, file => this._s3.deleteObject({ Bucket: this.parameters.bucket, Key: file.Key }), {
concurrency: 5
});
}
/**
* @inheritdoc
*/
async _cleanUsage(hash, uuid, attribute) {
if (!attribute) {
return this._cleanHash(hash);
}
// Dont clean data for now
let params = { Bucket: this.parameters.bucket, Key: this._getKey(hash, `${attribute}_${uuid}`) };
await this._s3.deleteObject(params);
}
/**
* @inheritdoc
*/
async _exists(hash) {
try {
await this._s3.headObject({ Bucket: this.parameters.bucket, Key: this._getKey(hash) });
return true;
}
catch (err) {
if (err.name !== "NotFound") {
throw err;
}
}
return false;
}
/**
* Get a head object
* @param hash
* @returns
*/
async _getS3(hash) {
try {
return await this._s3.headObject({ Bucket: this.parameters.bucket, Key: this._getKey(hash) });
}
catch (err) {
if (err.name !== "NotFound") {
throw err;
}
}
}
/**
* Get an object from s3 bucket
*
* @param key to get
* @param bucket to retrieve from or default bucket
* @returns
*/
async getObject(key, bucket) {
bucket = bucket || this.parameters.bucket;
let s3obj = new S3(this.parameters);
return (await s3obj.getObject({ Bucket: bucket, Key: key })).Body;
}
/**
*
* @param Bucket to iterate on
* @param Prefix to use
* @param callback to execute with each key
* @param filter regexp to execute on the key
*/
async forEachFile(Bucket, callback, Prefix = "", filter = undefined) {
let params = { Bucket, Prefix };
let page = 0;
let s3 = new S3(this.parameters);
do {
await s3.listObjectsV2(params).then(async ({ Contents, NextContinuationToken }) => {
params.ContinuationToken = NextContinuationToken;
for (let f in Contents) {
let { Key } = Contents[f];
if (filter && filter.exec(Key) === null) {
continue;
}
await callback(Key, page);
}
});
page++;
} while (params.ContinuationToken);
}
/**
* Add an object to S3 bucket
*
* @param key to add to
* @param body content of the object
* @param metadatas to put along the object
* @param bucket to use
*/
async putObject(key, body, metadatas = {}, bucket = this.parameters.bucket) {
let s3obj = new S3(this.parameters);
await s3obj.putObject({ Bucket: bucket, Key: key, Metadata: metadatas, Body: body });
}
/**
* @inheritdoc
*/
async store(object, property, file) {
this.checkMap(object, property);
await file.getHashes();
let data = await this._getS3(file.hash);
if (data === undefined) {
let s3metas = {};
s3metas["x-amz-meta-challenge"] = file.challenge;
let s3obj = new S3(this.parameters);
await s3obj.putObject({
Bucket: this.parameters.bucket,
Key: this._getKey(file.hash),
Metadata: s3metas,
Body: await BinaryService.streamToBuffer(await file.get())
});
}
// Set challenge aside for now
await this.putMarker(file.hash, `challenge_${file.challenge}`, "challenge");
await this.putMarker(file.hash, `${property}_${object.getUuid()}`, object.getStore().getName());
await this.uploadSuccess(object, property, file);
}
/**
* @inheritdoc
*/
getARNPolicy(_accountId) {
return {
Sid: this.constructor.name + this._name,
Effect: "Allow",
Action: [
"s3:AbortMultipartUpload",
"s3:DeleteObject",
"s3:DeleteObjectVersion",
"s3:GetObject",
"s3:GetObjectAcl",
"s3:GetObjectTagging",
"s3:GetObjectTorrent",
"s3:GetObjectVersion",
"s3:GetObjectVersionAcl",
"s3:GetObjectVersionTagging",
"s3:GetObjectVersionTorrent",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListBucketVersions",
"s3:ListMultipartUploadParts",
"s3:PutBucketAcl",
"s3:PutObject",
"s3:PutObjectAcl",
"s3:RestoreObject"
],
Resource: [`arn:aws:s3:::${this.parameters.bucket}`, `arn:aws:s3:::${this.parameters.bucket}/*`]
};
}
/**
* @inheritdoc
*/
getCloudFormation(deployer) {
if (this.parameters.CloudFormationSkip) {
return {};
}
let resources = {};
this.parameters.CloudFormation = this.parameters.CloudFormation || {};
this.parameters.CloudFormation.Bucket = this.parameters.CloudFormation.Bucket || {};
resources[this._name + "Bucket"] = {
Type: "AWS::S3::Bucket",
Properties: {
...this.parameters.CloudFormation.Bucket,
BucketName: this.parameters.bucket,
Tags: deployer.getDefaultTags(this.parameters.CloudFormation.Bucket.Tags)
}
};
// Add any Other resources with prefix of the service
return resources;
}
}
export { S3Binary };
//# sourceMappingURL=s3binary.js.map