@webda/gcp
Version:
Webda GCP Services implementation
302 lines • 10.1 kB
JavaScript
;
import { Storage as GCS } from "@google-cloud/storage";
import { BinaryParameters, CloudBinary, Throttler } from "@webda/core";
import { createReadStream } from "fs";
import * as mime from "mime-types";
import { Readable, Stream } from "stream";
export class StorageParameters extends BinaryParameters {
constructor(params, service) {
super(params, service);
this.prefix = "";
}
}
/**
* Storage handles the storage of binary on a Google Cloud Storage bucket
*
* See Binary the general interface
* @WebdaModda GoogleCloudStorage
*/
export default class Storage extends CloudBinary {
get storage() {
if (!this._storage) {
this._storage = new GCS();
}
return this._storage;
}
/**
* Load the parameters
*
* @param params
*/
loadParameters(params) {
return new StorageParameters(params, this);
}
/**
* @override
*/
async _get(info) {
const key = this._getKey(info.hash);
return this.getContent({ key });
}
/**
* @override
*/
getSignedUrlFromMap(map, expires, _context) {
return this.getSignedUrl({
key: this._getKey(map.hash),
expires,
action: "read"
});
}
/**
* Get object content in Buffer
* @param {StorageObject} params
*/
async getContent({ bucket, key }) {
return this.getStorageBucket(bucket).file(key).createReadStream();
}
/**
* Upload a local file to GCS bucket destination
*
* @param key to add to
* @param body content of the object
* @param metadatas to put along the object
* @param bucket to use
*/
async putObject(key, body, metadata = {}, bucket = this.parameters.bucket
//{ bucket, localPath, content, key }: PutObjectParams
) {
const file = this.getStorageBucket(bucket).file(key);
let contentType = "application/octet-stream";
let stream;
if (body instanceof Readable) {
stream = body;
}
else if (body instanceof Buffer) {
const bufferStream = new Stream.PassThrough();
bufferStream.end(body);
stream = bufferStream;
}
else {
contentType = mime.lookup(body, "application/octet-stream");
stream = createReadStream(body);
}
// NODE16-REFACTOR: const { pipeline } = require('stream/promises');
await new Promise((resolve, reject) => {
stream
.pipe(file.createWriteStream({
contentType
}))
.on("error", reject)
.on("finish", resolve);
});
// Save metadata now
if (metadata) {
await file.setMetadata({ metadata: metadata });
}
}
/**
* Return the Google Bucket directly
*
* @param bucket name of the bucket default to the one predefined
* @returns
*/
getStorageBucket(bucket = this.parameters.bucket) {
return this.storage.bucket(bucket);
}
/**
* @inheritdoc
*/
async _cleanUsage(hash, uuid, property) {
const suffix = property ? `${property}_${uuid}` : `_${uuid}`;
let files = (await this.getStorageBucket().getFiles({ prefix: this._getKey(hash, "") }))[0];
await Promise.all(files.filter(f => f.name.endsWith(suffix)).map(f => f.delete()));
// If no more usage, delete the data
files = files.filter(f => !f.name.endsWith(suffix));
if (files.length == 1) {
await Promise.all(files.map(f => f.delete()));
}
}
/**
* @inheritdoc
*/
async getUsageCount(hash) {
const [files] = await this.getStorageBucket().getFiles({
prefix: this._getKey(hash, "")
});
return files.filter(n => !n.name.endsWith("/data")).length;
}
/**
* @inheritdoc
*/
async store(object, property, file) {
this.checkMap(object, property);
await file.getHashes();
const [exists] = await this.getStorageBucket().file(this._getKey(file.hash)).exists();
// If data already exist no need to upload
if (!exists) {
await this.putObject(this._getKey(file.hash), await file.get(), {
...file.metadata,
challenge: file.challenge
});
}
await this.putMarker(file.hash, `${property}_${object.getUuid()}`, object.getStore().getName());
await this.uploadSuccess(object, property, file);
}
/**
* Delete a file inside an existing bucket
* @param {DeleteObjectParams} params
*/
async deleteObject({ bucket, key }) {
await this.getStorageBucket(bucket).file(key).delete();
this._webda.log("DEBUG", `gs://${bucket}/${key} deleted`);
}
/**
* Move a file from one bucket to an other
* @param {StorageObject} source
* @param {StorageObject} destination
*/
async moveObject(source, destination) {
const newFile = this.getStorageBucket(destination.bucket).file(destination.key);
this._webda.log("DEBUG", `moveObject gs://${source.bucket}/${source.key} => gs://${destination.bucket}/${destination.key}`);
await this.getStorageBucket(source.bucket).file(source.key).move(newFile);
}
/**
* @inheritdoc
*/
async putRedirectUrl(ctx) {
let body = await ctx.getRequestBody();
const { uuid, store, property } = ctx.getParameters();
let targetStore = this._verifyMapAndStore(ctx);
let object = await targetStore.get(uuid);
let base64String = Buffer.from(body.hash, "hex").toString("base64");
let params = {
bucket: this.parameters.bucket,
key: this._getKey(body.hash, "data"),
action: "write",
contentType: "application/octet-stream",
contentMd5: base64String,
extensionHeaders: {
"x-goog-meta-challenge": body.challenge
}
};
// List bucket to check if the file already exist
let challenge;
try {
let res = await this.getStorageBucket().file(params.key).getMetadata();
challenge = res[0].metadata.challenge;
}
catch (err) {
// Ignore error
}
await this.uploadSuccess(object, property, body);
await this.putMarker(body.hash, `${property}_${uuid}`, store);
// If the challenge is the same, no need to upload
if (challenge && challenge === body.challenge) {
return;
}
let url = await this.getSignedUrl(params);
// Re-upload, we should probably queue for recheck
return {
url,
method: "PUT",
headers: {
"Content-MD5": base64String,
"Content-Type": "application/octet-stream",
"x-goog-meta-challenge": body.challenge,
Host: new URL(url).host
}
};
}
/**
* Retrieve one signed URL to download the file in parameter
* @param {SignedUrlParams} params
* @returns {string} URL in order to download the file
*/
async getSignedUrl({ bucket, key, expires = 3600, action = "read", ...params }) {
const options = {
version: "v4",
action,
...params,
expires: Date.now() + expires * 1000
};
const [url] = await this.getStorageBucket(bucket).file(key).getSignedUrl(options);
this._webda.log("TRACE", `The signed url for ${bucket}/${key} is ${url}.`);
return url;
}
/**
* Retrieve mandatory headers if needed by the cloud provider (ie. Azure)
* Returns empty object if no headers are mandatory
* @returns {[key:string]: string} mandatory headers
*/
getSignedUrlHeaders() {
return {};
}
/**
* Retrieve public URL of this object
* @param {StorageModel} store
* @returns {string} Public URL in order to download the file
*/
async getPublicUrl({ bucket, key }) {
return this.getStorageBucket(bucket).file(key).publicUrl();
}
/**
* Fetch an object's metadata
* @param {StorageModel} store
* @returns {any} Metadata
*/
async getMeta({ bucket, key }) {
const [metadata] = await this.getStorageBucket(bucket).file(key).getMetadata();
return {
size: typeof metadata.size === "string" ? parseInt(metadata.size) : metadata.size,
contentType: metadata.contentType
};
}
/**
* Get a bucket size with a prefix
* @param bucket
* @param prefix on the bucket
* @param regex to filter files
*/
async getBucketSize(bucket, prefix, regex) {
let pageToken;
const throttler = new Throttler();
bucket ?? (bucket = this.parameters.bucket);
let size = 0;
let count = 0;
const dwl = f => {
return async () => {
try {
const metadata = (await f.getMetadata()).shift();
size += Number.parseInt(metadata.size);
count++;
}
catch (err) { }
};
};
do {
let [files, page, _] = await this.storage.bucket(bucket).getFiles({ maxResults: 1000, pageToken, prefix });
files.filter(f => (regex ? f.name.match(regex) : true)).forEach(f => throttler.queue(dwl(f)));
await throttler.wait();
pageToken = page?.pageToken;
} while (pageToken);
return {
size,
count
};
}
/**
* @inheritdoc
*/
async putMarker(hash, uuid, storeName) {
await this.getStorageBucket()
.file(this._getKey(hash, uuid))
.save("", {
metadata: {
webdaStore: storeName
}
});
}
}
export { Storage };
//# sourceMappingURL=storage.js.map