@webda/core
Version:
Expose API with Lambda
999 lines • 33.2 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
import * as crypto from "crypto";
import * as fs from "fs";
import * as mime from "mime-types";
import * as path from "path";
import { Readable } from "stream";
import { Core, Counter, WebdaError } from "../index.js";
import { NotEnumerable } from "../models/coremodel.js";
import { Store } from "../stores/store.js";
import { Service, ServiceParameters } from "./service.js";
/**
* Emitted if binary does not exist
*/
export class BinaryNotFoundError extends WebdaError.CodeError {
constructor(hash, storeName) {
super("BINARY_NOTFOUND", `Binary not found ${hash} BinaryService(${storeName})`);
}
}
/**
* Represent a file to store
* @WebdaSchema
*/
export class BinaryFile {
constructor(info) {
this.set(info);
}
/**
* Set the information
* @param info
*/
set(info) {
this.name = info.name;
this.challenge = info.challenge;
this.hash = info.hash;
this.mimetype = info.mimetype || "application/octet-stream";
this.metadata = info.metadata || {};
}
/**
* Retrieve a plain BinaryFileInfo object
* @returns
*/
toBinaryFileInfo() {
return {
hash: this.hash,
size: this.size,
mimetype: this.mimetype,
metadata: this.metadata,
challenge: this.challenge,
// Fallback on original name
name: this.name || this.originalname
};
}
/**
* Create hashes
* @param buffer
* @returns
*/
async getHashes() {
if (!this.hash) {
// Using MD5 as S3 content verification use md5
const hash = crypto.createHash("md5");
const challenge = crypto.createHash("md5");
const stream = await this.get();
challenge.update("WEBDA");
await new Promise((resolve, reject) => {
stream.on("error", err => reject(err));
stream.on("end", () => {
this.hash = hash.digest("hex");
this.challenge = challenge.digest("hex");
resolve();
});
stream.on("data", chunk => {
let buffer = Buffer.from(chunk);
hash.update(buffer);
challenge.update(buffer);
});
});
}
return {
hash: this.hash,
challenge: this.challenge
};
}
}
export class LocalBinaryFile extends BinaryFile {
constructor(filePath) {
super({
name: path.basename(filePath),
size: fs.statSync(filePath).size,
mimetype: mime.lookup(filePath) || "application/octet-stream"
});
this.path = filePath;
}
/**
* @override
*/
async get() {
return fs.createReadStream(this.path);
}
}
export class MemoryBinaryFile extends BinaryFile {
constructor(buffer, info = {}) {
super({
...info,
size: info.size || buffer.length,
name: info.name || "data.bin",
mimetype: info.mimetype || "application/octet-stream"
});
this.buffer = typeof buffer === "string" ? Buffer.from(buffer) : buffer;
}
/**
* @override
*/
async get() {
return Readable.from(this.buffer);
}
}
/**
* This is a map used to retrieve binary
*
* @class BinaryMap
*/
export class BinaryMap extends BinaryFile {
constructor(service, obj) {
super(obj);
this.set(obj);
this.__store = service;
}
/**
* Get the binary data
*
* @returns
*/
get() {
return this.__store.get(this);
}
/**
* Get into a buffer
*/
async getAsBuffer() {
return BinaryService.streamToBuffer(await this.get());
}
/**
* Download the binary to a path
*
* Shortcut to call {@link Binary.downloadTo} with current object
*
* @param filename
*/
async downloadTo(filename) {
return this.__store.downloadTo(this, filename);
}
/**
* Set the http context
* @param ctx
*/
setContext(ctx) {
this.__ctx = ctx;
}
}
__decorate([
NotEnumerable
], BinaryMap.prototype, "__ctx", void 0);
__decorate([
NotEnumerable
], BinaryMap.prototype, "__store", void 0);
/**
* One Binary instance
*/
export class Binary extends BinaryMap {
constructor(attribute, model) {
super(Core.get().getBinaryStore(model, attribute), model[attribute] || {});
this.empty = model[attribute] === undefined;
this.attribute = attribute;
this.model = model;
}
/**
* isEmpty
* @returns
*/
isEmpty() {
return this.empty;
}
/**
* Ensure empty is set correctly
* @param info
*/
set(info) {
super.set(info);
this.empty = false;
}
/**
* Replace the binary
* @param id
* @param ctx
* @returns
*/
async upload(file) {
await this.__store.store(this.model, this.attribute, file);
this.set(file);
}
/**
* Delete the binary, if you need to replace just use upload
*/
async delete() {
await this.__store.delete(this.model, this.attribute);
this.set({});
}
/**
* Return undefined if no hash
* @returns
*/
toJSON() {
if (!this.hash) {
return undefined;
}
return this;
}
}
__decorate([
NotEnumerable
], Binary.prototype, "model", void 0);
__decorate([
NotEnumerable
], Binary.prototype, "attribute", void 0);
__decorate([
NotEnumerable
], Binary.prototype, "empty", void 0);
/**
* Define a Binary map stored in a Binaries collection
*/
export class BinariesItem extends BinaryMap {
constructor(parent, info) {
super(parent.__service, info);
this.parent = parent;
}
/**
* Replace the binary
* @param id
* @param ctx
* @returns
*/
async upload(file) {
await this.parent.upload(file, this);
this.set(file);
}
/**
* Delete the binary, if you need to replace just use upload
*/
async delete() {
return this.parent.delete(this);
}
}
__decorate([
NotEnumerable
], BinariesItem.prototype, "parent", void 0);
/**
* Define a collection of Binary
*/
export class BinariesImpl extends Array {
assign(model, attribute) {
this.model = model;
this.attribute = attribute;
for (let binary of model[attribute] || []) {
this.push(binary);
}
this.__service = Core.get().getBinaryStore(model, attribute);
return this;
}
// Readonly methods
pop() {
throw new Error("Readonly");
}
slice() {
throw new Error("Readonly");
}
unshift() {
throw new Error("Readonly");
}
shift() {
throw new Error("Readonly");
}
push(...args) {
return super.push(...args.map(arg => (arg instanceof BinariesItem ? arg : new BinariesItem(this, arg))));
}
/**
* Upload a file to this model
* @param file
*/
async upload(file, replace) {
await this.__service.store(this.model, this.attribute, file);
// Should call the store first
super.push(new BinariesItem(this, file));
if (replace) {
await this.delete(replace);
}
}
/**
* Delete an item
* @param item
*/
async delete(item) {
let itemIndex = this.indexOf(item);
if (itemIndex === -1) {
throw new Error("Item not found");
}
await this.__service.delete(this.model, this.attribute, itemIndex);
itemIndex = this.indexOf(item);
if (itemIndex >= 0) {
// Call store delete here
this.splice(itemIndex, 1);
}
}
}
__decorate([
NotEnumerable
], BinariesImpl.prototype, "__service", void 0);
__decorate([
NotEnumerable
], BinariesImpl.prototype, "model", void 0);
__decorate([
NotEnumerable
], BinariesImpl.prototype, "attribute", void 0);
export class BinaryParameters extends ServiceParameters {
constructor(params, _service) {
super(params);
if (this.expose) {
this.expose.restrict = this.expose.restrict || {};
}
this.map ?? (this.map = {});
// Store all models in it by default
this.models ?? (this.models = {
"*": ["*"]
});
}
}
/**
* This is an abstract service to represent a storage of files
* The binary allow you to expose this service as HTTP
*
* It supports two modes:
* - attached to a CoreModel (attach, detach, reattach)
* - pure storage with no managed id (read, write, delete)
*
* As we have deduplication builtin you can get some stats
* - getUsageCount(hash)
* - getUsageCountForRaw()
* - getUsageCountForMap()
*
* The Binary storage should store only once a binary and reference every object that are used by this binary, so it can be cleaned.
*
*
* @see FileBinary
* @see S3Binary
*
* @exports
* @abstract
* @WebdaModda Binary
*/
export class BinaryService extends Service {
/**
* @override
*/
initMetrics() {
super.initMetrics();
this.metrics.upload = this.getMetric(Counter, {
name: "binary_upload",
help: "Number of binary upload"
});
this.metrics.delete = this.getMetric(Counter, {
name: "binary_delete",
help: "Number of binary deleted"
});
this.metrics.download = this.getMetric(Counter, {
name: "binary_download",
help: "Number of binary upload"
});
this.metrics.metadataUpdate = this.getMetric(Counter, {
name: "binary_metadata_update",
help: "Number of binary metadata updated"
});
}
/**
* Redirect to the temporary link to S3 object
* or return it if returnInfo=true
*
* @param ctx of the request
* @param returnInfo
*/
async httpGet(context) {
const returnInfo = context.getHttpContext().getRelativeUri().endsWith("/url");
const { uuid, index, property } = context.getParameters();
let targetStore = this._verifyMapAndStore(context);
let object = await targetStore.get(uuid);
if (!object || (Array.isArray(object[property]) && object[property].length <= index)) {
throw new WebdaError.NotFound("Object does not exist or attachment does not exist");
}
await object.checkAct(context, "get_binary");
const file = Array.isArray(object[property]) ? object[property][index] : object[property];
await this.emitSync("Binary.Get", {
object: file,
service: this,
context: context
});
let url = await this.getRedirectUrlFromObject(file, context);
// No url, we return the file
if (url === null) {
if (returnInfo) {
// Redirect to same url without /url
context.write({
Location: context
.getHttpContext()
.getAbsoluteUrl()
.replace(/\/url$/, ""),
Map: file
});
}
else {
// Output
context.writeHead(200, {
"Content-Type": file.mimetype === undefined ? "application/octet-steam" : file.mimetype,
"Content-Length": file.size
});
let readStream = await this.get(file);
await new Promise((resolve, reject) => {
// We replaced all the event handlers with a simple call to readStream.pipe()
context._stream.on("finish", resolve);
context._stream.on("error", reject);
readStream.pipe(context._stream);
});
}
}
else if (returnInfo) {
context.write({ Location: url, Map: file });
}
else {
context.writeHead(302, {
Location: url
});
}
}
/**
* Get a UrlFromObject
*
*/
async getRedirectUrlFromObject(binaryMap, _context, _expires = 30) {
return null;
}
/**
* Define if binary is managed by the store
* @param modelName
* @param attribute
* @returns -1 if not managed, 0 if managed but by default, 1 if managed and in the map, 2 if explicit with attribute and model
*/
handleBinary(modelName, attribute) {
let key = Object.keys(this.parameters.models).find(k => k === modelName);
if (key) {
// Explicit model
let attributes = this.parameters.models[key];
if (attributes.includes(attribute)) {
return 2;
}
else if (attributes.includes("*")) {
return 1;
}
}
// Default to all model - 593-594,598-599
key = Object.keys(this.parameters.models).find(k => k === "*");
if (!key) {
return -1;
}
let attributes = this.parameters.models[key];
if (attributes.includes(attribute)) {
return 1;
}
if (attributes.includes("*")) {
return 0;
}
return -1;
}
/**
* Get a binary
*
* @param {Object} info The reference stored in your target object
* @emits 'binaryGet'
*/
async get(info) {
await this.emitSync("Binary.Get", {
object: info,
service: this
});
this.metrics.download.inc();
return this._get(info);
}
/**
* Download a binary to a file
*
* @param {Object} info The reference stored in your target object
* @param {String} filepath to save the binary to
*/
async downloadTo(info, filename) {
await this.emitSync("Binary.Get", {
object: info,
service: this
});
this.metrics.download.inc();
let readStream = await this._get(info);
let writeStream = fs.createWriteStream(filename);
return new Promise((resolve, reject) => {
writeStream.on("finish", _src => {
return resolve();
});
writeStream.on("error", src => {
try {
fs.unlinkSync(filename);
// Stubing the fs module in ESM seems complicated for now
/* c8 ignore next 3 */
}
catch (err) {
this._webda.log("ERROR", err);
}
return reject(src);
});
readStream.pipe(writeStream);
});
}
/**
* @override
*/
resolve() {
super.resolve();
this.initMap(this.parameters.map);
return this;
}
/**
* Init the declared maps, adding reverse maps
*
* @param map
*/
initMap(map) {
if (map == undefined || map._init) {
return;
}
this._lowercaseMaps = {};
Object.keys(map).forEach(prop => {
this._lowercaseMaps[prop.toLowerCase()] = prop;
let reverseStore = this._webda.getService(prop);
if (reverseStore === undefined || !(reverseStore instanceof Store)) {
this._webda.log("WARN", "Can't setup mapping as store ", prop, " doesn't exist");
map[prop]["-onerror"] = "NoStore";
return;
}
for (let i in map[prop]) {
reverseStore.addReverseMap(map[prop][i], this);
}
// Cascade delete
reverseStore.on("Store.Deleted", async (evt) => {
let infos = [];
if (evt.object[map[prop]]) {
infos.push(...evt.object[map[prop]]);
}
await Promise.all(infos.map(info => this.cascadeDelete(info, evt.object.getUuid())));
});
});
}
/**
* Based on the raw Map init a BinaryMap
* @param obj
* @returns
*/
newModel(obj) {
return new BinaryMap(this, obj);
}
/**
* Read a stream to a buffer
*
* @param stream
* @returns
*/
static streamToBuffer(stream) {
// codesnippet from https://stackoverflow.com/questions/14269233/node-js-how-to-read-a-stream-into-a-buffer
const chunks = [];
return new Promise((resolve, reject) => {
stream.on("data", chunk => chunks.push(Buffer.from(chunk)));
stream.on("error", err => reject(err));
stream.on("end", () => resolve(Buffer.concat(chunks)));
});
}
/**
* Check if a map is defined
*
* @param name
* @param property
*/
checkMap(object, property) {
if (this.handleBinary(object.__type, property) !== -1) {
return;
}
// DELETE_IN_4.0.0
const name = object.getStore().getName();
let map = this.parameters.map[this._lowercaseMaps[name.toLowerCase()]];
if (map === undefined) {
throw new Error("Unknown mapping");
}
if (Array.isArray(map) && map.indexOf(property) === -1) {
throw new Error("Unknown mapping");
}
/*
// END_DELETE_IN_4.0.0
throw new Error("Unknown mapping");
// DELETE_IN_4.0.0
*/
// END_DELETE_IN_4.0.0
}
/**
* Ensure events are sent correctly after an upload and update the BinaryFileInfo in targetted object
*/
async uploadSuccess(object, property, fileInfo) {
let file;
// Ensure we do not have a full object
if (fileInfo["toBinaryFileInfo"] && typeof fileInfo["toBinaryFileInfo"] === "function") {
file = fileInfo["toBinaryFileInfo"]();
}
else {
file = fileInfo;
}
let additionalAttrs = Object.keys(file).filter(k => !["name", "size", "mimetype", "hash", "challenge", "metadata"].includes(k));
if (additionalAttrs.length > 0) {
throw new Error("Invalid file object it should be a plain BinaryFileInfo found additional properties: " +
additionalAttrs.join(","));
}
let object_uid = object.getUuid();
// Check if the file is already in the array then skip
if (Array.isArray(object[property]) && object[property].find(i => i.hash === file.hash)) {
return;
}
await this.emitSync("Binary.UploadSuccess", {
object: file,
service: this,
target: object
});
const relations = this.getWebda().getApplication().getRelations(object);
const cardinality = (relations.binaries || []).find(p => p.attribute === property)?.cardinality || "MANY";
if (cardinality === "MANY") {
await object.getStore().upsertItemToCollection(object_uid, property, file);
}
else {
await object.getStore().setAttribute(object_uid, property, file);
}
await this.emitSync("Binary.Create", {
object: file,
service: this,
target: object
});
this.metrics.upload.inc();
}
/**
*
* @param targetStore
* @param object
* @param property
* @param index
* @returns
*/
async deleteSuccess(object, property, index) {
let info = (index !== undefined ? object[property][index] : object[property]);
const relations = this.getWebda().getApplication().getRelations(object);
const cardinality = (relations.binaries || []).find(p => p.attribute === property)?.cardinality || "MANY";
let update;
if (cardinality === "MANY") {
update = object.getStore().deleteItemFromCollection(object.getUuid(), property, index, info.hash, "hash");
}
else {
object.getStore().removeAttribute(object.getUuid(), property);
}
await this.emitSync("Binary.Delete", {
object: info,
service: this
});
this.metrics.delete.inc();
return update;
}
/**
* Get file either from multipart post or raw
* @param req
* @returns
*/
async _getFile(req) {
let file = await req.getHttpContext().getRawBody(10 * 1024 * 1024);
// TODO Check if we have other type
return new MemoryBinaryFile(Buffer.from(file), {
mimetype: req.getHttpContext().getUniqueHeader("Content-Type", "application/octet-stream"),
size: parseInt(req.getHttpContext().getUniqueHeader("Content-Length")) || file.length,
name: req.getHttpContext().getUniqueHeader("X-Filename", "")
});
}
/**
* @override
*/
initRoutes() {
if (!this.parameters.expose) {
return;
}
this._initRoutes();
}
/**
* Init the Binary system routes
*
* Making sure parameters.expose exists prior
*/
_initRoutes() {
let url;
let name = this.getOperationName();
if (!this.parameters.expose.restrict.get) {
url = this.parameters.expose.url + "/{store}/{uuid}/{property}/{index}";
this.addRoute(url, ["GET"], this.httpGet, {
get: {
operationId: `get${name}Binary`,
description: "Download a binary linked to an object",
summary: "Download a binary",
responses: {
"200": {
description: "Binary stream"
},
"403": {
description: "You don't have permissions"
},
"404": {
description: "Object does not exist or attachment does not exist"
},
"412": {
description: "Provided hash does not match"
}
}
}
});
this.addRoute(url + "/url", ["GET"], this.httpGet, {
get: {
operationId: `get${name}BinaryInfo`,
description: "Return the url and information of a binary linked to an object",
summary: "GetInfo of a binary",
responses: {
"200": {
description: "Url and BinaryMap"
},
"403": {
description: "You don't have permissions"
},
"404": {
description: "Object does not exist or attachment does not exist"
},
"412": {
description: "Provided hash does not match"
}
}
}
});
}
if (!this.parameters.expose.restrict.create) {
// No need the index to add file
url = this.parameters.expose.url + "/{store}/{uuid}/{property}";
this.addRoute(url, ["POST"], this.httpRoute, {
post: {
operationId: `add${name}Binary`,
description: "Add a binary linked to an object",
summary: "Add a binary",
responses: {
"200": {
description: ""
},
"403": {
description: "You don't have permissions"
},
"404": {
description: "Object does not exist or attachment does not exist"
},
"412": {
description: "Provided hash does not match"
}
}
}
});
}
if (!this.parameters.expose.restrict.create) {
// Add file with challenge
url = this.parameters.expose.url + "/upload/{store}/{uuid}/{property}";
this.addRoute(url, ["PUT"], this.httpChallenge, {
put: {
operationId: `put${name}Binary`,
description: "Add a binary to an object after challenge",
summary: "Add a binary",
responses: {
"204": {
description: ""
},
"403": {
description: "You don't have permissions"
},
"404": {
description: "Object does not exist or attachment does not exist"
},
"412": {
description: "Provided hash does not match"
}
}
}
});
}
if (!this.parameters.expose.restrict.delete) {
// Need hash to avoid concurrent delete
url = this.parameters.expose.url + "/{store}/{uuid}/{property}/{index}/{hash}";
this.addRoute(url, ["DELETE"], this.httpRoute, {
delete: {
operationId: `delete${name}Binary`,
description: "Delete a binary linked to an object",
summary: "Delete a binary",
responses: {
"204": {
description: ""
},
"403": {
description: "You don't have permissions"
},
"404": {
description: "Object does not exist or attachment does not exist"
},
"412": {
description: "Provided hash does not match"
}
}
}
});
}
if (!this.parameters.expose.restrict.metadata) {
// Need hash to avoid concurrent delete
url = this.parameters.expose.url + "/{store}/{uuid}/{property}/{index}/{hash}";
this.addRoute(url, ["PUT"], this.httpRoute, {
put: {
operationId: `update${name}BinaryMetadata`,
description: "Update a binary metadata linked to an object",
summary: "Update a binary metadata",
responses: {
"204": {
description: ""
},
"403": {
description: "You don't have permissions"
},
"404": {
description: "Object does not exist or attachment does not exist"
},
"412": {
description: "Provided hash does not match"
}
}
}
});
}
}
/**
* Return the name of the service for OpenAPI
* @returns
*/
getOperationName() {
return this._name.toLowerCase() === "binary" ? "" : this._name;
}
/**
* Based on the request parameter verify it match a known mapping
* @param ctx
* @returns
*/
_verifyMapAndStore(ctx) {
// Check for model
if (ctx.parameter("model")) {
if (this.handleBinary(ctx.parameter("model"), ctx.parameter("property")) === -1) {
throw new WebdaError.NotFound("Model not managed by this store");
}
return this.getWebda().getModelStore(this.getWebda().getModel(ctx.parameter("model")));
}
let store = ctx.parameter("store").toLowerCase();
// To avoid any problem lowercase everything
let map = this.parameters.map[this._lowercaseMaps[store]];
if (map === undefined) {
throw new WebdaError.NotFound("Unknown map");
}
if (!map.includes(ctx.parameter("property"))) {
throw new WebdaError.NotFound("Unknown property");
}
let targetStore = this.getService(this._lowercaseMaps[store]);
if (targetStore === undefined) {
throw new WebdaError.NotFound("Unknown store");
}
return targetStore;
}
/**
* By default no challenge is managed so throws 404
*
* @param ctx
*/
async putRedirectUrl(_ctx) {
// Dont handle the redirect url
throw new WebdaError.NotFound("No redirect url");
}
/**
* Mechanism to add a data based on challenge
*/
async httpChallenge(ctx) {
let body = await ctx.getRequestBody();
if (!body.hash || !body.challenge) {
throw new WebdaError.BadRequest("Missing hash or challenge");
}
// First verify if map exist
let targetStore = this._verifyMapAndStore(ctx);
// Get the object
let object = await targetStore.get(ctx.parameter("uuid"), ctx);
if (object === undefined) {
throw new WebdaError.NotFound("Object does not exist");
}
await object.checkAct(ctx, "attach_binary");
let url = await this.putRedirectUrl(ctx);
let base64String = Buffer.from(body.hash, "hex").toString("base64");
ctx.write({
...url,
done: url === undefined,
md5: base64String
});
}
/**
* Manage the different routes
* @param ctx
*/
async httpRoute(ctx) {
// First verify if map exist
let targetStore = this._verifyMapAndStore(ctx);
// Get the object
let object = await targetStore.get(ctx.parameter("uuid"), ctx);
if (object === undefined) {
throw new WebdaError.NotFound("Object does not exist");
}
const { property, index } = ctx.getParameters();
// Current file - would be empty on creation
const file = Array.isArray(object[property]) ? object[property][index] : object[property];
// Check permissions
let action = "unknown";
if (ctx.getHttpContext().getMethod() === "DELETE") {
action = "detach_binary";
}
else if (ctx.getHttpContext().getMethod() === "POST") {
action = "attach_binary";
}
else if (ctx.getHttpContext().getMethod() === "PUT") {
action = "update_binary_metadata";
}
await object.checkAct(ctx, action);
// Now do the action
if (ctx.getHttpContext().getMethod() === "POST") {
await this.store(object, property, await this._getFile(ctx));
}
else {
if (file.hash !== ctx.parameter("hash")) {
throw new WebdaError.BadRequest("Hash does not match");
}
if (ctx.getHttpContext().getMethod() === "DELETE") {
await this.delete(object, property, index);
}
else if (ctx.getHttpContext().getMethod() === "PUT") {
let metadata = await ctx.getRequestBody();
// Limit metadata to 4kb
if (JSON.stringify(metadata).length >= 4096) {
throw new WebdaError.BadRequest("Metadata is too big: 4kb max");
}
let evt = {
service: this,
object: file,
target: object
};
await this.emitSync("Binary.MetadataUpdate", {
...evt,
metadata
});
file.metadata = metadata;
// Update mapper on purpose
await object.getStore().patch({
[object.__class.getUuidField()]: object.getUuid(),
[property]: object[property]
}, false);
this.metrics.metadataUpdate.inc();
await this.emitSync("Binary.MetadataUpdated", evt);
}
}
}
}
//# sourceMappingURL=binary.js.map