@nu-art/file-upload
Version:
File Uploader - Express & Typescript based backend framework
244 lines (243 loc) • 13.6 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ModuleBE_AssetsDB = exports.ModuleBE_AssetsDB_Class = exports.fileSizeValidator = exports.DefaultMimetypeValidator = void 0;
/*
* Permissions management system, define access level for each of
* your server apis, and restrict users by giving them access levels
*
* Copyright (C) 2020 Adam van der Kruk aka TacB0sS
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const ts_common_1 = require("@nu-art/ts-common");
const ModuleBE_AssetsTemp_1 = require("./ModuleBE_AssetsTemp");
const backend_1 = require("@nu-art/thunderstorm/backend");
const file_type_1 = require("file-type");
const ModuleBE_AssetsStorage_1 = require("./ModuleBE_AssetsStorage");
const shared_1 = require("../../shared");
const messages_1 = require("../core/messages");
const ModuleBE_AssetsDeleted_1 = require("./ModuleBE_AssetsDeleted");
const DefaultMimetypeValidator = async (file, doc) => {
const buffer = await file.read();
const fileType = await (0, file_type_1.fromBuffer)(buffer);
if (!fileType)
throw new ts_common_1.ImplementationMissingException(`No validator defined for asset of mimetype: ${doc.mimeType}`);
if (fileType.mime !== doc.mimeType)
throw new ts_common_1.BadImplementationException(`Original mimetype (${doc.mimeType}) does not match the resolved mimetype: (${fileType.mime})`);
return fileType;
};
exports.DefaultMimetypeValidator = DefaultMimetypeValidator;
const fileSizeValidator = async (file, metadata, minSizeInBytes = 0, maxSizeInBytes = ts_common_1.MB) => {
if (!metadata.size)
throw new ts_common_1.ThisShouldNotHappenException(`could not resolve metadata.size for file: ${file.path}`);
return metadata.size >= minSizeInBytes && metadata.size <= maxSizeInBytes;
};
exports.fileSizeValidator = fileSizeValidator;
class ModuleBE_AssetsDB_Class extends backend_1.ModuleBE_BaseDB {
constructor() {
super(shared_1.DBDef_Assets);
this.mimeTypeValidator = {};
this.fileValidator = {};
this.register = (key, validationConfig) => {
if (this.fileValidator[key] && this.fileValidator[key] !== validationConfig)
throw new ts_common_1.BadImplementationException(`File Validator already exists for key: ${key}`);
this.fileValidator[key] = validationConfig;
};
this.cleanup = async (interval = ts_common_1.Hour, module = ModuleBE_AssetsTemp_1.ModuleBE_AssetsTemp) => {
const entries = await module.query.custom({ where: { timestamp: { $lt: (0, ts_common_1.currentTimeMillis)() - interval } } });
await Promise.all(entries.map(async (dbAsset) => {
const file = await ModuleBE_AssetsStorage_1.ModuleBE_AssetsStorage.getFile(dbAsset);
if (!(await file.exists()))
return;
await file.delete();
}));
await module.delete.query({ where: { timestamp: { $lt: (0, ts_common_1.currentTimeMillis)() - interval } } });
};
this.getUrl = async (files) => {
var _a;
const bucketName = (_a = this.config) === null || _a === void 0 ? void 0 : _a.bucketName;
const bucket = await ModuleBE_AssetsStorage_1.ModuleBE_AssetsStorage.storage.getOrCreateBucket(bucketName);
return Promise.all(files.map(async (_file) => {
const key = _file.key || _file.mimeType;
// this will fail the entire request... should it?
if (!this.fileValidator[key])
throw new ts_common_1.ImplementationMissingException(`Missing validator for type ${key}`);
const _id = (0, ts_common_1.generateHex)(32);
const path = `${this.config.storagePath}/${_id}`;
const dbAsset = {
timestamp: (0, ts_common_1.currentTimeMillis)(),
_id,
feId: _file.feId,
name: _file.name,
ext: _file.name.substring(_file.name.toLowerCase().lastIndexOf('.') + 1),
mimeType: _file.mimeType,
key,
path,
bucketName: bucket.bucketName
};
if (_file.public)
dbAsset.public = _file.public;
const dbTempMeta = await ModuleBE_AssetsTemp_1.ModuleBE_AssetsTemp.set.item(dbAsset);
const fileWrapper = await bucket.getFile(dbTempMeta.path);
const url = await fileWrapper.getWriteSignedUrl(_file.mimeType, ts_common_1.Hour);
return {
signedUrl: url.signedUrl,
asset: dbTempMeta
};
}));
};
this.processAssetManually = async (feId) => {
let query = { limit: 1 };
if (feId)
query = { where: { feId } };
const unprocessedFiles = await ModuleBE_AssetsTemp_1.ModuleBE_AssetsTemp.query.custom(query);
return Promise.all(unprocessedFiles.map(asset => this.__processAsset(asset.path)));
};
this.__processAsset = async (filePath) => {
if (!filePath)
throw new ts_common_1.MUSTNeverHappenException('Missing file path');
if (!filePath.match(this.config.pathRegexp))
return this.logVerbose(`File was added to storage in path: ${filePath}, NOT via file uploader`);
this.logDebug(`Looking for file with path: ${filePath}`);
let dbTempAsset;
try {
dbTempAsset = await ModuleBE_AssetsTemp_1.ModuleBE_AssetsTemp.query.uniqueWhere({ path: filePath });
}
catch (e) {
return;
}
if (!dbTempAsset)
throw new ts_common_1.ThisShouldNotHappenException(`Could not find meta for file with path: ${filePath}`);
await this.notifyFrontend(shared_1.FileStatus.Processing, dbTempAsset);
this.logDebug(`Found temp meta with _id: ${dbTempAsset._id}`, dbTempAsset);
const validationConfig = this.fileValidator[dbTempAsset.key];
if (!validationConfig)
return this.notifyFrontend(shared_1.FileStatus.ErrorNoConfig, dbTempAsset);
let mimetypeValidator = exports.DefaultMimetypeValidator;
if (validationConfig.validator)
mimetypeValidator = validationConfig.validator;
if (!mimetypeValidator && validationConfig.fileType && validationConfig.fileType.includes(dbTempAsset.mimeType))
mimetypeValidator = this.mimeTypeValidator[dbTempAsset.mimeType];
if (!mimetypeValidator)
return this.notifyFrontend(shared_1.FileStatus.ErrorNoValidator, dbTempAsset);
const file = await ModuleBE_AssetsStorage_1.ModuleBE_AssetsStorage.getFile(dbTempAsset);
try {
const metadata = (await file.getDefaultMetadata()).metadata;
if (!metadata)
return this.notifyFrontend(shared_1.FileStatus.ErrorRetrievingMetadata, dbTempAsset);
await (0, exports.fileSizeValidator)(file, metadata, validationConfig.minSize, validationConfig.maxSize);
const fileType = await mimetypeValidator(file, dbTempAsset);
dbTempAsset.md5Hash = metadata.md5Hash;
if (fileType && dbTempAsset.ext !== fileType.ext) {
this.logWarning(`renaming the file extension name: ${dbTempAsset.ext} => ${fileType.ext}`);
dbTempAsset.ext = fileType.ext;
}
}
catch (e) {
//TODO delete the file and the temp doc
this.logError(`Error while processing asset: ${dbTempAsset.name}`, e);
return this.notifyFrontend(shared_1.FileStatus.ErrorWhileProcessing, dbTempAsset);
}
if (dbTempAsset.public) {
try {
// need to handle the response status!
await file.makePublic();
}
catch (e) {
return this.notifyFrontend(shared_1.FileStatus.ErrorMakingPublic, dbTempAsset);
}
}
const finalDbAsset = await this.runTransaction(async (transaction) => {
const duplicatedAssets = await this.query.custom({ where: { md5Hash: dbTempAsset.md5Hash } }, transaction);
if (duplicatedAssets.length && duplicatedAssets[0]) {
this.logWarning(`${dbTempAsset.feId} is a duplicated entry for ${duplicatedAssets[0]._id}`);
return Object.assign(Object.assign({}, duplicatedAssets[0]), { feId: dbTempAsset.feId });
}
const doc = this.collection.doc.item(dbTempAsset);
await ModuleBE_AssetsTemp_1.ModuleBE_AssetsTemp.delete.unique(dbTempAsset._id, transaction);
return await doc.set(dbTempAsset, transaction);
});
return this.notifyFrontend(shared_1.FileStatus.Completed, finalDbAsset);
};
this.notifyFrontend = async (status, asset, feId) => {
if (status !== shared_1.FileStatus.Completed && status !== shared_1.FileStatus.Processing) {
const message = `Error while processing asset: ${status}\n Failed on \n Asset: ${asset.feId}\n Key: ${asset.key}\n Type: ${asset.mimeType}\n File: ${asset.name}`;
throw new ts_common_1.ApiException(500, message);
}
this.logDebug(`notify FE about asset ${feId}: ${status}`);
return messages_1.PushMessageBE_FileUploadStatus.push({ status, asset }, { feId: feId || asset.feId });
};
this.setDefaultConfig(Object.assign(Object.assign({}, this.config), { storagePath: 'assets', pathRegexp: '^assets/.*', authKey: 'file-uploader' }));
}
async postWriteProcessing(data, actionType, transaction) {
const deletedItems = data.deleted;
if ((0, ts_common_1.exists)(deletedItems))
await ModuleBE_AssetsDeleted_1.ModuleBE_AssetsDeleted.create.all((0, ts_common_1.asArray)(deletedItems), transaction);
}
init() {
super.init();
(0, backend_1.addRoutes)([
(0, backend_1.createBodyServerApi)(shared_1.ApiDef_AssetUploader.vv1.getUploadUrl, this.getUrl)
]);
this.registerVersionUpgradeProcessor('1.0.1', async (assets) => {
assets.forEach(asset => {
// @ts-ignore
delete asset._audit;
});
});
const originalQuery = this.query;
this.query = Object.assign(Object.assign({}, originalQuery), { unique: async (_id, transaction) => {
var _a;
const dbAsset = await originalQuery.uniqueAssert(_id, transaction);
const signedUrl = (((_a = dbAsset.signedUrl) === null || _a === void 0 ? void 0 : _a.validUntil) || 0) > (0, ts_common_1.currentTimeMillis)() ? dbAsset.signedUrl : undefined;
if (!signedUrl) {
const url = await ModuleBE_AssetsStorage_1.ModuleBE_AssetsStorage.getReadSignedUrl(dbAsset);
dbAsset.signedUrl = {
url,
validUntil: (0, ts_common_1.currentTimeMillis)() + ts_common_1.Day - ts_common_1.Minute
};
}
return dbAsset;
} });
}
async getAssetsContent(assetIds) {
const assetsToSync = (0, ts_common_1.filterInstances)(await exports.ModuleBE_AssetsDB.query.all(assetIds));
const assetFiles = await Promise.all(assetsToSync.map(asset => ModuleBE_AssetsStorage_1.ModuleBE_AssetsStorage.getFile(asset)));
const assetContent = await Promise.all(assetFiles.map(asset => asset.read()));
return assetIds.map((id, index) => ({ asset: assetsToSync[index], content: assetContent[index] }));
}
registerTypeValidator(mimeType, validator) {
}
async queryUnique(where, transaction) {
var _a;
const dbAsset = await this.query.uniqueCustom({ where });
const signedUrl = (((_a = dbAsset.signedUrl) === null || _a === void 0 ? void 0 : _a.validUntil) || 0) > (0, ts_common_1.currentTimeMillis)() ? dbAsset.signedUrl : undefined;
if (!signedUrl) {
const url = await ModuleBE_AssetsStorage_1.ModuleBE_AssetsStorage.getReadSignedUrl(dbAsset);
dbAsset.signedUrl = {
url,
validUntil: (0, ts_common_1.currentTimeMillis)() + ts_common_1.Day - ts_common_1.Minute
};
}
return dbAsset;
}
__onCleanupSchedulerAct() {
return {
moduleKey: this.getName(),
interval: ts_common_1.Day,
cleanup: () => this.cleanup(),
};
}
}
exports.ModuleBE_AssetsDB_Class = ModuleBE_AssetsDB_Class;
exports.ModuleBE_AssetsDB = new ModuleBE_AssetsDB_Class();