@nu-art/file-upload
Version:
File Uploader - Express & Typescript based backend framework
244 lines • 12 kB
JavaScript
/*
* Permissions management system, define access level for each of
* your server apis, and restrict users by giving them access levels
*
* Copyright (C) 2020 Adam van der Kruk aka TacB0sS
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ApiException, asArray, BadImplementationException, currentTimeMillis, Day, exists, filterInstances, generateHex, Hour, ImplementationMissingException, MB, Minute, MUSTNeverHappenException, ThisShouldNotHappenException } from '@nu-art/ts-common';
import { ModuleBE_AssetsTemp } from './ModuleBE_AssetsTemp.js';
import { addRoutes, createBodyServerApi, ModuleBE_BaseDB } from '@nu-art/thunderstorm/backend/index';
import { ModuleBE_AssetsStorage } from './ModuleBE_AssetsStorage.js';
import { ApiDef_AssetUploader, DBDef_Assets, FileStatus } from '../../shared/index.js';
import { PushMessageBE_FileUploadStatus } from '../core/messages.js';
import { ModuleBE_AssetsDeleted } from './ModuleBE_AssetsDeleted.js';
import { fileTypeFromBuffer } from 'file-type';
export const DefaultMimetypeValidator = async (file, doc) => {
const buffer = await file.read();
const fileType = await fileTypeFromBuffer(buffer);
if (!fileType)
throw new ImplementationMissingException(`No validator defined for asset of mimetype: ${doc.mimeType}`);
if (fileType.mime !== doc.mimeType)
throw new BadImplementationException(`Original mimetype (${doc.mimeType}) does not match the resolved mimetype: (${fileType.mime})`);
return fileType;
};
export const fileSizeValidator = async (file, metadata, minSizeInBytes = 0, maxSizeInBytes = MB) => {
const fileSize = +(metadata.size ?? 0);
if (!fileSize)
throw new ThisShouldNotHappenException(`could not resolve metadata.size for file: ${file.path}`);
return fileSize >= minSizeInBytes && fileSize <= maxSizeInBytes;
};
export class ModuleBE_AssetsDB_Class extends ModuleBE_BaseDB {
constructor() {
super(DBDef_Assets);
this.setDefaultConfig({
...this.config,
storagePath: 'assets',
pathRegexp: '^assets/.*',
authKey: 'file-uploader'
});
}
mimeTypeValidator = {};
fileValidator = {};
async postWriteProcessing(data, actionType, transaction) {
const deletedItems = data.deleted;
if (exists(deletedItems))
await ModuleBE_AssetsDeleted.create.all(asArray(deletedItems), transaction);
}
init() {
super.init();
addRoutes([
createBodyServerApi(ApiDef_AssetUploader.vv1.getUploadUrl, this.getUrl)
]);
this.registerVersionUpgradeProcessor('1.0.1', async (assets) => {
assets.forEach(asset => {
// @ts-ignore
delete asset._audit;
});
});
const originalQuery = this.query;
this.query = {
...originalQuery,
unique: async (_id, transaction) => {
const dbAsset = await originalQuery.uniqueAssert(_id, transaction);
const signedUrl = (dbAsset.signedUrl?.validUntil || 0) > currentTimeMillis() ? dbAsset.signedUrl : undefined;
if (!signedUrl) {
const url = await ModuleBE_AssetsStorage.getReadSignedUrl(dbAsset);
dbAsset.signedUrl = {
url,
validUntil: currentTimeMillis() + Day - Minute
};
}
return dbAsset;
}
};
}
async getAssetsContent(assetIds) {
const assetsToSync = filterInstances(await ModuleBE_AssetsDB.query.all(assetIds));
const assetFiles = await Promise.all(assetsToSync.map(asset => ModuleBE_AssetsStorage.getFile(asset)));
const assetContent = await Promise.all(assetFiles.map(asset => asset.read()));
return assetIds.map((id, index) => ({ asset: assetsToSync[index], content: assetContent[index] }));
}
registerTypeValidator(mimeType, validator) {
}
async queryUnique(where, transaction) {
const dbAsset = await this.query.uniqueCustom({ where });
const signedUrl = (dbAsset.signedUrl?.validUntil || 0) > currentTimeMillis() ? dbAsset.signedUrl : undefined;
if (!signedUrl) {
const url = await ModuleBE_AssetsStorage.getReadSignedUrl(dbAsset);
dbAsset.signedUrl = {
url,
validUntil: currentTimeMillis() + Day - Minute
};
}
return dbAsset;
}
register = (key, validationConfig) => {
if (this.fileValidator[key] && this.fileValidator[key] !== validationConfig)
throw new BadImplementationException(`File Validator already exists for key: ${key}`);
this.fileValidator[key] = validationConfig;
};
__onCleanupSchedulerAct() {
return {
moduleKey: this.getName(),
interval: Day,
cleanup: () => this.cleanup(),
};
}
cleanup = async (interval = Hour, module = ModuleBE_AssetsTemp) => {
const entries = await module.query.custom({ where: { timestamp: { $lt: currentTimeMillis() - interval } } });
await Promise.all(entries.map(async (dbAsset) => {
const file = await ModuleBE_AssetsStorage.getFile(dbAsset);
if (!(await file.exists()))
return;
await file.delete();
}));
await module.delete.query({ where: { timestamp: { $lt: currentTimeMillis() - interval } } });
};
getUrl = async (files) => {
const bucketName = this.config?.bucketName;
const bucket = await ModuleBE_AssetsStorage.storage.getOrCreateBucket(bucketName);
return Promise.all(files.map(async (_file) => {
const key = _file.key || _file.mimeType;
// this will fail the entire request... should it?
if (!this.fileValidator[key])
throw new ImplementationMissingException(`Missing validator for type ${key}`);
const _id = generateHex(32);
const path = `${this.config.storagePath}/${_id}`;
const dbAsset = {
timestamp: currentTimeMillis(),
_id,
feId: _file.feId,
name: _file.name,
ext: _file.name.substring(_file.name.toLowerCase().lastIndexOf('.') + 1),
mimeType: _file.mimeType,
key,
path,
bucketName: bucket.bucketName
};
if (_file.public)
dbAsset.public = _file.public;
const dbTempMeta = await ModuleBE_AssetsTemp.set.item(dbAsset);
const fileWrapper = await bucket.getFile(dbTempMeta.path);
const url = await fileWrapper.getWriteSignedUrl(_file.mimeType, Hour);
return {
signedUrl: url.signedUrl,
asset: dbTempMeta
};
}));
};
processAssetManually = async (feId) => {
let query = { limit: 1 };
if (feId)
query = { where: { feId } };
const unprocessedFiles = await ModuleBE_AssetsTemp.query.custom(query);
return Promise.all(unprocessedFiles.map(asset => this.__processAsset(asset.path)));
};
__processAsset = async (filePath) => {
if (!filePath)
throw new MUSTNeverHappenException('Missing file path');
if (!filePath.match(this.config.pathRegexp))
return this.logVerbose(`File was added to storage in path: ${filePath}, NOT via file uploader`);
this.logDebug(`Looking for file with path: ${filePath}`);
let dbTempAsset;
try {
dbTempAsset = await ModuleBE_AssetsTemp.query.uniqueWhere({ path: filePath });
}
catch (e) {
return;
}
if (!dbTempAsset)
throw new ThisShouldNotHappenException(`Could not find meta for file with path: ${filePath}`);
await this.notifyFrontend(FileStatus.Processing, dbTempAsset);
this.logDebug(`Found temp meta with _id: ${dbTempAsset._id}`, dbTempAsset);
const validationConfig = this.fileValidator[dbTempAsset.key];
if (!validationConfig)
return this.notifyFrontend(FileStatus.ErrorNoConfig, dbTempAsset);
let mimetypeValidator = DefaultMimetypeValidator;
if (validationConfig.validator)
mimetypeValidator = validationConfig.validator;
if (!mimetypeValidator && validationConfig.fileType && validationConfig.fileType.includes(dbTempAsset.mimeType))
mimetypeValidator = this.mimeTypeValidator[dbTempAsset.mimeType];
if (!mimetypeValidator)
return this.notifyFrontend(FileStatus.ErrorNoValidator, dbTempAsset);
const file = await ModuleBE_AssetsStorage.getFile(dbTempAsset);
try {
const metadata = (await file.getMetadata());
if (!metadata)
return this.notifyFrontend(FileStatus.ErrorRetrievingMetadata, dbTempAsset);
await fileSizeValidator(file, metadata, validationConfig.minSize, validationConfig.maxSize);
const fileType = await mimetypeValidator(file, dbTempAsset);
dbTempAsset.md5Hash = metadata.md5Hash;
if (fileType && dbTempAsset.ext !== fileType.ext) {
this.logWarning(`renaming the file extension name: ${dbTempAsset.ext} => ${fileType.ext}`);
dbTempAsset.ext = fileType.ext;
}
}
catch (e) {
//TODO delete the file and the temp doc
this.logError(`Error while processing asset: ${dbTempAsset.name}`, e);
return this.notifyFrontend(FileStatus.ErrorWhileProcessing, dbTempAsset);
}
if (dbTempAsset.public) {
try {
// need to handle the response status!
await file.makePublic();
}
catch (e) {
return this.notifyFrontend(FileStatus.ErrorMakingPublic, dbTempAsset);
}
}
const finalDbAsset = await this.runTransaction(async (transaction) => {
const duplicatedAssets = await this.query.custom({ where: { md5Hash: dbTempAsset.md5Hash } }, transaction);
if (duplicatedAssets.length && duplicatedAssets[0]) {
this.logWarning(`${dbTempAsset.feId} is a duplicated entry for ${duplicatedAssets[0]._id}`);
return { ...duplicatedAssets[0], feId: dbTempAsset.feId };
}
const doc = this.collection.doc.item(dbTempAsset);
await ModuleBE_AssetsTemp.delete.unique(dbTempAsset._id, transaction);
return await doc.set(dbTempAsset, transaction);
});
return this.notifyFrontend(FileStatus.Completed, finalDbAsset);
};
notifyFrontend = async (status, asset, feId) => {
if (status !== FileStatus.Completed && status !== FileStatus.Processing) {
const message = `Error while processing asset: ${status}\n Failed on \n Asset: ${asset.feId}\n Key: ${asset.key}\n Type: ${asset.mimeType}\n File: ${asset.name}`;
throw new ApiException(500, message);
}
this.logDebug(`notify FE about asset ${feId}: ${status}`);
return PushMessageBE_FileUploadStatus.push({ status, asset }, { feId: feId || asset.feId });
};
}
export const ModuleBE_AssetsDB = new ModuleBE_AssetsDB_Class();
//# sourceMappingURL=ModuleBE_AssetsDB.js.map