@blocklet/uploader-server
Version:
blocklet upload server
439 lines (438 loc) • 14.6 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.fileExistBeforeUpload = fileExistBeforeUpload;
exports.getFileName = void 0;
exports.getFileNameParam = getFileNameParam;
exports.getLocalStorageFile = getLocalStorageFile;
exports.getMetaDataByFilePath = getMetaDataByFilePath;
exports.initLocalStorageServer = initLocalStorageServer;
exports.joinUrl = joinUrl;
exports.setHeaders = setHeaders;
var _server = require("@tus/server");
var _fileStore = require("@tus/file-store");
var _cron = _interopRequireDefault(require("@abtnode/cron"));
var _fs = require("fs");
var _path2 = _interopRequireDefault(require("path"));
var _crypto = _interopRequireDefault(require("crypto"));
var _mimeTypes = _interopRequireDefault(require("mime-types"));
var _urlJoin = _interopRequireDefault(require("url-join"));
var _pQueue = _interopRequireDefault(require("p-queue"));
var _utils = require("../utils");
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
const validFilePathInDirPath = (dirPath, filePath) => {
const fileName = _path2.default.basename(filePath);
if (!filePath.startsWith(dirPath) || _path2.default.join(dirPath, fileName) !== filePath) {
_utils.logger.error("Invalid file path: ", filePath);
throw new Error("Invalid file path");
}
return true;
};
function initLocalStorageServer({
path: _path,
onUploadFinish: _onUploadFinish,
onUploadCreate: _onUploadCreate,
express,
expiredUploadTime = 1e3 * 60 * 60 * 24 * 3,
// default 3 days expire
...restProps
}) {
const app = express();
const configstore = new RewriteFileConfigstore(_path);
const datastore = new RewriteFileStore({
directory: _path,
expirationPeriodInMilliseconds: expiredUploadTime,
configstore
});
const formatMetadata = uploadMetadata => {
const cloneUploadMetadata = {
...uploadMetadata
};
if (cloneUploadMetadata.metadata?.name?.indexOf("/") > -1 && cloneUploadMetadata.metadata?.relativePath?.indexOf("/") > -1) {
cloneUploadMetadata.metadata.name = cloneUploadMetadata.metadata.name.split("/").pop();
cloneUploadMetadata.metadata.filename = cloneUploadMetadata.metadata.name;
}
if (cloneUploadMetadata.id) {
const {
id,
metadata,
size
} = cloneUploadMetadata;
if (!cloneUploadMetadata?.runtime?.absolutePath) {
cloneUploadMetadata.runtime = {
...cloneUploadMetadata.runtime,
relativePath: metadata?.relativePath || cloneUploadMetadata.runtime?.relativePath,
absolutePath: _path2.default.join(_path, id),
size: size || cloneUploadMetadata.runtime?.size,
hashFileName: id,
originFileName: metadata?.filename || cloneUploadMetadata.runtime?.originFileName,
type: metadata?.type || cloneUploadMetadata.runtime?.type,
fileType: metadata?.filetype || cloneUploadMetadata.runtime?.fileType
};
} else {
const expectedPath = _path2.default.join(_path, id);
if (cloneUploadMetadata.runtime.absolutePath !== expectedPath) {
cloneUploadMetadata.runtime.absolutePath = expectedPath;
}
}
}
return cloneUploadMetadata;
};
const rewriteMetaDataFile = async uploadMetadata => {
uploadMetadata = formatMetadata(uploadMetadata);
const {
id
} = uploadMetadata;
if (!id) {
return;
}
const oldMetadata = formatMetadata(await configstore.get(id));
if (JSON.stringify(oldMetadata) !== JSON.stringify(uploadMetadata)) {
await configstore.set(id, uploadMetadata);
}
};
const onUploadCreate = async (req, res, uploadMetadata) => {
uploadMetadata = formatMetadata(uploadMetadata);
await rewriteMetaDataFile(uploadMetadata);
if (uploadMetadata.offset === 0 && uploadMetadata.size === 0) {
res.status(200);
res.setHeader("Location", joinUrl(req.headers["x-uploader-base-url"], uploadMetadata.id));
res.setHeader("Upload-Offset", 0);
res.setHeader("Upload-Length", 0);
res.setHeader("x-uploader-file-exist", true);
}
if (_onUploadCreate) {
const result = await _onUploadCreate(req, res, uploadMetadata);
return result;
}
return res;
};
const onUploadFinish = async (req, res, uploadMetadata) => {
res.setHeader("x-uploader-file-exist", true);
uploadMetadata = formatMetadata(uploadMetadata);
await rewriteMetaDataFile(uploadMetadata);
const fileType = uploadMetadata.metadata?.filetype || uploadMetadata.metadata?.type || "";
const fileName = uploadMetadata.metadata?.filename || uploadMetadata.metadata?.name || "";
const fileExt = _path2.default.extname(fileName).toLowerCase();
const isImageFile = fileType.startsWith("image/");
const isBinaryFile = ["zip", "gz", "tgz", "7z", "dmg", "pkg", "apk"].includes(fileExt.replace(".", ""));
if (isImageFile && !isBinaryFile && uploadMetadata.runtime?.absolutePath) {
try {
await (0, _utils.removeExifFromFile)(uploadMetadata.runtime.absolutePath);
} catch (err) {
_utils.logger.error("failed to remove EXIF from file", err);
}
}
if (_onUploadFinish) {
try {
const result = await _onUploadFinish(req, res, uploadMetadata);
return result;
} catch (err) {
_utils.logger.error("@blocklet/uploader: onUploadFinish error: ", err);
newServer.delete(uploadMetadata.id);
res.setHeader("x-uploader-file-exist", false);
throw err;
}
}
return res;
};
const newServer = new _server.Server({
path: "/",
// UNUSED
relativeLocation: true,
// respectForwardedHeaders: true,
namingFunction: req => {
const fileName = getFileName(req);
const filePath = _path2.default.join(_path, fileName);
validFilePathInDirPath(_path, filePath);
return fileName;
},
datastore,
onUploadFinish: async (req, res, uploadMetadata) => {
uploadMetadata = formatMetadata(uploadMetadata);
const result = await onUploadFinish(req, res, uploadMetadata);
if (result && !result.send) {
const body = typeof result === "string" ? result : JSON.stringify(result);
throw {
body,
status_code: 200
};
} else {
return result;
}
},
onUploadCreate,
...restProps
});
app.use((req, res, next) => {
req.uploaderProps = {
server: newServer,
onUploadFinish,
onUploadCreate
};
next();
});
_cron.default.init({
context: {},
jobs: [{
name: "auto-cleanup-expired-uploads",
time: "0 0 * * * *",
// each hour
fn: () => {
try {
newServer.cleanUpExpiredUploads().then(count => {
_utils.logger.info(`@blocklet/uploader: cleanup expired uploads done: ${count}`);
}).catch(err => {
_utils.logger.error(`@blocklet/uploader: cleanup expired uploads error`, err);
});
} catch (err) {
_utils.logger.error(`@blocklet/uploader: cleanup expired uploads error`, err);
}
},
options: {
runOnInit: false
}
}],
onError: err => {
_utils.logger.error("@blocklet/uploader: cleanup job failed", err);
}
});
newServer.delete = async key => {
try {
await configstore.delete(key);
await configstore.delete(key, false);
} catch (err) {
_utils.logger.error("@blocklet/uploader: delete error: ", err);
}
};
newServer.on(_server.EVENTS.POST_RECEIVE, async (req, res, uploadMetadata) => {
uploadMetadata = formatMetadata(uploadMetadata);
await rewriteMetaDataFile(uploadMetadata);
});
app.all("*", setHeaders, fileExistBeforeUpload, newServer.handle.bind(newServer));
newServer.handle = app;
return newServer;
}
const getFileName = req => {
const ext = req.headers["x-uploader-file-ext"];
const randomName = `${_crypto.default.randomBytes(16).toString("hex")}${ext ? `.${ext}` : ""}`;
return req.headers["x-uploader-file-name"] || randomName;
};
exports.getFileName = getFileName;
function getFileNameParam(req, res, {
isRequired = true
} = {}) {
let {
fileName
} = req.params;
if (!fileName) {
fileName = req.originalUrl.replace(req.baseUrl, "");
}
if (!fileName && isRequired) {
res.status(400).json({
error: 'Parameter "fileName" is required'
});
return;
}
return fileName;
}
function getLocalStorageFile({
server
}) {
return async (req, res, next) => {
const fileName = getFileNameParam(req, res);
const filePath = _path2.default.join(server.datastore.directory, fileName);
validFilePathInDirPath(server.datastore.directory, filePath);
const fileExists = await _fs.promises.stat(filePath).catch(() => false);
if (!fileExists) {
res.status(404).json({
error: "file not found"
});
return;
}
setHeaders(req, res);
const file = await _fs.promises.readFile(filePath);
res.send(file);
next?.();
};
}
function setHeaders(req, res, next) {
let {
method
} = req;
method = method.toUpperCase();
const fileName = getFileNameParam(req, res, {
isRequired: false
});
if (req.headers["x-uploader-endpoint-url"]) {
const query = new URL(req.headers["x-uploader-endpoint-url"]).searchParams;
req.query = {
...Object.fromEntries(query),
// query params convert to object
...req.query
};
}
if (method === "POST" && req.headers["x-uploader-base-url"]) {
req.baseUrl = req.headers["x-uploader-base-url"];
}
if (method === "GET" && fileName) {
const contentType = _mimeTypes.default.lookup(fileName);
if (contentType) {
res.setHeader("Content-Type", contentType);
}
(0, _utils.setPDFDownloadHeader)(req, res);
}
next?.();
}
async function fileExistBeforeUpload(req, res, next) {
let {
method,
uploaderProps
} = req;
method = method.toUpperCase();
if (["PATCH", "POST"].includes(method)) {
const _path = uploaderProps.server.datastore.directory;
const fileName = getFileName(req);
const filePath = _path2.default.join(_path, fileName);
validFilePathInDirPath(_path, filePath);
const isExist = await _fs.promises.stat(filePath).catch(() => false);
if (isExist) {
const metaData = await getMetaDataByFilePath(filePath);
if (isExist?.size >= 0 && isExist?.size === metaData?.size) {
const prepareUpload = method === "POST";
if (prepareUpload) {
res.status(200);
res.setHeader("Location", joinUrl(req.headers["x-uploader-base-url"], fileName));
res.setHeader("Upload-Offset", +metaData.offset);
res.setHeader("Upload-Length", +metaData.size);
}
if (req.headers["x-uploader-metadata"]) {
try {
const realMetaData = JSON.parse(req.headers["x-uploader-metadata"], (key, value) => {
if (typeof value === "string") {
return decodeURIComponent(value);
}
return value;
});
metaData.metadata = {
...metaData.metadata,
...realMetaData
};
} catch (err) {
_utils.logger.error("@blocklet/uploader: parse metadata error: ", err);
}
}
const uploadResult = await uploaderProps.onUploadFinish(req, res, metaData);
res.json(uploadResult);
return;
}
}
}
next?.();
}
async function getMetaDataByFilePath(filePath) {
const metaDataPath = `${filePath}.json`;
const isExist = await _fs.promises.stat(filePath).catch(() => false);
if (isExist) {
try {
const metaData = await _fs.promises.readFile(metaDataPath, "utf-8");
const metaDataJson = JSON.parse(metaData);
return metaDataJson;
} catch (err) {
_utils.logger.error("@blocklet/uploader: getMetaDataByPath error: ", err);
}
}
return null;
}
function joinUrl(...args) {
const realArgs = args.filter(Boolean).map(item => {
if (item === "/") {
return "";
}
return item;
});
return (0, _urlJoin.default)(...realArgs);
}
class RewriteFileConfigstore {
directory;
queue;
constructor(path2) {
this.directory = path2;
this.queue = new _pQueue.default({
concurrency: 1
});
}
async get(key) {
try {
const buffer = await this.queue.add(() => _fs.promises.readFile(this.resolve(key), "utf8"));
const metadata = JSON.parse(buffer);
if (metadata.offset !== metadata.size) {
const info = await _fs.promises.stat(this.resolve(key, false)).catch(() => false);
if (info?.size !== metadata?.offset) {
metadata.offset = info.size;
this.set(key, metadata);
}
}
return metadata;
} catch {
return void 0;
}
}
async set(key, value) {
if (value?.runtime) {
delete value.runtime;
}
if (value?.metadata?.runtime) {
delete value.metadata.runtime;
}
await this.queue.add(() => _fs.promises.writeFile(this.resolve(key), JSON.stringify(value)));
}
async safeDeleteFile(filePath) {
validFilePathInDirPath(this.directory, filePath);
try {
const isExist = await _fs.promises.stat(filePath).catch(() => false);
if (isExist) {
await _fs.promises.rm(filePath);
} else {
_utils.logger.log("Can not remove file, the file not exist: ", filePath);
}
} catch (err) {
_utils.logger.error("@blocklet/uploader: safeDeleteFile error: ", err);
}
}
async delete(key, isMetadata = true) {
try {
await this.queue.add(() => this.safeDeleteFile(this.resolve(key, isMetadata)));
} catch (err) {
_utils.logger.error("@blocklet/uploader: delete error: ", err);
}
}
async list() {
return this.queue.add(async () => {
const files = await _fs.promises.readdir(this.directory, {
withFileTypes: true
});
const promises = files.filter(file => file.isFile() && file.name.endsWith(".json")).map(file => {
return file.name.replace(".json", "");
});
return Promise.all(promises);
});
}
resolve(key, isMetadata = true) {
let fileKey = key;
if (isMetadata) {
fileKey = `${key}.json`;
}
return _path2.default.join(this.directory, fileKey);
}
}
class RewriteFileStore extends _fileStore.FileStore {
constructor(options) {
super(options);
}
async remove(key) {
this.configstore.delete(key);
this.configstore.delete(key, false);
}
}