@blocklet/uploader-server
Version:
blocklet upload server
310 lines (309 loc) • 9.66 kB
JavaScript
import axios from "axios";
import path from "path";
import joinUrl from "url-join";
import { isbot } from "isbot";
import component from "@blocklet/sdk/lib/component";
import { ImageBinDid } from "./constants.js";
import crypto from "crypto";
import { getSignData } from "@blocklet/sdk/lib/util/verify-sign";
import FormData from "form-data";
import omit from "lodash/omit";
import ms from "ms";
import mime from "mime-types";
import { existsSync, readdirSync, renameSync, statSync, unlinkSync, createReadStream, createWriteStream } from "fs";
import { join } from "path";
import ExifTransformer from "exif-be-gone";
export let logger = console;
const isProduction = process.env.NODE_ENV === "production" || process.env.ABT_NODE_SERVICE_ENV === "production";
if (process.env.BLOCKLET_LOG_DIR) {
try {
const initLogger = require("@blocklet/logger");
logger = initLogger("uploader-server");
logger.info("uploader-server logger init success");
} catch (error) {
}
}
const DEFAULT_TTL = 5 * 60 * 1e3;
const appUrl = process.env.BLOCKLET_APP_URL || "";
const trustedDomainsCache = {
timestamp: 0,
domains: []
};
export async function getTrustedDomainsCache({
forceUpdate = false,
ttl = DEFAULT_TTL
} = {}) {
if (!appUrl) {
return [];
}
const now = Date.now();
if (!forceUpdate && trustedDomainsCache.domains.length > 0) {
if (now - trustedDomainsCache.timestamp < ttl) {
return trustedDomainsCache.domains;
}
}
try {
if (!trustedDomainsCache?.domains?.length) {
trustedDomainsCache.domains = await axios.get(joinUrl(appUrl, "/.well-known/service/api/federated/getTrustedDomains")).then((res) => res.data);
trustedDomainsCache.timestamp = now;
}
} catch (error) {
}
return trustedDomainsCache.domains;
}
export async function checkTrustedReferer(req, res, next) {
if (!isProduction) {
return next?.();
}
if (isbot(req.get("user-agent"))) {
return next?.();
}
const referer = req.headers.referer;
if (!referer) {
return res.status(403).send("Access denied");
}
const allowedDomains = await getTrustedDomainsCache();
const refererHost = new URL(referer).hostname;
if (allowedDomains?.length && !allowedDomains.some((domain) => refererHost.includes(domain))) {
return res.status(403).send("Access denied");
}
next?.();
}
export async function proxyImageDownload(req, res, next) {
let { url } = {
...req.query,
...req.body
};
if (url) {
url = encodeURI(url);
try {
const { headers, data, status } = await axios({
method: "get",
url,
responseType: "stream",
timeout: 30 * 1e3,
headers: {
// Add common headers to mimic browser request
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}
});
if (data && status >= 200 && status < 302) {
res.setHeader("Content-Type", headers["content-type"]);
data.pipe(res);
} else {
throw new Error("download image error");
}
} catch (err) {
logger.error("Proxy url failed: ", err);
res.status(400).send("Proxy url failed");
}
} else {
res.status(400).send('Parameter "url" is required');
}
}
export function setPDFDownloadHeader(req, res) {
if (path.extname(req.path) === ".pdf") {
const filename = req.query?.filename ?? req?.path;
res.setHeader("Content-Disposition", `attachment; ${filename ? `filename="${filename}"` : ""}`);
}
}
export const getFileHash = async (filePath, maxBytes = 5 * 1024 * 1024) => {
const hash = crypto.createHash("md5");
const readStream = createReadStream(filePath, {
start: 0,
end: maxBytes - 1,
highWaterMark: 1024 * 1024
// 1MB chunks
});
for await (const chunk of readStream) {
hash.update(chunk.toString());
}
return hash.digest("hex");
};
export async function uploadToMediaKit({
filePath,
fileName,
base64,
extraComponentCallOptions
}) {
if (!filePath && !base64) {
throw new Error("filePath or base64 is required");
}
if (base64) {
if (!fileName) {
throw new Error("fileName is required when base64 is provided");
}
const res = await component.call({
name: ImageBinDid,
path: "/api/sdk/uploads",
data: {
base64,
filename: fileName
},
...omit(extraComponentCallOptions, ["name", "path", "data"])
});
return res;
}
if (filePath) {
const fileStream = createReadStream(filePath);
const filename = fileName || path.basename(filePath);
const form = new FormData();
const fileHash = await getFileHash(filePath);
form.append("file", fileStream);
form.append("filename", filename);
form.append("hash", fileHash);
const res = await component.call(
{
name: ImageBinDid,
path: "/api/sdk/uploads",
data: form,
headers: {
"x-component-upload-sig": getSignData({
data: {
filename,
hash: fileHash
},
method: "POST",
url: "/api/sdk/uploads",
params: extraComponentCallOptions?.params || {}
}).sig,
...extraComponentCallOptions?.headers
},
...omit(extraComponentCallOptions, ["name", "path", "data", "headers"])
},
{
retries: 0
}
);
return res;
}
}
export async function getMediaKitFileStream(filePath) {
const fileName = path.basename(filePath);
const res = await component.call({
name: ImageBinDid,
path: joinUrl("/uploads", fileName),
responseType: "stream",
method: "GET"
});
return res;
}
export function calculateCacheControl(maxAge = "365d", immutable = true) {
let maxAgeInSeconds = 31536e3;
if (typeof maxAge === "string") {
try {
const milliseconds = ms(maxAge);
maxAgeInSeconds = typeof milliseconds === "number" ? milliseconds / 1e3 : 31536e3;
} catch (e) {
logger.warn(`Invalid maxAge format: ${maxAge}, using default 1 year (31536000 seconds)`);
}
} else {
maxAgeInSeconds = maxAge;
}
const cacheControl = `public, max-age=${maxAgeInSeconds}`;
const cacheControlImmutable = `${cacheControl}, immutable`;
return {
cacheControl,
cacheControlImmutable,
maxAgeInSeconds
};
}
export function serveResource(req, res, next, resource, options = {}) {
try {
res.setHeader("Content-Type", resource.contentType);
res.setHeader("Content-Length", resource.size);
res.setHeader("Last-Modified", resource.mtime.toUTCString());
const { cacheControl, cacheControlImmutable } = calculateCacheControl(
options.maxAge || "365d",
options.immutable !== false
);
res.setHeader("Cache-Control", options.immutable === false ? cacheControl : cacheControlImmutable);
if (options.setHeaders && typeof options.setHeaders === "function") {
const statObj = { mtime: resource.mtime, size: resource.size };
options.setHeaders(res, resource.filePath, statObj);
}
const ifModifiedSince = req.headers["if-modified-since"];
if (ifModifiedSince) {
const ifModifiedSinceDate = new Date(ifModifiedSince);
if (resource.mtime <= ifModifiedSinceDate) {
res.statusCode = 304;
res.end();
return;
}
}
const fileStream = createReadStream(resource.filePath);
fileStream.on("error", (error) => {
logger.error(`Error streaming file ${resource.filePath}:`, error);
next(error);
});
fileStream.pipe(res);
} catch (error) {
logger.error("Error serving static file:", error);
next(error);
}
}
export function scanDirectory(directory, options = {}) {
const resourceMap = /* @__PURE__ */ new Map();
if (!existsSync(directory)) {
return resourceMap;
}
try {
const files = readdirSync(directory);
for (const file of files) {
const filePath = join(directory, file);
let stat;
try {
stat = statSync(filePath);
if (stat.isDirectory()) continue;
} catch (e) {
continue;
}
if (options.whitelist?.length && !options.whitelist.some((ext) => file.endsWith(ext))) {
continue;
}
if (options.blacklist?.length && options.blacklist.some((ext) => file.endsWith(ext))) {
continue;
}
const contentType = mime.lookup(filePath) || "application/octet-stream";
resourceMap.set(file, {
filePath,
dir: directory,
originDir: options.originDir || directory,
blockletInfo: options.blockletInfo || {},
whitelist: options.whitelist,
blacklist: options.blacklist,
mtime: stat.mtime,
size: stat.size,
contentType
});
}
} catch (err) {
logger.error(`Error scanning directory ${directory}:`, err);
}
return resourceMap;
}
export function getFileNameFromReq(req) {
const pathname = req.path || req.url?.split("?")[0];
return path.basename(decodeURIComponent(pathname || ""));
}
export const removeExifFromFile = async (filePath) => {
return new Promise(async (resolve, reject) => {
try {
statSync(filePath);
} catch (e) {
reject(e);
return;
}
const tempPath = filePath + ".tmp";
const reader = createReadStream(filePath);
const writer = createWriteStream(tempPath);
reader.pipe(new ExifTransformer()).pipe(writer).on("finish", () => {
renameSync(tempPath, filePath);
resolve();
}).on("error", (err) => {
logger.error("[exif-be-gone] failed", err);
unlinkSync(tempPath);
reject(err);
});
});
};