@blocklet/uploader-server/es/middlewares/local-storage.js

blocklet upload server

import { Server, EVENTS } from "@tus/server";
import { FileStore } from "@tus/file-store";
import cron from "@abtnode/cron";
import { promises as fs } from "fs";
import path from "path";
import crypto from "crypto";
import mime from "mime-types";
import joinUrlLib from "url-join";
import queue from "p-queue";
import { setPDFDownloadHeader, logger, removeExifFromFile } from "../utils.js";
const validFilePathInDirPath = (dirPath, filePath) => {
  const fileName = path.basename(filePath);
  if (!filePath.startsWith(dirPath) || path.join(dirPath, fileName) !== filePath) {
    logger.error("Invalid file path: ", filePath);
    throw new Error("Invalid file path");
  }
  return true;
};
export function initLocalStorageServer({
  path: _path,
  onUploadFinish: _onUploadFinish,
  onUploadCreate: _onUploadCreate,
  express,
  expiredUploadTime = 1e3 * 60 * 60 * 24 * 3,
  // default 3 days expire
  ...restProps
}) {
  const app = express();
  const configstore = new RewriteFileConfigstore(_path);
  const datastore = new RewriteFileStore({
    directory: _path,
    expirationPeriodInMilliseconds: expiredUploadTime,
    configstore
  });
  const formatMetadata = (uploadMetadata) => {
    const cloneUploadMetadata = {
      ...uploadMetadata
    };
    if (cloneUploadMetadata.metadata?.name?.indexOf("/") > -1 && cloneUploadMetadata.metadata?.relativePath?.indexOf("/") > -1) {
      cloneUploadMetadata.metadata.name = cloneUploadMetadata.metadata.name.split("/").pop();
      cloneUploadMetadata.metadata.filename = cloneUploadMetadata.metadata.name;
    }
    if (cloneUploadMetadata.id) {
      const { id, metadata, size } = cloneUploadMetadata;
      if (!cloneUploadMetadata?.runtime?.absolutePath) {
        cloneUploadMetadata.runtime = {
          ...cloneUploadMetadata.runtime,
          relativePath: metadata?.relativePath || cloneUploadMetadata.runtime?.relativePath,
          absolutePath: path.join(_path, id),
          size: size || cloneUploadMetadata.runtime?.size,
          hashFileName: id,
          originFileName: metadata?.filename || cloneUploadMetadata.runtime?.originFileName,
          type: metadata?.type || cloneUploadMetadata.runtime?.type,
          fileType: metadata?.filetype || cloneUploadMetadata.runtime?.fileType
        };
      } else {
        const expectedPath = path.join(_path, id);
        if (cloneUploadMetadata.runtime.absolutePath !== expectedPath) {
          cloneUploadMetadata.runtime.absolutePath = expectedPath;
        }
      }
    }
    return cloneUploadMetadata;
  };
  const rewriteMetaDataFile = async (uploadMetadata) => {
    uploadMetadata = formatMetadata(uploadMetadata);
    const { id } = uploadMetadata;
    if (!id) {
      return;
    }
    const oldMetadata = formatMetadata(await configstore.get(id));
    if (JSON.stringify(oldMetadata) !== JSON.stringify(uploadMetadata)) {
      await configstore.set(id, uploadMetadata);
    }
  };
  const onUploadCreate = async (req, res, uploadMetadata) => {
    uploadMetadata = formatMetadata(uploadMetadata);
    await rewriteMetaDataFile(uploadMetadata);
    if (uploadMetadata.offset === 0 && uploadMetadata.size === 0) {
      res.status(200);
      res.setHeader("Location", joinUrl(req.headers["x-uploader-base-url"], uploadMetadata.id));
      res.setHeader("Upload-Offset", 0);
      res.setHeader("Upload-Length", 0);
      res.setHeader("x-uploader-file-exist", true);
    }
    if (_onUploadCreate) {
      const result = await _onUploadCreate(req, res, uploadMetadata);
      return result;
    }
    return res;
  };
  const onUploadFinish = async (req, res, uploadMetadata) => {
    res.setHeader("x-uploader-file-exist", true);
    uploadMetadata = formatMetadata(uploadMetadata);
    await rewriteMetaDataFile(uploadMetadata);
    const fileType = uploadMetadata.metadata?.filetype || uploadMetadata.metadata?.type || "";
    const fileName = uploadMetadata.metadata?.filename || uploadMetadata.metadata?.name || "";
    const fileExt = path.extname(fileName).toLowerCase();
    const isImageFile = fileType.startsWith("image/");
    const isBinaryFile = ["zip", "gz", "tgz", "7z", "dmg", "pkg", "apk"].includes(fileExt.replace(".", ""));
    if (isImageFile && !isBinaryFile && uploadMetadata.runtime?.absolutePath) {
      try {
        await removeExifFromFile(uploadMetadata.runtime.absolutePath);
      } catch (err) {
        logger.error("failed to remove EXIF from file", err);
      }
    }
    if (_onUploadFinish) {
      try {
        const result = await _onUploadFinish(req, res, uploadMetadata);
        return result;
      } catch (err) {
        logger.error("@blocklet/uploader: onUploadFinish error: ", err);
        newServer.delete(uploadMetadata.id);
        res.setHeader("x-uploader-file-exist", false);
        throw err;
      }
    }
    return res;
  };
  const newServer = new Server({
    path: "/",
    // UNUSED
    relativeLocation: true,
    // respectForwardedHeaders: true,
    namingFunction: (req) => {
      const fileName = getFileName(req);
      const filePath = path.join(_path, fileName);
      validFilePathInDirPath(_path, filePath);
      return fileName;
    },
    datastore,
    onUploadFinish: async (req, res, uploadMetadata) => {
      uploadMetadata = formatMetadata(uploadMetadata);
      const result = await onUploadFinish(req, res, uploadMetadata);
      if (result && !result.send) {
        const body = typeof result === "string" ? result : JSON.stringify(result);
        throw { body, status_code: 200 };
      } else {
        return result;
      }
    },
    onUploadCreate,
    ...restProps
  });
  app.use((req, res, next) => {
    req.uploaderProps = {
      server: newServer,
      onUploadFinish,
      onUploadCreate
    };
    next();
  });
  cron.init({
    context: {},
    jobs: [
      {
        name: "auto-cleanup-expired-uploads",
        time: "0 0 * * * *",
        // each hour
        fn: () => {
          try {
            newServer.cleanUpExpiredUploads().then((count) => {
              logger.info(`@blocklet/uploader: cleanup expired uploads done: ${count}`);
            }).catch((err) => {
              logger.error(`@blocklet/uploader: cleanup expired uploads error`, err);
            });
          } catch (err) {
            logger.error(`@blocklet/uploader: cleanup expired uploads error`, err);
          }
        },
        options: { runOnInit: false }
      }
    ],
    onError: (err) => {
      logger.error("@blocklet/uploader: cleanup job failed", err);
    }
  });
  newServer.delete = async (key) => {
    try {
      await configstore.delete(key);
      await configstore.delete(key, false);
    } catch (err) {
      logger.error("@blocklet/uploader: delete error: ", err);
    }
  };
  newServer.on(EVENTS.POST_RECEIVE, async (req, res, uploadMetadata) => {
    uploadMetadata = formatMetadata(uploadMetadata);
    await rewriteMetaDataFile(uploadMetadata);
  });
  app.all("*", setHeaders, fileExistBeforeUpload, newServer.handle.bind(newServer));
  newServer.handle = app;
  return newServer;
}
export const getFileName = (req) => {
  const ext = req.headers["x-uploader-file-ext"];
  const randomName = `${crypto.randomBytes(16).toString("hex")}${ext ? `.${ext}` : ""}`;
  return req.headers["x-uploader-file-name"] || randomName;
};
export function getFileNameParam(req, res, { isRequired = true } = {}) {
  let { fileName } = req.params;
  if (!fileName) {
    fileName = req.originalUrl.replace(req.baseUrl, "");
  }
  if (!fileName && isRequired) {
    res.status(400).json({ error: 'Parameter "fileName" is required' });
    return;
  }
  return fileName;
}
export function getLocalStorageFile({ server }) {
  return async (req, res, next) => {
    const fileName = getFileNameParam(req, res);
    const filePath = path.join(server.datastore.directory, fileName);
    validFilePathInDirPath(server.datastore.directory, filePath);
    const fileExists = await fs.stat(filePath).catch(() => false);
    if (!fileExists) {
      res.status(404).json({ error: "file not found" });
      return;
    }
    setHeaders(req, res);
    const file = await fs.readFile(filePath);
    res.send(file);
    next?.();
  };
}
export function setHeaders(req, res, next) {
  let { method } = req;
  method = method.toUpperCase();
  const fileName = getFileNameParam(req, res, {
    isRequired: false
  });
  if (req.headers["x-uploader-endpoint-url"]) {
    const query = new URL(req.headers["x-uploader-endpoint-url"]).searchParams;
    req.query = {
      ...Object.fromEntries(query),
      // query params convert to object
      ...req.query
    };
  }
  if (method === "POST" && req.headers["x-uploader-base-url"]) {
    req.baseUrl = req.headers["x-uploader-base-url"];
  }
  if (method === "GET" && fileName) {
    const contentType = mime.lookup(fileName);
    if (contentType) {
      res.setHeader("Content-Type", contentType);
    }
    setPDFDownloadHeader(req, res);
  }
  next?.();
}
export async function fileExistBeforeUpload(req, res, next) {
  let { method, uploaderProps } = req;
  method = method.toUpperCase();
  if (["PATCH", "POST"].includes(method)) {
    const _path = uploaderProps.server.datastore.directory;
    const fileName = getFileName(req);
    const filePath = path.join(_path, fileName);
    validFilePathInDirPath(_path, filePath);
    const isExist = await fs.stat(filePath).catch(() => false);
    if (isExist) {
      const metaData = await getMetaDataByFilePath(filePath);
      if (isExist?.size >= 0 && isExist?.size === metaData?.size) {
        const prepareUpload = method === "POST";
        if (prepareUpload) {
          res.status(200);
          res.setHeader("Location", joinUrl(req.headers["x-uploader-base-url"], fileName));
          res.setHeader("Upload-Offset", +metaData.offset);
          res.setHeader("Upload-Length", +metaData.size);
        }
        if (req.headers["x-uploader-metadata"]) {
          try {
            const realMetaData = JSON.parse(req.headers["x-uploader-metadata"], (key, value) => {
              if (typeof value === "string") {
                return decodeURIComponent(value);
              }
              return value;
            });
            metaData.metadata = {
              ...metaData.metadata,
              ...realMetaData
            };
          } catch (err) {
            logger.error("@blocklet/uploader: parse metadata error: ", err);
          }
        }
        const uploadResult = await uploaderProps.onUploadFinish(req, res, metaData);
        res.json(uploadResult);
        return;
      }
    }
  }
  next?.();
}
export async function getMetaDataByFilePath(filePath) {
  const metaDataPath = `${filePath}.json`;
  const isExist = await fs.stat(filePath).catch(() => false);
  if (isExist) {
    try {
      const metaData = await fs.readFile(metaDataPath, "utf-8");
      const metaDataJson = JSON.parse(metaData);
      return metaDataJson;
    } catch (err) {
      logger.error("@blocklet/uploader: getMetaDataByPath error: ", err);
    }
  }
  return null;
}
export function joinUrl(...args) {
  const realArgs = args.filter(Boolean).map((item) => {
    if (item === "/") {
      return "";
    }
    return item;
  });
  return joinUrlLib(...realArgs);
}
class RewriteFileConfigstore {
  directory;
  queue;
  constructor(path2) {
    this.directory = path2;
    this.queue = new queue({ concurrency: 1 });
  }
  async get(key) {
    try {
      const buffer = await this.queue.add(() => fs.readFile(this.resolve(key), "utf8"));
      const metadata = JSON.parse(buffer);
      if (metadata.offset !== metadata.size) {
        const info = await fs.stat(this.resolve(key, false)).catch(() => false);
        if (info?.size !== metadata?.offset) {
          metadata.offset = info.size;
          this.set(key, metadata);
        }
      }
      return metadata;
    } catch {
      return void 0;
    }
  }
  async set(key, value) {
    if (value?.runtime) {
      delete value.runtime;
    }
    if (value?.metadata?.runtime) {
      delete value.metadata.runtime;
    }
    await this.queue.add(() => fs.writeFile(this.resolve(key), JSON.stringify(value)));
  }
  async safeDeleteFile(filePath) {
    validFilePathInDirPath(this.directory, filePath);
    try {
      const isExist = await fs.stat(filePath).catch(() => false);
      if (isExist) {
        await fs.rm(filePath);
      } else {
        logger.log("Can not remove file, the file not exist: ", filePath);
      }
    } catch (err) {
      logger.error("@blocklet/uploader: safeDeleteFile error: ", err);
    }
  }
  async delete(key, isMetadata = true) {
    try {
      await this.queue.add(() => this.safeDeleteFile(this.resolve(key, isMetadata)));
    } catch (err) {
      logger.error("@blocklet/uploader: delete error: ", err);
    }
  }
  async list() {
    return this.queue.add(async () => {
      const files = await fs.readdir(this.directory, { withFileTypes: true });
      const promises = files.filter((file) => file.isFile() && file.name.endsWith(".json")).map((file) => {
        return file.name.replace(".json", "");
      });
      return Promise.all(promises);
    });
  }
  resolve(key, isMetadata = true) {
    let fileKey = key;
    if (isMetadata) {
      fileKey = `${key}.json`;
    }
    return path.join(this.directory, fileKey);
  }
}
class RewriteFileStore extends FileStore {
  constructor(options) {
    super(options);
  }
  async remove(key) {
    this.configstore.delete(key);
    this.configstore.delete(key, false);
  }
}