ssr-to-html
Version:
Crawl a server rendered application and output html files
188 lines (187 loc) • 6.55 kB
JavaScript
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __markAsModule = (target) => __defProp(target, "__esModule", { value: true });
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __reExport = (target, module2, copyDefault, desc) => {
if (module2 && typeof module2 === "object" || typeof module2 === "function") {
for (let key of __getOwnPropNames(module2))
if (!__hasOwnProp.call(target, key) && (copyDefault || key !== "default"))
__defProp(target, key, { get: () => module2[key], enumerable: !(desc = __getOwnPropDesc(module2, key)) || desc.enumerable });
}
return target;
};
var __toESM = (module2, isNodeMode) => {
return __reExport(__markAsModule(__defProp(module2 != null ? __create(__getProtoOf(module2)) : {}, "default", !isNodeMode && module2 && module2.__esModule ? { get: () => module2.default, enumerable: true } : { value: module2, enumerable: true })), module2);
};
var __toCommonJS = /* @__PURE__ */ ((cache) => {
return (module2, temp) => {
return cache && cache.get(module2) || (temp = __reExport(__markAsModule({}), module2, 1), cache && cache.set(module2, temp), temp);
};
})(typeof WeakMap !== "undefined" ? /* @__PURE__ */ new WeakMap() : 0);
var src_exports = {};
__export(src_exports, {
default: () => crawl
});
var import_fs = __toESM(require("fs"));
var import_net = __toESM(require("net"));
var import_path = __toESM(require("path"));
var import_undici = require("undici");
var import_htmlparser2 = require("htmlparser2");
var import_abort_controller = require("abort-controller");
var import_start_server = __toESM(require("./start-server"));
const noop = () => {
};
const ignoredRels = /* @__PURE__ */ new Set(["nofollow", "enclosure", "external"]);
const contentType = "text/html";
async function crawl(opts) {
const { port = await getAvailablePort(), wait = 3e4 } = opts;
const origin = `http://localhost:${port}`;
const closeServer = await (0, import_start_server.default)(opts.cmd, port, wait);
const out = import_path.default.resolve(opts.out);
const notFoundPath = resolvePath(opts.notFoundPath || "/404/", origin);
const startPaths = opts.paths ? opts.paths.map((path) => resolvePath(path, origin)).concat(notFoundPath) : ["/", notFoundPath];
const seen = new Set(startPaths);
const client = new import_undici.Pool(origin);
let queue;
try {
queue = startPaths.map(visit);
while (queue.length) {
const pending = Promise.all(queue);
queue = [];
await pending;
}
} finally {
closeServer();
await client.close();
}
async function visit(path) {
var _a;
const parser = new import_htmlparser2.Parser({
onopentag(name, attrs) {
const href = resolveHref(name, attrs);
const path2 = href && resolvePath(href, origin);
if (path2 !== void 0 && !seen.has(path2)) {
seen.add(path2);
queue.push(visit(path2));
}
}
});
const dirname = import_path.default.join(out, path);
const abortController = new import_abort_controller.AbortController();
let fsWriter;
try {
const res = await client.request({
path,
method: "GET",
signal: abortController.signal,
headers: { accept: contentType }
});
if (!((_a = res.headers["content-type"]) == null ? void 0 : _a.includes(contentType))) {
res.body.on("error", noop);
abortController.abort();
return;
}
let redirect;
switch (res.statusCode) {
case 200:
break;
case 404:
if (path !== notFoundPath) {
redirect = origin + notFoundPath;
}
break;
case 301: {
redirect = res.headers.location;
const redirectPath = resolvePath(redirect, origin);
if (redirectPath && !seen.has(redirectPath)) {
seen.add(redirectPath);
queue.push(visit(redirectPath));
}
break;
}
default:
res.body.on("error", noop);
abortController.abort();
throw new Error(`Unexpected status code ${res.statusCode} was discovered while crawling.`);
}
await import_fs.default.promises.mkdir(dirname, { recursive: true }).catch(noop);
fsWriter = import_fs.default.createWriteStream(import_path.default.join(dirname, "index.html"));
if (redirect) {
res.body.on("error", noop);
abortController.abort();
fsWriter.write(`<!DOCTYPE html><meta http-equiv=Refresh content="0;url=${redirect.replace(/"/g, "(")}">`);
} else {
for await (const data of res.body) {
fsWriter.write(data);
parser.write(data);
}
}
} finally {
fsWriter == null ? void 0 : fsWriter.end();
parser.end();
}
}
}
function resolveHref(tagName, attrs) {
switch (tagName) {
case "a":
if (attrs.href && !(attrs.download || ignoredRels.has(attrs.rel))) {
return attrs.href;
}
break;
case "link":
if (attrs.href) {
switch (attrs.rel) {
case "alternate":
case "author":
case "canonical":
case "help":
case "license":
case "next":
case "prefetch":
case "prerender":
case "prev":
case "search":
case "tag":
return attrs.href;
case "preload":
if (attrs.as === "document") {
return attrs.href;
}
break;
}
}
break;
case "iframe":
return attrs.src;
}
}
function resolvePath(href, origin) {
const url = new URL(href, origin);
if (url.origin === origin) {
let { pathname } = url;
const lastChar = pathname.length - 1;
if (pathname[lastChar] !== "/") {
pathname += "/";
}
return pathname + url.search;
}
}
async function getAvailablePort() {
return new Promise((resolve) => {
const server = import_net.default.createServer().listen(0, () => {
const { port } = server.address();
server.close(() => resolve(port));
});
});
}
module.exports = __toCommonJS(src_exports);
// Annotate the CommonJS export names for ESM import in node:
0 && (module.exports = {});