ssr-to-html
Version:
Crawl a server rendered application and output html files
162 lines (159 loc) • 4.64 kB
JavaScript
import {
start_server_default
} from "./chunk-4NFP6AZR.mjs";
// src/index.ts
import fs from "fs";
import net from "net";
import nodePath from "path";
import { Pool } from "undici";
import { Parser } from "htmlparser2";
import { AbortController } from "abort-controller";
var noop = () => {
};
var ignoredRels = /* @__PURE__ */ new Set(["nofollow", "enclosure", "external"]);
var contentType = "text/html";
async function crawl(opts) {
const { port = await getAvailablePort(), wait = 3e4 } = opts;
const origin = `http://localhost:${port}`;
const closeServer = await start_server_default(opts.cmd, port, wait);
const out = nodePath.resolve(opts.out);
const notFoundPath = resolvePath(opts.notFoundPath || "/404/", origin);
const startPaths = opts.paths ? opts.paths.map((path) => resolvePath(path, origin)).concat(notFoundPath) : ["/", notFoundPath];
const seen = new Set(startPaths);
const client = new Pool(origin);
let queue;
try {
queue = startPaths.map(visit);
while (queue.length) {
const pending = Promise.all(queue);
queue = [];
await pending;
}
} finally {
closeServer();
await client.close();
}
async function visit(path) {
var _a;
const parser = new Parser({
onopentag(name, attrs) {
const href = resolveHref(name, attrs);
const path2 = href && resolvePath(href, origin);
if (path2 !== void 0 && !seen.has(path2)) {
seen.add(path2);
queue.push(visit(path2));
}
}
});
const dirname = nodePath.join(out, path);
const abortController = new AbortController();
let fsWriter;
try {
const res = await client.request({
path,
method: "GET",
signal: abortController.signal,
headers: { accept: contentType }
});
if (!((_a = res.headers["content-type"]) == null ? void 0 : _a.includes(contentType))) {
res.body.on("error", noop);
abortController.abort();
return;
}
let redirect;
switch (res.statusCode) {
case 200:
break;
case 404:
if (path !== notFoundPath) {
redirect = origin + notFoundPath;
}
break;
case 301: {
redirect = res.headers.location;
const redirectPath = resolvePath(redirect, origin);
if (redirectPath && !seen.has(redirectPath)) {
seen.add(redirectPath);
queue.push(visit(redirectPath));
}
break;
}
default:
res.body.on("error", noop);
abortController.abort();
throw new Error(`Unexpected status code ${res.statusCode} was discovered while crawling.`);
}
await fs.promises.mkdir(dirname, { recursive: true }).catch(noop);
fsWriter = fs.createWriteStream(nodePath.join(dirname, "index.html"));
if (redirect) {
res.body.on("error", noop);
abortController.abort();
fsWriter.write(`<!DOCTYPE html><meta http-equiv=Refresh content="0;url=${redirect.replace(/"/g, "(")}">`);
} else {
for await (const data of res.body) {
fsWriter.write(data);
parser.write(data);
}
}
} finally {
fsWriter == null ? void 0 : fsWriter.end();
parser.end();
}
}
}
function resolveHref(tagName, attrs) {
switch (tagName) {
case "a":
if (attrs.href && !(attrs.download || ignoredRels.has(attrs.rel))) {
return attrs.href;
}
break;
case "link":
if (attrs.href) {
switch (attrs.rel) {
case "alternate":
case "author":
case "canonical":
case "help":
case "license":
case "next":
case "prefetch":
case "prerender":
case "prev":
case "search":
case "tag":
return attrs.href;
case "preload":
if (attrs.as === "document") {
return attrs.href;
}
break;
}
}
break;
case "iframe":
return attrs.src;
}
}
function resolvePath(href, origin) {
const url = new URL(href, origin);
if (url.origin === origin) {
let { pathname } = url;
const lastChar = pathname.length - 1;
if (pathname[lastChar] !== "/") {
pathname += "/";
}
return pathname + url.search;
}
}
async function getAvailablePort() {
return new Promise((resolve) => {
const server = net.createServer().listen(0, () => {
const { port } = server.address();
server.close(() => resolve(port));
});
});
}
export {
crawl
};