website-scrap-engine
Version:
Configurable website scraper in typescript
71 lines • 3.58 kB
JavaScript
import path from 'node:path';
import URI from 'urijs';
import { ResourceType, urlOfSavePath } from '../resource.js';
import { escapePath } from '../util.js';
import { writeFile } from '../io.js';
export function getResourceBodyFromHtml(res, options) {
if (!res.meta.doc) {
return res.body;
}
if (options.cheerioSerialize) {
return res.meta.doc.html(options.cheerioSerialize);
}
return res.meta.doc.html();
}
export function redirectHtml(relativePath, encoding) {
// language=HTML
return `<html lang="en">
<head>
<meta charset="${encoding || 'utf8'}">
<meta http-equiv="refresh" content="0; url=${relativePath}">
<script>location.replace('${relativePath}' + location.hash);</script>
<title>Redirecting</title>
</head>
</html>`;
}
export async function saveHtmlToDisk(res, options, pipeline) {
var _a, _b, _c, _d;
if (res.type !== ResourceType.Html) {
return res;
}
const localRoot = (_a = res.localRoot) !== null && _a !== void 0 ? _a : options.localRoot;
// https://github.com/website-local/website-scrap-engine/issues/174
let mtime = void 0;
if (options.preferRemoteLastModifiedTime && ((_c = (_b = res.meta) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['last-modified'])) {
mtime = Date.parse((_d = res.meta.headers) === null || _d === void 0 ? void 0 : _d['last-modified']);
}
if (res.redirectedUrl && res.redirectedUrl !== res.url) {
if (res.redirectedSavePath) {
if (res.redirectedSavePath !== res.savePath) {
const replaceUri = URI(urlOfSavePath(res.redirectedSavePath))
.relativeTo(urlOfSavePath(res.savePath));
const relativePath = escapePath(replaceUri.toString());
await writeFile(path.join(localRoot, decodeURI(res.savePath)), redirectHtml(relativePath, res.encoding), res.encoding, mtime);
const body = getResourceBodyFromHtml(res, options);
await writeFile(path.join(localRoot, decodeURI(res.redirectedSavePath)), body, res.encoding, mtime);
}
else {
const body = getResourceBodyFromHtml(res, options);
const filePath = path.join(localRoot, decodeURI(res.savePath));
await writeFile(filePath, body, res.encoding, mtime);
}
return;
}
const redirectResource = await pipeline.createResource(ResourceType.Html, res.depth, res.redirectedUrl, res.url, localRoot, undefined, res.refSavePath);
if (redirectResource.replacePath) {
const relativePath = escapePath(redirectResource.replacePath);
const savePath = decodeURI(res.savePath);
await writeFile(path.join(localRoot, savePath), redirectHtml(relativePath, res.encoding), res.encoding, mtime);
const redirectedResource = await pipeline.createResource(ResourceType.Html, res.depth, res.redirectedUrl, res.refUrl, res.localRoot, res.encoding, undefined, ResourceType.Html);
const redirectedSavePath = decodeURI(redirectedResource.savePath);
const body = getResourceBodyFromHtml(res, options);
await writeFile(path.join(localRoot, redirectedSavePath), body, res.encoding, mtime);
return;
}
}
const body = getResourceBodyFromHtml(res, options);
const filePath = path.join(localRoot, decodeURI(res.savePath));
await writeFile(filePath, body, res.encoding, mtime);
return;
}
//# sourceMappingURL=save-html-to-disk.js.map