website-scrap-engine
Version:
Configurable website scraper in typescript
36 lines • 2.17 kB
JavaScript
import path from 'node:path';
import { writeFile } from '../io.js';
export async function saveResourceToDisk(res, options, pipeline) {
var _a, _b, _c, _d;
const localRoot = (_a = res.localRoot) !== null && _a !== void 0 ? _a : options.localRoot;
// https://github.com/website-local/website-scrap-engine/issues/174
let mtime = void 0;
if (options.preferRemoteLastModifiedTime && ((_c = (_b = res.meta) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['last-modified'])) {
mtime = Date.parse((_d = res.meta.headers) === null || _d === void 0 ? void 0 : _d['last-modified']);
}
if (res.redirectedUrl && res.redirectedUrl !== res.url) {
if (res.redirectedSavePath) {
if (res.redirectedSavePath !== res.savePath) {
const redirectedSavePath = decodeURI(res.redirectedSavePath);
await writeFile(path.join(localRoot, redirectedSavePath), res.body, res.encoding, mtime);
}
const savePath = decodeURI(res.savePath);
await writeFile(path.join(localRoot, savePath), res.body, res.encoding, mtime);
return;
}
const redirectResource = await pipeline.createResource(res.type, res.depth, res.url, res.redirectedUrl, localRoot, res.encoding, undefined, res.type);
// maybe we can try module:fs/promises.symlink first
if (redirectResource.replacePath) {
const savePath = decodeURI(res.savePath);
await writeFile(path.join(localRoot, savePath), res.body, res.encoding, mtime);
const redirectedResource = await pipeline.createResource(res.type, res.depth, res.redirectedUrl, res.refUrl, res.localRoot, res.encoding, undefined, res.type);
const redirectedSavePath = decodeURI(redirectedResource.savePath);
await writeFile(path.join(localRoot, redirectedSavePath), res.body, res.encoding, mtime);
return;
}
}
const filePath = path.join(localRoot, decodeURI(res.savePath));
await writeFile(filePath, res.body, res.encoding, mtime);
return;
}
//# sourceMappingURL=save-resource-to-disk.js.map