website-scrap-engine
Version:
Configurable website scraper in typescript
67 lines • 2.2 kB
JavaScript
import * as path from 'node:path';
import { promises } from 'node:fs';
import { ResourceType } from '../resource.js';
import { error as errorLogger } from '../logger/logger.js';
const FILE_PREFIX = 'file://';
export async function readOrCopyLocalResource(res, requestOptions, options) {
var _a;
if (res.body) {
return res;
}
if (!res.downloadLink.startsWith(FILE_PREFIX)) {
return res;
}
if (!res.downloadStartTimestamp) {
res.downloadStartTimestamp = Date.now();
res.waitTime = res.downloadStartTimestamp - res.createTimestamp;
}
let fileSrcPath = res.downloadLink.slice(FILE_PREFIX.length);
if (!fileSrcPath) {
return;
}
// index.html handling
let stats = void 0;
if (res.type === ResourceType.Html) {
stats = await promises.stat(fileSrcPath);
if (stats.isDirectory()) {
for (const index of ['index.html', 'index.htm']) {
if (await promises.access(fileSrcPath + '/' + index)
.then(() => true).catch(() => false)) {
fileSrcPath += '/' + index;
break;
}
}
}
}
if (res.type === ResourceType.StreamingBinary) {
const fileDestPath = path.join((_a = res.localRoot) !== null && _a !== void 0 ? _a : options.localRoot, res.savePath);
await promises.copyFile(fileSrcPath, fileDestPath);
}
else {
res.body = await promises.readFile(fileSrcPath, {
encoding: res.encoding
});
}
try {
if (!stats) {
stats = await promises.stat(fileSrcPath);
}
if (stats) {
res.meta.headers = {
'last-modified': stats.mtime.toISOString(),
'content-length': stats.size.toString()
};
}
}
catch (e) {
errorLogger.warn('stat ' + fileSrcPath, e);
}
res.finishTimestamp = Date.now();
res.downloadTime =
res.finishTimestamp - res.downloadStartTimestamp;
if (res.type === ResourceType.StreamingBinary) {
return;
}
return res;
}
//# sourceMappingURL=read-or-copy-local-resource.js.map