website-scrap-engine
Version:
Configurable website scraper in typescript
279 lines • 11.1 kB
JavaScript
import path from 'node:path';
import { constants, createWriteStream, promises as fs } from 'node:fs';
import { pipeline } from 'node:stream';
import { promisify } from 'node:util';
import got, { HTTPError } from 'got';
import { ResourceType } from '../resource.js';
import { mkdirRetry } from '../io.js';
import { error as errorLogger, retry as retryLogger } from '../logger/logger.js';
import { isUrlHttp } from '../util.js';
const promisifyPipeline = promisify(pipeline);
export function isBytesAccepted(acceptRange) {
if (!acceptRange) {
return false;
}
const ranges = acceptRange.split(',');
for (let i = 0; i < ranges.length; i++) {
if (ranges[i] === 'bytes') {
return true;
}
}
return false;
}
export function isSameRangeStart(rangeStart, contentRange) {
if (!contentRange) {
return false;
}
let ranges = contentRange.split(',');
ranges = ranges[0].split(' ');
if (ranges.length < 2 || !ranges[1]) {
return false;
}
ranges = ranges[1].split('-');
return +ranges[0] === rangeStart;
}
export async function streamingDownloadToFile(res, requestOptions) {
const savePath = path.join(res.localRoot, decodeURI(res.savePath));
try {
await fs.access(savePath, constants.W_OK);
}
catch (e) {
// force cast for typescript 4.4
if (e && e.code === 'ENOENT') {
await mkdirRetry(path.dirname(savePath));
}
else {
throw e;
}
}
// force cast for typescript
const options = Object.assign({}, requestOptions, {
isStream: true
});
let fileWriteStream;
return new Promise((resolve, reject) => {
let rangeIsSupported;
let rangeStart;
const makeRequest = (retryCount) => {
let isRetry = false;
if (!rangeIsSupported && options.headers) {
rangeStart = undefined;
delete options.headers.range;
}
else if (rangeStart && rangeIsSupported) {
if (!options.headers) {
options.headers = {};
}
options.headers.range = `bytes=${rangeStart}-`;
fileWriteStream = createWriteStream(savePath, {
flags: 'a',
start: rangeStart
});
}
const request = got.stream(res.downloadLink, options);
request.retryCount = retryCount;
request.once('response', async (response) => {
response.retryCount = retryCount;
res.meta.headers = response.headers;
if (rangeIsSupported === undefined) {
if (isBytesAccepted(response.headers['accept-ranges'])) {
rangeIsSupported = true;
}
}
if (rangeIsSupported && fileWriteStream && rangeStart &&
(response.statusCode !== 206 ||
!isSameRangeStart(rangeStart, response.headers['content-range']))) {
errorLogger.warn('Unexpected response for range', rangeStart, response.headers['content-range'], response.statusCode);
rangeIsSupported = false;
fileWriteStream.destroy();
fileWriteStream = undefined;
rangeStart = undefined;
}
if (response.request.isAborted) {
// Canceled while downloading
//- will throw a `CancelError` or `TimeoutError` error
return;
}
// Download body
if (!fileWriteStream) {
fileWriteStream = createWriteStream(savePath, {
flags: 'w'
});
}
try {
// Download body directly to file
await promisifyPipeline(request, fileWriteStream);
}
catch (_a) {
// The same error is caught below.
// See request.once('error')
return;
}
// if (request._isAboutToError) {
// return;
// }
resolve(response);
});
const destroyStream = () => {
if (fileWriteStream) {
if (rangeIsSupported) {
if (rangeStart) {
rangeStart += fileWriteStream.bytesWritten;
}
else {
rangeStart = fileWriteStream.bytesWritten;
}
}
else {
rangeStart = undefined;
}
fileWriteStream.destroy();
}
else {
rangeStart = undefined;
}
fileWriteStream = undefined;
};
const onError = (error) => {
// https://developer.mozilla.org/docs/Web/HTTP/Headers/Range
// https://developer.mozilla.org/docs/Web/HTTP/Status/416
if (error instanceof HTTPError && error.response.statusCode === 416) {
errorLogger.warn('Unexpected response for range', rangeStart, error.response.headers['content-range'], error.response.statusCode);
rangeIsSupported = false;
}
destroyStream();
const { options } = request;
if (error instanceof HTTPError && !options.throwHttpErrors) {
const { response } = error;
resolve(response);
return;
}
if (!isRetry) {
let retry = -1;
if (error && error.message === 'premature close') {
retryLogger.warn(retryCount, res.downloadLink, 'manually retry on premature close',
// eslint-disable-next-line @typescript-eslint/no-explicit-any
error.name, error.code, error.event, error.message);
retry = retryCount * 200;
}
// these events might be accidentally unhandled
// eslint-disable-next-line @typescript-eslint/no-explicit-any
if (error && !error.retryLimitExceeded &&
(error.name === 'RequestError' || error.name === 'TimeoutError') &&
// RequestError: Cannot read property 'request' of undefined
// at Object.exports.default (got\dist\source\core\utils\timed-out.js:56:23)
// error.code === undefined
(error.code === 'ETIMEDOUT' || error.code === undefined)) {
retryLogger.warn(retryCount, res.downloadLink,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
`manually retry on ${error.event} timeout`, error.name, error.code, error.message);
retry = retryCount * 300;
}
if (retry > 0) {
makeRequest(retryCount + 1);
return;
}
}
reject(error);
};
request.once('error', onError);
request.once('retry', (newRetryCount) => {
destroyStream();
isRetry = true;
makeRequest(newRetryCount);
});
};
makeRequest(0);
});
}
export async function optionallySetLastModifiedTime(res, options) {
var _a, _b, _c;
// https://github.com/website-local/website-scrap-engine/issues/174
let mtime = void 0;
if (options.preferRemoteLastModifiedTime && ((_b = (_a = res.meta) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['last-modified'])) {
mtime = Date.parse((_c = res.meta.headers) === null || _c === void 0 ? void 0 : _c['last-modified']);
}
// void and NaN check
if (mtime) {
const savePath = path.join(res.localRoot, decodeURI(res.savePath));
try {
await fs.utimes(savePath, mtime, mtime);
}
catch (e) {
errorLogger.warn('skipping utimes ' + savePath, e);
}
}
}
export async function downloadStreamingResource(res, requestOptions, options) {
if (res.body) {
return res;
}
if (res.type !== ResourceType.StreamingBinary) {
return res;
}
if (!res.downloadStartTimestamp) {
res.downloadStartTimestamp = Date.now();
res.waitTime = res.downloadStartTimestamp - res.createTimestamp;
}
await streamingDownloadToFile(res, requestOptions);
await optionallySetLastModifiedTime(res, options);
/// Not needed before
// res.finishTimestamp = Date.now();
// res.downloadTime =
// res.finishTimestamp - res.downloadStartTimestamp;
return;
}
export function downloadStreamingResourceWithHook(beforeDownload, afterDownload, downloadError) {
if (!beforeDownload && !afterDownload && !downloadError) {
return downloadStreamingResource;
}
return async (res, requestOptions, options, pipeline) => {
if (res.body) {
return res;
}
if (res.type !== ResourceType.StreamingBinary) {
return res;
}
if (!isUrlHttp(res.downloadLink)) {
return res;
}
let resource = res;
if (beforeDownload) {
resource = await beforeDownload(res, requestOptions, options, pipeline);
if (!resource) {
return;
}
if (resource.body) {
return resource;
}
if (resource.shouldBeDiscardedFromDownload) {
return;
}
}
if (!res.downloadStartTimestamp) {
res.downloadStartTimestamp = Date.now();
res.waitTime = res.downloadStartTimestamp - res.createTimestamp;
}
if (!downloadError) {
await streamingDownloadToFile(res, requestOptions);
await optionallySetLastModifiedTime(res, options);
}
else {
try {
await streamingDownloadToFile(res, requestOptions);
await optionallySetLastModifiedTime(res, options);
}
catch (e) {
await downloadError(e, res, requestOptions, options, pipeline);
}
}
res.finishTimestamp = Date.now();
res.downloadTime =
res.finishTimestamp - res.downloadStartTimestamp;
if (afterDownload) {
await afterDownload(res, requestOptions, options, pipeline);
}
return;
};
}
//# sourceMappingURL=download-streaming-resource.js.map