xenforo-dl/dist/lib/utils/Fetcher.js

XenForo Forum Downloader

var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
    if (kind === "m") throw new TypeError("Private method is not writable");
    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
};
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
};
var _Fetcher_instances, _Fetcher_logger, _Fetcher_cookie, _Fetcher_fetchWithRedirect, _Fetcher_commitDownload, _Fetcher_cleanupDownload, _Fetcher_setHeaders, _Fetcher_assertResponseOK;
import * as fs from 'fs';
import fetch, { AbortError, Request } from 'node-fetch';
import { pipeline } from 'stream/promises';
import { URL } from 'url';
import path from 'path';
import { commonLog } from './logging/Logger.js';
import { ensureDirSync } from 'fs-extra';
import { sleepBeforeExecute } from './Misc.js';
import contentDisposition from 'content-disposition';
export class FetcherError extends Error {
    constructor(message, url, fatal = false) {
        super(message);
        this.name = 'FetcherError';
        this.url = url;
        this.fatal = fatal;
    }
}
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0';
export default class Fetcher {
    constructor(logger, cookie) {
        _Fetcher_instances.add(this);
        this.name = 'Fetcher';
        _Fetcher_logger.set(this, void 0);
        _Fetcher_cookie.set(this, void 0);
        __classPrivateFieldSet(this, _Fetcher_logger, logger, "f");
        __classPrivateFieldSet(this, _Fetcher_cookie, cookie, "f");
    }
    static async getInstance(logger, cookie) {
        return new Fetcher(logger, cookie);
    }
    async fetchHTML(args, rt = 0) {
        const { url, maxRetries, retryInterval, signal } = args;
        try {
            const res = await __classPrivateFieldGet(this, _Fetcher_instances, "m", _Fetcher_fetchWithRedirect).call(this, url, 'GET', signal);
            return {
                html: await res.text(),
                lastURL: res.url
            };
        }
        catch (error) {
            if (error instanceof AbortError || (error instanceof FetcherError && error.fatal)) {
                throw error;
            }
            if (rt < maxRetries) {
                this.log('error', `Error fetching "${url}" - will retry: `, error);
                return sleepBeforeExecute(() => this.fetchHTML({ url, maxRetries, retryInterval, signal }, rt + 1), retryInterval);
            }
            const errMsg = error instanceof Error ? error.message : error;
            const retriedMsg = rt > 0 ? ` (retried ${rt} times)` : '';
            throw new FetcherError(`${errMsg}${retriedMsg}`, url);
        }
    }
    async fetchFilenameByHeaders(args, rt = 0) {
        const { url, maxRetries, retryInterval, signal } = args;
        const urlObj = new URL(url);
        try {
            const res = await __classPrivateFieldGet(this, _Fetcher_instances, "m", _Fetcher_fetchWithRedirect).call(this, url, 'HEAD', signal);
            const disposition = res.headers.get('content-disposition');
            if (disposition) {
                const parsedDisposition = contentDisposition.parse(disposition);
                const filename = parsedDisposition.parameters['filename'] || null;
                return filename;
            }
            return null;
        }
        catch (error) {
            if (error instanceof AbortError || (error instanceof FetcherError && error.fatal)) {
                throw error;
            }
            if (rt < maxRetries) {
                this.log('error', `Error fetching "${url}" (HEAD) - will retry: `, error);
                return sleepBeforeExecute(() => this.fetchFilenameByHeaders({ url, maxRetries, retryInterval, signal }, rt + 1), retryInterval);
            }
            const errMsg = error instanceof Error ? error.message : error;
            const retriedMsg = rt > 0 ? ` (retried ${rt} times)` : '';
            throw new FetcherError(`${errMsg}${retriedMsg}`, urlObj.toString());
        }
    }
    async downloadAttachment(params, rt = 0) {
        const { src, dest, maxRetries, retryInterval, signal } = params;
        const request = new Request(src, { method: 'GET' });
        __classPrivateFieldGet(this, _Fetcher_instances, "m", _Fetcher_setHeaders).call(this, request);
        const res = await __classPrivateFieldGet(this, _Fetcher_instances, "m", _Fetcher_fetchWithRedirect).call(this, src, 'GET', signal);
        try {
            if (__classPrivateFieldGet(this, _Fetcher_instances, "m", _Fetcher_assertResponseOK).call(this, res, src)) {
                const destFilePath = path.resolve(dest);
                const { dir: destDir, base: destFilename } = path.parse(destFilePath);
                const tmpFilePath = path.resolve(destDir, `${destFilename}.part`);
                try {
                    ensureDirSync(destDir);
                    this.log('debug', `Download: "${src}" -> "${tmpFilePath}"`);
                    await pipeline(res.body, fs.createWriteStream(tmpFilePath));
                    __classPrivateFieldGet(this, _Fetcher_instances, "m", _Fetcher_commitDownload).call(this, tmpFilePath, destFilePath);
                    return;
                }
                catch (error) {
                    __classPrivateFieldGet(this, _Fetcher_instances, "m", _Fetcher_cleanupDownload).call(this, tmpFilePath);
                    throw error;
                }
            }
        }
        catch (error) {
            if (error instanceof AbortError || (error instanceof FetcherError && error.fatal)) {
                throw error;
            }
            if (rt < maxRetries) {
                this.log('error', `Error downloading attachment from  "${src}" - will retry: `, error);
                return sleepBeforeExecute(() => this.downloadAttachment(params, rt + 1), retryInterval);
            }
            const errMsg = error instanceof Error ? error.message : error;
            const retriedMsg = rt > 0 ? ` (retried ${rt} times)` : '';
            throw new FetcherError(`${errMsg}${retriedMsg}`, src);
        }
        return undefined;
    }
    log(level, ...msg) {
        commonLog(__classPrivateFieldGet(this, _Fetcher_logger, "f"), level, this.name, ...msg);
    }
}
_Fetcher_logger = new WeakMap(), _Fetcher_cookie = new WeakMap(), _Fetcher_instances = new WeakSet(), _Fetcher_fetchWithRedirect = async function _Fetcher_fetchWithRedirect(url, method, signal, useCookie = true) {
    const request = new Request(url, { method });
    __classPrivateFieldGet(this, _Fetcher_instances, "m", _Fetcher_setHeaders).call(this, request, useCookie);
    const res = await fetch(request, { signal, redirect: 'manual' });
    if (res.status >= 300 && res.status < 400) {
        const toURL = res.headers.get('Location');
        if (toURL) {
            this.log('debug', `HTTP Redirect: "${request.url}" -> "${toURL}"`);
            const redirectWithCookie = new URL(url).host === new URL(toURL).host;
            return __classPrivateFieldGet(this, _Fetcher_instances, "m", _Fetcher_fetchWithRedirect).call(this, toURL, method, signal, redirectWithCookie);
        }
        // We should never arrive here!
        return fetch(request);
    }
    return res;
}, _Fetcher_commitDownload = function _Fetcher_commitDownload(tmpFilePath, destFilePath) {
    try {
        this.log('debug', `Commit: "${tmpFilePath}" -> "${destFilePath} (filesize: ${fs.lstatSync(tmpFilePath).size} bytes)`);
        fs.renameSync(tmpFilePath, destFilePath);
    }
    finally {
        __classPrivateFieldGet(this, _Fetcher_instances, "m", _Fetcher_cleanupDownload).call(this, tmpFilePath);
    }
}, _Fetcher_cleanupDownload = function _Fetcher_cleanupDownload(tmpFilePath) {
    try {
        if (fs.existsSync(tmpFilePath)) {
            this.log('debug', `Cleanup "${tmpFilePath}"`);
            fs.unlinkSync(tmpFilePath);
        }
    }
    catch (error) {
        this.log('error', `Cleanup error "${tmpFilePath}":`, error);
    }
}, _Fetcher_setHeaders = function _Fetcher_setHeaders(request, setCookie = true) {
    request.headers.set('User-Agent', USER_AGENT);
    if (__classPrivateFieldGet(this, _Fetcher_cookie, "f") && setCookie) {
        request.headers.set('Cookie', __classPrivateFieldGet(this, _Fetcher_cookie, "f"));
    }
}, _Fetcher_assertResponseOK = function _Fetcher_assertResponseOK(response, originURL, requireBody = true) {
    if (!response) {
        throw new FetcherError('No response', originURL);
    }
    if (!response.ok) {
        throw new FetcherError(`${response.status} - ${response.statusText}`, originURL);
    }
    if (requireBody && !response.body) {
        throw new FetcherError('Empty response body', originURL);
    }
    return true;
};
//# sourceMappingURL=Fetcher.js.map