node-csfd-api
Version:
ČSFD API in JavaScript. Amazing NPM library for scrapping csfd.cz :)
124 lines (123 loc) • 5.34 kB
JavaScript
const require_client = require("../anubis/client.cjs");
const require_errors = require("../errors.cjs");
const require_fetch_polyfill = require("./fetch.polyfill.cjs");
//#region src/fetchers/index.ts
const browserProfiles = [
{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Sec-Ch-Ua": "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"",
"Sec-Ch-Ua-Platform": "\"Windows\""
},
{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
"Sec-Ch-Ua": "\"Google Chrome\";v=\"130\", \"Chromium\";v=\"130\", \"Not_A Brand\";v=\"24\"",
"Sec-Ch-Ua-Platform": "\"Windows\""
},
{
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Sec-Ch-Ua": "\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"",
"Sec-Ch-Ua-Platform": "\"macOS\""
},
{
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
"Sec-Ch-Ua": "\"Google Chrome\";v=\"130\", \"Chromium\";v=\"130\", \"Not_A Brand\";v=\"24\"",
"Sec-Ch-Ua-Platform": "\"macOS\""
},
{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
"Sec-Ch-Ua": "\"Microsoft Edge\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"",
"Sec-Ch-Ua-Platform": "\"Windows\""
},
{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0",
"Sec-Ch-Ua": "\"Microsoft Edge\";v=\"130\", \"Chromium\";v=\"130\", \"Not_A Brand\";v=\"24\"",
"Sec-Ch-Ua-Platform": "\"Windows\""
}
];
const randomProfile = () => browserProfiles[Math.floor(Math.random() * browserProfiles.length)];
const baseHeaders = {
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Accept-Language": "cs-CZ,cs;q=0.9,en-US;q=0.8,en;q=0.7",
"Accept-Encoding": "gzip, deflate, br",
"Cache-Control": "max-age=0",
Connection: "keep-alive",
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1"
};
const anubis = require_client.createAnubisClient({ fetch: require_fetch_polyfill.fetchSafe });
/**
* The cached Anubis cookie, so it can be persisted between runs. Anubis binds
* it to the client IP for a week, so it is only reusable from the same address.
*/
const getAnubisCookie = () => anubis.getCookie();
/** Seed the cookie cache, e.g. from a previous run or a per-tenant store. */
const setAnubisCookie = (cookie) => anubis.setCookie(cookie);
/** Forget the cookie so the next request solves a fresh challenge. */
const resetAnubisCookie = () => anubis.reset();
const buildHeaders = (optionsRequest) => {
const mergedHeaders = new Headers({
...baseHeaders,
...randomProfile()
});
if (optionsRequest?.headers) new Headers(optionsRequest.headers).forEach((value, key) => mergedHeaders.set(key, value));
const cookie = anubis.getCookie();
if (cookie) {
const existing = mergedHeaders.get("Cookie");
mergedHeaders.set("Cookie", existing ? `${cookie}; ${existing}` : cookie);
}
return mergedHeaders;
};
/**
* Fetch a ČSFD page, passing an anti-bot challenge if one is served.
*
* @throws {CsfdError} when the page cannot be retrieved. Never returns a
* placeholder body: letting one reach the parsers turns a plain failure into an
* unrelated crash deep inside them.
*/
const fetchPage = async (url, optionsRequest) => {
const { headers: _, ...restOptions } = optionsRequest || {};
const doFetch = async () => {
let response;
try {
response = await require_fetch_polyfill.fetchSafe(url, {
credentials: anubis.usesPlatformCookieJar() ? "include" : "omit",
...restOptions,
headers: buildHeaders(optionsRequest)
});
} catch (e) {
throw new require_errors.CsfdError("network", url, `Request failed for url: ${url}`, { cause: e });
}
if (!response.ok) throw new require_errors.CsfdError(response.status === 404 ? "not-found" : "http", url, `Bad response ${response.status} for url: ${url}`, { status: response.status });
return response;
};
let response = await doFetch();
let html = await response.text();
if (anubis.isChallenge(html)) {
let passed = false;
let exchangeError;
try {
passed = await anubis.pass(html, response.headers, url, new Headers({
...baseHeaders,
...randomProfile()
}));
} catch (e) {
exchangeError = e;
}
if (passed) {
response = await doFetch();
html = await response.text();
}
if (anubis.isChallenge(html)) throw new require_errors.CsfdError("blocked", url, `Anti-bot challenge could not be passed for url: ${url}. You may be rate-limited or blocked by ČSFD.`, { cause: exchangeError });
}
return html;
};
//#endregion
exports.fetchPage = fetchPage;
exports.getAnubisCookie = getAnubisCookie;
exports.resetAnubisCookie = resetAnubisCookie;
exports.setAnubisCookie = setAnubisCookie;
//# sourceMappingURL=index.cjs.map