UNPKG

node-csfd-api

Version:

ČSFD API in JavaScript. Amazing NPM library for scrapping csfd.cz :)

github.com/bartholomej/node-csfd-api

bartholomej/node-csfd-api

1 lines • 8.46 kB
1
{"version":3,"file":"index.cjs","names":["createAnubisClient","fetchSafe","CsfdError"],"sources":["../../src/fetchers/index.ts"],"sourcesContent":["import { createAnubisClient } from '../anubis';\nimport { CsfdError } from '../errors';\nimport { fetchSafe } from './fetch.polyfill';\n\ninterface BrowserProfile {\n 'User-Agent': string;\n 'Sec-Ch-Ua': string;\n 'Sec-Ch-Ua-Platform': string;\n}\n\nconst browserProfiles: BrowserProfile[] = [\n // Chrome 131 / Windows\n {\n 'User-Agent':\n 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',\n 'Sec-Ch-Ua': '\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"',\n 'Sec-Ch-Ua-Platform': '\"Windows\"'\n },\n // Chrome 130 / Windows\n {\n 'User-Agent':\n 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',\n 'Sec-Ch-Ua': '\"Google Chrome\";v=\"130\", \"Chromium\";v=\"130\", \"Not_A Brand\";v=\"24\"',\n 'Sec-Ch-Ua-Platform': '\"Windows\"'\n },\n // Chrome 131 / macOS\n {\n 'User-Agent':\n 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',\n 'Sec-Ch-Ua': '\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"',\n 'Sec-Ch-Ua-Platform': '\"macOS\"'\n },\n // Chrome 130 / macOS\n {\n 'User-Agent':\n 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',\n 'Sec-Ch-Ua': '\"Google Chrome\";v=\"130\", \"Chromium\";v=\"130\", \"Not_A Brand\";v=\"24\"',\n 'Sec-Ch-Ua-Platform': '\"macOS\"'\n },\n // Edge 131 / Windows\n {\n 'User-Agent':\n 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0',\n 'Sec-Ch-Ua': '\"Microsoft Edge\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"',\n 'Sec-Ch-Ua-Platform': '\"Windows\"'\n },\n // Edge 130 / Windows\n {\n 'User-Agent':\n 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0',\n 'Sec-Ch-Ua': '\"Microsoft Edge\";v=\"130\", \"Chromium\";v=\"130\", \"Not_A Brand\";v=\"24\"',\n 'Sec-Ch-Ua-Platform': '\"Windows\"'\n }\n];\n\nconst randomProfile = (): BrowserProfile =>\n browserProfiles[Math.floor(Math.random() * browserProfiles.length)];\n\nconst baseHeaders = {\n Accept:\n 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',\n 'Accept-Language': 'cs-CZ,cs;q=0.9,en-US;q=0.8,en;q=0.7',\n 'Accept-Encoding': 'gzip, deflate, br',\n 'Cache-Control': 'max-age=0',\n Connection: 'keep-alive',\n 'Sec-Ch-Ua-Mobile': '?0',\n 'Sec-Fetch-Dest': 'document',\n 'Sec-Fetch-Mode': 'navigate',\n 'Sec-Fetch-Site': 'none',\n 'Sec-Fetch-User': '?1',\n 'Upgrade-Insecure-Requests': '1'\n};\n\nconst anubis = createAnubisClient({ fetch: fetchSafe });\n\n/**\n * The cached Anubis cookie, so it can be persisted between runs. Anubis binds\n * it to the client IP for a week, so it is only reusable from the same address.\n */\nexport const getAnubisCookie = (): string | null => anubis.getCookie();\n\n/** Seed the cookie cache, e.g. from a previous run or a per-tenant store. */\nexport const setAnubisCookie = (cookie: string | null): void => anubis.setCookie(cookie);\n\n/** Forget the cookie so the next request solves a fresh challenge. */\nexport const resetAnubisCookie = (): void => anubis.reset();\n\nconst buildHeaders = (optionsRequest?: RequestInit): Headers => {\n const mergedHeaders = new Headers({ ...baseHeaders, ...randomProfile() });\n\n // Merge any custom headers provided in the function arguments\n if (optionsRequest?.headers) {\n const reqHeaders = new Headers(optionsRequest.headers);\n reqHeaders.forEach((value, key) => mergedHeaders.set(key, value));\n }\n\n const cookie = anubis.getCookie();\n if (cookie) {\n const existing = mergedHeaders.get('Cookie');\n mergedHeaders.set('Cookie', existing ? `${cookie}; ${existing}` : cookie);\n }\n\n return mergedHeaders;\n};\n\n/**\n * Fetch a ČSFD page, passing an anti-bot challenge if one is served.\n *\n * @throws {CsfdError} when the page cannot be retrieved. Never returns a\n * placeholder body: letting one reach the parsers turns a plain failure into an\n * unrelated crash deep inside them.\n */\nexport const fetchPage = async (url: string, optionsRequest?: RequestInit): Promise<string> => {\n const { headers: _, ...restOptions } = optionsRequest || {};\n\n const doFetch = async (): Promise<Response> => {\n let response: Response;\n try {\n // Stay credential-less by default so no ambient session rides along; only\n // runtimes that hide Set-Cookie need their jar, and only once detected.\n response = await fetchSafe(url, {\n credentials: anubis.usesPlatformCookieJar() ? 'include' : 'omit',\n ...restOptions,\n headers: buildHeaders(optionsRequest)\n });\n } catch (e: unknown) {\n throw new CsfdError('network', url, `Request failed for url: ${url}`, { cause: e });\n }\n\n if (!response.ok) {\n throw new CsfdError(\n response.status === 404 ? 'not-found' : 'http',\n url,\n `Bad response ${response.status} for url: ${url}`,\n { status: response.status }\n );\n }\n return response;\n };\n\n let response = await doFetch();\n let html = await response.text();\n\n if (anubis.isChallenge(html)) {\n // A failure inside the exchange is just another way of not getting through,\n // so it is reported as `blocked` with the original error kept as the cause.\n let passed = false;\n let exchangeError: unknown;\n try {\n passed = await anubis.pass(\n html,\n response.headers,\n url,\n new Headers({ ...baseHeaders, ...randomProfile() })\n );\n } catch (e: unknown) {\n exchangeError = e;\n }\n\n if (passed) {\n response = await doFetch();\n html = await response.text();\n }\n\n if (anubis.isChallenge(html)) {\n throw new CsfdError(\n 'blocked',\n url,\n `Anti-bot challenge could not be passed for url: ${url}. You may be rate-limited or blocked by ČSFD.`,\n { cause: exchangeError }\n );\n }\n }\n\n return html;\n};\n"],"mappings":";;;;AAUA,MAAM,kBAAoC;CAExC;EACE,cACE;EACF,aAAa;EACb,sBAAsB;CACxB;CAEA;EACE,cACE;EACF,aAAa;EACb,sBAAsB;CACxB;CAEA;EACE,cACE;EACF,aAAa;EACb,sBAAsB;CACxB;CAEA;EACE,cACE;EACF,aAAa;EACb,sBAAsB;CACxB;CAEA;EACE,cACE;EACF,aAAa;EACb,sBAAsB;CACxB;CAEA;EACE,cACE;EACF,aAAa;EACb,sBAAsB;CACxB;AACF;AAEA,MAAM,sBACJ,gBAAgB,KAAK,MAAM,KAAK,OAAO,IAAI,gBAAgB,MAAM;AAEnE,MAAM,cAAc;CAClB,QACE;CACF,mBAAmB;CACnB,mBAAmB;CACnB,iBAAiB;CACjB,YAAY;CACZ,oBAAoB;CACpB,kBAAkB;CAClB,kBAAkB;CAClB,kBAAkB;CAClB,kBAAkB;CAClB,6BAA6B;AAC/B;AAEA,MAAM,SAASA,eAAAA,mBAAmB,EAAE,OAAOC,uBAAAA,UAAU,CAAC;;;;;AAMtD,MAAa,wBAAuC,OAAO,UAAU;;AAGrE,MAAa,mBAAmB,WAAgC,OAAO,UAAU,MAAM;;AAGvF,MAAa,0BAAgC,OAAO,MAAM;AAE1D,MAAM,gBAAgB,mBAA0C;CAC9D,MAAM,gBAAgB,IAAI,QAAQ;EAAE,GAAG;EAAa,GAAG,cAAc;CAAE,CAAC;CAGxE,IAAI,gBAAgB,SAElB,IADuB,QAAQ,eAAe,OACrC,CAAC,CAAC,SAAS,OAAO,QAAQ,cAAc,IAAI,KAAK,KAAK,CAAC;CAGlE,MAAM,SAAS,OAAO,UAAU;CAChC,IAAI,QAAQ;EACV,MAAM,WAAW,cAAc,IAAI,QAAQ;EAC3C,cAAc,IAAI,UAAU,WAAW,GAAG,OAAO,IAAI,aAAa,MAAM;CAC1E;CAEA,OAAO;AACT;;;;;;;;AASA,MAAa,YAAY,OAAO,KAAa,mBAAkD;CAC7F,MAAM,EAAE,SAAS,GAAG,GAAG,gBAAgB,kBAAkB,CAAC;CAE1D,MAAM,UAAU,YAA+B;EAC7C,IAAI;EACJ,IAAI;GAGF,WAAW,MAAMA,uBAAAA,UAAU,KAAK;IAC9B,aAAa,OAAO,sBAAsB,IAAI,YAAY;IAC1D,GAAG;IACH,SAAS,aAAa,cAAc;GACtC,CAAC;EACH,SAAS,GAAY;GACnB,MAAM,IAAIC,eAAAA,UAAU,WAAW,KAAK,2BAA2B,OAAO,EAAE,OAAO,EAAE,CAAC;EACpF;EAEA,IAAI,CAAC,SAAS,IACZ,MAAM,IAAIA,eAAAA,UACR,SAAS,WAAW,MAAM,cAAc,QACxC,KACA,gBAAgB,SAAS,OAAO,YAAY,OAC5C,EAAE,QAAQ,SAAS,OAAO,CAC5B;EAEF,OAAO;CACT;CAEA,IAAI,WAAW,MAAM,QAAQ;CAC7B,IAAI,OAAO,MAAM,SAAS,KAAK;CAE/B,IAAI,OAAO,YAAY,IAAI,GAAG;EAG5B,IAAI,SAAS;EACb,IAAI;EACJ,IAAI;GACF,SAAS,MAAM,OAAO,KACpB,MACA,SAAS,SACT,KACA,IAAI,QAAQ;IAAE,GAAG;IAAa,GAAG,cAAc;GAAE,CAAC,CACpD;EACF,SAAS,GAAY;GACnB,gBAAgB;EAClB;EAEA,IAAI,QAAQ;GACV,WAAW,MAAM,QAAQ;GACzB,OAAO,MAAM,SAAS,KAAK;EAC7B;EAEA,IAAI,OAAO,YAAY,IAAI,GACzB,MAAM,IAAIA,eAAAA,UACR,WACA,KACA,mDAAmD,IAAI,gDACvD,EAAE,OAAO,cAAc,CACzB;CAEJ;CAEA,OAAO;AACT"}