@the-convocation/twitter-scraper/dist/cycletls/cjs/index.cjs.map

A port of n0madic/twitter-scraper to Node.js.

{"version":3,"file":"index.cjs","sources":["../../../src/chrome-fingerprint.ts","../../../src/cycletls-fetch.ts"],"sourcesContent":["/**\n * Chrome version-dependent fingerprint constants.\n *\n * IMPORTANT: All constants in this file are tied to a specific Chrome version\n * (currently Chrome 144 on Windows 10). When bumping the Chrome version:\n *   1. Update CHROME_USER_AGENT with the new version string\n *   2. Update CHROME_SEC_CH_UA with the matching Client Hints\n *   3. Update CHROME_JA3 fingerprint (capture via tls.peet.ws)\n *   4. Update CHROME_JA4R fingerprint\n *   5. Update CHROME_HTTP2_FINGERPRINT settings frame\n *   6. Review CHROME_HEADER_ORDER if Chrome changes header ordering\n *   7. Update castle.ts DEFAULT_PROFILE if Chrome version affects fingerprint fields\n *\n * All values must be consistent with each other and match a real Chrome release.\n */\n\n/**\n * User-Agent string matching Chrome 144 on Windows 10.\n * Must be consistent across all requests and match the TLS fingerprint.\n */\nexport const CHROME_USER_AGENT =\n  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36';\n\n/**\n * Chrome Client Hints header matching the Chrome 144 user-agent.\n */\nexport const CHROME_SEC_CH_UA =\n  '\"Not(A:Brand\";v=\"8\", \"Chromium\";v=\"144\", \"Google Chrome\";v=\"144\"';\n\n/**\n * JA3 TLS fingerprint for Chrome 144.\n * Captured from a real Chrome 144 browser session via tls.peet.ws.\n */\nexport const CHROME_JA3 =\n  '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-10-35-16-11-51-27-65037-43-45-18-23-5-65281-13-17613,4588-29-23-24,0';\n\n/**\n * JA4r fingerprint for Chrome 144.\n * Format: t13d1516h2_CIPHERS_EXTENSIONS_SIG_ALGS\n */\nexport const CHROME_JA4R =\n  't13d1516h2_002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0012,0017,001b,0023,002b,002d,0033,44cd,fe0d,ff01_0403,0804,0401,0503,0805,0501,0806,0601';\n\n/**\n * Chrome 144 HTTP/2 fingerprint - mimics exact HTTP/2 SETTINGS frame.\n * Format: SETTINGS|window_size|unknown|priority_order\n */\nexport const CHROME_HTTP2_FINGERPRINT =\n  '1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p';\n\n/**\n * Exact header order that Chrome 144 uses.\n * Header ordering is critical for HTTP/2 fingerprint evasion — servers can detect\n * non-browser clients by checking if headers arrive in a non-standard order.\n */\nexport const CHROME_HEADER_ORDER = [\n  // HTTP/2 pseudo-headers (Chrome 144 order: method, authority, scheme, path)\n  ':method',\n  ':authority',\n  ':scheme',\n  ':path',\n  // Chrome Client Hints (mandatory for modern detection bypass)\n  'sec-ch-ua',\n  'sec-ch-ua-mobile',\n  'sec-ch-ua-platform',\n  // Standard browser headers\n  'upgrade-insecure-requests',\n  'user-agent',\n  'accept',\n  'origin',\n  'sec-fetch-site',\n  'sec-fetch-mode',\n  'sec-fetch-user',\n  'sec-fetch-dest',\n  'referer',\n  'accept-encoding',\n  'accept-language',\n  'priority',\n  // Authentication headers\n  'authorization',\n  'x-csrf-token',\n  'x-guest-token',\n  'x-twitter-auth-type',\n  'x-twitter-active-user',\n  'x-twitter-client-language',\n  'x-client-transaction-id',\n  'x-xp-forwarded-for',\n  // POST-specific\n  'content-type',\n  'cookie',\n];\n","import initCycleTLS from 'cycletls';\nimport { Headers } from 'headers-polyfill';\nimport debug from 'debug';\nimport {\n  CHROME_USER_AGENT,\n  CHROME_JA3,\n  CHROME_JA4R,\n  CHROME_HTTP2_FINGERPRINT,\n  CHROME_HEADER_ORDER,\n} from './chrome-fingerprint';\n\nconst log = debug('twitter-scraper:cycletls');\n\nlet cycleTLSInstance: Awaited<ReturnType<typeof initCycleTLS>> | null = null;\n\n/**\n * Initialize the CycleTLS instance. This should be called once before using the fetch wrapper.\n */\nexport async function initCycleTLSFetch() {\n  if (!cycleTLSInstance) {\n    log('Initializing CycleTLS...');\n    cycleTLSInstance = await initCycleTLS();\n    log('CycleTLS initialized successfully');\n  }\n  return cycleTLSInstance;\n}\n\n/**\n * Cleanup the CycleTLS instance. Call this when you're done making requests.\n */\nexport function cycleTLSExit() {\n  if (cycleTLSInstance) {\n    log('Exiting CycleTLS...');\n    cycleTLSInstance.exit();\n    cycleTLSInstance = null;\n  }\n}\n\n/**\n * A fetch-compatible wrapper around CycleTLS that mimics Chrome's TLS fingerprint\n * to bypass Cloudflare and other bot detection systems.\n */\nexport async function cycleTLSFetch(\n  input: RequestInfo | URL,\n  init?: RequestInit,\n): Promise<Response> {\n  const instance = await initCycleTLSFetch();\n\n  const url =\n    typeof input === 'string'\n      ? input\n      : input instanceof URL\n      ? input.toString()\n      : input.url;\n  const method = (init?.method || 'GET').toUpperCase();\n\n  log(`Making ${method} request to ${url}`);\n\n  // Extract headers from RequestInit\n  const headers: Record<string, string> = {};\n  if (init?.headers) {\n    if (init.headers instanceof Headers) {\n      init.headers.forEach((value, key) => {\n        headers[key] = value;\n      });\n    } else if (Array.isArray(init.headers)) {\n      init.headers.forEach(([key, value]) => {\n        headers[key] = value;\n      });\n    } else {\n      Object.assign(headers, init.headers);\n    }\n  }\n\n  // Convert body to string if needed\n  let body: string | undefined;\n  if (init?.body) {\n    if (typeof init.body === 'string') {\n      body = init.body;\n    } else if (init.body instanceof URLSearchParams) {\n      body = init.body.toString();\n    } else {\n      body = init.body.toString();\n    }\n  }\n\n  // All Chrome fingerprint constants imported from chrome-fingerprint.ts\n  const options = {\n    body,\n    headers,\n    ja3: CHROME_JA3,\n    ja4r: CHROME_JA4R,\n    http2Fingerprint: CHROME_HTTP2_FINGERPRINT,\n    headerOrder: CHROME_HEADER_ORDER,\n    orderAsProvided: true,\n    disableGrease: false,\n    userAgent: headers['user-agent'] || CHROME_USER_AGENT,\n  };\n\n  try {\n    const response = await instance(\n      url,\n      options,\n      method.toLowerCase() as\n        | 'get'\n        | 'post'\n        | 'put'\n        | 'delete'\n        | 'patch'\n        | 'head'\n        | 'options',\n    );\n\n    // Convert CycleTLS response to fetch Response\n    // CycleTLS returns headers as an object\n    const responseHeaders = new Headers();\n    if (response.headers) {\n      Object.entries(response.headers).forEach(([key, value]) => {\n        if (Array.isArray(value)) {\n          value.forEach((v) => {\n            responseHeaders.append(key, v);\n          });\n        } else if (typeof value === 'string') {\n          responseHeaders.set(key, value);\n        }\n      });\n    }\n\n    // Get response body - cycletls provides helper methods, but we need the raw text\n    // The response object has a text() method that returns the body as text\n    let responseBody = '';\n    if (typeof response.text === 'function') {\n      responseBody = await response.text();\n    } else if ((response as any).body) {\n      responseBody = (response as any).body;\n    }\n\n    // Create a proper Response object using standard Response constructor\n    const fetchResponse = new Response(responseBody, {\n      status: response.status,\n      statusText: '', // CycleTLS doesn't provide status text\n      headers: responseHeaders,\n    });\n\n    return fetchResponse;\n  } catch (error) {\n    log(`CycleTLS request failed: ${error}`);\n    throw error;\n  }\n}\n"],"names":["Headers"],"mappings":";;;;;;AAoBO,MAAM,iBACX,GAAA,iHAAA,CAAA;AAYK,MAAM,UACX,GAAA,yJAAA,CAAA;AAMK,MAAM,WACX,GAAA,qMAAA,CAAA;AAMK,MAAM,wBACX,GAAA,mDAAA,CAAA;AAOK,MAAM,mBAAsB,GAAA;AAAA;AAAA,EAEjC,SAAA;AAAA,EACA,YAAA;AAAA,EACA,SAAA;AAAA,EACA,OAAA;AAAA;AAAA,EAEA,WAAA;AAAA,EACA,kBAAA;AAAA,EACA,oBAAA;AAAA;AAAA,EAEA,2BAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,QAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,SAAA;AAAA,EACA,iBAAA;AAAA,EACA,iBAAA;AAAA,EACA,UAAA;AAAA;AAAA,EAEA,eAAA;AAAA,EACA,cAAA;AAAA,EACA,eAAA;AAAA,EACA,qBAAA;AAAA,EACA,uBAAA;AAAA,EACA,2BAAA;AAAA,EACA,yBAAA;AAAA,EACA,oBAAA;AAAA;AAAA,EAEA,cAAA;AAAA,EACA,QAAA;AACF,CAAA;;AC/EA,MAAM,GAAA,GAAM,MAAM,0BAA0B,CAAA,CAAA;AAE5C,IAAI,gBAAoE,GAAA,IAAA,CAAA;AAKxE,eAAsB,iBAAoB,GAAA;AACxC,EAAA,IAAI,CAAC,gBAAkB,EAAA;AACrB,IAAA,GAAA,CAAI,0BAA0B,CAAA,CAAA;AAC9B,IAAA,gBAAA,GAAmB,MAAM,YAAa,EAAA,CAAA;AACtC,IAAA,GAAA,CAAI,mCAAmC,CAAA,CAAA;AAAA,GACzC;AACA,EAAO,OAAA,gBAAA,CAAA;AACT,CAAA;AAKO,SAAS,YAAe,GAAA;AAC7B,EAAA,IAAI,gBAAkB,EAAA;AACpB,IAAA,GAAA,CAAI,qBAAqB,CAAA,CAAA;AACzB,IAAA,gBAAA,CAAiB,IAAK,EAAA,CAAA;AACtB,IAAmB,gBAAA,GAAA,IAAA,CAAA;AAAA,GACrB;AACF,CAAA;AAMsB,eAAA,aAAA,CACpB,OACA,IACmB,EAAA;AACnB,EAAM,MAAA,QAAA,GAAW,MAAM,iBAAkB,EAAA,CAAA;AAEzC,EAAM,MAAA,GAAA,GACJ,OAAO,KAAA,KAAU,QACb,GAAA,KAAA,GACA,iBAAiB,GACjB,GAAA,KAAA,CAAM,QAAS,EAAA,GACf,KAAM,CAAA,GAAA,CAAA;AACZ,EAAA,MAAM,MAAU,GAAA,CAAA,IAAA,EAAM,MAAU,IAAA,KAAA,EAAO,WAAY,EAAA,CAAA;AAEnD,EAAA,GAAA,CAAI,CAAU,OAAA,EAAA,MAAM,CAAe,YAAA,EAAA,GAAG,CAAE,CAAA,CAAA,CAAA;AAGxC,EAAA,MAAM,UAAkC,EAAC,CAAA;AACzC,EAAA,IAAI,MAAM,OAAS,EAAA;AACjB,IAAI,IAAA,IAAA,CAAK,mBAAmBA,uBAAS,EAAA;AACnC,MAAA,IAAA,CAAK,OAAQ,CAAA,OAAA,CAAQ,CAAC,KAAA,EAAO,GAAQ,KAAA;AACnC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACQ,MAAA,IAAA,KAAA,CAAM,OAAQ,CAAA,IAAA,CAAK,OAAO,CAAG,EAAA;AACtC,MAAA,IAAA,CAAK,QAAQ,OAAQ,CAAA,CAAC,CAAC,GAAA,EAAK,KAAK,CAAM,KAAA;AACrC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACI,MAAA;AACL,MAAO,MAAA,CAAA,MAAA,CAAO,OAAS,EAAA,IAAA,CAAK,OAAO,CAAA,CAAA;AAAA,KACrC;AAAA,GACF;AAGA,EAAI,IAAA,IAAA,CAAA;AACJ,EAAA,IAAI,MAAM,IAAM,EAAA;AACd,IAAI,IAAA,OAAO,IAAK,CAAA,IAAA,KAAS,QAAU,EAAA;AACjC,MAAA,IAAA,GAAO,IAAK,CAAA,IAAA,CAAA;AAAA,KACd,MAAA,IAAW,IAAK,CAAA,IAAA,YAAgB,eAAiB,EAAA;AAC/C,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KACrB,MAAA;AACL,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KAC5B;AAAA,GACF;AAGA,EAAA,MAAM,OAAU,GAAA;AAAA,IACd,IAAA;AAAA,IACA,OAAA;AAAA,IACA,GAAK,EAAA,UAAA;AAAA,IACL,IAAM,EAAA,WAAA;AAAA,IACN,gBAAkB,EAAA,wBAAA;AAAA,IAClB,WAAa,EAAA,mBAAA;AAAA,IACb,eAAiB,EAAA,IAAA;AAAA,IACjB,aAAe,EAAA,KAAA;AAAA,IACf,SAAA,EAAW,OAAQ,CAAA,YAAY,CAAK,IAAA,iBAAA;AAAA,GACtC,CAAA;AAEA,EAAI,IAAA;AACF,IAAA,MAAM,WAAW,MAAM,QAAA;AAAA,MACrB,GAAA;AAAA,MACA,OAAA;AAAA,MACA,OAAO,WAAY,EAAA;AAAA,KAQrB,CAAA;AAIA,IAAM,MAAA,eAAA,GAAkB,IAAIA,uBAAQ,EAAA,CAAA;AACpC,IAAA,IAAI,SAAS,OAAS,EAAA;AACpB,MAAO,MAAA,CAAA,OAAA,CAAQ,SAAS,OAAO,CAAA,CAAE,QAAQ,CAAC,CAAC,GAAK,EAAA,KAAK,CAAM,KAAA;AACzD,QAAI,IAAA,KAAA,CAAM,OAAQ,CAAA,KAAK,CAAG,EAAA;AACxB,UAAM,KAAA,CAAA,OAAA,CAAQ,CAAC,CAAM,KAAA;AACnB,YAAgB,eAAA,CAAA,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA;AAAA,WAC9B,CAAA,CAAA;AAAA,SACH,MAAA,IAAW,OAAO,KAAA,KAAU,QAAU,EAAA;AACpC,UAAgB,eAAA,CAAA,GAAA,CAAI,KAAK,KAAK,CAAA,CAAA;AAAA,SAChC;AAAA,OACD,CAAA,CAAA;AAAA,KACH;AAIA,IAAA,IAAI,YAAe,GAAA,EAAA,CAAA;AACnB,IAAI,IAAA,OAAO,QAAS,CAAA,IAAA,KAAS,UAAY,EAAA;AACvC,MAAe,YAAA,GAAA,MAAM,SAAS,IAAK,EAAA,CAAA;AAAA,KACrC,MAAA,IAAY,SAAiB,IAAM,EAAA;AACjC,MAAA,YAAA,GAAgB,QAAiB,CAAA,IAAA,CAAA;AAAA,KACnC;AAGA,IAAM,MAAA,aAAA,GAAgB,IAAI,QAAA,CAAS,YAAc,EAAA;AAAA,MAC/C,QAAQ,QAAS,CAAA,MAAA;AAAA,MACjB,UAAY,EAAA,EAAA;AAAA;AAAA,MACZ,OAAS,EAAA,eAAA;AAAA,KACV,CAAA,CAAA;AAED,IAAO,OAAA,aAAA,CAAA;AAAA,WACA,KAAO,EAAA;AACd,IAAI,GAAA,CAAA,CAAA,yBAAA,EAA4B,KAAK,CAAE,CAAA,CAAA,CAAA;AACvC,IAAM,MAAA,KAAA,CAAA;AAAA,GACR;AACF;;;;;"}