UNPKG

rsshub

Version:
552 lines (551 loc) • 22.9 kB
import { t as rofetch } from "./ofetch-C3ts-Hud.mjs"; import { t as config } from "./config-CCmw1BNE.mjs"; import { t as logger } from "./logger-BKy98Y8B.mjs"; import { t as proxyExport } from "./proxy-Dbs7fWph.mjs"; import { t as cache_default } from "./cache-BkqOokyU.mjs"; import { t as InvalidParameterError } from "./invalid-parameter-CGxhjomn.mjs"; import { t as ConfigNotFoundError } from "./config-not-found-fQ5mxvDU.mjs"; import { r as getTwitterUserCacheKey, t as api_default$2 } from "./api-DAI6sKK3.mjs"; import x, { ProxyAgent } from "undici"; import { Cookie, CookieJar } from "tough-cookie"; import queryString from "query-string"; import { CookieAgent, cookie } from "http-cookie-agent/undici"; //#region lib/routes/twitter/api/web-api/gql-id-resolver.ts const CACHE_KEY = "twitter:gql-query-ids"; const fallbackIds = { UserTweets: "eoJ5zbv51Z_KVl81v9PmLQ", UserByScreenName: "Gb-d6r0vxPOADdG62OEBpQ", HomeTimeline: "3b9_7tltt0hJRef-xm_3sw", HomeLatestTimeline: "m1G65W9TS1-g-AllrKKYDQ", UserTweetsAndReplies: "wc5DRl4VaW5lSqJ8YbftZQ", UserMedia: "2DC9TKrcUzwGC_QskSVl5w", UserByRestId: "xvmVfRLmnr1alc5f2dib0Q", SearchTimeline: "BGd0T_j7oVwlW5U79tO_0A", ListLatestTweetsTimeline: "jW040BLUjh8X6Tw2ODQufA", TweetDetail: "559hs_YZNV4IgA3Z6zIIuw" }; const operationNames = Object.keys(fallbackIds); async function fetchAndExtractIds() { const api = await rofetch("https://cdn.jsdelivr.net/gh/fa0311/TwitterInternalAPIDocument@master/docs/json/API.json"); const ids = {}; for (const name of operationNames) { const queryId = api?.graphql?.[name]?.queryId; if (typeof queryId === "string") ids[name] = queryId; } return ids; } let resolvePromise = null; async function resolveQueryIds() { const cached = await cache_default.get(CACHE_KEY); if (cached) try { const parsed = typeof cached === "string" ? JSON.parse(cached) : cached; if (parsed && typeof parsed === "object" && Object.keys(parsed).length > 0) { logger.debug("twitter gql-id-resolver: using cached query IDs"); return { ...fallbackIds, ...parsed }; } } catch {} if (!resolvePromise) resolvePromise = (async () => { try { logger.info("twitter gql-id-resolver: fetching fresh query IDs from TwitterInternalAPIDocument"); const ids = await fetchAndExtractIds(); if (Object.keys(ids).length > 0) { await cache_default.set(CACHE_KEY, JSON.stringify(ids), config.cache.contentExpire); const found = operationNames.filter((name) => ids[name]); const missing = operationNames.filter((name) => !Object.hasOwn(ids, name)); logger.debug(`twitter gql-id-resolver: resolved ${found.length}/${operationNames.length} query IDs. Missing: ${missing.join(", ") || "none"}`); } else logger.warn("twitter gql-id-resolver: failed to extract any query IDs, using fallback"); return ids; } catch (error) { logger.warn(`twitter gql-id-resolver: error fetching query IDs: ${error}. Using fallback.`); return {}; } finally { resolvePromise = null; } })(); const ids = await resolvePromise; return { ...fallbackIds, ...ids }; } function buildGqlMap(queryIds) { const map = {}; for (const name of operationNames) map[name] = `/graphql/${queryIds[name] || fallbackIds[name]}/${name}`; return map; } //#endregion //#region lib/routes/twitter/api/web-api/constants.ts const baseUrl = "https://x.com/i/api"; let gqlMap = buildGqlMap(fallbackIds); const initGqlMap = async () => { gqlMap = buildGqlMap(await resolveQueryIds()); }; const thirdPartySupportedAPI = [ "UserByScreenName", "UserByRestId", "UserTweets", "UserTweetsAndReplies", "ListLatestTweetsTimeline", "SearchTimeline", "UserMedia" ]; const gqlFeatureUser = { hidden_profile_subscriptions_enabled: true, rweb_tipjar_consumption_enabled: true, responsive_web_graphql_exclude_directive_enabled: true, verified_phone_label_enabled: false, subscriptions_verification_info_is_identity_verified_enabled: true, subscriptions_verification_info_verified_since_enabled: true, highlights_tweets_tab_ui_enabled: true, responsive_web_twitter_article_notes_tab_enabled: true, subscriptions_feature_can_gift_premium: true, creator_subscriptions_tweet_preview_api_enabled: true, responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, responsive_web_graphql_timeline_navigation_enabled: true }; const gqlFeatureFeed = { rweb_tipjar_consumption_enabled: true, responsive_web_graphql_exclude_directive_enabled: true, verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true, responsive_web_graphql_timeline_navigation_enabled: true, responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, communities_web_enable_tweet_community_results_fetch: true, c9s_tweet_anatomy_moderator_badge_enabled: true, articles_preview_enabled: true, responsive_web_edit_tweet_api_enabled: true, graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true, responsive_web_twitter_article_tweet_consumption_enabled: true, tweet_awards_web_tipping_enabled: false, creator_subscriptions_quote_tweet_preview_enabled: false, freedom_of_speech_not_reach_fetch_enabled: true, standardized_nudges_misinfo: true, tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, rweb_video_timestamps_enabled: true, longform_notetweets_rich_text_read_enabled: true, longform_notetweets_inline_media_enabled: true, responsive_web_enhance_cards_enabled: false }; const TweetDetailFeatures = { rweb_tipjar_consumption_enabled: true, responsive_web_graphql_exclude_directive_enabled: true, verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true, responsive_web_graphql_timeline_navigation_enabled: true, responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, communities_web_enable_tweet_community_results_fetch: true, c9s_tweet_anatomy_moderator_badge_enabled: true, articles_preview_enabled: true, responsive_web_edit_tweet_api_enabled: true, graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true, responsive_web_twitter_article_tweet_consumption_enabled: true, tweet_awards_web_tipping_enabled: false, creator_subscriptions_quote_tweet_preview_enabled: false, freedom_of_speech_not_reach_fetch_enabled: true, standardized_nudges_misinfo: true, tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, rweb_video_timestamps_enabled: true, longform_notetweets_rich_text_read_enabled: true, longform_notetweets_inline_media_enabled: true, responsive_web_enhance_cards_enabled: false }; const gqlFeatures = { UserByScreenName: gqlFeatureUser, UserByRestId: gqlFeatureUser, UserTweets: gqlFeatureFeed, UserTweetsAndReplies: gqlFeatureFeed, UserMedia: gqlFeatureFeed, SearchTimeline: gqlFeatureFeed, ListLatestTweetsTimeline: gqlFeatureFeed, HomeTimeline: gqlFeatureFeed, HomeLatestTimeline: TweetDetailFeatures, TweetDetail: TweetDetailFeatures, Likes: gqlFeatureFeed }; const bearerToken = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"; //#endregion //#region lib/routes/twitter/api/web-api/utils.ts let authTokenIndex = 0; const token2Cookie = async (token) => { const c = await cache_default.get(`twitter:cookie:${token}`); if (c) return c; const jar = new CookieJar(); await jar.setCookie(`auth_token=${token}`, "https://x.com"); try { const agent = proxyExport.proxyUri ? new ProxyAgent({ uri: proxyExport.proxyUri }).compose(cookie({ jar })) : new CookieAgent({ cookies: { jar } }); if (token) await rofetch("https://x.com", { dispatcher: agent }); else { const gt = (await rofetch("https://x.com/narendramodi?mx=2", { dispatcher: agent })).match(/document\.cookie="gt=(\d+)/)?.[1]; if (gt) jar.setCookieSync(`gt=${gt}`, "https://x.com"); } const cookie$1 = JSON.stringify(jar.serializeSync()); cache_default.set(`twitter:cookie:${token}`, cookie$1); return cookie$1; } catch { return ""; } }; const lockPrefix = "twitter:lock-token1:"; const getAuth = async (retry) => { if (!config.twitter.authToken || retry <= 0) return; const index = authTokenIndex++ % config.twitter.authToken.length; const token = config.twitter.authToken[index]; if (await cache_default.get(`${lockPrefix}${token}`, false)) { logger.debug(`twitter debug: twitter cookie for token ${token} is locked, retry: ${retry}`); await new Promise((resolve) => setTimeout(resolve, Math.random() * 500 + 500)); return await getAuth(retry - 1); } logger.debug(`twitter debug: lock twitter cookie for token ${token}`); await cache_default.set(`${lockPrefix}${token}`, "1", 20); return { token }; }; const twitterGot = async (url, params, options) => { const auth = await getAuth(30); if (!auth && !options?.allowNoAuth) throw new ConfigNotFoundError("No valid Twitter token found"); const requestUrl = `${url}?${queryString.stringify(params)}`; let cookie$2 = await token2Cookie(auth?.token); let dispatchers; if (cookie$2) { logger.debug(`twitter debug: got twitter cookie for token ${auth?.token}`); if (typeof cookie$2 === "string") cookie$2 = JSON.parse(cookie$2); const jar = CookieJar.deserializeSync(cookie$2); const agent = proxyExport.proxyUri ? new ProxyAgent({ uri: proxyExport.proxyUri }).compose(cookie({ jar })) : new CookieAgent({ cookies: { jar } }); if (proxyExport.proxyUri) logger.debug(`twitter debug: Proxying request: ${requestUrl}`); dispatchers = { jar, agent }; } else if (auth) throw new ConfigNotFoundError(`Twitter cookie for token ${auth?.token?.replace(/(\w{8})(\w+)/, (_, v1, v2) => v1 + "*".repeat(v2.length))} is not valid`); const jsonCookie = dispatchers ? Object.fromEntries(dispatchers.jar.getCookieStringSync(url).split(";").map((c) => Cookie.parse(c)?.toJSON()).map((c) => [c?.key, c?.value])) : {}; const response = await x.fetch(requestUrl, { headers: { authority: "x.com", accept: "*/*", "accept-language": "en-US,en;q=0.9", authorization: bearerToken, "cache-control": "no-cache", "content-type": "application/json", dnt: "1", pragma: "no-cache", referer: "https://x.com/", "x-twitter-active-user": "yes", "x-twitter-client-language": "en", "x-csrf-token": jsonCookie.ct0, ...auth?.token ? { "x-twitter-auth-type": "OAuth2Session" } : { "x-guest-token": jsonCookie.gt } }, dispatcher: dispatchers?.agent }); let responseData; try { responseData = await response.json(); } catch { responseData = null; } const remaining = response.headers.get("x-rate-limit-remaining"); const remainingInt = Number.parseInt(remaining || "0"); const reset = response.headers.get("x-rate-limit-reset"); logger.debug(`twitter debug: twitter rate limit remaining for token ${auth?.token} is ${remaining} and reset at ${reset}, auth: ${JSON.stringify(auth)}, status: ${response.status}, data: ${JSON.stringify(responseData?.data)}, cookie: ${JSON.stringify(dispatchers?.jar.serializeSync())}`); if (auth) if (remaining && remainingInt < 2 && reset) { const delay = ((/* @__PURE__ */ new Date(Number.parseInt(reset) * 1e3)).getTime() - Date.now()) / 1e3; logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}, will unlock after ${delay}s`); await cache_default.set(`${lockPrefix}${auth.token}`, "1", Math.ceil(delay) * 2); } else if (response.status === 429 || JSON.stringify(responseData?.data) === "{\"user\":{}}") { logger.debug(`twitter debug: twitter rate limit exceeded for token ${auth.token} with status ${response.status}`); await cache_default.set(`${lockPrefix}${auth.token}`, "1", 2e3); } else if (response.status === 403 || response.status === 401) { const tokenIndex = config.twitter.authToken?.indexOf(auth.token); if (tokenIndex !== void 0 && tokenIndex !== -1) config.twitter.authToken?.splice(tokenIndex, 1); logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`); await cache_default.set(`${lockPrefix}${auth.token}`, "1", 3600); } else { logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`); await cache_default.set(`${lockPrefix}${auth.token}`, "", 1); } if (response.status >= 400) throw new Error(`Twitter API error: ${response.status}`); if (auth?.token) { logger.debug(`twitter debug: update twitter cookie for token ${auth.token}`); await cache_default.set(`twitter:cookie:${auth.token}`, JSON.stringify(dispatchers?.jar.serializeSync()), config.cache.contentExpire); } return responseData; }; const paginationTweets = async (endpoint, userId, variables, path) => { const params = { variables: JSON.stringify({ ...variables, userId }), features: JSON.stringify(gqlFeatures[endpoint]) }; const fetchData = async () => { if (config.twitter.thirdPartyApi && thirdPartySupportedAPI.includes(endpoint)) { const { data } = await rofetch(`${config.twitter.thirdPartyApi}${gqlMap[endpoint]}`, { method: "GET", params, headers: { "accept-encoding": "gzip" } }); return data; } const { data } = await twitterGot(baseUrl + gqlMap[endpoint], params); return data; }; const getInstructions = (data) => { if (path) { let instructions = data; for (const p of path) instructions = instructions[p]; return instructions.instructions; } const userResult = data?.user?.result; const instructions = (userResult?.timeline?.timeline || userResult?.timeline?.timeline_v2 || userResult?.timeline_v2?.timeline)?.instructions; if (!instructions) logger.debug(`twitter debug: instructions not found in data: ${JSON.stringify(data)}`); return instructions; }; const instructions = getInstructions(await fetchData()); if (!instructions) return []; const moduleItems = instructions.find((i) => i.type === "TimelineAddToModule")?.moduleItems; const entries = instructions.find((i) => i.type === "TimelineAddEntries")?.entries; return entries.find((i) => i.entryId === "profile-grid-0")?.content?.items || moduleItems || entries || []; }; const hydrateLegacyUser = (legacy, tweet) => { const userResult = tweet.core?.user_results?.result; if (!userResult) return; legacy.user = { name: userResult.core?.name, screen_name: userResult.core?.screen_name, profile_image_url_https: userResult.avatar?.image_url }; }; function gatherLegacyFromData(entries, filterNested, userId) { const tweets = []; const filteredEntries = []; for (const entry of entries) { const entryId = entry.entryId; if (entryId) { if (entryId.startsWith("tweet-") || entryId.startsWith("profile-grid-0-tweet-")) filteredEntries.push(entry); if (filterNested && filterNested.some((f) => entryId.startsWith(f))) filteredEntries.push(...entry.content.items); } } for (const entry of filteredEntries) { if (!entry.entryId) continue; const content = entry.content || entry.item; let tweet = content?.content?.tweetResult?.result || content?.itemContent?.tweet_results?.result; if (tweet?.__typename === "TweetPreviewDisplay") { const preview = tweet.tweet; if (preview?.rest_id) { const userResult = preview.core?.user_results?.result; const fakeLegacy = { id_str: preview.rest_id, full_text: `[Subscribers Only] ${preview.text ?? ""}`, created_at: preview.created_at ?? "", entities: { urls: [], hashtags: [], symbols: [], user_mentions: [] }, user_id_str: userResult?.rest_id ?? "", favorite_count: preview.favorite_count ?? 0, reply_count: preview.reply_count ?? 0, retweet_count: preview.retweet_count ?? 0 }; hydrateLegacyUser(fakeLegacy, { core: preview.core, rest_id: preview.rest_id }); if (userId === void 0 || fakeLegacy.user_id_str === userId + "") tweets.push(fakeLegacy); } continue; } if (tweet && tweet.tweet) tweet = tweet.tweet; if (tweet) { const retweet = tweet.legacy?.retweeted_status_result?.result; for (const t of [tweet, retweet]) { if (!t?.legacy) continue; hydrateLegacyUser(t.legacy, t); t.legacy.id_str = t.rest_id; const quote = t.quoted_status_result?.result?.tweet || t.quoted_status_result?.result; if (quote?.legacy) { t.legacy.quoted_status = quote.legacy; hydrateLegacyUser(t.legacy.quoted_status, quote); } if (t.note_tweet) { const tmp = t.note_tweet.note_tweet_results.result; t.legacy.entities.hashtags = tmp.entity_set.hashtags; t.legacy.entities.symbols = tmp.entity_set.symbols; t.legacy.entities.urls = tmp.entity_set.urls; t.legacy.entities.user_mentions = tmp.entity_set.user_mentions; t.legacy.full_text = tmp.text; } } const legacy = tweet.legacy; if (legacy) { if (retweet) legacy.retweeted_status = retweet.legacy; if (userId === void 0 || legacy.user_id_str === userId + "") tweets.push(legacy); } } } return tweets; } //#endregion //#region lib/routes/twitter/api/web-api/api.ts const getUserData = (id) => cache_default.tryGet(`twitter-userdata-${id}`, () => { const params = { variables: id.startsWith("+") ? JSON.stringify({ userId: id.slice(1), withSafetyModeUserFields: true }) : JSON.stringify({ screen_name: id, withSafetyModeUserFields: true }), features: JSON.stringify(id.startsWith("+") ? gqlFeatures.UserByRestId : gqlFeatures.UserByScreenName), fieldToggles: JSON.stringify({ withAuxiliaryUserLabels: false }) }; if (config.twitter.thirdPartyApi) { const endpoint = id.startsWith("+") ? gqlMap.UserByRestId : gqlMap.UserByScreenName; return rofetch(`${config.twitter.thirdPartyApi}${endpoint}`, { method: "GET", params, headers: { "accept-encoding": "gzip" } }); } return twitterGot(`${baseUrl}${id.startsWith("+") ? gqlMap.UserByRestId : gqlMap.UserByScreenName}`, params, { allowNoAuth: !id.startsWith("+") }); }); const cacheTryGet = async (_id, params, operationName, func) => { const id = (await getUserData(_id)).data?.user?.result?.rest_id; if (id === void 0) { cache_default.set(`twitter-userdata-${_id}`, "", config.cache.contentExpire); throw new InvalidParameterError("User not found"); } return cache_default.tryGet(getTwitterUserCacheKey(id, operationName, params), () => func(id, params), config.cache.routeExpire, false); }; const getUserTweets = (id, params) => cacheTryGet(id, params, "getUserTweets", async (id, params = {}) => gatherLegacyFromData(await paginationTweets("UserTweets", id, { ...params, count: 20, includePromotedContent: true, withQuickPromoteEligibilityTweetFields: true, withVoice: true, withV2Timeline: true }))); const getUserTweetsAndReplies = (id, params) => cacheTryGet(id, params, "getUserTweetsAndReplies", async (id, params = {}) => gatherLegacyFromData(await paginationTweets("UserTweetsAndReplies", id, { ...params, count: 20, includePromotedContent: true, withCommunity: true, withVoice: true, withV2Timeline: true }), ["profile-conversation-"], id)); const getUserMedia = (id, params) => cacheTryGet(id, params, "getUserMedia", async (id, params = {}) => gatherLegacyFromData(await paginationTweets("UserMedia", id, { ...params, count: 20, includePromotedContent: false, withClientEventToken: false, withBirdwatchNotes: false, withVoice: true, withV2Timeline: true }))); const getUserLikes = (id, params) => cacheTryGet(id, params, "getUserLikes", async (id, params = {}) => gatherLegacyFromData(await paginationTweets("Likes", id, { ...params, includeHasBirdwatchNotes: false, includePromotedContent: false, withBirdwatchNotes: false, withVoice: false, withV2Timeline: true }))); const getUserTweet = (id, params) => cacheTryGet(id, params, "getUserTweet", async (id, params = {}) => gatherLegacyFromData(await paginationTweets("TweetDetail", id, { ...params, includeHasBirdwatchNotes: false, includePromotedContent: false, withBirdwatchNotes: false, withVoice: false, withV2Timeline: true }, ["threaded_conversation_with_injections_v2"]), ["homeConversation-", "conversationthread-"])); const getSearch = async (keywords, params) => gatherLegacyFromData(await paginationTweets("SearchTimeline", void 0, { ...params, rawQuery: keywords, count: 20, querySource: "typed_query", product: "Latest" }, [ "search_by_raw_query", "search_timeline", "timeline" ])); const getList = async (id, params) => gatherLegacyFromData(await paginationTweets("ListLatestTweetsTimeline", void 0, { ...params, listId: id, count: 20 }, [ "list", "tweets_timeline", "timeline" ]), ["listConversation-"]); const getUser = async (id) => { const userData = await getUserData(id); if (!userData.data.user) throw new InvalidParameterError("This account doesn't exist"); if (userData.data.user.result.__typename === "UserUnavailable") throw new InvalidParameterError(userData.data.user.result.message || "User is unavailable"); return { profile_image_url: userData.data?.user?.result?.avatar?.image_url, description: userData.data?.user?.result?.profile_bio?.description, ...userData.data?.user?.result?.core }; }; const getHomeTimeline = async (id, params) => gatherLegacyFromData(await paginationTweets("HomeTimeline", void 0, { ...params, count: 20, includePromotedContent: true, latestControlAvailable: true, requestContext: "launch", withCommunity: true }, ["home", "home_timeline_urt"])); const getHomeLatestTimeline = async (id, params) => gatherLegacyFromData(await paginationTweets("HomeLatestTimeline", void 0, { ...params, count: 20, includePromotedContent: true, latestControlAvailable: true, requestContext: "launch", withCommunity: true }, ["home", "home_timeline_urt"])); var api_default$1 = { getUser, getUserTweets, getUserTweetsAndReplies, getUserMedia, getUserLikes, getUserTweet, getSearch, getList, getHomeTimeline, getHomeLatestTimeline, init: initGqlMap }; //#endregion //#region lib/routes/twitter/api/index.ts const enableThirdPartyApi = config.twitter.thirdPartyApi; const enableWebApi = config.twitter.authToken; const enableDeveloperApi = config.twitter.consumerKey && config.twitter.consumerSecret; let api = { init: () => { throw new ConfigNotFoundError("Twitter API is not configured"); }, getUser: () => null, getUserTweets: () => null, getUserTweetsAndReplies: () => null, getUserMedia: () => null, getUserLikes: () => null, getUserTweet: () => null, getSearch: () => null, getList: () => null, getHomeTimeline: () => null, getHomeLatestTimeline: () => null }; if (enableThirdPartyApi || enableWebApi) api = api_default$1; else if (enableDeveloperApi) api = api_default$2; var api_default = api; //#endregion export { api_default as t };