UNPKG

agent-twitter-summary

Version:

A twitter client for agents

1,575 lines (1,557 loc) 209 kB
'use strict'; var toughCookie = require('tough-cookie'); var setCookie = require('set-cookie-parser'); var headersPolyfill = require('headers-polyfill'); var twitterApiV2 = require('twitter-api-v2'); var typebox = require('@sinclair/typebox'); var value = require('@sinclair/typebox/value'); var OTPAuth = require('otpauth'); var stringify = require('json-stable-stringify'); var events = require('events'); var WebSocket = require('ws'); var wrtc = require('@roamhq/wrtc'); var fs = require('fs'); var path = require('path'); var child_process = require('child_process'); var tls = require('node:tls'); var node_crypto = require('node:crypto'); function _interopNamespaceDefault(e) { var n = Object.create(null); if (e) { Object.keys(e).forEach(function (k) { if (k !== 'default') { var d = Object.getOwnPropertyDescriptor(e, k); Object.defineProperty(n, k, d.get ? d : { enumerable: true, get: function () { return e[k]; } }); } }); } n.default = e; return Object.freeze(n); } var OTPAuth__namespace = /*#__PURE__*/_interopNamespaceDefault(OTPAuth); var fs__namespace = /*#__PURE__*/_interopNamespaceDefault(fs); class ApiError extends Error { constructor(response, data, message) { super(message); this.response = response; this.data = data; } static async fromResponse(response) { let data = void 0; try { data = await response.json(); } catch { try { data = await response.text(); } catch { } } return new ApiError(response, data, `Response status: ${response.status}`); } } class Platform { async randomizeCiphers() { const platform = await Platform.importPlatform(); await platform?.randomizeCiphers(); } static async importPlatform() { { const { platform } = await Promise.resolve().then(function () { return index; }); return platform; } } } async function updateCookieJar(cookieJar, headers) { const setCookieHeader = headers.get("set-cookie"); if (setCookieHeader) { const cookies = setCookie.splitCookiesString(setCookieHeader); for (const cookie of cookies.map((c) => toughCookie.Cookie.parse(c))) { if (!cookie) continue; await cookieJar.setCookie( cookie, `${cookie.secure ? "https" : "http"}://${cookie.domain}${cookie.path}` ); } } else if (typeof document !== "undefined") { for (const cookie of document.cookie.split(";")) { const hardCookie = toughCookie.Cookie.parse(cookie); if (hardCookie) { await cookieJar.setCookie(hardCookie, document.location.toString()); } } } } const bearerToken = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"; async function requestApi(url, auth, method = "GET", platform = new Platform()) { const headers = new headersPolyfill.Headers(); await auth.installTo(headers, url); await platform.randomizeCiphers(); let res; do { try { res = await auth.fetch(url, { method, headers, credentials: "include" }); } catch (err) { if (!(err instanceof Error)) { throw err; } return { success: false, err: new Error("Failed to perform request.") }; } await updateCookieJar(auth.cookieJar(), res.headers); if (res.status === 429) { const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining"); const xRateLimitReset = res.headers.get("x-rate-limit-reset"); if (xRateLimitRemaining == "0" && xRateLimitReset) { const currentTime = (/* @__PURE__ */ new Date()).valueOf() / 1e3; const timeDeltaMs = 1e3 * (parseInt(xRateLimitReset) - currentTime); await new Promise((resolve) => setTimeout(resolve, timeDeltaMs)); } } } while (res.status === 429); if (!res.ok) { return { success: false, err: await ApiError.fromResponse(res) }; } const value = await res.json(); if (res.headers.get("x-rate-limit-incoming") == "0") { auth.deleteToken(); return { success: true, value }; } else { return { success: true, value }; } } function addApiFeatures(o) { return { ...o, rweb_lists_timeline_redesign_enabled: true, responsive_web_graphql_exclude_directive_enabled: true, verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true, responsive_web_graphql_timeline_navigation_enabled: true, responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, tweetypie_unmention_optimization_enabled: true, responsive_web_edit_tweet_api_enabled: true, graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true, tweet_awards_web_tipping_enabled: false, freedom_of_speech_not_reach_fetch_enabled: true, standardized_nudges_misinfo: true, longform_notetweets_rich_text_read_enabled: true, responsive_web_enhance_cards_enabled: false, subscriptions_verification_info_enabled: true, subscriptions_verification_info_reason_enabled: true, subscriptions_verification_info_verified_since_enabled: true, super_follow_badge_privacy_enabled: false, super_follow_exclusive_tweet_notifications_enabled: false, super_follow_tweet_api_enabled: false, super_follow_user_api_enabled: false, android_graphql_skip_api_media_color_palette: false, creator_subscriptions_subscription_count_enabled: false, blue_business_profile_image_shape_enabled: false, unified_cards_ad_metadata_container_dynamic_card_content_query_enabled: false }; } function addApiParams(params, includeTweetReplies) { params.set("include_profile_interstitial_type", "1"); params.set("include_blocking", "1"); params.set("include_blocked_by", "1"); params.set("include_followed_by", "1"); params.set("include_want_retweets", "1"); params.set("include_mute_edge", "1"); params.set("include_can_dm", "1"); params.set("include_can_media_tag", "1"); params.set("include_ext_has_nft_avatar", "1"); params.set("include_ext_is_blue_verified", "1"); params.set("include_ext_verified_type", "1"); params.set("skip_status", "1"); params.set("cards_platform", "Web-12"); params.set("include_cards", "1"); params.set("include_ext_alt_text", "true"); params.set("include_ext_limited_action_results", "false"); params.set("include_quote_count", "true"); params.set("include_reply_count", "1"); params.set("tweet_mode", "extended"); params.set("include_ext_collab_control", "true"); params.set("include_ext_views", "true"); params.set("include_entities", "true"); params.set("include_user_entities", "true"); params.set("include_ext_media_color", "true"); params.set("include_ext_media_availability", "true"); params.set("include_ext_sensitive_media_warning", "true"); params.set("include_ext_trusted_friends_metadata", "true"); params.set("send_error_codes", "true"); params.set("simple_quoted_tweet", "true"); params.set("include_tweet_replies", `${includeTweetReplies}`); params.set( "ext", "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,collab_control,vibe" ); return params; } function withTransform(fetchFn, transform) { return async (input, init) => { const fetchArgs = await transform?.request?.(input, init) ?? [ input, init ]; const res = await fetchFn(...fetchArgs); return await transform?.response?.(res) ?? res; }; } class TwitterGuestAuth { constructor(bearerToken, options) { this.options = options; this.fetch = withTransform(options?.fetch ?? fetch, options?.transform); this.bearerToken = bearerToken; this.jar = new toughCookie.CookieJar(); this.v2Client = null; } cookieJar() { return this.jar; } getV2Client() { return this.v2Client ?? null; } loginWithV2(appKey, appSecret, accessToken, accessSecret) { const v2Client = new twitterApiV2.TwitterApi({ appKey, appSecret, accessToken, accessSecret }); this.v2Client = v2Client; } isLoggedIn() { return Promise.resolve(false); } async me() { return void 0; } // eslint-disable-next-line @typescript-eslint/no-unused-vars login(_username, _password, _email) { return this.updateGuestToken(); } logout() { this.deleteToken(); this.jar = new toughCookie.CookieJar(); return Promise.resolve(); } deleteToken() { delete this.guestToken; delete this.guestCreatedAt; } hasToken() { return this.guestToken != null; } authenticatedAt() { if (this.guestCreatedAt == null) { return null; } return new Date(this.guestCreatedAt); } async installTo(headers) { if (this.shouldUpdate()) { await this.updateGuestToken(); } const token = this.guestToken; if (token == null) { throw new Error("Authentication token is null or undefined."); } headers.set("authorization", `Bearer ${this.bearerToken}`); headers.set("x-guest-token", token); const cookies = await this.getCookies(); const xCsrfToken = cookies.find((cookie) => cookie.key === "ct0"); if (xCsrfToken) { headers.set("x-csrf-token", xCsrfToken.value); } headers.set("cookie", await this.getCookieString()); } getCookies() { return this.jar.getCookies(this.getCookieJarUrl()); } getCookieString() { return this.jar.getCookieString(this.getCookieJarUrl()); } async removeCookie(key) { const store = this.jar.store; const cookies = await this.jar.getCookies(this.getCookieJarUrl()); for (const cookie of cookies) { if (!cookie.domain || !cookie.path) continue; store.removeCookie(cookie.domain, cookie.path, key); if (typeof document !== "undefined") { document.cookie = `${cookie.key}=; Max-Age=0; path=${cookie.path}; domain=${cookie.domain}`; } } } getCookieJarUrl() { return typeof document !== "undefined" ? document.location.toString() : "https://twitter.com"; } /** * Updates the authentication state with a new guest token from the Twitter API. */ async updateGuestToken() { const guestActivateUrl = "https://api.twitter.com/1.1/guest/activate.json"; const headers = new headersPolyfill.Headers({ Authorization: `Bearer ${this.bearerToken}`, Cookie: await this.getCookieString() }); const res = await this.fetch(guestActivateUrl, { method: "POST", headers, referrerPolicy: "no-referrer" }); await updateCookieJar(this.jar, res.headers); if (!res.ok) { throw new Error(await res.text()); } const o = await res.json(); if (o == null || o["guest_token"] == null) { throw new Error("guest_token not found."); } const newGuestToken = o["guest_token"]; if (typeof newGuestToken !== "string") { throw new Error("guest_token was not a string."); } this.guestToken = newGuestToken; this.guestCreatedAt = /* @__PURE__ */ new Date(); } /** * Returns if the authentication token needs to be updated or not. * @returns `true` if the token needs to be updated; `false` otherwise. */ shouldUpdate() { return !this.hasToken() || this.guestCreatedAt != null && this.guestCreatedAt < new Date((/* @__PURE__ */ new Date()).valueOf() - 3 * 60 * 60 * 1e3); } } function getAvatarOriginalSizeUrl(avatarUrl) { return avatarUrl ? avatarUrl.replace("_normal", "") : void 0; } function parseProfile(user, isBlueVerified) { const profile = { avatar: getAvatarOriginalSizeUrl(user.profile_image_url_https), banner: user.profile_banner_url, biography: user.description, followersCount: user.followers_count, followingCount: user.friends_count, friendsCount: user.friends_count, mediaCount: user.media_count, isPrivate: user.protected ?? false, isVerified: user.verified, likesCount: user.favourites_count, listedCount: user.listed_count, location: user.location, name: user.name, pinnedTweetIds: user.pinned_tweet_ids_str, tweetsCount: user.statuses_count, url: `https://twitter.com/${user.screen_name}`, userId: user.id_str, username: user.screen_name, isBlueVerified: isBlueVerified ?? false, canDm: user.can_dm }; if (user.created_at != null) { profile.joined = new Date(Date.parse(user.created_at)); } const urls = user.entities?.url?.urls; if (urls?.length != null && urls?.length > 0) { profile.website = urls[0].expanded_url; } return profile; } async function getProfile(username, auth) { const params = new URLSearchParams(); params.set( "variables", stringify({ screen_name: username, withSafetyModeUserFields: true }) ?? "" ); params.set( "features", stringify({ hidden_profile_likes_enabled: false, hidden_profile_subscriptions_enabled: false, // Auth-restricted responsive_web_graphql_exclude_directive_enabled: true, verified_phone_label_enabled: false, subscriptions_verification_info_is_identity_verified_enabled: false, subscriptions_verification_info_verified_since_enabled: true, highlights_tweets_tab_ui_enabled: true, creator_subscriptions_tweet_preview_api_enabled: true, responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, responsive_web_graphql_timeline_navigation_enabled: true }) ?? "" ); params.set("fieldToggles", stringify({ withAuxiliaryUserLabels: false }) ?? ""); const res = await requestApi( `https://twitter.com/i/api/graphql/G3KGOASz96M-Qu0nwmGXNg/UserByScreenName?${params.toString()}`, auth ); if (!res.success) { return res; } const { value } = res; const { errors } = value; if (errors != null && errors.length > 0) { return { success: false, err: new Error(errors[0].message) }; } if (!value.data || !value.data.user || !value.data.user.result) { return { success: false, err: new Error("User not found.") }; } const { result: user } = value.data.user; const { legacy } = user; if (user.rest_id == null || user.rest_id.length === 0) { return { success: false, err: new Error("rest_id not found.") }; } legacy.id_str = user.rest_id; if (legacy.screen_name == null || legacy.screen_name.length === 0) { return { success: false, err: new Error(`Either ${username} does not exist or is private.`) }; } return { success: true, value: parseProfile(user.legacy, user.is_blue_verified) }; } const idCache = /* @__PURE__ */ new Map(); async function getScreenNameByUserId(userId, auth) { const params = new URLSearchParams(); params.set( "variables", stringify({ userId, withSafetyModeUserFields: true }) ?? "" ); params.set( "features", stringify({ hidden_profile_subscriptions_enabled: true, rweb_tipjar_consumption_enabled: true, responsive_web_graphql_exclude_directive_enabled: true, verified_phone_label_enabled: false, highlights_tweets_tab_ui_enabled: true, responsive_web_twitter_article_notes_tab_enabled: true, subscriptions_feature_can_gift_premium: false, creator_subscriptions_tweet_preview_api_enabled: true, responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, responsive_web_graphql_timeline_navigation_enabled: true }) ?? "" ); const res = await requestApi( `https://twitter.com/i/api/graphql/xf3jd90KKBCUxdlI_tNHZw/UserByRestId?${params.toString()}`, auth ); if (!res.success) { return res; } const { value } = res; const { errors } = value; if (errors != null && errors.length > 0) { return { success: false, err: new Error(errors[0].message) }; } if (!value.data || !value.data.user || !value.data.user.result) { return { success: false, err: new Error("User not found.") }; } const { result: user } = value.data.user; const { legacy } = user; if (legacy.screen_name == null || legacy.screen_name.length === 0) { return { success: false, err: new Error( `Either user with ID ${userId} does not exist or is private.` ) }; } return { success: true, value: legacy.screen_name }; } async function getUserIdByScreenName(screenName, auth) { const cached = idCache.get(screenName); if (cached != null) { return { success: true, value: cached }; } const profileRes = await getProfile(screenName, auth); if (!profileRes.success) { return profileRes; } const profile = profileRes.value; if (profile.userId != null) { idCache.set(screenName, profile.userId); return { success: true, value: profile.userId }; } return { success: false, err: new Error("User ID is undefined.") }; } const TwitterUserAuthSubtask = typebox.Type.Object({ subtask_id: typebox.Type.String(), enter_text: typebox.Type.Optional(typebox.Type.Object({})) }); class TwitterUserAuth extends TwitterGuestAuth { constructor(bearerToken, options) { super(bearerToken, options); } async isLoggedIn() { const res = await requestApi( "https://api.twitter.com/1.1/account/verify_credentials.json", this ); if (!res.success) { return false; } const { value: verify } = res; this.userProfile = parseProfile( verify, verify.verified ); return verify && !verify.errors?.length; } async me() { if (this.userProfile) { return this.userProfile; } await this.isLoggedIn(); return this.userProfile; } async login(username, password, email, twoFactorSecret, appKey, appSecret, accessToken, accessSecret) { await this.updateGuestToken(); let next = await this.initLogin(); while ("subtask" in next && next.subtask) { if (next.subtask.subtask_id === "LoginJsInstrumentationSubtask") { next = await this.handleJsInstrumentationSubtask(next); } else if (next.subtask.subtask_id === "LoginEnterUserIdentifierSSO") { next = await this.handleEnterUserIdentifierSSO(next, username); } else if (next.subtask.subtask_id === "LoginEnterAlternateIdentifierSubtask") { next = await this.handleEnterAlternateIdentifierSubtask( next, email ); } else if (next.subtask.subtask_id === "LoginEnterPassword") { next = await this.handleEnterPassword(next, password); } else if (next.subtask.subtask_id === "AccountDuplicationCheck") { next = await this.handleAccountDuplicationCheck(next); } else if (next.subtask.subtask_id === "LoginTwoFactorAuthChallenge") { if (twoFactorSecret) { next = await this.handleTwoFactorAuthChallenge(next, twoFactorSecret); } else { throw new Error( "Requested two factor authentication code but no secret provided" ); } } else if (next.subtask.subtask_id === "LoginAcid") { next = await this.handleAcid(next, email); } else if (next.subtask.subtask_id === "LoginSuccessSubtask") { next = await this.handleSuccessSubtask(next); } else { throw new Error(`Unknown subtask ${next.subtask.subtask_id}`); } } if (appKey && appSecret && accessToken && accessSecret) { this.loginWithV2(appKey, appSecret, accessToken, accessSecret); } if ("err" in next) { throw next.err; } } async logout() { if (!this.isLoggedIn()) { return; } await requestApi( "https://api.twitter.com/1.1/account/logout.json", this, "POST" ); this.deleteToken(); this.jar = new toughCookie.CookieJar(); } async installCsrfToken(headers) { const cookies = await this.getCookies(); const xCsrfToken = cookies.find((cookie) => cookie.key === "ct0"); if (xCsrfToken) { headers.set("x-csrf-token", xCsrfToken.value); } } async installTo(headers) { headers.set("authorization", `Bearer ${this.bearerToken}`); headers.set("cookie", await this.getCookieString()); await this.installCsrfToken(headers); } async initLogin() { this.removeCookie("twitter_ads_id="); this.removeCookie("ads_prefs="); this.removeCookie("_twitter_sess="); this.removeCookie("zipbox_forms_auth_token="); this.removeCookie("lang="); this.removeCookie("bouncer_reset_cookie="); this.removeCookie("twid="); this.removeCookie("twitter_ads_idb="); this.removeCookie("email_uid="); this.removeCookie("external_referer="); this.removeCookie("ct0="); this.removeCookie("aa_u="); return await this.executeFlowTask({ flow_name: "login", input_flow_data: { flow_context: { debug_overrides: {}, start_location: { location: "splash_screen" } } } }); } async handleJsInstrumentationSubtask(prev) { return await this.executeFlowTask({ flow_token: prev.flowToken, subtask_inputs: [ { subtask_id: "LoginJsInstrumentationSubtask", js_instrumentation: { response: "{}", link: "next_link" } } ] }); } async handleEnterAlternateIdentifierSubtask(prev, email) { return await this.executeFlowTask({ flow_token: prev.flowToken, subtask_inputs: [ { subtask_id: "LoginEnterAlternateIdentifierSubtask", enter_text: { text: email, link: "next_link" } } ] }); } async handleEnterUserIdentifierSSO(prev, username) { return await this.executeFlowTask({ flow_token: prev.flowToken, subtask_inputs: [ { subtask_id: "LoginEnterUserIdentifierSSO", settings_list: { setting_responses: [ { key: "user_identifier", response_data: { text_data: { result: username } } } ], link: "next_link" } } ] }); } async handleEnterPassword(prev, password) { return await this.executeFlowTask({ flow_token: prev.flowToken, subtask_inputs: [ { subtask_id: "LoginEnterPassword", enter_password: { password, link: "next_link" } } ] }); } async handleAccountDuplicationCheck(prev) { return await this.executeFlowTask({ flow_token: prev.flowToken, subtask_inputs: [ { subtask_id: "AccountDuplicationCheck", check_logged_in_account: { link: "AccountDuplicationCheck_false" } } ] }); } async handleTwoFactorAuthChallenge(prev, secret) { const totp = new OTPAuth__namespace.TOTP({ secret }); let error; for (let attempts = 1; attempts < 4; attempts += 1) { try { return await this.executeFlowTask({ flow_token: prev.flowToken, subtask_inputs: [ { subtask_id: "LoginTwoFactorAuthChallenge", enter_text: { link: "next_link", text: totp.generate() } } ] }); } catch (err) { error = err; await new Promise((resolve) => setTimeout(resolve, 2e3 * attempts)); } } throw error; } async handleAcid(prev, email) { return await this.executeFlowTask({ flow_token: prev.flowToken, subtask_inputs: [ { subtask_id: "LoginAcid", enter_text: { text: email, link: "next_link" } } ] }); } async handleSuccessSubtask(prev) { return await this.executeFlowTask({ flow_token: prev.flowToken, subtask_inputs: [] }); } async executeFlowTask(data) { const onboardingTaskUrl = "https://api.twitter.com/1.1/onboarding/task.json"; const token = this.guestToken; if (token == null) { throw new Error("Authentication token is null or undefined."); } const headers = new headersPolyfill.Headers({ authorization: `Bearer ${this.bearerToken}`, cookie: await this.getCookieString(), "content-type": "application/json", "User-Agent": "Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36", "x-guest-token": token, "x-twitter-auth-type": "OAuth2Client", "x-twitter-active-user": "yes", "x-twitter-client-language": "en" }); await this.installCsrfToken(headers); const res = await this.fetch(onboardingTaskUrl, { credentials: "include", method: "POST", headers, body: JSON.stringify(data) }); await updateCookieJar(this.jar, res.headers); if (!res.ok) { return { status: "error", err: new Error(await res.text()) }; } const flow = await res.json(); if (flow?.flow_token == null) { return { status: "error", err: new Error("flow_token not found.") }; } if (flow.errors?.length) { return { status: "error", err: new Error( `Authentication error (${flow.errors[0].code}): ${flow.errors[0].message}` ) }; } if (typeof flow.flow_token !== "string") { return { status: "error", err: new Error("flow_token was not a string.") }; } const subtask = flow.subtasks?.length ? flow.subtasks[0] : void 0; value.Check(TwitterUserAuthSubtask, subtask); if (subtask && subtask.subtask_id === "DenyLoginSubtask") { return { status: "error", err: new Error("Authentication error: DenyLoginSubtask") }; } return { status: "success", subtask, flowToken: flow.flow_token }; } } async function* getUserTimeline(query, maxProfiles, fetchFunc) { let nProfiles = 0; let cursor = void 0; let consecutiveEmptyBatches = 0; while (nProfiles < maxProfiles) { const batch = await fetchFunc( query, maxProfiles, cursor ); const { profiles, next } = batch; cursor = next; if (profiles.length === 0) { consecutiveEmptyBatches++; if (consecutiveEmptyBatches > 5) break; } else consecutiveEmptyBatches = 0; for (const profile of profiles) { if (nProfiles < maxProfiles) yield profile; else break; nProfiles++; } if (!next) break; } } async function* getTweetTimeline(query, maxTweets, fetchFunc) { let nTweets = 0; let cursor = void 0; while (nTweets < maxTweets) { const batch = await fetchFunc( query, maxTweets, cursor ); const { tweets, next } = batch; if (tweets.length === 0) { break; } for (const tweet of tweets) { if (nTweets < maxTweets) { cursor = next; yield tweet; } else { break; } nTweets++; } } } function isFieldDefined(key) { return function(value) { return isDefined(value[key]); }; } function isDefined(value) { return value != null; } const reHashtag = /\B(\#\S+\b)/g; const reCashtag = /\B(\$\S+\b)/g; const reTwitterUrl = /https:(\/\/t\.co\/([A-Za-z0-9]|[A-Za-z]){10})/g; const reUsername = /\B(\@\S{1,15}\b)/g; function parseMediaGroups(media) { const photos = []; const videos = []; let sensitiveContent = void 0; for (const m of media.filter(isFieldDefined("id_str")).filter(isFieldDefined("media_url_https"))) { if (m.type === "photo") { photos.push({ id: m.id_str, url: m.media_url_https, alt_text: m.ext_alt_text }); } else if (m.type === "video") { videos.push(parseVideo(m)); } const sensitive = m.ext_sensitive_media_warning; if (sensitive != null) { sensitiveContent = sensitive.adult_content || sensitive.graphic_violence || sensitive.other; } } return { sensitiveContent, photos, videos }; } function parseVideo(m) { const video = { id: m.id_str, preview: m.media_url_https }; let maxBitrate = 0; const variants = m.video_info?.variants ?? []; for (const variant of variants) { const bitrate = variant.bitrate; if (bitrate != null && bitrate > maxBitrate && variant.url != null) { let variantUrl = variant.url; const stringStart = 0; const tagSuffixIdx = variantUrl.indexOf("?tag=10"); if (tagSuffixIdx !== -1) { variantUrl = variantUrl.substring(stringStart, tagSuffixIdx + 1); } video.url = variantUrl; maxBitrate = bitrate; } } return video; } function reconstructTweetHtml(tweet, photos, videos) { const media = []; let html = tweet.full_text ?? ""; html = html.replace(reHashtag, linkHashtagHtml); html = html.replace(reCashtag, linkCashtagHtml); html = html.replace(reUsername, linkUsernameHtml); html = html.replace(reTwitterUrl, unwrapTcoUrlHtml(tweet, media)); for (const { url } of photos) { if (media.indexOf(url) !== -1) { continue; } html += `<br><img src="${url}"/>`; } for (const { preview: url } of videos) { if (media.indexOf(url) !== -1) { continue; } html += `<br><img src="${url}"/>`; } html = html.replace(/\n/g, "<br>"); return html; } function linkHashtagHtml(hashtag) { return `<a href="https://twitter.com/hashtag/${hashtag.replace( "#", "" )}">${hashtag}</a>`; } function linkCashtagHtml(cashtag) { return `<a href="https://twitter.com/search?q=%24${cashtag.replace( "$", "" )}">${cashtag}</a>`; } function linkUsernameHtml(username) { return `<a href="https://twitter.com/${username.replace( "@", "" )}">${username}</a>`; } function unwrapTcoUrlHtml(tweet, foundedMedia) { return function(tco) { for (const entity of tweet.entities?.urls ?? []) { if (tco === entity.url && entity.expanded_url != null) { return `<a href="${entity.expanded_url}">${tco}</a>`; } } for (const entity of tweet.extended_entities?.media ?? []) { if (tco === entity.url && entity.media_url_https != null) { foundedMedia.push(entity.media_url_https); return `<br><a href="${tco}"><img src="${entity.media_url_https}"/></a>`; } } return tco; }; } function parseLegacyTweet(user, tweet) { if (tweet == null) { return { success: false, err: new Error("Tweet was not found in the timeline object.") }; } if (user == null) { return { success: false, err: new Error("User was not found in the timeline object.") }; } if (!tweet.id_str) { if (!tweet.conversation_id_str) { return { success: false, err: new Error("Tweet ID was not found in object.") }; } tweet.id_str = tweet.conversation_id_str; } const hashtags = tweet.entities?.hashtags ?? []; const mentions = tweet.entities?.user_mentions ?? []; const media = tweet.extended_entities?.media ?? []; const pinnedTweets = new Set( user.pinned_tweet_ids_str ?? [] ); const urls = tweet.entities?.urls ?? []; const { photos, videos, sensitiveContent } = parseMediaGroups(media); const tw = { bookmarkCount: tweet.bookmark_count, conversationId: tweet.conversation_id_str, id: tweet.id_str, hashtags: hashtags.filter(isFieldDefined("text")).map((hashtag) => hashtag.text), likes: tweet.favorite_count, mentions: mentions.filter(isFieldDefined("id_str")).map((mention) => ({ id: mention.id_str, username: mention.screen_name, name: mention.name })), name: user.name, permanentUrl: `https://twitter.com/${user.screen_name}/status/${tweet.id_str}`, photos, replies: tweet.reply_count, retweets: tweet.retweet_count, text: tweet.full_text, thread: [], urls: urls.filter(isFieldDefined("expanded_url")).map((url) => url.expanded_url), userId: tweet.user_id_str, username: user.screen_name, videos, isQuoted: false, isReply: false, isRetweet: false, isPin: false, sensitiveContent: false }; if (tweet.created_at) { tw.timeParsed = new Date(Date.parse(tweet.created_at)); tw.timestamp = Math.floor(tw.timeParsed.valueOf() / 1e3); } if (tweet.place?.id) { tw.place = tweet.place; } const quotedStatusIdStr = tweet.quoted_status_id_str; const inReplyToStatusIdStr = tweet.in_reply_to_status_id_str; const retweetedStatusIdStr = tweet.retweeted_status_id_str; const retweetedStatusResult = tweet.retweeted_status_result?.result; if (quotedStatusIdStr) { tw.isQuoted = true; tw.quotedStatusId = quotedStatusIdStr; } if (inReplyToStatusIdStr) { tw.isReply = true; tw.inReplyToStatusId = inReplyToStatusIdStr; } if (retweetedStatusIdStr || retweetedStatusResult) { tw.isRetweet = true; tw.retweetedStatusId = retweetedStatusIdStr; if (retweetedStatusResult) { const parsedResult = parseLegacyTweet( retweetedStatusResult?.core?.user_results?.result?.legacy, retweetedStatusResult?.legacy ); if (parsedResult.success) { tw.retweetedStatus = parsedResult.tweet; } } } const views = parseInt(tweet.ext_views?.count ?? ""); if (!isNaN(views)) { tw.views = views; } if (pinnedTweets.has(tweet.id_str)) { tw.isPin = true; } if (sensitiveContent) { tw.sensitiveContent = true; } tw.html = reconstructTweetHtml(tweet, tw.photos, tw.videos); return { success: true, tweet: tw }; } function parseResult(result) { const noteTweetResultText = result?.note_tweet?.note_tweet_results?.result?.text; if (result?.legacy && noteTweetResultText) { result.legacy.full_text = noteTweetResultText; } const tweetResult = parseLegacyTweet( result?.core?.user_results?.result?.legacy, result?.legacy ); if (!tweetResult.success) { return tweetResult; } if (!tweetResult.tweet.views && result?.views?.count) { const views = parseInt(result.views.count); if (!isNaN(views)) { tweetResult.tweet.views = views; } } const quotedResult = result?.quoted_status_result?.result; if (quotedResult) { if (quotedResult.legacy && quotedResult.rest_id) { quotedResult.legacy.id_str = quotedResult.rest_id; } const quotedTweetResult = parseResult(quotedResult); if (quotedTweetResult.success) { tweetResult.tweet.quotedStatus = quotedTweetResult.tweet; } } return tweetResult; } const expectedEntryTypes = ["tweet", "profile-conversation"]; function parseTimelineTweetsV2(timeline) { let bottomCursor; let topCursor; const tweets = []; const instructions = timeline.data?.user?.result?.timeline_v2?.timeline?.instructions ?? []; for (const instruction of instructions) { const entries = instruction.entries ?? []; for (const entry of entries) { const entryContent = entry.content; if (!entryContent) continue; if (entryContent.cursorType === "Bottom") { bottomCursor = entryContent.value; continue; } else if (entryContent.cursorType === "Top") { topCursor = entryContent.value; continue; } const idStr = entry.entryId; if (!expectedEntryTypes.some((entryType) => idStr.startsWith(entryType))) { continue; } if (entryContent.itemContent) { parseAndPush(tweets, entryContent.itemContent, idStr); } else if (entryContent.items) { for (const item of entryContent.items) { if (item.item?.itemContent) { parseAndPush(tweets, item.item.itemContent, idStr); } } } } } return { tweets, next: bottomCursor, previous: topCursor }; } function parseTimelineEntryItemContentRaw(content, entryId, isConversation = false) { let result = content.tweet_results?.result ?? content.tweetResult?.result; if (result?.__typename === "Tweet" || result?.__typename === "TweetWithVisibilityResults" && result?.tweet) { if (result?.__typename === "TweetWithVisibilityResults") result = result.tweet; if (result?.legacy) { result.legacy.id_str = result.rest_id ?? entryId.replace("conversation-", "").replace("tweet-", ""); } const tweetResult = parseResult(result); if (tweetResult.success) { if (isConversation) { if (content?.tweetDisplayType === "SelfThread") { tweetResult.tweet.isSelfThread = true; } } return tweetResult.tweet; } } return null; } function parseAndPush(tweets, content, entryId, isConversation = false) { const tweet = parseTimelineEntryItemContentRaw( content, entryId, isConversation ); if (tweet) { tweets.push(tweet); } } function parseThreadedConversation(conversation) { const tweets = []; const instructions = conversation.data?.threaded_conversation_with_injections_v2?.instructions ?? []; for (const instruction of instructions) { const entries = instruction.entries ?? []; for (const entry of entries) { const entryContent = entry.content?.itemContent; if (entryContent) { parseAndPush(tweets, entryContent, entry.entryId, true); } for (const item of entry.content?.items ?? []) { const itemContent = item.item?.itemContent; if (itemContent) { parseAndPush(tweets, itemContent, entry.entryId, true); } } } } for (const tweet of tweets) { if (tweet.inReplyToStatusId) { for (const parentTweet of tweets) { if (parentTweet.id === tweet.inReplyToStatusId) { tweet.inReplyToStatus = parentTweet; break; } } } if (tweet.isSelfThread && tweet.conversationId === tweet.id) { for (const childTweet of tweets) { if (childTweet.isSelfThread && childTweet.id !== tweet.id) { tweet.thread.push(childTweet); } } if (tweet.thread.length === 0) { tweet.isSelfThread = false; } } } return tweets; } function parseSearchTimelineTweets(timeline) { let bottomCursor; let topCursor; const tweets = []; const instructions = timeline.data?.search_by_raw_query?.search_timeline?.timeline?.instructions ?? []; for (const instruction of instructions) { if (instruction.type === "TimelineAddEntries" || instruction.type === "TimelineReplaceEntry") { if (instruction.entry?.content?.cursorType === "Bottom") { bottomCursor = instruction.entry.content.value; continue; } else if (instruction.entry?.content?.cursorType === "Top") { topCursor = instruction.entry.content.value; continue; } const entries = instruction.entries ?? []; for (const entry of entries) { const itemContent = entry.content?.itemContent; if (itemContent?.tweetDisplayType === "Tweet") { const tweetResultRaw = itemContent.tweet_results?.result; const tweetResult = parseLegacyTweet( tweetResultRaw?.core?.user_results?.result?.legacy, tweetResultRaw?.legacy ); if (tweetResult.success) { if (!tweetResult.tweet.views && tweetResultRaw?.views?.count) { const views = parseInt(tweetResultRaw.views.count); if (!isNaN(views)) { tweetResult.tweet.views = views; } } tweets.push(tweetResult.tweet); } } else if (entry.content?.cursorType === "Bottom") { bottomCursor = entry.content.value; } else if (entry.content?.cursorType === "Top") { topCursor = entry.content.value; } } } } return { tweets, next: bottomCursor, previous: topCursor }; } function parseSearchTimelineUsers(timeline) { let bottomCursor; let topCursor; const profiles = []; const instructions = timeline.data?.search_by_raw_query?.search_timeline?.timeline?.instructions ?? []; for (const instruction of instructions) { if (instruction.type === "TimelineAddEntries" || instruction.type === "TimelineReplaceEntry") { if (instruction.entry?.content?.cursorType === "Bottom") { bottomCursor = instruction.entry.content.value; continue; } else if (instruction.entry?.content?.cursorType === "Top") { topCursor = instruction.entry.content.value; continue; } const entries = instruction.entries ?? []; for (const entry of entries) { const itemContent = entry.content?.itemContent; if (itemContent?.userDisplayType === "User") { const userResultRaw = itemContent.user_results?.result; if (userResultRaw?.legacy) { const profile = parseProfile( userResultRaw.legacy, userResultRaw.is_blue_verified ); if (!profile.userId) { profile.userId = userResultRaw.rest_id; } profiles.push(profile); } } else if (entry.content?.cursorType === "Bottom") { bottomCursor = entry.content.value; } else if (entry.content?.cursorType === "Top") { topCursor = entry.content.value; } } } } return { profiles, next: bottomCursor, previous: topCursor }; } var SearchMode = /* @__PURE__ */ ((SearchMode2) => { SearchMode2[SearchMode2["Top"] = 0] = "Top"; SearchMode2[SearchMode2["Latest"] = 1] = "Latest"; SearchMode2[SearchMode2["Photos"] = 2] = "Photos"; SearchMode2[SearchMode2["Videos"] = 3] = "Videos"; SearchMode2[SearchMode2["Users"] = 4] = "Users"; return SearchMode2; })(SearchMode || {}); function searchTweets(query, maxTweets, searchMode, auth) { return getTweetTimeline(query, maxTweets, (q, mt, c) => { return fetchSearchTweets(q, mt, searchMode, auth, c); }); } function searchProfiles(query, maxProfiles, auth) { return getUserTimeline(query, maxProfiles, (q, mt, c) => { return fetchSearchProfiles(q, mt, auth, c); }); } async function fetchSearchTweets(query, maxTweets, searchMode, auth, cursor) { const timeline = await getSearchTimeline( query, maxTweets, searchMode, auth, cursor ); return parseSearchTimelineTweets(timeline); } async function fetchSearchProfiles(query, maxProfiles, auth, cursor) { const timeline = await getSearchTimeline( query, maxProfiles, 4 /* Users */, auth, cursor ); return parseSearchTimelineUsers(timeline); } async function getSearchTimeline(query, maxItems, searchMode, auth, cursor) { if (!auth.isLoggedIn()) { throw new Error("Scraper is not logged-in for search."); } if (maxItems > 50) { maxItems = 50; } const variables = { rawQuery: query, count: maxItems, querySource: "typed_query", product: "Top" }; const features = addApiFeatures({ longform_notetweets_inline_media_enabled: true, responsive_web_enhance_cards_enabled: false, responsive_web_media_download_video_enabled: false, responsive_web_twitter_article_tweet_consumption_enabled: false, tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, interactive_text_enabled: false, responsive_web_text_conversations_enabled: false, vibe_api_enabled: false }); const fieldToggles = { withArticleRichContentState: false }; if (cursor != null && cursor != "") { variables["cursor"] = cursor; } switch (searchMode) { case 1 /* Latest */: variables.product = "Latest"; break; case 2 /* Photos */: variables.product = "Photos"; break; case 3 /* Videos */: variables.product = "Videos"; break; case 4 /* Users */: variables.product = "People"; break; } const params = new URLSearchParams(); params.set("features", stringify(features) ?? ""); params.set("fieldToggles", stringify(fieldToggles) ?? ""); params.set("variables", stringify(variables) ?? ""); const res = await requestApi( `https://api.twitter.com/graphql/gkjsKepM6gl_HmFWoWKfgg/SearchTimeline?${params.toString()}`, auth ); if (!res.success) { throw res.err; } return res.value; } function parseRelationshipTimeline(timeline) { let bottomCursor; let topCursor; const profiles = []; const instructions = timeline.data?.user?.result?.timeline?.timeline?.instructions ?? []; for (const instruction of instructions) { if (instruction.type === "TimelineAddEntries" || instruction.type === "TimelineReplaceEntry") { if (instruction.entry?.content?.cursorType === "Bottom") { bottomCursor = instruction.entry.content.value; continue; } if (instruction.entry?.content?.cursorType === "Top") { topCursor = instruction.entry.content.value; continue; } const entries = instruction.entries ?? []; for (const entry of entries) { const itemContent = entry.content?.itemContent; if (itemContent?.userDisplayType === "User") { const userResultRaw = itemContent.user_results?.result; if (userResultRaw?.legacy) { const profile = parseProfile( userResultRaw.legacy, userResultRaw.is_blue_verified ); if (!profile.userId) { profile.userId = userResultRaw.rest_id; } profiles.push(profile); } } else if (entry.content?.cursorType === "Bottom") { bottomCursor = entry.content.value; } else if (entry.content?.cursorType === "Top") { topCursor = entry.content.value; } } } } return { profiles, next: bottomCursor, previous: topCursor }; } function getFollowing(userId, maxProfiles, auth) { return getUserTimeline(userId, maxProfiles, (q, mt, c) => { return fetchProfileFollowing(q, mt, auth, c); }); } function getFollowers(userId, maxProfiles, auth) { return getUserTimeline(userId, maxProfiles, (q, mt, c) => { return fetchProfileFollowers(q, mt, auth, c); }); } async function fetchProfileFollowing(userId, maxProfiles, auth, cursor) { const timeline = await getFollowingTimeline( userId, maxProfiles, auth, cursor ); return parseRelationshipTimeline(timeline); } async function fetchProfileFollowers(userId, maxProfiles, auth, cursor) { const timeline = await getFollowersTimeline( userId, maxProfiles, auth, cursor ); return parseRelationshipTimeline(timeline); } async function getFollowingTimeline(userId, maxItems, auth, cursor) { if (!auth.isLoggedIn()) { throw new Error("Scraper is not logged-in for profile following."); } if (maxItems > 50) { maxItems = 50; } const variables = { userId, count: maxItems, includePromotedContent: false }; const features = addApiFeatures({ responsive_web_twitter_article_tweet_consumption_enabled: false, tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, longform_notetweets_inline_media_enabled: true, responsive_web_media_download_video_enabled: false }); if (cursor != null && cursor != "") { variables["cursor"] = cursor; } const params = new URLSearchParams(); params.set("features", stringify(features) ?? ""); params.set("variables", stringify(variables) ?? ""); const res = await requestApi( `https://twitter.com/i/api/graphql/iSicc7LrzWGBgDPL0tM_TQ/Following?${params.toString()}`, auth ); if (!res.success) { throw res.err; } return res.value; } async function getFollowersTimeline(userId, maxItems, auth, cursor) { if (!auth.isLoggedIn()) { throw new Error("Scraper is not logged-in for profile followers."); } if (maxItems > 50) { maxItems = 50; } const variables = { userId, count: maxItems, includePromotedContent: false }; const features = addApiFeatures({ responsive_web_twitter_article_tweet_consumption_enabled: false, tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, longform_notetweets_inline_media_enabled: true, responsive_web_media_download_video_enabled: false }); if (cursor != null && cursor != "") { variables["cursor"] = cursor; } const params = new URLSearchParams(); params.set("features", stringify(features) ?? ""); params.set("variables", stringify(variables) ?? ""); const res = await requestApi( `https://twitter.com/i/api/graphql/rRXFSG5vR6drKr5M37YOTw/Followers?${params.toString()}`, auth ); if (!res.success) { throw res.err; } return res.value; } async function followUser(username, auth) { if (!await auth.isLoggedIn()) { throw new Error("Must be logged in to follow users"); } const userIdResult = await getUserIdByScreenName(username, auth); if (!userIdResult.success) { throw new Error(`Failed to get user ID: ${userIdResult.err.message}`); } const userId = userIdResult.value; const requestBody = { include_profile_interstitial_type: "1", skip_status: "true", user_id: userId }; const headers = new headersPolyfill.Headers({ "Content-Type": "application/x-www-form-urlencoded", Referer: `https://twitter.com/${username}`, "X-Twitter-Active-User": "yes", "X-Twitter-Auth-Type": "OAuth2Session", "X-Twitter-Client-Language": "en", Authorization: `Bearer ${bearerToken}` }); await auth.installTo(headers, "https://api.twitter.com/1.1/friendships/create.json"); const res = await auth.fetch( "https://api.twitter.com/1.1/friendships/create.json", { method: "POST", headers, body: new URLSearchParams(requestBody).toString(), credentials: "in