agent-twitter-summary
Version:
A twitter client for agents
1,575 lines (1,557 loc) • 209 kB
JavaScript
'use strict';
var toughCookie = require('tough-cookie');
var setCookie = require('set-cookie-parser');
var headersPolyfill = require('headers-polyfill');
var twitterApiV2 = require('twitter-api-v2');
var typebox = require('@sinclair/typebox');
var value = require('@sinclair/typebox/value');
var OTPAuth = require('otpauth');
var stringify = require('json-stable-stringify');
var events = require('events');
var WebSocket = require('ws');
var wrtc = require('@roamhq/wrtc');
var fs = require('fs');
var path = require('path');
var child_process = require('child_process');
var tls = require('node:tls');
var node_crypto = require('node:crypto');
function _interopNamespaceDefault(e) {
var n = Object.create(null);
if (e) {
Object.keys(e).forEach(function (k) {
if (k !== 'default') {
var d = Object.getOwnPropertyDescriptor(e, k);
Object.defineProperty(n, k, d.get ? d : {
enumerable: true,
get: function () { return e[k]; }
});
}
});
}
n.default = e;
return Object.freeze(n);
}
var OTPAuth__namespace = /*#__PURE__*/_interopNamespaceDefault(OTPAuth);
var fs__namespace = /*#__PURE__*/_interopNamespaceDefault(fs);
class ApiError extends Error {
constructor(response, data, message) {
super(message);
this.response = response;
this.data = data;
}
static async fromResponse(response) {
let data = void 0;
try {
data = await response.json();
} catch {
try {
data = await response.text();
} catch {
}
}
return new ApiError(response, data, `Response status: ${response.status}`);
}
}
class Platform {
async randomizeCiphers() {
const platform = await Platform.importPlatform();
await platform?.randomizeCiphers();
}
static async importPlatform() {
{
const { platform } = await Promise.resolve().then(function () { return index; });
return platform;
}
}
}
async function updateCookieJar(cookieJar, headers) {
const setCookieHeader = headers.get("set-cookie");
if (setCookieHeader) {
const cookies = setCookie.splitCookiesString(setCookieHeader);
for (const cookie of cookies.map((c) => toughCookie.Cookie.parse(c))) {
if (!cookie) continue;
await cookieJar.setCookie(
cookie,
`${cookie.secure ? "https" : "http"}://${cookie.domain}${cookie.path}`
);
}
} else if (typeof document !== "undefined") {
for (const cookie of document.cookie.split(";")) {
const hardCookie = toughCookie.Cookie.parse(cookie);
if (hardCookie) {
await cookieJar.setCookie(hardCookie, document.location.toString());
}
}
}
}
const bearerToken = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF";
async function requestApi(url, auth, method = "GET", platform = new Platform()) {
const headers = new headersPolyfill.Headers();
await auth.installTo(headers, url);
await platform.randomizeCiphers();
let res;
do {
try {
res = await auth.fetch(url, {
method,
headers,
credentials: "include"
});
} catch (err) {
if (!(err instanceof Error)) {
throw err;
}
return {
success: false,
err: new Error("Failed to perform request.")
};
}
await updateCookieJar(auth.cookieJar(), res.headers);
if (res.status === 429) {
const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
const xRateLimitReset = res.headers.get("x-rate-limit-reset");
if (xRateLimitRemaining == "0" && xRateLimitReset) {
const currentTime = (/* @__PURE__ */ new Date()).valueOf() / 1e3;
const timeDeltaMs = 1e3 * (parseInt(xRateLimitReset) - currentTime);
await new Promise((resolve) => setTimeout(resolve, timeDeltaMs));
}
}
} while (res.status === 429);
if (!res.ok) {
return {
success: false,
err: await ApiError.fromResponse(res)
};
}
const value = await res.json();
if (res.headers.get("x-rate-limit-incoming") == "0") {
auth.deleteToken();
return { success: true, value };
} else {
return { success: true, value };
}
}
function addApiFeatures(o) {
return {
...o,
rweb_lists_timeline_redesign_enabled: true,
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
creator_subscriptions_tweet_preview_api_enabled: true,
responsive_web_graphql_timeline_navigation_enabled: true,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
tweetypie_unmention_optimization_enabled: true,
responsive_web_edit_tweet_api_enabled: true,
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
view_counts_everywhere_api_enabled: true,
longform_notetweets_consumption_enabled: true,
tweet_awards_web_tipping_enabled: false,
freedom_of_speech_not_reach_fetch_enabled: true,
standardized_nudges_misinfo: true,
longform_notetweets_rich_text_read_enabled: true,
responsive_web_enhance_cards_enabled: false,
subscriptions_verification_info_enabled: true,
subscriptions_verification_info_reason_enabled: true,
subscriptions_verification_info_verified_since_enabled: true,
super_follow_badge_privacy_enabled: false,
super_follow_exclusive_tweet_notifications_enabled: false,
super_follow_tweet_api_enabled: false,
super_follow_user_api_enabled: false,
android_graphql_skip_api_media_color_palette: false,
creator_subscriptions_subscription_count_enabled: false,
blue_business_profile_image_shape_enabled: false,
unified_cards_ad_metadata_container_dynamic_card_content_query_enabled: false
};
}
function addApiParams(params, includeTweetReplies) {
params.set("include_profile_interstitial_type", "1");
params.set("include_blocking", "1");
params.set("include_blocked_by", "1");
params.set("include_followed_by", "1");
params.set("include_want_retweets", "1");
params.set("include_mute_edge", "1");
params.set("include_can_dm", "1");
params.set("include_can_media_tag", "1");
params.set("include_ext_has_nft_avatar", "1");
params.set("include_ext_is_blue_verified", "1");
params.set("include_ext_verified_type", "1");
params.set("skip_status", "1");
params.set("cards_platform", "Web-12");
params.set("include_cards", "1");
params.set("include_ext_alt_text", "true");
params.set("include_ext_limited_action_results", "false");
params.set("include_quote_count", "true");
params.set("include_reply_count", "1");
params.set("tweet_mode", "extended");
params.set("include_ext_collab_control", "true");
params.set("include_ext_views", "true");
params.set("include_entities", "true");
params.set("include_user_entities", "true");
params.set("include_ext_media_color", "true");
params.set("include_ext_media_availability", "true");
params.set("include_ext_sensitive_media_warning", "true");
params.set("include_ext_trusted_friends_metadata", "true");
params.set("send_error_codes", "true");
params.set("simple_quoted_tweet", "true");
params.set("include_tweet_replies", `${includeTweetReplies}`);
params.set(
"ext",
"mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,collab_control,vibe"
);
return params;
}
function withTransform(fetchFn, transform) {
return async (input, init) => {
const fetchArgs = await transform?.request?.(input, init) ?? [
input,
init
];
const res = await fetchFn(...fetchArgs);
return await transform?.response?.(res) ?? res;
};
}
class TwitterGuestAuth {
constructor(bearerToken, options) {
this.options = options;
this.fetch = withTransform(options?.fetch ?? fetch, options?.transform);
this.bearerToken = bearerToken;
this.jar = new toughCookie.CookieJar();
this.v2Client = null;
}
cookieJar() {
return this.jar;
}
getV2Client() {
return this.v2Client ?? null;
}
loginWithV2(appKey, appSecret, accessToken, accessSecret) {
const v2Client = new twitterApiV2.TwitterApi({
appKey,
appSecret,
accessToken,
accessSecret
});
this.v2Client = v2Client;
}
isLoggedIn() {
return Promise.resolve(false);
}
async me() {
return void 0;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
login(_username, _password, _email) {
return this.updateGuestToken();
}
logout() {
this.deleteToken();
this.jar = new toughCookie.CookieJar();
return Promise.resolve();
}
deleteToken() {
delete this.guestToken;
delete this.guestCreatedAt;
}
hasToken() {
return this.guestToken != null;
}
authenticatedAt() {
if (this.guestCreatedAt == null) {
return null;
}
return new Date(this.guestCreatedAt);
}
async installTo(headers) {
if (this.shouldUpdate()) {
await this.updateGuestToken();
}
const token = this.guestToken;
if (token == null) {
throw new Error("Authentication token is null or undefined.");
}
headers.set("authorization", `Bearer ${this.bearerToken}`);
headers.set("x-guest-token", token);
const cookies = await this.getCookies();
const xCsrfToken = cookies.find((cookie) => cookie.key === "ct0");
if (xCsrfToken) {
headers.set("x-csrf-token", xCsrfToken.value);
}
headers.set("cookie", await this.getCookieString());
}
getCookies() {
return this.jar.getCookies(this.getCookieJarUrl());
}
getCookieString() {
return this.jar.getCookieString(this.getCookieJarUrl());
}
async removeCookie(key) {
const store = this.jar.store;
const cookies = await this.jar.getCookies(this.getCookieJarUrl());
for (const cookie of cookies) {
if (!cookie.domain || !cookie.path) continue;
store.removeCookie(cookie.domain, cookie.path, key);
if (typeof document !== "undefined") {
document.cookie = `${cookie.key}=; Max-Age=0; path=${cookie.path}; domain=${cookie.domain}`;
}
}
}
getCookieJarUrl() {
return typeof document !== "undefined" ? document.location.toString() : "https://twitter.com";
}
/**
* Updates the authentication state with a new guest token from the Twitter API.
*/
async updateGuestToken() {
const guestActivateUrl = "https://api.twitter.com/1.1/guest/activate.json";
const headers = new headersPolyfill.Headers({
Authorization: `Bearer ${this.bearerToken}`,
Cookie: await this.getCookieString()
});
const res = await this.fetch(guestActivateUrl, {
method: "POST",
headers,
referrerPolicy: "no-referrer"
});
await updateCookieJar(this.jar, res.headers);
if (!res.ok) {
throw new Error(await res.text());
}
const o = await res.json();
if (o == null || o["guest_token"] == null) {
throw new Error("guest_token not found.");
}
const newGuestToken = o["guest_token"];
if (typeof newGuestToken !== "string") {
throw new Error("guest_token was not a string.");
}
this.guestToken = newGuestToken;
this.guestCreatedAt = /* @__PURE__ */ new Date();
}
/**
* Returns if the authentication token needs to be updated or not.
* @returns `true` if the token needs to be updated; `false` otherwise.
*/
shouldUpdate() {
return !this.hasToken() || this.guestCreatedAt != null && this.guestCreatedAt < new Date((/* @__PURE__ */ new Date()).valueOf() - 3 * 60 * 60 * 1e3);
}
}
function getAvatarOriginalSizeUrl(avatarUrl) {
return avatarUrl ? avatarUrl.replace("_normal", "") : void 0;
}
function parseProfile(user, isBlueVerified) {
const profile = {
avatar: getAvatarOriginalSizeUrl(user.profile_image_url_https),
banner: user.profile_banner_url,
biography: user.description,
followersCount: user.followers_count,
followingCount: user.friends_count,
friendsCount: user.friends_count,
mediaCount: user.media_count,
isPrivate: user.protected ?? false,
isVerified: user.verified,
likesCount: user.favourites_count,
listedCount: user.listed_count,
location: user.location,
name: user.name,
pinnedTweetIds: user.pinned_tweet_ids_str,
tweetsCount: user.statuses_count,
url: `https://twitter.com/${user.screen_name}`,
userId: user.id_str,
username: user.screen_name,
isBlueVerified: isBlueVerified ?? false,
canDm: user.can_dm
};
if (user.created_at != null) {
profile.joined = new Date(Date.parse(user.created_at));
}
const urls = user.entities?.url?.urls;
if (urls?.length != null && urls?.length > 0) {
profile.website = urls[0].expanded_url;
}
return profile;
}
async function getProfile(username, auth) {
const params = new URLSearchParams();
params.set(
"variables",
stringify({
screen_name: username,
withSafetyModeUserFields: true
}) ?? ""
);
params.set(
"features",
stringify({
hidden_profile_likes_enabled: false,
hidden_profile_subscriptions_enabled: false,
// Auth-restricted
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
subscriptions_verification_info_is_identity_verified_enabled: false,
subscriptions_verification_info_verified_since_enabled: true,
highlights_tweets_tab_ui_enabled: true,
creator_subscriptions_tweet_preview_api_enabled: true,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
responsive_web_graphql_timeline_navigation_enabled: true
}) ?? ""
);
params.set("fieldToggles", stringify({ withAuxiliaryUserLabels: false }) ?? "");
const res = await requestApi(
`https://twitter.com/i/api/graphql/G3KGOASz96M-Qu0nwmGXNg/UserByScreenName?${params.toString()}`,
auth
);
if (!res.success) {
return res;
}
const { value } = res;
const { errors } = value;
if (errors != null && errors.length > 0) {
return {
success: false,
err: new Error(errors[0].message)
};
}
if (!value.data || !value.data.user || !value.data.user.result) {
return {
success: false,
err: new Error("User not found.")
};
}
const { result: user } = value.data.user;
const { legacy } = user;
if (user.rest_id == null || user.rest_id.length === 0) {
return {
success: false,
err: new Error("rest_id not found.")
};
}
legacy.id_str = user.rest_id;
if (legacy.screen_name == null || legacy.screen_name.length === 0) {
return {
success: false,
err: new Error(`Either ${username} does not exist or is private.`)
};
}
return {
success: true,
value: parseProfile(user.legacy, user.is_blue_verified)
};
}
const idCache = /* @__PURE__ */ new Map();
async function getScreenNameByUserId(userId, auth) {
const params = new URLSearchParams();
params.set(
"variables",
stringify({
userId,
withSafetyModeUserFields: true
}) ?? ""
);
params.set(
"features",
stringify({
hidden_profile_subscriptions_enabled: true,
rweb_tipjar_consumption_enabled: true,
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
highlights_tweets_tab_ui_enabled: true,
responsive_web_twitter_article_notes_tab_enabled: true,
subscriptions_feature_can_gift_premium: false,
creator_subscriptions_tweet_preview_api_enabled: true,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
responsive_web_graphql_timeline_navigation_enabled: true
}) ?? ""
);
const res = await requestApi(
`https://twitter.com/i/api/graphql/xf3jd90KKBCUxdlI_tNHZw/UserByRestId?${params.toString()}`,
auth
);
if (!res.success) {
return res;
}
const { value } = res;
const { errors } = value;
if (errors != null && errors.length > 0) {
return {
success: false,
err: new Error(errors[0].message)
};
}
if (!value.data || !value.data.user || !value.data.user.result) {
return {
success: false,
err: new Error("User not found.")
};
}
const { result: user } = value.data.user;
const { legacy } = user;
if (legacy.screen_name == null || legacy.screen_name.length === 0) {
return {
success: false,
err: new Error(
`Either user with ID ${userId} does not exist or is private.`
)
};
}
return {
success: true,
value: legacy.screen_name
};
}
async function getUserIdByScreenName(screenName, auth) {
const cached = idCache.get(screenName);
if (cached != null) {
return { success: true, value: cached };
}
const profileRes = await getProfile(screenName, auth);
if (!profileRes.success) {
return profileRes;
}
const profile = profileRes.value;
if (profile.userId != null) {
idCache.set(screenName, profile.userId);
return {
success: true,
value: profile.userId
};
}
return {
success: false,
err: new Error("User ID is undefined.")
};
}
const TwitterUserAuthSubtask = typebox.Type.Object({
subtask_id: typebox.Type.String(),
enter_text: typebox.Type.Optional(typebox.Type.Object({}))
});
class TwitterUserAuth extends TwitterGuestAuth {
constructor(bearerToken, options) {
super(bearerToken, options);
}
async isLoggedIn() {
const res = await requestApi(
"https://api.twitter.com/1.1/account/verify_credentials.json",
this
);
if (!res.success) {
return false;
}
const { value: verify } = res;
this.userProfile = parseProfile(
verify,
verify.verified
);
return verify && !verify.errors?.length;
}
async me() {
if (this.userProfile) {
return this.userProfile;
}
await this.isLoggedIn();
return this.userProfile;
}
async login(username, password, email, twoFactorSecret, appKey, appSecret, accessToken, accessSecret) {
await this.updateGuestToken();
let next = await this.initLogin();
while ("subtask" in next && next.subtask) {
if (next.subtask.subtask_id === "LoginJsInstrumentationSubtask") {
next = await this.handleJsInstrumentationSubtask(next);
} else if (next.subtask.subtask_id === "LoginEnterUserIdentifierSSO") {
next = await this.handleEnterUserIdentifierSSO(next, username);
} else if (next.subtask.subtask_id === "LoginEnterAlternateIdentifierSubtask") {
next = await this.handleEnterAlternateIdentifierSubtask(
next,
email
);
} else if (next.subtask.subtask_id === "LoginEnterPassword") {
next = await this.handleEnterPassword(next, password);
} else if (next.subtask.subtask_id === "AccountDuplicationCheck") {
next = await this.handleAccountDuplicationCheck(next);
} else if (next.subtask.subtask_id === "LoginTwoFactorAuthChallenge") {
if (twoFactorSecret) {
next = await this.handleTwoFactorAuthChallenge(next, twoFactorSecret);
} else {
throw new Error(
"Requested two factor authentication code but no secret provided"
);
}
} else if (next.subtask.subtask_id === "LoginAcid") {
next = await this.handleAcid(next, email);
} else if (next.subtask.subtask_id === "LoginSuccessSubtask") {
next = await this.handleSuccessSubtask(next);
} else {
throw new Error(`Unknown subtask ${next.subtask.subtask_id}`);
}
}
if (appKey && appSecret && accessToken && accessSecret) {
this.loginWithV2(appKey, appSecret, accessToken, accessSecret);
}
if ("err" in next) {
throw next.err;
}
}
async logout() {
if (!this.isLoggedIn()) {
return;
}
await requestApi(
"https://api.twitter.com/1.1/account/logout.json",
this,
"POST"
);
this.deleteToken();
this.jar = new toughCookie.CookieJar();
}
async installCsrfToken(headers) {
const cookies = await this.getCookies();
const xCsrfToken = cookies.find((cookie) => cookie.key === "ct0");
if (xCsrfToken) {
headers.set("x-csrf-token", xCsrfToken.value);
}
}
async installTo(headers) {
headers.set("authorization", `Bearer ${this.bearerToken}`);
headers.set("cookie", await this.getCookieString());
await this.installCsrfToken(headers);
}
async initLogin() {
this.removeCookie("twitter_ads_id=");
this.removeCookie("ads_prefs=");
this.removeCookie("_twitter_sess=");
this.removeCookie("zipbox_forms_auth_token=");
this.removeCookie("lang=");
this.removeCookie("bouncer_reset_cookie=");
this.removeCookie("twid=");
this.removeCookie("twitter_ads_idb=");
this.removeCookie("email_uid=");
this.removeCookie("external_referer=");
this.removeCookie("ct0=");
this.removeCookie("aa_u=");
return await this.executeFlowTask({
flow_name: "login",
input_flow_data: {
flow_context: {
debug_overrides: {},
start_location: {
location: "splash_screen"
}
}
}
});
}
async handleJsInstrumentationSubtask(prev) {
return await this.executeFlowTask({
flow_token: prev.flowToken,
subtask_inputs: [
{
subtask_id: "LoginJsInstrumentationSubtask",
js_instrumentation: {
response: "{}",
link: "next_link"
}
}
]
});
}
async handleEnterAlternateIdentifierSubtask(prev, email) {
return await this.executeFlowTask({
flow_token: prev.flowToken,
subtask_inputs: [
{
subtask_id: "LoginEnterAlternateIdentifierSubtask",
enter_text: {
text: email,
link: "next_link"
}
}
]
});
}
async handleEnterUserIdentifierSSO(prev, username) {
return await this.executeFlowTask({
flow_token: prev.flowToken,
subtask_inputs: [
{
subtask_id: "LoginEnterUserIdentifierSSO",
settings_list: {
setting_responses: [
{
key: "user_identifier",
response_data: {
text_data: { result: username }
}
}
],
link: "next_link"
}
}
]
});
}
async handleEnterPassword(prev, password) {
return await this.executeFlowTask({
flow_token: prev.flowToken,
subtask_inputs: [
{
subtask_id: "LoginEnterPassword",
enter_password: {
password,
link: "next_link"
}
}
]
});
}
async handleAccountDuplicationCheck(prev) {
return await this.executeFlowTask({
flow_token: prev.flowToken,
subtask_inputs: [
{
subtask_id: "AccountDuplicationCheck",
check_logged_in_account: {
link: "AccountDuplicationCheck_false"
}
}
]
});
}
async handleTwoFactorAuthChallenge(prev, secret) {
const totp = new OTPAuth__namespace.TOTP({ secret });
let error;
for (let attempts = 1; attempts < 4; attempts += 1) {
try {
return await this.executeFlowTask({
flow_token: prev.flowToken,
subtask_inputs: [
{
subtask_id: "LoginTwoFactorAuthChallenge",
enter_text: {
link: "next_link",
text: totp.generate()
}
}
]
});
} catch (err) {
error = err;
await new Promise((resolve) => setTimeout(resolve, 2e3 * attempts));
}
}
throw error;
}
async handleAcid(prev, email) {
return await this.executeFlowTask({
flow_token: prev.flowToken,
subtask_inputs: [
{
subtask_id: "LoginAcid",
enter_text: {
text: email,
link: "next_link"
}
}
]
});
}
async handleSuccessSubtask(prev) {
return await this.executeFlowTask({
flow_token: prev.flowToken,
subtask_inputs: []
});
}
async executeFlowTask(data) {
const onboardingTaskUrl = "https://api.twitter.com/1.1/onboarding/task.json";
const token = this.guestToken;
if (token == null) {
throw new Error("Authentication token is null or undefined.");
}
const headers = new headersPolyfill.Headers({
authorization: `Bearer ${this.bearerToken}`,
cookie: await this.getCookieString(),
"content-type": "application/json",
"User-Agent": "Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36",
"x-guest-token": token,
"x-twitter-auth-type": "OAuth2Client",
"x-twitter-active-user": "yes",
"x-twitter-client-language": "en"
});
await this.installCsrfToken(headers);
const res = await this.fetch(onboardingTaskUrl, {
credentials: "include",
method: "POST",
headers,
body: JSON.stringify(data)
});
await updateCookieJar(this.jar, res.headers);
if (!res.ok) {
return { status: "error", err: new Error(await res.text()) };
}
const flow = await res.json();
if (flow?.flow_token == null) {
return { status: "error", err: new Error("flow_token not found.") };
}
if (flow.errors?.length) {
return {
status: "error",
err: new Error(
`Authentication error (${flow.errors[0].code}): ${flow.errors[0].message}`
)
};
}
if (typeof flow.flow_token !== "string") {
return {
status: "error",
err: new Error("flow_token was not a string.")
};
}
const subtask = flow.subtasks?.length ? flow.subtasks[0] : void 0;
value.Check(TwitterUserAuthSubtask, subtask);
if (subtask && subtask.subtask_id === "DenyLoginSubtask") {
return {
status: "error",
err: new Error("Authentication error: DenyLoginSubtask")
};
}
return {
status: "success",
subtask,
flowToken: flow.flow_token
};
}
}
async function* getUserTimeline(query, maxProfiles, fetchFunc) {
let nProfiles = 0;
let cursor = void 0;
let consecutiveEmptyBatches = 0;
while (nProfiles < maxProfiles) {
const batch = await fetchFunc(
query,
maxProfiles,
cursor
);
const { profiles, next } = batch;
cursor = next;
if (profiles.length === 0) {
consecutiveEmptyBatches++;
if (consecutiveEmptyBatches > 5) break;
} else consecutiveEmptyBatches = 0;
for (const profile of profiles) {
if (nProfiles < maxProfiles) yield profile;
else break;
nProfiles++;
}
if (!next) break;
}
}
async function* getTweetTimeline(query, maxTweets, fetchFunc) {
let nTweets = 0;
let cursor = void 0;
while (nTweets < maxTweets) {
const batch = await fetchFunc(
query,
maxTweets,
cursor
);
const { tweets, next } = batch;
if (tweets.length === 0) {
break;
}
for (const tweet of tweets) {
if (nTweets < maxTweets) {
cursor = next;
yield tweet;
} else {
break;
}
nTweets++;
}
}
}
function isFieldDefined(key) {
return function(value) {
return isDefined(value[key]);
};
}
function isDefined(value) {
return value != null;
}
const reHashtag = /\B(\#\S+\b)/g;
const reCashtag = /\B(\$\S+\b)/g;
const reTwitterUrl = /https:(\/\/t\.co\/([A-Za-z0-9]|[A-Za-z]){10})/g;
const reUsername = /\B(\@\S{1,15}\b)/g;
function parseMediaGroups(media) {
const photos = [];
const videos = [];
let sensitiveContent = void 0;
for (const m of media.filter(isFieldDefined("id_str")).filter(isFieldDefined("media_url_https"))) {
if (m.type === "photo") {
photos.push({
id: m.id_str,
url: m.media_url_https,
alt_text: m.ext_alt_text
});
} else if (m.type === "video") {
videos.push(parseVideo(m));
}
const sensitive = m.ext_sensitive_media_warning;
if (sensitive != null) {
sensitiveContent = sensitive.adult_content || sensitive.graphic_violence || sensitive.other;
}
}
return { sensitiveContent, photos, videos };
}
function parseVideo(m) {
const video = {
id: m.id_str,
preview: m.media_url_https
};
let maxBitrate = 0;
const variants = m.video_info?.variants ?? [];
for (const variant of variants) {
const bitrate = variant.bitrate;
if (bitrate != null && bitrate > maxBitrate && variant.url != null) {
let variantUrl = variant.url;
const stringStart = 0;
const tagSuffixIdx = variantUrl.indexOf("?tag=10");
if (tagSuffixIdx !== -1) {
variantUrl = variantUrl.substring(stringStart, tagSuffixIdx + 1);
}
video.url = variantUrl;
maxBitrate = bitrate;
}
}
return video;
}
function reconstructTweetHtml(tweet, photos, videos) {
const media = [];
let html = tweet.full_text ?? "";
html = html.replace(reHashtag, linkHashtagHtml);
html = html.replace(reCashtag, linkCashtagHtml);
html = html.replace(reUsername, linkUsernameHtml);
html = html.replace(reTwitterUrl, unwrapTcoUrlHtml(tweet, media));
for (const { url } of photos) {
if (media.indexOf(url) !== -1) {
continue;
}
html += `<br><img src="${url}"/>`;
}
for (const { preview: url } of videos) {
if (media.indexOf(url) !== -1) {
continue;
}
html += `<br><img src="${url}"/>`;
}
html = html.replace(/\n/g, "<br>");
return html;
}
function linkHashtagHtml(hashtag) {
return `<a href="https://twitter.com/hashtag/${hashtag.replace(
"#",
""
)}">${hashtag}</a>`;
}
function linkCashtagHtml(cashtag) {
return `<a href="https://twitter.com/search?q=%24${cashtag.replace(
"$",
""
)}">${cashtag}</a>`;
}
function linkUsernameHtml(username) {
return `<a href="https://twitter.com/${username.replace(
"@",
""
)}">${username}</a>`;
}
function unwrapTcoUrlHtml(tweet, foundedMedia) {
return function(tco) {
for (const entity of tweet.entities?.urls ?? []) {
if (tco === entity.url && entity.expanded_url != null) {
return `<a href="${entity.expanded_url}">${tco}</a>`;
}
}
for (const entity of tweet.extended_entities?.media ?? []) {
if (tco === entity.url && entity.media_url_https != null) {
foundedMedia.push(entity.media_url_https);
return `<br><a href="${tco}"><img src="${entity.media_url_https}"/></a>`;
}
}
return tco;
};
}
function parseLegacyTweet(user, tweet) {
if (tweet == null) {
return {
success: false,
err: new Error("Tweet was not found in the timeline object.")
};
}
if (user == null) {
return {
success: false,
err: new Error("User was not found in the timeline object.")
};
}
if (!tweet.id_str) {
if (!tweet.conversation_id_str) {
return {
success: false,
err: new Error("Tweet ID was not found in object.")
};
}
tweet.id_str = tweet.conversation_id_str;
}
const hashtags = tweet.entities?.hashtags ?? [];
const mentions = tweet.entities?.user_mentions ?? [];
const media = tweet.extended_entities?.media ?? [];
const pinnedTweets = new Set(
user.pinned_tweet_ids_str ?? []
);
const urls = tweet.entities?.urls ?? [];
const { photos, videos, sensitiveContent } = parseMediaGroups(media);
const tw = {
bookmarkCount: tweet.bookmark_count,
conversationId: tweet.conversation_id_str,
id: tweet.id_str,
hashtags: hashtags.filter(isFieldDefined("text")).map((hashtag) => hashtag.text),
likes: tweet.favorite_count,
mentions: mentions.filter(isFieldDefined("id_str")).map((mention) => ({
id: mention.id_str,
username: mention.screen_name,
name: mention.name
})),
name: user.name,
permanentUrl: `https://twitter.com/${user.screen_name}/status/${tweet.id_str}`,
photos,
replies: tweet.reply_count,
retweets: tweet.retweet_count,
text: tweet.full_text,
thread: [],
urls: urls.filter(isFieldDefined("expanded_url")).map((url) => url.expanded_url),
userId: tweet.user_id_str,
username: user.screen_name,
videos,
isQuoted: false,
isReply: false,
isRetweet: false,
isPin: false,
sensitiveContent: false
};
if (tweet.created_at) {
tw.timeParsed = new Date(Date.parse(tweet.created_at));
tw.timestamp = Math.floor(tw.timeParsed.valueOf() / 1e3);
}
if (tweet.place?.id) {
tw.place = tweet.place;
}
const quotedStatusIdStr = tweet.quoted_status_id_str;
const inReplyToStatusIdStr = tweet.in_reply_to_status_id_str;
const retweetedStatusIdStr = tweet.retweeted_status_id_str;
const retweetedStatusResult = tweet.retweeted_status_result?.result;
if (quotedStatusIdStr) {
tw.isQuoted = true;
tw.quotedStatusId = quotedStatusIdStr;
}
if (inReplyToStatusIdStr) {
tw.isReply = true;
tw.inReplyToStatusId = inReplyToStatusIdStr;
}
if (retweetedStatusIdStr || retweetedStatusResult) {
tw.isRetweet = true;
tw.retweetedStatusId = retweetedStatusIdStr;
if (retweetedStatusResult) {
const parsedResult = parseLegacyTweet(
retweetedStatusResult?.core?.user_results?.result?.legacy,
retweetedStatusResult?.legacy
);
if (parsedResult.success) {
tw.retweetedStatus = parsedResult.tweet;
}
}
}
const views = parseInt(tweet.ext_views?.count ?? "");
if (!isNaN(views)) {
tw.views = views;
}
if (pinnedTweets.has(tweet.id_str)) {
tw.isPin = true;
}
if (sensitiveContent) {
tw.sensitiveContent = true;
}
tw.html = reconstructTweetHtml(tweet, tw.photos, tw.videos);
return { success: true, tweet: tw };
}
function parseResult(result) {
const noteTweetResultText = result?.note_tweet?.note_tweet_results?.result?.text;
if (result?.legacy && noteTweetResultText) {
result.legacy.full_text = noteTweetResultText;
}
const tweetResult = parseLegacyTweet(
result?.core?.user_results?.result?.legacy,
result?.legacy
);
if (!tweetResult.success) {
return tweetResult;
}
if (!tweetResult.tweet.views && result?.views?.count) {
const views = parseInt(result.views.count);
if (!isNaN(views)) {
tweetResult.tweet.views = views;
}
}
const quotedResult = result?.quoted_status_result?.result;
if (quotedResult) {
if (quotedResult.legacy && quotedResult.rest_id) {
quotedResult.legacy.id_str = quotedResult.rest_id;
}
const quotedTweetResult = parseResult(quotedResult);
if (quotedTweetResult.success) {
tweetResult.tweet.quotedStatus = quotedTweetResult.tweet;
}
}
return tweetResult;
}
const expectedEntryTypes = ["tweet", "profile-conversation"];
function parseTimelineTweetsV2(timeline) {
let bottomCursor;
let topCursor;
const tweets = [];
const instructions = timeline.data?.user?.result?.timeline_v2?.timeline?.instructions ?? [];
for (const instruction of instructions) {
const entries = instruction.entries ?? [];
for (const entry of entries) {
const entryContent = entry.content;
if (!entryContent) continue;
if (entryContent.cursorType === "Bottom") {
bottomCursor = entryContent.value;
continue;
} else if (entryContent.cursorType === "Top") {
topCursor = entryContent.value;
continue;
}
const idStr = entry.entryId;
if (!expectedEntryTypes.some((entryType) => idStr.startsWith(entryType))) {
continue;
}
if (entryContent.itemContent) {
parseAndPush(tweets, entryContent.itemContent, idStr);
} else if (entryContent.items) {
for (const item of entryContent.items) {
if (item.item?.itemContent) {
parseAndPush(tweets, item.item.itemContent, idStr);
}
}
}
}
}
return { tweets, next: bottomCursor, previous: topCursor };
}
function parseTimelineEntryItemContentRaw(content, entryId, isConversation = false) {
let result = content.tweet_results?.result ?? content.tweetResult?.result;
if (result?.__typename === "Tweet" || result?.__typename === "TweetWithVisibilityResults" && result?.tweet) {
if (result?.__typename === "TweetWithVisibilityResults")
result = result.tweet;
if (result?.legacy) {
result.legacy.id_str = result.rest_id ?? entryId.replace("conversation-", "").replace("tweet-", "");
}
const tweetResult = parseResult(result);
if (tweetResult.success) {
if (isConversation) {
if (content?.tweetDisplayType === "SelfThread") {
tweetResult.tweet.isSelfThread = true;
}
}
return tweetResult.tweet;
}
}
return null;
}
function parseAndPush(tweets, content, entryId, isConversation = false) {
const tweet = parseTimelineEntryItemContentRaw(
content,
entryId,
isConversation
);
if (tweet) {
tweets.push(tweet);
}
}
function parseThreadedConversation(conversation) {
const tweets = [];
const instructions = conversation.data?.threaded_conversation_with_injections_v2?.instructions ?? [];
for (const instruction of instructions) {
const entries = instruction.entries ?? [];
for (const entry of entries) {
const entryContent = entry.content?.itemContent;
if (entryContent) {
parseAndPush(tweets, entryContent, entry.entryId, true);
}
for (const item of entry.content?.items ?? []) {
const itemContent = item.item?.itemContent;
if (itemContent) {
parseAndPush(tweets, itemContent, entry.entryId, true);
}
}
}
}
for (const tweet of tweets) {
if (tweet.inReplyToStatusId) {
for (const parentTweet of tweets) {
if (parentTweet.id === tweet.inReplyToStatusId) {
tweet.inReplyToStatus = parentTweet;
break;
}
}
}
if (tweet.isSelfThread && tweet.conversationId === tweet.id) {
for (const childTweet of tweets) {
if (childTweet.isSelfThread && childTweet.id !== tweet.id) {
tweet.thread.push(childTweet);
}
}
if (tweet.thread.length === 0) {
tweet.isSelfThread = false;
}
}
}
return tweets;
}
function parseSearchTimelineTweets(timeline) {
let bottomCursor;
let topCursor;
const tweets = [];
const instructions = timeline.data?.search_by_raw_query?.search_timeline?.timeline?.instructions ?? [];
for (const instruction of instructions) {
if (instruction.type === "TimelineAddEntries" || instruction.type === "TimelineReplaceEntry") {
if (instruction.entry?.content?.cursorType === "Bottom") {
bottomCursor = instruction.entry.content.value;
continue;
} else if (instruction.entry?.content?.cursorType === "Top") {
topCursor = instruction.entry.content.value;
continue;
}
const entries = instruction.entries ?? [];
for (const entry of entries) {
const itemContent = entry.content?.itemContent;
if (itemContent?.tweetDisplayType === "Tweet") {
const tweetResultRaw = itemContent.tweet_results?.result;
const tweetResult = parseLegacyTweet(
tweetResultRaw?.core?.user_results?.result?.legacy,
tweetResultRaw?.legacy
);
if (tweetResult.success) {
if (!tweetResult.tweet.views && tweetResultRaw?.views?.count) {
const views = parseInt(tweetResultRaw.views.count);
if (!isNaN(views)) {
tweetResult.tweet.views = views;
}
}
tweets.push(tweetResult.tweet);
}
} else if (entry.content?.cursorType === "Bottom") {
bottomCursor = entry.content.value;
} else if (entry.content?.cursorType === "Top") {
topCursor = entry.content.value;
}
}
}
}
return { tweets, next: bottomCursor, previous: topCursor };
}
function parseSearchTimelineUsers(timeline) {
let bottomCursor;
let topCursor;
const profiles = [];
const instructions = timeline.data?.search_by_raw_query?.search_timeline?.timeline?.instructions ?? [];
for (const instruction of instructions) {
if (instruction.type === "TimelineAddEntries" || instruction.type === "TimelineReplaceEntry") {
if (instruction.entry?.content?.cursorType === "Bottom") {
bottomCursor = instruction.entry.content.value;
continue;
} else if (instruction.entry?.content?.cursorType === "Top") {
topCursor = instruction.entry.content.value;
continue;
}
const entries = instruction.entries ?? [];
for (const entry of entries) {
const itemContent = entry.content?.itemContent;
if (itemContent?.userDisplayType === "User") {
const userResultRaw = itemContent.user_results?.result;
if (userResultRaw?.legacy) {
const profile = parseProfile(
userResultRaw.legacy,
userResultRaw.is_blue_verified
);
if (!profile.userId) {
profile.userId = userResultRaw.rest_id;
}
profiles.push(profile);
}
} else if (entry.content?.cursorType === "Bottom") {
bottomCursor = entry.content.value;
} else if (entry.content?.cursorType === "Top") {
topCursor = entry.content.value;
}
}
}
}
return { profiles, next: bottomCursor, previous: topCursor };
}
var SearchMode = /* @__PURE__ */ ((SearchMode2) => {
SearchMode2[SearchMode2["Top"] = 0] = "Top";
SearchMode2[SearchMode2["Latest"] = 1] = "Latest";
SearchMode2[SearchMode2["Photos"] = 2] = "Photos";
SearchMode2[SearchMode2["Videos"] = 3] = "Videos";
SearchMode2[SearchMode2["Users"] = 4] = "Users";
return SearchMode2;
})(SearchMode || {});
function searchTweets(query, maxTweets, searchMode, auth) {
return getTweetTimeline(query, maxTweets, (q, mt, c) => {
return fetchSearchTweets(q, mt, searchMode, auth, c);
});
}
function searchProfiles(query, maxProfiles, auth) {
return getUserTimeline(query, maxProfiles, (q, mt, c) => {
return fetchSearchProfiles(q, mt, auth, c);
});
}
async function fetchSearchTweets(query, maxTweets, searchMode, auth, cursor) {
const timeline = await getSearchTimeline(
query,
maxTweets,
searchMode,
auth,
cursor
);
return parseSearchTimelineTweets(timeline);
}
async function fetchSearchProfiles(query, maxProfiles, auth, cursor) {
const timeline = await getSearchTimeline(
query,
maxProfiles,
4 /* Users */,
auth,
cursor
);
return parseSearchTimelineUsers(timeline);
}
async function getSearchTimeline(query, maxItems, searchMode, auth, cursor) {
if (!auth.isLoggedIn()) {
throw new Error("Scraper is not logged-in for search.");
}
if (maxItems > 50) {
maxItems = 50;
}
const variables = {
rawQuery: query,
count: maxItems,
querySource: "typed_query",
product: "Top"
};
const features = addApiFeatures({
longform_notetweets_inline_media_enabled: true,
responsive_web_enhance_cards_enabled: false,
responsive_web_media_download_video_enabled: false,
responsive_web_twitter_article_tweet_consumption_enabled: false,
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
interactive_text_enabled: false,
responsive_web_text_conversations_enabled: false,
vibe_api_enabled: false
});
const fieldToggles = {
withArticleRichContentState: false
};
if (cursor != null && cursor != "") {
variables["cursor"] = cursor;
}
switch (searchMode) {
case 1 /* Latest */:
variables.product = "Latest";
break;
case 2 /* Photos */:
variables.product = "Photos";
break;
case 3 /* Videos */:
variables.product = "Videos";
break;
case 4 /* Users */:
variables.product = "People";
break;
}
const params = new URLSearchParams();
params.set("features", stringify(features) ?? "");
params.set("fieldToggles", stringify(fieldToggles) ?? "");
params.set("variables", stringify(variables) ?? "");
const res = await requestApi(
`https://api.twitter.com/graphql/gkjsKepM6gl_HmFWoWKfgg/SearchTimeline?${params.toString()}`,
auth
);
if (!res.success) {
throw res.err;
}
return res.value;
}
function parseRelationshipTimeline(timeline) {
let bottomCursor;
let topCursor;
const profiles = [];
const instructions = timeline.data?.user?.result?.timeline?.timeline?.instructions ?? [];
for (const instruction of instructions) {
if (instruction.type === "TimelineAddEntries" || instruction.type === "TimelineReplaceEntry") {
if (instruction.entry?.content?.cursorType === "Bottom") {
bottomCursor = instruction.entry.content.value;
continue;
}
if (instruction.entry?.content?.cursorType === "Top") {
topCursor = instruction.entry.content.value;
continue;
}
const entries = instruction.entries ?? [];
for (const entry of entries) {
const itemContent = entry.content?.itemContent;
if (itemContent?.userDisplayType === "User") {
const userResultRaw = itemContent.user_results?.result;
if (userResultRaw?.legacy) {
const profile = parseProfile(
userResultRaw.legacy,
userResultRaw.is_blue_verified
);
if (!profile.userId) {
profile.userId = userResultRaw.rest_id;
}
profiles.push(profile);
}
} else if (entry.content?.cursorType === "Bottom") {
bottomCursor = entry.content.value;
} else if (entry.content?.cursorType === "Top") {
topCursor = entry.content.value;
}
}
}
}
return { profiles, next: bottomCursor, previous: topCursor };
}
function getFollowing(userId, maxProfiles, auth) {
return getUserTimeline(userId, maxProfiles, (q, mt, c) => {
return fetchProfileFollowing(q, mt, auth, c);
});
}
function getFollowers(userId, maxProfiles, auth) {
return getUserTimeline(userId, maxProfiles, (q, mt, c) => {
return fetchProfileFollowers(q, mt, auth, c);
});
}
async function fetchProfileFollowing(userId, maxProfiles, auth, cursor) {
const timeline = await getFollowingTimeline(
userId,
maxProfiles,
auth,
cursor
);
return parseRelationshipTimeline(timeline);
}
async function fetchProfileFollowers(userId, maxProfiles, auth, cursor) {
const timeline = await getFollowersTimeline(
userId,
maxProfiles,
auth,
cursor
);
return parseRelationshipTimeline(timeline);
}
async function getFollowingTimeline(userId, maxItems, auth, cursor) {
if (!auth.isLoggedIn()) {
throw new Error("Scraper is not logged-in for profile following.");
}
if (maxItems > 50) {
maxItems = 50;
}
const variables = {
userId,
count: maxItems,
includePromotedContent: false
};
const features = addApiFeatures({
responsive_web_twitter_article_tweet_consumption_enabled: false,
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
longform_notetweets_inline_media_enabled: true,
responsive_web_media_download_video_enabled: false
});
if (cursor != null && cursor != "") {
variables["cursor"] = cursor;
}
const params = new URLSearchParams();
params.set("features", stringify(features) ?? "");
params.set("variables", stringify(variables) ?? "");
const res = await requestApi(
`https://twitter.com/i/api/graphql/iSicc7LrzWGBgDPL0tM_TQ/Following?${params.toString()}`,
auth
);
if (!res.success) {
throw res.err;
}
return res.value;
}
async function getFollowersTimeline(userId, maxItems, auth, cursor) {
if (!auth.isLoggedIn()) {
throw new Error("Scraper is not logged-in for profile followers.");
}
if (maxItems > 50) {
maxItems = 50;
}
const variables = {
userId,
count: maxItems,
includePromotedContent: false
};
const features = addApiFeatures({
responsive_web_twitter_article_tweet_consumption_enabled: false,
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
longform_notetweets_inline_media_enabled: true,
responsive_web_media_download_video_enabled: false
});
if (cursor != null && cursor != "") {
variables["cursor"] = cursor;
}
const params = new URLSearchParams();
params.set("features", stringify(features) ?? "");
params.set("variables", stringify(variables) ?? "");
const res = await requestApi(
`https://twitter.com/i/api/graphql/rRXFSG5vR6drKr5M37YOTw/Followers?${params.toString()}`,
auth
);
if (!res.success) {
throw res.err;
}
return res.value;
}
async function followUser(username, auth) {
if (!await auth.isLoggedIn()) {
throw new Error("Must be logged in to follow users");
}
const userIdResult = await getUserIdByScreenName(username, auth);
if (!userIdResult.success) {
throw new Error(`Failed to get user ID: ${userIdResult.err.message}`);
}
const userId = userIdResult.value;
const requestBody = {
include_profile_interstitial_type: "1",
skip_status: "true",
user_id: userId
};
const headers = new headersPolyfill.Headers({
"Content-Type": "application/x-www-form-urlencoded",
Referer: `https://twitter.com/${username}`,
"X-Twitter-Active-User": "yes",
"X-Twitter-Auth-Type": "OAuth2Session",
"X-Twitter-Client-Language": "en",
Authorization: `Bearer ${bearerToken}`
});
await auth.installTo(headers, "https://api.twitter.com/1.1/friendships/create.json");
const res = await auth.fetch(
"https://api.twitter.com/1.1/friendships/create.json",
{
method: "POST",
headers,
body: new URLSearchParams(requestBody).toString(),
credentials: "in