rapidapi-mcp-server
Version:
MCP server for discovering and assessing APIs from RapidAPI marketplace
149 lines (148 loc) • 6.84 kB
JavaScript
import puppeteer from 'puppeteer';
export class BrowserManager {
browser = null;
config;
constructor(config = {}) {
this.config = {
headless: true, // REQUIRED: Must be true in server environment (no X Windows)
timeout: 60000,
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
args: [
// EXACT working args from Puppeteer MCP (simpler is better!)
'--no-sandbox',
'--single-process',
'--no-zygote'
],
...config
};
}
async getBrowser() {
// ALWAYS create fresh browser instance to avoid anti-bot flagging
// Once a browser is flagged by RapidAPI, all pages from it hit CAPTCHA
if (this.browser) {
await this.browser.close();
this.browser = null;
}
const launchOptions = {
headless: this.config.headless,
args: this.config.args,
slowMo: 100, // Add slight delay between actions for human-like behavior
defaultViewport: null, // Use actual browser viewport
devtools: false
};
// Use custom Chrome executable path if provided
if (process.env.PUPPETEER_EXECUTABLE_PATH) {
launchOptions.executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
console.error(`Using custom Chrome executable: ${process.env.PUPPETEER_EXECUTABLE_PATH}`);
}
this.browser = await puppeteer.launch(launchOptions);
return this.browser;
}
async createPage() {
const browser = await this.getBrowser();
const page = await browser.newPage();
// Randomize viewport slightly for more human-like behavior
const baseWidth = 1920;
const baseHeight = 1080;
await page.setViewport({
width: baseWidth + Math.floor(Math.random() * 100) - 50,
height: baseHeight + Math.floor(Math.random() * 100) - 50
});
if (this.config.userAgent) {
await page.setUserAgent(this.config.userAgent);
}
// Set additional headers to appear more like a real browser
await page.setExtraHTTPHeaders({
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Upgrade-Insecure-Requests': '1',
'Cache-Control': 'no-cache',
'Pragma': 'no-cache'
});
// Remove webdriver property and other automation indicators
await page.evaluateOnNewDocument(() => {
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
});
// Mock chrome object
window.chrome = {
runtime: {}
};
// Hide automation indicators
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5],
});
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US', 'en'],
});
});
// Set default timeout
page.setDefaultTimeout(this.config.timeout || 60000);
return page;
}
async closeBrowser() {
if (this.browser) {
await this.browser.close();
this.browser = null;
}
}
async navigateWithRetry(page, url, maxRetries = 3) {
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
await page.goto(url, {
waitUntil: 'networkidle2',
timeout: this.config.timeout
});
return;
}
catch (error) {
console.error(`Navigation attempt ${attempt} failed:`, error);
if (attempt === maxRetries) {
throw new Error(`Failed to navigate to ${url} after ${maxRetries} attempts`);
}
// Wait before retrying
await new Promise(resolve => setTimeout(resolve, 2000 * attempt));
}
}
}
async waitForNetworkIdle(page, timeout = 5000) {
// Wait for network to be idle (no requests for 500ms)
// Puppeteer's networkidle2 waits for 500ms of no network activity
await new Promise(resolve => setTimeout(resolve, 2000));
}
async handleAntiBot(page) {
// Import ethical timing configuration
const { ETHICAL_RATE_LIMITS, getRandomDelay } = await import('./rate-limiting-config.js');
// Add human-like delay using ethical configuration
const delay = getRandomDelay(ETHICAL_RATE_LIMITS.antiDetectionDelay, ETHICAL_RATE_LIMITS.antiDetectionDelay + 3000);
await new Promise(resolve => setTimeout(resolve, delay));
// Simulate subtle human-like mouse movements (essential for headless detection avoidance)
const viewport = page.viewport();
if (viewport) {
await page.mouse.move(Math.random() * viewport.width, Math.random() * viewport.height, { steps: 3 } // Move in steps to appear more natural
);
}
// Check for common anti-bot patterns
const captchaElements = await page.$$('iframe[src*="captcha"], div[class*="captcha"], .cf-browser-verification, .challenge-form');
if (captchaElements.length > 0) {
console.error('ETHICAL USAGE: CAPTCHA detected - respecting website protection');
throw new Error('CAPTCHA detected - cannot proceed automatically per ethical usage policy');
}
// Check for Cloudflare protection
const cfElements = await page.$$('.cf-error-title, #cf-wrapper, .cf-browser-verification');
if (cfElements.length > 0) {
console.error('ETHICAL USAGE: Cloudflare protection detected - respecting rate limits');
throw new Error('Cloudflare protection detected - respecting website protection per ethical usage policy');
}
// Check for rate limiting indicators
const rateLimitElements = await page.$$('[class*="rate"], [class*="limit"], [id*="rate"], [id*="limit"]');
if (rateLimitElements.length > 0) {
const textContent = await page.evaluate(() => document.body.textContent || '');
if (textContent.toLowerCase().includes('rate limit') || textContent.toLowerCase().includes('too many requests')) {
console.error('ETHICAL USAGE: Rate limiting detected - backing off');
throw new Error('Rate limiting detected - respecting website limits per ethical usage policy');
}
}
}
}