@just-every/mcp-screenshot-website-fast
Version:
Fast screenshot capture tool for web pages - optimized for Claude Vision API
732 lines (731 loc) • 27.9 kB
JavaScript
import puppeteer from 'puppeteer';
import { logger } from '../utils/logger.js';
logger.debug('Screenshot module loaded');
let browser = null;
let browserLaunchPromise = null;
let lastActivityTime = Date.now();
let inactivityTimer = null;
const BROWSER_IDLE_TIMEOUT_MS = 60000;
const MIN_BROWSER_LIFETIME_MS = 5000;
function updateActivityTime() {
lastActivityTime = Date.now();
resetInactivityTimer();
}
function resetInactivityTimer() {
if (inactivityTimer) {
clearTimeout(inactivityTimer);
inactivityTimer = null;
}
if (!browser || !browser.isConnected()) {
return;
}
inactivityTimer = setTimeout(async () => {
const timeSinceLastActivity = Date.now() - lastActivityTime;
const browserAge = Date.now() - lastActivityTime;
if (timeSinceLastActivity >= BROWSER_IDLE_TIMEOUT_MS &&
browserAge >= MIN_BROWSER_LIFETIME_MS) {
logger.info(`Browser idle for ${timeSinceLastActivity}ms, closing to save resources...`);
await closeBrowser();
}
else {
resetInactivityTimer();
}
}, BROWSER_IDLE_TIMEOUT_MS);
inactivityTimer.unref();
}
async function launchBrowser() {
logger.info('Launching new browser instance...');
logger.debug('Puppeteer executable path:', puppeteer.executablePath());
logger.debug('Browser launch options:', {
headless: true,
args: ['--no-sandbox', '--disable-setuid-sandbox', '...etc'],
});
const newBrowser = await puppeteer.launch({
headless: true,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-accelerated-2d-canvas',
'--disable-gpu',
'--no-first-run',
'--no-zygote',
'--disable-background-timer-throttling',
'--disable-backgrounding-occluded-windows',
'--disable-renderer-backgrounding',
'--disable-features=TranslateUI',
'--disable-ipc-flooding-protection',
'--disable-default-apps',
'--no-default-browser-check',
],
ignoreDefaultArgs: ['--enable-automation'],
});
logger.info('Browser launched successfully');
logger.debug('Browser process PID:', newBrowser.process()?.pid);
newBrowser.on('disconnected', () => {
logger.warn('Browser disconnected event received');
logger.debug('Browser instance:', {
isConnected: newBrowser.isConnected(),
});
if (browser === newBrowser) {
browser = null;
browserLaunchPromise = null;
}
});
startHealthCheck();
logger.debug('Health check started');
updateActivityTime();
logger.debug('Activity tracking initialized');
return newBrowser;
}
async function getBrowser(forceRestart = false) {
logger.debug('getBrowser called', { forceRestart, hasBrowser: !!browser });
if (forceRestart && browser) {
logger.info('Force restarting browser...');
await closeBrowser();
}
if (browser && browser.isConnected()) {
return browser;
}
if (browserLaunchPromise) {
try {
browser = await browserLaunchPromise;
if (browser && browser.isConnected()) {
return browser;
}
}
catch (error) {
logger.error('Failed to wait for browser launch:', error);
browserLaunchPromise = null;
}
}
try {
logger.debug('Starting browser launch...');
browserLaunchPromise = launchBrowser();
browser = await browserLaunchPromise;
browserLaunchPromise = null;
logger.info('Browser obtained successfully');
return browser;
}
catch (error) {
logger.error('Failed to launch browser:', error.message);
logger.debug('Browser launch error details:', {
name: error.name,
stack: error.stack,
code: error.code,
});
browserLaunchPromise = null;
browser = null;
throw error;
}
}
export async function closeBrowser() {
logger.debug('closeBrowser called');
stopHealthCheck();
if (inactivityTimer) {
clearTimeout(inactivityTimer);
inactivityTimer = null;
}
if (browser && browser.isConnected()) {
logger.info('Closing browser...');
try {
await browser.close();
logger.info('Browser closed successfully');
}
catch (error) {
logger.error('Error closing browser:', error);
}
browser = null;
browserLaunchPromise = null;
}
else {
logger.debug('No browser to close or already disconnected');
}
}
let healthCheckInterval = null;
function startHealthCheck() {
if (healthCheckInterval)
return;
healthCheckInterval = setInterval(async () => {
if (browser && !browser.isConnected()) {
logger.warn('Browser health check failed - browser disconnected');
browser = null;
browserLaunchPromise = null;
}
else if (browser && browser.isConnected()) {
try {
const pages = await browser.pages();
logger.debug(`Health check: ${pages.length} pages open`);
if (pages.length > 1) {
logger.info(`Closing ${pages.length - 1} unused pages to free memory`);
for (let i = 1; i < pages.length; i++) {
await pages[i].close().catch(() => { });
}
}
}
catch (error) {
logger.error('Error during health check:', error);
}
}
}, 30000);
healthCheckInterval.unref();
}
function stopHealthCheck() {
if (healthCheckInterval) {
clearInterval(healthCheckInterval);
healthCheckInterval = null;
}
}
async function setupPage(browser) {
logger.debug('Creating new page...');
const page = await browser.newPage();
logger.debug('Page created successfully');
await page.setDefaultNavigationTimeout(60000);
await page.setDefaultTimeout(60000);
await page.setJavaScriptEnabled(true);
await page.setOfflineMode(false);
await page.setRequestInterception(true);
page.on('request', request => {
const resourceType = request.resourceType();
if (['font', 'media'].includes(resourceType)) {
request.abort();
}
else {
request.continue();
}
});
page.on('error', error => {
logger.error('Page crashed:', error.message);
logger.debug('Page crash details:', error);
});
page.on('pageerror', error => {
logger.warn('Page JavaScript error:', error.message || error);
});
page.on('frameattached', frame => {
logger.debug(`Frame attached: ${frame.url()}`);
});
page.on('framedetached', frame => {
logger.debug(`Frame detached: ${frame.url()}`);
});
page.on('framenavigated', frame => {
logger.debug(`Frame navigated: ${frame.url()}`);
});
return page;
}
async function navigateWithRetry(page, url, options, browserRestartCallback) {
const maxRetries = 3;
let lastError;
let currentPage = page;
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
if (currentPage.isClosed()) {
logger.warn('Page is closed, attempting to create new page...');
if (browserRestartCallback) {
currentPage = await browserRestartCallback();
}
else {
throw new Error('Page is closed and no recovery callback provided');
}
}
logger.info(`Navigating to ${url} (attempt ${attempt}/${maxRetries})...`);
const frameDetachedPromise = new Promise((_, reject) => {
const handler = () => reject(new Error('Frame detached during navigation'));
currentPage.once('framedetached', handler);
currentPage.once('load', () => currentPage.off('framedetached', handler));
});
await Promise.race([
currentPage.goto(url, {
waitUntil: options.waitUntil || 'domcontentloaded',
timeout: 60000,
}),
frameDetachedPromise,
]);
if (currentPage.isClosed()) {
throw new Error('Page was closed after navigation');
}
await currentPage.evaluate(() => new Promise(resolve => setTimeout(resolve, 100)));
return currentPage;
}
catch (error) {
lastError = error;
logger.warn(`Navigation attempt ${attempt} failed:`, error.message);
const needsBrowserRestart = error.message.includes('Protocol error') ||
error.message.includes('Target closed') ||
error.message.includes('Session closed') ||
error.message.includes('Browser disconnected') ||
error.message.includes('Navigation failed because browser has disconnected');
if (attempt < maxRetries) {
await new Promise(resolve => setTimeout(resolve, 1000 * attempt));
if (needsBrowserRestart && browserRestartCallback) {
logger.info('Critical error detected, restarting browser...');
try {
currentPage = await browserRestartCallback();
}
catch (restartError) {
logger.error('Failed to restart browser:', restartError);
throw restartError;
}
}
else if (currentPage.isClosed() && browserRestartCallback) {
logger.info('Page closed, creating new page...');
currentPage = await browserRestartCallback();
}
}
}
}
throw lastError || new Error('Navigation failed after retries');
}
export async function captureScreenshot(options) {
logger.info('captureScreenshot called with options:', {
url: options.url,
fullPage: options.fullPage,
viewport: options.viewport,
waitUntil: options.waitUntil,
waitFor: options.waitFor,
});
updateActivityTime();
if (options.fullPage !== false) {
logger.debug('Delegating to captureTiledScreenshot');
return captureTiledScreenshot(options);
}
logger.info(`Taking viewport screenshot of ${options.url}`);
let browser = null;
let page = null;
let attemptCount = 0;
const maxAttempts = 2;
while (attemptCount < maxAttempts) {
try {
attemptCount++;
browser = await getBrowser(attemptCount > 1);
page = await setupPage(browser);
const viewport = {
width: options.viewport?.width || 1072,
height: options.viewport?.height || 1072,
};
await page.setViewport(viewport);
const recoveryCallback = async () => {
logger.info('Recovering from error, creating new page...');
browser = await getBrowser(true);
const newPage = await setupPage(browser);
await newPage.setViewport(viewport);
return newPage;
};
page = await navigateWithRetry(page, options.url, options, recoveryCallback);
if (options.waitFor) {
await page.evaluate(ms => new Promise(resolve => setTimeout(resolve, ms)), options.waitFor);
}
const screenshot = (await page.screenshot({
type: 'png',
fullPage: false,
encoding: 'binary',
}));
const result = {
url: options.url,
screenshot,
timestamp: new Date(),
viewport,
format: 'png',
};
if (page && !page.isClosed()) {
await page.close().catch(() => { });
}
return result;
}
catch (error) {
logger.error(`Error taking screenshot (attempt ${attemptCount}/${maxAttempts}):`, error);
if (page && !page.isClosed()) {
await page.close().catch(() => { });
}
if (attemptCount >= maxAttempts) {
throw error;
}
logger.info('Retrying with fresh browser...');
await new Promise(resolve => setTimeout(resolve, 2000));
}
}
throw new Error('Failed to capture screenshot after all attempts');
}
export function getBrowserStats() {
return {
hasBrowser: !!browser,
isConnected: browser?.isConnected() || false,
lastActivityTime: new Date(lastActivityTime).toISOString(),
timeSinceLastActivity: Date.now() - lastActivityTime,
hasInactivityTimer: !!inactivityTimer,
idleTimeoutMs: BROWSER_IDLE_TIMEOUT_MS,
};
}
export async function warmupBrowser() {
logger.info('Warming up browser for faster first requests...');
try {
const warmupBrowser = await getBrowser();
logger.info('Browser warmed up successfully');
const page = await warmupBrowser.newPage();
await page.goto('data:text/html,<html><body>Warmup</body></html>', {
waitUntil: 'load',
timeout: 5000,
});
await page.close();
logger.debug('Browser warmup page test completed');
}
catch (error) {
logger.warn('Browser warmup failed (first request may be slower):', error);
}
}
process.on('SIGINT', async () => {
logger.debug('SIGINT received in screenshot module');
stopHealthCheck();
await closeBrowser();
process.exit(0);
});
process.on('SIGTERM', async () => {
logger.debug('SIGTERM received in screenshot module');
stopHealthCheck();
await closeBrowser();
process.exit(0);
});
process.on('exit', async () => {
logger.debug('Process exit in screenshot module');
stopHealthCheck();
await closeBrowser();
});
process.on('uncaughtException', async (error) => {
logger.error('Uncaught exception in screenshot module:', error);
stopHealthCheck();
await closeBrowser();
process.exit(1);
});
process.on('unhandledRejection', async (error) => {
logger.error('Unhandled rejection in screenshot module:', error);
stopHealthCheck();
await closeBrowser();
process.exit(1);
});
async function captureTiledScreenshot(options) {
const tileSize = options.viewport?.width || 1072;
logger.info(`Taking tiled screenshot of ${options.url}`);
let browser = null;
let page = null;
let attemptCount = 0;
const maxAttempts = 2;
while (attemptCount < maxAttempts) {
try {
attemptCount++;
browser = await getBrowser(attemptCount > 1);
page = await setupPage(browser);
await page.setViewport({
width: tileSize,
height: tileSize,
});
const recoveryCallback = async () => {
logger.info('Recovering from error, creating new page...');
browser = await getBrowser(true);
const newPage = await setupPage(browser);
await newPage.setViewport({
width: tileSize,
height: tileSize,
});
return newPage;
};
page = await navigateWithRetry(page, options.url, options, recoveryCallback);
if (options.waitFor) {
await page.evaluate(ms => new Promise(resolve => setTimeout(resolve, ms)), options.waitFor);
}
const fullPageHeight = await page.evaluate(() => {
return globalThis.document.documentElement
.scrollHeight;
});
logger.info('Capturing full page screenshot...');
const fullPageScreenshot = (await page.screenshot({
type: 'png',
fullPage: false,
encoding: 'binary',
clip: {
x: 0,
y: 0,
width: tileSize,
height: fullPageHeight,
},
}));
const sharp = await import('sharp');
const metadata = await sharp.default(fullPageScreenshot).metadata();
const dimensions = {
width: Math.min(metadata.width, tileSize),
height: metadata.height,
};
logger.info(`Full page dimensions: ${dimensions.width}x${dimensions.height} (viewport width: ${tileSize})`);
const cols = Math.ceil(dimensions.width / tileSize);
const rows = Math.ceil(dimensions.height / tileSize);
const tiles = [];
logger.info(`Creating ${rows}x${cols} tiles (${rows * cols} total)`);
for (let row = 0; row < rows; row++) {
for (let col = 0; col < cols; col++) {
const x = col * tileSize;
const y = row * tileSize;
const width = Math.min(tileSize, dimensions.width - x);
const height = Math.min(tileSize, dimensions.height - y);
const tileBuffer = await sharp
.default(fullPageScreenshot)
.extract({
left: x,
top: y,
width,
height,
})
.png()
.toBuffer();
tiles.push({
screenshot: tileBuffer,
index: row * cols + col,
row,
col,
x,
y,
width,
height,
});
logger.debug(`Created tile ${row},${col} at ${x},${y} (${width}x${height})`);
}
}
const result = {
url: options.url,
tiles,
timestamp: new Date(),
fullWidth: dimensions.width,
fullHeight: dimensions.height,
tileSize,
format: 'png',
};
if (page && !page.isClosed()) {
await page.close().catch(() => { });
}
return result;
}
catch (error) {
logger.error(`Error taking tiled screenshot (attempt ${attemptCount}/${maxAttempts}):`, error);
if (page && !page.isClosed()) {
await page.close().catch(() => { });
}
if (attemptCount >= maxAttempts) {
throw error;
}
logger.info('Retrying with fresh browser...');
await new Promise(resolve => setTimeout(resolve, 2000));
}
}
throw new Error('Failed to capture tiled screenshot after all attempts');
}
export async function captureScreencast(options) {
logger.info('captureScreencast called with options:', {
url: options.url,
duration: options.duration,
interval: options.interval,
viewport: options.viewport,
waitUntil: options.waitUntil,
waitFor: options.waitFor,
hasJsEvaluate: !!options.jsEvaluate,
});
updateActivityTime();
const frames = [];
const startTime = new Date();
let browser = null;
let page = null;
try {
browser = await getBrowser();
page = await setupPage(browser);
const viewport = {
width: options.viewport?.width || 1072,
height: options.viewport?.height || 1072,
};
await page.setViewport(viewport);
logger.info(`Starting screencast of ${options.url}`);
await page.goto(options.url, {
waitUntil: options.waitUntil || 'domcontentloaded',
timeout: 60000,
});
if (options.waitFor) {
await page.evaluate(ms => new Promise(resolve => setTimeout(resolve, ms)), options.waitFor);
}
let jsInstructionCount = 0;
const screenshotInterval = options.interval * 1000;
const jsExecutionInterval = 1000;
if (options.jsEvaluate) {
const jsInstructions = Array.isArray(options.jsEvaluate)
? options.jsEvaluate
: [options.jsEvaluate];
jsInstructionCount = jsInstructions.length;
logger.info(`Processing ${jsInstructionCount} JavaScript instruction(s) with ${screenshotInterval}ms screenshot intervals`);
const startTime = Date.now();
let nextJsIndex = 0;
let frameIndex = 0;
const jsDuration = jsInstructions.length * jsExecutionInterval;
while (Date.now() - startTime < jsDuration) {
const elapsed = Date.now() - startTime;
if (nextJsIndex < jsInstructions.length &&
elapsed >= nextJsIndex * jsExecutionInterval) {
logger.info(`Executing JavaScript instruction ${nextJsIndex + 1}/${jsInstructions.length}: ${jsInstructions[nextJsIndex].substring(0, 50)}...`);
try {
await page.evaluate(jsInstructions[nextJsIndex]);
logger.debug(`JavaScript instruction ${nextJsIndex + 1} completed`);
}
catch (error) {
logger.error(`JavaScript instruction ${nextJsIndex + 1} failed:`, error);
throw new Error(`Failed to execute JavaScript instruction ${nextJsIndex + 1}: ${error}`);
}
nextJsIndex++;
}
const screenshot = (await page.screenshot({
type: 'png',
fullPage: false,
encoding: 'binary',
}));
frames.push({
screenshot,
timestamp: new Date(),
index: frameIndex,
});
frameIndex++;
logger.debug(`Captured high-frequency frame ${frameIndex} at ${elapsed}ms`);
const nextScreenshotTime = startTime + frameIndex * screenshotInterval;
const waitTime = Math.max(0, nextScreenshotTime - Date.now());
if (waitTime > 0) {
await new Promise(resolve => setTimeout(resolve, waitTime));
}
}
jsInstructionCount = frameIndex;
}
const remainingDuration = options.duration * 1000 - jsInstructionCount * screenshotInterval;
const remainingFrames = Math.max(0, Math.floor(remainingDuration / screenshotInterval));
logger.info(`Captured ${jsInstructionCount} frames during JS execution. Capturing ${remainingFrames} additional frames at ${screenshotInterval}ms intervals for remaining ${remainingDuration}ms`);
for (let i = 0; i < remainingFrames; i++) {
const frameStart = Date.now();
const screenshot = (await page.screenshot({
type: 'png',
fullPage: false,
encoding: 'binary',
}));
const frameIndex = jsInstructionCount + i;
frames.push({
screenshot,
timestamp: new Date(),
index: frameIndex,
});
logger.debug(`Captured duration frame ${frameIndex + 1} (${i + 1}/${remainingFrames})`);
if (i < remainingFrames - 1) {
const elapsed = Date.now() - frameStart;
const waitTime = Math.max(0, screenshotInterval - elapsed);
if (waitTime > 0) {
await new Promise(resolve => setTimeout(resolve, waitTime));
}
}
}
const endTime = new Date();
const result = {
url: options.url,
frames,
startTime,
endTime,
duration: options.duration,
interval: options.interval,
viewport,
format: 'png',
};
logger.info(`Screencast completed: ${frames.length} frames captured`);
if (page && !page.isClosed()) {
await page.close().catch(() => { });
}
return result;
}
catch (error) {
logger.error('Error capturing screencast:', error);
if (page && !page.isClosed()) {
await page.close().catch(() => { });
}
throw error;
}
}
export async function captureConsole(options) {
logger.info('captureConsole called with options:', {
url: options.url,
jsCommand: options.jsCommand,
duration: options.duration,
waitUntil: options.waitUntil,
});
updateActivityTime();
const messages = [];
const startTime = new Date();
const duration = options.duration || 4;
let browser = null;
let page = null;
try {
browser = await getBrowser();
page = await setupPage(browser);
page.on('console', msg => {
const type = msg.type();
const text = msg.text();
const timestamp = new Date();
const args = [];
msg.args().forEach(arg => {
args.push(arg.toString());
});
messages.push({
type,
text,
timestamp,
args: args.length > 0 ? args : undefined,
});
logger.debug(`Console ${type}: ${text}`);
});
page.on('pageerror', error => {
messages.push({
type: 'error',
text: error.toString(),
timestamp: new Date(),
});
logger.debug(`Page error: ${error}`);
});
logger.info(`Starting console capture for ${options.url}`);
await page.goto(options.url, {
waitUntil: options.waitUntil || 'domcontentloaded',
timeout: 60000,
});
if (options.jsCommand) {
logger.info(`Executing JS command: ${options.jsCommand}`);
try {
await page.evaluate(options.jsCommand);
logger.debug('JS command executed successfully');
}
catch (error) {
logger.error('Failed to execute JS command:', error);
messages.push({
type: 'error',
text: `Failed to execute JS command: ${error}`,
timestamp: new Date(),
});
}
}
logger.info(`Capturing console for ${duration} seconds...`);
await new Promise(resolve => setTimeout(resolve, duration * 1000));
const endTime = new Date();
const result = {
url: options.url,
messages,
startTime,
endTime,
duration,
executedCommand: options.jsCommand,
};
logger.info(`Console capture completed: ${messages.length} messages captured`);
if (page && !page.isClosed()) {
await page.close().catch(() => { });
}
return result;
}
catch (error) {
logger.error('Error capturing console:', error);
if (page && !page.isClosed()) {
await page.close().catch(() => { });
}
throw error;
}
}