UNPKG

chrometools-mcp

Version:

MCP (Model Context Protocol) server for Chrome automation using Puppeteer. Persistent browser sessions, visual testing, Figma comparison, and design validation. Works seamlessly in WSL, Linux, and macOS.

943 lines (792 loc) 28.6 kB
/** * recorder/scenario-executor.js * * Executes recorded scenarios with: * 1. Action playback with error handling * 2. Parameter substitution * 3. Secret injection * 4. Dependency resolution and chaining * 5. Retry logic with fallback selectors */ import { resolveDependencies, checkDependencyCondition } from './dependency-resolver.js'; import { loadScenario, loadSecrets, loadIndex } from './scenario-storage.js'; /** * Execute scenario with dependencies * @param {string} scenarioName - Scenario to execute * @param {Object} page - Puppeteer page instance * @param {Object} params - Parameters for scenario * @param {Object} options - Execution options { executeDependencies, skipConditions, maxRetries, timeout } * @returns {Object} - Execution result */ export async function executeScenario(scenarioName, page, params = {}, options = {}) { const { executeDependencies = true, // NEW: Execute dependencies by default skipConditions = false, maxRetries = 3, timeout = 30000 } = options; const result = { success: false, scenarioName, executedScenarios: [], errors: [], outputs: {}, duration: 0 }; const startTime = Date.now(); try { // Load scenario index const scenarioIndex = await loadIndex(); let chain = [scenarioName]; // Default: execute only the requested scenario // Resolve and execute dependencies if enabled if (executeDependencies) { const resolution = resolveDependencies(scenarioName, scenarioIndex, { skipConditions }); if (resolution.errors.length > 0) { result.errors.push(...resolution.errors); return result; } chain = resolution.chain; // Use full dependency chain } // Execute chain in order for (const name of chain) { const scenario = await loadScenario(name); if (!scenario) { result.errors.push(`Scenario "${name}" not found`); return result; } // Check dependency conditions if (scenario.metadata?.dependencies) { for (const dep of scenario.metadata.dependencies) { if (dep.condition) { const context = { page, variables: params }; const shouldExecute = await checkDependencyCondition(dep.condition, context); if (!shouldExecute) { console.log(`Skipping scenario "${name}" due to condition`); continue; } } } } // Load secrets const secrets = await loadSecrets(name); // Merge secrets with params const executionParams = { ...params, ...secrets }; // Execute scenario const scenarioResult = await executeSingleScenario(scenario, page, executionParams, { maxRetries, timeout }); result.executedScenarios.push(name); if (!scenarioResult.success) { result.errors.push(...scenarioResult.errors); return result; } // Collect outputs for next scenarios if (scenarioResult.outputs) { Object.assign(result.outputs, scenarioResult.outputs); Object.assign(params, scenarioResult.outputs); } } result.success = true; } catch (error) { result.errors.push(`Execution failed: ${error.message}`); } finally { result.duration = Date.now() - startTime; } return result; } /** * Execute single scenario (without dependencies) * @param {Object} scenario - Scenario data * @param {Object} page - Puppeteer page * @param {Object} params - Parameters * @param {Object} options - Options * @returns {Object} - Execution result */ async function executeSingleScenario(scenario, page, params = {}, options = {}) { const { maxRetries = 3, timeout = 30000 } = options; const result = { success: false, errors: [], outputs: {}, actionResults: [] }; try { for (const action of scenario.chain) { // Substitute parameters in action const resolvedAction = substituteParameters(action, params); // Execute action with retry const actionResult = await executeActionWithRetry( resolvedAction, page, maxRetries, timeout ); result.actionResults.push(actionResult); if (!actionResult.success) { result.errors.push(`Action failed: ${actionResult.error}`); return result; } // Store outputs if action produces any if (actionResult.output) { Object.assign(result.outputs, actionResult.output); } } // Validate final URL if exitUrl is specified in metadata if (scenario.metadata?.exitUrl) { const currentUrl = page.url(); const expectedUrl = scenario.metadata.exitUrl; // Normalize URLs for comparison (remove trailing slashes, fragments) const normalizeUrl = (url) => { try { const parsed = new URL(url); // Remove fragment and trailing slash return `${parsed.origin}${parsed.pathname.replace(/\/$/, '')}${parsed.search}`; } catch { return url.replace(/\/$/, ''); } }; const normalizedCurrent = normalizeUrl(currentUrl); const normalizedExpected = normalizeUrl(expectedUrl); if (normalizedCurrent !== normalizedExpected) { result.errors.push( `❌ URL Validation Failed\n\n` + `Expected final URL: ${expectedUrl}\n` + `Actual final URL: ${currentUrl}\n\n` + `The scenario ended on a different page than expected.\n` + `This may indicate:\n` + ` - Navigation flow has changed\n` + ` - An action failed silently\n` + ` - Page redirected unexpectedly\n\n` + `💡 Suggestion: Check if the page flow or redirects have changed since recording.` ); return result; } // URL validation passed result.urlValidation = { success: true, expectedUrl, actualUrl: currentUrl }; } result.success = true; } catch (error) { result.errors.push(`Scenario execution error: ${error.message}`); } return result; } /** * Execute action with retry and fallback selectors */ async function executeActionWithRetry(action, page, maxRetries, timeout) { const result = { success: false, action: action.type, error: null, errorDetails: { attempts: [], selector: action.selector?.value || action.selector?.primary, context: null }, output: null, attempts: 0 }; for (let attempt = 1; attempt <= maxRetries; attempt++) { result.attempts = attempt; const attemptInfo = { number: attempt, selector: action.selector?.value || action.selector?.primary, error: null, timestamp: new Date().toISOString() }; try { // Execute action based on type const actionResult = await executeAction(action, page, timeout); result.success = true; result.output = actionResult.output; attemptInfo.success = true; result.errorDetails.attempts.push(attemptInfo); return result; } catch (error) { attemptInfo.error = error.message; attemptInfo.success = false; // Capture page context for error reporting if (attempt === maxRetries) { try { result.errorDetails.context = await capturePageContext(page, action); } catch (contextError) { console.error('Failed to capture page context:', contextError); } } result.errorDetails.attempts.push(attemptInfo); result.error = error.message; // If this is a selector error and we have fallbacks, try them if (action.selector?.fallbacks && action.selector.fallbacks.length > 0) { const fallback = action.selector.fallbacks[0]; console.log(`[Retry ${attempt}] Trying fallback selector: ${fallback}`); action.selector.value = fallback; action.selector.fallbacks = action.selector.fallbacks.slice(1); continue; } // If we have element description, try smartFindElement if (action.selector?.elementInfo?.text && attempt < maxRetries) { console.log(`[Retry ${attempt}] Selector failed, trying smartFindElement with description: ${action.selector.elementInfo.text}`); try { // Inject element finder utilities if not already done await page.evaluate(elementFinderUtilsCode); const smartResult = await page.evaluate((description) => { return window.smartFindElement({ description, maxResults: 3 }); }, action.selector.elementInfo.text); if (smartResult.candidates && smartResult.candidates.length > 0) { action.selector.value = smartResult.candidates[0].selector; action.selector.fallbacks = smartResult.candidates.slice(1).map(c => c.selector); console.log(`[Retry ${attempt}] Found alternative selector: ${action.selector.value}`); continue; } } catch (smartError) { console.error('[Retry] smartFindElement failed:', smartError.message); } } // Last attempt failed if (attempt === maxRetries) { // Create comprehensive error message result.error = formatDetailedError(action, result.errorDetails); return result; } // Wait before retry await new Promise(resolve => setTimeout(resolve, 1000)); } } return result; } /** * Capture page context for error reporting */ async function capturePageContext(page, action) { try { const context = { url: page.url(), title: await page.title(), elementExists: false, elementVisible: false, elementInfo: null, pageState: null }; const selector = action.selector?.value || action.selector?.primary; if (selector) { // Check if element exists context.elementExists = await page.evaluate((sel) => { return document.querySelector(sel) !== null; }, selector); // If exists, check visibility and get info if (context.elementExists) { context.elementInfo = await page.evaluate((sel) => { const el = document.querySelector(sel); const rect = el.getBoundingClientRect(); const styles = window.getComputedStyle(el); return { tagName: el.tagName, id: el.id, className: el.className, visible: rect.width > 0 && rect.height > 0 && styles.display !== 'none' && styles.visibility !== 'hidden', disabled: el.disabled || el.getAttribute('aria-disabled') === 'true', readonly: el.readOnly || el.getAttribute('aria-readonly') === 'true', position: { top: rect.top, left: rect.left, width: rect.width, height: rect.height }, styles: { display: styles.display, visibility: styles.visibility, opacity: styles.opacity, pointerEvents: styles.pointerEvents } }; }, selector); context.elementVisible = context.elementInfo.visible; } } // Get page state context.pageState = await page.evaluate(() => { return { readyState: document.readyState, hasModals: document.querySelector('[role="dialog"], .modal, .popup') !== null, hasOverlays: document.querySelector('.overlay, .backdrop') !== null, activeElement: document.activeElement ? { tagName: document.activeElement.tagName, id: document.activeElement.id, className: document.activeElement.className } : null }; }); return context; } catch (error) { return { error: error.message }; } } /** * Format detailed error message for AI agent */ function formatDetailedError(action, errorDetails) { const parts = [ `❌ Action "${action.type}" failed after ${errorDetails.attempts.length} attempts`, ``, `📍 Selector: ${errorDetails.selector}`, ]; if (errorDetails.context) { parts.push(``, `📄 Page Context:`); parts.push(` URL: ${errorDetails.context.url}`); parts.push(` Title: ${errorDetails.context.title}`); if (errorDetails.context.elementExists) { parts.push(``, `🔍 Element Found But:`); const info = errorDetails.context.elementInfo; if (!info.visible) { parts.push(` ⚠️ Element is NOT VISIBLE`); parts.push(` - Display: ${info.styles.display}`); parts.push(` - Visibility: ${info.styles.visibility}`); parts.push(` - Opacity: ${info.styles.opacity}`); parts.push(` - Size: ${info.position.width}x${info.position.height}`); } if (info.disabled) { parts.push(` ⚠️ Element is DISABLED`); } if (info.readonly && action.type === 'type') { parts.push(` ⚠️ Element is READONLY`); } if (info.styles.pointerEvents === 'none') { parts.push(` ⚠️ Element has pointer-events: none`); } } else { parts.push(``, `❌ Element NOT FOUND in DOM`); } if (errorDetails.context.pageState) { const state = errorDetails.context.pageState; if (state.hasModals) { parts.push(``, `⚠️ Page has open modal/dialog`); } if (state.hasOverlays) { parts.push(`⚠️ Page has overlay/backdrop`); } } } parts.push(``, `🔄 Retry History:`); errorDetails.attempts.forEach(attempt => { parts.push(` Attempt ${attempt.number}: ${attempt.error || 'Unknown error'}`); if (attempt.selector !== errorDetails.selector) { parts.push(` (tried selector: ${attempt.selector})`); } }); parts.push(``, `💡 Suggestions:`); if (!errorDetails.context?.elementExists) { parts.push(` - Check if page has fully loaded`); parts.push(` - Verify the selector is correct`); parts.push(` - Element might be dynamically added - add wait condition`); } else if (!errorDetails.context?.elementVisible) { parts.push(` - Element exists but is hidden - check CSS/JS conditions`); parts.push(` - Wait for element to become visible`); parts.push(` - Check if element is covered by modal/overlay`); } return parts.join('\n'); } /** * Execute single action */ async function executeAction(action, page, timeout) { const result = { output: null }; switch (action.type) { case 'click': await executeClick(action, page, timeout); break; case 'type': await executeType(action, page, timeout); break; case 'select': await executeSelect(action, page, timeout); break; case 'scroll': await executeScroll(action, page); break; case 'hover': await executeHover(action, page); break; case 'keypress': await executeKeypress(action, page); break; case 'wait': await executeWait(action, page); break; case 'upload': await executeUpload(action, page, timeout); break; case 'drag': await executeDrag(action, page); break; case 'navigate': await executeNavigate(action, page, timeout); break; case 'extract': result.output = await executeExtract(action, page); break; default: throw new Error(`Unknown action type: ${action.type}`); } return result; } /** * Action executors */ async function executeClick(action, page, timeout) { const selector = action.selector.value || action.selector.primary || action.selector; try { await page.waitForSelector(selector, { timeout, visible: true }); await page.click(selector); // Smart waiting after click if (action.data.requiresWait !== false) { await smartWaitAfterClick(page, action, timeout); } // Additional wait if specified if (action.data.waitAfter) { await new Promise(resolve => setTimeout(resolve, action.data.waitAfter)); } } catch (error) { throw new Error(`Failed to click "${selector}": ${error.message}`); } } /** * Smart waiting after click - waits for animations and network requests */ async function smartWaitAfterClick(page, action, timeout) { const startTime = Date.now(); const maxWaitTime = timeout || 30000; try { // Initial wait 500ms to let page respond await new Promise(resolve => setTimeout(resolve, 500)); // Check if there's any activity (animations, network, DOM changes) const hasActivity = await checkPageActivity(page); if (!hasActivity) { // No activity detected - we're done, fast exit console.log('[Smart Wait] No activity detected, skipping extended wait'); return; } // Activity detected - wait minimum 2 seconds console.log('[Smart Wait] Activity detected, waiting for completion'); const remainingMinWait = 2000 - 500; // Already waited 500ms if (remainingMinWait > 0) { await new Promise(resolve => setTimeout(resolve, remainingMinWait)); } // Wait for animations to complete await page.evaluate(() => { return new Promise((resolve) => { const checkAnimations = () => { // Check for CSS animations/transitions const elements = document.querySelectorAll('*'); let hasAnimations = false; for (const el of elements) { const computedStyle = window.getComputedStyle(el); const animations = computedStyle.getPropertyValue('animation-name'); const transitions = computedStyle.getPropertyValue('transition-property'); if ((animations && animations !== 'none') || (transitions && transitions !== 'none' && transitions !== 'all')) { hasAnimations = true; break; } } if (!hasAnimations) { resolve(); } else { setTimeout(checkAnimations, 100); } }; // Start checking immediately checkAnimations(); // Timeout after 3 seconds setTimeout(resolve, 3000); }); }); // Wait for network to be idle (no pending requests for 500ms) await Promise.race([ page.waitForNetworkIdle({ idleTime: 500, timeout: 5000 }), new Promise(resolve => setTimeout(resolve, 5000)) // Max 5 seconds for network ]); // Wait for any DOM changes to settle await page.evaluate(() => { return new Promise((resolve) => { let timeoutId; const observer = new MutationObserver(() => { clearTimeout(timeoutId); timeoutId = setTimeout(() => { observer.disconnect(); resolve(); }, 300); // 300ms of no DOM changes }); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['class', 'style', 'hidden', 'disabled'] }); // Start the timeout timeoutId = setTimeout(() => { observer.disconnect(); resolve(); }, 300); // Max wait 3 seconds setTimeout(() => { observer.disconnect(); resolve(); }, 3000); }); }); } catch (error) { // If smart wait fails, just log and continue console.error('[Smart Wait] Error during smart wait:', error.message); } // Ensure we don't exceed max wait time const elapsed = Date.now() - startTime; if (elapsed > maxWaitTime) { console.warn(`[Smart Wait] Exceeded max wait time (${maxWaitTime}ms)`); } } /** * Check if page has any ongoing activity (animations, network, DOM changes) * Returns true if activity detected, false otherwise */ async function checkPageActivity(page) { try { const activity = await page.evaluate(() => { // Check for animations const elements = document.querySelectorAll('*'); for (const el of elements) { const computedStyle = window.getComputedStyle(el); const animations = computedStyle.getPropertyValue('animation-name'); const transitions = computedStyle.getPropertyValue('transition-property'); if ((animations && animations !== 'none') || (transitions && transitions !== 'none' && transitions !== 'all')) { return { hasActivity: true, reason: 'animations' }; } } // Check for recent DOM changes (using performance API) if (window.performance && window.performance.getEntriesByType) { const entries = window.performance.getEntriesByType('measure'); if (entries.length > 0) { const recentEntries = entries.filter(e => performance.now() - e.startTime < 500 ); if (recentEntries.length > 0) { return { hasActivity: true, reason: 'performance_measures' }; } } } return { hasActivity: false, reason: 'none' }; }); if (activity.hasActivity) { console.log(`[Smart Wait] Activity detected: ${activity.reason}`); return true; } // Check for pending network requests using CDP try { const client = await page.target().createCDPSession(); await client.send('Network.enable'); // Give a moment for network requests to start await new Promise(resolve => setTimeout(resolve, 100)); const hasNetworkActivity = await new Promise((resolve) => { let requestCount = 0; const requestListener = () => { requestCount++; }; client.on('Network.requestWillBeSent', requestListener); setTimeout(() => { client.off('Network.requestWillBeSent', requestListener); client.detach().catch(() => {}); resolve(requestCount > 0); }, 200); }); if (hasNetworkActivity) { console.log('[Smart Wait] Network activity detected'); return true; } } catch (netError) { // Network check failed, assume no activity console.log('[Smart Wait] Network check failed, assuming no activity'); } return false; } catch (error) { // If check fails, assume there's activity to be safe console.error('[Smart Wait] Activity check failed, assuming activity exists:', error.message); return true; } } async function executeType(action, page, timeout) { const selector = action.selector.value || action.selector.primary || action.selector; try { await page.waitForSelector(selector, { timeout, visible: true }); // Check if element is editable const isEditable = await page.evaluate((sel) => { const el = document.querySelector(sel); if (!el) return false; const isInput = el.tagName === 'INPUT' || el.tagName === 'TEXTAREA'; const isContentEditable = el.isContentEditable; return isInput || isContentEditable; }, selector); if (!isEditable) { throw new Error(`Element "${selector}" is not editable (not an input, textarea, or contenteditable)`); } // Clear field if specified if (action.data.clearFirst !== false) { await page.click(selector, { clickCount: 3 }); await page.keyboard.press('Backspace'); } // Type text with optional delay await page.type(selector, action.data.text, { delay: action.data.delay || 0 }); } catch (error) { throw new Error(`Failed to type into "${selector}": ${error.message}`); } } async function executeSelect(action, page, timeout) { const selector = action.selector.value || action.selector.primary || action.selector; try { if (action.data.selectType === 'custom') { // Custom select (multi-step) for (const step of action.data.steps) { if (step.action === 'click') { await page.waitForSelector(step.selector, { timeout }); await page.click(step.selector); } else if (step.action === 'wait') { await new Promise(resolve => setTimeout(resolve, step.duration)); } } } else { // Native select await page.waitForSelector(selector, { timeout, visible: true }); // Verify it's a select element const isSelect = await page.evaluate((sel) => { const el = document.querySelector(sel); return el && el.tagName === 'SELECT'; }, selector); if (!isSelect) { throw new Error(`Element "${selector}" is not a <select> element`); } await page.select(selector, action.data.value); } } catch (error) { throw new Error(`Failed to select option in "${selector}": ${error.message}`); } } async function executeScroll(action, page) { const selector = action.selector.value || action.selector.primary || action.selector; await page.evaluate((selector, behavior) => { const element = document.querySelector(selector); if (element) { element.scrollIntoView({ behavior: behavior || 'auto', block: 'center' }); } }, selector, action.data.behavior); } async function executeHover(action, page) { const selector = action.selector.value || action.selector.primary || action.selector; await page.hover(selector); } async function executeKeypress(action, page) { const key = action.data.key; const modifiers = action.data.modifiers || []; // Press modifiers for (const mod of modifiers) { await page.keyboard.down(mod); } // Press key await page.keyboard.press(key); // Release modifiers for (const mod of modifiers.reverse()) { await page.keyboard.up(mod); } } async function executeWait(action, page) { if (action.data.waitType === 'selector') { await page.waitForSelector(action.data.selector, { timeout: action.data.duration }); } else { await new Promise(resolve => setTimeout(resolve, action.data.duration)); } } async function executeUpload(action, page, timeout) { const selector = action.selector.value || action.selector.primary || action.selector; const fileInput = await page.waitForSelector(selector, { timeout }); await fileInput.uploadFile(action.data.filePath); } async function executeDrag(action, page) { const { fromSelector, toSelector, fromX, fromY, toX, toY } = action.data; if (fromSelector && toSelector) { // Drag from element to element const from = await page.$(fromSelector); const to = await page.$(toSelector); const fromBox = await from.boundingBox(); const toBox = await to.boundingBox(); await page.mouse.move(fromBox.x + fromBox.width / 2, fromBox.y + fromBox.height / 2); await page.mouse.down(); await page.mouse.move(toBox.x + toBox.width / 2, toBox.y + toBox.height / 2); await page.mouse.up(); } else { // Drag by coordinates await page.mouse.move(fromX, fromY); await page.mouse.down(); await page.mouse.move(toX, toY); await page.mouse.up(); } } async function executeNavigate(action, page, timeout) { await page.goto(action.data.url, { waitUntil: action.data.waitUntil || 'networkidle2', timeout }); } async function executeExtract(action, page) { const { selector, attribute, multiple } = action.data; if (multiple) { return await page.$$eval(selector, (elements, attr) => { return elements.map(el => attr ? el.getAttribute(attr) : el.textContent.trim()); }, attribute); } else { return await page.$eval(selector, (el, attr) => { return attr ? el.getAttribute(attr) : el.textContent.trim(); }, attribute); } } /** * Substitute parameters in action * Replaces {{paramName}} with actual values */ function substituteParameters(action, params) { const resolved = JSON.parse(JSON.stringify(action)); // Substitute in action data if (resolved.data) { for (const [key, value] of Object.entries(resolved.data)) { if (typeof value === 'string') { resolved.data[key] = substituteString(value, params); } } } return resolved; } /** * Substitute {{param}} in string */ function substituteString(str, params) { return str.replace(/\{\{(\w+)\}\}/g, (match, paramName) => { if (params[paramName] !== undefined) { return params[paramName]; } return match; // Keep original if param not found }); } /** * Element finder utils code (to be injected into page) * Will be loaded from utils/element-finder-utils.js */ const elementFinderUtilsCode = ` // This will be populated from element-finder-utils.js browser-side code // For now, placeholder `;