UNPKG

claude-playwright

Version:

Seamless integration between Claude Code and Playwright MCP for efficient browser automation and testing

1 lines 93.2 kB
{"version":3,"sources":["../../src/core/enhanced-cache-integration.ts","../../src/core/bidirectional-cache.ts","../../src/core/smart-normalizer.ts","../../src/core/tiered-cache.ts","../../src/core/snapshot-cache.ts","../../src/core/cache-manager.ts"],"sourcesContent":["import { Page } from 'playwright';\nimport { BidirectionalCache } from './bidirectional-cache.js';\nimport { TieredCache } from './tiered-cache.js';\nimport { SnapshotCache } from './snapshot-cache.js';\nimport { CacheManager } from './cache-manager.js';\n\nexport class EnhancedCacheIntegration {\n private static instance: EnhancedCacheIntegration | null = null;\n private bidirectionalCache: BidirectionalCache;\n private tieredCache: TieredCache;\n private snapshotCache: SnapshotCache;\n private legacyCacheManager: CacheManager;\n private currentPage?: Page;\n private currentUrl: string = '';\n private currentProfile?: string;\n private navigationCount: number = 0;\n\n private constructor() {\n // Initialize new bidirectional cache system\n this.bidirectionalCache = new BidirectionalCache({\n maxSizeMB: 50,\n selectorTTL: 300000, // 5 minutes\n cleanupInterval: 60000, // 1 minute\n maxVariationsPerSelector: 15\n });\n \n // Add tiered cache layer (memory + SQLite)\n this.tieredCache = new TieredCache(this.bidirectionalCache, {\n memorySize: 100,\n memoryTTL: 300000, // 5 minutes\n preloadCommonSelectors: true\n });\n\n // Keep legacy cache manager for snapshots\n this.legacyCacheManager = new CacheManager({\n maxSizeMB: 25, // Reduced since selectors moved to bidirectional\n snapshotTTL: 1800000, // 30 minutes\n cleanupInterval: 60000\n });\n \n this.snapshotCache = new SnapshotCache(this.legacyCacheManager);\n \n console.error('[EnhancedCache] Initialized bidirectional cache system');\n }\n\n static getInstance(): EnhancedCacheIntegration {\n if (!EnhancedCacheIntegration.instance) {\n EnhancedCacheIntegration.instance = new EnhancedCacheIntegration();\n }\n return EnhancedCacheIntegration.instance;\n }\n\n setPage(page: Page, url: string, profile?: string): void {\n this.currentPage = page;\n this.currentUrl = url;\n this.currentProfile = profile;\n \n // Setup snapshot cache context\n this.snapshotCache.setContext(page, url, profile);\n \n // Setup page event listeners for cache invalidation\n this.setupPageListeners(page);\n \n console.error(`[EnhancedCache] Context set for ${url} (profile: ${profile || 'default'})`);\n }\n\n private setupPageListeners(page: Page): void {\n // Invalidate cache on navigation\n page.on('framenavigated', async (frame) => {\n if (frame === page.mainFrame()) {\n const newUrl = frame.url();\n if (newUrl !== this.currentUrl) {\n console.error(`[EnhancedCache] Navigation detected: ${this.currentUrl} -> ${newUrl}`);\n await this.handleNavigation(newUrl);\n }\n }\n });\n\n // Track DOM changes for snapshot invalidation\n page.on('load', async () => {\n console.error('[EnhancedCache] Page loaded, checking for changes...');\n await this.snapshotCache.invalidateIfChanged();\n });\n }\n\n private async handleNavigation(newUrl: string): Promise<void> {\n this.navigationCount++;\n \n // Invalidate memory cache for old URL\n await this.tieredCache.invalidateForUrl(this.currentUrl);\n \n // Update current URL\n this.currentUrl = newUrl;\n \n if (this.currentPage) {\n this.snapshotCache.setContext(this.currentPage, newUrl, this.currentProfile);\n }\n \n console.error(`[EnhancedCache] Navigation handled, count: ${this.navigationCount}`);\n }\n\n // Enhanced selector operations using bidirectional cache\n async getCachedSelector(description: string): Promise<any | null> {\n const result = await this.tieredCache.get(description, this.currentUrl);\n return result ? {\n selector: result.selector,\n strategy: 'cached',\n confidence: result.confidence,\n source: result.source\n } : null;\n }\n\n async cacheSelector(description: string, selector: string, strategy: string = 'css'): Promise<void> {\n await this.tieredCache.set(description, this.currentUrl, selector);\n console.error(`[EnhancedCache] Cached \"${description}\" -> ${selector}`);\n }\n\n // Enhanced wrapper for MCP operations\n async wrapSelectorOperation<T>(\n description: string,\n operation: (selector: string) => Promise<T>,\n fallbackSelector: string\n ): Promise<{ result: T; cached: boolean; performance: any }> {\n const startTime = Date.now();\n \n try {\n const wrappedResult = await this.tieredCache.wrapSelectorOperation(\n description,\n this.currentUrl,\n operation,\n fallbackSelector\n );\n \n const endTime = Date.now();\n return {\n ...wrappedResult,\n performance: {\n duration: endTime - startTime,\n cacheHit: wrappedResult.cached\n }\n };\n } catch (error) {\n const endTime = Date.now();\n console.error(`[EnhancedCache] Operation failed after ${endTime - startTime}ms:`, error);\n throw error;\n }\n }\n\n // Snapshot operations (legacy integration)\n async getCachedSnapshot(): Promise<any | null> {\n return await this.snapshotCache.getCachedSnapshot();\n }\n\n async cacheSnapshot(snapshot: any): Promise<void> {\n await this.snapshotCache.cacheSnapshot(snapshot);\n }\n\n async getOrCreateSnapshot(): Promise<any> {\n return await this.snapshotCache.getOrCreateSnapshot();\n }\n\n // Advanced learning capabilities\n async learnFromSuccess(description: string, selector: string, context?: any): Promise<void> {\n // Enhanced learning that considers context\n await this.tieredCache.set(description, this.currentUrl, selector);\n \n // Learn patterns\n if (context?.elementType || context?.attributes) {\n console.error(`[EnhancedCache] Learning with context: ${JSON.stringify(context)}`);\n // Future: Could train ML models here\n }\n }\n\n async suggestAlternatives(description: string): Promise<string[]> {\n // Get related selectors from cache\n try {\n const stats = await this.bidirectionalCache.getStats();\n // This could be enhanced to actually return similar selectors\n return [];\n } catch (error) {\n return [];\n }\n }\n\n // Comprehensive metrics\n async getMetrics(): Promise<any> {\n try {\n const tieredStats = this.tieredCache.getStats();\n const bidirectionalStats = await this.bidirectionalCache.getStats();\n \n return {\n overview: {\n currentUrl: this.currentUrl,\n currentProfile: this.currentProfile || 'default',\n navigationCount: this.navigationCount,\n systemType: 'bidirectional'\n },\n performance: {\n ...tieredStats.tiered,\n breakdown: tieredStats.breakdown\n },\n storage: {\n ...bidirectionalStats.storage,\n operations: bidirectionalStats.operations\n },\n recommendations: this.generateRecommendations(tieredStats, bidirectionalStats)\n };\n } catch (error) {\n console.error('[EnhancedCache] Metrics error:', error);\n return {\n error: 'Failed to retrieve metrics',\n currentUrl: this.currentUrl,\n navigationCount: this.navigationCount\n };\n }\n }\n\n private generateRecommendations(tieredStats: any, bidirectionalStats: any): string[] {\n const recommendations = [];\n \n if (tieredStats.tiered.overallHitRate < 50) {\n recommendations.push(\"Low hit rate - consider using more consistent selector descriptions\");\n }\n \n if (tieredStats.tiered.memoryHitRate < 30) {\n recommendations.push(\"Memory cache underutilized - selectors may be too varied\");\n }\n \n if (bidirectionalStats.storage?.avg_inputs_per_selector > 5) {\n recommendations.push(\"High variation per selector - check for pattern consistency\");\n }\n \n if (this.navigationCount > 10) {\n recommendations.push(\"Frequent navigation detected - cache is adapting well\");\n }\n \n return recommendations;\n }\n\n // Cache management operations\n async invalidateAll(): Promise<void> {\n await this.tieredCache.clear();\n console.error('[EnhancedCache] All caches cleared');\n }\n\n async invalidateForUrl(url: string): Promise<void> {\n await this.tieredCache.invalidateForUrl(url);\n console.error(`[EnhancedCache] Invalidated cache for ${url}`);\n }\n\n // Health check\n async healthCheck(): Promise<{ status: 'healthy' | 'degraded' | 'unhealthy'; details: any }> {\n try {\n const metrics = await this.getMetrics();\n const hitRate = metrics.performance?.overallHitRate || 0;\n \n if (hitRate > 70) {\n return { status: 'healthy', details: { hitRate, message: 'Cache performing well' } };\n } else if (hitRate > 40) {\n return { status: 'degraded', details: { hitRate, message: 'Cache hit rate below optimal' } };\n } else {\n return { status: 'unhealthy', details: { hitRate, message: 'Cache hit rate too low' } };\n }\n } catch (error) {\n return { \n status: 'unhealthy', \n details: { error: error.message, message: 'Cache system error' } \n };\n }\n }\n\n close(): void {\n this.tieredCache.close();\n this.legacyCacheManager.close();\n EnhancedCacheIntegration.instance = null;\n console.error('[EnhancedCache] System closed');\n }\n}","import Database from 'better-sqlite3';\nimport { SmartNormalizer, NormalizationResult } from './smart-normalizer.js';\nimport crypto from 'crypto';\nimport * as path from 'path';\nimport * as os from 'os';\nimport * as fs from 'fs';\n\ninterface SelectorCacheEntry {\n id?: number;\n selector: string;\n selector_hash: string;\n url: string;\n confidence: number;\n created_at: number;\n last_used: number;\n use_count: number;\n}\n\ninterface InputMappingEntry {\n id?: number;\n selector_hash: string;\n input: string;\n normalized_input: string;\n input_tokens: string; // JSON array\n url: string;\n success_count: number;\n last_used: number;\n confidence: number;\n learned_from: 'direct' | 'inferred' | 'pattern';\n}\n\ninterface CacheOptions {\n maxSizeMB?: number;\n selectorTTL?: number;\n cleanupInterval?: number;\n maxVariationsPerSelector?: number;\n}\n\ninterface LookupResult {\n selector: string;\n confidence: number;\n source: 'exact' | 'normalized' | 'reverse' | 'fuzzy';\n cached: boolean;\n}\n\nexport class BidirectionalCache {\n private db: Database.Database;\n private normalizer: SmartNormalizer;\n private cleanupTimer?: NodeJS.Timeout;\n private options: Required<CacheOptions>;\n private cacheDir: string;\n private stats = {\n hits: { exact: 0, normalized: 0, reverse: 0, fuzzy: 0 },\n misses: 0,\n sets: 0,\n learnings: 0\n };\n\n constructor(options: CacheOptions = {}) {\n this.options = {\n maxSizeMB: options.maxSizeMB ?? 50,\n selectorTTL: options.selectorTTL ?? 300000, // 5 minutes\n cleanupInterval: options.cleanupInterval ?? 60000, // 1 minute\n maxVariationsPerSelector: options.maxVariationsPerSelector ?? 20\n };\n\n this.normalizer = new SmartNormalizer();\n \n // Create cache directory\n this.cacheDir = path.join(os.homedir(), '.claude-playwright', 'cache');\n if (!fs.existsSync(this.cacheDir)) {\n fs.mkdirSync(this.cacheDir, { recursive: true });\n }\n\n // Initialize database\n const dbPath = path.join(this.cacheDir, 'bidirectional-cache.db');\n this.db = new Database(dbPath);\n this.db.pragma('journal_mode = WAL');\n this.db.pragma('synchronous = NORMAL');\n \n this.initializeDatabase();\n this.startCleanupTimer();\n }\n\n private initializeDatabase(): void {\n this.db.exec(`\n -- Enhanced selector cache table\n CREATE TABLE IF NOT EXISTS selector_cache_v2 (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n selector TEXT NOT NULL,\n selector_hash TEXT NOT NULL UNIQUE,\n url TEXT NOT NULL,\n confidence REAL DEFAULT 0.5,\n created_at INTEGER NOT NULL,\n last_used INTEGER NOT NULL,\n use_count INTEGER DEFAULT 1\n );\n\n -- Bidirectional input mappings\n CREATE TABLE IF NOT EXISTS input_mappings (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n selector_hash TEXT NOT NULL,\n input TEXT NOT NULL,\n normalized_input TEXT NOT NULL,\n input_tokens TEXT NOT NULL,\n url TEXT NOT NULL,\n success_count INTEGER DEFAULT 1,\n last_used INTEGER NOT NULL,\n confidence REAL DEFAULT 0.5,\n learned_from TEXT DEFAULT 'direct',\n FOREIGN KEY (selector_hash) REFERENCES selector_cache_v2(selector_hash),\n UNIQUE(selector_hash, normalized_input, url)\n );\n\n -- Performance indices\n CREATE INDEX IF NOT EXISTS idx_selector_hash_v2 ON selector_cache_v2(selector_hash);\n CREATE INDEX IF NOT EXISTS idx_url_v2 ON selector_cache_v2(url);\n CREATE INDEX IF NOT EXISTS idx_input_normalized ON input_mappings(normalized_input, url);\n CREATE INDEX IF NOT EXISTS idx_mapping_selector_hash ON input_mappings(selector_hash);\n CREATE INDEX IF NOT EXISTS idx_url_selector ON input_mappings(url, selector_hash);\n CREATE INDEX IF NOT EXISTS idx_tokens ON input_mappings(input_tokens);\n\n -- Migration: Copy from old cache if exists\n INSERT OR IGNORE INTO selector_cache_v2 (selector, selector_hash, url, confidence, created_at, last_used)\n SELECT data as selector, \n substr(cache_key, 1, 32) as selector_hash,\n url,\n 0.5 as confidence,\n created_at,\n accessed_at as last_used\n FROM cache \n WHERE cache_type = 'selector' \n AND NOT EXISTS (SELECT 1 FROM selector_cache_v2 WHERE selector_hash = substr(cache.cache_key, 1, 32));\n `);\n }\n\n async set(input: string, url: string, selector: string): Promise<void> {\n const now = Date.now();\n const selectorHash = this.createSelectorHash(selector);\n const normalizedResult = this.normalizer.normalize(input);\n\n try {\n // Begin transaction\n const transaction = this.db.transaction(() => {\n // 1. Store/update selector\n const selectorStmt = this.db.prepare(`\n INSERT INTO selector_cache_v2 \n (selector, selector_hash, url, confidence, created_at, last_used, use_count)\n VALUES (?, ?, ?, 0.5, ?, ?, 1)\n ON CONFLICT(selector_hash) DO UPDATE SET\n last_used = excluded.last_used,\n use_count = use_count + 1,\n confidence = MIN(confidence * 1.02, 1.0)\n `);\n \n selectorStmt.run(selector, selectorHash, url, now, now);\n\n // 2. Store input mapping\n const mappingStmt = this.db.prepare(`\n INSERT INTO input_mappings \n (selector_hash, input, normalized_input, input_tokens, url, last_used, learned_from)\n VALUES (?, ?, ?, ?, ?, ?, 'direct')\n ON CONFLICT(selector_hash, normalized_input, url) DO UPDATE SET\n success_count = success_count + 1,\n confidence = MIN(confidence * 1.05, 1.0),\n last_used = excluded.last_used,\n input = CASE \n WHEN length(excluded.input) > length(input) THEN excluded.input \n ELSE input \n END\n `);\n\n mappingStmt.run(\n selectorHash,\n input,\n normalizedResult.normalized,\n JSON.stringify(normalizedResult.tokens),\n url,\n now\n );\n });\n\n transaction();\n this.stats.sets++;\n\n // 3. Learn from related inputs (async, non-blocking)\n setImmediate(() => this.learnRelatedInputs(selectorHash, input, url, normalizedResult));\n\n } catch (error) {\n console.error('[BidirectionalCache] Set error:', error);\n }\n }\n\n async get(input: string, url: string): Promise<LookupResult | null> {\n const normalizedResult = this.normalizer.normalize(input);\n\n // Level 1: Exact Match\n const exactResult = await this.exactMatch(input, url);\n if (exactResult) {\n this.stats.hits.exact++;\n return { ...exactResult, source: 'exact', cached: true };\n }\n\n // Level 2: Normalized Match\n const normalizedMatch = await this.normalizedMatch(normalizedResult.normalized, url);\n if (normalizedMatch) {\n this.stats.hits.normalized++;\n return { ...normalizedMatch, source: 'normalized', cached: true };\n }\n\n // Level 3: Reverse Lookup\n const reverseMatch = await this.reverseLookup(normalizedResult, url);\n if (reverseMatch) {\n this.stats.hits.reverse++;\n return { ...reverseMatch, source: 'reverse', cached: true };\n }\n\n // Level 4: Fuzzy Match (typo tolerance)\n const fuzzyMatch = await this.fuzzyMatch(normalizedResult, url);\n if (fuzzyMatch) {\n this.stats.hits.fuzzy++;\n return { ...fuzzyMatch, source: 'fuzzy', cached: true };\n }\n\n this.stats.misses++;\n return null;\n }\n\n private async exactMatch(input: string, url: string): Promise<{ selector: string; confidence: number } | null> {\n try {\n const stmt = this.db.prepare(`\n SELECT sc.selector, im.confidence\n FROM input_mappings im\n JOIN selector_cache_v2 sc ON sc.selector_hash = im.selector_hash\n WHERE im.input = ? AND im.url = ?\n ORDER BY im.confidence DESC, im.success_count DESC\n LIMIT 1\n `);\n\n const result = stmt.get(input, url) as { selector: string; confidence: number } | undefined;\n if (result) {\n await this.updateUsage(result.selector, url);\n return result;\n }\n } catch (error) {\n console.error('[BidirectionalCache] Exact match error:', error);\n }\n return null;\n }\n\n private async normalizedMatch(normalized: string, url: string): Promise<{ selector: string; confidence: number } | null> {\n try {\n const stmt = this.db.prepare(`\n SELECT sc.selector, im.confidence\n FROM input_mappings im\n JOIN selector_cache_v2 sc ON sc.selector_hash = im.selector_hash\n WHERE im.normalized_input = ? AND im.url = ?\n ORDER BY im.confidence DESC, im.success_count DESC\n LIMIT 1\n `);\n\n const result = stmt.get(normalized, url) as { selector: string; confidence: number } | undefined;\n if (result) {\n await this.updateUsage(result.selector, url);\n return result;\n }\n } catch (error) {\n console.error('[BidirectionalCache] Normalized match error:', error);\n }\n return null;\n }\n\n private async reverseLookup(normalizedResult: NormalizationResult, url: string): Promise<{ selector: string; confidence: number } | null> {\n try {\n // Find selectors with similar token patterns\n const stmt = this.db.prepare(`\n SELECT \n sc.selector,\n im.confidence,\n im.input_tokens,\n im.success_count,\n GROUP_CONCAT(im.input) as all_inputs\n FROM input_mappings im\n JOIN selector_cache_v2 sc ON sc.selector_hash = im.selector_hash\n WHERE im.url = ?\n AND json_array_length(im.input_tokens) > 0\n GROUP BY sc.selector_hash\n ORDER BY im.confidence DESC, im.success_count DESC\n LIMIT 10\n `);\n\n const candidates = stmt.all(url) as Array<{\n selector: string;\n confidence: number;\n input_tokens: string;\n success_count: number;\n all_inputs: string;\n }>;\n\n let bestMatch: { selector: string; confidence: number } | null = null;\n let bestScore = 0;\n\n for (const candidate of candidates) {\n try {\n const candidateTokens = JSON.parse(candidate.input_tokens);\n const similarity = this.calculateJaccardSimilarity(\n normalizedResult.tokens,\n candidateTokens\n );\n\n // Boost score based on success count and confidence\n const boostedScore = similarity * \n (1 + Math.log(1 + candidate.success_count) * 0.1) *\n candidate.confidence;\n\n if (boostedScore > 0.6 && boostedScore > bestScore) {\n bestScore = boostedScore;\n bestMatch = {\n selector: candidate.selector,\n confidence: candidate.confidence * 0.9 // Slight penalty for reverse lookup\n };\n }\n } catch (e) {\n // Skip malformed JSON\n continue;\n }\n }\n\n if (bestMatch) {\n await this.updateUsage(bestMatch.selector, url);\n return bestMatch;\n }\n } catch (error) {\n console.error('[BidirectionalCache] Reverse lookup error:', error);\n }\n return null;\n }\n\n private async fuzzyMatch(normalizedResult: NormalizationResult, url: string): Promise<{ selector: string; confidence: number } | null> {\n try {\n const stmt = this.db.prepare(`\n SELECT sc.selector, im.normalized_input, im.confidence\n FROM input_mappings im\n JOIN selector_cache_v2 sc ON sc.selector_hash = im.selector_hash\n WHERE im.url = ?\n AND im.last_used > ?\n ORDER BY im.confidence DESC, im.success_count DESC\n LIMIT 20\n `);\n\n const candidates = stmt.all(url, Date.now() - 3600000) as Array<{ // Last hour only\n selector: string;\n normalized_input: string;\n confidence: number;\n }>;\n\n for (const candidate of candidates) {\n const distance = this.normalizer.damerauLevenshtein(\n normalizedResult.normalized,\n candidate.normalized_input\n );\n\n const maxDistance = Math.floor(normalizedResult.normalized.length / 8); // 12.5% tolerance\n \n if (distance <= maxDistance && distance > 0) {\n await this.updateUsage(candidate.selector, url);\n return {\n selector: candidate.selector,\n confidence: candidate.confidence * (1 - distance / 10) // Penalty for typos\n };\n }\n }\n } catch (error) {\n console.error('[BidirectionalCache] Fuzzy match error:', error);\n }\n return null;\n }\n\n private calculateJaccardSimilarity(set1: string[], set2: string[]): number {\n const intersection = set1.filter(x => set2.includes(x));\n const union = [...new Set([...set1, ...set2])];\n return intersection.length / union.length;\n }\n\n private createSelectorHash(selector: string): string {\n return crypto.createHash('md5').update(selector).digest('hex');\n }\n\n private async updateUsage(selector: string, url: string): Promise<void> {\n try {\n const now = Date.now();\n const selectorHash = this.createSelectorHash(selector);\n \n const stmt = this.db.prepare(`\n UPDATE selector_cache_v2 \n SET last_used = ?, use_count = use_count + 1\n WHERE selector_hash = ? AND url = ?\n `);\n \n stmt.run(now, selectorHash, url);\n } catch (error) {\n console.error('[BidirectionalCache] Update usage error:', error);\n }\n }\n\n private async learnRelatedInputs(\n selectorHash: string, \n newInput: string, \n url: string, \n normalizedResult: NormalizationResult\n ): Promise<void> {\n try {\n // Find other inputs for the same selector\n const stmt = this.db.prepare(`\n SELECT input, normalized_input, input_tokens, confidence\n FROM input_mappings\n WHERE selector_hash = ? AND url = ? AND input != ?\n AND success_count > 1\n ORDER BY confidence DESC\n LIMIT 5\n `);\n\n const related = stmt.all(selectorHash, url, newInput) as Array<{\n input: string;\n normalized_input: string;\n input_tokens: string;\n confidence: number;\n }>;\n\n for (const rel of related) {\n const pattern = this.findCommonPattern(newInput, rel.input);\n if (pattern && pattern.confidence > 0.7) {\n await this.saveLearnedPattern(pattern, selectorHash, url);\n this.stats.learnings++;\n }\n }\n } catch (error) {\n console.error('[BidirectionalCache] Learn related inputs error:', error);\n }\n }\n\n private findCommonPattern(input1: string, input2: string): { pattern: string; confidence: number } | null {\n // Simple pattern extraction - can be enhanced\n const norm1 = this.normalizer.normalize(input1);\n const norm2 = this.normalizer.normalize(input2);\n \n const commonTokens = norm1.tokens.filter(t => norm2.tokens.includes(t));\n if (commonTokens.length >= 2) {\n return {\n pattern: commonTokens.sort().join(' '),\n confidence: commonTokens.length / Math.max(norm1.tokens.length, norm2.tokens.length)\n };\n }\n \n return null;\n }\n\n private async saveLearnedPattern(\n pattern: { pattern: string; confidence: number },\n selectorHash: string,\n url: string\n ): Promise<void> {\n try {\n const now = Date.now();\n const stmt = this.db.prepare(`\n INSERT OR IGNORE INTO input_mappings\n (selector_hash, input, normalized_input, input_tokens, url, last_used, confidence, learned_from)\n VALUES (?, ?, ?, ?, ?, ?, ?, 'pattern')\n `);\n\n stmt.run(\n selectorHash,\n `Pattern: ${pattern.pattern}`,\n pattern.pattern,\n JSON.stringify(pattern.pattern.split(' ')),\n url,\n now,\n pattern.confidence,\n );\n } catch (error) {\n // Ignore conflicts - pattern might already exist\n }\n }\n\n private startCleanupTimer(): void {\n this.cleanupTimer = setInterval(() => {\n this.cleanup();\n }, this.options.cleanupInterval);\n }\n\n private cleanup(): void {\n try {\n const now = Date.now();\n \n // Remove expired entries\n const expiredStmt = this.db.prepare(`\n DELETE FROM input_mappings \n WHERE (last_used + ?) < ?\n `);\n expiredStmt.run(this.options.selectorTTL, now);\n\n // Limit variations per selector\n const limitStmt = this.db.prepare(`\n DELETE FROM input_mappings\n WHERE id NOT IN (\n SELECT id FROM (\n SELECT id, ROW_NUMBER() OVER (\n PARTITION BY selector_hash, url \n ORDER BY confidence DESC, success_count DESC, last_used DESC\n ) as rn\n FROM input_mappings\n ) WHERE rn <= ?\n )\n `);\n limitStmt.run(this.options.maxVariationsPerSelector);\n\n // Clean orphaned selectors\n const orphanStmt = this.db.prepare(`\n DELETE FROM selector_cache_v2\n WHERE selector_hash NOT IN (\n SELECT DISTINCT selector_hash FROM input_mappings\n )\n `);\n orphanStmt.run();\n\n } catch (error) {\n console.error('[BidirectionalCache] Cleanup error:', error);\n }\n }\n\n async getStats(): Promise<any> {\n try {\n const dbStats = this.db.prepare(`\n SELECT \n COUNT(DISTINCT sc.selector_hash) as unique_selectors,\n COUNT(im.id) as total_mappings,\n AVG(im.success_count) as avg_success_count,\n COUNT(im.id) * 1.0 / COUNT(DISTINCT sc.selector_hash) as avg_inputs_per_selector,\n SUM(CASE WHEN im.learned_from = 'inferred' THEN 1 ELSE 0 END) * 100.0 / COUNT(im.id) as learning_rate\n FROM input_mappings im\n JOIN selector_cache_v2 sc ON sc.selector_hash = im.selector_hash\n `).get();\n\n const hitRate = Object.values(this.stats.hits).reduce((a, b) => a + b, 0) / \n (Object.values(this.stats.hits).reduce((a, b) => a + b, 0) + this.stats.misses);\n\n return {\n performance: {\n hitRate: hitRate || 0,\n hits: this.stats.hits,\n misses: this.stats.misses,\n totalLookups: Object.values(this.stats.hits).reduce((a, b) => a + b, 0) + this.stats.misses\n },\n storage: dbStats,\n operations: {\n sets: this.stats.sets,\n learnings: this.stats.learnings\n }\n };\n } catch (error) {\n console.error('[BidirectionalCache] Get stats error:', error);\n return {};\n }\n }\n\n async clear(): Promise<void> {\n try {\n this.db.exec('DELETE FROM input_mappings');\n this.db.exec('DELETE FROM selector_cache_v2');\n \n // Reset stats\n this.stats = {\n hits: { exact: 0, normalized: 0, reverse: 0, fuzzy: 0 },\n misses: 0,\n sets: 0,\n learnings: 0\n };\n } catch (error) {\n console.error('[BidirectionalCache] Clear error:', error);\n }\n }\n\n close(): void {\n if (this.cleanupTimer) {\n clearInterval(this.cleanupTimer);\n }\n this.db.close();\n }\n}","import crypto from 'crypto';\n\ninterface PositionalKeyword {\n word: string;\n position: number;\n context?: string;\n}\n\ninterface InputFeatures {\n hasId: boolean;\n hasClass: boolean;\n hasQuoted: boolean;\n numbers: string[];\n positions: PositionalKeyword[];\n attributes: string[];\n wordCount: number;\n hasImperative: boolean;\n casePattern: 'lower' | 'upper' | 'mixed' | 'title';\n isNavigation: boolean;\n isFormAction: boolean;\n hasDataTestId: boolean;\n}\n\nexport interface NormalizationResult {\n normalized: string;\n tokens: string[];\n positions: PositionalKeyword[];\n features: InputFeatures;\n hash: string;\n}\n\nexport class SmartNormalizer {\n private readonly POSITION_KEYWORDS = [\n 'before', 'after', 'first', 'last', 'next', 'previous', \n 'above', 'below', 'top', 'bottom', 'left', 'right'\n ];\n \n private readonly RELATION_KEYWORDS = [\n 'in', 'of', 'from', 'to', 'with', 'by', 'for'\n ];\n \n private readonly STOP_WORDS = [\n 'the', 'a', 'an', 'and', 'or', 'but', 'at', 'on'\n ];\n \n private readonly ACTION_SYNONYMS = {\n 'click': ['click', 'press', 'tap', 'hit', 'select', 'choose'],\n 'type': ['type', 'enter', 'input', 'fill', 'write'],\n 'navigate': ['go', 'navigate', 'open', 'visit', 'load'],\n 'hover': ['hover', 'mouseover', 'move']\n };\n\n normalize(input: string): NormalizationResult {\n const original = input.trim();\n const features = this.extractFeatures(original);\n \n // Step 1: Basic cleanup\n let text = original.toLowerCase();\n \n // Step 2: Extract and preserve quoted strings\n const quotedStrings: string[] = [];\n text = text.replace(/([\"'])((?:(?!\\1)[^\\\\]|\\\\.)*)(\\1)/g, (match, quote, content) => {\n quotedStrings.push(content);\n return `QUOTED_${quotedStrings.length - 1}`;\n });\n \n // Step 3: Extract positional information\n const positions = this.extractPositions(text);\n \n // Step 4: Normalize actions to canonical forms\n text = this.normalizeActions(text);\n \n // Step 5: Remove common patterns\n text = this.removeCommonPatterns(text);\n \n // Step 6: Tokenize and filter\n const words = text.split(/\\s+/).filter(word => word.length > 0);\n const tokens = [];\n const preserved = [];\n \n for (let i = 0; i < words.length; i++) {\n const word = words[i];\n \n if (this.POSITION_KEYWORDS.includes(word)) {\n // Preserve positional keywords with context\n preserved.push({\n word,\n position: i,\n context: words[i + 1] || null\n });\n } else if (!this.STOP_WORDS.includes(word) && \n !this.RELATION_KEYWORDS.includes(word) &&\n !['button', 'field', 'element'].includes(word)) {\n tokens.push(word);\n }\n }\n \n // Step 7: Sort tokens for order-invariance (except preserved)\n tokens.sort();\n \n // Step 8: Build normalized string\n let normalized = tokens.join(' ');\n \n // Step 9: Add positional information\n if (preserved.length > 0) {\n const posInfo = preserved.map(p => \n `${p.word}${p.context ? '-' + p.context : ''}`\n ).join(',');\n normalized += ` _pos:${posInfo}`;\n }\n \n // Step 10: Add quoted content back\n if (quotedStrings.length > 0) {\n normalized += ` _quoted:${quotedStrings.join(',')}`;\n }\n \n const hash = crypto.createHash('md5').update(normalized).digest('hex');\n \n return {\n normalized,\n tokens,\n positions,\n features,\n hash\n };\n }\n\n extractFeatures(input: string): InputFeatures {\n const text = input.toLowerCase();\n \n return {\n hasId: /#[\\w-]+/.test(input),\n hasClass: /\\.[\\w-]+/.test(input),\n hasQuoted: /\"[^\"]+\"|'[^']+'/.test(input),\n numbers: (input.match(/\\d+/g) || []),\n positions: this.extractPositions(text),\n attributes: this.extractAttributes(input),\n wordCount: input.split(/\\s+/).length,\n hasImperative: /^(click|press|tap|select|enter|type|fill)/i.test(input),\n casePattern: this.detectCasePattern(input),\n isNavigation: /^(go|navigate|open|visit)/i.test(input),\n isFormAction: /(submit|enter|fill|type|input)/i.test(input),\n hasDataTestId: /data-test|testid|data-cy/i.test(input)\n };\n }\n\n private extractPositions(text: string): PositionalKeyword[] {\n const positions: PositionalKeyword[] = [];\n const words = text.split(/\\s+/);\n \n for (let i = 0; i < words.length; i++) {\n const word = words[i];\n if (this.POSITION_KEYWORDS.includes(word)) {\n positions.push({\n word,\n position: i,\n context: words[i + 1] || words[i - 1] || undefined\n });\n }\n }\n \n return positions;\n }\n\n private extractAttributes(input: string): string[] {\n const attributes = [];\n \n // Extract common attribute patterns\n const patterns = [\n /\\[([^\\]]+)\\]/g, // [attribute=value]\n /data-[\\w-]+/g, // data-testid\n /aria-[\\w-]+/g, // aria-label\n /role=\"[\\w-]+\"/g, // role=\"button\"\n /type=\"[\\w-]+\"/g, // type=\"submit\"\n /placeholder=\"[^\"]+\"/g // placeholder=\"text\"\n ];\n \n for (const pattern of patterns) {\n const matches = input.match(pattern);\n if (matches) {\n attributes.push(...matches);\n }\n }\n \n return attributes;\n }\n\n private detectCasePattern(input: string): 'lower' | 'upper' | 'mixed' | 'title' {\n const hasLower = /[a-z]/.test(input);\n const hasUpper = /[A-Z]/.test(input);\n \n if (!hasLower && hasUpper) return 'upper';\n if (hasLower && !hasUpper) return 'lower';\n \n // Check if it's title case\n const words = input.split(/\\s+/);\n const isTitleCase = words.every(word => \n /^[A-Z][a-z]*$/.test(word) || /^[a-z]+$/.test(word)\n );\n \n return isTitleCase ? 'title' : 'mixed';\n }\n\n private normalizeActions(text: string): string {\n for (const [canonical, synonyms] of Object.entries(this.ACTION_SYNONYMS)) {\n for (const synonym of synonyms) {\n const regex = new RegExp(`\\\\b${synonym}\\\\b`, 'g');\n text = text.replace(regex, canonical);\n }\n }\n return text;\n }\n\n private removeCommonPatterns(text: string): string {\n // Remove common prefixes and suffixes\n text = text.replace(/^(click|press|tap)(\\s+on)?(\\s+the)?/i, 'click');\n text = text.replace(/\\s+(button|element|field)$/i, '');\n text = text.replace(/button\\s+/i, '');\n \n // Remove articles and common words\n text = text.replace(/\\b(the|a|an)\\b/g, '');\n \n // Clean up punctuation\n text = text.replace(/[^\\w\\s#._-]/g, ' ');\n \n // Normalize whitespace\n text = text.replace(/\\s+/g, ' ').trim();\n \n return text;\n }\n\n // Utility methods for similarity\n calculateSimilarity(result1: NormalizationResult, result2: NormalizationResult): number {\n // Token-based Jaccard similarity\n const set1 = new Set(result1.tokens);\n const set2 = new Set(result2.tokens);\n const intersection = new Set([...set1].filter(x => set2.has(x)));\n const union = new Set([...set1, ...set2]);\n \n let similarity = intersection.size / union.size;\n \n // Boost for matching quoted strings\n const quoted1 = result1.normalized.match(/_quoted:([^_]*)/)?.[1] || '';\n const quoted2 = result2.normalized.match(/_quoted:([^_]*)/)?.[1] || '';\n if (quoted1 === quoted2 && quoted1.length > 0) {\n similarity += 0.2;\n }\n \n // Penalty for mismatched positions\n const pos1 = result1.normalized.match(/_pos:([^_]*)/)?.[1] || '';\n const pos2 = result2.normalized.match(/_pos:([^_]*)/)?.[1] || '';\n if (pos1 !== pos2 && (pos1.length > 0 || pos2.length > 0)) {\n similarity -= 0.3;\n }\n \n return Math.max(0, Math.min(1, similarity));\n }\n\n // Fuzzy matching for typo tolerance\n damerauLevenshtein(a: string, b: string): number {\n const da: { [key: string]: number } = {};\n const maxdist = a.length + b.length;\n const H: number[][] = [];\n \n H[-1] = [];\n H[-1][-1] = maxdist;\n \n for (let i = 0; i <= a.length; i++) {\n H[i] = [];\n H[i][-1] = maxdist;\n H[i][0] = i;\n }\n \n for (let j = 0; j <= b.length; j++) {\n H[-1][j] = maxdist;\n H[0][j] = j;\n }\n \n for (let i = 1; i <= a.length; i++) {\n let db = 0;\n for (let j = 1; j <= b.length; j++) {\n const k = da[b[j - 1]] || 0;\n const l = db;\n let cost = 1;\n if (a[i - 1] === b[j - 1]) {\n cost = 0;\n db = j;\n }\n \n H[i][j] = Math.min(\n H[i - 1][j] + 1, // insertion\n H[i][j - 1] + 1, // deletion\n H[i - 1][j - 1] + cost, // substitution\n H[k - 1][l - 1] + (i - k - 1) + 1 + (j - l - 1) // transposition\n );\n }\n da[a[i - 1]] = i;\n }\n \n return H[a.length][b.length];\n }\n\n // Create fuzzy variations for learning\n generateVariations(input: string): string[] {\n const variations = [input];\n const normalized = this.normalize(input);\n \n // Generate common variations\n variations.push(input.toLowerCase());\n variations.push(input.replace(/\\s+/g, ' ').trim());\n variations.push(input.replace(/^(click|press)\\s+/i, 'tap '));\n variations.push(input.replace(/\\s+button$/i, ''));\n \n // Token permutations (limited)\n if (normalized.tokens.length <= 4) {\n const permutations = this.generateTokenPermutations(normalized.tokens);\n variations.push(...permutations.slice(0, 3)); // Limit to 3 permutations\n }\n \n return [...new Set(variations)];\n }\n\n private generateTokenPermutations(tokens: string[]): string[] {\n if (tokens.length <= 1) return tokens;\n if (tokens.length > 4) return []; // Too many combinations\n \n const result: string[] = [];\n const permute = (arr: string[], start = 0) => {\n if (start === arr.length) {\n result.push(arr.join(' '));\n return;\n }\n \n for (let i = start; i < arr.length; i++) {\n [arr[start], arr[i]] = [arr[i], arr[start]];\n permute(arr, start + 1);\n [arr[start], arr[i]] = [arr[i], arr[start]]; // backtrack\n }\n };\n \n permute([...tokens]);\n return result;\n }\n}","import { LRUCache } from 'lru-cache';\nimport { BidirectionalCache } from './bidirectional-cache.js';\n\ninterface CacheEntry {\n selector: string;\n confidence: number;\n source: 'exact' | 'normalized' | 'reverse' | 'fuzzy';\n cached: boolean;\n timestamp: number;\n}\n\ninterface TieredCacheOptions {\n memorySize?: number;\n memoryTTL?: number;\n preloadCommonSelectors?: boolean;\n}\n\nexport class TieredCache {\n private memoryCache: LRUCache<string, CacheEntry>;\n private bidirectionalCache: BidirectionalCache;\n private stats = {\n memoryHits: 0,\n sqliteHits: 0,\n misses: 0,\n totalRequests: 0\n };\n\n constructor(\n bidirectionalCache: BidirectionalCache, \n options: TieredCacheOptions = {}\n ) {\n this.bidirectionalCache = bidirectionalCache;\n \n // Initialize LRU memory cache\n this.memoryCache = new LRUCache<string, CacheEntry>({\n max: options.memorySize ?? 100,\n ttl: options.memoryTTL ?? 300000, // 5 minutes\n updateAgeOnGet: true,\n updateAgeOnHas: true\n });\n\n // Preload common selectors if requested\n if (options.preloadCommonSelectors) {\n this.preloadCommonSelectors();\n }\n }\n\n async get(input: string, url: string): Promise<CacheEntry | null> {\n this.stats.totalRequests++;\n const memoryKey = this.createMemoryKey(input, url);\n\n // L1: Memory Cache (0.1ms)\n if (this.memoryCache.has(memoryKey)) {\n const cached = this.memoryCache.get(memoryKey);\n if (cached) {\n this.stats.memoryHits++;\n console.error(`[TieredCache] Memory HIT for \"${input}\": ${cached.selector} (${cached.source})`);\n return cached;\n }\n }\n\n // L2: SQLite Cache (1-5ms) \n const result = await this.bidirectionalCache.get(input, url);\n \n if (result) {\n this.stats.sqliteHits++;\n const cacheEntry: CacheEntry = {\n ...result,\n timestamp: Date.now()\n };\n \n // Store in memory for next time\n this.memoryCache.set(memoryKey, cacheEntry);\n \n // Also cache variations for better hit rate\n await this.cacheVariations(input, url, cacheEntry);\n \n console.error(`[TieredCache] SQLite HIT for \"${input}\": ${result.selector} (${result.source})`);\n return cacheEntry;\n }\n\n this.stats.misses++;\n console.error(`[TieredCache] MISS for \"${input}\"`);\n return null;\n }\n\n async set(input: string, url: string, selector: string): Promise<void> {\n // Store in SQLite (persistent)\n await this.bidirectionalCache.set(input, url, selector);\n \n // Store in memory (fast access)\n const memoryKey = this.createMemoryKey(input, url);\n const cacheEntry: CacheEntry = {\n selector,\n confidence: 0.8, // New entries start with good confidence\n source: 'exact',\n cached: true,\n timestamp: Date.now()\n };\n \n this.memoryCache.set(memoryKey, cacheEntry);\n \n // Preemptively cache common variations\n await this.cacheVariations(input, url, cacheEntry);\n \n console.error(`[TieredCache] STORED \"${input}\" → ${selector}`);\n }\n\n private createMemoryKey(input: string, url: string): string {\n return `${input.toLowerCase().trim()}|${url}`;\n }\n\n private async cacheVariations(input: string, url: string, cacheEntry: CacheEntry): Promise<void> {\n // Generate and cache common variations\n const variations = this.generateInputVariations(input);\n \n for (const variation of variations) {\n const varKey = this.createMemoryKey(variation, url);\n if (!this.memoryCache.has(varKey)) {\n // Store with slightly lower confidence\n const varEntry: CacheEntry = {\n ...cacheEntry,\n confidence: cacheEntry.confidence * 0.95,\n source: 'normalized'\n };\n this.memoryCache.set(varKey, varEntry);\n }\n }\n }\n\n private generateInputVariations(input: string): string[] {\n const variations = new Set<string>();\n \n // Basic variations\n variations.add(input.toLowerCase());\n variations.add(input.toLowerCase().trim());\n variations.add(input.replace(/\\s+/g, ' ').trim());\n \n // Remove articles\n variations.add(input.replace(/\\b(the|a|an)\\s+/gi, '').trim());\n \n // Action variations\n const actionMappings = [\n ['click', 'press', 'tap', 'hit'],\n ['type', 'enter', 'input', 'fill'],\n ['select', 'choose', 'pick']\n ];\n \n for (const synonyms of actionMappings) {\n for (let i = 0; i < synonyms.length; i++) {\n for (let j = 0; j < synonyms.length; j++) {\n if (i !== j) {\n const regex = new RegExp(`\\\\b${synonyms[i]}\\\\b`, 'gi');\n if (regex.test(input)) {\n variations.add(input.replace(regex, synonyms[j]));\n }\n }\n }\n }\n }\n \n // Remove \"button\" suffix\n variations.add(input.replace(/\\s+button\\s*$/i, '').trim());\n \n // Limit variations to prevent memory bloat\n return Array.from(variations).slice(0, 8);\n }\n\n private async preloadCommonSelectors(): Promise<void> {\n // This would preload frequently used selectors\n // Implementation depends on having historical data\n try {\n const stats = await this.bidirectionalCache.getStats();\n console.error(`[TieredCache] Preloading enabled - ${stats.storage?.unique_selectors || 0} selectors available`);\n } catch (error) {\n console.error('[TieredCache] Preload failed:', error);\n }\n }\n\n async invalidateForUrl(url: string): Promise<void> {\n // Clear memory cache entries for this URL\n const keysToDelete: string[] = [];\n \n for (const [key] of this.memoryCache.entries()) {\n if (key.endsWith(`|${url}`)) {\n keysToDelete.push(key);\n }\n }\n \n for (const key of keysToDelete) {\n this.memoryCache.delete(key);\n }\n \n console.error(`[TieredCache] Invalidated ${keysToDelete.length} memory entries for ${url}`);\n }\n\n async clear(): Promise<void> {\n this.memoryCache.clear();\n await this.bidirectionalCache.clear();\n \n // Reset stats\n this.stats = {\n memoryHits: 0,\n sqliteHits: 0,\n misses: 0,\n totalRequests: 0\n };\n \n console.error('[TieredCache] All caches cleared');\n }\n\n getStats(): any {\n const memoryHitRate = this.stats.totalRequests > 0 \n ? (this.stats.memoryHits / this.stats.totalRequests) * 100\n : 0;\n \n const sqliteHitRate = this.stats.totalRequests > 0\n ? (this.stats.sqliteHits / this.stats.totalRequests) * 100 \n : 0;\n \n const overallHitRate = this.stats.totalRequests > 0\n ? ((this.stats.memoryHits + this.stats.sqliteHits) / this.stats.totalRequests) * 100\n : 0;\n\n return {\n tiered: {\n memoryHitRate: Math.round(memoryHitRate * 10) / 10,\n sqliteHitRate: Math.round(sqliteHitRate * 10) / 10,\n overallHitRate: Math.round(overallHitRate * 10) / 10,\n totalRequests: this.stats.totalRequests,\n memorySize: this.memoryCache.size,\n memoryMax: this.memoryCache.max\n },\n breakdown: {\n memoryHits: this.stats.memoryHits,\n sqliteHits: this.stats.sqliteHits,\n misses: this.stats.misses\n }\n };\n }\n\n // Enhanced wrapper for MCP server integration\n async wrapSelectorOperation<T>(\n description: string,\n url: string,\n operation: (selector: string) => Promise<T>,\n fallbackSelector?: string\n ): Promise<{ result: T; cached: boolean; selector: string }> {\n \n // Try cache first\n const cached = await this.get(description, url);\n if (cached) {\n try {\n const result = await operation(cached.selector);\n console.error(`[TieredCache] Operation SUCCESS with cached selector: ${cached.selector}`);\n return { \n result, \n cached: true, \n selector: cached.selector \n };\n } catch (error) {\n console.error(`[TieredCache] Cached selector FAILED, invalidating: ${cached.selector}`);\n // Cached selector failed - remove it\n const memKey = this.createMemoryKey(description, url);\n this.memoryCache.delete(memKey);\n }\n }\n\n // Use fallback selector\n if (!fallbackSelector) {\n throw new Error(`No cached selector found for \"${description}\" and no fallback provided`);\n }\n\n try {\n const result = await operation(fallbackSelector);\n \n // Cache successful operation\n await this.set(description, url, fallbackSelector);\n console.error(`[TieredCache] Operation SUCCESS with fallback, now cached: ${fallbackSelector}`);\n \n return { \n result, \n cached: false, \n selector: fallbackSelector \n };\n } catch (error) {\n console.error(`[TieredCache] Fallback selector also FAILED: ${fallbackSelector}`);\n throw error;\n }\n }\n\n close(): void {\n this.memoryCache.clear();\n this.bidirectionalCache.close();\n }\n}","import { Page } from 'playwright';\nimport { CacheManager } from './cache-manager';\nimport crypto from 'crypto';\n\ninterface SnapshotEntry {\n snapshot: any;\n url: string;\n timestamp: number;\n domHash: string;\n viewportSize: { width: number; height: number };\n}\n\nexport class SnapshotCache {\n private cacheManager: CacheManager;\n private currentUrl: string = '';\n private currentProfile?: string;\n private page?: Page;\n private lastDomHash?: string;\n\n constructor(cacheManager: CacheManager) {\n this.cacheManager = cacheManager;\n }\n\n setContext(page: Page, url: string, profile?: string): void {\n this.page = page;\n this.currentUrl = url;\n this.currentProfile = profile;\n }\n\n async getCachedSnapshot(): Promise<any | null> {\n if (!this.page) return null;\n\n const domHash = await this.computeDomHash();\n const cacheKey = {\n url: this.currentUrl,\n domHash\n };\n\n const cached = await this.cacheMana