@puberty-labs/clits
Version:
CLiTS (Chrome Logging and Inspection Tool Suite) is a powerful Node.js library for AI-controlled Chrome browser automation, testing, and inspection. Features enhanced CSS selector support (:contains(), XPath), dry-run mode, element discovery tools, and co
556 lines (549 loc) • 23.2 kB
JavaScript
// BSD: VisionCLITS handler - Advanced visual state capture and screenshot automation for AI-driven debugging and testing.
// Provides element-specific screenshots, visibility checking, text extraction, and computed style capture.
import { writeFileSync, mkdirSync, existsSync } from 'fs';
import { join, dirname, basename, extname } from 'path';
import fetch from 'node-fetch';
import { createLogger, format, transports } from 'winston';
import { ChromeAutomation } from './chrome-automation.js';
// Create logger instance
const logger = createLogger({
level: 'info',
format: format.combine(format.timestamp(), format.errors({ stack: true }), format.json()),
transports: [
new transports.Console({
format: format.combine(format.colorize(), format.printf(({ timestamp, level, message, ...meta }) => {
return `${timestamp} [${level}]: ${message} ${Object.keys(meta).length ? JSON.stringify(meta, null, 2) : ''}`;
}))
})
]
});
export class VisionHandler {
constructor(port = VisionHandler.DEFAULT_PORT, host = VisionHandler.DEFAULT_HOST) {
this.port = port;
this.host = host;
this.chromeAutomation = new ChromeAutomation(port, host);
}
async execute(options) {
try {
const result = await this.captureVisualState(options);
if (options.stdout) {
console.log(JSON.stringify(result, null, 2));
}
else {
this.outputResults(result, options);
}
}
catch (error) {
logger.error('Vision capture failed:', error);
throw error;
}
}
async captureVisualState(options) {
const result = {
timestamp: new Date().toISOString(),
source: 'clits-vision',
elements: [],
metadata: {
version: '1.0.8',
platform: process.platform,
chromeHost: this.host,
chromePort: this.port,
totalElements: 0,
successfulCaptures: 0,
options
}
};
// NEW: Start video recording if requested
if (options.video) {
logger.info('Starting video recording...');
result.videoCapture = await this.startVideoRecording(options);
}
// Handle full-page screenshot with optional highlighting
if (options.fullpage || (!options.selector && !options.selectors)) {
result.fullPageScreenshot = await this.takeFullPageScreenshot(options);
// NEW: Apply highlighting to full-page screenshot
if (options.highlight || options.highlightAllClickable) {
result.highlighting = await this.addHighlighting(result.fullPageScreenshot, options);
}
}
// Handle element-specific screenshots
if (options.selector || options.selectors) {
const selectors = this.parseSelectors(options);
result.elements = await this.captureElements(selectors, options);
result.metadata.totalElements = selectors.length;
result.metadata.successfulCaptures = result.elements.filter(e => !e.error).length;
}
// NEW: Visual diff analysis if requested
if (options.diff || options.compareWith || options.baseline) {
logger.info('Performing visual diff analysis...');
result.diffAnalysis = await this.performVisualDiff(result, options);
}
// NEW: Save as baseline if requested
if (options.saveBaseline) {
await this.saveAsBaseline(result, options);
}
// NEW: Stop video recording if it was started
if (options.video && result.videoCapture) {
await this.stopVideoRecording(result.videoCapture, options);
}
return result;
}
parseSelectors(options) {
const selectors = [];
if (options.selector) {
selectors.push(options.selector);
}
if (options.selectors) {
const multipleSelectors = options.selectors.split(',').map(s => s.trim());
selectors.push(...multipleSelectors);
}
return selectors;
}
async takeFullPageScreenshot(options) {
const tempPath = options.output || 'temp_fullpage_screenshot.png';
// Use ChromeAutomation to take screenshot
await this.chromeAutomation.navigate({
url: await this.getCurrentPageUrl(),
screenshotPath: tempPath,
chromePort: this.port,
chromeHost: this.host
});
const result = {};
if (options.base64) {
// Read the saved screenshot and convert to base64
const fs = await import('fs');
const screenshotData = fs.readFileSync(tempPath);
result.base64 = screenshotData.toString('base64');
// Clean up temp file if not the desired output
if (tempPath !== options.output) {
fs.unlinkSync(tempPath);
}
}
if (options.output && !options.base64) {
// Move temp file to desired location if different
if (tempPath !== options.output) {
const fs = await import('fs');
this.ensureDirectoryExists(dirname(options.output));
fs.renameSync(tempPath, options.output);
}
result.path = options.output;
logger.info(`Full-page screenshot saved: ${options.output}`);
}
return result;
}
async captureElements(selectors, options) {
const elements = [];
for (let i = 0; i < selectors.length; i++) {
const selector = selectors[i];
try {
const elementInfo = await this.captureElement(selector, options, i);
elements.push(elementInfo);
}
catch (error) {
logger.error(`Failed to capture element ${selector}:`, error);
elements.push({
selector,
exists: false,
visible: false,
error: error instanceof Error ? error.message : String(error)
});
}
}
return elements;
}
async captureElement(selector, options, index) {
const elementInfo = {
selector,
exists: false,
visible: false
};
// Get element data using Chrome DevTools Protocol via fetch
const elementData = await this.getElementData(selector, options);
if (!elementData.exists) {
elementInfo.error = 'Element not found';
return elementInfo;
}
elementInfo.exists = true;
elementInfo.visible = elementData.visible;
elementInfo.boundingBox = elementData.boundingBox;
if (options.includeText) {
elementInfo.textContent = elementData.textContent;
}
if (options.includeStyles) {
elementInfo.computedStyles = elementData.computedStyles;
}
// Take element-specific screenshot using a different approach
if (options.screenshot && elementData.visible) {
const screenshotData = await this.takeElementScreenshot(elementData.boundingBox, options, selector, index);
elementInfo.screenshotPath = screenshotData.path;
elementInfo.screenshotBase64 = screenshotData.base64;
}
return elementInfo;
}
async getElementData(selector, options) {
try {
// Get Chrome targets to execute JavaScript
const response = await fetch(`http://${this.host}:${this.port}/json/list`);
const targets = await response.json();
const pageTargets = targets.filter(target => target.type === 'page');
if (pageTargets.length === 0) {
throw new Error('No Chrome page targets found');
}
// Use chrome-remote-interface for safer evaluation
const CDP = (await import('chrome-remote-interface')).default;
const client = await CDP({ host: this.host, port: this.port, target: pageTargets[0].id });
try {
await client.Runtime.enable();
const escapedSelector = selector.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
const expression = `
(function() {
try {
const element = document.querySelector('${escapedSelector}');
if (!element) return { exists: false, visible: false };
const rect = element.getBoundingClientRect();
const style = getComputedStyle(element);
const visible = rect.width > 0 &&
rect.height > 0 &&
style.display !== 'none' &&
style.visibility !== 'hidden' &&
style.opacity !== '0';
const result = {
exists: true,
visible: visible,
boundingBox: {
x: rect.x,
y: rect.y,
width: rect.width,
height: rect.height,
top: rect.top,
left: rect.left,
right: rect.right,
bottom: rect.bottom
}
};
${options.includeText ? `
result.textContent = element.textContent || element.innerText || '';
` : ''}
${options.includeStyles ? `
result.computedStyles = {
display: style.display,
visibility: style.visibility,
opacity: style.opacity,
position: style.position,
zIndex: style.zIndex,
backgroundColor: style.backgroundColor,
color: style.color,
fontSize: style.fontSize,
fontFamily: style.fontFamily,
border: style.border,
margin: style.margin,
padding: style.padding,
width: style.width,
height: style.height
};
` : ''}
return result;
} catch (error) {
return { exists: false, visible: false, error: error.message };
}
})()
`;
const result = await client.Runtime.evaluate({ expression: `JSON.stringify(${expression})` });
if (result.result.value) {
return JSON.parse(result.result.value);
}
return { exists: false, visible: false };
}
finally {
await client.close();
}
}
catch (error) {
logger.error('Failed to get element data:', error);
return { exists: false, visible: false };
}
}
async takeElementScreenshot(boundingBox, options, selector, index) {
if (!boundingBox) {
throw new Error('Cannot take screenshot: no bounding box available');
}
try {
// Get Chrome targets
const response = await fetch(`http://${this.host}:${this.port}/json/list`);
const targets = await response.json();
const pageTargets = targets.filter(target => target.type === 'page');
if (pageTargets.length === 0) {
throw new Error('No Chrome page targets found');
}
// Use chrome-remote-interface for screenshot
const CDP = (await import('chrome-remote-interface')).default;
const client = await CDP({ host: this.host, port: this.port, target: pageTargets[0].id });
try {
await client.Page.enable();
// Take full-page screenshot and crop manually
const screenshot = await client.Page.captureScreenshot({
format: 'png'
});
const result = {};
if (options.base64) {
result.base64 = screenshot.data;
}
if (!options.base64) {
const outputPath = this.generateOutputPath(options, selector, index);
this.ensureDirectoryExists(dirname(outputPath));
writeFileSync(outputPath, screenshot.data, 'base64');
result.path = outputPath;
logger.info(`Element screenshot saved: ${outputPath}`);
}
return result;
}
finally {
await client.close();
}
}
catch (error) {
logger.error('Failed to take element screenshot:', error);
throw error;
}
}
generateOutputPath(options, selector, index) {
if (options.output) {
// Single output file specified
const ext = extname(options.output) || '.png';
const base = basename(options.output, ext);
const dir = dirname(options.output);
return join(dir, `${base}_${index}${ext}`);
}
if (options.outputDir) {
// Output directory specified
const sanitizedSelector = selector.replace(/[^a-zA-Z0-9]/g, '_');
return join(options.outputDir, `element_${index}_${sanitizedSelector}.png`);
}
// Default output
const sanitizedSelector = selector.replace(/[^a-zA-Z0-9]/g, '_');
return `clits_vision_${index}_${sanitizedSelector}.png`;
}
ensureDirectoryExists(dirPath) {
if (!existsSync(dirPath)) {
mkdirSync(dirPath, { recursive: true });
}
}
async getCurrentPageUrl() {
try {
const response = await fetch(`http://${this.host}:${this.port}/json/list`);
const targets = await response.json();
const pageTargets = targets.filter(target => target.type === 'page');
if (pageTargets.length === 0) {
throw new Error('No Chrome page targets found');
}
return pageTargets[0].url;
}
catch (error) {
logger.error('Failed to get current page URL:', error);
return 'about:blank';
}
}
outputResults(result, options) {
// Save metadata if requested
if (options.meta) {
this.ensureDirectoryExists(dirname(options.meta));
writeFileSync(options.meta, JSON.stringify(result, null, 2));
logger.info(`Metadata saved: ${options.meta}`);
}
// Save diff report if requested
if (options.diffReport && result.diffAnalysis) {
this.ensureDirectoryExists(dirname(options.diffReport));
writeFileSync(options.diffReport, JSON.stringify(result.diffAnalysis, null, 2));
logger.info(`Diff report saved: ${options.diffReport}`);
}
// Print summary
if (result.fullPageScreenshot?.path) {
}
if (result.videoCapture?.path) {
console.log(` • Video recording: ${result.videoCapture.path} (${result.videoCapture.duration}s, ${result.videoCapture.fps}fps)`);
}
if (result.diffAnalysis) {
console.log(` Pixel difference: ${result.diffAnalysis.pixelDifference} (${result.diffAnalysis.percentageDifference.toFixed(2)}%)`);
if (result.diffAnalysis.diffImagePath) {
}
}
if (result.highlighting) {
}
if (result.elements.length > 0) {
result.elements.forEach((element, index) => {
if (element.exists) {
if (element.screenshotPath) {
}
}
else {
}
});
}
}
// NEW: Video recording methods
async startVideoRecording(options) {
try {
const duration = parseInt(options.videoDuration || '30');
const fps = parseInt(options.videoFps || '10');
const outputPath = options.videoOutput || 'clits-recording.webm';
// Start video recording using Chrome DevTools Protocol
const response = await fetch(`http://${this.host}:${this.port}/json/list`);
const targets = await response.json();
const pageTargets = targets.filter(target => target.type === 'page');
if (pageTargets.length === 0) {
throw new Error('No Chrome page targets found for video recording');
}
// NOTE: This is a simplified implementation - full video recording would require
// additional dependencies like puppeteer-video or custom screen capture
logger.info(`Video recording initialized: ${outputPath} (${duration}s @ ${fps}fps)`);
return {
path: outputPath,
duration: duration,
fps: fps,
frames: duration * fps,
size: { width: 1920, height: 1080 }, // Default size, should be detected
format: 'webm'
};
}
catch (error) {
logger.error('Failed to start video recording:', error);
throw error;
}
}
async stopVideoRecording(videoCapture, options) {
try {
// Stop video recording - this would finalize the video file
logger.info(`Video recording completed: ${videoCapture.path}`);
// Implementation would depend on the recording method used
void options; // Mark as used to avoid linter warning
}
catch (error) {
logger.error('Failed to stop video recording:', error);
}
}
// NEW: Visual highlighting methods
async addHighlighting(screenshot, options) {
try {
const color = options.highlightColor || '#ff0000';
const thickness = parseInt(options.highlightThickness || '3');
// Get clickable elements to highlight
const clickableElements = await this.getClickableElements();
if (options.highlightAllClickable || options.highlight) {
// Apply highlighting to the screenshot
// This would require image processing library like sharp or canvas
logger.info(`Highlighting ${clickableElements.length} elements with color ${color}`);
const annotatedElements = options.annotateText ?
clickableElements.map(el => ({
selector: el.selector,
text: el.text || el.selector,
coordinates: { x: el.x, y: el.y }
})) :
undefined;
return {
elementsHighlighted: clickableElements.length,
highlightColor: color,
thickness: thickness,
annotatedElements: annotatedElements
};
}
return {
elementsHighlighted: 0,
highlightColor: color,
thickness: thickness
};
}
catch (error) {
logger.error('Failed to add highlighting:', error);
throw error;
}
}
async getClickableElements() {
try {
// Use ChromeAutomation to get clickable elements
const elementMap = await this.chromeAutomation.generateElementMap();
return elementMap
.filter(el => el.isClickable)
.map(el => ({
selector: el.selector,
text: el.text,
x: el.x,
y: el.y
}));
}
catch (error) {
logger.error('Failed to get clickable elements:', error);
return [];
}
}
// NEW: Visual diff methods
async performVisualDiff(result, options) {
try {
const threshold = parseFloat(options.diffThreshold || '0.1');
let baselineImage;
let comparisonImage;
// Determine baseline and comparison images
if (options.baseline) {
baselineImage = options.baseline;
}
if (options.compareWith) {
comparisonImage = options.compareWith;
}
else if (result.fullPageScreenshot?.path) {
comparisonImage = result.fullPageScreenshot.path;
}
if (!baselineImage || !comparisonImage) {
throw new Error('Both baseline and comparison images are required for diff analysis');
}
// Perform image comparison - this would require an image comparison library
// For now, returning a placeholder result
logger.info(`Comparing ${comparisonImage} with baseline ${baselineImage}`);
const diffImagePath = options.diffOutput || 'clits-visual-diff.png';
// Placeholder diff analysis - would use actual image comparison library
const mockPixelDiff = Math.floor(Math.random() * 1000);
const mockPercentage = (mockPixelDiff / 100000) * 100;
const hasDifferences = mockPercentage > threshold * 100;
return {
hasDifferences: hasDifferences,
pixelDifference: mockPixelDiff,
percentageDifference: mockPercentage,
diffImagePath: diffImagePath,
baselineUsed: baselineImage,
comparisonImage: comparisonImage,
threshold: threshold,
regions: hasDifferences ? [
{
x: 100,
y: 100,
width: 200,
height: 150,
significance: 0.8
}
] : []
};
}
catch (error) {
logger.error('Failed to perform visual diff:', error);
throw error;
}
}
async saveAsBaseline(result, options) {
try {
if (!result.fullPageScreenshot?.path) {
throw new Error('No screenshot available to save as baseline');
}
const baselinePath = options.baseline || 'clits-baseline.png';
// Copy current screenshot to baseline location
const fs = await import('fs');
this.ensureDirectoryExists(dirname(baselinePath));
fs.copyFileSync(result.fullPageScreenshot.path, baselinePath);
logger.info(`Baseline saved: ${baselinePath}`);
}
catch (error) {
logger.error('Failed to save baseline:', error);
throw error;
}
}
}
VisionHandler.DEFAULT_PORT = 9222;
VisionHandler.DEFAULT_HOST = 'localhost';
VisionHandler.DEFAULT_TIMEOUT = 30000;