@presidio-dev/factifai-agent
Version:
An AI powered browser automation testing agent powered by LLMs.
1,340 lines (1,337 loc) • 882 kB
JavaScript
#!/usr/bin/env node
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __commonJS = (cb, mod) => function __require() {
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
};
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
// ../../node_modules/.pnpm/dotenv@16.5.0/node_modules/dotenv/package.json
var require_package = __commonJS({
"../../node_modules/.pnpm/dotenv@16.5.0/node_modules/dotenv/package.json"(exports2, module2) {
module2.exports = {
name: "dotenv",
version: "16.5.0",
description: "Loads environment variables from .env file",
main: "lib/main.js",
types: "lib/main.d.ts",
exports: {
".": {
types: "./lib/main.d.ts",
require: "./lib/main.js",
default: "./lib/main.js"
},
"./config": "./config.js",
"./config.js": "./config.js",
"./lib/env-options": "./lib/env-options.js",
"./lib/env-options.js": "./lib/env-options.js",
"./lib/cli-options": "./lib/cli-options.js",
"./lib/cli-options.js": "./lib/cli-options.js",
"./package.json": "./package.json"
},
scripts: {
"dts-check": "tsc --project tests/types/tsconfig.json",
lint: "standard",
pretest: "npm run lint && npm run dts-check",
test: "tap run --allow-empty-coverage --disable-coverage --timeout=60000",
"test:coverage": "tap run --show-full-coverage --timeout=60000 --coverage-report=lcov",
prerelease: "npm test",
release: "standard-version"
},
repository: {
type: "git",
url: "git://github.com/motdotla/dotenv.git"
},
homepage: "https://github.com/motdotla/dotenv#readme",
funding: "https://dotenvx.com",
keywords: [
"dotenv",
"env",
".env",
"environment",
"variables",
"config",
"settings"
],
readmeFilename: "README.md",
license: "BSD-2-Clause",
devDependencies: {
"@types/node": "^18.11.3",
decache: "^4.6.2",
sinon: "^14.0.1",
standard: "^17.0.0",
"standard-version": "^9.5.0",
tap: "^19.2.0",
typescript: "^4.8.4"
},
engines: {
node: ">=12"
},
browser: {
fs: false
}
};
}
});
// ../../node_modules/.pnpm/dotenv@16.5.0/node_modules/dotenv/lib/main.js
var require_main = __commonJS({
"../../node_modules/.pnpm/dotenv@16.5.0/node_modules/dotenv/lib/main.js"(exports2, module2) {
var fs7 = require("fs");
var path7 = require("path");
var os4 = require("os");
var crypto = require("crypto");
var packageJson = require_package();
var version = packageJson.version;
var LINE = /(?:^|^)\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/mg;
function parse(src) {
const obj = {};
let lines = src.toString();
lines = lines.replace(/\r\n?/mg, "\n");
let match;
while ((match = LINE.exec(lines)) != null) {
const key = match[1];
let value = match[2] || "";
value = value.trim();
const maybeQuote = value[0];
value = value.replace(/^(['"`])([\s\S]*)\1$/mg, "$2");
if (maybeQuote === '"') {
value = value.replace(/\\n/g, "\n");
value = value.replace(/\\r/g, "\r");
}
obj[key] = value;
}
return obj;
}
function _parseVault(options) {
const vaultPath = _vaultPath(options);
const result = DotenvModule.configDotenv({ path: vaultPath });
if (!result.parsed) {
const err = new Error(`MISSING_DATA: Cannot parse ${vaultPath} for an unknown reason`);
err.code = "MISSING_DATA";
throw err;
}
const keys = _dotenvKey(options).split(",");
const length = keys.length;
let decrypted;
for (let i = 0; i < length; i++) {
try {
const key = keys[i].trim();
const attrs = _instructions(result, key);
decrypted = DotenvModule.decrypt(attrs.ciphertext, attrs.key);
break;
} catch (error) {
if (i + 1 >= length) {
throw error;
}
}
}
return DotenvModule.parse(decrypted);
}
function _warn(message) {
console.log(`[dotenv@${version}][WARN] ${message}`);
}
function _debug(message) {
console.log(`[dotenv@${version}][DEBUG] ${message}`);
}
function _dotenvKey(options) {
if (options && options.DOTENV_KEY && options.DOTENV_KEY.length > 0) {
return options.DOTENV_KEY;
}
if (process.env.DOTENV_KEY && process.env.DOTENV_KEY.length > 0) {
return process.env.DOTENV_KEY;
}
return "";
}
function _instructions(result, dotenvKey) {
let uri;
try {
uri = new URL(dotenvKey);
} catch (error) {
if (error.code === "ERR_INVALID_URL") {
const err = new Error("INVALID_DOTENV_KEY: Wrong format. Must be in valid uri format like dotenv://:key_1234@dotenvx.com/vault/.env.vault?environment=development");
err.code = "INVALID_DOTENV_KEY";
throw err;
}
throw error;
}
const key = uri.password;
if (!key) {
const err = new Error("INVALID_DOTENV_KEY: Missing key part");
err.code = "INVALID_DOTENV_KEY";
throw err;
}
const environment = uri.searchParams.get("environment");
if (!environment) {
const err = new Error("INVALID_DOTENV_KEY: Missing environment part");
err.code = "INVALID_DOTENV_KEY";
throw err;
}
const environmentKey = `DOTENV_VAULT_${environment.toUpperCase()}`;
const ciphertext = result.parsed[environmentKey];
if (!ciphertext) {
const err = new Error(`NOT_FOUND_DOTENV_ENVIRONMENT: Cannot locate environment ${environmentKey} in your .env.vault file.`);
err.code = "NOT_FOUND_DOTENV_ENVIRONMENT";
throw err;
}
return { ciphertext, key };
}
function _vaultPath(options) {
let possibleVaultPath = null;
if (options && options.path && options.path.length > 0) {
if (Array.isArray(options.path)) {
for (const filepath of options.path) {
if (fs7.existsSync(filepath)) {
possibleVaultPath = filepath.endsWith(".vault") ? filepath : `${filepath}.vault`;
}
}
} else {
possibleVaultPath = options.path.endsWith(".vault") ? options.path : `${options.path}.vault`;
}
} else {
possibleVaultPath = path7.resolve(process.cwd(), ".env.vault");
}
if (fs7.existsSync(possibleVaultPath)) {
return possibleVaultPath;
}
return null;
}
function _resolveHome(envPath) {
return envPath[0] === "~" ? path7.join(os4.homedir(), envPath.slice(1)) : envPath;
}
function _configVault(options) {
const debug = Boolean(options && options.debug);
if (debug) {
_debug("Loading env from encrypted .env.vault");
}
const parsed = DotenvModule._parseVault(options);
let processEnv = process.env;
if (options && options.processEnv != null) {
processEnv = options.processEnv;
}
DotenvModule.populate(processEnv, parsed, options);
return { parsed };
}
function configDotenv(options) {
const dotenvPath = path7.resolve(process.cwd(), ".env");
let encoding = "utf8";
const debug = Boolean(options && options.debug);
if (options && options.encoding) {
encoding = options.encoding;
} else {
if (debug) {
_debug("No encoding is specified. UTF-8 is used by default");
}
}
let optionPaths = [dotenvPath];
if (options && options.path) {
if (!Array.isArray(options.path)) {
optionPaths = [_resolveHome(options.path)];
} else {
optionPaths = [];
for (const filepath of options.path) {
optionPaths.push(_resolveHome(filepath));
}
}
}
let lastError;
const parsedAll = {};
for (const path8 of optionPaths) {
try {
const parsed = DotenvModule.parse(fs7.readFileSync(path8, { encoding }));
DotenvModule.populate(parsedAll, parsed, options);
} catch (e) {
if (debug) {
_debug(`Failed to load ${path8} ${e.message}`);
}
lastError = e;
}
}
let processEnv = process.env;
if (options && options.processEnv != null) {
processEnv = options.processEnv;
}
DotenvModule.populate(processEnv, parsedAll, options);
if (lastError) {
return { parsed: parsedAll, error: lastError };
} else {
return { parsed: parsedAll };
}
}
function config(options) {
if (_dotenvKey(options).length === 0) {
return DotenvModule.configDotenv(options);
}
const vaultPath = _vaultPath(options);
if (!vaultPath) {
_warn(`You set DOTENV_KEY but you are missing a .env.vault file at ${vaultPath}. Did you forget to build it?`);
return DotenvModule.configDotenv(options);
}
return DotenvModule._configVault(options);
}
function decrypt(encrypted, keyStr) {
const key = Buffer.from(keyStr.slice(-64), "hex");
let ciphertext = Buffer.from(encrypted, "base64");
const nonce = ciphertext.subarray(0, 12);
const authTag = ciphertext.subarray(-16);
ciphertext = ciphertext.subarray(12, -16);
try {
const aesgcm = crypto.createDecipheriv("aes-256-gcm", key, nonce);
aesgcm.setAuthTag(authTag);
return `${aesgcm.update(ciphertext)}${aesgcm.final()}`;
} catch (error) {
const isRange = error instanceof RangeError;
const invalidKeyLength = error.message === "Invalid key length";
const decryptionFailed = error.message === "Unsupported state or unable to authenticate data";
if (isRange || invalidKeyLength) {
const err = new Error("INVALID_DOTENV_KEY: It must be 64 characters long (or more)");
err.code = "INVALID_DOTENV_KEY";
throw err;
} else if (decryptionFailed) {
const err = new Error("DECRYPTION_FAILED: Please check your DOTENV_KEY");
err.code = "DECRYPTION_FAILED";
throw err;
} else {
throw error;
}
}
}
function populate(processEnv, parsed, options = {}) {
const debug = Boolean(options && options.debug);
const override = Boolean(options && options.override);
if (typeof parsed !== "object") {
const err = new Error("OBJECT_REQUIRED: Please check the processEnv argument being passed to populate");
err.code = "OBJECT_REQUIRED";
throw err;
}
for (const key of Object.keys(parsed)) {
if (Object.prototype.hasOwnProperty.call(processEnv, key)) {
if (override === true) {
processEnv[key] = parsed[key];
}
if (debug) {
if (override === true) {
_debug(`"${key}" is already defined and WAS overwritten`);
} else {
_debug(`"${key}" is already defined and was NOT overwritten`);
}
}
} else {
processEnv[key] = parsed[key];
}
}
}
var DotenvModule = {
configDotenv,
_configVault,
_parseVault,
config,
decrypt,
parse,
populate
};
module2.exports.configDotenv = DotenvModule.configDotenv;
module2.exports._configVault = DotenvModule._configVault;
module2.exports._parseVault = DotenvModule._parseVault;
module2.exports.config = DotenvModule.config;
module2.exports.decrypt = DotenvModule.decrypt;
module2.exports.parse = DotenvModule.parse;
module2.exports.populate = DotenvModule.populate;
module2.exports = DotenvModule;
}
});
// ../playwright-core/dist/index.js
var require_dist = __commonJS({
"../playwright-core/dist/index.js"(exports2, module2) {
"use strict";
var __create2 = Object.create;
var __defProp2 = Object.defineProperty;
var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
var __getOwnPropNames2 = Object.getOwnPropertyNames;
var __getProtoOf2 = Object.getPrototypeOf;
var __hasOwnProp2 = Object.prototype.hasOwnProperty;
var __export2 = (target, all) => {
for (var name in all)
__defProp2(target, name, { get: all[name], enumerable: true });
};
var __copyProps2 = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames2(from))
if (!__hasOwnProp2.call(to, key) && key !== except)
__defProp2(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc2(from, key)) || desc.enumerable });
}
return to;
};
var __toESM2 = (mod, isNodeMode, target) => (target = mod != null ? __create2(__getProtoOf2(mod)) : {}, __copyProps2(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp2(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps2(__defProp2({}, "__esModule", { value: true }), mod);
var index_exports = {};
__export2(index_exports, {
BrowserService: () => BrowserService2,
SessionManager: () => SessionManager,
clear: () => clear2,
click: () => click2,
getCurrentUrl: () => getCurrentUrl3,
getVisibleElements: () => getVisibleElements,
goBack: () => goBack2,
goForward: () => goForward2,
markVisibleElements: () => markVisibleElements,
navigate: () => navigate2,
reload: () => reload2,
removeElementMarkers: () => removeElementMarkers,
scrollBy: () => scrollBy2,
scrollToNextChunk: () => scrollToNextChunk2,
scrollToPrevChunk: () => scrollToPrevChunk2,
setCursorVisibility: () => setCursorVisibility,
type: () => type2,
wait: () => wait2
});
module2.exports = __toCommonJS(index_exports);
var playwright = __toESM2(require("playwright"));
async function navigate2(sessionId, url) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
await page.goto(url);
return {
success: true
};
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error during navigation"
};
}
}
async function getVisibleElements(sessionId, maxTextLength) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
const elements = await page.evaluate((maxTextLength2) => {
const allElements = document.querySelectorAll("*");
const results = [];
const excludeTags = /* @__PURE__ */ new Set([
// Document structure
"html",
"body",
"head",
// Metadata and resources
"script",
"style",
"meta",
"link",
"title",
"noscript",
// Hidden elements
"template",
"slot",
"iframe",
// SVG/MathML internals
"defs",
"clippath",
"lineargradient",
"radialgradient",
"mask",
"pattern",
"stop",
// Hidden form elements
"input[type=hidden]",
// Web Components internals
"shadow",
"shadowroot",
// Non-visual semantic elements
"time",
"data",
"param",
"source",
"track",
// Document sections that are typically containers
"main",
"section",
"article",
"nav",
// Other non-visual elements
"base",
"command",
"datalist",
"optgroup",
// containers
"div"
]);
for (const el of Array.from(allElements)) {
const tagName = el.tagName.toLowerCase();
if (excludeTags.has(tagName)) continue;
if (tagName === "input" && el.type === "hidden")
continue;
const rect = el.getBoundingClientRect();
if (rect.width === 0 || rect.height === 0) continue;
if (rect.bottom < 0 || rect.right < 0 || rect.top > window.innerHeight || rect.left > window.innerWidth)
continue;
const style = window.getComputedStyle(el);
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0")
continue;
const x = Math.round(rect.left + rect.width / 2);
const y = Math.round(rect.top + rect.height / 2);
const topElementAtPoint = document.elementFromPoint(x, y);
if (!topElementAtPoint || !el.contains(topElementAtPoint) && topElementAtPoint !== el) {
continue;
}
const isFullyVisible = rect.top >= 0 && rect.left >= 0 && rect.bottom <= window.innerHeight && rect.right <= window.innerWidth;
if (!el.id && !el.className && (!el.textContent || el.textContent.trim() === "")) {
continue;
}
const elementObject = {
tagName: el.tagName.toLowerCase(),
coordinates: { x, y }
};
if (el.id) elementObject.id = el.id;
if (el.className) elementObject.className = el.className;
let textContent = el.textContent?.trim();
if (textContent) {
if (maxTextLength2 && textContent.length > maxTextLength2) {
textContent = textContent.substring(0, maxTextLength2) + "...";
}
elementObject.trimmedText = textContent;
}
results.push(elementObject);
}
return results;
}, maxTextLength || 0);
return {
success: true,
elements
};
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error getting visible elements"
};
}
}
async function getCurrentUrl3(sessionId) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
const url = page.url();
return { success: true, url };
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error getting current URL"
};
}
}
async function wait2(sessionId, seconds) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
await page.waitForTimeout(seconds * 1e3);
return { success: true };
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error during wait"
};
}
}
async function reload2(sessionId) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
await page.reload();
const url = page.url();
return { success: true, url };
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error during page reload"
};
}
}
async function goBack2(sessionId) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
await page.goBack();
const url = page.url();
return { success: true, url };
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error during navigation back"
};
}
}
async function goForward2(sessionId) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
await page.goForward();
const url = page.url();
return { success: true, url };
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error during navigation forward"
};
}
}
async function markVisibleElements(sessionId, options) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
const opts = {
boxColor: options?.boxColor || "red",
textColor: options?.textColor || "white",
backgroundColor: options?.backgroundColor || "blue",
maxElements: options?.maxElements || 100,
minTextLength: options?.minTextLength || 0,
removeExisting: options?.removeExisting !== false,
// Default to true
randomColors: options?.randomColors !== false
// Default to true
};
const markedCount = await page.evaluate((opts2) => {
if (opts2.removeExisting) {
const existingMarkers = document.querySelectorAll(".factifai-element-marker");
existingMarkers.forEach((marker) => marker.remove());
const existingLabelContainer = document.getElementById("factifai-label-container");
if (existingLabelContainer) {
existingLabelContainer.remove();
}
}
const labelContainer = document.createElement("div");
labelContainer.id = "factifai-label-container";
labelContainer.style.position = "fixed";
labelContainer.style.top = "0";
labelContainer.style.left = "0";
labelContainer.style.width = "100vw";
labelContainer.style.height = "100vh";
labelContainer.style.pointerEvents = "none";
labelContainer.style.zIndex = "2147483647";
document.body.appendChild(labelContainer);
const allElements = document.querySelectorAll("*");
let markedCount2 = 0;
const excludeTags = /* @__PURE__ */ new Set([
// Document structure
"html",
"body",
"head",
// Metadata and resources
"script",
"style",
"meta",
"link",
"title",
"noscript",
// Hidden elements
"template",
"slot",
"iframe",
// SVG/MathML internals
"defs",
"clippath",
"lineargradient",
"radialgradient",
"mask",
"pattern",
"stop",
// Hidden form elements
"input[type=hidden]",
// Web Components internals
"shadow",
"shadowroot",
// Non-visual semantic elements
"time",
"data",
"param",
"source",
"track",
// Document sections that are typically containers
"main",
"section",
"article",
"nav",
// Text elements (typically not interactive)
"p",
"span",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"label",
"strong",
"em",
"b",
"i",
"u",
"small",
"blockquote",
"q",
"cite",
"dfn",
"abbr",
"mark",
"del",
"ins",
"sub",
"sup",
"code",
"pre",
"br",
"hr",
"wbr",
"bdi",
"bdo",
"ruby",
"rt",
"rp",
"figcaption",
"figure",
"picture",
"summary",
// Other non-visual elements
"base",
"command",
"datalist",
"optgroup",
// Containers
"div"
]);
const includeSelectors = [
"a",
"button",
'input:not([type="hidden"])',
"select",
"textarea",
'[role="button"]',
'[role="link"]',
'[role="checkbox"]',
'[role="radio"]',
'[role="tab"]',
'[role="menuitem"]',
'[role="combobox"]',
'[role="option"]',
'[role="switch"]',
'[role="searchbox"]',
'[role="textbox"]',
"[onclick]",
"[onkeydown]",
"[onkeyup]",
"[onmousedown]",
"[onmouseup]",
'[tabindex]:not([tabindex="-1"])'
];
const interactiveElements = [];
for (const selector of includeSelectors) {
const elements = document.querySelectorAll(selector);
interactiveElements.push(...Array.from(elements));
}
const markedElements = [];
for (const el of interactiveElements) {
if (markedCount2 >= opts2.maxElements) break;
const tagName = el.tagName.toLowerCase();
if (excludeTags.has(tagName)) continue;
if (tagName === "input" && el.type === "hidden")
continue;
const rect = el.getBoundingClientRect();
if (rect.width === 0 || rect.height === 0) continue;
if (rect.bottom < 0 || rect.right < 0 || rect.top > window.innerHeight || rect.left > window.innerWidth)
continue;
if (rect.width < 5 || rect.height < 5) continue;
const style = window.getComputedStyle(el);
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0")
continue;
const x = Math.round(rect.left + rect.width / 2);
const y = Math.round(rect.top + rect.height / 2);
const topElementAtPoint = document.elementFromPoint(x, y);
if (!topElementAtPoint || !el.contains(topElementAtPoint) && topElementAtPoint !== el) {
continue;
}
const getRandomColor = () => {
const hue = Math.floor(Math.random() * 360);
const saturation = 90 + Math.floor(Math.random() * 10);
const lightness = 30 + Math.floor(Math.random() * 25);
return `hsl(${hue}, ${saturation}%, ${lightness}%)`;
};
const getContrastingTextColor = (bgColor) => {
const lightnessMatch = bgColor.match(/hsl\(\d+,\s*\d+%,\s*(\d+)%\)/);
if (lightnessMatch) {
const lightness = parseInt(lightnessMatch[1], 10);
return lightness < 50 ? "white" : "black";
}
return "white";
};
const elementColor = opts2.randomColors !== false ? getRandomColor() : opts2.boxColor;
const textColor = opts2.randomColors !== false ? getContrastingTextColor(elementColor) : opts2.textColor;
const marker = document.createElement("div");
marker.className = "factifai-element-marker";
Object.assign(marker.style, {
position: "fixed",
left: rect.left + "px",
top: rect.top + "px",
width: rect.width + "px",
height: rect.height + "px",
border: `3px solid ${elementColor}`,
// Thicker border with element color
pointerEvents: "none",
zIndex: "9000000",
boxSizing: "border-box",
background: "transparent"
});
++markedCount2;
const label = document.createElement("div");
label.textContent = markedCount2.toString();
Object.assign(label.style, {
position: "fixed",
// Fixed position relative to viewport
left: rect.right - 10 + "px",
// Position at the right edge of the element
top: rect.top - 10 + "px",
// Position at the top of the element
backgroundColor: elementColor,
// Same color as the border
color: textColor,
borderRadius: "50%",
width: "20px",
height: "20px",
display: "flex",
alignItems: "center",
justifyContent: "center",
fontSize: "12px",
fontWeight: "bold",
boxShadow: "0 0 3px rgba(0,0,0,0.5)",
// Add shadow for better visibility
outline: "1px solid white",
// White outline to help it stand out
zIndex: "2147483647"
// Maximum z-index
});
document.body.appendChild(marker);
labelContainer.appendChild(label);
markedElements.push({
labelNumber: markedCount2.toString(),
coordinates: { x, y }
});
}
return { markedCount: markedCount2, markedElements };
}, opts);
return {
success: true,
markedCount: markedCount.markedCount,
elements: markedCount.markedElements
};
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error marking visible elements"
};
}
}
async function removeElementMarkers(sessionId) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
await page.evaluate(() => {
const markers = document.querySelectorAll(".factifai-element-marker");
markers.forEach((marker) => marker.remove());
const labelContainer = document.getElementById("factifai-label-container");
if (labelContainer) {
labelContainer.remove();
}
});
return { success: true };
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error removing element markers"
};
}
}
var BrowserService2 = class _BrowserService {
constructor() {
this.browser = null;
this.contexts = /* @__PURE__ */ new Map();
this.pageStacks = /* @__PURE__ */ new Map();
}
static getInstance() {
if (!_BrowserService.instance) {
_BrowserService.instance = new _BrowserService();
}
return _BrowserService.instance;
}
async initBrowser() {
this.browser ??= await playwright.chromium.launch({
headless: false,
args: ["--window-size=1280,720"]
});
}
/**
* Set up listeners for new tab creation
*/
setupTabTracking(sessionId, context) {
context.on("page", async (page) => {
const pageStack = this.pageStacks.get(sessionId) || [];
pageStack.push(page);
this.pageStacks.set(sessionId, pageStack);
page.once("close", () => {
console.log(`Page closed in session ${sessionId}`);
const currentStack = this.pageStacks.get(sessionId) || [];
const index = currentStack.indexOf(page);
if (index > -1) {
currentStack.splice(index, 1);
this.pageStacks.set(sessionId, currentStack);
}
});
try {
await page.waitForLoadState("domcontentloaded", { timeout: 5e3 });
} catch (e) {
console.log(`Timeout waiting for page to load in session ${sessionId}`);
}
});
}
/**
* Gets the currently active page for the session (the last one in the stack)
*/
async getPage(sessionId) {
const pageStack = this.pageStacks.get(sessionId) || [];
if (pageStack.length > 0) {
return pageStack[pageStack.length - 1];
}
await this.initBrowser();
if (!this.contexts.has(sessionId)) {
const context = await this.browser.newContext();
this.contexts.set(sessionId, context);
this.setupTabTracking(sessionId, context);
}
const page = await this.contexts.get(sessionId).newPage();
this.pageStacks.set(sessionId, [page]);
return page;
}
/**
* Gets all pages for a session
*/
async getAllPages(sessionId) {
return this.pageStacks.get(sessionId) || [];
}
/**
* Gets the number of open tabs for a session
*/
async getTabCount(sessionId) {
const pageStack = this.pageStacks.get(sessionId) || [];
return pageStack.length;
}
/**
* Closes the currently active tab and switches focus to the previous tab
* @param sessionId The session identifier
* @returns Promise with success status
*/
async closeCurrentTab(sessionId) {
const pageStack = this.pageStacks.get(sessionId) || [];
if (pageStack.length <= 1) {
return false;
}
const currentTab = pageStack[pageStack.length - 1];
try {
await currentTab.close();
return true;
} catch (error) {
console.error("Error closing current tab:", error);
return false;
}
}
async takeScreenshot(sessionId, minWaitMs = 1e3) {
try {
const page = await this.getPage(sessionId);
const start = Date.now();
try {
await Promise.all([
page.waitForLoadState("load", { timeout: 5e3 }),
page.waitForLoadState("networkidle", { timeout: 5e3 }),
page.waitForFunction(() => document.readyState === "complete", {
timeout: 5e3
})
]);
} catch {
}
const elapsed = Date.now() - start;
if (elapsed < minWaitMs) {
await page.waitForTimeout(minWaitMs - elapsed);
}
const buffer = await page.screenshot({
type: "jpeg",
quality: 90,
fullPage: false,
timeout: 5e3
});
return buffer.toString("base64");
} catch (error) {
console.error("Screenshot error:", error);
return null;
}
}
/**
* Captures a screenshot and analyzes page elements
* @param sessionId The session identifier
* @returns Promise with browser inference data
*/
async captureScreenshotAndInfer(sessionId) {
try {
const page = await this.getPage(sessionId);
const base64Image = await this.takeScreenshot(sessionId);
if (!base64Image) {
throw new Error("Failed to capture screenshot");
}
const elements = await this.getAllPageElements(sessionId);
const combinedElements = [
...elements.clickableElements,
...elements.inputElements
];
let scrollPosition = 0;
let totalScroll = 0;
await page.evaluate(() => {
const position = window.scrollY;
const total = document.body.scrollHeight;
return { position, total };
}).then((result) => {
scrollPosition = result.position;
totalScroll = result.total;
});
console.log(combinedElements, "combinedElements");
return {
image: base64Image,
inference: combinedElements,
scrollPosition,
totalScroll,
originalImage: base64Image
};
} catch (error) {
console.error("Error in captureScreenshotAndInfer:", error);
return null;
}
}
/**
* Checks if an element at the given coordinates is visible
* @param sessionId The session identifier
* @param coordinate The x,y coordinates of the element
* @returns Promise with visibility information
*/
async checkIfElementIsVisible(sessionId, coordinate) {
try {
const page = await this.getPage(sessionId);
return await page.evaluate((coordinate2) => {
let result = null;
try {
const element = document.elementFromPoint(coordinate2.x, coordinate2.y);
if (!element) {
return {
isSuccess: false,
message: "No element found at the specified coordinates"
};
}
const { top: top2 } = element.getBoundingClientRect();
const needsScrolling = top2 > window.innerHeight || top2 < 0;
if (needsScrolling) {
element.scrollIntoView({ behavior: "smooth" });
}
result = {
top: top2,
isSuccess: true,
isConditionPassed: needsScrolling
};
} catch (e) {
result = {
isSuccess: false,
message: "Element not available on the visible viewport. Please check if the element is visible in the current viewport otherwise scroll the page to make the element visible in the viewport"
};
}
return result;
}, coordinate);
} catch (error) {
console.error("Error checking element visibility:", error);
return null;
}
}
/**
* Gets all clickable and input elements on the page
* @param sessionId The session identifier
* @returns Promise with clickable and input elements
*/
async getAllPageElements(sessionId) {
try {
const page = await this.getPage(sessionId);
return await page.evaluate(() => {
const clickableSelectors = `a, button, [role], [onclick], input[type="submit"], input[type="button"]`;
const inputSelectors = `input:not([type="submit"]):not([type="button"]), textarea, [contenteditable="true"], select`;
const uniqueClickableElements = Array.from(
document.querySelectorAll(clickableSelectors)
);
const uniqueInputElements = Array.from(
document.querySelectorAll(inputSelectors)
);
function checkIfElementIsVisuallyVisible(element, centerX, centerY) {
const topElement = document.elementFromPoint(centerX, centerY);
return !(topElement !== element && !element.contains(topElement));
}
function elementVisibility(element) {
const checkVisibilityResult = "checkVisibility" in element ? element.checkVisibility({
checkOpacity: true,
checkVisibilityCSS: true,
contentVisibilityAuto: true,
opacityProperty: true,
visibilityProperty: true
}) : true;
const style = getComputedStyle(element);
const notHiddenByCSS = style.display !== "none" && style.visibility !== "hidden" && parseFloat(style.opacity) > 0;
const notHiddenAttribute = !element.hidden;
return checkVisibilityResult && notHiddenByCSS && notHiddenAttribute;
}
function getElementInfo(element) {
const { top: top2, left: left2, bottom: bottom2, right: right2, width, height } = element.getBoundingClientRect();
const attributes = {};
const { innerHeight, innerWidth } = window;
const isVisibleInCurrentViewPort = top2 >= 0 && left2 >= 0 && bottom2 <= innerHeight && right2 <= innerWidth;
Array.from(element.attributes).forEach((attr) => {
attributes[attr.name] = attr.value;
});
return elementVisibility(element) ? {
type: element.type || element.tagName.toLowerCase(),
tagName: element.tagName.toLowerCase(),
text: element.textContent?.trim(),
placeholder: element.placeholder,
coordinate: {
x: Math.round(left2 + width / 2),
y: Math.round(top2 + height / 2)
},
attributes,
isVisibleInCurrentViewPort,
isVisuallyVisible: checkIfElementIsVisuallyVisible(
element,
Math.round(left2 + width / 2),
Math.round(top2 + height / 2)
)
} : null;
}
return {
clickableElements: uniqueClickableElements.map(getElementInfo).filter(Boolean),
inputElements: uniqueInputElements.map(getElementInfo).filter(Boolean)
};
});
} catch (error) {
console.error("Error getting page elements:", error);
return { clickableElements: [], inputElements: [] };
}
}
async closePage(sessionId) {
const pageStack = this.pageStacks.get(sessionId) || [];
for (const page of pageStack) {
await page.close().catch((e) => console.error(`Error closing page: ${e}`));
}
this.pageStacks.delete(sessionId);
if (this.contexts.has(sessionId)) {
await this.contexts.get(sessionId).close().catch((e) => console.error(`Error closing context: ${e}`));
this.contexts.delete(sessionId);
}
}
async closeAll() {
if (this.browser) {
await this.browser.close();
this.browser = null;
this.contexts.clear();
this.pageStacks.clear();
}
}
/**
* Takes a screenshot with elements marked with numbered bounding boxes
* @param sessionId The session identifier
* @param options Optional configuration for marking elements and screenshot
* @returns Promise with screenshot image and marked elements data
*/
async takeMarkedScreenshot(sessionId, options) {
try {
const markResult = await markVisibleElements(sessionId, {
boxColor: options?.boxColor,
textColor: options?.textColor,
backgroundColor: options?.backgroundColor,
maxElements: options?.maxElements,
minTextLength: options?.minTextLength,
removeExisting: options?.removeExisting,
randomColors: options?.randomColors
});
const minWaitMs = options?.minWaitMs || 1e3;
const screenshot = await this.takeScreenshot(sessionId, minWaitMs);
if (options?.removeAfter) {
await removeElementMarkers(sessionId);
}
return {
image: screenshot,
elements: markResult.elements || []
};
} catch (error) {
console.error("Error in takeMarkedScreenshot:", error);
return { image: null, elements: [] };
}
}
};
var SessionManager = class {
constructor() {
this.browserService = BrowserService2.getInstance();
}
async createSession() {
const sessionId = this.generateSessionId();
await this.browserService.getPage(sessionId);
return sessionId;
}
async closeSession(sessionId) {
try {
await this.browserService.closePage(sessionId);
} catch (error) {
console.error(`Error closing session ${sessionId}:`, error);
}
}
generateSessionId() {
return Date.now().toString() + Math.random().toString(36).substring(2, 15);
}
};
var sessionCursors = /* @__PURE__ */ new Map();
async function createOrGetCursor(page) {
return page.evaluate(() => {
let cursor = document.getElementById("__ai_cursor__");
if (!cursor) {
cursor = document.createElement("div");
cursor.id = "__ai_cursor__";
Object.assign(cursor.style, {
position: "fixed",
width: "20px",
height: "20px",
borderRadius: "50%",
backgroundColor: "rgba(255, 50, 50, 0.8)",
// Striking red
border: "2px solid white",
// White stroke
boxShadow: "0 0 0 1px black",
// Black outline for contrast
zIndex: "9999999",
// Very top layer
pointerEvents: "none",
// Don't interfere with clicks
transform: "translate(-50%, -50%)",
// Center the cursor at point
transition: "left 0.2s, top 0.2s"
// Smooth movement
});
document.body.appendChild(cursor);
}
return cursor !== null;
});
}
async function updateCursorPosition(page, x, y) {
await page.evaluate(({ x: x2, y: y2 }) => {
const cursor = document.getElementById("__ai_cursor__");
if (cursor) {
cursor.style.left = `${x2}px`;
cursor.style.top = `${y2}px`;
cursor.animate([
{ transform: "translate(-50%, -50%) scale(1)" },
{ transform: "translate(-50%, -50%) scale(1.3)" },
{ transform: "translate(-50%, -50%) scale(1)" }
], {
duration: 300,
easing: "ease-out"
});
}
}, { x, y });
}
async function updateCursorAtPosition(page, sessionId, x, y) {
if (!sessionCursors.has(sessionId)) {
sessionCursors.set(sessionId, true);
}
if (sessionCursors.get(sessionId)) {
await createOrGetCursor(page);
await updateCursorPosition(page, x, y);
}
}
async function setCursorVisibility(sessionId, visible) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
sessionCursors.set(sessionId, visible);
await page.evaluate((visible2) => {
const cursor = document.getElementById("__ai_cursor__");
if (cursor) {
cursor.style.display = visible2 ? "block" : "none";
}
}, visible);
return { success: true };
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error during cursor visibility operation"
};
}
}
async function click2(sessionId, selectorOrCoordinates) {
const browserService = BrowserService2.getInstance();
try {
const page = await browserService.getPage(sessionId);
if (typeof selectorOrCoordinates === "string") {
const coords = await page.evaluate((selector) => {
const element = document.querySelector(selector);
if (!element) return null;
const rect = element.getBoundingClientRect();
return {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2
};
}, selectorOrCoordinates);
if (coords) {
await updateCursorAtPosition(page, sessionId, coords.x, coords.y);
}
await page.click(selectorOrCoordinates);
} else {
const { x, y } = selectorOrCoordinates;
await updateCursorAtPosition(page, sessionId, x, y);
await page.mouse.click(x, y);
}
return {
success: true
};
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error during click operation"
};
}
}
async function type2(sessionId, text) {
const browserService = BrowserService2.g