playwright-selector-finder
Version:
A tool for finding and interacting with elements using Playwright's vision locators
168 lines (167 loc) • 7.36 kB
JavaScript
"use strict";
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getSelector = exports.type = exports.hover = exports.drag = exports.click = exports.snapshot = void 0;
const zod_1 = require("zod");
const zod_to_json_schema_1 = __importDefault(require("zod-to-json-schema"));
const utils_1 = require("./utils");
exports.snapshot = {
schema: {
name: 'browser_snapshot',
description: 'Capture accessibility snapshot of the current page, this is better than screenshot',
inputSchema: (0, zod_to_json_schema_1.default)(zod_1.z.object({})),
},
handle: async (context) => {
return await (0, utils_1.captureAriaSnapshot)(await context.ensurePage());
},
};
const elementSchema = zod_1.z.object({
element: zod_1.z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
ref: zod_1.z.string().describe('Exact target element reference from the page snapshot'),
});
exports.click = {
schema: {
name: 'browser_click',
description: 'Perform click on a web page',
inputSchema: (0, zod_to_json_schema_1.default)(elementSchema),
},
handle: async (context, params) => {
const validatedParams = elementSchema.parse(params);
return (0, utils_1.runAndWait)(context, `"${validatedParams.element}" clicked`, page => refLocator(page, validatedParams.ref).click(), true);
},
};
const dragSchema = zod_1.z.object({
startElement: zod_1.z.string().describe('Human-readable source element description used to obtain the permission to interact with the element'),
startRef: zod_1.z.string().describe('Exact source element reference from the page snapshot'),
endElement: zod_1.z.string().describe('Human-readable target element description used to obtain the permission to interact with the element'),
endRef: zod_1.z.string().describe('Exact target element reference from the page snapshot'),
});
exports.drag = {
schema: {
name: 'browser_drag',
description: 'Perform drag and drop between two elements',
inputSchema: (0, zod_to_json_schema_1.default)(dragSchema),
},
handle: async (context, params) => {
const validatedParams = dragSchema.parse(params);
return (0, utils_1.runAndWait)(context, `Dragged "${validatedParams.startElement}" to "${validatedParams.endElement}"`, async (page) => {
const startLocator = refLocator(page, validatedParams.startRef);
const endLocator = refLocator(page, validatedParams.endRef);
await startLocator.dragTo(endLocator);
}, true);
},
};
exports.hover = {
schema: {
name: 'browser_hover',
description: 'Hover over element on page',
inputSchema: (0, zod_to_json_schema_1.default)(elementSchema),
},
handle: async (context, params) => {
const validatedParams = elementSchema.parse(params);
return (0, utils_1.runAndWait)(context, `Hovered over "${validatedParams.element}"`, page => refLocator(page, validatedParams.ref).hover(), true);
},
};
const typeSchema = elementSchema.extend({
text: zod_1.z.string().describe('Text to type into the element'),
submit: zod_1.z.boolean().describe('Whether to submit entered text (press Enter after)'),
});
exports.type = {
schema: {
name: 'browser_type',
description: 'Type text into editable element',
inputSchema: (0, zod_to_json_schema_1.default)(typeSchema),
},
handle: async (context, params) => {
const validatedParams = typeSchema.parse(params);
return await (0, utils_1.runAndWait)(context, `Typed "${validatedParams.text}" into "${validatedParams.element}"`, async (page) => {
const locator = refLocator(page, validatedParams.ref);
await locator.fill(validatedParams.text);
if (validatedParams.submit)
await locator.press('Enter');
}, true);
},
};
function refLocator(page, ref) {
return page.locator(`aria-ref=${ref}`);
}
// New schema for get_selector tool
const getSelectorSchema = zod_1.z.object({
prompt: zod_1.z.string().describe('Natural language description of the element to find (e.g., "login button", "email input field")'),
});
// New tool to get selector from natural language
exports.getSelector = {
schema: {
name: 'get_selector',
description: 'Get selector for an element based on natural language description',
inputSchema: (0, zod_to_json_schema_1.default)(getSelectorSchema),
},
handle: async (context, params) => {
const validatedParams = getSelectorSchema.parse(params);
const page = await context.ensurePage();
// Get the accessibility snapshot
const snapshot = await page.accessibility.snapshot();
if (!snapshot) {
return {
content: [{ type: 'text', text: 'No elements found matching the description.' }],
isError: true,
};
}
// Function to find the most relevant node based on the prompt
const findRelevantNode = (node, prompt) => {
// Convert prompt to lowercase for case-insensitive matching
const promptLower = prompt.toLowerCase();
// Check if current node matches
const name = (node.name || '').toLowerCase();
const role = (node.role || '').toLowerCase();
if (name.includes(promptLower) || role.includes(promptLower)) {
return node;
}
// Check children
if (node.children) {
for (const child of node.children) {
const found = findRelevantNode(child, prompt);
if (found)
return found;
}
}
return null;
};
// Find the relevant node
const relevantNode = findRelevantNode(snapshot, validatedParams.prompt);
if (!relevantNode) {
return {
content: [{ type: 'text', text: 'No matching element found for: ' + validatedParams.prompt }],
isError: true,
};
}
// Return the selector
return {
content: [{
type: 'text',
text: JSON.stringify({
selector: `aria-ref=${relevantNode.ref}`,
description: relevantNode.name || relevantNode.role,
role: relevantNode.role
})
}],
};
},
};