donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
352 lines • 14.6 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AuditTool = exports.AuditGptSchema = exports.AuditCoreSchema = void 0;
exports.runAudit = runAudit;
const playwright_1 = __importDefault(require("@axe-core/playwright"));
const v4_1 = require("zod/v4");
const originalGotoRegistry_1 = require("../lib/page/originalGotoRegistry");
const ToolSchema_1 = require("../models/ToolSchema");
const Logger_1 = require("../utils/Logger");
const TargetUtils_1 = require("../utils/TargetUtils");
const AssertTool_1 = require("./AssertTool");
const Tool_1 = require("./Tool");
// ---------------------------------------------------------------------------
// Schemas
// ---------------------------------------------------------------------------
// Each check is a flat object with an `enabled` field (default true). When
// `enabled` is false the remaining fields are ignored. This avoids Zod union
// schemas which don't serialize reliably through all tool-call paths.
exports.AuditCoreSchema = v4_1.z.object({
url: v4_1.z
.string()
.optional()
.describe('URL to navigate to before running checks. Omit to audit the current page.'),
pageLoad: v4_1.z
.object({
enabled: v4_1.z
.boolean()
.optional()
.describe('Whether to run this check. Defaults to true.'),
assertion: v4_1.z
.string()
.optional()
.describe('Custom assertion describing what "fully loaded" means for this page.'),
retries: v4_1.z
.number()
.optional()
.describe('Number of retry attempts. Defaults to 3.'),
retryDelaySeconds: v4_1.z
.number()
.optional()
.describe('Seconds between retries. Defaults to 5.'),
waitUntil: v4_1.z
.enum(['load', 'domcontentloaded', 'networkidle', 'commit'])
.optional()
.describe('When to consider navigation complete. Defaults to load.'),
})
.optional()
.describe('Page load check — verifies the page has fully loaded with no spinners or skeleton screens.'),
accessibility: v4_1.z
.object({
enabled: v4_1.z
.boolean()
.optional()
.describe('Whether to run this check. Defaults to true.'),
})
.optional()
.describe('Accessibility check — runs axe-core.'),
uniqueIds: v4_1.z
.object({
enabled: v4_1.z
.boolean()
.optional()
.describe('Whether to run this check. Defaults to true.'),
})
.optional()
.describe('Checks that all DOM id attributes are unique.'),
uniqueTestIds: v4_1.z
.object({
enabled: v4_1.z
.boolean()
.optional()
.describe('Whether to run this check. Defaults to true.'),
attributes: v4_1.z
.array(v4_1.z.string())
.optional()
.describe('Attributes to check. Defaults to data-testid, data-test-id, data-test, data-cy, data-qa.'),
})
.optional()
.describe('Checks that test ID attributes (data-testid and common variants) are unique.'),
consoleErrors: v4_1.z
.object({
enabled: v4_1.z
.boolean()
.optional()
.describe('Whether to run this check. Defaults to true.'),
ignore: v4_1.z
.array(v4_1.z.string())
.optional()
.describe('Regex patterns — errors matching any pattern are excluded.'),
})
.optional()
.describe('Flags JavaScript console errors and uncaught exceptions captured during page load.'),
networkErrors: v4_1.z
.object({
enabled: v4_1.z
.boolean()
.optional()
.describe('Whether to run this check. Defaults to true.'),
ignore: v4_1.z
.array(v4_1.z.string())
.optional()
.describe('URL regex patterns — matching requests are excluded.'),
})
.optional()
.describe('Flags failed network requests (4xx/5xx responses, request failures).'),
});
exports.AuditGptSchema = v4_1.z.object({
...ToolSchema_1.BaseGptArgsSchema.shape,
...exports.AuditCoreSchema.shape,
});
// ---------------------------------------------------------------------------
// Tool
// ---------------------------------------------------------------------------
const IMPACT_RANK = {
minor: 0,
moderate: 1,
serious: 2,
critical: 3,
};
class AuditTool extends Tool_1.Tool {
constructor() {
super(AuditTool.NAME, 'Run a comprehensive audit of the current webpage covering accessibility, duplicate IDs, console errors, and network failures.', exports.AuditCoreSchema, exports.AuditGptSchema, false, undefined, ['web']);
}
async call(context, parameters) {
const report = await runAudit(context, parameters);
// Build a human-readable summary for the LLM / report timeline.
const sections = [];
if (!report.pageLoad.passed) {
sections.push(`Page load: FAILED — ${report.pageLoad.error}`);
}
if (!report.accessibility.passed) {
sections.push(`Accessibility: ${report.accessibility.violations.length} critical violation(s)`);
}
if (!report.uniqueIds.passed) {
sections.push(`Duplicate IDs: ${report.uniqueIds.duplicates.map((d) => d.id).join(', ')}`);
}
if (!report.uniqueTestIds.passed) {
sections.push(`Duplicate test IDs: ${report.uniqueTestIds.duplicates.map((d) => `${d.attribute}="${d.value}"`).join(', ')}`);
}
if (!report.consoleErrors.passed) {
sections.push(`Console errors: ${report.consoleErrors.errors.length} error(s)`);
}
if (!report.networkErrors.passed) {
sections.push(`Network errors: ${report.networkErrors.errors.length} failure(s)`);
}
const forLlm = report.passed
? 'Audit passed — all checks clean.'
: `Audit failed:\n${sections.join('\n')}`;
// Always isSuccessful: true — the tool executed correctly regardless of
// whether the page passed the audit. report.passed carries the audit outcome.
return {
isSuccessful: true,
forLlm,
metadata: report,
};
}
async callFromGpt(context, parameters) {
return this.call(context, parameters);
}
}
exports.AuditTool = AuditTool;
AuditTool.NAME = 'audit';
// ---------------------------------------------------------------------------
// Core audit logic (extracted so it can be tested independently)
// ---------------------------------------------------------------------------
async function runAudit(context, params) {
const page = (0, TargetUtils_1.webPage)(context);
const startedAt = Date.now();
// Snapshot the log buffer length so we only consider entries added
// *during* this audit.
const store = Logger_1.loggingContext.getStore();
const logEntriesBefore = store?.logBuffer?.snapshot().entries.length ?? 0;
const isEnabled = (check) => check?.enabled !== false;
// Navigate or reload so that console/network events from the initial
// page load land in the log buffer within this audit's diff window.
// Use the original (un-enhanced) goto to avoid recording a nested
// goToWebpage tool call inside the audit.
const waitUntil = params.pageLoad?.waitUntil ?? 'load';
if (params.url) {
const goto = (0, originalGotoRegistry_1.hasOriginalGoto)(page)
? (0, originalGotoRegistry_1.getOriginalGoto)(page).bind(page)
: page.goto.bind(page);
await goto(params.url, { waitUntil });
}
else {
await page.reload({ waitUntil });
}
// Capture the URL right after navigation (including any redirects)
// rather than at the end when subsequent checks may have changed it.
const auditedPageUrl = page.url();
// 1. Page load check — delegates to AssertTool for AI-powered evaluation.
let pageLoadResult = { passed: true };
if (isEnabled(params.pageLoad)) {
const assertion = params.pageLoad?.assertion ??
'The page has fully loaded — there are no loading spinners, skeleton screens, or "loading" indicators visible, and meaningful content is displayed.';
const assertResult = await new AssertTool_1.AssertTool().call(context, {
assertionToTestFor: assertion,
retries: params.pageLoad?.retries ?? 3,
retryWaitSeconds: params.pageLoad?.retryDelaySeconds ?? 5,
});
if (!assertResult.isSuccessful) {
pageLoadResult = {
passed: false,
error: assertResult.forLlm,
};
}
}
// 2. Accessibility — run axe-core and filter by impact threshold.
let accessibilityResult = {
passed: true,
violations: [],
};
if (isEnabled(params.accessibility)) {
const minRank = IMPACT_RANK['critical'];
const axe = new playwright_1.default({ page });
const axeResult = await axe.analyze();
const fullResult = {
violations: axeResult.violations,
incomplete: axeResult.incomplete,
passCount: axeResult.passes.length,
ignoredRuleCount: axeResult.inapplicable.length,
totalRuleCount: axeResult.violations.length +
axeResult.incomplete.length +
axeResult.passes.length +
axeResult.inapplicable.length,
};
const filtered = fullResult.violations.filter((v) => IMPACT_RANK[v.impact ?? 'minor'] >= minRank);
accessibilityResult = {
passed: filtered.length === 0,
violations: filtered,
};
}
// 3. Unique id attributes.
let uniqueIdsResult = {
passed: true,
duplicates: [],
};
if (isEnabled(params.uniqueIds)) {
const duplicateIds = await page.evaluate(() => {
const counts = {};
document.querySelectorAll('[id]').forEach((el) => {
const id = el.id;
if (id) {
counts[id] = (counts[id] || 0) + 1;
}
});
return Object.entries(counts)
.filter(([, count]) => count > 1)
.map(([id, count]) => ({ id, count }));
});
uniqueIdsResult = {
passed: duplicateIds.length === 0,
duplicates: duplicateIds,
};
}
// 4. Unique data-testid (and common variants) attributes.
let uniqueTestIdsResult = {
passed: true,
duplicates: [],
};
if (isEnabled(params.uniqueTestIds)) {
const testIdAttributes = params.uniqueTestIds?.attributes ?? [
'data-testid',
'data-test-id',
'data-test',
'data-cy',
'data-qa',
];
const duplicateTestIds = await page.evaluate((attrs) => {
const counts = {};
for (const attr of attrs) {
counts[attr] = {};
document.querySelectorAll(`[${attr}]`).forEach((el) => {
const value = el.getAttribute(attr);
if (value) {
counts[attr][value] = (counts[attr][value] || 0) + 1;
}
});
}
return Object.entries(counts).flatMap(([attribute, valueCounts]) => Object.entries(valueCounts)
.filter(([, count]) => count > 1)
.map(([value, count]) => ({ attribute, value, count })));
}, testIdAttributes);
uniqueTestIdsResult = {
passed: duplicateTestIds.length === 0,
duplicates: duplicateTestIds,
};
}
// 5. Console errors — diff the log buffer from before the audit started.
let consoleErrorsResult = {
passed: true,
errors: [],
};
if (isEnabled(params.consoleErrors)) {
const ignorePatterns = (params.consoleErrors?.ignore ?? []).map((s) => new RegExp(s));
const allEntries = store?.logBuffer?.snapshot().entries ?? [];
const errors = allEntries
.slice(logEntriesBefore)
.filter((e) => e.source === 'browser' && e.level === 'error')
.map((e) => ({
message: e.message,
source: e.url ?? undefined,
}))
.filter((e) => !ignorePatterns.some((p) => p.test(e.message)));
consoleErrorsResult = { passed: errors.length === 0, errors };
}
// 6. Network errors — diff the log buffer from before the audit started.
let networkErrorsResult = {
passed: true,
errors: [],
};
if (isEnabled(params.networkErrors)) {
const ignorePatterns = (params.networkErrors?.ignore ?? []).map((s) => new RegExp(s));
const allEntries = store?.logBuffer?.snapshot().entries ?? [];
const errors = allEntries
.slice(logEntriesBefore)
.filter((e) => e.source === 'network' && (e.level === 'error' || e.level === 'warn'))
.map((e) => ({
url: e.url ?? e.message,
method: e.method ?? undefined,
statusCode: e.status ?? undefined,
failureReason: e.failureReason ?? undefined,
}))
.filter((e) => !ignorePatterns.some((p) => p.test(e.url)));
networkErrorsResult = { passed: errors.length === 0, errors };
}
const logEntriesAfter = store?.logBuffer?.snapshot().entries.length ?? 0;
return {
passed: pageLoadResult.passed &&
accessibilityResult.passed &&
uniqueIdsResult.passed &&
uniqueTestIdsResult.passed &&
consoleErrorsResult.passed &&
networkErrorsResult.passed,
page: auditedPageUrl,
startedAt,
completedAt: Date.now(),
logStartIndex: logEntriesBefore,
logEndIndex: logEntriesAfter,
options: params,
pageLoad: pageLoadResult,
accessibility: accessibilityResult,
uniqueIds: uniqueIdsResult,
uniqueTestIds: uniqueTestIdsResult,
consoleErrors: consoleErrorsResult,
networkErrors: networkErrorsResult,
};
}
//# sourceMappingURL=AuditTool.js.map