donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
1,334 lines (1,310 loc) • 80.7 kB
JavaScript
#!/usr/bin/env node
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports._forTesting = void 0;
/**
* @fileoverview Donobu Test Runner CLI shim.
*
* Extends the Playwright CLI with Donobu-specific hooks. In addition to proxying
* `npx playwright test`, the shim orchestrates failure triage by collecting
* evidence during the test run and generating treatment plans once execution
* completes.
*
* Current behavior:
* - Accepts the `test` sub-command (default when omitted) plus any Playwright flags.
* - Passes Donobu-specific flags `--no-triage`, `--triage-output-dir`, `--clear-ai-cache`,
* and `--auto-heal`.
* - Supports `npx donobu heal --plan <file>` for applying previously generated treatment plans.
* - Captures failure evidence into a run-scoped directory and, when possible,
* generates treatment plans after Playwright exits.
* - Invokes Playwright via `npx playwright ...`, streaming stdout/stderr directly.
* - Propagates exit codes and termination signals so CI pipelines behave like the
* upstream Playwright CLI.
*/
const child_process_1 = require("child_process");
const crypto_1 = require("crypto");
const fs_1 = require("fs");
const os = __importStar(require("os"));
const path = __importStar(require("path"));
const v4_1 = require("zod/v4");
const envVars_1 = require("../envVars");
const gptClients_1 = require("../lib/test/fixtures/gptClients");
const donobuTestStack_1 = require("../lib/test/utils/donobuTestStack");
const triageTestFailure_1 = require("../lib/test/utils/triageTestFailure");
const merge_1 = require("../reporter/merge");
const model_1 = require("../reporter/model");
const render_1 = require("../reporter/render");
const renderMarkdown_1 = require("../reporter/renderMarkdown");
const renderPullRequestBody_1 = require("../reporter/renderPullRequestBody");
const renderSlack_1 = require("../reporter/renderSlack");
const slack_1 = require("../reporter/slack");
const Logger_1 = require("../utils/Logger");
const FAILURE_EVIDENCE_PREFIX = 'failure-evidence-';
const TREATMENT_PLAN_PREFIX = 'treatment-plan-';
const PLAYWRIGHT_JSON_REPORT_FILENAME = 'report.json';
/**
* Execute `npx playwright` while wiring Donobu-specific environment controls.
* Streams stdout/stderr to the current process so our CLI mirrors the native
* Playwright experience, and forwards termination signals to keep CI parity.
*/
async function runPlaywright(args, envOverrides = {}) {
Logger_1.appLogger.debug(`Running Playwright with args: ${JSON.stringify(args)}`);
if (Object.keys(envOverrides).length > 0) {
Logger_1.appLogger.debug(`Playwright env overrides: ${JSON.stringify(envOverrides, null, 2)}`);
}
return new Promise((resolve, reject) => {
const childEnv = { ...process.env, ...envOverrides };
const child = (0, child_process_1.spawn)('npx', ['playwright', ...args], {
stdio: 'inherit',
shell: process.platform === 'win32',
env: childEnv,
});
child.once('error', (error) => {
reject(error);
});
child.once('close', (code, signal) => {
if (signal) {
process.kill(process.pid, signal);
return;
}
resolve(code ?? 0);
});
});
}
/**
* Peel off Donobu-managed flags from the CLI invocation while keeping the rest
* untouched so they can be forwarded directly to Playwright.
*
* Supports both `--flag value` and `--flag=value` syntaxes because folks often
* copy commands from CI logs where spacing varies.
*/
function parseDonobuArgs(args) {
const passthroughArgs = [];
let triageEnabled = true;
let triageOutputDir;
let clearAiCache = false;
let autoHeal = false;
let passthroughMode = false;
for (let i = 0; i < args.length; i += 1) {
const arg = args[i];
if (passthroughMode) {
passthroughArgs.push(arg);
continue;
}
if (arg === '--') {
passthroughArgs.push(arg);
passthroughMode = true;
continue;
}
if (arg === '--no-triage') {
triageEnabled = false;
continue;
}
if (arg === '--clear-ai-cache') {
clearAiCache = true;
continue;
}
if (arg === '--auto-heal') {
autoHeal = true;
continue;
}
if (arg === '--triage-output-dir') {
const value = args[i + 1];
if (value) {
triageOutputDir = value;
i += 1;
}
else {
Logger_1.appLogger.warn('Missing value for --triage-output-dir; ignoring flag.');
}
continue;
}
if (arg.startsWith('--triage-output-dir=')) {
const [, value] = arg.split('=', 2);
if (value) {
triageOutputDir = value;
}
else {
Logger_1.appLogger.warn('Missing value for --triage-output-dir; ignoring flag.');
}
continue;
}
passthroughArgs.push(arg);
}
return {
passthroughArgs,
options: {
triageEnabled,
triageOutputDir,
clearAiCache,
autoHeal,
},
};
}
/**
* Playwright expects the first argument to be a sub-command like `test`. Users
* can omit it when using Donobu, so we normalize the argv before proxying.
*/
function normalizePlaywrightArgs(args) {
if (args.length === 0) {
return ['test'];
}
const [first, ...rest] = args;
if (first === 'test') {
return ['test', ...rest];
}
if (first.startsWith('-')) {
return ['test', first, ...rest];
}
Logger_1.appLogger.error(`Unsupported command "${first}". Expected "test".`);
process.exit(1);
}
function parseTestCommandArgs(rawArgs) {
/**
* `npx donobu test ...` and `npx donobu ...` both route here. We strip off the
* optional explicit `test` token, parse Donobu flags, then rebuild the argv
* we intend to hand to Playwright.
*/
let args = rawArgs;
let explicitTestCommand = false;
if (args[0] === 'test') {
explicitTestCommand = true;
args = args.slice(1);
}
const { passthroughArgs, options } = parseDonobuArgs(args);
const normalized = normalizePlaywrightArgs(explicitTestCommand ? ['test', ...passthroughArgs] : passthroughArgs);
return {
options,
playwrightArgs: normalized,
};
}
/**
* Playwright writes all artifacts under `--output`. If the user does not supply
* it we default to `<repo>/test-results`, matching Playwright's behaviour.
*/
function resolvePlaywrightOutputDir(playwrightArgs) {
let outputDir;
for (let i = 0; i < playwrightArgs.length; i += 1) {
const arg = playwrightArgs[i];
if (arg === '--output' || arg === '-o') {
outputDir = playwrightArgs[i + 1];
i += 1;
continue;
}
if (arg.startsWith('--output=')) {
outputDir = arg.slice('--output='.length);
}
}
if (!outputDir) {
return path.resolve(process.cwd(), 'test-results');
}
return path.isAbsolute(outputDir)
? outputDir
: path.resolve(process.cwd(), outputDir);
}
function buildTriageRunId() {
const isoSafe = new Date().toISOString().replace(/[:.]/g, '-');
return `${isoSafe}-${(0, crypto_1.randomUUID)().slice(0, 8)}`;
}
/**
* Prepare the folder structure Donobu expects for collecting evidence during a
* Playwright run. This is invoked for both regular test runs and heal retries.
*/
async function prepareTriageContext(playwrightOutputDir, options) {
const outputBaseDir = options.triageOutputDir
? path.resolve(options.triageOutputDir)
: path.join(playwrightOutputDir, 'donobu-triage');
const runId = buildTriageRunId();
const runDir = path.join(outputBaseDir, runId);
await fs_1.promises.mkdir(runDir, { recursive: true });
return { runId, runDir, outputBaseDir };
}
/**
* Translate runtime flags into environment variables consumed by lower layers.
* Using env vars keeps the boundaries loose: the Playwright test harness can
* detect triage/auto-heal mode without taking a dependency on this CLI.
*/
function createRunEnvOverrides(params) {
const envOverrides = {};
if (params.clearAiCache) {
envOverrides.DONOBU_PAGE_AI_CLEAR_CACHE = '1';
}
if (params.triageEnabled && params.triageContext) {
envOverrides.DONOBU_TRIAGE_RUN_DIR = params.triageContext.runDir;
envOverrides.DONOBU_TRIAGE_RUN_ID = params.triageContext.runId;
envOverrides.DONOBU_TRIAGE_OUTPUT_BASE_DIR =
params.triageContext.outputBaseDir;
}
else {
envOverrides.DONOBU_TRIAGE_DISABLED = '1';
}
return envOverrides;
}
async function ensureDirectory(dirPath) {
await fs_1.promises.mkdir(dirPath, { recursive: true });
}
/**
* Allocate a temporary sandbox for the auto-heal rerun. All transient
* Playwright output and triage artifacts live here until we copy the pieces we
* want to keep back into the workspace.
*/
async function createAutoHealStagingArea() {
const rootDir = await fs_1.promises.mkdtemp(path.join(os.tmpdir(), 'donobu-auto-heal-'));
const playwrightOutputDir = path.join(rootDir, 'playwright-output');
const triageBaseDir = path.join(rootDir, 'triage');
await ensureDirectory(playwrightOutputDir);
await ensureDirectory(triageBaseDir);
const dispose = async () => {
try {
await fs_1.promises.rm(rootDir, { recursive: true, force: true });
}
catch (error) {
Logger_1.appLogger.warn(`Failed to clean up auto-heal staging directory ${rootDir}.`, error);
}
};
return {
rootDir,
playwrightOutputDir,
triageBaseDir,
dispose,
};
}
const noopAsync = async () => { };
/**
* Ensure Playwright receives a JSON reporter without clobbering reporters
* defined via CLI flags or the project's Playwright config. Creates a temporary
* wrapper config when necessary so we can append the JSON reporter alongside
* user-specified reporters.
*/
async function ensureJsonReporter(originalArgs, options) {
const args = [...originalArgs];
// For branches that don't use the config wrapper, the JSON reporter has no
// explicit `outputFile` and Playwright falls back to env vars — which we set
// in applyJsonReportEnv to (playwrightOutputDir, PLAYWRIGHT_JSON_REPORT_FILENAME).
// Either honour an explicit override or reconstruct that path here.
const envDerivedJsonOutputFile = options.jsonOutputFile ??
path.join(options.playwrightOutputDir, PLAYWRIGHT_JSON_REPORT_FILENAME);
const argInjection = injectJsonReporterIntoArgs(args);
if (argInjection.reporterFlagFound) {
return {
args,
cleanup: noopAsync,
resolveJsonReporterInfo: async () => ({
userHadJson: argInjection.userHadJson,
jsonOutputFile: envDerivedJsonOutputFile,
}),
};
}
const configPath = await resolvePlaywrightConfigPath(args);
if (!configPath) {
if (!hasReporterArg(args)) {
// Only inject the reporter name; the output file path is handled via the
// PLAYWRIGHT_JSON_OUTPUT_NAME / PLAYWRIGHT_JSON_OUTPUT_DIR env vars set by
// applyJsonReportEnv(). Using `json=/path` here would cause Playwright to
// treat the whole value as a custom reporter module path (MODULE_NOT_FOUND).
args.push('--reporter=json');
}
Logger_1.appLogger.debug('No Playwright config detected; falling back to CLI --reporter=json injection.');
return {
args,
cleanup: noopAsync,
resolveJsonReporterInfo: async () => ({
userHadJson: false,
jsonOutputFile: envDerivedJsonOutputFile,
}),
};
}
const wrapper = await createConfigWrapperWithJsonReporter(configPath, {
jsonOutputFile: options.jsonOutputFile,
});
Logger_1.appLogger.debug(`Augmenting Playwright config at ${configPath} with temporary wrapper ${wrapper.configPath} to ensure JSON reporter.`);
const strippedArgs = stripConfigArgs(args);
const finalArgs = insertConfigArg(strippedArgs, wrapper.configPath);
return {
args: finalArgs,
cleanup: wrapper.cleanup,
resolveJsonReporterInfo: wrapper.resolveJsonReporterInfo,
};
}
function hasReporterArg(args) {
for (const arg of args) {
if (arg === '--') {
break;
}
if (arg === '--reporter' ||
arg === '-r' ||
arg.startsWith('--reporter=') ||
arg.startsWith('-r=')) {
return true;
}
}
return false;
}
function injectJsonReporterIntoArgs(args) {
let reporterFlagFound = false;
let userHadJson = false;
for (let i = 0; i < args.length; i += 1) {
const arg = args[i];
if (arg === '--') {
break;
}
if (arg === '--reporter' || arg === '-r') {
reporterFlagFound = true;
const valueIndex = i + 1;
if (valueIndex < args.length) {
const { value, changed } = ensureReporterValueHasJson(args[valueIndex]);
args[valueIndex] = value;
if (!changed) {
userHadJson = true;
}
}
i += 1;
continue;
}
if (arg.startsWith('--reporter=') || arg.startsWith('-r=')) {
reporterFlagFound = true;
const [prefix, rawValue] = arg.split('=', 2);
const { value, changed } = ensureReporterValueHasJson(rawValue ?? '');
args[i] = `${prefix}=${value}`;
if (!changed) {
userHadJson = true;
}
}
}
return { reporterFlagFound, userHadJson };
}
function ensureReporterValueHasJson(value) {
const segments = value
.split(',')
.map((segment) => segment.trim())
.filter((segment) => segment.length > 0);
const hasJson = segments.some((segment) => segment.split('=')[0].trim() === 'json');
if (hasJson) {
return { value, changed: false };
}
// Always use plain 'json' — the output file path is controlled by env vars
// (PLAYWRIGHT_JSON_OUTPUT_NAME / PLAYWRIGHT_JSON_OUTPUT_DIR). Appending
// `json=/path` would make Playwright try to require() it as a module.
segments.push('json');
return { value: segments.join(','), changed: true };
}
async function resolvePlaywrightConfigPath(args) {
const fromArgs = extractConfigPathFromArgs(args);
if (fromArgs) {
return path.isAbsolute(fromArgs)
? fromArgs
: path.resolve(process.cwd(), fromArgs);
}
const candidates = [
'playwright.config.ts',
'playwright.config.mts',
'playwright.config.cts',
'playwright.config.js',
'playwright.config.mjs',
'playwright.config.cjs',
];
for (const candidate of candidates) {
try {
const resolved = path.resolve(process.cwd(), candidate);
await fs_1.promises.access(resolved, fs_1.constants.F_OK);
Logger_1.appLogger.debug(`Detected Playwright config candidate at ${resolved} for JSON reporter injection.`);
return resolved;
}
catch {
// continue scanning other candidates
}
}
return null;
}
function extractConfigPathFromArgs(args) {
for (let i = 0; i < args.length; i += 1) {
const arg = args[i];
if (arg === '--') {
break;
}
if (arg === '--config' || arg === '-c') {
const value = args[i + 1];
return value ?? null;
}
if (arg.startsWith('--config=') || arg.startsWith('-c=')) {
return arg.split('=', 2)[1] ?? null;
}
}
return null;
}
function stripConfigArgs(args) {
const result = [];
for (let i = 0; i < args.length; i += 1) {
const arg = args[i];
if (arg === '--') {
result.push(...args.slice(i));
break;
}
if (arg === '--config' || arg === '-c') {
i += 1;
continue;
}
if (arg.startsWith('--config=') || arg.startsWith('-c=')) {
continue;
}
result.push(arg);
}
return result;
}
function insertConfigArg(args, configPath) {
const dashDashIndex = args.indexOf('--');
if (dashDashIndex === -1) {
return [...args, '--config', configPath];
}
return [
...args.slice(0, dashDashIndex),
'--config',
configPath,
...args.slice(dashDashIndex),
];
}
async function createConfigWrapperWithJsonReporter(originalConfigPath, options = {}) {
const stagingDir = await fs_1.promises.mkdtemp(path.join(os.tmpdir(), 'donobu-playwright-config-'));
const wrapperPath = path.join(stagingDir, 'playwright.config.cjs');
const sentinelPath = path.join(stagingDir, '.donobu-user-had-json');
const content = buildConfigWrapperContent(originalConfigPath, sentinelPath, options.jsonOutputFile);
await fs_1.promises.writeFile(wrapperPath, content, 'utf-8');
const cleanup = async () => {
try {
await fs_1.promises.rm(stagingDir, { recursive: true, force: true });
}
catch (error) {
Logger_1.appLogger.warn(`Failed to remove temporary Playwright config at ${stagingDir}.`, error);
}
};
// The wrapper writes the sentinel inside its top-level config-load code, so
// it exists by the time Playwright has loaded the config. Read it after the
// wrapped run finishes and before `cleanup` tears the staging dir down.
const resolveJsonReporterInfo = async () => {
try {
const raw = await fs_1.promises.readFile(sentinelPath, 'utf8');
const parsed = JSON.parse(raw);
return {
userHadJson: parsed?.userHadJson === true,
jsonOutputFile: typeof parsed?.jsonOutputFile === 'string' &&
parsed.jsonOutputFile.length > 0
? parsed.jsonOutputFile
: null,
};
}
catch {
// Sentinel missing or malformed (wrapper never loaded, or already
// cleaned up). Err on the side of "no user JSON" so we don't leave a
// giant artifact behind.
return { userHadJson: false, jsonOutputFile: null };
}
};
return { configPath: wrapperPath, cleanup, resolveJsonReporterInfo };
}
function buildConfigWrapperContent(originalConfigPath, userHadJsonSentinelPath, jsonOutputFileOverride) {
const sanitisedPath = originalConfigPath.replace(/\\/g, '\\\\');
const sentinelPath = userHadJsonSentinelPath.replace(/\\/g, '\\\\');
const forcedJsonPath = jsonOutputFileOverride
? jsonOutputFileOverride.replace(/\\/g, '\\\\')
: null;
const defaultJsonName = PLAYWRIGHT_JSON_REPORT_FILENAME;
const forcedLiteral = forcedJsonPath ? `'${forcedJsonPath}'` : 'null';
return `"use strict";
const path = require('path');
const forcedJsonOutputFile = ${forcedLiteral};
const userHadJsonSentinelPath = '${sentinelPath}';
function loadBaseConfig() {
const imported = require('${sanitisedPath}');
return imported && imported.__esModule ? imported.default : imported;
}
const configDir = path.dirname('${sanitisedPath}');
const originalCwd = process.cwd();
let baseConfig;
try {
process.chdir(configDir);
baseConfig = loadBaseConfig();
} finally {
process.chdir(originalCwd);
}
const normalizedConfig = { ...baseConfig };
function absolutify(value) {
if (typeof value !== 'string') {
return value;
}
if (path.isAbsolute(value)) {
return value;
}
return path.resolve(configDir, value);
}
if (normalizedConfig.testDir) {
normalizedConfig.testDir = absolutify(normalizedConfig.testDir);
} else {
normalizedConfig.testDir = configDir;
}
if (normalizedConfig.outputDir) {
normalizedConfig.outputDir = absolutify(normalizedConfig.outputDir);
}
if (normalizedConfig.snapshotDir) {
normalizedConfig.snapshotDir = absolutify(normalizedConfig.snapshotDir);
}
if (normalizedConfig.expect && typeof normalizedConfig.expect === 'object') {
const expectConfig = { ...normalizedConfig.expect };
if (expectConfig.outputDir) {
expectConfig.outputDir = absolutify(expectConfig.outputDir);
}
if (expectConfig.snapshotDir) {
expectConfig.snapshotDir = absolutify(expectConfig.snapshotDir);
}
normalizedConfig.expect = expectConfig;
}
if (Array.isArray(normalizedConfig.projects)) {
normalizedConfig.projects = normalizedConfig.projects.map((project) => {
const projectConfig = { ...project };
if (projectConfig.testDir) {
projectConfig.testDir = absolutify(projectConfig.testDir);
}
if (projectConfig.outputDir) {
projectConfig.outputDir = absolutify(projectConfig.outputDir);
}
if (projectConfig.snapshotDir) {
projectConfig.snapshotDir = absolutify(projectConfig.snapshotDir);
}
if (
projectConfig.use &&
typeof projectConfig.use === 'object' &&
projectConfig.use !== null
) {
const useConfig = { ...projectConfig.use };
if (typeof useConfig.storageState === 'string') {
useConfig.storageState = absolutify(useConfig.storageState);
}
if (
typeof useConfig.baseURL === 'string' &&
!/^https?:/i.test(useConfig.baseURL)
) {
useConfig.baseURL = absolutify(useConfig.baseURL);
}
projectConfig.use = useConfig;
}
return projectConfig;
});
}
const reporters = Array.isArray(normalizedConfig.reporter)
? normalizedConfig.reporter.map((entry) =>
Array.isArray(entry) ? [...entry] : entry,
)
: normalizedConfig.reporter
? [normalizedConfig.reporter]
: [];
function computeEnvDerivedJsonOutputFile() {
const outputDir = process.env.PLAYWRIGHT_JSON_OUTPUT_DIR
? path.resolve(process.cwd(), process.env.PLAYWRIGHT_JSON_OUTPUT_DIR)
: path.resolve(configDir, 'test-results');
const outputName =
process.env.PLAYWRIGHT_JSON_OUTPUT_NAME || '${defaultJsonName}';
return path.isAbsolute(outputName)
? outputName
: path.join(outputDir, outputName);
}
const hasJsonReporter = reporters.some((entry) => {
if (!entry) {
return false;
}
if (typeof entry === 'string') {
return entry
.split(',')
.map((segment) => segment.trim())
.filter((segment) => segment.length > 0)
.some((segment) => segment.split('=')[0] === 'json');
}
if (Array.isArray(entry) && entry.length > 0) {
const name = typeof entry[0] === 'string' ? entry[0] : '';
return name === 'json';
}
return false;
});
if (!hasJsonReporter) {
reporters.push([
'json',
{ outputFile: forcedJsonOutputFile || computeEnvDerivedJsonOutputFile() },
]);
}
// Resolve a reporter name to an absolute path so that Playwright can find it
// even though this shim lives in a temp directory with no node_modules.
// Built-in reporters and already-absolute/relative paths are passed through unchanged.
const _builtInReporters = new Set(['dot', 'line', 'list', 'junit', 'json', 'html', 'null', 'github', 'blob', 'markdown']);
function resolveReporterName(name) {
if (!name || _builtInReporters.has(name) || path.isAbsolute(name) || name.startsWith('.')) {
return name;
}
try {
return require.resolve(name, { paths: [configDir] });
} catch {
return name;
}
}
const normalisedReporters = reporters.map((entry) => {
if (typeof entry === 'string') {
if (!forcedJsonOutputFile) {
return entry;
}
const segments = entry
.split(',')
.map((segment) => segment.trim())
.filter((segment) => segment.length > 0);
const rewritten = segments.map((segment) => {
const [name] = segment.split('=', 2);
if (name === 'json') {
return \`json=\${forcedJsonOutputFile}\`;
}
return segment;
});
return rewritten.join(',');
}
if (
Array.isArray(entry) &&
entry.length > 1 &&
entry[1] &&
typeof entry[1] === 'object'
) {
const options = { ...entry[1] };
if (options.outputFile) {
options.outputFile = absolutify(options.outputFile);
}
if (options.outputFolder) {
options.outputFolder = absolutify(options.outputFolder);
}
if (entry[0] === 'json' && forcedJsonOutputFile) {
options.outputFile = forcedJsonOutputFile;
}
return [resolveReporterName(entry[0]), options];
}
if (Array.isArray(entry) && entry.length > 0) {
const name = typeof entry[0] === 'string' ? entry[0] : '';
if (name === 'json' && forcedJsonOutputFile) {
return [entry[0], { outputFile: forcedJsonOutputFile }];
}
return [resolveReporterName(entry[0]), ...entry.slice(1)];
}
return entry;
});
// Walk the final reporter list to surface the JSON output path the orchestrator
// should treat as the merge target — Playwright's JSON reporter accepts the
// path via either reporter options or env var, so checking both keeps us in
// sync with whatever Playwright will actually do.
function extractJsonOutputFile(rep) {
for (const entry of rep) {
if (Array.isArray(entry) && entry[0] === 'json') {
if (
entry[1] &&
typeof entry[1] === 'object' &&
typeof entry[1].outputFile === 'string'
) {
return entry[1].outputFile;
}
return computeEnvDerivedJsonOutputFile();
}
if (typeof entry === 'string') {
const segments = entry
.split(',')
.map((s) => s.trim())
.filter((s) => s.length > 0);
for (const segment of segments) {
const eqIndex = segment.indexOf('=');
const name = eqIndex === -1 ? segment : segment.slice(0, eqIndex);
const value = eqIndex === -1 ? '' : segment.slice(eqIndex + 1);
if (name === 'json') {
return value ? absolutify(value) : computeEnvDerivedJsonOutputFile();
}
}
}
}
return null;
}
// Tell the parent orchestrator (a) whether the user's own config defined a
// JSON reporter and (b) the exact path that JSON will land at. The path lets
// the orchestrator merge auto-heal results onto the user's file directly,
// without filesystem-scanning heuristics that can pick the wrong file.
try {
require('fs').writeFileSync(
userHadJsonSentinelPath,
JSON.stringify({
userHadJson: hasJsonReporter,
jsonOutputFile: extractJsonOutputFile(normalisedReporters),
}),
'utf8',
);
} catch (_) {
// Non-fatal — the orchestrator falls back to "no user JSON" if missing.
}
module.exports = {
...normalizedConfig,
reporter: normalisedReporters,
};
`;
}
/**
* Copy the canonical Playwright JSON report into a Donobu-controlled location.
* We keep a stable snapshot because Playwright may delete the file between
* retries or when the `--output` folder is cleaned.
*/
async function copyJsonReport(outputDir, destinationPath, options = {}) {
const candidatePaths = new Set();
const envDefinedPath = resolveEnvJsonReportPath(options.envOverrides);
if (envDefinedPath) {
candidatePaths.add(envDefinedPath);
}
candidatePaths.add(path.join(outputDir, PLAYWRIGHT_JSON_REPORT_FILENAME));
(options.additionalCandidates ?? []).forEach((candidate) => {
if (candidate) {
candidatePaths.add(candidate);
}
});
for (const sourcePath of candidatePaths) {
const copied = await tryCopyReport(sourcePath, destinationPath);
if (copied) {
return { sourcePath, destinationPath };
}
}
const fallbackSource = await findJsonReportInDir(outputDir);
if (fallbackSource) {
const copied = await tryCopyReport(fallbackSource, destinationPath);
if (copied) {
return { sourcePath: fallbackSource, destinationPath };
}
}
return null;
}
function resolveEnvJsonReportPath(envOverrides) {
if (!envOverrides) {
return null;
}
const outputDir = envOverrides.PLAYWRIGHT_JSON_OUTPUT_DIR;
const outputName = envOverrides.PLAYWRIGHT_JSON_OUTPUT_NAME;
if (!outputDir || !outputName) {
return null;
}
const resolvedDir = path.isAbsolute(outputDir)
? outputDir
: path.resolve(process.cwd(), outputDir);
return path.isAbsolute(outputName)
? outputName
: path.join(resolvedDir, outputName);
}
async function tryCopyReport(sourcePath, destinationPath) {
try {
await fs_1.promises.access(sourcePath, fs_1.constants.F_OK);
}
catch {
return false;
}
await ensureDirectory(path.dirname(destinationPath));
try {
await fs_1.promises.copyFile(sourcePath, destinationPath);
return true;
}
catch (error) {
Logger_1.appLogger.warn(`Failed to copy Playwright JSON report from ${sourcePath} to ${destinationPath}.`, error);
return false;
}
}
/** Files we write into the Playwright output dir ourselves. They happen to
* share `suites`-shaped JSON with Playwright's own report, so the scan
* fallback must exclude them or it can pick our own state file (or a
* previous heal-run copy) as the "user's JSON report". */
function isDonobuInternalJsonFile(fileName) {
return (fileName === model_1.DONOBU_REPORT_STATE_FILENAME ||
fileName.startsWith('donobu-auto-heal-report-') ||
fileName.startsWith('donobu-heal-report-') ||
fileName === 'donobu-heal-merged-report.json');
}
async function findJsonReportInDir(outputDir) {
let entries;
try {
entries = await fs_1.promises.readdir(outputDir);
}
catch {
return null;
}
const candidates = entries
.filter((entry) => entry.endsWith('.json'))
.filter((entry) => !isDonobuInternalJsonFile(entry))
.sort((a, b) => {
const aScore = a.includes('report') ? 0 : 1;
const bScore = b.includes('report') ? 0 : 1;
if (aScore !== bScore) {
return aScore - bScore;
}
return a.localeCompare(b);
});
for (const fileName of candidates) {
const fullPath = path.join(outputDir, fileName);
if (await isLikelyPlaywrightReport(fullPath)) {
return fullPath;
}
}
return null;
}
async function isLikelyPlaywrightReport(filePath) {
try {
const raw = await fs_1.promises.readFile(filePath, 'utf8');
const parsed = JSON.parse(raw);
return (!!parsed && typeof parsed === 'object' && Array.isArray(parsed.suites));
}
catch {
return false;
}
}
/**
* Remove the JSON artifacts Donobu wrote purely for its own use:
*
* - The Donobu reporter state file (`.donobu-report-state.json`) — internal
* IPC between the HTML/Markdown/Slack reporters and this orchestrator.
* Always safe to delete once orchestration finishes; nothing else reads it.
* - The Playwright JSON report — when the user did not configure a JSON
* reporter themselves, this file is purely Donobu's by-product (we forced
* the JSON reporter on to drive treatment plans and the merge step).
*
* Best-effort: missing files are ignored. Any failure is non-fatal — these
* cleanups don't affect test exit code.
*/
async function cleanupForceInjectedJsonArtifacts(params) {
const targets = [
path.join(params.playwrightOutputDir, model_1.DONOBU_REPORT_STATE_FILENAME),
];
if (!params.userHadJson && params.jsonOutputFile) {
targets.push(params.jsonOutputFile);
}
await Promise.all(targets.map(async (target) => {
try {
await fs_1.promises.rm(target, { force: true });
}
catch (error) {
Logger_1.appLogger.debug(`Failed to remove force-injected JSON artifact at ${target}: ${error}`);
}
}));
}
/**
* Donobu always wants Playwright's JSON reporter enabled so we can build
* treatment plans. If the user did not explicitly configure it we add the
* environment defaults for location and filename.
*/
function applyJsonReportEnv(env, outputDir) {
if (!env.PLAYWRIGHT_JSON_OUTPUT_DIR) {
env.PLAYWRIGHT_JSON_OUTPUT_DIR = outputDir;
}
if (!env.PLAYWRIGHT_JSON_OUTPUT_NAME) {
env.PLAYWRIGHT_JSON_OUTPUT_NAME = PLAYWRIGHT_JSON_REPORT_FILENAME;
}
}
/**
* Inspect generated treatment plans to determine whether an automated rerun is
* viable. A plan qualifies if it explicitly opted into automation directives.
*/
function evaluateAutoHealEligibility(plans) {
const eligiblePlans = plans.filter((record) => {
const directives = record.plan.automationDirectives;
return (record.plan.shouldRetryAutomation === true &&
directives !== undefined &&
Object.keys(directives).length > 0);
});
const clearPageAiCache = eligiblePlans.some((record) => record.plan.automationDirectives?.clearPageAiCache === true);
const directives = derivePlaywrightDirectiveArgs(eligiblePlans.map((record) => ({
plan: record.plan,
testCase: record.evidence.failureContext.testCase,
})));
return {
eligiblePlans,
clearPageAiCache,
directives,
};
}
/**
* Coalesce directives from one or more treatment plans into a single Playwright
* invocation. Multiple failed tests can be healed in a single rerun, so we
* gather all relevant files/projects/titles here.
*/
function derivePlaywrightDirectiveArgs(descriptors) {
const targetFiles = new Set();
const targetProjects = new Set();
const targetTitles = new Set();
const additionalArgs = [];
for (const descriptor of descriptors) {
const directives = descriptor.plan.automationDirectives;
if (!directives) {
continue;
}
const fileCandidate = normalizeSpecPath(directives.targetTestFile ?? descriptor.testCase.file);
if (fileCandidate) {
targetFiles.add(fileCandidate);
}
const projectCandidate = directives.targetProject ?? descriptor.testCase.projectName;
if (projectCandidate && !looksLikePath(projectCandidate)) {
targetProjects.add(projectCandidate);
}
if (descriptor.testCase.title) {
targetTitles.add(descriptor.testCase.title);
}
if (directives.additionalPlaywrightArgs) {
directives.additionalPlaywrightArgs.forEach((arg) => {
additionalArgs.push(arg);
});
}
}
return {
files: Array.from(targetFiles),
projects: Array.from(targetProjects),
grepPattern: targetTitles.size > 0
? Array.from(targetTitles)
.map((title) => escapeRegex(title))
.join('|')
: undefined,
extras: additionalArgs,
};
}
// We match test titles via `--grep`, so ensure literal characters are escaped.
function escapeRegex(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
// Some teams name projects after directories (e.g. `projects/mobile`); treat those as file paths.
function looksLikePath(value) {
return value.includes('/') || value.includes('\\');
}
/**
* Convert Playwright's test file references into something relative to the
* current workspace when possible. Relative paths make persisted plans easier
* to share between developers on different machines.
*/
function normalizeSpecPath(specPath) {
if (!specPath) {
return specPath;
}
const absolute = path.isAbsolute(specPath)
? specPath
: path.resolve(process.cwd(), specPath);
const relative = path.relative(process.cwd(), absolute);
return relative.startsWith('..') ? absolute : relative || specPath;
}
// The original CLI may have explicit spec files; keep them when no directive overrides exist.
function extractOriginalFiles(args) {
return args.slice(1).filter((arg) => !arg.startsWith('--') && arg !== 'test');
}
/**
* Preserve most user-provided Playwright flags (e.g. `--config`, `--workers`).
* We only strip flags we know we're going to replace (projects, grep, reporter).
*/
function extractPreservedOptions(args) {
return args.slice(1).filter((arg) => {
if (!arg.startsWith('--')) {
return false;
}
const optionName = arg.startsWith('--') ? arg.split('=')[0] : arg;
return (optionName !== '--project' &&
!optionName.startsWith('--project=') &&
optionName !== '--grep' &&
optionName !== '--reporter' &&
!optionName.startsWith('--grep=') &&
!optionName.startsWith('--reporter='));
});
}
/**
* Merge the user's original Playwright command with the automation directives
* suggested by treatment plans. Directives win over the original arguments but
* we keep the rest of the flags intact.
*/
function buildPlaywrightArgsWithDirectives(originalArgs, directives) {
const files = directives.files.length > 0
? directives.files
: extractOriginalFiles(originalArgs);
const preservedOptions = extractPreservedOptions(originalArgs);
const projectArgs = directives.projects.map((project) => `--project=${project}`);
const grepArgs = directives.grepPattern
? ['--grep', directives.grepPattern]
: [];
const finalArgs = [
'test',
...files,
...projectArgs,
...grepArgs,
...directives.extras,
...preservedOptions,
];
return finalArgs;
}
/**
* Force Playwright to emit artifacts into a specific directory, replacing any
* existing `--output` flag in the argv. Used by auto-heal so the rerun does not
* overwrite the original failure artifacts.
*/
function overrideOutputDir(args, outputDir) {
const rewritten = [];
let outputInjected = false;
for (let i = 0; i < args.length; i += 1) {
const arg = args[i];
if (arg === '--output' || arg === '-o') {
outputInjected = true;
rewritten.push('--output', outputDir);
i += 1; // Skip the value that followed the original flag.
continue;
}
if (arg.startsWith('--output=')) {
outputInjected = true;
rewritten.push(`--output=${outputDir}`);
continue;
}
rewritten.push(arg);
}
if (!outputInjected) {
rewritten.push('--output', outputDir);
}
return rewritten;
}
/**
* Treatment plans are persisted to disk between runs so engineers can apply
* them later. This schema guards against schema drift when the file is
* reloaded by the `heal` command.
*/
const PersistedTreatmentPlanFileSchema = v4_1.z.object({
generatedAtIso: v4_1.z.string(),
plan: triageTestFailure_1.TreatmentPlan,
failure: v4_1.z.object({
testCase: v4_1.z.object({
title: v4_1.z.string(),
file: v4_1.z.string().optional(),
projectName: v4_1.z.string().optional(),
}),
runId: v4_1.z.string().nullable().optional(),
runDirectory: v4_1.z.string().optional(),
evidencePath: v4_1.z.string(),
}),
originalPlaywrightArgs: v4_1.z.array(v4_1.z.string()).default([]),
reportPath: v4_1.z.string().optional(),
});
/**
* Read the evidence that Playwright (and Donobu's test fixtures) produced for a
* failing test. Evidence includes the Playwright context plus Donobu metadata.
*/
async function loadFailureEvidence(filePath) {
try {
const raw = await fs_1.promises.readFile(filePath, 'utf8');
return JSON.parse(raw);
}
catch (error) {
Logger_1.appLogger.error(`Failed to read test-failure evidence at ${filePath}`, error);
return null;
}
}
/**
* After Playwright exits we iterate over any captured failure evidence,
* generate treatment plans (via GPT when available, otherwise heuristics), and
* persist the results next to the evidence files. When a Donobu flow ID is
* available in the evidence, the plan is also saved to all registered Donobu
* persistence backends (e.g. SQLite, Donobu API) via the shared DonobuStack.
*
* This runs for the initial test pass and for later heal attempts.
*/
async function postProcessTriageRun(context, originalPlaywrightArgs, reportPath) {
const generatedPlans = [];
const originalArgsSnapshot = [...originalPlaywrightArgs];
let entries;
try {
entries = await fs_1.promises.readdir(context.runDir);
}
catch (error) {
Logger_1.appLogger.error(`Unable to read test-failure triage directory ${context.runDir}.`, error);
return generatedPlans;
}
const evidenceFiles = entries
.filter((entry) => entry.startsWith(FAILURE_EVIDENCE_PREFIX) && entry.endsWith('.json'))
.sort();
if (evidenceFiles.length === 0) {
Logger_1.appLogger.info(`No failure evidence found in ${context.runDir}.`);
return generatedPlans;
}
let gptClient = null;
try {
gptClient = await (0, gptClients_1.getOrCreateDefaultGptClient)();
}
catch (error) {
Logger_1.appLogger.warn('Unable to instantiate GPT client for treatment plans; will use heuristic fallback.', error);
}
for (const fileName of evidenceFiles) {
const evidencePath = path.join(context.runDir, fileName);
const evidence = await loadFailureEvidence(evidencePath);
if (!evidence) {
continue;
}
const testLabel = evidence.failureContext.testCase.title ??
evidence.failureContext.testCase.file ??
'unknown test';
if (evidence.failureContext.testCase.autoHealEnabled === false) {
Logger_1.appLogger.info(`Skipping treatment plan for "${testLabel}" — auto-heal is disabled for this test's project.`);
continue;
}
Logger_1.appLogger.info(`Detected test failure for "${testLabel}". Generating treatment plan to facilitate healing...`);
const heuristicFallback = () => {
const h = evidence.failureContext.heuristics;
return {
failureSummary: h.failureSummary,
failureReason: h.failureReason,
confidence: h.confidence,
observedIndicators: h.evidence,
remediationSteps: h.remediationSteps,
additionalDataRequests: h.additionalDataRequests,
shouldRetryAutomation: h.shouldRetryAutomation,
requiresCodeChange: h.requiresCodeChange,
requiresProductFix: h.requiresProductFix,
notes: h.notes,
};
};
let plan;
if (gptClient) {
try {
plan = await (0, triageTestFailure_1.generateTreatmentPlanFromEvidence)(gptClient, evidence);
}
catch (error) {
Logger_1.appLogger.warn(`GPT triage failed for ${fileName}; using heuristic fallback.`, error);
plan = heuristicFallback();
}
}
else {
plan = heuristicFallback();
}
if (plan.automationDirectives &&
Object.keys(plan.automationDirectives).length > 0) {
const enrichedDirectives = { ...plan.automationDirectives };
const targetFile = normalizeSpecPath(evidence.failureContext.testCase.file);
const targetProject = evidence.failureContext.testCase.projectName;
if (targetFile && !enrichedDirectives.targetTestFile) {
enrichedDirectives.targetTestFile = targetFile;
}
if (targetProject && !enrichedDirectives.targetProject) {
enrichedDirectives.targetProject = targetProject;
}
plan = {
...plan,
automationDirectives: enrichedDirectives,
};
}
const planFileName = fileName.replace(FAILURE_EVIDENCE_PREFIX, TREATMENT_PLAN_PREFIX);
const planPath = path.join(context.runDir, planFileName);
const persisted = {
generatedAtIso: new Date().toISOString(),
plan,
failure: {
testCase: evidence.failureContext.testCase,
runId: evidence.runId,
runDirectory: evidence.runDirectory,
evidencePath,
},
originalPlaywrightArgs: originalArgsSnapshot,
reportPath,
};
const persistedJson = JSON.stringify(persisted, null, 2);
await fs_1.promises.writeFile(planPath, persistedJson, 'utf8');
Logger_1.appLogger.info(`Saved test failure treatment plan for "${testLabel}" to "${planPath}"`);
const flowId = evidence.failureContext.donobuFlow?.metadata?.id;
if (flowId) {
try {
const donobu = await (0, donobuTestStack_1.getOrCreateDonobuStack)();
const persistence = await donobu.flowsPersistenceRegistry.get();
const planBuffer = Buffer.from(persistedJson, 'utf8');
const fileId = 'treatment-plan.json';
await persistence.setFlowFile(flowId, fileId, planBuffer);
Logger_1.appLogger.info(`Persisted treatment plan for "${testLabel}" as "${fileId}" to Donobu stack (flow: ${flowId}).`);
}
catch (error) {
Logger_1.appLogger.warn(`Failed to persist treatment plan for "${testLabel}" to Donobu stack (flow: ${flowId}); filesystem copy is still available.`, error);
}
}
generatedPlans.push({
plan,
planPath,
evidence,
evidencePath,
generatedAtIso: persisted.generatedAtIso,
originalPlaywrightArgs: [...originalArgsSnapshot],
reportPath,
});
}
return generatedPlans;
}
/**
* Optionally launch a second Playwright run using the automation directives
* produced by one or more treatment plans. This is the "auto-heal" feature
* where Donobu tries to repair failures on its own. The rerun happens in a
* temporary staging area so the user-visible Playwright output stays clean.
*/
async function attemptAutoHealRun(params) {
const evaluation = evaluateAutoHealEligibility(params.generatedPlans);
Logger_1.appLogger.info(`Auto-heal directives resolved to: ${JSON.stringify(evaluation.directives)}`);
if (evaluation.eligiblePlans.length === 0) {
Logger_1.appLogger.info('Auto-heal requested but no treatment plan provided actionable directives; skipping rerun.');
return { attempted: false, exitCode: params.currentExitCode };
}
const staging = await createAutoHealStagingArea();
let healExitCode = params.currentExitCode;
const healOptions = {
...params.options,
clearAiCache: params.options.clearAiCache || evaluation.clearPageAiCache === true,
autoHeal: false,
triageOutputDir: staging.triageBaseDir,
};
if (evaluation.clearPageAiCache && !params.options.clearAiCache) {
Logger_1.appLogger.info('Auto-heal: clearing Page.AI cache as recommended by the treatment plan.');
}
const healArgsWithDirectives = buildPlaywrightArgsWithDirectives(params.playwrightArgs, evaluation.directives);
const healArgsForRun = overrideOutputDir(healArgsWithDirectives, staging.playwrightOutputDir);
let healTriageContext = null;
let healTriageEnabled = healOptions.triageEnabled;
try {
if (healTriageEnabled) {
try {
healTriageContext = await prepareTriageContext(staging.playwrightOutputDir, healOptions);
}
catch (error) {
Logger_1.appLogger.error('Auto-heal: failed to prepare triage artifacts for the rerun. Continuing without triage.', error);
healTriageEnabled = false;
}
}
const envOverrides = createRunEnvOverrides({
clearAiCache: healOptions.clearAiCache,
triageEnabled: healTriageEnabled,
triageContext: healTriageContext,
});
applyJsonReportEnv(envOverrides, staging.playwrightOutputDir);
// Flag downstream systems so they know this invocation came from auto-heal.
envOverrides.DONOBU_AUTO_HEAL_ACTIVE