donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
303 lines (301 loc) • 14.9 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.selfHeal = selfHeal;
const fs = __importStar(require("fs"));
const v4_1 = require("zod/v4");
const DonobuFlowsManager_1 = require("../../../managers/DonobuFlowsManager");
const CodeGenerationOptions_1 = require("../../../models/CodeGenerationOptions");
const AnalyzePageTextTool_1 = require("../../../tools/AnalyzePageTextTool");
const AssertPageTextTool_1 = require("../../../tools/AssertPageTextTool");
const AssertTool_1 = require("../../../tools/AssertTool");
const ChangeWebBrowserTabTool_1 = require("../../../tools/ChangeWebBrowserTabTool");
const ChooseSelectOptionTool_1 = require("../../../tools/ChooseSelectOptionTool");
const ClickTool_1 = require("../../../tools/ClickTool");
const CreateBrowserCookieReportTool_1 = require("../../../tools/CreateBrowserCookieReportTool");
const DetectBrokenLinksTool_1 = require("../../../tools/DetectBrokenLinksTool");
const GoForwardOrBackTool_1 = require("../../../tools/GoForwardOrBackTool");
const GoToWebpageTool_1 = require("../../../tools/GoToWebpageTool");
const HandleBrowserDialogTool_1 = require("../../../tools/HandleBrowserDialogTool");
const HoverOverElementTool_1 = require("../../../tools/HoverOverElementTool");
const InputFakerTool_1 = require("../../../tools/InputFakerTool");
const InputRandomizedEmailAddressTool_1 = require("../../../tools/InputRandomizedEmailAddressTool");
const InputTextTool_1 = require("../../../tools/InputTextTool");
const MarkObjectiveCompleteTool_1 = require("../../../tools/MarkObjectiveCompleteTool");
const MarkObjectiveNotCompletableTool_1 = require("../../../tools/MarkObjectiveNotCompletableTool");
const PressKeyTool_1 = require("../../../tools/PressKeyTool");
const RunAccessibilityTestTool_1 = require("../../../tools/RunAccessibilityTestTool");
const ScrollPageTool_1 = require("../../../tools/ScrollPageTool");
const SummarizeLearningsTool_1 = require("../../../tools/SummarizeLearningsTool");
const WaitTool_1 = require("../../../tools/WaitTool");
const BrowserUtils_1 = require("../../../utils/BrowserUtils");
const Logger_1 = require("../../../utils/Logger");
const donobuTestStack_1 = require("./donobuTestStack");
const TestFileUpdater_1 = require("./TestFileUpdater");
/**
* Self-heals a test that has failed by creating and running a new autonomous
* flow using the failed test's code as part of the overall objective. If the
* new flow is successful, it uses the TestFileUpdater to update the specific
* failed test case in the original test file.
*
* @param gptClient The GPT client to use for running the new autonomous flow.
* @param testInfo The test information object that contains the failed test's
* annotations and other metadata.
* @param donobuFlowMetadata The metadata of the flow that is being self-healed.
*/
async function selfHeal(gptClient, testInfo, page) {
const donobuFlowMetadata = page._dnb.donobuFlowMetadata;
if (!donobuFlowMetadata) {
Logger_1.appLogger.warn('Will not self-heal due to no test flow metadata being found.');
return;
}
try {
const testFilePath = testInfo.file;
const originalTestCode = await fs.promises.readFile(testFilePath, 'utf8');
let postHealedOverallObjective;
let envVarNames;
if (!donobuFlowMetadata.overallObjective) {
// This means we are likely dealing with trying to heal a test that was
// not created using Donobu. So, we need to infer two things given the
// original test code:
// 1) What an overall objective might be, had it been created using Donobu.
// 2) What environment variables should we allow usage of.
const tmp = await distillObjectiveAndEnvVars(gptClient, originalTestCode);
postHealedOverallObjective = tmp.overallObjective;
envVarNames = tmp.environmentVariableNames;
}
else {
postHealedOverallObjective = donobuFlowMetadata.overallObjective;
envVarNames = donobuFlowMetadata.envVars ?? [];
}
const donobu = await (0, donobuTestStack_1.getOrCreateDonobuStack)();
const selfHealTimeoutMs = 300000; // 5 minutes
testInfo.setTimeout(testInfo.timeout + selfHealTimeoutMs);
Logger_1.appLogger.info(`Extending the test timeout ${selfHealTimeoutMs}ms for self-healing...`);
Logger_1.appLogger.info('Attempting to self-heal by creating a new autonomous flow...');
const allowedTools = await buildToolList(donobuFlowMetadata, donobu.toolRegistry);
const objectiveWhileSelfHealing = `
Use tool calling to follow along with the intent of this Playwright test.
The test had failed, so some parts may need to be adjusted along the way, but
you must adhere to its intent. Things that are okay to alter are selectors,
dismissing unexpected modals, fixing obvious mistakes, etc. Things that are not
okay to alter would be to make a test pass by removing its assertions, or
altering the intent of the test like making a test for a checkout flow to
instead be a test for login, etc. As you follow along the test, be sure to
retain assertions (et. al.) by mapping them to an appropriate tool call where
possible.
Here is the original Playwright test code...
\`\`\`
${originalTestCode}
\`\`\`
`.trim();
const newFlowParams = {
target: donobuFlowMetadata.target,
web: donobuFlowMetadata.web,
overallObjective: objectiveWhileSelfHealing,
name: donobuFlowMetadata.name ?? undefined,
envVars: envVarNames,
customTools: donobuFlowMetadata.customTools ?? undefined,
allowedTools: allowedTools,
callbackUrl: undefined,
maxToolCalls: DonobuFlowsManager_1.DonobuFlowsManager.DEFAULT_MAX_TOOL_CALLS,
gptConfigNameOverride: undefined,
defaultMessageDuration: undefined,
initialRunMode: 'AUTONOMOUS',
isControlPanelEnabled: false,
toolCallsOnStart: undefined,
resultJsonSchema: undefined,
};
await restoreBrowserContext(page.context(), page._dnb.initialBrowserState);
const newFlow = await donobu.flowsManager.createFlow(newFlowParams, gptClient, page.context());
Logger_1.appLogger.info('Running the new autonomous flow...');
await newFlow.job;
// Revert the objective of the self-healing flow back to the original
// objective now that the flow is complete. We do this so that the
// generated test code for the self-healed run looks clean, rather than
// having it embed the entire original test code file as its objective.
const persistence = await donobu.flowsPersistenceRegistry.get();
await persistence.setFlowMetadata({
...newFlow.donobuFlow.metadata,
overallObjective: postHealedOverallObjective,
});
if (newFlow.donobuFlow.metadata.state === 'SUCCESS') {
Logger_1.appLogger.info('Self-heal succeeded! The new autonomous flow completed successfully. Generating revised test...');
const newTestCode = await donobu.flowsManager.getFlowAsPlaywrightScript(newFlow.donobuFlow.metadata.id, getSelfHealingOptions(testInfo));
// Save the fixed test as an attachment
await testInfo.attach('fixed-test.ts', {
body: newTestCode,
contentType: 'application/typescript',
});
testInfo.annotations.push({
type: 'self-healed',
});
if (testFilePath) {
try {
Logger_1.appLogger.info(`Updating test file: ${testFilePath}`);
const updateSuccess = await TestFileUpdater_1.TestFileUpdater.updateTestCase(testFilePath, testInfo.title, newTestCode);
if (updateSuccess) {
Logger_1.appLogger.info(`Successfully updated test file with fixed test code.`);
}
else {
Logger_1.appLogger.warn(`Test file update did not complete successfully.`);
}
}
catch (fileUpdateError) {
Logger_1.appLogger.error(`Error updating test file ${testFilePath}:`, fileUpdateError);
}
}
else {
Logger_1.appLogger.warn('Could not update test file: test file path is not available.');
}
}
else {
Logger_1.appLogger.error('Self-heal failed! The new autonomous flow did not complete successfully.');
}
}
catch (error) {
Logger_1.appLogger.error('Error creating new flow when attempting to self heal:', error);
return;
}
}
function getSelfHealingOptions(testInfo) {
const defaultCodeGenerationOptions = {
areElementIdsVolatile: false,
disableSelectorFailover: false,
playwrightScriptVariant: 'ai',
// The other options are not relevant to self-healing.
};
if ('selfHealingOptions' in testInfo.project.metadata) {
try {
return CodeGenerationOptions_1.CodeGenerationOptionsSchema.parse(testInfo.project.metadata.selfHealingOptions);
}
catch (error) {
Logger_1.appLogger.warn(`Malformed selfHealingOptions, will use default value of: ${JSON.stringify(defaultCodeGenerationOptions, null, 2)}`, error);
return defaultCodeGenerationOptions;
}
}
return defaultCodeGenerationOptions;
}
async function restoreBrowserContext(browserContext, browserState) {
const pages = browserContext.pages();
for (let i = 0; i < pages.length; ++i) {
try {
await pages[i].close();
}
catch (error) {
Logger_1.appLogger.error('Failed to close page when attempting to reset browser context', error);
}
}
await browserContext.clearCookies();
await browserContext.addCookies(browserState.cookies);
await BrowserUtils_1.BrowserUtils.attachSessionStorageToBrowserContext(browserContext, browserState);
}
async function distillObjectiveAndEnvVars(gptClient, originalTestCode) {
try {
const systemPrompt = `
You will be given the file contents of a Playwright test.
Your job is to state the "overall objective" of the test and to return the
names of all environment variables used or referenced by the test.
A good "overall objective" describes the overall intent of a test, as if given
to someone who is tasked with completing it as a goal.
`.trim();
const responseSchema = v4_1.z.object({
overallObjective: v4_1.z
.string()
.describe('The overall objective of the test.'),
environmentVariableNames: v4_1.z
.array(v4_1.z.string())
.describe('The list of environment variables (by name) used or referenced in the test.'),
});
const resp = await gptClient.getStructuredOutput([
{
type: 'system',
text: systemPrompt,
},
{
type: 'user',
items: [
{
type: 'text',
text: originalTestCode,
},
],
},
], responseSchema);
return responseSchema.parse(resp.output);
}
catch (error) {
Logger_1.appLogger.warn('Failed to distill an objective and environment variable names for the test, will use a generic objective!', error);
return {
overallObjective: 'Follow the test steps',
environmentVariableNames: [],
};
}
}
async function buildToolList(donobuFlowMetadata, toolRegistry) {
const baseTools = [
AnalyzePageTextTool_1.AnalyzePageTextTool.NAME,
AssertTool_1.AssertTool.NAME,
AssertPageTextTool_1.AssertPageTextTool.NAME,
CreateBrowserCookieReportTool_1.CreateBrowserCookieReportTool.NAME,
ChangeWebBrowserTabTool_1.ChangeWebBrowserTabTool.NAME,
ChooseSelectOptionTool_1.ChooseSelectOptionTool.NAME,
ClickTool_1.ClickTool.NAME,
DetectBrokenLinksTool_1.DetectBrokenLinksTool.NAME,
GoForwardOrBackTool_1.GoForwardOrBackTool.NAME,
GoToWebpageTool_1.GoToWebpageTool.NAME,
HandleBrowserDialogTool_1.HandleBrowserDialogTool.NAME,
HoverOverElementTool_1.HoverOverElementTool.NAME,
InputFakerTool_1.InputFakerTool.NAME,
InputRandomizedEmailAddressTool_1.InputRandomizedEmailAddressTool.NAME,
InputTextTool_1.InputTextTool.NAME,
MarkObjectiveCompleteTool_1.MarkObjectiveCompleteTool.NAME,
MarkObjectiveNotCompletableTool_1.MarkObjectiveNotCompletableTool.NAME,
PressKeyTool_1.PressKeyTool.NAME,
RunAccessibilityTestTool_1.RunAccessibilityTestTool.NAME,
ScrollPageTool_1.ScrollPageTool.NAME,
SummarizeLearningsTool_1.SummarizeLearningsTool.NAME,
WaitTool_1.WaitTool.NAME,
];
// Always allow tools from plugins.
const toolsFromPlugins = toolRegistry.pluginTools().map((t) => t.name);
const defaultTools = new Set([
...(donobuFlowMetadata.allowedTools ? donobuFlowMetadata.allowedTools : []),
...baseTools,
...toolsFromPlugins,
]);
return [...defaultTools];
}
//# sourceMappingURL=selfHealing.js.map