donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
279 lines • 12 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.TestsManager = void 0;
const crypto_1 = require("crypto");
const CannotDeleteRunningFlowException_1 = require("../exceptions/CannotDeleteRunningFlowException");
const SuiteNotFoundException_1 = require("../exceptions/SuiteNotFoundException");
const TestNotFoundException_1 = require("../exceptions/TestNotFoundException");
const buildProvenance_1 = require("../utils/buildProvenance");
const displayName_1 = require("../utils/displayName");
const FederatedPagination_1 = require("./FederatedPagination");
class TestsManager {
constructor(testsPersistenceRegistry, suitesPersistenceRegistry, flowsManager) {
this.testsPersistenceRegistry = testsPersistenceRegistry;
this.suitesPersistenceRegistry = suitesPersistenceRegistry;
this.flowsManager = flowsManager;
}
async createTest(params) {
const testId = (0, crypto_1.randomUUID)();
const web = params.web
? {
browser: params.web.browser ?? { using: { type: 'device' } },
targetWebsite: params.web.targetWebsite,
}
: undefined;
const testMetadata = {
id: testId,
metadataVersion: 1,
name: (0, displayName_1.getDisplayName)({ name: params.name ?? null, web }),
suiteId: params.suiteId ?? null,
nextRunMode: params.nextRunMode ?? 'AUTONOMOUS',
target: params.target,
web,
envVars: params.envVars ?? null,
customTools: params.customTools ?? null,
overallObjective: params.overallObjective ?? null,
callbackUrl: params.callbackUrl ?? null,
allowedTools: params.allowedTools ?? [],
resultJsonSchema: params.resultJsonSchema ?? null,
maxToolCalls: params.maxToolCalls ?? null,
videoDisabled: params.videoDisabled,
provenance: (0, buildProvenance_1.buildProvenance)('DONOBU_STUDIO'),
};
// If the test is part of a suite, write it to the same persistence layer
// as the suite — otherwise the suite_id foreign key fails (in SQLite)
// or leaves a dangling reference (in non-DB layers). When standalone,
// fall back to the primary layer.
const persistence = await this.resolveLayerForTestCreate(testMetadata.suiteId);
await persistence.createTest(testMetadata);
return testMetadata;
}
async getTestById(testId) {
for (const persistence of await this.testsPersistenceRegistry.getAll()) {
try {
return await persistence.getTestById(testId);
}
catch (error) {
if (!(error instanceof TestNotFoundException_1.TestNotFoundException)) {
throw error;
}
}
}
throw TestNotFoundException_1.TestNotFoundException.forId(testId);
}
/**
* Picks the tests persistence layer to use when creating a new test.
*
* - If `suiteId` is null/undefined: use the primary tests layer.
* - If `suiteId` is set: look up the suite's layer key and use the matching
* tests layer. If no tests layer matches the suite's key (rare — would
* require asymmetric registry config), fall back to the primary tests
* layer; the FK won't hold but at least the test is persisted somewhere.
* - If `suiteId` is set but the suite doesn't exist anywhere: fall back
* to the primary tests layer (the SQLite FK will reject if applicable;
* non-DB layers will accept the dangling reference).
*/
async resolveLayerForTestCreate(suiteId) {
if (!suiteId) {
return this.testsPersistenceRegistry.get();
}
let suiteLayerKey;
try {
suiteLayerKey = await this.findSuiteLayerKey(suiteId);
}
catch (error) {
if (!(error instanceof SuiteNotFoundException_1.SuiteNotFoundException)) {
throw error;
}
return this.testsPersistenceRegistry.get();
}
const matched = await this.testsPersistenceRegistry.getByKey(suiteLayerKey);
return matched ?? (await this.testsPersistenceRegistry.get());
}
async findSuiteLayerKey(suiteId) {
for (const { key, persistence, } of await this.suitesPersistenceRegistry.getEntries()) {
try {
await persistence.getSuiteById(suiteId);
return key;
}
catch (error) {
if (!(error instanceof SuiteNotFoundException_1.SuiteNotFoundException)) {
throw error;
}
}
}
throw SuiteNotFoundException_1.SuiteNotFoundException.forId(suiteId);
}
async getTests(query) {
const layers = (await this.testsPersistenceRegistry.getAll()).map((persistence) => ({
getItems: (q) => persistence.getTests(q),
}));
const sortBy = query.sortBy ?? 'created_at';
const desc = (query.sortOrder ?? 'desc') === 'desc';
// TestMetadata has no `created_at` field — fall back to "now" so newly
// returned items sort to the most-recent end. For other sort columns,
// map the snake_case API name to the camelCase JS field.
const fieldFor = (test) => {
switch (sortBy) {
case 'created_at':
return Date.now();
case 'name':
return test.name ?? '';
case 'suite_id':
return test.suiteId ?? '';
case 'next_run_mode':
return test.nextRunMode;
case 'flow_count':
return test.flowCount ?? 0;
case 'latest_flow_created_at':
return test.latestFlow?.startedAt ?? Date.now();
default:
return '';
}
};
return (0, FederatedPagination_1.federatedList)(layers, query, (a, b) => {
const aKey = fieldFor(a);
const bKey = fieldFor(b);
if (aKey < bKey) {
return desc ? 1 : -1;
}
if (aKey > bKey) {
return desc ? -1 : 1;
}
return 0;
});
}
/**
* Update a test in the persistence layer where it exists.
* Iterates layers, updates in the first one that has it, ignores
* TestNotFoundExceptions from others.
*/
async updateTest(testMetadata) {
for (const persistence of await this.testsPersistenceRegistry.getAll()) {
try {
await persistence.updateTest(testMetadata);
return;
}
catch (error) {
if (!(error instanceof TestNotFoundException_1.TestNotFoundException)) {
throw error;
}
}
}
throw TestNotFoundException_1.TestNotFoundException.forId(testMetadata.id);
}
/**
* Delete a test from all persistence layers.
*
* After deleting, cascade-deletes any flows that belong to this test.
* This mirrors the DB-level ON DELETE CASCADE for non-DB persistence
* layers (Volatile, S3, GCS).
*/
async deleteTest(testId) {
for (const persistence of await this.testsPersistenceRegistry.getAll()) {
try {
await persistence.deleteTest(testId);
}
catch (e) {
// Ignore TestNotFoundException errors from layers that don't have this test.
if (!(e instanceof TestNotFoundException_1.TestNotFoundException)) {
throw e;
}
}
}
// Cascade-delete flows belonging to this test. Paginate through all flows
// since `getFlows` caps each page at 100.
const flowsNotDeleted = [];
let pageToken = undefined;
do {
const { items, nextPageToken } = await this.flowsManager.getFlows({
testId,
pageToken,
});
for (const flow of items) {
try {
await this.flowsManager.deleteFlowById(flow.id);
}
catch (error) {
if (!(error instanceof CannotDeleteRunningFlowException_1.CannotDeleteRunningFlowException)) {
throw error;
}
flowsNotDeleted.push(flow);
}
}
pageToken = nextPageToken;
} while (pageToken);
if (flowsNotDeleted.length > 0) {
// TODO: Just warn here? There will be orphaned flows.
console.warn(`Failed to delete ${flowsNotDeleted.length} flows belonging to test ${testId}: ${flowsNotDeleted.map((f) => f.id).join(', ')}`);
}
}
/**
* Gets the list of tool calls to invoke when starting a new flow for the
* given test, based on the most recent successful flow for the test.
*
* @param test - The test to get the tool calls for
*
* @returns The tool calls to use when executing the test
*
* @throws {Error} if no previous successful flow is found
*/
async getTestToolCalls(test) {
let previousSuccesfulFlow = undefined;
let pageToken = undefined;
while (!previousSuccesfulFlow) {
const { items: previousFlows, nextPageToken } = await this.flowsManager.getFlows({
testId: test.id,
sortBy: 'created_at',
sortOrder: 'desc',
pageToken,
});
previousSuccesfulFlow = previousFlows.find((f) => f.state === 'SUCCESS');
if (!previousSuccesfulFlow && !nextPageToken) {
throw new Error(`No previous successful flow found for test ${test.id}`);
}
pageToken = nextPageToken;
}
return this.flowsManager.getToolCallsForRerun(previousSuccesfulFlow, {
areElementIdsVolatile: false,
disableSelectorFailover: false,
});
}
/**
* Creates a new flow (config) for the given test, which should be passed to
* `flowsManager.createFlow` to execute the test. The returned config's
* `testId` is set, which `createFlow` uses to route the flow to the same
* persistence layer as the test.
*
* @param testId - The ID of the test to create a new flow for
*
* @returns A new flow configuration
*/
async getNewFlowFromTest(testId) {
const test = await this.getTestById(testId);
let toolCallsOnStart = [];
let runMode = test.nextRunMode;
if (runMode === 'DETERMINISTIC') {
try {
toolCallsOnStart = await this.getTestToolCalls(test);
}
catch (error) {
// Fallback to AUTONOMOUS is only viable when the test has an
// overallObjective for the agent to work toward. For tests without
// one (e.g. Playwright-imported tests, or AI tests where the user
// cleared the objective), AUTONOMOUS would just fail downstream
// with a misleading "overallObjective is required" — propagate the
// original error instead so the user sees the real cause.
if ((test.overallObjective?.trim().length ?? 0) === 0) {
throw error;
}
runMode = 'AUTONOMOUS';
}
}
const newFlowConfig = this.flowsManager.getFlowFromConfigAndToolCalls(test.name ?? `Flow for test ${test.id}`, runMode, test, toolCallsOnStart);
newFlowConfig.testId = test.id;
return newFlowConfig;
}
}
exports.TestsManager = TestsManager;
//# sourceMappingURL=TestsManager.js.map