donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
839 lines (836 loc) • 40.3 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getDonobuSqliteDatabase = getDonobuSqliteDatabase;
exports.closeDonobuSqliteDatabase = closeDonobuSqliteDatabase;
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
const fs = __importStar(require("fs/promises"));
const path_1 = __importDefault(require("path"));
const LockFile = __importStar(require("proper-lockfile"));
const v4_1 = require("zod/v4");
const FlowMetadata_1 = require("../models/FlowMetadata");
const displayName_1 = require("../utils/displayName");
const Logger_1 = require("../utils/Logger");
const MiscUtils_1 = require("../utils/MiscUtils");
const normalizeFlowMetadata_1 = require("./normalizeFlowMetadata");
const TestConfigHash_1 = require("./TestConfigHash");
let instance = null;
const Hex32RegExp = /^[a-f0-9]{32}$/i;
/**
* ##################################################################
* # #
* # WARNING! WARNING! WARNING! WARNING! WARNING! WARNING! #
* # #
* ##################################################################
*
* NEVER change existing migrations. This list is APPEND-ONLY! If you do not
* abide by this, YOU WILL BREAK OUR USERS!
*/
const migrations = [
{
// Setup the Donobu agent configs table.
version: 1,
up: (db) => {
db.exec(`
CREATE TABLE IF NOT EXISTS agent_configs (
agent TEXT PRIMARY KEY,
config_name TEXT NULL
);
INSERT OR IGNORE INTO agent_configs (agent, config_name)
VALUES ('flow-runner', NULL);`);
},
},
{
version: 2,
up: (db) => {
// Setup the Donobu flows-related tables.
db.exec(`
CREATE TABLE IF NOT EXISTS flow_metadata (
id TEXT PRIMARY KEY,
name TEXT NULL,
metadata TEXT NOT NULL,
created_at INTEGER NOT NULL,
run_mode TEXT NOT NULL,
state TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS tool_calls (
id TEXT PRIMARY KEY,
flow_id TEXT NOT NULL,
created_at INTEGER NOT NULL,
tool_call TEXT NOT NULL,
FOREIGN KEY (flow_id) REFERENCES flow_metadata(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS binary_files (
id TEXT PRIMARY KEY,
flow_id TEXT NOT NULL,
file_id TEXT NOT NULL,
mime_type TEXT NOT NULL,
content BLOB,
FOREIGN KEY (flow_id) REFERENCES flow_metadata(id) ON DELETE CASCADE,
UNIQUE(flow_id, file_id)
);
-- Indices for better performance
CREATE INDEX IF NOT EXISTS idx_flow_metadata_name ON flow_metadata(name);
CREATE INDEX IF NOT EXISTS idx_flow_metadata_created_at ON flow_metadata(created_at);
CREATE INDEX IF NOT EXISTS idx_flow_metadata_run_mode ON flow_metadata(run_mode);
CREATE INDEX IF NOT EXISTS idx_flow_metadata_state ON flow_metadata(state);
CREATE INDEX IF NOT EXISTS idx_flow_metadata_run_mode_state ON flow_metadata(run_mode, state);
CREATE INDEX IF NOT EXISTS idx_tool_calls_flow_id_started ON tool_calls(flow_id, created_at);
CREATE INDEX IF NOT EXISTS idx_tool_calls_created_at ON tool_calls(created_at);
CREATE INDEX IF NOT EXISTS idx_binary_files_flow_file ON binary_files(flow_id, file_id);
-- Enable foreign key support
PRAGMA foreign_keys = ON;`);
},
},
{
// Setup the GPT configs table.
version: 3,
up: (db) => {
db.exec(`
CREATE TABLE IF NOT EXISTS gpt_configs (
name TEXT PRIMARY KEY,
config TEXT NOT NULL
);`);
},
},
{
// Update the flow JSON metadata schema.
version: 4,
up: (db) => {
// For each row in the flow_metadata table...
// 1. Load the metadata field.
// 2. Parse it as JSON data.
// 3. Update the JSON data such that...
// a. There is a new field called "browser" that is an object.
// b. The "initialBrowserState" field is moved to the "browser" object as "initialState".
// c. The "persistBrowserState" field is moved to the "browser" object as "persistState".
// d. There is a field in all "browser" objects called "using" that is an object.
// e. If there is a non-null "remoteBrowserInstanceUrl" field, then...
// i. The "remoteBrowserInstanceUrl" field is moved to the "using" object as "url".
// ii. The "type" field in the "using" object is set to "remoteInstance".
// f. If the "remoteBrowserInstanceUrl" field is null, then...
// i. The "deviceName" field is moved to the "using" object. If "deviceName" is null,
// then it is defaulted to "Desktop Chromium".
// ii. The "headless" field is moved to the "using" object.
// iii. The "type" field in the "using" object is set to "device".
// g. The original top-level "initialBrowserState" field is removed.
// h. The original top-level "persistBrowserState" field is removed.
// i. The original top-level "remoteBrowserInstanceUrl" field is removed.
// j. The original top-level "deviceName" field is removed.
// k. The original top-level "headless" field is removed.
// 4. Save the JSON data back to the metadata field.
// Get all rows from the flow_metadata table.
const flows = db
.prepare('SELECT id, metadata FROM flow_metadata')
.all();
// Prepare the update statement.
const updateStmt = db.prepare('UPDATE flow_metadata SET metadata = ? WHERE id = ?');
// Process each flow's metadata.
for (const flow of flows) {
try {
// Parse the metadata JSON.
const metadata = JSON.parse(flow.metadata);
// Create the new browser object structure.
metadata.browser = {
initialState: metadata.initialBrowserState || null,
persistState: metadata.persistBrowserState || false,
using: {},
};
// Determine the browser type and populate the 'using' object.
if (metadata.remoteBrowserInstanceUrl) {
// Remote browser instance configuration.
metadata.browser.using = {
type: 'remoteInstance',
url: metadata.remoteBrowserInstanceUrl,
};
}
else {
// Local device configuration.
metadata.browser.using = {
type: 'device',
deviceName: metadata.deviceName || 'Desktop Chromium',
headless: metadata.headless !== undefined ? metadata.headless : false,
};
}
// Remove the old fields.
delete metadata.initialBrowserState;
delete metadata.persistBrowserState;
delete metadata.remoteBrowserInstanceUrl;
delete metadata.deviceName;
delete metadata.headless;
// Update the database with the modified metadata.
updateStmt.run(JSON.stringify(metadata), flow.id);
}
catch (error) {
Logger_1.appLogger.error(`Error updating metadata for flow ${flow.id}:`, error);
// Continue processing other flows even if one fails.
}
}
},
},
{
version: 5,
up: (db) => {
// For each row in the flow_metadata table...
// 1. Load the metadata field.
// 2. Parse it as JSON data.
// 3. Rename the "defaultToolTipDurationMilliseconds" field to "defaultMessageDuration".
// 4. Save the JSON data back to the metadata field.
// Get all rows from the flow_metadata table
const flows = db
.prepare('SELECT id, metadata FROM flow_metadata')
.all();
// Prepare the update statement
const updateStmt = db.prepare('UPDATE flow_metadata SET metadata = ? WHERE id = ?');
// Process each flow's metadata
for (const flow of flows) {
try {
// Parse the metadata JSON
const metadata = JSON.parse(flow.metadata);
// Rename defaultToolTipDurationMilliseconds to defaultMessageDuration if it exists
if ('defaultToolTipDurationMilliseconds' in metadata) {
metadata.defaultMessageDuration =
metadata.defaultToolTipDurationMilliseconds;
delete metadata.defaultToolTipDurationMilliseconds;
}
// Update the database with the modified metadata
updateStmt.run(JSON.stringify(metadata), flow.id);
}
catch (error) {
Logger_1.appLogger.error(`Error updating metadata for flow ${flow.id}:`, error);
// Continue processing other flows even if one fails
}
}
},
},
{
version: 6,
up: (db) => {
// Create the environment_data table to store cross-flow data
db.exec(`
CREATE TABLE IF NOT EXISTS environment_data (
name TEXT PRIMARY KEY,
value TEXT NOT NULL
);
`);
},
},
{
version: 7,
up: (db) => {
// For each row in the flow_metadata table...
// 1. Load the metadata field.
// 2. Parse it as JSON data.
// 3. Rename the "maxIterations" field to "maxToolCalls".
// If the existing "maxIterations" value is 0, map it to null for "maxToolCalls".
// 4. Remove the "iterations" field.
// 5. Save the JSON data back to the metadata field.
// Get all rows from the flow_metadata table
const flows = db
.prepare('SELECT id, metadata FROM flow_metadata')
.all();
// Prepare the update statement
const updateStmt = db.prepare('UPDATE flow_metadata SET metadata = ? WHERE id = ?');
// Process each flow's metadata
for (const flow of flows) {
try {
// Parse the metadata JSON
const metadata = JSON.parse(flow.metadata);
// Rename maxIterations to maxToolCalls if it exists
if ('maxIterations' in metadata) {
// If maxIterations is 0, map it to null, otherwise keep the value
metadata.maxToolCalls =
metadata.maxIterations === 0 ? null : metadata.maxIterations;
delete metadata.maxIterations;
}
// Remove the iterations field if it exists
if ('iterations' in metadata) {
delete metadata.iterations;
}
// Update the database with the modified metadata
updateStmt.run(JSON.stringify(metadata), flow.id);
}
catch (error) {
Logger_1.appLogger.error(`Error updating metadata for flow ${flow.id}:`, error);
// Continue processing other flows even if one fails
}
}
},
},
{
version: 8,
up: (db) => {
// For each row in the tool_calls table with a toolName of 'scrollPage'...
// 1. Load the tool_call field.
// 2. Parse it as JSON data.
// 3. If the tool_call object's .outcome.metadata is null or undefined,
// set it to {"element":["html > body"],"frame":null,"usedSelector":"html>body"}
// 4. Save the updated data back to the tool_call field.
const toolCalls = db
.prepare('SELECT id, tool_call FROM tool_calls')
.all();
// Prepare the update statement
const updateStmt = db.prepare('UPDATE tool_calls SET tool_call = ? WHERE id = ?');
// Process each tool call
for (const toolCall of toolCalls) {
try {
// Parse the tool_call JSON
const toolCallData = JSON.parse(toolCall.tool_call);
// Check if this is actually a scrollPage tool call and needs updating
if (toolCallData.toolName === 'scrollPage') {
// Ensure outcome object exists
if (!toolCallData.outcome) {
toolCallData.outcome = {};
}
// If metadata is null or undefined, set the default value
if (!toolCallData.outcome.metadata) {
toolCallData.outcome.metadata = {
element: ['html > body'],
frame: null,
usedSelector: 'html > body',
};
}
// Update the database with the modified tool_call
updateStmt.run(JSON.stringify(toolCallData), toolCall.id);
}
}
catch (error) {
Logger_1.appLogger.error(`Error updating tool_call for ID ${toolCall.id}:`, error);
// Continue processing other tool calls even if one fails
}
}
},
},
{
// Restructure flow metadata to the { target, web } pattern.
// Moves top-level `browser` and `targetWebsite` into `web: { browser, targetWebsite }`
// and adds a `target: 'web'` discriminant. All pre-existing flows are web flows.
version: 9,
up: (db) => {
const flows = db
.prepare('SELECT id, metadata FROM flow_metadata')
.all();
const updateStmt = db.prepare('UPDATE flow_metadata SET metadata = ? WHERE id = ?');
for (const flow of flows) {
try {
const metadata = JSON.parse(flow.metadata);
// Skip if already migrated (has `target` field).
if (metadata.target) {
continue;
}
metadata.target = 'web';
metadata.web = {
browser: metadata.browser,
targetWebsite: metadata.targetWebsite ?? '',
};
// Remove old top-level fields.
delete metadata.browser;
delete metadata.targetWebsite;
updateStmt.run(JSON.stringify(metadata), flow.id);
}
catch (error) {
Logger_1.appLogger.error(`Error updating metadata for flow ${flow.id}:`, error);
}
}
},
},
{
// Create suite_metadata and test_metadata tables, and add test_id FK to
// flow_metadata. This is the schema foundation for the Test Suites feature
// where tests are the reusable config and flows are individual runs.
version: 10,
up: (db) => {
db.exec(`
CREATE TABLE IF NOT EXISTS suite_metadata (
id TEXT PRIMARY KEY,
name TEXT NULL,
metadata TEXT NOT NULL,
created_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_suite_metadata_name ON suite_metadata(name);
CREATE INDEX IF NOT EXISTS idx_suite_metadata_created_at ON suite_metadata(created_at);
CREATE TABLE IF NOT EXISTS test_metadata (
id TEXT PRIMARY KEY,
name TEXT NULL,
metadata TEXT NOT NULL,
created_at INTEGER NOT NULL,
suite_id TEXT NULL,
next_run_mode TEXT NULL,
FOREIGN KEY (suite_id) REFERENCES suite_metadata(id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_test_metadata_name ON test_metadata(name);
CREATE INDEX IF NOT EXISTS idx_test_metadata_created_at ON test_metadata(created_at);
CREATE INDEX IF NOT EXISTS idx_test_metadata_suite_id ON test_metadata(suite_id);
ALTER TABLE flow_metadata ADD COLUMN test_id TEXT NULL REFERENCES test_metadata(id) ON DELETE CASCADE;
CREATE INDEX IF NOT EXISTS idx_flow_metadata_test_id ON flow_metadata(test_id);
`);
},
},
{
// Create the ai_queries table to store AI decision-cycle records (clean
// and annotated screenshots, interactable elements, optional error).
version: 11,
up: (db) => {
db.exec(`
CREATE TABLE IF NOT EXISTS ai_queries (
id TEXT PRIMARY KEY,
flow_id TEXT NOT NULL,
started_at INTEGER NOT NULL,
ai_query TEXT NOT NULL,
FOREIGN KEY (flow_id) REFERENCES flow_metadata(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_ai_queries_flow_id_started_at ON ai_queries(flow_id, started_at);
`);
},
},
{
// Back-fill tests for orphan flows (test_id IS NULL). Flows with identical
// RunConfig-derived fields are grouped under a single new test whose id is
// the MD5 hash of the canonical-sorted config JSON. See TestConfigHash.ts.
version: 12,
up: (db) => {
// 1. Fetch all flows with null test_id, oldest first.
const flows = v4_1.z
.array(v4_1.z.object({ id: v4_1.z.string(), metadata: v4_1.z.string() }))
.parse(db
.prepare('SELECT id, metadata FROM flow_metadata WHERE test_id IS NULL ORDER BY created_at ASC')
.all());
if (flows.length === 0) {
return;
}
const groups = new Map();
for (const flow of flows) {
try {
const metadata = FlowMetadata_1.FlowMetadataSchema.parse((0, normalizeFlowMetadata_1.normalizeFlowMetadata)(JSON.parse(flow.metadata)));
const hash = (0, TestConfigHash_1.hashTestConfig)(metadata);
const groupedFlow = { id: flow.id, metadata };
const existing = groups.get(hash);
if (existing) {
existing.flows.push(groupedFlow);
}
else {
groups.set(hash, { flows: [groupedFlow] });
}
}
catch (error) {
// Fail open: skip flows we can't parse/normalize so one bad row
// doesn't block the rest of the migration.
Logger_1.appLogger.error(`Migration 12: skipping flow ${flow.id} during grouping:`, error);
}
}
// 4. Create a test for each group.
const insertTestStmt = db.prepare(`
INSERT INTO test_metadata (id, name, metadata, created_at, suite_id, next_run_mode)
VALUES (@id, @name, @metadata, @createdAt, @suiteId, @nextRunMode)
`);
const updateFlowStmt = db.prepare('UPDATE flow_metadata SET test_id = ?, metadata = ? WHERE id = ?');
for (const [hash, group] of groups) {
try {
// Flows were inserted oldest-first, so the last entry is newest.
const newest = group.flows[group.flows.length - 1].metadata;
// `maxToolCalls` is null on every deterministic flow, so prefer the
// most recent autonomous flow's value. Fall back to the newest flow's
// value (likely null) if no autonomous flow exists in the group.
const newestAutonomous = [...group.flows]
.reverse()
.find((f) => f.metadata.runMode === 'AUTONOMOUS')?.metadata;
const maxToolCalls = newestAutonomous?.maxToolCalls ?? newest.maxToolCalls;
const testName = (0, displayName_1.getDisplayName)(newest, 'Untitled Test');
const testMetadata = {
id: hash,
name: testName,
target: newest.target,
web: newest.web,
envVars: newest.envVars,
customTools: newest.customTools,
overallObjective: newest.overallObjective,
allowedTools: newest.allowedTools,
resultJsonSchema: newest.resultJsonSchema,
callbackUrl: newest.callbackUrl,
videoDisabled: newest.videoDisabled,
maxToolCalls,
metadataVersion: newest.metadataVersion,
createdWithDonobuVersion: newest.createdWithDonobuVersion,
suiteId: null,
nextRunMode: 'DETERMINISTIC',
};
insertTestStmt.run({
id: hash,
name: testName,
metadata: JSON.stringify(testMetadata),
createdAt: Date.now(),
suiteId: null,
nextRunMode: 'DETERMINISTIC',
});
// 5. Link each flow to its test (both test_id column and testId in JSON).
for (const groupedFlow of group.flows) {
try {
groupedFlow.metadata.testId = hash;
updateFlowStmt.run(hash, JSON.stringify(groupedFlow.metadata), groupedFlow.id);
}
catch (error) {
Logger_1.appLogger.error(`Migration 12: failed to link flow ${groupedFlow.id} to test ${hash}:`, error);
}
}
}
catch (error) {
// Fail open: skip groups we can't materialize as tests.
Logger_1.appLogger.error(`Migration 12: skipping test group ${hash}:`, error);
}
}
},
},
{
version: 13,
up: (db) => {
// 1. Fetch all flows, oldest first.
const flows = v4_1.z
.array(v4_1.z.object({
id: v4_1.z.string(),
test_id: v4_1.z.string().nullable(),
metadata: v4_1.z.string(),
}))
.parse(db
.prepare('SELECT id, test_id, metadata FROM flow_metadata ORDER BY created_at ASC')
.all())
// We'll only consider those flows without tests, or those migrated by
// migration 12. Fortunately, only flows that were created as part of
// migration 12 have test IDs that are 32-character hexidecimals (tests
// generated in the app use UUIDs, and tests generated by the SDK use
// the Playwright testId, both of which contain hypens).
.filter((flow) => flow.test_id === null || Hex32RegExp.test(flow.test_id));
if (flows.length === 0) {
return;
}
// 2. Build a config hash for each flow and group by hash.
const groups = new Map();
// If 2 flows previously hashed to different tests, but now have the same
// hash (due to the improvements to hashTestConfig), and the user executed
// the test that will now be merged into the other one, we'll need to
// correctly move over the new flow using the same mapping, even if the
// hash changes (because the user edited the test, say). So we'll need to
// keep track of the mapping of old hash to new hash.
const oldHashToNewHash = new Map();
// We'll also want to delete the old test if it's no longer referenced by
// any flows.
const testsToDelete = new Set();
for (const flow of flows) {
try {
const flowMetadata = FlowMetadata_1.FlowMetadataSchema.parse((0, normalizeFlowMetadata_1.normalizeFlowMetadata)(JSON.parse(flow.metadata)));
/*
* There are 7 possible scenarios, since the hashing algorithm was
* improved since migration 12, and some flows will have a new hash,
* matching them with flows that were previously added to a distinct
* test:
*
* 1. The flow is a new (orphaned) flow, with no testId: use the hash
* to find or create its test.
* 2. The flow was previously migrated by migration 12, and correctly:
* use the hash, which should match the existing testId, to keep it
* in the existing test.
* 3. The flow was previously migrated by migration 12, but
* incorrectly (old hash): use the new hash to add it to the
* correct test.
* 4. The flow was created from a previously-migrated test, which was
* created from a correct hash, and the test was not modified: use
* the hash, which should match the existing testId, to keep it in
* the existing test.
* 5. The flow was created from a previously-migrated test, which was
* created from a correct hash, and the test was modified: since
* the test was modified, this will be a brand-new hash, but we
* want to keep the flow in this test, so use the existing testId
* to keep it in the existing test.
* 6. The flow was created from a previously-migrated test, which was
* created from an incorrect hash, and the test was not modified:
* use the new hash to add it to the correct test.
* 7. The flow was created from a previously-migrated test, which was
* created from an incorrect hash, and the test was modified: since
* the test was modified, this will be a brand-new hash, but we
* want to keep it with the other flows from this test, so use a
* old-hash-to-new-hash map to assign it the correct test based on
* its current testId.
*
* We have no way of distinguishing between scenarios 5 and 7, since
* they both produce brand-new hashes, so we'll need to keep track of
* the mapping of old hash to new hash (based on the flows that
* existed prior to migration 12) to assign the flow to the correct
* test. Since flows are handled in oldest-to-newest order, the map
* should be populated by migration 12 flows prior to new flows being
* encountered.
*/
let hash = (0, TestConfigHash_1.hashTestConfig)(flowMetadata);
if (flowMetadata.testId && flowMetadata.testId !== hash) {
// Scenario 3, 5, 6
if (oldHashToNewHash.has(flowMetadata.testId)) {
hash = oldHashToNewHash.get(flowMetadata.testId);
}
else if (groups.has(flowMetadata.testId)) {
// Scenario 5
hash = flowMetadata.testId;
}
else {
// Scenario 3 (first time)
oldHashToNewHash.set(flowMetadata.testId, hash);
testsToDelete.add(flowMetadata.testId);
}
}
const existing = groups.get(hash);
if (existing) {
existing.push(flowMetadata);
}
else {
groups.set(hash, [flowMetadata]);
}
}
catch (error) {
// Fail open: skip flows we can't parse/normalize so one bad row
// doesn't block the rest of the migration.
Logger_1.appLogger.error(`Migration 13: skipping flow ${flow.id} during grouping:`, error);
}
}
Logger_1.appLogger.info(`Migration 13: found ${groups.size} test groups`);
// 3. Create a test for each group.
const insertTestStmt = db.prepare(`
INSERT OR IGNORE INTO test_metadata (id, name, metadata, created_at, suite_id, next_run_mode)
VALUES (@id, @name, @metadata, @createdAt, @suiteId, @nextRunMode)
`);
const updateFlowStmt = db.prepare('UPDATE flow_metadata SET test_id = ?, metadata = ? WHERE id = ?');
const deleteTestStmt = db.prepare('DELETE FROM test_metadata WHERE id = ?');
const countFlowsByTestIdStmt = db.prepare('SELECT COUNT(*) FROM flow_metadata WHERE test_id = ?');
let testsCreated = 0;
let testsSkipped = 0;
let flowsLinked = 0;
let failures = 0;
let testsDeleted = 0;
for (const [hash, flows] of groups) {
try {
// Flows were inserted oldest-first, so the last entry is newest.
const newestFlow = flows[flows.length - 1];
// `maxToolCalls` is null on every deterministic flow, so prefer the
// most recent autonomous flow's value. Fall back to the newest flow's
// value (likely null) if no autonomous flow exists in the group.
const newestAutonomous = [...flows]
.reverse()
.find((f) => f.runMode === 'AUTONOMOUS');
const maxToolCalls = newestAutonomous?.maxToolCalls ?? newestFlow.maxToolCalls;
const testName = (0, displayName_1.getDisplayName)(newestFlow, 'Untitled Test');
const testMetadata = {
id: hash,
name: testName,
target: newestFlow.target,
web: newestFlow.web,
envVars: newestFlow.envVars,
customTools: newestFlow.customTools,
overallObjective: newestFlow.overallObjective,
allowedTools: newestFlow.allowedTools,
resultJsonSchema: newestFlow.resultJsonSchema,
callbackUrl: newestFlow.callbackUrl,
videoDisabled: newestFlow.videoDisabled,
maxToolCalls,
metadataVersion: newestFlow.metadataVersion,
createdWithDonobuVersion: newestFlow.createdWithDonobuVersion,
suiteId: null,
nextRunMode: 'DETERMINISTIC',
};
const insertResult = insertTestStmt.run({
id: hash,
name: testName,
metadata: JSON.stringify(testMetadata),
createdAt: Date.now(),
suiteId: null,
nextRunMode: 'DETERMINISTIC',
});
if (insertResult.changes > 0) {
testsCreated++;
}
else {
testsSkipped++;
}
// 4. Link each flow to its test (both test_id column and testId in JSON).
for (const groupedFlow of flows) {
try {
groupedFlow.testId = hash;
updateFlowStmt.run(hash, JSON.stringify(groupedFlow), groupedFlow.id);
flowsLinked++;
}
catch (error) {
failures++;
Logger_1.appLogger.error(`Migration 13: failed to link flow ${groupedFlow.id} to test ${hash}:`, error);
}
}
}
catch (error) {
// Fail open: skip groups we can't materialize as tests.
failures++;
Logger_1.appLogger.error(`Migration 13: skipping test group ${hash}:`, error);
}
}
// 5. Delete any tests that are no longer referenced by any flows.
for (const testId of testsToDelete) {
// Verify that there are no flows still referencing the test.
const flowsReferencingTest = countFlowsByTestIdStmt.get(testId);
if (Number(flowsReferencingTest) > 0) {
Logger_1.appLogger.warn(`Migration 13: skipping deletion of test ${testId} because it is still referenced by ${flowsReferencingTest} flow(s).`);
continue;
}
deleteTestStmt.run(testId);
testsDeleted++;
}
Logger_1.appLogger.info(`Migration 13: done:
- Created ${testsCreated} test(s)
- Skipped ${testsSkipped} pre-existing test(s)
- Linked ${flowsLinked} flow(s)
- Deleted ${testsDeleted} test(s)`);
if (failures > 0) {
Logger_1.appLogger.warn(`Migration 13: ${failures} failure(s) (see errors above).`);
}
},
},
];
/**
* Create the SQL schema migrations table that can be used to manage table
* schemas across Donobu releases.
*/
function initMigrationsTable(db) {
db.exec(`
CREATE TABLE IF NOT EXISTS schema_migrations (
version INTEGER PRIMARY KEY,
applied_at TEXT NOT NULL
);
`);
}
function applyMigrations(db) {
// First, validate that no duplicate version numbers exist in the migrations
// array.
const versionCounts = new Map();
const duplicateVersions = [];
// Count occurrences of each version.
for (const migration of migrations) {
const count = (versionCounts.get(migration.version) || 0) + 1;
versionCounts.set(migration.version, count);
// If we've seen this version more than once, add it to duplicates.
if (count > 1 && !duplicateVersions.includes(migration.version)) {
duplicateVersions.push(migration.version);
}
}
// If we found duplicates, throw an error.
if (duplicateVersions.length > 0) {
throw new Error(`Duplicate migration version(s) detected: ${duplicateVersions.join(', ')}. ` +
'Each migration must have a unique version number.');
}
const getCurrentVersionStmt = db.prepare('SELECT MAX(version) as current_version FROM schema_migrations');
const recordMigrationStmt = db.prepare('INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)');
const runPendingMigrations = db.transaction(() => {
const result = getCurrentVersionStmt.get();
const currentVersion = result.current_version || 0;
const pendingMigrations = migrations
.filter((m) => m.version > currentVersion)
.sort((a, b) => a.version - b.version);
if (pendingMigrations.length === 0) {
return;
}
for (const migration of pendingMigrations) {
Logger_1.appLogger.debug(`Applying migration ${migration.version}...`);
migration.up(db);
recordMigrationStmt.run(migration.version, new Date().toISOString());
}
});
// Ensure only one worker can evaluate/apply pending migrations at a time.
runPendingMigrations.exclusive();
}
/**
* Returns the singleton SQLite database instance.
* Creates the database on first call and initializes it with proper settings.
*
* When multiple processes (e.g. Playwright workers) start concurrently, they
* can all race into this function at the same time. SQLite's built-in busy
* timeout is insufficient here because `PRAGMA journal_mode = WAL` requires
* an exclusive file lock that is not retryable via the busy handler. We use
* a cross-process file lock (`proper-lockfile`) to serialize initialization
* so only one process creates/configures the database at a time.
*/
async function getDonobuSqliteDatabase() {
if (instance) {
return instance;
}
const dbDirectory = MiscUtils_1.MiscUtils.baseWorkingDirectory();
await fs.mkdir(dbDirectory, { recursive: true });
const dbFilepath = path_1.default.join(dbDirectory, 'database.sqlite');
const lockFilepath = path_1.default.join(dbDirectory, 'database.sqlite.init');
// Create the lock target file if it doesn't exist — proper-lockfile requires
// the file to exist before locking.
await fs.writeFile(lockFilepath, '', { flag: 'a' });
const release = await LockFile.lock(lockFilepath, {
retries: {
retries: 10,
minTimeout: 100,
maxTimeout: 2000,
},
});
try {
// Another process may have completed initialization while we waited for
// the lock, so re-check the singleton.
if (instance) {
return instance;
}
// Initialize database.
const donobuSqliteDb = new better_sqlite3_1.default(dbFilepath);
// Set up database configuration.
donobuSqliteDb.pragma('journal_mode = WAL');
donobuSqliteDb.pragma('foreign_keys = ON');
// Set up tables.
initMigrationsTable(donobuSqliteDb);
applyMigrations(donobuSqliteDb);
// Update global database handle.
instance = donobuSqliteDb;
return instance;
}
finally {
await release();
}
}
/**
* Closes the database connection. Should be called during application shutdown.
*/
function closeDonobuSqliteDatabase() {
if (instance) {
instance.close();
instance = null;
}
}
//# sourceMappingURL=DonobuSqliteDb.js.map