donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
288 lines • 12.7 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.FlowsPersistenceSqlite = void 0;
const crypto_1 = require("crypto");
const FlowNotFoundException_1 = require("../../exceptions/FlowNotFoundException");
const BrowserStorageState_1 = require("../../models/BrowserStorageState");
const MiscUtils_1 = require("../../utils/MiscUtils");
const normalizeFlowMetadata_1 = require("../normalizeFlowMetadata");
/**
* A persistence implementation that uses SQLite for all data storage, including binary files.
*/
class FlowsPersistenceSqlite {
constructor(db) {
this.db = db;
}
static async create(db) {
return new FlowsPersistenceSqlite(db);
}
async setEnvironmentDatum(key, value) {
if (!key) {
throw new Error('Key cannot be empty');
}
const stmt = this.db.prepare('INSERT OR REPLACE INTO environment_data (name, value) VALUES (?, ?)');
stmt.run(key, value);
}
async deleteEnvironmentDatum(key) {
if (!key) {
return; // No-op if key is empty
}
const stmt = this.db.prepare('DELETE FROM environment_data WHERE name = ?');
stmt.run(key);
}
async getEnvironmentDatum(key) {
if (!key) {
return undefined;
}
const stmt = this.db.prepare('SELECT value FROM environment_data WHERE name = ?');
const row = stmt.get(key);
return row ? row.value : undefined;
}
/**
* Get all environment data.
*/
async getEnvironmentData() {
const result = {};
const stmt = this.db.prepare('SELECT name, value FROM environment_data');
const rows = stmt.all();
for (const row of rows) {
result[row.name] = row.value;
}
return result;
}
async setFlowMetadata(flowMetadata) {
const upsertStmt = this.db.prepare(`
INSERT INTO flow_metadata (id, name, metadata, created_at, run_mode, state, test_id)
VALUES (@id, @name, @metadata, @createdAt, @runMode, @state, @testId)
ON CONFLICT(id) DO UPDATE SET
name = excluded.name,
metadata = excluded.metadata,
created_at = excluded.created_at,
run_mode = excluded.run_mode,
state = excluded.state,
test_id = excluded.test_id
`);
const saveFlow = this.db.transaction((flow) => {
upsertStmt.run({
id: flow.id,
name: flow.name,
metadata: JSON.stringify(flow),
createdAt: flow.startedAt || Date.now(),
runMode: flow.runMode,
state: flow.state,
testId: flow.testId ?? null,
});
});
// Acquire the write lock up front so concurrent writers queue instead of throwing SQLITE_BUSY.
saveFlow.immediate(flowMetadata);
}
async getFlowMetadataById(flowId) {
const stmt = this.db.prepare('SELECT metadata FROM flow_metadata WHERE id = ?');
const row = stmt.get(flowId);
if (!row) {
throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
}
return (0, normalizeFlowMetadata_1.normalizeFlowMetadata)(JSON.parse(row.metadata));
}
async getFlowMetadataByName(flowName) {
const stmt = this.db.prepare('SELECT metadata FROM flow_metadata WHERE name = ? ORDER BY created_at DESC LIMIT 1');
const row = stmt.get(flowName);
if (!row) {
throw FlowNotFoundException_1.FlowNotFoundException.forName(flowName);
}
return (0, normalizeFlowMetadata_1.normalizeFlowMetadata)(JSON.parse(row.metadata));
}
async getFlowsMetadata(query) {
// Sanitize inputs
const validLimit = Math.max(1, Math.min(100, query.limit || 100));
const offset = query.pageToken ? parseInt(query.pageToken, 10) || 0 : 0;
// Build the WHERE clause dynamically based on query parameters
let whereConditions = [];
let params = [];
// partialName takes precedence over name if both are provided.
if (query.partialName) {
whereConditions.push('name LIKE ?');
params.push(`%${query.partialName}%`);
}
else if (query.name) {
whereConditions.push('name = ?');
params.push(query.name);
}
if (query.runMode) {
whereConditions.push('run_mode = ?');
params.push(query.runMode);
}
if (query.state) {
whereConditions.push('state = ?');
params.push(query.state);
}
// Add startedAfter condition if provided
if (query.startedAfter !== undefined) {
whereConditions.push('created_at >= ?');
params.push(query.startedAfter);
}
// Add startedBefore condition if provided
if (query.startedBefore !== undefined) {
whereConditions.push('created_at <= ?');
params.push(query.startedBefore);
}
// Add testId condition if provided
if (query.testId) {
whereConditions.push('test_id = ?');
params.push(query.testId);
}
if (query.orphaned === true) {
whereConditions.push('test_id IS NULL');
}
else if (query.orphaned === false) {
whereConditions.push('test_id IS NOT NULL');
}
// Build the full SQL query
let sql = 'SELECT metadata FROM flow_metadata';
if (whereConditions.length > 0) {
sql += ' WHERE ' + whereConditions.join(' AND ');
}
const sortCol = query.sortBy ?? 'created_at';
const sortDir = query.sortOrder === 'asc' ? 'ASC' : 'DESC';
sql += ` ORDER BY ${sortCol} ${sortDir} LIMIT ? OFFSET ?`;
// Add limit and offset to params
params.push(validLimit + 1);
params.push(offset);
// Execute the query with parameters
const stmt = this.db.prepare(sql);
const rows = stmt.all(...params);
// Check if there are more results by fetching one extra
const hasMore = rows.length > validLimit;
const results = hasMore ? rows.slice(0, validLimit) : rows;
// Generate next token if needed
const nextPageToken = hasMore ? `${offset + validLimit}` : undefined;
return {
items: results.map((row) => (0, normalizeFlowMetadata_1.normalizeFlowMetadata)(JSON.parse(row.metadata))),
nextPageToken,
};
}
async saveScreenShot(flowId, bytes) {
// Ensure flow exists.
await this.getFlowMetadataById(flowId);
const imageType = MiscUtils_1.MiscUtils.detectImageType(bytes);
const mimeType = `image/${imageType}`;
const hash = (0, crypto_1.createHash)('sha256').update(bytes).digest('hex').slice(0, 12);
const epochMillis = Date.now();
// Generate a unique filename.
const fileId = `${epochMillis}.${hash}.${imageType}`;
const binaryId = `${flowId}-${fileId}`;
// `fileId` embeds a content hash, so a (flow_id, file_id) collision
// implies the same bytes are already persisted. `INSERT OR IGNORE`
// makes the save idempotent — important when two screenshots taken in
// the same millisecond happen to be byte-identical (e.g. a clean and
// "annotated" screenshot where annotation was a no-op).
const stmt = this.db.prepare('INSERT OR IGNORE INTO binary_files (id, flow_id, file_id, mime_type, content) VALUES (?, ?, ?, ?, ?)');
stmt.run(binaryId, flowId, fileId, mimeType, bytes);
return fileId;
}
async getScreenShot(flowId, screenShotId) {
const stmt = this.db.prepare('SELECT mime_type, content FROM binary_files WHERE flow_id = ? AND file_id = ?');
const row = stmt.get(flowId, screenShotId);
if (row?.mime_type.startsWith('image')) {
return row.content;
}
else {
return null;
}
}
async setToolCall(flowId, toolCall) {
const stmt = this.db.prepare('INSERT OR REPLACE INTO tool_calls (id, flow_id, created_at, tool_call) VALUES (?, ?, ?, ?)');
stmt.run(toolCall.id, flowId, toolCall.startedAt, JSON.stringify(toolCall));
}
async getToolCalls(flowId) {
// Check if flow exists
await this.getFlowMetadataById(flowId);
const stmt = this.db.prepare('SELECT tool_call FROM tool_calls WHERE flow_id = ? ORDER BY created_at');
const rows = stmt.all(flowId);
return rows.map((row) => JSON.parse(row.tool_call));
}
async deleteToolCall(flowId, toolCallId) {
// Ensure the flow exists so we can return a consistent error if not.
await this.getFlowMetadataById(flowId);
const stmt = this.db.prepare('DELETE FROM tool_calls WHERE flow_id = ? AND id = ?');
stmt.run(flowId, toolCallId);
}
async setAiQuery(flowId, aiQuery) {
const stmt = this.db.prepare('INSERT OR REPLACE INTO ai_queries (id, flow_id, started_at, ai_query) VALUES (?, ?, ?, ?)');
stmt.run(aiQuery.id, flowId, aiQuery.startedAt, JSON.stringify(aiQuery));
}
async getAiQueries(flowId) {
await this.getFlowMetadataById(flowId);
const stmt = this.db.prepare('SELECT ai_query FROM ai_queries WHERE flow_id = ? ORDER BY started_at');
const rows = stmt.all(flowId);
return rows.map((row) => JSON.parse(row.ai_query));
}
async setVideo(flowId, bytes) {
// Ensure flow exists.
await this.getFlowMetadataById(flowId);
const binaryId = `${flowId}-${FlowsPersistenceSqlite.VIDEO_FILE_ID}`;
const stmt = this.db.prepare('INSERT OR REPLACE INTO binary_files (id, flow_id, file_id, mime_type, content) VALUES (?, ?, ?, ?, ?)');
stmt.run(binaryId, flowId, FlowsPersistenceSqlite.VIDEO_FILE_ID, FlowsPersistenceSqlite.VIDEO_MIME_TYPE, bytes);
}
async getVideoSegment(flowId, startOffset, length) {
const stmt = this.db.prepare('SELECT content FROM binary_files WHERE flow_id = ? AND file_id = ?');
const row = stmt.get(flowId, FlowsPersistenceSqlite.VIDEO_FILE_ID);
if (!row) {
return null;
}
const videoBuffer = row.content;
const totalLength = videoBuffer.length;
if (startOffset >= totalLength) {
return null;
}
const adjustedLength = Math.min(length, totalLength - startOffset);
const segmentBuffer = Buffer.alloc(adjustedLength);
// Copy the segment from the full video buffer.
videoBuffer.copy(segmentBuffer, 0, startOffset, startOffset + adjustedLength);
return {
bytes: segmentBuffer,
totalLength: totalLength,
startOffset: startOffset,
};
}
async getFlowFile(flowId, fileId) {
const stmt = this.db.prepare('SELECT content FROM binary_files WHERE flow_id = ? AND file_id = ?');
const row = stmt.get(flowId, fileId);
return row ? row.content : null;
}
async setFlowFile(flowId, fileId, fileBytes) {
// Ensure flow exists
await this.getFlowMetadataById(flowId);
const binaryId = `${flowId}-${fileId}`;
const mimeType = MiscUtils_1.MiscUtils.inferMimeType(fileId);
const stmt = this.db.prepare('INSERT OR REPLACE INTO binary_files (id, flow_id, file_id, mime_type, content) VALUES (?, ?, ?, ?, ?)');
stmt.run(binaryId, flowId, fileId, mimeType, fileBytes);
}
async setBrowserState(flowId, browserState) {
const serializedBrowserState = Buffer.from(JSON.stringify(browserState), 'utf-8');
await this.setFlowFile(flowId, FlowsPersistenceSqlite.BROWSER_STATE_FILE_ID, serializedBrowserState);
}
async getBrowserState(flowId) {
const browserStateRaw = await this.getFlowFile(flowId, FlowsPersistenceSqlite.BROWSER_STATE_FILE_ID);
if (browserStateRaw) {
return BrowserStorageState_1.BrowserStorageStateSchema.parse(JSON.parse(browserStateRaw.toString('utf-8')));
}
else {
return null;
}
}
async deleteFlow(flowId) {
// Delete from database with cascading delete to related tables
const stmt = this.db.prepare('DELETE FROM flow_metadata WHERE id = ?');
stmt.run(flowId);
}
// Database management
close() {
this.db.close();
}
}
exports.FlowsPersistenceSqlite = FlowsPersistenceSqlite;
FlowsPersistenceSqlite.VIDEO_MIME_TYPE = 'video/webm';
FlowsPersistenceSqlite.BROWSER_STATE_FILE_ID = 'browserstate.json';
FlowsPersistenceSqlite.VIDEO_FILE_ID = 'video.webm';
//# sourceMappingURL=FlowsPersistenceSqlite.js.map