donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
426 lines • 18.4 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.FlowsPersistenceGoogleCloudStorage = void 0;
const FlowNotFoundException_1 = require("../exceptions/FlowNotFoundException");
const Logger_1 = require("../utils/Logger");
const JsonUtils_1 = require("../utils/JsonUtils");
/**
* A persistence implementation that uses Google Cloud Storage.
*
* If you are running locally, be sure to have the Google Cloud CLI installed
* so that you can authenticate using `gcloud auth application-default login`
* before running Donobu.
*
* If Donobu is being run in the Google cloud, be sure that the service account
* in which it is being run has permissions to read and write the Google Cloud
* Storage bucket specified as a constructor argument.
*/
class FlowsPersistenceGoogleCloudStorage {
/**
* Dynamically imports the Google Cloud Storage library.
* This helps to delay loading the library until it's actually needed.
*/
static async importGoogleCloudStorage() {
try {
// Dynamic import to delay loading until needed
const module = await import('@google-cloud/storage');
return {
Storage: module.Storage,
};
}
catch (error) {
Logger_1.appLogger.error('Failed to import Google Cloud Storage', error);
throw new Error('Failed to import Google Cloud Storage library');
}
}
static async create(bucketName) {
// Only import the Google Cloud Storage library when creating an instance
const { Storage } = await FlowsPersistenceGoogleCloudStorage.importGoogleCloudStorage();
const storage = new Storage();
const bucket = storage.bucket(bucketName);
return bucket
.exists()
.then(([exists]) => {
if (!exists) {
throw new Error(`Bucket ${bucketName} does not exist.`);
}
else {
return bucket;
}
})
.then((bucket) => {
return new FlowsPersistenceGoogleCloudStorage(bucket);
});
}
constructor(bucket) {
this.bucket = bucket;
}
async saveMetadata(flowMetadata) {
const objectName = `${flowMetadata.id}/${FlowsPersistenceGoogleCloudStorage.METADATA_FILENAME}`;
await this.uploadString(objectName, JSON.stringify(flowMetadata), 'application/json');
// Update the name index if a name exists
if (flowMetadata.name) {
await this.updateNameIndex(flowMetadata.id, flowMetadata.name);
}
// Update runMode index
await this.updateRunModeIndex(flowMetadata.id, flowMetadata.runMode);
// Update state index
await this.updateStateIndex(flowMetadata.id, flowMetadata.state);
// Update combined runMode+state index
await this.updateRunModeStateIndex(flowMetadata.id, flowMetadata.runMode, flowMetadata.state);
}
async getMetadataByFlowId(flowId) {
const objectName = `${flowId}/${FlowsPersistenceGoogleCloudStorage.METADATA_FILENAME}`;
const jsonData = await this.downloadString(objectName);
if (!jsonData) {
throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
}
return JSON.parse(jsonData);
}
async getMetadataByFlowName(flowName) {
if (!flowName) {
throw FlowNotFoundException_1.FlowNotFoundException.forName(flowName);
}
const indexKey = `${FlowsPersistenceGoogleCloudStorage.NAME_INDEX_PREFIX}${encodeURIComponent(flowName)}`;
const flowId = await this.downloadString(indexKey);
if (!flowId) {
throw FlowNotFoundException_1.FlowNotFoundException.forName(flowName);
}
try {
return await this.getMetadataByFlowId(flowId);
}
catch (error) {
// If the index points to a non-existent flow, clean up the index
Logger_1.appLogger.warn(`Flow referenced in name index not found: ${flowId}`, error);
// Delete the stale index entry
const indexFile = this.bucket.file(indexKey);
await indexFile.delete().catch((e) => {
Logger_1.appLogger.warn(`Failed to delete stale name index: ${indexKey}`, e);
});
throw FlowNotFoundException_1.FlowNotFoundException.forName(flowName);
}
}
async savePngScreenShot(flowId, bytes) {
const filename = `${new Date().toISOString()}${FlowsPersistenceGoogleCloudStorage.SCREENSHOT_FILENAME_SUFFIX}`;
const objectName = `${flowId}/${filename}`;
await this.uploadBytes(objectName, bytes, 'image/png');
return filename;
}
async getPngScreenShot(flowId, screenShotId) {
return this.getFlowFile(flowId, screenShotId);
}
async saveToolCall(flowId, toolCall) {
const filename = `${toolCall.startedAt}${FlowsPersistenceGoogleCloudStorage.TOOL_CALL_FILENAME_SUFFIX}`;
const objectName = `${flowId}/${filename}`;
await this.uploadString(objectName, JSON.stringify(toolCall), 'application/json');
}
async getToolCalls(flowId) {
const prefix = `${flowId}/`;
const [files] = await this.bucket.getFiles({ prefix });
const toolCalls = [];
for (const file of files) {
if (file.name.endsWith(FlowsPersistenceGoogleCloudStorage.TOOL_CALL_FILENAME_SUFFIX)) {
const [content] = await file.download();
const toolCall = JSON.parse(content.toString());
toolCalls.push(toolCall);
}
}
if (toolCalls.length === 0) {
throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
}
return toolCalls.sort((a, b) => (a.startedAt || 0) - (b.startedAt || 0));
}
async getFlows(query) {
const { limit = 20, pageToken, name, runMode, state, startedAfter, startedBefore, } = query || {};
const validLimit = Math.min(Math.max(1, limit), 100);
// Determine which index to use based on query params
let prefix = '';
let usedIndex = false;
if (name) {
// If querying by name, directly fetch that specific flow
try {
const flow = await this.getMetadataByFlowName(name);
// Apply other filters
let includeFlow = true;
if (runMode && flow.runMode !== runMode) {
includeFlow = false;
}
if (state && flow.state !== state) {
includeFlow = false;
}
// Apply date range filters
if (startedAfter !== undefined &&
(flow.startedAt || 0) < startedAfter) {
includeFlow = false;
}
if (startedBefore !== undefined &&
(flow.startedAt || 0) > startedBefore) {
includeFlow = false;
}
if (includeFlow) {
return { items: [flow], nextPageToken: undefined };
}
else {
return { items: [], nextPageToken: undefined };
}
}
catch (_e) {
return { items: [], nextPageToken: undefined };
}
}
else if (runMode && state) {
// Use combined index if both runMode and state are specified
prefix = `${FlowsPersistenceGoogleCloudStorage.RUN_MODE_STATE_INDEX_PREFIX}${runMode}_${state}/`;
usedIndex = true;
}
else if (runMode) {
// Use runMode index
prefix = `${FlowsPersistenceGoogleCloudStorage.RUN_MODE_INDEX_PREFIX}${runMode}/`;
usedIndex = true;
}
else if (state) {
// Use state index
prefix = `${FlowsPersistenceGoogleCloudStorage.STATE_INDEX_PREFIX}${state}/`;
usedIndex = true;
}
// Prepare query options
const queryOptions = {
maxResults: validLimit * 2,
pageToken: pageToken,
prefix: usedIndex ? prefix : undefined,
};
// Get files based on query
const [files, nextQueryObj] = await this.bucket.getFiles(queryOptions);
// Extract flow IDs - either from index files or from metadata files
let flowIds = [];
if (usedIndex) {
// If using an index, extract flow IDs from index file names
flowIds = files.map((file) => {
// Extract flowId from the index file path
return file.name.substring(prefix.length);
});
}
else {
// If not using an index, filter to metadata files
const metadataFiles = files.filter((file) => file.name.endsWith(`/${FlowsPersistenceGoogleCloudStorage.METADATA_FILENAME}`));
// Extract flow IDs from metadata file paths
flowIds = metadataFiles.map((file) => {
return file.name.substring(0, file.name.indexOf('/'));
});
}
// Fetch metadata for each flow
let flows = [];
for (const flowId of flowIds) {
try {
const flowMetadata = await this.getMetadataByFlowId(flowId);
// Apply remaining filters
let includeFlow = true;
// Apply date range filters
if (startedAfter !== undefined &&
(flowMetadata.startedAt || 0) < startedAfter) {
includeFlow = false;
}
if (startedBefore !== undefined &&
(flowMetadata.startedAt || 0) > startedBefore) {
includeFlow = false;
}
// Double-check the indexed fields (in case index is out of sync)
if (runMode && flowMetadata.runMode !== runMode) {
includeFlow = false;
}
if (state && flowMetadata.state !== state) {
includeFlow = false;
}
if (includeFlow) {
flows.push(flowMetadata);
}
}
catch (e) {
Logger_1.appLogger.warn(`Flow not found: ${flowId}`, e);
}
}
// Sort by creation date (newest first)
flows.sort((a, b) => (b.startedAt || 0) - (a.startedAt || 0));
// Apply limit
const limitedFlows = flows.slice(0, validLimit);
const hasMore = flows.length > validLimit;
return {
items: limitedFlows,
nextPageToken: hasMore ? nextQueryObj?.pageToken : undefined,
};
}
async setVideo(flowId, bytes) {
const objectName = `${flowId}/${FlowsPersistenceGoogleCloudStorage.VIDEO_FILENAME}`;
await this.uploadBytes(objectName, bytes, 'video/webm');
}
async getVideoSegment(flowId, startOffset, length) {
const objectName = `${flowId}/${FlowsPersistenceGoogleCloudStorage.VIDEO_FILENAME}`;
const file = this.bucket.file(objectName);
const [exists] = await file.exists();
if (!exists) {
return null;
}
const [metadata] = await file.getMetadata();
const totalLength = Number(metadata.size);
const adjustedLength = Math.min(length, totalLength - startOffset);
if (adjustedLength <= 0) {
return null;
}
try {
const [content] = await file.download({
start: startOffset,
end: startOffset + adjustedLength - 1,
});
return {
bytes: content,
totalLength: totalLength,
startOffset: startOffset,
};
}
catch (e) {
Logger_1.appLogger.error(`Error reading video segment for flowId ${flowId}: ${e}`);
return null;
}
}
async getFlowFile(flowId, fileId) {
const objectName = `${flowId}/${fileId}`;
const file = this.bucket.file(objectName);
const [exists] = await file.exists();
if (!exists) {
return null;
}
const [content] = await file.download();
return content;
}
async setFlowFile(flowId, fileId, fileBytes) {
const objectName = `${flowId}/${fileId}`;
await this.uploadBytes(objectName, fileBytes, 'application/octet-stream');
}
async setBrowserState(flowId, browserState) {
const serializedBrowserState = Buffer.from(JSON.stringify(browserState), 'utf-8');
await this.setFlowFile(flowId, FlowsPersistenceGoogleCloudStorage.BROWSER_STATE_FILENAME, serializedBrowserState);
}
async getBrowserState(flowId) {
const browserStateRaw = await this.getFlowFile(flowId, FlowsPersistenceGoogleCloudStorage.BROWSER_STATE_FILENAME);
if (browserStateRaw) {
const browserState = JsonUtils_1.JsonUtils.jsonStringToJsonObject(browserStateRaw.toString('utf-8'));
if (browserState) {
return browserState;
}
else {
throw new Error(`Cannot load malformed browser state from flow ${flowId}`);
}
}
else {
return null;
}
}
async deleteFlow(flowId) {
try {
// Get flow metadata to find its fields for index cleanup
const metadata = await this.getMetadataByFlowId(flowId);
// Delete the name index entry if name exists
if (metadata.name) {
const indexKey = `${FlowsPersistenceGoogleCloudStorage.NAME_INDEX_PREFIX}${encodeURIComponent(metadata.name)}`;
const indexFile = this.bucket.file(indexKey);
const [exists] = await indexFile.exists();
if (exists) {
await indexFile.delete();
}
}
// Delete runMode index
const runModeIndexKey = `${FlowsPersistenceGoogleCloudStorage.RUN_MODE_INDEX_PREFIX}${metadata.runMode}/${flowId}`;
const runModeIndexFile = this.bucket.file(runModeIndexKey);
const [runModeExists] = await runModeIndexFile.exists();
if (runModeExists) {
await runModeIndexFile.delete();
}
// Delete state index
const stateIndexKey = `${FlowsPersistenceGoogleCloudStorage.STATE_INDEX_PREFIX}${metadata.state}/${flowId}`;
const stateIndexFile = this.bucket.file(stateIndexKey);
const [stateExists] = await stateIndexFile.exists();
if (stateExists) {
await stateIndexFile.delete();
}
// Delete combined index
const combinedIndexKey = `${FlowsPersistenceGoogleCloudStorage.RUN_MODE_STATE_INDEX_PREFIX}${metadata.runMode}_${metadata.state}/${flowId}`;
const combinedIndexFile = this.bucket.file(combinedIndexKey);
const [combinedExists] = await combinedIndexFile.exists();
if (combinedExists) {
await combinedIndexFile.delete();
}
}
catch (_error) {
// Pass - couldn't load metadata to clean up indexes. This likely means
// the flow has already been deleted.
}
// Delete all flow files
const prefix = `${flowId}/`;
const [files] = await this.bucket.getFiles({ prefix });
if (files.length > 0) {
await Promise.all(files.map((file) => file.delete()));
}
}
async uploadString(objectName, data, contentType) {
const file = this.bucket.file(objectName);
await file.save(data, {
contentType,
metadata: { contentType },
});
}
async uploadBytes(objectName, bytes, contentType) {
const file = this.bucket.file(objectName);
await file.save(bytes, {
contentType,
metadata: { contentType },
});
}
async downloadString(objectName) {
const file = this.bucket.file(objectName);
const [exists] = await file.exists();
if (!exists) {
return null;
}
const [content] = await file.download();
return content.toString();
}
/**
* Update name-to-id index.
*/
async updateNameIndex(flowId, flowName) {
const indexKey = `${FlowsPersistenceGoogleCloudStorage.NAME_INDEX_PREFIX}${encodeURIComponent(flowName)}`;
await this.uploadString(indexKey, flowId, 'text/plain');
}
/**
* Update runMode-to-id index.
*/
async updateRunModeIndex(flowId, runMode) {
const indexKey = `${FlowsPersistenceGoogleCloudStorage.RUN_MODE_INDEX_PREFIX}${runMode}/${flowId}`;
await this.uploadString(indexKey, flowId, 'text/plain');
}
/**
* Update state-to-id index.
*/
async updateStateIndex(flowId, state) {
const indexKey = `${FlowsPersistenceGoogleCloudStorage.STATE_INDEX_PREFIX}${state}/${flowId}`;
await this.uploadString(indexKey, flowId, 'text/plain');
}
/**
* Update combined runMode+state-to-id index.
*/
async updateRunModeStateIndex(flowId, runMode, state) {
const indexKey = `${FlowsPersistenceGoogleCloudStorage.RUN_MODE_STATE_INDEX_PREFIX}${runMode}_${state}/${flowId}`;
await this.uploadString(indexKey, flowId, 'text/plain');
}
}
exports.FlowsPersistenceGoogleCloudStorage = FlowsPersistenceGoogleCloudStorage;
FlowsPersistenceGoogleCloudStorage.NAME_INDEX_PREFIX = '_name-index/';
FlowsPersistenceGoogleCloudStorage.RUN_MODE_INDEX_PREFIX = '_run-mode-index/';
FlowsPersistenceGoogleCloudStorage.STATE_INDEX_PREFIX = '_state-index/';
FlowsPersistenceGoogleCloudStorage.RUN_MODE_STATE_INDEX_PREFIX = '_run-mode-state-index/';
FlowsPersistenceGoogleCloudStorage.METADATA_FILENAME = 'metadata.json';
FlowsPersistenceGoogleCloudStorage.BROWSER_STATE_FILENAME = 'browserstate.json';
FlowsPersistenceGoogleCloudStorage.SCREENSHOT_FILENAME_SUFFIX = '.screenshot.png';
FlowsPersistenceGoogleCloudStorage.TOOL_CALL_FILENAME_SUFFIX = '.tool-call.json';
FlowsPersistenceGoogleCloudStorage.VIDEO_FILENAME = 'video.webm';
//# sourceMappingURL=FlowsPersistenceGoogleCloudStorage.js.map