donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
235 lines • 13 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.FlowsPersistenceFactoryImpl = void 0;
const promises_1 = __importDefault(require("fs/promises"));
const path_1 = __importDefault(require("path"));
const NoApiAccessTokenPresentException_1 = require("../exceptions/NoApiAccessTokenPresentException");
const FlowsPersistenceGoogleCloudStorage_1 = require("./FlowsPersistenceGoogleCloudStorage");
const FlowsPersistenceFilesystem_1 = require("./FlowsPersistenceFilesystem");
const FlowsPersistenceSupabase_1 = require("./FlowsPersistenceSupabase");
const SupabaseClient_1 = require("../clients/SupabaseClient");
const DonobuDeploymentEnvironment_1 = require("../models/DonobuDeploymentEnvironment");
const FlowsPersistenceAwsS3_1 = require("./FlowsPersistenceAwsS3");
const FlowsPersistenceSqlite_1 = require("./FlowsPersistenceSqlite");
const Logger_1 = require("../utils/Logger");
const MiscUtils_1 = require("../utils/MiscUtils");
const DonobuSqliteDb_1 = require("./DonobuSqliteDb");
const envVars_1 = require("../envVars");
/**
* A factory class for creating new FlowsPersistence instances. The type of persistence
* objects created varies depending on this factory's constructor arguments and the current API
* request context.
*/
class FlowsPersistenceFactoryImpl {
/**
* Creates an instance.
*
* @param deploymentEnvironment The deployment environment for this application. If this is set to
* a value of DonobuDeploymentEnvironment.DONOBU_HOSTED_MULTI_TENANT, then the
* supabaseJwtSecretKey must also be specified.
* @param awsS3Bucket The name of the AWS S3 bucket to use for persistence.
* @param awsS3Region The region of the AWS S3 bucket.
* @param googleCloudStorageBucket The name of the Google Cloud storage bucket to use for
* persistence. Note that no special authentication is performed, so the current environment
* must be able to reach said bucket. If deployed in Google Cloud, this may be done
* automatically. If running locally, this may mean you have to log in to Google Cloud using
* the Google Cloud CLI before running this application.
* @param supabaseJwtSecretKey The Supabase JWT secret key used to generate new JWTs for the
* purposes of authentication. This key is extremely sensitive! This parameter MUST be
* specified if deploymentEnvironment is set to DonobuDeploymentEnvironment.DONOBU_HOSTED_MULTI_TENANT.
* @param requestContextHolder The object used to get the current API request context.
*/
constructor(deploymentEnvironment, awsS3Bucket, awsS3Region, googleCloudStorageBucket, supabaseJwtSecretKey, requestContextHolder) {
this.deploymentEnvironment = deploymentEnvironment;
this.awsS3Bucket = awsS3Bucket;
this.awsS3Region = awsS3Region;
this.googleCloudStorageBucket = googleCloudStorageBucket;
this.supabaseJwtSecretKey = supabaseJwtSecretKey;
this.requestContextHolder = requestContextHolder;
if (deploymentEnvironment ===
DonobuDeploymentEnvironment_1.DonobuDeploymentEnvironment.DONOBU_HOSTED_MULTI_TENANT &&
!supabaseJwtSecretKey) {
throw new Error('supabaseJwtSecretKey cannot be null if deploymentEnvironment is set to DONOBU_HOSTED_MULTI_TENANT.');
}
}
/**
* Creates an instance, fetching the values for googleCloudStorageBucket and
* supabaseJwtSecretKey from the environment variables.
*/
static async fromEnvironment(deploymentEnvironment, requestContextHolder) {
if (deploymentEnvironment === DonobuDeploymentEnvironment_1.DonobuDeploymentEnvironment.LOCAL) {
await FlowsPersistenceFactoryImpl.migrateToSqlite();
}
const awsS3Bucket = process.env[envVars_1.ENV_VAR_NAMES.AWS_S3_BUCKET];
const awsS3Region = process.env[envVars_1.ENV_VAR_NAMES.AWS_S3_REGION];
const googleCloudStorageBucket = process.env[envVars_1.ENV_VAR_NAMES.GOOGLE_CLOUD_STORAGE_BUCKET];
const supabaseJwtSecretKey = process.env[envVars_1.ENV_VAR_NAMES.SUPABASE_JWT_SECRET_KEY];
return new FlowsPersistenceFactoryImpl(deploymentEnvironment, awsS3Bucket ? awsS3Bucket : null, awsS3Region ? awsS3Region : null, googleCloudStorageBucket ? googleCloudStorageBucket : null, supabaseJwtSecretKey ? supabaseJwtSecretKey : null, requestContextHolder);
}
/**
* Creates a new FlowsPersistence instance.
*
* @throws NoApiAccessTokenPresentException If the deploymentEnvironment has a value of
* DonobuDeploymentEnvironment.DONOBU_HOSTED_MULTI_TENANT, and the current RequestContext
* accessToken is null or invalid.
*/
async createPersistenceLayer() {
const requestContext = this.requestContextHolder.get();
if (this.awsS3Bucket && this.awsS3Region) {
return new FlowsPersistenceAwsS3_1.FlowsPersistenceAwsS3(this.awsS3Bucket, this.awsS3Region);
}
else if (this.googleCloudStorageBucket) {
return FlowsPersistenceGoogleCloudStorage_1.FlowsPersistenceGoogleCloudStorage.create(this.googleCloudStorageBucket);
}
else if (this.deploymentEnvironment === DonobuDeploymentEnvironment_1.DonobuDeploymentEnvironment.LOCAL) {
return await FlowsPersistenceSqlite_1.FlowsPersistenceSqlite.create((0, DonobuSqliteDb_1.getDonobuSqliteDatabase)());
}
else if (this.deploymentEnvironment ===
DonobuDeploymentEnvironment_1.DonobuDeploymentEnvironment.DONOBU_HOSTED_MULTI_TENANT) {
if (!requestContext.accessToken) {
throw new NoApiAccessTokenPresentException_1.NoApiAccessTokenPresentException();
}
return new FlowsPersistenceSupabase_1.FlowsPersistenceSupabase(SupabaseClient_1.SupabaseClient.createClient(requestContext.accessToken, this.supabaseJwtSecretKey));
}
throw new Error('No valid flow persistence implementation available!');
}
/** Returns a list of all valid persistence layers. Note that the returned list may be empty! */
async createPersistenceLayers() {
const flowsPersistences = [];
const requestContext = this.requestContextHolder.get();
if (this.googleCloudStorageBucket) {
flowsPersistences.push(await FlowsPersistenceGoogleCloudStorage_1.FlowsPersistenceGoogleCloudStorage.create(this.googleCloudStorageBucket));
}
if (this.deploymentEnvironment ===
DonobuDeploymentEnvironment_1.DonobuDeploymentEnvironment.DONOBU_HOSTED_MULTI_TENANT &&
requestContext.accessToken &&
this.supabaseJwtSecretKey) {
flowsPersistences.push(new FlowsPersistenceSupabase_1.FlowsPersistenceSupabase(SupabaseClient_1.SupabaseClient.createClient(requestContext.accessToken, this.supabaseJwtSecretKey)));
}
else if (this.deploymentEnvironment === DonobuDeploymentEnvironment_1.DonobuDeploymentEnvironment.LOCAL) {
flowsPersistences.push(await FlowsPersistenceSqlite_1.FlowsPersistenceSqlite.create((0, DonobuSqliteDb_1.getDonobuSqliteDatabase)()));
}
return flowsPersistences;
}
static async migrateToSqlite() {
const legacyFlowsDir = path_1.default.join(MiscUtils_1.MiscUtils.baseWorkingDirectory(), 'flows');
// Check if the `flows` directory exists.
try {
await promises_1.default.access(legacyFlowsDir);
}
catch (_error) {
// No legacy flows to migrate.
return;
}
const legacyPersistence = await FlowsPersistenceFilesystem_1.FlowsPersistenceFilesystem.create(MiscUtils_1.MiscUtils.baseWorkingDirectory());
const sqlitePersistence = await FlowsPersistenceSqlite_1.FlowsPersistenceSqlite.create((0, DonobuSqliteDb_1.getDonobuSqliteDatabase)());
try {
// Use pagination to fetch all flows.
let allFlows = [];
let nextPageToken;
// Collect all flows using pagination.
do {
const result = await legacyPersistence.getFlows({
limit: 100, // Use large batch size for migration
pageToken: nextPageToken,
});
allFlows.push(...result.items);
nextPageToken = result.nextPageToken;
} while (nextPageToken);
Logger_1.appLogger.info(`Migrating ${allFlows.length} flows to SQLite...`);
for (const metadata of allFlows) {
try {
await sqlitePersistence.saveMetadata(metadata);
// Migrate tool calls for each flow...
const toolCalls = (await legacyPersistence.getToolCalls(metadata.id)).map((t) => FlowsPersistenceFactoryImpl.revampToolCall(t));
for (const toolCall of toolCalls) {
await sqlitePersistence.saveToolCall(metadata.id, toolCall);
// Migrate the images of each tool call...
if (toolCall.postCallImageId) {
const image = await legacyPersistence.getFlowFile(metadata.id, toolCall.postCallImageId);
if (image) {
await sqlitePersistence.setFlowFile(metadata.id, toolCall.postCallImageId, image);
}
}
}
// Migrate browser state for each flow...
const browserState = await legacyPersistence.getBrowserState(metadata.id);
if (browserState) {
await sqlitePersistence.setBrowserState(metadata.id, browserState);
}
// Migrate the video of each flow...
const flowVideo = await legacyPersistence.getVideoSegment(metadata.id, 0, Number.MAX_SAFE_INTEGER);
if (flowVideo) {
await sqlitePersistence.setVideo(metadata.id, flowVideo.bytes);
}
Logger_1.appLogger.info(`Successfully migrated flow ${metadata.id}`);
}
catch (error) {
Logger_1.appLogger.error(`Failed to migrate flow ${metadata.id} to SQLite`, error);
}
}
// Only remove the directory if migration was successful
if (allFlows.length > 0) {
Logger_1.appLogger.info('Migration complete. Removing legacy flows directory...');
await promises_1.default.rm(legacyPersistence.flowsDirectory, {
recursive: true,
force: true,
});
}
}
catch (error) {
Logger_1.appLogger.error('Failed to migrate flows to SQLite', error);
}
}
/**
* Update a given tool call to conform with the latest expected schema.
*/
static revampToolCall(toolCall) {
const outcomeMetadata = toolCall.outcome.metadata;
const isLegacyMetadata = (outcomeMetadata?.elementSelectorCandidatesForReplay ?? false)
? true
: false;
if (isLegacyMetadata) {
// Remap the legacy selector metadata to the new format.
outcomeMetadata.element =
outcomeMetadata.elementSelectorCandidatesForReplay;
outcomeMetadata.frame = outcomeMetadata.frameSelectorForReplay;
delete outcomeMetadata.elementSelectorCandidatesForReplay;
delete outcomeMetadata.frameSelectorForReplay;
}
const isLegacyRerun = (toolCall.parameters.selectorForReplay ?? false) ? true : false;
if (isLegacyRerun) {
// Remap the legacy rerun params to the new format.
toolCall.parameters.selector = {
element: toolCall.parameters.selectorForReplay
.elementSelectorCandidatesForReplay,
frame: toolCall.parameters.selectorForReplay.frameSelectorForReplay,
};
delete toolCall.parameters.selectorForReplay;
}
else if (toolCall.parameters.selectorForReplay === null) {
delete toolCall.parameters.selectorForReplay;
}
// Remap legacy tool names.
if (toolCall.toolName === 'clickButton') {
return {
...toolCall,
toolName: 'click',
};
}
else if (toolCall.toolName === 'navigateToWebpage') {
return {
...toolCall,
toolName: 'goToWebpage',
};
}
else {
return toolCall;
}
}
}
exports.FlowsPersistenceFactoryImpl = FlowsPersistenceFactoryImpl;
//# sourceMappingURL=FlowsPersistenceFactoryImpl.js.map