donobu
Version:
Create browser automations with an LLM agent and replay them as Playwright scripts.
445 lines • 18.2 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.FlowsPersistenceAwsS3 = void 0;
const client_s3_1 = require("@aws-sdk/client-s3");
const FlowNotFoundException_1 = require("../exceptions/FlowNotFoundException");
const Logger_1 = require("../utils/Logger");
const JsonUtils_1 = require("../utils/JsonUtils");
/**
* A persistence implementation that uses AWS S3.
* Ensure AWS credentials are properly configured in your environment,
* or provide them directly in the constructor.
*/
class FlowsPersistenceAwsS3 {
/**
* Creates a new instance of the AWS S3 persistence layer.
*
* @param bucketName The name of the S3 bucket to use for storage
* @param region AWS region (defaults to 'us-east-1')
* @param credentials Optional AWS credentials (if not provided, will use environment variables)
*/
constructor(bucketName, region, credentials) {
this.bucketName = bucketName;
this.s3Client = new client_s3_1.S3Client({
region,
credentials,
});
}
async saveMetadata(flowMetadata) {
const objectKey = `${flowMetadata.id}/${FlowsPersistenceAwsS3.METADATA_FILENAME}`;
// Save the main metadata file.
await this.s3Client.send(new client_s3_1.PutObjectCommand({
Bucket: this.bucketName,
Key: objectKey,
Body: JSON.stringify(flowMetadata, null, 2),
ContentType: 'application/json',
}));
// Update the name index if a name exists.
if (flowMetadata.name) {
await this.updateNameIndex(flowMetadata.id, flowMetadata.name);
}
// Update runMode index
await this.updateRunModeIndex(flowMetadata.id, flowMetadata.runMode);
// Update state index
await this.updateStateIndex(flowMetadata.id, flowMetadata.state);
// Update combined runMode+state index
await this.updateRunModeStateIndex(flowMetadata.id, flowMetadata.runMode, flowMetadata.state);
}
async getMetadataByFlowId(flowId) {
const objectKey = `${flowId}/${FlowsPersistenceAwsS3.METADATA_FILENAME}`;
const response = await this.s3Client.send(new client_s3_1.GetObjectCommand({
Bucket: this.bucketName,
Key: objectKey,
}));
if (!response.Body) {
throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
}
const data = await this.streamToBuffer(response.Body);
return JSON.parse(data.toString('utf-8'));
}
async getMetadataByFlowName(flowName) {
if (!flowName) {
throw FlowNotFoundException_1.FlowNotFoundException.forName('null');
}
const indexKey = `${FlowsPersistenceAwsS3.NAME_INDEX_PREFIX}${encodeURIComponent(flowName)}`;
// Look up the flow ID in the name index
const response = await this.s3Client.send(new client_s3_1.GetObjectCommand({
Bucket: this.bucketName,
Key: indexKey,
}));
if (!response.Body) {
throw FlowNotFoundException_1.FlowNotFoundException.forName(flowName);
}
// Extract the flow ID from the index entry
const data = await this.streamToBuffer(response.Body);
const flowId = data.toString('utf-8');
// Get the flow metadata using the ID
return await this.getMetadataByFlowId(flowId);
}
async savePngScreenShot(flowId, bytes) {
const filename = `${new Date().toISOString()}${FlowsPersistenceAwsS3.SCREENSHOT_FILENAME_SUFFIX}`;
const objectKey = `${flowId}/${filename}`;
await this.s3Client.send(new client_s3_1.PutObjectCommand({
Bucket: this.bucketName,
Key: objectKey,
Body: bytes,
ContentType: 'image/png',
}));
return filename;
}
async getPngScreenShot(flowId, screenShotId) {
const objectKey = `${flowId}/${screenShotId}`;
try {
const response = await this.s3Client.send(new client_s3_1.GetObjectCommand({
Bucket: this.bucketName,
Key: objectKey,
}));
if (!response.Body || response.ContentType !== 'image/png') {
return null;
}
return await this.streamToBuffer(response.Body);
}
catch (_error) {
return null;
}
}
async saveToolCall(flowId, toolCall) {
const filename = `${toolCall.startedAt}${FlowsPersistenceAwsS3.TOOL_CALL_FILENAME_SUFFIX}`;
const objectKey = `${flowId}/${filename}`;
await this.s3Client.send(new client_s3_1.PutObjectCommand({
Bucket: this.bucketName,
Key: objectKey,
Body: JSON.stringify(toolCall, null, 2),
ContentType: 'application/json',
}));
}
async getToolCalls(flowId) {
const prefix = `${flowId}/`;
const response = await this.s3Client.send(new client_s3_1.ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: prefix,
}));
if (!response.Contents || response.Contents.length === 0) {
throw FlowNotFoundException_1.FlowNotFoundException.forId(flowId);
}
const toolCallFiles = response.Contents.filter((item) => item.Key &&
item.Key.endsWith(FlowsPersistenceAwsS3.TOOL_CALL_FILENAME_SUFFIX));
if (toolCallFiles.length === 0) {
return [];
}
const toolCalls = await Promise.all(toolCallFiles.map(async (file) => {
const response = await this.s3Client.send(new client_s3_1.GetObjectCommand({
Bucket: this.bucketName,
Key: file.Key,
}));
if (!response.Body) {
return null;
}
const data = await this.streamToBuffer(response.Body);
return JSON.parse(data.toString('utf-8'));
}));
return toolCalls
.filter((toolCall) => toolCall !== null)
.sort((a, b) => (a.startedAt || 0) - (b.startedAt || 0));
}
async getFlows(query) {
const { limit = 20, pageToken, name, runMode, state, startedAfter, startedBefore, } = query || {};
const validLimit = Math.min(Math.max(1, limit), 100);
// Determine which index to use based on query params
let prefix = '';
let usedIndex = false;
if (name) {
// If querying by name, directly fetch that specific flow
try {
const flow = await this.getMetadataByFlowName(name);
// Apply other filters
let includeFlow = true;
if (runMode && flow.runMode !== runMode) {
includeFlow = false;
}
if (state && flow.state !== state) {
includeFlow = false;
}
// Apply date range filters
if (startedAfter !== undefined &&
(flow.startedAt || 0) < startedAfter) {
includeFlow = false;
}
if (startedBefore !== undefined &&
(flow.startedAt || 0) > startedBefore) {
includeFlow = false;
}
if (includeFlow) {
return { items: [flow], nextPageToken: undefined };
}
else {
return { items: [], nextPageToken: undefined };
}
}
catch (_error) {
return { items: [], nextPageToken: undefined };
}
}
else if (runMode && state) {
// Use combined index if both runMode and state are specified
prefix = `${FlowsPersistenceAwsS3.RUN_MODE_STATE_INDEX_PREFIX}${runMode}_${state}/`;
usedIndex = true;
}
else if (runMode) {
// Use runMode index
prefix = `${FlowsPersistenceAwsS3.RUN_MODE_INDEX_PREFIX}${runMode}/`;
usedIndex = true;
}
else if (state) {
// Use state index
prefix = `${FlowsPersistenceAwsS3.STATE_INDEX_PREFIX}${state}/`;
usedIndex = true;
}
// Configure the list command
const listCommand = new client_s3_1.ListObjectsV2Command({
Bucket: this.bucketName,
MaxKeys: validLimit * 2, // Request more than needed for filtering
ContinuationToken: pageToken,
Prefix: usedIndex ? prefix : undefined,
});
const response = await this.s3Client.send(listCommand);
if (!response.Contents) {
return { items: [] };
}
// Process objects based on whether we're using an index or not
let flowIds = [];
if (usedIndex) {
// If using an index, extract flow IDs from index file keys
flowIds = response.Contents.filter((item) => item.Key).map((item) => {
const key = item.Key;
return key.substring(prefix.length);
});
}
else {
// If not using an index, filter to metadata files
const metadataFiles = response.Contents.filter((item) => item.Key &&
item.Key.endsWith(`/${FlowsPersistenceAwsS3.METADATA_FILENAME}`));
// Extract flow IDs from metadata file paths
flowIds = metadataFiles.map((file) => {
const key = file.Key;
return key.substring(0, key.indexOf('/'));
});
}
// Fetch metadata for each flow and apply remaining filters
const flows = [];
for (const flowId of flowIds) {
try {
const flowMetadata = await this.getMetadataByFlowId(flowId);
// Apply remaining filters
let includeFlow = true;
// Apply date range filters
if (startedAfter !== undefined &&
(flowMetadata.startedAt || 0) < startedAfter) {
includeFlow = false;
}
if (startedBefore !== undefined &&
(flowMetadata.startedAt || 0) > startedBefore) {
includeFlow = false;
}
// Double-check indexed fields (in case index is out of sync)
if (runMode && flowMetadata.runMode !== runMode) {
includeFlow = false;
}
if (state && flowMetadata.state !== state) {
includeFlow = false;
}
if (includeFlow) {
flows.push(flowMetadata);
}
}
catch (e) {
Logger_1.appLogger.warn(`Flow not found: ${flowId}`, e);
}
}
// Sort by creation date (newest first) - fixing the sort order
flows.sort((a, b) => (b.startedAt || 0) - (a.startedAt || 0));
// Apply limit
const limitedFlows = flows.slice(0, validLimit);
const hasMore = flows.length > validLimit;
return {
items: limitedFlows,
nextPageToken: hasMore ? response.NextContinuationToken : undefined,
};
}
async setVideo(flowId, bytes) {
const objectKey = `${flowId}/${FlowsPersistenceAwsS3.VIDEO_FILENAME}`;
await this.s3Client.send(new client_s3_1.PutObjectCommand({
Bucket: this.bucketName,
Key: objectKey,
Body: bytes,
ContentType: 'video/webm',
}));
}
async getVideoSegment(flowId, startOffset, length) {
const objectKey = `${flowId}/${FlowsPersistenceAwsS3.VIDEO_FILENAME}`;
// First, get the total size of the video file
const headResponse = await this.s3Client.send(new client_s3_1.HeadObjectCommand({
Bucket: this.bucketName,
Key: objectKey,
}));
const totalLength = headResponse.ContentLength || 0;
const adjustedLength = Math.min(length, totalLength - startOffset);
if (adjustedLength <= 0) {
return null;
}
const range = `bytes=${startOffset}-${startOffset + adjustedLength - 1}`;
const response = await this.s3Client.send(new client_s3_1.GetObjectCommand({
Bucket: this.bucketName,
Key: objectKey,
Range: range,
}));
if (!response.Body) {
return null;
}
const buffer = await this.streamToBuffer(response.Body);
return {
bytes: buffer,
totalLength: totalLength,
startOffset: startOffset,
};
}
async getFlowFile(flowId, fileId) {
const objectKey = `${flowId}/${fileId}`;
const response = await this.s3Client.send(new client_s3_1.GetObjectCommand({
Bucket: this.bucketName,
Key: objectKey,
}));
if (!response.Body) {
return null;
}
return await this.streamToBuffer(response.Body);
}
async setFlowFile(flowId, fileId, fileBytes) {
const objectKey = `${flowId}/${fileId}`;
await this.s3Client.send(new client_s3_1.PutObjectCommand({
Bucket: this.bucketName,
Key: objectKey,
Body: fileBytes,
ContentType: 'application/octet-stream',
}));
}
async setBrowserState(flowId, browserState) {
const serializedBrowserState = Buffer.from(JSON.stringify(browserState), 'utf-8');
await this.setFlowFile(flowId, FlowsPersistenceAwsS3.BROWSER_STATE_FILENAME, serializedBrowserState);
}
async getBrowserState(flowId) {
const browserStateRaw = await this.getFlowFile(flowId, FlowsPersistenceAwsS3.BROWSER_STATE_FILENAME);
if (browserStateRaw) {
const browserState = JsonUtils_1.JsonUtils.jsonStringToJsonObject(browserStateRaw.toString('utf-8'));
if (browserState) {
return browserState;
}
else {
throw new Error(`Cannot load malformed browser state from flow ${flowId}`);
}
}
else {
return null;
}
}
async deleteFlow(flowId) {
// Get flow metadata to find its name.
const metadata = await this.getMetadataByFlowId(flowId);
// Delete the name index entry if name exists.
if (metadata.name) {
const indexKey = `${FlowsPersistenceAwsS3.NAME_INDEX_PREFIX}${encodeURIComponent(metadata.name)}`;
await this.s3Client.send(new client_s3_1.DeleteObjectCommand({
Bucket: this.bucketName,
Key: indexKey,
}));
}
const prefix = `${flowId}/`;
// List all objects with the flow prefix.
const response = await this.s3Client.send(new client_s3_1.ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: prefix,
}));
if (!response.Contents || response.Contents.length === 0) {
return;
}
// Delete each object.
for (const object of response.Contents) {
if (object.Key) {
await this.s3Client.send(new client_s3_1.DeleteObjectCommand({
Bucket: this.bucketName,
Key: object.Key,
}));
}
}
}
/**
* Utility method to stream S3 object data to a Buffer
*/
async streamToBuffer(stream) {
return new Promise((resolve, reject) => {
const chunks = [];
stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
stream.on('error', reject);
stream.on('end', () => resolve(Buffer.concat(chunks)));
});
}
/**
* Update name-to-id index.
*/
async updateNameIndex(flowId, flowName) {
const indexKey = `${FlowsPersistenceAwsS3.NAME_INDEX_PREFIX}${encodeURIComponent(flowName)}`;
await this.s3Client.send(new client_s3_1.PutObjectCommand({
Bucket: this.bucketName,
Key: indexKey,
Body: flowId,
ContentType: 'text/plain',
}));
}
/**
* Update runMode-to-id index.
*/
async updateRunModeIndex(flowId, runMode) {
const indexKey = `${FlowsPersistenceAwsS3.RUN_MODE_INDEX_PREFIX}${runMode}/${flowId}`;
await this.s3Client.send(new client_s3_1.PutObjectCommand({
Bucket: this.bucketName,
Key: indexKey,
Body: flowId,
ContentType: 'text/plain',
}));
}
/**
* Update state-to-id index.
*/
async updateStateIndex(flowId, state) {
const indexKey = `${FlowsPersistenceAwsS3.STATE_INDEX_PREFIX}${state}/${flowId}`;
await this.s3Client.send(new client_s3_1.PutObjectCommand({
Bucket: this.bucketName,
Key: indexKey,
Body: flowId,
ContentType: 'text/plain',
}));
}
/**
* Update combined runMode+state-to-id index.
*/
async updateRunModeStateIndex(flowId, runMode, state) {
const indexKey = `${FlowsPersistenceAwsS3.RUN_MODE_STATE_INDEX_PREFIX}${runMode}_${state}/${flowId}`;
await this.s3Client.send(new client_s3_1.PutObjectCommand({
Bucket: this.bucketName,
Key: indexKey,
Body: flowId,
ContentType: 'text/plain',
}));
}
}
exports.FlowsPersistenceAwsS3 = FlowsPersistenceAwsS3;
FlowsPersistenceAwsS3.NAME_INDEX_PREFIX = '_name-index/';
FlowsPersistenceAwsS3.RUN_MODE_INDEX_PREFIX = '_run-mode-index/';
FlowsPersistenceAwsS3.STATE_INDEX_PREFIX = '_state-index/';
FlowsPersistenceAwsS3.RUN_MODE_STATE_INDEX_PREFIX = '_run-mode-state-index/';
FlowsPersistenceAwsS3.METADATA_FILENAME = 'metadata.json';
FlowsPersistenceAwsS3.BROWSER_STATE_FILENAME = 'browserstate.json';
FlowsPersistenceAwsS3.SCREENSHOT_FILENAME_SUFFIX = '.screenshot.png';
FlowsPersistenceAwsS3.TOOL_CALL_FILENAME_SUFFIX = '.tool-call.json';
FlowsPersistenceAwsS3.VIDEO_FILENAME = 'video.webm';
//# sourceMappingURL=FlowsPersistenceAwsS3.js.map