@wix-pilot/core
Version:
A flexible plugin that drives your tests with human-written commands, enhanced by the power of large language models (LLMs)
263 lines (262 loc) • 12.2 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AutoPerformer = void 0;
const extractTaggedOutputs_1 = require("../../common/extract/extractTaggedOutputs");
const format_utils_1 = require("./reviews/format-utils");
const logger_1 = __importDefault(require("../../common/logger"));
class AutoPerformer {
promptCreator;
stepPerformer;
promptHandler;
screenCapturer;
cacheHandler;
snapshotComparator;
constructor(promptCreator, stepPerformer, promptHandler, screenCapturer, cacheHandler, snapshotComparator) {
this.promptCreator = promptCreator;
this.stepPerformer = stepPerformer;
this.promptHandler = promptHandler;
this.screenCapturer = screenCapturer;
this.cacheHandler = cacheHandler;
this.snapshotComparator = snapshotComparator;
}
extractReviewOutput(text) {
const { summary, findings, score } = (0, extractTaggedOutputs_1.extractAutoPilotReviewOutputs)(text);
if (!summary) {
return null;
}
return {
summary,
findings: findings
? findings
.split("\n")
.map((finding) => finding.replace(/^- /, "").trim())
: undefined,
score,
};
}
formatReviewSection(review, typeObject) {
const summaryMessage = review.score
? `${review.summary} (Score: ${review.score})`
: review.summary;
const formattedResults = [];
formattedResults.push({ message: "\n", isBold: false, color: "white" });
formattedResults.push({
message: `${typeObject.title.toUpperCase()}: `,
isBold: true,
color: "green",
});
formattedResults.push(...(0, format_utils_1.parseFormattedText)(summaryMessage));
if (review.findings?.length) {
formattedResults.push({ message: "\n", isBold: false, color: "white" });
review.findings.forEach((finding) => {
formattedResults.push({ message: "\n", isBold: false, color: "white" });
formattedResults.push({
message: " - ",
isBold: false,
color: "white",
});
formattedResults.push(...(0, format_utils_1.parseFormattedText)(finding));
});
}
return formattedResults;
}
async analyseScreenAndCreatePilotStep(goal, previousSteps, screenCapture, reviewSectionTypes, maxAttempts = 2) {
const cacheKey = this.cacheHandler.generateCacheKey({
goal,
previousSteps,
});
if (this.cacheHandler.isCacheInUse() && cacheKey) {
const cacheResult = await this.getValueFromCache(cacheKey, screenCapture, reviewSectionTypes);
if (cacheResult) {
return cacheResult;
}
}
const analysisProgress = logger_1.default.startProgress({
actionLabel: "ANALYZE",
successLabel: "READY",
failureLabel: "FAILED",
}, {
message: "Analyzing current screen content and structure",
isBold: true,
color: "whiteBright",
});
const lastError = null;
let lastScreenDescription = "";
let lastAction = "";
for (let attempt = 0; attempt < maxAttempts; attempt++) {
try {
const prompt = this.promptCreator.createPrompt(goal, screenCapture.viewHierarchy ?? "Unknown view hierarchy", !!screenCapture.snapshot, previousSteps, reviewSectionTypes);
const promptResult = await this.promptHandler.runPrompt(prompt, screenCapture.snapshot);
const outputs = (0, extractTaggedOutputs_1.extractAutoPilotStepOutputs)(promptResult, reviewSectionTypes);
// These fields are required according to the extractor, so we can safely assert they exist
const thoughts = outputs.thoughts;
lastScreenDescription = outputs.screenDescription;
lastAction = outputs.action;
const plan = { action: lastAction, thoughts };
const goalAchieved = !!outputs.goalSummary;
analysisProgress.stop("success", {
message: "Screen analysis complete, next action determined",
isBold: true,
color: "green",
});
// Log thoughts with formatted text
const formattedThoughts = (0, format_utils_1.parseFormattedText)(thoughts);
logger_1.default.labeled("THOUGHTS").info(...formattedThoughts);
const review = {};
if (reviewSectionTypes && reviewSectionTypes.length > 0) {
reviewSectionTypes.forEach((reviewType) => {
const reviewContent = outputs[reviewType.title];
if (reviewContent) {
const extractedReview = this.extractReviewOutput(reviewContent);
if (extractedReview) {
review[reviewType.title] = extractedReview;
}
}
});
}
const hasReviews = Object.keys(review).length > 0;
if (hasReviews) {
this.logReviews(lastScreenDescription, review, reviewSectionTypes);
}
const summary = outputs.goalSummary;
if (this.cacheHandler.isCacheInUse() && cacheKey) {
const snapshotHashes = await this.cacheHandler.generateHashes(screenCapture);
const cacheValue = {
screenDescription: lastScreenDescription,
plan,
review,
goalAchieved,
summary,
};
this.cacheHandler.addToTemporaryCacheSnapshotBased(cacheKey, cacheValue, snapshotHashes);
}
return {
screenDescription: lastScreenDescription,
plan,
review,
goalAchieved,
summary,
};
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : error;
logger_1.default
.labeled("ERROR")
.error(`Analysis attempt ${attempt}/${maxAttempts} failed: ${errorMessage}`);
if (attempt < maxAttempts) {
logger_1.default.labeled("RETRYING").warn("Initiating new analysis attempt");
previousSteps = [
...previousSteps,
{
screenDescription: lastScreenDescription,
step: lastAction,
error: errorMessage,
},
];
}
else {
analysisProgress.stop("failure", `Screen analysis failed: ${errorMessage}`);
throw lastError;
}
}
}
throw new Error("Analysis failed to reach a decision");
}
async perform(goal, reviewSectionTypes) {
const maxSteps = 100;
const previousSteps = [];
const pilotSteps = [];
const report = { goal, steps: [] };
// Create the overall goal progress with minimal labels
const mainProgress = logger_1.default.startProgress({
actionLabel: "GOAL",
successLabel: "DONE",
failureLabel: "FAILED",
}, {
message: goal,
isBold: true,
color: "whiteBright",
});
for (let step = 0; step < maxSteps; step++) {
// Capture the screen silently (without separate logging)
const screenCaptureWithoutHighlight = await this.screenCapturer.capture(false);
// Analyze the screen and plan the next step
const stepReport = await this.analyseScreenAndCreatePilotStep(goal, [...previousSteps], screenCaptureWithoutHighlight, reviewSectionTypes ? reviewSectionTypes : undefined);
if (stepReport.goalAchieved) {
report.summary = stepReport.summary;
report.review = stepReport.review;
// Format summary with our formatting parser if available
const formattedSummary = report.summary
? (0, format_utils_1.parseFormattedText)(report.summary)
: [
{
message: "Goal completed successfully",
isBold: false,
color: "green",
},
];
// Complete the main progress
mainProgress.stop("success", {
message: "Success",
isBold: true,
color: "green",
});
logger_1.default.labeled("SUMMARY").info(...formattedSummary);
break;
}
const screenCaptureWithHighlight = await this.screenCapturer.capture(true);
const { code, result } = await this.stepPerformer.perform(stepReport.plan.action, [...pilotSteps], screenCaptureWithHighlight);
report.steps.push({ code, ...stepReport });
pilotSteps.push({ step: stepReport.plan.action, code, result });
previousSteps.push({
screenDescription: stepReport.screenDescription,
step: stepReport.plan.action,
review: stepReport.review,
});
if (step === maxSteps - 1) {
mainProgress.stop("warn", {
message: "Limit reached",
isBold: true,
color: "yellow",
});
}
}
return report;
}
logReviews(screenDescription, review, reviewSectionTypes) {
const allReviewComponents = [];
allReviewComponents.push(...(0, format_utils_1.parseFormattedText)(screenDescription));
Object.keys(review).forEach((reviewType) => {
if (review[reviewType]) {
const reviewTypeConfig = reviewSectionTypes?.find((rt) => rt.title.toLowerCase() === reviewType.toLowerCase());
if (reviewTypeConfig !== undefined) {
allReviewComponents.push(...this.formatReviewSection(review[reviewType], reviewTypeConfig));
}
}
});
logger_1.default.labeled("REVIEWING").info(...allReviewComponents);
}
async getValueFromCache(cacheKey, screenCapture, reviewSectionTypes) {
const cachedValues = this.cacheHandler.getFromPersistentCache(cacheKey);
if (!cachedValues) {
return undefined;
}
const snapshotHashes = await this.snapshotComparator.generateHashes(screenCapture);
const matchingEntry = this.cacheHandler.findMatchingCacheEntrySnapshotBased(cachedValues, snapshotHashes);
const cachedReport = matchingEntry?.value;
if (cachedReport && cachedReport.plan && cachedReport.plan.thoughts) {
logger_1.default.labeled("CACHE").info("Using cached analysis result");
const formattedThoughts = (0, format_utils_1.parseFormattedText)(cachedReport.plan.thoughts);
logger_1.default.labeled("THOUGHTS").info(...formattedThoughts);
const hasReviews = cachedReport.review && Object.keys(cachedReport.review).length > 0;
if (hasReviews && reviewSectionTypes && cachedReport.review) {
this.logReviews(cachedReport.screenDescription, cachedReport.review, reviewSectionTypes);
}
}
return cachedReport;
}
}
exports.AutoPerformer = AutoPerformer;