@hyperbrowser/agent
Version:
Hyperbrowsers Web Agent
193 lines (192 loc) • 7.55 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.runAgentTask = void 0;
const fs_1 = __importDefault(require("fs"));
const dom_1 = require("../../context-providers/dom");
const retry_1 = require("../../utils/retry");
const sleep_1 = require("../../utils/sleep");
const types_1 = require("../../types/index");
const types_2 = require("../../types/index");
const error_1 = require("../error");
const builder_1 = require("../messages/builder");
const structured_output_1 = require("../llms/structured-output");
const system_prompt_1 = require("../messages/system-prompt");
const zod_1 = require("zod");
const actions_1 = require("../actions");
const sharp_1 = __importDefault(require("sharp"));
const compositeScreenshot = async (page, overlay) => {
const screenshot = await page.screenshot();
const responseBuffer = await (0, sharp_1.default)(screenshot)
.composite([{ input: Buffer.from(overlay, "base64") }])
.png()
.toBuffer();
return responseBuffer.toString("base64");
};
const getActionSchema = (actions) => {
const zodDefs = actions.map((action) => zod_1.z.object({
type: zod_1.z.nativeEnum([action.type]),
params: action.actionParams,
actionDescription: zod_1.z
.string()
.describe("Describe why you are performing this action and what you aim to perform with this action."),
}));
return zod_1.z.union([zodDefs[0], zodDefs[1], ...zodDefs.splice(2)]);
};
const getActionHandler = (actions, type) => {
const foundAction = actions.find((actions) => actions.type === type);
if (foundAction) {
return foundAction.run;
}
else {
throw new actions_1.ActionNotFoundError(type);
}
};
const runAction = async (action, domState, page, ctx) => {
const actionCtx = {
domState,
page,
tokenLimit: ctx.tokenLimit,
llm: ctx.llm,
debugDir: ctx.debugDir,
mcpClient: ctx.mcpClient || undefined,
variables: Object.values(ctx.variables),
};
const actionType = action.type;
const actionHandler = getActionHandler(ctx.actions, action.type);
if (!actionHandler) {
return {
success: false,
message: `Unknown action type: ${actionType}`,
};
}
try {
return await actionHandler(actionCtx, action.params);
}
catch (error) {
return {
success: false,
message: `Action ${action.type} failed: ${error}`,
};
}
};
const runAgentTask = async (ctx, taskState, params) => {
const taskId = taskState.id;
const debugDir = params?.debugDir || `debug/${taskId}`;
if (ctx.debug) {
console.log(`Debugging task ${taskId} in ${debugDir}`);
}
if (!taskState) {
throw new error_1.HyperagentError(`Task ${taskId} not found`);
}
taskState.status = types_2.TaskStatus.RUNNING;
if (!ctx.llm) {
throw new error_1.HyperagentError("LLM not initialized");
}
const llmStructured = ctx.llm.withStructuredOutput((0, types_1.AgentOutputFn)(getActionSchema(ctx.actions)), {
method: (0, structured_output_1.getStructuredOutputMethod)(ctx.llm),
});
const baseMsgs = [{ role: "system", content: system_prompt_1.SYSTEM_PROMPT }];
let output = "";
const page = taskState.startingPage;
let currStep = 0;
while (true) {
// Status Checks
if (taskState.status == types_2.TaskStatus.PAUSED) {
await (0, sleep_1.sleep)(100);
continue;
}
if (types_1.endTaskStatuses.has(taskState.status)) {
break;
}
if (params?.maxSteps && currStep >= params.maxSteps) {
taskState.status = types_2.TaskStatus.CANCELLED;
break;
}
const debugStepDir = `${debugDir}/step-${currStep}`;
if (ctx.debug) {
fs_1.default.mkdirSync(debugStepDir, { recursive: true });
}
// Get DOM State
const domState = await (0, retry_1.retry)({ func: () => (0, dom_1.getDom)(page) });
if (!domState) {
console.log("no dom state, waiting 1 second.");
await (0, sleep_1.sleep)(1000);
continue;
}
const trimmedScreenshot = await compositeScreenshot(page, domState.screenshot.startsWith("data:image/png;base64,")
? domState.screenshot.slice("data:image/png;base64,".length)
: domState.screenshot);
// Store Dom State for Debugging
if (ctx.debug) {
fs_1.default.mkdirSync(debugDir, { recursive: true });
fs_1.default.writeFileSync(`${debugStepDir}/elems.txt`, domState.domState);
if (trimmedScreenshot) {
fs_1.default.writeFileSync(`${debugStepDir}/screenshot.png`, Buffer.from(trimmedScreenshot, "base64"));
}
}
// Build Agent Step Messages
const msgs = await (0, builder_1.buildAgentStepMessages)(baseMsgs, taskState.steps, taskState.task, page, domState, trimmedScreenshot, Object.values(ctx.variables));
// Store Agent Step Messages for Debugging
if (ctx.debug) {
fs_1.default.writeFileSync(`${debugStepDir}/msgs.json`, JSON.stringify(msgs, null, 2));
}
// Invoke LLM
const agentOutput = await (0, retry_1.retry)({
func: () => llmStructured.invoke(msgs),
});
params?.debugOnAgentOutput?.(agentOutput);
// Status Checks
if (taskState.status == types_2.TaskStatus.PAUSED) {
await (0, sleep_1.sleep)(100);
continue;
}
if (types_1.endTaskStatuses.has(taskState.status)) {
break;
}
// Run Actions
const agentStepActions = agentOutput.actions;
const actionOutputs = [];
for (const action of agentStepActions) {
if (action.type === "complete") {
taskState.status = types_2.TaskStatus.COMPLETED;
const actionDefinition = ctx.actions.find((actionDefinition) => actionDefinition.type === "complete");
if (actionDefinition) {
output =
(await actionDefinition.completeAction?.(action.params)) ??
"No complete action found";
}
else {
output = "No complete action found";
}
}
const actionOutput = await runAction(action, domState, page, ctx);
actionOutputs.push(actionOutput);
await (0, sleep_1.sleep)(2000); // TODO: look at this - smarter page loading
}
const step = {
idx: currStep,
agentOutput: agentOutput,
actionOutputs,
};
taskState.steps.push(step);
await params?.onStep?.(step);
currStep = currStep + 1;
if (ctx.debug) {
fs_1.default.writeFileSync(`${debugStepDir}/stepOutput.json`, JSON.stringify(step, null, 2));
}
}
const taskOutput = {
status: taskState.status,
steps: taskState.steps,
output,
};
if (ctx.debug) {
fs_1.default.writeFileSync(`${debugDir}/taskOutput.json`, JSON.stringify(taskOutput, null, 2));
}
await params?.onComplete?.(taskOutput);
return taskOutput;
};
exports.runAgentTask = runAgentTask;