askui
Version:
Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on
235 lines (234 loc) • 9.63 kB
JavaScript
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
import { ToolCollection } from './tools/base';
import { logger } from '../../../lib/logger';
import { Base64Image } from '../../../utils/base_64_image/base-64-image';
export class ClaudeAgent {
constructor(predictActResponseFunction) {
this.predictActResponseFunction = predictActResponseFunction;
this.maxTokens = 4096;
this.onlyNMostRecentImages = 3;
this.imageTruncationThreshold = 10;
this.systemPrompt = '';
this.model = 'claude-sonnet-4-20250514';
this.betas = ['computer-use-2025-01-24'];
this._toolCollection = undefined;
this.tools = [];
this.history = {};
this.toolChoice = {
type: 'auto',
};
}
setToolChoice(toolChoice) {
this.toolChoice = toolChoice;
}
setTools(tools) {
this._toolCollection = undefined;
this.tools = tools;
}
addTool(tool) {
this._toolCollection = undefined;
this.tools.push(tool);
}
listToolNames() {
return this.tools.map((tool) => tool.ToolName);
}
removeToolByName(toolName) {
this.tools = this.tools.filter((tool) => tool.ToolName !== toolName);
}
setSystemPrompt(systemPrompt) {
const enhancedPrompt = `${systemPrompt}
If you cannot complete a request due to safety concerns, please:
1. Explain what specific aspect is problematic
2. Suggest alternative approaches that would be acceptable
3. Provide partial assistance where possible within guidelines.
Raise an exception After you have provided the above information. include the error message in the exception.
`;
this.systemPrompt = enhancedPrompt;
}
IsConfigured() {
return this.tools.length > 0 && this.systemPrompt !== '';
}
get toolCollection() {
if (!this._toolCollection) {
this._toolCollection = new ToolCollection(this.tools);
}
return this._toolCollection;
}
setHistory(key, messages) {
this.history[key] = messages;
}
getHistory(key) {
return this.history[key] || [];
}
act(goal, imagePathOrBase64String, options) {
return __awaiter(this, void 0, void 0, function* () {
if (!goal.trim()) {
throw new Error('Goal cannot be empty');
}
if (!this.IsConfigured()) {
throw new Error('Claude agent is not configured. Please configure the agent first.');
}
const messages = [];
if (options === null || options === void 0 ? void 0 : options.agentHistory) {
messages.push(...options.agentHistory);
}
if (options === null || options === void 0 ? void 0 : options.chatId) {
messages.push(...this.getHistory(options.chatId));
}
// Add the new goal as a user message
const userContent = [{
type: 'text',
text: goal,
}];
if (imagePathOrBase64String !== undefined) {
const image = yield Base64Image.fromPathOrString(imagePathOrBase64String);
const imageString = image.toString(false);
userContent.push({
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: imageString,
},
});
}
messages.push({
content: userContent,
role: 'user',
});
if (this.onlyNMostRecentImages) {
ClaudeAgent.filterNMostRecentImages(messages, this.onlyNMostRecentImages, this.imageTruncationThreshold);
}
while (true) {
const response = yield this.predictActResponseFunction({
max_tokens: this.maxTokens,
messages,
model: this.model,
system: this.systemPrompt,
tools: (new ToolCollection(this.tools).toParams()),
betas: this.betas,
tool_choice: this.toolChoice,
});
if (response.stop_reason === 'refusal') {
const refusalMessage = response.content
.filter(block => block.type === 'text')
.map(block => block.text)
.join(' ');
throw new Error(`Agent refused to answer: ${refusalMessage || 'The request violates agent\'s usage policies'}`);
}
messages.push({
content: response.content,
role: 'assistant',
});
const toolResultContent = [];
for (const block of response.content) {
if (block.type === 'tool_use') {
logger.debug(`Agent will execute tool '${block.name}' with input '${JSON.stringify(block.input)}'`);
const toolUseBlock = block;
const result = yield this.toolCollection.run(toolUseBlock.name, toolUseBlock.input);
const toolResult = this.makeApiToolResult(result, toolUseBlock.id);
toolResultContent.push(toolResult);
}
if (block.type === 'thinking') {
logger.debug(`Agent is thinking: ${block.thinking}`);
}
if (block.type === 'text') {
logger.info(`Agent: ${block.text}`);
}
}
if (toolResultContent.length === 0) {
if (options === null || options === void 0 ? void 0 : options.chatId) {
this.setHistory(options.chatId, messages);
}
return messages;
}
messages.push({
role: 'user',
content: toolResultContent,
});
}
});
}
makeApiToolResult(result, toolUseId) {
const content = [];
let isError = false;
if (result.error) {
isError = true;
content.push({
type: 'text',
text: this.maybePrependSystemToolResult(result, result.error),
});
}
else {
if (result.output) {
content.push({
type: 'text',
text: this.maybePrependSystemToolResult(result, result.output),
});
}
if (result.base64Images) {
for (const base64Image of result.base64Images) {
content.push({
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: base64Image,
},
});
}
}
}
return {
type: 'tool_result',
tool_use_id: toolUseId,
content,
is_error: isError,
};
}
maybePrependSystemToolResult(result, text) {
if (result.system) {
return `<system>${result.system}</system>\n${text}`;
}
return text;
}
static filterNMostRecentImages(messages, imagesToKeep, minRemovalThreshold = 10) {
const toolResultBlocks = messages
.flatMap((message) => (Array.isArray(message.content) ? message.content : []))
.filter((item) => typeof item === 'object'
&& 'type' in item
&& item.type === 'tool_result');
const totalImages = toolResultBlocks.reduce((count, result) => {
const content = Array.isArray(result.content) ? result.content : [];
return count + content.filter((item) => typeof item === 'object'
&& 'type' in item
&& item.type === 'image').length;
}, 0);
let imagesToRemove = totalImages - imagesToKeep;
imagesToRemove -= imagesToRemove % minRemovalThreshold;
if (imagesToRemove <= 0)
return;
for (const result of toolResultBlocks) {
if (Array.isArray(result.content)) {
const newContent = result.content.filter((item) => {
if ('type' in item && item.type === 'image') {
if (imagesToRemove > 0) {
imagesToRemove -= 1;
return false;
}
}
return true;
});
result.content = newContent;
}
}
}
}