magnitude-core
Version:
Magnitude e2e testing agent
1,253 lines (1,197 loc) • 158 kB
JavaScript
import { setLogLevel } from '@boundaryml/baml/logging';
import { BamlRuntime, BamlCtxManager, toBamlError, BamlStream, ThrowIfVersionMismatch, Collector, ClientRegistry, Image as Image$1 } from '@boundaryml/baml';
import pino from 'pino';
import EventEmitter from 'eventemitter3';
import z$1, { z, ZodObject } from 'zod';
import { TypeBuilder as TypeBuilder$1 } from '@boundaryml/baml/type_builder';
import { zodToJsonSchema } from 'zod-to-json-schema';
import sharp from 'sharp';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { execSync } from 'node:child_process';
import cuid2 from '@paralleldrive/cuid2';
import { PostHog } from 'posthog-node';
import { createRequire } from 'node:module';
import { createHash } from 'crypto';
import { chromium } from 'playwright';
import objectHash from 'object-hash';
import crypto from 'node:crypto';
import { vl } from 'moondream';
import { bold, cyanBright, blueBright, gray } from 'ansis';
const fileMap = {
"behaviorTests.baml": `
test DoNotOverplan1 {
functions [CreatePartialRecipe]
args {
screenshot {
url "https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/do_not_overplan_1.png"
}
step {
description #"Create a new company"#
checks [
#"Company added successfully"#
]
testData {
data [
]
other #"Make up the first 2 values and use defaults for the rest"#
}
}
previousActions [
]
}
@@assert( one_action, {{ this.actions|length == 1 }} )
@@assert( not_marked_finished, {{ this.finished == false }} )
}
test DoNotOverplan2 {
// Especially with the test data, planner might be tempted to click create company and also plan to fill form fields.
// BUT we do not want it to do that - it should only click the button.
functions [CreatePartialRecipe]
args {
screenshot {
url "https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/do_not_overplan_2.png"
}
step {
description #"Create a new company"#
checks [
#"Company added successfully"#
]
testData {
data [
]
other #"Make up the first 2 values and use defaults for the rest"#
}
}
previousActions [
#"
{
"variant": "click",
"target": "Companies option in the left sidebar menu"
}
"#
]
}
@@assert( one_action, {{ this.actions|length == 1 }} )
@@assert( not_marked_finished, {{ this.finished == false }} )
}
test OptimalPlanning1 {
functions [CreatePartialRecipe]
args {
screenshot {
url "https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/do_not_underplan_1.png"
}
step {
description #"Create a new company"#
checks [
#"Company added successfully"#
]
testData {
data [
]
other #"Make up the first 2 values and use defaults for the rest"#
}
}
previousActions [
#"
{
"variant": "click",
"target": "Companies option in the left sidebar menu"
}
"#,
#"
{
"variant": "click",
"target": "Add Company button in the top right corner"
}
"#
]
}
// optimal is type name, type domain, click save, mark finished
@@assert( optimal_actions, {{ this.actions|length == 3 }} )
// @@assert( not_marked_finished, {{ this.finished == false }} )
}
// test CheckContextRemove1 {
// // Manual for now - ideally we'd want to verify that executor can check the returned description successfully
// functions [RemoveImplicitCheckContext]
// args {
// screenshot {
// url "https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/check_context_remove_1.png"
// }
// check #"Company added successfully"#
// previousActions [
// #"
// {
// "variant": "click",
// "target": "Companies option in the left sidebar menu"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "'Add Company' button in the top right corner of the Companies page"
// }
// "#,
// #"
// {
// "variant": "type",
// "target": "Name input field",
// "content": "Acme Solutions"
// }
// "#,
// #"
// {
// "variant": "type",
// "target": "Domain input field",
// "content": "acmesolutions.com"
// }
// "#,
// #"
// {
// "variant": "type",
// "target": "Logo URL input field",
// "content": "https://example.com/image.jpg"
// }
// "#,
// #"
// {
// "variant": "type",
// "target": "Industry input field",
// "content": "Technology"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "Size dropdown menu"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "1-10 option in the Size dropdown"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "Revenue dropdown menu"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "<$100K option in the Revenue dropdown"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "ICP Fit dropdown menu"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "Medium option in the ICP Fit dropdown"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "Est. ARR dropdown menu"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "<$10K option in the Est. ARR dropdown"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "Connection Strength dropdown menu"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "Weak option in the Connection Strength dropdown"
// }
// "#,
// #"
// {
// "variant": "click",
// "target": "Save button"
// }
// "#
// ]
// }
// }
test TargetGrounding1 {
// This test is a screenshot of Magnitude dasboard. Goal is to go to test playground.
// Originally failed because target was described as "TC-0 Playground Test Case button"
// This aligns more with the card shown rather that the specific button that actually needs to be clicked
// which has text "Experiment Now (Free!)"
// We want the planner to ground the targets in very specific details when available, such as the text on a button.
// This test asserts that the generated target includes specific text from the button, at least "Experiment Now"
functions [CreatePartialRecipe]
args {
screenshot {
url "https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/target_grounding_1.png"
}
step {
description #"Go to test playground"#
checks [
]
testData {
data [
]
other ""
}
}
previousActions [
]
}
@@assert( includes_button_text, {{ "Experiment Now (Free!)" in this.actions[0].target }} )
}`,
"clients.baml": '// Learn more about clients at https://docs.boundaryml.com/docs/snippets/clients/overview\n\nclient<llm> SonnetBedrock {\n provider aws-bedrock\n retry_policy Exponential\n options {\n inference_configuration {\n temperature 0.0\n }\n model_id "anthropic.claude-3-5-sonnet-20240620-v1:0"\n }\n}\n\nclient<llm> SonnetAnthropic {\n provider anthropic\n retry_policy Exponential\n options {\n model "claude-3-5-sonnet-20240620"\n api_key env.ANTHROPIC_API_KEY\n temperature 0.0\n }\n}\n\nclient<llm> GeminiPro {\n provider "google-ai"\n options {\n api_key env.GOOGLE_API_KEY\n model "gemini-2.5-pro-preview-05-06"\n generationConfig {\n temperature 0.0\n }\n }\n}\n\nclient<llm> GeminiProOpenRouter {\n provider "openai-generic"\n options {\n base_url "https://openrouter.ai/api/v1"\n api_key env.OPENROUTER_API_KEY\n model "google/gemini-2.5-pro-preview-03-25"\n temperature 0.0\n }\n}\n\nclient<llm> GeminiFlash {\n provider "openai-generic"\n options {\n base_url "https://openrouter.ai/api/v1"\n api_key env.OPENROUTER_API_KEY\n model "google/gemini-2.5-flash-preview"\n temperature 0.0\n }\n}\n\nclient<llm> GPT {\n provider openai\n options {\n model "gpt-4.1-2025-04-14"\n api_key env.OPENAI_API_KEY\n temperature 0.0\n }\n}\n\n// client<llm> Macro {\n// provider openai\n// options {\n// model "gpt-4o"\n// api_key env.OPENAI_API_KEY\n// temperature 0.0\n// }\n// }\n\nclient<llm> NovitaLlamaMaverick {\n provider "openai-generic"\n retry_policy Exponential\n options {\n base_url "https://api.novita.ai/v3/openai"\n api_key env.NOVITA_API_KEY\n model "meta-llama/llama-4-maverick-17b-128e-instruct-fp8"\n temperature 0.0\n logprobs true\n }\n}\n\nclient<llm> Molmo {\n provider "openai-generic"\n retry_policy Exponential\n options {\n base_url env.MOLMO_VLLM_BASE_URL\n api_key env.MOLMO_VLLM_API_KEY\n model "Molmo-7B-D-0924"\n temperature 0.0\n logprobs true\n }\n}\n\nretry_policy Exponential {\n max_retries 5\n strategy {\n type exponential_backoff\n delay_ms 500\n multiplier 2.0\n max_delay_ms 10000\n }\n}\n\n// Default used by macro.ts\nretry_policy DefaultRetryPolicy {\n max_retries 5\n strategy {\n type exponential_backoff\n delay_ms 500\n multiplier 2.0\n max_delay_ms 10000\n }\n}\n',
"diagnosis.baml": `
// class FailureClassification {
// classification "bug" | "misalignment"
// }
class BugFailureClassification {
reasoning string
classification "bug"
title string
expectedResult string
actualResult string
severity "critical" | "high" | "medium" | "low"
}
class MisalignmentClassification {
reasoning string
classification "misalignment"
fault "test" | "agent" @description(#"
Is this likely a problem with the test case or some issue with the agent?
"#)
message string @description(#"
If fault of test case:
Message to the developer who wrote the test case to help them understand what may have happened and how they might be able to fix it.
Be simple, direct, and informative.
If fault of agent:
Message to the developer who wrote the test case (not the person who wrote the agent) to help them understand what may have happened, and suggest possible ways to adjust the test case to accommodate the issue.
"#)
}
// class FailureClassification {
// reasoning
// classification "bug" | "misalignment"
// }
template_string BaseMeta #"
You are observing the execution of an LLM agent that runs test cases.
This agent executes test cases by observing screenshots from the browser, acting out steps, and verifying checks.
"#
// should this be given the original or adjusted check?
// should this be given more ctx of the overall step or test case?
// this is a tricky prompt, but its not as critical to get right as others
// it does become more important if we rely on classify as bug/misalign then correct minor misaligns only if not bug
// ^ if we need this then we should think of a more isolated/logical prompt to detect misalignments
// function ClassifyCheckFailure (context: BrowserExecutionContext, check: string) -> BugFailureClassification | MisalignmentClassification {
// client SonnetAnthropic
// prompt #"
// {{ _.role("system") }}
// {{ BaseMeta() }}
// The agent just marked a check as failed. Your job is to figure out why.
// Either:
// (1) The web application actually has a bug in it
// or
// (2) There's some misalignment between the test case and what is in the interface
// If there is a bug, please break it down in detail.
// If it is a misalignment, please describe what you think happened.
// Use the provided history of actions the agent took as well as the most recent screenshot to help you identify what happened.
// {{ ctx.output_format }}
// {{ _.role("user") }}
// The "check" that was marked as failed:
// {{ check }}
// {{ DescribeBrowserExecutionContext(context) }}
// "#
// }
// function DiagnoseTargetNotFound (screenshot: image, step: TestStep, target: string, previousActions: string[]) -> BugFailureClassification | MisalignmentClassification {
// client SonnetAnthropic
// prompt #"
// {{ _.role("system") }}
// {{ BaseMeta() }}
// The agent had an issue acting out a step because it could not find a target. Your job is to figure out why.
// Either:
// (1) The web application actually has a bug in it
// or
// (2) There's some misalignment between the test case and what is in the interface
// If there is a bug, please break it down in detail.
// If it is a misalignment, please describe what you think happened.
// Use the provided history of actions the agent took as well as the most recent screenshot to help you identify what happened.
// {{ ctx.output_format }}
// {{ _.role("user") }}
// The history of previous actions:
// {%if previousActions %}
// (there are none)
// {%endif%}
// {%for action in previousActions%}
// {{ action }}
// {%endfor%}
// The step that failed: <step>{{ step.description }}</step>
// Target that could not be found: <target>{{ target }}</target>
// Current screenshot:
// {{screenshot}}
// "#
// }`,
"extract.baml": '// if primitive, populate key "data"\n// if array, populate key data with that array\n// else fill with object fields\nclass ExtractedData {\n @@dynamic\n}\n\nfunction ExtractData (instructions: string, screenshot: image, domContent: string) -> ExtractedData {\n client GeminiPro\n prompt #"\n {{ _.role("system") }}\n Based on the browser screenshot and page content, extract data according to these instructions:\n <instructions>{{ instructions }}</instructions>\n \n {{ ctx.output_format }}\n\n {{ _.role("user") }}\n\n {{ domContent }}\n\n {{ screenshot }}\n "#\n}',
"generators.baml": '// This helps use auto generate libraries you can use in the language of\n// your choice. You can have multiple generators if you use multiple languages.\n// Just ensure that the output_dir is different for each generator.\ngenerator target {\n // Valid values: "python/pydantic", "typescript", "ruby/sorbet", "rest/openapi"\n output_type "typescript"\n\n // Where the generated code will be saved (relative to baml_src/)\n output_dir "../src/ai"\n\n // The version of the BAML package you have installed (e.g. same version as your baml-py or @boundaryml/baml).\n // The BAML VSCode extension version should also match this version.\n version "0.90.2"\n\n // Valid values: "sync", "async"\n // This controls what `b.FunctionName()` will be (sync or async).\n default_client_mode async\n}\n',
"memory.baml": `// BamlImage class is assumed to be defined and available from your BAML setup
// For example, it might be part of a core BAML library or generated separately.
// If not, you might need to define it or import it if it's in another baml file.
// For now, we assume 'Image' is a known type to BAML.
// class BamlThought {
// variant "thought"
// timestamp string
// message string
// }
// class BamlTurn {
// variant "turn"
// timestamp string
// action string
// content (string | image)[] // multimedia content "chunks"
// }
// class Observation {
// source string
// content (string | image)[] // multimedia content "chunks"
// }
class ConnectorInstructions {
connectorId string
instructions string
//content (string | image)[]
}
class ModularMemoryContext {
instructions string? // additional task-level or agent-level instructions
//history (BamlThought | BamlTurn)[]
//observations Observation[]
observationContent (string | image)[] // complete rendered observation content as multimedia content "chunks"
//currentTimestamp string
connectorInstructions ConnectorInstructions[]
}
`,
"planner.baml": 'class PartialRecipe {\n // this CoT pipeline is not thoroughly tested against alternatives\n // observations string? @description(#"Any key observations about past actions or current state"#)\n // meta_reasoning string @description(#"Reflect on the current state of task execution with respect to your own abilities as an agent"#)\n // reasoning string @description(#"Consider what you can see right now and what actions you can plan without guessing"#)\n\n // Simplifying CoT - seemed to be causing underplanning on Claude\n //observations string @description(#"What important clues or key information are worth considering?"#)\n reasoning string @description(#"What is the most actions you can safely take at once?"#)\n\n //actions ActionIntent[]\n @@dynamic\n //finished bool\n}\n\n\n// render "chunked" multimedia content of strings or images with no added whitespace between\ntemplate_string RenderContent(content: (string | image)[]) #"\n {% for chunk in content -%}{{chunk}}{%- endfor %}\n"#\n\n\n// template_string DescribeModularMemoryContext(context: ModularMemoryContext) #"\n// {% for entry in context.history %}\n// {% if entry.variant == "thought" %}\n// [{{entry.timestamp}}] {{entry.message}}\n// {% elif entry.variant == "turn" %}\n// [{{entry.timestamp}}] {{entry.action}}\n// {{ RenderContent(entry.content) }}\n// {% endif %}\n// {% endfor %}\n// "#\n\n// unused\ntemplate_string HybridMeta #"\n <meta>\n You are a web agent powered by (1) a powerful LLM (you) and (2) a small vision model (Moondream).\n You operate by planning out actions over time to try and complete a certain task.\n To do this successfully you may need to consider and adjust for your own limitations as an agent, so here is some information about how you work:\n - You plan several actions ahead of time without trying to guess, which are then executed\n - You are then given a new screenshot of the webpage which is the result of all actions taken\n - This occurs in a loop until you complete your task\n - Mouse movements require a "target", which is located by Moondream as a specific coordinate on the page\n - Moondream may sometimes fail to locate the target you expected it to.\n - Therefore, actions may not always be completely successful and you may need to try different variations\n </meta>\n"#\n\ntemplate_string DescribeConnectorInstructions(memory: ModularMemoryContext) #"\n {% for conn in memory.connectorInstructions %}\n <{{ conn.connectorId }}>\n {{- conn.instructions -}}\n </{{ conn.connectorId }}>\n {% endfor %}\n"#\n\ntemplate_string InstructionsWithContext(memory: ModularMemoryContext, instructions: string) #"\n {{ _.role("system") }}\n <instructions>\n {{ instructions }}\n {%if memory.instructions %}\n\n {{ memory.instructions }}\n {%endif%}\n </instructions>\n {{ DescribeConnectorInstructions(memory) }}\n {{ ctx.output_format }}\n\n {{ _.role("user") }}\n\n {{ RenderContent(memory.observationContent) }}\n"# \n\ntemplate_string TaskInstructions(task: string) #"\n Plan out actions that should be executed in order to complete the task:\n <task>\n {{ task }}\n </task>\n\n Execute the task serially.\n\n Plan out as many actions as possible, but stopping at the point where you will need to observe to plan further.\n"#\n\n// TODO: connector system instructions\nfunction CreatePartialRecipe (memory: ModularMemoryContext, task: string) -> PartialRecipe {\n client GeminiPro\n prompt #"\n {{ InstructionsWithContext(memory, TaskInstructions(task)) }}\n "# \n}\n\n// class EvaluatedCheck {\n// reasoning string\n// passes bool\n// }\nclass QueryResponse {\n @@dynamic\n}\n\nfunction QueryMemory(memory: ModularMemoryContext, query: string) -> QueryResponse {\n client GeminiPro\n prompt #"\n {{ InstructionsWithContext(memory, query) }}\n "#\n}\n\n\n// template_string CheckInstructions(check: string) #"\n// Given the actions of an LLM agent executing a test case, and a screenshot taken afterwards, evaluate whether the provided check "passes" i.e. holds true or not.\n\n// Check to evaluate:\n// <check>{{ check }}</check>\n// "#\n\n// function EvaluateCheck (context: ModularMemoryContext, check: string) -> EvaluatedCheck {\n// client GeminiPro\n// prompt #"\n// {{ InstructionsWithContext(context, CheckInstructions(check)) }}\n// "#\n// }\n\n// Moondream struggles to handle converted check, this is a bypass to use big model to evaluate check directly\n// function EvaluateCheck (context: BrowserExecutionContext, check: string) -> EvaluatedCheck {\n// client GeminiPro\n// prompt #"\n// {{ _.role("system") }}\n// Given the actions of an LLM agent executing a test case, and a screenshot taken afterwards, evaluate whether the provided check "passes" i.e. holds true or not.\n// {{ ctx.output_format }}\n\n// {{ _.role("user")}}\n// Check to evaluate:\n// <check>{{ check }}</check>\n\n// {{ DescribeBrowserExecutionContext(context) }}\n// "# \n// }\n'
};
const getBamlFiles = () => {
return fileMap;
};
const env = { ...process.env };
const DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME = BamlRuntime.fromFiles(
"baml_src",
getBamlFiles(),
env
);
const DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX = new BamlCtxManager(DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME);
const logger = pino({
level: process.env.MAGNITUDE_LOG_LEVEL || "warn",
transport: process.stdout.isTTY ? {
target: "pino-pretty",
options: {
colorize: !process.env.NO_COLOR,
translateTime: "SYS:HH:MM:ss.l",
ignore: "pid,hostname"
}
} : void 0
}).child({
name: "agent"
});
function cleanNestedObject(obj) {
return Object.fromEntries(
Object.entries(obj).filter(([_, value]) => value !== null && value !== void 0).map(([key, value]) => [
key,
typeof value === "object" ? cleanNestedObject(value) : value
])
);
}
function convertToBamlClientOptions(client) {
const temp = "temperature" in client.options ? client.options.temperature ?? 0 : 0;
let options;
if (client.provider === "anthropic") {
options = {
api_key: client.options.apiKey,
model: client.options.model,
temperature: temp
};
} else if (client.provider === "aws-bedrock") {
options = {
model_id: client.options.model,
inference_configuration: {
temperature: temp
}
};
} else if (client.provider === "google-ai") {
options = {
base_url: client.options.baseUrl,
model: client.options.model,
api_key: client.options.apiKey,
generationConfig: {
temperature: temp
//thinking_budget: 0
}
};
} else if (client.provider === "vertex-ai") {
options = {
location: client.options.location,
base_url: client.options.baseUrl,
project_id: client.options.projectId,
credentials: client.options.credentials,
model: client.options.model,
generationConfig: {
temperature: temp
}
};
} else if (client.provider === "openai") {
options = {
api_key: client.options.apiKey,
model: client.options.model,
temperature: temp
};
} else if (client.provider === "openai-generic") {
options = {
base_url: client.options.baseUrl,
api_key: client.options.apiKey,
model: client.options.model,
temperature: temp,
headers: client.options.headers
};
} else if (client.provider === "azure-openai") {
options = {
resource_name: client.options.resourceName,
deployment_id: client.options.deploymentId,
api_version: client.options.apiVersion,
api_key: client.options.apiKey
};
} else {
throw new Error(`Invalid provider: ${client.provider}`);
}
return cleanNestedObject(options);
}
function tryDeriveUIGroundedClient() {
if (process.env.ANTHROPIC_API_KEY) {
return {
provider: "anthropic",
options: {
// TODO: do more testing on best claude model for visuals
// model: 'claude-3-5-sonnet-20240620', // <- definitely not, pre computer use
// model: 'claude-3-5-sonnet-20241022', // <- not great on rescaling res
//model: 'claude-3-7-sonnet-latest', // <- underplans
model: "claude-sonnet-4-20250514",
// <- underplans, also supposedly worse at visual reasoning
apiKey: process.env.ANTHROPIC_API_KEY
}
};
} else {
return null;
}
}
function isClaude(llm) {
if ("model" in llm.options) {
const model = llm.options.model;
if (model.includes("claude")) return true;
}
return false;
}
const DEFAULT_BROWSER_AGENT_TEMP = 0.2;
function buildDefaultBrowserAgentOptions({ agentOptions, browserOptions }) {
const envLlm = tryDeriveUIGroundedClient();
let llm = agentOptions.llm ?? envLlm;
const grounding = browserOptions.grounding;
if (!llm) {
throw new Error("No LLM configured or available from environment. Set environment variable ANTHROPIC_API_KEY and try again. See https://docs.magnitude.run/customizing/llm-configuration for details");
}
let llmOptions = { temperature: DEFAULT_BROWSER_AGENT_TEMP, ...llm?.options ?? {} };
llm = { ...llm, options: llmOptions };
let virtualScreenDimensions = null;
if (isClaude(llm)) {
virtualScreenDimensions = { width: 1024, height: 768 };
}
return {
agentOptions: { ...agentOptions, llm },
browserOptions: { ...browserOptions, grounding: grounding ?? void 0, virtualScreenDimensions: virtualScreenDimensions ?? void 0 }
};
}
class AsyncHttpRequest {
constructor(runtime, ctxManager) {
this.runtime = runtime;
this.ctxManager = ctxManager;
}
async CreatePartialRecipe(memory, task, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return await this.runtime.buildRequest(
"CreatePartialRecipe",
{
"memory": memory,
"task": task
},
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
false,
env
);
} catch (error) {
throw toBamlError(error);
}
}
async ExtractData(instructions, screenshot, domContent, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return await this.runtime.buildRequest(
"ExtractData",
{
"instructions": instructions,
"screenshot": screenshot,
"domContent": domContent
},
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
false,
env
);
} catch (error) {
throw toBamlError(error);
}
}
async QueryMemory(memory, query, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return await this.runtime.buildRequest(
"QueryMemory",
{
"memory": memory,
"query": query
},
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
false,
env
);
} catch (error) {
throw toBamlError(error);
}
}
}
class AsyncHttpStreamRequest {
constructor(runtime, ctxManager) {
this.runtime = runtime;
this.ctxManager = ctxManager;
}
async CreatePartialRecipe(memory, task, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return await this.runtime.buildRequest(
"CreatePartialRecipe",
{
"memory": memory,
"task": task
},
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
true,
env
);
} catch (error) {
throw toBamlError(error);
}
}
async ExtractData(instructions, screenshot, domContent, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return await this.runtime.buildRequest(
"ExtractData",
{
"instructions": instructions,
"screenshot": screenshot,
"domContent": domContent
},
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
true,
env
);
} catch (error) {
throw toBamlError(error);
}
}
async QueryMemory(memory, query, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return await this.runtime.buildRequest(
"QueryMemory",
{
"memory": memory,
"query": query
},
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
true,
env
);
} catch (error) {
throw toBamlError(error);
}
}
}
class LlmResponseParser {
constructor(runtime, ctxManager) {
this.runtime = runtime;
this.ctxManager = ctxManager;
}
CreatePartialRecipe(llmResponse, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return this.runtime.parseLlmResponse(
"CreatePartialRecipe",
llmResponse,
false,
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
env
);
} catch (error) {
throw toBamlError(error);
}
}
ExtractData(llmResponse, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return this.runtime.parseLlmResponse(
"ExtractData",
llmResponse,
false,
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
env
);
} catch (error) {
throw toBamlError(error);
}
}
QueryMemory(llmResponse, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return this.runtime.parseLlmResponse(
"QueryMemory",
llmResponse,
false,
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
env
);
} catch (error) {
throw toBamlError(error);
}
}
}
class LlmStreamParser {
constructor(runtime, ctxManager) {
this.runtime = runtime;
this.ctxManager = ctxManager;
}
CreatePartialRecipe(llmResponse, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return this.runtime.parseLlmResponse(
"CreatePartialRecipe",
llmResponse,
true,
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
env
);
} catch (error) {
throw toBamlError(error);
}
}
ExtractData(llmResponse, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return this.runtime.parseLlmResponse(
"ExtractData",
llmResponse,
true,
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
env
);
} catch (error) {
throw toBamlError(error);
}
}
QueryMemory(llmResponse, __baml_options__) {
try {
const env = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
return this.runtime.parseLlmResponse(
"QueryMemory",
llmResponse,
true,
this.ctxManager.cloneContext(),
__baml_options__?.tb?.__tb(),
__baml_options__?.clientRegistry,
env
);
} catch (error) {
throw toBamlError(error);
}
}
}
class BamlAsyncClient {
runtime;
ctxManager;
streamClient;
httpRequest;
httpStreamRequest;
llmResponseParser;
llmStreamParser;
bamlOptions;
constructor(runtime, ctxManager, bamlOptions) {
this.runtime = runtime;
this.ctxManager = ctxManager;
this.streamClient = new BamlStreamClient(runtime, ctxManager, bamlOptions);
this.httpRequest = new AsyncHttpRequest(runtime, ctxManager);
this.httpStreamRequest = new AsyncHttpStreamRequest(runtime, ctxManager);
this.llmResponseParser = new LlmResponseParser(runtime, ctxManager);
this.llmStreamParser = new LlmStreamParser(runtime, ctxManager);
this.bamlOptions = bamlOptions || {};
}
withOptions(bamlOptions) {
return new BamlAsyncClient(this.runtime, this.ctxManager, bamlOptions);
}
get stream() {
return this.streamClient;
}
get request() {
return this.httpRequest;
}
get streamRequest() {
return this.httpStreamRequest;
}
get parse() {
return this.llmResponseParser;
}
get parseStream() {
return this.llmStreamParser;
}
async CreatePartialRecipe(memory, task, __baml_options__) {
try {
const options = { ...this.bamlOptions, ...__baml_options__ || {} };
const collector = options.collector ? Array.isArray(options.collector) ? options.collector : [options.collector] : [];
const env = options.env ? { ...process.env, ...options.env } : { ...process.env };
const raw = await this.runtime.callFunction(
"CreatePartialRecipe",
{
"memory": memory,
"task": task
},
this.ctxManager.cloneContext(),
options.tb?.__tb(),
options.clientRegistry,
collector,
env
);
return raw.parsed(false);
} catch (error) {
throw toBamlError(error);
}
}
async ExtractData(instructions, screenshot, domContent, __baml_options__) {
try {
const options = { ...this.bamlOptions, ...__baml_options__ || {} };
const collector = options.collector ? Array.isArray(options.collector) ? options.collector : [options.collector] : [];
const env = options.env ? { ...process.env, ...options.env } : { ...process.env };
const raw = await this.runtime.callFunction(
"ExtractData",
{
"instructions": instructions,
"screenshot": screenshot,
"domContent": domContent
},
this.ctxManager.cloneContext(),
options.tb?.__tb(),
options.clientRegistry,
collector,
env
);
return raw.parsed(false);
} catch (error) {
throw toBamlError(error);
}
}
async QueryMemory(memory, query, __baml_options__) {
try {
const options = { ...this.bamlOptions, ...__baml_options__ || {} };
const collector = options.collector ? Array.isArray(options.collector) ? options.collector : [options.collector] : [];
const env = options.env ? { ...process.env, ...options.env } : { ...process.env };
const raw = await this.runtime.callFunction(
"QueryMemory",
{
"memory": memory,
"query": query
},
this.ctxManager.cloneContext(),
options.tb?.__tb(),
options.clientRegistry,
collector,
env
);
return raw.parsed(false);
} catch (error) {
throw toBamlError(error);
}
}
}
class BamlStreamClient {
runtime;
ctxManager;
bamlOptions;
constructor(runtime, ctxManager, bamlOptions) {
this.runtime = runtime;
this.ctxManager = ctxManager;
this.bamlOptions = bamlOptions || { env: { ...process.env } };
}
CreatePartialRecipe(memory, task, __baml_options__) {
try {
const options = { ...this.bamlOptions, ...__baml_options__ || {} };
const collector = options.collector ? Array.isArray(options.collector) ? options.collector : [options.collector] : [];
const env = options.env ? { ...process.env, ...options.env } : { ...process.env };
const raw = this.runtime.streamFunction(
"CreatePartialRecipe",
{
"memory": memory,
"task": task
},
void 0,
this.ctxManager.cloneContext(),
options.tb?.__tb(),
options.clientRegistry,
collector,
env
);
return new BamlStream(
raw,
(a) => a,
(a) => a,
this.ctxManager.cloneContext()
);
} catch (error) {
throw toBamlError(error);
}
}
ExtractData(instructions, screenshot, domContent, __baml_options__) {
try {
const options = { ...this.bamlOptions, ...__baml_options__ || {} };
const collector = options.collector ? Array.isArray(options.collector) ? options.collector : [options.collector] : [];
const env = options.env ? { ...process.env, ...options.env } : { ...process.env };
const raw = this.runtime.streamFunction(
"ExtractData",
{
"instructions": instructions,
"screenshot": screenshot,
"domContent": domContent
},
void 0,
this.ctxManager.cloneContext(),
options.tb?.__tb(),
options.clientRegistry,
collector,
env
);
return new BamlStream(
raw,
(a) => a,
(a) => a,
this.ctxManager.cloneContext()
);
} catch (error) {
throw toBamlError(error);
}
}
QueryMemory(memory, query, __baml_options__) {
try {
const options = { ...this.bamlOptions, ...__baml_options__ || {} };
const collector = options.collector ? Array.isArray(options.collector) ? options.collector : [options.collector] : [];
const env = options.env ? { ...process.env, ...options.env } : { ...process.env };
const raw = this.runtime.streamFunction(
"QueryMemory",
{
"memory": memory,
"query": query
},
void 0,
this.ctxManager.cloneContext(),
options.tb?.__tb(),
options.clientRegistry,
collector,
env
);
return new BamlStream(
raw,
(a) => a,
(a) => a,
this.ctxManager.cloneContext()
);
} catch (error) {
throw toBamlError(error);
}
}
}
const b = new BamlAsyncClient(DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME, DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX);
const traceAsync = DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX.traceFnAsync.bind(DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX);
DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX.traceFnSync.bind(DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX);
DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX.upsertTags.bind(DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX);
const version = "0.90.2";
ThrowIfVersionMismatch(version);
class TypeBuilder {
tb;
BugFailureClassification;
ConnectorInstructions;
ExtractedData;
MisalignmentClassification;
ModularMemoryContext;
PartialRecipe;
QueryResponse;
constructor() {
this.tb = new TypeBuilder$1({
classes: /* @__PURE__ */ new Set([
"BugFailureClassification",
"ConnectorInstructions",
"ExtractedData",
"MisalignmentClassification",
"ModularMemoryContext",
"PartialRecipe",
"QueryResponse"
]),
enums: /* @__PURE__ */ new Set([]),
runtime: DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME
});
this.BugFailureClassification = this.tb.classViewer("BugFailureClassification", [
"reasoning",
"classification",
"title",
"expectedResult",
"actualResult",
"severity"
]);
this.ConnectorInstructions = this.tb.classViewer("ConnectorInstructions", [
"connectorId",
"instructions"
]);
this.ExtractedData = this.tb.classBuilder("ExtractedData", []);
this.MisalignmentClassification = this.tb.classViewer("MisalignmentClassification", [
"reasoning",
"classification",
"fault",
"message"
]);
this.ModularMemoryContext = this.tb.classViewer("ModularMemoryContext", [
"instructions",
"observationContent",
"connectorInstructions"
]);
this.PartialRecipe = this.tb.classBuilder("PartialRecipe", [
"reasoning"
]);
this.QueryResponse = this.tb.classBuilder("QueryResponse", []);
}
__tb() {
return this.tb._tb();
}
string() {
return this.tb.string();
}
literalString(value) {
return this.tb.literalString(value);
}
literalInt(value) {
return this.tb.literalInt(value);
}
literalBool(value) {
return this.tb.literalBool(value);
}
int() {
return this.tb.int();
}
float() {
return this.tb.float();
}
bool() {
return this.tb.bool();
}
list(type) {
return this.tb.list(type);
}
null() {
return this.tb.null();
}
map(key, value) {
return this.tb.map(key, value);
}
union(types) {
return this.tb.union(types);
}
addClass(name) {
return this.tb.addClass(name);
}
addEnum(name) {
return this.tb.addEnum(name);
}
addBaml(baml) {
this.tb.addBaml(baml);
}
}
function randomName() {
const chars = "abcdefghijklmnopqrstuvwxyz";
let result = "";
for (let i = 0; i < 12; i++) {
result += chars.charAt(Math.floor(Math.random() * chars.length));
}
return result;
}
class SchemaAdder {
tb;
rootSchema;
refCache = {};
constructor(tb, rootSchema) {
this.tb = tb;
this.rootSchema = rootSchema;
}
_parseObject(jsonSchema) {
if (jsonSchema.type !== "object") {
throw new Error(`_parseObject called with non-object type: ${jsonSchema.type}`);
}
let name = jsonSchema.title;
if (name === void 0 || typeof name !== "string") {
name = randomName();
}
const newCls = this.tb.addClass(name);
const requiredFields = jsonSchema.required || [];
if (!Array.isArray(requiredFields) || !requiredFields.every((rf) => typeof rf === "string")) {
throw new Error(`'required' property in object '${name}' must be an array of strings.`);
}
const properties = jsonSchema.properties;
if (properties && typeof properties === "object") {
for (const [fieldName, fieldSchemaUntyped] of Object.entries(properties)) {
if (typeof fieldSchemaUntyped !== "object" || fieldSchemaUntyped === null) {
console.warn(`Property schema for '${fieldName}' in class '${name}' is not a valid object/schema. Skipping.`);
continue;
}
const fieldSchema = fieldSchemaUntyped;
const defaultValue = fieldSchema.default;
let fieldType;
if (fieldSchema.properties === void 0 && fieldSchema.type === "object") {
console.warn(
`Field '${fieldName}' in class '${name}' uses generic dict type (object without properties) which defaults to map<string, string>. If a more specific type is needed, please provide a specific schema with properties.`
);
fieldType = this.tb.map(this.tb.string(), this.tb.string());
} else {
fieldType = this.parse(fieldSchema);
}
if (!requiredFields.includes(fieldName)) {
if (defaultValue === void 0) {
fieldType = fieldType.optional();
}
}
const propertyBuilder = newCls.addProperty(fieldName, fieldType);
let fieldDescription = fieldSchema.description;
if (typeof fieldDescription === "string") {
let finalDescription = fieldDescription.trim();
if (defaultValue !== void 0) {
finalDescription = `${finalDescription}
Default: ${JSON.stringify(defaultValue)}`;
finalDescription = finalDescription.trim();
}
if (finalDescription.length > 0) {
propertyBuilder.description(finalDescription);
}
} else if (defaultValue !== void 0) {
const defaultDesc = `Default: ${JSON.stringify(defaultValue)}`;
propertyBuilder.description(defaultDesc.trim());
}
}
}
return newCls.type();
}
_parseString(jsonSchema) {
if (jsonSchema.type !== "string") {
throw new Error(`_parseString called with non-string type: ${jsonSchema.type}`);
}
const title = jsonSchema.title;
const enumValues = jsonSchema.enum;
if (enumValues) {
if (!Array.isArray(enumValues)) {
throw new Error(`'enum' property for string type '${title || "anonymous"}' must be an array.`);
}
const stringEnumValues = [];
for (const val of enumValues) {
stringEnumValues.push(String(val));
}
if (title === void 0 || typeof title !== "string") {
if (stringEnumValues.length === 0) {
console.warn(`Anonymous enum (string type with 'enum' but no 'title') has no values. Defaulting to plain string type.`);
return this.tb.string();
}
return this.tb.union(stringEnumValues.map((value) => this.tb.literalString(value)));
}
const newEnum = this.tb.addEnum(title);
if (stringEnumValues.length === 0) {
console.warn(`Enum '${title}' has no values. An empty enum was created.`);
}
for (const value of stringEnumValues) {
newEnum.addValue(value);
}
return newEnum.type();
}
return this.tb.string();
}
_parseArray(jsonSchema) {
if (jsonSchema.type !== "array") {
throw new Error(`_parseArray called with non-array type: ${jsonSchema.type}`);
}
const itemsSchema = jsonSchema.items;
if (itemsSchema === void 0) {
throw new Error(`Array field '${jsonSchema.title || "untitled array"}' is missing 'items' definition.`);
}
if (Array.isArray(itemsSchema) || typeof itemsSchema !== "object" || itemsSchema === null) {
throw new Error(`'items' property for array '${jsonSchema.title || "untitled array"}' must be a single schema object. Tuple/array types for 'items' are not supported by this converter (matching Python script's direct parsing of items).`);
}
return this.parse(itemsSchema).list();
}
_loadRef(ref) {
if (!ref.startsWith("#/")) {
throw new Error(`Only local references are supported: ${ref}`);
}
if (this.refCache[ref]) {
return this.refCache[ref];
}
const pathParts = ref.substring(2).split("/");
if (pathParts.length !== 2 || !pathParts[0] || !pathParts[1]) {
throw new Error(`Unsupported $ref format: '${ref}'. Expected format like '#/collectionKey/definitionKey' matching Python logic.`);
}
const collectionKey = pathParts[0];
const definitionKey = pathParts[1];
const collection = this.rootSchema[collectionKey];
if (typeof collection !== "object" || collection === null) {
throw new Error(`Reference collection '${collectionKey}' for $ref '${ref}' not found or not an object in the root schema.`);
}
if (!collection.hasOwnProperty(definitionKey)) {
throw new Error(`Reference item '${definitionKe