magnitude-core/dist/ai/baml_client/inlinedbaml.js

Magnitude e2e testing agent

/*************************************************************************************************

Welcome to Baml! To use this generated code, please run one of the following:

$ npm install @boundaryml/baml
$ yarn add @boundaryml/baml
$ pnpm add @boundaryml/baml

*************************************************************************************************/
// This file was generated by BAML: please do not edit it. Instead, edit the
// BAML files and re-generate this code using: baml-cli generate
// You can install baml-cli with:
//  $ npm install @boundaryml/baml
//
/* eslint-disable */
// tslint:disable
// @ts-nocheck
// biome-ignore format: autogenerated code
const fileMap = {
    "behaviorTests.baml": "\ntest DoNotOverplan1 {\n    functions [CreatePartialRecipe]\n    args {\n        screenshot {\n            url \"https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/do_not_overplan_1.png\"\n        }\n        step {\n            description #\"Create a new company\"#\n            checks [\n                #\"Company added successfully\"#\n            ]\n            testData {\n                data [\n\n                ]\n                other #\"Make up the first 2 values and use defaults for the rest\"#\n            }\n        }\n        previousActions [\n\n        ]\n    }\n    @@assert( one_action, {{ this.actions|length == 1 }} )\n    @@assert( not_marked_finished, {{ this.finished == false }} )\n}\n\n\ntest DoNotOverplan2 {\n    // Especially with the test data, planner might be tempted to click create company and also plan to fill form fields.\n    // BUT we do not want it to do that - it should only click the button.\n    functions [CreatePartialRecipe]\n    args {\n        screenshot {\n            url \"https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/do_not_overplan_2.png\"\n        }\n        step {\n            description #\"Create a new company\"#\n            checks [\n                #\"Company added successfully\"#\n            ]\n            testData {\n                data [\n\n                ]\n                other #\"Make up the first 2 values and use defaults for the rest\"#\n            }\n        }\n        previousActions [\n            #\"\n            {\n                \"variant\": \"click\",\n                \"target\": \"Companies option in the left sidebar menu\"\n            }\n            \"#\n        ]\n    }\n    @@assert( one_action, {{ this.actions|length == 1 }} )\n    @@assert( not_marked_finished, {{ this.finished == false }} )\n}\n\ntest OptimalPlanning1 {\n    functions [CreatePartialRecipe]\n    args {\n        screenshot {\n            url \"https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/do_not_underplan_1.png\"\n        }\n        step {\n            description #\"Create a new company\"#\n            checks [\n                #\"Company added successfully\"#\n            ]\n            testData {\n                data [\n\n                ]\n                other #\"Make up the first 2 values and use defaults for the rest\"#\n            }\n        }\n        previousActions [\n        #\"\n            {\n                \"variant\": \"click\",\n                \"target\": \"Companies option in the left sidebar menu\"\n            }\n        \"#,\n        #\"\n            {\n                \"variant\": \"click\",\n                \"target\": \"Add Company button in the top right corner\"\n            }\n        \"#\n        ]\n        \n    }\n    // optimal is type name, type domain, click save, mark finished\n    @@assert( optimal_actions, {{ this.actions|length == 3 }} )\n    // @@assert( not_marked_finished, {{ this.finished == false }} )\n}\n\n\n\n// test CheckContextRemove1 {\n//     // Manual for now - ideally we'd want to verify that executor can check the returned description successfully\n//     functions [RemoveImplicitCheckContext]\n//     args {\n//         screenshot {\n//             url \"https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/check_context_remove_1.png\"\n//         }\n//         check #\"Company added successfully\"#\n//         previousActions [\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"Companies option in the left sidebar menu\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"'Add Company' button in the top right corner of the Companies page\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"type\",\n//                     \"target\": \"Name input field\",\n//                     \"content\": \"Acme Solutions\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"type\",\n//                     \"target\": \"Domain input field\",\n//                     \"content\": \"acmesolutions.com\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"type\",\n//                     \"target\": \"Logo URL input field\",\n//                     \"content\": \"https://example.com/image.jpg\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"type\",\n//                     \"target\": \"Industry input field\",\n//                     \"content\": \"Technology\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"Size dropdown menu\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"1-10 option in the Size dropdown\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"Revenue dropdown menu\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"<$100K option in the Revenue dropdown\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"ICP Fit dropdown menu\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"Medium option in the ICP Fit dropdown\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"Est. ARR dropdown menu\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"<$10K option in the Est. ARR dropdown\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"Connection Strength dropdown menu\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"Weak option in the Connection Strength dropdown\"\n//                 }\n//             \"#,\n//             #\"\n//                 {\n//                     \"variant\": \"click\",\n//                     \"target\": \"Save button\"\n//                 }\n//             \"#\n//         ]\n//     }\n// }\n\ntest TargetGrounding1 {\n    // This test is a screenshot of Magnitude dasboard. Goal is to go to test playground.\n    // Originally failed because target was described as \"TC-0 Playground Test Case button\"\n    // This aligns more with the card shown rather that the specific button that actually needs to be clicked\n    // which has text \"Experiment Now (Free!)\"\n    // We want the planner to ground the targets in very specific details when available, such as the text on a button.\n    // This test asserts that the generated target includes specific text from the button, at least \"Experiment Now\"\n    functions [CreatePartialRecipe]\n    args {\n        screenshot {\n            url \"https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/target_grounding_1.png\"\n        }\n        step {\n            description #\"Go to test playground\"#\n            checks [\n            ]\n            testData {\n                data [\n\n                ]\n                other \"\"\n            }\n        }\n        previousActions [\n        ]\n    }\n    @@assert( includes_button_text, {{ \"Experiment Now (Free!)\" in this.actions[0].target }} )\n}",
    "clients.baml": "// Learn more about clients at https://docs.boundaryml.com/docs/snippets/clients/overview\n\nclient<llm> SonnetBedrock {\n    provider aws-bedrock\n    retry_policy Exponential\n    options {\n        inference_configuration {\n            temperature 0.0\n        }\n        model_id \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n    }\n}\n\nclient<llm> SonnetAnthropic {\n    provider anthropic\n    retry_policy Exponential\n    options {\n        model \"claude-sonnet-4-20250514\"\n        api_key env.ANTHROPIC_API_KEY\n        temperature 0.0\n        allowed_role_metadata [\"cache_control\"]\n        headers {\n            \"anthropic-beta\" \"prompt-caching-2024-07-31\"\n        }\n    }\n}\n\nclient<llm> GeminiPro {\n    provider \"google-ai\"\n    options {\n        api_key env.GOOGLE_API_KEY\n        model \"gemini-2.5-pro-preview-05-06\"\n        generationConfig {\n            temperature 0.0\n        }\n    }\n}\n\nclient<llm> GeminiProOpenRouter {\n    provider \"openai-generic\"\n    options {\n        base_url \"https://openrouter.ai/api/v1\"\n        api_key env.OPENROUTER_API_KEY\n        model \"google/gemini-2.5-pro-preview-03-25\"\n        temperature 0.0\n    }\n}\n\nclient<llm> GeminiFlash {\n    provider \"openai-generic\"\n    options {\n        base_url \"https://openrouter.ai/api/v1\"\n        api_key env.OPENROUTER_API_KEY\n        model \"google/gemini-2.5-flash-preview\"\n        temperature 0.0\n    }\n}\n\nclient<llm> GPT {\n    provider openai\n    options {\n        model \"gpt-4.1-2025-04-14\"\n        api_key env.OPENAI_API_KEY\n        temperature 0.0\n    }\n}\n\n// client<llm> Macro {\n//     provider openai\n//     options {\n//         model \"gpt-4o\"\n//         api_key env.OPENAI_API_KEY\n//         temperature 0.0\n//     }\n// }\n\nclient<llm> NovitaLlamaMaverick {\n    provider \"openai-generic\"\n    retry_policy Exponential\n    options {\n        base_url \"https://api.novita.ai/v3/openai\"\n        api_key env.NOVITA_API_KEY\n        model \"meta-llama/llama-4-maverick-17b-128e-instruct-fp8\"\n        temperature 0.0\n        logprobs true\n    }\n}\n\nclient<llm> Molmo {\n    provider \"openai-generic\"\n    retry_policy Exponential\n    options {\n        base_url env.MOLMO_VLLM_BASE_URL\n        api_key env.MOLMO_VLLM_API_KEY\n        model \"Molmo-7B-D-0924\"\n        temperature 0.0\n        logprobs true\n    }\n}\n\nretry_policy Exponential {\n    max_retries 5\n    strategy {\n        type exponential_backoff\n        delay_ms 500\n        multiplier 2.0\n        max_delay_ms 10000\n    }\n}\n\n// Default used by macro.ts\nretry_policy DefaultRetryPolicy {\n    max_retries 5\n    strategy {\n        type exponential_backoff\n        delay_ms 500\n        multiplier 2.0\n        max_delay_ms 10000\n    }\n}\n",
    "diagnosis.baml": "\n// class FailureClassification {\n//     classification \"bug\" | \"misalignment\"\n// }\n\nclass BugFailureClassification {\n    reasoning string\n    classification \"bug\"\n    title string\n    expectedResult string\n    actualResult string\n    severity \"critical\" | \"high\" | \"medium\" | \"low\"\n}\n\nclass MisalignmentClassification {\n    reasoning string\n    classification \"misalignment\"\n\n    fault \"test\" | \"agent\" @description(#\"\n        Is this likely a problem with the test case or some issue with the agent?\n    \"#)\n    \n    message string @description(#\"\n        If fault of test case:\n        Message to the developer who wrote the test case to help them understand what may have happened and how they might be able to fix it.\n        Be simple, direct, and informative.\n        If fault of agent:\n        Message to the developer who wrote the test case (not the person who wrote the agent) to help them understand what may have happened, and suggest possible ways to adjust the test case to accommodate the issue.\n    \"#)\n}\n\n// class FailureClassification {\n//     reasoning\n//     classification \"bug\" | \"misalignment\"\n// }\n\ntemplate_string BaseMeta #\"\n    You are observing the execution of an LLM agent that runs test cases.\n    This agent executes test cases by observing screenshots from the browser, acting out steps, and verifying checks.\n\"#\n\n// should this be given the original or adjusted check?\n// should this be given more ctx of the overall step or test case?\n// this is a tricky prompt, but its not as critical to get right as others\n// it does become more important if we rely on classify as bug/misalign then correct minor misaligns only if not bug\n// ^ if we need this then we should think of a more isolated/logical prompt to detect misalignments\n// function ClassifyCheckFailure (context: BrowserExecutionContext, check: string) -> BugFailureClassification | MisalignmentClassification {\n//     client SonnetAnthropic\n//     prompt #\"\n//         {{ _.role(\"system\") }}\n//         {{ BaseMeta() }}\n\n//         The agent just marked a check as failed. Your job is to figure out why.\n\n//         Either:\n//         (1) The web application actually has a bug in it\n//         or\n//         (2) There's some misalignment between the test case and what is in the interface\n\n//         If there is a bug, please break it down in detail.\n//         If it is a misalignment, please describe what you think happened.\n\n//         Use the provided history of actions the agent took as well as the most recent screenshot to help you identify what happened.\n\n//         {{ ctx.output_format }}\n\n//         {{ _.role(\"user\") }}\n\n//         The \"check\" that was marked as failed:\n//         {{ check }}\n\n//         {{ DescribeBrowserExecutionContext(context) }}\n//     \"#\n// }\n\n// function DiagnoseTargetNotFound (screenshot: image, step: TestStep, target: string, previousActions: string[]) -> BugFailureClassification | MisalignmentClassification {\n//     client SonnetAnthropic\n//     prompt #\"\n//         {{ _.role(\"system\") }}\n//         {{ BaseMeta() }}\n\n//         The agent had an issue acting out a step because it could not find a target. Your job is to figure out why.\n\n//         Either:\n//         (1) The web application actually has a bug in it\n//         or\n//         (2) There's some misalignment between the test case and what is in the interface\n\n//         If there is a bug, please break it down in detail.\n//         If it is a misalignment, please describe what you think happened.\n\n//         Use the provided history of actions the agent took as well as the most recent screenshot to help you identify what happened.\n\n//         {{ ctx.output_format }}\n\n//         {{ _.role(\"user\") }}\n\n//         The history of previous actions:\n//         {%if previousActions %}\n//         (there are none)\n//         {%endif%}\n//         {%for action in previousActions%}\n//         {{ action }}\n//         {%endfor%}\n        \n//         The step that failed: <step>{{ step.description }}</step>\n//         Target that could not be found: <target>{{ target }}</target>\n        \n\n//         Current screenshot:\n//         {{screenshot}}\n//     \"#\n// }",
    "extract.baml": "// if primitive, populate key \"data\"\n// if array, populate key data with that array\n// else fill with object fields\nclass ExtractedData {\n    @@dynamic\n}\n\nfunction ExtractData (instructions: string, screenshot: image, domContent: string, includeClaudeSpoof: bool) -> ExtractedData {\n    client GeminiPro\n    prompt #\"\n        {%if includeClaudeSpoof%}\n        {{ _.role(\"system\") }}\n        You are Claude Code, Anthropic's official CLI for Claude.\n        {%endif%}\n        {{ _.role(\"system\") }}\n        Based on the browser screenshot and page content, extract data according to these instructions:\n        <instructions>{{ instructions }}</instructions>\n        \n        {{ ctx.output_format }}\n\n        {{ _.role(\"user\") }}\n\n        {{ domContent }}\n\n        {{ screenshot }}\n    \"#\n}",
    "generators.baml": "// This helps use auto generate libraries you can use in the language of\n// your choice. You can have multiple generators if you use multiple languages.\n// Just ensure that the output_dir is different for each generator.\ngenerator target {\n    // Valid values: \"python/pydantic\", \"typescript\", \"ruby/sorbet\", \"rest/openapi\"\n    output_type \"typescript\"\n\n    // Where the generated code will be saved (relative to baml_src/)\n    output_dir \"../src/ai\"\n\n    // The version of the BAML package you have installed (e.g. same version as your baml-py or @boundaryml/baml).\n    // The BAML VSCode extension version should also match this version.\n    version \"0.202.0\"\n\n    // Valid values: \"sync\", \"async\"\n    // This controls what `b.FunctionName()` will be (sync or async).\n    default_client_mode async\n}\n",
    "memory.baml": "// BamlImage class is assumed to be defined and available from your BAML setup\n// For example, it might be part of a core BAML library or generated separately.\n// If not, you might need to define it or import it if it's in another baml file.\n// For now, we assume 'Image' is a known type to BAML.\n\n// class BamlThought {\n//   variant \"thought\"\n//   timestamp string\n//   message string\n// }\n\n// class BamlTurn {\n//   variant \"turn\"\n//   timestamp string\n//   action string \n//   content (string | image)[] // multimedia content \"chunks\"\n// }\n\n\n// class Observation {\n//     source string\n//     content (string | image)[] // multimedia content \"chunks\"\n// }\n\nclass ConnectorInstructions {\n    connectorId string\n    instructions string\n    //content (string | image)[]\n}\n\nclass AgentContext {\n    instructions string? // additional task-level or agent-level instructions\n    //history (BamlThought | BamlTurn)[]\n    //observations Observation[]\n    //observationContent (string | image)[] // complete rendered observation content as multimedia content \"chunks\"\n    observationContent MultiMediaMessage[]\n    //currentTimestamp string\n    connectorInstructions ConnectorInstructions[]\n}\n",
    "planner.baml": "class PartialRecipe {\n    // this CoT pipeline is not thoroughly tested against alternatives\n    // observations string? @description(#\"Any key observations about past actions or current state\"#)\n    // meta_reasoning string @description(#\"Reflect on the current state of task execution with respect to your own abilities as an agent\"#)\n    // reasoning string @description(#\"Consider what you can see right now and what actions you can plan without guessing\"#)\n\n    // Simplifying CoT - seemed to be causing underplanning on Claude\n    //observations string @description(#\"What important clues or key information are worth considering?\"#)\n    reasoning string @description(#\"What is the most actions you can safely take at once?\"#)\n\n    //actions ActionIntent[]\n    @@dynamic\n    //finished bool\n}\n\n\n// render \"chunked\" multimedia content of strings or images with no added whitespace between\n// template_string RenderContent(content: (string | image)[]) #\"\n//     {% for chunk in content -%}{{chunk}}{%- endfor %}\n// \"#\n\ntemplate_string RenderMultiMediaContentParts(parts: MultiMediaContentPart[]) #\"\n    {% for part in parts -%}{{part}}{%- endfor %}\n\"#\n\ntemplate_string RenderMessages(messages: MultiMediaMessage[]) #\"\n    {% for message in messages -%}\n    {%if message.cacheControl %}\n    {{ _.role(message.role, cache_control={\"type\": \"ephemeral\"}) }}\n    {{ RenderMultiMediaContentParts(message.content) }}\n    {%else%}\n    {{ _.role(message.role) }}\n    {{ RenderMultiMediaContentParts(message.content) }}\n    {%endif%}\n    {%- endfor %}\n\"#\n\n\n// template_string DescribeModularMemoryContext(context: ModularMemoryContext) #\"\n//     {% for entry in context.history %}\n//         {% if entry.variant == \"thought\" %}\n//             [{{entry.timestamp}}] {{entry.message}}\n//         {% elif entry.variant == \"turn\" %}\n//             [{{entry.timestamp}}] {{entry.action}}\n//             {{ RenderContent(entry.content) }}\n//         {% endif %}\n//     {% endfor %}\n// \"#\n\n// unused\ntemplate_string HybridMeta #\"\n    <meta>\n    You are a web agent powered by (1) a powerful LLM (you) and (2) a small vision model (Moondream).\n    You operate by planning out actions over time to try and complete a certain task.\n    To do this successfully you may need to consider and adjust for your own limitations as an agent, so here is some information about how you work:\n    - You plan several actions ahead of time without trying to guess, which are then executed\n    - You are then given a new screenshot of the webpage which is the result of all actions taken\n    - This occurs in a loop until you complete your task\n    - Mouse movements require a \"target\", which is located by Moondream as a specific coordinate on the page\n    - Moondream may sometimes fail to locate the target you expected it to.\n    - Therefore, actions may not always be completely successful and you may need to try different variations\n    </meta>\n\"#\n\ntemplate_string DescribeConnectorInstructions(memory: AgentContext) #\"\n    {% for conn in memory.connectorInstructions %}\n    <{{ conn.connectorId }}>\n    {{- conn.instructions -}}\n    </{{ conn.connectorId }}>\n    {% endfor %}\n\"#\n\ntemplate_string InstructionsWithContext(memory: AgentContext, instructions: string) #\"\n    {{ _.role(\"system\", cache_control={\"type\": \"ephemeral\"}) }}\n    <instructions>\n    {{ instructions }}\n    {%if memory.instructions %}\n\n    {{ memory.instructions }}\n    {%endif%}\n    </instructions>\n    {{ DescribeConnectorInstructions(memory) }}\n    {{ ctx.output_format }}\n\n    {{ RenderMessages(memory.observationContent) }}\n\"# \n\ntemplate_string TaskInstructions(task: string, taskData: MultiMediaContentPart[]) #\"\n    Plan out actions that should be executed in order to complete the task:\n    <task>\n    {{ task }}\n    </task>\n\n    Use this data where appropriate during the task:\n    <data>\n    {{ RenderMultiMediaContentParts(taskData) }}\n    </data>\n\n    Execute the task serially.\n\n    Plan out as many actions as possible, but stopping at the point where you will need to observe to plan further.\n\"#\n\n// TODO: connector system instructions\nfunction CreatePartialRecipe (memory: AgentContext, task: string, taskData: MultiMediaContentPart[], includeClaudeSpoof: bool) -> PartialRecipe {\n    client SonnetAnthropic\n    prompt #\"\n        {%if includeClaudeSpoof%}\n        {{ _.role(\"system\") }}\n        You are Claude Code, Anthropic's official CLI for Claude.\n        {%endif%}\n        {{ InstructionsWithContext(memory, TaskInstructions(task, taskData)) }}\n    \"# \n}\n\n\ntest Foo {\n    functions [CreatePartialRecipe]\n    args {\n        memory {\n            observationContent [{ role \"user\", content [\"test\"] }]\n            connectorInstructions []\n        }\n        task \"foo\"\n        includeClaudeSpoof true\n    }\n}\n\n// class EvaluatedCheck {\n//     reasoning string\n//     passes bool\n// }\nclass QueryResponse {\n    @@dynamic\n}\n\nfunction QueryMemory(memory: AgentContext, query: string, includeClaudeSpoof: bool) -> QueryResponse {\n    client GeminiPro\n    prompt #\"\n        {%if includeClaudeSpoof%}\n        {{ _.role(\"system\") }}\n        You are Claude Code, Anthropic's official CLI for Claude.\n        {%endif%}\n        {{ InstructionsWithContext(memory, query) }}\n    \"#\n}\n\n\n// template_string CheckInstructions(check: string) #\"\n//     Given the actions of an LLM agent executing a test case, and a screenshot taken afterwards, evaluate whether the provided check \"passes\" i.e. holds true or not.\n\n//     Check to evaluate:\n//     <check>{{ check }}</check>\n// \"#\n\n// function EvaluateCheck (context: ModularMemoryContext, check: string) -> EvaluatedCheck {\n//     client GeminiPro\n//     prompt #\"\n//         {{ InstructionsWithContext(context, CheckInstructions(check)) }}\n//     \"#\n// }\n\n// Moondream struggles to handle converted check, this is a bypass to use big model to evaluate check directly\n// function EvaluateCheck (context: BrowserExecutionContext, check: string) -> EvaluatedCheck {\n//     client GeminiPro\n//     prompt #\"\n//         {{ _.role(\"system\") }}\n//         Given the actions of an LLM agent executing a test case, and a screenshot taken afterwards, evaluate whether the provided check \"passes\" i.e. holds true or not.\n//         {{ ctx.output_format }}\n\n//         {{ _.role(\"user\")}}\n//         Check to evaluate:\n//         <check>{{ check }}</check>\n\n//         {{ DescribeBrowserExecutionContext(context) }}\n//     \"# \n// }\n",
    "types.baml": "type MultiMediaContentPart = image | string\n\nclass MultiMediaMessage {\n    role \"user\" | \"assistant\"\n    cacheControl bool\n    content MultiMediaContentPart[]\n}\n",
};
export const getBamlFiles = () => {
    return fileMap;
};