magnitude-core/dist/index.js

Magnitude e2e testing agent

'use strict';

var logging = require('@boundaryml/baml/logging');
var baml = require('@boundaryml/baml');
var pino = require('pino');
var EventEmitter = require('eventemitter3');
var z = require('zod');
var sharp = require('sharp');
var fs = require('node:fs');
var path = require('node:path');
var os = require('node:os');
var node_child_process = require('node:child_process');
var cuid2 = require('@paralleldrive/cuid2');
var posthogNode = require('posthog-node');
var node_module = require('node:module');
var crypto = require('crypto');
var fs$1 = require('fs');
var os$1 = require('os');
var path$1 = require('path');
var ansis = require('ansis');
var type_builder = require('@boundaryml/baml/type_builder');
var zodToJsonSchema = require('zod-to-json-schema');
var playwright = require('playwright');
var objectHash = require('object-hash');
var crypto$1 = require('node:crypto');
var moondream = require('moondream');
var magnitudeExtract = require('magnitude-extract');

var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
function _interopNamespaceDefault(e) {
  var n = Object.create(null);
  if (e) {
    Object.keys(e).forEach(function (k) {
      if (k !== 'default') {
        var d = Object.getOwnPropertyDescriptor(e, k);
        Object.defineProperty(n, k, d.get ? d : {
          enumerable: true,
          get: function () { return e[k]; }
        });
      }
    });
  }
  n.default = e;
  return Object.freeze(n);
}

var fs__namespace = /*#__PURE__*/_interopNamespaceDefault(fs);
var path__namespace = /*#__PURE__*/_interopNamespaceDefault(path);
var os__namespace = /*#__PURE__*/_interopNamespaceDefault(os);

const fileMap = {
  "behaviorTests.baml": `
test DoNotOverplan1 {
    functions [CreatePartialRecipe]
    args {
        screenshot {
            url "https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/do_not_overplan_1.png"
        }
        step {
            description #"Create a new company"#
            checks [
                #"Company added successfully"#
            ]
            testData {
                data [

                ]
                other #"Make up the first 2 values and use defaults for the rest"#
            }
        }
        previousActions [

        ]
    }
    @@assert( one_action, {{ this.actions|length == 1 }} )
    @@assert( not_marked_finished, {{ this.finished == false }} )
}


test DoNotOverplan2 {
    // Especially with the test data, planner might be tempted to click create company and also plan to fill form fields.
    // BUT we do not want it to do that - it should only click the button.
    functions [CreatePartialRecipe]
    args {
        screenshot {
            url "https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/do_not_overplan_2.png"
        }
        step {
            description #"Create a new company"#
            checks [
                #"Company added successfully"#
            ]
            testData {
                data [

                ]
                other #"Make up the first 2 values and use defaults for the rest"#
            }
        }
        previousActions [
            #"
            {
                "variant": "click",
                "target": "Companies option in the left sidebar menu"
            }
            "#
        ]
    }
    @@assert( one_action, {{ this.actions|length == 1 }} )
    @@assert( not_marked_finished, {{ this.finished == false }} )
}

test OptimalPlanning1 {
    functions [CreatePartialRecipe]
    args {
        screenshot {
            url "https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/do_not_underplan_1.png"
        }
        step {
            description #"Create a new company"#
            checks [
                #"Company added successfully"#
            ]
            testData {
                data [

                ]
                other #"Make up the first 2 values and use defaults for the rest"#
            }
        }
        previousActions [
        #"
            {
                "variant": "click",
                "target": "Companies option in the left sidebar menu"
            }
        "#,
        #"
            {
                "variant": "click",
                "target": "Add Company button in the top right corner"
            }
        "#
        ]
        
    }
    // optimal is type name, type domain, click save, mark finished
    @@assert( optimal_actions, {{ this.actions|length == 3 }} )
    // @@assert( not_marked_finished, {{ this.finished == false }} )
}



// test CheckContextRemove1 {
//     // Manual for now - ideally we'd want to verify that executor can check the returned description successfully
//     functions [RemoveImplicitCheckContext]
//     args {
//         screenshot {
//             url "https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/check_context_remove_1.png"
//         }
//         check #"Company added successfully"#
//         previousActions [
//             #"
//                 {
//                     "variant": "click",
//                     "target": "Companies option in the left sidebar menu"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "'Add Company' button in the top right corner of the Companies page"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "type",
//                     "target": "Name input field",
//                     "content": "Acme Solutions"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "type",
//                     "target": "Domain input field",
//                     "content": "acmesolutions.com"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "type",
//                     "target": "Logo URL input field",
//                     "content": "https://example.com/image.jpg"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "type",
//                     "target": "Industry input field",
//                     "content": "Technology"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "Size dropdown menu"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "1-10 option in the Size dropdown"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "Revenue dropdown menu"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "<$100K option in the Revenue dropdown"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "ICP Fit dropdown menu"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "Medium option in the ICP Fit dropdown"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "Est. ARR dropdown menu"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "<$10K option in the Est. ARR dropdown"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "Connection Strength dropdown menu"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "Weak option in the Connection Strength dropdown"
//                 }
//             "#,
//             #"
//                 {
//                     "variant": "click",
//                     "target": "Save button"
//                 }
//             "#
//         ]
//     }
// }

test TargetGrounding1 {
    // This test is a screenshot of Magnitude dasboard. Goal is to go to test playground.
    // Originally failed because target was described as "TC-0 Playground Test Case button"
    // This aligns more with the card shown rather that the specific button that actually needs to be clicked
    // which has text "Experiment Now (Free!)"
    // We want the planner to ground the targets in very specific details when available, such as the text on a button.
    // This test asserts that the generated target includes specific text from the button, at least "Experiment Now"
    functions [CreatePartialRecipe]
    args {
        screenshot {
            url "https://magnitude-test-screenshots.s3.us-east-1.amazonaws.com/target_grounding_1.png"
        }
        step {
            description #"Go to test playground"#
            checks [
            ]
            testData {
                data [

                ]
                other ""
            }
        }
        previousActions [
        ]
    }
    @@assert( includes_button_text, {{ "Experiment Now (Free!)" in this.actions[0].target }} )
}`,
  "clients.baml": '// Learn more about clients at https://docs.boundaryml.com/docs/snippets/clients/overview\n\nclient<llm> SonnetBedrock {\n    provider aws-bedrock\n    retry_policy Exponential\n    options {\n        inference_configuration {\n            temperature 0.0\n        }\n        model_id "anthropic.claude-3-5-sonnet-20240620-v1:0"\n    }\n}\n\nclient<llm> SonnetAnthropic {\n    provider anthropic\n    retry_policy Exponential\n    options {\n        model "claude-sonnet-4-20250514"\n        api_key env.ANTHROPIC_API_KEY\n        temperature 0.0\n        allowed_role_metadata ["cache_control"]\n        headers {\n            "anthropic-beta" "prompt-caching-2024-07-31"\n        }\n    }\n}\n\nclient<llm> GeminiPro {\n    provider "google-ai"\n    options {\n        api_key env.GOOGLE_API_KEY\n        model "gemini-2.5-pro-preview-05-06"\n        generationConfig {\n            temperature 0.0\n        }\n    }\n}\n\nclient<llm> GeminiProOpenRouter {\n    provider "openai-generic"\n    options {\n        base_url "https://openrouter.ai/api/v1"\n        api_key env.OPENROUTER_API_KEY\n        model "google/gemini-2.5-pro-preview-03-25"\n        temperature 0.0\n    }\n}\n\nclient<llm> GeminiFlash {\n    provider "openai-generic"\n    options {\n        base_url "https://openrouter.ai/api/v1"\n        api_key env.OPENROUTER_API_KEY\n        model "google/gemini-2.5-flash-preview"\n        temperature 0.0\n    }\n}\n\nclient<llm> GPT {\n    provider openai\n    options {\n        model "gpt-4.1-2025-04-14"\n        api_key env.OPENAI_API_KEY\n        temperature 0.0\n    }\n}\n\n// client<llm> Macro {\n//     provider openai\n//     options {\n//         model "gpt-4o"\n//         api_key env.OPENAI_API_KEY\n//         temperature 0.0\n//     }\n// }\n\nclient<llm> NovitaLlamaMaverick {\n    provider "openai-generic"\n    retry_policy Exponential\n    options {\n        base_url "https://api.novita.ai/v3/openai"\n        api_key env.NOVITA_API_KEY\n        model "meta-llama/llama-4-maverick-17b-128e-instruct-fp8"\n        temperature 0.0\n        logprobs true\n    }\n}\n\nclient<llm> Molmo {\n    provider "openai-generic"\n    retry_policy Exponential\n    options {\n        base_url env.MOLMO_VLLM_BASE_URL\n        api_key env.MOLMO_VLLM_API_KEY\n        model "Molmo-7B-D-0924"\n        temperature 0.0\n        logprobs true\n    }\n}\n\nretry_policy Exponential {\n    max_retries 5\n    strategy {\n        type exponential_backoff\n        delay_ms 500\n        multiplier 2.0\n        max_delay_ms 10000\n    }\n}\n\n// Default used by macro.ts\nretry_policy DefaultRetryPolicy {\n    max_retries 5\n    strategy {\n        type exponential_backoff\n        delay_ms 500\n        multiplier 2.0\n        max_delay_ms 10000\n    }\n}\n',
  "diagnosis.baml": `
// class FailureClassification {
//     classification "bug" | "misalignment"
// }

class BugFailureClassification {
    reasoning string
    classification "bug"
    title string
    expectedResult string
    actualResult string
    severity "critical" | "high" | "medium" | "low"
}

class MisalignmentClassification {
    reasoning string
    classification "misalignment"

    fault "test" | "agent" @description(#"
        Is this likely a problem with the test case or some issue with the agent?
    "#)
    
    message string @description(#"
        If fault of test case:
        Message to the developer who wrote the test case to help them understand what may have happened and how they might be able to fix it.
        Be simple, direct, and informative.
        If fault of agent:
        Message to the developer who wrote the test case (not the person who wrote the agent) to help them understand what may have happened, and suggest possible ways to adjust the test case to accommodate the issue.
    "#)
}

// class FailureClassification {
//     reasoning
//     classification "bug" | "misalignment"
// }

template_string BaseMeta #"
    You are observing the execution of an LLM agent that runs test cases.
    This agent executes test cases by observing screenshots from the browser, acting out steps, and verifying checks.
"#

// should this be given the original or adjusted check?
// should this be given more ctx of the overall step or test case?
// this is a tricky prompt, but its not as critical to get right as others
// it does become more important if we rely on classify as bug/misalign then correct minor misaligns only if not bug
// ^ if we need this then we should think of a more isolated/logical prompt to detect misalignments
// function ClassifyCheckFailure (context: BrowserExecutionContext, check: string) -> BugFailureClassification | MisalignmentClassification {
//     client SonnetAnthropic
//     prompt #"
//         {{ _.role("system") }}
//         {{ BaseMeta() }}

//         The agent just marked a check as failed. Your job is to figure out why.

//         Either:
//         (1) The web application actually has a bug in it
//         or
//         (2) There's some misalignment between the test case and what is in the interface

//         If there is a bug, please break it down in detail.
//         If it is a misalignment, please describe what you think happened.

//         Use the provided history of actions the agent took as well as the most recent screenshot to help you identify what happened.

//         {{ ctx.output_format }}

//         {{ _.role("user") }}

//         The "check" that was marked as failed:
//         {{ check }}

//         {{ DescribeBrowserExecutionContext(context) }}
//     "#
// }

// function DiagnoseTargetNotFound (screenshot: image, step: TestStep, target: string, previousActions: string[]) -> BugFailureClassification | MisalignmentClassification {
//     client SonnetAnthropic
//     prompt #"
//         {{ _.role("system") }}
//         {{ BaseMeta() }}

//         The agent had an issue acting out a step because it could not find a target. Your job is to figure out why.

//         Either:
//         (1) The web application actually has a bug in it
//         or
//         (2) There's some misalignment between the test case and what is in the interface

//         If there is a bug, please break it down in detail.
//         If it is a misalignment, please describe what you think happened.

//         Use the provided history of actions the agent took as well as the most recent screenshot to help you identify what happened.

//         {{ ctx.output_format }}

//         {{ _.role("user") }}

//         The history of previous actions:
//         {%if previousActions %}
//         (there are none)
//         {%endif%}
//         {%for action in previousActions%}
//         {{ action }}
//         {%endfor%}
        
//         The step that failed: <step>{{ step.description }}</step>
//         Target that could not be found: <target>{{ target }}</target>
        

//         Current screenshot:
//         {{screenshot}}
//     "#
// }`,
  "extract.baml": `// if primitive, populate key "data"
// if array, populate key data with that array
// else fill with object fields
class ExtractedData {
    @@dynamic
}

function ExtractData (instructions: string, screenshot: image, domContent: string, includeClaudeSpoof: bool) -> ExtractedData {
    client GeminiPro
    prompt #"
        {%if includeClaudeSpoof%}
        {{ _.role("system") }}
        You are Claude Code, Anthropic's official CLI for Claude.
        {%endif%}
        {{ _.role("system") }}
        Based on the browser screenshot and page content, extract data according to these instructions:
        <instructions>{{ instructions }}</instructions>
        
        {{ ctx.output_format }}

        {{ _.role("user") }}

        {{ domContent }}

        {{ screenshot }}
    "#
}`,
  "generators.baml": '// This helps use auto generate libraries you can use in the language of\n// your choice. You can have multiple generators if you use multiple languages.\n// Just ensure that the output_dir is different for each generator.\ngenerator target {\n    // Valid values: "python/pydantic", "typescript", "ruby/sorbet", "rest/openapi"\n    output_type "typescript"\n\n    // Where the generated code will be saved (relative to baml_src/)\n    output_dir "../src/ai"\n\n    // The version of the BAML package you have installed (e.g. same version as your baml-py or @boundaryml/baml).\n    // The BAML VSCode extension version should also match this version.\n    version "0.201.0"\n\n    // Valid values: "sync", "async"\n    // This controls what `b.FunctionName()` will be (sync or async).\n    default_client_mode async\n}\n',
  "memory.baml": `// BamlImage class is assumed to be defined and available from your BAML setup
// For example, it might be part of a core BAML library or generated separately.
// If not, you might need to define it or import it if it's in another baml file.
// For now, we assume 'Image' is a known type to BAML.

// class BamlThought {
//   variant "thought"
//   timestamp string
//   message string
// }

// class BamlTurn {
//   variant "turn"
//   timestamp string
//   action string 
//   content (string | image)[] // multimedia content "chunks"
// }


// class Observation {
//     source string
//     content (string | image)[] // multimedia content "chunks"
// }

class ConnectorInstructions {
    connectorId string
    instructions string
    //content (string | image)[]
}

class AgentContext {
    instructions string? // additional task-level or agent-level instructions
    //history (BamlThought | BamlTurn)[]
    //observations Observation[]
    //observationContent (string | image)[] // complete rendered observation content as multimedia content "chunks"
    observationContent MultiMediaMessage[]
    //currentTimestamp string
    connectorInstructions ConnectorInstructions[]
}
`,
  "planner.baml": `class PartialRecipe {
    // this CoT pipeline is not thoroughly tested against alternatives
    // observations string? @description(#"Any key observations about past actions or current state"#)
    // meta_reasoning string @description(#"Reflect on the current state of task execution with respect to your own abilities as an agent"#)
    // reasoning string @description(#"Consider what you can see right now and what actions you can plan without guessing"#)

    // Simplifying CoT - seemed to be causing underplanning on Claude
    //observations string @description(#"What important clues or key information are worth considering?"#)
    reasoning string @description(#"What is the most actions you can safely take at once?"#)

    //actions ActionIntent[]
    @@dynamic
    //finished bool
}


// render "chunked" multimedia content of strings or images with no added whitespace between
// template_string RenderContent(content: (string | image)[]) #"
//     {% for chunk in content -%}{{chunk}}{%- endfor %}
// "#

template_string RenderMultiMediaContentParts(parts: MultiMediaContentPart[]) #"
    {% for part in parts -%}{{part}}{%- endfor %}
"#

template_string RenderMessages(messages: MultiMediaMessage[]) #"
    {% for message in messages -%}
    {%if message.cacheControl %}
    {{ _.role(message.role, cache_control={"type": "ephemeral"}) }}
    {{ RenderMultiMediaContentParts(message.content) }}
    {%else%}
    {{ _.role(message.role) }}
    {{ RenderMultiMediaContentParts(message.content) }}
    {%endif%}
    {%- endfor %}
"#


// template_string DescribeModularMemoryContext(context: ModularMemoryContext) #"
//     {% for entry in context.history %}
//         {% if entry.variant == "thought" %}
//             [{{entry.timestamp}}] {{entry.message}}
//         {% elif entry.variant == "turn" %}
//             [{{entry.timestamp}}] {{entry.action}}
//             {{ RenderContent(entry.content) }}
//         {% endif %}
//     {% endfor %}
// "#

// unused
template_string HybridMeta #"
    <meta>
    You are a web agent powered by (1) a powerful LLM (you) and (2) a small vision model (Moondream).
    You operate by planning out actions over time to try and complete a certain task.
    To do this successfully you may need to consider and adjust for your own limitations as an agent, so here is some information about how you work:
    - You plan several actions ahead of time without trying to guess, which are then executed
    - You are then given a new screenshot of the webpage which is the result of all actions taken
    - This occurs in a loop until you complete your task
    - Mouse movements require a "target", which is located by Moondream as a specific coordinate on the page
    - Moondream may sometimes fail to locate the target you expected it to.
    - Therefore, actions may not always be completely successful and you may need to try different variations
    </meta>
"#

template_string DescribeConnectorInstructions(memory: AgentContext) #"
    {% for conn in memory.connectorInstructions %}
    <{{ conn.connectorId }}>
    {{- conn.instructions -}}
    </{{ conn.connectorId }}>
    {% endfor %}
"#

template_string InstructionsWithContext(memory: AgentContext, instructions: string) #"
    {{ _.role("system", cache_control={"type": "ephemeral"}) }}
    <instructions>
    {{ instructions }}
    {%if memory.instructions %}

    {{ memory.instructions }}
    {%endif%}
    </instructions>
    {{ DescribeConnectorInstructions(memory) }}
    {{ ctx.output_format }}

    {{ RenderMessages(memory.observationContent) }}
"# 

template_string TaskInstructions(task: string, taskData: MultiMediaContentPart[]) #"
    Plan out actions that should be executed in order to complete the task:
    <task>
    {{ task }}
    </task>

    Use this data where appropriate during the task:
    <data>
    {{ RenderMultiMediaContentParts(taskData) }}
    </data>

    Execute the task serially.

    Plan out as many actions as possible, but stopping at the point where you will need to observe to plan further.
"#

// TODO: connector system instructions
function CreatePartialRecipe (memory: AgentContext, task: string, taskData: MultiMediaContentPart[], includeClaudeSpoof: bool) -> PartialRecipe {
    client SonnetAnthropic
    prompt #"
        {%if includeClaudeSpoof%}
        {{ _.role("system") }}
        You are Claude Code, Anthropic's official CLI for Claude.
        {%endif%}
        {{ InstructionsWithContext(memory, TaskInstructions(task, taskData)) }}
    "# 
}


test Foo {
    functions [CreatePartialRecipe]
    args {
        memory {
            observationContent [{ role "user", content ["test"] }]
            connectorInstructions []
        }
        task "foo"
        includeClaudeSpoof true
    }
}

// class EvaluatedCheck {
//     reasoning string
//     passes bool
// }
class QueryResponse {
    @@dynamic
}

function QueryMemory(memory: AgentContext, query: string, includeClaudeSpoof: bool) -> QueryResponse {
    client GeminiPro
    prompt #"
        {%if includeClaudeSpoof%}
        {{ _.role("system") }}
        You are Claude Code, Anthropic's official CLI for Claude.
        {%endif%}
        {{ InstructionsWithContext(memory, query) }}
    "#
}


// template_string CheckInstructions(check: string) #"
//     Given the actions of an LLM agent executing a test case, and a screenshot taken afterwards, evaluate whether the provided check "passes" i.e. holds true or not.

//     Check to evaluate:
//     <check>{{ check }}</check>
// "#

// function EvaluateCheck (context: ModularMemoryContext, check: string) -> EvaluatedCheck {
//     client GeminiPro
//     prompt #"
//         {{ InstructionsWithContext(context, CheckInstructions(check)) }}
//     "#
// }

// Moondream struggles to handle converted check, this is a bypass to use big model to evaluate check directly
// function EvaluateCheck (context: BrowserExecutionContext, check: string) -> EvaluatedCheck {
//     client GeminiPro
//     prompt #"
//         {{ _.role("system") }}
//         Given the actions of an LLM agent executing a test case, and a screenshot taken afterwards, evaluate whether the provided check "passes" i.e. holds true or not.
//         {{ ctx.output_format }}

//         {{ _.role("user")}}
//         Check to evaluate:
//         <check>{{ check }}</check>

//         {{ DescribeBrowserExecutionContext(context) }}
//     "# 
// }
`,
  "types.baml": 'type MultiMediaContentPart = image | string\n\nclass MultiMediaMessage {\n    role "user" | "assistant"\n    cacheControl bool\n    content MultiMediaContentPart[]\n}\n'
};
const getBamlFiles = () => {
  return fileMap;
};

const env = { ...process.env };
const DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME = baml.BamlRuntime.fromFiles(
  "baml_src",
  getBamlFiles(),
  env
);
const DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_CTX = new baml.BamlCtxManager(DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME);

const logger = pino({
  level: process.env.MAGNITUDE_LOG_LEVEL || "warn",
  transport: process.stdout.isTTY ? {
    target: "pino-pretty",
    options: {
      colorize: !process.env.NO_COLOR,
      translateTime: "SYS:HH:MM:ss.l",
      ignore: "pid,hostname"
    }
  } : void 0
}).child({
  name: "agent"
});

function fnv1a32Hex(str, seed = 2166136261) {
  const FNV_PRIME_32 = 16777619;
  let hash = seed;
  const encoder = new TextEncoder();
  const bytes = encoder.encode(str);
  for (let i = 0; i < bytes.length; i++) {
    hash ^= bytes[i];
    hash = hash * FNV_PRIME_32 | 0;
  }
  const hexString = (hash >>> 0).toString(16);
  return hexString.padStart(8, "0");
}

class Image {
  /**
   * Wrapper for a Sharp image with conveniences to go to/from base64, convert to BAML, or serialize as JSON
   */
  // represents the start of a pipeline
  img;
  // Cached metadata property for sync access + required width/height properties
  //private metadata: Sharp['metadata'] & { width: number, height: number };
  //private content: string;
  //private mediaType: ImageMediaType;
  //constructor(type: 'url' | 'base64', content: string, mediaType: ImageMediaType) {
  constructor(img) {
    this.img = img;
  }
  static fromBase64(base64) {
    const base64Data = base64.replace(/^data:.*?;base64,/, "");
    return new Image(sharp(Buffer.from(base64Data, "base64")));
  }
  async getFormat() {
    const format = (await this.img.clone().metadata()).format;
    if (!format) throw new Error("Unable to get image format");
    return format;
  }
  /**
   * Convert the image to a JSON representation
   */
  async toJson() {
    return {
      type: "media",
      //mediaType: this.mediaType,//`image/${this.mediaType}`,
      format: await this.getFormat(),
      storage: "base64",
      base64: await this.toBase64()
      //this.content
    };
  }
  async toBase64() {
    const base64data = (await this.img.clone().toBuffer()).toString("base64");
    return base64data;
  }
  async toBaml() {
    const format = await this.getFormat();
    const data = await this.toBase64();
    return baml.Image.fromBase64(`image/${format}`, data);
  }
  async saveToFile(filepath) {
    await this.img.clone().toFile(filepath);
  }
  async getDimensions() {
    const { info: { width, height } } = await this.img.clone().toBuffer({ resolveWithObject: true });
    if (!width || !height) throw new Error("Unable to get dimensions from image");
    return { width, height };
  }
  async resize(width, height) {
    const resizedImage = new Image(await this.img.clone().resize({
      // Round width/height since sometimes they are floats due to rounding errors - sharp will throw if not integers
      width: Math.round(width),
      height: Math.round(height),
      fit: "fill",
      // exact size, no cropping
      kernel: sharp.kernel.lanczos3
    }));
    return resizedImage;
  }
}

async function observableDataToJson(data) {
  if (data instanceof Image) {
    return await data.toJson();
  }
  if (typeof data === "string" || typeof data === "number" || typeof data === "boolean") {
    return { type: "primitive", content: data };
  }
  if (data === void 0) {
    return void 0;
  }
  if (data === null) {
    return null;
  }
  if (Array.isArray(data)) {
    return Promise.all(
      data.filter((item) => item !== void 0).map((item) => observableDataToJson(item))
    );
  }
  if (typeof data === "object") {
    const processedObject = {};
    for (const key in data) {
      if (Object.prototype.hasOwnProperty.call(data, key)) {
        const value = data[key];
        const processedValue = observableDataToJson(value);
        if (processedValue !== void 0) {
          processedObject[key] = processedValue;
        }
      }
    }
    return processedObject;
  }
  return void 0;
}
async function jsonToObservableData(data) {
  if (data === null || data === void 0) {
    return data;
  }
  if (Array.isArray(data)) {
    return Promise.all(data.map((item) => jsonToObservableData(item)));
  }
  if (typeof data === "object") {
    if ("type" in data && typeof data.type === "string") {
      switch (data.type) {
        case "media":
          const media = data;
          if (media.storage === "base64") {
            return Image.fromBase64(media.base64);
          }
          throw new Error(`Unsupported media storage type: ${media.storage}`);
        case "primitive":
          return data.content;
      }
    }
    const result = {};
    const keys = Object.keys(data);
    const values = await Promise.all(
      keys.map((key) => jsonToObservableData(data[key]))
    );
    keys.forEach((key, index) => {
      result[key] = values[index];
    });
    return result;
  }
  throw new Error(`Invalid MultiMediaJson format: Unexpected primitive value '${data}'.`);
}

async function buildXmlPartsRecursive(data, indentLevel, partsList, isInsideList = false) {
  const indent = "  ".repeat(indentLevel);
  if (data instanceof Image) {
    const bamlImg = await data.toBaml();
    if (bamlImg) {
      partsList.push(bamlImg);
    }
    return;
  }
  if (data instanceof baml.Image) {
    partsList.push(data);
    return;
  }
  if (typeof data === "string" || typeof data === "number" || typeof data === "boolean" || data === null) {
    partsList.push(String(data));
    return;
  }
  if (data === void 0) {
    return;
  }
  if (Array.isArray(data)) {
    for (let index = 0; index < data.length; index++) {
      await buildXmlPartsRecursive(data[index], indentLevel, partsList, true);
      if (index < data.length - 1) {
        partsList.push("\n");
      }
    }
    return;
  }
  if (typeof data === "object" && data !== null) {
    const objectEntries = Object.entries(data).filter(([, val]) => val !== void 0);
    objectEntries.forEach(async ([key, value], entryIndex) => {
      const tagName = key;
      const currentValueParts = [];
      await buildXmlPartsRecursive(value, indentLevel + 1, currentValueParts, false);
      const mergedValueParts = [];
      let currentStr = "";
      for (const part of currentValueParts) {
        if (typeof part === "string") {
          currentStr += part;
        } else {
          if (currentStr) mergedValueParts.push(currentStr);
          currentStr = "";
          mergedValueParts.push(part);
        }
      }
      if (currentStr) mergedValueParts.push(currentStr);
      if (mergedValueParts.length === 1 && mergedValueParts[0] instanceof baml.Image) {
        partsList.push(`${indent}<${tagName}>`);
        partsList.push(mergedValueParts[0]);
        partsList.push(`</${tagName}>`);
      } else if (mergedValueParts.length === 1 && typeof mergedValueParts[0] === "string") {
        const contentStr = mergedValueParts[0];
        if (contentStr.includes("\n")) {
          partsList.push(`${indent}<${tagName}>
${contentStr}
${indent}</${tagName}>`);
        } else {
          partsList.push(`${indent}<${tagName}>${contentStr}</${tagName}>`);
        }
      } else {
        partsList.push(`${indent}<${tagName}>
`);
        mergedValueParts.forEach((part, partIdx) => {
          partsList.push(part);
          if (partIdx < mergedValueParts.length - 1) {
            partsList.push("\n");
          }
        });
        partsList.push(`
${indent}</${tagName}>`);
      }
      if (entryIndex < objectEntries.length - 1) {
        partsList.push("\n");
      }
    });
    return;
  }
  throw new Error(`Object type not supported for LLM context: ${typeof data}`);
}
async function renderXmlParts(data) {
  const rawList = [];
  await buildXmlPartsRecursive(data, 0, rawList);
  if (rawList.length === 0) {
    return [];
  }
  const mergedList = [];
  let currentString = "";
  for (const item of rawList) {
    if (typeof item === "string") {
      currentString += item;
    } else {
      if (currentString.length > 0) {
        mergedList.push(currentString);
      }
      currentString = "";
      mergedList.push(item);
    }
  }
  if (currentString.length > 0) {
    mergedList.push(currentString);
  }
  return mergedList;
}

async function buildJsonPartsRecursive(data, partsList, indent, currentLevel = 0) {
  if (data instanceof Image) {
    const bamlImg = await data.toBaml();
    if (bamlImg) {
      partsList.push(bamlImg);
    }
    return;
  }
  if (data instanceof baml.Image) {
    partsList.push(data);
    return;
  }
  if (data === null) {
    partsList.push("null");
    return;
  }
  if (data === void 0) {
    return;
  }
  if (typeof data === "string") {
    partsList.push(JSON.stringify(data));
    return;
  }
  if (typeof data === "number" || typeof data === "boolean") {
    partsList.push(String(data));
    return;
  }
  const newline = indent > 0 ? "\n" : "";
  const spacing = indent > 0 ? " ".repeat(currentLevel * indent) : "";
  const nextSpacing = indent > 0 ? " ".repeat((currentLevel + 1) * indent) : "";
  if (Array.isArray(data)) {
    partsList.push("[");
    for (let index = 0; index < data.length; index++) {
      const item = data[index];
      if (item !== void 0) {
        if (index > 0) {
          partsList.push(",");
        }
        if (indent > 0) {
          partsList.push(newline + nextSpacing);
        } else if (index > 0) {
          partsList.push(" ");
        }
        await buildJsonPartsRecursive(item, partsList, indent, currentLevel + 1);
      }
    }
    if (data.length > 0 && indent > 0) {
      partsList.push(newline + spacing);
    }
    partsList.push("]");
    return;
  }
  if (typeof data === "object" && data !== null) {
    partsList.push("{");
    const entries = Object.entries(data).filter(([, val]) => val !== void 0);
    for (let index = 0; index < entries.length; index++) {
      const [key, value] = entries[index];
      if (index > 0) {
        partsList.push(",");
      }
      if (indent > 0) {
        partsList.push(newline + nextSpacing);
      } else if (index > 0) {
        partsList.push(" ");
      }
      partsList.push(JSON.stringify(key));
      partsList.push(": ");
      await buildJsonPartsRecursive(value, partsList, indent, currentLevel + 1);
    }
    if (entries.length > 0 && indent > 0) {
      partsList.push(newline + spacing);
    }
    partsList.push("}");
    return;
  }
  throw new Error(`Object type not supported for JSON rendering: ${typeof data}`);
}
async function renderJsonParts(data, indent) {
  const rawList = [];
  await buildJsonPartsRecursive(data, rawList, indent);
  if (rawList.length === 0) {
    return [];
  }
  const mergedList = [];
  let currentString = "";
  for (const item of rawList) {
    if (typeof item === "string") {
      currentString += item;
    } else {
      if (currentString.length > 0) {
        mergedList.push(currentString);
      }
      currentString = "";
      mergedList.push(item);
    }
  }
  if (currentString.length > 0) {
    mergedList.push(currentString);
  }
  return mergedList;
}

async function renderContentParts(data, options) {
  if (options.mode === "json") {
    return await renderJsonParts(data, options.indent);
  } else if (options.mode === "xml") {
    return await renderXmlParts(data);
  } else {
    throw new Error(`Invalid render mode ${options.mode}`);
  }
}

class Observation {
  source;
  // where this observation came from
  role;
  timestamp;
  // time this observation was made
  content;
  // the arbitrarily-structured multimedia content of this observation
  retention;
  constructor(source, role, content, retention, timestamp) {
    this.source = source;
    this.role = role;
    this.content = content;
    this.timestamp = timestamp ?? Date.now();
    this.retention = retention;
  }
  static fromConnector(connectorId, content, options) {
    return new Observation(`connector:${connectorId}`, "user", content, options);
  }
  static fromActionTaken(actionId, content, options) {
    return new Observation(`action:taken:${actionId}`, "user", content, options);
  }
  static fromActionResult(actionId, content, options) {
    return new Observation(`action:result:${actionId}`, "user", content, options);
  }
  static fromThought(content, options) {
    return new Observation(`thought`, "user", content, options);
  }
  toString() {
    return JSON.stringify(this.content);
  }
  async toJson() {
    return await observableDataToJson(this.content);
  }
  async render(options) {
    return {
      role: this.role,
      cacheControl: options?.cacheControl ?? false,
      content: [...options?.prefix ?? [], ...await renderContentParts(this.content, { mode: "json", indent: 2 }), ...options?.postfix ?? []]
    };
  }
  // async renderContentParts(): Promise<MultiMediaContentPart[]> {
  //     return await renderParts(this.content);
  // }
  // async toContext(): Promise<BamlRenderable[]> {
  //     return await observableDataToContext(this.data);
  // }
  async hash() {
    const stringifiedContent = JSON.stringify(await this.toJson());
    return fnv1a32Hex(stringifiedContent);
  }
  async equals(obs) {
    return await this.hash() == await obs.hash();
  }
}

class AgentError extends Error {
  //public readonly failure: FailureDescriptor;
  options;
  constructor(message, options = {}) {
    super(message);
    this.options = {
      variant: options.variant ?? "unknown",
      adaptable: options.adaptable ?? false
    };
  }
}

async function maskObservations(observations, freezeMask) {
  const frozenCount = freezeMask?.length ?? 0;
  const mask = new Array(observations.length).fill(true);
  if (freezeMask) {
    for (let i = 0; i < frozenCount && i < observations.length; i++) {
      mask[i] = freezeMask[i];
    }
  }
  const observationsByType = /* @__PURE__ */ new Map();
  observations.forEach((obs, index) => {
    if (obs.retention && obs.retention.type) {
      const type = obs.retention.type;
      if (!observationsByType.has(type)) {
        observationsByType.set(type, {
          frozenIndices: [],
          unfrozenIndices: [],
          frozenObs: [],
          unfrozenObs: [],
          limit: obs.retention.limit,
          dedupe: obs.retention.dedupe
        });
      }
      const typeData = observationsByType.get(type);
      if (index < frozenCount) {
        typeData.frozenIndices.push(index);
        typeData.frozenObs.push(obs);
      } else {
        typeData.unfrozenIndices.push(index);
        typeData.unfrozenObs.push(obs);
      }
    }
  });
  for (const [type, data] of observationsByType.entries()) {
    let visibleUnfrozenIndices = [...data.unfrozenIndices];
    if (data.dedupe && data.unfrozenObs.length > 1) {
      const dedupedIndices = [];
      dedupedIndices.unshift(data.unfrozenIndices[data.unfrozenIndices.length - 1]);
      for (let i = data.unfrozenIndices.length - 2; i >= 0; i--) {
        const currentObs = data.unfrozenObs[i];
        const lastKeptIdx = dedupedIndices[0];
        const lastKeptObs = observations[lastKeptIdx];
        if (!await currentObs.equals(lastKeptObs)) {
          dedupedIndices.unshift(data.unfrozenIndices[i]);
        }
      }
      visibleUnfrozenIndices = dedupedIndices;
    }
    if (data.dedupe && freezeMask) {
      const frozenToPreserve = /* @__PURE__ */ new Set();
      for (let i = 0; i < data.frozenIndices.length; i++) {
        const frozenIdx = data.frozenIndices[i];
        if (mask[frozenIdx]) {
          const frozenObs = data.frozenObs[i];
          for (const unfrozenObs of data.unfrozenObs) {
            if (await frozenObs.equals(unfrozenObs)) {
              frozenToPreserve.add(frozenIdx);
              break;
            }
          }
        }
      }
      frozenToPreserve.forEach((idx) => {
        mask[idx] = true;
      });
    }
    if (data.limit !== void 0 && data.limit >= 0) {
      if (data.limit === 0) {
        visibleUnfrozenIndices = [];
      } else {
        visibleUnfrozenIndices = visibleUnfrozenIndices.slice(-data.limit);
      }
    }
    const visibleSet = new Set(visibleUnfrozenIndices);
    data.unfrozenIndices.forEach((idx) => {
      if (!visibleSet.has(idx)) {
        mask[idx] = false;
      }
    });
  }
  return mask;
}
function applyMask(observations, mask) {
  if (observations.length !== mask.length) {
    throw new Error(`Mask length (${mask.length}) must match observations length (${observations.length})`);
  }
  const result = [];
  observations.forEach((observation, index) => {
    if (mask[index]) {
      result.push({ observation, index });
    }
  });
  return result;
}

const CACHE_CONTROL_LIMIT = 3;
class AgentMemory {
  //public readonly events: EventEmitter<AgentMemoryEvents> = new EventEmitter();
  options;
  // Custom instructions relating to this memory instance (e.g. agent-level and/or task-level instructions)
  //public readonly instructions: string | null;
  observations = [];
  //private freezeState?: FreezeState;
  freezeMask;
  cacheControlIndices = [];
  constructor(options) {
    this.options = {
      instructions: options?.instructions ?? null,
      promptCaching: options?.promptCaching ?? false,
      //optimizeForPromptCaching: false,
      thoughtLimit: options?.thoughtLimit ?? 20
    };
  }
  get instructions() {
    return this.options.instructions;
  }
  async render() {
    if (this.options.promptCaching && this.cacheControlIndices.length >= CACHE_CONTROL_LIMIT) {
      this.freezeMask = void 0;
      this.cacheControlIndices = [];
    }
    const mask = await maskObservations(this.observations, this.freezeMask);
    const visibleObservations = applyMask(this.observations, mask);
    const lastVisible = visibleObservations.at(-1);
    if (lastVisible) this.cacheControlIndices.push(lastVisible.index);
    let messages = [];
    for (const { observation, index } of visibleObservations) {
      const message = await observation.render({
        prefix: observation.source.startsWith("action:taken") || observation.source.startsWith("thought") ? [`[${new Date(observation.timestamp).toTimeString().split(" ")[0]}]: `] : [],
        cacheControl: this.options.promptCaching && this.cacheControlIndices.includes(index)
      });
      messages.push(message);
    }
    if (this.options.promptCaching) {
      this.freezeMask = mask;
    }
    return messages;
  }
  isEmpty() {
    return this.observations.length === 0;
  }
  recordThought(content) {
    this.observations.push(
      Observation.fromThought(content, { type: "thought", limit: this.options.thoughtLimit })
    );
  }
  recordObservation(obs) {
    this.observations.push(obs);
  }
  getLastThoughtMessage() {
    for (let i = this.observations.length - 1; i >= 0; i--) {
      const obs = this.observations[i];
      if (obs.source.startsWith("thought")) return obs.toString();
    }
    return null;
  }
  async toJSON() {
    const observations = [];
    for (const observation of this.observations) {
      observations.push({
        source: observation.source,
        role: observation.role,
        timestamp: observation.timestamp,
        data: await observableDataToJson(observation.content),
        options: observation.retention
      });
    }
    return {
      // TODO: include other options as well
      ...this.options.instructions ? { instructions: this.options.instructions } : {},
      observations
    };
  }
  // TODO: turn into class static method / rework cons
  async loadJSON(data) {
    const observations = [];
    for (const observation of data.observations) {
      observations.push(new Observation(
        observation.source,
        observation.role,
        await jsonToObservableData(observation.data),
        observation.options,
        observation.timestamp
      ));
    }
    this.observations = observations;
  }
}

function createAction(action) {
  return {
    name: action.name,
    description: action.description,
    schema: action.schema ?? z.z.object({}),
    resolver: action.resolver,
    render: action.render ?? ((action2) => JSON.stringify(action2))
  };
}

const doneAction = createAction({
  name: "task:done",
  description: "ONLY once you have seen sufficient evidence of the task's completion, mark it as done",
  //Use once sure that task is finished',//'Designate current task as finished',// Do not use until you can verify the task is completed.
  schema: z.z.object({
    evidence: z.z.string().describe(`Specific observed evidence that verifies the task's completion. Do NOT predict this evidence.`)
  }),
  resolver: async ({ agent }) => {
    agent.queueDone();
  },
  render: () => `\u2713 done`
});
const failAction = createAction({
  name: "task:fail",
  description: "Use if task was attempted but does not seem possible. Use common sense",
  //'Designate current task as infeasible',
  schema: z.z.object({}),
  resolver: async ({ agent }) => {
    throw new AgentError(`Task failed: ${agent.memory.getLastThoughtMessage() ?? "No thought recorded"}`);
  },
  render: () => `\u2715 fail`
});
const taskActions = [
  doneAction,
  failAction
];

class AsyncHttpRequest {
  constructor(runtime, ctxManager) {
    this.runtime = runtime;
    this.ctxManager = ctxManager;
  }
  async CreatePartialRecipe(memory, task, taskData, includeClaudeSpoof, __baml_options__) {
    try {
      const rawEnv = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
      const env = Object.fromEntries(
        Object.entries(rawEnv).filter(([_, value]) => value !== void 0)
      );
      return await this.runtime.buildRequest(
        "CreatePartialRecipe",
        {
          "memory": memory,
          "task": task,
          "taskData": taskData,
          "includeClaudeSpoof": includeClaudeSpoof
        },
        this.ctxManager.cloneContext(),
        __baml_options__?.tb?.__tb(),
        __baml_options__?.clientRegistry,
        false,
        env
      );
    } catch (error) {
      throw baml.toBamlError(error);
    }
  }
  async ExtractData(instructions, screenshot, domContent, includeClaudeSpoof, __baml_options__) {
    try {
      const rawEnv = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
      const env = Object.fromEntries(
        Object.entries(rawEnv).filter(([_, value]) => value !== void 0)
      );
      return await this.runtime.buildRequest(
        "ExtractData",
        {
          "instructions": instructions,
          "screenshot": screenshot,
          "domContent": domContent,
          "includeClaudeSpoof": includeClaudeSpoof
        },
        this.ctxManager.cloneContext(),
        __baml_options__?.tb?.__tb(),
        __baml_options__?.clientRegistry,
        false,
        env
      );
    } catch (error) {
      throw baml.toBamlError(error);
    }
  }
  async QueryMemory(memory, query, includeClaudeSpoof, __baml_options__) {
    try {
      const rawEnv = __baml_options__?.env ? { ...process.env, ...__baml_options__.env } : { ...process.env };
      const env = Object.fromEntries(
        Object.entries(rawEnv).filter(([_, value]) => value !== void 0)
      );
      return await this.runtime.buildRequest(
        "QueryMemory",
        {
          "memory": memory,
          "query": query,
          "includeClaudeSpoof": includeClaudeSpoof
        },
        this.ctxManager.cloneContext(),
        __baml_options__?.tb?.__tb(),
        __baml_options__?.clientRegistry,
        false,
        env
      );
    } catch (error) {
      throw baml.toBamlError(error);
    }
  }
}
class AsyncHttpStreamRequest {
  constructor(runtime, ctxManager) {
    this.runtime = runtime;
    this.ctxManager = ctxManager;
  }
  async CreatePartialRecipe(memory, task, taskData, includeClaudeSpoof, __baml_options__) {
    try {
      const rawEn