UNPKG

magnitude-core

Version:
826 lines (789 loc) 26.5 kB
import EventEmitter from 'eventemitter3'; import z$1, { Schema, ZodTypeAny, z, ZodSchema } from 'zod'; import { Image as Image$1 } from '@boundaryml/baml'; import sharp, { Sharp } from 'sharp'; import { BrowserContext, Page, PageScreenshotOptions, Browser, BrowserContextOptions, LaunchOptions } from 'playwright'; import { PostHog } from 'posthog-node'; import pino from 'pino'; type Base64Image = `data:image/${'jpeg' | 'png' | 'gif'};base64,${string}`; type WebAction = NavigateWebAction | ClickWebAction | TypeWebAction | ScrollWebAction | SwitchTabWebAction; interface NavigateWebAction { variant: 'load'; url: string; } interface ClickWebAction { variant: 'click'; x: number; y: number; } interface TypeWebAction { variant: 'type'; x: number; y: number; content: string; } interface ScrollWebAction { variant: 'scroll'; x: number; y: number; deltaX: number; deltaY: number; } type SwitchTabWebAction = { variant: 'tab'; index: number; }; interface PixelCoordinate { x: number; y: number; } /** * "Intents" are natural language descriptors that align 1:1 with executable web actions. * Micro converts intents into web actions. */ interface Action { variant: string; [key: string]: any; } type ActionIntent = ClickIntent | TypeIntent | ScrollIntent | SwitchTabIntent; type Intent = ActionIntent | CheckIntent; interface ClickIntent { variant: 'click'; target: string; } interface TypeIntent { variant: 'type'; target: string; content: string; } interface ScrollIntent { variant: 'scroll'; target: string; deltaX: number; deltaY: number; } type SwitchTabIntent = SwitchTabWebAction; interface CheckIntent { variant: 'check'; checks: string[]; } type BrowserAgentRole = 'act' | 'extract' | 'query'; declare const allBrowserAgentRoles: BrowserAgentRole[]; type LLMClient = (AnthropicClient | ClaudeCodeClient | BedrockClient | GoogleAIClient | GoogleVertexClient | OpenAIClient | OpenAIGenericClient | AzureOpenAIClient) & { roles?: BrowserAgentRole[]; }; interface AnthropicClient { provider: 'anthropic'; options: { model: string; apiKey?: string; temperature?: number; promptCaching?: boolean; }; } interface ClaudeCodeClient { provider: 'claude-code'; options: { model: string; temperature?: number; promptCaching?: boolean; }; } interface BedrockClient { provider: 'aws-bedrock'; options: { model: string; temperature?: number; }; } interface GoogleAIClient { provider: 'google-ai'; options: { model: string; apiKey?: string; temperature?: number; baseUrl?: string; }; } interface GoogleVertexClient { provider: 'vertex-ai'; options: { model: string; location?: string; baseUrl?: string; projectId?: string; credentials?: string | object; anthropicVersion?: string; temperature?: number; }; } interface OpenAIClient { provider: 'openai'; options: { model: string; apiKey?: string; temperature?: number; }; } interface AzureOpenAIClient { provider: 'azure-openai'; options: { resourceName: string; deploymentId: string; apiVersion: string; apiKey: string; }; } interface OpenAIGenericClient { provider: 'openai-generic'; options: { model: string; baseUrl: string; apiKey?: string; temperature?: number; headers?: Record<string, string>; }; } interface LLMClientIdentifier { provider: string; model: string; } interface ModelUsage { llm: LLMClientIdentifier; inputTokens: number; outputTokens: number; cacheWriteInputTokens?: number; cacheReadInputTokens?: number; inputCost?: number; outputCost?: number; } interface AgentEvents { 'start': () => void; 'stop': () => void; 'thought': (thought: string) => void; 'actStarted': (task: string, options: ActOptions) => void; 'actDone': (task: string, options: ActOptions) => void; 'actionStarted': (action: Action) => void; 'actionDone': (action: Action) => void; 'pause': () => void; 'resume': () => void; 'tokensUsed': (usage: ModelUsage) => void; } type StoredMedia = { type: 'media'; format: string; storage: 'base64'; base64: string; }; interface StoredPrimitive { type: 'primitive'; content: string | boolean | number; } type MultiMediaPrimitive = StoredMedia | StoredPrimitive | undefined | null; type MultiMediaArray = Array<MultiMediaJson>; type MultiMediaObject = { [key: string]: MultiMediaJson; }; type MultiMediaJson = MultiMediaPrimitive | MultiMediaArray | MultiMediaObject; declare class Image { /** * Wrapper for a Sharp image with conveniences to go to/from base64, convert to BAML, or serialize as JSON */ private img; constructor(img: Sharp); static fromBase64(base64: string): Image; getFormat(): Promise<keyof sharp.FormatEnum>; /** * Convert the image to a JSON representation */ toJson(): Promise<StoredMedia>; toBase64(): Promise<string>; toBaml(): Promise<Image$1>; saveToFile(filepath: string): Promise<void>; getDimensions(): Promise<{ width: number; height: number; }>; resize(width: number, height: number): Promise<Image>; } /************************************************************************************************* Welcome to Baml! To use this generated code, please run one of the following: $ npm install @boundaryml/baml $ yarn add @boundaryml/baml $ pnpm add @boundaryml/baml *************************************************************************************************/ interface AgentContext { instructions?: string | null; observationContent: MultiMediaMessage[]; connectorInstructions: ConnectorInstructions[]; } interface ConnectorInstructions { connectorId: string; instructions: string; } interface MultiMediaMessage { role: "user" | "assistant"; cacheControl: boolean; content: (Image$1 | string)[]; } type MultiMediaContentPart = Image$1 | string; type ObservableDataPrimitive = Image | string | number | boolean | null | undefined; type ObservableDataArray = Array<RenderableContent>; type ObservableDataObject = { [key: string]: RenderableContent; }; type RenderableContent = ObservableDataPrimitive | ObservableDataArray | ObservableDataObject; type ObservationRole = 'user' | 'assistant'; interface ObservationRetentionOptions { type: string; limit?: number; dedupe?: boolean; } type ObservationSource = `connector:${string}` | `action:taken:${string}` | `action:result:${string}` | `thought`; declare class Observation { readonly source: ObservationSource; readonly role: ObservationRole; readonly timestamp: number; readonly content: RenderableContent; readonly retention?: ObservationRetentionOptions; constructor(source: ObservationSource, role: ObservationRole, content: RenderableContent, retention?: ObservationRetentionOptions, timestamp?: number); static fromConnector(connectorId: string, content: RenderableContent, options?: ObservationRetentionOptions): Observation; static fromActionTaken(actionId: string, content: RenderableContent, options?: ObservationRetentionOptions): Observation; static fromActionResult(actionId: string, content: RenderableContent, options?: ObservationRetentionOptions): Observation; static fromThought(content: RenderableContent, options?: ObservationRetentionOptions): Observation; toString(): string; toJson(): Promise<MultiMediaJson>; render(options?: { prefix?: MultiMediaContentPart[]; postfix?: MultiMediaContentPart[]; cacheControl?: boolean; }): Promise<MultiMediaMessage>; hash(): Promise<string>; equals(obs: Observation): Promise<boolean>; } interface ActionDefinition<T> { name: string; description?: string; schema: Schema<T>; resolver: ({ input, agent }: { input: T; agent: Agent; }) => Promise<void | RenderableContent>; render: (action: T) => string; } declare function createAction<S extends ZodTypeAny>(action: { name: string; description?: string; schema?: S; resolver: ({ input, agent }: { input: z.infer<S>; agent: Agent; }) => Promise<void | RenderableContent>; render?: (action: z.infer<S>) => string; }): ActionDefinition<z.infer<S>>; type ActionPayload<A extends ActionDefinition<any>> = { name: A['name']; } & z.infer<A['schema']>; interface AgentConnector { id: string; onStart?(): Promise<void>; onStop?(): Promise<void>; getActionSpace?(): ActionDefinition<any>[]; collectObservations?(): Promise<Observation[]>; getInstructions?(): Promise<void | string>; } interface SerializedAgentMemory { instructions?: string; observations: { source: ObservationSource; role: ObservationRole; timestamp: number; data: MultiMediaJson; options?: ObservationRetentionOptions; }[]; } interface AgentMemoryOptions { instructions?: string | null; promptCaching?: boolean; thoughtLimit?: number; } interface MemoryRenderOptions { } declare class AgentMemory { private options; private observations; private freezeMask?; private cacheControlIndices; constructor(options?: AgentMemoryOptions); get instructions(): string | null; render(options?: MemoryRenderOptions): Promise<MultiMediaMessage[]>; simpleRender(): Promise<(Image$1 | string)[]>; isEmpty(): boolean; recordThought(content: string): void; recordObservation(obs: Observation): void; getLastThoughtMessage(): string | null; toJSON(): Promise<SerializedAgentMemory>; loadJSON(data: SerializedAgentMemory): Promise<void>; } declare function mergeMessages(messages: MultiMediaMessage[]): MultiMediaMessage[]; interface ModelHarnessEvents { 'tokensUsed': (usage: ModelUsage) => {}; } declare class MultiModelHarness { /** * Delegates model responsibilites to different LLMs and consolidates their usage */ private roles; private uniqueModels; readonly events: EventEmitter<ModelHarnessEvents>; constructor(clients: LLMClient[]); setup(): Promise<void>; describe(): string; partialAct<T>(context: AgentContext, task: string, data: MultiMediaContentPart[], actionVocabulary: ActionDefinition<T>[]): Promise<{ reasoning: string; actions: Action[]; }>; extract<T extends z$1.Schema>(instructions: string, schema: T, screenshot: Image, domContent: string): Promise<z$1.infer<T>>; query<T extends z$1.Schema>(context: AgentContext, query: string, schema: T): Promise<z$1.infer<T>>; get numUniqueModels(): number; } interface AgentOptions { llm?: LLMClient | LLMClient[]; connectors?: AgentConnector[]; actions?: ActionDefinition<any>[]; prompt?: string | null; telemetry?: boolean; } interface ActOptions { prompt?: string; data?: RenderableContent; memory?: AgentMemory; } declare class Agent { private options; private connectors; private actions; private memoryOptions; readonly models: MultiModelHarness; readonly events: EventEmitter<AgentEvents>; private doneActing; private _paused; private _pauseResolve; protected latestTaskMemory: AgentMemory; constructor(baseConfig?: Partial<AgentOptions>); getConnector<C extends AgentConnector>(connectorClass: new (...args: any[]) => C): C | undefined; require<C extends AgentConnector>(connectorClass: new (...args: any[]) => C): C; start(): Promise<void>; identifyAction(action: Action): ActionDefinition<any>; exec(action: Action, memory?: AgentMemory): Promise<void>; protected _recordConnectorObservations(memory: AgentMemory): Promise<void>; get memory(): AgentMemory; act(taskOrSteps: string | string[], options?: ActOptions): Promise<void>; _traceAct(task: string, memory: AgentMemory, options?: ActOptions): Promise<void>; private _buildContext; _act(description: string, memory: AgentMemory, options?: ActOptions): Promise<void>; query<T extends z$1.Schema>(query: string, schema: T): Promise<z$1.infer<T>>; queueDone(): Promise<void>; private _waitIfPaused; pause(): void; resume(): void; get paused(): boolean; stop(): Promise<void>; } interface ActionVisualizerOptions { showCursor?: boolean; showHoverCircle?: boolean; showClickRipple?: boolean; showDragLine?: boolean; showTypeEffects?: boolean; } declare class ActionVisualizer { private options; private context; private page; private cursor; private mouseEffects; private typeEffects; constructor(context: BrowserContext, options: ActionVisualizerOptions); setup(): Promise<void>; setActivePage(page: Page): Promise<void>; moveVirtualCursor(x: number, y: number): Promise<void>; hideAll(): Promise<void>; showAll(): Promise<void>; } interface TabState { activeTab: number; tabs: { title: string; url: string; }[]; } interface WebHarnessOptions { virtualScreenDimensions?: { width: number; height: number; }; visuals?: ActionVisualizerOptions; switchTabsOnActivity?: boolean; } interface WebHarnessEvents { 'activePageChanged': (page: Page) => Promise<void>; } declare class WebHarness { /** * Executes web actions on a page * Not responsible for browser lifecycle */ readonly context: BrowserContext; private options; private stability; readonly visualizer: ActionVisualizer; private transformer; private tabs; readonly events: EventEmitter<WebHarnessEvents>; constructor(context: BrowserContext, options?: WebHarnessOptions); setActivePage(page: Page): Promise<void>; retrieveTabState(): Promise<TabState>; start(): Promise<void>; stop(): Promise<void>; get page(): Page; screenshot(options?: PageScreenshotOptions): Promise<Image>; _type(content: string): Promise<void>; transformCoordinates({ x, y }: { x: number; y: number; }): Promise<{ x: number; y: number; }>; click({ x, y }: { x: number; y: number; }, options?: { transform: boolean; }): Promise<void>; private _click; rightClick({ x, y }: { x: number; y: number; }, options?: { transform: boolean; }): Promise<void>; doubleClick({ x, y }: { x: number; y: number; }, options?: { transform: boolean; }): Promise<void>; drag({ x1, y1, x2, y2 }: { x1: number; y1: number; x2: number; y2: number; }, options?: { transform: boolean; }): Promise<void>; type({ content }: { content: string; }): Promise<void>; clickAndType({ x, y, content }: { x: number; y: number; content: string; }, options?: { transform: boolean; }): Promise<void>; scroll({ x, y, deltaX, deltaY }: { x: number; y: number; deltaX: number; deltaY: number; }, options?: { transform: boolean; }): Promise<void>; switchTab({ index }: { index: number; }): Promise<void>; newTab(): Promise<void>; navigate(url: string): Promise<void>; selectAll(): Promise<void>; enter(): Promise<void>; backspace(): Promise<void>; tab(): Promise<void>; goBack(): Promise<void>; executeAction(action: WebAction): Promise<void>; waitForStability(timeout?: number): Promise<void>; } type BrowserOptions = { instance: Browser; contextOptions?: BrowserContextOptions; } | { cdp: string; contextOptions?: BrowserContextOptions; } | { launchOptions?: LaunchOptions; contextOptions?: BrowserContextOptions; } | { context: BrowserContext; }; declare class BrowserProvider { private activeBrowsers; private logger; private constructor(); static getInstance(): BrowserProvider; private _launchOrReuseBrowser; _createAndTrackContext(options: BrowserOptions): Promise<BrowserContext>; newContext(options?: BrowserOptions): Promise<BrowserContext>; private _applyEmulationSettings; } interface BrowserConnectorOptions { browser?: BrowserOptions; url?: string; virtualScreenDimensions?: { width: number; height: number; }; minScreenshots?: number; visuals?: ActionVisualizerOptions; } interface BrowserConnectorStateData { screenshot: Image; tabs: TabState; } declare class BrowserConnector implements AgentConnector { readonly id: string; private harness; private options; private browser?; private context; private logger; constructor(options?: BrowserConnectorOptions); onStart(): Promise<void>; onStop(): Promise<void>; getActionSpace(): ActionDefinition<any>[]; getHarness(): WebHarness; private captureCurrentState; transformScreenshot(screenshot: Image): Promise<Image>; getLastScreenshot(): Promise<Image>; collectObservations(): Promise<Observation[]>; getInstructions(): Promise<void | string>; } declare function startBrowserAgent(options?: AgentOptions & BrowserConnectorOptions & { narrate?: boolean; }): Promise<BrowserAgent>; type ExtractedOutput = string | number | boolean | bigint | Date | null | undefined | { [key: string]: ExtractedOutput; } | ExtractedOutput[]; interface BrowserAgentEvents { 'nav': (url: string) => void; 'extractStarted': (instructions: string, schema: ZodSchema) => void; 'extractDone': (instructions: string, data: ExtractedOutput) => void; } declare class BrowserAgent extends Agent { readonly browserAgentEvents: EventEmitter<BrowserAgentEvents>; constructor({ agentOptions, browserOptions }: { agentOptions?: Partial<AgentOptions>; browserOptions?: BrowserConnectorOptions; }); get page(): Page; get context(): BrowserContext; nav(url: string): Promise<void>; extract<T extends Schema>(instructions: string, schema: T): Promise<z$1.infer<T>>; } /** * Generic desktop automation interface. * Implementations can use any desktop automation technology * (Lume, PyAutoGUI, Windows UI Automation, etc.) */ interface DesktopInterface { click(x: number, y: number): Promise<void>; rightClick(x: number, y: number): Promise<void>; doubleClick(x: number, y: number): Promise<void>; moveCursor(x: number, y: number): Promise<void>; drag(fromX: number, fromY: number, toX: number, toY: number): Promise<void>; scroll(x: number, y: number, deltaX: number, deltaY: number): Promise<void>; type(text: string): Promise<void>; key(key: string): Promise<void>; hotkey(keys: string[]): Promise<void>; screenshot(): Promise<Buffer>; getScreenSize(): Promise<{ width: number; height: number; }>; navigate?(url: string): Promise<void>; getActiveWindow?(): Promise<{ title: string; app: string; }>; getOpenWindows?(): Promise<Array<{ title: string; app: string; isActive: boolean; }>>; focusWindow?(title: string): Promise<void>; openApplication?(name: string): Promise<void>; closeApplication?(name: string): Promise<void>; } interface DesktopConnectorOptions { desktopInterface: DesktopInterface; virtualScreenDimensions?: { width: number; height: number; }; minScreenshots?: number; } declare class DesktopConnector implements AgentConnector { readonly id: string; private desktopInterface; private options; private logger; constructor(options: DesktopConnectorOptions); onStart(): Promise<void>; onStop(): Promise<void>; getActionSpace(): ActionDefinition<any>[]; collectObservations(): Promise<Observation[]>; private formatWindowInfo; getInterface(): DesktopInterface; } interface AgentErrorOptions { variant?: string; adaptable?: boolean; } declare class AgentError extends Error { readonly options: Required<AgentErrorOptions>; constructor(message: string, options?: AgentErrorOptions); } /** * Represents any reason why a test case could have failed, for example: * - Step could not be completed * - Check did not pass * - Could not navigate to starting URL * - Time or action based timeout * - Operation cancelled by signal * - ... */ type FailureDescriptor = BugDetectedFailure | MisalignmentFailure | NetworkFailure | BrowserFailure | RateLimitFailure | ApiKeyFailure | UnknownFailure | CancelledFailure; type BugSeverity = 'critical' | 'high' | 'medium' | 'low'; /** * Step and check failures are classified into one of: * BugDetectedFailure: seems to be something wrong with the application itself * MisalignmentFailure: seems to be a discrepency between the test case and the interface * OR the agent did not properly recognize the relationship between test case and interface */ interface BugDetectedFailure { variant: 'bug'; title: string; expectedResult: string; actualResult: string; severity: BugSeverity; } interface MisalignmentFailure { /** * Major misalignment: when a step/check fails due to: * 1. Poorly written step/check that is completely unrelated to the interface * 2. Or interface has changed so much that step/check no longer applicable * 3. Planner did not do good enough job adjusting recipe for minor misalignment * Misalignment could be due to a poorly written test case OR bad agent behavior. */ variant: 'misalignment'; message: string; } interface NetworkFailure { /** * For example, failure to connect to starting URL, or any other network errors * that would completely prevent the test from executing. */ variant: 'network'; message: string; } interface BrowserFailure { /** * E.g. something goes wrong with playwright interactions, any DOM manipulation, etc. */ variant: 'browser'; message: string; } interface RateLimitFailure { variant: 'rate_limit'; message: string; } interface ApiKeyFailure { variant: 'api_key'; message: string; } interface UnknownFailure { variant: 'unknown'; message: string; } interface CancelledFailure { /** * Operation was cancelled, typically by an AbortSignal from a controlling process (e.g., test runner pool). */ variant: 'cancelled'; } type RetryMode = { mode: 'retry_on_partial_message'; errorSubstrings: string[]; } | { mode: 'retry_all'; }; type RetryParams = { retryLimit: number; delayMs: number; showWarnOnRetry: boolean; }; type RetryOptions = RetryMode & RetryParams; declare function retryOnErrorIsSuccess<T>(fnToRetry: () => Promise<T>, retryOptions: RetryMode & Partial<RetryParams>): Promise<boolean>; declare function retryOnError<T>(fnToRetry: () => Promise<T>, retryOptions: RetryMode & Partial<RetryParams>): Promise<T>; /** * Performs a deep comparison between two values to determine if they are equivalent. * * @param a The first value to compare. * @param b The second value to compare. * @param cache A WeakMap to handle circular references. Should not be provided by the caller. * @returns `true` if the values are deeply equal, `false` otherwise. */ declare function deepEquals(a: any, b: any, cache?: WeakMap<object, WeakSet<object>>): boolean; interface TestDataEntry { key: string; value: string; sensitive: boolean; } interface TestData { data?: TestDataEntry[]; other?: string; } interface TestStepDefinition { description: string; checks: string[]; testData: TestData; } interface TestCaseDefinition { url: string; steps: TestStepDefinition[]; recipe?: Intent[]; } type TestCaseResult = SuccessfulTestCaseResult | FailedTestCaseResult; interface SuccessfulTestCaseResult { passed: true; recipe: Intent[]; } interface FailedTestCaseResult { passed: false; failure: FailureDescriptor; } declare const createId: () => string; declare const posthog: PostHog; declare function getMachineId(): string; declare function getCodebaseId(): string | undefined; declare function sendTelemetry(eventName: string, properties: Record<string, any>): Promise<void>; declare function buildDefaultBrowserAgentOptions({ agentOptions, browserOptions }: { agentOptions: AgentOptions; browserOptions: BrowserConnectorOptions; }): { agentOptions: AgentOptions; browserOptions: BrowserConnectorOptions; }; declare const logger: pino.Logger<never, boolean>; export { Agent, AgentError, AgentMemory, BrowserAgent, BrowserConnector, BrowserProvider, DesktopConnector, Observation, WebHarness, allBrowserAgentRoles, buildDefaultBrowserAgentOptions, createAction, createId, deepEquals, getCodebaseId, getMachineId, logger, mergeMessages, posthog, retryOnError, retryOnErrorIsSuccess, sendTelemetry, startBrowserAgent }; export type { ActOptions, Action, ActionDefinition, ActionIntent, ActionPayload, AgentConnector, AgentErrorOptions, AgentEvents, AgentMemoryOptions, AgentOptions, AnthropicClient, ApiKeyFailure, AzureOpenAIClient, Base64Image, BedrockClient, BrowserAgentRole, BrowserConnectorOptions, BrowserConnectorStateData, BrowserFailure, BrowserOptions, BugDetectedFailure, BugSeverity, CancelledFailure, CheckIntent, ClaudeCodeClient, ClickIntent, ClickWebAction, DesktopConnectorOptions, DesktopInterface, FailedTestCaseResult, FailureDescriptor, GoogleAIClient, GoogleVertexClient, Intent, LLMClient, LLMClientIdentifier, MemoryRenderOptions, MisalignmentFailure, ModelUsage, NavigateWebAction, NetworkFailure, ObservableDataArray, ObservableDataObject, ObservableDataPrimitive, ObservationRetentionOptions, ObservationRole, ObservationSource, OpenAIClient, OpenAIGenericClient, PixelCoordinate, RateLimitFailure, RenderableContent, RetryMode, RetryOptions, RetryParams, ScrollIntent, ScrollWebAction, SerializedAgentMemory, SuccessfulTestCaseResult, SwitchTabIntent, SwitchTabWebAction, TestCaseDefinition, TestCaseResult, TestData, TestDataEntry, TestStepDefinition, TypeIntent, TypeWebAction, UnknownFailure, WebAction, WebHarnessEvents, WebHarnessOptions };