@mastra/core/dist/evals/run/index.d.ts

import type { CoreMessage } from '../../_types/@internal_ai-sdk-v4/dist/index.d.ts';
import type { Agent, AgentExecutionOptions, AgentMemoryOption, AiMessageType, UIMessageWithMetadata } from '../../agent/index.js';
import type { ObservabilityContext } from '../../observability/index.js';
import type { RequestContext } from '../../request-context/index.js';
import type { WorkflowResult, WorkflowRunStartOptions } from '../../workflows/types.js';
import type { AnyWorkflow } from '../../workflows/workflow.js';
import { Workflow } from '../../workflows/workflow.js';
import type { MastraScorer } from '../base.js';
type WorkflowRunOptions = WorkflowRunStartOptions & {
    initialState?: any;
};
type AgentInputType = string | string[] | CoreMessage[] | AiMessageType[] | UIMessageWithMetadata[];
type RunEvalsDataItemBase = {
    groundTruth?: any;
    expectedTrajectory?: any;
    requestContext?: RequestContext;
    startOptions?: WorkflowRunOptions;
} & Partial<ObservabilityContext>;
/**
 * A single turn in a multi-turn conversation with optional per-turn assertions.
 * Per-turn gates/scorers evaluate ONLY that turn's input and output, so a broken
 * turn fails that turn instead of being averaged into a holistic score.
 */
export type EvalTurn = {
    /** The input sent to the agent for this turn. */
    input: AgentInputType;
    /** Gates that must score 1.0 for this turn. A failing turn gate fails the run. */
    gates?: MastraScorer<any, any, any, any>[];
    /** Scorers (optionally with thresholds) evaluated against this turn only. */
    scorers?: ScorerEntry[];
};
type RunEvalsDataItem<TTarget = unknown> = TTarget extends Agent ? (RunEvalsDataItemBase & {
    input: AgentInputType;
    inputs?: never;
    turns?: never;
}) | (RunEvalsDataItemBase & {
    input?: AgentInputType;
    /**
     * Multi-turn inputs. When provided, each entry is sent sequentially to the agent
     * on the same thread. Scorers see the accumulated output from all turns.
     * Only supported for Agent targets (not Workflows).
     */
    inputs: AgentInputType[];
    turns?: never;
}) | (RunEvalsDataItemBase & {
    input?: never;
    inputs?: never;
    /**
     * Multi-turn conversation with per-turn assertions. Each turn is sent sequentially
     * on the same thread; its `gates`/`scorers` evaluate only that turn's output.
     * Only supported for Agent targets (not Workflows).
     */
    turns: EvalTurn[];
}) : TTarget extends Workflow<any, any> ? RunEvalsDataItemBase & {
    input: any;
    inputs?: never;
    turns?: never;
} : RunEvalsDataItemBase & {
    input: unknown;
    inputs?: never;
    turns?: never;
};
export type WorkflowScorerConfig = {
    /** Scorers that evaluate the overall workflow input/output */
    workflow?: MastraScorer<any, any, any, any>[];
    /** Scorers that evaluate individual workflow steps by step ID */
    steps?: Record<string, MastraScorer<any, any, any, any>[]>;
    /** Scorers that evaluate the workflow's step execution trajectory */
    trajectory?: MastraScorer<any, any, any, any>[];
};
export type AgentScorerConfig = {
    /** Scorers that evaluate the full agent input/output */
    agent?: MastraScorer<any, any, any, any>[];
    /** Scorers that evaluate the agent's tool call trajectory */
    trajectory?: MastraScorer<any, any, any, any>[];
};
/** Threshold configuration: a number implies minimum, or an object with min/max bounds. */
export type ThresholdConfig = number | {
    min?: number;
    max?: number;
};
/** A scorer with an associated pass/fail threshold. */
export type ScorerWithThreshold = {
    scorer: MastraScorer<any, any, any, any>;
    /** A number implies minimum threshold. Use { min, max } for range-based checks. */
    threshold: ThresholdConfig;
};
/** A scorer entry: either a bare scorer or one with a threshold. */
export type ScorerEntry = MastraScorer<any, any, any, any> | ScorerWithThreshold;
/** Result of a gate evaluation for a single data item. */
export type GateResult = {
    id: string;
    passed: boolean;
    score: number;
};
/** Verdict of an eval run. */
export type EvalVerdict = 'passed' | 'scored' | 'failed';
/** Per-turn assertion results, aggregated by turn index across data items. */
export type TurnResult = {
    /** Zero-based turn index within the conversation. */
    index: number;
    /** Per-gate results for this turn (averaged across data items). */
    gateResults?: GateResult[];
    /** Per-threshold-scorer results for this turn (averaged across data items). */
    thresholdResults?: Array<{
        id: string;
        passed: boolean;
        averageScore: number;
        threshold: ThresholdConfig;
    }>;
    /** Average bare-scorer scores for this turn, keyed by scorer id. */
    scores?: Record<string, number>;
};
type RunEvalsResult = {
    scores: Record<string, any>;
    summary: {
        totalItems: number;
    };
    /** Present when `gates` or threshold-bearing scorers (top-level or per-turn) are provided. */
    verdict?: EvalVerdict;
    /** Per-gate results (averaged across all data items). */
    gateResults?: GateResult[];
    /** Per-threshold-scorer results (averaged across all data items). */
    thresholdResults?: Array<{
        id: string;
        passed: boolean;
        averageScore: number;
        threshold: ThresholdConfig;
    }>;
    /** Per-turn assertion results, present when any data item uses `turns` with gates/scorers. */
    turnResults?: TurnResult[];
};
/**
 * Agent execution options accepted by runEvals. Identical to the agent's own options
 * except `thread` is optional on `memory`: runEvals generates and injects a thread per
 * data item (multi-turn shares one thread across its turns), so callers only need to
 * supply a `resource` when they want a specific one — they don't have to pass a
 * placeholder thread that runEvals would immediately replace.
 */
type RunEvalsAgentOptions = Omit<AgentExecutionOptions<any>, 'scorers' | 'returnScorerData' | 'requestContext' | 'memory'> & {
    memory?: Omit<AgentMemoryOption, 'thread'> & {
        thread?: AgentMemoryOption['thread'];
    };
};
export declare function runEvals<TAgent extends Agent>(config: {
    data: RunEvalsDataItem<TAgent>[];
    /** Gates: scorers that must score 1.0 for the run to pass. */
    gates: MastraScorer<any, any, any, any>[];
    scorers?: ScorerEntry[];
    target: TAgent;
    targetOptions?: RunEvalsAgentOptions;
    onItemComplete?: (params: {
        item: RunEvalsDataItem<TAgent>;
        targetResult: Awaited<ReturnType<Agent['generate']>>;
        scorerResults: Record<string, any>;
    }) => void | Promise<void>;
    concurrency?: number;
}): Promise<RunEvalsResult>;
export declare function runEvals<TAgent extends Agent>(config: {
    data: RunEvalsDataItem<TAgent>[];
    scorers: ScorerEntry[];
    target: TAgent;
    /** Gates: scorers that must score 1.0 for the run to pass. */
    gates?: MastraScorer<any, any, any, any>[];
    targetOptions?: RunEvalsAgentOptions;
    onItemComplete?: (params: {
        item: RunEvalsDataItem<TAgent>;
        targetResult: Awaited<ReturnType<Agent['generate']>>;
        scorerResults: Record<string, any>;
    }) => void | Promise<void>;
    concurrency?: number;
}): Promise<RunEvalsResult>;
export declare function runEvals<TWorkflow extends AnyWorkflow>(config: {
    data: RunEvalsDataItem<TWorkflow>[];
    scorers: MastraScorer<any, any, any, any>[];
    target: TWorkflow;
    targetOptions?: WorkflowRunOptions;
    onItemComplete?: (params: {
        item: RunEvalsDataItem<TWorkflow>;
        targetResult: WorkflowResult<any, any, any, any>;
        scorerResults: Record<string, any>;
    }) => void | Promise<void>;
    concurrency?: number;
}): Promise<RunEvalsResult>;
export declare function runEvals<TWorkflow extends AnyWorkflow>(config: {
    data: RunEvalsDataItem<TWorkflow>[];
    scorers: WorkflowScorerConfig;
    target: TWorkflow;
    targetOptions?: WorkflowRunOptions;
    onItemComplete?: (params: {
        item: RunEvalsDataItem<TWorkflow>;
        targetResult: WorkflowResult<any, any, any, any>;
        scorerResults: {
            workflow?: Record<string, any>;
            steps?: Record<string, Record<string, any>>;
            trajectory?: Record<string, any>;
        };
    }) => void | Promise<void>;
    concurrency?: number;
}): Promise<RunEvalsResult>;
export declare function runEvals<TAgent extends Agent>(config: {
    data: RunEvalsDataItem<TAgent>[];
    scorers: AgentScorerConfig;
    target: TAgent;
    targetOptions?: RunEvalsAgentOptions;
    onItemComplete?: (params: {
        item: RunEvalsDataItem<TAgent>;
        targetResult: Awaited<ReturnType<Agent['generate']>>;
        scorerResults: {
            agent?: Record<string, any>;
            trajectory?: Record<string, any>;
        };
    }) => void | Promise<void>;
    concurrency?: number;
}): Promise<RunEvalsResult>;
export {};
//# sourceMappingURL=index.d.ts.map