@mastra/core/dist/evals/run/index.d.ts

import type { CoreMessage } from '../../_types/@internal_ai-sdk-v4/dist/index.d.ts';
import type { Agent, AgentExecutionOptions, AiMessageType, UIMessageWithMetadata } from '../../agent/index.js';
import type { ObservabilityContext } from '../../observability/index.js';
import type { RequestContext } from '../../request-context/index.js';
import type { WorkflowResult, WorkflowRunStartOptions } from '../../workflows/types.js';
import type { AnyWorkflow } from '../../workflows/workflow.js';
import { Workflow } from '../../workflows/workflow.js';
import type { MastraScorer } from '../base.js';
type WorkflowRunOptions = WorkflowRunStartOptions & {
    initialState?: any;
};
type RunEvalsDataItem<TTarget = unknown> = {
    input: TTarget extends Workflow<any, any> ? any : TTarget extends Agent ? string | string[] | CoreMessage[] | AiMessageType[] | UIMessageWithMetadata[] : unknown;
    groundTruth?: any;
    expectedTrajectory?: any;
    requestContext?: RequestContext;
    startOptions?: WorkflowRunOptions;
} & Partial<ObservabilityContext>;
export type WorkflowScorerConfig = {
    /** Scorers that evaluate the overall workflow input/output */
    workflow?: MastraScorer<any, any, any, any>[];
    /** Scorers that evaluate individual workflow steps by step ID */
    steps?: Record<string, MastraScorer<any, any, any, any>[]>;
    /** Scorers that evaluate the workflow's step execution trajectory */
    trajectory?: MastraScorer<any, any, any, any>[];
};
export type AgentScorerConfig = {
    /** Scorers that evaluate the full agent input/output */
    agent?: MastraScorer<any, any, any, any>[];
    /** Scorers that evaluate the agent's tool call trajectory */
    trajectory?: MastraScorer<any, any, any, any>[];
};
/** Threshold configuration: a number implies minimum, or an object with min/max bounds. */
export type ThresholdConfig = number | {
    min?: number;
    max?: number;
};
/** A scorer with an associated pass/fail threshold. */
export type ScorerWithThreshold = {
    scorer: MastraScorer<any, any, any, any>;
    /** A number implies minimum threshold. Use { min, max } for range-based checks. */
    threshold: ThresholdConfig;
};
/** A scorer entry: either a bare scorer or one with a threshold. */
export type ScorerEntry = MastraScorer<any, any, any, any> | ScorerWithThreshold;
/** Result of a gate evaluation for a single data item. */
export type GateResult = {
    id: string;
    passed: boolean;
    score: number;
};
/** Verdict of an eval run. */
export type EvalVerdict = 'passed' | 'scored' | 'failed';
type RunEvalsResult = {
    scores: Record<string, any>;
    summary: {
        totalItems: number;
    };
    /** Present when `gates` or threshold-bearing scorers are provided. */
    verdict?: EvalVerdict;
    /** Per-gate results (averaged across all data items). */
    gateResults?: GateResult[];
    /** Per-threshold-scorer results (averaged across all data items). */
    thresholdResults?: Array<{
        id: string;
        passed: boolean;
        averageScore: number;
        threshold: ThresholdConfig;
    }>;
};
export declare function runEvals<TAgent extends Agent>(config: {
    data: RunEvalsDataItem<TAgent>[];
    scorers: ScorerEntry[];
    target: TAgent;
    /** Gates: scorers that must score 1.0 for the run to pass. */
    gates?: MastraScorer<any, any, any, any>[];
    targetOptions?: Omit<AgentExecutionOptions<any>, 'scorers' | 'returnScorerData' | 'requestContext'>;
    onItemComplete?: (params: {
        item: RunEvalsDataItem<TAgent>;
        targetResult: Awaited<ReturnType<Agent['generate']>>;
        scorerResults: Record<string, any>;
    }) => void | Promise<void>;
    concurrency?: number;
}): Promise<RunEvalsResult>;
export declare function runEvals<TWorkflow extends AnyWorkflow>(config: {
    data: RunEvalsDataItem<TWorkflow>[];
    scorers: MastraScorer<any, any, any, any>[];
    target: TWorkflow;
    targetOptions?: WorkflowRunOptions;
    onItemComplete?: (params: {
        item: RunEvalsDataItem<TWorkflow>;
        targetResult: WorkflowResult<any, any, any, any>;
        scorerResults: Record<string, any>;
    }) => void | Promise<void>;
    concurrency?: number;
}): Promise<RunEvalsResult>;
export declare function runEvals<TWorkflow extends AnyWorkflow>(config: {
    data: RunEvalsDataItem<TWorkflow>[];
    scorers: WorkflowScorerConfig;
    target: TWorkflow;
    targetOptions?: WorkflowRunOptions;
    onItemComplete?: (params: {
        item: RunEvalsDataItem<TWorkflow>;
        targetResult: WorkflowResult<any, any, any, any>;
        scorerResults: {
            workflow?: Record<string, any>;
            steps?: Record<string, Record<string, any>>;
            trajectory?: Record<string, any>;
        };
    }) => void | Promise<void>;
    concurrency?: number;
}): Promise<RunEvalsResult>;
export declare function runEvals<TAgent extends Agent>(config: {
    data: RunEvalsDataItem<TAgent>[];
    scorers: AgentScorerConfig;
    target: TAgent;
    targetOptions?: Omit<AgentExecutionOptions<any>, 'scorers' | 'returnScorerData' | 'requestContext'>;
    onItemComplete?: (params: {
        item: RunEvalsDataItem<TAgent>;
        targetResult: Awaited<ReturnType<Agent['generate']>>;
        scorerResults: {
            agent?: Record<string, any>;
            trajectory?: Record<string, any>;
        };
    }) => void | Promise<void>;
    concurrency?: number;
}): Promise<RunEvalsResult>;
export {};
//# sourceMappingURL=index.d.ts.map