@mastra/core/dist/manager-CosKYh4m.js

import { i as MastraError } from "./error-MjDSls8S.js";
import { a as resolveObservabilityContext } from "./observability-Cz-X7NF_.js";
import { f as EntityType } from "./utils-DxsDNzD2.js";
import { a as RequestContext } from "./request-context-p_Tq-4EM.js";
import { t as deepEqual } from "./deep-equal-D_JPy4kj.js";
import { b as isSupportedLanguageModel } from "./trip-wire-csEv7lz7.js";
import { Jt as ScorerRunError, Kt as validateAndSaveScore } from "./agent-Dj30gJa3.js";
import { mn as extractTrajectoryFromTrace, pn as extractTrajectory } from "./constants-BfpAlX25.js";
import { isZodType } from "@mastra/schema-compat";
import { zodToJsonSchema } from "@mastra/schema-compat/zod-to-json";
//#region src/datasets/experiment/events.ts
var ExperimentEventDispatcher = class {
	abortController = new AbortController();
	#observer;
	#experimentId;
	#sequence = 0;
	#tail = Promise.resolve();
	#failure;
	get failure() {
		return this.#failure;
	}
	constructor(experimentId, observer) {
		this.#experimentId = experimentId;
		this.#observer = observer;
	}
	emit(input) {
		const event = {
			...input,
			version: 1,
			sequence: ++this.#sequence,
			timestamp: (/* @__PURE__ */ new Date()).toISOString()
		};
		const delivery = this.#tail.then(async () => {
			if (this.#failure) throw this.#failure;
			try {
				await this.#observer(event);
			} catch (error) {
				this.#failure = new MastraError({
					id: "EXPERIMENT_EVENT_OBSERVER_FAILED",
					domain: "EVAL",
					category: "USER",
					details: {
						experimentId: this.#experimentId,
						eventType: event.type,
						eventSequence: event.sequence
					},
					text: `Experiment event observer failed while handling "${event.type}".`
				}, error);
				this.abortController.abort(this.#failure);
				throw this.#failure;
			}
		});
		this.#tail = delivery.catch(() => {});
		return delivery;
	}
};
function toExperimentJsonValue(value, seen = /* @__PURE__ */ new WeakSet()) {
	if (value === null || typeof value === "string" || typeof value === "boolean") return value;
	if (typeof value === "number") return Number.isFinite(value) ? value : null;
	if (typeof value === "bigint") return value.toString();
	if (typeof value === "undefined" || typeof value === "function" || typeof value === "symbol") return null;
	if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value.toISOString();
	if (value instanceof Error) return {
		name: value.name,
		message: value.message
	};
	if (typeof value !== "object") return String(value);
	if (seen.has(value)) return null;
	seen.add(value);
	if (Array.isArray(value)) {
		const result = value.map((entry) => toExperimentJsonValue(entry, seen));
		seen.delete(value);
		return result;
	}
	const result = {};
	for (const [key, entry] of Object.entries(value)) result[key] = toExperimentJsonValue(entry, seen);
	seen.delete(value);
	return result;
}
function createItemCompletedEvent(base, itemIndex, result, traceId) {
	return {
		...base,
		type: "experiment.item.completed",
		itemIndex,
		itemId: result.itemId,
		itemVersion: result.itemVersion,
		status: result.error ? "failed" : "succeeded",
		input: toExperimentJsonValue(result.input),
		output: toExperimentJsonValue(result.output),
		groundTruth: toExperimentJsonValue(result.groundTruth),
		error: toExperimentJsonValue(result.error),
		persistenceError: toExperimentJsonValue(result.persistenceError ?? null),
		scores: toExperimentJsonValue(result.scores),
		toolMockReport: toExperimentJsonValue(result.toolMockReport ?? null),
		retryCount: result.retryCount,
		startedAt: result.startedAt.toISOString(),
		completedAt: result.completedAt.toISOString(),
		traceId
	};
}
//#endregion
//#region src/datasets/experiment/tool-mocks.ts
/** Deterministic failure codes surfaced via `ExecutionResult.error.code`. */
const TOOL_MOCK_MISMATCH = "TOOL_MOCK_MISMATCH";
const TOOL_MOCK_EXHAUSTED = "TOOL_MOCK_EXHAUSTED";
const TOOL_MOCK_NOT_DECLARED = "TOOL_MOCK_NOT_DECLARED";
/**
* Per-item mock matcher. Built fresh for each item run; consumption is tracked
* in local state so repeated `(toolName, args)` mocks are served top-to-bottom.
*
* Tool execution must be forced sequential while an item has mocks so that
* ordered consumption is deterministic (the matcher itself is order-sensitive).
*/
var ToolMockMatcher = class {
	unmockedToolPolicy;
	#entries;
	#served = [];
	#liveCalls = [];
	#failure;
	constructor(mocks, unmockedToolPolicy = "allow") {
		this.unmockedToolPolicy = unmockedToolPolicy;
		this.#entries = (mocks ?? []).map((mock, mockIndex) => ({
			mockIndex,
			toolName: mock.toolName,
			args: mock.args,
			output: mock.output,
			matchArgs: mock.matchArgs ?? "strict",
			consumed: false
		}));
	}
	/** True when the item declares at least one mock (tool execution should run sequentially). */
	get hasMocks() {
		return this.#entries.length > 0;
	}
	/**
	* Resolve a single tool call:
	* - no mock for this tool → `live`
	* - unconsumed mock whose args match (deep-equal for `strict`, always for
	*   `ignore`) → `serve`
	* - tool is mocked but no unconsumed entry matches → `fail`
	*   (`TOOL_MOCK_EXHAUSTED` if args matched but all consumed, else `TOOL_MOCK_MISMATCH`)
	*/
	resolve(toolName, args) {
		if (this.#failure) return {
			kind: "fail",
			code: this.#failure.code
		};
		const candidates = this.#entries.filter((entry) => entry.toolName === toolName);
		if (candidates.length === 0) {
			if (this.unmockedToolPolicy === "deny") {
				this.#failure = {
					code: TOOL_MOCK_NOT_DECLARED,
					toolName,
					args
				};
				return {
					kind: "fail",
					code: TOOL_MOCK_NOT_DECLARED
				};
			}
			this.#liveCalls.push({
				toolName,
				args
			});
			return { kind: "live" };
		}
		const argsMatch = (entry) => entry.matchArgs === "ignore" || deepEqual(entry.args, args);
		const next = candidates.find((entry) => !entry.consumed && argsMatch(entry));
		if (next) {
			next.consumed = true;
			this.#served.push({
				mockIndex: next.mockIndex,
				toolName,
				args
			});
			return {
				kind: "serve",
				output: next.output
			};
		}
		const code = candidates.some((entry) => argsMatch(entry)) ? TOOL_MOCK_EXHAUSTED : TOOL_MOCK_MISMATCH;
		this.#failure ??= {
			code,
			toolName,
			args
		};
		return {
			kind: "fail",
			code
		};
	}
	/** Build the diagnostic report for this item run. */
	report() {
		const unconsumed = this.#entries.filter((entry) => !entry.consumed).map((entry) => ({
			mockIndex: entry.mockIndex,
			toolName: entry.toolName,
			args: entry.args
		}));
		return {
			served: this.#served,
			unconsumed,
			liveCalls: this.#liveCalls,
			...this.#failure ? { failure: this.#failure } : {}
		};
	}
};
//#endregion
//#region src/datasets/experiment/executor.ts
/**
* Execute a dataset item against a scorer (LLM-as-judge calibration).
* item.input should contain exactly what the scorer expects - direct passthrough.
* For calibration: item.input = { input, output, groundTruth } (user structures it)
*/
async function executeScorer(scorer, item) {
	try {
		const result = await scorer.run(item.input);
		const score = typeof result.score === "number" && !isNaN(result.score) ? result.score : null;
		if (score === null && result.score !== void 0) console.warn(`Scorer ${scorer.id} returned invalid score: ${result.score}`);
		return {
			output: {
				score,
				reason: typeof result.reason === "string" ? result.reason : null
			},
			error: null,
			traceId: null
		};
	} catch (error) {
		return {
			output: null,
			error: {
				message: error instanceof Error ? error.message : String(error),
				stack: error instanceof Error ? error.stack : void 0
			},
			traceId: null
		};
	}
}
/** Maximum number of suspend/resume cycles to prevent infinite loops */
const MAX_RESUME_CYCLES = 10;
/**
* Execute a dataset item against a target (agent, workflow, scorer, processor).
* Phase 2: agent/workflow. Phase 4: scorer. Processor deferred.
*/
async function executeTarget(target, targetType, item, options) {
	try {
		const signal = options?.signal;
		if (signal?.aborted) throw signal.reason ?? new DOMException("The operation was aborted.", "AbortError");
		let executionPromise;
		switch (targetType) {
			case "agent":
				executionPromise = executeAgent(target, item, signal, options?.requestContext, options?.experimentId, options?.versions, options?.toolMocks, options?.unmockedToolPolicy);
				break;
			case "workflow":
				executionPromise = executeWorkflow(target, item, options?.requestContext);
				break;
			case "scorer":
				executionPromise = executeScorer(target, item);
				break;
			case "processor": throw new Error(`Target type '${targetType}' not yet supported.`);
			default: throw new Error(`Unknown target type: ${targetType}`);
		}
		if (signal) return await raceWithSignal(executionPromise, signal);
		return await executionPromise;
	} catch (error) {
		return {
			output: null,
			error: {
				message: error instanceof Error ? error.message : String(error),
				stack: error instanceof Error ? error.stack : void 0
			},
			traceId: null
		};
	}
}
/**
* Race a promise against an AbortSignal. Rejects with the signal's reason when aborted.
*/
function raceWithSignal(promise, signal) {
	if (signal.aborted) return Promise.reject(signal.reason ?? new DOMException("The operation was aborted.", "AbortError"));
	return new Promise((resolve, reject) => {
		const onAbort = () => {
			reject(signal.reason ?? new DOMException("The operation was aborted.", "AbortError"));
		};
		signal.addEventListener("abort", onAbort, { once: true });
		promise.then((value) => {
			signal.removeEventListener("abort", onAbort);
			resolve(value);
		}, (err) => {
			signal.removeEventListener("abort", onAbort);
			reject(err);
		});
	});
}
/**
* Execute a dataset item against an agent.
* Uses generate() for both v1 and v2 models.
*/
async function executeAgent(agent, item, signal, requestContext, experimentId, versions, toolMocks, unmockedToolPolicy) {
	const model = await agent.getModel();
	const input = item.input;
	const reqCtx = requestContext ? new RequestContext(Object.entries(requestContext)) : void 0;
	const tracingOptions = experimentId ? { metadata: { experimentId } } : void 0;
	const matcher = new ToolMockMatcher(toolMocks, unmockedToolPolicy);
	const shouldInterceptTools = matcher.hasMocks || matcher.unmockedToolPolicy === "deny";
	const mockAbort = shouldInterceptTools ? new AbortController() : void 0;
	const mockHooks = shouldInterceptTools ? buildToolMockHooks(agent, matcher, mockAbort) : void 0;
	const generateSignal = mockAbort && signal ? AbortSignal.any([signal, mockAbort.signal]) : mockAbort?.signal ?? signal;
	const mockConcurrency = shouldInterceptTools ? { toolCallConcurrency: 1 } : void 0;
	let rawResult;
	try {
		rawResult = isSupportedLanguageModel(model) ? await agent.generate(input, {
			scorers: {},
			returnScorerData: true,
			abortSignal: generateSignal,
			...reqCtx ? { requestContext: reqCtx } : {},
			...tracingOptions ? { tracingOptions } : {},
			...versions ? { versions } : {},
			...mockHooks ? { hooks: mockHooks } : {},
			...mockConcurrency ?? {}
		}) : await agent.generateLegacy(input, {
			scorers: {},
			returnScorerData: true,
			abortSignal: generateSignal,
			...reqCtx ? { requestContext: reqCtx } : {},
			...tracingOptions ? { tracingOptions } : {},
			...mockHooks ? { hooks: mockHooks } : {},
			...mockConcurrency ?? {}
		});
	} catch (error) {
		const mockReport = shouldInterceptTools ? matcher.report() : void 0;
		if (mockReport?.failure) return toolMockFailureResult(mockReport, null);
		throw error;
	}
	const result = rawResult;
	const traceId = result.traceId ?? null;
	const scoringData = result.scoringData;
	const toolMockReport = shouldInterceptTools ? matcher.report() : void 0;
	if (toolMockReport?.failure) return toolMockFailureResult(toolMockReport, traceId);
	return {
		output: {
			text: result.text,
			object: result.object,
			toolCalls: result.toolCalls,
			toolResults: result.toolResults,
			sources: result.sources,
			files: result.files,
			usage: result.usage,
			reasoningText: result.reasoningText,
			traceId,
			error: result.error ?? null
		},
		error: null,
		traceId,
		scorerInput: scoringData?.input,
		scorerOutput: scoringData?.output,
		...toolMockReport ? { toolMockReport } : {}
	};
}
/** Build the deterministic, non-retryable failure result for a mis-called mock. */
function toolMockFailureResult(report, traceId) {
	const failure = report.failure;
	return {
		output: null,
		error: {
			message: failure.code === "TOOL_MOCK_NOT_DECLARED" ? `Tool "${failure.toolName}" was called without a declared mock (${failure.code}).` : `Mocked tool "${failure.toolName}" was called with arguments that did not match an available mock (${failure.code}).`,
			code: failure.code
		},
		traceId,
		toolMockReport: report
	};
}
/**
* Compose item-level tool mocks with the agent's configured tool hooks into a
* single set of run-level hooks.
*
* Composition order (per spec):
*  1. User `beforeToolCall` (if `{ proceed: false }`, short-circuit — the mock is
*     left unconsumed and reported as such; user `afterToolCall` is NOT called,
*     matching the agent's own short-circuit behavior).
*  2. Mock matcher — `serve` returns the mocked output; `fail` aborts the run so
*     the model cannot call any further (possibly unmocked, side-effecting) tools
*     live; `live` falls through to the real tool.
*  3. User `afterToolCall` runs for served mocks (the agent skips its own on
*     short-circuit, so it is invoked here to honor the documented composition).
*
* Ordered consumption of repeated `(toolName, args)` mocks is deterministic because
* the caller forces `toolCallConcurrency: 1` when mocks exist, so tool calls arrive
* (and consume) in the provider's call order — no mutex needed.
*/
function buildToolMockHooks(agent, matcher, mockAbort) {
	const userHooks = agent.getConfiguredToolHooks();
	return {
		beforeToolCall: async (context) => {
			const userResult = await userHooks?.beforeToolCall?.(context);
			if (userResult?.proceed === false) return userResult;
			const resolution = matcher.resolve(context.toolName, context.input);
			if (resolution.kind === "serve") {
				await userHooks?.afterToolCall?.({
					...context,
					output: resolution.output
				});
				return {
					proceed: false,
					output: resolution.output
				};
			}
			if (resolution.kind === "fail") {
				mockAbort.abort(/* @__PURE__ */ new Error(`Tool mock failure for "${context.toolName}" (${resolution.code})`));
				return {
					proceed: false,
					output: { error: resolution.code }
				};
			}
		},
		afterToolCall: userHooks?.afterToolCall
	};
}
/**
* Extract resume data from item fields and metadata.
*
* Checks top-level `resumeSteps`/`resumeData` first (inline data path),
* then falls back to `metadata.resumeSteps`/`metadata.resumeData` (storage-backed path).
*
* Supports two shapes:
* 1. Keyed by step ID: `resumeSteps: { "step-id": <payload> }`
*    Used when the workflow may suspend on multiple steps and each needs distinct data.
* 2. Flat payload: `resumeData: <payload>`
*    Used when the workflow has a single suspended step (auto-detected).
*/
function extractResumeData(item) {
	return {
		perStep: item.resumeSteps !== void 0 ? item.resumeSteps : item.metadata?.resumeSteps,
		flat: item.resumeData !== void 0 ? item.resumeData : item.metadata?.resumeData
	};
}
/**
* Execute a dataset item against a workflow.
* Creates a run with scorers disabled to avoid double-scoring.
*
* When the workflow suspends, checks for resume data in `item.metadata`
* (via `resumeSteps` keyed by step ID or `resumeData` for single-step workflows)
* and automatically resumes. Loops through multiple suspend/resume cycles up to
* MAX_RESUME_CYCLES to support multi-step suspend workflows.
*
* Mirrors `executeWorkflow` in evals/run so dataset experiments and runEvals
* produce the same observability spans and scoring data for workflow targets.
*/
async function executeWorkflow(workflow, item, requestContext) {
	const reqCtx = requestContext ? new RequestContext(Object.entries(requestContext)) : void 0;
	const observabilityContext = resolveObservabilityContext({});
	const run = await workflow.createRun({ disableScorers: true });
	let result = await run.start({
		inputData: item.input,
		...reqCtx ? { requestContext: reqCtx } : {},
		...observabilityContext
	});
	const { perStep, flat } = extractResumeData(item);
	if (perStep !== void 0 || flat !== void 0) {
		let cycle = 0;
		while (result.status === "suspended" && cycle < MAX_RESUME_CYCLES) {
			cycle++;
			const suspendedPaths = result.suspended ?? [];
			if (suspendedPaths.length === 0) break;
			const firstSuspendedStep = suspendedPaths[0]?.[0];
			if (!firstSuspendedStep) break;
			const perStepValue = perStep?.[firstSuspendedStep];
			const stepResumeData = perStepValue !== void 0 ? perStepValue : flat;
			if (stepResumeData === void 0) break;
			result = await run.resume({
				resumeData: stepResumeData,
				step: firstSuspendedStep,
				...reqCtx ? { requestContext: reqCtx } : {},
				...observabilityContext
			});
		}
	}
	return handleWorkflowResult(result);
}
/**
* Map a terminal WorkflowResult to an ExecutionResult.
* Uses a loose `result: any` parameter because WorkflowResult is heavily generic;
* status-narrowing guards below keep accesses safe.
*/
function handleWorkflowResult(result) {
	const traceId = result.traceId ?? null;
	const spanId = result.spanId ?? null;
	if (result.status === "success") return {
		output: result.result,
		error: null,
		traceId,
		spanId,
		stepResults: result.steps,
		stepExecutionPath: result.stepExecutionPath
	};
	if (result.status === "failed") return {
		output: null,
		error: {
			message: result.error?.message ?? "Workflow failed",
			stack: result.error?.stack
		},
		traceId,
		spanId,
		stepResults: result.steps,
		stepExecutionPath: result.stepExecutionPath
	};
	if (result.status === "tripwire") return {
		output: null,
		error: { message: `Workflow tripwire: ${result.tripwire?.reason ?? "Unknown reason"}` },
		traceId,
		spanId,
		stepResults: result.steps,
		stepExecutionPath: result.stepExecutionPath
	};
	if (result.status === "suspended") return {
		output: result.suspendPayload ?? null,
		error: { message: "Workflow suspended — provide resume data via item.resumeSteps/item.resumeData (or metadata.resumeSteps/metadata.resumeData) to auto-resume" },
		traceId,
		spanId,
		stepResults: result.steps,
		stepExecutionPath: result.stepExecutionPath
	};
	if (result.status === "paused") return {
		output: null,
		error: { message: "Workflow paused - not yet supported in dataset experiments" },
		traceId,
		spanId,
		stepResults: result.steps,
		stepExecutionPath: result.stepExecutionPath
	};
	return {
		output: null,
		error: { message: `Workflow ended with unexpected status: ${result.status}` },
		traceId,
		spanId
	};
}
//#endregion
//#region src/datasets/experiment/scorer.ts
function toScorerTargetEntityType(targetType) {
	switch (targetType) {
		case "agent": return EntityType.AGENT;
		case "workflow": return EntityType.WORKFLOW_RUN;
		case "scorer": return EntityType.SCORER;
		default: return;
	}
}
function getItemScorerById(mastra, scorerId) {
	try {
		return mastra.getScorerById(scorerId) ?? null;
	} catch {
		return null;
	}
}
/**
* Resolve scorers from mixed array of instances and string IDs.
* String IDs are looked up from Mastra's scorer registry.
*/
function resolveScorers(mastra, scorers) {
	if (!scorers || scorers.length === 0) return [];
	return scorers.map((scorer) => {
		if (typeof scorer === "string") {
			const resolved = mastra.getScorerById(scorer);
			if (!resolved) {
				console.warn(`Scorer not found: ${scorer}`);
				return null;
			}
			return resolved;
		}
		return scorer;
	}).filter((s) => s !== null);
}
const EXPERIMENT_ITEM_SCORER_NOT_FOUND = "EXPERIMENT_ITEM_SCORER_NOT_FOUND";
/**
* Create a run-scoped resolver for item scorer IDs. Resolutions, including misses,
* are cached so concurrent items hydrate a stored scorer at most once per run.
*/
function createItemScorerResolver(mastra) {
	const resolutionCache = /* @__PURE__ */ new Map();
	const resolveById = (scorerId) => {
		const cached = resolutionCache.get(scorerId);
		if (cached) return cached;
		const resolution = (async () => {
			let scorer = getItemScorerById(mastra, scorerId);
			if (scorer) return scorer;
			const editor = mastra.getEditor?.();
			if (editor) {
				try {
					await editor.scorer.getById(scorerId);
				} catch {}
				scorer = getItemScorerById(mastra, scorerId);
			}
			return scorer;
		})();
		resolutionCache.set(scorerId, resolution);
		return resolution;
	};
	return async (scorerIds) => {
		const uniqueIds = [...new Set(scorerIds)];
		const resolved = await Promise.all(uniqueIds.map(async (id) => ({
			id,
			scorer: await resolveById(id)
		})));
		return {
			scorers: resolved.flatMap(({ scorer }) => scorer ? [scorer] : []),
			missingIds: resolved.flatMap(({ id, scorer }) => scorer ? [] : [id])
		};
	};
}
/**
* Attempt to extract a Trajectory from the observability trace store.
* Falls back to undefined if storage is unavailable or the trace has no spans.
*/
async function extractTrajectoryFromStorage(storage, traceId) {
	if (!storage || !traceId) return void 0;
	try {
		const observabilityStore = await storage.getStore("observability");
		if (!observabilityStore) return void 0;
		const trace = await observabilityStore.getTrace({ traceId });
		if (!trace?.spans?.length) return void 0;
		return extractTrajectoryFromTrace(trace.spans);
	} catch {
		return;
	}
}
/**
* Run all scorers for a single item result.
* Errors are isolated per scorer - one failing scorer doesn't affect others.
* Trajectory scorers (scorer.type === 'trajectory') receive a pre-extracted
* Trajectory as their output, mirroring the dispatch runEvals performs.
*
* `persistScores: false` suppresses score writes while leaving `storage` usable
* for reads. The two are deliberately separate parameters: `storage` is also the
* source for trajectory extraction, so nulling it to stop writes would silently
* downgrade trajectory scorers to the raw-message fallback.
*/
async function runScorersForItem(scorers, item, output, storage, runId, targetType, targetId, itemId, scorerInput, scorerOutput, traceId, workflowData, persistScores = true) {
	if (scorers.length === 0) return [];
	const hasTrajectoryScorer = scorers.some((s) => s.type === "trajectory");
	let trajectoryOutput;
	if (hasTrajectoryScorer) trajectoryOutput = await extractTrajectoryFromStorage(storage, traceId) ?? (scorerOutput ? extractTrajectory(scorerOutput) : { steps: [] });
	const targetCorrelationContext = {
		...traceId ? { traceId } : {},
		entityType: toScorerTargetEntityType(targetType),
		entityId: targetId,
		entityName: targetId,
		experimentId: runId
	};
	return (await Promise.allSettled(scorers.map(async (scorer) => {
		const { result, promptMetadata } = await runScorerSafe(scorer, item, output, scorerInput, scorerOutput, targetType, traceId, targetCorrelationContext, scorer.type === "trajectory" ? trajectoryOutput : void 0, workflowData, persistScores);
		if (persistScores && storage && result.error === null && result.score !== null) try {
			await validateAndSaveScore(storage, {
				scorerId: scorer.id,
				score: result.score,
				reason: result.reason ?? void 0,
				input: item.input,
				output,
				additionalContext: item.metadata,
				entityType: targetType.toUpperCase(),
				entityId: itemId,
				source: "TEST",
				runId,
				traceId,
				scorer: {
					id: scorer.id,
					name: scorer.name,
					description: scorer.description ?? "",
					hasJudge: !!scorer.judge
				},
				entity: {
					id: targetId,
					name: targetId
				},
				...promptMetadata
			});
		} catch (saveError) {
			console.warn(`Failed to save score for scorer ${scorer.id}:`, saveError);
		}
		return result;
	}))).map((s, i) => {
		if (s.status === "fulfilled") return s.value;
		const scorer = scorers[i];
		return {
			scorerId: scorer.id,
			scorerName: scorer.name,
			score: null,
			reason: null,
			error: String(s.reason),
			targetScope: scorer.type === "trajectory" ? "trajectory" : "span"
		};
	});
}
function extractScorerRunFields(scoreResult) {
	if (typeof scoreResult !== "object" || scoreResult === null) return {
		score: null,
		reason: null,
		promptMetadata: {}
	};
	const fields = scoreResult;
	const str = (key) => typeof fields[key] === "string" ? fields[key] : void 0;
	const obj = (key) => {
		const value = fields[key];
		return typeof value === "object" && value !== null ? value : void 0;
	};
	return {
		score: typeof fields.score === "number" ? fields.score : null,
		reason: typeof fields.reason === "string" ? fields.reason : null,
		promptMetadata: {
			generateScorePrompt: str("generateScorePrompt"),
			generateReasonPrompt: str("generateReasonPrompt"),
			preprocessStepResult: obj("preprocessStepResult"),
			preprocessPrompt: str("preprocessPrompt"),
			analyzeStepResult: obj("analyzeStepResult"),
			analyzePrompt: str("analyzePrompt")
		}
	};
}
/**
* Run a single scorer safely, catching any errors.
* Returns both the ScorerResult and prompt metadata for DB persistence.
* When trajectoryOutput is provided the scorer receives it as run.output,
* honoring the type: 'trajectory' contract.
*/
async function runScorerSafe(scorer, item, output, scorerInput, scorerOutput, targetType, targetTraceId, targetCorrelationContext, trajectoryOutput, workflowData, persistScores = true) {
	try {
		const effectiveOutput = trajectoryOutput ?? scorerOutput ?? output;
		const effectiveScope = trajectoryOutput ? "trajectory" : "span";
		const targetMetadata = !trajectoryOutput && workflowData && (workflowData.stepResults || workflowData.stepExecutionPath) ? {
			...workflowData.stepResults ? { stepResults: workflowData.stepResults } : {},
			...workflowData.stepExecutionPath ? { stepExecutionPath: workflowData.stepExecutionPath } : {}
		} : void 0;
		const scoreResult = await scorer.run({
			input: scorerInput ?? item.input,
			output: effectiveOutput,
			groundTruth: item.groundTruth,
			scoreSource: "experiment",
			targetScope: effectiveScope,
			targetEntityType: toScorerTargetEntityType(targetType),
			targetTraceId,
			...workflowData?.spanId ? { targetSpanId: workflowData.spanId } : {},
			...targetCorrelationContext ? { targetCorrelationContext } : {},
			...targetMetadata ? { targetMetadata } : {},
			_internal: { emitObservabilityScore: persistScores }
		});
		if (typeof scoreResult !== "object" || scoreResult === null) return {
			result: {
				scorerId: scorer.id,
				scorerName: scorer.name,
				score: null,
				reason: null,
				error: `Scorer ${scorer.name} (${scorer.id}) returned invalid result: expected object, got ${scoreResult === null ? "null" : typeof scoreResult} (${String(scoreResult)})`
			},
			promptMetadata: {}
		};
		const { score, reason, promptMetadata } = extractScorerRunFields(scoreResult);
		return {
			result: {
				scorerId: scorer.id,
				scorerName: scorer.name,
				score,
				reason,
				error: null,
				targetScope: effectiveScope
			},
			promptMetadata
		};
	} catch (error) {
		if (error instanceof ScorerRunError) {
			const { score, reason, promptMetadata } = extractScorerRunFields(error.result);
			return {
				result: {
					scorerId: scorer.id,
					scorerName: scorer.name,
					score,
					reason,
					error: error.message,
					failedStep: error.failedStep,
					completedSteps: error.completedSteps,
					targetScope: trajectoryOutput ? "trajectory" : "span"
				},
				promptMetadata
			};
		}
		return {
			result: {
				scorerId: scorer.id,
				scorerName: scorer.name,
				score: null,
				reason: null,
				error: error instanceof Error ? error.message : String(error),
				targetScope: trajectoryOutput ? "trajectory" : "span"
			},
			promptMetadata: {}
		};
	}
}
/**
* Resolve step-scoped scorers from a `Record<stepId, (MastraScorer | string)[]>`.
* String IDs are looked up from Mastra's scorer registry; missing IDs are skipped
* with a warning (matching `resolveScorers`).
*/
function resolveStepScorers(mastra, stepsConfig) {
	if (!stepsConfig) return {};
	const resolved = {};
	for (const [stepId, scorers] of Object.entries(stepsConfig)) {
		const stepScorers = resolveScorers(mastra, scorers);
		if (stepScorers.length > 0) resolved[stepId] = stepScorers;
	}
	return resolved;
}
/**
* Run step-scoped scorers for a single workflow item. Mirrors the per-step
* dispatch in `runEvals`: each scorer runs against `stepResult.payload` and
* `stepResult.output`, with `targetScope: 'span'` and
* `targetEntityType: WORKFLOW_STEP`. The returned `ScorerResult` carries the
* originating `stepId` so callers can disambiguate per-step results in the
* flat `scores` array. Steps whose result is missing or did not succeed
* surface as an error `ScorerResult` rather than disappearing silently.
*
* Errors are isolated per scorer (consistent with `runScorersForItem`); a
* failing scorer produces a `ScorerResult` with `error` set, not a throw.
*/
async function runStepScorersForItem(stepScorers, item, workflowData, storage, runId, targetType, targetId, itemId, traceId, persistScores = true) {
	const stepIds = Object.keys(stepScorers);
	if (stepIds.length === 0) return [];
	const results = [];
	const stepResults = workflowData?.stepResults;
	for (const stepId of stepIds) {
		const scorers = stepScorers[stepId];
		const stepResult = stepResults?.[stepId];
		if (!stepResult || stepResult.status !== "success" || stepResult.output === void 0) {
			for (const scorer of scorers) results.push({
				scorerId: scorer.id,
				scorerName: scorer.name,
				score: null,
				reason: null,
				error: `Step "${stepId}" did not produce a successful output (status: ${stepResult?.status ?? "missing"})`,
				targetScope: "span",
				stepId
			});
			continue;
		}
		const stepInput = stepResult.payload !== void 0 ? stepResult.payload : item.input;
		const stepOutput = stepResult.output;
		const targetCorrelationContext = {
			...traceId ? { traceId } : {},
			entityType: EntityType.WORKFLOW_STEP,
			entityId: stepId,
			entityName: stepId,
			experimentId: runId
		};
		const settled = await Promise.allSettled(scorers.map(async (scorer) => {
			try {
				const scoreResult = await scorer.run({
					input: stepInput,
					output: stepOutput,
					groundTruth: item.groundTruth,
					scoreSource: "experiment",
					targetScope: "span",
					targetEntityType: EntityType.WORKFLOW_STEP,
					targetTraceId: traceId,
					...targetCorrelationContext ? { targetCorrelationContext } : {},
					_internal: { emitObservabilityScore: persistScores }
				});
				if (typeof scoreResult !== "object" || scoreResult === null) return {
					scorerId: scorer.id,
					scorerName: scorer.name,
					score: null,
					reason: null,
					error: `Scorer ${scorer.name} (${scorer.id}) returned invalid result on step ${stepId}`,
					targetScope: "span",
					stepId
				};
				const fields = scoreResult;
				const score = typeof fields.score === "number" ? fields.score : null;
				const reason = typeof fields.reason === "string" ? fields.reason : null;
				if (persistScores && storage && score !== null) try {
					await validateAndSaveScore(storage, {
						scorerId: scorer.id,
						score,
						reason: reason ?? void 0,
						input: stepInput,
						output: stepOutput,
						additionalContext: {
							...item.metadata,
							stepId
						},
						entityType: "WORKFLOW_STEP",
						entityId: itemId,
						source: "TEST",
						runId,
						traceId,
						scorer: {
							id: scorer.id,
							name: scorer.name,
							description: scorer.description ?? "",
							hasJudge: !!scorer.judge
						},
						entity: {
							id: targetId,
							name: targetId
						}
					});
				} catch (saveError) {
					console.warn(`Failed to save score for step scorer ${scorer.id} on ${stepId}:`, saveError);
				}
				return {
					scorerId: scorer.id,
					scorerName: scorer.name,
					score,
					reason,
					error: null,
					targetScope: "span",
					stepId
				};
			} catch (error) {
				if (error instanceof ScorerRunError) {
					const { score, reason } = extractScorerRunFields(error.result);
					return {
						scorerId: scorer.id,
						scorerName: scorer.name,
						score,
						reason,
						error: error.message,
						failedStep: error.failedStep,
						completedSteps: error.completedSteps,
						targetScope: "span",
						stepId
					};
				}
				return {
					scorerId: scorer.id,
					scorerName: scorer.name,
					score: null,
					reason: null,
					error: error instanceof Error ? error.message : String(error),
					targetScope: "span",
					stepId
				};
			}
		}));
		for (let i = 0; i < settled.length; i++) {
			const s = settled[i];
			if (s.status === "fulfilled") results.push(s.value);
			else {
				const scorer = scorers[i];
				results.push({
					scorerId: scorer.id,
					scorerName: scorer.name,
					score: null,
					reason: null,
					error: String(s.reason),
					targetScope: "span",
					stepId
				});
			}
		}
	}
	return results;
}
//#endregion
//#region src/datasets/experiment/analytics/aggregate.ts
/**
* Compute the arithmetic mean of an array of numbers.
*
* @param values - Array of numbers to average
* @returns Mean value, or 0 if array is empty
*/
function computeMean(values) {
	if (values.length === 0) return 0;
	return values.reduce((acc, val) => acc + val, 0) / values.length;
}
/**
* Compute aggregate statistics for a set of scores.
*
* Metrics:
* - errorRate: proportion of items with null scores (errors)
* - passRate: proportion of scored items meeting threshold
* - avgScore: mean of non-null scores
*
* @param scores - Score records from storage
* @param passThreshold - Absolute threshold for pass (score >= threshold)
* @returns ScorerStats with all computed metrics
*/
function computeScorerStats(scores, passThreshold = .5) {
	const totalItems = scores.length;
	if (totalItems === 0) return {
		errorRate: 0,
		errorCount: 0,
		passRate: 0,
		passCount: 0,
		avgScore: 0,
		scoreCount: 0,
		totalItems: 0
	};
	const validScores = [];
	let errorCount = 0;
	for (const score of scores) if (score.score === null || score.score === void 0) errorCount++;
	else validScores.push(score.score);
	const scoreCount = validScores.length;
	const errorRate = errorCount / totalItems;
	const passCount = validScores.filter((s) => s >= passThreshold).length;
	const passRate = scoreCount > 0 ? passCount / scoreCount : 0;
	const avgScore = computeMean(validScores);
	return {
		errorRate,
		errorCount,
		passRate,
		passCount,
		avgScore,
		scoreCount,
		totalItems
	};
}
/**
* Determine if a score delta represents a regression.
*
* @param delta - Score difference (experiment B - experiment A)
* @param threshold - Absolute threshold for regression detection
* @param direction - Score direction ('higher-is-better' or 'lower-is-better')
* @returns True if delta represents a regression
*
* @example
* // Higher is better (default): negative delta is bad
* isRegression(-0.1, 0.05, 'higher-is-better') // true (dropped more than 0.05)
* isRegression(-0.01, 0.05, 'higher-is-better') // false (within tolerance)
*
* // Lower is better: positive delta is bad
* isRegression(0.1, 0.05, 'lower-is-better') // true (increased more than 0.05)
*/
function isRegression(delta, threshold, direction = "higher-is-better") {
	if (direction === "higher-is-better") return delta < -threshold;
	else return delta > threshold;
}
//#endregion
//#region src/datasets/experiment/analytics/compare.ts
/**
* Default threshold when not specified: no tolerance for regression.
*/
const DEFAULT_THRESHOLD = {
	value: 0,
	direction: "higher-is-better"
};
/**
* Default pass threshold for computing pass rate.
*/
const DEFAULT_PASS_THRESHOLD = .5;
/**
* Compare two experiments to detect score regressions.
*
* @param mastra - Mastra instance for storage access
* @param config - Comparison configuration
* @returns ComparisonResult with per-scorer and per-item comparisons
*
* @example
* ```typescript
* const result = await compareExperiments(mastra, {
*   experimentIdA: 'baseline-experiment-id',
*   experimentIdB: 'candidate-experiment-id',
*   thresholds: {
*     'accuracy': { value: 0.05, direction: 'higher-is-better' },
*     'latency': { value: 100, direction: 'lower-is-better' },
*   },
* });
*
* if (result.hasRegression) {
*   console.log('Quality regression detected!');
* }
* ```
*/
async function compareExperiments(mastra, config) {
	const { experimentIdA, experimentIdB, thresholds = {} } = config;
	const warnings = [];
	const storage = mastra.getStorage();
	if (!storage) throw new Error("Storage not configured. Configure storage in Mastra instance.");
	const experimentsStore = await storage.getStore("experiments");
	const scoresStore = await storage.getStore("scores");
	if (!experimentsStore) throw new Error("ExperimentsStorage not configured.");
	if (!scoresStore) throw new Error("ScoresStorage not configured.");
	const [experimentA, experimentB] = await Promise.all([experimentsStore.getExperimentById({ id: experimentIdA }), experimentsStore.getExperimentById({ id: experimentIdB })]);
	if (!experimentA) throw new Error(`Experiment not found: ${experimentIdA}`);
	if (!experimentB) throw new Error(`Experiment not found: ${experimentIdB}`);
	const versionMismatch = experimentA.datasetVersion !== experimentB.datasetVersion;
	if (versionMismatch) warnings.push(`Experiments have different dataset versions: ${experimentA.datasetVersion} vs ${experimentB.datasetVersion}`);
	const [resultsA, resultsB] = await Promise.all([experimentsStore.listExperimentResults({
		experimentId: experimentIdA,
		pagination: {
			page: 0,
			perPage: false
		}
	}), experimentsStore.listExperimentResults({
		experimentId: experimentIdB,
		pagination: {
			page: 0,
			perPage: false
		}
	})]);
	const [scoresA, scoresB] = await Promise.all([scoresStore.listScoresByRunId({
		runId: experimentIdA,
		pagination: {
			page: 0,
			perPage: false
		}
	}), scoresStore.listScoresByRunId({
		runId: experimentIdB,
		pagination: {
			page: 0,
			perPage: false
		}
	})]);
	if (resultsA.results.length === 0 && resultsB.results.length === 0) {
		warnings.push("Both experiments have no results.");
		return buildEmptyResult(experimentA, experimentB, versionMismatch, warnings);
	}
	if (resultsA.results.length === 0) warnings.push("Experiment A has no results.");
	if (resultsB.results.length === 0) warnings.push("Experiment B has no results.");
	const itemIdsA = new Set(resultsA.results.map((r) => r.itemId));
	const itemIdsB = new Set(resultsB.results.map((r) => r.itemId));
	if ([...itemIdsA].filter((id) => itemIdsB.has(id)).length === 0) warnings.push("No overlapping items between experiments.");
	const scoresMapA = groupScoresByScorerAndItem(scoresA.scores);
	const scoresMapB = groupScoresByScorerAndItem(scoresB.scores);
	const allScorerIds = /* @__PURE__ */ new Set([...Object.keys(scoresMapA), ...Object.keys(scoresMapB)]);
	const scorers = {};
	let hasRegression = false;
	for (const scorerId of allScorerIds) {
		const scorerScoresA = scoresMapA[scorerId] ?? {};
		const scorerScoresB = scoresMapB[scorerId] ?? {};
		const scoresArrayA = Object.values(scorerScoresA);
		const scoresArrayB = Object.values(scorerScoresB);
		const thresholdConfig = thresholds[scorerId] ?? DEFAULT_THRESHOLD;
		const threshold = thresholdConfig.value;
		const direction = thresholdConfig.direction ?? "higher-is-better";
		const statsA = computeScorerStats(scoresArrayA, DEFAULT_PASS_THRESHOLD);
		const statsB = computeScorerStats(scoresArrayB, DEFAULT_PASS_THRESHOLD);
		const delta = statsB.avgScore - statsA.avgScore;
		const regressed = isRegression(delta, threshold, direction);
		if (regressed) hasRegression = true;
		scorers[scorerId] = {
			statsA,
			statsB,
			delta,
			regressed,
			threshold
		};
	}
	const allItemIds = /* @__PURE__ */ new Set([...itemIdsA, ...itemIdsB]);
	const items = [];
	for (const itemId of allItemIds) {
		const inBothExperiments = itemIdsA.has(itemId) && itemIdsB.has(itemId);
		const itemScoresA = {};
		const itemScoresB = {};
		for (const scorerId of allScorerIds) {
			const scoreA = scoresMapA[scorerId]?.[itemId];
			const scoreB = scoresMapB[scorerId]?.[itemId];
			itemScoresA[scorerId] = scoreA?.score ?? null;
			itemScoresB[scorerId] = scoreB?.score ?? null;
		}
		items.push({
			itemId,
			inBothExperiments,
			scoresA: itemScoresA,
			scoresB: itemScoresB
		});
	}
	return {
		experimentA: {
			id: experimentA.id,
			datasetVersion: experimentA.datasetVersion
		},
		experimentB: {
			id: experimentB.id,
			datasetVersion: experimentB.datasetVersion
		},
		versionMismatch,
		hasRegression,
		scorers,
		items,
		warnings
	};
}
/**
* Group scores by scorer ID, then by item ID.
*/
function groupScoresByScorerAndItem(scores) {
	const result = {};
	for (const score of scores) {
		const scorerId = score.scorerId;
		const itemId = score.entityId;
		if (!result[scorerId]) result[scorerId] = {};
		result[scorerId][itemId] = score;
	}
	return result;
}
/**
* Build an empty comparison result for edge cases.
*/
function buildEmptyResult(experimentA, experimentB, versionMismatch, warnings) {
	return {
		experimentA: {
			id: experimentA.id,
			datasetVersion: experimentA.datasetVersion
		},
		experimentB: {
			id: experimentB.id,
			datasetVersion: experimentB.datasetVersion
		},
		versionMismatch,
		hasRegression: false,
		scorers: {},
		items: [],
		warnings
	};
}
//#endregion
//#region src/datasets/experiment/index.ts
/**
* Run a dataset experiment against a target with optional scoring.
*
* Executes all items in the dataset concurrently (up to maxConcurrency) against
* the specified target (agent or workflow). Optionally applies scorers to each
* result and persists both results and scores to storage.
*
* @param mastra - Mastra instance for storage and target resolution
* @param config - Experiment configuration
* @returns ExperimentSummary with results and scores
*
* @example
* ```typescript
* const summary = await runExperiment(mastra, {
*   datasetId: 'my-dataset',
*   targetType: 'agent',
*   targetId: 'my-agent',
*   scorers: [accuracyScorer, latencyScorer],
*   maxConcurrency: 10,
* });
* console.log(`${summary.succeededCount}/${summary.totalItems} succeeded`);
* ```
*/
async function runExperiment(mastra, config) {
	const { datasetId, targetType, targetId, scorers: scorerInput, version, maxConcurrency = 5, signal, onEvent, itemTimeout, maxRetries = 0, experimentId: providedExperimentId, name, description, metadata, requestContext: globalRequestContext, agentVersion, versions, persistence } = config;
	const persistExperiments = persistence?.experiments !== "none";
	const persistScores = persistence?.scores !== "none";
	const startedAt = /* @__PURE__ */ new Date();
	const experimentId = providedExperimentId ?? crypto.randomUUID();
	const eventDispatcher = onEvent ? new ExperimentEventDispatcher(experimentId, onEvent) : void 0;
	const executionSignal = eventDispatcher ? signal ? AbortSignal.any([signal, eventDispatcher.abortController.signal]) : eventDispatcher.abortController.signal : signal;
	const eventTarget = {
		type: targetType ?? "task",
		id: targetId ?? "inline"
	};
	const storage = mastra.getStorage();
	const datasetsStore = await storage?.getStore("datasets");
	const experimentsStore = persistExperiments ? await storage?.getStore("experiments") : void 0;
	const markFailedForObserverError = async (counts) => {
		if (!experimentsStore) return;
		try {
			await experimentsStore.updateExperiment({
				id: experimentId,
				status: "failed",
				...counts
			});
		} catch {}
	};
	const markFailedOnSetupError = async (err) => {
		if (providedExperimentId && experimentsStore) try {
			await experimentsStore.updateExperiment({
				id: experimentId,
				status: "failed",
				completedAt: /* @__PURE__ */ new Date()
			});
		} catch (updateErr) {
			mastra.getLogger()?.error(`Failed to mark experiment ${experimentId} as failed: ${updateErr}`);
		}
		throw err;
	};
	let items;
	let datasetVersion;
	let datasetRecord;
	try {
		if (config.data) {
			items = (typeof config.data === "function" ? await config.data() : config.data).map((dataItem) => {
				return {
					id: dataItem.id ?? crypto.randomUUID(),
					datasetVersion: null,
					input: dataItem.input,
					groundTruth: dataItem.groundTruth,
					requestContext: dataItem.requestContext,
					metadata: dataItem.metadata,
					resumeSteps: dataItem.resumeSteps,
					resumeData: dataItem.resumeData,
					toolMocks: dataItem.toolMocks,
					unmockedToolPolicy: dataItem.unmockedToolPolicy,
					scorerIds: dataItem.scorerIds
				};
			});
			datasetVersion = null;
		} else if (datasetId) {
			if (!datasetsStore) throw new Error("DatasetsStorage not configured. Configure storage in Mastra instance.");
			datasetRecord = await datasetsStore.getDatasetById({
				id: datasetId,
				filters: config.filters
			});
			if (!datasetRecord) throw new MastraError({
				id: "DATASET_NOT_FOUND",
				text: `Dataset not found: ${datasetId}`,
				domain: "STORAGE",
				category: "USER"
			});
			datasetVersion = version ?? datasetRecord.version;
			const versionItems = await datasetsStore.getItemsByVersion({
				datasetId,
				version: datasetVersion
			});
			if (versionItems.length === 0) throw new MastraError({
				id: "EXPERIMENT_NO_ITEMS",
				text: `No items in dataset ${datasetId} at version ${datasetVersion}`,
				domain: "STORAGE",
				category: "USER"
			});
			items = versionItems.map((v) => ({
				id: v.id,
				datasetVersion: v.datasetVersion,
				input: v.input,
				groundTruth: v.groundTruth,
				requestContext: v.requestContext,
				metadata: v.metadata,
				toolMocks: v.toolMocks,
				unmockedToolPolicy: v.unmockedToolPolicy,
				scorerIds: v.scorerIds
			}));
		} else throw new Error("No data source: provide datasetId or data");
	} catch (err) {
		await markFailedOnSetupError(err);
		throw err;
	}
	let execFn;
	try {
		if (config.task) {
			const taskFn = config.task;
			execFn = async (item, itemSignal) => {
				try {
					return {
						output: await taskFn({
							input: item.input,
							mastra,
							groundTruth: item.groundTruth,
							metadata: item.metadata,
							signal: itemSignal
						}),
						error: null,
						traceId: null
					};
				} catch (err) {
					return {
						output: null,
						error: {
							message: err instanceof Error ? err.message : String(err),
							stack: err instanceof Error ? err.stack : void 0
						},
						traceId: null
					};
				}
			};
		} else if (targetType && targetId) {
			const resolved = await resolveTarget(mastra, targetType, targetId, agentVersion);
			if (!resolved) throw new Error(`Target not found: ${targetType}/${targetId}`);
			const { target } = resolved;
			execFn = (item, itemSignal) => {
				const mergedRequestContext = globalRequestContext || item.requestContext ? {
					...globalRequestContext,
					...item.requestContext
				} : void 0;
				return executeTarget(target, targetType, item, {
					signal: itemSignal,
					requestContext: mergedRequestContext,
					experimentId,
					versions,
					toolMocks: targetType === "agent" ? item.toolMocks : void 0,
					unmockedToolPolicy: targetType === "agent" ? item.unmockedToolPolicy ?? config.unmockedToolPolicy ?? "allow" : void 0
				});
			};
		} else throw new Error("No task: provide targetType+targetId or task");
	} catch (err) {
		await markFailedOnSetupError(err);
		throw err;
	}
	const itemsWithToolMocks = items.filter((item) => item.toolMocks?.length).length;
	if (targetType !== "agent" && itemsWithToolMocks > 0) mastra.getLogger()?.warn(`Experiment target is "${config.task ? "task" : targetType}" but ${itemsWithToolMocks} of ${items.length} dataset items declare toolMocks. Tool mocks only apply to agent targets and will be ignored.`);
	const hasRunLevelScorers = scorerInput !== void 0;
	let stepsConfigInput;
	let flatScorerInput;
	if (scorerInput !== void 0) if (Array.isArray(scorerInput)) flatScorerInput = scorerInput;
	else {
		flatScorerInput = [];
		if ("agent" in scorerInput && scorerInput.agent) flatScorerInput.push(...scorerInput.agent);
		if ("workflow" in scorerInput && scorerInput.workflow) flatScorerInput.push(...scorerInput.workflow);
		if ("trajectory" in scorerInput && scorerInput.trajectory) flatScorerInput.push(...scorerInput.trajectory);
		if ("steps" in scorerInput && scorerInput.steps) stepsConfigInput = scorerInput.steps;
	}
	if (flatScorerInput?.length) {
		const seen = /* @__PURE__ */ new Set();
		flatScorerInput = flatScorerInput.filter((entry) => {
			if (typeof entry !== "string") return true;
			if (seen.has(entry)) return false;
			seen.add(entry);
			return true;
		});
	}
	const runLevelScorers = hasRunLevelScorers ? resolveScorers(mastra, flatScorerInput) : [];
	const runLevelStepScorers = hasRunLevelScorers ? resolveStepScorers(mastra, stepsConfigInput) : {};
	const resolveItemScorers = createItemScorerResolver(mastra);
	const datasetScorers = !hasRunLevelScorers && items.some((item) => item.scorerIds === void 0) ? resolveScorers(mastra, [...new Set(datasetRecord?.scorerIds ?? [])]) : [];
	if (experimentsStore) {
		if (!providedExperimentId) await experimentsStore.createExperiment({
			id: experimentId,
			name,
			description,
			metadata,
			datasetId: datasetId ?? null,
			datasetVersion,
			targetType: targetType ?? "agent",
			targetId: targetId ?? "inline",
			totalItems: items.length,
			agentVersion,
			organizationId: datasetRecord?.organizationId ?? null,
			projectId: datasetRecord?.projectId ?? null
		});
		await experimentsStore.updateExperiment({
			id: experimentId,
			status: "running",
			totalItems: items.length,
			startedAt
		});
	}
	if (eventDispatcher) try {
		await eventDispatcher.emit({
			type: "experiment.run.started",
			experimentId,
			target: eventTarget,
			status: "running",
			datasetId: datasetRecord?.id ?? null,
			datasetVersion,
			totalItems: items.length
		});
	} catch (observerError) {