@mastra/core/dist/evals/scoreTraces/index.cjs

Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
const require_rolldown_runtime = require("../../rolldown-runtime-uwYp4b74.cjs");
const require_error = require("../../error-B-e62x-A.cjs");
const require_utils = require("../../utils-CNiGU0Uf.cjs");
require("../../tracing-BUrUJwCM.cjs");
const require_agent = require("../../agent-DCD4MApC.cjs");
const require_constants = require("../../constants-CHm1eNBE.cjs");
let zod_v4 = require("zod/v4");
let p_map = require("p-map");
p_map = require_rolldown_runtime.__toESM(p_map, 1);
//#region src/evals/scoreTraces/scoreTraces.ts
async function scoreTraces({ scorerId, targets, mastra }) {
	const workflow = mastra.__getInternalWorkflow("__batch-scoring-traces");
	try {
		await (await workflow.createRun()).start({ inputData: {
			targets,
			scorerId
		} });
	} catch (error) {
		const mastraError = new require_error.MastraError({
			category: "SYSTEM",
			domain: "SCORER",
			id: "MASTRA_SCORER_FAILED_TO_RUN_TRACE_SCORING",
			details: {
				scorerId,
				targets: JSON.stringify(targets)
			}
		}, error);
		mastra.getLogger()?.trackException(mastraError);
	}
}
//#endregion
//#region src/evals/scoreTraces/utils.ts
function isSpanMessage(value) {
	return typeof value === "object" && value !== null && "role" in value && typeof value.role === "string" && "content" in value;
}
function hasMessagesArray(value) {
	return typeof value === "object" && value !== null && "messages" in value && Array.isArray(value.messages);
}
function hasTextProperty(value) {
	return typeof value === "object" && value !== null && "text" in value;
}
/**
* Build a hierarchical span tree with efficient lookup maps
*/
function buildSpanTree(spans) {
	const spanMap = /* @__PURE__ */ new Map();
	const childrenMap = /* @__PURE__ */ new Map();
	const rootSpans = [];
	for (const span of spans) spanMap.set(span.spanId, span);
	for (const span of spans) if (span.parentSpanId == null) rootSpans.push(span);
	else {
		const siblings = childrenMap.get(span.parentSpanId) || [];
		siblings.push(span);
		childrenMap.set(span.parentSpanId, siblings);
	}
	for (const children of childrenMap.values()) children.sort((a, b) => new Date(a.startedAt).getTime() - new Date(b.startedAt).getTime());
	rootSpans.sort((a, b) => new Date(a.startedAt).getTime() - new Date(b.startedAt).getTime());
	return {
		spanMap,
		childrenMap,
		rootSpans
	};
}
/**
* Extract children spans of a specific type
*/
function getChildrenOfType(spanTree, parentSpanId, spanType) {
	return (spanTree.childrenMap.get(parentSpanId) || []).filter((span) => span.spanType === spanType);
}
/**
* Normalize message content to string format
* For arrays with multiple text parts, returns only the last text part (AI SDK convention)
*/
function normalizeMessageContent(content) {
	if (typeof content === "string") return content;
	const textParts = content.filter((part) => part.type === "text");
	return textParts.length > 0 ? textParts[textParts.length - 1]?.text || "" : "";
}
/**
* Create MastraDBMessage directly from span message data
*/
function createMastraDBMessage(message, createdAt, id = "") {
	const contentText = normalizeMessageContent(message.content);
	return {
		id,
		role: message.role,
		content: {
			format: 2,
			parts: [{
				type: "text",
				text: contentText
			}],
			content: contentText
		},
		createdAt: new Date(createdAt)
	};
}
/**
* Extract input messages from agent run span
*/
function extractInputMessages(agentSpan) {
	const input = agentSpan.input;
	if (typeof input === "string") return [createMastraDBMessage({
		role: "user",
		content: input
	}, agentSpan.startedAt)];
	if (Array.isArray(input)) return input.filter(isSpanMessage).map((msg) => createMastraDBMessage(msg, agentSpan.startedAt));
	if (hasMessagesArray(input)) return input.messages.filter(isSpanMessage).map((msg) => createMastraDBMessage(msg, agentSpan.startedAt));
	return [];
}
/**
* Extract system messages from LLM span
*/
function extractSystemMessages(llmSpan) {
	const input = llmSpan.input;
	if (!hasMessagesArray(input)) return [];
	return input.messages.filter((msg) => isSpanMessage(msg) && msg.role === "system").map((msg) => ({
		role: "system",
		content: normalizeMessageContent(msg.content)
	}));
}
/**
* Extract conversation history (remembered messages) from LLM span
* Excludes system messages and the current input message
*/
function extractRememberedMessages(llmSpan, currentInputContent) {
	const input = llmSpan.input;
	if (!hasMessagesArray(input)) return [];
	return input.messages.filter(isSpanMessage).filter((msg) => msg.role !== "system").filter((msg) => normalizeMessageContent(msg.content) !== currentInputContent).map((msg) => createMastraDBMessage(msg, llmSpan.startedAt));
}
/**
* Reconstruct tool invocations from tool call spans
*/
function reconstructToolInvocations(spanTree, parentSpanId) {
	return getChildrenOfType(spanTree, parentSpanId, "tool_call").map((toolSpan) => ({
		toolCallId: toolSpan.spanId,
		toolName: toolSpan.entityName ?? toolSpan.entityId ?? "unknown",
		toolId: toolSpan.entityId,
		args: toolSpan.input || {},
		result: toolSpan.output || {},
		state: "result"
	}));
}
/**
* Validate trace structure and throw descriptive errors
*/
function validateTrace(trace) {
	if (!trace) throw new Error("Trace is null or undefined");
	if (!trace.spans || !Array.isArray(trace.spans)) throw new Error("Trace must have a spans array");
	if (trace.spans.length === 0) throw new Error("Trace has no spans");
	const spanIds = new Set(trace.spans.map((span) => span.spanId));
	for (const span of trace.spans) if (span.parentSpanId && !spanIds.has(span.parentSpanId)) throw new Error(`Span ${span.spanId} references non-existent parent ${span.parentSpanId}`);
}
/**
* Find the most recent model span that contains conversation history
*/
function findPrimaryLLMSpan(spanTree, rootAgentSpan) {
	const directLLMSpans = getChildrenOfType(spanTree, rootAgentSpan.spanId, "model_generation");
	if (directLLMSpans.length > 0) return directLLMSpans[0];
	throw new Error("No model generation span found in trace");
}
/**
* Extract common trace validation and span tree building logic
*/
function prepareTraceForTransformation(trace) {
	validateTrace(trace);
	const spanTree = buildSpanTree(trace.spans);
	const rootAgentSpan = spanTree.rootSpans.find((span) => span.spanType === "agent_run");
	if (!rootAgentSpan) throw new Error("No root agent_run span found in trace");
	return {
		spanTree,
		rootAgentSpan
	};
}
function transformTraceToScorerInputAndOutput(trace) {
	const { spanTree, rootAgentSpan } = prepareTraceForTransformation(trace);
	if (!rootAgentSpan.output) throw new Error("Root agent span has no output");
	const primaryLLMSpan = findPrimaryLLMSpan(spanTree, rootAgentSpan);
	const inputMessages = extractInputMessages(rootAgentSpan);
	const systemMessages = extractSystemMessages(primaryLLMSpan);
	const input = {
		inputMessages,
		rememberedMessages: extractRememberedMessages(primaryLLMSpan, inputMessages[0]?.content.content || ""),
		systemMessages,
		taggedSystemMessages: {}
	};
	const toolInvocations = reconstructToolInvocations(spanTree, rootAgentSpan.spanId);
	const responseText = hasTextProperty(rootAgentSpan.output) ? rootAgentSpan.output.text ?? "" : "";
	const parts = [];
	for (const toolInvocation of toolInvocations) parts.push({
		type: "tool-invocation",
		toolInvocation
	});
	if (responseText.trim()) parts.push({
		type: "text",
		text: responseText
	});
	return {
		input,
		output: [{
			id: "",
			role: "assistant",
			content: {
				format: 2,
				parts,
				content: responseText,
				toolInvocations
			},
			createdAt: new Date(rootAgentSpan.endedAt || rootAgentSpan.startedAt)
		}]
	};
}
//#endregion
//#region src/evals/scoreTraces/scoreTracesWorkflow.ts
const getTraceStep = require_agent.createStep({
	id: "__process-trace-scoring",
	inputSchema: zod_v4.z.object({
		targets: zod_v4.z.array(zod_v4.z.object({
			traceId: zod_v4.z.string(),
			spanId: zod_v4.z.string().optional()
		})),
		scorerId: zod_v4.z.string()
	}),
	outputSchema: zod_v4.z.any(),
	execute: async ({ inputData, mastra }) => {
		const logger = mastra.getLogger();
		if (!logger) console.warn("[scoreTracesWorkflow] Logger not initialized: no debug or error logs will be recorded for scoring traces.");
		const storage = mastra.getStorage();
		if (!storage) {
			const mastraError = new require_error.MastraError({
				id: "MASTRA_STORAGE_NOT_FOUND_FOR_TRACE_SCORING",
				domain: require_error.ErrorDomain.STORAGE,
				category: require_error.ErrorCategory.SYSTEM,
				text: "Storage not found for trace scoring",
				details: { scorerId: inputData.scorerId }
			});
			logger?.trackException(mastraError);
			return;
		}
		let scorer;
		try {
			scorer = mastra.getScorerById(inputData.scorerId);
		} catch (error) {
			const mastraError = new require_error.MastraError({
				id: "MASTRA_SCORER_NOT_FOUND_FOR_TRACE_SCORING",
				domain: require_error.ErrorDomain.SCORER,
				category: require_error.ErrorCategory.SYSTEM,
				text: `Scorer not found for trace scoring`,
				details: { scorerId: inputData.scorerId }
			}, error);
			logger?.trackException(mastraError);
			return;
		}
		await (0, p_map.default)(inputData.targets, async (target) => {
			try {
				await scoreTrace({
					storage,
					scorer,
					target
				});
			} catch (error) {
				const mastraError = new require_error.MastraError({
					id: "MASTRA_SCORER_FAILED_TO_RUN_SCORER_ON_TRACE",
					domain: require_error.ErrorDomain.SCORER,
					category: require_error.ErrorCategory.SYSTEM,
					details: {
						scorerId: scorer.id,
						spanId: target.spanId || "",
						traceId: target.traceId
					}
				}, error);
				logger?.trackException(mastraError);
			}
		}, { concurrency: 3 });
	}
});
/**
* Derive score tenancy from a span. On spans, `resourceId` carries the project
* scope, so it maps to the score's `projectId` field.
*/
function getSpanTenancy(span) {
	const tenancy = {};
	if (span.organizationId) tenancy.organizationId = span.organizationId;
	if (span.resourceId) tenancy.projectId = span.resourceId;
	return tenancy;
}
function isScoreTraceReferenceTarget(target) {
	return "traceId" in target;
}
function resolveTargetSpan({ trace, spanId }) {
	const span = spanId ? trace.spans.find((candidateSpan) => candidateSpan.spanId === spanId) : trace.spans.find((candidateSpan) => candidateSpan.parentSpanId === null);
	if (!span) throw new Error(`Span not found for scoring, traceId: ${trace.traceId}, spanId: ${spanId ?? "Not provided"}`);
	return span;
}
/** Resolve the target span for a trace/target pair. */
async function resolveTraceAndSpan({ storage, target }) {
	if (!isScoreTraceReferenceTarget(target)) return {
		trace: target.trace,
		span: resolveTargetSpan({
			trace: target.trace,
			spanId: target.spanId
		})
	};
	const observabilityStore = await storage.getStore("observability");
	if (!observabilityStore) throw new require_error.MastraError({
		id: "MASTRA_OBSERVABILITY_STORAGE_NOT_AVAILABLE",
		domain: require_error.ErrorDomain.STORAGE,
		category: require_error.ErrorCategory.SYSTEM,
		text: "Observability storage domain is not available"
	});
	const trace = await observabilityStore.getTrace({ traceId: target.traceId });
	if (!trace) throw new Error(`Trace not found for scoring, traceId: ${target.traceId}`);
	return {
		trace,
		span: resolveTargetSpan({
			trace,
			spanId: target.spanId
		})
	};
}
/**
* Run a scorer against an already-resolved trace + span.
*
* Span tenancy (`organizationId`, `resourceId` → `projectId`) is threaded into
* the scorer run so any score the scorer emits is correctly multi-tenant.
*/
async function runScorerForTrace({ scorer, trace, span }) {
	const tenancy = getSpanTenancy(span);
	const scorerRun = buildScorerRun({
		scorerType: scorer.type === "agent" ? "agent" : void 0,
		trace,
		targetSpan: span
	});
	return scorer.run({
		...scorerRun,
		scoreSource: "trace",
		targetScope: "span",
		targetEntityType: require_utils.getEntityTypeForSpan(span),
		targetTraceId: trace.traceId,
		targetSpanId: span.spanId,
		targetCorrelationContext: {
			traceId: trace.traceId,
			spanId: span.spanId,
			...tenancy.organizationId ? { organizationId: tenancy.organizationId } : {},
			...tenancy.projectId ? { resourceId: tenancy.projectId } : {}
		},
		targetMetadata: {
			...tenancy.organizationId ? { organizationId: tenancy.organizationId } : {},
			...tenancy.projectId ? { projectId: tenancy.projectId } : {}
		}
	});
}
/**
* Resolve a trace/span target, run the scorer, and persist the resulting score.
*/
async function scoreTrace({ storage, scorer, target, batchId, datasetId, datasetItemId }) {
	const { trace, span } = await resolveTraceAndSpan({
		storage,
		target
	});
	const tenancy = getSpanTenancy(span);
	const savedScoreRecord = await validateAndSaveScore({
		storage,
		scorerResult: {
			...await runScorerForTrace({
				scorer,
				trace,
				span
			}),
			scorer: {
				id: scorer.id,
				name: scorer.name || scorer.id,
				description: scorer.description,
				hasJudge: !!scorer.judge
			},
			traceId: trace.traceId,
			spanId: span.spanId,
			entityId: span.entityId || span.entityName || "unknown",
			entityType: span.spanType,
			entity: {
				traceId: span.traceId,
				spanId: span.spanId
			},
			source: "TEST",
			scorerId: scorer.id,
			...tenancy.organizationId ? { organizationId: tenancy.organizationId } : {},
			...tenancy.projectId ? { projectId: tenancy.projectId } : {},
			...batchId ? { batchId } : {},
			...datasetId ? { datasetId } : {},
			...datasetItemId ? { datasetItemId } : {}
		}
	});
	await attachScoreToSpan({
		storage,
		span,
		scoreRecord: savedScoreRecord
	});
	return savedScoreRecord;
}
function toBatchResultError(error) {
	if (error instanceof Error) return error;
	return new Error(typeof error === "string" ? error : "Unknown scoreTraceBatch error");
}
async function scoreTraceBatch({ storage, scorer, targets, batchId, datasetId, concurrency = 3 }) {
	const results = await (0, p_map.default)(targets, async (target, index) => {
		try {
			const score = await scoreTrace({
				storage,
				scorer,
				target,
				batchId,
				datasetId,
				datasetItemId: target.datasetItemId
			});
			const spanId = score.spanId ?? target.spanId;
			if (!spanId) throw new Error(`Persisted score is missing spanId for traceId: ${target.traceId}`);
			return {
				ok: true,
				index,
				traceId: score.traceId ?? target.traceId,
				spanId,
				...target.datasetItemId ? { datasetItemId: target.datasetItemId } : {},
				score
			};
		} catch (error) {
			return {
				ok: false,
				index,
				traceId: target.traceId,
				...target.spanId ? { spanId: target.spanId } : {},
				...target.datasetItemId ? { datasetItemId: target.datasetItemId } : {},
				error: toBatchResultError(error)
			};
		}
	}, { concurrency });
	const scoredCount = results.filter((result) => result.ok).length;
	return {
		...batchId ? { batchId } : {},
		...datasetId ? { datasetId } : {},
		scoredCount,
		failedCount: results.length - scoredCount,
		results
	};
}
/**
* @deprecated Legacy scores-store path. New score emission should use `mastra.observability.addScore()`.
*/
async function validateAndSaveScore({ storage, scorerResult }) {
	const scoresStore = await storage.getStore("scores");
	if (!scoresStore) throw new require_error.MastraError({
		id: "MASTRA_SCORES_STORAGE_NOT_AVAILABLE",
		domain: require_error.ErrorDomain.STORAGE,
		category: require_error.ErrorCategory.SYSTEM,
		text: "Scores storage domain is not available"
	});
	const payloadToSave = require_constants.saveScorePayloadSchema.parse(scorerResult);
	return (await scoresStore.saveScore(payloadToSave)).score;
}
function buildScorerRun({ scorerType, trace, targetSpan }) {
	if (scorerType === "agent") {
		const { input, output } = transformTraceToScorerInputAndOutput(trace);
		return {
			input,
			output
		};
	}
	return {
		input: targetSpan.input,
		output: targetSpan.output
	};
}
/**
* @deprecated Legacy score-attach path. New score emission should use `mastra.observability.addScore()`
* which autmatically attach scores to spans.
*/
async function attachScoreToSpan({ storage, span, scoreRecord }) {
	const observabilityStore = await storage.getStore("observability");
	if (!observabilityStore) throw new require_error.MastraError({
		id: "MASTRA_OBSERVABILITY_STORAGE_NOT_AVAILABLE",
		domain: require_error.ErrorDomain.STORAGE,
		category: require_error.ErrorCategory.SYSTEM,
		text: "Observability storage domain is not available"
	});
	try {
		const existingLinks = span.links || [];
		const link = {
			type: "score",
			scoreId: scoreRecord.id,
			scorerId: scoreRecord.scorerId ?? scoreRecord.scorer?.id,
			score: scoreRecord.score,
			createdAt: scoreRecord.createdAt
		};
		await observabilityStore.updateSpan({
			spanId: span.spanId,
			traceId: span.traceId,
			updates: { links: [...existingLinks, link] }
		});
	} catch {}
}
const scoreTracesWorkflow = require_agent.createWorkflow$1({
	id: "__batch-scoring-traces",
	inputSchema: zod_v4.z.object({
		targets: zod_v4.z.array(zod_v4.z.object({
			traceId: zod_v4.z.string(),
			spanId: zod_v4.z.string().optional()
		})),
		scorerId: zod_v4.z.string()
	}),
	outputSchema: zod_v4.z.any(),
	steps: [getTraceStep],
	options: {
		validateInputs: false,
		tracingPolicy: { internal: 1 }
	}
});
scoreTracesWorkflow.then(getTraceStep).commit();
//#endregion
exports.scoreTrace = scoreTrace;
exports.scoreTraceBatch = scoreTraceBatch;
exports.scoreTraces = scoreTraces;
exports.scoreTracesWorkflow = scoreTracesWorkflow;

//# sourceMappingURL=index.cjs.map