@mastra/core
Version:
518 lines (517 loc) • 16.9 kB
JavaScript
import { i as MastraError, n as ErrorDomain, t as ErrorCategory } from "../../error-MjDSls8S.js";
import { i as getEntityTypeForSpan } from "../../utils-DxsDNzD2.js";
import "../../tracing-Bm0k4FBA.js";
import { an as createWorkflow, in as createStep } from "../../agent-Dj30gJa3.js";
import { _n as saveScorePayloadSchema } from "../../constants-BfpAlX25.js";
import { z } from "zod/v4";
import pMap from "p-map";
//#region src/evals/scoreTraces/scoreTraces.ts
async function scoreTraces({ scorerId, targets, mastra }) {
const workflow = mastra.__getInternalWorkflow("__batch-scoring-traces");
try {
await (await workflow.createRun()).start({ inputData: {
targets,
scorerId
} });
} catch (error) {
const mastraError = new MastraError({
category: "SYSTEM",
domain: "SCORER",
id: "MASTRA_SCORER_FAILED_TO_RUN_TRACE_SCORING",
details: {
scorerId,
targets: JSON.stringify(targets)
}
}, error);
mastra.getLogger()?.trackException(mastraError);
}
}
//#endregion
//#region src/evals/scoreTraces/utils.ts
function isSpanMessage(value) {
return typeof value === "object" && value !== null && "role" in value && typeof value.role === "string" && "content" in value;
}
function hasMessagesArray(value) {
return typeof value === "object" && value !== null && "messages" in value && Array.isArray(value.messages);
}
function hasTextProperty(value) {
return typeof value === "object" && value !== null && "text" in value;
}
/**
* Build a hierarchical span tree with efficient lookup maps
*/
function buildSpanTree(spans) {
const spanMap = /* @__PURE__ */ new Map();
const childrenMap = /* @__PURE__ */ new Map();
const rootSpans = [];
for (const span of spans) spanMap.set(span.spanId, span);
for (const span of spans) if (span.parentSpanId == null) rootSpans.push(span);
else {
const siblings = childrenMap.get(span.parentSpanId) || [];
siblings.push(span);
childrenMap.set(span.parentSpanId, siblings);
}
for (const children of childrenMap.values()) children.sort((a, b) => new Date(a.startedAt).getTime() - new Date(b.startedAt).getTime());
rootSpans.sort((a, b) => new Date(a.startedAt).getTime() - new Date(b.startedAt).getTime());
return {
spanMap,
childrenMap,
rootSpans
};
}
/**
* Extract children spans of a specific type
*/
function getChildrenOfType(spanTree, parentSpanId, spanType) {
return (spanTree.childrenMap.get(parentSpanId) || []).filter((span) => span.spanType === spanType);
}
/**
* Normalize message content to string format
* For arrays with multiple text parts, returns only the last text part (AI SDK convention)
*/
function normalizeMessageContent(content) {
if (typeof content === "string") return content;
const textParts = content.filter((part) => part.type === "text");
return textParts.length > 0 ? textParts[textParts.length - 1]?.text || "" : "";
}
/**
* Create MastraDBMessage directly from span message data
*/
function createMastraDBMessage(message, createdAt, id = "") {
const contentText = normalizeMessageContent(message.content);
return {
id,
role: message.role,
content: {
format: 2,
parts: [{
type: "text",
text: contentText
}],
content: contentText
},
createdAt: new Date(createdAt)
};
}
/**
* Extract input messages from agent run span
*/
function extractInputMessages(agentSpan) {
const input = agentSpan.input;
if (typeof input === "string") return [createMastraDBMessage({
role: "user",
content: input
}, agentSpan.startedAt)];
if (Array.isArray(input)) return input.filter(isSpanMessage).map((msg) => createMastraDBMessage(msg, agentSpan.startedAt));
if (hasMessagesArray(input)) return input.messages.filter(isSpanMessage).map((msg) => createMastraDBMessage(msg, agentSpan.startedAt));
return [];
}
/**
* Extract system messages from LLM span
*/
function extractSystemMessages(llmSpan) {
const input = llmSpan.input;
if (!hasMessagesArray(input)) return [];
return input.messages.filter((msg) => isSpanMessage(msg) && msg.role === "system").map((msg) => ({
role: "system",
content: normalizeMessageContent(msg.content)
}));
}
/**
* Extract conversation history (remembered messages) from LLM span
* Excludes system messages and the current input message
*/
function extractRememberedMessages(llmSpan, currentInputContent) {
const input = llmSpan.input;
if (!hasMessagesArray(input)) return [];
return input.messages.filter(isSpanMessage).filter((msg) => msg.role !== "system").filter((msg) => normalizeMessageContent(msg.content) !== currentInputContent).map((msg) => createMastraDBMessage(msg, llmSpan.startedAt));
}
/**
* Reconstruct tool invocations from tool call spans
*/
function reconstructToolInvocations(spanTree, parentSpanId) {
return getChildrenOfType(spanTree, parentSpanId, "tool_call").map((toolSpan) => ({
toolCallId: toolSpan.spanId,
toolName: toolSpan.entityName ?? toolSpan.entityId ?? "unknown",
toolId: toolSpan.entityId,
args: toolSpan.input || {},
result: toolSpan.output || {},
state: "result"
}));
}
/**
* Validate trace structure and throw descriptive errors
*/
function validateTrace(trace) {
if (!trace) throw new Error("Trace is null or undefined");
if (!trace.spans || !Array.isArray(trace.spans)) throw new Error("Trace must have a spans array");
if (trace.spans.length === 0) throw new Error("Trace has no spans");
const spanIds = new Set(trace.spans.map((span) => span.spanId));
for (const span of trace.spans) if (span.parentSpanId && !spanIds.has(span.parentSpanId)) throw new Error(`Span ${span.spanId} references non-existent parent ${span.parentSpanId}`);
}
/**
* Find the most recent model span that contains conversation history
*/
function findPrimaryLLMSpan(spanTree, rootAgentSpan) {
const directLLMSpans = getChildrenOfType(spanTree, rootAgentSpan.spanId, "model_generation");
if (directLLMSpans.length > 0) return directLLMSpans[0];
throw new Error("No model generation span found in trace");
}
/**
* Extract common trace validation and span tree building logic
*/
function prepareTraceForTransformation(trace) {
validateTrace(trace);
const spanTree = buildSpanTree(trace.spans);
const rootAgentSpan = spanTree.rootSpans.find((span) => span.spanType === "agent_run");
if (!rootAgentSpan) throw new Error("No root agent_run span found in trace");
return {
spanTree,
rootAgentSpan
};
}
function transformTraceToScorerInputAndOutput(trace) {
const { spanTree, rootAgentSpan } = prepareTraceForTransformation(trace);
if (!rootAgentSpan.output) throw new Error("Root agent span has no output");
const primaryLLMSpan = findPrimaryLLMSpan(spanTree, rootAgentSpan);
const inputMessages = extractInputMessages(rootAgentSpan);
const systemMessages = extractSystemMessages(primaryLLMSpan);
const input = {
inputMessages,
rememberedMessages: extractRememberedMessages(primaryLLMSpan, inputMessages[0]?.content.content || ""),
systemMessages,
taggedSystemMessages: {}
};
const toolInvocations = reconstructToolInvocations(spanTree, rootAgentSpan.spanId);
const responseText = hasTextProperty(rootAgentSpan.output) ? rootAgentSpan.output.text ?? "" : "";
const parts = [];
for (const toolInvocation of toolInvocations) parts.push({
type: "tool-invocation",
toolInvocation
});
if (responseText.trim()) parts.push({
type: "text",
text: responseText
});
return {
input,
output: [{
id: "",
role: "assistant",
content: {
format: 2,
parts,
content: responseText,
toolInvocations
},
createdAt: new Date(rootAgentSpan.endedAt || rootAgentSpan.startedAt)
}]
};
}
//#endregion
//#region src/evals/scoreTraces/scoreTracesWorkflow.ts
const getTraceStep = createStep({
id: "__process-trace-scoring",
inputSchema: z.object({
targets: z.array(z.object({
traceId: z.string(),
spanId: z.string().optional()
})),
scorerId: z.string()
}),
outputSchema: z.any(),
execute: async ({ inputData, mastra }) => {
const logger = mastra.getLogger();
if (!logger) console.warn("[scoreTracesWorkflow] Logger not initialized: no debug or error logs will be recorded for scoring traces.");
const storage = mastra.getStorage();
if (!storage) {
const mastraError = new MastraError({
id: "MASTRA_STORAGE_NOT_FOUND_FOR_TRACE_SCORING",
domain: ErrorDomain.STORAGE,
category: ErrorCategory.SYSTEM,
text: "Storage not found for trace scoring",
details: { scorerId: inputData.scorerId }
});
logger?.trackException(mastraError);
return;
}
let scorer;
try {
scorer = mastra.getScorerById(inputData.scorerId);
} catch (error) {
const mastraError = new MastraError({
id: "MASTRA_SCORER_NOT_FOUND_FOR_TRACE_SCORING",
domain: ErrorDomain.SCORER,
category: ErrorCategory.SYSTEM,
text: `Scorer not found for trace scoring`,
details: { scorerId: inputData.scorerId }
}, error);
logger?.trackException(mastraError);
return;
}
await pMap(inputData.targets, async (target) => {
try {
await scoreTrace({
storage,
scorer,
target
});
} catch (error) {
const mastraError = new MastraError({
id: "MASTRA_SCORER_FAILED_TO_RUN_SCORER_ON_TRACE",
domain: ErrorDomain.SCORER,
category: ErrorCategory.SYSTEM,
details: {
scorerId: scorer.id,
spanId: target.spanId || "",
traceId: target.traceId
}
}, error);
logger?.trackException(mastraError);
}
}, { concurrency: 3 });
}
});
/**
* Derive score tenancy from a span. On spans, `resourceId` carries the project
* scope, so it maps to the score's `projectId` field.
*/
function getSpanTenancy(span) {
const tenancy = {};
if (span.organizationId) tenancy.organizationId = span.organizationId;
if (span.resourceId) tenancy.projectId = span.resourceId;
return tenancy;
}
function isScoreTraceReferenceTarget(target) {
return "traceId" in target;
}
function resolveTargetSpan({ trace, spanId }) {
const span = spanId ? trace.spans.find((candidateSpan) => candidateSpan.spanId === spanId) : trace.spans.find((candidateSpan) => candidateSpan.parentSpanId === null);
if (!span) throw new Error(`Span not found for scoring, traceId: ${trace.traceId}, spanId: ${spanId ?? "Not provided"}`);
return span;
}
/** Resolve the target span for a trace/target pair. */
async function resolveTraceAndSpan({ storage, target }) {
if (!isScoreTraceReferenceTarget(target)) return {
trace: target.trace,
span: resolveTargetSpan({
trace: target.trace,
spanId: target.spanId
})
};
const observabilityStore = await storage.getStore("observability");
if (!observabilityStore) throw new MastraError({
id: "MASTRA_OBSERVABILITY_STORAGE_NOT_AVAILABLE",
domain: ErrorDomain.STORAGE,
category: ErrorCategory.SYSTEM,
text: "Observability storage domain is not available"
});
const trace = await observabilityStore.getTrace({ traceId: target.traceId });
if (!trace) throw new Error(`Trace not found for scoring, traceId: ${target.traceId}`);
return {
trace,
span: resolveTargetSpan({
trace,
spanId: target.spanId
})
};
}
/**
* Run a scorer against an already-resolved trace + span.
*
* Span tenancy (`organizationId`, `resourceId` → `projectId`) is threaded into
* the scorer run so any score the scorer emits is correctly multi-tenant.
*/
async function runScorerForTrace({ scorer, trace, span }) {
const tenancy = getSpanTenancy(span);
const scorerRun = buildScorerRun({
scorerType: scorer.type === "agent" ? "agent" : void 0,
trace,
targetSpan: span
});
return scorer.run({
...scorerRun,
scoreSource: "trace",
targetScope: "span",
targetEntityType: getEntityTypeForSpan(span),
targetTraceId: trace.traceId,
targetSpanId: span.spanId,
targetCorrelationContext: {
traceId: trace.traceId,
spanId: span.spanId,
...tenancy.organizationId ? { organizationId: tenancy.organizationId } : {},
...tenancy.projectId ? { resourceId: tenancy.projectId } : {}
},
targetMetadata: {
...tenancy.organizationId ? { organizationId: tenancy.organizationId } : {},
...tenancy.projectId ? { projectId: tenancy.projectId } : {}
}
});
}
/**
* Resolve a trace/span target, run the scorer, and persist the resulting score.
*/
async function scoreTrace({ storage, scorer, target, batchId, datasetId, datasetItemId }) {
const { trace, span } = await resolveTraceAndSpan({
storage,
target
});
const tenancy = getSpanTenancy(span);
const savedScoreRecord = await validateAndSaveScore({
storage,
scorerResult: {
...await runScorerForTrace({
scorer,
trace,
span
}),
scorer: {
id: scorer.id,
name: scorer.name || scorer.id,
description: scorer.description,
hasJudge: !!scorer.judge
},
traceId: trace.traceId,
spanId: span.spanId,
entityId: span.entityId || span.entityName || "unknown",
entityType: span.spanType,
entity: {
traceId: span.traceId,
spanId: span.spanId
},
source: "TEST",
scorerId: scorer.id,
...tenancy.organizationId ? { organizationId: tenancy.organizationId } : {},
...tenancy.projectId ? { projectId: tenancy.projectId } : {},
...batchId ? { batchId } : {},
...datasetId ? { datasetId } : {},
...datasetItemId ? { datasetItemId } : {}
}
});
await attachScoreToSpan({
storage,
span,
scoreRecord: savedScoreRecord
});
return savedScoreRecord;
}
function toBatchResultError(error) {
if (error instanceof Error) return error;
return new Error(typeof error === "string" ? error : "Unknown scoreTraceBatch error");
}
async function scoreTraceBatch({ storage, scorer, targets, batchId, datasetId, concurrency = 3 }) {
const results = await pMap(targets, async (target, index) => {
try {
const score = await scoreTrace({
storage,
scorer,
target,
batchId,
datasetId,
datasetItemId: target.datasetItemId
});
const spanId = score.spanId ?? target.spanId;
if (!spanId) throw new Error(`Persisted score is missing spanId for traceId: ${target.traceId}`);
return {
ok: true,
index,
traceId: score.traceId ?? target.traceId,
spanId,
...target.datasetItemId ? { datasetItemId: target.datasetItemId } : {},
score
};
} catch (error) {
return {
ok: false,
index,
traceId: target.traceId,
...target.spanId ? { spanId: target.spanId } : {},
...target.datasetItemId ? { datasetItemId: target.datasetItemId } : {},
error: toBatchResultError(error)
};
}
}, { concurrency });
const scoredCount = results.filter((result) => result.ok).length;
return {
...batchId ? { batchId } : {},
...datasetId ? { datasetId } : {},
scoredCount,
failedCount: results.length - scoredCount,
results
};
}
/**
* @deprecated Legacy scores-store path. New score emission should use `mastra.observability.addScore()`.
*/
async function validateAndSaveScore({ storage, scorerResult }) {
const scoresStore = await storage.getStore("scores");
if (!scoresStore) throw new MastraError({
id: "MASTRA_SCORES_STORAGE_NOT_AVAILABLE",
domain: ErrorDomain.STORAGE,
category: ErrorCategory.SYSTEM,
text: "Scores storage domain is not available"
});
const payloadToSave = saveScorePayloadSchema.parse(scorerResult);
return (await scoresStore.saveScore(payloadToSave)).score;
}
function buildScorerRun({ scorerType, trace, targetSpan }) {
if (scorerType === "agent") {
const { input, output } = transformTraceToScorerInputAndOutput(trace);
return {
input,
output
};
}
return {
input: targetSpan.input,
output: targetSpan.output
};
}
/**
* @deprecated Legacy score-attach path. New score emission should use `mastra.observability.addScore()`
* which autmatically attach scores to spans.
*/
async function attachScoreToSpan({ storage, span, scoreRecord }) {
const observabilityStore = await storage.getStore("observability");
if (!observabilityStore) throw new MastraError({
id: "MASTRA_OBSERVABILITY_STORAGE_NOT_AVAILABLE",
domain: ErrorDomain.STORAGE,
category: ErrorCategory.SYSTEM,
text: "Observability storage domain is not available"
});
try {
const existingLinks = span.links || [];
const link = {
type: "score",
scoreId: scoreRecord.id,
scorerId: scoreRecord.scorerId ?? scoreRecord.scorer?.id,
score: scoreRecord.score,
createdAt: scoreRecord.createdAt
};
await observabilityStore.updateSpan({
spanId: span.spanId,
traceId: span.traceId,
updates: { links: [...existingLinks, link] }
});
} catch {}
}
const scoreTracesWorkflow = createWorkflow({
id: "__batch-scoring-traces",
inputSchema: z.object({
targets: z.array(z.object({
traceId: z.string(),
spanId: z.string().optional()
})),
scorerId: z.string()
}),
outputSchema: z.any(),
steps: [getTraceStep],
options: {
validateInputs: false,
tracingPolicy: { internal: 1 }
}
});
scoreTracesWorkflow.then(getTraceStep).commit();
//#endregion
export { scoreTrace, scoreTraceBatch, scoreTraces, scoreTracesWorkflow };
//# sourceMappingURL=index.js.map