@mastra/core
Version:
Mastra is a framework for building AI-powered applications and agents with a modern TypeScript stack.
874 lines (869 loc) • 29.8 kB
JavaScript
'use strict';
var chunkACQ5CVFF_cjs = require('./chunk-ACQ5CVFF.cjs');
var chunkXD7CRAEV_cjs = require('./chunk-XD7CRAEV.cjs');
var chunk2E7FPUYL_cjs = require('./chunk-2E7FPUYL.cjs');
var chunkG3JYQ2UI_cjs = require('./chunk-G3JYQ2UI.cjs');
var chunk4U7ZLI36_cjs = require('./chunk-4U7ZLI36.cjs');
// src/mastra/hooks.ts
function toScorerTargetEntityType(entityType) {
switch (entityType) {
case "AGENT":
return chunkG3JYQ2UI_cjs.EntityType.AGENT;
case "WORKFLOW":
return chunkG3JYQ2UI_cjs.EntityType.WORKFLOW_RUN;
default:
return void 0;
}
}
function createOnScorerHook(mastra) {
return async (hookData) => {
const storage = mastra.getStorage();
if (!storage) {
mastra.getLogger()?.warn("Storage not found, skipping score validation and saving");
return;
}
const entityId = hookData.entity.id;
const entityType = hookData.entityType;
const scorer = hookData.scorer;
const scorerId = scorer.id;
if (!scorerId) {
mastra.getLogger()?.warn("Scorer ID not found, skipping score validation and saving");
return;
}
try {
const scorerToUse = await findScorer(mastra, entityId, entityType, scorerId);
if (!scorerToUse) {
throw new chunk4U7ZLI36_cjs.MastraError({
id: "MASTRA_SCORER_NOT_FOUND",
domain: "MASTRA" /* MASTRA */,
category: "USER" /* USER */,
text: `Scorer with ID ${scorerId} not found`
});
}
let input = hookData.input;
let output = hookData.output;
const { structuredOutput, ...rest } = hookData;
const currentSpan = hookData.tracingContext?.currentSpan;
const traceId = currentSpan?.isValid ? currentSpan.traceId : void 0;
const spanId = currentSpan?.isValid ? currentSpan.id : void 0;
const targetCorrelationContext = currentSpan?.isValid ? currentSpan.getCorrelationContext?.() : void 0;
const targetMetadata = currentSpan?.isValid && currentSpan.metadata ? { ...currentSpan.metadata } : void 0;
const runResult = await scorerToUse.scorer.run({
...rest,
input,
output,
scoreSource: "live",
targetScope: "span",
targetEntityType: toScorerTargetEntityType(entityType),
targetTraceId: traceId,
targetSpanId: spanId,
targetCorrelationContext,
targetMetadata
});
const payload = {
...rest,
...runResult,
entityId,
scorerId,
spanId,
traceId,
scorer: {
...rest.scorer,
hasJudge: !!scorerToUse.scorer.judge
},
metadata: {
structuredOutput: !!structuredOutput
}
};
await validateAndSaveScore(storage, payload);
} catch (error) {
const mastraError = new chunk4U7ZLI36_cjs.MastraError(
{
id: "MASTRA_SCORER_FAILED_TO_RUN_HOOK",
domain: "SCORER" /* SCORER */,
category: "USER" /* USER */,
details: {
scorerId,
entityId,
entityType
}
},
error
);
mastra.getLogger()?.trackException(mastraError);
}
};
}
async function validateAndSaveScore(storage, payload) {
const scoresStore = await storage.getStore("scores");
if (!scoresStore) {
throw new chunk4U7ZLI36_cjs.MastraError({
id: "MASTRA_SCORES_STORAGE_NOT_AVAILABLE",
domain: "STORAGE" /* STORAGE */,
category: "SYSTEM" /* SYSTEM */,
text: "Scores storage domain is not available"
});
}
const payloadToSave = chunkXD7CRAEV_cjs.saveScorePayloadSchema.parse(payload);
await scoresStore.saveScore(payloadToSave);
}
async function findScorer(mastra, entityId, entityType, scorerId) {
let scorerToUse;
if (entityType === "AGENT") {
try {
const agent = mastra.getAgentById(entityId);
const scorers = await agent.listScorers();
for (const [_, scorer] of Object.entries(scorers)) {
if (scorer.scorer.id === scorerId) {
scorerToUse = scorer;
break;
}
}
} catch {
try {
const storedAgent = await mastra.getEditor()?.agent.getById(entityId) ?? null;
if (storedAgent) {
const scorers = await storedAgent.listScorers();
for (const [_, scorer] of Object.entries(scorers)) {
if (scorer.scorer.id === scorerId) {
scorerToUse = scorer;
break;
}
}
}
} catch {
}
}
} else if (entityType === "WORKFLOW") {
const scorers = await mastra.getWorkflowById(entityId).listScorers();
for (const [_, scorer] of Object.entries(scorers)) {
if (scorer.scorer.id === scorerId) {
scorerToUse = scorer;
break;
}
}
}
if (!scorerToUse) {
const mastraRegisteredScorer = mastra.getScorerById(scorerId);
scorerToUse = mastraRegisteredScorer ? { scorer: mastraRegisteredScorer } : void 0;
}
return scorerToUse;
}
// src/evals/run/scorerAccumulator.ts
var ScoreAccumulator = class {
flatScores = {};
workflowScores = {};
stepScores = {};
agentScores = {};
trajectoryScores = {};
addScores(scorerResults) {
const isWorkflowScores = "steps" in scorerResults || "workflow" in scorerResults;
const isAgentScores = "agent" in scorerResults;
const hasTrajectory = "trajectory" in scorerResults;
if (isWorkflowScores) {
this.addWorkflowScores(scorerResults);
} else if (isAgentScores || hasTrajectory) {
this.addAgentScores(scorerResults);
} else {
this.addFlatScores(scorerResults);
}
}
addFlatScores(scorerResults) {
for (const [scorerName, result] of Object.entries(scorerResults)) {
if (!this.flatScores[scorerName]) {
this.flatScores[scorerName] = [];
}
this.flatScores[scorerName].push(result.score);
}
}
addWorkflowScores(scorerResults) {
if ("workflow" in scorerResults && scorerResults.workflow) {
for (const [scorerName, result] of Object.entries(scorerResults.workflow)) {
if (!this.workflowScores[scorerName]) {
this.workflowScores[scorerName] = [];
}
this.workflowScores[scorerName].push(result.score);
}
}
if ("steps" in scorerResults && scorerResults.steps) {
for (const [stepId, stepResults] of Object.entries(scorerResults.steps)) {
if (!this.stepScores[stepId]) {
this.stepScores[stepId] = {};
}
for (const [scorerName, result] of Object.entries(stepResults)) {
if (!this.stepScores[stepId][scorerName]) {
this.stepScores[stepId][scorerName] = [];
}
this.stepScores[stepId][scorerName].push(result.score);
}
}
}
if ("trajectory" in scorerResults && scorerResults.trajectory) {
for (const [scorerName, result] of Object.entries(scorerResults.trajectory)) {
if (!this.trajectoryScores[scorerName]) {
this.trajectoryScores[scorerName] = [];
}
this.trajectoryScores[scorerName].push(result.score);
}
}
}
addAgentScores(scorerResults) {
if ("agent" in scorerResults && scorerResults.agent) {
for (const [scorerName, result] of Object.entries(scorerResults.agent)) {
if (!this.agentScores[scorerName]) {
this.agentScores[scorerName] = [];
}
this.agentScores[scorerName].push(result.score);
}
}
if ("trajectory" in scorerResults && scorerResults.trajectory) {
for (const [scorerName, result] of Object.entries(scorerResults.trajectory)) {
if (!this.trajectoryScores[scorerName]) {
this.trajectoryScores[scorerName] = [];
}
this.trajectoryScores[scorerName].push(result.score);
}
}
}
addStepScores(stepScorerResults) {
for (const [stepId, stepResults] of Object.entries(stepScorerResults)) {
if (!this.stepScores[stepId]) {
this.stepScores[stepId] = {};
}
for (const [scorerName, result] of Object.entries(stepResults)) {
if (!this.stepScores[stepId][scorerName]) {
this.stepScores[stepId][scorerName] = [];
}
this.stepScores[stepId][scorerName].push(result.score);
}
}
}
getAverageScores() {
const result = {};
for (const [scorerName, scoreArray] of Object.entries(this.flatScores)) {
result[scorerName] = this.getAverageScore(scoreArray);
}
if (Object.keys(this.workflowScores).length > 0) {
result.workflow = {};
for (const [scorerName, scoreArray] of Object.entries(this.workflowScores)) {
result.workflow[scorerName] = this.getAverageScore(scoreArray);
}
}
if (Object.keys(this.stepScores).length > 0) {
result.steps = {};
for (const [stepId, stepScorers] of Object.entries(this.stepScores)) {
result.steps[stepId] = {};
for (const [scorerName, scoreArray] of Object.entries(stepScorers)) {
result.steps[stepId][scorerName] = this.getAverageScore(scoreArray);
}
}
}
if (Object.keys(this.agentScores).length > 0) {
result.agent = {};
for (const [scorerName, scoreArray] of Object.entries(this.agentScores)) {
result.agent[scorerName] = this.getAverageScore(scoreArray);
}
}
if (Object.keys(this.trajectoryScores).length > 0) {
result.trajectory = {};
for (const [scorerName, scoreArray] of Object.entries(this.trajectoryScores)) {
result.trajectory[scorerName] = this.getAverageScore(scoreArray);
}
}
return result;
}
getAverageScore(scoreArray) {
if (scoreArray.length > 0) {
return scoreArray.reduce((a, b) => a + b, 0) / scoreArray.length;
} else {
return 0;
}
}
};
// src/evals/run/index.ts
async function runEvals(config) {
const { data, scorers, target, targetOptions, onItemComplete, concurrency = 1 } = config;
validateEvalsInputs(data, scorers, target);
let totalItems = 0;
const scoreAccumulator = new ScoreAccumulator();
const mastra = target.getMastraInstance?.() || target.mastra;
const storage = mastra?.getStorage();
const pMap = (await import('p-map')).default;
await pMap(
data,
async (item) => {
const targetResult = await executeTarget(target, item, targetOptions);
const scorerResults = await runScorers(scorers, targetResult, item, storage);
scoreAccumulator.addScores(scorerResults);
if (storage) {
await saveScoresToStorage({
storage,
scorerResults,
target,
item,
mastra
});
}
if (onItemComplete) {
await onItemComplete({
item,
targetResult,
scorerResults
});
}
totalItems++;
},
{ concurrency }
);
return {
scores: scoreAccumulator.getAverageScores(),
summary: {
totalItems
}
};
}
function isWorkflow(target) {
return target instanceof chunkACQ5CVFF_cjs.Workflow;
}
function isWorkflowScorerConfig(scorers) {
return typeof scorers === "object" && !Array.isArray(scorers) && ("workflow" in scorers || "steps" in scorers || "trajectory" in scorers && !("agent" in scorers));
}
function isAgentScorerConfig(scorers) {
return typeof scorers === "object" && !Array.isArray(scorers) && ("agent" in scorers || "trajectory" in scorers && !("workflow" in scorers) && !("steps" in scorers));
}
function validateEvalsInputs(data, scorers, target) {
if (data.length === 0) {
throw new chunk4U7ZLI36_cjs.MastraError({
domain: "SCORER",
id: "RUN_EXPERIMENT_FAILED_NO_DATA_PROVIDED",
category: "USER",
text: "Failed to run experiment: Data array is empty"
});
}
for (let i = 0; i < data.length; i++) {
const item = data[i];
if (!item || typeof item !== "object" || !("input" in item)) {
throw new chunk4U7ZLI36_cjs.MastraError({
domain: "SCORER",
id: "INVALID_DATA_ITEM",
category: "USER",
text: `Invalid data item at index ${i}: must have 'input' properties`
});
}
}
if (Array.isArray(scorers)) {
if (scorers.length === 0) {
throw new chunk4U7ZLI36_cjs.MastraError({
domain: "SCORER",
id: "NO_SCORERS_PROVIDED",
category: "USER",
text: "At least one scorer must be provided"
});
}
} else if (isWorkflow(target) && isWorkflowScorerConfig(scorers)) {
const hasScorers = scorers.workflow && scorers.workflow.length > 0 || scorers.steps && Object.keys(scorers.steps).length > 0 || scorers.trajectory && scorers.trajectory.length > 0;
if (!hasScorers) {
throw new chunk4U7ZLI36_cjs.MastraError({
domain: "SCORER",
id: "NO_SCORERS_PROVIDED",
category: "USER",
text: "At least one workflow, step, or trajectory scorer must be provided"
});
}
} else if (!isWorkflow(target) && isAgentScorerConfig(scorers)) {
const hasScorers = scorers.agent && scorers.agent.length > 0 || scorers.trajectory && scorers.trajectory.length > 0;
if (!hasScorers) {
throw new chunk4U7ZLI36_cjs.MastraError({
domain: "SCORER",
id: "NO_SCORERS_PROVIDED",
category: "USER",
text: "At least one agent or trajectory scorer must be provided"
});
}
} else if (!isWorkflow(target) && !Array.isArray(scorers) && !isAgentScorerConfig(scorers)) {
throw new chunk4U7ZLI36_cjs.MastraError({
domain: "SCORER",
id: "INVALID_AGENT_SCORERS",
category: "USER",
text: "Agent scorers must be an array of scorers or an AgentScorerConfig"
});
}
}
async function executeTarget(target, item, targetOptions) {
try {
if (isWorkflow(target)) {
return await executeWorkflow(target, item, targetOptions);
} else {
return await executeAgent(
target,
item,
targetOptions
);
}
} catch (error) {
throw new chunk4U7ZLI36_cjs.MastraError(
{
domain: "SCORER",
id: "RUN_EXPERIMENT_TARGET_FAILED_TO_GENERATE_RESULT",
category: "USER",
text: "Failed to run experiment: Error generating result from target",
details: {
item: JSON.stringify(item)
}
},
error
);
}
}
async function executeWorkflow(target, item, targetOptions) {
const observabilityContext = chunk2E7FPUYL_cjs.resolveObservabilityContext(item);
const run = await target.createRun({ disableScorers: true });
const workflowResult = await run.start({
...targetOptions,
...item.startOptions,
inputData: item.input,
requestContext: item.requestContext,
...observabilityContext
});
return {
traceId: workflowResult.traceId,
spanId: workflowResult.spanId,
entityType: chunkG3JYQ2UI_cjs.EntityType.WORKFLOW_RUN,
scoringData: {
input: item.input,
output: workflowResult.status === "success" ? workflowResult.result : void 0,
stepResults: workflowResult.steps,
stepExecutionPath: workflowResult.stepExecutionPath
}
};
}
async function executeAgent(agent, item, targetOptions) {
const observabilityContext = chunk2E7FPUYL_cjs.resolveObservabilityContext(item);
const model = await agent.getModel();
if (chunkACQ5CVFF_cjs.isSupportedLanguageModel(model)) {
const { structuredOutput, ...restOptions } = targetOptions ?? {};
const baseOptions = {
...restOptions,
...observabilityContext,
scorers: {},
returnScorerData: true,
requestContext: item.requestContext
};
const result = structuredOutput ? await agent.generate(item.input, { ...baseOptions, structuredOutput }) : await agent.generate(item.input, baseOptions);
return {
...result,
entityType: chunkG3JYQ2UI_cjs.EntityType.AGENT
};
} else {
const result = await agent.generateLegacy(item.input, {
scorers: {},
returnScorerData: true,
requestContext: item.requestContext,
...observabilityContext
});
return {
...result,
entityType: chunkG3JYQ2UI_cjs.EntityType.AGENT
};
}
}
async function extractTrajectoryFromTraceStore(storage, traceId, spanId) {
if (!storage || !traceId) return void 0;
try {
const observabilityStore = await storage.getStore("observability");
if (!observabilityStore) return void 0;
const trace = await observabilityStore.getTrace({ traceId });
if (!trace?.spans?.length) return void 0;
return chunkXD7CRAEV_cjs.extractTrajectoryFromTrace(trace.spans, spanId);
} catch {
return void 0;
}
}
async function runScorers(scorers, targetResult, item, storage) {
const scorerResults = {};
const targetTraceId = targetResult.traceId;
const targetEntityType = targetResult.entityType;
if (Array.isArray(scorers)) {
for (const scorer of scorers) {
try {
const score = await scorer.run({
input: targetResult.scoringData?.input,
output: targetResult.scoringData?.output,
groundTruth: item.groundTruth,
requestContext: item.requestContext,
scoreSource: "experiment",
targetScope: "span",
targetEntityType,
targetTraceId,
targetSpanId: targetResult.spanId
});
scorerResults[scorer.id] = score;
} catch (error) {
throw new chunk4U7ZLI36_cjs.MastraError(
{
domain: "SCORER",
id: "RUN_EXPERIMENT_SCORER_FAILED_TO_SCORE_RESULT",
category: "USER",
text: `Failed to run experiment: Error running scorer ${scorer.id}`,
details: {
scorerId: scorer.id,
item: JSON.stringify(item)
}
},
error
);
}
}
} else if (isAgentScorerConfig(scorers)) {
if (scorers.agent) {
const agentScorerResults = {};
for (const scorer of scorers.agent) {
try {
const score = await scorer.run({
input: targetResult.scoringData?.input,
output: targetResult.scoringData?.output,
groundTruth: item.groundTruth,
requestContext: item.requestContext,
scoreSource: "experiment",
targetScope: "span",
targetEntityType,
targetTraceId,
targetSpanId: targetResult.spanId
});
agentScorerResults[scorer.id] = score;
} catch (error) {
throw new chunk4U7ZLI36_cjs.MastraError(
{
domain: "SCORER",
id: "RUN_EXPERIMENT_SCORER_FAILED_TO_SCORE_RESULT",
category: "USER",
text: `Failed to run experiment: Error running agent scorer ${scorer.id}`,
details: {
scorerId: scorer.id,
item: JSON.stringify(item)
}
},
error
);
}
}
if (Object.keys(agentScorerResults).length > 0) {
scorerResults.agent = agentScorerResults;
}
}
if (scorers.trajectory) {
const trajectoryScorerResults = {};
const traceTrajectory = await extractTrajectoryFromTraceStore(storage, targetResult.traceId, targetResult.spanId);
const rawOutput = targetResult.scoringData?.output;
const trajectory = traceTrajectory ?? (rawOutput ? chunkXD7CRAEV_cjs.extractTrajectory(rawOutput) : { steps: [] });
for (const scorer of scorers.trajectory) {
try {
const score = await scorer.run({
input: targetResult.scoringData?.input,
output: trajectory,
groundTruth: item.groundTruth,
expectedTrajectory: item.expectedTrajectory,
requestContext: item.requestContext,
scoreSource: "experiment",
targetScope: "trajectory",
targetEntityType,
targetTraceId,
targetSpanId: targetResult.spanId
});
trajectoryScorerResults[scorer.id] = score;
} catch (error) {
throw new chunk4U7ZLI36_cjs.MastraError(
{
domain: "SCORER",
id: "RUN_EXPERIMENT_SCORER_FAILED_TO_SCORE_TRAJECTORY",
category: "USER",
text: `Failed to run experiment: Error running trajectory scorer ${scorer.id}`,
details: {
scorerId: scorer.id,
item: JSON.stringify(item)
}
},
error
);
}
}
if (Object.keys(trajectoryScorerResults).length > 0) {
scorerResults.trajectory = trajectoryScorerResults;
}
}
} else {
if (scorers.workflow) {
const workflowScorerResults = {};
for (const scorer of scorers.workflow) {
const score = await scorer.run({
input: targetResult.scoringData.input,
output: targetResult.scoringData.output,
groundTruth: item.groundTruth,
requestContext: item.requestContext,
scoreSource: "experiment",
targetScope: "span",
targetEntityType,
targetTraceId,
targetSpanId: targetResult.spanId
});
workflowScorerResults[scorer.id] = score;
}
if (Object.keys(workflowScorerResults).length > 0) {
scorerResults.workflow = workflowScorerResults;
}
}
if (scorers.steps) {
const stepScorerResults = {};
for (const [stepId, stepScorers] of Object.entries(scorers.steps)) {
const stepResult = targetResult.scoringData.stepResults?.[stepId];
if (stepResult?.status === "success" && stepResult.output !== void 0) {
const stepResults = {};
for (const scorer of stepScorers) {
try {
const score = await scorer.run({
input: stepResult.payload !== void 0 ? stepResult.payload : targetResult.scoringData.input,
output: stepResult.output,
groundTruth: item.groundTruth,
requestContext: item.requestContext,
scoreSource: "experiment",
targetScope: "span",
targetEntityType: chunkG3JYQ2UI_cjs.EntityType.WORKFLOW_STEP,
targetTraceId
});
stepResults[scorer.id] = score;
} catch (error) {
throw new chunk4U7ZLI36_cjs.MastraError(
{
domain: "SCORER",
id: "RUN_EXPERIMENT_SCORER_FAILED_TO_SCORE_STEP_RESULT",
category: "USER",
text: `Failed to run experiment: Error running scorer ${scorer.id} on step ${stepId}`,
details: {
scorerId: scorer.id,
stepId
}
},
error
);
}
}
if (Object.keys(stepResults).length > 0) {
stepScorerResults[stepId] = stepResults;
}
}
}
if (Object.keys(stepScorerResults).length > 0) {
scorerResults.steps = stepScorerResults;
}
}
if (scorers.trajectory) {
const trajectoryScorerResults = {};
const traceTrajectory = await extractTrajectoryFromTraceStore(storage, targetResult.traceId, targetResult.spanId);
let trajectory = traceTrajectory;
if (!trajectory) {
const stepResults = targetResult.scoringData?.stepResults;
const stepExecutionPath = targetResult.scoringData?.stepExecutionPath;
trajectory = stepResults ? chunkXD7CRAEV_cjs.extractWorkflowTrajectory(stepResults, stepExecutionPath) : { steps: [] };
}
for (const scorer of scorers.trajectory) {
try {
const score = await scorer.run({
input: targetResult.scoringData?.input,
output: trajectory,
groundTruth: item.groundTruth,
expectedTrajectory: item.expectedTrajectory,
requestContext: item.requestContext,
scoreSource: "experiment",
targetScope: "trajectory",
targetEntityType: chunkG3JYQ2UI_cjs.EntityType.TRAJECTORY,
targetTraceId,
targetSpanId: targetResult.spanId
});
trajectoryScorerResults[scorer.id] = score;
} catch (error) {
throw new chunk4U7ZLI36_cjs.MastraError(
{
domain: "SCORER",
id: "RUN_EXPERIMENT_SCORER_FAILED_TO_SCORE_WORKFLOW_TRAJECTORY",
category: "USER",
text: `Failed to run experiment: Error running workflow trajectory scorer ${scorer.id}`,
details: {
scorerId: scorer.id,
item: JSON.stringify(item)
}
},
error
);
}
}
if (Object.keys(trajectoryScorerResults).length > 0) {
scorerResults.trajectory = trajectoryScorerResults;
}
}
}
return scorerResults;
}
async function saveScoresToStorage({
storage,
scorerResults,
target,
item,
mastra
}) {
const entityId = target.id;
const entityType = isWorkflow(target) ? "WORKFLOW" : "AGENT";
const isStructuredWorkflowResult = "workflow" in scorerResults || "steps" in scorerResults;
const isStructuredAgentResult = "agent" in scorerResults || "trajectory" in scorerResults;
if (!isStructuredWorkflowResult && !isStructuredAgentResult) {
for (const [scorerId, scoreResult] of Object.entries(scorerResults)) {
if (scoreResult && typeof scoreResult === "object" && "score" in scoreResult) {
await saveSingleScore({
storage,
scoreResult,
scorerId,
entityId,
entityType,
mastra,
target,
item
});
}
}
} else if (isStructuredAgentResult) {
if (scorerResults.agent) {
for (const [scorerId, scoreResult] of Object.entries(scorerResults.agent)) {
if (scoreResult && typeof scoreResult === "object" && "score" in scoreResult) {
await saveSingleScore({
storage,
scoreResult,
scorerId,
entityId,
entityType: "AGENT",
mastra,
target,
item
});
}
}
}
if (scorerResults.trajectory) {
for (const [scorerId, scoreResult] of Object.entries(scorerResults.trajectory)) {
if (scoreResult && typeof scoreResult === "object" && "score" in scoreResult) {
await saveSingleScore({
storage,
scoreResult,
scorerId,
entityId,
entityType: "TRAJECTORY",
mastra,
target,
item
});
}
}
}
} else {
if (scorerResults.workflow) {
for (const [scorerId, scoreResult] of Object.entries(scorerResults.workflow)) {
if (scoreResult && typeof scoreResult === "object" && "score" in scoreResult) {
await saveSingleScore({
storage,
scoreResult,
scorerId,
entityId,
entityType: "WORKFLOW",
mastra,
target,
item
});
}
}
}
if (scorerResults.steps) {
for (const [stepId, stepScorers] of Object.entries(scorerResults.steps)) {
for (const [scorerId, scoreResult] of Object.entries(stepScorers)) {
if (scoreResult && typeof scoreResult === "object" && "score" in scoreResult) {
await saveSingleScore({
storage,
scoreResult,
scorerId,
entityId: stepId,
entityType: "STEP",
mastra,
target,
item
});
}
}
}
}
}
}
async function saveSingleScore({
storage,
scoreResult,
scorerId,
entityId,
entityType,
mastra,
target,
item
}) {
try {
let scorer = mastra?.getScorerById?.(scorerId);
if (!scorer) {
const targetScorers = await target.listScorers?.();
if (targetScorers) {
for (const [_, scorerEntry] of Object.entries(targetScorers)) {
if (scorerEntry.scorer?.id === scorerId) {
scorer = scorerEntry.scorer;
break;
}
}
}
}
let traceId;
let spanId;
if (item.tracingContext?.currentSpan && item.tracingContext.currentSpan.isValid) {
spanId = item.tracingContext.currentSpan.id;
traceId = item.tracingContext.currentSpan.traceId;
}
const additionalContext = {};
if (item.groundTruth !== void 0) {
additionalContext.groundTruth = item.groundTruth;
}
const payload = {
...scoreResult,
scorerId,
entityId,
entityType,
source: "TEST",
scorer: {
id: scorer?.id || scorerId,
name: scorer?.name || scorerId,
description: scorer?.description || "",
type: scorer?.type || "unknown",
...scorer ? { hasJudge: !!scorer.judge } : {}
},
entity: {
id: target.id,
name: target.name || target.id
},
// Include requestContext from item
requestContext: item.requestContext ? Object.fromEntries(item.requestContext.entries()) : void 0,
// Include additionalContext with groundTruth
additionalContext: Object.keys(additionalContext).length > 0 ? additionalContext : void 0,
// Include tracing information
traceId,
spanId
};
await validateAndSaveScore(storage, payload);
} catch (error) {
mastra?.getLogger?.()?.warn?.(`Failed to save score for scorer ${scorerId}:`, error);
}
}
exports.createOnScorerHook = createOnScorerHook;
exports.runEvals = runEvals;
exports.validateAndSaveScore = validateAndSaveScore;
//# sourceMappingURL=chunk-CCTAEUR3.cjs.map
//# sourceMappingURL=chunk-CCTAEUR3.cjs.map