@mastra/core/dist/evals/scoreTraces/index.cjs

'use strict';

var chunkCJAAPSS7_cjs = require('../../chunk-CJAAPSS7.cjs');
var chunkPLKGTK7L_cjs = require('../../chunk-PLKGTK7L.cjs');
var chunkLP4WZA6D_cjs = require('../../chunk-LP4WZA6D.cjs');
var chunkXSOONORA_cjs = require('../../chunk-XSOONORA.cjs');
var pMap = require('p-map');
var v4 = require('zod/v4');

function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }

var pMap__default = /*#__PURE__*/_interopDefault(pMap);

// src/evals/scoreTraces/scoreTraces.ts
async function scoreTraces({
  scorerId,
  targets,
  mastra
}) {
  const workflow = mastra.__getInternalWorkflow("__batch-scoring-traces");
  try {
    const run = await workflow.createRun();
    await run.start({ inputData: { targets, scorerId } });
  } catch (error) {
    const mastraError = new chunkXSOONORA_cjs.MastraError(
      {
        category: "SYSTEM",
        domain: "SCORER",
        id: "MASTRA_SCORER_FAILED_TO_RUN_TRACE_SCORING",
        details: {
          scorerId,
          targets: JSON.stringify(targets)
        }
      },
      error
    );
    mastra.getLogger()?.trackException(mastraError);
  }
}

// src/evals/scoreTraces/utils.ts
function isSpanMessage(value) {
  return typeof value === "object" && value !== null && "role" in value && typeof value.role === "string" && "content" in value;
}
function hasMessagesArray(value) {
  return typeof value === "object" && value !== null && "messages" in value && Array.isArray(value.messages);
}
function hasTextProperty(value) {
  return typeof value === "object" && value !== null && "text" in value;
}
function buildSpanTree(spans) {
  const spanMap = /* @__PURE__ */ new Map();
  const childrenMap = /* @__PURE__ */ new Map();
  const rootSpans = [];
  for (const span of spans) {
    spanMap.set(span.spanId, span);
  }
  for (const span of spans) {
    if (span.parentSpanId == null) {
      rootSpans.push(span);
    } else {
      const siblings = childrenMap.get(span.parentSpanId) || [];
      siblings.push(span);
      childrenMap.set(span.parentSpanId, siblings);
    }
  }
  for (const children of childrenMap.values()) {
    children.sort((a, b) => new Date(a.startedAt).getTime() - new Date(b.startedAt).getTime());
  }
  rootSpans.sort((a, b) => new Date(a.startedAt).getTime() - new Date(b.startedAt).getTime());
  return { spanMap, childrenMap, rootSpans };
}
function getChildrenOfType(spanTree, parentSpanId, spanType) {
  const children = spanTree.childrenMap.get(parentSpanId) || [];
  return children.filter((span) => span.spanType === spanType);
}
function normalizeMessageContent(content) {
  if (typeof content === "string") {
    return content;
  }
  const textParts = content.filter((part) => part.type === "text");
  return textParts.length > 0 ? textParts[textParts.length - 1]?.text || "" : "";
}
function createMastraDBMessage(message, createdAt, id = "") {
  const contentText = normalizeMessageContent(message.content);
  const role = message.role;
  return {
    id,
    role,
    content: {
      format: 2,
      parts: [{ type: "text", text: contentText }],
      content: contentText
    },
    createdAt: new Date(createdAt)
  };
}
function extractInputMessages(agentSpan) {
  const input = agentSpan.input;
  if (typeof input === "string") {
    return [
      createMastraDBMessage(
        {
          role: "user",
          content: input
        },
        agentSpan.startedAt
      )
    ];
  }
  if (Array.isArray(input)) {
    const messages = input.filter(isSpanMessage);
    return messages.map((msg) => createMastraDBMessage(msg, agentSpan.startedAt));
  }
  if (hasMessagesArray(input)) {
    const messages = input.messages.filter(isSpanMessage);
    return messages.map((msg) => createMastraDBMessage(msg, agentSpan.startedAt));
  }
  return [];
}
function extractSystemMessages(llmSpan) {
  const input = llmSpan.input;
  if (!hasMessagesArray(input)) {
    return [];
  }
  return input.messages.filter((msg) => isSpanMessage(msg) && msg.role === "system").map((msg) => ({
    role: "system",
    content: normalizeMessageContent(msg.content)
  }));
}
function extractRememberedMessages(llmSpan, currentInputContent) {
  const input = llmSpan.input;
  if (!hasMessagesArray(input)) {
    return [];
  }
  const filtered = input.messages.filter(isSpanMessage);
  const messages = filtered.filter((msg) => msg.role !== "system").filter((msg) => normalizeMessageContent(msg.content) !== currentInputContent);
  return messages.map((msg) => createMastraDBMessage(msg, llmSpan.startedAt));
}
function reconstructToolInvocations(spanTree, parentSpanId) {
  const toolSpans = getChildrenOfType(spanTree, parentSpanId, "tool_call" /* TOOL_CALL */);
  return toolSpans.map((toolSpan) => ({
    toolCallId: toolSpan.spanId,
    toolName: toolSpan.entityName ?? toolSpan.entityId ?? "unknown",
    toolId: toolSpan.entityId,
    args: toolSpan.input || {},
    result: toolSpan.output || {},
    state: "result"
  }));
}
function validateTrace(trace) {
  if (!trace) {
    throw new Error("Trace is null or undefined");
  }
  if (!trace.spans || !Array.isArray(trace.spans)) {
    throw new Error("Trace must have a spans array");
  }
  if (trace.spans.length === 0) {
    throw new Error("Trace has no spans");
  }
  const spanIds = new Set(trace.spans.map((span) => span.spanId));
  for (const span of trace.spans) {
    if (span.parentSpanId && !spanIds.has(span.parentSpanId)) {
      throw new Error(`Span ${span.spanId} references non-existent parent ${span.parentSpanId}`);
    }
  }
}
function findPrimaryLLMSpan(spanTree, rootAgentSpan) {
  const directLLMSpans = getChildrenOfType(spanTree, rootAgentSpan.spanId, "model_generation" /* MODEL_GENERATION */);
  if (directLLMSpans.length > 0) {
    return directLLMSpans[0];
  }
  throw new Error("No model generation span found in trace");
}
function prepareTraceForTransformation(trace) {
  validateTrace(trace);
  const spanTree = buildSpanTree(trace.spans);
  const rootAgentSpan = spanTree.rootSpans.find((span) => span.spanType === "agent_run");
  if (!rootAgentSpan) {
    throw new Error("No root agent_run span found in trace");
  }
  return { spanTree, rootAgentSpan };
}
function transformTraceToScorerInputAndOutput(trace) {
  const { spanTree, rootAgentSpan } = prepareTraceForTransformation(trace);
  if (!rootAgentSpan.output) {
    throw new Error("Root agent span has no output");
  }
  const primaryLLMSpan = findPrimaryLLMSpan(spanTree, rootAgentSpan);
  const inputMessages = extractInputMessages(rootAgentSpan);
  const systemMessages = extractSystemMessages(primaryLLMSpan);
  const currentInputContent = inputMessages[0]?.content.content || "";
  const rememberedMessages = extractRememberedMessages(primaryLLMSpan, currentInputContent);
  const input = {
    inputMessages,
    rememberedMessages,
    systemMessages,
    taggedSystemMessages: {}
    // Todo: Support tagged system messages
  };
  const toolInvocations = reconstructToolInvocations(spanTree, rootAgentSpan.spanId);
  const responseText = hasTextProperty(rootAgentSpan.output) ? rootAgentSpan.output.text ?? "" : "";
  const parts = [];
  for (const toolInvocation of toolInvocations) {
    parts.push({
      type: "tool-invocation",
      toolInvocation
    });
  }
  if (responseText.trim()) {
    parts.push({
      type: "text",
      text: responseText
    });
  }
  const responseMessage = {
    id: "",
    role: "assistant",
    content: {
      format: 2,
      parts,
      // Type assertion needed due to providerMetadata optional field
      content: responseText,
      toolInvocations
      // Always include, even if empty array
    },
    createdAt: new Date(rootAgentSpan.endedAt || rootAgentSpan.startedAt)
  };
  const output = [responseMessage];
  return {
    input,
    output
  };
}

// src/evals/scoreTraces/scoreTracesWorkflow.ts
var getTraceStep = chunkCJAAPSS7_cjs.createStep2({
  id: "__process-trace-scoring",
  inputSchema: v4.z.object({
    targets: v4.z.array(
      v4.z.object({
        traceId: v4.z.string(),
        spanId: v4.z.string().optional()
      })
    ),
    scorerId: v4.z.string()
  }),
  outputSchema: v4.z.any(),
  execute: async ({ inputData, mastra }) => {
    const logger = mastra.getLogger();
    if (!logger) {
      console.warn(
        "[scoreTracesWorkflow] Logger not initialized: no debug or error logs will be recorded for scoring traces."
      );
    }
    const storage = mastra.getStorage();
    if (!storage) {
      const mastraError = new chunkXSOONORA_cjs.MastraError({
        id: "MASTRA_STORAGE_NOT_FOUND_FOR_TRACE_SCORING",
        domain: chunkXSOONORA_cjs.ErrorDomain.STORAGE,
        category: chunkXSOONORA_cjs.ErrorCategory.SYSTEM,
        text: "Storage not found for trace scoring",
        details: {
          scorerId: inputData.scorerId
        }
      });
      logger?.trackException(mastraError);
      return;
    }
    let scorer;
    try {
      scorer = mastra.getScorerById(inputData.scorerId);
    } catch (error) {
      const mastraError = new chunkXSOONORA_cjs.MastraError(
        {
          id: "MASTRA_SCORER_NOT_FOUND_FOR_TRACE_SCORING",
          domain: chunkXSOONORA_cjs.ErrorDomain.SCORER,
          category: chunkXSOONORA_cjs.ErrorCategory.SYSTEM,
          text: `Scorer not found for trace scoring`,
          details: {
            scorerId: inputData.scorerId
          }
        },
        error
      );
      logger?.trackException(mastraError);
      return;
    }
    await pMap__default.default(
      inputData.targets,
      async (target) => {
        try {
          await scoreTrace({ storage, scorer, target });
        } catch (error) {
          const mastraError = new chunkXSOONORA_cjs.MastraError(
            {
              id: "MASTRA_SCORER_FAILED_TO_RUN_SCORER_ON_TRACE",
              domain: chunkXSOONORA_cjs.ErrorDomain.SCORER,
              category: chunkXSOONORA_cjs.ErrorCategory.SYSTEM,
              details: {
                scorerId: scorer.id,
                spanId: target.spanId || "",
                traceId: target.traceId
              }
            },
            error
          );
          logger?.trackException(mastraError);
        }
      },
      { concurrency: 3 }
    );
  }
});
function getSpanTenancy(span) {
  const tenancy = {};
  if (span.organizationId) {
    tenancy.organizationId = span.organizationId;
  }
  if (span.resourceId) {
    tenancy.projectId = span.resourceId;
  }
  return tenancy;
}
function isScoreTraceReferenceTarget(target) {
  return "traceId" in target;
}
function resolveTargetSpan({ trace, spanId }) {
  const span = spanId ? trace.spans.find((candidateSpan) => candidateSpan.spanId === spanId) : trace.spans.find((candidateSpan) => candidateSpan.parentSpanId === null);
  if (!span) {
    throw new Error(`Span not found for scoring, traceId: ${trace.traceId}, spanId: ${spanId ?? "Not provided"}`);
  }
  return span;
}
async function resolveTraceAndSpan({
  storage,
  target
}) {
  if (!isScoreTraceReferenceTarget(target)) {
    return {
      trace: target.trace,
      span: resolveTargetSpan({ trace: target.trace, spanId: target.spanId })
    };
  }
  const observabilityStore = await storage.getStore("observability");
  if (!observabilityStore) {
    throw new chunkXSOONORA_cjs.MastraError({
      id: "MASTRA_OBSERVABILITY_STORAGE_NOT_AVAILABLE",
      domain: chunkXSOONORA_cjs.ErrorDomain.STORAGE,
      category: chunkXSOONORA_cjs.ErrorCategory.SYSTEM,
      text: "Observability storage domain is not available"
    });
  }
  const trace = await observabilityStore.getTrace({ traceId: target.traceId });
  if (!trace) {
    throw new Error(`Trace not found for scoring, traceId: ${target.traceId}`);
  }
  return { trace, span: resolveTargetSpan({ trace, spanId: target.spanId }) };
}
async function runScorerForTrace({
  scorer,
  trace,
  span
}) {
  const tenancy = getSpanTenancy(span);
  const scorerRun = buildScorerRun({
    scorerType: scorer.type === "agent" ? "agent" : void 0,
    trace,
    targetSpan: span
  });
  return scorer.run({
    ...scorerRun,
    scoreSource: "trace",
    targetScope: "span",
    targetEntityType: chunkLP4WZA6D_cjs.getEntityTypeForSpan(span),
    targetTraceId: trace.traceId,
    targetSpanId: span.spanId,
    targetCorrelationContext: {
      traceId: trace.traceId,
      spanId: span.spanId,
      ...tenancy.organizationId ? { organizationId: tenancy.organizationId } : {},
      ...tenancy.projectId ? { resourceId: tenancy.projectId } : {}
    },
    targetMetadata: {
      ...tenancy.organizationId ? { organizationId: tenancy.organizationId } : {},
      ...tenancy.projectId ? { projectId: tenancy.projectId } : {}
    }
  });
}
async function scoreTrace({
  storage,
  scorer,
  target,
  batchId,
  datasetId,
  datasetItemId
}) {
  const { trace, span } = await resolveTraceAndSpan({ storage, target });
  const tenancy = getSpanTenancy(span);
  const result = await runScorerForTrace({ scorer, trace, span });
  const scorerResult = {
    ...result,
    scorer: {
      id: scorer.id,
      name: scorer.name || scorer.id,
      description: scorer.description,
      hasJudge: !!scorer.judge
    },
    traceId: trace.traceId,
    spanId: span.spanId,
    entityId: span.entityId || span.entityName || "unknown",
    entityType: span.spanType,
    entity: { traceId: span.traceId, spanId: span.spanId },
    source: "TEST",
    scorerId: scorer.id,
    ...tenancy.organizationId ? { organizationId: tenancy.organizationId } : {},
    ...tenancy.projectId ? { projectId: tenancy.projectId } : {},
    ...batchId ? { batchId } : {},
    ...datasetId ? { datasetId } : {},
    ...datasetItemId ? { datasetItemId } : {}
  };
  const savedScoreRecord = await validateAndSaveScore({ storage, scorerResult });
  await attachScoreToSpan({ storage, span, scoreRecord: savedScoreRecord });
  return savedScoreRecord;
}
function toBatchResultError(error) {
  if (error instanceof Error) {
    return error;
  }
  return new Error(typeof error === "string" ? error : "Unknown scoreTraceBatch error");
}
async function scoreTraceBatch({
  storage,
  scorer,
  targets,
  batchId,
  datasetId,
  concurrency = 3
}) {
  const results = await pMap__default.default(
    targets,
    async (target, index) => {
      try {
        const score = await scoreTrace({
          storage,
          scorer,
          target,
          batchId,
          datasetId,
          datasetItemId: target.datasetItemId
        });
        const spanId = score.spanId ?? target.spanId;
        if (!spanId) {
          throw new Error(`Persisted score is missing spanId for traceId: ${target.traceId}`);
        }
        return {
          ok: true,
          index,
          traceId: score.traceId ?? target.traceId,
          spanId,
          ...target.datasetItemId ? { datasetItemId: target.datasetItemId } : {},
          score
        };
      } catch (error) {
        return {
          ok: false,
          index,
          traceId: target.traceId,
          ...target.spanId ? { spanId: target.spanId } : {},
          ...target.datasetItemId ? { datasetItemId: target.datasetItemId } : {},
          error: toBatchResultError(error)
        };
      }
    },
    { concurrency }
  );
  const scoredCount = results.filter((result) => result.ok).length;
  return {
    ...batchId ? { batchId } : {},
    ...datasetId ? { datasetId } : {},
    scoredCount,
    failedCount: results.length - scoredCount,
    results
  };
}
async function validateAndSaveScore({ storage, scorerResult }) {
  const scoresStore = await storage.getStore("scores");
  if (!scoresStore) {
    throw new chunkXSOONORA_cjs.MastraError({
      id: "MASTRA_SCORES_STORAGE_NOT_AVAILABLE",
      domain: chunkXSOONORA_cjs.ErrorDomain.STORAGE,
      category: chunkXSOONORA_cjs.ErrorCategory.SYSTEM,
      text: "Scores storage domain is not available"
    });
  }
  const payloadToSave = chunkPLKGTK7L_cjs.saveScorePayloadSchema.parse(scorerResult);
  const result = await scoresStore.saveScore(payloadToSave);
  return result.score;
}
function buildScorerRun({
  scorerType,
  trace,
  targetSpan
}) {
  if (scorerType === "agent") {
    const { input, output } = transformTraceToScorerInputAndOutput(trace);
    return { input, output };
  }
  return { input: targetSpan.input, output: targetSpan.output };
}
async function attachScoreToSpan({
  storage,
  span,
  scoreRecord
}) {
  const observabilityStore = await storage.getStore("observability");
  if (!observabilityStore) {
    throw new chunkXSOONORA_cjs.MastraError({
      id: "MASTRA_OBSERVABILITY_STORAGE_NOT_AVAILABLE",
      domain: chunkXSOONORA_cjs.ErrorDomain.STORAGE,
      category: chunkXSOONORA_cjs.ErrorCategory.SYSTEM,
      text: "Observability storage domain is not available"
    });
  }
  try {
    const existingLinks = span.links || [];
    const link = {
      type: "score",
      scoreId: scoreRecord.id,
      scorerId: scoreRecord.scorerId ?? scoreRecord.scorer?.id,
      score: scoreRecord.score,
      createdAt: scoreRecord.createdAt
    };
    await observabilityStore.updateSpan({
      spanId: span.spanId,
      traceId: span.traceId,
      updates: { links: [...existingLinks, link] }
    });
  } catch {
  }
}
var scoreTracesWorkflow = chunkCJAAPSS7_cjs.createWorkflow2({
  id: "__batch-scoring-traces",
  inputSchema: v4.z.object({
    targets: v4.z.array(
      v4.z.object({
        traceId: v4.z.string(),
        spanId: v4.z.string().optional()
      })
    ),
    scorerId: v4.z.string()
  }),
  outputSchema: v4.z.any(),
  steps: [getTraceStep],
  options: {
    validateInputs: false,
    // Internal batch-scoring plumbing — hide its workflow spans from exported
    // traces. Any user-facing work invoked from steps (e.g. the scorer's own
    // SCORER_RUN span) keeps its own policy and remains visible.
    tracingPolicy: {
      internal: 1 /* WORKFLOW */
    }
  }
});
scoreTracesWorkflow.then(getTraceStep).commit();

exports.scoreTrace = scoreTrace;
exports.scoreTraceBatch = scoreTraceBatch;
exports.scoreTraces = scoreTraces;
exports.scoreTracesWorkflow = scoreTracesWorkflow;
//# sourceMappingURL=index.cjs.map
//# sourceMappingURL=index.cjs.map