@mastra/core
Version:
Mastra is a framework for building AI-powered applications and agents with a modern TypeScript stack.
279 lines • 10.1 kB
TypeScript
/**
* Network Completion Scorers
*
* Completion checks are just MastraScorers that return 0 (failed) or 1 (passed).
* This unifies completion checking with the evaluation system.
*
* @example
* ```typescript
* import { createScorer } from '@mastra/core/evals';
*
* // Simple completion scorer
* const testsScorer = createScorer({
* id: 'tests',
* description: 'Run unit tests',
* }).generateScore(async ({ run }) => {
* const result = await exec('npm test');
* return result.exitCode === 0 ? 1 : 0;
* });
*
* // Use in network
* await agent.network(messages, {
* completion: {
* scorers: [testsScorer],
* },
* });
* ```
*/
import type { MastraDBMessage, Agent } from '../../agent/index.js';
import type { StructuredOutputOptions } from '../../agent/types.js';
import type { MastraScorer } from '../../evals/base.js';
import type { NetworkChunkType } from '../../stream/types.js';
/**
* Runtime context passed to completion scoring.
* Available via run.input when using a completion scorer.
*/
export interface CompletionContext {
/** Current iteration number (1-based) */
iteration: number;
/** Maximum iterations allowed */
maxIterations?: number;
/** All messages in the conversation thread */
messages: MastraDBMessage[];
/** The original task/prompt that started this network run */
originalTask: string;
/** Which primitive was selected this iteration */
selectedPrimitive: {
id: string;
type: 'agent' | 'workflow' | 'tool' | 'none';
};
/** The prompt/input sent to the selected primitive */
primitivePrompt: string;
/** Result from the primitive execution */
primitiveResult: string;
/** Name of the network/routing agent */
networkName: string;
/** ID of the current run */
runId: string;
/** Current thread ID (if using memory) */
threadId?: string;
/** Resource ID (if using memory) */
resourceId?: string;
/** Custom context from the request */
customContext?: Record<string, unknown>;
}
/**
* Result of running a single scorer.
* Scorers just evaluate pass/fail - they don't generate the final result.
*/
export interface ScorerResult {
/** The score (0 = failed, 1 = passed) */
score: number;
/** Whether this scorer passed (score === 1) */
passed: boolean;
/** Reason from the scorer (why it passed/failed) */
reason?: string;
/** Scorer ID */
scorerId: string;
/** Scorer name */
scorerName: string;
/** Duration in ms */
duration: number;
/** Final result generated by the scorer (if any) */
finalResult?: string;
}
/**
* Configuration for network completion.
*/
export interface CompletionConfig {
/**
* Scorers to run to determine if the task is complete.
* Each scorer should return 0 (not complete) or 1 (complete).
*
* @example
* ```typescript
* completion: {
* scorers: [testsScorer, buildScorer],
* }
* ```
*/
scorers?: MastraScorer<any, any, any, any>[];
/**
* How to combine scorer results:
* - 'all': All scorers must pass (score = 1) (default)
* - 'any': At least one scorer must pass
*/
strategy?: 'all' | 'any';
/**
* Maximum time for all scorers (ms)
* Default: 600000 (10 minutes)
*/
timeout?: number;
/**
* Run scorers in parallel (default: true)
*/
parallel?: boolean;
/**
* Called after scorers run with results
*/
onComplete?: (results: CompletionRunResult) => void | Promise<void>;
/**
* Suppress the completion feedback message from being saved to memory.
* When true, the "#### Completion Check Results" message will not be
* persisted, preventing it from appearing in subsequent iterations or
* history. Useful for cleaner conversation threads.
* Default: false
*/
suppressFeedback?: boolean;
}
/**
* Result of running completion checks.
*
* Completion checks just evaluate "is this done?" - they don't generate the final result.
* The final result comes from the agent network's primitives.
*/
export interface CompletionRunResult {
/** Whether the task is complete (based on strategy) */
complete: boolean;
/** Reason for completion/failure */
completionReason?: string;
/** Individual scorer results */
scorers: ScorerResult[];
/** Total duration of all checks */
totalDuration: number;
/** Whether checks timed out */
timedOut: boolean;
}
/** @deprecated Use CompletionContext instead */
export type CheckContext = CompletionContext;
/** @deprecated Use CompletionConfig instead */
export type NetworkValidationConfig = CompletionConfig;
/** @deprecated Use CompletionRunResult instead */
export type CheckRunResult = CompletionRunResult;
/** @deprecated Use CompletionRunResult instead */
export type ValidationRunResult = CompletionRunResult;
/**
* Runs all completion scorers according to the configuration
*/
export declare function runCompletionScorers(scorers: MastraScorer<any, any, any, any>[], context: CompletionContext, options?: {
strategy?: 'all' | 'any';
parallel?: boolean;
timeout?: number;
}): Promise<CompletionRunResult>;
/** @deprecated Use runCompletionScorers instead */
export declare function runChecks(scorers: MastraScorer<any, any, any, any>[], context: CompletionContext, options?: {
strategy?: 'all' | 'any';
parallel?: boolean;
timeout?: number;
}): Promise<CompletionRunResult>;
/** @deprecated Use runCompletionScorers instead */
export declare function runValidation(config: CompletionConfig, context: CompletionContext): Promise<CompletionRunResult>;
/**
* Formats scorer results into a message for the LLM
*/
export declare function formatCompletionFeedback(result: CompletionRunResult, maxIterationReached: boolean): string;
/** @deprecated Use formatCompletionFeedback instead */
export declare const formatCheckFeedback: typeof formatCompletionFeedback;
/** @deprecated Use formatCompletionFeedback instead */
export declare const formatValidationFeedback: typeof formatCompletionFeedback;
/**
* Runs the default LLM completion check.
* Just evaluates "is this done?" - does NOT generate the final result.
*
* @internal Used by the network loop when no scorers are configured
*/
export declare function runDefaultCompletionCheck(agent: Agent, context: CompletionContext, streamContext?: {
writer?: {
write: (chunk: NetworkChunkType) => Promise<void>;
};
stepId?: string;
runId?: string;
}, abortSignal?: AbortSignal, onAbort?: (event: any) => Promise<void> | void): Promise<ScorerResult>;
/**
* Generates and streams the final result after custom scorers have passed.
* Unlike runDefaultCompletionCheck, this doesn't evaluate completion - it only generates the result.
*
* @internal Used by the network loop after custom scorers pass
*/
export declare function generateFinalResult(agent: Agent, context: CompletionContext, streamContext?: {
writer?: {
write: (chunk: NetworkChunkType) => Promise<void>;
};
stepId?: string;
runId?: string;
}, abortSignal?: AbortSignal, onAbort?: (event: any) => Promise<void> | void): Promise<string | undefined>;
/**
* Result type for structured final result generation
*/
export interface StructuredFinalResult<OUTPUT = undefined> {
/** Text result (for backward compatibility) */
text?: string;
/** Structured object result when user schema is provided */
object?: OUTPUT;
}
/**
* Generates a structured final result using the user-provided schema.
* This is called when the network has structuredOutput option configured.
*
* @internal Used by the network loop when structuredOutput is provided
*/
export declare function generateStructuredFinalResult<OUTPUT extends {}>(agent: Agent, context: CompletionContext, structuredOutputOptions: StructuredOutputOptions<OUTPUT>, streamContext?: {
writer?: {
write: (chunk: NetworkChunkType) => Promise<void>;
};
stepId?: string;
runId?: string;
}, abortSignal?: AbortSignal, onAbort?: (event: any) => Promise<void> | void): Promise<StructuredFinalResult<OUTPUT>>;
export { createScorer } from '../../evals/base.js';
/**
* Runtime context passed to stream/generate completion scoring.
* This is a simplified version of CompletionContext for tool-based supervisor patterns.
*/
export interface StreamCompletionContext {
/** Current iteration number (1-based) */
iteration: number;
/** Maximum iterations allowed (maxSteps) */
maxIterations?: number;
/** The original user message/task that started this execution */
originalTask: string;
/** Current output text from the LLM */
currentText: string;
/** Tool calls made in this iteration */
toolCalls: Array<{
name: string;
args: Record<string, unknown>;
}>;
/** Tool results from this iteration */
toolResults: Array<{
name: string;
result: unknown;
}>;
/** ID of the current run */
runId: string;
/** Current thread ID (if using memory) */
threadId?: string;
/** Resource ID (if using memory) */
resourceId?: string;
/** Agent ID */
agentId?: string;
/** Agent name */
agentName?: string;
/** Custom context from the request */
customContext?: Record<string, unknown>;
messages: MastraDBMessage[];
}
/**
* Runs completion scorers for stream/generate execution.
* Adapts the StreamCompletionContext to work with existing scorers.
*/
export declare function runStreamCompletionScorers(scorers: MastraScorer<any, any, any, any>[], context: StreamCompletionContext, options?: {
strategy?: 'all' | 'any';
parallel?: boolean;
timeout?: number;
}): Promise<CompletionRunResult>;
/**
* Formats stream completion feedback for the LLM.
* Similar to formatCompletionFeedback but tailored for stream context.
*/
export declare function formatStreamCompletionFeedback(result: CompletionRunResult, maxIterationReached: boolean): string;
//# sourceMappingURL=validation.d.ts.map