@vfarcic/dot-ai
Version:
AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance
47 lines • 1.27 kB
TypeScript
/**
* Standard Evaluator Interface Following OpenAI Evals Pattern
*
* Based on OpenAI Evals framework standards:
* - Each evaluator has a name and description
* - evaluate() method takes input, output, and optional ideal
* - Returns standardized EvaluationScore
*/
export interface EvaluationScore {
key: string;
score: number;
comment?: string;
confidence?: number;
}
export interface EvaluationSample {
input: Record<string, unknown>;
output: string;
ideal?: unknown;
metadata?: Record<string, unknown>;
}
export interface PerformanceMetrics {
duration_ms: number;
input_tokens: number;
output_tokens: number;
total_tokens: number;
cost_usd?: number;
iterations?: number;
tool_calls_executed?: number;
cache_hit_rate?: number;
model_version: string;
}
export interface EvaluationResult {
sample_id: string;
model: string;
timestamp: string;
quality_scores: Record<string, EvaluationScore>;
performance: PerformanceMetrics;
efficiency: {
quality_per_second: number;
quality_per_dollar?: number;
quality_per_token: number;
};
input: Record<string, unknown>;
output: string;
ideal?: unknown;
}
//# sourceMappingURL=base.d.ts.map