@wavequery/conductor/dist/esm/evaluation/evaluators.js

Modular LLM orchestration framework

import { Logger, LogLevel } from "@/utils/logger";
export class BaseEvaluator {
    constructor(config) {
        this.config = config;
        this.logger = new Logger({
            level: LogLevel.DEBUG,
            prefix: "Evaluator",
        });
    }
}
export class AutomatedEvaluator extends BaseEvaluator {
    async evaluate(input, expected) {
        const results = [];
        const startTime = Date.now();
        for (const metric of this.config.metrics) {
            if (metric.type !== "automated")
                continue;
            try {
                const result = await this.evaluateMetric(metric, input, expected);
                results.push(result);
            }
            catch (error) {
                this.logger.error(`Metric ${metric.name} failed:`, error);
            }
        }
        return results;
    }
    async evaluateMetric(metric, input, expected) {
        switch (metric.name) {
            case "responseTime":
                return this.evaluateResponseTime(input);
            case "tokenUsage":
                return this.evaluateTokenUsage(input);
            case "errorRate":
                return this.evaluateErrorRate(input);
            default:
                throw new Error(`Unknown metric: ${metric.name}`);
        }
    }
    async evaluateResponseTime(input) {
        const duration = input.metrics?.duration || 0;
        const score = Math.min(1, 5000 / duration);
        return {
            metricName: "responseTime",
            score,
            metadata: { duration },
            timestamp: new Date(),
        };
    }
    async evaluateTokenUsage(input) {
        const tokens = input.metrics?.tokens || 0;
        const score = Math.min(1, 2000 / tokens);
        return {
            metricName: "tokenUsage",
            score,
            metadata: { tokens },
            timestamp: new Date(),
        };
    }
    async evaluateErrorRate(input) {
        const success = input.success === true;
        return {
            metricName: "errorRate",
            score: success ? 1 : 0,
            metadata: { error: input.error },
            timestamp: new Date(),
        };
    }
}
export class HumanEvaluator extends BaseEvaluator {
    async evaluate(input, feedback) {
        const results = [];
        for (const metric of this.config.metrics) {
            if (metric.type !== "human")
                continue;
            const score = feedback[metric.name] || 0;
            results.push({
                metricName: metric.name,
                score,
                metadata: { feedback: feedback[`${metric.name}_feedback`] },
                timestamp: new Date(),
                evaluator: feedback.evaluator,
            });
        }
        return results;
    }
}
export class CustomEvaluator extends BaseEvaluator {
    constructor(config) {
        super(config);
        this.customMetrics = new Map();
    }
    registerMetric(name, evaluator) {
        this.customMetrics.set(name, evaluator);
    }
    async evaluate(input, expected) {
        const results = [];
        for (const [name, evaluator] of this.customMetrics) {
            try {
                const score = await evaluator(input, expected);
                results.push({
                    metricName: name,
                    score,
                    metadata: { input, expected },
                    timestamp: new Date(),
                });
            }
            catch (error) {
                this.logger.error(`Custom metric ${name} failed:`, error);
            }
        }
        return results;
    }
}
//# sourceMappingURL=evaluators.js.map