@wavequery/conductor/dist/cjs/evaluation/example/eval-runner-example.js

Modular LLM orchestration framework

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.SQLEvaluationRunner = void 0;
class SQLEvaluationRunner {
    constructor(service, options) {
        this.service = service;
        this.options = options;
    }
    async evaluateTestCase(testCase) {
        const startTime = Date.now();
        try {
            const generated = await this.service.processQuestion(testCase.question, testCase.context);
            const metrics = {};
            for (const metric of this.options.metrics) {
                metrics[metric.name] = metric.compute({ ...testCase, query: generated.query }, testCase);
            }
            return {
                testCase,
                metrics,
                duration: Date.now() - startTime,
            };
        }
        catch (error) {
            return {
                testCase,
                metrics: {},
                duration: Date.now() - startTime,
                error,
            };
        }
    }
    async evaluateTestSuite(testCases) {
        if (this.options.parallel) {
            return Promise.all(testCases.map((tc) => this.evaluateTestCase(tc)));
        }
        const results = [];
        for (const testCase of testCases) {
            results.push(await this.evaluateTestCase(testCase));
        }
        return results;
    }
}
exports.SQLEvaluationRunner = SQLEvaluationRunner;
//# sourceMappingURL=eval-runner-example.js.map