@convo-lang/convo-lang/src/lib/testing/ConvoModelTestManager.js

The language of AI

import { pushBehaviorSubjectAry } from "@iyio/common";
import { BehaviorSubject } from "rxjs";
import { convoAnyModelName } from "../convo-lib.js";
import { convoCompletionService, convoConversationConverterProvider } from "../convo.deps.js";
import { ConvoModelTester } from "./ConvoModelTester.js";
export class ConvoModelTestManager {
    get testResultsSubject() { return this._testResults; }
    get testResults() { return this._testResults.value; }
    constructor({ models, parallel = false, printUpdates = false, verbose = false, defaultModel = false, tests, }) {
        this._testResults = new BehaviorSubject([]);
        this.printUpdate = () => {
            const grid = {};
            for (const r of this.testResults) {
                const row = grid[r.model] ?? (grid[r.model] = {});
                row['model'] = r.model;
                row['ms'] = r.durationMs;
                row[r.testName] = r.passed;
                if (r.errorMessage) {
                    row[`${r.testName}_err`] = r.errorMessage.substring(0, 40);
                }
            }
            const rows = Object.values(grid);
            rows.sort((a, b) => a['model'].localeCompare(b['model']));
            console.table(rows);
        };
        this.options = {
            models,
            parallel,
            printUpdates,
            verbose,
            defaultModel,
            tests,
        };
        if (printUpdates) {
            this._testResults.subscribe(this.printUpdate);
        }
    }
    printResults(errorsOnly) {
        for (const r of this.testResults) {
            if (r.passed && errorsOnly) {
                continue;
            }
            console.log(`\n--------------------- ${r.model}:${r.testName} ${r.passed ? 'passed' : 'failed'}`);
            const p = { ...r };
            delete p.debugOutput;
            delete p.convo;
            console.log(p);
            if (r.debugOutput) {
                console.log('<debugOutput>');
                console.log(r.debugOutput.join('\n'));
                console.log('</debugOutput>');
            }
            if (r.convo) {
                console.log('convo>');
                console.log(r.convo);
                console.log('</convo>');
            }
            if (r.errorMessage) {
                console.log('errorMessage>');
                console.log(r.errorMessage);
                console.log('</errorMessage>');
            }
            console.log(`\n--------------------- end ${r.model}:${r.testName} \n\n`);
        }
    }
    async runTestAsync() {
        const testModelAsync = (model) => {
            const tester = new ConvoModelTester({
                manager: this,
                model: model.name,
                tests: this.options.tests,
                convoOptions: {
                    completionService: convoCompletionService.all(),
                    converters: convoConversationConverterProvider.all(),
                },
            });
            if (!this.testNames) {
                this.testNames = tester.getTestNames();
            }
            return tester.runAllTestsAsync();
        };
        try {
            const models = [...this.options.models];
            if (this.options.defaultModel) {
                models.push({ name: convoAnyModelName });
            }
            if (this.options.parallel) {
                await Promise.all(models.map(testModelAsync));
            }
            else {
                for (const model of models) {
                    await testModelAsync(model);
                }
            }
            this.testResults.sort((a, b) => `${a.model}:${a.testName}`.localeCompare(`${b.model}:${b.testName}`));
        }
        catch (ex) {
            console.log(`testing failed`, ex);
            process.exit(1);
        }
    }
    publishResult(result) {
        pushBehaviorSubjectAry(this._testResults, result);
    }
}
//# sourceMappingURL=ConvoModelTestManager.js.map