@convo-lang/convo-lang
Version:
The language of AI
111 lines • 4.06 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ConvoModelTestManager = void 0;
const convo_lang_1 = require("@convo-lang/convo-lang");
const common_1 = require("@iyio/common");
const rxjs_1 = require("rxjs");
const ConvoModelTester_1 = require("./ConvoModelTester");
class ConvoModelTestManager {
options;
_testResults = new rxjs_1.BehaviorSubject([]);
get testResultsSubject() { return this._testResults; }
get testResults() { return this._testResults.value; }
constructor({ models, parallel = false, printUpdates = false, verbose = false, defaultModel = false, tests, }) {
this.options = {
models,
parallel,
printUpdates,
verbose,
defaultModel,
tests,
};
if (printUpdates) {
this._testResults.subscribe(this.printUpdate);
}
}
printUpdate = () => {
const grid = {};
for (const r of this.testResults) {
const row = grid[r.model] ?? (grid[r.model] = {});
row['model'] = r.model;
row['ms'] = r.durationMs;
row[r.testName] = r.passed;
if (r.errorMessage) {
row[`${r.testName}_err`] = r.errorMessage.substring(0, 40);
}
}
const rows = Object.values(grid);
rows.sort((a, b) => a['model'].localeCompare(b['model']));
console.table(rows);
};
printResults(errorsOnly) {
for (const r of this.testResults) {
if (r.passed && errorsOnly) {
continue;
}
console.log(`\n--------------------- ${r.model}:${r.testName} ${r.passed ? 'passed' : 'failed'}`);
const p = { ...r };
delete p.debugOutput;
delete p.convo;
console.log(p);
if (r.debugOutput) {
console.log('<debugOutput>');
console.log(r.debugOutput.join('\n'));
console.log('</debugOutput>');
}
if (r.convo) {
console.log('convo>');
console.log(r.convo);
console.log('</convo>');
}
if (r.errorMessage) {
console.log('errorMessage>');
console.log(r.errorMessage);
console.log('</errorMessage>');
}
console.log(`\n--------------------- end ${r.model}:${r.testName} \n\n`);
}
}
testNames;
async runTestAsync() {
const testModelAsync = (model) => {
const tester = new ConvoModelTester_1.ConvoModelTester({
manager: this,
model: model.name,
tests: this.options.tests,
convoOptions: {
completionService: convo_lang_1.convoCompletionService.all(),
converters: convo_lang_1.convoConversationConverterProvider.all(),
},
});
if (!this.testNames) {
this.testNames = tester.getTestNames();
}
return tester.runAllTestsAsync();
};
try {
const models = [...this.options.models];
if (this.options.defaultModel) {
models.push({ name: convo_lang_1.convoAnyModelName });
}
if (this.options.parallel) {
await Promise.all(models.map(testModelAsync));
}
else {
for (const model of models) {
await testModelAsync(model);
}
}
this.testResults.sort((a, b) => `${a.model}:${a.testName}`.localeCompare(`${b.model}:${b.testName}`));
}
catch (ex) {
console.log(`testing failed`, ex);
process.exit(1);
}
}
publishResult(result) {
(0, common_1.pushBehaviorSubjectAry)(this._testResults, result);
}
}
exports.ConvoModelTestManager = ConvoModelTestManager;
//# sourceMappingURL=ConvoModelTestManager.js.map