@convo-lang/convo-lang
Version:
The language of AI
267 lines (252 loc) • 8.83 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ConvoModelTester = void 0;
const convo_lang_1 = require("@convo-lang/convo-lang");
const common_1 = require("@iyio/common");
const json5_1 = require("@iyio/json5");
const rxjs_1 = require("rxjs");
class ConvoModelTester {
_testResults = new rxjs_1.BehaviorSubject([]);
get testResultsSubject() { return this._testResults; }
get testResults() { return this._testResults.value; }
options;
constructor({ initConvo, convoOptions = {}, model, skipDefineModel = false, manager, tests, }) {
this.options = {
initConvo,
convoOptions,
model,
skipDefineModel,
manager,
tests,
};
}
async runAllTestsAsync() {
for (const name of this.getTestNames()) {
await this['test_' + name]();
}
}
getTestNames() {
const names = [];
for (const e in this) {
if (e.startsWith('test_')) {
names.push(e.substring(5));
}
}
if (this.options.tests) {
return names.filter(t => this.options.tests?.includes(t));
}
else {
return names;
}
}
createDebugger = (output) => (...values) => {
for (const v of values) {
try {
const type = typeof v;
if (type === 'string') {
output.push(v);
}
else if (type === 'object') {
try {
output.push(JSON.stringify(v, null, 4));
}
catch {
try {
output.push(JSON.stringify(v, (0, common_1.createJsonRefReplacer)(), 4));
}
catch {
output.push(v + '');
}
}
}
else {
output.push(v + '');
}
}
catch {
output.push('!! Unable to push debug output');
}
}
};
createConvo(debug, convoOptions) {
const convo = new convo_lang_1.Conversation({
disableAutoFlatten: true,
debug,
...this.options.convoOptions,
...convoOptions
});
if (!this.options.skipDefineModel && this.options.model !== convo_lang_1.convoAnyModelName) {
convo.append(`> define\n__model=${JSON.stringify(this.options.model)}`);
}
convo.append(`> define\n__debug=true\n__trackModel=true`);
if (this.options.initConvo) {
convo.append(this.options.initConvo);
}
return convo;
}
async runTestAsync({ name, expectedRoles, returnCount, format, isJsonArray, disablePublishResult, callFunction, functionCall = callFunction ? true : false, returnValue, convoOptions, }, source) {
const debugOutput = [];
const log = this.createDebugger(debugOutput);
log(`## test start ${this.options.model}:${name}`);
const convo = this.createConvo(log, convoOptions);
const startTime = Date.now();
try {
convo.append(source);
const r = await convo.completeAsync();
const last = r.message;
if (expectedRoles) {
for (let i = 0; i < expectedRoles.length; i++) {
const role = expectedRoles[i];
const msg = r.messages[i];
if (!msg) {
throw new Error(`Returned result does not have a message at index ${i} for checking role`);
}
if (role === null || role === undefined) {
continue;
}
if (msg.role !== role) {
throw new Error(`Expected returned message at index (${i}) to have a role of ${msg.role}`);
}
}
if (expectedRoles.length > r.messages.length) {
throw new Error(`Expected returned messages has more messages that expected roles`);
}
}
if (returnCount !== undefined && returnCount !== r.messages.length) {
throw new Error(`Expected (${returnCount}) returned messages but found (${r.messages.length})`);
}
if (format && last?.format !== format) {
throw new Error(`Expected returned message to have a format of (${format})`);
}
if (format === null && last?.format) {
throw new Error('Expected returned message to not have a format');
}
const jsonValue = last?.format === 'json' ? (0, json5_1.parseJson5)(last.content ?? '') : undefined;
if (isJsonArray && !Array.isArray(jsonValue)) {
throw new Error(`Expected JSON array`);
}
if (callFunction && r.lastFnCall?.name !== callFunction) {
throw new Error(`Expected (${callFunction}) function to be called`);
}
if (functionCall && !r.lastFnCall) {
throw new Error(`Expected function to be called`);
}
if (returnValue !== undefined && returnValue !== r.lastFnCall?.returnValue) {
throw new Error(`Expected return value was not returned.\nExpected:${JSON.stringify(returnValue)}\nReturned:${JSON.stringify(r.lastFnCall?.returnValue)}`);
}
const endConvo = convo.convo;
log(`## test pass ${this.options.model}:${name}`);
const result = {
model: this.options.model,
testName: name,
convo: endConvo,
passed: true,
debugOutput,
durationMs: Date.now() - startTime,
};
if (!disablePublishResult) {
this.publishResult(result);
}
return result;
}
catch (ex) {
const endConvo = convo.convo;
log(`## test failed ${this.options.model}:${name}`);
const errorMessage = (0, common_1.getErrorMessage)(ex);
const result = {
model: this.options.model,
testName: name,
convo: endConvo,
passed: false,
errorMessage,
errorStack: ex?.stack,
debugOutput,
durationMs: Date.now() - startTime,
};
if (!disablePublishResult) {
this.publishResult(result);
}
return result;
}
}
publishResult(result) {
(0, common_1.pushBehaviorSubjectAry)(this._testResults, result);
this.options.manager?.publishResult(result);
}
test_sayHi = () => {
return this.runTestAsync({
name: 'sayHi',
description: 'Test to see if a simple message of hi completes',
format: null,
expectedRoles: ['assistant']
}, /*convo*/ `
> user
hi
`);
};
test_jsonSchema = () => {
return this.runTestAsync({
name: 'jsonSchema',
description: 'Test JSON mode using a schema',
format: 'json',
}, /*convo*/ `
> define
Car=struct(
name: string
topSpeedMPH: number
color: string
)
@json Car
> user
A fast red car
`);
};
test_jsonArray = () => {
return this.runTestAsync({
name: 'jsonArray',
description: 'Test JSON mode for returning an array',
format: 'json',
isJsonArray: true,
}, /*convo*/ `
> define
Planet=struct(
name: string
# Distance from sun in miles
distanceFromSun: number;
)
@json Planet[]
> user
List the planets in our solar system
`);
};
test_callAddNumbers = () => {
return this.runTestAsync({
name: 'callAddNumbers',
description: 'Calls a functions to add numbers',
callFunction: 'addNumbers',
returnValue: 19,
}, /*convo*/ `
> addNumbers(
a:number
b:number
) -> (
return(add(a b))
)
> user
Add 7 plus 12 by calling the addNumbers function
`);
};
test_assistantFirst = () => {
return this.runTestAsync({
name: 'assistantFirst',
description: 'Starts a conversation with an assistant message first',
}, /*convo*/ `
> assistant
How can I help you?
> user
What color is the sky on a sunny day?
`);
};
}
exports.ConvoModelTester = ConvoModelTester;
//# sourceMappingURL=ConvoModelTester.js.map