@ovotech/genesys-web-messaging-tester-cli
Version:
185 lines (184 loc) • 11.3 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createAiTestCommand = void 0;
const commander_1 = require("commander");
const fs_1 = require("fs");
const genesys_web_messaging_tester_1 = require("@ovotech/genesys-web-messaging-tester");
const ui_1 = require("./ui");
const validateSessionConfig_1 = require("./validateSessionConfig");
const shouldEndConversation_1 = require("./prompt/shouldEndConversation");
const readableFileValidator_1 = require("../../fileSystem/readableFileValidator");
const yamlFileReader_1 = require("../../fileSystem/yamlFileReader");
const validatePromptScript_1 = require("./testScript/validatePromptScript");
const CommandExpectedlyFailedError_1 = require("../CommandExpectedlyFailedError");
const modelTypes_1 = require("./testScript/modelTypes");
const googleAi = __importStar(require("./chatCompletionClients/googleVertexAi/createChatCompletionClient"));
const openAi = __importStar(require("./chatCompletionClients/chatGpt/createChatCompletionClient"));
const validateOpenAiEnvVariables_1 = require("./chatCompletionClients/chatGpt/validateOpenAiEnvVariables");
const promptGenerator_1 = require("./prompt/generation/promptGenerator");
const writableDirValidator_1 = require("../../fileSystem/writableDirValidator");
const saveOutputJson_1 = require("./output/saveOutputJson");
const node_fs_1 = require("node:fs");
const validateProjectLocationConfig_1 = require("./chatCompletionClients/googleVertexAi/validateProjectLocationConfig");
function createAiTestCommand({ command = new commander_1.Command(), ui = new ui_1.Ui(), openAiCreateChatCompletionClient = openAi.createChatCompletionClient, googleAiCreateChatCompletionClient = googleAi.createChatCompletionClient, webMessengerSessionFactory = (config) => new genesys_web_messaging_tester_1.WebMessengerGuestSession(config, { IsAutomatedTest: 'true' }), conversationFactory = (session) => new genesys_web_messaging_tester_1.Conversation(session), processEnv = process.env, fsReadFileSync = fs_1.readFileSync, fsWriteFileSync = node_fs_1.writeFileSync, fsAccessSync = fs_1.accessSync, } = {}) {
const yamlFileReader = (0, yamlFileReader_1.createYamlFileReader)(fsReadFileSync);
if (!ui) {
throw new Error('UI must be defined');
}
return command
.command('ai')
.description('Perform testing using Generative AI')
.argument('<filePath>', 'Path of the YAML test-script file', (0, readableFileValidator_1.readableFileValidator)(fsAccessSync))
.option('-id, --deployment-id <deploymentId>', "Web Messenger Deployment's ID")
.option('-r, --region <region>', 'Region of Genesys instance that hosts the Web Messenger Deployment')
.option('-o, --out-dir <outDir>', 'Directory to store output', (0, writableDirValidator_1.writableDirValidator)(fsAccessSync))
.option('-o, --origin <origin>', 'Origin domain used for restricting Web Messenger Deployment')
.action(async (testScriptPath, options) => {
const outputConfig = command.configureOutput();
if (!outputConfig?.writeOut || !outputConfig?.writeErr) {
throw new Error('No writeOut and/or writeErr');
}
// 1. Read YAML file
let testScriptFileContents;
try {
testScriptFileContents = yamlFileReader(testScriptPath);
}
catch (error) {
outputConfig.writeErr(ui.errorReadingTestScriptFile(error));
throw new CommandExpectedlyFailedError_1.CommandExpectedlyFailedError();
}
// 2. Validate Test Script
const testScriptValidationResults = (0, validatePromptScript_1.validatePromptScript)(testScriptFileContents);
// TODO Update scenario validation object to match
if (testScriptValidationResults.error) {
outputConfig.writeErr(ui.validatingPromptScriptFailed(testScriptValidationResults.error));
throw new CommandExpectedlyFailedError_1.CommandExpectedlyFailedError();
}
// 3. Merge session config from args and Test Script - args take priority
const { validPromptScript } = testScriptValidationResults;
const mergedSessionConfig = {
deploymentId: options.deploymentId ?? validPromptScript?.config?.deploymentId,
region: options.region ?? validPromptScript?.config?.region,
origin: options.origin ?? validPromptScript?.config?.origin,
};
// 4. Validate session config
const sessionConfigValidationResults = (0, validateSessionConfig_1.validateSessionConfig)(mergedSessionConfig);
if (!sessionConfigValidationResults.validSessionConfig) {
outputConfig.writeErr(ui.validatingSessionConfigFailed(sessionConfigValidationResults.error));
throw new CommandExpectedlyFailedError_1.CommandExpectedlyFailedError();
}
let chatCompletionClient = null;
switch (validPromptScript.config.ai.provider) {
case modelTypes_1.SupportedAiProviders.ChatGpt: {
const openAiEnvValidationResult = (0, validateOpenAiEnvVariables_1.validateOpenAiEnvVariables)(processEnv);
if (!openAiEnvValidationResult.openAiKey) {
outputConfig.writeErr(ui.validatingOpenAiEnvValidationFailed(openAiEnvValidationResult.error));
throw new CommandExpectedlyFailedError_1.CommandExpectedlyFailedError();
}
const chatGptConfig = validPromptScript.config.ai.config ?? {};
chatCompletionClient = openAiCreateChatCompletionClient(chatGptConfig, openAiEnvValidationResult.openAiKey);
break;
}
case modelTypes_1.SupportedAiProviders.GoogleVertexAi: {
const googleAiConfig = validPromptScript.config.ai.config;
// Override location and project with environment variables
const projectLocationValidationResult = (0, validateProjectLocationConfig_1.validateProjectLocationConfig)({
project: processEnv['VERTEX_AI_PROJECT'] ?? googleAiConfig?.project,
location: processEnv['VERTEX_AI_LOCATION'] ?? googleAiConfig?.location,
});
if (!projectLocationValidationResult.value) {
outputConfig.writeErr(ui.validatingGcpProjectLocationConfigFailed(sessionConfigValidationResults.error));
throw new CommandExpectedlyFailedError_1.CommandExpectedlyFailedError();
}
chatCompletionClient = googleAiCreateChatCompletionClient({
...googleAiConfig,
...projectLocationValidationResult.value,
});
break;
}
default:
outputConfig.writeErr(ui.errorDeterminingAiProvider());
throw new CommandExpectedlyFailedError_1.CommandExpectedlyFailedError();
}
// 5. Preflight check of AI library
const preflightCheckResult = await chatCompletionClient.preflightCheck();
if (!preflightCheckResult.ok) {
outputConfig.writeErr(ui.preflightCheckFailure(chatCompletionClient.getProviderName(), preflightCheckResult));
throw new CommandExpectedlyFailedError_1.CommandExpectedlyFailedError();
}
const totalScenarios = Object.keys(validPromptScript?.scenarios).length;
if (totalScenarios > 1) {
outputConfig.writeErr(ui.onlyOnePromptSupported(totalScenarios));
throw new CommandExpectedlyFailedError_1.CommandExpectedlyFailedError();
}
const scenario = Object.entries(validPromptScript?.scenarios)[0][1];
const scenarioName = Object.entries(validPromptScript?.scenarios)[0][0];
const session = webMessengerSessionFactory(sessionConfigValidationResults.validSessionConfig);
new genesys_web_messaging_tester_1.SessionTranscriber(session, { nameForClient: 'AI', nameForServer: 'Chatbot' }).on('messageTranscribed', (msg) => {
outputConfig.writeOut(ui.messageTranscribed(msg));
});
const convo = conversationFactory(session);
const messages = [];
const generatedPrompt = (0, promptGenerator_1.promptGenerator)(scenario.setup);
if (generatedPrompt.placeholdersFound) {
outputConfig.writeOut(ui.displayPlaceholders(generatedPrompt));
outputConfig.writeOut(ui.conversationStartHeader());
}
let endConversation = {
hasEnded: false,
};
do {
const utterance = await chatCompletionClient.predict(generatedPrompt.prompt, messages);
if (utterance) {
messages.push(utterance);
await convo.sendText(utterance.content);
}
else {
messages.push({ role: 'customer', content: '' });
}
endConversation = (0, shouldEndConversation_1.shouldEndConversation)(messages, scenario.setup.terminatingPhrases.fail, scenario.setup.terminatingPhrases.pass);
if (!endConversation.hasEnded) {
// TODO Allow time to wait to be customised
const chatBotResponses = await convo.waitForResponses(3000);
messages.push({ role: 'bot', content: chatBotResponses.join('\n') });
}
endConversation = (0, shouldEndConversation_1.shouldEndConversation)(messages, scenario.setup.terminatingPhrases.fail, scenario.setup.terminatingPhrases.pass);
// TODO Handle bot ending conversation
} while (!endConversation.hasEnded);
session.close();
outputConfig.writeOut(ui.testResult(endConversation));
if (options.outDir) {
const outputResult = (0, saveOutputJson_1.saveOutputJs)(scenarioName, { messages, generatedPrompt, result: endConversation }, options.outDir, fsWriteFileSync);
if (!outputResult.successful) {
outputConfig.writeErr(ui.savingOutputFailed(outputResult));
}
}
if (endConversation.reason.type === 'fail') {
throw new CommandExpectedlyFailedError_1.CommandExpectedlyFailedError();
}
});
}
exports.createAiTestCommand = createAiTestCommand;