@microsoft/teams-ai
Version:
SDK focused on building AI based applications for Microsoft Teams.
380 lines • 19.6 kB
JavaScript
"use strict";
/**
* @module teams-ai
*/
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.LLMClient = void 0;
const internals_1 = require("../internals");
const MemoryFork_1 = require("../MemoryFork");
const prompts_1 = require("../prompts");
const StreamingResponse_1 = require("../StreamingResponse");
const tokenizers_1 = require("../tokenizers");
const validators_1 = require("../validators");
/**
* LLMClient class that's used to complete prompts.
* @remarks
* Each wave, at a minimum needs to be configured with a `client`, `prompt`, and `prompt_options`.
*
* Configuring the wave to use a `validator` is optional but recommended. The primary benefit to
* using LLMClient is it's response validation and automatic response repair features. The
* validator acts as guard and guarantees that you never get an malformed response back from the
* model. At least not without it being flagged as an `invalid_response`.
*
* Using the `JSONResponseValidator`, for example, guarantees that you only ever get a valid
* object back from `completePrompt()`. In fact, you'll get back a fully parsed object and any
* additional response text from the model will be dropped. If you give the `JSONResponseValidator`
* a JSON Schema, you will get back a strongly typed and validated instance of an object in
* the returned `response.message.content`.
*
* When a validator detects a bad response from the model, it gives the model "feedback" as to the
* problem it detected with its response and more importantly an instruction that tells the model
* how it should repair the problem. This puts the wave into a special repair mode where it first
* forks the memory for the conversation and then has a side conversation with the model in an
* effort to get it to repair its response. By forking the conversation, this isolates the bad
* response and prevents it from contaminating the main conversation history. If the response can
* be repaired, the wave will un-fork the memory and use the repaired response in place of the
* original bad response. To the model it's as if it never made a mistake which is important for
* future turns with the model. If the response can't be repaired, a response status of
* `invalid_response` will be returned.
*
* When using a well designed validator, like the `JSONResponseValidator`, the wave can typically
* repair a bad response in a single additional model call. Sometimes it takes a couple of calls
* to effect a repair and occasionally it won't be able to repair it at all. If your prompt is
* well designed and you only occasionally see failed repair attempts, I'd recommend just calling
* the wave a second time. Given the stochastic nature of these models, there's a decent chance
* it won't make the same mistake on the second call. A well designed prompt coupled with a well
* designed validator should get the reliability of calling these models somewhere close to 99%
* reliable.
*
* This "feedback" technique works with all the GPT-3 generation of models and I've tested it with
* `text-davinci-003`, `gpt-3.5-turbo`, and `gpt-4`. There's a good chance it will work with other
* open source models like `LLaMA` and Googles `Bard` but I have yet to test it with those models.
*
* LLMClient supports OpenAI's functions feature and can validate the models response against the
* schema for the supported functions. When an LLMClient is configured with both a `OpenAIModel`
* and a `FunctionResponseValidator`, the model will be cloned and configured to send the
* validators configured list of functions with the request. There's no need to separately
* configure the models `functions` list, but if you do, the models functions list will be sent
* instead.
* @template TContent Optional. Type of message content returned for a 'success' response. The `response.message.content` field will be of type TContent. Defaults to `any`.
*/
class LLMClient {
_startStreamingMessage;
_endStreamHandler;
_enableFeedbackLoop;
_feedbackLoopType;
/**
* Configured options for this LLMClient instance.
*/
options;
/**
* Creates a new `LLMClient` instance.
* @param {LLMClientOptions<TContent>} options - Options to configure the instance with.
*/
constructor(options) {
this.options = Object.assign({
history_variable: 'conversation.history',
input_variable: 'temp.input',
max_history_messages: 10,
max_repair_attempts: 3,
logRepairs: false
}, options);
// Create validator to use
if (!this.options.validator) {
this.options.validator = new validators_1.DefaultResponseValidator();
}
// Create tokenizer to use
if (!this.options.tokenizer) {
this.options.tokenizer = new tokenizers_1.GPTTokenizer();
}
this._startStreamingMessage = options.startStreamingMessage;
this._endStreamHandler = options.endStreamHandler;
this._enableFeedbackLoop = options.enableFeedbackLoop;
this._feedbackLoopType = options.feedbackLoopType;
}
/**
* Completes a prompt.
* @remarks
* The `input` parameter is optional but if passed in, will be assigned to memory using the
* configured `input_variable`. If it's not passed in an attempt will be made to read it
* from memory so passing it in or assigning to memory works. In either case, the `input`
* variable is only used when constructing a user message that, will be added to the
* conversation history and formatted like `{ role: 'user', content: input }`.
*
* It's important to note that if you want the users input sent to the model as part of the
* prompt, you will need to add a `UserMessage` section to your prompt. The wave does not do
* anything to modify your prompt, except when performing repairs and those changes are
* temporary.
*
* When the model successfully returns a valid (or repaired) response, a 'user' message (if
* input was detected) and 'assistant' message will be automatically added to the conversation
* history. You can disable that behavior by setting `max_history_messages` to `0`.
*
* The response returned by `completePrompt()` will be strongly typed by the validator you're
* using. The `DefaultResponseValidator` returns a `string` and the `JSONResponseValidator`
* will return either an `object` or if a JSON Schema is provided, an instance of `TContent`.
* When using a custom validator, the validator is return any type of content it likes.
*
* A successful response is indicated by `response.status == 'success'` and the content can be
* accessed via `response.message.content`. If a response is invalid it will have a
* `response.status == 'invalid_response'` and the `response.message` will be a string containing
* the validator feedback message. There are other status codes for various errors and in all
* cases except `success` the `response.message` will be of type `string`.
* @template TContent Optional. Type of message content returned for a 'success' response. The `response.message.content` field will be of type TContent. Defaults to `any`.
* @param {TurnContext} context - Current turn context.
* @param {Memory} memory - An interface for accessing state values.
* @param {PromptFunctions} functions - Functions to use when rendering the prompt.
* @returns {Promise<PromptResponse<TContent>>} A `PromptResponse` with the status and message.
*/
async completePrompt(context, memory, functions) {
// Define event handlers
let streamer;
const beforeCompletion = (ctx, memory, functions, tokenizer, template, streaming) => {
// Ignore events for other contexts
if (context !== ctx) {
return;
}
// Check for a streaming response
if (streaming) {
// Attach to any existing streamer
// - see tool call note below to understand.
streamer = memory.getValue('temp.streamer');
if (!streamer) {
// Create streamer and send initial message
streamer = new StreamingResponse_1.StreamingResponse(context);
memory.setValue('temp.streamer', streamer);
if (this._enableFeedbackLoop != null) {
streamer.setFeedbackLoop(this._enableFeedbackLoop);
if (this._feedbackLoopType) {
streamer.setFeedbackLoopType(this._feedbackLoopType);
}
}
streamer.setGeneratedByAILabel(true);
if (this._startStreamingMessage) {
streamer.queueInformativeUpdate(this._startStreamingMessage);
}
}
}
};
const chunkReceived = (ctx, memory, chunk) => {
// Ignore events for other contexts
if (context !== ctx || !streamer) {
return;
}
const citations = chunk.delta?.context?.citations ?? undefined;
if (citations) {
streamer.setCitations(citations);
}
// Ignore calls without content
// - This is typically because the chunk represents a tool call.
// - See the note below for why we're handling tool calls this way.
if (!chunk.delta?.content) {
return;
}
// Send text chunk to client
const text = chunk.delta?.content;
if (text.length > 0) {
streamer.queueTextChunk(text);
}
};
// Subscribe to model events
if (this.options.model.events) {
this.options.model.events.on('beforeCompletion', beforeCompletion);
this.options.model.events.on('chunkReceived', chunkReceived);
if (this._endStreamHandler) {
this.options.model.events.on('responseReceived', this._endStreamHandler);
}
}
try {
// Complete the prompt
const response = await this.callCompletePrompt(context, memory, functions);
// Handle streaming responses
if (streamer) {
// Tool call handling
// - We need to keep the streamer around during tool calls so we're just letting them return as normal
// messages minus the message content. The text content is being streamed to the client in chunks.
// - When the tool call completes we'll call back into ActionPlanner and end up re-attaching to the
// streamer. This will result in us continuing to stream the response to the client.
if (Array.isArray(response.message?.action_calls)) {
// Ensure content is empty for tool calls
response.message.content = '';
}
else {
if (response.status == 'success') {
// Delete message from response to avoid sending it twice
delete response.message;
}
// End the stream and remove pointer from memory
// - We're not listening for the response received event because we can't await the completion of events.
await streamer.endStream();
memory.deleteValue('temp.streamer');
}
}
return response;
}
finally {
// Unsubscribe from model events
if (this.options.model.events) {
this.options.model.events.off('beforeCompletion', beforeCompletion);
this.options.model.events.off('chunkReceived', chunkReceived);
if (this._endStreamHandler) {
this.options.model.events.off('responseReceived', this._endStreamHandler);
}
}
}
}
/**
* @param {TurnContext} context - Current turn context.
* @param {Memory} memory - An interface for accessing state values.
* @param {PromptFunctions} functions - Functions to use when rendering the prompt.
* @returns {Promise<PromptResponse<TContent>>} A `PromptResponse` with the status and message.
* @private
*/
async callCompletePrompt(context, memory, functions) {
const { model, template, tokenizer, validator, max_repair_attempts, history_variable, input_variable } = this.options;
try {
// Ask client to complete prompt
const response = (await model.completePrompt(context, memory, functions, tokenizer, template));
if (response.status !== 'success') {
// The response isn't valid so we don't care that the messages type is potentially incorrect.
return response;
}
// Get input message
let inputMsg = response.input;
if (!inputMsg && input_variable) {
const content = memory.getValue(input_variable) ?? '';
inputMsg = { role: 'user', content };
}
// Validate response
const validation = await validator.validateResponse(context, memory, tokenizer, response, max_repair_attempts);
if (validation.valid) {
// Update content
if (Object.prototype.hasOwnProperty.call(validation, 'value')) {
response.message.content = validation.value;
}
// Update history and return
this.addMessageToHistory(memory, history_variable, inputMsg);
this.addMessageToHistory(memory, history_variable, response.message);
return response;
}
// Bail out if we're not repairing
if (max_repair_attempts <= 0) {
return response;
}
// Fork the conversation history
const fork = new MemoryFork_1.MemoryFork(memory);
// Log repair attempts
if (this.options.logRepairs) {
console.log(internals_1.Colorize.title('REPAIRING RESPONSE:'));
console.log(internals_1.Colorize.output(response.message.content ?? ''));
}
// Attempt to repair response
const repair = await this.repairResponse(context, fork, functions, response, validation, max_repair_attempts);
// Log repair success
if (this.options.logRepairs) {
if (repair.status === 'success') {
console.log(internals_1.Colorize.success('Response Repaired'));
}
else {
console.log(internals_1.Colorize.error('Response Repair Failed'));
}
}
// Update history with repaired response if successful.
// - conversation history will be left unchanged if the repair failed.
// - we never want to save an invalid response to conversation history.
// - the caller can take further corrective action, including simply re-trying.
if (repair.status === 'success') {
this.addMessageToHistory(memory, history_variable, inputMsg);
this.addMessageToHistory(memory, history_variable, repair.message);
}
return repair;
}
catch (err) {
return {
status: 'error',
input: undefined,
error: err
};
}
}
/**
* @param {Memory} memory - Current memory.
* @param {string} variable - Variable to fetch value from memory.
* @param {Message<any> | Message<any>[]} message - The Message to be added to history.
* @private
*/
addMessageToHistory(memory, variable, message) {
if (variable) {
const history = memory.getValue(variable) ?? [];
if (Array.isArray(message)) {
history.push(...message);
}
else {
history.push(message);
}
if (history.length > this.options.max_history_messages) {
history.splice(0, history.length - this.options.max_history_messages);
}
while (history.length > 0 && history[0].role === 'tool') {
history.shift();
}
memory.setValue(variable, history);
}
}
/**
* @param {TurnContext} context - The current TurnContext
* @param {MemoryFork} fork - The current fork of memory to be repaired.
* @param {PromptFunctions} functions - Functions to use.
* @param {PromptResponse<TContent>} response - The response that needs repairing.
* @param {Validation} validation - The Validation object to be used during repair.
* @param {number} remaining_attempts - The number of remaining attempts.
* @returns {Promise<PromptResponse<TContent>>} - The repaired response.
* @private
*/
async repairResponse(context, fork, functions, response, validation, remaining_attempts) {
const { model, template, tokenizer, validator, history_variable } = this.options;
// Add response and feedback to repair history
const feedback = validation.feedback ?? 'The response was invalid. Try another strategy.';
this.addMessageToHistory(fork, `${history_variable}-repair`, response.message);
this.addMessageToHistory(fork, `${history_variable}-repair`, { role: 'user', content: feedback });
// Append repair history to prompt
const repairTemplate = Object.assign({}, template, {
prompt: new prompts_1.Prompt([template.prompt, new prompts_1.ConversationHistory(`${history_variable}-repair`)])
});
// Log the repair
if (this.options.logRepairs) {
console.log(internals_1.Colorize.value('feedback', feedback));
}
// Ask client to complete prompt
const repairResponse = (await model.completePrompt(context, fork, functions, tokenizer, repairTemplate));
if (repairResponse.status !== 'success') {
return repairResponse;
}
// Validate response
validation = await validator.validateResponse(context, fork, tokenizer, repairResponse, remaining_attempts);
if (validation.valid) {
// Update content
if (Object.prototype.hasOwnProperty.call(validation, 'value')) {
repairResponse.message.content = validation.value;
}
return repairResponse;
}
// Are we out of attempts?
remaining_attempts--;
if (remaining_attempts <= 0) {
return {
status: 'invalid_response',
input: undefined,
error: new Error(`Reached max model response repair attempts. Last feedback given to model: "${feedback}"`)
};
}
// Try next attempt
return await this.repairResponse(context, fork, functions, repairResponse, validation, remaining_attempts);
}
}
exports.LLMClient = LLMClient;
//# sourceMappingURL=LLMClient.js.map