@microsoft/teams-ai
Version:
SDK focused on building AI based applications for Microsoft Teams.
234 lines • 10.2 kB
JavaScript
;
/**
* @module teams-ai
*/
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.AzureContentSafetyModerator = exports.ModerationSeverity = void 0;
const internals_1 = require("../internals");
const AI_1 = require("../AI");
const OpenAIModerator_1 = require("./OpenAIModerator");
// Export the moderation severity
var internals_2 = require("../internals");
Object.defineProperty(exports, "ModerationSeverity", { enumerable: true, get: function () { return internals_2.ModerationSeverity; } });
const defaultHarmCategories = ['Hate', 'Sexual', 'SelfHarm', 'Violence'];
/**
* An Azure OpenAI moderator that uses OpenAI's moderation API to review prompts and plans for safety.
* @remarks
* This moderation can be configured to review the input from the user, output from the model, or both.
* @template TState Optional. Type of the applications turn state.
*/
class AzureContentSafetyModerator extends OpenAIModerator_1.OpenAIModerator {
_contentSafetyOptions;
_azureContentSafetyClient;
_azureContentSafetyCategories = {};
/**
* Creates a new instance of the OpenAI based moderator.
* @param {AzureOpenAIModeratorOptions} options Configuration options for the moderator.
*/
constructor(options) {
// Create the moderator options
const moderatorOptions = {
apiKey: options.apiKey,
moderate: options.moderate ?? 'both',
endpoint: options.endpoint,
apiVersion: options.apiVersion,
organization: options.organization,
model: options.model
};
super(moderatorOptions);
// Create the Azure OpenAI Content Safety client
this._azureContentSafetyClient = this.createClient(moderatorOptions);
// Construct the content safety categories
let categories = [];
if (options.categories) {
options.categories.forEach((category) => {
categories.push(category.category);
this._azureContentSafetyCategories[category.category] = category;
});
}
else {
categories = Object.assign([], defaultHarmCategories);
this._azureContentSafetyCategories.Hate = {
category: 'Hate',
severity: 6
};
this._azureContentSafetyCategories.Sexual = {
category: 'Sexual',
severity: 6
};
this._azureContentSafetyCategories.SelfHarm = {
category: 'SelfHarm',
severity: 6
};
this._azureContentSafetyCategories.Violence = {
category: 'Violence',
severity: 6
};
}
// Create the content safety request
this._contentSafetyOptions = {
categories: categories ?? defaultHarmCategories,
blocklistNames: options.blocklistNames ?? [],
haltOnBlocklistHit: options.haltOnBlocklistHit ?? options.breakByBlocklists ?? false
};
}
/**
* Creates a new instance of the Azure OpenAI client.
* @protected
* @param {OpenAIModeratorOptions} options The options for the moderator.
* @returns {AzureOpenAIClient} The Azure OpenAI client.
*/
createClient(options) {
return new internals_1.AzureOpenAIClient({
apiKey: options.apiKey,
endpoint: options.endpoint,
apiVersion: options.apiVersion ?? '2023-10-01',
headerKey: 'Ocp-Apim-Subscription-Key'
});
}
/**
* @protected
* @param {string} input The input to moderate.
* @returns {Promise<CreateModerationResponseResultsInner | undefined>} The moderation results.
* This method is called by the moderator to moderate the input.
* @template TState Optional. Type of the applications turn state.
*/
async createModeration(input) {
const res = (await this._azureContentSafetyClient.createModeration({
text: input,
...this._contentSafetyOptions
}));
if (!res.data) {
return undefined;
}
const predicate = (category) => {
return (c) => {
return (c.category === category &&
c.severity > 0 &&
c.severity <= this._azureContentSafetyCategories[category].severity);
};
};
const hate = res.data.categoriesAnalysis.find(predicate('Hate'));
const selfHarm = res.data.categoriesAnalysis.find(predicate('SelfHarm'));
const sexual = res.data.categoriesAnalysis.find(predicate('Sexual'));
const violence = res.data.categoriesAnalysis.find(predicate('Violence'));
// Create the moderation results
const result = {
flagged: !!hate || !!selfHarm || !!sexual || !!violence,
categories: {
hate: !!hate,
'hate/threatening': !!hate,
'self-harm': !!selfHarm,
sexual: !!sexual,
'sexual/minors': !!sexual,
violence: !!violence,
'violence/graphic': !!violence
},
category_scores: {
// Normalize the scores to be between 0 and 1
hate: (hate?.severity ?? 0) / internals_1.ModerationSeverity.High,
'hate/threatening': (hate?.severity ?? 0) / internals_1.ModerationSeverity.High,
'self-harm': (selfHarm?.severity ?? 0) / internals_1.ModerationSeverity.High,
sexual: (sexual?.severity ?? 0) / internals_1.ModerationSeverity.High,
'sexual/minors': (sexual?.severity ?? 0) / internals_1.ModerationSeverity.High,
violence: (violence?.severity ?? 0) / internals_1.ModerationSeverity.High,
'violence/graphic': (violence?.severity ?? 0) / internals_1.ModerationSeverity.High
}
};
return result;
}
/**
* Reviews an incoming prompt for safety violations.
* @param {TurnContext} context - Context for the current turn of conversation.
* @param {TState} state - Application state for the current turn of conversation.
* @returns {Promise<Plan | undefined>} An undefined value to approve the prompt or a new plan to redirect to if not approved.
*/
async reviewInput(context, state) {
switch (this.options.moderate) {
case 'input':
case 'both': {
const input = state.temp.input ?? context.activity.text;
const result = await this.createModeration(input);
if (result) {
if (result.flagged) {
// Input flagged
// console.info(`ReviewPrompt: Azure Content Safety Result: ${JSON.stringify(result)}`);
return {
type: 'plan',
commands: [
{
type: 'DO',
action: AI_1.AI.FlaggedInputActionName,
parameters: result
}
]
};
}
}
else {
// Rate limited
return {
type: 'plan',
commands: [{ type: 'DO', action: AI_1.AI.HttpErrorActionName, parameters: {} }]
};
}
break;
}
}
return undefined;
}
/**
* Reviews the SAY commands generated by the planner for safety violations.
* @param {TurnContext} context - Context for the current turn of conversation.
* @param {TState} state - Application state for the current turn of conversation.
* @param {Plan} plan - Plan generated by the planner.
* @returns {Promise<Plan>} The plan to execute. Either the current plan passed in for review or a new plan.
*/
async reviewOutput(context, state, plan) {
switch (this.options.moderate) {
case 'output':
case 'both':
for (let i = 0; i < plan.commands.length; i++) {
const cmd = plan.commands[i];
if (cmd.type == 'SAY') {
const predictedSayCommand = cmd;
const output = predictedSayCommand.response;
const result = await this.createModeration(output.content || '');
if (result) {
if (result.flagged) {
// Output flagged
// console.info(`ReviewPlan: Azure Content Safety Result: ${JSON.stringify(result)}`);
return {
type: 'plan',
commands: [
{
type: 'DO',
action: AI_1.AI.FlaggedOutputActionName,
parameters: result
}
]
};
}
}
else {
// Rate limited
return {
type: 'plan',
commands: [
{ type: 'DO', action: AI_1.AI.HttpErrorActionName, parameters: {} }
]
};
}
}
}
break;
}
return plan;
}
}
exports.AzureContentSafetyModerator = AzureContentSafetyModerator;
//# sourceMappingURL=AzureContentSafetyModerator.js.map