ai-ctrf
Version:
Generate AI summaries of test results using a wide range of AI models like OpenAI, Anthropic, Gemini, Mistral, Grok, DeepSeek, Azure, Perplexity, and OpenRouter
83 lines (82 loc) • 5.05 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.azureOpenAI = azureOpenAI;
exports.azureFailedTestSummary = azureFailedTestSummary;
const openai_1 = require("openai");
const common_1 = require("../common");
const consolidated_summary_1 = require("../consolidated-summary");
const constants_1 = require("../constants");
function azureOpenAI(systemPrompt, prompt, args) {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b;
const apiKey = process.env.AZURE_OPENAI_API_KEY;
const endpoint = process.env.AZURE_OPENAI_ENDPOINT;
const deployment = args.deploymentId || process.env.AZURE_OPENAI_DEPLOYMENT_NAME;
if (!apiKey || !endpoint || !deployment) {
console.error('Missing Azure OpenAI configuration. Please set AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, and AZURE_OPENAI_DEPLOYMENT_NAME environment variables or provide them as arguments.');
return null;
}
const client = new openai_1.AzureOpenAI({ endpoint, apiKey, deployment });
try {
const response = yield client.chat.completions.create(Object.assign(Object.assign({ model: "gpt-4o", messages: [
{ role: "system", content: systemPrompt },
{ role: "user", content: (0, common_1.stripAnsi)(prompt) },
], max_tokens: args.maxTokens || null, frequency_penalty: args.frequencyPenalty, presence_penalty: args.presencePenalty }, (args.temperature !== undefined ? { temperature: args.temperature } : {})), (args.topP !== undefined ? { top_p: args.topP } : {})));
return ((_b = (_a = response.choices[0]) === null || _a === void 0 ? void 0 : _a.message) === null || _b === void 0 ? void 0 : _b.content) || null;
}
catch (error) {
console.error(`Error invoking Azure OpenAI`, error);
return null;
}
});
}
function azureFailedTestSummary(report_1, args_1, file_1) {
return __awaiter(this, arguments, void 0, function* (report, args, file, log = false) {
const failedTests = report.results.tests.filter(test => test.status === 'failed');
failedTests.forEach(test => {
if (test.extra) {
delete test.extra;
}
});
let logged = false;
let messageCount = 0;
for (const test of failedTests) {
if (args.maxMessages && messageCount >= args.maxMessages) {
break;
}
const prompt = `Report:\n${JSON.stringify(test, null, 2)}.\n\nTool: ${report.results.tool.name}.\n\nPlease provide a human-readable failure summary that explains why you think the test might have failed and ways to fix it.`;
const systemPrompt = args.systemPrompt || constants_1.FAILED_TEST_SUMMARY_SYSTEM_PROMPT_CURRENT;
const response = yield azureOpenAI(systemPrompt, prompt, args);
if (response) {
test.ai = response;
messageCount++;
if (args.log && !logged) {
console.log(`\n─────────────────────────────────────────────────────────────────────────────────────────────────────────────`);
console.log(`✨ AI Test Reporter Summary`);
console.log(`─────────────────────────────────────────────────────────────────────────────────────────────────────────────\n`);
logged = true;
}
if (args.log) {
console.log(`❌ Failed Test: ${test.name}\n`);
console.log(`${response}\n`);
}
}
}
if (args.consolidate) {
yield (0, consolidated_summary_1.generateConsolidatedSummary)(report, "azure", args);
}
if (file) {
(0, common_1.saveUpdatedReport)(file, report);
}
return report;
});
}