ai-ctrf
Version:
Generate AI summaries of test results using a wide range of AI models like OpenAI, Anthropic, Gemini, Mistral, Grok, DeepSeek, Azure, Perplexity, and OpenRouter
100 lines (99 loc) • 6.5 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.generateConsolidatedSummary = generateConsolidatedSummary;
const openai_1 = require("./models/openai");
const claude_1 = require("./models/claude");
const azure_openai_1 = require("./models/azure-openai");
const grok_1 = require("./models/grok");
const deepseek_1 = require("./models/deepseek");
const gemini_1 = require("./models/gemini");
const perplexity_1 = require("./models/perplexity");
const openrouter_1 = require("./models/openrouter");
const bedrock_1 = require("./models/bedrock");
// import { CONSOLIDATED_SUMMARY_SYSTEM_PROMPT } from "./constants";
function generateConsolidatedSummary(report, model, args) {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
const failedTests = report.results.tests.filter((test) => test.status === 'failed');
const aiSummaries = [];
for (const test of failedTests) {
if (test.ai != null && test.ai.trim() !== '') {
aiSummaries.push(`Test Name: ${test.name}\nAI Summary: ${test.ai}\n`);
}
}
const systemPrompt = `You are tasked with summarizing the results of a test run that contains test failures. Your goal is to provide a concise, high-level overview of what went wrong in the test run. Focus on identifying patterns or root causes that might explain why these tests failed. Keep the summary brief and informative, without repeating the test details or providing step-by-step instructions. Avoid unnecessary verbosity and focus on delivering actionable insights.
Avoid:
- Including any code in your response.
- Adding generic conclusions or advice such as "By following these steps..."
- headings, bullet points, or special formatting.`;
const consolidatedPrompt = `The following tests failed in the suite:\n\n${aiSummaries.join('\n')}\n\nA total of ${failedTests.length} tests failed in this test suite. Please provide a high-level summary of what went wrong across the suite and suggest what might be the root causes or patterns.`;
// const systemPrompt = CONSOLIDATED_SUMMARY_SYSTEM_PROMPT;
// const consolidatedPrompt = `Analyze these ${failedTests.length} test failures from our test suite:
// Test Environment: ${report.results.environment || 'Not specified'}
// Test Tool: ${report.results.tool.name}
// Total Tests Run: ${report.results.tests.length}
// Failed Tests: ${failedTests.length}
// Failed Test Details:
// ${aiSummaries.join("\n")}
// Key Questions to Address:
// 1. What patterns or common themes exist across these failures?
// 3. Is there evidence of a broader system change causing these failures?
// 4. How should I prioritize the issues to address based on the failure patterns?
// Please provide a high-level analysis of the test suite failures, focusing on systemic issues and patterns.`;
let consolidatedSummary = '';
if (model === 'openai') {
consolidatedSummary =
(_a = (yield (0, openai_1.openAI)(systemPrompt, consolidatedPrompt, args))) !== null && _a !== void 0 ? _a : '';
}
else if (model === 'claude') {
consolidatedSummary =
(_b = (yield (0, claude_1.claudeAI)(systemPrompt, consolidatedPrompt, args))) !== null && _b !== void 0 ? _b : '';
}
else if (model === 'azure') {
consolidatedSummary =
(_c = (yield (0, azure_openai_1.azureOpenAI)(systemPrompt, consolidatedPrompt, args))) !== null && _c !== void 0 ? _c : '';
}
else if (model === 'grok') {
consolidatedSummary =
(_d = (yield (0, grok_1.grokAI)(systemPrompt, consolidatedPrompt, args))) !== null && _d !== void 0 ? _d : '';
}
else if (model === 'deepseek') {
consolidatedSummary =
(_e = (yield (0, deepseek_1.deepseekAI)(systemPrompt, consolidatedPrompt, args))) !== null && _e !== void 0 ? _e : '';
}
else if (model === 'gemini') {
consolidatedSummary =
(_f = (yield (0, gemini_1.gemini)(systemPrompt, consolidatedPrompt, args))) !== null && _f !== void 0 ? _f : '';
}
else if (model === 'perplexity') {
consolidatedSummary =
(_g = (yield (0, perplexity_1.perplexity)(systemPrompt, consolidatedPrompt, args))) !== null && _g !== void 0 ? _g : '';
}
else if (model === 'openrouter') {
consolidatedSummary =
(_h = (yield (0, openrouter_1.openRouter)(systemPrompt, consolidatedPrompt, args))) !== null && _h !== void 0 ? _h : '';
}
else if (model === 'bedrock') {
consolidatedSummary =
(_j = (yield (0, bedrock_1.bedrock)(systemPrompt, consolidatedPrompt, args))) !== null && _j !== void 0 ? _j : '';
}
if (consolidatedSummary !== '') {
if (args.log === true) {
console.log(`\n─────────────────────────────────────────────────────────────────────────────────────────────────────────────\n`);
console.log(`📝 Overall Summary:\n`);
console.log(`${consolidatedSummary}\n`);
}
report.results.extra = (_k = report.results.extra) !== null && _k !== void 0 ? _k : {};
report.results.extra.ai = consolidatedSummary;
}
});
}