ai-ctrf
Version:
Generate AI summaries of test results using a wide range of AI models like OpenAI, Anthropic, Gemini, Mistral, Grok, DeepSeek, Azure, Perplexity, OpenRouter, and custom OpenAI-compatible APIs
102 lines (101 loc) • 6.05 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.openRouter = openRouter;
exports.openRouterFailedTestSummary = openRouterFailedTestSummary;
const openai_1 = __importDefault(require("openai"));
const common_1 = require("../common");
const consolidated_summary_1 = require("../consolidated-summary");
const constants_1 = require("../constants");
function openRouter(systemPrompt, prompt, args) {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b, _c, _d, _e, _f, _g;
const client = new openai_1.default({
apiKey: process.env.OPENROUTER_API_KEY,
baseURL: 'https://openrouter.ai/api/v1',
defaultHeaders: {
'HTTP-Referer': (_a = process.env.OPENROUTER_REFERER) !== null && _a !== void 0 ? _a : 'http://localhost:3000',
'X-Title': 'AI Test Reporter',
},
});
try {
const response = yield client.chat.completions.create(Object.assign(Object.assign(Object.assign({ model: (_b = args.model) !== null && _b !== void 0 ? _b : 'openai/gpt-3.5-turbo', messages: [
{ role: 'system', content: systemPrompt },
{ role: 'user', content: (0, common_1.stripAnsi)(prompt) },
], max_tokens: (_c = args.maxTokens) !== null && _c !== void 0 ? _c : undefined, temperature: (_d = args.temperature) !== null && _d !== void 0 ? _d : undefined }, (args.frequencyPenalty !== undefined && {
frequency_penalty: args.frequencyPenalty,
})), (args.presencePenalty !== undefined && {
presence_penalty: args.presencePenalty,
})), (args.topP !== undefined && { top_p: args.topP })));
if (((_g = (_f = (_e = response === null || response === void 0 ? void 0 : response.choices) === null || _e === void 0 ? void 0 : _e[0]) === null || _f === void 0 ? void 0 : _f.message) === null || _g === void 0 ? void 0 : _g.content) == null) {
return null;
}
return response.choices[0].message.content;
}
catch (error) {
console.error(`Error invoking OpenRouter`, error);
return null;
}
});
}
function openRouterFailedTestSummary(report_1, args_1, file_1) {
return __awaiter(this, arguments, void 0, function* (report, args, file, log = false) {
var _a;
const failedTests = report.results.tests.filter((test) => test.status === 'failed');
failedTests.forEach((test) => {
if (test.extra != null) {
delete test.extra;
}
});
let logged = false;
let messageCount = 0;
for (const test of failedTests) {
if (args.maxMessages != null && messageCount >= args.maxMessages) {
break;
}
let prompt = `Report:\n${JSON.stringify(test, null, 2)}.\n\nTool:${report.results.tool.name}.\n\n Please provide a human-readable failure summary that explains why you think the test might have failed and ways to fix`;
if (args.additionalPromptContext != null &&
args.additionalPromptContext !== '') {
prompt += `\n\nAdditional Context:\n${args.additionalPromptContext}`;
}
let systemPrompt = (_a = args.systemPrompt) !== null && _a !== void 0 ? _a : constants_1.FAILED_TEST_SUMMARY_SYSTEM_PROMPT_CURRENT;
if (args.additionalSystemPromptContext != null &&
args.additionalSystemPromptContext !== '') {
systemPrompt += `\n\n${args.additionalSystemPromptContext}`;
}
const response = yield openRouter(systemPrompt, prompt, args);
if (response != null) {
test.ai = response;
messageCount++;
if (args.log === true && !logged) {
console.log('\n─────────────────────────────────────────────────────────────────────────────────────────────────────────────');
console.log('✨ AI Test Reporter Summary');
console.log('─────────────────────────────────────────────────────────────────────────────────────────────────────────────\n');
logged = true;
}
if (args.log === true) {
console.log(`❌ Failed Test: ${test.name}\n`);
console.log(`${response}\n`);
}
}
}
if (args.consolidate === true) {
yield (0, consolidated_summary_1.generateConsolidatedSummary)(report, 'openrouter', args);
}
if (file !== undefined) {
(0, common_1.saveUpdatedReport)(file, report);
}
return report;
});
}