ai-ctrf
Version:
Generate AI summaries of test results using a wide range of AI models like OpenAI, Anthropic, Gemini, Mistral, Grok, DeepSeek, Azure, Perplexity, and OpenRouter
281 lines (280 loc) • 10.3 kB
JavaScript
#!/usr/bin/env node
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.openRouterFailedTestSummary = exports.perplexityFailedTestSummary = exports.geminiFailedTestSummary = exports.mistralFailedTestSummary = exports.deepseekFailedTestSummary = exports.grokFailedTestSummary = exports.azureFailedTestSummary = exports.claudeFailedTestSummary = exports.openAIFailedTestSummary = void 0;
const yargs_1 = __importDefault(require("yargs/yargs"));
const helpers_1 = require("yargs/helpers");
const openai_1 = require("./models/openai");
Object.defineProperty(exports, "openAIFailedTestSummary", { enumerable: true, get: function () { return openai_1.openAIFailedTestSummary; } });
const azure_openai_1 = require("./models/azure-openai");
Object.defineProperty(exports, "azureFailedTestSummary", { enumerable: true, get: function () { return azure_openai_1.azureFailedTestSummary; } });
const common_1 = require("./common");
const claude_1 = require("./models/claude");
Object.defineProperty(exports, "claudeFailedTestSummary", { enumerable: true, get: function () { return claude_1.claudeFailedTestSummary; } });
const grok_1 = require("./models/grok");
Object.defineProperty(exports, "grokFailedTestSummary", { enumerable: true, get: function () { return grok_1.grokFailedTestSummary; } });
const deepseek_1 = require("./models/deepseek");
Object.defineProperty(exports, "deepseekFailedTestSummary", { enumerable: true, get: function () { return deepseek_1.deepseekFailedTestSummary; } });
const mistral_1 = require("./models/mistral");
Object.defineProperty(exports, "mistralFailedTestSummary", { enumerable: true, get: function () { return mistral_1.mistralFailedTestSummary; } });
const gemini_1 = require("./models/gemini");
Object.defineProperty(exports, "geminiFailedTestSummary", { enumerable: true, get: function () { return gemini_1.geminiFailedTestSummary; } });
const perplexity_1 = require("./models/perplexity");
Object.defineProperty(exports, "perplexityFailedTestSummary", { enumerable: true, get: function () { return perplexity_1.perplexityFailedTestSummary; } });
const openrouter_1 = require("./models/openrouter");
Object.defineProperty(exports, "openRouterFailedTestSummary", { enumerable: true, get: function () { return openrouter_1.openRouterFailedTestSummary; } });
const constants_1 = require("./constants");
const argv = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
.command('openai <file>', 'Generate test summary from a CTRF report', (yargs) => {
return yargs.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'OpenAI model to use',
type: 'string',
default: 'gpt-4o',
});
})
.command('claude <file>', 'Generate test summary from a CTRF report', (yargs) => {
return yargs.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Claude model to use',
type: 'string',
default: 'claude-3-5-sonnet-20240620',
});
})
.command('azure-openai <file>', 'Generate test summary from a CTRF report using Azure OpenAI', (yargs) => {
return yargs.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('deploymentId', {
describe: 'Deployment ID for Azure OpenAI',
type: 'string',
})
.option('model', {
describe: 'Model to use',
type: 'string',
default: 'gpt-4o',
});
})
.command('grok <file>', 'Generate test summary from a CTRF report using Grok', (yargs) => {
return yargs.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Grok model to use',
type: 'string',
default: 'grok-2-latest',
});
})
.command('deepseek <file>', 'Generate test summary from a CTRF report using DeepSeek', (yargs) => {
return yargs.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'DeepSeek model to use',
type: 'string',
default: 'deepseek-reasoner',
});
})
.command('mistral <file>', 'Generate test summary from a CTRF report using Mistral', (yargs) => {
return yargs.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Mistral model to use',
type: 'string',
default: 'mistral-medium',
});
})
.command('gemini <file>', 'Generate test summary from a CTRF report using Google Gemini', (yargs) => {
return yargs.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Gemini model to use',
type: 'string',
default: 'gemini-pro',
});
})
.command('perplexity <file>', 'Generate test summary from a CTRF report using Perplexity', (yargs) => {
return yargs.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Perplexity model to use',
type: 'string',
default: 'pplx-7b-online',
});
})
.command('openrouter <file>', 'Generate test summary from a CTRF report using OpenRouter', (yargs) => {
return yargs.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'OpenRouter model to use',
type: 'string',
default: 'anthropic/claude-3-opus',
});
})
.option('systemPrompt', {
describe: 'System prompt to guide the AI',
type: 'string',
default: constants_1.FAILED_TEST_SUMMARY_SYSTEM_PROMPT_CURRENT,
})
.option('frequencyPenalty', {
describe: 'Frequency penalty parameter for the model',
type: 'number',
default: 0,
})
.option('maxTokens', {
describe: 'Maximum number of tokens to generate',
type: 'number',
})
.option('presencePenalty', {
describe: 'Presence penalty parameter for the model',
type: 'number',
default: 0,
})
.option('temperature', {
describe: 'Sampling temperature',
type: 'number',
conflicts: 'topP',
})
.option('topP', {
describe: 'Top-p sampling parameter',
type: 'number',
conflicts: 'temperature',
})
.option('log', {
describe: 'Log the AI responses to the console',
type: 'boolean',
default: true,
})
.option('maxMessages', {
describe: 'Limit the number of failing tests to send for summarization in the LLM request. This helps avoid overwhelming the model when dealing with reports that have many failing tests.',
type: 'number',
default: 10,
})
.option('consolidate', {
describe: 'Consolidate and summarize multiple AI summaries into a higher-level overview',
type: 'boolean',
default: true,
})
.help()
.alias('help', 'h')
.parseSync();
const file = argv.file || "ctrf/ctrf-report.json";
if (argv._.includes('openai') && argv.file) {
try {
const report = (0, common_1.validateCtrfFile)(argv.file);
if (report !== null) {
(0, openai_1.openAIFailedTestSummary)(report, argv, file, true);
}
}
catch (error) {
console.error('Failed to read file:', error);
}
}
else if (argv._.includes('claude') && argv.file) {
try {
const report = (0, common_1.validateCtrfFile)(argv.file);
if (report !== null) {
(0, claude_1.claudeFailedTestSummary)(report, argv, file, true);
}
}
catch (error) {
console.error('Failed to read file:', error);
}
}
else if (argv._.includes('azure-openai') && argv.file) {
try {
const report = (0, common_1.validateCtrfFile)(argv.file);
if (report !== null) {
(0, azure_openai_1.azureFailedTestSummary)(report, argv, file, true);
}
}
catch (error) {
console.error('Failed to read file:', error);
}
}
else if (argv._.includes('grok') && argv.file) {
try {
const report = (0, common_1.validateCtrfFile)(argv.file);
if (report !== null) {
(0, grok_1.grokFailedTestSummary)(report, argv, file, true);
}
}
catch (error) {
console.error('Failed to read file:', error);
}
}
else if (argv._.includes('deepseek') && argv.file) {
try {
const report = (0, common_1.validateCtrfFile)(argv.file);
if (report !== null) {
(0, deepseek_1.deepseekFailedTestSummary)(report, argv, file, true);
}
}
catch (error) {
console.error('Failed to read file:', error);
}
}
else if (argv._.includes('mistral') && argv.file) {
try {
const report = (0, common_1.validateCtrfFile)(argv.file);
if (report !== null) {
(0, mistral_1.mistralFailedTestSummary)(report, argv, file, true);
}
}
catch (error) {
console.error('Failed to read file:', error);
}
}
else if (argv._.includes('gemini') && argv.file) {
try {
const report = (0, common_1.validateCtrfFile)(argv.file);
if (report !== null) {
(0, gemini_1.geminiFailedTestSummary)(report, argv, file, true);
}
}
catch (error) {
console.error('Failed to read file:', error);
}
}
else if (argv._.includes('perplexity') && argv.file) {
try {
const report = (0, common_1.validateCtrfFile)(argv.file);
if (report !== null) {
(0, perplexity_1.perplexityFailedTestSummary)(report, argv, file, true);
}
}
catch (error) {
console.error('Failed to read file:', error);
}
}
else if (argv._.includes('openrouter') && argv.file) {
try {
const report = (0, common_1.validateCtrfFile)(argv.file);
if (report !== null) {
(0, openrouter_1.openRouterFailedTestSummary)(report, argv, file, true);
}
}
catch (error) {
console.error('Failed to read file:', error);
}
}