ai-ctrf
Version:
Generate AI summaries of test results using a wide range of AI models like OpenAI, Anthropic, Gemini, Mistral, Grok, DeepSeek, Azure, Perplexity, OpenRouter, and custom OpenAI-compatible APIs
300 lines (299 loc) • 12.6 kB
JavaScript
#!/usr/bin/env node
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
var _a;
Object.defineProperty(exports, "__esModule", { value: true });
exports.generateJsonSummary = exports.customFailedTestSummary = exports.ollamaFailedTestSummary = exports.bedrockFailedTestSummary = exports.openRouterFailedTestSummary = exports.perplexityFailedTestSummary = exports.geminiFailedTestSummary = exports.mistralFailedTestSummary = exports.deepseekFailedTestSummary = exports.grokFailedTestSummary = exports.azureOpenAIFailedTestSummary = exports.claudeFailedTestSummary = exports.openAIFailedTestSummary = void 0;
const yargs_1 = __importDefault(require("yargs/yargs"));
const helpers_1 = require("yargs/helpers");
const common_1 = require("./common");
const constants_1 = require("./constants");
const json_summary_1 = require("./json-summary");
const azure_openai_1 = require("./models/azure-openai");
Object.defineProperty(exports, "azureOpenAIFailedTestSummary", { enumerable: true, get: function () { return azure_openai_1.azureOpenAIFailedTestSummary; } });
const bedrock_1 = require("./models/bedrock");
Object.defineProperty(exports, "bedrockFailedTestSummary", { enumerable: true, get: function () { return bedrock_1.bedrockFailedTestSummary; } });
const claude_1 = require("./models/claude");
Object.defineProperty(exports, "claudeFailedTestSummary", { enumerable: true, get: function () { return claude_1.claudeFailedTestSummary; } });
const deepseek_1 = require("./models/deepseek");
Object.defineProperty(exports, "deepseekFailedTestSummary", { enumerable: true, get: function () { return deepseek_1.deepseekFailedTestSummary; } });
const gemini_1 = require("./models/gemini");
Object.defineProperty(exports, "geminiFailedTestSummary", { enumerable: true, get: function () { return gemini_1.geminiFailedTestSummary; } });
const grok_1 = require("./models/grok");
Object.defineProperty(exports, "grokFailedTestSummary", { enumerable: true, get: function () { return grok_1.grokFailedTestSummary; } });
const mistral_1 = require("./models/mistral");
Object.defineProperty(exports, "mistralFailedTestSummary", { enumerable: true, get: function () { return mistral_1.mistralFailedTestSummary; } });
const ollama_1 = require("./models/ollama");
Object.defineProperty(exports, "ollamaFailedTestSummary", { enumerable: true, get: function () { return ollama_1.ollamaFailedTestSummary; } });
const openai_1 = require("./models/openai");
Object.defineProperty(exports, "openAIFailedTestSummary", { enumerable: true, get: function () { return openai_1.openAIFailedTestSummary; } });
const openrouter_1 = require("./models/openrouter");
Object.defineProperty(exports, "openRouterFailedTestSummary", { enumerable: true, get: function () { return openrouter_1.openRouterFailedTestSummary; } });
const perplexity_1 = require("./models/perplexity");
Object.defineProperty(exports, "perplexityFailedTestSummary", { enumerable: true, get: function () { return perplexity_1.perplexityFailedTestSummary; } });
const custom_1 = require("./models/custom");
Object.defineProperty(exports, "customFailedTestSummary", { enumerable: true, get: function () { return custom_1.customFailedTestSummary; } });
const argv = (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
.command('openai <file>', 'Generate test summary from a CTRF report', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'OpenAI model to use',
type: 'string',
default: 'gpt-4o',
});
})
.command('claude <file>', 'Generate test summary from a CTRF report', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Claude model to use',
type: 'string',
default: 'claude-3-5-sonnet-20240620',
});
})
.command('ollama <file>', 'Generate test summary from a CTRF report using Ollama', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Ollama model to use',
type: 'string',
default: 'llama2',
});
})
.command('azure-openai <file>', 'Generate test summary from a CTRF report using Azure OpenAI', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('deploymentId', {
describe: 'Deployment ID for Azure OpenAI',
type: 'string',
})
.option('model', {
describe: 'Model to use',
type: 'string',
default: 'gpt-4o',
});
})
.command('grok <file>', 'Generate test summary from a CTRF report using Grok', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Grok model to use',
type: 'string',
default: 'grok-2-latest',
});
})
.command('deepseek <file>', 'Generate test summary from a CTRF report using DeepSeek', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'DeepSeek model to use',
type: 'string',
default: 'deepseek-reasoner',
});
})
.command('mistral <file>', 'Generate test summary from a CTRF report using Mistral', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Mistral model to use',
type: 'string',
default: 'mistral-medium',
});
})
.command('gemini <file>', 'Generate test summary from a CTRF report using Google Gemini', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Gemini model to use',
type: 'string',
default: 'gemini-pro',
});
})
.command('perplexity <file>', 'Generate test summary from a CTRF report using Perplexity', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Perplexity model to use',
type: 'string',
default: 'pplx-7b-online',
});
})
.command('openrouter <file>', 'Generate test summary from a CTRF report using OpenRouter', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'OpenRouter model to use',
type: 'string',
default: 'anthropic/claude-3-opus',
});
})
.command('bedrock <file>', 'Generate test summary from a CTRF report using Amazon Bedrock Claude', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('model', {
describe: 'Bedrock model to use',
type: 'string',
default: 'anthropic.claude-3-5-sonnet-20240620-v1:0',
});
})
.command('custom <file>', 'Generate test summary from a CTRF report using a custom OpenAI-compatible API', (yargs) => {
return yargs
.positional('file', {
describe: 'Path to the CTRF file',
type: 'string',
})
.option('url', {
describe: 'Base URL for the custom OpenAI-compatible API (e.g., http://localhost:8080/v1)',
type: 'string',
demandOption: false,
})
.option('model', {
describe: 'Model to use',
type: 'string',
default: 'gpt-4o',
});
})
.option('systemPrompt', {
describe: 'System prompt to guide the AI',
type: 'string',
default: constants_1.FAILED_TEST_SUMMARY_SYSTEM_PROMPT_CURRENT,
})
.option('additionalSystemPromptContext', {
describe: 'Additional context to append to the system prompt',
type: 'string',
})
.option('additionalPromptContext', {
describe: 'Additional context to append to the user prompt',
type: 'string',
})
.option('frequencyPenalty', {
describe: 'Frequency penalty parameter for the model',
type: 'number',
default: 0,
})
.option('maxTokens', {
describe: 'Maximum number of tokens to generate',
type: 'number',
})
.option('presencePenalty', {
describe: 'Presence penalty parameter for the model',
type: 'number',
default: 0,
})
.option('temperature', {
describe: 'Sampling temperature',
type: 'number',
conflicts: 'topP',
})
.option('topP', {
describe: 'Top-p sampling parameter',
type: 'number',
conflicts: 'temperature',
})
.option('log', {
describe: 'Log the AI responses to the console',
type: 'boolean',
default: true,
})
.option('maxMessages', {
describe: 'Limit the number of failing tests to send for summarization in the LLM request. This helps avoid overwhelming the model when dealing with reports that have many failing tests.',
type: 'number',
default: 10,
})
.option('consolidate', {
describe: 'Consolidate and summarize multiple AI summaries into a higher-level overview',
type: 'boolean',
default: true,
})
.option('json-analysis', {
describe: 'Generate structured JSON analysis with categorized issues (code, timeout, application) and recommendations',
type: 'boolean',
default: false,
})
.help()
.alias('help', 'h')
.parseSync();
const file = (_a = argv.file) !== null && _a !== void 0 ? _a : 'ctrf/ctrf-report.json';
const executeCommand = (command, modelFunction) => __awaiter(void 0, void 0, void 0, function* () {
if (argv._.includes(command) && argv.file != null) {
try {
const report = (0, common_1.validateCtrfFile)(argv.file);
if (report !== null) {
yield modelFunction(report, argv, file, true);
if (argv.jsonAnalysis === true) {
const result = yield (0, json_summary_1.generateJsonSummary)(report, command, argv, argv.customUrl);
if (result !== null) {
console.log(JSON.stringify(result, null, 2));
}
}
}
}
catch (error) {
console.error('Failed to read file:', error);
}
}
});
if (argv.url != null) {
argv.customUrl = argv.url;
}
void Promise.resolve().then(() => __awaiter(void 0, void 0, void 0, function* () {
yield executeCommand('openai', openai_1.openAIFailedTestSummary);
yield executeCommand('claude', claude_1.claudeFailedTestSummary);
yield executeCommand('azure-openai', azure_openai_1.azureOpenAIFailedTestSummary);
yield executeCommand('grok', grok_1.grokFailedTestSummary);
yield executeCommand('deepseek', deepseek_1.deepseekFailedTestSummary);
yield executeCommand('mistral', mistral_1.mistralFailedTestSummary);
yield executeCommand('gemini', gemini_1.geminiFailedTestSummary);
yield executeCommand('perplexity', perplexity_1.perplexityFailedTestSummary);
yield executeCommand('openrouter', openrouter_1.openRouterFailedTestSummary);
yield executeCommand('bedrock', bedrock_1.bedrockFailedTestSummary);
yield executeCommand('ollama', ollama_1.ollamaFailedTestSummary);
yield executeCommand('custom', custom_1.customFailedTestSummary);
}));
var json_summary_2 = require("./json-summary");
Object.defineProperty(exports, "generateJsonSummary", { enumerable: true, get: function () { return json_summary_2.generateJsonSummary; } });