@craftapit/tester
Version:
A focused, LLM-powered testing framework for natural language test scenarios
152 lines (151 loc) • 7.04 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.runScenarios = runScenarios;
const fs = __importStar(require("fs/promises"));
const path = __importStar(require("path"));
const TestExecutor_1 = require("../../core/TestExecutor");
const config_1 = require("../utils/config");
const formatter_1 = require("../utils/formatter");
const logger_1 = require("../../utils/logger");
const chalk_1 = __importDefault(require("chalk"));
const ora_1 = __importDefault(require("ora"));
async function runScenarios(directory, options) {
const logger = new logger_1.Logger('CLI');
const spinner = (0, ora_1.default)('Initializing test executor...').start();
try {
// Load config from file or use defaults with CLI overrides
const config = await (0, config_1.loadConfig)(options.config, options);
// Find all scenario files
spinner.text = `Finding scenarios in ${directory}...`;
const scenarioPaths = await findScenarios(directory, options.recursive);
if (scenarioPaths.length === 0) {
spinner.fail(`No scenario files found in ${directory}`);
process.exit(1);
}
spinner.succeed(`Found ${scenarioPaths.length} scenario files`);
// Initialize test executor
const executor = new TestExecutor_1.TestExecutor(config);
// Initialize appropriate adapters based on config
await initializeAdapters(executor, config);
// Run all scenarios
spinner.text = `Running ${scenarioPaths.length} scenarios...`;
const results = await executor.runScenarios(scenarioPaths);
// Format and display results
spinner.stop();
console.log((0, formatter_1.formatResults)(results));
console.log(chalk_1.default.bold('\nSummary:'));
console.log(`Total: ${results.total}`);
console.log(`Passed: ${chalk_1.default.green(results.passed)}`);
console.log(`Failed: ${chalk_1.default.red(results.failed)}`);
// Exit with appropriate code
process.exit(results.failed === 0 ? 0 : 1);
}
catch (error) {
spinner.fail('Error running scenarios');
logger.error('Error running scenarios:', error);
process.exit(1);
}
}
async function findScenarios(directory, recursive = false) {
const scenarioPaths = [];
async function scanDirectory(dir) {
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory() && recursive) {
await scanDirectory(fullPath);
}
else if (entry.isFile() && entry.name.endsWith('.md')) {
// Check if it's a scenario file by looking for scenario markers
const content = await fs.readFile(fullPath, 'utf-8');
if (content.includes('## Scenario:') || content.includes('### Steps')) {
scenarioPaths.push(fullPath);
}
}
}
}
await scanDirectory(directory);
return scenarioPaths;
}
/**
* Helper function to initialize adapters based on config
*/
async function initializeAdapters(executor, config) {
// Register LLM adapter
if (config.llm?.provider === 'anthropic') {
const { AnthropicAdapter } = await Promise.resolve().then(() => __importStar(require('../../adapters/AnthropicAdapter')));
executor.registerAdapter('llm', new AnthropicAdapter({
apiKey: config.llm.apiKey,
model: config.llm.model || 'claude-2'
}));
}
else if (config.llm?.provider === 'openai') {
const { OpenAIAdapter } = await Promise.resolve().then(() => __importStar(require('../../adapters/OpenAIAdapter')));
executor.registerAdapter('llm', new OpenAIAdapter({
apiKey: config.llm.apiKey,
model: config.llm.model || 'gpt-4'
}));
}
else if (config.llm?.provider === 'ollama') {
const { OllamaAdapter } = await Promise.resolve().then(() => __importStar(require('../../adapters/OllamaAdapter')));
executor.registerAdapter('llm', new OllamaAdapter({
baseUrl: config.llm.baseUrl || process.env.OLLAMA_URL || 'http://localhost:11434',
model: config.llm.model || process.env.OLLAMA_MODEL || 'phi4:14b-fp16',
contextSize: config.llm.contextSize || parseInt(process.env.CONTEXT_SIZE || '16384', 10),
dynamicContextSizing: config.llm.dynamicContextSizing !== undefined
? config.llm.dynamicContextSizing
: process.env.DYNAMIC_SIZING?.toLowerCase() !== 'false'
}));
}
// Register browser adapter if needed
if (config.browser) {
const { BrowserAdapter } = await Promise.resolve().then(() => __importStar(require('../../adapters/BrowserAdapter')));
executor.registerAdapter('browser', new BrowserAdapter(config.browser));
}
// Register API adapter if needed
if (config.api) {
const { APIAdapter } = await Promise.resolve().then(() => __importStar(require('../../adapters/APIAdapter')));
executor.registerAdapter('api', new APIAdapter(config.api));
}
// Register database adapter if needed
if (config.database) {
const { DatabaseAdapter } = await Promise.resolve().then(() => __importStar(require('../../adapters/DatabaseAdapter')));
executor.registerAdapter('database', new DatabaseAdapter(config.database));
}
}