@craftapit/tester
Version:
A focused, LLM-powered testing framework for natural language test scenarios
367 lines (366 loc) • 14.6 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.TestRunner = void 0;
exports.createTestRunner = createTestRunner;
const path = __importStar(require("path"));
const fs = __importStar(require("fs"));
const CapabilityRegistry_1 = require("./CapabilityRegistry");
const TestExecutor_1 = require("./TestExecutor");
const ScenarioParser_1 = require("./ScenarioParser");
const AnthropicAdapter_1 = require("../adapters/AnthropicAdapter");
const OpenAIAdapter_1 = require("../adapters/OpenAIAdapter");
const OllamaAdapter_1 = require("../adapters/OllamaAdapter");
const CraftacoderAdapter_1 = require("../adapters/CraftacoderAdapter");
/**
* Test runner for executing test scenarios
*/
class TestRunner {
/**
* Create a new test runner
* @param config Test runner configuration
*/
constructor(config = {}) {
this.config = {
caching: true,
timeout: 60000,
verbose: false,
llmAdapter: 'craftacoder',
...config
};
this.registry = new CapabilityRegistry_1.CapabilityRegistry({
cachingEnabled: this.config.caching,
cacheFilePath: this.config.cachePath || path.join(process.cwd(), '.craft-a-tester', 'test-cache.json')
});
this.llmAdapter = this.createLLMAdapter();
this.registry.setLLMAdapter(this.llmAdapter);
// Create the scenario parser
this.parser = new ScenarioParser_1.ScenarioParser();
// Register the LLM adapter
this.registry.registerAdapter('llm', this.llmAdapter);
// Register additional adapters
if (this.config.adapters) {
Object.entries(this.config.adapters).forEach(([name, adapter]) => {
this.registry.registerAdapter(name, adapter);
});
}
// Register addons
if (this.config.addons) {
this.config.addons.forEach(addon => {
addon.registerCapabilities(this.registry);
});
}
this.executor = new TestExecutor_1.TestExecutor({}, this.registry);
// Make sure we register the adapters with the executor as well
this.executor.registerAdapter('llm', this.llmAdapter);
if (this.config.adapters) {
Object.entries(this.config.adapters).forEach(([name, adapter]) => {
this.executor.registerAdapter(name, adapter);
});
}
}
/**
* Create an LLM adapter based on the configuration
*/
createLLMAdapter() {
if (this.config.llmAdapter === 'custom' && this.config.customLLMAdapter) {
return this.config.customLLMAdapter;
}
const adapterConfig = this.config.llmAdapterConfig || {};
switch (this.config.llmAdapter) {
case 'craftacoder':
return new CraftacoderAdapter_1.CraftacoderAdapter({
apiKey: adapterConfig.apiKey || process.env.CRAFTACODER_API_KEY,
baseUrl: adapterConfig.baseUrl || process.env.CRAFTACODER_API_BASE || 'http://localhost:3000',
model: adapterConfig.model || process.env.CRAFTACODER_MODEL || 'claude-3-sonnet-20240229',
provider: adapterConfig.provider || process.env.CRAFTACODER_PROVIDER || 'anthropic'
});
case 'anthropic':
return new AnthropicAdapter_1.AnthropicAdapter({
apiKey: adapterConfig.apiKey || process.env.ANTHROPIC_API_KEY,
baseUrl: adapterConfig.baseUrl || process.env.ANTHROPIC_API_BASE || 'https://api.anthropic.com',
model: adapterConfig.model || process.env.ANTHROPIC_MODEL || 'claude-3-sonnet-20240229'
});
case 'openai':
return new OpenAIAdapter_1.OpenAIAdapter({
apiKey: adapterConfig.apiKey || process.env.OPENAI_API_KEY,
baseUrl: adapterConfig.baseUrl || process.env.OPENAI_API_BASE,
model: adapterConfig.model || process.env.OPENAI_MODEL || 'gpt-4'
});
case 'ollama':
return new OllamaAdapter_1.OllamaAdapter({
baseUrl: adapterConfig.baseUrl || process.env.OLLAMA_URL || 'http://localhost:11434',
model: adapterConfig.model || process.env.OLLAMA_MODEL || 'phi4:14b-fp16',
contextSize: adapterConfig.contextSize || parseInt(process.env.CONTEXT_SIZE || '16384', 10),
dynamicContextSizing: adapterConfig.dynamicContextSizing !== undefined
? adapterConfig.dynamicContextSizing
: process.env.DYNAMIC_SIZING?.toLowerCase() !== 'false'
});
default:
throw new Error(`Unsupported LLM adapter type: ${this.config.llmAdapter}`);
}
}
/**
* Initialize the test runner
* This will initialize all adapters
*/
async initialize() {
if (this.config.verbose) {
console.log('Initializing test runner...');
}
try {
await this.llmAdapter.initialize();
if (this.config.adapters) {
for (const [name, adapter] of Object.entries(this.config.adapters)) {
if (this.config.verbose) {
console.log(`Initializing adapter: ${name}`);
}
await adapter.initialize();
}
}
if (this.config.verbose) {
console.log('Test runner initialized');
}
}
catch (error) {
console.error('Failed to initialize test runner:', error);
throw error;
}
}
/**
* Run a test from a file path
* @param filePath Path to the test file
* @returns Test results
*/
async runTestFile(filePath) {
if (this.config.verbose) {
console.log(`Running test file: ${filePath}`);
}
try {
const content = await fs.promises.readFile(filePath, 'utf-8');
return this.runTest(content);
}
catch (error) {
console.error(`Failed to run test file ${filePath}:`, error);
throw error;
}
}
/**
* Run a test from a string
* @param test Test content as a string
* @returns Test results
*/
async runTest(test) {
if (this.config.verbose) {
console.log('Running test...');
}
try {
// Create a timeout for the test
let timeoutId = null;
// Parse the test string into a scenario
const scenario = this.parser.parse(test);
const results = await Promise.race([
this.executor.executeScenario(scenario),
new Promise((_, reject) => {
timeoutId = setTimeout(() => {
reject(new Error(`Test execution timed out after ${this.config.timeout}ms`));
}, this.config.timeout);
})
]);
// Clear the timeout
if (timeoutId) {
clearTimeout(timeoutId);
}
if (this.config.verbose) {
this.logTestResults(results);
}
return results;
}
catch (error) {
console.error('Test execution failed:', error);
throw error;
}
}
/**
* Run all tests in a directory
* @param directory Directory containing test files
* @param pattern File pattern to match (default: '**\*.md')
* @returns Test results by file path
*/
async runTestDirectory(directory, pattern = '**/*.md') {
if (this.config.verbose) {
console.log(`Running tests in directory: ${directory}`);
}
try {
const testFiles = await this.discoverTestFiles(directory, pattern);
if (testFiles.length === 0) {
console.warn(`No test files found in ${directory} matching pattern ${pattern}`);
return {};
}
if (this.config.verbose) {
console.log(`Found ${testFiles.length} test files`);
}
const results = {};
for (const file of testFiles) {
try {
if (this.config.verbose) {
console.log(`Running test file: ${file.path}`);
}
const content = file.content || await fs.promises.readFile(file.path, 'utf-8');
results[file.path] = await this.runTest(content);
}
catch (error) {
console.error(`Failed to run test file ${file.path}:`, error);
results[file.path] = {
passed: false,
steps: [],
error: error instanceof Error ? error.message : String(error)
};
}
}
if (this.config.verbose) {
this.logTestResultsSummary(results);
}
return results;
}
catch (error) {
console.error(`Failed to run tests in directory ${directory}:`, error);
throw error;
}
}
/**
* Discover test files in a directory
* @param directory Directory to search
* @param pattern File pattern to match
*/
async discoverTestFiles(directory, pattern) {
// For now, we'll use a simple recursive search
// In a full implementation, this would use a proper glob library
const files = [];
const scanDirectory = async (dir) => {
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
await scanDirectory(fullPath);
}
else if (entry.isFile() && entry.name.endsWith('.md')) {
files.push({ path: fullPath });
}
}
};
await scanDirectory(directory);
return files;
}
/**
* Log test results
* @param results Test results
*/
logTestResults(results) {
console.log('\nTest Results:');
console.log(`Passed: ${results.passed}`);
console.log(`Total steps: ${results.steps.length}`);
console.log(`Passed steps: ${results.steps.filter(s => s.status === 'passed').length}`);
console.log(`Failed steps: ${results.steps.filter(s => s.status === 'failed').length}`);
console.log(`Skipped steps: ${results.steps.filter(s => s.status === 'skipped').length}`);
if (results.error) {
console.log(`Error: ${results.error}`);
}
console.log('\nDetailed Results:');
results.steps.forEach((step, index) => {
console.log(`Step ${index + 1}: ${step.description}`);
console.log(` Status: ${step.status}`);
if (step.error) {
console.log(` Error: ${step.error}`);
}
});
}
/**
* Log test results summary
* @param results Test results by file path
*/
logTestResultsSummary(results) {
console.log('\nTest Results Summary:');
const totalTests = Object.keys(results).length;
const passedTests = Object.values(results).filter(r => r.passed).length;
const failedTests = totalTests - passedTests;
console.log(`Total tests: ${totalTests}`);
console.log(`Passed tests: ${passedTests}`);
console.log(`Failed tests: ${failedTests}`);
if (failedTests > 0) {
console.log('\nFailed Tests:');
Object.entries(results)
.filter(([_, result]) => !result.passed)
.forEach(([path, result]) => {
console.log(`- ${path}: ${result.error || 'Failed steps'}`);
});
}
}
/**
* Clean up the test runner
* This will clean up all adapters
*/
async cleanup() {
if (this.config.verbose) {
console.log('Cleaning up test runner...');
}
try {
await this.llmAdapter.cleanup();
if (this.config.adapters) {
for (const [name, adapter] of Object.entries(this.config.adapters)) {
if (this.config.verbose) {
console.log(`Cleaning up adapter: ${name}`);
}
await adapter.cleanup();
}
}
if (this.config.verbose) {
console.log('Test runner cleaned up');
}
}
catch (error) {
console.error('Failed to clean up test runner:', error);
}
}
}
exports.TestRunner = TestRunner;
/**
* Create a test runner with the given configuration
* @param config Test runner configuration
* @returns A test runner instance
*/
function createTestRunner(config) {
return new TestRunner(config);
}