@craftapit/tester
Version:
A focused, LLM-powered testing framework for natural language test scenarios
470 lines (462 loc) • 17.3 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.CapabilityRegistry = void 0;
const VectorStore_1 = require("../utils/VectorStore");
const SimpleEmbedding_1 = require("../utils/SimpleEmbedding");
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
/**
* Registry for addon capabilities with vector-based caching
*/
class CapabilityRegistry {
constructor(options = {}) {
/**
* Map of adapter types to adapter instances
*/
this.adapters = new Map();
/**
* Map of capability names to capability definitions
*/
this.capabilities = new Map();
/**
* LLM adapter for resolving capabilities
*/
this.llmAdapter = null;
/**
* Cache of feedback entries
*/
this.feedback = [];
this.cachingEnabled = options.cachingEnabled ?? true;
this.cacheFilePath = options.cacheFilePath || path.join(process.cwd(), '.craft-a-tester', 'capability-cache.json');
const embeddingDimension = options.embeddingDimension || 384; // Smaller dimension is faster
this.vectorStore = new VectorStore_1.VectorStore(embeddingDimension);
this.embedder = new SimpleEmbedding_1.SimpleEmbedding(embeddingDimension);
// Create cache directory if it doesn't exist
const cacheDir = path.dirname(this.cacheFilePath);
if (this.cachingEnabled) {
if (!fs.existsSync(cacheDir)) {
try {
fs.mkdirSync(cacheDir, { recursive: true });
}
catch (error) {
console.warn(`Failed to create cache directory: ${error}`);
this.cachingEnabled = false;
}
}
}
// Load cache if it exists
this.loadCache();
}
/**
* Set the LLM adapter for capability resolution
*/
setLLMAdapter(llmAdapter) {
this.llmAdapter = llmAdapter;
}
/**
* Register an adapter
*/
registerAdapter(type, adapter) {
if (this.adapters.has(type)) {
console.warn(`Overwriting existing adapter for type ${type}`);
}
this.adapters.set(type, adapter);
console.log(`Registered adapter for type ${type}`);
}
/**
* Get an adapter by type
*/
getAdapter(type) {
return this.adapters.get(type) || null;
}
/**
* Register a capability
*/
registerCapability(capability) {
if (this.capabilities.has(capability.name)) {
console.warn(`Overwriting existing capability ${capability.name}`);
}
this.capabilities.set(capability.name, capability);
console.log(`Registered capability ${capability.name}`);
// Create embeddings for each example
if (this.cachingEnabled && capability.examples.length > 0) {
console.log(`Creating embeddings for ${capability.examples.length} examples of ${capability.name}`);
capability.examples.forEach((example, index) => {
const vector = this.embedder.embed(example);
const id = `${capability.name}_example_${index}`;
this.vectorStore.addVector(id, vector, {
capabilityName: capability.name,
type: 'example',
text: example
});
});
// Save the updated cache
this.saveCache();
}
}
/**
* Get a capability by name
*/
getCapability(name) {
return this.capabilities.get(name) || null;
}
/**
* Get all capabilities
*/
getAllCapabilities() {
return Array.from(this.capabilities.values());
}
/**
* Get capability context for LLM
*/
getCapabilityContext() {
const capabilities = this.getAllCapabilities();
if (capabilities.length === 0) {
return 'No capabilities available.';
}
let context = `Available capabilities (${capabilities.length}):\n\n`;
capabilities.forEach(capability => {
context += `## ${capability.name}\n`;
context += `${capability.descriptions[0]}\n\n`;
if (capability.parameters && capability.parameters.length > 0) {
context += 'Parameters:\n';
capability.parameters.forEach(param => {
context += `- ${param.name} (${param.type}${param.required ? ', required' : ''}): ${param.description}\n`;
});
context += '\n';
}
context += 'Examples:\n';
capability.examples.forEach(example => {
context += `- "${example}"\n`;
});
context += '\n';
});
return context;
}
/**
* Find capability for action using LLM or vector cache
*/
async findCapabilityForAction(description) {
if (this.capabilities.size === 0) {
console.warn('No capabilities registered');
return null;
}
// Check if we have a cached capability resolution for a similar action
if (this.cachingEnabled) {
const cachedResult = this.findCachedCapability(description);
if (cachedResult && cachedResult.confidence > 0.85) {
console.log(`Found cached capability match: ${cachedResult.capability.name} with confidence ${cachedResult.confidence.toFixed(2)}`);
return {
capability: cachedResult.capability,
parameters: cachedResult.parameters || [],
confidence: cachedResult.confidence
};
}
}
// If no good cache match or caching disabled, use LLM
if (!this.llmAdapter) {
throw new Error('No LLM adapter set for capability resolution');
}
const prompt = this.buildCapabilityResolutionPrompt(description);
try {
const response = await this.llmAdapter.complete(prompt);
const result = this.parseCapabilityResolutionResponse(response);
// Cache the result if successful
if (result && this.cachingEnabled) {
this.cacheCapabilityResolution(description, result);
}
return result;
}
catch (error) {
console.error('Error resolving capability:', error);
return null;
}
}
/**
* Find a cached capability match for the given description
*/
findCachedCapability(description) {
// Skip if no capabilities registered
if (this.capabilities.size === 0) {
return null;
}
// Generate embedding for the description
const embedding = this.embedder.embed(description);
// Find nearest neighbors in the vector store
const nearestResults = this.vectorStore.findNearest(embedding, 3);
// If we have no results, return null
if (nearestResults.length === 0) {
return null;
}
// Check if we have a good match (high similarity)
const bestMatch = nearestResults[0];
// For saved actions (not examples), we can directly use the stored parameters
if (bestMatch.metadata.type === 'action' && bestMatch.similarity > 0.85) {
const capability = this.getCapability(bestMatch.metadata.capabilityName);
if (capability) {
return {
capability,
parameters: bestMatch.metadata.parameters,
confidence: bestMatch.similarity
};
}
}
// For example matches, find the best matching capability
const candidateResults = nearestResults.filter(r => r.similarity > 0.8);
if (candidateResults.length === 0) {
return null;
}
// Count capabilities by frequency in the top results
const capabilityCounts = {};
for (const result of candidateResults) {
const capName = result.metadata.capabilityName;
capabilityCounts[capName] = (capabilityCounts[capName] || 0) + 1;
}
// Find the most frequent capability
let mostFrequentCapability = '';
let highestCount = 0;
for (const [capName, count] of Object.entries(capabilityCounts)) {
if (count > highestCount) {
mostFrequentCapability = capName;
highestCount = count;
}
}
if (mostFrequentCapability) {
const capability = this.getCapability(mostFrequentCapability);
if (capability) {
return {
capability,
confidence: bestMatch.similarity
};
}
}
return null;
}
/**
* Cache a capability resolution result
*/
cacheCapabilityResolution(description, result) {
// Generate embedding for the description
const embedding = this.embedder.embed(description);
// Store in vector database
const id = `action_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
this.vectorStore.addVector(id, embedding, {
capabilityName: result.capability.name,
type: 'action',
text: description,
parameters: result.parameters,
confidence: result.confidence,
timestamp: Date.now()
});
// Save the updated cache
this.saveCache();
}
/**
* Build prompt for capability resolution
*/
buildCapabilityResolutionPrompt(description) {
const context = this.getCapabilityContext();
return `
# Capability Resolution
You need to identify the most appropriate testing capability for the following test step:
"${description}"
Here are the available capabilities:
${context}
## Task
1. Analyze the test step and find the most appropriate capability.
2. Determine what parameters should be passed to the capability.
3. Provide your confidence level (0.0 to 1.0) in this match.
Format your response as a JSON object with the following structure:
{
"capability": "capabilityName",
"parameters": [param1, param2, ...],
"confidence": 0.95,
"reasoning": "Brief explanation of why this capability was selected"
}
`;
}
/**
* Parse LLM response for capability resolution
*/
parseCapabilityResolutionResponse(response) {
try {
// Extract JSON from response
const jsonMatch = response.match(/\{[\s\S]*\}/);
if (!jsonMatch) {
console.error('No JSON found in response');
return null;
}
const json = JSON.parse(jsonMatch[0]);
const { capability: capabilityName, parameters, confidence } = json;
if (!capabilityName || !parameters || typeof confidence !== 'number') {
console.error('Invalid resolution format', json);
return null;
}
const capability = this.capabilities.get(capabilityName);
if (!capability) {
console.error(`Capability ${capabilityName} not found`);
return null;
}
return {
capability,
parameters,
confidence
};
}
catch (error) {
console.error('Error parsing capability resolution:', error);
return null;
}
}
/**
* Execute a capability
*/
async executeCapability(name, parameters) {
const capability = this.capabilities.get(name);
if (!capability) {
throw new Error(`Capability ${name} not found`);
}
return capability.handler(...parameters);
}
/**
* Provide feedback on a capability resolution
*/
async provideFeedback(feedback) {
this.feedback.push({
...feedback,
timestamp: feedback.timestamp || Date.now()
});
console.log(`Received ${feedback.quality} feedback for step ${feedback.stepId}`);
// Store feedback with embedding if caching is enabled
if (this.cachingEnabled && feedback.description) {
const embedding = this.embedder.embed(feedback.description);
// Store in vector database
const id = `feedback_${feedback.stepId || Date.now()}`;
this.vectorStore.addVector(id, embedding, {
type: 'feedback',
quality: feedback.quality,
capabilityName: feedback.capabilityName,
text: feedback.description,
parameters: feedback.parameters,
timestamp: feedback.timestamp || Date.now()
});
// If this is negative feedback with cleanup strategy, clean up incorrect cache entries
if (feedback.quality === 'incorrect' && feedback.cacheStrategy === 'cleanup' && feedback.capabilityName && feedback.description) {
this.cleanupIncorrectEntries(feedback.description, feedback.capabilityName);
}
// Save the updated cache
this.saveCache();
}
}
/**
* Clean up incorrect capability selections by removing similar cache entries
*/
cleanupIncorrectEntries(description, wrongCapabilityName) {
// Generate embedding for the description
const embedding = this.embedder.embed(description);
// Find similar vectors
const similarVectors = this.vectorStore.findNearest(embedding, 5);
// Get all vectors from the store
const allVectors = this.vectorStore.getAllVectors();
// Find and remove similar vectors that point to the wrong capability
const vectorsToRemove = [];
for (const similar of similarVectors) {
if (similar.similarity > 0.9 &&
similar.metadata.type === 'action' &&
similar.metadata.capabilityName === wrongCapabilityName) {
vectorsToRemove.push(similar.id);
}
}
// Remove the identified vectors
for (const id of vectorsToRemove) {
this.vectorStore.removeVector(id);
}
if (vectorsToRemove.length > 0) {
console.log(`Cleaned up ${vectorsToRemove.length} similar cache entries with wrong capability`);
}
}
/**
* Get feedback for analysis
*/
getFeedback() {
return [...this.feedback];
}
/**
* Save the cache to disk
*/
saveCache() {
if (!this.cachingEnabled) {
return;
}
try {
const cacheData = this.vectorStore.saveToJson();
fs.writeFileSync(this.cacheFilePath, cacheData, 'utf8');
console.log(`Cache saved to ${this.cacheFilePath}`);
}
catch (error) {
console.warn(`Failed to save cache: ${error}`);
}
}
/**
* Load the cache from disk
*/
loadCache() {
if (!this.cachingEnabled) {
return;
}
try {
if (fs.existsSync(this.cacheFilePath)) {
const cacheData = fs.readFileSync(this.cacheFilePath, 'utf8');
this.vectorStore.loadFromJson(cacheData);
console.log(`Loaded ${this.vectorStore.size()} cached vectors from ${this.cacheFilePath}`);
}
}
catch (error) {
console.warn(`Failed to load cache: ${error}`);
}
}
/**
* Clear the cache
*/
clearCache() {
this.vectorStore.clear();
console.log('Cache cleared');
if (this.cachingEnabled) {
this.saveCache();
}
}
}
exports.CapabilityRegistry = CapabilityRegistry;