shipdeck/lib/ultimate/anthropic/tests.js

Ship MVPs in 48 hours. Fix bugs in 30 seconds. The command deck for developers who ship.

/**
 * Comprehensive Tests for Anthropic API Integration
 * Production-ready test suite covering all functionality
 */

const { 
  AnthropicIntegration, 
  AnthropicClient, 
  AgentExecutor, 
  ConfigManager,
  TokenManager,
  validateApiKeyFormat
} = require('./index');

class TestRunner {
  constructor() {
    this.tests = [];
    this.passed = 0;
    this.failed = 0;
    this.skipped = 0;
  }

  /**
   * Add a test
   */
  test(name, testFn, options = {}) {
    this.tests.push({ name, testFn, options });
  }

  /**
   * Run all tests
   */
  async run(options = {}) {
    const startTime = Date.now();
    console.log('🧪 Running Anthropic API Integration Tests...\n');

    for (const test of this.tests) {
      try {
        // Skip tests that require API key if not available
        if (test.options.requiresApiKey && !process.env.ANTHROPIC_API_KEY) {
          console.log(`⏭️  ${test.name} (skipped - no API key)`);
          this.skipped++;
          continue;
        }

        const testStartTime = Date.now();
        await test.testFn();
        const duration = Date.now() - testStartTime;
        
        console.log(`✅ ${test.name} (${duration}ms)`);
        this.passed++;
        
      } catch (error) {
        console.log(`❌ ${test.name}`);
        console.log(`   Error: ${error.message}`);
        if (options.verbose) {
          console.log(`   Stack: ${error.stack}`);
        }
        this.failed++;
      }
    }

    const totalTime = Date.now() - startTime;
    const total = this.passed + this.failed + this.skipped;
    
    console.log(`\n📊 Test Results:`);
    console.log(`   Total: ${total}`);
    console.log(`   Passed: ${this.passed}`);
    console.log(`   Failed: ${this.failed}`);
    console.log(`   Skipped: ${this.skipped}`);
    console.log(`   Time: ${totalTime}ms`);
    
    if (this.failed > 0) {
      console.log(`\n❌ ${this.failed} test(s) failed`);
      process.exit(1);
    } else {
      console.log(`\n✅ All tests passed!`);
    }
  }

  /**
   * Assert helper
   */
  assert(condition, message) {
    if (!condition) {
      throw new Error(message || 'Assertion failed');
    }
  }

  /**
   * Assert equal helper
   */
  assertEqual(actual, expected, message) {
    if (actual !== expected) {
      throw new Error(message || `Expected ${expected}, got ${actual}`);
    }
  }

  /**
   * Assert throws helper
   */
  async assertThrows(fn, message) {
    let threw = false;
    try {
      await fn();
    } catch (error) {
      threw = true;
    }
    if (!threw) {
      throw new Error(message || 'Expected function to throw');
    }
  }
}

// Create test runner instance
const runner = new TestRunner();

// Test utility functions
runner.test('Utility Functions - API Key Validation', () => {
  runner.assert(validateApiKeyFormat('sk-ant-api03-test123'), 'Valid API key should pass');
  runner.assert(!validateApiKeyFormat('invalid-key'), 'Invalid API key should fail');
  runner.assert(!validateApiKeyFormat(''), 'Empty string should fail');
  runner.assert(!validateApiKeyFormat(null), 'Null should fail');
});

// Test ConfigManager
runner.test('ConfigManager - Initialization', () => {
  const config = new ConfigManager();
  const defaultConfig = config.getConfig();
  
  runner.assert(defaultConfig.anthropic, 'Should have anthropic config');
  runner.assert(defaultConfig.usage, 'Should have usage config');
  runner.assert(defaultConfig.agents, 'Should have agents config');
});

runner.test('ConfigManager - API Key Management', () => {
  const config = new ConfigManager();
  
  // Test setting API key
  config.setApiKey('sk-ant-api03-test123');
  runner.assert(config.hasApiKey(), 'Should have API key after setting');
  
  // Test getting masked key
  const maskedKey = config.getApiKey(true);
  runner.assert(maskedKey.includes('*'), 'Masked key should contain asterisks');
  runner.assert(maskedKey.startsWith('sk-ant-api03'), 'Should show beginning of key');
});

runner.test('ConfigManager - Usage Tracking', () => {
  const config = new ConfigManager();
  
  // Reset usage stats to ensure clean test
  config.resetUsageStats();
  
  // Test usage update
  const initialUsage = config.getTodayUsage();
  config.updateUsage({
    inputTokens: 100,
    outputTokens: 50,
    requests: 1,
    cost: 0.01,
    errors: 0
  });
  
  const updatedUsage = config.getTodayUsage();
  runner.assertEqual(updatedUsage.inputTokens, initialUsage.inputTokens + 100, 'Input tokens should be updated');
  runner.assertEqual(updatedUsage.requests, initialUsage.requests + 1, 'Request count should be updated');
});

runner.test('ConfigManager - Daily Limits', () => {
  const config = new ConfigManager();
  
  // Set daily limit
  config.setDailyLimit(10.0, true);
  
  // Check limit status
  const limitCheck = config.checkDailyLimit();
  runner.assert(limitCheck.limit === 10.0, 'Should set correct limit');
  runner.assert(!limitCheck.exceeded, 'Should not be exceeded initially');
});

// Test TokenManager
runner.test('TokenManager - Token Estimation', () => {
  const tokenManager = new TokenManager();
  
  const text = 'Hello world, this is a test message.';
  const tokens = tokenManager.estimateTokens(text);
  
  runner.assert(tokens > 0, 'Should estimate positive tokens');
  runner.assert(tokens < text.length, 'Tokens should be less than character count');
});

runner.test('TokenManager - Message Token Estimation', () => {
  const tokenManager = new TokenManager();
  
  const message = {
    role: 'user',
    content: 'Write a simple function that adds two numbers.'
  };
  
  const tokens = tokenManager.estimateMessageTokens(message);
  runner.assert(tokens > 0, 'Should estimate positive tokens for message');
});

runner.test('TokenManager - Context Window Check', () => {
  const tokenManager = new TokenManager();
  
  const messages = [
    { role: 'system', content: 'You are a helpful assistant.' },
    { role: 'user', content: 'Hello!' }
  ];
  
  const analysis = tokenManager.checkContextWindow(messages);
  runner.assert(analysis.inputTokens > 0, 'Should calculate input tokens');
  runner.assert(analysis.fitsInContext, 'Simple messages should fit in context');
  runner.assert(analysis.utilizationPercent >= 0, 'Should calculate utilization');
});

runner.test('TokenManager - Conversation Optimization', () => {
  const tokenManager = new TokenManager();
  
  // Test basic optimization functionality
  const shortMessages = [
    { role: 'system', content: 'You are a helpful assistant.' },
    { role: 'user', content: 'Hello!' },
    { role: 'assistant', content: 'Hi there!' }
  ];
  
  // Test with messages that fit in context
  const shortResult = tokenManager.optimizeConversation(shortMessages);
  runner.assert(!shortResult.optimized, 'Short conversation should not be optimized');
  runner.assertEqual(shortResult.messages.length, shortMessages.length, 'Short conversation should remain unchanged');
  
  // Test token estimation works
  const tokens = tokenManager.estimateConversationTokens(shortMessages);
  runner.assert(tokens > 0, 'Should estimate positive tokens for conversation');
});

// Test AnthropicClient (without API calls)
runner.test('AnthropicClient - Initialization', () => {
  // Test with mock API key
  const client = new AnthropicClient({ apiKey: 'sk-ant-api03-test123' });
  
  runner.assert(client.apiKey === 'sk-ant-api03-test123', 'Should set API key');
  runner.assert(client.model, 'Should have default model');
  runner.assert(client.config, 'Should have config');
});

runner.test('AnthropicClient - Model Configuration', () => {
  const client = new AnthropicClient({ 
    apiKey: 'sk-ant-api03-test123',
    model: 'claude-3-5-haiku-20241022'
  });
  
  runner.assertEqual(client.model, 'claude-3-5-haiku-20241022', 'Should set specified model');
  
  const modelInfo = client.getModelInfo();
  runner.assert(modelInfo.maxTokens > 0, 'Should have token limit');
  runner.assert(modelInfo.inputCostPer1K > 0, 'Should have input cost');
});

runner.test('AnthropicClient - Cost Estimation', () => {
  const client = new AnthropicClient({ apiKey: 'sk-ant-api03-test123' });
  
  const estimate = client.estimateMessageCost('Hello world');
  runner.assert(estimate.estimatedInputTokens > 0, 'Should estimate input tokens');
  runner.assert(estimate.estimatedMinCost > 0, 'Should estimate minimum cost');
  runner.assert(estimate.estimatedMaxCost > estimate.estimatedMinCost, 'Max cost should be higher than min');
});

// Test AgentExecutor
runner.test('AgentExecutor - Agent Information', () => {
  const executor = new AgentExecutor({ anthropic: { apiKey: 'sk-ant-api03-test123' } });
  
  const agents = executor.getAvailableAgents();
  runner.assert(agents.length > 0, 'Should have available agents');
  
  const backendAgent = agents.find(a => a.type === 'backend-architect');
  runner.assert(backendAgent, 'Should have backend-architect agent');
  runner.assert(backendAgent.config.maxTokens > 0, 'Agent should have token limit');
});

runner.test('AgentExecutor - Cost Estimation', () => {
  const executor = new AgentExecutor({ anthropic: { apiKey: 'sk-ant-api03-test123' } });
  
  const estimate = executor.estimateAgentCost('backend-architect', 'Design a REST API');
  runner.assert(estimate.estimatedInputTokens > 0, 'Should estimate tokens');
  runner.assert(estimate.model, 'Should specify model');
});

// Test AnthropicIntegration
runner.test('AnthropicIntegration - Initialization', () => {
  const integration = new AnthropicIntegration({ apiKey: 'sk-ant-api03-test123' });
  
  runner.assert(integration.client, 'Should have client');
  runner.assert(integration.executor, 'Should have executor');
  runner.assert(integration.tokenManager, 'Should have token manager');
  runner.assert(integration.configManager, 'Should have config manager');
});

runner.test('AnthropicIntegration - Agent Information', () => {
  const integration = new AnthropicIntegration({ apiKey: 'sk-ant-api03-test123' });
  
  const agents = integration.getAgents();
  runner.assert(agents.length > 0, 'Should return available agents');
  
  const models = integration.getModels();
  runner.assert(models.length > 0, 'Should return available models');
});

runner.test('AnthropicIntegration - Configuration Management', () => {
  const integration = new AnthropicIntegration({ apiKey: 'sk-ant-api03-test123' });
  
  const config = integration.getConfig();
  runner.assert(config.anthropic, 'Should have anthropic config');
  
  // Test config update
  const updatedConfig = integration.updateConfig({
    agents: { autoGenerateTests: false }
  });
  runner.assert(updatedConfig.agents.autoGenerateTests === false, 'Should update config');
});

// Tests requiring actual API key
runner.test('API Key Validation', async () => {
  const integration = new AnthropicIntegration();
  const validation = await integration.client.validateApiKey();
  
  runner.assert(typeof validation.valid === 'boolean', 'Should return validation result');
  if (validation.valid) {
    console.log('   ✅ API key is valid and working');
  } else {
    console.log('   ℹ️  API key validation result:', validation);
  }
}, { requiresApiKey: true });

runner.test('Simple Agent Execution', async () => {
  const integration = new AnthropicIntegration();
  
  const result = await integration.execute('backend-architect', 'Say hello in one sentence', {
    maxTokens: 50,
    temperature: 0.1
  });
  
  runner.assert(result.content, 'Should return content');
  runner.assert(result.usage, 'Should return usage info');
  runner.assert(result.metadata, 'Should return metadata');
  
  console.log(`   📊 Used ${result.usage.input_tokens} input + ${result.usage.output_tokens} output tokens`);
}, { requiresApiKey: true });

runner.test('Streaming Agent Execution', async () => {
  const integration = new AnthropicIntegration();
  
  const stream = await integration.executeStream('ai-engineer', 'Explain AI in one paragraph', {
    maxTokens: 100
  });
  
  let content = '';
  let tokenInfo = null;
  
  for await (const chunk of stream) {
    if (chunk.type === 'content') {
      content = chunk.fullContent;
    } else if (chunk.type === 'usage') {
      tokenInfo = chunk.tokens;
    }
  }
  
  runner.assert(content.length > 0, 'Should receive content via streaming');
  runner.assert(tokenInfo, 'Should receive token usage info');
  
  console.log(`   📊 Streamed ${tokenInfo.input + tokenInfo.output} total tokens`);
}, { requiresApiKey: true });

runner.test('Health Check', async () => {
  const integration = new AnthropicIntegration();
  
  const health = await integration.healthCheck();
  runner.assert(typeof health.overall === 'boolean', 'Should return overall health status');
  runner.assert(typeof health.apiKey === 'boolean', 'Should check API key');
  runner.assert(typeof health.connectivity === 'boolean', 'Should check connectivity');
  
  if (health.overall) {
    console.log('   ✅ All systems healthy');
  } else {
    console.log('   ⚠️  Health check issues:', health.configIssues || health.error);
  }
}, { requiresApiKey: true });

runner.test('Cost Estimation Accuracy', async () => {
  const integration = new AnthropicIntegration();
  
  // Get cost estimate
  const estimate = integration.estimateCost('test-writer-fixer', 'Write a simple test', {
    maxTokens: 200
  });
  
  // Execute and compare
  const result = await integration.execute('test-writer-fixer', 'Write a simple test', {
    maxTokens: 200
  });
  
  const actualCost = result.metadata.estimatedCost;
  const estimatedCost = estimate.estimatedMinCost;
  
  runner.assert(actualCost >= estimatedCost, 'Actual cost should be at least minimum estimate');
  runner.assert(actualCost <= estimate.estimatedMaxCost, 'Actual cost should not exceed maximum estimate');
  
  const accuracy = Math.abs(actualCost - estimatedCost) / actualCost;
  console.log(`   📊 Cost estimation accuracy: ${((1 - accuracy) * 100).toFixed(1)}%`);
}, { requiresApiKey: true });

// Export test runner for external use
module.exports = { TestRunner, runner };

// Run tests if this file is executed directly
if (require.main === module) {
  runner.run({ verbose: process.argv.includes('--verbose') });
}