cmte/src/processors/test-api-dry.js

Design by Committee™ except it's just you and LLMs

/**
 * Test script for the --apidry and --lite options
 * 
 * This script tests the functionality of API dry run and lite model options
 * 
 * Usage: npm run test:apidry
 */

import path from 'path';
import fs from 'fs/promises';
import getClaudeClient from "../core/llm/claude-adapter.js";
import logger from "../utils/logger.js";
async function saveFileWithDir(filePath, content) {
  await fs.mkdir(path.dirname(filePath), {
    recursive: true
  });
  await fs.writeFile(filePath, content);
}
async function main() {
  try {
    logger.info('Testing --apidry and --lite options');

    // Create test directory
    const testDir = 'test/api-dry';
    await fs.mkdir(testDir, {
      recursive: true
    });

    // Create a complex prompt with multiple sections and code blocks
    const prompt = `
# API Analysis Task

## Background
We need to design a new API for our e-commerce system. The API should support the following operations:
- Product listing
- Product details
- Shopping cart management
- Checkout process
- Order history

## Current System
Our current system uses REST APIs but we're considering GraphQL for the new implementation.

## Code Example (Current REST API)
\`\`\`javascript
// Get product details
app.get('/api/products/:id', (req, res) => {
  const productId = req.params.id;
  const product = db.getProduct(productId);
  res.json(product);
});

// Add to cart
app.post('/api/cart', (req, res) => {
  const { productId, quantity } = req.body;
  const cart = cartService.addToCart(req.user.id, productId, quantity);
  res.json(cart);
});
\`\`\`

## Database Schema
\`\`\`sql
CREATE TABLE products (
  id SERIAL PRIMARY KEY,
  name VARCHAR(255) NOT NULL,
  description TEXT,
  price DECIMAL(10, 2) NOT NULL,
  inventory INTEGER NOT NULL
);

CREATE TABLE orders (
  id SERIAL PRIMARY KEY,
  user_id INTEGER REFERENCES users(id),
  total DECIMAL(10, 2) NOT NULL,
  status VARCHAR(50) NOT NULL,
  created_at TIMESTAMP DEFAULT NOW()
);
\`\`\`

## Variables to Consider
- Number of users: {{user_count}}
- Number of products: {{product_count}}
- Average order value: {{avg_order_value}}
- Peak requests per second: {{peak_rps}}

## Request
Please analyze the best API approach for our needs. Consider REST vs GraphQL tradeoffs.
`;

    // Output paths
    const apiDryOutputPath = path.join(testDir, 'api-dry-response.o.xml');
    const apiDryPromptPath = path.join(testDir, 'api-dry-response.sent.xml');
    const liteOutputPath = path.join(testDir, 'lite-response.o.xml');
    const litePromptPath = path.join(testDir, 'lite-response.sent.xml');

    // Get Claude client
    const claude = getClaudeClient();

    // Test 1: API Dry Run
    logger.info('1. Testing --apidry option');

    // Clear any existing files first
    try {
      await fs.unlink(apiDryOutputPath);
      await fs.unlink(apiDryPromptPath);
    } catch (e) {
      // Ignore errors if files don't exist
    }

    // Call Claude with apiDryRun option
    const apiDryResponse = await claude.respond(prompt, {
      apiDryRun: true,
      outputPath: apiDryOutputPath
    });

    // Check if prompt file exists
    const apiDryPromptExists = await fs.stat(apiDryPromptPath).then(() => true).catch(() => false);
    if (apiDryPromptExists) {
      logger.info('✅ API dry run prompt file was created successfully');

      // Read the content to verify
      const savedPrompt = await fs.readFile(apiDryPromptPath, 'utf-8');
      if (savedPrompt.includes('Compressed Prompt for API Dry Run')) {
        logger.info('✅ API dry run prompt content is correct (compressed format)');
      } else {
        logger.error('❌ API dry run prompt is not in compressed format');
      }
    } else {
      logger.error('❌ API dry run prompt file was not created');
    }

    // Check the response
    if (apiDryResponse.length < 500) {
      logger.info('✅ API dry run response is concise as expected');
      logger.info(`Response: ${apiDryResponse}`);
    } else {
      logger.error('❌ API dry run response is too verbose');
    }

    // Test 2: Lite Model
    logger.info('2. Testing --lite option');

    // Clear any existing files first
    try {
      await fs.unlink(liteOutputPath);
      await fs.unlink(litePromptPath);
    } catch (e) {
      // Ignore errors if files don't exist
    }

    // Simplify the prompt for lite test to save tokens/cost
    const shortPrompt = "What is GraphQL and how does it differ from REST APIs? Keep it brief.";

    // Call Claude with lite option
    try {
      const liteStartTime = Date.now();
      const liteResponse = await claude.respond(shortPrompt, {
        lite: true,
        savePrompt: true,
        outputPath: liteOutputPath
      });
      const liteEndTime = Date.now();
      const liteDuration = liteEndTime - liteStartTime;
      logger.info(`✅ Lite model response received in ${liteDuration}ms`);
      logger.info(`First 100 chars: ${liteResponse.substring(0, 100)}...`);

      // Check if prompt file exists
      const litePromptExists = await fs.stat(litePromptPath).then(() => true).catch(() => false);
      if (litePromptExists) {
        logger.info('✅ Lite prompt file was created successfully');
      } else {
        logger.error('❌ Lite prompt file was not created');
      }
    } catch (error) {
      logger.error('❌ Error using lite model', {
        error
      });
    }

    // Test 3: Combine API Dry Run with Lite Model
    logger.info('3. Testing combined --apidry --lite options');
    const combinedOutputPath = path.join(testDir, 'combined-response.o.xml');
    try {
      const combinedResponse = await claude.respond(shortPrompt, {
        apiDryRun: true,
        lite: true,
        outputPath: combinedOutputPath
      });
      logger.info('✅ Combined options worked successfully');
      logger.info(`Response: ${combinedResponse}`);
    } catch (error) {
      logger.error('❌ Error with combined options', {
        error
      });
    }
    logger.info('Tests completed');
  } catch (error) {
    logger.error('Test failed', {
      error
    });
    process.exit(1);
  }
}
main();