cmte/src/core/llm/__tests__/local-llm-adapter.test.js

Design by Committee™ except it's just you and LLMs

import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { LocalLLMAdapter } from "../local-llm-adapter.js";

// Mock logger
vi.mock('../../../utils/logger.js', () => ({
  logger: {
    debug: vi.fn(),
    info: vi.fn(),
    warn: vi.fn(),
    error: vi.fn()
  }
}));

import { logger } from '../../../utils/logger.js'; // Import real logger if needed
import 'dotenv/config'; // Load .env variables

// Conditionally describe or skip based on environment variables and health check
const SKIP_INTEGRATION = !!process.env.SKIP_LLM_INTEGRATION_TESTS;
const ENV_VARS_PRESENT = process.env.LOCAL_LLM_URL && process.env.LOCAL_LLM_MODEL;

// Create a temporary adapter to check health
let isLLMAccessible = false;
if (!SKIP_INTEGRATION && ENV_VARS_PRESENT) {
  const healthCheckAdapter = new LocalLLMAdapter({
    provider: 'local'
  });
  
  try {
    isLLMAccessible = await healthCheckAdapter.healthCheck();
    if (!isLLMAccessible) {
      console.log('[INFO] Skipping LocalLLMAdapter Integration Tests: Local LLM is not accessible.');
    }
  } catch (error) {
    console.log('[INFO] Skipping LocalLLMAdapter Integration Tests: Local LLM health check failed.');
    console.log(`[DEBUG] Health check error: ${error.message}`);
    isLLMAccessible = false;
  }
}

const describeOrSkip = (SKIP_INTEGRATION || !ENV_VARS_PRESENT || !isLLMAccessible) ? describe.skip : describe;

if (SKIP_INTEGRATION) {
  console.log('[INFO] Skipping LocalLLMAdapter Integration Tests: SKIP_LLM_INTEGRATION_TESTS is set.');
} else if (!ENV_VARS_PRESENT) {
  console.log('[INFO] Skipping LocalLLMAdapter Integration Tests: LOCAL_LLM_URL or LOCAL_LLM_MODEL not set.');
}

describeOrSkip('LocalLLMAdapter Integration Tests', () => {
  let adapter;

  beforeEach(() => {
    // Reset mocks (if any others exist, keep this)
    vi.clearAllMocks(); // Keep in case other mocks are used/added later

    // Create adapter instance - LET IT USE ENV VARS by default
    adapter = new LocalLLMAdapter({
      provider: 'local',
      // model: 'test-model', // REMOVED - Use LOCAL_LLM_MODEL from .env via adapter default
      lite: false,
      apiDryRun: false,
      // baseUrl: 'http://localhost:1234/v1' // REMOVED - Use LOCAL_LLM_URL from .env via adapter default
    });
    // Optional: Increase timeout for tests hitting network
    vi.setConfig({ testTimeout: 30000 }); // 30 seconds timeout
  });

  afterEach(() => {
    vi.clearAllMocks(); // Keep for consistency
    vi.setConfig({ testTimeout: 5000 }); // Reset timeout if changed
  });

  describe('healthCheck', () => {
    it('should return true when local LLM is available and model exists', async () => {
      // Assuming 'test-model' or a default model exists on the local server
      // This now makes a real HTTP request
      const result = await adapter.healthCheck();
      expect(result).toBe(true);
      // Cannot check mockFetch calls anymore
    });

    // Note: Testing failure cases without mocks is harder.
    // These tests might need adjustment or rely on specific server states.
    it('should return false when local LLM server is not running', async () => {
      // Temporarily point to a non-existent server to test connection failure
      const badAdapter = new LocalLLMAdapter({
        provider: 'local', model: 'test-model', baseUrl: 'http://localhost:9999/v1'
      });
      try {
        const result = await badAdapter.healthCheck();
        expect(result).toBe(false);
      } catch (e) {
        // Depending on fetch implementation, it might throw or return false
        logger.warn(`Health check failed as expected: ${e.message}`);
        expect(e).toBeDefined(); // Or expect result to be false if fetch handles it gracefully
      }
    });

    it('should return false when the specified model does not exist', async () => {
       const adapterWithBadModel = new LocalLLMAdapter({
         provider: 'local',
         model: 'non-existent-model-abcxyz', // Use a name unlikely to exist
         baseUrl: 'http://localhost:1234/v1'
       });
       const result = await adapterWithBadModel.healthCheck();
       expect(result).toBe(false);
    });


  });

  describe('completeMessages', () => {
    const testMessages = [{ role: 'user', content: 'Say "Hello"' }];

    it('should handle successful message completion (hitting real LLM)', async () => {
      // Removed mock setup for streaming response
      // This now makes a real HTTP request
      let result;
      try {
        result = await adapter.completeMessages(testMessages, {
          noStreaming: true // Disable streaming for more reliable test
        });
        expect(result).toBeDefined();
        expect(result.length).toBeGreaterThan(0);
        // Make assertion more flexible for real LLM response
        expect(result.toLowerCase()).toContain('hello');
        logger.info(`Local LLM Response: ${result}`);
      } catch (error) {
         logger.error(`Error during completeMessages test: ${error}`);
         // Re-throw the error to fail the test clearly
         throw error;
      }
    }, 60000); // Increase timeout to 60 seconds

    it('should handle API dry run mode', async () => {
      adapter = new LocalLLMAdapter({
        provider: 'local',
        model: 'test-model',
        lite: false,
        apiDryRun: true,
        baseUrl: process.env.LOCAL_LLM_URL || 'http://localhost:1234'
      });
      const result = await adapter.completeMessages([{
        role: 'user',
        content: '# Test Prompt\n\nThis is a test.\n\n```typescript\nconst x = 1;\n```'
      }]);
      expect(result).toContain('# Compressed Prompt for API Dry Run');
      expect(result).toContain('[user] # Test Prompt...');
      expect(result).toContain('Code blocks: typescript: 1');
    });

    it('should use custom model config when provided (hitting real LLM)', async () => {
      const customModel = process.env.LOCAL_LLM_MODEL;
      const adapterWithCustom = new LocalLLMAdapter({
        provider: 'local',
        noStreaming: true // Disable streaming for more reliable test
      });

      let result;
      try {
        result = await adapterWithCustom.completeMessages(testMessages, {
          temperature: 0.5,
          maxTokens: 50,
          model: customModel,
          noStreaming: true
        });
        expect(result).toBeDefined();
        expect(result.length).toBeGreaterThan(0);
        expect(result.toLowerCase()).toContain('hello');
        logger.info(`Local LLM Response (custom model ${customModel}): ${result}`);
      } catch (error) {
        logger.error(`Error during completeMessages test with custom model: ${error}`);
        throw error;
      }
    }, 60000); // Increase timeout to 60 seconds
  });
});