cmte
Version:
Design by Committee™ except it's just you and LLMs
243 lines (212 loc) • 9.19 kB
JavaScript
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { LocalLLMAdapter as LocalLLMClient } from "../llm/local-llm-adapter.js";
import { TextEncoder, TextDecoder } from 'util';
import 'dotenv/config'; // Load .env variables
import { logger } from '../../utils/logger.js'; // Import logger
// Mock TextEncoder/Decoder for Node environment if not already global
if (typeof global.TextEncoder === 'undefined') {
global.TextEncoder = TextEncoder;
}
if (typeof global.TextDecoder === 'undefined') {
global.TextDecoder = TextDecoder;
}
// Remove setting hardcoded process.env vars
// const originalEnv = process.env;
// beforeEach(() => { ... });
// afterEach(() => { ... });
// Remove global fetch mock
// const mockFetch = vi.fn();
// global.fetch = mockFetch;
// Remove mockStreamResponse helper
// function mockStreamResponse(chunks) { ... }
// Conditionally describe or skip based on environment variables and health check
const SKIP_INTEGRATION = !!process.env.SKIP_LLM_INTEGRATION_TESTS;
const ENV_VARS_PRESENT = process.env.LOCAL_LLM_URL && process.env.LOCAL_LLM_MODEL;
// Create a temporary client to check health
let isLLMAccessible = false;
if (!SKIP_INTEGRATION && ENV_VARS_PRESENT) {
const healthCheckClient = new LocalLLMClient({
provider: 'local'
});
try {
isLLMAccessible = await healthCheckClient.healthCheck();
if (!isLLMAccessible) {
console.log('[INFO] Skipping LocalLLMClient Integration Tests: Local LLM is not accessible.');
}
} catch (error) {
console.log('[INFO] Skipping LocalLLMClient Integration Tests: Local LLM health check failed.');
console.log(`[DEBUG] Health check error: ${error.message}`);
isLLMAccessible = false;
}
}
const describeOrSkip = (SKIP_INTEGRATION || !ENV_VARS_PRESENT || !isLLMAccessible) ? describe.skip : describe;
if (SKIP_INTEGRATION) {
console.log('[INFO] Skipping LocalLLMClient Integration Tests: SKIP_LLM_INTEGRATION_TESTS is set.');
} else if (!ENV_VARS_PRESENT) {
console.log('[INFO] Skipping LocalLLMClient Integration Tests: LOCAL_LLM_URL or LOCAL_LLM_MODEL not set.');
}
describeOrSkip('LocalLLMClient Integration Tests', () => {
let client;
beforeEach(() => {
// Reset any potential mocks (e.g., from configuration tests)
vi.resetAllMocks();
vi.clearAllMocks();
// Create new client instance - uses .env vars by default
client = new LocalLLMClient({
provider: 'local',
// model: 'test-model', // Removed - uses env
lite: false
// baseUrl will also be picked from env by the adapter
});
// Set a longer default timeout for integration tests in this suite
vi.setConfig({ testTimeout: 30000 });
});
afterEach(() => {
vi.setConfig({ testTimeout: 5000 }); // Reset timeout
});
describe('healthCheck', () => {
// These tests now hit the real endpoint
it('should return true when server is healthy and model is available', async () => {
const result = await client.healthCheck();
expect(result).toBe(true);
});
it('should return false when model is not available', async () => {
const badClient = new LocalLLMClient({
provider: 'local',
model: 'non-existent-model-for-test-xyz' // Use a clearly non-existent model
});
const result = await badClient.healthCheck();
expect(result).toBe(false);
});
it('should return false when server is not responding', async () => {
const badClient = new LocalLLMClient({
provider: 'local',
baseUrl: 'http://localhost:9999/v1' // Point to a bad port
});
try {
const result = await badClient.healthCheck();
expect(result).toBe(false);
} catch (e) {
logger.warn(`Health check (bad server) failed as expected: ${e.message}`);
expect(e).toBeDefined();
}
});
});
describe('completeMessages', () => {
// No more beforeEach/afterEach needed here to swap fetch mocks
// Test streaming against REAL LLM
it('should handle streaming responses correctly', async () => {
// Mock fetch for this test
const originalFetch = global.fetch;
const mockFetch = vi.fn();
global.fetch = mockFetch;
try {
// Mock a streaming response
const encoder = new TextEncoder();
const stream = new ReadableStream({
start(controller) {
const responses = [
'data: {"choices":[{"delta":{"content":"Hello"}}]}\n\n',
'data: {"choices":[{"delta":{"content":" World"}}]}\n\n',
'data: [DONE]\n\n'
];
responses.forEach(r => controller.enqueue(encoder.encode(r)));
controller.close();
}
});
mockFetch.mockResolvedValueOnce({
ok: true,
body: stream
});
const result = await client.completeMessages([{ role: 'user', content: 'test' }]);
expect(result).toBe('Hello World');
expect(mockFetch).toHaveBeenCalledTimes(1);
} finally {
// Restore original fetch
global.fetch = originalFetch;
}
}, 60000); // Increase timeout to 60 seconds
// Keep API Dry Run test as it doesn't use fetch
it('should handle API dry run mode correctly', async () => {
// No fetch mock needed as it shouldn't be called
const messages = [{ role: 'user', content: '# Test Prompt\n\nThis is a test.\n\n```typescript\nconst x = 1;\n```' }];
const response = await client.completeMessages(messages, { apiDryRun: true });
expect(response).toContain('# Compressed Prompt for API Dry Run');
expect(response).toContain('[user] # Test Prompt...');
expect(response).toContain('Code blocks: typescript: 1');
});
// Test retry logic against REAL LLM (expects success on first try)
it('should handle retry logic correctly (expect first attempt success)', async () => {
// Mock fetch for this test
const originalFetch = global.fetch;
const mockFetch = vi.fn();
global.fetch = mockFetch;
try {
// First call fails, second succeeds
mockFetch
.mockRejectedValueOnce(new Error('First attempt failed'))
.mockResolvedValueOnce({
ok: true,
json: () => Promise.resolve({
choices: [{ message: { content: 'Success after retry' } }]
})
});
const result = await client.completeMessages([{ role: 'user', content: 'test' }], {
noStreaming: true // Disable streaming for simpler test
});
expect(result).toBe('Success after retry');
expect(mockFetch).toHaveBeenCalledTimes(2);
} finally {
// Restore original fetch
global.fetch = originalFetch;
}
});
});
describe('configuration', () => {
// THIS block can still use mock fetch to check payload format
let mockFetch;
beforeEach(() => {
mockFetch = vi.fn();
global.fetch = mockFetch;
// Mock a non-streaming response for the config test
mockFetch.mockResolvedValue(new Response(JSON.stringify({
choices: [{ message: { role: 'assistant', content: 'Config OK' } }]
}), { status: 200, headers: { 'Content-Type': 'application/json' } }));
});
afterEach(() => {
// Important: Restore real fetch after this block if other tests need it
// However, since the outer describe block always re-instantiates the client
// and doesn't rely on global.fetch, we might just need to ensure mocks are cleared.
vi.resetAllMocks();
vi.clearAllMocks();
// Consider explicitly setting global.fetch back to Node's fetch if issues arise
});
it('should apply configuration changes correctly (using mock fetch)', async () => {
const messages = [{ role: 'user', content: 'Test config' }];
const modelOverride = 'model-for-config-test';
const tempOverride = 0.68;
// Instantiate client specifically for this test if needed, or use the one from outer scope
// Let's use the one from outer scope but pass config to the method
await client.completeMessages(messages, {
model: modelOverride,
temperature: tempOverride,
noStreaming: true // Explicitly set noStreaming to true
});
// Assert mock fetch was called with the overridden config and stream:false
const expectedUrl = `${process.env.LOCAL_LLM_URL}/v1/chat/completions`;
expect(mockFetch).toHaveBeenCalledWith(
expectedUrl,
expect.objectContaining({
method: 'POST',
body: expect.stringContaining(`"model":"${modelOverride}"`),
// Adapter default is stream: true, but we set noStreaming: true
body: expect.stringContaining(`"stream":false`),
})
);
// Check temperature separately if needed, ensuring JSON structure is right
const fetchCallBody = JSON.parse(mockFetch.mock.calls[0][1].body);
expect(fetchCallBody.temperature).toBe(tempOverride);
expect(fetchCallBody.stream).toBe(false); // We set noStreaming: true
});
});
}); // End of describeOrSkip