@quantumai/quantum-cli-core
Version:
Quantum CLI Core - Multi-LLM Collaboration System
399 lines • 19.6 kB
JavaScript
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { validateCollaborationConfig, mergeCollaborationConfigs, loadCollaborationConfigFromEnv, loadCollaborationConfigFromFile, writeDefaultCollaborationConfig, DEFAULT_COLLABORATION_CONFIG, DEFAULT_PROVIDERS, } from './collaboration-config.js';
describe('CollaborationConfig', () => {
let originalEnv;
beforeEach(() => {
originalEnv = { ...process.env };
});
afterEach(() => {
process.env = originalEnv;
vi.restoreAllMocks();
});
describe('validateCollaborationConfig', () => {
it('should validate a complete valid configuration', () => {
const config = {
enabled: true,
autoVerifyThreshold: 0.7,
showConfidenceScores: true,
maxCostPerQuery: 0.05,
maxProviders: 2,
consensusThreshold: 0.8,
verificationMode: 'smart',
fallbackBehavior: 'primary',
maxLatency: 5000,
parallelExecution: true,
cacheResponses: true,
minConfidenceScore: 0.6,
requireConsensus: false,
diversityBonus: 0.1,
providers: [
{
id: 'test-provider',
name: 'Test Provider',
type: 'openai',
enabled: true,
maxTokens: 2048,
temperature: 0.1,
costPerToken: 0.00001,
},
],
};
const result = validateCollaborationConfig(config);
expect(result.valid).toBe(true);
expect(result.errors).toHaveLength(0);
expect(result.config).toMatchObject(config);
});
it('should apply defaults for missing values', () => {
const config = { enabled: true };
const result = validateCollaborationConfig(config);
expect(result.valid).toBe(true);
expect(result.config.autoVerifyThreshold).toBe(DEFAULT_COLLABORATION_CONFIG.autoVerifyThreshold);
expect(result.config.maxCostPerQuery).toBe(DEFAULT_COLLABORATION_CONFIG.maxCostPerQuery);
expect(result.config.verificationMode).toBe(DEFAULT_COLLABORATION_CONFIG.verificationMode);
});
it('should validate threshold ranges', () => {
const invalidConfigs = [
{ autoVerifyThreshold: -0.1 },
{ autoVerifyThreshold: 1.5 },
{ consensusThreshold: -0.5 },
{ consensusThreshold: 2.0 },
{ minConfidenceScore: -1 },
{ minConfidenceScore: 1.1 },
];
invalidConfigs.forEach((config) => {
const result = validateCollaborationConfig(config);
expect(result.valid, `Should be invalid for config: ${JSON.stringify(config)}`).toBe(false);
expect(result.errors.length).toBeGreaterThan(0);
});
});
it('should validate cost constraints', () => {
const invalidConfigs = [
{ maxCostPerQuery: -0.01 },
{ maxCostPerQuery: 'invalid' },
];
invalidConfigs.forEach((config) => {
const result = validateCollaborationConfig(config);
expect(result.valid).toBe(false);
expect(result.errors.some((e) => e.field === 'maxCostPerQuery')).toBe(true);
});
});
it('should validate provider limits', () => {
const invalidConfigs = [
{ maxProviders: 0 },
{ maxProviders: -1 },
{ maxProviders: 1.5 },
{ maxProviders: 'two' },
];
invalidConfigs.forEach((config) => {
const result = validateCollaborationConfig(config);
expect(result.valid).toBe(false);
expect(result.errors.some((e) => e.field === 'maxProviders')).toBe(true);
});
});
it('should validate latency settings', () => {
const invalidConfigs = [
{ maxLatency: 500 }, // Too low (< 1000ms)
{ maxLatency: -1000 },
{ maxLatency: 1.5 },
];
invalidConfigs.forEach((config) => {
const result = validateCollaborationConfig(config);
expect(result.valid).toBe(false);
expect(result.errors.some((e) => e.field === 'maxLatency')).toBe(true);
});
});
it('should validate verification modes', () => {
const validModes = ['automatic', 'manual', 'smart'];
const invalidModes = ['auto', 'always', 'never', 'random'];
validModes.forEach((mode) => {
const result = validateCollaborationConfig({
verificationMode: mode,
});
expect(result.valid, `Should be valid for mode: ${mode}`).toBe(true);
});
invalidModes.forEach((mode) => {
const result = validateCollaborationConfig({
verificationMode: mode,
});
expect(result.valid, `Should be invalid for mode: ${mode}`).toBe(false);
});
});
it('should validate fallback behaviors', () => {
const validBehaviors = ['primary', 'random', 'best'];
const invalidBehaviors = ['secondary', 'cheapest', 'fastest'];
validBehaviors.forEach((behavior) => {
const result = validateCollaborationConfig({
fallbackBehavior: behavior,
});
expect(result.valid, `Should be valid for behavior: ${behavior}`).toBe(true);
});
invalidBehaviors.forEach((behavior) => {
const result = validateCollaborationConfig({
fallbackBehavior: behavior,
});
expect(result.valid, `Should be invalid for behavior: ${behavior}`).toBe(false);
});
});
it('should validate provider configurations', () => {
const invalidProviders = [
{ name: 'Test', type: 'openai' }, // Missing id
{ id: 'test', type: 'openai' }, // Missing name
{ id: 'test', name: 'Test' }, // Missing type
{ id: 'test', name: 'Test', type: 'openai', maxTokens: -100 }, // Invalid maxTokens
{ id: 'test', name: 'Test', type: 'openai', temperature: -1 }, // Invalid temperature
{ id: 'test', name: 'Test', type: 'openai', temperature: 3 }, // Temperature too high
{ id: 'test', name: 'Test', type: 'openai', costPerToken: -0.01 }, // Negative cost
];
invalidProviders.forEach((provider, index) => {
const config = { providers: [provider] };
const result = validateCollaborationConfig(config);
expect(result.valid, `Should be invalid for provider ${index}: ${JSON.stringify(provider)}`).toBe(false);
});
});
it('should detect duplicate provider IDs', () => {
const config = {
providers: [
{ id: 'test', name: 'Test 1', type: 'openai' },
{ id: 'test', name: 'Test 2', type: 'anthropic' }, // Duplicate ID
],
};
const result = validateCollaborationConfig(config);
expect(result.valid).toBe(false);
expect(result.errors.some((e) => e.message.includes('unique'))).toBe(true);
});
it('should provide detailed error messages', () => {
const config = {
autoVerifyThreshold: 2.0,
maxProviders: -1,
verificationMode: 'invalid-mode',
};
const result = validateCollaborationConfig(config);
expect(result.valid).toBe(false);
expect(result.errors.length).toBeGreaterThanOrEqual(3);
result.errors.forEach((error) => {
expect(error.field).toBeDefined();
expect(error.message).toBeDefined();
expect(error.value).toBeDefined();
});
});
});
describe('mergeCollaborationConfigs', () => {
it('should merge multiple configurations with proper precedence', () => {
const base = { enabled: false, maxCostPerQuery: 0.01 };
const override1 = { enabled: true, autoVerifyThreshold: 0.8 };
const override2 = { maxCostPerQuery: 0.05, showConfidenceScores: true };
const result = mergeCollaborationConfigs(base, override1, override2);
expect(result.enabled).toBe(true); // From override1
expect(result.autoVerifyThreshold).toBe(0.8); // From override1
expect(result.maxCostPerQuery).toBe(0.05); // From override2 (last wins)
expect(result.showConfidenceScores).toBe(true); // From override2
});
it('should merge provider arrays without duplication', () => {
const config1 = {
providers: [{ id: 'provider1', name: 'Provider 1', type: 'openai' }],
};
const config2 = {
providers: [
{ id: 'provider1', name: 'Provider 1 Updated', type: 'openai' }, // Same ID
{ id: 'provider2', name: 'Provider 2', type: 'anthropic' }, // New ID
],
};
const result = mergeCollaborationConfigs(config1, config2);
// Should have 2 providers (provider1 updated, provider2 added)
expect(result.providers).toHaveLength(2);
expect(result.providers?.find((p) => p.id === 'provider1')?.name).toBe('Provider 1 Updated');
expect(result.providers?.find((p) => p.id === 'provider2')).toBeDefined();
});
it('should return valid configuration after merging', () => {
const config1 = { enabled: true };
const config2 = { autoVerifyThreshold: 0.9 };
const result = mergeCollaborationConfigs(config1, config2);
const validation = validateCollaborationConfig(result);
expect(validation.valid).toBe(true);
});
});
describe('loadCollaborationConfigFromEnv', () => {
it('should load configuration from environment variables', () => {
process.env.ENABLE_COLLABORATION = 'true';
process.env.COLLABORATION_AUTO_VERIFY_THRESHOLD = '0.8';
process.env.COLLABORATION_MAX_COST_PER_QUERY = '0.1';
process.env.COLLABORATION_VERIFICATION_MODE = 'automatic';
process.env.COLLABORATION_SHOW_CONFIDENCE = 'true';
const config = loadCollaborationConfigFromEnv();
expect(config.enabled).toBe(true);
expect(config.autoVerifyThreshold).toBe(0.8);
expect(config.maxCostPerQuery).toBe(0.1);
expect(config.verificationMode).toBe('automatic');
expect(config.showConfidenceScores).toBe(true);
});
it('should load provider configurations from environment', () => {
process.env.OPENAI_API_KEY = 'test-openai-key';
process.env.ANTHROPIC_API_KEY = 'test-anthropic-key';
process.env.OPENAI_MODEL = 'gpt-4-turbo';
process.env.ANTHROPIC_MODEL = 'claude-3-opus';
const config = loadCollaborationConfigFromEnv();
expect(config.providers).toBeDefined();
expect(config.providers?.length).toBe(2);
const openaiProvider = config.providers?.find((p) => p.type === 'openai');
expect(openaiProvider?.apiKey).toBe('test-openai-key');
expect(openaiProvider?.modelName).toBe('gpt-4-turbo');
const anthropicProvider = config.providers?.find((p) => p.type === 'anthropic');
expect(anthropicProvider?.apiKey).toBe('test-anthropic-key');
expect(anthropicProvider?.modelName).toBe('claude-3-opus');
});
it('should handle invalid environment values gracefully', () => {
process.env.COLLABORATION_AUTO_VERIFY_THRESHOLD = 'invalid-number';
process.env.COLLABORATION_MAX_COST_PER_QUERY = 'not-a-number';
process.env.COLLABORATION_VERIFICATION_MODE = 'invalid-mode';
const config = loadCollaborationConfigFromEnv();
// Should skip invalid values
expect(config.autoVerifyThreshold).toBeUndefined();
expect(config.maxCostPerQuery).toBeUndefined();
expect(config.verificationMode).toBeUndefined();
});
it('should return empty config when no environment variables are set', () => {
// Clear all collaboration-related env vars
Object.keys(process.env).forEach((key) => {
if (key.startsWith('COLLABORATION_') ||
key.startsWith('OPENAI_') ||
key.startsWith('ANTHROPIC_')) {
delete process.env[key];
}
});
const config = loadCollaborationConfigFromEnv();
expect(Object.keys(config)).toHaveLength(0);
});
});
describe('loadCollaborationConfigFromFile', () => {
it('should return empty config when file does not exist', () => {
// Mock fs.existsSync to return false
const mockExistsSync = vi.fn().mockReturnValue(false);
vi.doMock('fs', () => ({ existsSync: mockExistsSync }));
const config = loadCollaborationConfigFromFile();
expect(config).toEqual({});
});
it('should handle file read errors gracefully', () => {
// Mock fs to throw error
const mockExistsSync = vi.fn().mockReturnValue(true);
const mockReadFileSync = vi.fn().mockImplementation(() => {
throw new Error('File read error');
});
vi.doMock('fs', () => ({
existsSync: mockExistsSync,
readFileSync: mockReadFileSync,
}));
const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => { });
const config = loadCollaborationConfigFromFile();
expect(config).toEqual({});
expect(consoleSpy).toHaveBeenCalled();
consoleSpy.mockRestore();
});
});
describe('writeDefaultCollaborationConfig', () => {
it('should create default configuration file', () => {
const mockMkdirSync = vi.fn();
const mockWriteFileSync = vi.fn();
const mockExistsSync = vi.fn().mockReturnValue(false);
vi.doMock('fs', () => ({
existsSync: mockExistsSync,
mkdirSync: mockMkdirSync,
writeFileSync: mockWriteFileSync,
}));
const result = writeDefaultCollaborationConfig();
expect(result).toBe(true);
expect(mockMkdirSync).toHaveBeenCalled();
expect(mockWriteFileSync).toHaveBeenCalled();
});
it('should handle write errors gracefully', () => {
const mockMkdirSync = vi.fn().mockImplementation(() => {
throw new Error('Permission denied');
});
vi.doMock('fs', () => ({
existsSync: vi.fn().mockReturnValue(false),
mkdirSync: mockMkdirSync,
}));
const consoleSpy = vi
.spyOn(console, 'error')
.mockImplementation(() => { });
const result = writeDefaultCollaborationConfig();
expect(result).toBe(false);
expect(consoleSpy).toHaveBeenCalled();
consoleSpy.mockRestore();
});
});
describe('DEFAULT_COLLABORATION_CONFIG', () => {
it('should have valid default values', () => {
const validation = validateCollaborationConfig(DEFAULT_COLLABORATION_CONFIG);
expect(validation.valid).toBe(true);
});
it('should have sensible default thresholds', () => {
expect(DEFAULT_COLLABORATION_CONFIG.autoVerifyThreshold).toBeGreaterThan(0);
expect(DEFAULT_COLLABORATION_CONFIG.autoVerifyThreshold).toBeLessThan(1);
expect(DEFAULT_COLLABORATION_CONFIG.maxCostPerQuery).toBeGreaterThan(0);
expect(DEFAULT_COLLABORATION_CONFIG.consensusThreshold).toBeGreaterThan(0.5);
});
});
describe('DEFAULT_PROVIDERS', () => {
it('should have valid provider configurations', () => {
DEFAULT_PROVIDERS.forEach((provider, index) => {
const validation = validateCollaborationConfig({
providers: [provider],
});
expect(validation.valid, `Provider ${index} should be valid: ${provider.name}`).toBe(true);
});
});
it('should have unique provider IDs', () => {
const ids = DEFAULT_PROVIDERS.map((p) => p.id);
const uniqueIds = new Set(ids);
expect(uniqueIds.size).toBe(ids.length);
});
it('should include major LLM providers', () => {
const types = DEFAULT_PROVIDERS.map((p) => p.type);
expect(types).toContain('gemini');
expect(types).toContain('openai');
expect(types).toContain('anthropic');
});
});
describe('integration tests', () => {
it('should handle complex configuration scenarios', () => {
// Test real-world scenario: file config + env overrides
const fileConfig = {
enabled: false,
autoVerifyThreshold: 0.6,
providers: [
{ id: 'file-provider', name: 'File Provider', type: 'openai' },
],
};
process.env.ENABLE_COLLABORATION = 'true';
process.env.COLLABORATION_AUTO_VERIFY_THRESHOLD = '0.8';
process.env.OPENAI_API_KEY = 'env-key';
const envConfig = loadCollaborationConfigFromEnv();
const merged = mergeCollaborationConfigs(fileConfig, envConfig);
expect(merged.enabled).toBe(true); // From env
expect(merged.autoVerifyThreshold).toBe(0.8); // From env
expect(merged.providers?.length).toBeGreaterThan(1); // File + env providers
});
it('should maintain configuration consistency across operations', () => {
const originalConfig = {
enabled: true,
autoVerifyThreshold: 0.7,
verificationMode: 'smart',
providers: DEFAULT_PROVIDERS,
};
// Validate, merge with empty, validate again
const validation1 = validateCollaborationConfig(originalConfig);
const merged = mergeCollaborationConfigs(originalConfig, {});
const validation2 = validateCollaborationConfig(merged);
expect(validation1.valid).toBe(true);
expect(validation2.valid).toBe(true);
expect(merged.enabled).toBe(originalConfig.enabled);
expect(merged.autoVerifyThreshold).toBe(originalConfig.autoVerifyThreshold);
});
});
});
//# sourceMappingURL=collaboration-config.test.js.map