quantum-cli-core
Version:
Quantum CLI Core - Multi-LLM Collaboration System
222 lines • 9.76 kB
JavaScript
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { DualModelEngine } from './dual-engine.js';
import { BaseLLMProvider } from './providers/base-provider.js';
import { UncertaintyLevel, } from './types.js';
import * as featureFlags from './feature-flags.js';
// Mock the feature flags module
vi.mock('./feature-flags.js');
// Mock provider implementation
class MockProvider extends BaseLLMProvider {
id;
capabilities;
mockResponse;
constructor(id, capabilities, mockResponse) {
super({ id, name: id, type: 'mock', capabilities });
this.id = id;
this.capabilities = capabilities;
this.mockResponse = mockResponse;
}
async generate() {
return this.mockResponse;
}
async validateCredentials() {
return true;
}
async *generateStream() {
yield this.mockResponse.content;
}
async validateConfiguration() {
return true;
}
}
describe('DualModelEngine', () => {
let primaryProvider;
let secondaryProvider;
let engine;
let config;
beforeEach(() => {
vi.resetAllMocks();
// Enable collaboration by default
vi.mocked(featureFlags.isCollaborationEnabled).mockReturnValue(true);
config = {
uncertaintyThreshold: 0.7,
maxCostPerQuery: 0.05,
autoVerifyThreshold: 0.7,
alignmentThreshold: 0.8,
enableSynthesis: true,
};
});
describe('generateWithVerification', () => {
it('should use primary model only when confidence is high', async () => {
const primaryResponse = {
providerId: 'primary',
content: 'This is a clear and confident response.',
confidence: 0.95,
};
primaryProvider = new MockProvider('primary', ['test'], primaryResponse);
secondaryProvider = new MockProvider('secondary', ['test'], {
...primaryResponse,
providerId: 'secondary',
});
const collaborationConfig = {
enabled: true,
verificationMode: 'smart',
autoVerifyThreshold: 0.8, // Higher than uncertainty level to skip verification
maxCostPerQuery: 0.05,
providers: [],
};
engine = new DualModelEngine(primaryProvider, secondaryProvider, config, collaborationConfig);
const result = await engine.generateWithVerification('test query');
expect(result.primary).toEqual(primaryResponse);
expect(result.secondary).toBeUndefined();
expect(result.confidence).toBe(0.95);
expect(result.uncertaintyLevel).toBe(UncertaintyLevel.LOW);
});
it('should trigger verification when explicitly requested', async () => {
const primaryResponse = {
providerId: 'primary',
content: 'This is a standard response.',
confidence: 0.7,
};
const secondaryResponse = {
providerId: 'secondary',
content: 'Here is a verified approach that should work correctly.',
confidence: 0.9,
};
primaryProvider = new MockProvider('primary', ['test'], primaryResponse);
secondaryProvider = new MockProvider('secondary', ['test'], secondaryResponse);
const collaborationConfig = {
enabled: true,
verificationMode: 'manual', // Manual mode requires explicit verification
autoVerifyThreshold: 0.6,
maxCostPerQuery: 0.05,
providers: [],
};
engine = new DualModelEngine(primaryProvider, secondaryProvider, config, collaborationConfig);
// Explicitly request verification
const result = await engine.generateWithVerification('test query', undefined, { verify: true });
expect(result.primary).toEqual(primaryResponse);
expect(result.secondary).toEqual(secondaryResponse);
expect(result.verified).toBe(true);
});
it('should fallback to primary model when collaboration is disabled', async () => {
vi.mocked(featureFlags.isCollaborationEnabled).mockReturnValue(false);
const primaryResponse = {
providerId: 'primary',
content: 'Maybe this could work, I think.',
confidence: 0.6,
};
primaryProvider = new MockProvider('primary', ['test'], primaryResponse);
secondaryProvider = new MockProvider('secondary', ['test'], {
...primaryResponse,
providerId: 'secondary',
});
const collaborationConfig = {
enabled: true,
verificationMode: 'automatic',
autoVerifyThreshold: 0.1,
maxCostPerQuery: 0.05,
providers: [],
};
engine = new DualModelEngine(primaryProvider, secondaryProvider, config, collaborationConfig);
const result = await engine.generateWithVerification('test query');
expect(result.primary).toEqual(primaryResponse);
expect(result.secondary).toBeUndefined();
expect(result.confidence).toBe(0.6);
});
it('should calculate alignment between responses', async () => {
const primaryResponse = {
providerId: 'primary',
content: 'Use React hooks for state management',
confidence: 0.8,
};
const secondaryResponse = {
providerId: 'secondary',
content: 'React hooks are good for state management',
confidence: 0.85,
};
primaryProvider = new MockProvider('primary', ['test'], primaryResponse);
secondaryProvider = new MockProvider('secondary', ['test'], secondaryResponse);
const collaborationConfig = {
enabled: true,
verificationMode: 'automatic', // Always verify for this test
autoVerifyThreshold: 0.1, // Low threshold to ensure verification
maxCostPerQuery: 0.05,
providers: [],
};
engine = new DualModelEngine(primaryProvider, secondaryProvider, config, collaborationConfig);
const result = await engine.generateWithVerification('How to manage state in React?');
expect(result.alignment).toBeDefined();
expect(result.alignment).toBeGreaterThan(0.5); // High alignment expected
});
it('should handle errors gracefully', async () => {
const primaryResponse = {
providerId: 'primary',
content: 'Standard response',
confidence: 0.9,
};
primaryProvider = new MockProvider('primary', ['test'], primaryResponse);
secondaryProvider = new MockProvider('secondary', ['test'], primaryResponse);
// Mock secondary provider to throw error
vi.spyOn(secondaryProvider, 'generate').mockRejectedValueOnce(new Error('API Error'));
const collaborationConfig = {
enabled: true,
verificationMode: 'manual',
autoVerifyThreshold: 0.6,
maxCostPerQuery: 0.05,
providers: [],
};
engine = new DualModelEngine(primaryProvider, secondaryProvider, config, collaborationConfig);
// Should throw when explicitly requesting verification
await expect(engine.generateWithVerification('test query', undefined, {
verify: true,
})).rejects.toThrow('API Error');
});
});
describe('uncertainty detection', () => {
it('should detect high uncertainty from language patterns', async () => {
const responses = [
{
content: "I'm not sure, but maybe this could work",
expectedLevel: UncertaintyLevel.HIGH,
},
{
content: 'This is the correct approach',
expectedLevel: UncertaintyLevel.LOW,
},
{
content: 'This might be one way, but there are several options',
expectedLevel: UncertaintyLevel.HIGH,
},
];
for (const { content, expectedLevel } of responses) {
const primaryResponse = {
providerId: 'primary',
content,
confidence: 0.8,
};
primaryProvider = new MockProvider('primary', ['test'], primaryResponse);
secondaryProvider = new MockProvider('secondary', ['test'], {
...primaryResponse,
providerId: 'secondary',
});
const collaborationConfig = {
enabled: true,
verificationMode: 'automatic',
autoVerifyThreshold: 0.1,
maxCostPerQuery: 0.05,
providers: [],
};
engine = new DualModelEngine(primaryProvider, secondaryProvider, config, collaborationConfig);
const result = await engine.generateWithVerification('test');
expect(result.uncertaintyLevel).toBe(expectedLevel);
}
});
});
});
//# sourceMappingURL=dual-engine.test.js.map