@quantumai/quantum-cli-core
Version:
Quantum CLI Core - Multi-LLM Collaboration System
267 lines • 12.6 kB
JavaScript
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect, beforeEach } from 'vitest';
import { UncertaintyDetector } from './uncertainty-detector.js';
import { UncertaintyLevel } from '../types.js';
describe('UncertaintyDetector', () => {
let detector;
beforeEach(() => {
detector = new UncertaintyDetector();
});
describe('detectUncertainty', () => {
it('should detect high uncertainty with multiple patterns', () => {
const testCases = [
{
text: "I'm not sure, but maybe this could work, possibly.",
expectedLevel: UncertaintyLevel.HIGH,
description: 'Multiple uncertainty indicators',
},
{
text: 'I think this might be one approach, but there are several ways to handle this.',
expectedLevel: UncertaintyLevel.HIGH,
description: 'Mixed uncertainty patterns',
},
{
text: 'This could be difficult and probably requires more analysis.',
expectedLevel: UncertaintyLevel.HIGH,
description: 'Qualification patterns',
},
];
testCases.forEach(({ text, expectedLevel, description }) => {
const result = detector.detect(text);
expect(result.level, `Failed for: ${description}`).toBe(expectedLevel);
expect(result.reasons.length, `Should have reasons for: ${description}`).toBeGreaterThan(0);
});
});
it('should detect medium uncertainty with single patterns', () => {
const testCases = [
{
text: 'I think this is the right approach.',
expectedLevel: UncertaintyLevel.MEDIUM,
description: 'Single "I think" pattern',
},
{
text: 'This might work for your use case.',
expectedLevel: UncertaintyLevel.MEDIUM,
description: 'Single "might" pattern',
},
{
text: 'There are several ways to implement this feature.',
expectedLevel: UncertaintyLevel.MEDIUM,
description: 'Single options pattern',
},
];
testCases.forEach(({ text, expectedLevel, description }) => {
const result = detector.detect(text);
expect(result.level, `Failed for: ${description}`).toBe(expectedLevel);
expect(result.reasons.length, `Should have exactly one reason for: ${description}`).toBe(1);
});
});
it('should detect low uncertainty with confident language', () => {
const testCases = [
{
text: 'This is the correct implementation.',
expectedLevel: UncertaintyLevel.LOW,
description: 'Definitive statement',
},
{
text: 'You should use React hooks for state management.',
expectedLevel: UncertaintyLevel.LOW,
description: 'Clear recommendation',
},
{
text: 'The solution works by creating a new instance and calling the method.',
expectedLevel: UncertaintyLevel.LOW,
description: 'Technical explanation',
},
];
testCases.forEach(({ text, expectedLevel, description }) => {
const result = detector.detect(text);
expect(result.level, `Failed for: ${description}`).toBe(expectedLevel);
expect(result.reasons.length, `Should have no reasons for: ${description}`).toBe(0);
});
});
it('should detect critical uncertainty with extreme hesitation', () => {
const text = "I'm really not sure about this, maybe it could work, but there are many different approaches, and it's probably quite complex.";
const result = detector.detect(text);
expect(result.level).toBe(UncertaintyLevel.CRITICAL);
expect(result.reasons.length).toBeGreaterThan(2);
expect(result.suggestedAction).toBe('verify');
});
it('should provide appropriate suggested actions', () => {
const testCases = [
{
text: 'This is definitely the right way.',
expectedAction: 'accept',
},
{
text: 'I think this might work.',
expectedAction: 'compare',
},
{
text: "I'm not sure, maybe this could work, but there are several options.",
expectedAction: 'verify',
},
];
testCases.forEach(({ text, expectedAction }) => {
const result = detector.detect(text);
expect(result.suggestedAction).toBe(expectedAction);
});
});
it('should handle edge cases', () => {
const edgeCases = [
{ text: '', expected: UncertaintyLevel.LOW },
{ text: ' ', expected: UncertaintyLevel.LOW },
{
text: 'Maybe maybe maybe maybe maybe',
expected: UncertaintyLevel.CRITICAL,
},
{
text: 'I think I think I think',
expected: UncertaintyLevel.CRITICAL,
},
];
edgeCases.forEach(({ text, expected }) => {
const result = detector.detect(text);
expect(result.level).toBe(expected);
});
});
});
describe('analyzeConfidence', () => {
it('should analyze confidence patterns in technical responses', () => {
const testCases = [
{
text: 'This solution will definitely work because it follows established patterns.',
expectedConfidence: 0.9,
description: 'High confidence technical response',
},
{
text: 'This approach should work, though you might need to adjust the configuration.',
expectedConfidence: 0.7,
description: 'Medium confidence with caveats',
},
{
text: 'I guess this could work, but I am not really sure.',
expectedConfidence: 0.3,
description: 'Low confidence response',
},
];
testCases.forEach(({ text, expectedConfidence, description }) => {
const confidence = detector.detect(text);
expect(confidence, `Failed for: ${description}`).toBeCloseTo(expectedConfidence, 1);
});
});
});
describe('detectComplexity', () => {
it('should detect complexity indicators', () => {
const testCases = [
{
text: 'You need to configure multiple systems, handle edge cases, and implement error handling.',
expectedComplexity: 'high',
description: 'Multiple steps and considerations',
},
{
text: 'Just call the API and parse the response.',
expectedComplexity: 'low',
description: 'Simple two-step process',
},
{
text: 'This involves setting up authentication, configuring the database, and implementing the business logic.',
expectedComplexity: 'medium',
description: 'Moderate number of steps',
},
];
testCases.forEach(({ text, expectedComplexity, description }) => {
const complexity = detector.detect(text);
expect(complexity, `Failed for: ${description}`).toBe(expectedComplexity);
});
});
});
describe('contextualized detection', () => {
it('should consider query context in uncertainty detection', () => {
const text = 'This might work.';
const securityContext = detector.detect(text, { domain: 'security' });
const generalContext = detector.detect(text, { domain: 'general' });
// Security contexts should have higher uncertainty thresholds
expect(securityContext.level).toBeGreaterThanOrEqual(generalContext.level);
});
it('should adjust uncertainty based on query criticality', () => {
const text = 'This approach should work.';
const highCriticalityResult = detector.detect(text, {
criticality: 'high',
});
const lowCriticalityResult = detector.detect(text, {
criticality: 'low',
});
expect(highCriticalityResult.level).toBeGreaterThanOrEqual(lowCriticalityResult.level);
});
});
describe('performance', () => {
it('should process text efficiently', () => {
const longText = 'This is a very long text. '.repeat(1000);
const startTime = performance.now();
detector.detect(longText);
const endTime = performance.now();
// Should process within reasonable time (under 100ms for 1000 repeated phrases)
expect(endTime - startTime).toBeLessThan(100);
});
it('should handle concurrent detection calls', async () => {
const texts = [
'I think this might work.',
'This is definitely correct.',
'Maybe we should consider alternatives.',
'This solution works perfectly.',
'I am not sure about this approach.',
];
const promises = texts.map((text) => Promise.resolve(detector.detect(text)));
const results = await Promise.all(promises);
expect(results).toHaveLength(5);
results.forEach((result) => {
expect(result).toHaveProperty('level');
expect(result).toHaveProperty('reasons');
});
});
});
describe('language pattern recognition', () => {
it('should recognize various uncertainty markers', () => {
const patterns = [
// Epistemic uncertainty
{ text: 'I believe this should work', pattern: 'epistemic' },
{ text: 'It seems like this is correct', pattern: 'epistemic' },
// Modal uncertainty
{ text: 'This could potentially work', pattern: 'modal' },
{ text: 'It might be possible to implement', pattern: 'modal' },
// Hedging
{ text: 'This is somewhat complex', pattern: 'hedging' },
{ text: 'It is relatively straightforward', pattern: 'hedging' },
// Alternatives
{
text: 'There are multiple ways to solve this',
pattern: 'alternatives',
},
{ text: 'You have several options here', pattern: 'alternatives' },
];
patterns.forEach(({ text, pattern }) => {
const result = detector.detect(text);
expect(result.reasons.length, `Should detect ${pattern} pattern in: ${text}`).toBeGreaterThan(0);
});
});
it('should not be fooled by false positives', () => {
const falsePositives = [
'I think therefore I am', // Philosophical statement
'Maybe tomorrow is a function name', // Technical reference
'The perhaps variable stores the result', // Technical naming
'This API might return different results', // Describing API behavior, not uncertainty
];
falsePositives.forEach((text) => {
const result = detector.detect(text);
// These should have low uncertainty as they're not expressing hesitation
expect(result.level, `False positive for: ${text}`).toBeLessThanOrEqual(UncertaintyLevel.MEDIUM);
});
});
});
});
//# sourceMappingURL=uncertainty-detector.test.js.map