quantum-cli-core
Version:
Quantum CLI Core - Multi-LLM Collaboration System
315 lines • 16.9 kB
JavaScript
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, it, expect } from 'vitest';
import { UseCaseAnalyzer } from './use-case-analyzer.js';
import { QueryType } from './types.js';
describe('UseCaseAnalyzer', () => {
describe('analyzeQuery', () => {
it('should correctly classify code queries', () => {
const queries = [
'How do I implement a binary search algorithm in Python?',
'Debug this JavaScript function that returns undefined',
'Create a REST API endpoint using Express.js',
'Optimize this SQL query for better performance',
'Write a unit test for this React component',
];
queries.forEach((query) => {
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.queryType).toBe(QueryType.CODE);
expect(analysis.confidence).toBeGreaterThan(0.3);
expect(analysis.detectedPatterns.length).toBeGreaterThan(0);
});
});
it('should correctly classify creative queries', () => {
const queries = [
'Write a short story about time travel',
'Create marketing copy for a new product launch',
'Compose a poem about artificial intelligence',
'Draft a blog post about sustainable living',
'Generate creative slogans for a coffee shop',
];
queries.forEach((query) => {
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.queryType).toBe(QueryType.CREATIVE);
expect(analysis.confidence).toBeGreaterThan(0.3);
});
});
it('should correctly classify analysis queries', () => {
const queries = [
'Analyze the pros and cons of remote work',
'Compare React vs Vue.js for web development',
'Evaluate the performance metrics of our application',
'What are the trends in AI development?',
'Examine the impact of climate change on agriculture',
];
queries.forEach((query) => {
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.queryType).toBe(QueryType.ANALYSIS);
expect(analysis.confidence).toBeGreaterThan(0.3);
});
});
it('should correctly classify security queries', () => {
const queries = [
'How to implement OAuth2 authentication?',
'Identify security vulnerabilities in this code',
'Best practices for password hashing',
'GDPR compliance checklist for web applications',
'Prevent SQL injection attacks in PHP',
];
queries.forEach((query) => {
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.queryType).toBe(QueryType.SECURITY);
expect(analysis.confidence).toBeGreaterThan(0.3);
});
});
it('should detect domain correctly', () => {
const webDevQuery = 'How to create a React component with useState hook?';
const analysis = UseCaseAnalyzer.analyzeQuery(webDevQuery);
expect(analysis.domain).toBe('web-development');
expect(analysis.frameworks).toContain('React');
});
it('should detect frameworks and languages', () => {
const query = 'Build a Node.js Express API with TypeScript and PostgreSQL';
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.frameworks).toContain('Node.js');
expect(analysis.languages).toContain('TypeScript');
expect(analysis.domain).toBe('web-development');
});
it('should determine complexity correctly', () => {
const simpleQuery = 'What is a function in Python?';
const complexQuery = 'Design a distributed microservices architecture with event sourcing and CQRS patterns';
const simpleAnalysis = UseCaseAnalyzer.analyzeQuery(simpleQuery);
const complexAnalysis = UseCaseAnalyzer.analyzeQuery(complexQuery);
expect(simpleAnalysis.complexity).toBe('simple');
expect(complexAnalysis.complexity).toBe('complex');
});
it('should detect special requirements', () => {
const query = 'Build a real-time secure trading system with low latency requirements';
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.specialRequirements).toContain('real-time');
expect(analysis.specialRequirements).toContain('secure');
expect(analysis.specialRequirements).toContain('low-latency');
});
});
describe('getRecommendations', () => {
it('should provide recommendations for code queries', () => {
const query = 'Implement a machine learning model using TensorFlow';
const recommendations = UseCaseAnalyzer.getRecommendations(query);
expect(recommendations.analysis.queryType).toBe(QueryType.CODE);
expect(recommendations.primaryRecommendation).toBeDefined();
expect(recommendations.alternatives.length).toBeGreaterThan(0);
expect(recommendations.explanation).toBeDefined();
// Should have meaningful scores
expect(recommendations.primaryRecommendation.score).toBeGreaterThan(0);
expect(recommendations.primaryRecommendation.confidence).toBeGreaterThan(0);
});
it('should respect cost constraints', () => {
const query = 'Write a simple hello world function';
const lowBudgetRecs = UseCaseAnalyzer.getRecommendations(query, 0.001);
const highBudgetRecs = UseCaseAnalyzer.getRecommendations(query, 1.0);
// Low budget should prefer cheaper models
expect(lowBudgetRecs.primaryRecommendation.estimatedCost).toBeLessThanOrEqual(0.001);
// High budget recommendations might be different
expect(highBudgetRecs.primaryRecommendation).toBeDefined();
});
it('should consider latency preferences', () => {
const query = 'Real-time data processing system';
const fastRecs = UseCaseAnalyzer.getRecommendations(query, undefined, 1000);
const normalRecs = UseCaseAnalyzer.getRecommendations(query);
// Fast preference should prioritize low-latency models
expect(fastRecs.primaryRecommendation.estimatedLatency).toBeLessThanOrEqual(1000);
});
it('should provide different recommendations for different query types', () => {
const codeQuery = 'Implement a binary search algorithm';
const creativeQuery = 'Write a science fiction short story';
const codeRecs = UseCaseAnalyzer.getRecommendations(codeQuery);
const creativeRecs = UseCaseAnalyzer.getRecommendations(creativeQuery);
expect(codeRecs.analysis.queryType).toBe(QueryType.CODE);
expect(creativeRecs.analysis.queryType).toBe(QueryType.CREATIVE);
// Different models might be recommended for different types
// This depends on the model characteristics data
});
it('should include pros and cons in recommendations', () => {
const query = 'Complex distributed system architecture design';
const recommendations = UseCaseAnalyzer.getRecommendations(query);
expect(recommendations.primaryRecommendation.tradeoffs.pros).toBeDefined();
expect(recommendations.primaryRecommendation.tradeoffs.cons).toBeDefined();
// Should have some meaningful tradeoffs for complex queries
expect(recommendations.primaryRecommendation.tradeoffs.pros.length +
recommendations.primaryRecommendation.tradeoffs.cons.length).toBeGreaterThan(0);
});
});
describe('compareModelsForUseCase', () => {
it('should compare models for specific use cases', () => {
const query = 'Create a web application with React and Node.js';
const modelIds = ['gemini-2.5-pro', 'gpt-4', 'claude-3-sonnet'];
const comparison = UseCaseAnalyzer.compareModelsForUseCase(modelIds, query);
expect(comparison).toHaveLength(3);
expect(comparison[0].recommendation.score).toBeGreaterThanOrEqual(comparison[1].recommendation.score);
expect(comparison[1].recommendation.score).toBeGreaterThanOrEqual(comparison[2].recommendation.score);
// All models should have valid recommendations
comparison.forEach((item) => {
expect(item.modelId).toBeDefined();
expect(item.recommendation.score).toBeGreaterThanOrEqual(0);
expect(item.recommendation.score).toBeLessThanOrEqual(100);
expect(item.recommendation.estimatedCost).toBeGreaterThan(0);
expect(item.recommendation.estimatedLatency).toBeGreaterThan(0);
});
});
it('should throw error for non-existent models', () => {
const query = 'Test query';
const modelIds = ['non-existent-model'];
expect(() => {
UseCaseAnalyzer.compareModelsForUseCase(modelIds, query);
}).toThrow('Model not found: non-existent-model');
});
});
describe('Pattern Detection', () => {
it('should detect code patterns correctly', () => {
const codeQueries = [
'function calculateSum(a, b) { return a + b; }',
'class MyClass extends BaseClass {}',
'const result = array.map(item => item.value)',
'if (condition) { console.log("true"); }',
'try { doSomething(); } catch (error) { handleError(error); }',
];
codeQueries.forEach((query) => {
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.queryType).toBe(QueryType.CODE);
expect(analysis.detectedPatterns.length).toBeGreaterThan(0);
});
});
it('should detect framework-specific patterns', () => {
const frameworkQueries = [
{ query: 'useState hook in React component', expected: ['React'] },
{ query: 'Vue composition API with reactivity', expected: ['Vue'] },
{
query: 'Angular service with dependency injection',
expected: ['Angular'],
},
{
query: 'Express.js middleware for authentication',
expected: ['Node.js'],
},
{ query: 'Django models and views setup', expected: ['Python'] },
];
frameworkQueries.forEach(({ query, expected }) => {
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expected.forEach((framework) => {
expect(analysis.frameworks).toContain(framework);
});
});
});
it('should detect domain-specific vocabulary', () => {
const domainQueries = [
{
query: 'machine learning model training with neural networks',
domain: 'machine-learning',
},
{
query: 'e-commerce shopping cart and payment processing',
domain: 'e-commerce',
},
{
query: 'financial portfolio risk assessment and trading algorithms',
domain: 'finance',
},
{
query: 'patient data management and medical diagnosis system',
domain: 'healthcare',
},
{
query: 'Docker containerization and Kubernetes deployment',
domain: 'devops',
},
];
domainQueries.forEach(({ query, domain }) => {
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.domain).toBe(domain);
});
});
it('should handle ambiguous queries gracefully', () => {
const ambiguousQueries = [
'help me',
'what do you think?',
'can you assist?',
'I need some advice',
];
ambiguousQueries.forEach((query) => {
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.queryType).toBe(QueryType.GENERAL);
expect(analysis.confidence).toBeLessThan(0.8); // Should have lower confidence
});
});
});
describe('Complexity Detection', () => {
it('should classify queries by complexity correctly', () => {
const complexityTests = [
{ query: 'What is Python?', expected: 'simple' },
{
query: 'How to implement a REST API with authentication and rate limiting?',
expected: 'medium',
},
{
query: 'Design a microservices architecture with event sourcing, CQRS, distributed tracing, and fault tolerance mechanisms including circuit breakers, bulkheads, and graceful degradation patterns',
expected: 'complex',
},
];
complexityTests.forEach(({ query, expected }) => {
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.complexity).toBe(expected);
});
});
});
describe('Special Requirements Detection', () => {
it('should detect performance requirements', () => {
const query = 'Build a high-performance real-time trading system with microsecond latency';
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.specialRequirements).toContain('real-time');
expect(analysis.specialRequirements).toContain('high-performance');
expect(analysis.specialRequirements).toContain('low-latency');
});
it('should detect security requirements', () => {
const query = 'Secure payment processing system with PCI compliance';
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.specialRequirements).toContain('secure');
expect(analysis.specialRequirements).toContain('compliant');
});
it('should detect cost sensitivity', () => {
const query = 'Build a cost-effective budget-friendly solution';
const analysis = UseCaseAnalyzer.analyzeQuery(query);
expect(analysis.specialRequirements).toContain('cost-sensitive');
});
});
describe('Recommendation Quality', () => {
it('should provide higher scores for suitable models', () => {
const codeQuery = 'Implement advanced algorithms and data structures';
const recommendations = UseCaseAnalyzer.getRecommendations(codeQuery);
// Should prefer models with strong code generation capabilities
expect(recommendations.primaryRecommendation.score).toBeGreaterThan(50);
// Should have reasoning
expect(recommendations.primaryRecommendation.reasoning.length).toBeGreaterThan(0);
});
it('should consider cost-performance tradeoffs', () => {
const simpleQuery = 'What is 2+2?';
const complexQuery = 'Design a distributed consensus algorithm for blockchain';
const simpleRecs = UseCaseAnalyzer.getRecommendations(simpleQuery);
const complexRecs = UseCaseAnalyzer.getRecommendations(complexQuery);
// Simple queries might prefer cost-effective models
// Complex queries might prefer higher-quality models
expect(simpleRecs.primaryRecommendation).toBeDefined();
expect(complexRecs.primaryRecommendation).toBeDefined();
});
it('should provide meaningful explanations', () => {
const query = 'Build a React application with TypeScript and PostgreSQL';
const recommendations = UseCaseAnalyzer.getRecommendations(query);
expect(recommendations.explanation).toBeDefined();
expect(recommendations.explanation.length).toBeGreaterThan(50);
expect(recommendations.explanation).toContain(recommendations.analysis.queryType);
});
});
});
//# sourceMappingURL=use-case-analyzer.test.js.map