@quantumai/quantum-cli-core
Version:
Quantum CLI Core - Multi-LLM Collaboration System
235 lines (229 loc) • 9.18 kB
JavaScript
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { UncertaintyLevel, } from './types.js';
import { isCollaborationEnabled } from './feature-flags.js';
import { validateCollaborationConfig, } from '../config/collaboration-config.js';
import { UncertaintyDetector } from './detection/uncertainty-detector.js';
export class DualModelEngine {
primaryModel;
secondaryModel;
config;
collaborationConfig;
uncertaintyDetector;
constructor(primaryModel, secondaryModel, config = {}, collaborationConfig) {
this.primaryModel = primaryModel;
this.secondaryModel = secondaryModel;
this.config = {
autoVerifyThreshold: config.autoVerifyThreshold ?? 0.7,
maxCostPerQuery: config.maxCostPerQuery ?? 0.05,
uncertaintyThreshold: config.uncertaintyThreshold ?? 0.6,
alignmentThreshold: config.alignmentThreshold ?? 0.8,
enableSynthesis: config.enableSynthesis ?? true,
...config,
};
// Validate and apply collaboration configuration
const validationResult = validateCollaborationConfig(collaborationConfig || {});
if (!validationResult.valid) {
console.warn('Collaboration config validation errors:', validationResult.errors);
}
this.collaborationConfig = validationResult.config;
this.uncertaintyDetector = new UncertaintyDetector();
}
/**
* Update collaboration configuration
*/
updateConfiguration(newConfig) {
const validationResult = validateCollaborationConfig({
...this.collaborationConfig,
...newConfig,
});
if (!validationResult.valid) {
throw new Error(`Invalid configuration: ${validationResult.errors.map((e) => e.message).join(', ')}`);
}
this.collaborationConfig = validationResult.config;
}
/**
* Get current configuration
*/
getConfiguration() {
return { ...this.collaborationConfig };
}
/**
* Check if verification should be performed based on configuration
*/
shouldVerify(options) {
if (!isCollaborationEnabled() || !this.collaborationConfig.enabled) {
return false;
}
// Explicit verification request
if (options?.verify) {
return true;
}
// Auto-verification based on configuration
return this.collaborationConfig.verificationMode === 'automatic';
}
/**
* Generate response with optional verification from secondary model
*/
async generateWithVerification(prompt, context, options) {
const startTime = Date.now();
if (!this.shouldVerify(options)) {
// Fallback to primary model only
const response = await this.primaryModel.generate(prompt, options);
return {
content: response.content,
primaryResponse: response.content,
verified: false,
confidence: response.confidence,
uncertaintyLevel: UncertaintyLevel.LOW,
primary: response,
metadata: {
duration: Date.now() - startTime,
modelsUsed: [this.primaryModel.getId()],
},
};
}
// 1. Generate primary response
const primaryResponse = await this.primaryModel.generate(prompt, options);
// 2. Detect uncertainty
const uncertainty = this.detectUncertainty(primaryResponse.content);
// 3. Decide if verification is needed based on configuration
const shouldVerifyBasedOnUncertainty = uncertainty.level >=
(this.collaborationConfig.autoVerifyThreshold || 0.7);
const shouldVerify = options?.verify === true || // Explicit request
this.collaborationConfig.verificationMode === 'automatic' || // Always verify in auto mode
(this.collaborationConfig.verificationMode === 'smart' &&
shouldVerifyBasedOnUncertainty); // Smart mode with uncertainty
if (!shouldVerify) {
return {
content: primaryResponse.content,
primaryResponse: primaryResponse.content,
verified: false,
confidence: primaryResponse.confidence,
uncertaintyLevel: uncertainty.level,
primary: primaryResponse,
metadata: {
duration: Date.now() - startTime,
modelsUsed: [this.primaryModel.getId()],
},
};
}
// 4. Check cost limits before proceeding
const estimatedCost = this.estimateQueryCost();
if (estimatedCost > (this.collaborationConfig.maxCostPerQuery || 0.05)) {
console.warn(`Query cost (${estimatedCost}) exceeds limit, skipping verification`);
return {
content: primaryResponse.content,
primaryResponse: primaryResponse.content,
verified: false,
confidence: primaryResponse.confidence,
uncertaintyLevel: uncertainty.level,
primary: primaryResponse,
metadata: {
duration: Date.now() - startTime,
modelsUsed: [this.primaryModel.getId()],
costLimitExceeded: true,
},
};
}
// 5. Get secondary opinion
const verificationPrompt = this.buildVerificationPrompt(prompt, primaryResponse.content);
const secondaryResponse = await this.secondaryModel.generate(verificationPrompt, options);
// 6. Synthesize responses
return this.synthesizeResponses(primaryResponse, secondaryResponse, uncertainty, Date.now() - startTime);
}
/**
* Estimate the cost of a query with verification
*/
estimateQueryCost() {
const primaryCost = this.primaryModel.calculateCost(1000); // Estimate 1000 tokens
const secondaryCost = this.secondaryModel.calculateCost(1000);
return primaryCost + secondaryCost;
}
/**
* Build verification prompt for secondary model
*/
buildVerificationPrompt(originalPrompt, primaryResponse) {
return `
Please review and verify the following response to ensure accuracy and completeness.
Original Question: ${originalPrompt}
Response to Verify: ${primaryResponse}
Please provide:
1. Your assessment of the accuracy
2. Any corrections or improvements
3. Alternative approaches if applicable
`;
}
/**
* Combine primary and secondary responses into a verified response
*/
synthesizeResponses(primary, secondary, uncertainty, duration) {
// Calculate alignment between responses
const alignment = this.calculateAlignment(primary.content, secondary.content);
// Determine final confidence
const finalConfidence = alignment > 0.8
? Math.max(primary.confidence, secondary.confidence)
: (primary.confidence + secondary.confidence) / 2;
const synthesizedContent = this.createSynthesizedContent(primary, secondary, alignment);
return {
content: synthesizedContent || primary.content,
primaryResponse: primary.content,
secondaryResponse: secondary.content,
verified: true,
confidence: finalConfidence,
uncertaintyLevel: uncertainty.level,
uncertaintyReasons: uncertainty.reasons,
alignment,
primary,
secondary,
metadata: {
duration,
modelsUsed: [this.primaryModel.getId(), this.secondaryModel.getId()],
comparison: true,
},
};
}
/**
* Calculate how well two responses align (0-1 score)
*/
calculateAlignment(content1, content2) {
// Simple implementation - can be improved with semantic similarity
const words1 = new Set(content1.toLowerCase().split(/\s+/));
const words2 = new Set(content2.toLowerCase().split(/\s+/));
const intersection = new Set([...words1].filter((x) => words2.has(x)));
const union = new Set([...words1, ...words2]);
return intersection.size / union.size;
}
/**
* Create synthesized content when both models provide responses
*/
createSynthesizedContent(primary, secondary, alignment) {
if (alignment > 0.9) {
// High agreement - use primary
return undefined;
}
// Models disagree - present both perspectives
return `
Based on analysis from multiple models:
Primary Assessment:
${primary.content}
Secondary Verification:
${secondary.content}
Alignment Score: ${(alignment * 100).toFixed(1)}%
`;
}
/**
* Detect uncertainty in a response
*/
detectUncertainty(content) {
const result = this.uncertaintyDetector.detect(content);
return {
level: result.level,
reasons: result.reasons,
};
}
}
//# sourceMappingURL=dual-engine.js.map