route-claudecode
Version:
Advanced routing and transformation system for Claude Code outputs to multiple AI providers
845 lines (827 loc) • 34.9 kB
JavaScript
;
/**
* Gemini Tool Calling Test Matrix Generator
* Generates comprehensive test matrices for systematic validation
*
* Project owner: Jason Zhang
*
* This generator creates exhaustive test coverage for Gemini tool calling pipeline,
* covering all possible combinations of:
* - Tool formats (Anthropic, OpenAI)
* - Tool counts (single, multiple)
* - Schema complexities (simple, nested, arrays)
* - Configuration modes (AUTO, ANY, NONE)
* - Model variations (gemini-2.5-flash, gemini-2.5-pro)
* - Error conditions and edge cases
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.GeminiTestMatrixGenerator = void 0;
const promises_1 = __importDefault(require("fs/promises"));
const path_1 = __importDefault(require("path"));
const logger_1 = require("../utils/logger");
class GeminiTestMatrixGenerator {
outputDir;
constructor(outputDir = './test-matrices') {
this.outputDir = outputDir;
}
/**
* Generate comprehensive test matrix for Gemini tool calling
*/
async generateComprehensiveMatrix() {
const testCases = [];
// 1. Generate functional test cases
testCases.push(...this.generateFunctionalTests());
// 2. Generate integration test cases
testCases.push(...this.generateIntegrationTests());
// 3. Generate performance test cases
testCases.push(...this.generatePerformanceTests());
// 4. Generate edge case test cases
testCases.push(...this.generateEdgeCaseTests());
// 5. Generate error handling test cases
testCases.push(...this.generateErrorHandlingTests());
const matrix = {
matrixId: `gemini-tool-calling-${Date.now()}`,
generatedAt: new Date().toISOString(),
pipelineType: 'gemini-tool-calling',
totalTestCases: testCases.length,
categories: {
functional: testCases.filter(tc => tc.category === 'functional').length,
integration: testCases.filter(tc => tc.category === 'integration').length,
performance: testCases.filter(tc => tc.category === 'performance').length,
edgeCase: testCases.filter(tc => tc.category === 'edge-case').length,
errorHandling: testCases.filter(tc => tc.category === 'error-handling').length
},
priorities: {
critical: testCases.filter(tc => tc.priority === 'critical').length,
high: testCases.filter(tc => tc.priority === 'high').length,
medium: testCases.filter(tc => tc.priority === 'medium').length,
low: testCases.filter(tc => tc.priority === 'low').length
},
estimatedTotalTime: testCases.reduce((sum, tc) => sum + tc.estimatedDuration, 0),
testCases
};
await this.saveMatrix(matrix);
logger_1.logger.info('Gemini test matrix generated successfully', {
totalTests: matrix.totalTestCases,
criticalTests: matrix.priorities.critical,
estimatedTime: `${Math.round(matrix.estimatedTotalTime / 60)}min`
});
return matrix;
}
/**
* Generate functional test cases covering basic tool calling scenarios
*/
generateFunctionalTests() {
const tests = [];
// Anthropic format tools
tests.push({
id: 'func-001',
name: 'Anthropic单工具基础调用',
description: '验证Anthropic格式单个工具的基本调用功能',
category: 'functional',
priority: 'critical',
expectedOutcome: 'success',
testData: {
request: {
model: 'gemini-2.5-flash',
messages: [{
role: 'user',
content: 'What is the weather like in Tokyo? Use the weather tool to check.'
}],
tools: [{
name: 'get_weather',
description: 'Get current weather for a location',
input_schema: {
type: 'object',
properties: {
location: { type: 'string', description: 'City name' },
units: { type: 'string', enum: ['celsius', 'fahrenheit'], description: 'Temperature units' }
},
required: ['location']
}
}]
},
expectedBehavior: [
'toolConfig应包含functionCallingConfig',
'functionCallingConfig.mode应为AUTO',
'tools应正确转换为Gemini SDK格式',
'响应应包含tool_use类型内容',
'stop_reason应为tool_use'
]
},
validationCriteria: [
'request.toolConfig.functionCallingConfig.mode === "AUTO"',
'response.content.some(c => c.type === "tool_use")',
'response.stop_reason === "tool_use"',
'response.content[0].name === "get_weather"',
'response.content[0].input.location === "Tokyo"'
],
relatedIssues: ['toolConfig缺失问题', 'stop_reason映射错误'],
estimatedDuration: 30
});
// OpenAI format tools
tests.push({
id: 'func-002',
name: 'OpenAI单工具基础调用',
description: '验证OpenAI格式单个工具的基本调用功能',
category: 'functional',
priority: 'critical',
expectedOutcome: 'success',
testData: {
request: {
model: 'gemini-2.5-pro',
messages: [{
role: 'user',
content: 'Search for information about Node.js latest features. Use the search tool.'
}],
tools: [{
type: 'function',
function: {
name: 'search_web',
description: 'Search the web for information',
parameters: {
type: 'object',
properties: {
query: { type: 'string', description: 'Search query' },
max_results: { type: 'integer', description: 'Maximum number of results', default: 5 }
},
required: ['query']
}
}
}]
},
expectedBehavior: [
'OpenAI格式工具应正确转换为Gemini格式',
'function.parameters应正确映射为Gemini的parameters',
'toolConfig应正确配置',
'响应应包含工具调用'
]
},
validationCriteria: [
'request.tools[0].functionDeclarations[0].name === "search_web"',
'request.tools[0].functionDeclarations[0].parameters.type === "object"',
'response.content.some(c => c.type === "tool_use" && c.name === "search_web")'
],
estimatedDuration: 30
});
// Multi-tool scenarios
tests.push({
id: 'func-003',
name: '多工具混合格式调用',
description: '验证同时使用Anthropic和OpenAI格式多个工具的调用',
category: 'functional',
priority: 'high',
expectedOutcome: 'success',
testData: {
request: {
model: 'gemini-2.5-flash',
messages: [{
role: 'user',
content: 'Help me plan a trip to Paris. Check the weather and search for attractions.'
}],
tools: [
{
name: 'get_weather',
description: 'Get weather information',
input_schema: {
type: 'object',
properties: { location: { type: 'string' } },
required: ['location']
}
},
{
type: 'function',
function: {
name: 'search_attractions',
description: 'Search for tourist attractions',
parameters: {
type: 'object',
properties: {
city: { type: 'string' },
category: { type: 'string', enum: ['museum', 'restaurant', 'landmark'] }
},
required: ['city']
}
}
}
]
},
expectedBehavior: [
'两种格式工具都应正确转换',
'allowedFunctionNames应包含两个工具名',
'可能调用一个或多个工具',
'工具调用顺序应合理'
]
},
validationCriteria: [
'request.toolConfig.functionCallingConfig.allowedFunctionNames.length === 2',
'request.tools[0].functionDeclarations.length === 2'
],
estimatedDuration: 45
});
return tests;
}
/**
* Generate integration test cases covering end-to-end scenarios
*/
generateIntegrationTests() {
const tests = [];
tests.push({
id: 'int-001',
name: '完整工具调用流水线测试',
description: '验证从请求到响应的完整工具调用流水线',
category: 'integration',
priority: 'critical',
expectedOutcome: 'success',
testData: {
request: {
model: 'gemini-2.5-flash',
messages: [{
role: 'user',
content: 'Calculate the area of a circle with radius 5 units.'
}],
tools: [{
name: 'calculate_area',
description: 'Calculate the area of a circle',
input_schema: {
type: 'object',
properties: {
shape: { type: 'string', enum: ['circle'] },
radius: { type: 'number', description: 'Radius in units' }
},
required: ['shape', 'radius']
}
}]
},
expectedBehavior: [
'STD-8-STEP-PIPELINE所有步骤成功',
'输入预处理正确应用补丁',
'路由选择正确模型和提供商',
'请求转换符合Gemini SDK格式',
'API调用返回有效响应',
'响应预处理正确解析工具调用',
'响应转换为标准Anthropic格式',
'输出后处理提供完整响应'
]
},
validationCriteria: [
'all_pipeline_steps_successful === true',
'tool_call_detected_in_preprocessing === true',
'gemini_sdk_format_valid === true',
'api_response_contains_function_call === true',
'final_response.stop_reason === "tool_use"'
],
estimatedDuration: 120
});
tests.push({
id: 'int-002',
name: '多轮对话工具调用测试',
description: '验证多轮对话中的工具调用和状态维护',
category: 'integration',
priority: 'high',
expectedOutcome: 'success',
testData: {
request: {
model: 'gemini-2.5-pro',
messages: [
{
role: 'user',
content: 'What is the weather in London?'
},
{
role: 'assistant',
content: [
{ type: 'text', text: 'I\'ll check the weather in London for you.' },
{ type: 'tool_use', id: 'toolu_123', name: 'get_weather', input: { location: 'London' } }
]
},
{
role: 'tool',
tool_call_id: 'toolu_123',
content: 'Temperature: 15°C, Condition: Cloudy, Humidity: 80%'
},
{
role: 'user',
content: 'What about in Paris?'
}
],
tools: [{
name: 'get_weather',
description: 'Get weather information',
input_schema: {
type: 'object',
properties: { location: { type: 'string' } },
required: ['location']
}
}]
},
expectedBehavior: [
'对话历史正确转换为Gemini格式',
'工具结果正确处理为用户消息',
'第二次工具调用应该针对Paris',
'上下文信息应该保持'
]
},
validationCriteria: [
'request.contents.length >= 4',
'request.contents.some(c => c.role === "user" && c.parts[0].text.includes("Tool"))',
'response.content.some(c => c.type === "tool_use" && c.input.location === "Paris")'
],
estimatedDuration: 90
});
return tests;
}
/**
* Generate performance test cases
*/
generatePerformanceTests() {
const tests = [];
tests.push({
id: 'perf-001',
name: '大量工具定义性能测试',
description: '验证处理大量工具定义时的性能表现',
category: 'performance',
priority: 'medium',
expectedOutcome: 'success',
testData: {
request: {
model: 'gemini-2.5-pro',
messages: [{
role: 'user',
content: 'I need help with various calculations and searches.'
}],
tools: this.generateLargeToolSet(20) // 20个工具
},
expectedBehavior: [
'响应时间应小于5秒',
'内存使用应合理',
'工具转换应高效',
'所有工具应正确转换'
]
},
validationCriteria: [
'response_time_ms < 5000',
'memory_usage_mb < 100',
'request.tools[0].functionDeclarations.length === 20',
'all_tools_converted_successfully === true'
],
estimatedDuration: 60
});
tests.push({
id: 'perf-002',
name: '复杂Schema处理性能测试',
description: '验证处理复杂嵌套Schema时的性能',
category: 'performance',
priority: 'medium',
expectedOutcome: 'success',
testData: {
request: {
model: 'gemini-2.5-flash',
messages: [{
role: 'user',
content: 'Create a complex data structure for me.'
}],
tools: [{
name: 'create_complex_structure',
description: 'Create a complex nested data structure',
input_schema: this.generateComplexSchema()
}]
},
expectedBehavior: [
'Schema清理应高效完成',
'不支持的字段应被移除',
'嵌套结构应保持完整',
'处理时间应合理'
]
},
validationCriteria: [
'schema_cleaning_time_ms < 100',
'cleaned_schema_valid === true',
'nested_structure_preserved === true'
],
estimatedDuration: 45
});
return tests;
}
/**
* Generate edge case test cases
*/
generateEdgeCaseTests() {
const tests = [];
tests.push({
id: 'edge-001',
name: '空工具列表处理',
description: '验证工具列表为空时的正确处理',
category: 'edge-case',
priority: 'medium',
expectedOutcome: 'success',
testData: {
request: {
model: 'gemini-2.5-flash',
messages: [{
role: 'user',
content: 'Just have a normal conversation with me.'
}],
tools: []
},
expectedBehavior: [
'toolConfig不应被设置',
'正常文本响应',
'stop_reason应为end_turn'
]
},
validationCriteria: [
'request.toolConfig === undefined',
'request.tools === undefined',
'response.stop_reason === "end_turn"',
'response.content.every(c => c.type === "text")'
],
estimatedDuration: 20
});
tests.push({
id: 'edge-002',
name: '无效工具格式处理',
description: '验证遇到无效工具格式时的错误处理',
category: 'edge-case',
priority: 'high',
expectedOutcome: 'controlled-failure',
testData: {
request: {
model: 'gemini-2.5-flash',
messages: [{
role: 'user',
content: 'Use an invalid tool.'
}],
tools: [{
invalid: 'format',
missing: 'name_and_schema'
}]
},
expectedBehavior: [
'应抛出清晰的错误信息',
'错误应指出具体问题',
'系统应优雅降级'
]
},
validationCriteria: [
'error_thrown === true',
'error_message.includes("Invalid tool format")',
'system_remained_stable === true'
],
estimatedDuration: 25
});
tests.push({
id: 'edge-003',
name: '极长工具描述处理',
description: '验证处理极长工具描述时的表现',
category: 'edge-case',
priority: 'low',
expectedOutcome: 'success',
testData: {
request: {
model: 'gemini-2.5-pro',
messages: [{
role: 'user',
content: 'Use the tool with very long description.'
}],
tools: [{
name: 'long_description_tool',
description: 'A'.repeat(5000), // 5000 字符描述
input_schema: {
type: 'object',
properties: { input: { type: 'string' } }
}
}]
},
expectedBehavior: [
'长描述应正确处理',
'不应影响功能',
'可能需要截断处理'
]
},
validationCriteria: [
'tool_description_processed === true',
'function_remains_callable === true'
],
estimatedDuration: 30
});
return tests;
}
/**
* Generate error handling test cases
*/
generateErrorHandlingTests() {
const tests = [];
tests.push({
id: 'error-001',
name: 'API配额超限处理',
description: '验证API配额超限时的错误处理和重试机制',
category: 'error-handling',
priority: 'high',
expectedOutcome: 'controlled-failure',
testData: {
request: {
model: 'gemini-2.5-flash',
messages: [{
role: 'user',
content: 'This should trigger quota error in testing.'
}],
tools: [{
name: 'test_tool',
description: 'A test tool',
input_schema: {
type: 'object',
properties: { input: { type: 'string' } }
}
}]
},
expectedBehavior: [
'429错误应被正确识别',
'重试机制应激活',
'Key轮换应发生',
'最终应提供有意义的错误信息'
]
},
validationCriteria: [
'error_type === "quota_exceeded"',
'retry_attempts > 0',
'key_rotation_occurred === true',
'user_friendly_error_message !== null'
],
estimatedDuration: 180
});
tests.push({
id: 'error-002',
name: '网络超时处理',
description: '验证网络超时时的处理机制',
category: 'error-handling',
priority: 'medium',
expectedOutcome: 'graceful-error',
testData: {
request: {
model: 'gemini-2.5-pro',
messages: [{
role: 'user',
content: 'This request should timeout.'
}],
tools: [{
name: 'timeout_tool',
description: 'A tool that causes timeout',
input_schema: {
type: 'object',
properties: { delay: { type: 'integer' } }
}
}]
},
expectedBehavior: [
'超时应在合理时间内触发',
'应提供超时错误信息',
'资源应正确清理'
]
},
validationCriteria: [
'error_type === "timeout"',
'timeout_duration_ms <= 60000',
'resources_cleaned === true'
],
estimatedDuration: 120
});
return tests;
}
/**
* Generate large tool set for performance testing
*/
generateLargeToolSet(count) {
const tools = [];
for (let i = 1; i <= count; i++) {
tools.push({
name: `tool_${i}`,
description: `Test tool number ${i} for performance testing`,
input_schema: {
type: 'object',
properties: {
input: { type: 'string', description: `Input for tool ${i}` },
options: {
type: 'object',
properties: {
option1: { type: 'string' },
option2: { type: 'number' }
}
}
},
required: ['input']
}
});
}
return tools;
}
/**
* Generate complex nested schema for performance testing
*/
generateComplexSchema() {
return {
type: 'object',
properties: {
user: {
type: 'object',
properties: {
personal_info: {
type: 'object',
properties: {
name: { type: 'string' },
age: { type: 'integer', minimum: 0, maximum: 150 },
addresses: {
type: 'array',
items: {
type: 'object',
properties: {
type: { type: 'string', enum: ['home', 'work', 'other'] },
street: { type: 'string' },
city: { type: 'string' },
country: { type: 'string' },
postal_code: { type: 'string' },
coordinates: {
type: 'object',
properties: {
latitude: { type: 'number' },
longitude: { type: 'number' }
},
required: ['latitude', 'longitude']
}
},
required: ['type', 'street', 'city']
}
},
contacts: {
type: 'array',
items: {
type: 'object',
properties: {
type: { type: 'string', enum: ['email', 'phone', 'social'] },
value: { type: 'string' },
primary: { type: 'boolean', default: false },
metadata: {
type: 'object',
additionalProperties: true
}
},
required: ['type', 'value']
}
}
},
required: ['name']
},
preferences: {
type: 'object',
properties: {
categories: {
type: 'array',
items: { type: 'string' }
},
settings: {
type: 'object',
properties: {
notifications: {
type: 'object',
properties: {
email: { type: 'boolean' },
sms: { type: 'boolean' },
push: { type: 'boolean' }
}
},
privacy: {
type: 'object',
properties: {
public_profile: { type: 'boolean' },
data_sharing: { type: 'boolean' }
}
}
}
}
}
}
},
required: ['personal_info']
}
},
required: ['user']
};
}
/**
* Save test matrix to file
*/
async saveMatrix(matrix) {
await promises_1.default.mkdir(this.outputDir, { recursive: true });
const filename = `${matrix.matrixId}.json`;
const filepath = path_1.default.join(this.outputDir, filename);
await promises_1.default.writeFile(filepath, JSON.stringify(matrix, null, 2));
logger_1.logger.info('Test matrix saved successfully', {
filepath,
totalTests: matrix.totalTestCases
});
}
/**
* Generate test scripts from matrix
*/
async generateTestScripts(matrix) {
const scriptPaths = [];
const scriptsDir = path_1.default.join(this.outputDir, 'generated-scripts');
await promises_1.default.mkdir(scriptsDir, { recursive: true });
// Group tests by category
const categories = ['functional', 'integration', 'performance', 'edge-case', 'error-handling'];
for (const category of categories) {
const categoryTests = matrix.testCases.filter(tc => tc.category === category);
if (categoryTests.length === 0)
continue;
const scriptContent = this.generateCategoryScript(category, categoryTests);
const scriptFilename = `test-gemini-${category.replace('-', '_')}.js`;
const scriptPath = path_1.default.join(scriptsDir, scriptFilename);
await promises_1.default.writeFile(scriptPath, scriptContent);
scriptPaths.push(scriptPath);
}
logger_1.logger.info('Test scripts generated', {
scriptCount: scriptPaths.length,
scriptsDir
});
return scriptPaths;
}
/**
* Generate test script for specific category
*/
generateCategoryScript(category, tests) {
return `#!/usr/bin/env node
/**
* Generated Gemini Tool Calling ${category.toUpperCase()} Tests
* Auto-generated from test matrix
*
* Project owner: Jason Zhang
*/
const axios = require('axios');
const fs = require('fs').promises;
const BASE_URL = 'http://localhost:5502';
async function runGemini${category.charAt(0).toUpperCase() + category.slice(1).replace('-', '')}Tests() {
console.log('🧪 Running Gemini ${category} tests');
console.log('=' .repeat(60));
const testCases = ${JSON.stringify(tests, null, 2)};
const results = [];
for (const testCase of testCases) {
console.log(\`\\n📋 Test: \${testCase.name}\`);
console.log(\`Description: \${testCase.description}\`);
console.log(\`Priority: \${testCase.priority}\`);
console.log(\`Expected: \${testCase.expectedOutcome}\`);
const startTime = Date.now();
try {
const response = await axios.post(\`\${BASE_URL}/v1/chat/completions\`, testCase.testData.request, {
headers: { 'Content-Type': 'application/json' },
timeout: testCase.estimatedDuration * 1000
});
const endTime = Date.now();
const duration = endTime - startTime;
console.log(\`✅ Test passed (\${duration}ms)\`);
results.push({
testId: testCase.id,
name: testCase.name,
status: 'passed',
duration,
response: response.data
});
} catch (error) {
const endTime = Date.now();
const duration = endTime - startTime;
const status = testCase.expectedOutcome === 'controlled-failure' || testCase.expectedOutcome === 'graceful-error' ? 'passed' : 'failed';
console.log(\`\${status === 'passed' ? '✅' : '❌'} Test \${status} (\${duration}ms)\`);
results.push({
testId: testCase.id,
name: testCase.name,
status,
duration,
error: error.message
});
}
}
console.log(\`\\n📊 Test Summary:\`);
console.log(\`Total: \${results.length}\`);
console.log(\`Passed: \${results.filter(r => r.status === 'passed').length}\`);
console.log(\`Failed: \${results.filter(r => r.status === 'failed').length}\`);
return results;
}
if (require.main === module) {
runGemini${category.charAt(0).toUpperCase() + category.slice(1).replace('-', '')}Tests()
.then(results => {
console.log('\\n✅ ${category} tests completed');
process.exit(0);
})
.catch(error => {
console.error('\\n❌ ${category} tests failed:', error.message);
process.exit(1);
});
}
module.exports = { runGemini${category.charAt(0).toUpperCase() + category.slice(1).replace('-', '')}Tests };`;
}
}
exports.GeminiTestMatrixGenerator = GeminiTestMatrixGenerator;
exports.default = GeminiTestMatrixGenerator;
//# sourceMappingURL=gemini-test-matrix-generator.js.map