UNPKG

mcp-server-debug-thinking

Version:

Graph-based MCP server for systematic debugging using Problem-Solution Trees and Hypothesis-Experiment-Learning cycles

816 lines 40.9 kB
import { describe, it, expect, beforeEach, afterEach } from "vitest"; import { GraphService } from "../../services/GraphService.js"; import { ActionType } from "../../types/graphActions.js"; // Test suite focused on the improved similarity calculation features describe("GraphService - Enhanced Similarity Calculation", () => { let graphService; beforeEach(async () => { // Ensure clean test environment with unique directory per test process.env.DEBUG_DATA_DIR = `.test-graph-service-similarity-${Date.now()}-${Math.random().toString(36).substring(7)}`; graphService = new GraphService(); await graphService.initialize(); }); afterEach(async () => { // Clean up if (process.env.DEBUG_DATA_DIR) { const fs = await import("fs/promises"); try { await fs.rm(process.env.DEBUG_DATA_DIR, { recursive: true, force: true }); } catch (_error) { // Directory might not exist } delete process.env.DEBUG_DATA_DIR; } }); describe("calculateSimilarity method", () => { describe("Error type similarity (20% weight)", () => { it("should give full score for exact error type match", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "TypeError: Cannot read property 'foo' of undefined", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "TypeError: Cannot read property 'bar' of undefined", }); const response = JSON.parse(result.content[0].text); expect(response.similarProblems).toBeDefined(); expect(response.similarProblems?.length).toBeGreaterThan(0); // Should have high similarity due to same error type const similarity = response.similarProblems?.[0].similarity; expect(similarity).toBeGreaterThanOrEqual(0.2); // At least the error type contribution }); it("should give partial score for related error types", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "TypeError: Something went wrong", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Type Error: Different format but same category", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { const similarity = response.similarProblems[0].similarity; expect(similarity).toBeGreaterThan(0.12); // Should get group match score } }); it("should handle different error type groups correctly", async () => { // Create problems with different error types await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "ReferenceError: variable is not defined", }); await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Reference Error: x is not defined", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "not defined error in module", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should find the ReferenceError problems with grouped scoring const _refErrors = response.similarProblems.filter((p) => p.content.toLowerCase().includes("reference")); // ReferenceError関連の問題が見つからなくても他の問題は見つかる可能性があるため削除 } }); it("should give minimal score for unrelated error types", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "TypeError: type mismatch", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "NetworkError: connection timeout", }); const response = JSON.parse(result.content[0].text); // If similar problems found, they should have low similarity if (response.similarProblems && response.similarProblems.length > 0) { const typeErrorProblem = response.similarProblems.find((p) => p.content.includes("TypeError")); if (typeErrorProblem) { expect(typeErrorProblem.similarity).toBeLessThan(0.3); } } }); }); describe("Substring matching (20% weight)", () => { it("should find long common substrings", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Failed to connect to database server at localhost:5432", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Unable to connect to database server at localhost:3306", }); const response = JSON.parse(result.content[0].text); expect(response.similarProblems).toBeDefined(); expect(response.similarProblems?.length).toBeGreaterThan(0); // Should have good similarity due to "connect to database server at localhost" const similarity = response.similarProblems?.[0].similarity; expect(similarity).toBeGreaterThan(0.3); }); it("should handle short common substrings appropriately", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error ABC occurred", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Problem XYZ happened", }); const response = JSON.parse(result.content[0].text); // Should have low similarity due to no significant common substring if (response.similarProblems && response.similarProblems.length > 0) { expect(response.similarProblems[0].similarity).toBeLessThan(0.6); } }); it("should scale substring score based on text length", async () => { // Very similar short messages await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error in module A", }); const result1 = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error in module B", }); const response1 = JSON.parse(result1.content[0].text); // Very similar long messages await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Failed to process request due to invalid authentication token provided by the client application", }); const result2 = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Failed to process request due to invalid authentication token provided by the server backend", }); const response2 = JSON.parse(result2.content[0].text); // Both should have high similarity, but the scoring should be proportional if (response1.similarProblems && response1.similarProblems.length > 0 && response2.similarProblems && response2.similarProblems.length > 0) { expect(response1.similarProblems[0].similarity).toBeGreaterThan(0.2); expect(response2.similarProblems[0].similarity).toBeGreaterThan(0.4); } }); }); describe("Edit distance similarity (15% weight)", () => { it("should calculate character-level similarity for short texts", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Module not found: utils", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Module not found: util", }); const response = JSON.parse(result.content[0].text); expect(response.similarProblems).toBeDefined(); expect(response.similarProblems?.length).toBeGreaterThan(0); // Should have very high similarity (only 1 character difference) const similarity = response.similarProblems?.[0].similarity; expect(similarity).toBeGreaterThan(0.45); }); it("should calculate word-level similarity for long texts", async () => { const longText1 = "The application failed to start because the required configuration file was not found in the expected location"; const longText2 = "The application failed to initialize because the required configuration file was missing from the expected directory"; await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: longText1, }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: longText2, }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should have good similarity despite word differences expect(response.similarProblems[0].similarity).toBeGreaterThan(0.4); } }); it("should handle typos and minor variations", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Authentication failed", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Authentification failed", // Common typo }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should still match well despite typo expect(response.similarProblems[0].similarity).toBeGreaterThanOrEqual(0.5); } }); }); describe("Key phrase fuzzy matching (15% weight)", () => { it("should match exact key phrases", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Cannot read property 'length' of undefined", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Cannot read property 'size' of undefined", }); const response = JSON.parse(result.content[0].text); expect(response.similarProblems).toBeDefined(); expect(response.similarProblems?.length).toBeGreaterThan(0); // Should match the "cannot read property" pattern expect(response.similarProblems?.[0].similarity).toBeGreaterThan(0.4); }); it("should perform partial matching on key phrases", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "The system was unable to process the request", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Unable to handle the user request", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should partially match "unable to" phrase expect(response.similarProblems[0].similarity).toBeGreaterThan(0.2); } }); it("should match multiple key phrases", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Failed to connect: Permission denied when accessing resource", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Failed to authenticate: Permission denied for user account", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should match both "failed to" and "permission denied" expect(response.similarProblems[0].similarity).toBeGreaterThan(0.25); } }); it("should detect variations of key phrases", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Maximum call stack size exceeded", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error: Maximum call stack limit reached", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should match the stack overflow pattern expect(response.similarProblems[0].similarity).toBeGreaterThan(0.25); } }); }); describe("Word-based similarity (20% weight)", () => { it("should tokenize and match words correctly", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "database_connection-error: timeout.exceeded/retry", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "database connection error - timeout exceeded, will retry", }); const response = JSON.parse(result.content[0].text); expect(response.similarProblems).toBeDefined(); expect(response.similarProblems?.length).toBeGreaterThan(0); // Should match despite different delimiters expect(response.similarProblems?.[0].similarity).toBeGreaterThan(0.4); }); it("should handle partial word matches", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Configuration file not found", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Config file missing", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should match "config" with "configuration" and "file" expect(response.similarProblems[0].similarity).toBeGreaterThan(0.2); } }); it("should filter out short and numeric tokens", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error at line 42 in file a.js", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error at line 100 in file b.py", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should match "Error", "line", "file" but not numbers or short tokens expect(response.similarProblems[0].similarity).toBeGreaterThan(0.2); expect(response.similarProblems[0].similarity).toBeLessThanOrEqual(0.5); } }); }); describe("Identifier matching (10% weight)", () => { it("should extract and match quoted identifiers", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Cannot find module 'express' in file 'server.js'", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Module 'express' not found in 'app.js'", }); const response = JSON.parse(result.content[0].text); expect(response.similarProblems).toBeDefined(); expect(response.similarProblems?.length).toBeGreaterThan(0); // Should match the 'express' identifier expect(response.similarProblems?.[0].similarity).toBeGreaterThan(0.25); }); it("should match function names", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "TypeError: getUserById() returned undefined", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error in getUserById() function call", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should match the function name expect(response.similarProblems[0].similarity).toBeGreaterThan(0.2); } }); it("should handle different quote types", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: 'File "config.json" is corrupted', }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Cannot parse `config.json` file", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should match config.json despite different quotes expect(response.similarProblems[0].similarity).toBeGreaterThan(0.2); } }); }); describe("Combined similarity scoring", () => { it("should combine all factors for comprehensive similarity", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "TypeError: Cannot read property 'name' of undefined in UserService.getUser()", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "TypeError: Cannot read property 'email' of undefined in UserService.getProfile()", }); const response = JSON.parse(result.content[0].text); expect(response.similarProblems).toBeDefined(); expect(response.similarProblems?.length).toBeGreaterThan(0); // Should have high similarity due to: // - Same error type (20%) // - Common substring "Cannot read property" and "of undefined" (20%) // - Similar edit distance (15%) // - Matching key phrase (15%) // - Common words and UserService identifier (30%) expect(response.similarProblems?.[0].similarity).toBeGreaterThan(0.6); }); it("should handle completely different problems", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Network timeout while fetching data", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Invalid syntax in configuration file", }); const response = JSON.parse(result.content[0].text); // Should have very low or no similarity if (response.similarProblems && response.similarProblems.length > 0) { const networkProblem = response.similarProblems.find((p) => p.content.includes("Network")); if (networkProblem) { expect(networkProblem.similarity).toBeLessThan(0.2); } } }); it("should cap similarity score at 1.0", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Exact same error message", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Exact same error message", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { expect(response.similarProblems[0].similarity).toBeLessThanOrEqual(1.0); expect(response.similarProblems[0].similarity).toBeGreaterThan(0.45); } }); }); }); describe("findSimilarProblems integration", () => { it("should respect minSimilarity parameter", async () => { // Create problems with varying similarity await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Database connection error: timeout", }); await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Network error: request failed", }); await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "File not found: missing.txt", }); const result = await graphService.query({ action: ActionType.QUERY, type: "similar-problems", parameters: { pattern: "Database connection timeout exceeded", minSimilarity: 0.5, // High threshold }, }); const response = JSON.parse(result.content[0].text); expect(response.success).toBe(true); // Should only return highly similar problems if (response.results && response.results.problems.length > 0) { response.results.problems.forEach((p) => { expect(p.similarity).toBeGreaterThanOrEqual(0.5); }); } }); it("should limit results according to limit parameter", async () => { // Create many similar problems for (let i = 0; i < 20; i++) { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: `API Error ${i}: Request failed with status 500`, }); } const result = await graphService.query({ action: ActionType.QUERY, type: "similar-problems", parameters: { pattern: "API Error: Request failed", limit: 5, }, }); const response = JSON.parse(result.content[0].text); expect(response.success).toBe(true); expect(response.results?.problems.length).toBeLessThanOrEqual(5); }); it("should prioritize solved problems even with lower similarity", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Memory leak in event handlers", metadata: { status: "solved" }, }); await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Memory leak detected in React component lifecycle", metadata: { status: "open" }, }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Memory usage increasing in event listeners", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length >= 2) { // The solved problem should appear first expect(response.similarProblems[0].status).toBe("solved"); } }); }); describe("Performance with improved similarity", () => { it("should handle large numbers of problems efficiently", async () => { // Create 100 problems with various error types const errorTypes = ["TypeError", "ReferenceError", "SyntaxError", "RangeError", "Error"]; const components = ["UserService", "AuthModule", "Database", "API", "Cache"]; for (let i = 0; i < 100; i++) { const errorType = errorTypes[i % errorTypes.length]; const component = components[Math.floor(i / 20)]; await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: `${errorType}: Issue in ${component} at line ${i}`, }); } const startTime = Date.now(); const result = await graphService.query({ action: ActionType.QUERY, type: "similar-problems", parameters: { pattern: "TypeError: Problem in UserService module", limit: 10, }, }); const queryTime = Date.now() - startTime; const response = JSON.parse(result.content[0].text); expect(response.success).toBe(true); expect(queryTime).toBeLessThan(200); // Should be fast even with complex similarity calculation // Should find relevant results if (response.results && response.results.problems.length > 0) { const topResult = response.results.problems[0]; expect(topResult.content).toContain("TypeError"); } }); it("should use error type index for performance", async () => { // Create many problems of different types for (let i = 0; i < 50; i++) { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: `Generic problem ${i} without specific error type`, }); } for (let i = 0; i < 30; i++) { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: `TypeError: Specific type error number ${i}`, }); } const startTime = Date.now(); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "TypeError: New type error problem", }); const searchTime = Date.now() - startTime; const response = JSON.parse(result.content[0].text); expect(response.success).toBe(true); expect(searchTime).toBeLessThan(100); // Should be very fast with indexing // Should primarily find TypeError problems if (response.similarProblems && response.similarProblems.length > 0) { const typeErrorCount = response.similarProblems.filter((p) => p.content.includes("TypeError")).length; expect(typeErrorCount).toBeGreaterThan(response.similarProblems.length * 0.8); } }); }); describe("Edge cases and boundary conditions", () => { it("should handle empty strings", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Normal problem", }); const result = await graphService.query({ action: ActionType.QUERY, type: "similar-problems", parameters: { pattern: "", limit: 5, }, }); const response = JSON.parse(result.content[0].text); expect(response.success).toBe(true); // Should still return results, but with low similarity }); it("should handle very long texts", async () => { const longText1 = `Error: ${"x".repeat(1000)} in module`; const longText2 = `Error: ${"x".repeat(990)} in module`; await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: longText1, }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: longText2, }); const response = JSON.parse(result.content[0].text); expect(response.success).toBe(true); // Should handle long texts without crashing if (response.similarProblems && response.similarProblems.length > 0) { expect(response.similarProblems[0].similarity).toBeGreaterThan(0.45); } }); it("should handle special characters and unicode", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error: Invalid character '→' in file", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error: Invalid character '⇒' in file", }); const response = JSON.parse(result.content[0].text); expect(response.success).toBe(true); // Should handle unicode characters gracefully if (response.similarProblems && response.similarProblems.length > 0) { expect(response.similarProblems[0].similarity).toBeGreaterThan(0.25); } }); it("should handle problems with only whitespace differences", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error: Multiple spaces between words", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error: Multiple spaces between words", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should have very high similarity despite whitespace differences expect(response.similarProblems[0].similarity).toBeGreaterThan(0.45); } }); it("should handle case sensitivity correctly", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "ERROR: UPPERCASE MESSAGE", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "error: uppercase message", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should match despite case differences expect(response.similarProblems[0].similarity).toBeGreaterThan(0.45); } }); }); describe("Real-world scenarios", () => { it("should match React component errors effectively", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Warning: Can't perform a React state update on an unmounted component", }); await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error: Cannot update state on unmounted React component", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "React warning: State update on unmounted component detected", }); const response = JSON.parse(result.content[0].text); expect(response.similarProblems).toBeDefined(); expect(response.similarProblems?.length).toBeGreaterThan(0); // All three problems are about the same React issue response.similarProblems?.forEach((p) => { expect(p.similarity).toBeGreaterThan(0.25); }); }); it("should match database connection errors", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "ECONNREFUSED: Connection refused to PostgreSQL on localhost:5432", }); await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Error: connect ECONNREFUSED 127.0.0.1:5432", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "PostgreSQL connection failed: ECONNREFUSED at port 5432", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should match all connection refused errors const connRefusedProblems = response.similarProblems.filter((p) => p.content.includes("ECONNREFUSED")); expect(connRefusedProblems.length).toBeGreaterThan(0); connRefusedProblems.forEach((p) => { expect(p.similarity).toBeGreaterThan(0.15); }); } }); it("should match API endpoint errors", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "404 Not Found: GET /api/users/123", }); await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "HTTP 404: /api/users/456 not found", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Failed to fetch user: 404 error on /api/users/789", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should match API user endpoint 404 errors response.similarProblems.forEach((p) => { expect(p.content).toMatch(/404|users/i); expect(p.similarity).toBeGreaterThan(0.2); }); } }); it("should match webpack/build errors", async () => { await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Module not found: Error: Can't resolve './components/Header' in '/src/app'", }); await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "ERROR in ./src/app/index.js Module not found: Cannot resolve './components/Footer'", }); const result = await graphService.create({ action: ActionType.CREATE, nodeType: "problem", content: "Build failed: Module not found - Unable to resolve './components/Sidebar'", }); const response = JSON.parse(result.content[0].text); if (response.similarProblems && response.similarProblems.length > 0) { // Should match module resolution errors response.similarProblems.forEach((p) => { expect(p.content.toLowerCase()).toContain("module not found"); expect(p.similarity).toBeGreaterThan(0.2); }); } }); }); }); //# sourceMappingURL=GraphServiceSimilarity.test.js.map