@skyramp/mcp
Version:
Skyramp MCP (Model Context Protocol) Server - AI-powered test generation and execution
199 lines (189 loc) • 9.31 kB
JavaScript
import { z } from "zod";
import { EnhancedDriftAnalysisService } from "../../services/DriftAnalysisService.js";
import { logger } from "../../utils/logger.js";
import { StateManager, } from "../../utils/AnalysisStateManager.js";
import path from "path";
import { AnalyticsService } from "../../services/AnalyticsService.js";
const TOOL_NAME = "skyramp_analyze_test_drift";
/**
* Register the Skyramp test drift analysis tool with the MCP server
*
* This tool analyzes code drift by comparing the current codebase state
* to the baseline when a test was created, helping identify tests that
* may need updates due to code changes.
*/
export function registerAnalyzeTestDriftTool(server) {
server.registerTool(TOOL_NAME, {
description: `Analyze code drift for specific Skyramp tests by comparing current code to baseline.
**PREREQUISITE:** Call \`skyramp_discover_tests\` first
**WHEN TO USE THIS TOOL:**
- After calling \`skyramp_discover_tests\`
**STATE FILE MODE:**
- Provide stateFile from skyramp_discover_tests
- Returns summary + updated stateFile path (reduces token usage by 98%+)
- Pass updated stateFile to next tool in chain
**LLM-Powered Analysis Strategy**:
- Filters changed files to ONLY those relevant to the test type
- For API tests: Extracts/generates API schema from code if none exists
- For UI/E2E tests: Extracts UI structure and routes from components
- Calculates drift score based on ACTUAL impact
**CRITICAL DRIFT DETECTION STRATEGY**
- For API tests:
* Checks for explicit API schema (OpenAPI/Swagger)
* If none exists, EXTRACTS schema from code (route handlers, decorators, controllers)
* Compares baseline vs current schema
* Compare the current param types with the baseline param types to check for breaking changes
* Compare the current response types with the baseline response types to check for breaking changes
- For UI/E2E tests:
* Extracts UI component structure
* Identifies routes and navigation patterns
* Tracks selector and element changes
**Drift Score** (0-100):
- 0-20: Minimal impact
- 21-40: Low impact
- 41-60: Medium impact
- 61-80: High impact (breaking changes likely)
- 81-100: Critical impact (major breaking changes)
**NEXT STEP:** Call \`skyramp_execute_tests_batch\` if user explicitly asks to execute tests, then call \`skyramp_calculate_health_scores\`
**Output:**
{summary, stateFile, sessionId, stateFileSize, message} with drift analysis scores, breaking changes detected, affected files, and recommendations.`,
inputSchema: {
stateFile: z
.string()
.describe("Path to state file from skyramp_discover_tests (required)"),
},
}, async (args) => {
let errorResult;
try {
// Load tests from state file
const stateManager = StateManager.fromStatePath(args.stateFile);
const stateData = await stateManager.readData();
const tests = stateData?.tests || [];
const fullState = await stateManager.readFullState();
const repositoryPath = fullState?.metadata.repositoryPath || "";
if (!tests || tests.length === 0) {
errorResult = {
content: [
{
type: "text",
text: JSON.stringify({
error: "State file is empty or invalid",
stateFile: args.stateFile,
}, null, 2),
},
],
isError: true,
};
return errorResult;
}
logger.info(`Loaded ${tests.length} tests from state file: ${args.stateFile}`);
// Validate repositoryPath
if (!repositoryPath || typeof repositoryPath !== "string") {
errorResult = {
content: [
{
type: "text",
text: JSON.stringify({
error: "repositoryPath not found in state file metadata",
details: "State file must contain repositoryPath in metadata",
}, null, 2),
},
],
isError: true,
};
return errorResult;
}
const absoluteRepoPath = path.resolve(repositoryPath);
// Map tests to include metadata (avoid recalculating test type)
const testMetadata = tests.map((t) => ({
testFile: t.testFile,
testType: t.testType,
}));
// Analyze all tests in batch
logger.info(`Analyzing drift for ${testMetadata.length} tests in batch mode`);
const driftService = new EnhancedDriftAnalysisService();
const driftResults = await driftService.analyzeBatchDrift(testMetadata, absoluteRepoPath, {
includeApiSchema: true,
includeDependencies: true,
});
// Enrich test objects with drift data (modify in place)
const enrichedTests = tests.map((test) => {
const driftData = driftResults.find((d) => d.testFile === test.testFile);
if (driftData) {
return {
...test,
drift: {
lastCommit: driftData.lastCommit,
currentCommit: driftData.currentCommit,
driftScore: driftData.driftScore,
changes: driftData.changes,
affectedFiles: driftData.affectedFiles,
apiSchemaChanges: driftData.apiSchemaChanges,
uiComponentChanges: driftData.uiComponentChanges,
analysisTimestamp: driftData.analysisTimestamp,
recommendations: driftData.recommendations,
},
};
}
return test;
});
// Calculate summary statistics
const testsWithDrift = enrichedTests.filter((t) => t.drift && t.drift.driftScore >= 0 && t.drift.lastCommit !== "");
const testsWithoutGitHistory = enrichedTests.filter((t) => t.drift && t.drift.lastCommit === "");
const testsWithErrors = enrichedTests.filter((t) => t.drift && t.drift.driftScore < 0);
const summary = {
totalTests: enrichedTests.length,
averageDriftScore: testsWithDrift.length > 0
? testsWithDrift.reduce((sum, t) => sum + (t.drift?.driftScore || 0), 0) / testsWithDrift.length
: 0,
highDrift: enrichedTests.filter((t) => t.drift && t.drift.driftScore >= 60).length,
mediumDrift: enrichedTests.filter((t) => t.drift && t.drift.driftScore >= 40 && t.drift.driftScore < 60).length,
lowDrift: enrichedTests.filter((t) => t.drift && t.drift.driftScore > 0 && t.drift.driftScore < 40).length,
noDrift: enrichedTests.filter((t) => t.drift && t.drift.driftScore === 0).length,
noGitHistory: testsWithoutGitHistory.length,
driftErrors: testsWithErrors.length,
};
logger.info(`Batch drift analysis completed. ${summary.totalTests} tests analyzed ` +
`(${summary.noGitHistory} without git history, ${summary.driftErrors} errors)`);
// Save to state file
await stateManager.writeData({ tests: enrichedTests }, {
repositoryPath: absoluteRepoPath,
step: "drift",
});
const stateSize = await stateManager.getSizeFormatted();
logger.info(`Saved ${enrichedTests.length} tests with drift data to state file: ${stateManager.getStatePath()} (${stateSize})`);
const responseData = {
summary,
stateFile: stateManager.getStatePath(),
sessionId: stateManager.getSessionId(),
stateFileSize: stateSize,
message: `Drift analysis complete. Analyzed ${summary.totalTests} tests. Pass stateFile to skyramp_execute_tests_batch or skyramp_calculate_health_scores.`,
generatedAt: new Date().toISOString(),
};
return {
content: [
{
type: "text",
text: JSON.stringify(responseData, null, 2),
},
],
};
}
catch (error) {
logger.error(`Test drift analysis failed: ${error.message}`, error);
errorResult = {
content: [
{
type: "text",
text: error.message,
},
],
isError: true,
};
return errorResult;
}
finally {
AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, {});
}
});
}