@skyramp/mcp
Version:
Skyramp MCP (Model Context Protocol) Server - AI-powered test generation and execution
657 lines (656 loc) • 25.8 kB
JavaScript
import * as fs from "fs";
import { logger } from "../utils/logger.js";
export class TestHealthService {
/**
* Generate comprehensive health report for tests
*/
async generateHealthReport(tests, driftResults) {
logger.info(`Generating health report for ${tests.length} tests`);
const healthAnalyses = [];
const recommendations = [];
// Analyze each test
for (const test of tests) {
const driftData = driftResults?.find((d) => d.testFile === test.testFile);
const analysis = await this.analyzeTestHealth(test.testFile, test.execution, driftData, test.apiSchema);
healthAnalyses.push(analysis);
recommendations.push(analysis.recommendation);
}
// Calculate summary statistics
const summary = {
totalTests: tests.length,
healthy: healthAnalyses.filter((a) => a.healthScore.status === "healthy")
.length,
atRisk: healthAnalyses.filter((a) => a.healthScore.status === "at_risk")
.length,
broken: healthAnalyses.filter((a) => a.healthScore.status === "broken")
.length,
unknown: healthAnalyses.filter((a) => a.healthScore.status === "unknown")
.length,
averageHealthScore: healthAnalyses.length > 0
? Math.round(healthAnalyses.reduce((sum, a) => sum + a.healthScore.overall, 0) / healthAnalyses.length)
: 0,
};
// Analyze coverage if API schemas are available
let coverage;
const apiSchemas = tests
.map((t) => t.apiSchema)
.filter((s) => s);
if (apiSchemas.length > 0) {
try {
coverage = await this.analyzeCoverage(tests, apiSchemas[0]);
}
catch (error) {
logger.error(`Failed to analyze coverage: ${error.message}`);
}
}
return {
summary,
tests: healthAnalyses,
recommendations: this.prioritizeRecommendations(recommendations),
coverage,
generatedAt: new Date().toISOString(),
};
}
/**
* Analyze health of a single test
*/
async analyzeTestHealth(testFile, execution, drift, apiSchema) {
// Calculate execution score
const executionScore = execution
? this.calculateExecutionScore(execution)
: undefined;
// Calculate health score
const healthScore = this.calculateHealthScore(executionScore?.score, drift?.driftScore);
// Identify issues
const issues = this.identifyIssues(execution, drift);
// Extract API endpoint info from test
const apiEndpoint = apiSchema
? await this.extractEndpointFromTest(testFile, apiSchema)
: undefined;
// Generate recommendation
const recommendation = this.generateRecommendation(testFile, healthScore, drift?.driftScore, execution, issues, apiEndpoint);
return {
testFile,
healthScore,
issues,
recommendation,
executionData: execution
? {
passed: execution.passed,
duration: execution.duration,
errors: execution.errors,
warnings: execution.warnings,
}
: undefined,
driftData: drift
? {
driftScore: drift.driftScore,
changes: drift.changes?.length || 0,
affectedFiles: drift.affectedFiles?.files?.length || 0,
}
: undefined,
apiEndpoint,
};
}
/**
* Calculate execution score (0-100)
*/
calculateExecutionScore(execution) {
let score;
let status;
if (execution.crashed) {
score = 0;
status = "crashed";
}
else if (execution.passed) {
if (execution.warnings.length > 0) {
// Passed with warnings: 50-100 based on warning count
score = Math.max(50, 100 - execution.warnings.length * 10);
status = "passed_with_warnings";
}
else {
score = 100;
status = "passed";
}
}
else if (execution.errors.length > 0) {
// Failed with errors: 30-50 based on error count
score = Math.max(30, 50 - execution.errors.length * 5);
status = "failed";
}
else if (execution.duration > 280000) {
// Near timeout (5 min = 300000ms)
score = 20;
status = "timeout";
}
else {
score = 30;
status = "failed";
}
return {
score,
status,
hasWarnings: execution.warnings.length > 0,
hasErrors: execution.errors.length > 0,
crashed: execution.crashed,
};
}
/**
* Calculate overall health score
*
* Health status is primarily drift-based with optional execution refinement:
* - Healthy: drift < 20 (and execution >= 80 if available)
* - At Risk: drift 20-40 (and execution >= 60 if available)
* - Broken: drift >= 40 (or execution < 60 if available)
*/
calculateHealthScore(executionScore, driftScore) {
let overall;
let calculationMethod;
if (executionScore !== undefined && driftScore !== undefined) {
// Combined: 60% execution, 40% drift (inverted)
overall = Math.round(0.6 * executionScore + 0.4 * (100 - driftScore));
calculationMethod = "combined";
}
else if (executionScore !== undefined) {
overall = executionScore;
calculationMethod = "execution_only";
}
else if (driftScore !== undefined) {
overall = 100 - driftScore;
calculationMethod = "drift_only";
}
else {
overall = 0;
calculationMethod = "drift_only";
}
// Determine health status (primarily drift-based, execution is optional refinement)
// User requirements interpretation:
// - Healthy: drift < 20 AND (no execution OR execution >= 80)
// - At Risk: drift 20-40 AND (no execution OR execution >= 60)
// - Broken: drift >= 40 OR execution < 60
let status;
const drift = driftScore !== undefined ? driftScore : -1; // -1 means no drift data
const exec = executionScore !== undefined ? executionScore : -1; // -1 means no execution data
if (drift === -1 && exec === -1) {
// No data available
status = "unknown";
}
else if (drift !== -1) {
// Drift-based status (primary)
if (drift >= 40) {
// High drift (>= 40): Broken regardless of execution
status = "broken";
}
else if (drift >= 20 && drift < 40) {
// Medium drift (20-40): At risk, unless execution is very poor
status = exec !== -1 && exec < 60 ? "broken" : "at_risk";
}
else {
// Low drift (< 20): Healthy, but execution can downgrade to at_risk
// Since code hasn't changed much, even failing tests are "at_risk" not "broken"
if (exec !== -1 && exec < 80) {
status = "at_risk"; // Any execution issues with low drift → at_risk
}
else {
status = "healthy";
}
}
}
else if (exec !== -1) {
// Only execution data available (no drift)
if (exec >= 80) {
status = "healthy";
}
else if (exec >= 60) {
status = "at_risk";
}
else {
status = "broken";
}
}
else {
status = "unknown";
}
return {
overall,
executionScore,
driftScore,
status,
calculationMethod,
};
}
/**
* Identify specific issues with a test
*/
identifyIssues(execution, drift) {
const issues = [];
// Check execution issues
if (execution) {
if (execution.crashed) {
issues.push({
type: "crash",
severity: "critical",
description: "Test crashed during execution",
details: execution.errors.join("; "),
});
}
else if (execution.errors.length > 0) {
issues.push({
type: "test_failures",
severity: "high",
description: `Test failed with ${execution.errors.length} error(s)`,
details: execution.errors.slice(0, 3).join("; "),
});
}
if (execution.duration > 280000) {
issues.push({
type: "timeout",
severity: "medium",
description: "Test approaching timeout threshold",
details: `Duration: ${(execution.duration / 1000).toFixed(1)}s`,
});
}
}
// Check drift issues
if (drift &&
drift.changes &&
Array.isArray(drift.changes) &&
drift.changes.length > 0) {
const hasCodeChanges = drift.changes.some((c) => ["code_change", "function_changed", "class_changed"].includes(c.type));
if (hasCodeChanges) {
issues.push({
type: "code_changes",
severity: "medium",
description: "Code changes detected in dependencies",
details: `${drift.affectedFiles?.files.length || 0} file(s) changed`,
});
}
const endpointsRemoved = drift.changes.filter((c) => c.type === "endpoint_removed");
if (endpointsRemoved.length > 0) {
issues.push({
type: "endpoints_removed",
severity: "high",
description: `${endpointsRemoved.length} API endpoint(s) removed`,
details: endpointsRemoved.map((c) => c.description).join("; "),
});
}
const schemaChanges = drift.changes.filter((c) => ["endpoint_modified", "authentication_changed"].includes(c.type));
if (schemaChanges.length > 0) {
issues.push({
type: "schema_changes",
severity: "high",
description: "API schema changes detected",
details: schemaChanges.map((c) => c.description).join("; "),
});
}
const authChanged = drift.changes.some((c) => c.type === "authentication_changed");
if (authChanged) {
issues.push({
type: "authentication_changed",
severity: "critical",
description: "Authentication mechanism changed",
});
}
}
return issues;
}
/**
* Generate recommendation for a test
*
* Recommendation logic (primarily drift-based):
* 1. IF drift > 70: REGENERATE (HIGH priority)
* 2. ELSE IF endpoint missing: DELETE (HIGH priority)
* 3. ELSE IF 30 < drift <= 70: UPDATE (MEDIUM priority)
* 4. ELSE IF drift > 10: VERIFY (LOW priority)
* 5. ELSE: VERIFY (LOW priority, healthy test)
*
* Execution failures enhance rationale but don't change primary action
*/
generateRecommendation(testFile, healthScore, driftScore, execution, issues, apiEndpoint) {
const drift = driftScore !== undefined ? driftScore : -1; // -1 means no drift data
let action;
let priority;
let rationale;
let estimatedWork = "SMALL";
// Handle missing drift data first
if (drift === -1) {
// No drift data available - base recommendation on health status and execution
if (healthScore.status === "unknown") {
action = "VERIFY";
priority = "MEDIUM";
rationale = "Unable to analyze drift - manual verification recommended";
estimatedWork = "SMALL";
// If execution data shows failure, escalate
if (execution && !execution.passed) {
priority = "HIGH";
rationale =
"Drift analysis unavailable and test is failing - investigate immediately";
estimatedWork = "MEDIUM";
}
}
else if (execution && !execution.passed) {
// No drift data but test is failing
action = "UPDATE";
priority = "HIGH";
rationale =
"Test is failing but drift analysis unavailable - review test logic and dependencies";
estimatedWork = "MEDIUM";
}
else if (execution && execution.passed) {
// No drift data but test is passing
action = "VERIFY";
priority = "LOW";
rationale =
"Drift analysis unavailable but test is passing - periodic verification recommended";
estimatedWork = "SMALL";
}
else {
// No drift data, no execution data
action = "VERIFY";
priority = "MEDIUM";
rationale = "No drift or execution data available - analysis needed";
estimatedWork = "SMALL";
}
}
else if (drift > 70) {
// High drift -> REGENERATE
action = "REGENERATE";
priority = "HIGH";
rationale =
"High drift detected - significant code changes since test creation";
estimatedWork = "MEDIUM";
// Enhance rationale with test failures if present
if (execution && !execution.passed) {
rationale += ". Test is also failing";
}
// Add specific issues
if (issues && issues.length > 0) {
const criticalIssues = issues.filter((i) => ["critical", "high"].includes(i.severity));
if (criticalIssues.length > 0) {
rationale += `. Critical issues: ${criticalIssues
.map((i) => i.description)
.join(", ")}`;
}
}
}
else if (apiEndpoint?.exists === false) {
// Endpoint removed from schema -> DELETE
action = "DELETE";
priority = "HIGH";
rationale = "Endpoint no longer exists in API schema";
estimatedWork = "SMALL";
}
else if (drift > 30 && drift <= 70) {
// Moderate drift -> UPDATE
action = "UPDATE";
priority = "MEDIUM";
rationale =
"Moderate drift detected - related code changes may affect test";
estimatedWork = "SMALL";
// Enhance with schema changes
const schemaChanges = issues?.filter((i) => [
"schema_changes",
"endpoints_removed",
"authentication_changed",
].includes(i.type));
if (schemaChanges && schemaChanges.length > 0) {
rationale += `. Schema changes: ${schemaChanges
.map((i) => i.description)
.join(", ")}`;
estimatedWork = "MEDIUM";
}
// Note execution failures
if (execution && !execution.passed) {
rationale += ". Test is also failing - requires immediate attention";
priority = "HIGH"; // Escalate priority for failing tests
}
}
else if (drift > 10) {
// Low drift -> VERIFY
action = "VERIFY";
priority = "LOW";
rationale = "Minor changes detected - test likely still valid";
estimatedWork = "SMALL";
// Note execution status for context
if (execution) {
if (!execution.passed) {
rationale += ". However, test is failing - review needed";
priority = "MEDIUM"; // Escalate for failing tests
}
else if (execution.warnings.length > 0) {
rationale += ". Test passed with warnings";
}
}
}
else {
// Minimal/no drift -> VERIFY
action = "VERIFY";
priority = "LOW";
rationale = "Test appears healthy - periodic verification recommended";
estimatedWork = "SMALL";
// Handle edge case: low drift but test is failing
if (execution && !execution.passed) {
action = "UPDATE";
priority = "HIGH";
rationale =
"Test is failing despite low drift - may indicate environmental issues or test flakiness";
estimatedWork = "MEDIUM";
}
}
// Determine endpoint status
let endpointStatus;
if (apiEndpoint === undefined) {
endpointStatus = undefined;
}
else if (apiEndpoint.exists) {
endpointStatus = "exists";
}
else {
endpointStatus = "missing";
}
return {
testFile,
action,
priority,
rationale,
estimatedWork,
issues,
details: {
driftScore: drift,
executionPassed: execution?.passed,
endpointStatus,
},
};
}
/**
* Parse OpenAPI schema and extract endpoints
*/
async parseApiSchema(schemaPath) {
let schema;
try {
if (schemaPath.startsWith("http://") ||
schemaPath.startsWith("https://")) {
// Fetch from URL
const response = await fetch(schemaPath);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
schema = await response.json();
}
else {
// Read from file
const content = fs.readFileSync(schemaPath, "utf-8");
schema = JSON.parse(content);
}
}
catch (error) {
logger.error(`Failed to parse API schema: ${error.message}`);
throw new Error(`Could not parse API schema at ${schemaPath}`);
}
const endpoints = [];
const paths = schema.paths || {};
for (const [pathStr, pathItem] of Object.entries(paths)) {
for (const [method, operation] of Object.entries(pathItem)) {
if (["get", "post", "put", "delete", "patch"].includes(method)) {
const op = operation;
const authRequired = this.checkAuthRequired(op, schema);
endpoints.push({
path: pathStr,
method: method.toUpperCase(),
operationId: op.operationId,
authRequired,
parameters: op.parameters || [],
requestBody: op.requestBody,
responses: op.responses,
});
}
}
}
return endpoints;
}
/**
* Check if endpoint requires authentication
*/
checkAuthRequired(operation, schema) {
// Check security at operation level
if (operation.security && operation.security.length > 0) {
return true;
}
// Check security at schema level
if (schema.security && schema.security.length > 0) {
return true;
}
return false;
}
/**
* Analyze test coverage
*/
async analyzeCoverage(tests, apiSchema) {
const endpoints = await this.parseApiSchema(apiSchema);
const coverage = [];
for (const endpoint of endpoints) {
const endpointKey = `${endpoint.method} ${endpoint.path}`;
const coveredBy = tests.filter((t) => this.testCoversEndpoint(t.testFile, endpoint));
coverage.push({
endpoint: endpointKey,
method: endpoint.method,
covered: coveredBy.length > 0,
testFiles: coveredBy.map((t) => t.testFile),
endpointInfo: endpoint,
});
}
const coveredEndpoints = coverage.filter((c) => c.covered).length;
const uncoveredEndpoints = endpoints.filter((ep) => !coverage.find((c) => c.endpoint === `${ep.method} ${ep.path}`)
?.covered);
return {
totalEndpoints: endpoints.length,
coveredEndpoints,
coveragePercentage: endpoints.length > 0
? Math.round((coveredEndpoints / endpoints.length) * 100)
: 0,
uncoveredEndpoints,
coverage,
};
}
/**
* Check if test covers endpoint
*/
testCoversEndpoint(testFile, endpoint) {
try {
const content = fs.readFileSync(testFile, "utf-8");
// Convert OpenAPI-style path to regex, e.g. /users/{id} -> /users/[^/]+
// Replace path parameters with regex pattern, escaping the path parts but not the regex itself
const pathRegexString = endpoint.path
.split(/(\{[^}]+\})/) // Split on path parameters, keeping them in the result
.map((part, index) => {
if (part.match(/^\{[^}]+\}$/)) {
// This is a path parameter like {id}, replace with regex pattern
return "[^/]+";
}
else {
// This is a literal path part, escape special regex chars
return part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
})
.join("");
// pathRegexString is already escaped, so we can use it directly
const pathRegex = new RegExp(pathRegexString);
const method = endpoint.method.toUpperCase();
const methodLower = endpoint.method.toLowerCase();
// Check if path matches
if (!pathRegex.test(content)) {
return false;
}
// More precise method detection patterns
const methodPatterns = [
// Python: method="PUT" or method='PUT'
new RegExp(`method\\s*=\\s*["']${method}["']`, "i"),
// JavaScript/TypeScript: method: 'PUT' or .put( or .PUT(
new RegExp(`method\\s*:\\s*["']${method}["']`, "i"),
new RegExp(`\\.${methodLower}\\s*\\(`, "i"),
// HTTP request patterns: PUT /path
new RegExp(`${method}\\s+${pathRegexString}`, "i"),
// Axios/fetch: { method: 'PUT' }
new RegExp(`["']method["']\\s*:\\s*["']${method}["']`, "i"),
// RestAssured/Java: .put()
new RegExp(`\\.${methodLower}\\(`, "i"),
// Go: http.MethodPut or "PUT"
new RegExp(`http\\.Method${method.charAt(0) + methodLower.slice(1)}`, "i"),
new RegExp(`["']${method}["']`, "i"),
];
// Check if any pattern matches
return methodPatterns.some((pattern) => pattern.test(content));
}
catch (error) {
return false;
}
}
/**
* Extract endpoint information from test file
*/
async extractEndpointFromTest(testFile, apiSchema) {
try {
const content = fs.readFileSync(testFile, "utf-8");
const endpoints = await this.parseApiSchema(apiSchema);
// Find matching endpoint
for (const endpoint of endpoints) {
if (this.testCoversEndpoint(testFile, endpoint)) {
return {
path: endpoint.path,
method: endpoint.method,
exists: true,
};
}
}
return { path: "unknown", method: "unknown", exists: false };
}
catch (error) {
return undefined;
}
}
/**
* Prioritize and sort recommendations
*/
prioritizeRecommendations(recommendations) {
const priorityOrder = {
CRITICAL: 0,
HIGH: 1,
MEDIUM: 2,
LOW: 3,
};
return recommendations.sort((a, b) => {
const priorityDiff = priorityOrder[a.priority] - priorityOrder[b.priority];
if (priorityDiff !== 0)
return priorityDiff;
// Within same priority, sort by action
const actionOrder = {
REGENERATE: 0,
DELETE: 1,
UPDATE: 2,
ADD: 3,
VERIFY: 4,
};
return actionOrder[a.action] - actionOrder[b.action];
});
}
}