@skyramp/mcp
Version:
Skyramp MCP (Model Context Protocol) Server - AI-powered test generation and execution
661 lines (660 loc) • 27.8 kB
JavaScript
import Docker from "dockerode";
import path from "path";
import fs from "fs";
import { Writable } from "stream";
import { stripVTControlCharacters } from "util";
import { logger } from "../utils/logger.js";
const DEFAULT_TIMEOUT = 300000; // 5 minutes
const MAX_CONCURRENT_EXECUTIONS = 5;
const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.8";
const DOCKER_PLATFORM = "linux/amd64";
const EXECUTION_PROGRESS_INTERVAL = 10000; // 10 seconds between progress updates during execution
// Files and directories to exclude when mounting workspace to Docker container
export const EXCLUDED_MOUNT_ITEMS = [
"package-lock.json",
"package.json",
"node_modules",
];
/**
* Find the start index of a comment in a line, ignoring comment delimiters inside strings
* Returns -1 if no comment is found outside of strings
*/
function findCommentStart(line) {
let inSingleQuote = false;
let inDoubleQuote = false;
let escaped = false;
for (let i = 0; i < line.length; i++) {
const char = line[i];
const nextChar = i + 1 < line.length ? line[i + 1] : "";
// Handle escape sequences
if (escaped) {
escaped = false;
continue;
}
if (char === "\\") {
escaped = true;
continue;
}
// Track string boundaries
if (char === "'" && !inDoubleQuote) {
inSingleQuote = !inSingleQuote;
continue;
}
if (char === '"' && !inSingleQuote) {
inDoubleQuote = !inDoubleQuote;
continue;
}
// Check for comments only if not in a string
if (!inSingleQuote && !inDoubleQuote) {
if (char === "/" && nextChar === "/") {
return i;
}
if (char === "#") {
return i;
}
}
}
return -1; // No comment found
}
/**
* Filter out comments from code lines
* Returns array of non-comment lines with inline comments removed
*/
function filterComments(lines) {
const nonCommentLines = [];
let inMultiLineComment = false;
let inPythonMultiLineComment = false;
for (let line of lines) {
const trimmed = line.trim();
// Check for Python multi-line string comments (""" or ''')
// Count occurrences to handle single-line strings like '''comment'''
const tripleDoubleCount = (trimmed.match(/"""/g) || []).length;
const tripleSingleCount = (trimmed.match(/'''/g) || []).length;
if (tripleDoubleCount > 0) {
// Odd number: opening or closing a multi-line comment (toggle state)
// Even number: complete string on same line (e.g., """comment""" - don't toggle)
if (tripleDoubleCount % 2 === 1) {
inPythonMultiLineComment = !inPythonMultiLineComment;
}
continue; // Skip any line with triple quotes
}
if (tripleSingleCount > 0) {
if (tripleSingleCount % 2 === 1) {
inPythonMultiLineComment = !inPythonMultiLineComment;
}
continue;
}
if (inPythonMultiLineComment) {
continue;
}
// Check for multi-line comment start/end (/* */)
if (trimmed.includes("/*")) {
if (trimmed.includes("*/")) {
// Single-line multi-line comment (e.g., /* comment */)
// Don't change state, just skip the line
continue;
}
else {
// Opening a multi-line comment
inMultiLineComment = true;
continue;
}
}
if (inMultiLineComment) {
if (trimmed.includes("*/")) {
inMultiLineComment = false;
}
continue;
}
// Skip single-line comments
if (trimmed.startsWith("//") || trimmed.startsWith("#")) {
continue;
}
// Remove inline comments from the line before processing
// Use helper function to avoid removing comment delimiters inside strings
const commentIndex = findCommentStart(line);
if (commentIndex >= 0) {
line = line.substring(0, commentIndex);
}
nonCommentLines.push(line);
}
return nonCommentLines;
}
/**
* Detect session file paths referenced in test files
* Looks for storageState patterns in TypeScript/JavaScript/Python/Java/C# test files
* Excludes matches found in comments
*/
function detectSessionFiles(testFilePath) {
try {
const content = fs.readFileSync(testFilePath, "utf-8");
const lines = content.split("\n");
const sessionFiles = [];
// Pattern for TypeScript/JavaScript: storageState: '/path/to/file' or storageState: "/path/to/file"
const tsJsPattern = /storageState:\s*['"]([^'"]+)['"]/g;
// Pattern for Python: storage_state='/path/to/file' or storage_state="/path/to/file"
const pythonPattern = /storage_state\s*=\s*['"]([^'"]+)['"]/g;
// Pattern for Java: setStorageState(Paths.get("path")) or setStorageState("path")
// Enforces proper parenthesis matching: first ) is required, second ) is required when using Paths.get
const javaPattern = /setStorageState(?:Path)?\s*\(\s*(?:Paths\.get\s*\(\s*)?['"]([^'"]+)['"]\s*\)(?:\s*\))?/g;
// Pattern for C#: StorageStatePath = "path" or StorageState = "path"
const csharpPattern = /StorageState(?:Path)?\s*=\s*['"]([^'"]+)['"]/g;
// Filter out comments
const codeLines = filterComments(lines);
// Process each non-comment line
for (const line of codeLines) {
// Try all patterns on this line
let match;
tsJsPattern.lastIndex = 0;
while ((match = tsJsPattern.exec(line)) !== null) {
sessionFiles.push(match[1]);
}
pythonPattern.lastIndex = 0;
while ((match = pythonPattern.exec(line)) !== null) {
sessionFiles.push(match[1]);
}
javaPattern.lastIndex = 0;
while ((match = javaPattern.exec(line)) !== null) {
sessionFiles.push(match[1]);
}
csharpPattern.lastIndex = 0;
while ((match = csharpPattern.exec(line)) !== null) {
sessionFiles.push(match[1]);
}
}
return sessionFiles;
}
catch (error) {
logger.error(`Failed to detect session files in ${testFilePath}`, {
error,
});
return [];
}
}
export class TestExecutionService {
docker;
imageReady = null;
constructor() {
this.docker = new Docker();
}
/**
* Execute multiple tests in parallel batches
*/
async executeBatch(testOptions) {
logger.info(`Starting batch execution of ${testOptions.length} tests`);
const startTime = Date.now();
const results = [];
// Execute tests in batches to control concurrency
for (let i = 0; i < testOptions.length; i += MAX_CONCURRENT_EXECUTIONS) {
const batch = testOptions.slice(i, i + MAX_CONCURRENT_EXECUTIONS);
logger.debug(`Executing batch ${Math.floor(i / MAX_CONCURRENT_EXECUTIONS) + 1} (${batch.length} tests)`);
const batchPromises = batch.map((options) => this.executeTest(options).catch((error) => {
// If execution fails completely, return error result
return {
testFile: options.testFile,
passed: false,
executedAt: new Date().toISOString(),
duration: 0,
errors: [error.message],
warnings: [],
crashed: true,
};
}));
const batchResults = await Promise.all(batchPromises);
results.push(...batchResults);
}
const totalDuration = Date.now() - startTime;
const passed = results.filter((r) => r.passed).length;
const failed = results.filter((r) => !r.passed && !r.crashed).length;
const crashed = results.filter((r) => r.crashed).length;
logger.info(`Batch execution complete: ${passed} passed, ${failed} failed, ${crashed} crashed`);
return {
totalTests: testOptions.length,
passed,
failed,
crashed,
totalDuration,
results,
};
}
/**
* Execute a single test
* @param options Test execution options
* @param onProgress Optional callback for progress updates
*/
async executeTest(options, onProgress) {
const startTime = Date.now();
const executedAt = new Date().toISOString();
logger.debug(`Executing test: ${options.testFile}`);
// Check and ensure Docker image is ready with progress reporting
await onProgress?.({
phase: "docker-check",
message: "Checking Docker image availability...",
percent: 5,
});
const imageResult = await this.ensureDockerImage(onProgress);
if (imageResult.cached) {
await onProgress?.({
phase: "docker-check",
message: "Using cached Docker image",
percent: 20,
details: { cached: true },
});
}
else {
await onProgress?.({
phase: "docker-pull",
message: "Docker image pull completed",
percent: 20,
details: { cached: false },
});
}
// Report preparing phase
await onProgress?.({
phase: "preparing",
message: "Validating workspace and test file...",
percent: 25,
});
// Validate workspace path - use accessSync for better validation
const workspacePath = path.resolve(options.workspacePath);
try {
fs.accessSync(workspacePath, fs.constants.R_OK);
}
catch (err) {
throw new Error(`Workspace path does not exist or is not readable: ${workspacePath}`);
}
// Validate test file - use basename for safer filename extraction
const filename = path.basename(options.testFile);
if (!filename) {
throw new Error("Invalid test file path: could not extract filename");
}
if (!fs.existsSync(options.testFile)) {
throw new Error(`Test file does not exist: ${options.testFile}`);
}
await onProgress?.({
phase: "preparing",
message: "Preparing Docker container configuration...",
percent: 30,
});
const containerMountPath = "/home/user";
const dockerSocketPath = "/var/run/docker.sock";
// Calculate relative test file path
let testFilePath = path.relative(workspacePath, options.testFile);
testFilePath = path.resolve(containerMountPath, testFilePath);
// Prepare Docker command
const command = [
"/root/runner.sh",
options.language,
testFilePath,
options.testType,
];
// Prepare host config with mounts
const hostConfig = {
ExtraHosts: ["host.docker.internal:host-gateway"],
Mounts: [
{
Type: "bind",
Target: dockerSocketPath,
Source: dockerSocketPath,
},
],
};
// Mount workspace files (excluding unnecessary items)
const workspaceFiles = fs.readdirSync(workspacePath);
const filesToMount = workspaceFiles.filter((file) => !EXCLUDED_MOUNT_ITEMS.includes(file));
hostConfig.Mounts?.push(...filesToMount.map((file) => ({
Type: "bind",
Target: path.join(containerMountPath, file),
Source: path.join(workspacePath, file),
})));
// Detect and mount session files
const sessionFiles = detectSessionFiles(options.testFile);
const mountedPaths = new Set(); // Track mounted file paths to prevent duplicates
for (const sessionFile of sessionFiles) {
let sessionFileSource;
let sessionFileTarget;
if (path.isAbsolute(sessionFile)) {
// Absolute path: mount at the same absolute path in container
sessionFileSource = sessionFile;
sessionFileTarget = sessionFile;
}
else {
// Relative path: resolve from test file directory on host
const testFileDir = path.dirname(options.testFile);
sessionFileSource = path.resolve(testFileDir, sessionFile);
// Mount at the corresponding relative path in the container
const testFileDirInContainer = path.dirname(testFilePath);
sessionFileTarget = path.resolve(testFileDirInContainer, sessionFile);
}
// Check if session file exists on host
if (fs.existsSync(sessionFileSource)) {
// Only mount if we haven't already mounted this path
if (!mountedPaths.has(sessionFileTarget)) {
logger.info(` docker run -v ${sessionFileSource}:${sessionFileTarget}:ro ...`);
hostConfig.Mounts?.push({
Type: "bind",
Target: sessionFileTarget,
Source: sessionFileSource,
ReadOnly: true,
});
mountedPaths.add(sessionFileTarget);
}
}
else {
logger.error(`✗ Session file not found: ${sessionFileSource}`);
logger.error(` Referenced in test as: ${sessionFile}`);
}
}
// Handle playwright save storage path
let saveStorageTargetPath = "";
if (options.playwrightSaveStoragePath) {
let saveStorageSource;
let saveStorageTarget;
if (path.isAbsolute(options.playwrightSaveStoragePath)) {
// Absolute path: use as-is
saveStorageSource = path.dirname(options.playwrightSaveStoragePath);
saveStorageTarget = path.dirname(options.playwrightSaveStoragePath);
saveStorageTargetPath = options.playwrightSaveStoragePath;
}
else {
// Relative path: resolve from workspace
const absolutePath = path.resolve(workspacePath, options.playwrightSaveStoragePath);
saveStorageSource = path.dirname(absolutePath);
saveStorageTarget = path.join(containerMountPath, path.dirname(options.playwrightSaveStoragePath));
saveStorageTargetPath = path.join(containerMountPath, options.playwrightSaveStoragePath);
}
// Ensure the directory exists on host
if (!fs.existsSync(saveStorageSource)) {
fs.mkdirSync(saveStorageSource, { recursive: true });
}
// Mount the directory as writable (not read-only)
if (!mountedPaths.has(saveStorageTarget)) {
logger.info(` docker run -v ${saveStorageSource}:${saveStorageTarget} ...`);
hostConfig.Mounts?.push({
Type: "bind",
Target: saveStorageTarget,
Source: saveStorageSource,
ReadOnly: false,
});
mountedPaths.add(saveStorageTarget);
}
}
// Prepare environment variables
const env = [
`SKYRAMP_TEST_TOKEN=${options.token || ""}`,
"SKYRAMP_IN_DOCKER=true",
];
// Add save storage path to environment if provided
if (saveStorageTargetPath) {
env.push(`PLAYWRIGHT_SAVE_STORAGE_PATH=${saveStorageTargetPath}`);
}
if (process.env.SKYRAMP_DEBUG) {
env.push(`SKYRAMP_DEBUG=${process.env.SKYRAMP_DEBUG}`);
}
if (process.env.API_KEY) {
env.push(`API_KEY=${process.env.API_KEY}`);
}
// Capture output
let output = "";
class DockerStream extends Writable {
_write(data, encode, cb) {
output += data.toString();
cb();
}
}
const stream = new DockerStream();
try {
// Report executing phase
await onProgress?.({
phase: "executing",
message: "Starting test execution in Docker container...",
percent: 40,
});
let statusCode = 0;
let containerRef = null;
// Start periodic progress reporter during execution
// Runs in parallel with docker.run() to keep the AI agent informed
let progressIntervalHandle;
if (onProgress) {
let progressTick = 0;
progressIntervalHandle = setInterval(() => {
progressTick++;
const elapsed = Math.floor((Date.now() - startTime) / 1000);
// Progress moves from 40% to 79% during execution phase
// Slowly increment to show activity without reaching 80% (reserved for processing)
const percent = Math.min(40 + progressTick * 2, 79);
onProgress({
phase: "executing",
message: `Test running... (${elapsed}s elapsed)`,
percent,
}).catch(() => {
// Ignore progress notification errors
});
}, EXECUTION_PROGRESS_INTERVAL);
}
// Run container with timeout
const executionPromise = this.docker
.run(EXECUTOR_DOCKER_IMAGE, command, stream, {
Env: env,
HostConfig: hostConfig,
WorkingDir: containerMountPath, // Explicitly set working directory
})
.then(function (data) {
const result = data[0];
const container = data[1];
containerRef = container; // Capture container reference for cleanup
stream.end();
statusCode = result.StatusCode;
logger.debug("Docker container execution completed");
return container.remove();
})
.then(function () {
logger.debug("Docker container removed successfully");
containerRef = null; // Container already removed
return statusCode;
})
.catch(function (err) {
logger.error("Docker container execution failed", {
error: err.message,
});
throw err;
});
// Apply timeout
const timeout = options.timeout || DEFAULT_TIMEOUT;
let timeoutHandle;
const timeoutPromise = new Promise((_, reject) => {
timeoutHandle = setTimeout(() => reject(new Error(`Test execution timeout after ${timeout}ms`)), timeout);
});
try {
statusCode = await Promise.race([executionPromise, timeoutPromise]);
// Clear timers on successful completion
if (timeoutHandle)
clearTimeout(timeoutHandle);
if (progressIntervalHandle)
clearInterval(progressIntervalHandle);
}
catch (error) {
// Clear all timers on error
if (timeoutHandle)
clearTimeout(timeoutHandle);
if (progressIntervalHandle)
clearInterval(progressIntervalHandle);
// Cleanup container on timeout or other errors
if (containerRef !== null) {
const container = containerRef;
try {
logger.warning("Cleaning up orphaned Docker container after error");
await container.stop({ t: 5 }); // Give 5 seconds to stop gracefully
await container.remove({ force: true });
logger.debug("Orphaned container cleaned up successfully");
}
catch (cleanupError) {
logger.error("Failed to cleanup orphaned container", {
error: cleanupError.message,
});
}
}
throw error; // Re-throw the original error
}
// Report processing phase
await onProgress?.({
phase: "processing",
message: "Test execution completed, processing results...",
percent: 80,
});
const duration = Date.now() - startTime;
const cleanOutput = stripVTControlCharacters(output);
logger.info("Test execution completed", {
output: cleanOutput.substring(0, 200) +
(cleanOutput.length > 200 ? "..." : ""),
statusCode,
});
// Parse errors and warnings from output
const errors = this.parseErrors(cleanOutput);
const warnings = this.parseWarnings(cleanOutput);
// Check if test crashed
const crashed = statusCode !== 0 && statusCode !== 1;
const result = {
testFile: options.testFile,
passed: statusCode === 0,
executedAt,
duration,
errors,
warnings,
crashed,
output: cleanOutput,
exitCode: statusCode,
};
// Report completion
await onProgress?.({
phase: "processing",
message: result.passed
? "Test passed successfully"
: "Test execution completed with failures",
percent: 100,
});
logger.debug(`Test ${result.passed ? "passed" : "failed"}: ${options.testFile}`);
return result;
}
catch (error) {
const duration = Date.now() - startTime;
const cleanOutput = stripVTControlCharacters(output);
logger.error(`Test execution error: ${error.message}`);
return {
testFile: options.testFile,
passed: false,
executedAt,
duration,
errors: [error.message],
warnings: [],
crashed: true,
output: cleanOutput,
};
}
}
/**
* Ensure Docker image is available
* @param onProgress Optional callback for progress updates during pull
* @returns Object indicating whether image was cached or pulled
*/
async ensureDockerImage(onProgress) {
try {
const images = await this.docker.listImages();
const imageExists = images.some((img) => (img.RepoTags && img.RepoTags.includes(EXECUTOR_DOCKER_IMAGE)) ||
(img.RepoDigests && img.RepoDigests.includes(EXECUTOR_DOCKER_IMAGE)));
if (imageExists) {
logger.debug(`Docker image ${EXECUTOR_DOCKER_IMAGE} already available`);
return { cached: true };
}
logger.info(`Pulling Docker image ${EXECUTOR_DOCKER_IMAGE}...`);
await onProgress?.({
phase: "docker-pull",
message: `Pulling Docker image ${EXECUTOR_DOCKER_IMAGE}...`,
percent: 10,
details: { cached: false },
});
await new Promise((resolve, reject) => {
this.docker.pull(EXECUTOR_DOCKER_IMAGE, { platform: DOCKER_PLATFORM }, (err, stream) => {
if (err)
return reject(err);
if (!stream)
return reject(new Error("No stream received from docker pull"));
this.docker.modem.followProgress(stream, (err, res) => {
if (err)
return reject(err);
logger.info(`Docker image ${EXECUTOR_DOCKER_IMAGE} pulled successfully`);
resolve(res);
},
// Progress callback for each layer
(event) => {
if (event.status && onProgress) {
const progressMsg = event.progress || event.status;
const layerId = event.id || "";
// Calculate approximate progress (10-19% range during pull)
let percent = 10;
if (event.progressDetail?.current &&
event.progressDetail?.total) {
const layerProgress = event.progressDetail.current / event.progressDetail.total;
percent = 10 + Math.floor(layerProgress * 9); // 10-19%
}
onProgress({
phase: "docker-pull",
message: `${event.status}${layerId ? ` [${layerId}]` : ""}: ${progressMsg}`,
percent,
details: {
cached: false,
pullProgress: {
current: event.progressDetail?.current,
total: event.progressDetail?.total,
layer: layerId,
},
},
}).catch(() => {
// Ignore progress notification errors
});
}
});
});
});
return { cached: false };
}
catch (error) {
logger.error(`Failed to ensure Docker image: ${error.message}`);
throw new Error(`Docker image setup failed: ${error.message}`);
}
}
/**
* Parse errors from test output
*/
parseErrors(output) {
const errors = [];
const lines = output.split("\n");
for (const line of lines) {
// Common error patterns
if (line.includes("Error:") ||
line.includes("ERROR") ||
line.includes("AssertionError") ||
line.includes("Exception") ||
line.includes("FAILED") ||
line.includes("Traceback")) {
errors.push(line.trim());
}
}
return errors.slice(0, 20); // Limit to first 20 errors
}
/**
* Parse warnings from test output
*/
parseWarnings(output) {
const warnings = [];
const lines = output.split("\n");
for (const line of lines) {
if (line.includes("Warning:") ||
line.includes("WARN") ||
line.includes("Deprecated")) {
warnings.push(line.trim());
}
}
return warnings.slice(0, 10); // Limit to first 10 warnings
}
}