@ursamu/core/docs/DEBUGGING_AND_TESTING.md

Ursamu - Modular MUD Engine with sandboxed scripting and plugin system

# Debugging and Testing Guide

This comprehensive guide covers debugging techniques, testing strategies, and troubleshooting methods for Ursamu.

## Table of Contents

- [Debugging Overview](#debugging-overview)
- [Debug Configuration](#debug-configuration)
- [Logging and Monitoring](#logging-and-monitoring)
- [Interactive Debugging](#interactive-debugging)
- [Performance Profiling](#performance-profiling)
- [Testing Strategies](#testing-strategies)
- [Unit Testing](#unit-testing)
- [Integration Testing](#integration-testing)
- [Load Testing](#load-testing)
- [Debugging Plugins](#debugging-plugins)
- [Debugging Commands](#debugging-commands)
- [Hot-Reload Debugging](#hot-reload-debugging)
- [Network Debugging](#network-debugging)
- [Database Debugging](#database-debugging)
- [Common Issues](#common-issues)
- [Troubleshooting Tools](#troubleshooting-tools)
- [Production Debugging](#production-debugging)

## Debugging Overview

The MUD engine provides comprehensive debugging capabilities through:

### Debug Levels
```typescript
enum DebugLevel {
  TRACE = 0,
  DEBUG = 1,
  INFO = 2,
  WARN = 3,
  ERROR = 4,
  FATAL = 5
}
```

### Debug Categories
- **Engine Core** - Main engine systems
- **Plugins** - Plugin loading and execution
- **Commands** - Command processing and execution
- **Network** - Protocol and connection handling
- **Database** - Data persistence and queries
- **Hot-Reload** - File watching and reloading
- **Virtual Machine** - Script execution and sandboxing

## Debug Configuration

### Environment Variables

```bash
# General debugging
DEBUG=ursamu:*                    # Enable all debug output
DEBUG=ursamu:core,ursamu:plugins     # Enable specific categories
DEBUG_LEVEL=debug              # Set minimum debug level
DEBUG_COLOR=true               # Enable colorized output

# Component-specific debugging
DEBUG_PLUGINS=true             # Plugin debugging
DEBUG_COMMANDS=true            # Command debugging
DEBUG_HOTRELOAD=true           # Hot-reload debugging
DEBUG_NETWORK=true             # Network debugging
DEBUG_VM=true                  # Virtual machine debugging

# Performance debugging
DEBUG_PERFORMANCE=true         # Performance monitoring
DEBUG_MEMORY=true              # Memory usage tracking
DEBUG_TIMINGS=true             # Execution timing
```

### Configuration File

```json
// config/debug.json
{
  "debug": {
    "enabled": true,
    "level": "debug",
    "categories": {
      "core": true,
      "plugins": true,
      "commands": true,
      "network": false,
      "database": true,
      "hotreload": true,
      "vm": true
    },
    "output": {
      "console": true,
      "file": "logs/debug.log",
      "structured": true,
      "timestamps": true,
      "colors": true
    },
    "performance": {
      "enabled": true,
      "slowQueryThreshold": 100,
      "slowCommandThreshold": 1000,
      "memoryTrackingInterval": 30000
    }
  }
}
```

### Runtime Debug Control

```typescript
// Enable/disable debugging at runtime
import { DebugManager } from './src/debug/DebugManager.js';

const debugManager = new DebugManager();

// Toggle debug categories
debugManager.enable('plugins');
debugManager.disable('network');

// Set debug level
debugManager.setLevel('debug');

// Get debug status
const status = debugManager.getStatus();
```

## Logging and Monitoring

### Structured Logging

```typescript
import { Logger } from './src/utils/Logger.js';

export class ExamplePlugin implements Plugin {
  private logger: Logger;

  async onLoad(context: PluginContext): Promise<void> {
    this.logger = context.logger.createChild('ExamplePlugin');

    // Structured logging
    this.logger.info('Plugin loading', {
      pluginName: this.name,
      version: this.version,
      loadTime: Date.now()
    });

    // Context-aware logging
    this.logger.debug('Registering commands', {
      commands: ['example', 'test'],
      category: 'setup'
    });

    // Error logging with stack traces
    try {
      await this.riskyOperation();
    } catch (error) {
      this.logger.error('Failed to initialize plugin', {
        error: error.message,
        stack: error.stack,
        operation: 'riskyOperation'
      });
    }
  }

  private async commandHandler(player: Player, args: string[]): Promise<CommandResult> {
    const timer = this.logger.startTimer();

    this.logger.debug('Command execution started', {
      player: player.name,
      command: 'example',
      args
    });

    try {
      const result = await this.executeCommand(player, args);
      
      timer.done('Command completed', {
        success: result.success,
        duration: timer.duration
      });

      return result;
    } catch (error) {
      this.logger.error('Command execution failed', {
        player: player.name,
        error: error.message,
        duration: timer.duration
      });
      throw error;
    }
  }
}
```

### Log Analysis

```bash
# Real-time log monitoring
tail -f logs/debug.log | grep -E "(ERROR|WARN)"

# Search for specific patterns
grep -i "plugin.*error" logs/*.log

# Analyze performance logs
awk '/Command.*duration/ { print $0 }' logs/debug.log | sort -k6 -n

# Extract plugin loading times
grep "Plugin.*loaded" logs/debug.log | \
  sed 's/.*loaded in \([0-9]*\)ms.*/\1/' | \
  awk '{ sum += $1; count++ } END { print "Average:", sum/count, "ms" }'
```

### Monitoring Dashboard

```typescript
// Real-time debug dashboard
export class DebugDashboard {
  private metrics = new Map<string, any>();
  private connections = new Set<WebSocket>();

  startServer(port: number = 3001): void {
    const server = new WebSocketServer({ port });
    
    server.on('connection', (ws) => {
      this.connections.add(ws);
      
      // Send current metrics
      ws.send(JSON.stringify({
        type: 'metrics',
        data: Object.fromEntries(this.metrics)
      }));
      
      ws.on('close', () => {
        this.connections.delete(ws);
      });
    });

    // Update metrics every second
    setInterval(() => {
      this.updateMetrics();
      this.broadcastMetrics();
    }, 1000);
  }

  private updateMetrics(): void {
    const engine = MUDEngine.getInstance();
    
    this.metrics.set('players', {
      online: engine.playerManager.getOnlineCount(),
      total: engine.playerManager.getTotalCount()
    });

    this.metrics.set('performance', {
      memory: process.memoryUsage(),
      uptime: process.uptime(),
      cpu: process.cpuUsage()
    });

    this.metrics.set('plugins', {
      loaded: engine.pluginManager.getLoadedCount(),
      active: engine.pluginManager.getActiveCount(),
      errors: engine.pluginManager.getErrorCount()
    });
  }

  private broadcastMetrics(): void {
    const data = JSON.stringify({
      type: 'metrics',
      data: Object.fromEntries(this.metrics),
      timestamp: Date.now()
    });

    for (const ws of this.connections) {
      if (ws.readyState === WebSocket.OPEN) {
        ws.send(data);
      }
    }
  }
}
```

## Interactive Debugging

### Debug Console

```typescript
// Built-in debug console
export class DebugConsole {
  private commands = new Map<string, Function>();

  constructor(private engine: MUDEngine) {
    this.registerDebugCommands();
  }

  private registerDebugCommands(): void {
    // Plugin debugging
    this.commands.set('plugins', () => {
      const plugins = this.engine.pluginManager.getAll();
      return plugins.map(p => ({
        name: p.name,
        version: p.version,
        status: p.status,
        loaded: p.loadTime,
        errors: p.errors.length
      }));
    });

    // Player debugging
    this.commands.set('players', () => {
      return this.engine.playerManager.getOnlinePlayers().map(p => ({
        name: p.name,
        id: p.id,
        room: p.getCurrentRoom()?.id,
        connected: Date.now() - p.connectionTime
      }));
    });

    // Performance debugging
    this.commands.set('perf', () => {
      return {
        memory: process.memoryUsage(),
        cpu: process.cpuUsage(),
        uptime: process.uptime(),
        eventLoop: this.getEventLoopLag()
      };
    });

    // Command debugging
    this.commands.set('commands', (pattern?: string) => {
      const commands = this.engine.commandRegistry.getAll();
      const filtered = pattern ? 
        commands.filter(c => c.name.includes(pattern)) : 
        commands;
      
      return filtered.map(c => ({
        name: c.name,
        category: c.category,
        executions: c.executionCount,
        errors: c.errorCount,
        avgTime: c.averageExecutionTime
      }));
    });

    // Hot-reload debugging
    this.commands.set('hotreload', () => {
      const status = this.engine.hotReload.getStatus();
      const history = this.engine.hotReload.getHistory(5);
      
      return {
        active: status.active,
        reloads: status.totalReloads,
        failures: status.failedReloads,
        recentHistory: history
      };
    });
  }

  async execute(command: string, ...args: any[]): Promise<any> {
    const handler = this.commands.get(command);
    if (!handler) {
      throw new Error(`Unknown debug command: ${command}`);
    }

    return await handler(...args);
  }

  getAvailableCommands(): string[] {
    return Array.from(this.commands.keys());
  }
}

// Usage in REPL
const console = new DebugConsole(engine);
await console.execute('plugins');
await console.execute('commands', 'attack');
```

### Breakpoint System

```typescript
// Conditional breakpoints
export class Breakpoints {
  private static breakpoints = new Map<string, Function>();

  static set(id: string, condition: () => boolean): void {
    this.breakpoints.set(id, condition);
  }

  static check(id: string, context?: any): boolean {
    const condition = this.breakpoints.get(id);
    if (!condition) return false;

    if (condition()) {
      console.log(`Breakpoint hit: ${id}`, context);
      debugger; // Trigger debugger
      return true;
    }

    return false;
  }

  static remove(id: string): void {
    this.breakpoints.delete(id);
  }
}

// Usage in code
export class CommandProcessor {
  async execute(player: Player, input: string): Promise<CommandResult> {
    // Conditional breakpoint
    Breakpoints.check('command-execute', {
      player: player.name,
      command: input,
      room: player.getCurrentRoom()?.id
    });

    // Set breakpoint condition
    Breakpoints.set('slow-command', () => {
      return Date.now() - startTime > 1000; // Break on slow commands
    });

    const result = await this.processCommand(player, input);
    
    Breakpoints.check('slow-command');
    
    return result;
  }
}
```

## Performance Profiling

### CPU Profiling

```typescript
import { createRequire } from 'module';
const require = createRequire(import.meta.url);
const v8Profiler = require('v8-profiler-next');

export class CPUProfiler {
  private profiling = false;
  private startTime = 0;

  start(title: string): void {
    if (this.profiling) {
      throw new Error('Profiler already running');
    }

    this.profiling = true;
    this.startTime = Date.now();
    v8Profiler.startProfiling(title);
  }

  stop(title: string): any {
    if (!this.profiling) {
      throw new Error('Profiler not running');
    }

    const profile = v8Profiler.stopProfiling(title);
    this.profiling = false;

    const duration = Date.now() - this.startTime;
    
    return {
      duration,
      profile: profile.export(),
      summary: this.analyzeProfil(profile)
    };
  }

  private analyzeProfil(profile: any): any {
    // Analyze CPU profile data
    const nodes = profile.nodes || [];
    const samples = profile.samples || [];
    
    // Find hotspots
    const hotspots = nodes
      .filter(node => node.hitCount > 0)
      .sort((a, b) => b.hitCount - a.hitCount)
      .slice(0, 10);

    return {
      totalSamples: samples.length,
      hotspots: hotspots.map(node => ({
        function: node.callFrame.functionName,
        file: node.callFrame.url,
        line: node.callFrame.lineNumber,
        hits: node.hitCount
      }))
    };
  }
}

// Usage
const profiler = new CPUProfiler();

profiler.start('command-processing');
await processCommands();
const results = profiler.stop('command-processing');

console.log('Profiling Results:', results.summary);
```

### Memory Profiling

```typescript
export class MemoryProfiler {
  private snapshots: any[] = [];

  takeSnapshot(label: string): void {
    const snapshot = {
      label,
      timestamp: Date.now(),
      memory: process.memoryUsage(),
      heap: v8.getHeapSnapshot ? v8.getHeapSnapshot() : null
    };

    this.snapshots.push(snapshot);
  }

  compareSnapshots(before: string, after: string): any {
    const beforeSnapshot = this.snapshots.find(s => s.label === before);
    const afterSnapshot = this.snapshots.find(s => s.label === after);

    if (!beforeSnapshot || !afterSnapshot) {
      throw new Error('Snapshots not found');
    }

    const memoryDiff = {
      rss: afterSnapshot.memory.rss - beforeSnapshot.memory.rss,
      heapUsed: afterSnapshot.memory.heapUsed - beforeSnapshot.memory.heapUsed,
      heapTotal: afterSnapshot.memory.heapTotal - beforeSnapshot.memory.heapTotal,
      external: afterSnapshot.memory.external - beforeSnapshot.memory.external
    };

    return {
      duration: afterSnapshot.timestamp - beforeSnapshot.timestamp,
      memoryDiff,
      leakSuspected: memoryDiff.heapUsed > 10 * 1024 * 1024 // 10MB threshold
    };
  }

  detectLeaks(): any[] {
    if (this.snapshots.length < 2) {
      return [];
    }

    const leaks = [];
    
    for (let i = 1; i < this.snapshots.length; i++) {
      const comparison = this.compareSnapshots(
        this.snapshots[i-1].label,
        this.snapshots[i].label
      );

      if (comparison.leakSuspected) {
        leaks.push({
          from: this.snapshots[i-1].label,
          to: this.snapshots[i].label,
          ...comparison
        });
      }
    }

    return leaks;
  }
}

// Usage
const memProfiler = new MemoryProfiler();

memProfiler.takeSnapshot('before-plugin-load');
await loadPlugins();
memProfiler.takeSnapshot('after-plugin-load');

const comparison = memProfiler.compareSnapshots('before-plugin-load', 'after-plugin-load');
console.log('Memory impact of plugin loading:', comparison);
```

### Performance Metrics

```typescript
export class PerformanceMonitor {
  private metrics = new Map<string, any>();
  private timers = new Map<string, number>();

  startTimer(name: string): void {
    this.timers.set(name, Date.now());
  }

  endTimer(name: string): number {
    const startTime = this.timers.get(name);
    if (!startTime) {
      throw new Error(`Timer '${name}' not found`);
    }

    const duration = Date.now() - startTime;
    this.timers.delete(name);

    this.recordMetric(name, duration);
    return duration;
  }

  recordMetric(name: string, value: number): void {
    if (!this.metrics.has(name)) {
      this.metrics.set(name, {
        count: 0,
        sum: 0,
        min: Infinity,
        max: -Infinity,
        avg: 0
      });
    }

    const metric = this.metrics.get(name);
    metric.count++;
    metric.sum += value;
    metric.min = Math.min(metric.min, value);
    metric.max = Math.max(metric.max, value);
    metric.avg = metric.sum / metric.count;
  }

  getMetric(name: string): any {
    return this.metrics.get(name) || null;
  }

  getAllMetrics(): any {
    return Object.fromEntries(this.metrics);
  }

  reset(name?: string): void {
    if (name) {
      this.metrics.delete(name);
    } else {
      this.metrics.clear();
    }
  }
}

// Usage with decorators
function monitor(metricName?: string) {
  return function(target: any, propertyKey: string, descriptor: PropertyDescriptor) {
    const originalMethod = descriptor.value;
    const name = metricName || `${target.constructor.name}.${propertyKey}`;

    descriptor.value = async function(...args: any[]) {
      const monitor = PerformanceMonitor.getInstance();
      monitor.startTimer(name);

      try {
        const result = await originalMethod.apply(this, args);
        return result;
      } finally {
        monitor.endTimer(name);
      }
    };

    return descriptor;
  };
}

// Usage
export class CommandProcessor {
  @monitor('command-processing')
  async processCommand(player: Player, input: string): Promise<CommandResult> {
    // Command processing logic
  }
}
```

## Testing Strategies

### Test Structure

```
tests/
├── unit/                  # Unit tests
│   ├── core/             # Core engine tests
│   ├── plugins/          # Plugin tests
│   ├── commands/         # Command tests
│   └── utils/            # Utility tests
├── integration/          # Integration tests
│   ├── plugin-system/   # Plugin system integration
│   ├── hot-reload/      # Hot-reload integration
│   └── protocols/       # Protocol integration
├── e2e/                 # End-to-end tests
│   ├── gameplay/        # Gameplay scenarios
│   ├── admin/           # Admin functionality
│   └── load/            # Load testing
├── fixtures/            # Test data and mocks
├── helpers/             # Test utilities
└── performance/         # Performance tests
```

### Test Configuration

```json
// jest.config.js
{
  "preset": "ts-jest",
  "testEnvironment": "node",
  "setupFilesAfterEnv": ["<rootDir>/tests/setup.ts"],
  "testMatch": [
    "**/tests/**/*.test.ts",
    "**/src/**/__tests__/**/*.test.ts"
  ],
  "collectCoverageFrom": [
    "src/**/*.ts",
    "!src/**/*.d.ts",
    "!src/**/__tests__/**",
    "!src/**/index.ts"
  ],
  "coverageThreshold": {
    "global": {
      "branches": 80,
      "functions": 80,
      "lines": 80,
      "statements": 80
    }
  },
  "testTimeout": 30000,
  "maxWorkers": 4
}
```

## Unit Testing

### Plugin Testing

```typescript
// tests/unit/plugins/example-plugin.test.ts
import { ExamplePlugin } from '../../../src/plugins/ExamplePlugin.js';
import { createMockContext, createMockPlayer } from '../../helpers/mocks.js';

describe('ExamplePlugin', () => {
  let plugin: ExamplePlugin;
  let mockContext: MockPluginContext;

  beforeEach(() => {
    plugin = new ExamplePlugin();
    mockContext = createMockContext();
  });

  afterEach(() => {
    jest.clearAllMocks();
  });

  describe('Plugin Lifecycle', () => {
    test('should load successfully', async () => {
      await plugin.onLoad(mockContext);
      
      expect(mockContext.logger.info).toHaveBeenCalledWith(
        expect.stringContaining('loaded successfully')
      );
      expect(mockContext.engine.commandRegistry.register).toHaveBeenCalled();
    });

    test('should handle load errors gracefully', async () => {
      mockContext.engine.commandRegistry.register.mockRejectedValue(
        new Error('Registration failed')
      );

      await expect(plugin.onLoad(mockContext)).rejects.toThrow('Registration failed');
    });

    test('should unload cleanly', async () => {
      await plugin.onLoad(mockContext);
      await plugin.onUnload(mockContext);
      
      expect(mockContext.engine.commandRegistry.unregister).toHaveBeenCalled();
      expect(mockContext.eventBus.off).toHaveBeenCalled();
    });
  });

  describe('Command Handling', () => {
    beforeEach(async () => {
      await plugin.onLoad(mockContext);
    });

    test('should execute example command', async () => {
      const player = createMockPlayer('TestPlayer');
      const result = await plugin.exampleCommand(player, ['test', 'args']);

      expect(result.success).toBe(true);
      expect(result.message).toContain('TestPlayer');
      expect(player.send).toHaveBeenCalled();
    });

    test('should handle command errors', async () => {
      const player = createMockPlayer('TestPlayer');
      player.getCurrentRoom.mockReturnValue(null);

      const result = await plugin.exampleCommand(player, []);
      
      expect(result.success).toBe(false);
      expect(result.message).toContain('error');
    });
  });

  describe('Event Handling', () => {
    test('should handle player login events', async () => {
      await plugin.onLoad(mockContext);
      
      const player = createMockPlayer('TestPlayer');
      const loginHandler = mockContext.eventBus.on.mock.calls
        .find(call => call[0] === 'player:login')[1];
      
      await loginHandler(player);
      
      // Verify welcome message sent after delay
      await new Promise(resolve => setTimeout(resolve, 1100));
      expect(player.send).toHaveBeenCalledWith(
        expect.stringContaining('Welcome')
      );
    });
  });
});
```

### Command Testing

```typescript
// tests/unit/commands/take.test.ts
import { takeCommand } from '../../../src/commands/basic-commands.js';
import { createMockPlayer, createMockRoom, createMockItem } from '../../helpers/mocks.js';

describe('takeCommand', () => {
  let player: MockPlayer;
  let room: MockRoom;
  let context: MockCommandContext;

  beforeEach(() => {
    player = createMockPlayer('TestPlayer');
    room = createMockRoom('test-room');
    context = createMockContext(player, room);
  });

  test('should take item from room', async () => {
    const sword = createMockItem('iron sword');
    room.findItem.mockReturnValue(sword);
    player.canCarry.mockReturnValue(true);

    const result = await takeCommand.handler(player, ['sword'], context);

    expect(result.success).toBe(true);
    expect(room.removeItem).toHaveBeenCalledWith(sword, 1);
    expect(player.addItem).toHaveBeenCalledWith(sword, 1);
    expect(room.broadcast).toHaveBeenCalled();
  });

  test('should handle item not found', async () => {
    room.findItem.mockReturnValue(null);

    const result = await takeCommand.handler(player, ['nonexistent'], context);

    expect(result.success).toBe(false);
    expect(result.message).toContain("don't see");
  });

  test('should handle untakeable items', async () => {
    const fixture = createMockItem('heavy boulder');
    fixture.canTake = false;
    room.findItem.mockReturnValue(fixture);

    const result = await takeCommand.handler(player, ['boulder'], context);

    expect(result.success).toBe(false);
    expect(result.message).toContain("can't take");
  });

  test('should handle carrying capacity', async () => {
    const heavyItem = createMockItem('anvil');
    room.findItem.mockReturnValue(heavyItem);
    player.canCarry.mockReturnValue(false);

    const result = await takeCommand.handler(player, ['anvil'], context);

    expect(result.success).toBe(false);
    expect(result.message).toContain('too heavy');
  });

  test('should handle quantity', async () => {
    const coins = createMockItem('gold coins');
    coins.quantity = 100;
    room.findItem.mockReturnValue(coins);
    player.canCarry.mockReturnValue(true);

    const result = await takeCommand.handler(player, ['coins', '50'], context);

    expect(result.success).toBe(true);
    expect(room.removeItem).toHaveBeenCalledWith(coins, 50);
    expect(player.addItem).toHaveBeenCalledWith(coins, 50);
  });
});
```

## Integration Testing

### Plugin System Integration

```typescript
// tests/integration/plugin-system.test.ts
import { MUDEngine } from '../../src/index.js';
import { ExamplePlugin } from '../../src/plugins/ExamplePlugin.js';

describe('Plugin System Integration', () => {
  let engine: MUDEngine;

  beforeEach(async () => {
    engine = new MUDEngine({
      database: ':memory:',
      protocols: [],
      debug: { enabled: false }
    });
    
    await engine.start();
  });

  afterEach(async () => {
    await engine.stop();
  });

  test('should load and activate plugin', async () => {
    const plugin = new ExamplePlugin();
    
    await engine.pluginManager.loadPlugin(plugin);
    
    expect(engine.pluginManager.isLoaded(plugin.name)).toBe(true);
    expect(engine.pluginManager.isActive(plugin.name)).toBe(true);
    
    // Verify commands were registered
    const exampleCmd = engine.commandRegistry.get('example');
    expect(exampleCmd).toBeDefined();
  });

  test('should handle plugin dependencies', async () => {
    const dependentPlugin = new DependentPlugin();
    dependentPlugin.dependencies = ['example-plugin'];
    
    // Should fail without dependency
    await expect(
      engine.pluginManager.loadPlugin(dependentPlugin)
    ).rejects.toThrow('Missing dependency');
    
    // Should succeed with dependency
    await engine.pluginManager.loadPlugin(new ExamplePlugin());
    await engine.pluginManager.loadPlugin(dependentPlugin);
    
    expect(engine.pluginManager.isLoaded(dependentPlugin.name)).toBe(true);
  });

  test('should unload plugins cleanly', async () => {
    const plugin = new ExamplePlugin();
    await engine.pluginManager.loadPlugin(plugin);
    
    await engine.pluginManager.unloadPlugin(plugin.name);
    
    expect(engine.pluginManager.isLoaded(plugin.name)).toBe(false);
    expect(engine.commandRegistry.get('example')).toBeUndefined();
  });
});
```

### Hot-Reload Integration

```typescript
// tests/integration/hot-reload.test.ts
import { promises as fs } from 'fs';
import { join } from 'path';
import { MUDEngine } from '../../src/index.js';

describe('Hot-Reload Integration', () => {
  let engine: MUDEngine;
  let tempDir: string;

  beforeEach(async () => {
    tempDir = join(tmpdir(), `hotreload-test-${Date.now()}`);
    await fs.mkdir(tempDir, { recursive: true });

    engine = new MUDEngine({
      hotReload: {
        enabled: true,
        watchPaths: [tempDir],
        debounceMs: 100
      }
    });

    await engine.start();
  });

  afterEach(async () => {
    await engine.stop();
    await fs.rm(tempDir, { recursive: true, force: true });
  });

  test('should reload on file changes', async () => {
    const testFile = join(tempDir, 'test.ts');
    const originalContent = 'export const test = "original";';
    
    await fs.writeFile(testFile, originalContent);
    
    // Wait for initial compilation
    await waitForReload(engine, 1);
    
    // Modify file
    await fs.writeFile(testFile, 'export const test = "modified";');
    
    // Wait for reload
    await waitForReload(engine, 2);
    
    const status = engine.hotReload.getStatus();
    expect(status.totalReloads).toBe(2);
    expect(status.successfulReloads).toBe(2);
  });

  test('should handle compilation errors', async () => {
    const testFile = join(tempDir, 'error.ts');
    const invalidContent = 'export const test = "unclosed string';
    
    await fs.writeFile(testFile, invalidContent);
    
    await waitForReload(engine, 1);
    
    const status = engine.hotReload.getStatus();
    expect(status.failedReloads).toBe(1);
  });

  test('should preserve state during reload', async () => {
    // Create plugin with state
    const pluginFile = join(tempDir, 'stateful-plugin.ts');
    const pluginContent = `
      export class StatefulPlugin {
        private counter = 0;
        
        increment() { this.counter++; }
        getCounter() { return this.counter; }
      }
    `;
    
    await fs.writeFile(pluginFile, pluginContent);
    await waitForReload(engine, 1);
    
    // Interact with plugin to create state
    const plugin = engine.pluginManager.get('stateful-plugin');
    plugin.increment();
    plugin.increment();
    
    // Modify plugin file
    const modifiedContent = pluginContent.replace('counter++', 'counter += 2');
    await fs.writeFile(pluginFile, modifiedContent);
    await waitForReload(engine, 2);
    
    // State should be preserved
    const reloadedPlugin = engine.pluginManager.get('stateful-plugin');
    expect(reloadedPlugin.getCounter()).toBe(2);
  });
});

async function waitForReload(engine: MUDEngine, expectedReloads: number): Promise<void> {
  return new Promise((resolve) => {
    const checkStatus = () => {
      const status = engine.hotReload.getStatus();
      if (status.totalReloads >= expectedReloads) {
        resolve();
      } else {
        setTimeout(checkStatus, 50);
      }
    };
    
    checkStatus();
  });
}
```

## Load Testing

### Connection Load Testing

```typescript
// tests/performance/load.test.ts
import WebSocket from 'ws';
import { MUDEngine } from '../../src/index.js';

describe('Load Testing', () => {
  let engine: MUDEngine;

  beforeAll(async () => {
    engine = new MUDEngine({
      protocols: {
        websocket: { port: 8081, enabled: true }
      }
    });
    
    await engine.start();
  });

  afterAll(async () => {
    await engine.stop();
  });

  test('should handle multiple concurrent connections', async () => {
    const connectionCount = 100;
    const connections: WebSocket[] = [];
    
    // Create connections
    for (let i = 0; i < connectionCount; i++) {
      const ws = new WebSocket('ws://localhost:8081/ws');
      connections.push(ws);
    }

    // Wait for all connections to open
    await Promise.all(
      connections.map(ws => new Promise(resolve => {
        ws.on('open', resolve);
      }))
    );

    expect(connections.length).toBe(connectionCount);
    expect(engine.protocolManager.getConnectionCount()).toBe(connectionCount);

    // Clean up
    connections.forEach(ws => ws.close());
  }, 30000);

  test('should handle high command throughput', async () => {
    const commandCount = 1000;
    const commands = Array.from({ length: commandCount }, (_, i) => `test ${i}`);
    
    const player = await engine.playerManager.createTestPlayer('LoadTester');
    
    const startTime = Date.now();
    
    const results = await Promise.all(
      commands.map(cmd => engine.commandProcessor.execute(player, cmd))
    );
    
    const duration = Date.now() - startTime;
    const commandsPerSecond = commandCount / (duration / 1000);
    
    console.log(`Processed ${commandCount} commands in ${duration}ms (${commandsPerSecond.toFixed(2)} cmd/s)`);
    
    expect(results.every(r => r.success)).toBe(true);
    expect(commandsPerSecond).toBeGreaterThan(100); // At least 100 commands per second
  });

  test('should maintain performance under memory pressure', async () => {
    const iterations = 10;
    const itemsPerIteration = 1000;
    
    for (let i = 0; i < iterations; i++) {
      // Create many objects to stress memory
      const items = Array.from({ length: itemsPerIteration }, (_, j) => 
        engine.itemManager.createItem({ name: `item_${i}_${j}`, type: 'test' })
      );
      
      const startTime = Date.now();
      
      // Perform operations
      const player = await engine.playerManager.createTestPlayer(`Tester${i}`);
      items.forEach(item => player.addItem(item));
      
      const operationTime = Date.now() - startTime;
      
      console.log(`Iteration ${i + 1}: ${operationTime}ms for ${itemsPerIteration} operations`);
      
      // Performance shouldn't degrade significantly
      expect(operationTime).toBeLessThan(5000); // 5 seconds max
      
      // Clean up
      await engine.playerManager.removePlayer(player.id);
    }
  });
});
```

## Debugging Plugins

### Plugin Debug Utilities

```typescript
// src/debug/PluginDebugger.ts
export class PluginDebugger {
  private breakpoints = new Map<string, Function>();
  private watchedProperties = new Map<string, any>();

  constructor(private plugin: Plugin) {
    this.setupDebugging();
  }

  private setupDebugging(): void {
    // Wrap plugin methods with debugging
    const methods = Object.getOwnPropertyNames(Object.getPrototypeOf(this.plugin))
      .filter(name => typeof (this.plugin as any)[name] === 'function');

    for (const method of methods) {
      this.wrapMethod(method);
    }
  }

  private wrapMethod(methodName: string): void {
    const originalMethod = (this.plugin as any)[methodName];
    
    (this.plugin as any)[methodName] = async (...args: any[]) => {
      console.log(`[${this.plugin.name}] Calling ${methodName}`, args);
      
      const startTime = Date.now();
      
      try {
        const result = await originalMethod.apply(this.plugin, args);
        const duration = Date.now() - startTime;
        
        console.log(`[${this.plugin.name}] ${methodName} completed in ${duration}ms`, result);
        
        return result;
      } catch (error) {
        console.error(`[${this.plugin.name}] ${methodName} failed:`, error);
        throw error;
      }
    };
  }

  setBreakpoint(condition: string, callback?: Function): void {
    this.breakpoints.set(condition, callback || (() => debugger));
  }

  watchProperty(propertyPath: string): void {
    const obj = this.resolvePropertyPath(propertyPath);
    const property = propertyPath.split('.').pop()!;
    
    let value = obj[property];
    
    Object.defineProperty(obj, property, {
      get: () => value,
      set: (newValue) => {
        console.log(`[${this.plugin.name}] Property ${propertyPath} changed:`, value, '->', newValue);
        value = newValue;
      }
    });
  }

  private resolvePropertyPath(path: string): any {
    return path.split('.').reduce((obj, prop) => obj[prop], this.plugin);
  }

  getState(): any {
    return {
      name: this.plugin.name,
      version: this.plugin.version,
      status: this.plugin.status,
      properties: this.getPublicProperties(),
      breakpoints: Array.from(this.breakpoints.keys()),
      watchedProperties: Array.from(this.watchedProperties.keys())
    };
  }

  private getPublicProperties(): any {
    const properties: any = {};
    
    for (const key in this.plugin) {
      if (!key.startsWith('_') && typeof (this.plugin as any)[key] !== 'function') {
        properties[key] = (this.plugin as any)[key];
      }
    }
    
    return properties;
  }
}

// Usage
const debugger = new PluginDebugger(myPlugin);
debugger.setBreakpoint('player.name === "admin"');
debugger.watchProperty('commandCount');
```

## Common Issues

### Memory Leaks

```typescript
// Debug memory leaks in plugins
export class MemoryLeakDetector {
  private snapshots: any[] = [];
  private intervals: NodeJS.Timeout[] = [];

  startMonitoring(plugin: Plugin): void {
    const interval = setInterval(() => {
      const snapshot = this.takeSnapshot(plugin);
      this.snapshots.push(snapshot);
      
      // Keep only last 10 snapshots
      if (this.snapshots.length > 10) {
        this.snapshots.shift();
      }
      
      // Check for leaks
      if (this.snapshots.length > 3) {
        this.analyzeLeaks(plugin);
      }
    }, 30000); // Every 30 seconds
    
    this.intervals.push(interval);
  }

  private takeSnapshot(plugin: Plugin): any {
    return {
      timestamp: Date.now(),
      memory: process.memoryUsage(),
      pluginState: this.getPluginMemoryUsage(plugin),
      eventListeners: this.countEventListeners(plugin)
    };
  }

  private getPluginMemoryUsage(plugin: Plugin): any {
    // Estimate plugin memory usage
    const state = JSON.stringify(plugin);
    return {
      stateSize: state.length,
      objectCount: (state.match(/\{/g) || []).length
    };
  }

  private countEventListeners(plugin: Plugin): number {
    // Count event listeners if plugin uses EventEmitter
    if (plugin instanceof EventEmitter) {
      return plugin.listenerCount();
    }
    return 0;
  }

  private analyzeLeaks(plugin: Plugin): void {
    const recent = this.snapshots.slice(-3);
    const trend = recent.map(s => s.memory.heapUsed);
    
    // Check if memory is consistently growing
    if (trend[2] > trend[1] && trend[1] > trend[0]) {
      const growth = trend[2] - trend[0];
      if (growth > 10 * 1024 * 1024) { // 10MB growth
        console.warn(`Potential memory leak in plugin ${plugin.name}:`, {
          growth: `${Math.round(growth / 1024 / 1024)}MB`,
          trend
        });
      }
    }
  }
}
```

### Performance Issues

```typescript
// Debug performance bottlenecks
export class PerformanceAnalyzer {
  private measurements: Map<string, number[]> = new Map();

  measure<T>(name: string, fn: () => Promise<T>): Promise<T> {
    const start = Date.now();
    
    return fn().then(result => {
      const duration = Date.now() - start;
      this.recordMeasurement(name, duration);
      return result;
    });
  }

  private recordMeasurement(name: string, duration: number): void {
    if (!this.measurements.has(name)) {
      this.measurements.set(name, []);
    }
    
    const measurements = this.measurements.get(name)!;
    measurements.push(duration);
    
    // Keep only recent measurements
    if (measurements.length > 100) {
      measurements.shift();
    }
  }

  analyzeBottlenecks(): any[] {
    const bottlenecks = [];
    
    for (const [name, measurements] of this.measurements) {
      const avg = measurements.reduce((a, b) => a + b, 0) / measurements.length;
      const max = Math.max(...measurements);
      const p95 = this.percentile(measurements, 95);
      
      if (avg > 100 || max > 1000) { // Slow operations
        bottlenecks.push({
          name,
          average: Math.round(avg),
          maximum: max,
          p95: Math.round(p95),
          samples: measurements.length
        });
      }
    }
    
    return bottlenecks.sort((a, b) => b.average - a.average);
  }

  private percentile(values: number[], p: number): number {
    const sorted = values.slice().sort((a, b) => a - b);
    const index = Math.ceil((p / 100) * sorted.length) - 1;
    return sorted[index];
  }
}
```

## Troubleshooting Tools

### Health Check System

```typescript
// Comprehensive system health checking
export class HealthChecker {
  private checks: Map<string, Function> = new Map();

  constructor(private engine: MUDEngine) {
    this.registerDefaultChecks();
  }

  private registerDefaultChecks(): void {
    // Database connectivity
    this.checks.set('database', async () => {
      try {
        await this.engine.database.query('SELECT 1');
        return { status: 'healthy', message: 'Database connected' };
      } catch (error) {
        return { status: 'unhealthy', message: `Database error: ${error.message}` };
      }
    });

    // Plugin health
    this.checks.set('plugins', async () => {
      const plugins = this.engine.pluginManager.getAll();
      const unhealthy = plugins.filter(p => p.status === 'error');
      
      if (unhealthy.length > 0) {
        return {
          status: 'unhealthy',
          message: `${unhealthy.length} plugins in error state`,
          details: unhealthy.map(p => p.name)
        };
      }
      
      return { status: 'healthy', message: `${plugins.length} plugins loaded` };
    });

    // Memory usage
    this.checks.set('memory', async () => {
      const usage = process.memoryUsage();
      const usedMB = Math.round(usage.heapUsed / 1024 / 1024);
      const totalMB = Math.round(usage.heapTotal / 1024 / 1024);
      
      if (usedMB > 512) { // 512MB threshold
        return {
          status: 'warning',
          message: `High memory usage: ${usedMB}/${totalMB}MB`
        };
      }
      
      return { status: 'healthy', message: `Memory usage: ${usedMB}/${totalMB}MB` };
    });

    // Event loop lag
    this.checks.set('eventloop', async () => {
      const lag = await this.measureEventLoopLag();
      
      if (lag > 100) { // 100ms threshold
        return { status: 'warning', message: `Event loop lag: ${lag}ms` };
      }
      
      return { status: 'healthy', message: `Event loop lag: ${lag}ms` };
    });
  }

  async runAllChecks(): Promise<any> {
    const results: any = {};
    
    for (const [name, check] of this.checks) {
      try {
        results[name] = await check();
      } catch (error) {
        results[name] = {
          status: 'error',
          message: `Health check failed: ${error.message}`
        };
      }
    }

    const overallStatus = this.determineOverallStatus(results);
    
    return {
      status: overallStatus,
      timestamp: new Date().toISOString(),
      checks: results
    };
  }

  private determineOverallStatus(results: any): string {
    const statuses = Object.values(results).map((r: any) => r.status);
    
    if (statuses.includes('error') || statuses.includes('unhealthy')) {
      return 'unhealthy';
    }
    
    if (statuses.includes('warning')) {
      return 'warning';
    }
    
    return 'healthy';
  }

  private async measureEventLoopLag(): Promise<number> {
    return new Promise(resolve => {
      const start = Date.now();
      setImmediate(() => {
        resolve(Date.now() - start);
      });
    });
  }
}
```

This comprehensive debugging and testing guide provides all the tools and techniques needed to maintain a robust, well-tested MUD engine. The examples progress from basic debugging techniques to advanced performance analysis and production monitoring.