@push.rocks/smartproxy/ts/proxies/smart-proxy/smart-proxy.ts

A powerful proxy package with unified route-based configuration for high traffic management. Features include SSL/TLS support, flexible routing patterns, WebSocket handling, advanced security options, and automatic ACME certificate management.

import * as plugins from '../../plugins.js';
import { logger } from '../../core/utils/logger.js';

// Rust bridge and helpers
import { RustProxyBridge } from './rust-proxy-bridge.js';
import { RoutePreprocessor } from './route-preprocessor.js';
import { SocketHandlerServer } from './socket-handler-server.js';
import { DatagramHandlerServer } from './datagram-handler-server.js';
import { ChallengeProviderRelayServer } from './challenge-provider-relay-server.js';
import { RustMetricsAdapter } from './rust-metrics-adapter.js';

// Route management
import { SharedRouteManager as RouteManager } from '../../core/routing/route-manager.js';
import { RouteValidator } from './utils/route-validator.js';
import { buildRustProxyOptions } from './utils/rust-config.js';
import { generateDefaultCertificate } from './utils/default-cert-generator.js';
import { Mutex } from './utils/mutex.js';
import { ConcurrencySemaphore } from './utils/concurrency-semaphore.js';

// Types
import type { ISmartProxyOptions, ISmartProxySecurityPolicy, TSmartProxyCertProvisionObject, IAcmeOptions, ICertProvisionEventComms, ICertificateIssuedEvent, ICertificateFailedEvent, ISmartProxyCertProvisionCertificate, IActiveConnectionSnapshot, IActiveConnectionSnapshotOptions } from './models/interfaces.js';
import type { IRouteConfig } from './models/route-types.js';
import type { IMetrics } from './models/metrics-types.js';
import type { IRustCertificateStatus, IRustChallengeOptions, IRustProxyOptions, IRustStatistics } from './models/rust-types.js';

type TChallengeProvider = plugins.smartchallenge.IChallengeProvider;

/**
 * SmartProxy - Rust-backed proxy engine with TypeScript configuration API.
 *
 * All networking (TCP, TLS, HTTP reverse proxy, connection management, security)
 * is handled by the Rust binary. TypeScript is only:
 * - The npm module interface (types, route helpers)
 * - The thin IPC wrapper (this class)
 * - Socket-handler callback relay (for JS-defined handlers)
 * - Certificate provisioning callbacks (certProvisionFunction)
 */
export class SmartProxy extends plugins.EventEmitter {
  public settings: ISmartProxyOptions;
  public routeManager: RouteManager;

  private bridge: RustProxyBridge;
  private preprocessor: RoutePreprocessor;
  private socketHandlerServer: SocketHandlerServer | null = null;
  private datagramHandlerServer: DatagramHandlerServer | null = null;
  private challengeProviderRelayServer: ChallengeProviderRelayServer | null = null;
  private challengeProviders = new Map<string, TChallengeProvider>();
  private challengeRuntimeOptions?: IRustChallengeOptions;
  private metricsAdapter: RustMetricsAdapter;
  private nftablesManager: InstanceType<typeof plugins.smartnftables.SmartNftables> | null = null;
  private routeUpdateLock: Mutex;
  private stopping = false;
  private certProvisionPromise: Promise<void> | null = null;

  constructor(settingsArg: ISmartProxyOptions) {
    super();

    // Apply defaults
    this.settings = {
      ...settingsArg,
      initialDataTimeout: settingsArg.initialDataTimeout || 60_000,
      socketTimeout: settingsArg.socketTimeout || 60_000,
      maxConnectionLifetime: settingsArg.maxConnectionLifetime || 3_600_000,
      inactivityTimeout: settingsArg.inactivityTimeout || 75_000,
      gracefulShutdownTimeout: settingsArg.gracefulShutdownTimeout || 30_000,
      maxConnectionsPerIP: settingsArg.maxConnectionsPerIP || 100,
      connectionRateLimitPerMinute: settingsArg.connectionRateLimitPerMinute || 300,
      keepAliveTreatment: settingsArg.keepAliveTreatment || 'standard',
      keepAliveInactivityMultiplier: settingsArg.keepAliveInactivityMultiplier || 4,
      extendedKeepAliveLifetime: settingsArg.extendedKeepAliveLifetime || 3_600_000,
    };

    // Normalize ACME options
    if (this.settings.acme) {
      if (this.settings.acme.accountEmail && !this.settings.acme.email) {
        this.settings.acme.email = this.settings.acme.accountEmail;
      }
      this.settings.acme = {
        enabled: this.settings.acme.enabled !== false,
        port: this.settings.acme.port || 80,
        email: this.settings.acme.email,
        useProduction: this.settings.acme.useProduction || false,
        renewThresholdDays: this.settings.acme.renewThresholdDays || 30,
        autoRenew: this.settings.acme.autoRenew !== false,
        skipConfiguredCerts: this.settings.acme.skipConfiguredCerts || false,
        renewCheckIntervalHours: this.settings.acme.renewCheckIntervalHours || 24,
        routeForwards: this.settings.acme.routeForwards || [],
        ...this.settings.acme,
      };
    }

    // Validate routes
    if (this.settings.routes?.length) {
      const validation = RouteValidator.validateRoutes(this.settings.routes);
      if (!validation.valid) {
        RouteValidator.logValidationErrors(validation.errors);
        throw new Error(`Initial route validation failed: ${validation.errors.size} route(s) have errors`);
      }
    }

    // Create logger adapter
    const loggerAdapter = {
      debug: (message: string, data?: any) => logger.log('debug', message, data),
      info: (message: string, data?: any) => logger.log('info', message, data),
      warn: (message: string, data?: any) => logger.log('warn', message, data),
      error: (message: string, data?: any) => logger.log('error', message, data),
    };

    // Initialize components
    this.routeManager = new RouteManager({
      logger: loggerAdapter,
      enableDetailedLogging: this.settings.enableDetailedLogging,
      routes: this.settings.routes,
    });

    this.bridge = new RustProxyBridge();
    this.preprocessor = new RoutePreprocessor();
    this.metricsAdapter = new RustMetricsAdapter(
      this.bridge,
      this.settings.metrics?.sampleIntervalMs ?? 1000
    );
    this.routeUpdateLock = new Mutex();
  }

  /**
   * Register a runtime challenge provider family. Routes reference providerId + challengeType;
   * deployment wiring and provider secrets stay outside route configs.
   */
  public registerChallengeProvider(providerId: string, provider: TChallengeProvider): void {
    if (!providerId || typeof providerId !== 'string') {
      throw new Error('Challenge providerId must be a non-empty string');
    }
    this.challengeProviders.set(providerId, provider);
  }

  /**
   * Start the proxy.
   * Spawns the Rust binary, configures socket relay if needed, sends routes, handles cert provisioning.
   */
  public async start(): Promise<void> {
    await this.validateChallengeRoutes(this.settings.routes);

    let didSpawn = false;
    try {
      // Spawn Rust binary
      const spawned = await this.bridge.spawn();
      if (!spawned) {
        throw new Error(
          'RustProxy binary not found. Set SMARTPROXY_RUST_BINARY env var, install the platform package, ' +
          'or build locally with: pnpm build'
        );
      }
      didSpawn = true;

      // Handle unexpected exit (only emits error if not intentionally stopping)
      this.bridge.removeAllListeners('exit');
      this.bridge.on('exit', (code: number | null, signal: string | null) => {
        if (this.stopping) return;
        logger.log('error', `RustProxy exited unexpectedly (code=${code}, signal=${signal})`, { component: 'smart-proxy' });
        this.emit('error', new Error(`RustProxy exited (code=${code}, signal=${signal})`));
      });

      const hasChallengeRoutes = this.hasChallengeRoutes(this.settings.routes);
      if (hasChallengeRoutes) {
        await this.ensureChallengeProviderRelay();
      }

      // Check if any routes need TS-side handling (socket handlers, dynamic functions)
      const hasHandlerRoutes = this.settings.routes.some(
        (r) =>
          (r.action.type === 'socket-handler' && r.action.socketHandler) ||
          r.action.targets?.some((t) => typeof t.host === 'function' || typeof t.port === 'function')
      );

      // Start socket handler relay server (but don't tell Rust yet - proxy not started)
      if (hasHandlerRoutes) {
        this.socketHandlerServer = new SocketHandlerServer(this.preprocessor);
        await this.socketHandlerServer.start();
      }

      // Check if any routes need datagram handler relay (UDP socket-handler routes)
      const hasDatagramHandlers = this.settings.routes.some(
        (r) => r.action.type === 'socket-handler' && r.action.datagramHandler
      );
      if (hasDatagramHandlers) {
        const dgPath = `/tmp/smartproxy-dgram-relay-${process.pid}.sock`;
        this.datagramHandlerServer = new DatagramHandlerServer(dgPath, this.preprocessor);
        await this.datagramHandlerServer.start();
      }

      // Preprocess routes (strip JS functions, convert socket-handler routes)
      const rustRoutes = this.preprocessor.preprocessForRust(this.settings.routes);

      // When certProvisionFunction handles cert provisioning,
      // disable Rust's built-in ACME to prevent race condition.
      let acmeForRust = this.settings.acme;
      if (this.settings.certProvisionFunction && acmeForRust?.enabled) {
        acmeForRust = { ...acmeForRust, enabled: false };
        logger.log('info', 'Rust ACME disabled — certProvisionFunction will handle certificate provisioning', { component: 'smart-proxy' });
      }

      // Build Rust config
      const config = this.buildRustConfig(rustRoutes, acmeForRust);

      // Start the Rust proxy
      await this.bridge.startProxy(config);

      // Now that Rust proxy is running, configure socket handler relay
      if (this.socketHandlerServer) {
        await this.bridge.setSocketHandlerRelay(this.socketHandlerServer.getSocketPath());
      }

      // Configure challenge provider relay. The path is also present in startup config,
      // but this hot setter keeps runtime changes explicit and mirrors other relays.
      if (this.challengeProviderRelayServer) {
        await this.bridge.setChallengeProviderRelay(
          this.challengeProviderRelayServer.getSocketPath(),
          this.challengeRuntimeOptions,
        );
      }

      // Configure datagram handler relay
      if (this.datagramHandlerServer) {
        await this.bridge.setDatagramHandlerRelay(this.datagramHandlerServer.getSocketPath());
      }

      // Load default self-signed fallback certificate (domain: '*')
      if (!this.settings.disableDefaultCert) {
        try {
          const defaultCert = generateDefaultCertificate();
          await this.bridge.loadCertificate('*', defaultCert.cert, defaultCert.key);
          logger.log('info', 'Default self-signed fallback certificate loaded', { component: 'smart-proxy' });
        } catch (err) {
          logger.log('warn', `Failed to generate default certificate: ${(err as Error).message}`, { component: 'smart-proxy' });
        }
      }

      // Load consumer-stored certificates
      const preloadedDomains = new Set<string>();
      if (this.settings.certStore) {
        try {
          const stored = await this.settings.certStore.loadAll();
          for (const entry of stored) {
            await this.bridge.loadCertificate(entry.domain, entry.publicKey, entry.privateKey, entry.ca);
            preloadedDomains.add(entry.domain);
          }
          logger.log('info', `Loaded ${stored.length} certificate(s) from consumer store`, { component: 'smart-proxy' });
        } catch (err) {
          logger.log('warn', `Failed to load certificates from consumer store: ${(err as Error).message}`, { component: 'smart-proxy' });
        }
      }

      // Apply NFTables rules for routes using nftables forwarding engine
      await this.applyNftablesRules(this.settings.routes);

      // Start metrics polling BEFORE cert provisioning — the Rust engine is already
      // running and accepting connections, so metrics should be available immediately.
      // Cert provisioning can hang indefinitely (e.g. DNS-01 ACME timeouts) and must
      // not block metrics collection.
      this.metricsAdapter.startPolling();

      logger.log('info', 'SmartProxy started (Rust engine)', { component: 'smart-proxy' });

      // Fire-and-forget cert provisioning — Rust engine is already running and serving traffic.
      // Events (certificate-issued / certificate-failed) fire independently per domain.
      this.certProvisionPromise = this.provisionCertificatesViaCallback(preloadedDomains)
        .catch((err) => logger.log('error', `Unexpected error in cert provisioning: ${err.message}`, { component: 'smart-proxy' }));
    } catch (err) {
      await this.cleanupRuntimeResourcesAfterStartFailure(didSpawn);
      throw err;
    }
  }

  /**
   * Stop the proxy.
   */
  public async stop(): Promise<void> {
    logger.log('info', 'SmartProxy shutting down...', { component: 'smart-proxy' });
    this.stopping = true;

    // Wait for in-flight cert provisioning to bail out (it checks this.stopping)
    if (this.certProvisionPromise) {
      await this.certProvisionPromise;
      this.certProvisionPromise = null;
    }

    // Clean up NFTables rules
    if (this.nftablesManager) {
      await this.nftablesManager.cleanup();
      this.nftablesManager = null;
    }

    // Stop metrics polling
    this.metricsAdapter.stopPolling();

    // Remove exit listener before killing to avoid spurious error events
    this.bridge.removeAllListeners('exit');

    // Stop Rust proxy
    try {
      await this.bridge.stopProxy();
    } catch {
      // Ignore if already stopped
    }
    this.bridge.kill();

    // Stop socket handler relay
    if (this.socketHandlerServer) {
      await this.socketHandlerServer.stop();
      this.socketHandlerServer = null;
    }

    // Stop datagram handler relay
    if (this.datagramHandlerServer) {
      await this.datagramHandlerServer.stop();
      this.datagramHandlerServer = null;
    }

    // Stop challenge provider relay
    if (this.challengeProviderRelayServer) {
      await this.challengeProviderRelayServer.stop();
      this.challengeProviderRelayServer = null;
    }
    this.challengeRuntimeOptions = undefined;

    logger.log('info', 'SmartProxy shutdown complete.', { component: 'smart-proxy' });
  }

  /**
   * Update routes atomically.
   */
  public async updateRoutes(newRoutes: IRouteConfig[]): Promise<void> {
    await this.routeUpdateLock.runExclusive(async () => {
      await this.validateChallengeRoutes(newRoutes);

      // Validate before starting relays so failed route updates do not leak runtime resources.
      const validation = RouteValidator.validateRoutes(newRoutes);
      if (!validation.valid) {
        RouteValidator.logValidationErrors(validation.errors);
        throw new Error(`Route validation failed: ${validation.errors.size} route(s) have errors`);
      }

      const hasChallengeRoutes = this.hasChallengeRoutes(newRoutes);
      const challengeRelayWasStarted = Boolean(this.challengeProviderRelayServer);

      // Preprocess for Rust
      const rustRoutes = this.preprocessor.preprocessForRust(newRoutes);

      try {
        if (hasChallengeRoutes) {
          await this.ensureChallengeProviderRelay();
          await this.bridge.setChallengeProviderRelay(
            this.challengeProviderRelayServer!.getSocketPath(),
            this.challengeRuntimeOptions,
          );
        }

        // Send to Rust
        await this.bridge.updateRoutes(rustRoutes);
      } catch (err) {
        if (hasChallengeRoutes && !challengeRelayWasStarted && this.challengeProviderRelayServer) {
          await this.bridge.setChallengeProviderRelay('').catch(() => undefined);
          await this.challengeProviderRelayServer.stop();
          this.challengeProviderRelayServer = null;
          this.challengeRuntimeOptions = undefined;
        }
        throw err;
      }

      // Update local route manager
      this.routeManager.updateRoutes(newRoutes);

      // Update socket handler relay if handler routes changed
      const hasHandlerRoutes = newRoutes.some(
        (r) =>
          (r.action.type === 'socket-handler' && r.action.socketHandler) ||
          r.action.targets?.some((t) => typeof t.host === 'function' || typeof t.port === 'function')
      );

      if (hasHandlerRoutes && !this.socketHandlerServer) {
        this.socketHandlerServer = new SocketHandlerServer(this.preprocessor);
        await this.socketHandlerServer.start();
        await this.bridge.setSocketHandlerRelay(this.socketHandlerServer.getSocketPath());
      } else if (!hasHandlerRoutes && this.socketHandlerServer) {
        await this.socketHandlerServer.stop();
        this.socketHandlerServer = null;
      }

      // Update datagram handler relay if datagram handler routes changed
      const hasDatagramHandlers = newRoutes.some(
        (r) => r.action.type === 'socket-handler' && r.action.datagramHandler
      );

      if (hasDatagramHandlers && !this.datagramHandlerServer) {
        const dgPath = `/tmp/smartproxy-dgram-relay-${process.pid}.sock`;
        this.datagramHandlerServer = new DatagramHandlerServer(dgPath, this.preprocessor);
        await this.datagramHandlerServer.start();
        await this.bridge.setDatagramHandlerRelay(this.datagramHandlerServer.getSocketPath());
      } else if (!hasDatagramHandlers && this.datagramHandlerServer) {
        await this.datagramHandlerServer.stop();
        this.datagramHandlerServer = null;
      }

      if (!hasChallengeRoutes && this.challengeProviderRelayServer) {
        await this.bridge.setChallengeProviderRelay('').catch((err) => {
          logger.log('warn', `Failed to clear challenge provider relay in Rust: ${(err as Error).message}`, { component: 'smart-proxy' });
        });
        await this.challengeProviderRelayServer.stop();
        this.challengeProviderRelayServer = null;
        this.challengeRuntimeOptions = undefined;
      }

      // Update NFTables rules
      if (this.nftablesManager) {
        await this.nftablesManager.cleanup();
        this.nftablesManager = null;
      }
      await this.applyNftablesRules(newRoutes);

      // Update stored routes
      this.settings.routes = newRoutes;

      logger.log('info', `Routes updated (${newRoutes.length} routes)`, { component: 'smart-proxy' });
    });

    // Fire-and-forget cert provisioning outside the mutex — routes are already updated,
    // cert provisioning doesn't need the route update lock and may be slow.
    this.certProvisionPromise = this.provisionCertificatesViaCallback()
      .catch((err) => logger.log('error', `Unexpected error in cert provisioning after route update: ${err.message}`, { component: 'smart-proxy' }));
  }

  /**
   * Update the global ingress security policy without changing routes.
   * The Rust engine applies this before route selection and backend connection.
   */
  public async updateSecurityPolicy(policy: ISmartProxySecurityPolicy): Promise<void> {
    this.settings.securityPolicy = policy;
    await this.bridge.setSecurityPolicy(policy);
  }

  /**
   * Provision a certificate for a named route.
   */
  public async provisionCertificate(routeName: string): Promise<void> {
    await this.bridge.provisionCertificate(routeName);
  }

  /**
   * Force renewal of a certificate.
   */
  public async renewCertificate(routeName: string): Promise<void> {
    await this.bridge.renewCertificate(routeName);
  }

  /**
   * Get certificate status for a route (async - calls Rust).
   */
  public async getCertificateStatus(routeName: string): Promise<IRustCertificateStatus | null> {
    return this.bridge.getCertificateStatus(routeName);
  }

  /**
   * Get the metrics interface.
   */
  public getMetrics(): IMetrics {
    return this.metricsAdapter;
  }

  /**
   * Get sanitized active connection snapshots from the Rust engine.
   */
  public async getActiveConnectionSnapshots(
    options: IActiveConnectionSnapshotOptions = {},
  ): Promise<IActiveConnectionSnapshot[]> {
    return this.bridge.getActiveConnectionSnapshots(options);
  }

  /**
   * Get statistics (async - calls Rust).
   */
  public async getStatistics(): Promise<IRustStatistics> {
    return this.bridge.getStatistics();
  }

  /**
   * Add a listening port at runtime.
   */
  public async addListeningPort(port: number): Promise<void> {
    await this.bridge.addListeningPort(port);
  }

  /**
   * Remove a listening port at runtime.
   */
  public async removeListeningPort(port: number): Promise<void> {
    await this.bridge.removeListeningPort(port);
  }

  /**
   * Get all currently listening ports (async - calls Rust).
   */
  public async getListeningPorts(): Promise<number[]> {
    if (!this.bridge.running) return [];
    return this.bridge.getListeningPorts();
  }

  /**
   * Get eligible domains for ACME certificates (sync - reads local routes).
   */
  public getEligibleDomainsForCertificates(): string[] {
    const domains: string[] = [];
    for (const route of this.settings.routes || []) {
      if (!route.match.domains) continue;
      if (
        route.action.type !== 'forward' ||
        !route.action.tls ||
        route.action.tls.mode === 'passthrough' ||
        route.action.tls.certificate !== 'auto'
      )
        continue;

      const routeDomains = Array.isArray(route.match.domains) ? route.match.domains : [route.match.domains];
      const eligible = routeDomains.filter((d) => !d.includes('*') && this.isValidDomain(d));
      domains.push(...eligible);
    }
    return domains;
  }

  /**
   * Get NFTables status.
   */
  public getNfTablesStatus(): plugins.smartnftables.INftStatus | null {
    return this.nftablesManager?.status() ?? null;
  }

  // --- Private helpers ---

  private async cleanupRuntimeResourcesAfterStartFailure(didSpawn: boolean): Promise<void> {
    this.metricsAdapter.stopPolling();

    if (this.nftablesManager) {
      await this.nftablesManager.cleanup().catch((err) => {
        logger.log('warn', `Failed to clean NFTables after start failure: ${(err as Error).message}`, { component: 'smart-proxy' });
      });
      this.nftablesManager = null;
    }

    if (this.socketHandlerServer) {
      await this.socketHandlerServer.stop().catch((err) => {
        logger.log('warn', `Failed to stop socket handler relay after start failure: ${(err as Error).message}`, { component: 'smart-proxy' });
      });
      this.socketHandlerServer = null;
    }

    if (this.datagramHandlerServer) {
      await this.datagramHandlerServer.stop().catch((err) => {
        logger.log('warn', `Failed to stop datagram handler relay after start failure: ${(err as Error).message}`, { component: 'smart-proxy' });
      });
      this.datagramHandlerServer = null;
    }

    if (this.challengeProviderRelayServer) {
      await this.challengeProviderRelayServer.stop().catch((err) => {
        logger.log('warn', `Failed to stop challenge provider relay after start failure: ${(err as Error).message}`, { component: 'smart-proxy' });
      });
      this.challengeProviderRelayServer = null;
      this.challengeRuntimeOptions = undefined;
    }

    if (didSpawn) {
      this.bridge.removeAllListeners('exit');
      await this.bridge.stopProxy().catch(() => undefined);
      this.bridge.kill();
    }
  }

  /**
   * Apply NFTables rules for routes using the nftables forwarding engine.
   */
  private async applyNftablesRules(routes: IRouteConfig[]): Promise<void> {
    const nftRoutes = routes.filter(r => r.action.forwardingEngine === 'nftables');
    if (nftRoutes.length === 0) return;

    const tableName = nftRoutes.find(r => r.action.nftables?.tableName)?.action.nftables?.tableName ?? 'smartproxy';
    const nft = new plugins.smartnftables.SmartNftables({ tableName });
    await nft.initialize();

    for (const route of nftRoutes) {
      const routeId = route.name || 'unnamed';
      const targets = route.action.targets;
      if (!targets) continue;

      const nftOpts = route.action.nftables;
      const protocol: plugins.smartnftables.TNftProtocol = (nftOpts?.protocol as any) ?? 'tcp';
      const preserveSourceIP = nftOpts?.preserveSourceIP ?? false;

      const ports = Array.isArray(route.match.ports)
        ? route.match.ports.flatMap(p => typeof p === 'number' ? [p] : [])
        : typeof route.match.ports === 'number' ? [route.match.ports] : [];

      for (const target of targets) {
        const targetHost = Array.isArray(target.host) ? target.host[0] : target.host;
        if (typeof targetHost !== 'string') continue;

        for (const sourcePort of ports) {
          const targetPort = typeof target.port === 'number' ? target.port : sourcePort;
          await nft.nat.addPortForwarding(`${routeId}-${sourcePort}-${targetPort}`, {
            sourcePort,
            targetHost,
            targetPort,
            protocol,
            preserveSourceIP,
          });
        }
      }
    }

    this.nftablesManager = nft;
    logger.log('info', `Applied NFTables rules for ${nftRoutes.length} route(s)`, { component: 'smart-proxy' });
  }

  /**
   * Build the Rust configuration object from TS settings.
   */
  private buildRustConfig(routes: IRustProxyOptions['routes'], acmeOverride?: IAcmeOptions): IRustProxyOptions {
    return buildRustProxyOptions(this.settings, routes, acmeOverride, this.challengeRuntimeOptions);
  }

  private hasChallengeRoutes(routes: IRouteConfig[]): boolean {
    return routes.some((route) => Boolean(route.security?.challenge));
  }

  private async ensureChallengeProviderRelay(): Promise<void> {
    const runtimeOptions = this.settings.challenge || {};
    if (!this.challengeProviderRelayServer) {
      this.challengeProviderRelayServer = new ChallengeProviderRelayServer(this.challengeProviders, {
        providerTimeoutMs: runtimeOptions.relayTimeoutMs ?? 5_000,
      });
      await this.challengeProviderRelayServer.start();
    }
    if (!this.challengeRuntimeOptions) {
      this.challengeRuntimeOptions = {
        cookieSigningKey: runtimeOptions.cookieSigningKey || plugins.crypto.randomBytes(32).toString('base64url'),
        pendingCookieName: runtimeOptions.pendingCookieName || '__smartproxy_challenge_pending',
        clearanceCookieName: runtimeOptions.clearanceCookieName || '__smartproxy_clearance',
        reservedPathPrefix: runtimeOptions.reservedPathPrefix || '/.well-known/smartproxy-challenge',
        relaySocketPath: this.challengeProviderRelayServer.getSocketPath(),
        relayTimeoutMs: runtimeOptions.relayTimeoutMs ?? 5_000,
        pendingTtlSeconds: runtimeOptions.pendingTtlSeconds ?? 300,
      };
    } else {
      this.challengeRuntimeOptions = {
        ...this.challengeRuntimeOptions,
        relaySocketPath: this.challengeProviderRelayServer.getSocketPath(),
      };
    }
  }

  private async validateChallengeRoutes(routes: IRouteConfig[]): Promise<void> {
    const manifestCache = new Map<string, plugins.smartchallenge.IChallengeProviderManifest>();
    const errors: string[] = [];

    for (const route of routes) {
      const challenge = route.security?.challenge;
      if (!challenge) continue;
      const routeName = route.name || route.id || 'unnamed route';

      if (!challenge.providerId || typeof challenge.providerId !== 'string') {
        errors.push(`${routeName}: challenge.providerId must be a non-empty string`);
      }
      if (!challenge.challengeType || typeof challenge.challengeType !== 'string') {
        errors.push(`${routeName}: challenge.challengeType must be a non-empty string`);
      }
      if (!route.name && !route.id) {
        errors.push(`${routeName}: challenge routes must set a stable route name or id`);
      }
      this.validateChallengeIntentShape(challenge, errors, routeName);
      if (route.action.type !== 'forward') {
        errors.push(`${routeName}: challenge routes must use forward actions`);
      }
      if (route.action.forwardingEngine === 'nftables') {
        errors.push(`${routeName}: challenge routes cannot use nftables forwarding`);
      }
      if (route.action.tls?.mode === 'passthrough') {
        errors.push(`${routeName}: challenge routes cannot use TLS passthrough`);
      }
      for (const target of route.action.targets || []) {
        if (typeof target.host === 'function' || typeof target.port === 'function') {
          errors.push(`${routeName}: challenge routes cannot use dynamic target host or port functions`);
        }
        if (target.tls?.mode === 'passthrough') {
          errors.push(`${routeName}: challenge routes cannot use target TLS passthrough`);
        }
      }
      if (route.match.transport === 'udp' || route.match.transport === 'all') {
        errors.push(`${routeName}: challenge routes currently require HTTP-visible TCP handling`);
      }
      if (route.match.protocol !== 'http') {
        errors.push(`${routeName}: challenge routes must set match.protocol to 'http'`);
      }
      this.collectForbiddenChallengeKeys(challenge, `security.challenge`, errors, routeName);

      const provider = this.challengeProviders.get(challenge.providerId);
      if (!provider) {
        errors.push(`${routeName}: challenge provider '${challenge.providerId}' is not registered`);
        continue;
      }
      let manifest = manifestCache.get(challenge.providerId);
      if (!manifest) {
        manifest = await provider.getManifest();
        manifestCache.set(challenge.providerId, manifest);
      }
      if (!manifest.challengeTypes.some((type) => type.challengeType === challenge.challengeType)) {
        errors.push(`${routeName}: challenge provider '${challenge.providerId}' does not support type '${challenge.challengeType}'`);
      }
    }

    if (errors.length > 0) {
      throw new Error(`Challenge route validation failed:\n${errors.map((error) => `- ${error}`).join('\n')}`);
    }
  }

  private validateChallengeIntentShape(challengeArg: unknown, errors: string[], routeName: string): void {
    if (!this.isPlainRecord(challengeArg)) {
      errors.push(`${routeName}: security.challenge must be an object`);
      return;
    }

    this.validateAllowedChallengeKeys(
      challengeArg,
      'security.challenge',
      new Set(['providerId', 'challengeType', 'policyRef', 'settings', 'applyTo', 'clearance']),
      errors,
      routeName,
    );

    const settings = challengeArg.settings;
    if (settings !== undefined && !this.isPlainRecord(settings)) {
      errors.push(`${routeName}: security.challenge.settings must be an object when set`);
    }

    const applyTo = challengeArg.applyTo;
    if (applyTo !== undefined) {
      if (!this.isPlainRecord(applyTo)) {
        errors.push(`${routeName}: security.challenge.applyTo must be an object when set`);
      } else {
        this.validateAllowedChallengeKeys(
          applyTo,
          'security.challenge.applyTo',
          new Set(['methods', 'browserNavigationsOnly', 'includePaths', 'excludePaths']),
          errors,
          routeName,
        );
      }
    }

    const clearance = challengeArg.clearance;
    if (clearance !== undefined) {
      if (!this.isPlainRecord(clearance)) {
        errors.push(`${routeName}: security.challenge.clearance must be an object when set`);
      } else {
        this.validateAllowedChallengeKeys(
          clearance,
          'security.challenge.clearance',
          new Set(['ttlSeconds', 'bindToHost', 'bindToRoute', 'bindToIp']),
          errors,
          routeName,
        );
      }
    }
  }

  private validateAllowedChallengeKeys(
    valueArg: Record<string, unknown>,
    pathArg: string,
    allowedKeysArg: Set<string>,
    errors: string[],
    routeName: string,
  ): void {
    for (const key of Object.keys(valueArg)) {
      if (!allowedKeysArg.has(key)) {
        errors.push(`${routeName}: ${pathArg}.${key} is not part of the persisted challenge intent shape`);
      }
    }
  }

  private isPlainRecord(valueArg: unknown): valueArg is Record<string, unknown> {
    return Boolean(valueArg)
      && typeof valueArg === 'object'
      && !Array.isArray(valueArg);
  }

  private collectForbiddenChallengeKeys(value: unknown, path: string, errors: string[], routeName: string): void {
    if (!value || typeof value !== 'object') return;
    const forbiddenKeyPattern = /(endpoint|url|uri|host|hostname|address|ipAddress|port|secret|token|password|credential|apiKey|apikey|accessKey|privateKey|keyfile|certfile|deployment|container|socketPath)/i;
    for (const [key, nestedValue] of Object.entries(value as Record<string, unknown>)) {
      const childPath = `${path}.${key}`;
      if (forbiddenKeyPattern.test(key) && !['providerId', 'challengeType', 'policyRef', 'bindToHost', 'bindToIp', 'includePaths', 'excludePaths'].includes(key)) {
        errors.push(`${routeName}: ${childPath} looks like runtime wiring or secret material and must not be stored in route config`);
      }
      if (typeof nestedValue === 'string' && this.looksLikeRuntimeChallengeValue(nestedValue) && !['includePaths', 'excludePaths'].includes(key)) {
        errors.push(`${routeName}: ${childPath} contains endpoint/IP-style runtime wiring and must not be stored in route config`);
      }
      if (nestedValue && typeof nestedValue === 'object') {
        this.collectForbiddenChallengeKeys(nestedValue, childPath, errors, routeName);
      }
    }
  }

  private looksLikeRuntimeChallengeValue(valueArg: string): boolean {
    return /^https?:\/\//i.test(valueArg)
      || /^wss?:\/\//i.test(valueArg)
      || /(?:^|\b)(?:\d{1,3}\.){3}\d{1,3}(?::\d+)?(?:\b|$)/.test(valueArg)
      || /^[a-z0-9.-]+:\d+$/i.test(valueArg);
  }

  /**
   * For routes with certificate: 'auto', call certProvisionFunction if set.
   * If the callback returns a cert object, load it into Rust.
   * If it returns 'http01', let Rust handle ACME.
   */
  private async provisionCertificatesViaCallback(skipDomains: Set<string> = new Set()): Promise<void> {
    const provisionFn = this.settings.certProvisionFunction;
    if (!provisionFn) return;

    // Phase 1: Collect all unique (domain, route) pairs that need provisioning
    const seen = new Set<string>(skipDomains);
    const tasks: Array<{ domain: string; route: IRouteConfig }> = [];

    for (const route of this.settings.routes) {
      if (route.action.tls?.certificate !== 'auto') continue;
      if (!route.match.domains) continue;

      const rawDomains = Array.isArray(route.match.domains) ? route.match.domains : [route.match.domains];
      const certDomains = this.normalizeDomainsForCertProvisioning(rawDomains);

      for (const domain of certDomains) {
        if (seen.has(domain)) continue;
        seen.add(domain);
        tasks.push({ domain, route });
      }
    }

    if (tasks.length === 0) return;

    // Phase 2: Process all domains in parallel with concurrency limit
    const concurrency = this.settings.certProvisionConcurrency ?? 4;
    const semaphore = new ConcurrencySemaphore(concurrency);

    const promises = tasks.map(async ({ domain, route }) => {
      await semaphore.acquire();
      try {
        await this.provisionSingleDomain(domain, route, provisionFn);
      } finally {
        semaphore.release();
      }
    });

    await Promise.allSettled(promises);
  }

  /**
   * Provision a single domain's certificate via the callback.
   * Includes per-domain timeout and shutdown checks.
   */
  private async provisionSingleDomain(
    domain: string,
    route: IRouteConfig,
    provisionFn: (domain: string, eventComms: ICertProvisionEventComms) => Promise<TSmartProxyCertProvisionObject>,
  ): Promise<void> {
    if (this.stopping) return;

    let expiryDate: string | undefined;
    let source = 'certProvisionFunction';

    const eventComms: ICertProvisionEventComms = {
      log: (msg) => logger.log('info', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
      warn: (msg) => logger.log('warn', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
      error: (msg) => logger.log('error', `[certProvision ${domain}] ${msg}`, { component: 'smart-proxy' }),
      setExpiryDate: (date) => { expiryDate = date.toISOString(); },
      setSource: (s) => { source = s; },
    };

    const timeoutMs = this.settings.certProvisionTimeout ?? 300_000; // 5 min default

    try {
      const result: TSmartProxyCertProvisionObject = await this.withTimeout(
        provisionFn(domain, eventComms),
        timeoutMs,
        `Certificate provisioning timed out for ${domain} after ${timeoutMs}ms`,
      );

      if (this.stopping) return;

      if (result === 'http01') {
        if (route.name) {
          try {
            await this.bridge.provisionCertificate(route.name);
            logger.log('info', `Triggered Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' });
          } catch (provisionErr: any) {
            logger.log('warn', `Cannot provision cert for ${domain} — callback returned 'http01' but Rust ACME failed: ${provisionErr.message}. ` +
              'Note: Rust ACME is disabled when certProvisionFunction is set.', { component: 'smart-proxy' });
          }
        }
        return;
      }

      if (result && typeof result === 'object') {
        if (this.stopping) return;

        const certObj = result as ISmartProxyCertProvisionCertificate;
        await this.bridge.loadCertificate(
          domain,
          certObj.publicKey,
          certObj.privateKey,
          certObj.ca,
        );
        logger.log('info', `Certificate loaded via provision function for ${domain}`, { component: 'smart-proxy' });

        // Persist to consumer store
        if (this.settings.certStore?.save) {
          try {
            await this.settings.certStore.save(domain, certObj.publicKey, certObj.privateKey, certObj.ca);
          } catch (storeErr: any) {
            logger.log('warn', `certStore.save() failed for ${domain}: ${storeErr.message}`, { component: 'smart-proxy' });
          }
        }

        this.emit('certificate-issued', {
          domain,
          expiryDate: expiryDate || (certObj.validUntil ? new Date(certObj.validUntil).toISOString() : undefined),
          source,
        } satisfies ICertificateIssuedEvent);
      }
    } catch (err: any) {
      logger.log('warn', `certProvisionFunction failed for ${domain}: ${err.message}`, { component: 'smart-proxy' });

      this.emit('certificate-failed', {
        domain,
        error: err.message,
        source,
      } satisfies ICertificateFailedEvent);

      // Fallback to ACME if enabled and route has a name
      if (this.settings.certProvisionFallbackToAcme !== false && route.name) {
        try {
          await this.bridge.provisionCertificate(route.name);
          logger.log('info', `Falling back to Rust ACME for ${domain} (route: ${route.name})`, { component: 'smart-proxy' });
        } catch (acmeErr: any) {
          logger.log('warn', `ACME fallback also failed for ${domain}: ${acmeErr.message}` +
            (this.settings.disableDefaultCert
              ? ' — TLS will fail for this domain (disableDefaultCert is true)'
              : ' — default self-signed fallback cert will be used'), { component: 'smart-proxy' });
        }
      }
    }
  }

  /**
   * Race a promise against a timeout. Rejects with the given message if the timeout fires first.
   */
  private withTimeout<T>(promise: Promise<T>, ms: number, message: string): Promise<T> {
    return new Promise<T>((resolve, reject) => {
      const timer = setTimeout(() => reject(new Error(message)), ms);
      promise.then(
        (val) => { clearTimeout(timer); resolve(val); },
        (err) => { clearTimeout(timer); reject(err); },
      );
    });
  }

  /**
   * Normalize routing glob patterns into valid domain identifiers for cert provisioning.
   * - `*nevermind.cloud` → `['nevermind.cloud', '*.nevermind.cloud']`
   * - `*.lossless.digital` → `['*.lossless.digital']` (already valid wildcard)
   * - `code.foss.global` → `['code.foss.global']` (plain domain)
   * - `*mid*.example.com` → skipped with warning (unsupported glob)
   */
  private normalizeDomainsForCertProvisioning(rawDomains: string[]): string[] {
    const result: string[] = [];
    for (const raw of rawDomains) {
      // Plain domain — no glob characters
      if (!raw.includes('*')) {
        result.push(raw);
        continue;
      }

      // Valid wildcard: *.example.com
      if (raw.startsWith('*.') && !raw.slice(2).includes('*')) {
        result.push(raw);
        continue;
      }

      // Routing glob like *example.com (leading star, no dot after it)
      // Convert to bare domain + wildcard pair
      if (raw.startsWith('*') && !raw.startsWith('*.') && !raw.slice(1).includes('*')) {
        const baseDomain = raw.slice(1); // Remove leading *
        result.push(baseDomain);
        result.push(`*.${baseDomain}`);
        continue;
      }

      // Unsupported glob pattern (e.g. *mid*.example.com)
      logger.log('warn', `Skipping unsupported glob pattern for cert provisioning: ${raw}`, { component: 'smart-proxy' });
    }
    return result;
  }

  private isValidDomain(domain: string): boolean {
    if (!domain || domain.length === 0) return false;
    if (domain.includes('*')) return false;
    const validDomainRegex =
      /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/;
    return validDomainRegex.test(domain);
  }
}