underpost/src/server/underpost-gateway.js

Underpost Platform — end-to-end CI/CD and application-delivery toolchain CLI. Covers bare metal, Kubernetes, K3s, kubeadm, LXD, container/image orchestration, secrets, databases, cron jobs, monitoring, SSH, runners, PWA + Workbox delivery, and release orc

/**
 * The centralized gateway infrastructure service.
 *
 * One Nginx deployment is the cluster's single edge utility layer: it serves
 * every host's status pages, maintenance pages and shared edge contexts, and it
 * reverse-proxies the application workloads whose errors it is asked to
 * intercept. Application runtimes stay agnostic — they return a standard status
 * code or become unreachable, and nothing about status page delivery lives in
 * them.
 *
 * Interception is Nginx's `proxy_intercept_errors`, not an Envoy response
 * substitution, because only it satisfies all three constraints at once: the
 * document is served from disk so its size is unbounded, the upstream's status
 * code is preserved, and the client's URI never changes. Envoy's own mechanisms
 * substitute an inline body capped at 4096 bytes, and Envoy cannot re-dispatch a
 * request to another cluster once the upstream has answered.
 *
 * Layout under the Nginx root:
 *   <root>/<host>/<path>/status-pages/<status>/index.html
 *   <root>/<host>/<path>/<context>/...
 *   <root>/conf.d/<host>.conf                    generated server blocks
 * where `<path>` is the proxy sub-path with `/` written as `root`, so
 * `www.cyberiaonline.com` + `/` + 404 becomes
 * `www.cyberiaonline.com/root/status-pages/404/index.html`.
 *
 * @module src/server/underpost-gateway.js
 * @namespace UnderpostGateway
 */

import crypto from 'node:crypto';
import fs from 'fs-extra';
import nodePath from 'node:path';
import { timer } from '../client/components/core/CommonJs.js';
import { instanceStatusPageEntriesFactory, loadConfServerJson, loadReplicas } from './conf.js';
import { compressionConfFactory, compressionModulesConfFactory, nginxImageFactory } from './underpost-compression.js';
import Underpost from '../index.js';
import { loggerFactory } from './logger.js';
import { shellExec } from './process.js';

const logger = loggerFactory(import.meta);

/**
 * @constant UNDERPOST_GATEWAY
 * @description Identity of the shared gateway workload. One deployment serves
 * every host of every deploy, so these names are cluster-wide constants rather
 * than per-deploy.
 * @memberof UnderpostGateway
 */
const UNDERPOST_GATEWAY = {
  name: 'underpost-gateway',
  serviceName: 'underpost-gateway-service',
  configMapName: 'underpost-gateway-nginx',
  claimName: 'pvc-underpost-gateway',
  volumeName: 'pv-underpost-gateway',
  image: nginxImageFactory(),
  root: '/var/www/static',
  port: 80,
  healthPath: '/healthz',
  defaultHostDir: 'default',
  confDir: 'conf.d',
  // kube-dns's conventional ClusterIP; overridden from the live Service.
  resolver: '10.96.0.10',
};

/**
 * @method staticPathSegmentFactory
 * @description Folds a proxy sub-path into one directory name. `/` has no
 * directory of its own, so it is written as `root`; anything else keeps its
 * segments joined by `-` to stay a single level under the host.
 * @param {string} [path] - Proxy sub-path (`/`, `/peer`, `/app`).
 * @returns {string} Directory name.
 * @memberof UnderpostGateway
 */
const staticPathSegmentFactory = (path = '/') => {
  const segment = `${path || '/'}`.replace(/^\/+|\/+$/g, '').replace(/\//g, '-');
  return segment || 'root';
};

/**
 * @method kubernetesUpstreamFactory
 * @description Qualifies a short Kubernetes Service name for Nginx's runtime DNS resolver.
 *
 * Nginx uses the configured `resolver` whenever `proxy_pass` contains a
 * variable. That resolver does not apply the pod's `/etc/resolv.conf` search
 * suffixes, so `service-name:80` fails even though the same short name works in
 * wget/curl. A fully-qualified Service DNS name is unambiguous and continues to
 * resolve after the Service is recreated.
 *
 * Already-qualified hosts and IP literals are kept as supplied.
 * @param {string} upstream - `host:port`.
 * @param {string} [namespace] - Kubernetes namespace containing the Service.
 * @returns {string} Runtime-resolvable upstream.
 * @memberof UnderpostGateway
 */
const kubernetesUpstreamFactory = (upstream, namespace = 'default') => {
  const value = `${upstream || ''}`.trim();
  const separator = value.lastIndexOf(':');
  if (separator < 1) return value;
  const host = value.slice(0, separator);
  const port = value.slice(separator + 1);
  if (host.includes('.') || host === 'localhost' || /^\d{1,3}(?:\.\d{1,3}){3}$/.test(host)) return value;
  return `${host}.${namespace}.svc.cluster.local:${port}`;
};

/**
 * @method nginxTokenFactory
 * @description A sub-path as an identifier nginx accepts. Variable names admit
 * only word characters, so the `-` that {@link UnderpostGateway.staticPathSegmentFactory}
 * joins multi-segment paths with cannot appear in one.
 * @param {string} [path] - Proxy sub-path.
 * @returns {string} Identifier-safe token.
 * @memberof UnderpostGateway
 */
const nginxTokenFactory = (path = '/') => staticPathSegmentFactory(path).replace(/[^a-zA-Z0-9]/g, '_');

/**
 * @method staticLocationFactory
 * @description The three forms every placed document is addressed by: where it
 * sits under the root, the directory a prefix rewrite targets, and the exact URL
 * a full-path rewrite targets.
 *
 * `dir` is what routes normally use. A `ReplacePrefixMatch` onto the directory
 * lets one rule cover the document *and* everything beside it — `/maintenance`
 * resolves through `try_files $uri/index.html`, while `/maintenance/logo.png`
 * resolves through `$uri` — which is why the layout keeps each context in its
 * own directory rather than as a bare file.
 * @param {string} host - Hostname the document belongs to.
 * @param {string} [path] - Proxy sub-path the document belongs to.
 * @param {string} context - Directory under the sub-path (`status-pages/404`, `maintenance`).
 * @param {string} [file] - Document within the context.
 * @returns {{ assetPath: string, dir: string, url: string }} Root-relative path, prefix target, full-path target.
 * @memberof UnderpostGateway
 */
const staticLocationFactory = ({ host, path = '/', context, file = 'index.html' }) => {
  const dir = `${host}/${staticPathSegmentFactory(path)}/${context}`;
  const assetPath = `${dir}/${file}`;
  return { assetPath, dir: `/${dir}`, url: `/${assetPath}` };
};

/**
 * @method statusPageAssetPathFactory
 * @description Location of one host's status page.
 * @param {string} host - Hostname the page belongs to.
 * @param {string} [path] - Proxy sub-path the page belongs to.
 * @param {string|number} status - HTTP status code.
 * @returns {{ assetPath: string, dir: string, url: string }} See {@link UnderpostGateway.staticLocationFactory}.
 * @memberof UnderpostGateway
 */
const statusPageAssetPathFactory = ({ host, path = '/', status }) =>
  staticLocationFactory({ host, path, context: `status-pages/${status}` });

/**
 * @method statusPageBuildSegment
 * @description Where a status page is built inside the client bundle, relative
 * to that client's served root.
 *
 * Deliberately not `<status>/index.html`. A status page is an edge artifact, and
 * a document sitting on the runtime's own `/<status>` route makes it an
 * application route too: the runtime then has a page to serve — or to redirect to
 * — for its own errors, which is exactly the URI change the edge exists to
 * prevent. Kept under the same `status-pages` name the gateway layout uses, so
 * both sides read one convention.
 * @param {string|number} status - HTTP status code.
 * @returns {string} Bundle-relative path of the document.
 * @memberof UnderpostGateway
 */
const statusPageBuildSegment = (status) => `status-pages/${status}/index.html`;

/**
 * @method defaultStatusPagePath
 * @description Root-relative location of the shared fallback document.
 * @param {string|number} [status] - HTTP status code.
 * @returns {string} Root-relative path.
 * @memberof UnderpostGateway
 */
const defaultStatusPagePath = (status = 404) => `${UNDERPOST_GATEWAY.defaultHostDir}/status-pages/${status}/index.html`;

/**
 * @method nginxConfFactory
 * @description Renders the server config.
 *
 * The gateway always rewrites into the layout before forwarding, so `try_files`
 * only has to resolve a path that is already root-relative: the document itself
 * for an asset request, then `index.html` beneath it for a directory — which is
 * what a `ReplacePrefixMatch` onto a context directory produces.
 *
 * A miss ends on the shared default page, so a host with nothing on disk still
 * answers with a deliberate document — but through `error_page`, which keeps the
 * 404 status rather than presenting the fallback as the host's own page with a
 * 200. That status is what stops the document being stored: the PWA service
 * worker caches navigations for hours and would otherwise keep serving the
 * fallback long after the host's real page landed in the tree.
 *
 * Compression is applied here rather than further out because this is the last
 * hop that holds a response body in the clear — and, for every host whose
 * errors are intercepted, the hop every site path already passes through. See
 * {@link module:src/server/underpost-compression.js}.
 * @param {string} [resolver] - Cluster DNS ClusterIP.
 * @param {object} [compression] - Overrides for the compression policy.
 * @returns {string} nginx.conf contents.
 * @memberof UnderpostGateway
 */
const nginxConfFactory = ({ resolver = UNDERPOST_GATEWAY.resolver, compression = {} } = {}) => {
  const policy = { ...compression, staticRoot: true };
  const modules = compressionModulesConfFactory(policy);
  const compress = compressionConfFactory(policy);
  return `worker_processes auto;
error_log /dev/stderr warn;
pid /tmp/nginx.pid;
${modules ? `${modules}\n` : ''}
events {
  worker_connections 1024;
}

http {
  include /etc/nginx/mime.types;
  default_type text/html;
  sendfile on;
  tcp_nopush on;
  server_tokens off;
${compress ? `\n${compress}\n` : ''}
  log_format concise '$remote_addr "$request" $status $body_bytes_sent "$host"';
  access_log /dev/stdout concise;

  # Websocket upgrades must be forwarded verbatim; a proxied hop that drops the
  # Connection header leaves the client holding a half-open socket.
  map $http_upgrade $connection_upgrade {
    default upgrade;
    ''      close;
  }

  # Cluster DNS as a literal address — nginx cannot resolve its own resolver.
  # Needed because every upstream is passed through a variable: without it nginx
  # resolves a Service name once at start-up and keeps the address for the life
  # of the process, and a redeployed workload gets a new ClusterIP.
  resolver ${resolver} valid=10s ipv6=off;

  # Per-host server blocks, written into the volume by the deploy that owns the
  # host. They live beside the documents rather than in this ConfigMap because
  # the workload is shared: one deploy must not rewrite another's routing.
  include ${UNDERPOST_GATEWAY.root}/${UNDERPOST_GATEWAY.confDir}/*.conf;

  server {
    listen ${UNDERPOST_GATEWAY.port} default_server;
    server_name _;
    root ${UNDERPOST_GATEWAY.root};

    error_page 404 /${defaultStatusPagePath(404)};

    # Probes are the only traffic that would otherwise dominate the log.
    location = ${UNDERPOST_GATEWAY.healthPath} {
      access_log off;
      add_header Content-Type text/plain;
      return 200 'ok';
    }

    location = /${defaultStatusPagePath(404)} {
      internal;
      add_header Cache-Control 'no-store' always;
    }

    location / {
      add_header Cache-Control 'public, max-age=60';
      try_files $uri $uri/index.html =404;
    }
  }
}
`;
};

/**
 * @method statusPageLocationsFactory
 * @description The internal locations an intercepted status resolves to, and the
 * `error_page` lines that reach them.
 *
 * Each is `internal`, so a client cannot request the document at its storage
 * path — it is only ever reached by interception, which is what keeps the
 * client's URI unchanged. The `=` form is deliberately absent: `error_page 404
 * /x` preserves the upstream's status, while `error_page 404 = /x` would rewrite
 * it to the status of the page itself.
 * @param {string} host - Hostname the documents belong to.
 * @param {string} [path] - Proxy sub-path the documents belong to.
 * @param {Object<string,string>} statuses - Status code → context directory under the sub-path.
 * @returns {{errorPages: string, locations: string}} Rendered `error_page` directives and their locations.
 * @memberof UnderpostGateway
 */
const statusPageLocationsFactory = ({ host, path = '/', statuses }) => {
  const entries = Object.entries(statuses);
  const named = (status) => `@status_${nginxTokenFactory(path)}_${status}`;
  return {
    errorPages: entries.map(([status]) => `    error_page ${status} ${named(status)};`).join('\n'),
    locations: entries
      .map(
        ([status, context]) => `  location ${named(status)} {
    add_header Cache-Control 'no-store' always;
    try_files /${staticLocationFactory({ host, path, context }).assetPath} =${status};
  }`,
      )
      .join('\n'),
  };
};

/**
 * @method hostServerConfFactory
 * @description Renders one host's server block: every proxied sub-path, and the
 * documents its errors are intercepted with.
 *
 * `proxy_intercept_errors` is the whole mechanism. The upstream answers 404 or
 * dies, Nginx swaps in the document from disk, and the client sees its own URI
 * with the upstream's status code — no redirect, no size ceiling, and nothing
 * for the application runtime to implement. A sub-path that declares no status
 * page is proxied untouched, so an API keeps returning its own error bodies.
 * @param {string} host - Hostname this block serves.
 * @param {Array<object>} routes - `{ path, upstream, statuses, stripPrefix }` per proxied
 *   sub-path; `statuses` maps a status code to the context directory holding its
 *   document, and `stripPrefix` drops the sub-path before dialling the upstream.
 * @returns {string} nginx server block, or an empty string when the host proxies nothing.
 * @memberof UnderpostGateway
 */
const hostServerConfFactory = ({ host, routes = [], namespace = 'default' }) => {
  const proxied = routes.filter((route) => route.upstream);
  if (proxied.length === 0) return '';
  // Longest sub-path first: nginx prefix locations are longest-match, but the
  // emitted order keeps the block readable next to the HTTPRoute it mirrors.
  const sorted = [...proxied].sort((a, b) => (b.path || '/').length - (a.path || '/').length);
  const blocks = sorted.map((route) => {
    const { errorPages, locations } = statusPageLocationsFactory({
      host,
      path: route.path,
      statuses: route.statuses || {},
    });
    const intercept = errorPages ? `    proxy_intercept_errors on;\n${errorPages}` : '    proxy_intercept_errors off;';
    const path = route.path || '/';
    // The upstream is dialled through a variable, so nginx forwards the request
    // URI verbatim and a prefix strip has to be an explicit rewrite. `break`
    // keeps it inside this location instead of re-running location matching.
    const rewrite = route.stripPrefix && path !== '/' ? `    rewrite ^${path}/?(.*)$ /$1 break;\n` : '';
    return {
      path,
      upstream: kubernetesUpstreamFactory(route.upstream, namespace),
      intercept,
      locations,
      rewrite,
    };
  });
  return `
server {
  listen ${UNDERPOST_GATEWAY.port};
  server_name ${host};
  root ${UNDERPOST_GATEWAY.root};

  # This host's own documents, served from disk before anything is proxied.
  # Every path in the layout begins with the hostname, which is exactly what the
  # gateway rewrites a status or context route onto — and a longer prefix than
  # the proxied root below, so nginx prefers it. Without this location the
  # rewritten document path falls into the proxy and is sent to the application,
  # which has no such route: the app answers 404, and an app that redirects its
  # own 404s turns that into a loop between the route and the rewrite.
  location /${host}/ {
    add_header Cache-Control 'public, max-age=60';
    try_files $uri $uri/index.html =404;
  }

${blocks
  .map(
    ({ path, upstream, intercept, rewrite }) => `  location ${path} {
    proxy_http_version 1.1;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;
    # Websockets: the upgrade must survive this hop or a client that negotiated
    # one at the edge is left holding a half-open connection.
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    set $upstream_${nginxTokenFactory(path)} ${upstream};
${intercept}
${rewrite}    proxy_pass http://$upstream_${nginxTokenFactory(path)};
  }`,
  )
  .join('\n\n')}

${blocks
  .map(({ locations }) => locations)
  .filter(Boolean)
  .join('\n')}
}
`;
};

/**
 * @method underpostGatewayManifestsFactory
 * @description Renders the whole workload: the Nginx config, the hostPath volume
 * holding the documents, the deployment and the Service routes target.
 * @param {string} [namespace] - Kubernetes namespace.
 * @param {string} hostPath - Node directory backing the static root.
 * @param {string} [nodeName] - Node the hostPath volume is pinned to.
 * @param {string} [storage] - Volume size.
 * @param {object} [compression] - Overrides for the compression policy.
 * @returns {string} Multi-document YAML.
 * @memberof UnderpostGateway
 */
const underpostGatewayManifestsFactory = ({
  namespace = 'default',
  hostPath,
  nodeName = '',
  storage = '1Gi',
  resolver,
  compression = {},
} = {}) => {
  const nginxConf = nginxConfFactory({ resolver, compression });
  // The config is mounted with `subPath`, which Kubernetes never refreshes in
  // place, and the pod template is otherwise identical across rebuilds — so
  // without this annotation an edited nginx.conf reaches the ConfigMap and
  // nothing else, and the running Nginx keeps serving under the previous
  // layout for the life of the pod.
  const configHash = crypto.createHash('sha256').update(nginxConf).digest('hex').slice(0, 16);
  return `
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: ${UNDERPOST_GATEWAY.configMapName}
  namespace: ${namespace}
data:
  nginx.conf: |
${nginxConf
  .replace(/\n$/, '')
  .split('\n')
  .map((line) => (line.length > 0 ? `    ${line}` : ''))
  .join('\n')}
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: ${UNDERPOST_GATEWAY.volumeName}
spec:
  capacity:
    storage: ${storage}
  accessModes:
    - ReadOnlyMany
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: manual${
    nodeName
      ? `
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: kubernetes.io/hostname
              operator: In
              values:
                - ${nodeName}`
      : ''
  }
  claimRef:
    apiVersion: v1
    kind: PersistentVolumeClaim
    name: ${UNDERPOST_GATEWAY.claimName}
    namespace: ${namespace}
  hostPath:
    path: ${hostPath}
    type: DirectoryOrCreate
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: ${UNDERPOST_GATEWAY.claimName}
  namespace: ${namespace}
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: manual
  volumeName: ${UNDERPOST_GATEWAY.volumeName}
  resources:
    requests:
      storage: ${storage}
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${UNDERPOST_GATEWAY.name}
  namespace: ${namespace}
  labels:
    app: ${UNDERPOST_GATEWAY.name}
spec:
  replicas: 1
  selector:
    matchLabels:
      app: ${UNDERPOST_GATEWAY.name}
  template:
    metadata:
      labels:
        app: ${UNDERPOST_GATEWAY.name}
      annotations:
        underpost.net/nginx-conf-hash: '${configHash}'
    spec:
      containers:
        - name: nginx
          image: ${UNDERPOST_GATEWAY.image}
          ports:
            - containerPort: ${UNDERPOST_GATEWAY.port}
          resources:
            requests:
              cpu: 10m
              memory: 16Mi
            limits:
              cpu: 200m
              memory: 128Mi
          readinessProbe:
            httpGet:
              path: ${UNDERPOST_GATEWAY.healthPath}
              port: ${UNDERPOST_GATEWAY.port}
            initialDelaySeconds: 2
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: ${UNDERPOST_GATEWAY.healthPath}
              port: ${UNDERPOST_GATEWAY.port}
            initialDelaySeconds: 10
            periodSeconds: 20
          volumeMounts:
            - name: nginx-conf
              mountPath: /etc/nginx/nginx.conf
              subPath: nginx.conf
            - name: static-root
              mountPath: ${UNDERPOST_GATEWAY.root}
              readOnly: true
      volumes:
        - name: nginx-conf
          configMap:
            name: ${UNDERPOST_GATEWAY.configMapName}
        - name: static-root
          persistentVolumeClaim:
            claimName: ${UNDERPOST_GATEWAY.claimName}
---
apiVersion: v1
kind: Service
metadata:
  name: ${UNDERPOST_GATEWAY.serviceName}
  namespace: ${namespace}
  labels:
    app: ${UNDERPOST_GATEWAY.name}
spec:
  type: ClusterIP
  selector:
    app: ${UNDERPOST_GATEWAY.name}
  ports:
    - name: http
      protocol: TCP
      port: ${UNDERPOST_GATEWAY.port}
      targetPort: ${UNDERPOST_GATEWAY.port}
`;
};

/**
 * @method writeStaticAsset
 * @description Places one document in the node directory backing the static
 * root. The deploy runs on that node, so the file is copied directly rather than
 * shipped through the API server — which is also what keeps a page of any size
 * out of the cluster's object store.
 * @param {string} hostRoot - Node directory backing the static root.
 * @param {string} assetPath - Root-relative destination.
 * @param {string} sourcePath - File to copy.
 * @returns {boolean} True when the document was placed.
 * @memberof UnderpostGateway
 */
const writeStaticAsset = ({ hostRoot, assetPath, sourcePath }) => {
  if (!sourcePath || !fs.existsSync(sourcePath) || fs.statSync(sourcePath).size === 0) return false;
  const target = nodePath.join(hostRoot, assetPath);
  // sudo: the node directory is root-owned, and the deploy may run unprivileged.
  shellExec(`sudo mkdir -p ${nodePath.dirname(target)}`, { silent: true });
  shellExec(`sudo cp -f ${sourcePath} ${target}`, { silent: true });
  return true;
};

/**
 * @method syncStaticAssetFromPod
 * @description Pulls one document out of the running workload and places it in
 * the static tree.
 *
 * The pod is the authority for these artifacts, not the host: several clients
 * are built from sources that only exist inside the container (cloned from the
 * private repo at start-up), so the host's `public/` tree is both incomplete and
 * as old as the last host-side build. Copying from the pod is what makes the
 * placed document match what the workload would actually have served.
 * @param {string} podName - Workload pod holding the built artifact.
 * @param {string} [namespace] - Pod namespace.
 * @param {string} [container] - Container within the pod.
 * @param {string} sourcePath - Absolute path of the artifact inside the container.
 * @param {string} hostRoot - Node directory backing the static root.
 * @param {string} assetPath - Root-relative destination.
 * @returns {boolean} True when the document was placed.
 * @memberof UnderpostGateway
 */
const syncStaticAssetFromPod = ({ podName, namespace = 'default', container, sourcePath, hostRoot, assetPath }) => {
  const target = nodePath.join(hostRoot, assetPath);
  const containerFlag = container ? ` -c ${container}` : '';
  // Keyed by destination, not by basename: every document in the layout is an
  // `index.html`, so a shared staging name lets one asset's copy be mistaken
  // for another's.
  const staged = nodePath.join(
    '/tmp',
    `underpost-gateway-${crypto.createHash('sha256').update(assetPath).digest('hex').slice(0, 12)}-${process.pid}`,
  );
  // Staged through /tmp because `kubectl cp` runs unprivileged while the node
  // directory is root-owned; the move is the only step that needs sudo.
  fs.removeSync(staged);
  shellExec(`kubectl cp ${namespace}/${podName}:${sourcePath} ${staged}${containerFlag} 2>/dev/null || true`, {
    silent: true,
    silentOnError: true,
  });
  if (!fs.existsSync(staged) || fs.statSync(staged).size === 0) {
    fs.removeSync(staged);
    return false;
  }
  shellExec(`sudo mkdir -p ${nodePath.dirname(target)}`, { silent: true });
  shellExec(`sudo cp -f ${staged} ${target}`, { silent: true });
  fs.removeSync(staged);
  logger.info('Static asset synced from workload', { podName, sourcePath, assetPath });
  return true;
};

/**
 * @method writeHostServerConf
 * @description Writes one host's server block into a directory, or removes it
 * when there is nothing to serve.
 *
 * A build artifact and nothing more: it touches no cluster, so generating
 * manifests works with no cluster running at all. Installing the block into the
 * live gateway and reloading it is {@link UnderpostGateway.installGatewayConf}'s
 * job, on the apply path where a cluster is a precondition.
 * @param {string} confDir - Directory the block is written to.
 * @param {string} host - Hostname the block serves.
 * @param {string} conf - Rendered block from {@link UnderpostGateway.hostServerConfFactory}; empty removes it.
 * @returns {boolean} True when the file changed.
 * @memberof UnderpostGateway
 */
const writeHostServerConf = ({ confDir, host, conf }) => {
  const target = nodePath.join(confDir, `${host}.conf`);
  const current = fs.existsSync(target) ? fs.readFileSync(target, 'utf8') : '';
  if (current === (conf || '')) return false;
  if (!conf) {
    fs.removeSync(target);
    return true;
  }
  fs.mkdirpSync(confDir);
  fs.writeFileSync(target, conf, 'utf8');
  return true;
};

/**
 * @method hostInstanceRegistryPathFactory
 * @description Path of the descriptor set last published for a host.
 *
 * Kept beside the host's server block because it describes the same object, and
 * given a non-`.conf` suffix so {@link UnderpostGateway.installGatewayConf}
 * never installs it into Nginx.
 * @param {string} confDir - Directory holding the built blocks.
 * @param {string} host - Hostname.
 * @returns {string} File path.
 * @memberof UnderpostGateway
 */
const hostInstanceRegistryPathFactory = ({ confDir, host }) => nodePath.join(confDir, `${host}.instances.json`);

/**
 * @method readHostInstanceRegistry
 * @description The instance descriptors last published for a host.
 *
 * The conf declares what *should* run; this records what the host was last
 * rendered with, which is the only place a variant's descriptor survives being
 * removed from the conf while its workload is still up. Unreadable or malformed
 * content is treated as absent: a broken registry must not block a deploy, it
 * just means nothing extra is preserved.
 * @param {string} confDir - Directory holding the built blocks.
 * @param {string} host - Hostname.
 * @returns {Array<object>} Descriptors, or an empty list.
 * @memberof UnderpostGateway
 */
const readHostInstanceRegistry = ({ confDir, host }) => {
  const target = hostInstanceRegistryPathFactory({ confDir, host });
  if (!fs.existsSync(target)) return [];
  try {
    const parsed = JSON.parse(fs.readFileSync(target, 'utf8'));
    return Array.isArray(parsed) ? parsed.filter((entry) => entry?.id) : [];
  } catch (error) {
    logger.warn('Ignoring unreadable host instance registry', { target, message: error.message });
    return [];
  }
};

/**
 * @method writeHostInstanceRegistry
 * @description Records the descriptors a host was just rendered with.
 * @param {string} confDir - Directory holding the built blocks.
 * @param {string} host - Hostname.
 * @param {Array<object>} instances - Descriptors used for this render.
 * @returns {boolean} True when the file changed.
 * @memberof UnderpostGateway
 */
const writeHostInstanceRegistry = ({ confDir, host, instances = [] }) => {
  const target = hostInstanceRegistryPathFactory({ confDir, host });
  const next = `${JSON.stringify(instances, null, 2)}\n`;
  const current = fs.existsSync(target) ? fs.readFileSync(target, 'utf8') : '';
  if (current === next) return false;
  fs.mkdirpSync(confDir);
  fs.writeFileSync(target, next, 'utf8');
  return true;
};

/**
 * @method installGatewayConf
 * @description Installs the built server blocks into the shared gateway and
 * reloads Nginx.
 *
 * The blocks live in the volume rather than the ConfigMap because the workload is
 * shared by every deploy: a ConfigMap would make one deploy's apply rewrite
 * another's routing, and a `subPath` mount would never refresh anyway. Reloading
 * signals the running master, so the config lands without dropping a connection.
 *
 * A block that does not parse would take the whole edge down on reload, so the
 * config is validated first and the previous content is put back if it fails —
 * leaving the running Nginx exactly as it was.
 * @param {string} hostRoot - Node directory backing the gateway root.
 * @param {string} confSourceDir - Directory holding the built blocks.
 * @param {string} [namespace] - Namespace holding the workload.
 * @returns {boolean} True when Nginx was reloaded with the new config.
 * @throws {Error} When Nginx rejects the candidate config or cannot reload. A
 *   rejected candidate is restored before the error is raised.
 * @memberof UnderpostGateway
 */
const installGatewayConf = ({ hostRoot, confSourceDir, namespace = 'default' }) => {
  if (!fs.existsSync(confSourceDir)) return false;
  const blocks = fs.readdirSync(confSourceDir).filter((name) => name.endsWith('.conf'));
  if (blocks.length === 0) return false;
  const confDir = nodePath.join(hostRoot, UNDERPOST_GATEWAY.confDir);
  // sudo: the node directory is root-owned, and the deploy may run unprivileged.
  shellExec(`sudo mkdir -p ${confDir}`, { silent: true });
  const previous = Object.fromEntries(
    blocks.map((name) => {
      const target = nodePath.join(confDir, name);
      return [name, fs.existsSync(target) ? fs.readFileSync(target, 'utf8') : null];
    }),
  );
  const restorePrevious = () => {
    for (const [name, content] of Object.entries(previous)) {
      const target = nodePath.join(confDir, name);
      if (content === null) shellExec(`sudo rm -f ${target}`, { silent: true });
      else {
        const staged = nodePath.join('/tmp', `underpost-gateway-restore-${name}-${process.pid}`);
        fs.writeFileSync(staged, content, 'utf8');
        shellExec(`sudo cp -f ${staged} ${target}`, { silent: true });
        fs.removeSync(staged);
      }
    }
  };
  for (const name of blocks)
    shellExec(`sudo cp -f ${nodePath.join(confSourceDir, name)} ${nodePath.join(confDir, name)}`, { silent: true });
  const test = shellExec(`kubectl exec -n ${namespace} deploy/${UNDERPOST_GATEWAY.name} -- nginx -t 2>&1`, {
    stdout: true,
    silent: true,
    silentOnError: true,
  });
  if (!`${test}`.includes('successful')) {
    restorePrevious();
    logger.error('Gateway config rejected; the previous config was restored and Nginx left running', {
      blocks,
      test: `${test}`.trim().split('\n').slice(-3).join(' '),
    });
    throw new Error(
      `Gateway config rejected for ${blocks.join(', ')}: ${`${test}`.trim().split('\n').slice(-3).join(' ')}`,
    );
  }
  try {
    shellExec(`kubectl exec -n ${namespace} deploy/${UNDERPOST_GATEWAY.name} -- nginx -s reload`, {
      silent: true,
    });
  } catch (error) {
    restorePrevious();
    // The running master normally retains its old config when a reload signal
    // fails. Re-signal after restoring so even a partial reload converges back
    // to the last validated state.
    shellExec(`kubectl exec -n ${namespace} deploy/${UNDERPOST_GATEWAY.name} -- nginx -s reload`, {
      silent: true,
      silentOnError: true,
    });
    throw error;
  }
  logger.info('Gateway config installed and reloaded', { blocks, confDir });
  return true;
};

/**
 * @method seedDefaultStatusPage
 * @description Writes the shared fallback document `nginx.conf` serves through
 * `error_page`, so a host with nothing on disk gets a deliberate page instead of
 * Nginx's stock error. Never overwrites an existing document — an operator may
 * have replaced it.
 * @param {string} hostRoot - Node directory backing the static root.
 * @returns {boolean} True when the document was written.
 * @memberof UnderpostGateway
 */
const seedDefaultStatusPage = (hostRoot) => {
  // The base config includes `<confDir>/*.conf`; the directory has to exist
  // before any deploy contributes a block to it.
  shellExec(`sudo mkdir -p ${nodePath.join(hostRoot, UNDERPOST_GATEWAY.confDir)}`, { silent: true });
  const target = nodePath.join(hostRoot, defaultStatusPagePath(404));
  if (fs.existsSync(target)) return false;
  const document = `<!doctype html>
<html lang="en">
  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <title>404 Not Found</title>
  </head>
  <body>
    <h1>404</h1>
    <p>The requested resource was not found.</p>
  </body>
</html>
`;
  shellExec(`sudo mkdir -p ${nodePath.dirname(target)}`, { silent: true });
  shellExec(`sudo tee ${target} > /dev/null <<'EOF'\n${document}EOF\n`, { silent: true });
  return true;
};

/**
 * @method gatewayStaticAssetExists
 * @description Whether a document is present and non-empty under the gateway
 * root. An empty file is treated as absent: it is what a half-finished copy
 * leaves behind, and serving it would answer a status page with a blank body.
 * @param {string} hostRoot - Node directory backing the gateway root.
 * @param {string} assetPath - Root-relative path of the document.
 * @returns {boolean} True when the document can be served.
 * @memberof UnderpostGateway
 */
const gatewayStaticAssetExists = ({ hostRoot, assetPath }) => {
  const target = `${hostRoot}/${assetPath}`;
  return fs.existsSync(target) && fs.statSync(target).size > 0;
};

/**
 * @method pwaFallbackChecksFactory
 * @description The fallback probes a deploy's own hosts are verified with, read
 * from `conf.server.json` and `conf.ssr.json`.
 *
 * One probe per host/sub-path that declares a maintenance view, because that is
 * the document an unreachable workload has to answer with — the condition the
 * edge must satisfy before any application is deployed behind it.
 *
 * `loadReplicas` expansion and the `singleReplica` skip mirror
 * `buildProxyRouter`/`buildManifest` exactly: a plain `replicas` path (no
 * `singleReplica`) is built and routed by this same deploy id, so its fallback
 * is checked too; a `singleReplica` canonical path is never built or routed
 * under this deploy id at all — it is each replica's own, separate deploy id —
 * so probing it here would poll a document that structurally cannot exist.
 * @param {string} deployId - Deploy id whose conf declares the views.
 * @returns {Array<{host: string, path: string, assetPath: string, kind: string}>} One probe per declaring sub-path.
 * @memberof UnderpostGateway
 */
const pwaFallbackChecksFactory = (deployId) => {
  const confServer = loadReplicas(deployId, loadConfServerJson(`./engine-private/conf/${deployId}/conf.server.json`));
  const confSSRPath = `./engine-private/conf/${deployId}/conf.ssr.json`;
  const confSSR = fs.existsSync(confSSRPath) ? JSON.parse(fs.readFileSync(confSSRPath, 'utf8')) : {};
  const checks = [];
  for (const host of Object.keys(confServer))
    for (const path of Object.keys(confServer[host])) {
      if (confServer[host][path].singleReplica) continue;
      const maintenance = Underpost.deploy
        .edgeRouteEntriesFactory({ confServer, confSSR, host, path })
        .find((entry) => entry.context === 'maintenance');
      if (maintenance)
        checks.push({
          host,
          path,
          assetPath: maintenance.assetPath,
          kind: maintenance.kind,
        });
    }
  return checks;
};

/**
 * @method instanceFallbackChecksFactory
 * @description The fallback probes a set of instances are verified with, one per
 * host and sub-path.
 *
 * Deduplicated on `host + path`: a variant that declares several status pages is
 * still one route to probe, and probing it once per declared status would report
 * the same reachability several times.
 * @param {Array<object>} [instances] - Expanded instance entries.
 * @returns {Array<{host: string, path: string, assetPath: string, kind: string}>} One probe per instance sub-path.
 * @memberof UnderpostGateway
 */
const instanceFallbackChecksFactory = (instances = []) => {
  const checks = new Map();
  for (const entry of instanceStatusPageEntriesFactory({ instances }))
    if (!checks.has(`${entry.host}${entry.path}`))
      checks.set(`${entry.host}${entry.path}`, {
        host: entry.host,
        path: entry.path,
        assetPath: entry.assetPath,
        kind: `status:${entry.status}`,
      });
  return [...checks.values()];
};

/**
 * @method assertStaticAssets
 * @description Fails the deploy when a configured document reached neither the
 * placement pass nor the gateway root.
 *
 * Deliberately fatal. A missing document is not visible at deploy time — the
 * routes are accepted, the workload is healthy, and the gap only surfaces later
 * as a shared default page in place of the host's own. Refusing to continue is
 * what turns that into an immediate, attributable failure.
 * @param {Array<object>} records - Placement records carrying `source` and `assetPath`.
 * @param {string} hostRoot - Node directory backing the gateway root.
 * @param {string} label - Workflow name used in the thrown message.
 * @returns {Array<object>} The records, unchanged, when every document is present.
 * @throws {Error} When any configured document is absent.
 * @memberof UnderpostGateway
 */
const assertStaticAssets = ({ records, hostRoot, label }) => {
  const missing = records.filter(
    (entry) => !entry.source && !gatewayStaticAssetExists({ hostRoot, assetPath: entry.assetPath }),
  );
  if (missing.length > 0)
    throw new Error(
      `[${label}] Static gateway bootstrap is missing configured assets: ` +
        missing.map((entry) => entry.assetPath).join(', '),
    );
  return records;
};

/**
 * @method placeInstanceStaticAssets
 * @description Places every instance's declared status page in the gateway root
 * and asserts the result.
 *
 * The documents come from the project each instance runs, so this is the only
 * pass that can supply them; anything still missing afterwards would leave a
 * route pointing at a document that cannot exist.
 * @param {Array<object>} instances - Expanded instance entries.
 * @param {object} options - Deploy/run options (gateway root, namespace).
 * @param {string} label - Workflow name used in the thrown message.
 * @returns {Array<object>} One record per document, with where it came from.
 * @memberof UnderpostGateway
 */
const placeInstanceStaticAssets = ({ instances, options, label }) => {
  const hostRoot = Underpost.deploy.underpostGatewayRootFactory(options);
  const records = instanceStatusPageEntriesFactory({ instances }).map((entry) => ({
    ...entry,
    source: writeStaticAsset({ hostRoot, assetPath: entry.assetPath, sourcePath: entry.sourcePath }) ? 'project' : null,
  }));
  return assertStaticAssets({ records, hostRoot, label });
};

/**
 * @method gatewayFallbackProbeRunner
 * @description Proves the edge answers each configured fallback with the exact
 * document on disk, before any application is deployed behind it.
 *
 * The assertion is on the response, not on the manifests: an accepted route and a
 * Programmed Gateway say nothing about which body a client receives, and every
 * failure this pipeline has had was invisible in object status. The expected body
 * is hashed from the file the config points at, so a probe cannot pass against a
 * shared default page.
 *
 * With no workload deployed yet the upstream is unreachable, so the wanted status
 * is an upstream failure carrying the configured document — which is the whole
 * contract being verified. Polling absorbs the reconciliation window in which the
 * data plane still serves the previous generation.
 *
 * `gatewayStatusRunner` is injected rather than imported so this module never
 * depends on the runner collection that calls it.
 * @param {Array<object>} checks - Probes from {@link UnderpostGateway.pwaFallbackChecksFactory} or {@link UnderpostGateway.instanceFallbackChecksFactory}.
 * @param {object} options - Deploy/run options (namespace, dev, gatewayApi).
 * @param {string} label - Workflow name used in log lines and thrown messages.
 * @param {Function} gatewayStatusRunner - `(hosts, options) => Promise<{programmed: boolean, servesHttps: boolean}>`.
 * @returns {Promise<Array<object>>} One result per probe.
 * @throws {Error} When the gateway is not operational, or any probe fails.
 * @memberof UnderpostGateway
 */
const gatewayFallbackProbeRunner = async ({ checks, options, label, gatewayStatusRunner }) => {
  if (!options.gatewayApi || checks.length === 0) return [];
  const namespace = options.namespace || 'default';
  shellExec(`kubectl rollout status deployment/${UNDERPOST_GATEWAY.name} -n ${namespace} --timeout=5m`);
  const hosts = [...new Set(checks.map((check) => check.host))];
  const gatewayStatus = await gatewayStatusRunner(hosts.join(','), { ...options, namespace });
  if (!gatewayStatus.programmed || (options.dev && !gatewayStatus.servesHttps))
    throw new Error(`[${label}] Gateway is not operational before application deployment`);

  const hostRoot = Underpost.deploy.underpostGatewayRootFactory(options);
  const failures = [];
  const results = [];
  for (const check of checks) {
    const expectedPath = `${hostRoot}/${check.assetPath}`;
    const expectedHash = gatewayStaticAssetExists({ hostRoot, assetPath: check.assetPath })
      ? crypto.createHash('sha256').update(fs.readFileSync(expectedPath)).digest('hex')
      : '';
    let body = '';
    let status = '';
    let actualHash = '';
    let passed = false;
    let attempts = 0;
    // Gateway and HTTPRoute status can still show the previous generation for a
    // short reconciliation window. Poll the actual response until the intended
    // fallback is observable instead of racing the controller once.
    for (attempts = 1; attempts <= 30; attempts++) {
      if (options.dev) {
        const url = `https://${check.host}${check.path || '/'}`;
        const curl = `curl -sSk --noproxy '*' --resolve ${check.host}:443:127.0.0.1`;
        body = shellExec(`${curl} ${url}`, { stdout: true, silent: true, silentOnError: true });
        status = shellExec(`${curl} -o /dev/null -w '%{http_code}' ${url}`, {
          stdout: true,
          silent: true,
          silentOnError: true,
        }).trim();
      } else {
        const request = `http://127.0.0.1${check.path || '/'}`;
        body = shellExec(
          `kubectl exec -n ${namespace} deploy/${UNDERPOST_GATEWAY.name} -- sh -c ` +
            `"wget -q -O - -T 10 --header 'Host: ${check.host}' ${request} 2>/dev/null || true"`,
          { stdout: true, silent: true, silentOnError: true },
        );
        const headers = shellExec(
          `kubectl exec -n ${namespace} deploy/${UNDERPOST_GATEWAY.name} -- sh -c ` +
            `"wget -S -O /dev/null -T 10 --header 'Host: ${check.host}' ${request} 2>&1 || true"`,
          { stdout: true, silent: true, silentOnError: true },
        );
        status = [...headers.matchAll(/HTTP\/[0-9.]+\s+([0-9]{3})/g)].pop()?.[1] || '';
      }
      actualHash = crypto
        .createHash('sha256')
        .update(body || '')
        .digest('hex');
      passed = /^50[234]$/.test(status) && !!expectedHash && actualHash === expectedHash;
      if (passed) break;
      if (attempts < 30) await timer(2000);
    }
    const result = {
      ...check,
      status,
      bodyMatchesConfiguredAsset: actualHash === expectedHash,
      attempts,
      passed,
    };
    results.push(result);
    if (!passed) failures.push(result);
  }
  logger.info(`[${label}] Pre-runtime fallback probes`, { results });
  if (failures.length > 0) throw new Error(`[${label}] ${failures.length}/${checks.length} fallback probes failed`);
  return results;
};

export {
  UNDERPOST_GATEWAY,
  assertStaticAssets,
  gatewayFallbackProbeRunner,
  gatewayStaticAssetExists,
  hostInstanceRegistryPathFactory,
  hostServerConfFactory,
  installGatewayConf,
  instanceFallbackChecksFactory,
  placeInstanceStaticAssets,
  pwaFallbackChecksFactory,
  readHostInstanceRegistry,
  writeHostInstanceRegistry,
  kubernetesUpstreamFactory,
  underpostGatewayManifestsFactory,
  nginxConfFactory,
  seedDefaultStatusPage,
  staticLocationFactory,
  staticPathSegmentFactory,
  statusPageAssetPathFactory,
  statusPageBuildSegment,
  statusPageLocationsFactory,
  syncStaticAssetFromPod,
  writeHostServerConf,
  writeStaticAsset,
};