UNPKG

@fntio/infra-client

Version:

Self-hosted AWS access portal: credential vending, target-account trust, and grant-driven private networking as CDK stacks and constructs.

858 lines (857 loc) • 48.1 kB
"use strict"; var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { if (k2 === undefined) k2 = k; var desc = Object.getOwnPropertyDescriptor(m, k); if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { desc = { enumerable: true, get: function() { return m[k]; } }; } Object.defineProperty(o, k2, desc); }) : (function(o, m, k, k2) { if (k2 === undefined) k2 = k; o[k2] = m[k]; })); var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { Object.defineProperty(o, "default", { enumerable: true, value: v }); }) : function(o, v) { o["default"] = v; }); var __importStar = (this && this.__importStar) || (function () { var ownKeys = function(o) { ownKeys = Object.getOwnPropertyNames || function (o) { var ar = []; for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k; return ar; }; return ownKeys(o); }; return function (mod) { if (mod && mod.__esModule) return mod; var result = {}; if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]); __setModuleDefault(result, mod); return result; }; })(); Object.defineProperty(exports, "__esModule", { value: true }); exports.NetworkStack = void 0; const cdk = __importStar(require("aws-cdk-lib")); const ec2 = __importStar(require("aws-cdk-lib/aws-ec2")); const ecs = __importStar(require("aws-cdk-lib/aws-ecs")); const efs = __importStar(require("aws-cdk-lib/aws-efs")); const elbv2 = __importStar(require("aws-cdk-lib/aws-elasticloadbalancingv2")); const s3 = __importStar(require("aws-cdk-lib/aws-s3")); const acm = __importStar(require("aws-cdk-lib/aws-certificatemanager")); const route53 = __importStar(require("aws-cdk-lib/aws-route53")); const targets = __importStar(require("aws-cdk-lib/aws-route53-targets")); const logs = __importStar(require("aws-cdk-lib/aws-logs")); const iam = __importStar(require("aws-cdk-lib/aws-iam")); const events = __importStar(require("aws-cdk-lib/aws-events")); const eventsTargets = __importStar(require("aws-cdk-lib/aws-events-targets")); const cloudwatch = __importStar(require("aws-cdk-lib/aws-cloudwatch")); const cwactions = __importStar(require("aws-cdk-lib/aws-cloudwatch-actions")); const cr = __importStar(require("aws-cdk-lib/custom-resources")); const app_function_1 = require("./constructs/app-function"); const scratch_tmpfs_1 = require("./constructs/scratch-tmpfs"); const mirrored_image_1 = require("./constructs/mirrored-image"); const naming_1 = require("../lambdacode/network/naming"); const audit_log_1 = require("./audit-log"); /** The /api/* block MUST outrank every site's forward rule or Headscale's * admin API goes public. Site priorities: stable per-id hashes in 100–989. */ const BLOCK_ADMIN_PRIORITY = 10; const MAX_SITES = 90; // stays inside the ALB's 100-rules-per-listener quota /** Self-hosted Headscale, ONE PER SITE, sharing one cluster/EFS/ALB. Control * planes never carry user traffic; admin ops via ECS Exec — no admin port. */ class NetworkStack extends cdk.Stack { sites; /** Mints DEVICE keys for any site (siteId argument — its only caller, the * vending API role, binds userName to the verified JWT). */ deviceKeyMinter; /** Per-site CONNECTOR minters; each ignores input and is pinned to its * site's control plane by env. Safe to grant cross-account. */ connectorKeyMinters = {}; constructor(scope, id, props) { super(scope, id, props); const prefix = props.resourcePrefix; this.sites = props.sites; if (props.sites.length === 0) throw new Error("NetworkStack needs ≥1 site."); if (props.sites.length > MAX_SITES) { throw new Error(`NetworkStack supports at most ${MAX_SITES} sites.`); } // Public-subnets-only: outbound internet without NAT gateways to pay for. const vpc = new ec2.Vpc(this, "Vpc", { maxAzs: 2, natGateways: 0, // Tasks request their public IP explicitly (assignPublicIp); the subnet // itself hands none out (Security Hub EC2.15). subnetConfiguration: [ { name: "public", subnetType: ec2.SubnetType.PUBLIC, mapPublicIpOnLaunch: false }, ], // Explicit, so it never depends on the consumer's cdk.json flags (EC2.2). restrictDefaultSecurityGroup: true, }); // REJECT-only keeps the volume small; Security Hub EC2.6 asks for no more. vpc.addFlowLog("RejectFlowLog", { trafficType: ec2.FlowLogTrafficType.REJECT, destination: ec2.FlowLogDestination.toCloudWatchLogs(new logs.LogGroup(this, "VpcFlowLogs", { retention: logs.RetentionDays.THREE_MONTHS, })), }); // Headscale state is SQLite: one shared EFS, one ACCESS POINT PER SITE. // Each site's directory is precious — losing it re-enrolls that site. const fileSystem = new efs.FileSystem(this, "StateFs", { vpc, encrypted: true, // Security Hub EFS.2/EFS.7 (the state is megabytes). Not for sandboxes: // recovery points would outlive the destroyed stack for 35 days. enableAutomaticBackups: props.prod, removalPolicy: props.prod ? cdk.RemovalPolicy.RETAIN : cdk.RemovalPolicy.DESTROY, }); const cluster = new ecs.Cluster(this, "Cluster", { vpc, // Explicit either way, so turning the knob off actually disables it. containerInsightsV2: (props.containerInsights ?? props.prod) ? ecs.ContainerInsights.ENABLED : ecs.ContainerInsights.DISABLED, }); // Private DNS so the key minters/reconciler reach each Headscale WITHOUT // the public ALB (where /api/* is 403-blocked below). const namespace = cluster.addDefaultCloudMapNamespace({ name: "tailnet.internal", }); const zone = route53.HostedZone.fromHostedZoneAttributes(this, "Zone", { hostedZoneId: props.hostedZoneId, zoneName: props.zoneName, }); // One wildcard covers every site — adding one changes no cert, no DNS. const certificate = new acm.Certificate(this, "WildcardCert", { domainName: `*.${props.subdomain}.${props.zoneName}`, validation: acm.CertificateValidation.fromDns(zone), }); // The access-log bucket policy names the Region's ELB delivery account, and // a foreign-region connector VPC is placed by comparing Regions. if (cdk.Token.isUnresolved(this.region)) { throw new Error("NetworkStack needs an explicit env.region (ALB access logs, connector VPC placement)."); } const alb = new elbv2.ApplicationLoadBalancer(this, "Alb", { vpc, internetFacing: true, // Headscale clients long-poll; the default 60s idle would churn them. idleTimeout: cdk.Duration.seconds(3600), dropInvalidHeaderFields: true, // Security Hub ELB.4 deletionProtection: props.prod, // ELB.6 — disable before a prod teardown }); // ELB.5. SSE-S3 only: the ELB log-delivery principal cannot use a KMS key. const albLogs = new s3.Bucket(this, "AlbAccessLogs", { blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, enforceSSL: true, encryption: s3.BucketEncryption.S3_MANAGED, objectOwnership: s3.ObjectOwnership.BUCKET_OWNER_ENFORCED, lifecycleRules: [ { expiration: cdk.Duration.days(90), abortIncompleteMultipartUploadAfter: cdk.Duration.days(7) }, ], removalPolicy: props.prod ? cdk.RemovalPolicy.RETAIN : cdk.RemovalPolicy.DESTROY, autoDeleteObjects: !props.prod, // ACL-based delivery into the hub's log bucket: no policy that would // reference this bucket from DataStack (a cross-stack cycle). serverAccessLogsBucket: props.accessLogsBucket, serverAccessLogsPrefix: props.accessLogsBucket && "s3/alb-logs/", }); alb.logAccessLogs(albLogs, "alb"); // No port-80 listener at all: Security Hub ELB.18 counts even a redirect // listener as plaintext, and every client is handed an https URL. const listener = alb.addListener("Https", { port: 443, certificates: [certificate], sslPolicy: elbv2.SslPolicy.TLS13_12_RES_PQ, // ELB.17 no longer accepts TLS13_RES; TLS 1.2+ only open: true, // The wildcard record sends EVERY name here — an explicit 404 turns a // mistyped --login-server into a message instead of a hang. defaultAction: elbv2.ListenerAction.fixedResponse(404, { contentType: "text/plain", messageBody: "No site exists at this hostname.", }), }); // The admin API must not exist on the internet, for ANY site — admin // traffic goes Lambda → Cloud Map → task. Must outrank every site rule. listener.addAction("BlockAdminApi", { priority: BLOCK_ADMIN_PRIORITY, conditions: [elbv2.ListenerCondition.pathPatterns(["/api/*"])], action: elbv2.ListenerAction.fixedResponse(403, { contentType: "text/plain", messageBody: "Forbidden", }), }); new route53.ARecord(this, "WildcardAlias", { zone, recordName: `*.${props.subdomain}`, target: route53.RecordTarget.fromAlias(new targets.LoadBalancerTarget(alb)), }); /** siteId → VPC-internal admin API URL (Cloud Map, never the ALB). */ const apiHostOf = (siteId) => `headscale-${siteId}.${namespace.namespaceName}`; const apiUrlOf = (siteId) => `https://${apiHostOf(siteId)}:8080`; /** siteId → the SSM parameter the site's task publishes its current TLS certificate to; * the Lambdas verify the admin API against it (the pair is self-signed, re-minted per task start). */ const tlsParamOf = (siteId) => `/${prefix}/headscale-tls-${siteId}`; const tlsParamArnOf = (siteId) => this.formatArn({ service: "ssm", resource: "parameter", resourceName: `${prefix}/headscale-tls-${siteId}` }); const grantTlsRead = (fn, siteIds) => fn.addToRolePolicy(new iam.PolicyStatement({ actions: ["ssm:GetParameter"], resources: siteIds.map(tlsParamArnOf) })); /** siteId → the site's Headscale admin API key (Secrets Manager), minted/ * adopted by this stack's Custom::HeadscaleApiKey. */ const apiKeyOf = (siteId) => cdk.SecretValue.secretsManager(`${prefix}/headscale-apikey-${siteId}`).unsafeUnwrap(); const keyRotation = String(this.node.tryGetContext("headscaleKeyRotation") ?? "0"); const services = {}; const bootstrapInputs = {}; for (const site of props.sites) { if (site.priority <= BLOCK_ADMIN_PRIORITY) { throw new Error(`Site "${site.id}" priority ${site.priority} would outrank the ` + `admin-API block rule (${BLOCK_ADMIN_PRIORITY}).`); } } // Pinned: we ship control-plane updates deliberately, never :latest. One mirrored copy serves every site. // The -debug variant adds busybox (/busybox/sh): the server container runs // its own bootstrap, so no sidecar needs to reach its Unix socket. const HEADSCALE = "ghcr.io/juanfont/headscale:v0.26.1-debug"; // Every container is non-root (distroless `nonroot`) on a read-only root. // No shared task volume (a Fargate task volume is root-owned whatever the // image says): config and TLS pair sit on the EFS mount, whose // access point acts as uid 0 for any container user; the socket on tmpfs. // The CLI finds the config through the cwd, so ECS Exec needs no flag. const NONROOT = "65532"; const STATE = "/var/lib/headscale"; const SCRATCH = "/tmp"; // Debian slim ships the openssl CLI (via ca-certificates); busybox has none. const INIT_IMAGE = "public.ecr.aws/docker/library/python:3.13-slim"; const DERP_MAP_URL = "https://controlplane.tailscale.com/derpmap/default"; const DERP_FALLBACK = `${STATE}/derp-fallback.yaml`; // Saves the live relay map as the fallback; a failed or empty download keeps the old file and // never fails the task start. yaml.v3 matches lowercased Go field names and needs int region keys. const saveDerpMap = String.raw `import json, os, sys, time, urllib.request url, out = sys.argv[1], sys.argv[2] def y(v, ids=False): if isinstance(v, dict): return "{" + ", ".join(f"{int(k) if ids else json.dumps(k.lower())}: {y(x, k == 'Regions')}" for k, x in v.items()) + "}" if isinstance(v, list): return "[" + ", ".join(y(x) for x in v) + "]" return json.dumps(v) for attempt in range(3): try: with urllib.request.urlopen(url, timeout=10) as r: regions = json.load(r)["Regions"] if not regions: raise ValueError("no regions") with open(out + ".tmp", "w") as f: f.write(y({"Regions": regions}) + "\n") os.replace(out + ".tmp", out) print(f"relay map saved: {len(regions)} regions") break except Exception as e: print(f"relay map download failed ({type(e).__name__}): {e}") time.sleep(2) else: print("relay map not refreshed; keeping " + ("the saved copy" if os.path.exists(out) else "no saved copy"))`; const headscaleMirror = props.mirrorImages ? (0, mirrored_image_1.mirroredImage)(this, "HeadscaleImage", HEADSCALE) : undefined; const headscaleImage = (taskDef) => { if (!headscaleMirror) return ecs.ContainerImage.fromRegistry(HEADSCALE); taskDef.node.addDependency(headscaleMirror.copy); return headscaleMirror.image; }; props.sites.forEach((site) => { const accessPoint = fileSystem.addAccessPoint(`StateAp-${site.id}`, { path: `/headscale-${site.id}`, createAcl: { ownerUid: "0", ownerGid: "0", permissions: "755" }, posixUser: { uid: "0", gid: "0" }, }); const taskDef = new ecs.FargateTaskDefinition(this, `TaskDef-${site.id}`, { cpu: 256, memoryLimitMiB: 512, }); fileSystem.grantRootAccess(taskDef.taskRole); taskDef.taskRole.addToPrincipalPolicy(new iam.PolicyStatement({ actions: ["ssm:PutParameter"], resources: [tlsParamArnOf(site.id)] })); taskDef.addVolume({ name: "state", efsVolumeConfiguration: { fileSystemId: fileSystem.fileSystemId, transitEncryption: "ENABLED", authorizationConfig: { accessPointId: accessPoint.accessPointId, iam: "ENABLED", }, }, }); // Rendered by an init container at task start — keeps us on the stock // image (Headscale has no complete env-var interface). const configYaml = [ `server_url: ${site.controlUrl}`, `listen_addr: 0.0.0.0:8080`, `metrics_listen_addr: 127.0.0.1:9090`, `grpc_listen_addr: 127.0.0.1:50443`, `noise:`, ` private_key_path: /var/lib/headscale/noise_private.key`, `prefixes:`, ` v4: 100.64.0.0/10`, ` v6: fd7a:115c:a1e0::/48`, // Embedded DERP stays DISABLED: it needs UDP/3478 (STUN), which an // ALB cannot serve — enabling it would force an NLB redesign. // v0.26.1 swaps in an EMPTY map when a refresh download fails (#83). It merges the saved // copy and the URL region by region (the URL wins per region ID), so a region Tailscale // removes stays served from the copy until the next control-server start rewrites it. `derp:`, ` server:`, ` enabled: false`, ` paths:`, ` - ${DERP_FALLBACK}`, ` urls:`, ` - ${DERP_MAP_URL}`, ` auto_update_enabled: true`, ` update_frequency: 24h`, `disable_check_updates: true`, `ephemeral_node_inactivity_timeout: 30m`, `database:`, ` type: sqlite`, ` sqlite:`, ` path: /var/lib/headscale/db.sqlite`, `log:`, ` level: info`, // ACLs live in Headscale's DB and are PUT by the reconciler — never a // hand-edited policy file. Deny-by-default once a policy is set. `policy:`, ` mode: database`, `dns:`, ` magic_dns: true`, // Distinct per site — also what lets the desktop shell tell its saved // tailscale profiles apart when switching sites. ` base_domain: ${site.id}.tail.internal`, ` nameservers:`, ` global:`, ` - 1.1.1.1`, // Split DNS: PER SITE — overlapping VPCs collide on resolver IPs too. ...(Object.keys(site.splitDns).length ? [ ` split:`, ...Object.entries(site.splitDns).flatMap(([domain, servers]) => [ ` ${domain}:`, ...servers.map((ip) => ` - ${ip}`), ]), ] : []), `unix_socket: ${SCRATCH}/headscale.sock`, // Self-signed per task start: the ALB encrypts the hop but never // validates a target's certificate (ELB.21/.22); the Lambdas verify // against the copy the task publishes to SSM. `tls_cert_path: ${STATE}/tls.crt`, `tls_key_path: ${STATE}/tls.key`, ].join("\n"); const initContainer = taskDef.addContainer("write-config", { image: ecs.ContainerImage.fromRegistry(INIT_IMAGE), essential: false, command: [ "sh", "-ec", `cat > ${STATE}/config.yaml <<'EOF'\n${configYaml}\nEOF\n` + `openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 ` + `-subj "/CN=${apiHostOf(site.id)}" -addext "subjectAltName=DNS:${apiHostOf(site.id)},DNS:${site.fqdn}" ` + `-keyout ${STATE}/tls.key -out ${STATE}/tls.crt\n` + `python3 -I - ${DERP_MAP_URL} ${DERP_FALLBACK} <<'PY' || echo "relay map step failed"\n${saveDerpMap}\nPY`, ], user: NONROOT, readonlyRootFilesystem: true, logging: ecs.LogDrivers.awsLogs({ streamPrefix: `headscale-${site.id}-init`, logRetention: logs.RetentionDays.ONE_MONTH, }), }); initContainer.addMountPoints({ containerPath: STATE, sourceVolume: "state", readOnly: false }); // Serve, then — once the socket answers — the fresh-site CLI steps the // Custom::HeadscaleApiKey resource reads from this container's log stream // ("already exists" on every later start is fine). The trap hands the // task's SIGTERM to the server so SQLite closes cleanly. const serveScript = [ "headscale serve &", "pid=$!", // Re-wait after forwarding: the interrupted wait returns at once and // PID 1 exiting would SIGKILL the server before SQLite closes. "trap 'kill -TERM $pid; wait $pid' TERM INT", "n=0", "until headscale users list >/dev/null 2>&1; do", ' n=$((n+1)); if [ $n -ge 150 ]; then echo "headscale did not come up"; kill -TERM $pid; wait $pid; exit 1; fi', " sleep 2", "done", "headscale users create infra || true", // Marker line the Custom::HeadscaleApiKey handler matches on. 'echo "APIKEY=$(headscale apikeys create --expiration 87600h -o json)"', "wait $pid", ].join("\n"); const headscale = taskDef.addContainer("headscale", { image: headscaleImage(taskDef), entryPoint: ["/busybox/sh", "-c", serveScript], workingDirectory: STATE, user: NONROOT, readonlyRootFilesystem: true, linuxParameters: (0, scratch_tmpfs_1.scratchTmpfs)(this, `Tmpfs-${site.id}-headscale`, 64), // A label (not an env var — cdk-nag ECS2) so the rotation stamp // still forces a new task revision, hence a fresh key in the log. dockerLabels: { rotation: keyRotation }, portMappings: [{ containerPort: 8080 }], // The CLI answers only once the server serves (no curl in the image); // generous so a slow start never trips a restart loop. healthCheck: { command: ["CMD", "headscale", "users", "list"], interval: cdk.Duration.seconds(60), timeout: cdk.Duration.seconds(15), retries: 5, startPeriod: cdk.Duration.seconds(120), }, logging: ecs.LogDrivers.awsLogs({ streamPrefix: `headscale-${site.id}`, logRetention: logs.RetentionDays.ONE_MONTH, }), }); headscale.addMountPoints({ containerPath: STATE, sourceVolume: "state", readOnly: false }); // Publishes the fresh certificate before the server takes traffic, so a // Lambda's next call verifies against the pair this task actually serves. const publishCert = taskDef.addContainer("publish-cert", { image: ecs.ContainerImage.fromRegistry("public.ecr.aws/aws-cli/aws-cli:2.15.17"), essential: false, // Region as an argument, not an env var (cdk-nag ECS2); the uid has no // passwd entry here, which the CLI tolerates (no ~ to read or write). command: [ "--region", this.region, "ssm", "put-parameter", "--name", tlsParamOf(site.id), "--type", "String", "--overwrite", "--value", `file://${STATE}/tls.crt`, ], user: NONROOT, readonlyRootFilesystem: true, linuxParameters: (0, scratch_tmpfs_1.scratchTmpfs)(this, `Tmpfs-${site.id}-publish`, 16), logging: ecs.LogDrivers.awsLogs({ streamPrefix: `headscale-${site.id}-init`, logRetention: logs.RetentionDays.ONE_MONTH, }), }); publishCert.addMountPoints({ containerPath: STATE, sourceVolume: "state", readOnly: true }); publishCert.addContainerDependencies({ container: initContainer, condition: ecs.ContainerDependencyCondition.SUCCESS, }); headscale.addContainerDependencies({ container: publishCert, condition: ecs.ContainerDependencyCondition.SUCCESS, }); const service = new ecs.FargateService(this, `Service-${site.id}`, { cluster, taskDefinition: taskDef, desiredCount: 1, // SQLite on EFS = exactly one writer, ever: replace, don't overlap. minHealthyPercent: 0, maxHealthyPercent: 100, assignPublicIp: true, // public subnets, no NAT — needed for image pull vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC }, circuitBreaker: { rollback: true }, enableExecuteCommand: true, // admin CLI access — no admin port exposed cloudMapOptions: { name: `headscale-${site.id}` }, }); fileSystem.connections.allowDefaultPortFrom(service); services[site.id] = service; bootstrapInputs[site.id] = { logGroup: headscale.logDriverConfig.options["awslogs-group"], streamPrefix: `headscale-${site.id}`, }; listener.addTargets(`Headscale-${site.id}`, { priority: site.priority, conditions: [elbv2.ListenerCondition.hostHeaders([site.fqdn])], port: 8080, protocol: elbv2.ApplicationProtocol.HTTPS, targets: [service], healthCheck: { path: "/health", healthyHttpCodes: "200", protocol: elbv2.Protocol.HTTPS }, deregistrationDelay: cdk.Duration.seconds(30), }); new cdk.CfnOutput(this, `ControlUrl-${site.id}`, { value: site.controlUrl, }); }); const minterBase = { timeout: cdk.Duration.seconds(10), vpc, vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC }, allowPublicSubnet: true, // no NAT here; the fns need no internet, only VPC DNS }; // CFN re-resolves the secret only when the consuming RESOURCE changes — // rotate with `-c headscaleKeyRotation=$(date +%s)` or env keeps the old key. /** siteId → {url, key} map the device minter and reconciler read. */ const sitesEnv = JSON.stringify(Object.fromEntries(props.sites.map((s) => [s.id, { url: apiUrlOf(s.id), certParam: tlsParamOf(s.id), key: apiKeyOf(s.id) }]))); // Stores each site's server-minted admin API key (adopting a hand-made // secret); everything that resolves the secret into env waits on this. const bootstrapFn = new app_function_1.AppFunction(this, "HeadscaleApiKeyFn", { description: "Deploy-time helper: stores each VPN site's control-server admin API key in Secrets Manager.", asset: "headscale-apikey", timeout: cdk.Duration.minutes(14), }); const secretArn = (siteId) => `arn:aws:secretsmanager:${this.region}:${this.account}:secret:${prefix}/headscale-apikey-${siteId}-*`; bootstrapFn.addToRolePolicy(new iam.PolicyStatement({ actions: ["ecs:DescribeTasks", "ecs:DescribeServices", "ecs:ListTasks"], resources: ["*"], })); bootstrapFn.addToRolePolicy(new iam.PolicyStatement({ actions: ["logs:GetLogEvents"], resources: props.sites.map((s) => `arn:aws:logs:${this.region}:${this.account}:log-group:${bootstrapInputs[s.id].logGroup}:*`), })); bootstrapFn.addToRolePolicy(new iam.PolicyStatement({ actions: [ "secretsmanager:GetSecretValue", "secretsmanager:CreateSecret", "secretsmanager:PutSecretValue", ], resources: props.sites.map((s) => secretArn(s.id)), })); const bootstrapProvider = new cr.Provider(this, "HeadscaleApiKeyProvider", { onEventHandler: bootstrapFn, }); const bootstrapBySite = {}; for (const site of props.sites) { const { logGroup, streamPrefix } = bootstrapInputs[site.id]; const service = services[site.id]; const resource = new cdk.CustomResource(this, `HeadscaleBootstrap-${site.id}`, { serviceToken: bootstrapProvider.serviceToken, resourceType: "Custom::HeadscaleApiKey", properties: { SiteId: site.id, SecretName: `${prefix}/headscale-apikey-${site.id}`, ClusterArn: cluster.clusterArn, ServiceName: service.serviceName, LogGroupName: logGroup, StreamPrefix: streamPrefix, Rotation: keyRotation, }, }); // Resource-level edge on purpose: a construct-level one pulls the // service SG's ingress (referencing the minters' SGs) into a cycle. resource.node.addDependency(service.node.defaultChild); bootstrapBySite[site.id] = resource; new cdk.CfnOutput(this, `HeadscaleApiKey-${site.id}`, { value: resource.getAttString("Source"), }); } const bootstraps = Object.values(bootstrapBySite); /** Make only a function's own CloudFormation resource wait for the key. */ const waitForKey = (fn, ...on) => { const cfn = fn.node.defaultChild; for (const r of on) cfn.addResourceDependency(r.node.defaultChild); }; // ONE device minter for all sites: its only caller (the vending API role) // binds userName to the verified JWT, and users enroll wherever granted. this.deviceKeyMinter = new app_function_1.AppFunction(this, "DeviceKeyMinter", { ...minterBase, functionName: (0, naming_1.deviceKeyMinterFnName)(prefix), asset: "keyminter", description: "Mints a one-time VPN enrollment key for a signed-in user's device.", handler: "deviceHandler", environment: { HEADSCALE_SITES: sitesEnv, HEADSCALE_KEY_ROTATION: keyRotation, NETWORK_RECONCILER_FN_NAME: (0, naming_1.aclReconcilerFnName)(prefix), }, }); waitForKey(this.deviceKeyMinter, ...bootstraps); grantTlsRead(this.deviceKeyMinter, props.sites.map((s) => s.id)); // A user's first mint on a site starts a reconciler pass (#76). By name, through the VPC's Lambda endpoint. this.deviceKeyMinter.addToRolePolicy(new iam.PolicyStatement({ actions: ["lambda:InvokeFunction"], resources: [ this.formatArn({ service: "lambda", resource: "function", resourceName: (0, naming_1.aclReconcilerFnName)(prefix), arnFormat: cdk.ArnFormat.COLON_RESOURCE_NAME, }), ], })); // One connector minter PER SITE, env-pinned, input ignored — which site a // connector joins = which minter its account may invoke. for (const site of props.sites) { const minter = new app_function_1.AppFunction(this, `ConnectorKeyMinter-${site.id}`, { ...minterBase, functionName: (0, naming_1.connectorKeyMinterFnName)(prefix, site.id), asset: "keyminter", description: `Mints VPN enrollment keys for connectors joining site ${site.id}.`, handler: "connectorHandler", environment: { HEADSCALE_API_URL: apiUrlOf(site.id), HEADSCALE_API_KEY: apiKeyOf(site.id), HEADSCALE_TLS_PARAM: tlsParamOf(site.id), HEADSCALE_USER_ID: "1", // the `infra` machine-owner user HEADSCALE_KEY_ROTATION: keyRotation, }, }); waitForKey(minter, bootstrapBySite[site.id]); grantTlsRead(minter, [site.id]); this.connectorKeyMinters[site.id] = minter; services[site.id].connections.allowFrom(minter, ec2.Port.tcp(8080)); services[site.id].connections.allowFrom(this.deviceKeyMinter, ec2.Port.tcp(8080)); new cdk.CfnOutput(this, `ConnectorKeyMinterArn-${site.id}`, { value: minter.functionArn, }); } const reconcilerRole = new iam.Role(this, "AclReconcilerRole", { roleName: (0, naming_1.aclReconcilerRoleName)(prefix), assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"), managedPolicies: [ iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaVPCAccessExecutionRole"), ], }); // Every account's connector VPCs by name and (foreign) region — the reconciler's fan-out targets, and // the (account, region) pairs the out-of-VPC relay may address (why a relay: wiki Private Network, // "Connector VPCs in another region"). const connectorVpcs = {}; // `<account>#<name or empty>` → the routes config declares for that VPC: all the reconciler will approve for it. const connectorRoutes = {}; for (const s of props.sites) { for (const [account, list] of Object.entries(s.connectorVpcs)) { for (const vpc of list) connectorRoutes[`${account}#${vpc.name ?? ""}`] = vpc.routes; connectorVpcs[account] = list.map((vpc) => { // JSON.stringify drops the undefineds: an unnamed hub-region VPC travels as `{}`. return { name: vpc.name, region: vpc.region && vpc.region !== this.region ? vpc.region : undefined, }; }); } } const relayRegions = [ ...new Set(Object.values(connectorVpcs) .flat() .map((c) => c.region) .filter((r) => Boolean(r))), ].sort(); const reconciler = new app_function_1.AppFunction(this, "AclReconciler", { functionName: (0, naming_1.aclReconcilerFnName)(prefix), description: "Keeps each VPN site's access rules, routes and security-group rules in line with the portal's grants.", asset: "acl-reconciler", handler: "handler", role: reconcilerRole, // Per-site budget: the pass runs the SG-manager AND tailnet-resolver // fan-outs (each per-account invoke aborts at 10s) plus up to ~6s of // sequential local DNS fallbacks per unresolvable dnsName resource. timeout: cdk.Duration.seconds(Math.min(90 * props.sites.length, 900)), // One pass at a time: the API kicks a pass on every dev-share create/stop, and two passes racing on // the whole-policy PUT could re-publish a rule the later one just removed. reservedConcurrentExecutions: 1, vpc, vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC }, allowPublicSubnet: true, // like the key minters: needs VPC DNS, not internet environment: { RESOURCE_PREFIX: prefix, // by-name SG-manager target derivation HEADSCALE_SITES: sitesEnv, HEADSCALE_KEY_ROTATION: keyRotation, // Only this account's names may be resolved from inside this VPC (Cloud Map, plain DNS): // another account's private zone can shadow a label of the hub's. HUB_ACCOUNT: this.account, // accountNumber → siteId: THE site-membership source of truth. SITE_ACCOUNTS: JSON.stringify(Object.fromEntries(props.sites.flatMap((s) => s.accounts.map((a) => [a, s.id])))), // accountNumber → its connector VPCs ({ name?, region? }; region only outside this region). CONNECTOR_VPCS: JSON.stringify(connectorVpcs), CONNECTOR_ROUTES: JSON.stringify(connectorRoutes), // siteId → displayName: how the activity log names a site. SITE_NAMES: JSON.stringify(Object.fromEntries(props.sites.map((s) => [s.id, s.displayName]))), RESOURCES_TABLE_NAME: props.resourcesTable.tableName, ACCESS_TABLE_NAME: props.accessTable.tableName, GROUPS_TABLE_NAME: props.groupsTable.tableName, // Resolvers stay reachable on :53 for everyone ON THAT SITE, or // name-based access dies before any granted port is tried. SPLIT_DNS_BY_SITE: JSON.stringify(Object.fromEntries(props.sites.map((s) => [ s.id, [...new Set(Object.values(s.splitDns).flat())], ]))), ...(props.auditLog ? (0, audit_log_1.auditLogEnv)(props.auditLog) : {}), }, }); if (props.auditLog) (0, audit_log_1.grantAuditLogWrite)(reconciler, props.auditLog); waitForKey(reconciler, ...bootstraps); grantTlsRead(reconciler, props.sites.map((s) => s.id)); for (const site of props.sites) { services[site.id].connections.allowFrom(reconciler, ec2.Port.tcp(8080)); } // ReadWrite: the reconciler writes per-site status sentinels into Resources. props.resourcesTable.grantReadWriteData(reconciler); props.accessTable.grantReadData(reconciler); props.groupsTable.grantReadData(reconciler); // Cloud Map re-resolution of resource dnsNames (Fargate IP drift): // DiscoverInstances has no resource-level scoping. reconciler.addToRolePolicy(new iam.PolicyStatement({ actions: ["servicediscovery:DiscoverInstances"], resources: ["*"], })); // SG-manager and tailnet-resolver invokes by well-known name in ANY account — // reaching one still requires that account's resource policy (pinned to this role). reconciler.addToRolePolicy(new iam.PolicyStatement({ actions: ["lambda:InvokeFunction"], resources: [ `arn:aws:lambda:${this.region}:*:function:${(0, naming_1.sgManagerFnName)(prefix)}`, // Named connector VPCs suffix the manager; per-site (and per-name) // resolvers — the wildcards cover every one. `arn:aws:lambda:${this.region}:*:function:${(0, naming_1.sgManagerFnName)(prefix, "*")}`, `arn:aws:lambda:${this.region}:*:function:${(0, naming_1.tailnetResolverFnName)(prefix, "*")}`, ], })); // The relay's invoke-only role. Connector accounts in another region pin their trust to it // by name, so it exists even without a relay (Lambda validates the principal at permission time). const relayRole = new iam.Role(this, "AclReconcilerRelayRole", { roleName: (0, naming_1.aclReconcilerRelayRoleName)(prefix), assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"), managedPolicies: [ iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaBasicExecutionRole"), ], }); if (relayRegions.length) { relayRole.addToPolicy(new iam.PolicyStatement({ actions: ["lambda:InvokeFunction"], resources: relayRegions.flatMap((region) => [ `arn:aws:lambda:${region}:*:function:${(0, naming_1.sgManagerFnName)(prefix)}`, `arn:aws:lambda:${region}:*:function:${(0, naming_1.sgManagerFnName)(prefix, "*")}`, `arn:aws:lambda:${region}:*:function:${(0, naming_1.tailnetResolverFnName)(prefix, "*")}`, ]), })); const relay = new app_function_1.AppFunction(this, "InvokeRelay", { functionName: (0, naming_1.aclReconcilerRelayFnName)(prefix), description: "Forwards the VPN reconciler's calls to connector functions in other regions.", asset: "invoke-relay", handler: "handler", role: relayRole, // The reconciler's per-target budget (INVOKE_TIMEOUT_MS) plus this function's own start. timeout: cdk.Duration.seconds(12), environment: { RESOURCE_PREFIX: prefix, CONNECTOR_VPCS: JSON.stringify(connectorVpcs), }, }); reconciler.addToRolePolicy(new iam.PolicyStatement({ actions: ["lambda:InvokeFunction"], resources: [relay.functionArn], })); } // The Lambdas have no internet path — DiscoverInstances needs Cloud Map's // data endpoint. One subnet = one ENI (~$8/mo) serving the whole VPC. vpc.addInterfaceEndpoint("CloudMapDataEndpoint", { service: new ec2.InterfaceVpcEndpointAwsService("data-servicediscovery"), subnets: { subnets: [vpc.publicSubnets[0]] }, }); // Lambda invokes (the SG-managers, the device minter's reconciler start) need Lambda's API from this // no-internet VPC — same one-ENI pattern as the Cloud Map endpoint. vpc.addInterfaceEndpoint("LambdaEndpoint", { service: ec2.InterfaceVpcEndpointAwsService.LAMBDA, subnets: { subnets: [vpc.publicSubnets[0]] }, }); // The minters and the reconciler read each site's TLS certificate from // SSM before calling Headscale; without this endpoint that read hangs. vpc.addInterfaceEndpoint("SsmEndpoint", { service: ec2.InterfaceVpcEndpointAwsService.SSM, subnets: { subnets: [vpc.publicSubnets[0]] }, }); // DynamoDB from inside the VPC without internet: free gateway endpoint. vpc.addGatewayEndpoint("DynamoDbEndpoint", { service: ec2.GatewayVpcEndpointAwsService.DYNAMODB, }); new events.Rule(this, "ReconcileSchedule", { schedule: events.Schedule.rate(cdk.Duration.minutes(5)), targets: [new eventsTargets.LambdaFunction(reconciler)], }); // A broken reconciler freezes reachability INCLUDING revocations; one // failure self-heals next pass, so alarm on two consecutive. if (props.alertTopic) { const notify = new cwactions.SnsAction(props.alertTopic); const alarm = new cloudwatch.Alarm(this, "ReconcilerErrors", { metric: reconciler.metricErrors({ period: cdk.Duration.minutes(15) }), threshold: 2, evaluationPeriods: 1, comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, alarmDescription: "The ACL reconciler has failed 2+ consecutive passes — tailnet " + "reachability is frozen at the last synced state, so grant changes " + "AND revocations are not taking effect. Check its CloudWatch logs " + "and the Headscale services.", }); alarm.addAlarmAction(notify); alarm.addOkAction(notify); // Zombie: "online" but silent — forwards on STALE rules and never learns new device keys. EMF // metric from the reconciler, summed; a node a live replacement superseded is not in it (#70). const staleAlarm = new cloudwatch.Alarm(this, "StaleConnectors", { metric: new cloudwatch.Metric({ namespace: "InfraClient/Network", metricName: "StaleConnectors", statistic: "Maximum", period: cdk.Duration.minutes(15), }), threshold: 1, evaluationPeriods: 1, comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, alarmDescription: "A connector node with approved routes has been OFFLINE 15+ " + "minutes with no online replacement in its account and VPC " + "(zombie): it enforces stale rules and won't learn " + "re-enrolled devices. Restart it: aws ecs update-service " + "--force-new-deployment on the connector service; the reconciler " + "auto-approves the replacement's routes.", }); staleAlarm.addAlarmAction(notify); staleAlarm.addOkAction(notify); const derpMapFailures = { namespace: "InfraClient/Network", metricName: "DerpMapLoadFailures" }; for (const [siteId, { logGroup }] of Object.entries(bootstrapInputs)) { new logs.MetricFilter(this, `DerpMapLoadFailures-${siteId}`, { logGroup: logs.LogGroup.fromLogGroupName(this, `ControlServerLogs-${siteId}`, logGroup), filterPattern: logs.FilterPattern.literal('"Could not load DERP map"'), metricNamespace: derpMapFailures.namespace, metricName: derpMapFailures.metricName, metricValue: "1", }); } const derpMapAlarm = new cloudwatch.Alarm(this, "DerpMapLoadFailed", { metric: new cloudwatch.Metric({ ...derpMapFailures, statistic: "Sum", period: cdk.Duration.minutes(5), }), threshold: 1, evaluationPeriods: 1, comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING, alarmDescription: "A VPN control server could not load its relay (DERP) map: the " + "download from controlplane.tailscale.com failed, or the saved " + "fallback copy on EFS is missing. With a saved copy, nodes keep " + "relays; without one, the site has none until the next refresh. " + "Check the site's control-server log group. Without a saved copy, " + "force a new deployment of its service once the URL loads again. " + "OK only means no new failure was logged; a refresh runs every 24 h.", }); derpMapAlarm.addAlarmAction(notify); derpMapAlarm.addOkAction(notify); } new cdk.CfnOutput(this, "ClusterName", { value: cluster.clusterName }); new cdk.CfnOutput(this, "DeviceKeyMinterArn", { value: this.deviceKeyMinter.functionArn, }); } /** Grant invoke on ONE SITE's connector minter (machine keys only — never * the device minter). Prefer a task-role ARN over a whole-account id. */ grantConnectorAccount(siteId, connectorPrincipal) { const minter = this.connectorKeyMinters[siteId]; if (!minter) throw new Error(`No such site: ${siteId}`); const principal = connectorPrincipal.includes(":") ? new iam.ArnPrincipal(connectorPrincipal) // specific role ARN (preferred) : new iam.AccountPrincipal(connectorPrincipal); // whole account (coarser) // Construct ids must be alphanumeric; keep them stable + unique per grant. const suffix = connectorPrincipal.replace(/[^a-zA-Z0-9]/g, "").slice(-24); minter.addPermission(`ConnectorInvoke${suffix}`, { principal, action: "lambda:InvokeFunction", }); } } exports.NetworkStack = NetworkStack;