@fntio/infra-client
Version:
Self-hosted AWS access portal: credential vending, target-account trust, and grant-driven private networking as CDK stacks and constructs.
858 lines (857 loc) • 48.1 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.NetworkStack = void 0;
const cdk = __importStar(require("aws-cdk-lib"));
const ec2 = __importStar(require("aws-cdk-lib/aws-ec2"));
const ecs = __importStar(require("aws-cdk-lib/aws-ecs"));
const efs = __importStar(require("aws-cdk-lib/aws-efs"));
const elbv2 = __importStar(require("aws-cdk-lib/aws-elasticloadbalancingv2"));
const s3 = __importStar(require("aws-cdk-lib/aws-s3"));
const acm = __importStar(require("aws-cdk-lib/aws-certificatemanager"));
const route53 = __importStar(require("aws-cdk-lib/aws-route53"));
const targets = __importStar(require("aws-cdk-lib/aws-route53-targets"));
const logs = __importStar(require("aws-cdk-lib/aws-logs"));
const iam = __importStar(require("aws-cdk-lib/aws-iam"));
const events = __importStar(require("aws-cdk-lib/aws-events"));
const eventsTargets = __importStar(require("aws-cdk-lib/aws-events-targets"));
const cloudwatch = __importStar(require("aws-cdk-lib/aws-cloudwatch"));
const cwactions = __importStar(require("aws-cdk-lib/aws-cloudwatch-actions"));
const cr = __importStar(require("aws-cdk-lib/custom-resources"));
const app_function_1 = require("./constructs/app-function");
const scratch_tmpfs_1 = require("./constructs/scratch-tmpfs");
const mirrored_image_1 = require("./constructs/mirrored-image");
const naming_1 = require("../lambdacode/network/naming");
const audit_log_1 = require("./audit-log");
/** The /api/* block MUST outrank every site's forward rule or Headscale's
* admin API goes public. Site priorities: stable per-id hashes in 100–989. */
const BLOCK_ADMIN_PRIORITY = 10;
const MAX_SITES = 90; // stays inside the ALB's 100-rules-per-listener quota
/** Self-hosted Headscale, ONE PER SITE, sharing one cluster/EFS/ALB. Control
* planes never carry user traffic; admin ops via ECS Exec — no admin port. */
class NetworkStack extends cdk.Stack {
sites;
/** Mints DEVICE keys for any site (siteId argument — its only caller, the
* vending API role, binds userName to the verified JWT). */
deviceKeyMinter;
/** Per-site CONNECTOR minters; each ignores input and is pinned to its
* site's control plane by env. Safe to grant cross-account. */
connectorKeyMinters = {};
constructor(scope, id, props) {
super(scope, id, props);
const prefix = props.resourcePrefix;
this.sites = props.sites;
if (props.sites.length === 0)
throw new Error("NetworkStack needs ≥1 site.");
if (props.sites.length > MAX_SITES) {
throw new Error(`NetworkStack supports at most ${MAX_SITES} sites.`);
}
// Public-subnets-only: outbound internet without NAT gateways to pay for.
const vpc = new ec2.Vpc(this, "Vpc", {
maxAzs: 2,
natGateways: 0,
// Tasks request their public IP explicitly (assignPublicIp); the subnet
// itself hands none out (Security Hub EC2.15).
subnetConfiguration: [
{ name: "public", subnetType: ec2.SubnetType.PUBLIC, mapPublicIpOnLaunch: false },
],
// Explicit, so it never depends on the consumer's cdk.json flags (EC2.2).
restrictDefaultSecurityGroup: true,
});
// REJECT-only keeps the volume small; Security Hub EC2.6 asks for no more.
vpc.addFlowLog("RejectFlowLog", {
trafficType: ec2.FlowLogTrafficType.REJECT,
destination: ec2.FlowLogDestination.toCloudWatchLogs(new logs.LogGroup(this, "VpcFlowLogs", {
retention: logs.RetentionDays.THREE_MONTHS,
})),
});
// Headscale state is SQLite: one shared EFS, one ACCESS POINT PER SITE.
// Each site's directory is precious — losing it re-enrolls that site.
const fileSystem = new efs.FileSystem(this, "StateFs", {
vpc,
encrypted: true,
// Security Hub EFS.2/EFS.7 (the state is megabytes). Not for sandboxes:
// recovery points would outlive the destroyed stack for 35 days.
enableAutomaticBackups: props.prod,
removalPolicy: props.prod
? cdk.RemovalPolicy.RETAIN
: cdk.RemovalPolicy.DESTROY,
});
const cluster = new ecs.Cluster(this, "Cluster", {
vpc,
// Explicit either way, so turning the knob off actually disables it.
containerInsightsV2: (props.containerInsights ?? props.prod)
? ecs.ContainerInsights.ENABLED
: ecs.ContainerInsights.DISABLED,
});
// Private DNS so the key minters/reconciler reach each Headscale WITHOUT
// the public ALB (where /api/* is 403-blocked below).
const namespace = cluster.addDefaultCloudMapNamespace({
name: "tailnet.internal",
});
const zone = route53.HostedZone.fromHostedZoneAttributes(this, "Zone", {
hostedZoneId: props.hostedZoneId,
zoneName: props.zoneName,
});
// One wildcard covers every site — adding one changes no cert, no DNS.
const certificate = new acm.Certificate(this, "WildcardCert", {
domainName: `*.${props.subdomain}.${props.zoneName}`,
validation: acm.CertificateValidation.fromDns(zone),
});
// The access-log bucket policy names the Region's ELB delivery account, and
// a foreign-region connector VPC is placed by comparing Regions.
if (cdk.Token.isUnresolved(this.region)) {
throw new Error("NetworkStack needs an explicit env.region (ALB access logs, connector VPC placement).");
}
const alb = new elbv2.ApplicationLoadBalancer(this, "Alb", {
vpc,
internetFacing: true,
// Headscale clients long-poll; the default 60s idle would churn them.
idleTimeout: cdk.Duration.seconds(3600),
dropInvalidHeaderFields: true, // Security Hub ELB.4
deletionProtection: props.prod, // ELB.6 — disable before a prod teardown
});
// ELB.5. SSE-S3 only: the ELB log-delivery principal cannot use a KMS key.
const albLogs = new s3.Bucket(this, "AlbAccessLogs", {
blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
enforceSSL: true,
encryption: s3.BucketEncryption.S3_MANAGED,
objectOwnership: s3.ObjectOwnership.BUCKET_OWNER_ENFORCED,
lifecycleRules: [
{ expiration: cdk.Duration.days(90), abortIncompleteMultipartUploadAfter: cdk.Duration.days(7) },
],
removalPolicy: props.prod ? cdk.RemovalPolicy.RETAIN : cdk.RemovalPolicy.DESTROY,
autoDeleteObjects: !props.prod,
// ACL-based delivery into the hub's log bucket: no policy that would
// reference this bucket from DataStack (a cross-stack cycle).
serverAccessLogsBucket: props.accessLogsBucket,
serverAccessLogsPrefix: props.accessLogsBucket && "s3/alb-logs/",
});
alb.logAccessLogs(albLogs, "alb");
// No port-80 listener at all: Security Hub ELB.18 counts even a redirect
// listener as plaintext, and every client is handed an https URL.
const listener = alb.addListener("Https", {
port: 443,
certificates: [certificate],
sslPolicy: elbv2.SslPolicy.TLS13_12_RES_PQ, // ELB.17 no longer accepts TLS13_RES; TLS 1.2+ only
open: true,
// The wildcard record sends EVERY name here — an explicit 404 turns a
// mistyped --login-server into a message instead of a hang.
defaultAction: elbv2.ListenerAction.fixedResponse(404, {
contentType: "text/plain",
messageBody: "No site exists at this hostname.",
}),
});
// The admin API must not exist on the internet, for ANY site — admin
// traffic goes Lambda → Cloud Map → task. Must outrank every site rule.
listener.addAction("BlockAdminApi", {
priority: BLOCK_ADMIN_PRIORITY,
conditions: [elbv2.ListenerCondition.pathPatterns(["/api/*"])],
action: elbv2.ListenerAction.fixedResponse(403, {
contentType: "text/plain",
messageBody: "Forbidden",
}),
});
new route53.ARecord(this, "WildcardAlias", {
zone,
recordName: `*.${props.subdomain}`,
target: route53.RecordTarget.fromAlias(new targets.LoadBalancerTarget(alb)),
});
/** siteId → VPC-internal admin API URL (Cloud Map, never the ALB). */
const apiHostOf = (siteId) => `headscale-${siteId}.${namespace.namespaceName}`;
const apiUrlOf = (siteId) => `https://${apiHostOf(siteId)}:8080`;
/** siteId → the SSM parameter the site's task publishes its current TLS certificate to;
* the Lambdas verify the admin API against it (the pair is self-signed, re-minted per task start). */
const tlsParamOf = (siteId) => `/${prefix}/headscale-tls-${siteId}`;
const tlsParamArnOf = (siteId) => this.formatArn({ service: "ssm", resource: "parameter", resourceName: `${prefix}/headscale-tls-${siteId}` });
const grantTlsRead = (fn, siteIds) => fn.addToRolePolicy(new iam.PolicyStatement({ actions: ["ssm:GetParameter"], resources: siteIds.map(tlsParamArnOf) }));
/** siteId → the site's Headscale admin API key (Secrets Manager), minted/
* adopted by this stack's Custom::HeadscaleApiKey. */
const apiKeyOf = (siteId) => cdk.SecretValue.secretsManager(`${prefix}/headscale-apikey-${siteId}`).unsafeUnwrap();
const keyRotation = String(this.node.tryGetContext("headscaleKeyRotation") ?? "0");
const services = {};
const bootstrapInputs = {};
for (const site of props.sites) {
if (site.priority <= BLOCK_ADMIN_PRIORITY) {
throw new Error(`Site "${site.id}" priority ${site.priority} would outrank the ` +
`admin-API block rule (${BLOCK_ADMIN_PRIORITY}).`);
}
}
// Pinned: we ship control-plane updates deliberately, never :latest. One mirrored copy serves every site.
// The -debug variant adds busybox (/busybox/sh): the server container runs
// its own bootstrap, so no sidecar needs to reach its Unix socket.
const HEADSCALE = "ghcr.io/juanfont/headscale:v0.26.1-debug";
// Every container is non-root (distroless `nonroot`) on a read-only root.
// No shared task volume (a Fargate task volume is root-owned whatever the
// image says): config and TLS pair sit on the EFS mount, whose
// access point acts as uid 0 for any container user; the socket on tmpfs.
// The CLI finds the config through the cwd, so ECS Exec needs no flag.
const NONROOT = "65532";
const STATE = "/var/lib/headscale";
const SCRATCH = "/tmp";
// Debian slim ships the openssl CLI (via ca-certificates); busybox has none.
const INIT_IMAGE = "public.ecr.aws/docker/library/python:3.13-slim";
const DERP_MAP_URL = "https://controlplane.tailscale.com/derpmap/default";
const DERP_FALLBACK = `${STATE}/derp-fallback.yaml`;
// Saves the live relay map as the fallback; a failed or empty download keeps the old file and
// never fails the task start. yaml.v3 matches lowercased Go field names and needs int region keys.
const saveDerpMap = String.raw `import json, os, sys, time, urllib.request
url, out = sys.argv[1], sys.argv[2]
def y(v, ids=False):
if isinstance(v, dict):
return "{" + ", ".join(f"{int(k) if ids else json.dumps(k.lower())}: {y(x, k == 'Regions')}" for k, x in v.items()) + "}"
if isinstance(v, list):
return "[" + ", ".join(y(x) for x in v) + "]"
return json.dumps(v)
for attempt in range(3):
try:
with urllib.request.urlopen(url, timeout=10) as r:
regions = json.load(r)["Regions"]
if not regions:
raise ValueError("no regions")
with open(out + ".tmp", "w") as f:
f.write(y({"Regions": regions}) + "\n")
os.replace(out + ".tmp", out)
print(f"relay map saved: {len(regions)} regions")
break
except Exception as e:
print(f"relay map download failed ({type(e).__name__}): {e}")
time.sleep(2)
else:
print("relay map not refreshed; keeping " + ("the saved copy" if os.path.exists(out) else "no saved copy"))`;
const headscaleMirror = props.mirrorImages ? (0, mirrored_image_1.mirroredImage)(this, "HeadscaleImage", HEADSCALE) : undefined;
const headscaleImage = (taskDef) => {
if (!headscaleMirror)
return ecs.ContainerImage.fromRegistry(HEADSCALE);
taskDef.node.addDependency(headscaleMirror.copy);
return headscaleMirror.image;
};
props.sites.forEach((site) => {
const accessPoint = fileSystem.addAccessPoint(`StateAp-${site.id}`, {
path: `/headscale-${site.id}`,
createAcl: { ownerUid: "0", ownerGid: "0", permissions: "755" },
posixUser: { uid: "0", gid: "0" },
});
const taskDef = new ecs.FargateTaskDefinition(this, `TaskDef-${site.id}`, {
cpu: 256,
memoryLimitMiB: 512,
});
fileSystem.grantRootAccess(taskDef.taskRole);
taskDef.taskRole.addToPrincipalPolicy(new iam.PolicyStatement({ actions: ["ssm:PutParameter"], resources: [tlsParamArnOf(site.id)] }));
taskDef.addVolume({
name: "state",
efsVolumeConfiguration: {
fileSystemId: fileSystem.fileSystemId,
transitEncryption: "ENABLED",
authorizationConfig: {
accessPointId: accessPoint.accessPointId,
iam: "ENABLED",
},
},
});
// Rendered by an init container at task start — keeps us on the stock
// image (Headscale has no complete env-var interface).
const configYaml = [
`server_url: ${site.controlUrl}`,
`listen_addr: 0.0.0.0:8080`,
`metrics_listen_addr: 127.0.0.1:9090`,
`grpc_listen_addr: 127.0.0.1:50443`,
`noise:`,
` private_key_path: /var/lib/headscale/noise_private.key`,
`prefixes:`,
` v4: 100.64.0.0/10`,
` v6: fd7a:115c:a1e0::/48`,
// Embedded DERP stays DISABLED: it needs UDP/3478 (STUN), which an
// ALB cannot serve — enabling it would force an NLB redesign.
// v0.26.1 swaps in an EMPTY map when a refresh download fails (#83). It merges the saved
// copy and the URL region by region (the URL wins per region ID), so a region Tailscale
// removes stays served from the copy until the next control-server start rewrites it.
`derp:`,
` server:`,
` enabled: false`,
` paths:`,
` - ${DERP_FALLBACK}`,
` urls:`,
` - ${DERP_MAP_URL}`,
` auto_update_enabled: true`,
` update_frequency: 24h`,
`disable_check_updates: true`,
`ephemeral_node_inactivity_timeout: 30m`,
`database:`,
` type: sqlite`,
` sqlite:`,
` path: /var/lib/headscale/db.sqlite`,
`log:`,
` level: info`,
// ACLs live in Headscale's DB and are PUT by the reconciler — never a
// hand-edited policy file. Deny-by-default once a policy is set.
`policy:`,
` mode: database`,
`dns:`,
` magic_dns: true`,
// Distinct per site — also what lets the desktop shell tell its saved
// tailscale profiles apart when switching sites.
` base_domain: ${site.id}.tail.internal`,
` nameservers:`,
` global:`,
` - 1.1.1.1`,
// Split DNS: PER SITE — overlapping VPCs collide on resolver IPs too.
...(Object.keys(site.splitDns).length
? [
` split:`,
...Object.entries(site.splitDns).flatMap(([domain, servers]) => [
` ${domain}:`,
...servers.map((ip) => ` - ${ip}`),
]),
]
: []),
`unix_socket: ${SCRATCH}/headscale.sock`,
// Self-signed per task start: the ALB encrypts the hop but never
// validates a target's certificate (ELB.21/.22); the Lambdas verify
// against the copy the task publishes to SSM.
`tls_cert_path: ${STATE}/tls.crt`,
`tls_key_path: ${STATE}/tls.key`,
].join("\n");
const initContainer = taskDef.addContainer("write-config", {
image: ecs.ContainerImage.fromRegistry(INIT_IMAGE),
essential: false,
command: [
"sh",
"-ec",
`cat > ${STATE}/config.yaml <<'EOF'\n${configYaml}\nEOF\n` +
`openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 ` +
`-subj "/CN=${apiHostOf(site.id)}" -addext "subjectAltName=DNS:${apiHostOf(site.id)},DNS:${site.fqdn}" ` +
`-keyout ${STATE}/tls.key -out ${STATE}/tls.crt\n` +
`python3 -I - ${DERP_MAP_URL} ${DERP_FALLBACK} <<'PY' || echo "relay map step failed"\n${saveDerpMap}\nPY`,
],
user: NONROOT,
readonlyRootFilesystem: true,
logging: ecs.LogDrivers.awsLogs({
streamPrefix: `headscale-${site.id}-init`,
logRetention: logs.RetentionDays.ONE_MONTH,
}),
});
initContainer.addMountPoints({ containerPath: STATE, sourceVolume: "state", readOnly: false });
// Serve, then — once the socket answers — the fresh-site CLI steps the
// Custom::HeadscaleApiKey resource reads from this container's log stream
// ("already exists" on every later start is fine). The trap hands the
// task's SIGTERM to the server so SQLite closes cleanly.
const serveScript = [
"headscale serve &",
"pid=$!",
// Re-wait after forwarding: the interrupted wait returns at once and
// PID 1 exiting would SIGKILL the server before SQLite closes.
"trap 'kill -TERM $pid; wait $pid' TERM INT",
"n=0",
"until headscale users list >/dev/null 2>&1; do",
' n=$((n+1)); if [ $n -ge 150 ]; then echo "headscale did not come up"; kill -TERM $pid; wait $pid; exit 1; fi',
" sleep 2",
"done",
"headscale users create infra || true",
// Marker line the Custom::HeadscaleApiKey handler matches on.
'echo "APIKEY=$(headscale apikeys create --expiration 87600h -o json)"',
"wait $pid",
].join("\n");
const headscale = taskDef.addContainer("headscale", {
image: headscaleImage(taskDef),
entryPoint: ["/busybox/sh", "-c", serveScript],
workingDirectory: STATE,
user: NONROOT,
readonlyRootFilesystem: true,
linuxParameters: (0, scratch_tmpfs_1.scratchTmpfs)(this, `Tmpfs-${site.id}-headscale`, 64),
// A label (not an env var — cdk-nag ECS2) so the rotation stamp
// still forces a new task revision, hence a fresh key in the log.
dockerLabels: { rotation: keyRotation },
portMappings: [{ containerPort: 8080 }],
// The CLI answers only once the server serves (no curl in the image);
// generous so a slow start never trips a restart loop.
healthCheck: {
command: ["CMD", "headscale", "users", "list"],
interval: cdk.Duration.seconds(60),
timeout: cdk.Duration.seconds(15),
retries: 5,
startPeriod: cdk.Duration.seconds(120),
},
logging: ecs.LogDrivers.awsLogs({
streamPrefix: `headscale-${site.id}`,
logRetention: logs.RetentionDays.ONE_MONTH,
}),
});
headscale.addMountPoints({ containerPath: STATE, sourceVolume: "state", readOnly: false });
// Publishes the fresh certificate before the server takes traffic, so a
// Lambda's next call verifies against the pair this task actually serves.
const publishCert = taskDef.addContainer("publish-cert", {
image: ecs.ContainerImage.fromRegistry("public.ecr.aws/aws-cli/aws-cli:2.15.17"),
essential: false,
// Region as an argument, not an env var (cdk-nag ECS2); the uid has no
// passwd entry here, which the CLI tolerates (no ~ to read or write).
command: [
"--region", this.region,
"ssm", "put-parameter",
"--name", tlsParamOf(site.id),
"--type", "String",
"--overwrite",
"--value", `file://${STATE}/tls.crt`,
],
user: NONROOT,
readonlyRootFilesystem: true,
linuxParameters: (0, scratch_tmpfs_1.scratchTmpfs)(this, `Tmpfs-${site.id}-publish`, 16),
logging: ecs.LogDrivers.awsLogs({
streamPrefix: `headscale-${site.id}-init`,
logRetention: logs.RetentionDays.ONE_MONTH,
}),
});
publishCert.addMountPoints({ containerPath: STATE, sourceVolume: "state", readOnly: true });
publishCert.addContainerDependencies({
container: initContainer,
condition: ecs.ContainerDependencyCondition.SUCCESS,
});
headscale.addContainerDependencies({
container: publishCert,
condition: ecs.ContainerDependencyCondition.SUCCESS,
});
const service = new ecs.FargateService(this, `Service-${site.id}`, {
cluster,
taskDefinition: taskDef,
desiredCount: 1,
// SQLite on EFS = exactly one writer, ever: replace, don't overlap.
minHealthyPercent: 0,
maxHealthyPercent: 100,
assignPublicIp: true, // public subnets, no NAT — needed for image pull
vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
circuitBreaker: { rollback: true },
enableExecuteCommand: true, // admin CLI access — no admin port exposed
cloudMapOptions: { name: `headscale-${site.id}` },
});
fileSystem.connections.allowDefaultPortFrom(service);
services[site.id] = service;
bootstrapInputs[site.id] = {
logGroup: headscale.logDriverConfig.options["awslogs-group"],
streamPrefix: `headscale-${site.id}`,
};
listener.addTargets(`Headscale-${site.id}`, {
priority: site.priority,
conditions: [elbv2.ListenerCondition.hostHeaders([site.fqdn])],
port: 8080,
protocol: elbv2.ApplicationProtocol.HTTPS,
targets: [service],
healthCheck: { path: "/health", healthyHttpCodes: "200", protocol: elbv2.Protocol.HTTPS },
deregistrationDelay: cdk.Duration.seconds(30),
});
new cdk.CfnOutput(this, `ControlUrl-${site.id}`, {
value: site.controlUrl,
});
});
const minterBase = {
timeout: cdk.Duration.seconds(10),
vpc,
vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
allowPublicSubnet: true, // no NAT here; the fns need no internet, only VPC DNS
};
// CFN re-resolves the secret only when the consuming RESOURCE changes —
// rotate with `-c headscaleKeyRotation=$(date +%s)` or env keeps the old key.
/** siteId → {url, key} map the device minter and reconciler read. */
const sitesEnv = JSON.stringify(Object.fromEntries(props.sites.map((s) => [s.id, { url: apiUrlOf(s.id), certParam: tlsParamOf(s.id), key: apiKeyOf(s.id) }])));
// Stores each site's server-minted admin API key (adopting a hand-made
// secret); everything that resolves the secret into env waits on this.
const bootstrapFn = new app_function_1.AppFunction(this, "HeadscaleApiKeyFn", {
description: "Deploy-time helper: stores each VPN site's control-server admin API key in Secrets Manager.",
asset: "headscale-apikey",
timeout: cdk.Duration.minutes(14),
});
const secretArn = (siteId) => `arn:aws:secretsmanager:${this.region}:${this.account}:secret:${prefix}/headscale-apikey-${siteId}-*`;
bootstrapFn.addToRolePolicy(new iam.PolicyStatement({
actions: ["ecs:DescribeTasks", "ecs:DescribeServices", "ecs:ListTasks"],
resources: ["*"],
}));
bootstrapFn.addToRolePolicy(new iam.PolicyStatement({
actions: ["logs:GetLogEvents"],
resources: props.sites.map((s) => `arn:aws:logs:${this.region}:${this.account}:log-group:${bootstrapInputs[s.id].logGroup}:*`),
}));
bootstrapFn.addToRolePolicy(new iam.PolicyStatement({
actions: [
"secretsmanager:GetSecretValue",
"secretsmanager:CreateSecret",
"secretsmanager:PutSecretValue",
],
resources: props.sites.map((s) => secretArn(s.id)),
}));
const bootstrapProvider = new cr.Provider(this, "HeadscaleApiKeyProvider", {
onEventHandler: bootstrapFn,
});
const bootstrapBySite = {};
for (const site of props.sites) {
const { logGroup, streamPrefix } = bootstrapInputs[site.id];
const service = services[site.id];
const resource = new cdk.CustomResource(this, `HeadscaleBootstrap-${site.id}`, {
serviceToken: bootstrapProvider.serviceToken,
resourceType: "Custom::HeadscaleApiKey",
properties: {
SiteId: site.id,
SecretName: `${prefix}/headscale-apikey-${site.id}`,
ClusterArn: cluster.clusterArn,
ServiceName: service.serviceName,
LogGroupName: logGroup,
StreamPrefix: streamPrefix,
Rotation: keyRotation,
},
});
// Resource-level edge on purpose: a construct-level one pulls the
// service SG's ingress (referencing the minters' SGs) into a cycle.
resource.node.addDependency(service.node.defaultChild);
bootstrapBySite[site.id] = resource;
new cdk.CfnOutput(this, `HeadscaleApiKey-${site.id}`, {
value: resource.getAttString("Source"),
});
}
const bootstraps = Object.values(bootstrapBySite);
/** Make only a function's own CloudFormation resource wait for the key. */
const waitForKey = (fn, ...on) => {
const cfn = fn.node.defaultChild;
for (const r of on)
cfn.addResourceDependency(r.node.defaultChild);
};
// ONE device minter for all sites: its only caller (the vending API role)
// binds userName to the verified JWT, and users enroll wherever granted.
this.deviceKeyMinter = new app_function_1.AppFunction(this, "DeviceKeyMinter", {
...minterBase,
functionName: (0, naming_1.deviceKeyMinterFnName)(prefix),
asset: "keyminter",
description: "Mints a one-time VPN enrollment key for a signed-in user's device.",
handler: "deviceHandler",
environment: {
HEADSCALE_SITES: sitesEnv,
HEADSCALE_KEY_ROTATION: keyRotation,
NETWORK_RECONCILER_FN_NAME: (0, naming_1.aclReconcilerFnName)(prefix),
},
});
waitForKey(this.deviceKeyMinter, ...bootstraps);
grantTlsRead(this.deviceKeyMinter, props.sites.map((s) => s.id));
// A user's first mint on a site starts a reconciler pass (#76). By name, through the VPC's Lambda endpoint.
this.deviceKeyMinter.addToRolePolicy(new iam.PolicyStatement({
actions: ["lambda:InvokeFunction"],
resources: [
this.formatArn({
service: "lambda",
resource: "function",
resourceName: (0, naming_1.aclReconcilerFnName)(prefix),
arnFormat: cdk.ArnFormat.COLON_RESOURCE_NAME,
}),
],
}));
// One connector minter PER SITE, env-pinned, input ignored — which site a
// connector joins = which minter its account may invoke.
for (const site of props.sites) {
const minter = new app_function_1.AppFunction(this, `ConnectorKeyMinter-${site.id}`, {
...minterBase,
functionName: (0, naming_1.connectorKeyMinterFnName)(prefix, site.id),
asset: "keyminter",
description: `Mints VPN enrollment keys for connectors joining site ${site.id}.`,
handler: "connectorHandler",
environment: {
HEADSCALE_API_URL: apiUrlOf(site.id),
HEADSCALE_API_KEY: apiKeyOf(site.id),
HEADSCALE_TLS_PARAM: tlsParamOf(site.id),
HEADSCALE_USER_ID: "1", // the `infra` machine-owner user
HEADSCALE_KEY_ROTATION: keyRotation,
},
});
waitForKey(minter, bootstrapBySite[site.id]);
grantTlsRead(minter, [site.id]);
this.connectorKeyMinters[site.id] = minter;
services[site.id].connections.allowFrom(minter, ec2.Port.tcp(8080));
services[site.id].connections.allowFrom(this.deviceKeyMinter, ec2.Port.tcp(8080));
new cdk.CfnOutput(this, `ConnectorKeyMinterArn-${site.id}`, {
value: minter.functionArn,
});
}
const reconcilerRole = new iam.Role(this, "AclReconcilerRole", {
roleName: (0, naming_1.aclReconcilerRoleName)(prefix),
assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"),
managedPolicies: [
iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaVPCAccessExecutionRole"),
],
});
// Every account's connector VPCs by name and (foreign) region — the reconciler's fan-out targets, and
// the (account, region) pairs the out-of-VPC relay may address (why a relay: wiki Private Network,
// "Connector VPCs in another region").
const connectorVpcs = {};
// `<account>#<name or empty>` → the routes config declares for that VPC: all the reconciler will approve for it.
const connectorRoutes = {};
for (const s of props.sites) {
for (const [account, list] of Object.entries(s.connectorVpcs)) {
for (const vpc of list)
connectorRoutes[`${account}#${vpc.name ?? ""}`] = vpc.routes;
connectorVpcs[account] = list.map((vpc) => {
// JSON.stringify drops the undefineds: an unnamed hub-region VPC travels as `{}`.
return {
name: vpc.name,
region: vpc.region && vpc.region !== this.region ? vpc.region : undefined,
};
});
}
}
const relayRegions = [
...new Set(Object.values(connectorVpcs)
.flat()
.map((c) => c.region)
.filter((r) => Boolean(r))),
].sort();
const reconciler = new app_function_1.AppFunction(this, "AclReconciler", {
functionName: (0, naming_1.aclReconcilerFnName)(prefix),
description: "Keeps each VPN site's access rules, routes and security-group rules in line with the portal's grants.",
asset: "acl-reconciler",
handler: "handler",
role: reconcilerRole,
// Per-site budget: the pass runs the SG-manager AND tailnet-resolver
// fan-outs (each per-account invoke aborts at 10s) plus up to ~6s of
// sequential local DNS fallbacks per unresolvable dnsName resource.
timeout: cdk.Duration.seconds(Math.min(90 * props.sites.length, 900)),
// One pass at a time: the API kicks a pass on every dev-share create/stop, and two passes racing on
// the whole-policy PUT could re-publish a rule the later one just removed.
reservedConcurrentExecutions: 1,
vpc,
vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
allowPublicSubnet: true, // like the key minters: needs VPC DNS, not internet
environment: {
RESOURCE_PREFIX: prefix, // by-name SG-manager target derivation
HEADSCALE_SITES: sitesEnv,
HEADSCALE_KEY_ROTATION: keyRotation,
// Only this account's names may be resolved from inside this VPC (Cloud Map, plain DNS):
// another account's private zone can shadow a label of the hub's.
HUB_ACCOUNT: this.account,
// accountNumber → siteId: THE site-membership source of truth.
SITE_ACCOUNTS: JSON.stringify(Object.fromEntries(props.sites.flatMap((s) => s.accounts.map((a) => [a, s.id])))),
// accountNumber → its connector VPCs ({ name?, region? }; region only outside this region).
CONNECTOR_VPCS: JSON.stringify(connectorVpcs),
CONNECTOR_ROUTES: JSON.stringify(connectorRoutes),
// siteId → displayName: how the activity log names a site.
SITE_NAMES: JSON.stringify(Object.fromEntries(props.sites.map((s) => [s.id, s.displayName]))),
RESOURCES_TABLE_NAME: props.resourcesTable.tableName,
ACCESS_TABLE_NAME: props.accessTable.tableName,
GROUPS_TABLE_NAME: props.groupsTable.tableName,
// Resolvers stay reachable on :53 for everyone ON THAT SITE, or
// name-based access dies before any granted port is tried.
SPLIT_DNS_BY_SITE: JSON.stringify(Object.fromEntries(props.sites.map((s) => [
s.id,
[...new Set(Object.values(s.splitDns).flat())],
]))),
...(props.auditLog ? (0, audit_log_1.auditLogEnv)(props.auditLog) : {}),
},
});
if (props.auditLog)
(0, audit_log_1.grantAuditLogWrite)(reconciler, props.auditLog);
waitForKey(reconciler, ...bootstraps);
grantTlsRead(reconciler, props.sites.map((s) => s.id));
for (const site of props.sites) {
services[site.id].connections.allowFrom(reconciler, ec2.Port.tcp(8080));
}
// ReadWrite: the reconciler writes per-site status sentinels into Resources.
props.resourcesTable.grantReadWriteData(reconciler);
props.accessTable.grantReadData(reconciler);
props.groupsTable.grantReadData(reconciler);
// Cloud Map re-resolution of resource dnsNames (Fargate IP drift):
// DiscoverInstances has no resource-level scoping.
reconciler.addToRolePolicy(new iam.PolicyStatement({
actions: ["servicediscovery:DiscoverInstances"],
resources: ["*"],
}));
// SG-manager and tailnet-resolver invokes by well-known name in ANY account —
// reaching one still requires that account's resource policy (pinned to this role).
reconciler.addToRolePolicy(new iam.PolicyStatement({
actions: ["lambda:InvokeFunction"],
resources: [
`arn:aws:lambda:${this.region}:*:function:${(0, naming_1.sgManagerFnName)(prefix)}`,
// Named connector VPCs suffix the manager; per-site (and per-name)
// resolvers — the wildcards cover every one.
`arn:aws:lambda:${this.region}:*:function:${(0, naming_1.sgManagerFnName)(prefix, "*")}`,
`arn:aws:lambda:${this.region}:*:function:${(0, naming_1.tailnetResolverFnName)(prefix, "*")}`,
],
}));
// The relay's invoke-only role. Connector accounts in another region pin their trust to it
// by name, so it exists even without a relay (Lambda validates the principal at permission time).
const relayRole = new iam.Role(this, "AclReconcilerRelayRole", {
roleName: (0, naming_1.aclReconcilerRelayRoleName)(prefix),
assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"),
managedPolicies: [
iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaBasicExecutionRole"),
],
});
if (relayRegions.length) {
relayRole.addToPolicy(new iam.PolicyStatement({
actions: ["lambda:InvokeFunction"],
resources: relayRegions.flatMap((region) => [
`arn:aws:lambda:${region}:*:function:${(0, naming_1.sgManagerFnName)(prefix)}`,
`arn:aws:lambda:${region}:*:function:${(0, naming_1.sgManagerFnName)(prefix, "*")}`,
`arn:aws:lambda:${region}:*:function:${(0, naming_1.tailnetResolverFnName)(prefix, "*")}`,
]),
}));
const relay = new app_function_1.AppFunction(this, "InvokeRelay", {
functionName: (0, naming_1.aclReconcilerRelayFnName)(prefix),
description: "Forwards the VPN reconciler's calls to connector functions in other regions.",
asset: "invoke-relay",
handler: "handler",
role: relayRole,
// The reconciler's per-target budget (INVOKE_TIMEOUT_MS) plus this function's own start.
timeout: cdk.Duration.seconds(12),
environment: {
RESOURCE_PREFIX: prefix,
CONNECTOR_VPCS: JSON.stringify(connectorVpcs),
},
});
reconciler.addToRolePolicy(new iam.PolicyStatement({
actions: ["lambda:InvokeFunction"],
resources: [relay.functionArn],
}));
}
// The Lambdas have no internet path — DiscoverInstances needs Cloud Map's
// data endpoint. One subnet = one ENI (~$8/mo) serving the whole VPC.
vpc.addInterfaceEndpoint("CloudMapDataEndpoint", {
service: new ec2.InterfaceVpcEndpointAwsService("data-servicediscovery"),
subnets: { subnets: [vpc.publicSubnets[0]] },
});
// Lambda invokes (the SG-managers, the device minter's reconciler start) need Lambda's API from this
// no-internet VPC — same one-ENI pattern as the Cloud Map endpoint.
vpc.addInterfaceEndpoint("LambdaEndpoint", {
service: ec2.InterfaceVpcEndpointAwsService.LAMBDA,
subnets: { subnets: [vpc.publicSubnets[0]] },
});
// The minters and the reconciler read each site's TLS certificate from
// SSM before calling Headscale; without this endpoint that read hangs.
vpc.addInterfaceEndpoint("SsmEndpoint", {
service: ec2.InterfaceVpcEndpointAwsService.SSM,
subnets: { subnets: [vpc.publicSubnets[0]] },
});
// DynamoDB from inside the VPC without internet: free gateway endpoint.
vpc.addGatewayEndpoint("DynamoDbEndpoint", {
service: ec2.GatewayVpcEndpointAwsService.DYNAMODB,
});
new events.Rule(this, "ReconcileSchedule", {
schedule: events.Schedule.rate(cdk.Duration.minutes(5)),
targets: [new eventsTargets.LambdaFunction(reconciler)],
});
// A broken reconciler freezes reachability INCLUDING revocations; one
// failure self-heals next pass, so alarm on two consecutive.
if (props.alertTopic) {
const notify = new cwactions.SnsAction(props.alertTopic);
const alarm = new cloudwatch.Alarm(this, "ReconcilerErrors", {
metric: reconciler.metricErrors({ period: cdk.Duration.minutes(15) }),
threshold: 2,
evaluationPeriods: 1,
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
alarmDescription: "The ACL reconciler has failed 2+ consecutive passes — tailnet " +
"reachability is frozen at the last synced state, so grant changes " +
"AND revocations are not taking effect. Check its CloudWatch logs " +
"and the Headscale services.",
});
alarm.addAlarmAction(notify);
alarm.addOkAction(notify);
// Zombie: "online" but silent — forwards on STALE rules and never learns new device keys. EMF
// metric from the reconciler, summed; a node a live replacement superseded is not in it (#70).
const staleAlarm = new cloudwatch.Alarm(this, "StaleConnectors", {
metric: new cloudwatch.Metric({
namespace: "InfraClient/Network",
metricName: "StaleConnectors",
statistic: "Maximum",
period: cdk.Duration.minutes(15),
}),
threshold: 1,
evaluationPeriods: 1,
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
alarmDescription: "A connector node with approved routes has been OFFLINE 15+ " +
"minutes with no online replacement in its account and VPC " +
"(zombie): it enforces stale rules and won't learn " +
"re-enrolled devices. Restart it: aws ecs update-service " +
"--force-new-deployment on the connector service; the reconciler " +
"auto-approves the replacement's routes.",
});
staleAlarm.addAlarmAction(notify);
staleAlarm.addOkAction(notify);
const derpMapFailures = { namespace: "InfraClient/Network", metricName: "DerpMapLoadFailures" };
for (const [siteId, { logGroup }] of Object.entries(bootstrapInputs)) {
new logs.MetricFilter(this, `DerpMapLoadFailures-${siteId}`, {
logGroup: logs.LogGroup.fromLogGroupName(this, `ControlServerLogs-${siteId}`, logGroup),
filterPattern: logs.FilterPattern.literal('"Could not load DERP map"'),
metricNamespace: derpMapFailures.namespace,
metricName: derpMapFailures.metricName,
metricValue: "1",
});
}
const derpMapAlarm = new cloudwatch.Alarm(this, "DerpMapLoadFailed", {
metric: new cloudwatch.Metric({
...derpMapFailures,
statistic: "Sum",
period: cdk.Duration.minutes(5),
}),
threshold: 1,
evaluationPeriods: 1,
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
alarmDescription: "A VPN control server could not load its relay (DERP) map: the " +
"download from controlplane.tailscale.com failed, or the saved " +
"fallback copy on EFS is missing. With a saved copy, nodes keep " +
"relays; without one, the site has none until the next refresh. " +
"Check the site's control-server log group. Without a saved copy, " +
"force a new deployment of its service once the URL loads again. " +
"OK only means no new failure was logged; a refresh runs every 24 h.",
});
derpMapAlarm.addAlarmAction(notify);
derpMapAlarm.addOkAction(notify);
}
new cdk.CfnOutput(this, "ClusterName", { value: cluster.clusterName });
new cdk.CfnOutput(this, "DeviceKeyMinterArn", {
value: this.deviceKeyMinter.functionArn,
});
}
/** Grant invoke on ONE SITE's connector minter (machine keys only — never
* the device minter). Prefer a task-role ARN over a whole-account id. */
grantConnectorAccount(siteId, connectorPrincipal) {
const minter = this.connectorKeyMinters[siteId];
if (!minter)
throw new Error(`No such site: ${siteId}`);
const principal = connectorPrincipal.includes(":")
? new iam.ArnPrincipal(connectorPrincipal) // specific role ARN (preferred)
: new iam.AccountPrincipal(connectorPrincipal); // whole account (coarser)
// Construct ids must be alphanumeric; keep them stable + unique per grant.
const suffix = connectorPrincipal.replace(/[^a-zA-Z0-9]/g, "").slice(-24);
minter.addPermission(`ConnectorInvoke${suffix}`, {
principal,
action: "lambda:InvokeFunction",
});
}
}
exports.NetworkStack = NetworkStack;