@mesh-tech/mesh-cli/dist/src/utils/tailscale.js

CLI for Mesh platform development utilities

import * as fs from "node:fs";
import * as net from "node:net";
import * as os from "node:os";
import * as path from "node:path";
import { spawn, execFileSync } from "node:child_process";
import { CONFIG_DIR, atomicWriteFileSync } from "../commands/login.js";
import { resolveStableMeshBin } from "./aws-auth.js";
import { logInfo, logWarn } from "./log.js";
import { MeshCliError, renderErrorBody } from "./errors.js";
import { assertSignalablePid } from "./pid.js";
import { getPlatformBastionInfo } from "./bastion.js";
import { resolveTunnelTargets, resolveElbIp, tenantPortIndex, PORT_BLOCK } from "./tailscale-targets.js";
import { createForwarder } from "./socks-forward.js";
import { mintRunnerToken, readRunnerManifest, writeRunnerManifest, clearRunnerManifest, getProcessGroupId, discoverRunnerPids, startControlServer, verifyRunnerOwnership, pollOwnership, findSquattedPorts, buildSquatterError, } from "./tunnel-ownership.js";
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
export function tenantStateDir(tenant) {
    return path.join(CONFIG_DIR, "tailscale", tenant);
}
export function socketPath(tenant) {
    return path.join(tenantStateDir(tenant), "tailscaled.sock");
}
export function logPath(tenant) {
    return path.join(tenantStateDir(tenant), "tailscaled.log");
}
export function statePath(tenant) {
    return path.join(tenantStateDir(tenant), "tunnel-state.json");
}
export function daemonMetaPath(tenant) {
    return path.join(tenantStateDir(tenant), "daemon.json");
}
export function runnerManifestPath(tenant) {
    return path.join(tenantStateDir(tenant), "runner.json");
}
export function readTunnelState(tenant) {
    const p = statePath(tenant);
    if (!fs.existsSync(p))
        return null;
    try {
        return JSON.parse(fs.readFileSync(p, "utf8"));
    }
    catch {
        return null;
    }
}
export function writeTunnelState(tenant, state) {
    fs.mkdirSync(tenantStateDir(tenant), { recursive: true });
    atomicWriteFileSync(statePath(tenant), JSON.stringify(state, null, 2), 0o600);
}
export function clearTunnelState(tenant) {
    try {
        fs.unlinkSync(statePath(tenant));
    }
    catch {
    }
}
export function readDaemonMeta(tenant) {
    const p = daemonMetaPath(tenant);
    if (!fs.existsSync(p))
        return null;
    try {
        return JSON.parse(fs.readFileSync(p, "utf8"));
    }
    catch {
        return null;
    }
}
export function writeDaemonMeta(tenant, meta) {
    fs.mkdirSync(tenantStateDir(tenant), { recursive: true });
    atomicWriteFileSync(daemonMetaPath(tenant), JSON.stringify(meta, null, 2), 0o600);
}
export function clearDaemonMeta(tenant) {
    try {
        fs.unlinkSync(daemonMetaPath(tenant));
    }
    catch {
    }
}
export function parseDaemonState(statusJson) {
    try {
        const j = JSON.parse(statusJson);
        return { backendState: j.BackendState ?? "Unknown", selfName: j.Self?.HostName };
    }
    catch {
        return { backendState: "Unknown" };
    }
}
export function tailscaledBinPath() {
    return resolveBrewBin("tailscaled");
}
export function tailscaleBinPath() {
    return resolveBrewBin("tailscale");
}
function resolveBrewBin(name) {
    const brew = `/opt/homebrew/bin/${name}`;
    if (fs.existsSync(brew))
        return brew;
    try {
        const p = execFileSync("which", [name], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim();
        if (p)
            return p;
    }
    catch {
    }
    throw new Error(`${name} not found — install the standalone binary: brew install tailscale`);
}
export function daemonState(tenant) {
    if (!fs.existsSync(socketPath(tenant)))
        return { backendState: "Down" };
    try {
        const out = execFileSync(tailscaleBinPath(), ["--socket", socketPath(tenant), "status", "--json"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
        return parseDaemonState(out);
    }
    catch {
        return { backendState: "Down" };
    }
}
export function parseSocksPortFromArgv(argv) {
    const m = argv.match(/--socks5-server[=\s]+(?:127\.0\.0\.1|localhost):(\d+)/);
    if (!m)
        return null;
    const port = Number(m[1]);
    return Number.isInteger(port) && port > 0 && port <= 65535 ? port : null;
}
export function findRunningDaemon(tenant) {
    const sock = socketPath(tenant);
    let out;
    try {
        out = execFileSync("ps", ["ax", "-o", "pid=,command="], {
            encoding: "utf8",
            stdio: ["ignore", "pipe", "ignore"],
        });
    }
    catch {
        return null;
    }
    for (const line of out.split("\n")) {
        if (!line.includes("tailscaled") || !line.includes(`--socket=${sock}`))
            continue;
        const m = line.trim().match(/^(\d+)\s+(.*)$/);
        if (!m)
            continue;
        return { pid: Number(m[1]), socksPort: parseSocksPortFromArgv(m[2]) };
    }
    return null;
}
export function portAccepts(port, timeoutMs = 400) {
    return new Promise((resolve) => {
        const s = new net.Socket();
        let done = false;
        const fin = (v) => {
            if (done)
                return;
            done = true;
            s.destroy();
            resolve(v);
        };
        s.setTimeout(timeoutMs);
        s.once("connect", () => fin(true));
        s.once("timeout", () => fin(false));
        s.once("error", () => fin(false));
        s.connect(port, "127.0.0.1");
    });
}
export async function ensureDaemon(tenant, opts) {
    if (daemonState(tenant).backendState !== "Down") {
        if (await portAccepts(opts.socksPort)) {
            writeDaemonMeta(tenant, {
                socksPort: opts.socksPort,
                pid: readDaemonMeta(tenant)?.pid ?? findRunningDaemon(tenant)?.pid ?? 0,
            });
            return opts.socksPort;
        }
        const running = findRunningDaemon(tenant);
        if (running?.socksPort && (await portAccepts(running.socksPort))) {
            writeDaemonMeta(tenant, { socksPort: running.socksPort, pid: running.pid });
            return running.socksPort;
        }
        stopDaemon(tenant);
        for (let i = 0; i < 12 && daemonState(tenant).backendState !== "Down"; i++) {
            await sleep(250);
        }
    }
    const dir = tenantStateDir(tenant);
    fs.mkdirSync(dir, { recursive: true });
    const log = fs.openSync(logPath(tenant), "a");
    try {
        const child = spawn(tailscaledBinPath(), [
            "--tun=userspace-networking",
            `--socket=${socketPath(tenant)}`,
            `--statedir=${dir}`,
            `--socks5-server=127.0.0.1:${opts.socksPort}`,
            "--port=0",
        ], {
            detached: true,
            stdio: ["ignore", log, log],
            env: { ...process.env, AWS_REGION: opts.region },
        });
        child.unref();
        if (child.pid)
            writeDaemonMeta(tenant, { socksPort: opts.socksPort, pid: child.pid });
    }
    finally {
        fs.closeSync(log);
    }
    for (let i = 0; i < 40; i++) {
        await sleep(250);
        if (daemonState(tenant).backendState !== "Down")
            return opts.socksPort;
    }
    throw new Error(`tailscaled did not start for tenant '${tenant}' — see ${logPath(tenant)}`);
}
export function stopDaemon(tenant) {
    const meta = readDaemonMeta(tenant) ?? (() => {
        const running = findRunningDaemon(tenant);
        return running ? { socksPort: running.socksPort ?? 0, pid: running.pid } : null;
    })();
    if (meta?.pid) {
        try {
            assertSignalablePid(meta.pid, {
                what: `the VPN daemon pid recorded for tenant '${tenant}'`,
                file: daemonMetaPath(tenant),
            });
            process.kill(meta.pid, "SIGTERM");
        }
        catch (err) {
            if (err instanceof MeshCliError)
                logWarn(renderErrorBody(err));
        }
    }
    for (const p of [socketPath(tenant)]) {
        try {
            fs.unlinkSync(p);
        }
        catch {
        }
    }
    clearDaemonMeta(tenant);
}
export function buildTailscaleUpArgs(opts) {
    const args = [
        "--socket", opts.socket, "up",
        "--login-server", opts.loginServer,
        "--accept-routes",
        "--hostname", opts.hostname,
    ];
    if (opts.preAuthKey)
        args.push("--authkey", opts.preAuthKey);
    return args;
}
export async function joinHeadscale(tenant, loginServer, opts = {}) {
    if (daemonState(tenant).backendState === "Running")
        return;
    const hostname = `mesh-${tenant}-${os.hostname().split(".")[0]}`;
    const child = spawn(tailscaleBinPath(), buildTailscaleUpArgs({ socket: socketPath(tenant), loginServer, hostname, preAuthKey: opts.preAuthKey }), { stdio: ["ignore", "pipe", "pipe"] });
    if (!opts.preAuthKey) {
        let buf = "";
        let urlShown = false;
        const capture = (d) => {
            buf += d.toString();
            if (urlShown)
                return;
            const m = buf.match(/https?:\/\/\S+\/register\/\S+/);
            if (m) {
                urlShown = true;
                logInfo(`\n  This machine isn't registered with the platform VPN yet.\n` +
                    `  Open this URL in a browser to authorize it (one-time, sign in with Zitadel):\n\n` +
                    `      ${m[0]}\n\n` +
                    `  Waiting for approval (up to 5 min)…`);
            }
        };
        child.stdout.on("data", capture);
        child.stderr.on("data", capture);
    }
    for (let i = 0; i < 600; i++) {
        await sleep(500);
        if (daemonState(tenant).backendState === "Running") {
            child.kill();
            return;
        }
    }
    child.kill();
    throw new Error(`VPN join not completed for tenant '${tenant}' within 5 min. Re-run \`mesh vpn tunnel up\` to resume — the daemon is still up.`);
}
export function logout(tenant) {
    try {
        execFileSync(tailscaleBinPath(), ["--socket", socketPath(tenant), "logout"], { stdio: "ignore" });
    }
    catch {
    }
}
export function tunnelTargetsEqual(a, b) {
    const key = (ts) => ts.map((t) => `${t.name}:${t.localPort}:${t.targetIp}:${t.remotePort}`).sort().join(",");
    return key(a) === key(b);
}
export function supervisorTargetsMatch(state, targets) {
    if (!state || !state.supervisorPid)
        return false;
    return tunnelTargetsEqual(state.tunnels, targets);
}
function pidAlive(pid) {
    if (!pid)
        return false;
    try {
        process.kill(pid, 0);
        return true;
    }
    catch {
        return false;
    }
}
export function killSupervisor(pid, source) {
    assertSignalablePid(pid, source ?? { what: "tunnel supervisor pid" });
    try {
        process.kill(-pid, "SIGTERM");
    }
    catch {
        try {
            process.kill(pid, "SIGTERM");
        }
        catch {
        }
    }
}
export async function verifyTunnelOwnership(tenant, expected) {
    return verifyRunnerOwnership({
        manifest: readRunnerManifest(runnerManifestPath(tenant)),
        expectedSocksPort: expected.socksPort,
        expectedPorts: expected.ports,
    });
}
function awaitOwnership(tenant, expected, deadlineMs) {
    return pollOwnership(() => verifyTunnelOwnership(tenant, expected), deadlineMs);
}
export function stopTunnelRunners(tenant, prior) {
    if (prior?.managed === "launchd")
        uninstallLaunchAgent(tenant);
    const killOrWarn = (pid, source) => {
        try {
            killSupervisor(pid, source);
        }
        catch (err) {
            if (!(err instanceof MeshCliError))
                throw err;
            logWarn(renderErrorBody(err));
        }
    };
    if (prior?.supervisorPid)
        killOrWarn(prior.supervisorPid, {
            what: `the tunnel supervisor pid recorded for tenant '${tenant}'`,
            file: statePath(tenant),
        });
    const manifest = readRunnerManifest(runnerManifestPath(tenant));
    if (manifest)
        killOrWarn(manifest.pgid || manifest.pid, {
            what: `the tunnel runner process group recorded for tenant '${tenant}'`,
            file: runnerManifestPath(tenant),
        });
    for (const pid of discoverRunnerPids(tenant, process.pid))
        killOrWarn(pid, { what: `a tunnel runner pid discovered from 'ps' for tenant '${tenant}'` });
    clearRunnerManifest(runnerManifestPath(tenant));
}
async function assertTunnelPortsFree(tenant, ports) {
    let held = ports;
    for (let i = 0; i < 8; i++) {
        const stillHeld = [];
        for (const p of held) {
            if (await portAccepts(p, 250))
                stillHeld.push(p);
        }
        held = stillHeld;
        if (held.length === 0)
            return;
        await sleep(250);
    }
    const squatted = await findSquattedPorts(held, (p) => portAccepts(p, 250));
    throw new Error(buildSquatterError(tenant, squatted, "still bound after teardown"));
}
export async function ensureForwarders(state) {
    const prior = readTunnelState(state.tenant);
    const expected = { socksPort: state.socksPort, ports: state.tunnels.map((t) => t.localPort) };
    const runnerAlive = prior?.managed === "launchd"
        ? fs.existsSync(launchAgentPath(state.tenant))
        : pidAlive(prior?.supervisorPid);
    if (prior &&
        runnerAlive &&
        tunnelTargetsEqual(prior.tunnels, state.tunnels) &&
        prior.socksPort === state.socksPort &&
        (await awaitOwnership(state.tenant, expected, 1000)).ok) {
        state.supervisorPid = prior.supervisorPid;
        state.managed = prior.managed;
        writeTunnelState(state.tenant, state);
        return prior.supervisorPid ?? 0;
    }
    stopTunnelRunners(state.tenant, prior);
    await assertTunnelPortsFree(state.tenant, expected.ports);
    state.supervisorPid = null;
    const meshBin = resolveStableMeshBin(process.argv[1]);
    if (launchdAvailable()) {
        state.managed = "launchd";
        writeTunnelState(state.tenant, state);
        if (path.isAbsolute(meshBin) && installLaunchAgent(state.tenant, meshBin)) {
            if ((await awaitOwnership(state.tenant, expected, 8000)).ok)
                return 0;
            uninstallLaunchAgent(state.tenant);
            await assertTunnelPortsFree(state.tenant, expected.ports);
            logInfo("launchd runner did not take ownership of the tunnel ports — using a detached runner.");
        }
        else {
            logInfo("launchd install failed — using a detached self-healing runner.");
        }
    }
    state.managed = "detached";
    writeTunnelState(state.tenant, state);
    const child = spawn(meshBin, ["vpn", "tunnel", "__run", state.tenant], {
        detached: true,
        stdio: "ignore",
    });
    child.on("error", (err) => logWarn(`tunnel supervisor failed to start (${err.message}) — falling back to SSM.`));
    child.unref();
    state.supervisorPid = child.pid ?? null;
    writeTunnelState(state.tenant, state);
    const last = await awaitOwnership(state.tenant, expected, 5000);
    if (last.ok)
        return child.pid ?? -1;
    const squatted = await findSquattedPorts(expected.ports, (p) => portAccepts(p, 250));
    if (squatted.length > 0) {
        throw new Error(buildSquatterError(state.tenant, squatted, `ownership check: ${last.reason}`));
    }
    const bindErrors = Object.entries(last.resp?.bindErrors ?? {})
        .map(([port, code]) => `port ${port}: ${code}`)
        .join(", ");
    throw new Error(`Tailscale forwarders did not take ownership for tenant '${state.tenant}' ` +
        `(${last.reason}${bindErrors ? `; ${bindErrors}` : ""}) — see ${logPath(state.tenant)}.`);
}
export function rewriteTunnelsForTargets(devOutput, targets) {
    const tunnels = { ...devOutput.tunnels };
    for (const t of targets) {
        tunnels[t.name] = { host: "localhost", port: t.localPort };
    }
    return { ...devOutput, tunnels };
}
export function tailscaleAvailable() {
    try {
        tailscaledBinPath();
        return true;
    }
    catch {
        return false;
    }
}
export async function ensureTailscaleTunnels(tenant, env, opts) {
    const socksPort = await ensureDaemon(tenant, { region: opts.region, socksPort: opts.socksPort });
    await joinHeadscale(tenant, opts.loginServer, { preAuthKey: opts.preAuthKey });
    const bastion = await getPlatformBastionInfo(tenant, env, opts.region);
    if (!bastion) {
        throw new Error(`Could not read the platform bastion for ${tenant}/${env} (check AWS credentials).`);
    }
    const portOffset = tenantPortIndex(tenant) * PORT_BLOCK;
    const targets = resolveTunnelTargets(bastion.services, resolveElbIp, portOffset);
    if (targets.length === 0) {
        throw new Error(`No resolvable VPC tunnels found for ${tenant}/${env}.`);
    }
    const state = {
        tenant,
        env,
        socksPort,
        loginServer: opts.loginServer,
        supervisorPid: null,
        startedAt: new Date().toISOString(),
        tunnels: targets,
    };
    await ensureForwarders(state);
    return targets;
}
export async function startTailscaleTunnels(devOutput, ctx) {
    const targets = await ensureTailscaleTunnels(ctx.tenant, ctx.env, {
        region: ctx.region,
        loginServer: ctx.loginServer,
        socksPort: ctx.socksPort,
        preAuthKey: ctx.preAuthKey,
    });
    return rewriteTunnelsForTargets(devOutput, targets);
}
export function launchAgentLabel(tenant) {
    return `tech.mesh.tunnel.${tenant}`;
}
function xmlEscape(s) {
    return s
        .replace(/&/g, "&amp;")
        .replace(/</g, "&lt;")
        .replace(/>/g, "&gt;");
}
export function renderLaunchAgentPlist(args) {
    const pathEnv = args.pathEnv ?? "/usr/bin:/bin:/usr/sbin:/sbin";
    return `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
  <key>Label</key>
  <string>${launchAgentLabel(args.tenant)}</string>
  <key>ProgramArguments</key>
  <array>
    <string>${xmlEscape(args.nodeBin)}</string>
    <string>${xmlEscape(args.meshBin)}</string>
    <string>vpn</string>
    <string>tunnel</string>
    <string>__run</string>
    <string>${xmlEscape(args.tenant)}</string>
  </array>
  <key>EnvironmentVariables</key>
  <dict>
    <key>PATH</key>
    <string>${xmlEscape(pathEnv)}</string>
  </dict>
  <key>RunAtLoad</key>
  <true/>
  <key>KeepAlive</key>
  <dict>
    <key>SuccessfulExit</key>
    <false/>
  </dict>
  <key>StandardOutPath</key>
  <string>${xmlEscape(args.logPath)}</string>
  <key>StandardErrorPath</key>
  <string>${xmlEscape(args.logPath)}</string>
</dict>
</plist>
`;
}
export function forwardersNeedingRebind(targets, listening) {
    return targets.filter((t) => !listening.has(t.localPort));
}
export function launchAgentPath(tenant) {
    return path.join(os.homedir(), "Library", "LaunchAgents", `${launchAgentLabel(tenant)}.plist`);
}
function guiDomain() {
    return `gui/${process.getuid?.() ?? 0}`;
}
let _launchdAvailable;
export function launchdAvailable() {
    if (_launchdAvailable !== undefined)
        return _launchdAvailable;
    try {
        execFileSync("which", ["launchctl"], { stdio: "ignore" });
        execFileSync("launchctl", ["print", guiDomain()], { stdio: "ignore" });
        _launchdAvailable = true;
    }
    catch {
        _launchdAvailable = false;
    }
    return _launchdAvailable;
}
export function installLaunchAgent(tenant, meshBin) {
    const plistPath = launchAgentPath(tenant);
    try {
        let realMeshBin = meshBin;
        try {
            realMeshBin = fs.realpathSync(meshBin);
        }
        catch {
        }
        let pathEnv = "/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin";
        try {
            pathEnv = `${path.dirname(tailscaledBinPath())}:${pathEnv}`;
        }
        catch {
        }
        fs.mkdirSync(path.dirname(plistPath), { recursive: true });
        fs.writeFileSync(plistPath, renderLaunchAgentPlist({
            tenant,
            meshBin: realMeshBin,
            nodeBin: process.execPath,
            logPath: path.join(tenantStateDir(tenant), "runner.log"),
            pathEnv,
        }));
        try {
            execFileSync("launchctl", ["bootout", guiDomain(), plistPath], { stdio: "ignore" });
        }
        catch {
        }
        execFileSync("launchctl", ["bootstrap", guiDomain(), plistPath], { stdio: "ignore" });
        return true;
    }
    catch {
        return false;
    }
}
export function uninstallLaunchAgent(tenant) {
    const plistPath = launchAgentPath(tenant);
    try {
        execFileSync("launchctl", ["bootout", guiDomain(), plistPath], { stdio: "ignore" });
    }
    catch {
    }
    try {
        fs.unlinkSync(plistPath);
    }
    catch {
    }
}
export async function runTunnels(tenant) {
    const st = readTunnelState(tenant);
    if (!st) {
        uninstallLaunchAgent(tenant);
        clearRunnerManifest(runnerManifestPath(tenant));
        process.exit(0);
    }
    const token = mintRunnerToken();
    const servers = new Map();
    const bindErrors = new Map();
    const spawnFor = (t) => {
        servers.get(t.localPort)?.close();
        bindErrors.delete(t.localPort);
        const server = createForwarder({
            listenPort: t.localPort,
            socksPort: st.socksPort,
            targetIp: t.targetIp,
            targetPort: t.remotePort,
            onError: (err) => {
                bindErrors.set(t.localPort, err.code ?? err.message);
                logWarn(`forwarder ${t.name} failed on 127.0.0.1:${t.localPort}: ${err.code ?? err.message}`);
            },
        });
        server.on("listening", () => bindErrors.delete(t.localPort));
        servers.set(t.localPort, server);
    };
    for (const t of st.tunnels)
        spawnFor(t);
    const control = await startControlServer(() => ({
        token,
        tenant,
        pid: process.pid,
        socksPort: st.socksPort,
        listening: st.tunnels.filter((t) => servers.get(t.localPort)?.listening).map((t) => t.localPort),
        bindErrors: Object.fromEntries([...bindErrors].map(([p, c]) => [String(p), c])),
    }));
    writeRunnerManifest(runnerManifestPath(tenant), {
        tenant,
        pid: process.pid,
        pgid: getProcessGroupId(process.pid) ?? process.pid,
        token,
        controlPort: control.port,
        socksPort: st.socksPort,
        ports: st.tunnels.map((t) => t.localPort),
        startedAt: new Date().toISOString(),
    });
    const shutdown = () => {
        control.server.close();
        for (const s of servers.values())
            s.close();
        clearRunnerManifest(runnerManifestPath(tenant), token);
        process.exit(0);
    };
    process.on("SIGTERM", shutdown);
    process.on("SIGINT", shutdown);
    for (;;) {
        await sleep(5000);
        try {
            if (daemonState(tenant).backendState === "Down") {
                await ensureDaemon(tenant, {
                    region: process.env.AWS_REGION ?? "us-east-2",
                    socksPort: st.socksPort,
                });
            }
            const listening = new Set();
            for (const t of st.tunnels) {
                if (servers.get(t.localPort)?.listening)
                    listening.add(t.localPort);
            }
            for (const t of forwardersNeedingRebind(st.tunnels, listening))
                spawnFor(t);
        }
        catch {
        }
    }
}