diff --git a/hosts/heather/astrnet.nix b/hosts/heather/astrnet.nix index 1113df1..60f1070 100644 --- a/hosts/heather/astrnet.nix +++ b/hosts/heather/astrnet.nix @@ -1,52 +1,27 @@ -# astrnet.nix — AstrNet peer "michaotic". Generated by astrnet-join.sh. +# astrnet.nix — AstrNet stub peer "michaotic". Generated by astrnet-join.sh. # -# Identity 10.0.0.9 / 2a05:f480:2400:1cf7::9, ASN 4200000009, two WireGuard -# legs that THIS host dials: -# wg-mic-bd8 -> bd8816 (Moscow) PRIMARY, pair 39 -# wg-mic-der -> alderamin (Helsinki) backup, pair 38 +# What this host is to the mesh: identity 10.0.0.9 / 2a05:f480:2400:1cf7::9, +# ASN 4200000009, two WireGuard legs that THIS host dials: +# wg-mic-der -> alderamin (Helsinki) primary, pair 38 +# wg-mic-bd8 -> bd8816 (Moscow) backup, pair 39 (paths come prepended) +# It announces only its identity and accepts only mesh address space; it +# never becomes transit and never takes a default route from the mesh. # -# Egress works like an AstrNet client: both legs announce a default route, -# and this host sends all its traffic into the mesh. The far end splits it: -# RU destinations leave via the RU exit, everything else via the -# international exits. Exceptions stay on the WAN: WireGuard's own packets, -# replies from the WAN address (inbound services keep working), and the cloud -# metadata service. If both BGP sessions drop, the BGP default is withdrawn -# and the WAN default takes over on its own. -# -# The WAN values below were detected at install time. Everything else is -# assigned by the AstrNet admin; do not renumber it. +# All values are assigned by the AstrNet admin. Do not renumber anything +# here without them: the far ends are configured to match exactly. { config, lib, pkgs, ... }: let keyFile = "/var/lib/astrnet/wg.key"; id4 = "10.0.0.9"; id6 = "2a05:f480:2400:1cf7::9"; - # ── Detected WAN ── - wanIf = "enp7s0"; - wanGw4 = "10.130.0.1"; - wanIp4 = "10.130.0.36"; - wanGw6 = ""; # "" when there is no IPv6 WAN - wanIp6 = ""; - - # Table holding the WAN default for the exceptions above, and the mark on - # WireGuard's outer UDP that selects it (without it the tunnels would try - # to reach their own endpoints through the mesh default). - wanTable = "100"; - wgMark = "0x4d49"; - wanRules = - [ { fam = "-4"; sel = "fwmark ${wgMark}"; } - { fam = "-4"; sel = "from ${wanIp4}"; } - ] - ++ lib.optional (wanGw6 != "") { fam = "-6"; sel = "fwmark ${wgMark}"; } - ++ lib.optional (wanGw6 != "" && wanIp6 != "") { fam = "-6"; sel = "from ${wanIp6}"; }; - mkLeg = { listenPort, ips, publicKey, endpoint }: { inherit ips listenPort; privateKeyFile = keyFile; mtu = 1380; - fwMark = wgMark; # Routing is BGP's job. allowedIPs 0/0 only PERMITS any source through - # the tunnel; with routes off it installs nothing. + # the tunnel; with routes off, it installs nothing. Turning routes on + # would hijack this host's default route. allowedIPsAsRoutes = false; peers = [{ inherit publicKey endpoint; @@ -55,35 +30,30 @@ let }]; }; in { - # DHCP keeps the WAN address/DNS but not a competing kernel default route. - # FRR otherwise prefers the existing kernel route over the BGP default; - # WAN failback is provided by policy table 100 below. - networking.dhcpcd.extraConfig = "nogateway"; - networking.wireguard.interfaces = { # The fe80:: link-locals are required: without one FRR's IPv6 BGP # next-hop selection fails and the v6 session never leaves Connect. - wg-mic-bd8 = mkLeg { - listenPort = 52039; - ips = [ "10.99.0.158/30" "fd53:4023:2326:99::9e/126" "fe80::2/64" ]; - publicKey = "QIvX+avvJqaHxuksGWb6Hx8fv7R6VI5wpaGqLme6Fgw="; - endpoint = "195.133.60.217:52039"; - }; wg-mic-der = mkLeg { listenPort = 52038; ips = [ "10.99.0.154/30" "fd53:4023:2326:99::9a/126" "fe80::2/64" ]; publicKey = "a4FyXuoI16TxZxv407lSzJV2YFJdq6FgRAtGu+oMzEY="; endpoint = "31.77.105.157:52038"; }; + wg-mic-bd8 = mkLeg { + listenPort = 52039; + ips = [ "10.99.0.158/30" "fd53:4023:2326:99::9e/126" "fe80::2/64" ]; + publicKey = "QIvX+avvJqaHxuksGWb6Hx8fv7R6VI5wpaGqLme6Fgw="; + endpoint = "195.133.60.217:52039"; + }; }; # Explicit owner decision: allow all AstrNet members to access Heather's # host services over either WG leg. This does NOT open additional WAN ports, - # enable forwarding from AstrNet. Egress routing is configured below. + # enable forwarding from AstrNet, or route public traffic through the mesh. # WireGuard peers use allowedIPs 0/0 as an ACL, so do not rely on a packet's # source address to identify a particular AstrNet client here. systemd.services.astrnet-guard = { - description = "Allow AstrNet ingress and Docker replies; block new transit"; + description = "Allow AstrNet host ingress; block AstrNet transit"; wantedBy = [ "multi-user.target" ]; requiredBy = [ "wireguard-wg-mic-der.service" "wireguard-wg-mic-bd8.service" ]; before = [ "wireguard-wg-mic-der.service" "wireguard-wg-mic-bd8.service" ]; @@ -97,21 +67,11 @@ in { type filter hook input priority -150; policy accept; # Full inbound access from AstrNet to Heather, IPv4 and IPv6. iifname { "wg-mic-der", "wg-mic-bd8" } accept - # bgpd listens on 0.0.0.0 and ::; never expose BGP on public WAN. - iifname "enp7s0" tcp dport 179 drop } chain forward { type filter hook forward priority -150; policy accept; - # Only replies to Docker-initiated connections, never new mesh transit. - iifname { "wg-mic-der", "wg-mic-bd8" } ct state established,related accept iifname { "wg-mic-der", "wg-mic-bd8" } drop } - chain postrouting { - type nat hook postrouting priority 99; policy accept; - # Docker's own MASQUERADE would use the tunnel transit IP, which - # AstrNet exits do not NAT. Use the advertised mesh identity instead. - oifname { "wg-mic-der", "wg-mic-bd8" } ip saddr { 172.17.0.0/16, 172.18.0.0/16, 172.19.0.0/16 } snat to ${id4} - } } RULES ''; @@ -136,71 +96,12 @@ in { preStop = "ip link del mesh0 || true"; }; - # Traffic that must keep using the WAN even while the mesh default is up. - systemd.services.astrnet-wan-policy = { - description = "AstrNet: keep tunnel, inbound replies and metadata on the WAN"; - wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" ]; - wants = [ "network-online.target" ]; - before = [ "frr.service" ]; - requiredBy = [ "frr.service" ]; - path = [ pkgs.iproute2 ]; - serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; - unitConfig.OnFailure = "astrnet-wan-recovery.service"; - script = '' - ip route replace default via ${wanGw4} dev ${wanIf} table ${wanTable} - '' + lib.optionalString (wanGw6 != "") '' - ip -6 route replace default via ${wanGw6} dev ${wanIf} table ${wanTable} - '' + '' - # The cloud metadata endpoint must bypass Docker's 169.254/16 link-local - # routes; a rule at priority 1000 would never be reached. - ip -4 rule del to 169.254.169.254 lookup ${wanTable} priority 998 2>/dev/null || true - ip -4 rule add to 169.254.169.254 lookup ${wanTable} priority 998 - # Specific routes in main (the local subnet, mesh space) always win, so - # the WAN table below only ever decides where a DEFAULT would go. - for fam in -4 -6; do - ip $fam rule del lookup main suppress_prefixlength 0 priority 999 2>/dev/null || true - ip $fam rule add lookup main suppress_prefixlength 0 priority 999 - done - '' + lib.concatMapStrings (r: '' - ip ${r.fam} rule del ${r.sel} lookup ${wanTable} priority 1000 2>/dev/null || true - ip ${r.fam} rule add ${r.sel} lookup ${wanTable} priority 1000 - '') wanRules + '' - # Main BGP default wins; WAN table 100 is used if BGP withdraws it. - ip -4 rule del lookup main priority 2000 2>/dev/null || true - ip -4 rule add lookup main priority 2000 - ip -4 rule del lookup ${wanTable} priority 2001 2>/dev/null || true - ip -4 rule add lookup ${wanTable} priority 2001 - # A kernel default in main outranks BGP in FRR itself, regardless of - # Linux route metric. Remove it only AFTER the fallback rule is live. - ip -4 route del default via ${wanGw4} dev ${wanIf} 2>/dev/null || true - ''; - preStop = '' - # On rollback restore the plain WAN path first. - ip -4 route replace default via ${wanGw4} dev ${wanIf} metric 1002 || true - ip -4 rule del lookup main priority 2000 || true - ip -4 rule del lookup ${wanTable} priority 2001 || true - ip -4 rule del to 169.254.169.254 lookup ${wanTable} priority 998 || true - ip -4 rule del lookup main suppress_prefixlength 0 priority 999 || true - ip -6 rule del lookup main suppress_prefixlength 0 priority 999 || true - '' + lib.concatMapStrings (r: '' - ip ${r.fam} rule del ${r.sel} lookup ${wanTable} priority 1000 || true - '') wanRules; - }; - - # If policy setup fails, leave a normal WAN default rather than a stranded host. - systemd.services.astrnet-wan-recovery = { - description = "Restore YC WAN route if AstrNet policy fails"; - path = [ pkgs.iproute2 ]; - serviceConfig.Type = "oneshot"; - script = "ip -4 route replace default via ${wanGw4} dev ${wanIf} metric 1002"; - }; - # BGP only inside the tunnels, never on the public interface. - networking.firewall.interfaces."wg-mic-bd8".allowedTCPPorts = [ 179 ]; networking.firewall.interfaces."wg-mic-der".allowedTCPPorts = [ 179 ]; - # Traffic can return on the leg that is not the current best route. - # Strict reverse-path filtering drops that; loose keeps the spoof check. + networking.firewall.interfaces."wg-mic-bd8".allowedTCPPorts = [ 179 ]; + # Two legs means replies can arrive on the leg that is not the best route + # back. Strict reverse-path filtering drops those; loose keeps the spoof + # check (a route must exist) without caring which leg it points at. networking.firewall.checkReversePath = lib.mkDefault "loose"; networking.networkmanager.unmanaged = @@ -213,55 +114,33 @@ in { ip prefix-list OWN-V4 seq 5 permit ${id4}/32 ipv6 prefix-list OWN-V6 seq 5 permit ${id6}/128 ! - ! Accept the default the mesh hands us, plus mesh space. Nothing else: - ! public prefixes are the far end's business (it does the RU/intl - ! split), and the mesh carries anycast /32s for public services that - ! must not be installed here. - ip prefix-list MESH-V4 seq 1 permit 0.0.0.0/0 - ! Preserve Heather's narrower accepted mesh space (not YC 10.130/24). + ! Accept mesh space only: never a default, never public prefixes. The + ! mesh carries anycast /32s for public services (DNS resolvers etc.); + ! accepting those would steer this host's own traffic into AstrNet. + ! It also keeps the tunnel endpoints themselves out of the tunnels. + ! Narrow to AstrNet address space; do not accept Yandex Cloud 10.130/24. ip prefix-list MESH-V4 seq 5 permit 10.0.0.0/24 ge 24 le 32 ip prefix-list MESH-V4 seq 10 permit 10.0.1.0/24 ge 24 le 32 ip prefix-list MESH-V4 seq 15 permit 10.0.2.0/24 ge 24 le 32 ip prefix-list MESH-V4 seq 20 permit 10.99.0.0/22 ge 22 le 32 - ipv6 prefix-list MESH-V6 seq 1 permit ::/0 ipv6 prefix-list MESH-V6 seq 5 permit 2a05:f480:2400:1cf7::/64 le 128 ipv6 prefix-list MESH-V6 seq 10 permit fd53:4023:2326::/48 le 128 ! - ! bd8816 is primary (same city): higher local-preference inbound... - route-map IN-BD8-V4 permit 10 - match ip address prefix-list MESH-V4 - set local-preference 200 - exit - route-map IN-BD8-V6 permit 10 - match ipv6 address prefix-list MESH-V6 - set local-preference 200 - exit - route-map IN-DER-V4 permit 10 + route-map MESH-IN-V4 permit 10 match ip address prefix-list MESH-V4 exit - route-map IN-DER-V6 permit 10 + route-map MESH-IN-V6 permit 10 match ipv6 address prefix-list MESH-V6 exit - ! - ! ...and a longer path via alderamin outbound, so replies from the rest - ! of the mesh come back through bd8816 too. - route-map OUT-BD8-V4 permit 10 + route-map MESH-OUT-V4 permit 10 match ip address prefix-list OWN-V4 exit - route-map OUT-BD8-V6 permit 10 + route-map MESH-OUT-V6 permit 10 match ipv6 address prefix-list OWN-V6 exit - route-map OUT-DER-V4 permit 10 - match ip address prefix-list OWN-V4 - set as-path prepend 4200000009 4200000009 - exit - route-map OUT-DER-V6 permit 10 - match ipv6 address prefix-list OWN-V6 - set as-path prepend 4200000009 4200000009 - exit ! - ! Source everything routed via BGP (the mesh default included) from the - ! identity; the exits NAT that address. + ! Source mesh traffic from the identity, not from whichever tunnel /30 + ! it leaves by; otherwise replies break whenever the path changes. route-map SET-MESH-SRC permit 10 set src ${id4} exit @@ -275,39 +154,40 @@ in { bgp router-id ${id4} no bgp ebgp-requires-policy no bgp default ipv4-unicast - neighbor 10.99.0.157 remote-as 4200000005 - neighbor 10.99.0.157 description bd8816-v4-wg-mic-bd8 - neighbor 10.99.0.157 update-source 10.99.0.158 + bgp bestpath as-path multipath-relax neighbor 10.99.0.153 remote-as 4200000012 neighbor 10.99.0.153 description alderamin-v4-wg-mic-der neighbor 10.99.0.153 update-source 10.99.0.154 - neighbor fd53:4023:2326:99::9d remote-as 4200000005 - neighbor fd53:4023:2326:99::9d description bd8816-v6-wg-mic-bd8 - neighbor fd53:4023:2326:99::9d interface wg-mic-bd8 - neighbor fd53:4023:2326:99::9d update-source fd53:4023:2326:99::9e + neighbor 10.99.0.157 remote-as 4200000005 + neighbor 10.99.0.157 description bd8816-v4-wg-mic-bd8 + neighbor 10.99.0.157 update-source 10.99.0.158 neighbor fd53:4023:2326:99::99 remote-as 4200000012 neighbor fd53:4023:2326:99::99 description alderamin-v6-wg-mic-der neighbor fd53:4023:2326:99::99 interface wg-mic-der neighbor fd53:4023:2326:99::99 update-source fd53:4023:2326:99::9a + neighbor fd53:4023:2326:99::9d remote-as 4200000005 + neighbor fd53:4023:2326:99::9d description bd8816-v6-wg-mic-bd8 + neighbor fd53:4023:2326:99::9d interface wg-mic-bd8 + neighbor fd53:4023:2326:99::9d update-source fd53:4023:2326:99::9e ! address-family ipv4 unicast network ${id4}/32 - neighbor 10.99.0.157 activate - neighbor 10.99.0.157 route-map IN-BD8-V4 in - neighbor 10.99.0.157 route-map OUT-BD8-V4 out neighbor 10.99.0.153 activate - neighbor 10.99.0.153 route-map IN-DER-V4 in - neighbor 10.99.0.153 route-map OUT-DER-V4 out + neighbor 10.99.0.153 route-map MESH-IN-V4 in + neighbor 10.99.0.153 route-map MESH-OUT-V4 out + neighbor 10.99.0.157 activate + neighbor 10.99.0.157 route-map MESH-IN-V4 in + neighbor 10.99.0.157 route-map MESH-OUT-V4 out exit-address-family ! address-family ipv6 unicast network ${id6}/128 - neighbor fd53:4023:2326:99::9d activate - neighbor fd53:4023:2326:99::9d route-map IN-BD8-V6 in - neighbor fd53:4023:2326:99::9d route-map OUT-BD8-V6 out neighbor fd53:4023:2326:99::99 activate - neighbor fd53:4023:2326:99::99 route-map IN-DER-V6 in - neighbor fd53:4023:2326:99::99 route-map OUT-DER-V6 out + neighbor fd53:4023:2326:99::99 route-map MESH-IN-V6 in + neighbor fd53:4023:2326:99::99 route-map MESH-OUT-V6 out + neighbor fd53:4023:2326:99::9d activate + neighbor fd53:4023:2326:99::9d route-map MESH-IN-V6 in + neighbor fd53:4023:2326:99::9d route-map MESH-OUT-V6 out exit-address-family exit '';