heather: route outbound via AstrNet with WAN failback

hosts/heather/astrnet.nix

@@ -1,27 +1,52 @@
-# astrnet.nix — AstrNet stub peer "michaotic". Generated by astrnet-join.sh.
+# astrnet.nix — AstrNet peer "michaotic". Generated by astrnet-join.sh.
 #
-# What this host is to the mesh: identity 10.0.0.9 / 2a05:f480:2400:1cf7::9,
+# Identity 10.0.0.9 / 2a05:f480:2400:1cf7::9, ASN 4200000009, two WireGuard
-# ASN 4200000009, two WireGuard legs that THIS host dials:
+# legs that THIS host dials:
-#   wg-mic-der -> alderamin (Helsinki)  primary, pair 38
+#   wg-mic-bd8 -> bd8816    (Moscow)    PRIMARY, pair 39
-#   wg-mic-bd8 -> bd8816    (Moscow)    backup,  pair 39 (paths come prepended)
+#   wg-mic-der -> alderamin (Helsinki)  backup,  pair 38
-# It announces only its identity and accepts only mesh address space; it
-# never becomes transit and never takes a default route from the mesh.
 #
-# All values are assigned by the AstrNet admin. Do not renumber anything
+# Egress works like an AstrNet client: both legs announce a default route,
-# here without them: the far ends are configured to match exactly.
+# and this host sends all its traffic into the mesh. The far end splits it:
+# RU destinations leave via the RU exit, everything else via the
+# international exits. Exceptions stay on the WAN: WireGuard's own packets,
+# replies from the WAN address (inbound services keep working), and the cloud
+# metadata service. If both BGP sessions drop, the BGP default is withdrawn
+# and the WAN default takes over on its own.
+#
+# The WAN values below were detected at install time. Everything else is
+# assigned by the AstrNet admin; do not renumber it.
 { config, lib, pkgs, ... }:
 let
   keyFile = "/var/lib/astrnet/wg.key";
   id4 = "10.0.0.9";
   id6 = "2a05:f480:2400:1cf7::9";
 
+  # ── Detected WAN ──
+  wanIf  = "enp7s0";
+  wanGw4 = "10.130.0.1";
+  wanIp4 = "10.130.0.36";
+  wanGw6 = "";   # "" when there is no IPv6 WAN
+  wanIp6 = "";
+
+  # Table holding the WAN default for the exceptions above, and the mark on
+  # WireGuard's outer UDP that selects it (without it the tunnels would try
+  # to reach their own endpoints through the mesh default).
+  wanTable = "100";
+  wgMark = "0x4d49";
+  wanRules =
+    [ { fam = "-4"; sel = "fwmark ${wgMark}"; }
+      { fam = "-4"; sel = "from ${wanIp4}"; }
+    ]
+    ++ lib.optional (wanGw6 != "") { fam = "-6"; sel = "fwmark ${wgMark}"; }
+    ++ lib.optional (wanGw6 != "" && wanIp6 != "") { fam = "-6"; sel = "from ${wanIp6}"; };
+
   mkLeg = { listenPort, ips, publicKey, endpoint }: {
     inherit ips listenPort;
     privateKeyFile = keyFile;
     mtu = 1380;
+    fwMark = wgMark;
     # Routing is BGP's job. allowedIPs 0/0 only PERMITS any source through
-    # the tunnel; with routes off, it installs nothing. Turning routes on
+    # the tunnel; with routes off it installs nothing.
-    # would hijack this host's default route.
     allowedIPsAsRoutes = false;
     peers = [{
       inherit publicKey endpoint;
@@ -33,23 +58,23 @@ in {
   networking.wireguard.interfaces = {
     # The fe80:: link-locals are required: without one FRR's IPv6 BGP
     # next-hop selection fails and the v6 session never leaves Connect.
-    wg-mic-der = mkLeg {
-      listenPort = 52038;
-      ips = [ "10.99.0.154/30" "fd53:4023:2326:99::9a/126" "fe80::2/64" ];
-      publicKey = "a4FyXuoI16TxZxv407lSzJV2YFJdq6FgRAtGu+oMzEY=";
-      endpoint = "31.77.105.157:52038";
-    };
     wg-mic-bd8 = mkLeg {
       listenPort = 52039;
       ips = [ "10.99.0.158/30" "fd53:4023:2326:99::9e/126" "fe80::2/64" ];
       publicKey = "QIvX+avvJqaHxuksGWb6Hx8fv7R6VI5wpaGqLme6Fgw=";
       endpoint = "195.133.60.217:52039";
     };
+    wg-mic-der = mkLeg {
+      listenPort = 52038;
+      ips = [ "10.99.0.154/30" "fd53:4023:2326:99::9a/126" "fe80::2/64" ];
+      publicKey = "a4FyXuoI16TxZxv407lSzJV2YFJdq6FgRAtGu+oMzEY=";
+      endpoint = "31.77.105.157:52038";
+    };
   };
 
   # Explicit owner decision: allow all AstrNet members to access Heather's
   # host services over either WG leg. This does NOT open additional WAN ports,
-  # enable forwarding from AstrNet, or route public traffic through the mesh.
+  # enable forwarding from AstrNet. Egress routing is configured below.
   # WireGuard peers use allowedIPs 0/0 as an ACL, so do not rely on a packet's
   # source address to identify a particular AstrNet client here.
   systemd.services.astrnet-guard = {
@@ -96,12 +121,49 @@ in {
     preStop = "ip link del mesh0 || true";
   };
 
+  # Traffic that must keep using the WAN even while the mesh default is up.
+  systemd.services.astrnet-wan-policy = {
+    description = "AstrNet: keep tunnel, inbound replies and metadata on the WAN";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    before = [ "frr.service" ];
+    requiredBy = [ "frr.service" ];
+    path = [ pkgs.iproute2 ];
+    serviceConfig = { Type = "oneshot"; RemainAfterExit = true; };
+    script = ''
+      ip route replace default via ${wanGw4} dev ${wanIf} table ${wanTable}
+    '' + lib.optionalString (wanGw6 != "") ''
+      ip -6 route replace default via ${wanGw6} dev ${wanIf} table ${wanTable}
+    '' + ''
+      # The cloud metadata endpoint must bypass Docker's 169.254/16 link-local
+      # routes; a rule at priority 1000 would never be reached.
+      ip -4 rule del to 169.254.169.254 lookup ${wanTable} priority 998 2>/dev/null || true
+      ip -4 rule add to 169.254.169.254 lookup ${wanTable} priority 998
+      # Specific routes in main (the local subnet, mesh space) always win, so
+      # the WAN table below only ever decides where a DEFAULT would go.
+      for fam in -4 -6; do
+        ip $fam rule del lookup main suppress_prefixlength 0 priority 999 2>/dev/null || true
+        ip $fam rule add lookup main suppress_prefixlength 0 priority 999
+      done
+    '' + lib.concatMapStrings (r: ''
+      ip ${r.fam} rule del ${r.sel} lookup ${wanTable} priority 1000 2>/dev/null || true
+      ip ${r.fam} rule add ${r.sel} lookup ${wanTable} priority 1000
+    '') wanRules;
+    preStop = ''
+      ip -4 rule del to 169.254.169.254 lookup ${wanTable} priority 998 || true
+      ip -4 rule del lookup main suppress_prefixlength 0 priority 999 || true
+      ip -6 rule del lookup main suppress_prefixlength 0 priority 999 || true
+    '' + lib.concatMapStrings (r: ''
+      ip ${r.fam} rule del ${r.sel} lookup ${wanTable} priority 1000 || true
+    '') wanRules;
+  };
+
   # BGP only inside the tunnels, never on the public interface.
-  networking.firewall.interfaces."wg-mic-der".allowedTCPPorts = [ 179 ];
   networking.firewall.interfaces."wg-mic-bd8".allowedTCPPorts = [ 179 ];
-  # Two legs means replies can arrive on the leg that is not the best route
+  networking.firewall.interfaces."wg-mic-der".allowedTCPPorts = [ 179 ];
-  # back. Strict reverse-path filtering drops those; loose keeps the spoof
+  # Traffic can return on the leg that is not the current best route.
-  # check (a route must exist) without caring which leg it points at.
+  # Strict reverse-path filtering drops that; loose keeps the spoof check.
   networking.firewall.checkReversePath = lib.mkDefault "loose";
 
   networking.networkmanager.unmanaged =
@@ -114,33 +176,55 @@ in {
       ip prefix-list OWN-V4 seq 5 permit ${id4}/32
       ipv6 prefix-list OWN-V6 seq 5 permit ${id6}/128
       !
-      ! Accept mesh space only: never a default, never public prefixes. The
+      ! Accept the default the mesh hands us, plus mesh space. Nothing else:
-      ! mesh carries anycast /32s for public services (DNS resolvers etc.);
+      ! public prefixes are the far end's business (it does the RU/intl
-      ! accepting those would steer this host's own traffic into AstrNet.
+      ! split), and the mesh carries anycast /32s for public services that
-      ! It also keeps the tunnel endpoints themselves out of the tunnels.
+      ! must not be installed here.
-      ! Narrow to AstrNet address space; do not accept Yandex Cloud 10.130/24.
+      ip prefix-list MESH-V4 seq 1 permit 0.0.0.0/0
+      ! Preserve Heather's narrower accepted mesh space (not YC 10.130/24).
       ip prefix-list MESH-V4 seq 5 permit 10.0.0.0/24 ge 24 le 32
       ip prefix-list MESH-V4 seq 10 permit 10.0.1.0/24 ge 24 le 32
       ip prefix-list MESH-V4 seq 15 permit 10.0.2.0/24 ge 24 le 32
       ip prefix-list MESH-V4 seq 20 permit 10.99.0.0/22 ge 22 le 32
+      ipv6 prefix-list MESH-V6 seq 1 permit ::/0
       ipv6 prefix-list MESH-V6 seq 5 permit 2a05:f480:2400:1cf7::/64 le 128
       ipv6 prefix-list MESH-V6 seq 10 permit fd53:4023:2326::/48 le 128
       !
-      route-map MESH-IN-V4 permit 10
+      ! bd8816 is primary (same city): higher local-preference inbound...
+      route-map IN-BD8-V4 permit 10
+       match ip address prefix-list MESH-V4
+       set local-preference 200
+      exit
+      route-map IN-BD8-V6 permit 10
+       match ipv6 address prefix-list MESH-V6
+       set local-preference 200
+      exit
+      route-map IN-DER-V4 permit 10
        match ip address prefix-list MESH-V4
       exit
-      route-map MESH-IN-V6 permit 10
+      route-map IN-DER-V6 permit 10
        match ipv6 address prefix-list MESH-V6
       exit
-      route-map MESH-OUT-V4 permit 10
+      !
+      ! ...and a longer path via alderamin outbound, so replies from the rest
+      ! of the mesh come back through bd8816 too.
+      route-map OUT-BD8-V4 permit 10
        match ip address prefix-list OWN-V4
       exit
-      route-map MESH-OUT-V6 permit 10
+      route-map OUT-BD8-V6 permit 10
        match ipv6 address prefix-list OWN-V6
       exit
+      route-map OUT-DER-V4 permit 10
+       match ip address prefix-list OWN-V4
+       set as-path prepend 4200000009 4200000009
+      exit
+      route-map OUT-DER-V6 permit 10
+       match ipv6 address prefix-list OWN-V6
+       set as-path prepend 4200000009 4200000009
+      exit
       !
-      ! Source mesh traffic from the identity, not from whichever tunnel /30
+      ! Source everything routed via BGP (the mesh default included) from the
-      ! it leaves by; otherwise replies break whenever the path changes.
+      ! identity; the exits NAT that address.
       route-map SET-MESH-SRC permit 10
        set src ${id4}
       exit
@@ -154,40 +238,39 @@ in {
        bgp router-id ${id4}
        no bgp ebgp-requires-policy
        no bgp default ipv4-unicast
-       bgp bestpath as-path multipath-relax
-       neighbor 10.99.0.153 remote-as 4200000012
-       neighbor 10.99.0.153 description alderamin-v4-wg-mic-der
-       neighbor 10.99.0.153 update-source 10.99.0.154
        neighbor 10.99.0.157 remote-as 4200000005
        neighbor 10.99.0.157 description bd8816-v4-wg-mic-bd8
        neighbor 10.99.0.157 update-source 10.99.0.158
-       neighbor fd53:4023:2326:99::99 remote-as 4200000012
+       neighbor 10.99.0.153 remote-as 4200000012
-       neighbor fd53:4023:2326:99::99 description alderamin-v6-wg-mic-der
+       neighbor 10.99.0.153 description alderamin-v4-wg-mic-der
-       neighbor fd53:4023:2326:99::99 interface wg-mic-der
+       neighbor 10.99.0.153 update-source 10.99.0.154
-       neighbor fd53:4023:2326:99::99 update-source fd53:4023:2326:99::9a
        neighbor fd53:4023:2326:99::9d remote-as 4200000005
        neighbor fd53:4023:2326:99::9d description bd8816-v6-wg-mic-bd8
        neighbor fd53:4023:2326:99::9d interface wg-mic-bd8
        neighbor fd53:4023:2326:99::9d update-source fd53:4023:2326:99::9e
+       neighbor fd53:4023:2326:99::99 remote-as 4200000012
+       neighbor fd53:4023:2326:99::99 description alderamin-v6-wg-mic-der
+       neighbor fd53:4023:2326:99::99 interface wg-mic-der
+       neighbor fd53:4023:2326:99::99 update-source fd53:4023:2326:99::9a
        !
        address-family ipv4 unicast
         network ${id4}/32
-        neighbor 10.99.0.153 activate
-        neighbor 10.99.0.153 route-map MESH-IN-V4 in
-        neighbor 10.99.0.153 route-map MESH-OUT-V4 out
         neighbor 10.99.0.157 activate
-        neighbor 10.99.0.157 route-map MESH-IN-V4 in
+        neighbor 10.99.0.157 route-map IN-BD8-V4 in
-        neighbor 10.99.0.157 route-map MESH-OUT-V4 out
+        neighbor 10.99.0.157 route-map OUT-BD8-V4 out
+        neighbor 10.99.0.153 activate
+        neighbor 10.99.0.153 route-map IN-DER-V4 in
+        neighbor 10.99.0.153 route-map OUT-DER-V4 out
        exit-address-family
        !
        address-family ipv6 unicast
         network ${id6}/128
-        neighbor fd53:4023:2326:99::99 activate
-        neighbor fd53:4023:2326:99::99 route-map MESH-IN-V6 in
-        neighbor fd53:4023:2326:99::99 route-map MESH-OUT-V6 out
         neighbor fd53:4023:2326:99::9d activate
-        neighbor fd53:4023:2326:99::9d route-map MESH-IN-V6 in
+        neighbor fd53:4023:2326:99::9d route-map IN-BD8-V6 in
-        neighbor fd53:4023:2326:99::9d route-map MESH-OUT-V6 out
+        neighbor fd53:4023:2326:99::9d route-map OUT-BD8-V6 out
+        neighbor fd53:4023:2326:99::99 activate
+        neighbor fd53:4023:2326:99::99 route-map IN-DER-V6 in
+        neighbor fd53:4023:2326:99::99 route-map OUT-DER-V6 out
        exit-address-family
       exit
     '';