mikl/nix-config
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

heather: route outbound via AstrNet with WAN failback

Browse source
This commit is contained in:
heather 2026-09-26 23:57:46 +00:00
commit b0d1de279f
1 changed files with 134 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

185
hosts/heather/astrnet.nix
View file

@ -1,27 +1,52 @@
# astrnet.nix — AstrNet stub peer "michaotic". Generated by astrnet-join.sh.
# astrnet.nix — AstrNet peer "michaotic". Generated by astrnet-join.sh.
#
# What this host is to the mesh: identity 10.0.0.9 / 2a05:f480:2400:1cf7::9,
# ASN 4200000009, two WireGuard legs that THIS host dials:
# wg-mic-der -> alderamin (Helsinki) primary, pair 38
# wg-mic-bd8 -> bd8816 (Moscow) backup, pair 39 (paths come prepended)
# It announces only its identity and accepts only mesh address space; it
# never becomes transit and never takes a default route from the mesh.
# Identity 10.0.0.9 / 2a05:f480:2400:1cf7::9, ASN 4200000009, two WireGuard
# legs that THIS host dials:
# wg-mic-bd8 -> bd8816 (Moscow) PRIMARY, pair 39
# wg-mic-der -> alderamin (Helsinki) backup, pair 38
#
# All values are assigned by the AstrNet admin. Do not renumber anything
# here without them: the far ends are configured to match exactly.
# Egress works like an AstrNet client: both legs announce a default route,
# and this host sends all its traffic into the mesh. The far end splits it:
# RU destinations leave via the RU exit, everything else via the
# international exits. Exceptions stay on the WAN: WireGuard's own packets,
# replies from the WAN address (inbound services keep working), and the cloud
# metadata service. If both BGP sessions drop, the BGP default is withdrawn
# and the WAN default takes over on its own.
#
# The WAN values below were detected at install time. Everything else is
# assigned by the AstrNet admin; do not renumber it.
{ config, lib, pkgs, ... }:
let
keyFile = "/var/lib/astrnet/wg.key";
id4 = "10.0.0.9";
id6 = "2a05:f480:2400:1cf7::9";
# ── Detected WAN ──
wanIf = "enp7s0";
wanGw4 = "10.130.0.1";
wanIp4 = "10.130.0.36";
wanGw6 = ""; # "" when there is no IPv6 WAN
wanIp6 = "";
# Table holding the WAN default for the exceptions above, and the mark on
# WireGuard's outer UDP that selects it (without it the tunnels would try
# to reach their own endpoints through the mesh default).
wanTable = "100";
wgMark = "0x4d49";
wanRules =
[ { fam = "-4"; sel = "fwmark ${wgMark}"; }
{ fam = "-4"; sel = "from ${wanIp4}"; }
]
++ lib.optional (wanGw6 != "") { fam = "-6"; sel = "fwmark ${wgMark}"; }
++ lib.optional (wanGw6 != "" && wanIp6 != "") { fam = "-6"; sel = "from ${wanIp6}"; };
mkLeg = { listenPort, ips, publicKey, endpoint }: {
inherit ips listenPort;
privateKeyFile = keyFile;
mtu = 1380;
fwMark = wgMark;
# Routing is BGP's job. allowedIPs 0/0 only PERMITS any source through
# the tunnel; with routes off, it installs nothing. Turning routes on
# would hijack this host's default route.
# the tunnel; with routes off it installs nothing.
allowedIPsAsRoutes = false;
peers = [{
inherit publicKey endpoint;
@ -33,23 +58,23 @@ in {
networking.wireguard.interfaces = {
# The fe80:: link-locals are required: without one FRR's IPv6 BGP
# next-hop selection fails and the v6 session never leaves Connect.
wg-mic-der = mkLeg {
listenPort = 52038;
ips = [ "10.99.0.154/30" "fd53:4023:2326:99::9a/126" "fe80::2/64" ];
publicKey = "a4FyXuoI16TxZxv407lSzJV2YFJdq6FgRAtGu+oMzEY=";
endpoint = "31.77.105.157:52038";
};
wg-mic-bd8 = mkLeg {
listenPort = 52039;
ips = [ "10.99.0.158/30" "fd53:4023:2326:99::9e/126" "fe80::2/64" ];
publicKey = "QIvX+avvJqaHxuksGWb6Hx8fv7R6VI5wpaGqLme6Fgw=";
endpoint = "195.133.60.217:52039";
};
wg-mic-der = mkLeg {
listenPort = 52038;
ips = [ "10.99.0.154/30" "fd53:4023:2326:99::9a/126" "fe80::2/64" ];
publicKey = "a4FyXuoI16TxZxv407lSzJV2YFJdq6FgRAtGu+oMzEY=";
endpoint = "31.77.105.157:52038";
};
};
# Explicit owner decision: allow all AstrNet members to access Heather's
# host services over either WG leg. This does NOT open additional WAN ports,
# enable forwarding from AstrNet, or route public traffic through the mesh.
# enable forwarding from AstrNet. Egress routing is configured below.
# WireGuard peers use allowedIPs 0/0 as an ACL, so do not rely on a packet's
# source address to identify a particular AstrNet client here.
systemd.services.astrnet-guard = {
@ -96,12 +121,49 @@ in {
preStop = "ip link del mesh0 || true";
};
# Traffic that must keep using the WAN even while the mesh default is up.
systemd.services.astrnet-wan-policy = {
description = "AstrNet: keep tunnel, inbound replies and metadata on the WAN";
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
before = [ "frr.service" ];
requiredBy = [ "frr.service" ];
path = [ pkgs.iproute2 ];
serviceConfig = { Type = "oneshot"; RemainAfterExit = true; };
script = ''
ip route replace default via ${wanGw4} dev ${wanIf} table ${wanTable}
'' + lib.optionalString (wanGw6 != "") ''
ip -6 route replace default via ${wanGw6} dev ${wanIf} table ${wanTable}
'' + ''
# The cloud metadata endpoint must bypass Docker's 169.254/16 link-local
# routes; a rule at priority 1000 would never be reached.
ip -4 rule del to 169.254.169.254 lookup ${wanTable} priority 998 2>/dev/null || true
ip -4 rule add to 169.254.169.254 lookup ${wanTable} priority 998
# Specific routes in main (the local subnet, mesh space) always win, so
# the WAN table below only ever decides where a DEFAULT would go.
for fam in -4 -6; do
ip $fam rule del lookup main suppress_prefixlength 0 priority 999 2>/dev/null || true
ip $fam rule add lookup main suppress_prefixlength 0 priority 999
done
'' + lib.concatMapStrings (r: ''
ip ${r.fam} rule del ${r.sel} lookup ${wanTable} priority 1000 2>/dev/null || true
ip ${r.fam} rule add ${r.sel} lookup ${wanTable} priority 1000
'') wanRules;
preStop = ''
ip -4 rule del to 169.254.169.254 lookup ${wanTable} priority 998 || true
ip -4 rule del lookup main suppress_prefixlength 0 priority 999 || true
ip -6 rule del lookup main suppress_prefixlength 0 priority 999 || true
'' + lib.concatMapStrings (r: ''
ip ${r.fam} rule del ${r.sel} lookup ${wanTable} priority 1000 || true
'') wanRules;
};
# BGP only inside the tunnels, never on the public interface.
networking.firewall.interfaces."wg-mic-der".allowedTCPPorts = [ 179 ];
networking.firewall.interfaces."wg-mic-bd8".allowedTCPPorts = [ 179 ];
# Two legs means replies can arrive on the leg that is not the best route
# back. Strict reverse-path filtering drops those; loose keeps the spoof
# check (a route must exist) without caring which leg it points at.
networking.firewall.interfaces."wg-mic-der".allowedTCPPorts = [ 179 ];
# Traffic can return on the leg that is not the current best route.
# Strict reverse-path filtering drops that; loose keeps the spoof check.
networking.firewall.checkReversePath = lib.mkDefault "loose";
networking.networkmanager.unmanaged =
@ -114,33 +176,55 @@ in {
ip prefix-list OWN-V4 seq 5 permit ${id4}/32
ipv6 prefix-list OWN-V6 seq 5 permit ${id6}/128
!
! Accept mesh space only: never a default, never public prefixes. The
! mesh carries anycast /32s for public services (DNS resolvers etc.);
! accepting those would steer this host's own traffic into AstrNet.
! It also keeps the tunnel endpoints themselves out of the tunnels.
! Narrow to AstrNet address space; do not accept Yandex Cloud 10.130/24.
! Accept the default the mesh hands us, plus mesh space. Nothing else:
! public prefixes are the far end's business (it does the RU/intl
! split), and the mesh carries anycast /32s for public services that
! must not be installed here.
ip prefix-list MESH-V4 seq 1 permit 0.0.0.0/0
! Preserve Heather's narrower accepted mesh space (not YC 10.130/24).
ip prefix-list MESH-V4 seq 5 permit 10.0.0.0/24 ge 24 le 32
ip prefix-list MESH-V4 seq 10 permit 10.0.1.0/24 ge 24 le 32
ip prefix-list MESH-V4 seq 15 permit 10.0.2.0/24 ge 24 le 32
ip prefix-list MESH-V4 seq 20 permit 10.99.0.0/22 ge 22 le 32
ipv6 prefix-list MESH-V6 seq 1 permit ::/0
ipv6 prefix-list MESH-V6 seq 5 permit 2a05:f480:2400:1cf7::/64 le 128
ipv6 prefix-list MESH-V6 seq 10 permit fd53:4023:2326::/48 le 128
!
route-map MESH-IN-V4 permit 10
! bd8816 is primary (same city): higher local-preference inbound...
route-map IN-BD8-V4 permit 10
match ip address prefix-list MESH-V4
set local-preference 200
exit
route-map IN-BD8-V6 permit 10
match ipv6 address prefix-list MESH-V6
set local-preference 200
exit
route-map IN-DER-V4 permit 10
match ip address prefix-list MESH-V4
exit
route-map MESH-IN-V6 permit 10
route-map IN-DER-V6 permit 10
match ipv6 address prefix-list MESH-V6
exit
route-map MESH-OUT-V4 permit 10
!
! ...and a longer path via alderamin outbound, so replies from the rest
! of the mesh come back through bd8816 too.
route-map OUT-BD8-V4 permit 10
match ip address prefix-list OWN-V4
exit
route-map MESH-OUT-V6 permit 10
route-map OUT-BD8-V6 permit 10
match ipv6 address prefix-list OWN-V6
exit
route-map OUT-DER-V4 permit 10
match ip address prefix-list OWN-V4
set as-path prepend 4200000009 4200000009
exit
route-map OUT-DER-V6 permit 10
match ipv6 address prefix-list OWN-V6
set as-path prepend 4200000009 4200000009
exit
!
! Source mesh traffic from the identity, not from whichever tunnel /30
! it leaves by; otherwise replies break whenever the path changes.
! Source everything routed via BGP (the mesh default included) from the
! identity; the exits NAT that address.
route-map SET-MESH-SRC permit 10
set src ${id4}
exit
@ -154,40 +238,39 @@ in {
bgp router-id ${id4}
no bgp ebgp-requires-policy
no bgp default ipv4-unicast
bgp bestpath as-path multipath-relax
neighbor 10.99.0.153 remote-as 4200000012
neighbor 10.99.0.153 description alderamin-v4-wg-mic-der
neighbor 10.99.0.153 update-source 10.99.0.154
neighbor 10.99.0.157 remote-as 4200000005
neighbor 10.99.0.157 description bd8816-v4-wg-mic-bd8
neighbor 10.99.0.157 update-source 10.99.0.158
neighbor fd53:4023:2326:99::99 remote-as 4200000012
neighbor fd53:4023:2326:99::99 description alderamin-v6-wg-mic-der
neighbor fd53:4023:2326:99::99 interface wg-mic-der
neighbor fd53:4023:2326:99::99 update-source fd53:4023:2326:99::9a
neighbor 10.99.0.153 remote-as 4200000012
neighbor 10.99.0.153 description alderamin-v4-wg-mic-der
neighbor 10.99.0.153 update-source 10.99.0.154
neighbor fd53:4023:2326:99::9d remote-as 4200000005
neighbor fd53:4023:2326:99::9d description bd8816-v6-wg-mic-bd8
neighbor fd53:4023:2326:99::9d interface wg-mic-bd8
neighbor fd53:4023:2326:99::9d update-source fd53:4023:2326:99::9e
neighbor fd53:4023:2326:99::99 remote-as 4200000012
neighbor fd53:4023:2326:99::99 description alderamin-v6-wg-mic-der
neighbor fd53:4023:2326:99::99 interface wg-mic-der
neighbor fd53:4023:2326:99::99 update-source fd53:4023:2326:99::9a
!
address-family ipv4 unicast
network ${id4}/32
neighbor 10.99.0.153 activate
neighbor 10.99.0.153 route-map MESH-IN-V4 in
neighbor 10.99.0.153 route-map MESH-OUT-V4 out
neighbor 10.99.0.157 activate
neighbor 10.99.0.157 route-map MESH-IN-V4 in
neighbor 10.99.0.157 route-map MESH-OUT-V4 out
neighbor 10.99.0.157 route-map IN-BD8-V4 in
neighbor 10.99.0.157 route-map OUT-BD8-V4 out
neighbor 10.99.0.153 activate
neighbor 10.99.0.153 route-map IN-DER-V4 in
neighbor 10.99.0.153 route-map OUT-DER-V4 out
exit-address-family
!
address-family ipv6 unicast
network ${id6}/128
neighbor fd53:4023:2326:99::99 activate
neighbor fd53:4023:2326:99::99 route-map MESH-IN-V6 in
neighbor fd53:4023:2326:99::99 route-map MESH-OUT-V6 out
neighbor fd53:4023:2326:99::9d activate
neighbor fd53:4023:2326:99::9d route-map MESH-IN-V6 in
neighbor fd53:4023:2326:99::9d route-map MESH-OUT-V6 out
neighbor fd53:4023:2326:99::9d route-map IN-BD8-V6 in
neighbor fd53:4023:2326:99::9d route-map OUT-BD8-V6 out
neighbor fd53:4023:2326:99::99 activate
neighbor fd53:4023:2326:99::99 route-map IN-DER-V6 in
neighbor fd53:4023:2326:99::99 route-map OUT-DER-V6 out
exit-address-family
exit
'';