Failover, and existing connections

#!/bin/sh
# ./failover-test.sh OUTFILE: break ISP1's upstream for 60 s while lanhost runs load.sh (new web requests,
# plus one long TCP connection to listen.sh on the server), then show what each saw. Writes out/OUTFILE.
cd "$(dirname "$0")"
f=out/$1; mkdir -p out
x() { docker compose exec -T "$@"; }
x server sh -c 'pkill -x nc; setsid listen.sh > /tmp/tcp.log 2>&1 < /dev/null &'
x lanhost sh -c 'setsid load.sh > /dev/null 2>&1 < /dev/null &'
sleep 10
printf 'root@isp1:~# date +%%T; ip link set eth1 down   # ISP1 loses its upstream\n' >> $f
x isp1 sh -c 'date +%T; ip link set eth1 down' >> $f
sleep 60
printf '\nroot@isp1:~# date +%%T; ip link set eth1 up\n' >> $f
x isp1 sh -c 'date +%T; ip link set eth1 up; ip route replace 198.51.100.0/30 via 192.0.2.2' >> $f
sleep 25
x lanhost sh -c 'while pgrep -x load.sh >/dev/null && ! [ -s /tmp/nc.txt ]; do sleep 1; done; sleep 1'
printf '\nroot@lanhost:~# cat web.txt   # a new web request every half second: time and HTTP status (000 = no answer)\n' >> $f
x lanhost sh -c 'cat /tmp/web.txt' >> $f
printf '\nroot@server:~# sed -n "8,13p;\$p" tcp.log   # the long-lived connection, one line a second\n' >> $f
x server sh -c 'sed -n "8,13p;\$p" /tmp/tcp.log' >> $f
printf '\nroot@lanhost:~# cat nc.txt   # when the sending side gave up\n' >> $f
x lanhost sh -c 'cat /tmp/nc.txt' >> $f

• Lines 7–8: Start the listener on the server, and the web requests and long connection on lanhost.
• Lines 10–14: Break ISP1's upstream for a minute, then restore it (and its route, which went with the link).
• Lines 16–22: Collect what each side saw.

#!/bin/sh
# On lanhost: a new web request every half second (web.txt), and one long TCP connection sending a line a second.
for i in $(seq 1 160); do printf '%s ' "$(date +%T)"; curl -s -m 1 -o /dev/null -w '%{http_code}\n' http://192.0.2.10/; sleep 0.5; done > /tmp/web.txt 2>&1 &
for i in $(seq 1 90); do echo "line $i sent $(date +%T)"; sleep 1; done | nc -q 1 192.0.2.10 9000
echo "nc exited at $(date +%T) with status $?" > /tmp/nc.txt

• Line 3: A new web request every half second, logged with the status (000 means no answer).
• Lines 4–5: One long TCP connection, sending a line a second.

root@isp1:~# date +%T; ip link set eth1 down   # ISP1 loses its upstream
09:31:00

root@isp1:~# date +%T; ip link set eth1 up
09:32:00

root@lanhost:~# cat web.txt   # a new web request every half second: time and HTTP status (000 = no answer)
[... 17 lines trimmed: every request answered 200 ...]
09:30:59 200
09:30:59 200
09:31:00 200
09:31:00 000
09:31:01 000
09:31:01 000
09:31:03 000
09:31:04 000
09:31:05 000
09:31:06 000
09:31:07 000
09:31:08 000
09:31:10 000
09:31:11 000
09:31:12 000
09:31:13 000
09:31:15 200
09:31:15 200
09:31:16 200
[... 124 lines trimmed: every request answered 200 ...]

root@server:~# sed -n "8,13p;\$p" tcp.log   # the long-lived connection, one line a second
line 8 sent 09:30:57, received 09:30:57
line 9 sent 09:30:58, received 09:30:58
line 10 sent 09:30:59, received 09:30:59
line 11 sent 09:31:00, received 09:31:00
line 12 sent 09:31:01, received 09:32:08
line 13 sent 09:31:02, received 09:32:08
line 90 sent 09:32:19, received 09:32:19

• Lines 1–5: ISP1's upstream down for exactly a minute.
• Lines 12–24: New requests fail for about 14 seconds…
• Lines 25–27: …then work through ISP2.
• Lines 35–36: The long connection stalled: sent at 09:31:01, delivered at 09:32:08, after ISP1 came back.

root@server:~# grep '09:3[12]:' /tmp/http.log | awk '{print $1, $5}' | uniq -c -w 14
      1 203.0.113.2 09:31:00]
     91 198.51.100.2 09:31:15]
     45 203.0.113.2 09:32:02]

• Line 3: ISP2's address from 09:31:15.
• Line 4: Back to ISP1 at 09:32:02, two seconds after it recovered.

set load-balancing wan flush-connections

root@isp1:~# date +%T; ip link set eth1 down   # ISP1 loses its upstream
09:34:56

root@isp1:~# date +%T; ip link set eth1 up
09:35:56

root@lanhost:~# cat web.txt   # a new web request every half second: time and HTTP status (000 = no answer)
[... 17 lines trimmed: every request answered 200 ...]
09:34:55 200
09:34:55 200
09:34:56 200
09:34:56 000
09:34:57 000
09:34:57 000
09:34:59 000
09:35:00 000
09:35:02 000
09:35:03 000
09:35:04 000
09:35:05 000
09:35:07 000
09:35:08 000
09:35:09 000
09:35:10 000
09:35:11 000
09:35:13 000
09:35:13 000
09:35:15 200
09:35:15 200
09:35:16 200
[... 121 lines trimmed: every request answered 200 ...]

root@server:~# sed -n "8,13p;\$p" tcp.log   # the long-lived connection, one line a second
line 8 sent 09:34:53, received 09:34:53
line 9 sent 09:34:54, received 09:34:54
line 10 sent 09:34:55, received 09:34:55
line 11 sent 09:34:56, received 09:34:56
line 11 sent 09:34:56, received 09:34:56

root@lanhost:~# cat nc.txt   # when the sending side gave up
nc exited at 09:35:16 with status 0

• Lines 34–38: The long connection ends at line 11: it was cut, not stalled.
• Lines 40–41: The sender gave up three seconds after traffic moved to ISP2, so it could reconnect at once.

vyos@R1:~$ sudo nft list table ip vyos_wanloadbalance
table ip vyos_wanloadbalance {
	chain wlb_nat_postrouting {
		type nat hook postrouting priority srcnat - 1; policy accept;
	}

	chain wlb_mangle_prerouting {
		type filter hook prerouting priority mangle; policy accept;
		iifname "eth0" ct state new ct mark set 0x000000c9
		iifname "eth1" ct state new ct mark set 0x000000ca
		iifname "eth2" ct state new counter packets 0 bytes 0 jump wlb_mangle_isp_eth0
		iifname "eth2" counter packets 11 bytes 1004 meta mark set ct mark
	}

	chain wlb_mangle_output {
		type filter hook output priority mangle; policy accept;
	}

	chain wlb_mangle_isp_eth0 {
		meta mark set 0x000000c9 ct mark set 0x000000c9 counter packets 0 bytes 0 accept
	}

	chain wlb_mangle_isp_eth1 {
		meta mark set 0x000000ca ct mark set 0x000000ca counter packets 0 bytes 0 accept
	}
}
vyos@R1:~$ ip rule
0:	from all lookup local
32764:	from all fwmark 0xca lookup 202
32765:	from all fwmark 0xc9 lookup 201
32766:	from all lookup main
32767:	from all lookup default
vyos@R1:~$ ip route show table 201
default via 203.0.113.1 dev eth0
vyos@R1:~$ ip route show table 202
default via 198.51.100.1 dev eth1

• Lines 9–10: Connections arriving from an ISP are marked for it (sticky inbound, from the next section).
• Line 11: New LAN connections jump to the current ISP's chain…
• Line 12: …and every LAN packet takes its connection's mark.
• Lines 19–25: 0xc9 means ISP1, 0xca ISP2.
• Lines 27–36: Marked packets use a routing table with that ISP's default route.