From 799c1978fafc07fae8a15a42c8535b70e6c69e6e Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 4 Sep 2024 12:17:44 -0700 Subject: [PATCH] tun: add method for disabling TCP GRO on Linux torvalds/linux@e269d79c7d35aa3808b1f3c1737d63dab504ddc8 broke virtio_net TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then resolved later in torvalds/linux@89add40066f9ed9abe5f7f886fe5789ff7e0c50e. The offending commit was pulled into various LTS releases. Updates tailscale/tailscale#13041 Signed-off-by: Jordan Whited --- tun/offload_linux.go | 18 ++++----- tun/offload_linux_test.go | 82 ++++++++++++++++++++++----------------- tun/tun.go | 23 +++++++++-- tun/tun_linux.go | 45 ++++++++++++++++++--- 4 files changed, 114 insertions(+), 54 deletions(-) diff --git a/tun/offload_linux.go b/tun/offload_linux.go index fe34401..fb6ac5b 100644 --- a/tun/offload_linux.go +++ b/tun/offload_linux.go @@ -763,7 +763,7 @@ const ( udp6GROCandidate ) -func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType { +func packetIsGROCandidate(b []byte, gro groDisablementFlags) groCandidateType { if len(b) < 28 { return notGROCandidate } @@ -772,17 +772,17 @@ func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType { // IPv4 packets w/IP options do not coalesce return notGROCandidate } - if b[9] == unix.IPPROTO_TCP && len(b) >= 40 { + if b[9] == unix.IPPROTO_TCP && len(b) >= 40 && gro.canTCPGRO() { return tcp4GROCandidate } - if b[9] == unix.IPPROTO_UDP && canUDPGRO { + if b[9] == unix.IPPROTO_UDP && gro.canUDPGRO() { return udp4GROCandidate } } else if b[0]>>4 == 6 { - if b[6] == unix.IPPROTO_TCP && len(b) >= 60 { + if b[6] == unix.IPPROTO_TCP && len(b) >= 60 && gro.canTCPGRO() { return tcp6GROCandidate } - if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && canUDPGRO { + if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && gro.canUDPGRO() { return udp6GROCandidate } } @@ -875,15 +875,15 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool) // handleGRO evaluates bufs for GRO, and writes the indices of the resulting // packets into toWrite. toWrite, tcpTable, and udpTable should initially be // empty (but non-nil), and are passed in to save allocs as the caller may reset -// and recycle them across vectors of packets. canUDPGRO indicates if UDP GRO is -// supported. -func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, canUDPGRO bool, toWrite *[]int) error { +// and recycle them across vectors of packets. gro indicates if TCP and UDP GRO +// are supported/enabled. +func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, gro groDisablementFlags, toWrite *[]int) error { for i := range bufs { if offset < virtioNetHdrLen || offset > len(bufs[i])-1 { return errors.New("invalid offset") } var result groResult - switch packetIsGROCandidate(bufs[i][offset:], canUDPGRO) { + switch packetIsGROCandidate(bufs[i][offset:], gro) { case tcp4GROCandidate: result = tcpGRO(bufs, offset, i, tcpTable, false) case tcp6GROCandidate: diff --git a/tun/offload_linux_test.go b/tun/offload_linux_test.go index 91f3941..4070378 100644 --- a/tun/offload_linux_test.go +++ b/tun/offload_linux_test.go @@ -286,11 +286,11 @@ func Fuzz_handleGRO(f *testing.F) { pkt9 := udp6Packet(ip6PortA, ip6PortB, 100) pkt10 := udp6Packet(ip6PortA, ip6PortB, 100) pkt11 := udp6Packet(ip6PortA, ip6PortC, 100) - f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, true, offset) - f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, canUDPGRO bool, offset int) { + f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, 0, offset) + f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, gro int, offset int) { pkts := [][]byte{pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11} toWrite := make([]int, 0, len(pkts)) - handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), canUDPGRO, &toWrite) + handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), groDisablementFlags(gro), &toWrite) if len(toWrite) > len(pkts) { t.Errorf("len(toWrite): %d > len(pkts): %d", len(toWrite), len(pkts)) } @@ -311,7 +311,7 @@ func Test_handleGRO(t *testing.T) { tests := []struct { name string pktsIn [][]byte - canUDPGRO bool + gro groDisablementFlags wantToWrite []int wantLens []int wantErr bool @@ -331,7 +331,7 @@ func Test_handleGRO(t *testing.T) { udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 }, - true, + 0, []int{0, 1, 2, 4, 5, 7, 9}, []int{240, 228, 128, 140, 260, 160, 248}, false, @@ -351,7 +351,7 @@ func Test_handleGRO(t *testing.T) { udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 }, - false, + udpGRODisabled, []int{0, 1, 2, 4, 5, 7, 8, 9, 10}, []int{240, 128, 128, 140, 260, 160, 128, 148, 148}, false, @@ -368,7 +368,7 @@ func Test_handleGRO(t *testing.T) { tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 201), // v6 flow 1 tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 301), // v6 flow 1 }, - true, + 0, []int{0, 2, 4, 6}, []int{240, 240, 260, 260}, false, @@ -383,7 +383,7 @@ func Test_handleGRO(t *testing.T) { udp4Packet(ip4PortA, ip4PortB, 100), udp4Packet(ip4PortA, ip4PortB, 100), }, - true, + 0, []int{0, 1, 3, 4}, []int{140, 240, 128, 228}, false, @@ -395,7 +395,7 @@ func Test_handleGRO(t *testing.T) { tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // v4 flow 1 seq 1 len 100 tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 201), // v4 flow 1 seq 201 len 100 }, - true, + 0, []int{0}, []int{340}, false, @@ -412,7 +412,7 @@ func Test_handleGRO(t *testing.T) { fields.TTL++ }), }, - true, + 0, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -429,7 +429,7 @@ func Test_handleGRO(t *testing.T) { fields.TOS++ }), }, - true, + 0, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -446,7 +446,7 @@ func Test_handleGRO(t *testing.T) { fields.Flags = 1 }), }, - true, + 0, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -463,7 +463,7 @@ func Test_handleGRO(t *testing.T) { fields.Flags = 2 }), }, - true, + 0, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -480,7 +480,7 @@ func Test_handleGRO(t *testing.T) { fields.HopLimit++ }), }, - true, + 0, []int{0, 1, 2, 3}, []int{160, 160, 148, 148}, false, @@ -497,7 +497,7 @@ func Test_handleGRO(t *testing.T) { fields.TrafficClass++ }), }, - true, + 0, []int{0, 1, 2, 3}, []int{160, 160, 148, 148}, false, @@ -507,7 +507,7 @@ func Test_handleGRO(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { toWrite := make([]int, 0, len(tt.pktsIn)) - err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), tt.canUDPGRO, &toWrite) + err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), tt.gro, &toWrite) if err != nil { if tt.wantErr { return @@ -552,99 +552,111 @@ func Test_packetIsGROCandidate(t *testing.T) { udp6TooShort := udp6[:47] tests := []struct { - name string - b []byte - canUDPGRO bool - want groCandidateType + name string + b []byte + gro groDisablementFlags + want groCandidateType }{ { "tcp4", tcp4, - true, + 0, tcp4GROCandidate, }, + { + "tcp4 no support", + tcp4, + tcpGRODisabled, + notGROCandidate, + }, { "tcp6", tcp6, - true, + 0, tcp6GROCandidate, }, + { + "tcp6 no support", + tcp6, + tcpGRODisabled, + notGROCandidate, + }, { "udp4", udp4, - true, + 0, udp4GROCandidate, }, { "udp4 no support", udp4, - false, + udpGRODisabled, notGROCandidate, }, { "udp6", udp6, - true, + 0, udp6GROCandidate, }, { "udp6 no support", udp6, - false, + udpGRODisabled, notGROCandidate, }, { "udp4 too short", udp4TooShort, - true, + 0, notGROCandidate, }, { "udp6 too short", udp6TooShort, - true, + 0, notGROCandidate, }, { "tcp4 too short", tcp4TooShort, - true, + 0, notGROCandidate, }, { "tcp6 too short", tcp6TooShort, - true, + 0, notGROCandidate, }, { "invalid IP version", []byte{0x00}, - true, + 0, notGROCandidate, }, { "invalid IP header len", ip4InvalidHeaderLen, - true, + 0, notGROCandidate, }, { "ip4 invalid protocol", ip4InvalidProtocol, - true, + 0, notGROCandidate, }, { "ip6 invalid protocol", ip6InvalidProtocol, - true, + 0, notGROCandidate, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := packetIsGROCandidate(tt.b, tt.canUDPGRO); got != tt.want { + if got := packetIsGROCandidate(tt.b, tt.gro); got != tt.want { t.Errorf("packetIsGROCandidate() = %v, want %v", got, tt.want) } }) diff --git a/tun/tun.go b/tun/tun.go index d3c5012..719a606 100644 --- a/tun/tun.go +++ b/tun/tun.go @@ -52,10 +52,25 @@ type Device interface { BatchSize() int } -type LinuxDevice interface { +// GRODevice is a Device extended with methods for disabling GRO. Certain OS +// versions may have offload bugs. Where these bugs negatively impact throughput +// or break connectivity entirely we can use these methods to disable the +// related offload. +// +// Linux has the following known, GRO bugs. +// +// torvalds/linux@e269d79c7d35aa3808b1f3c1737d63dab504ddc8 broke virtio_net +// TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then +// resolved later in +// torvalds/linux@89add40066f9ed9abe5f7f886fe5789ff7e0c50e. The offending +// commit was pulled into various LTS releases. +// +// UDP GRO writes end up blackholing/dropping packets destined for a +// vxlan/geneve interface on kernel versions prior to 6.8.5. +type GRODevice interface { Device - // DisableUDPGRO disables UDP GRO if it is enabled. Certain device drivers - // (e.g. vxlan, geneve) do not properly handle coalesced UDP packets later - // in the stack, resulting in packet loss. + // DisableUDPGRO disables UDP GRO if it is enabled. DisableUDPGRO() + // DisableTCPGRO disables TCP GRO if it is enabled. + DisableTCPGRO() } diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 664eecc..4a03387 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -38,7 +38,6 @@ type NativeTun struct { statusListenersShutdown chan struct{} batchSize int vnetHdr bool - udpGSO bool closeOnce sync.Once @@ -53,7 +52,30 @@ type NativeTun struct { toWrite []int tcpGROTable *tcpGROTable udpGROTable *udpGROTable - udpGRO bool + gro groDisablementFlags +} + +type groDisablementFlags int + +const ( + tcpGRODisabled groDisablementFlags = 1 << iota + udpGRODisabled +) + +func (g *groDisablementFlags) disableTCPGRO() { + *g |= tcpGRODisabled +} + +func (g *groDisablementFlags) canTCPGRO() bool { + return (*g)&tcpGRODisabled == 0 +} + +func (g *groDisablementFlags) disableUDPGRO() { + *g |= udpGRODisabled +} + +func (g *groDisablementFlags) canUDPGRO() bool { + return (*g)&udpGRODisabled == 0 } func (tun *NativeTun) File() *os.File { @@ -346,7 +368,7 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) { ) tun.toWrite = tun.toWrite[:0] if tun.vnetHdr { - err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.udpGRO, &tun.toWrite) + err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.gro, &tun.toWrite) if err != nil { return 0, err } @@ -462,9 +484,19 @@ func (tun *NativeTun) BatchSize() int { return tun.batchSize } +// DisableUDPGRO disables UDP GRO if it is enabled. See the GRODevice interface +// for cases where it should be called. func (tun *NativeTun) DisableUDPGRO() { tun.writeOpMu.Lock() - tun.udpGRO = false + tun.gro.disableUDPGRO() + tun.writeOpMu.Unlock() +} + +// DisableTCPGRO disables TCP GRO if it is enabled. See the GRODevice interface +// for cases where it should be called. +func (tun *NativeTun) DisableTCPGRO() { + tun.writeOpMu.Lock() + tun.gro.disableTCPGRO() tun.writeOpMu.Unlock() } @@ -503,8 +535,9 @@ func (tun *NativeTun) initFromFlags(name string) error { tun.batchSize = conn.IdealBatchSize // tunUDPOffloads were added in Linux v6.2. We do not return an // error if they are unsupported at runtime. - tun.udpGSO = unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) == nil - tun.udpGRO = tun.udpGSO + if unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) != nil { + tun.gro.disableUDPGRO() + } } else { tun.batchSize = 1 }