eburet-oss/wireguard-go-awg2-lx
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tun: add method for disabling TCP GRO on Linux

Browse source 
torvalds/linux@e269d79c7d broke virtio_net
TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then
resolved later in
torvalds/linux@89add40066. The offending
commit was pulled into various LTS releases.

Updates tailscale/tailscale#13041

Signed-off-by: Jordan Whited <jordan@tailscale.com>
This commit is contained in:
Jordan Whited 2024-09-04 12:17:44 -07:00 • committed by Jordan Whited
parent 71393c576b
commit 799c1978fa
4 changed files with 114 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

18
tun/offload_linux.go
View file

@ -763,7 +763,7 @@ const (
udp6GROCandidate udp6GROCandidate
) )
func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType { func packetIsGROCandidate(b []byte, gro groDisablementFlags) groCandidateType {
if len(b) < 28 { if len(b) < 28 {
return notGROCandidate return notGROCandidate
} }
@ -772,17 +772,17 @@ func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType {
// IPv4 packets w/IP options do not coalesce // IPv4 packets w/IP options do not coalesce
return notGROCandidate return notGROCandidate
} }
if b[9] == unix.IPPROTO_TCP && len(b) >= 40 { if b[9] == unix.IPPROTO_TCP && len(b) >= 40 && gro.canTCPGRO() {
return tcp4GROCandidate return tcp4GROCandidate
} }
if b[9] == unix.IPPROTO_UDP && canUDPGRO { if b[9] == unix.IPPROTO_UDP && gro.canUDPGRO() {
return udp4GROCandidate return udp4GROCandidate
} }
} else if b[0]>>4 == 6 { } else if b[0]>>4 == 6 {
if b[6] == unix.IPPROTO_TCP && len(b) >= 60 { if b[6] == unix.IPPROTO_TCP && len(b) >= 60 && gro.canTCPGRO() {
return tcp6GROCandidate return tcp6GROCandidate
} }
if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && canUDPGRO { if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && gro.canUDPGRO() {
return udp6GROCandidate return udp6GROCandidate
} }
} }
@ -875,15 +875,15 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool)
// handleGRO evaluates bufs for GRO, and writes the indices of the resulting // handleGRO evaluates bufs for GRO, and writes the indices of the resulting
// packets into toWrite. toWrite, tcpTable, and udpTable should initially be // packets into toWrite. toWrite, tcpTable, and udpTable should initially be
// empty (but non-nil), and are passed in to save allocs as the caller may reset // empty (but non-nil), and are passed in to save allocs as the caller may reset
// and recycle them across vectors of packets. canUDPGRO indicates if UDP GRO is // and recycle them across vectors of packets. gro indicates if TCP and UDP GRO
// supported. // are supported/enabled.
func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, canUDPGRO bool, toWrite *[]int) error { func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, gro groDisablementFlags, toWrite *[]int) error {
for i := range bufs { for i := range bufs {
if offset < virtioNetHdrLen || offset > len(bufs[i])-1 { if offset < virtioNetHdrLen || offset > len(bufs[i])-1 {
return errors.New("invalid offset") return errors.New("invalid offset")
} }
var result groResult var result groResult
switch packetIsGROCandidate(bufs[i][offset:], canUDPGRO) { switch packetIsGROCandidate(bufs[i][offset:], gro) {
case tcp4GROCandidate: case tcp4GROCandidate:
result = tcpGRO(bufs, offset, i, tcpTable, false) result = tcpGRO(bufs, offset, i, tcpTable, false)
case tcp6GROCandidate: case tcp6GROCandidate:

76
tun/offload_linux_test.go
View file

@ -286,11 +286,11 @@ func Fuzz_handleGRO(f *testing.F) {
pkt9 := udp6Packet(ip6PortA, ip6PortB, 100) pkt9 := udp6Packet(ip6PortA, ip6PortB, 100)
pkt10 := udp6Packet(ip6PortA, ip6PortB, 100) pkt10 := udp6Packet(ip6PortA, ip6PortB, 100)
pkt11 := udp6Packet(ip6PortA, ip6PortC, 100) pkt11 := udp6Packet(ip6PortA, ip6PortC, 100)
f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, true, offset) f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, 0, offset)
f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, canUDPGRO bool, offset int) { f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, gro int, offset int) {
pkts := [][]byte{pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11} pkts := [][]byte{pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11}
toWrite := make([]int, 0, len(pkts)) toWrite := make([]int, 0, len(pkts))
handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), canUDPGRO, &toWrite) handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), groDisablementFlags(gro), &toWrite)
if len(toWrite) > len(pkts) { if len(toWrite) > len(pkts) {
t.Errorf("len(toWrite): %d > len(pkts): %d", len(toWrite), len(pkts)) t.Errorf("len(toWrite): %d > len(pkts): %d", len(toWrite), len(pkts))
} }
@ -311,7 +311,7 @@ func Test_handleGRO(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
pktsIn [][]byte pktsIn [][]byte
canUDPGRO bool gro groDisablementFlags
wantToWrite []int wantToWrite []int
wantLens []int wantLens []int
wantErr bool wantErr bool
@ -331,7 +331,7 @@ func Test_handleGRO(t *testing.T) {
udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1
udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1
}, },
true, 0,
[]int{0, 1, 2, 4, 5, 7, 9}, []int{0, 1, 2, 4, 5, 7, 9},
[]int{240, 228, 128, 140, 260, 160, 248}, []int{240, 228, 128, 140, 260, 160, 248},
false, false,
@ -351,7 +351,7 @@ func Test_handleGRO(t *testing.T) {
udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1
udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1
}, },
false, udpGRODisabled,
[]int{0, 1, 2, 4, 5, 7, 8, 9, 10}, []int{0, 1, 2, 4, 5, 7, 8, 9, 10},
[]int{240, 128, 128, 140, 260, 160, 128, 148, 148}, []int{240, 128, 128, 140, 260, 160, 128, 148, 148},
false, false,
@ -368,7 +368,7 @@ func Test_handleGRO(t *testing.T) {
tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 201), // v6 flow 1 tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 201), // v6 flow 1
tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 301), // v6 flow 1 tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 301), // v6 flow 1
}, },
true, 0,
[]int{0, 2, 4, 6}, []int{0, 2, 4, 6},
[]int{240, 240, 260, 260}, []int{240, 240, 260, 260},
false, false,
@ -383,7 +383,7 @@ func Test_handleGRO(t *testing.T) {
udp4Packet(ip4PortA, ip4PortB, 100), udp4Packet(ip4PortA, ip4PortB, 100),
udp4Packet(ip4PortA, ip4PortB, 100), udp4Packet(ip4PortA, ip4PortB, 100),
}, },
true, 0,
[]int{0, 1, 3, 4}, []int{0, 1, 3, 4},
[]int{140, 240, 128, 228}, []int{140, 240, 128, 228},
false, false,
@ -395,7 +395,7 @@ func Test_handleGRO(t *testing.T) {
tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // v4 flow 1 seq 1 len 100 tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // v4 flow 1 seq 1 len 100
tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 201), // v4 flow 1 seq 201 len 100 tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 201), // v4 flow 1 seq 201 len 100
}, },
true, 0,
[]int{0}, []int{0},
[]int{340}, []int{340},
false, false,
@ -412,7 +412,7 @@ func Test_handleGRO(t *testing.T) {
fields.TTL++ fields.TTL++
}), }),
}, },
true, 0,
[]int{0, 1, 2, 3}, []int{0, 1, 2, 3},
[]int{140, 140, 128, 128}, []int{140, 140, 128, 128},
false, false,
@ -429,7 +429,7 @@ func Test_handleGRO(t *testing.T) {
fields.TOS++ fields.TOS++
}), }),
}, },
true, 0,
[]int{0, 1, 2, 3}, []int{0, 1, 2, 3},
[]int{140, 140, 128, 128}, []int{140, 140, 128, 128},
false, false,
@ -446,7 +446,7 @@ func Test_handleGRO(t *testing.T) {
fields.Flags = 1 fields.Flags = 1
}), }),
}, },
true, 0,
[]int{0, 1, 2, 3}, []int{0, 1, 2, 3},
[]int{140, 140, 128, 128}, []int{140, 140, 128, 128},
false, false,
@ -463,7 +463,7 @@ func Test_handleGRO(t *testing.T) {
fields.Flags = 2 fields.Flags = 2
}), }),
}, },
true, 0,
[]int{0, 1, 2, 3}, []int{0, 1, 2, 3},
[]int{140, 140, 128, 128}, []int{140, 140, 128, 128},
false, false,
@ -480,7 +480,7 @@ func Test_handleGRO(t *testing.T) {
fields.HopLimit++ fields.HopLimit++
}), }),
}, },
true, 0,
[]int{0, 1, 2, 3}, []int{0, 1, 2, 3},
[]int{160, 160, 148, 148}, []int{160, 160, 148, 148},
false, false,
@ -497,7 +497,7 @@ func Test_handleGRO(t *testing.T) {
fields.TrafficClass++ fields.TrafficClass++
}), }),
}, },
true, 0,
[]int{0, 1, 2, 3}, []int{0, 1, 2, 3},
[]int{160, 160, 148, 148}, []int{160, 160, 148, 148},
false, false,
@ -507,7 +507,7 @@ func Test_handleGRO(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
toWrite := make([]int, 0, len(tt.pktsIn)) toWrite := make([]int, 0, len(tt.pktsIn))
err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), tt.canUDPGRO, &toWrite) err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), tt.gro, &toWrite)
if err != nil { if err != nil {
if tt.wantErr { if tt.wantErr {
return return
@ -554,97 +554,109 @@ func Test_packetIsGROCandidate(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
b []byte b []byte
canUDPGRO bool gro groDisablementFlags
want groCandidateType want groCandidateType
}{ }{
{ {
"tcp4", "tcp4",
tcp4, tcp4,
true, 0,
tcp4GROCandidate, tcp4GROCandidate,
}, },
{
"tcp4 no support",
tcp4,
tcpGRODisabled,
notGROCandidate,
},
{ {
"tcp6", "tcp6",
tcp6, tcp6,
true, 0,
tcp6GROCandidate, tcp6GROCandidate,
}, },
{
"tcp6 no support",
tcp6,
tcpGRODisabled,
notGROCandidate,
},
{ {
"udp4", "udp4",
udp4, udp4,
true, 0,
udp4GROCandidate, udp4GROCandidate,
}, },
{ {
"udp4 no support", "udp4 no support",
udp4, udp4,
false, udpGRODisabled,
notGROCandidate, notGROCandidate,
}, },
{ {
"udp6", "udp6",
udp6, udp6,
true, 0,
udp6GROCandidate, udp6GROCandidate,
}, },
{ {
"udp6 no support", "udp6 no support",
udp6, udp6,
false, udpGRODisabled,
notGROCandidate, notGROCandidate,
}, },
{ {
"udp4 too short", "udp4 too short",
udp4TooShort, udp4TooShort,
true, 0,
notGROCandidate, notGROCandidate,
}, },
{ {
"udp6 too short", "udp6 too short",
udp6TooShort, udp6TooShort,
true, 0,
notGROCandidate, notGROCandidate,
}, },
{ {
"tcp4 too short", "tcp4 too short",
tcp4TooShort, tcp4TooShort,
true, 0,
notGROCandidate, notGROCandidate,
}, },
{ {
"tcp6 too short", "tcp6 too short",
tcp6TooShort, tcp6TooShort,
true, 0,
notGROCandidate, notGROCandidate,
}, },
{ {
"invalid IP version", "invalid IP version",
[]byte{0x00}, []byte{0x00},
true, 0,
notGROCandidate, notGROCandidate,
}, },
{ {
"invalid IP header len", "invalid IP header len",
ip4InvalidHeaderLen, ip4InvalidHeaderLen,
true, 0,
notGROCandidate, notGROCandidate,
}, },
{ {
"ip4 invalid protocol", "ip4 invalid protocol",
ip4InvalidProtocol, ip4InvalidProtocol,
true, 0,
notGROCandidate, notGROCandidate,
}, },
{ {
"ip6 invalid protocol", "ip6 invalid protocol",
ip6InvalidProtocol, ip6InvalidProtocol,
true, 0,
notGROCandidate, notGROCandidate,
}, },
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
if got := packetIsGROCandidate(tt.b, tt.canUDPGRO); got != tt.want { if got := packetIsGROCandidate(tt.b, tt.gro); got != tt.want {
t.Errorf("packetIsGROCandidate() = %v, want %v", got, tt.want) t.Errorf("packetIsGROCandidate() = %v, want %v", got, tt.want)
} }
}) })

23
tun/tun.go
View file

@ -52,10 +52,25 @@ type Device interface {
BatchSize() int BatchSize() int
} }
type LinuxDevice interface { // GRODevice is a Device extended with methods for disabling GRO. Certain OS
// versions may have offload bugs. Where these bugs negatively impact throughput
// or break connectivity entirely we can use these methods to disable the
// related offload.
//
// Linux has the following known, GRO bugs.
//
// torvalds/linux@e269d79c7d35aa3808b1f3c1737d63dab504ddc8 broke virtio_net
// TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then
// resolved later in
// torvalds/linux@89add40066f9ed9abe5f7f886fe5789ff7e0c50e. The offending
// commit was pulled into various LTS releases.
//
// UDP GRO writes end up blackholing/dropping packets destined for a
// vxlan/geneve interface on kernel versions prior to 6.8.5.
type GRODevice interface {
Device Device
// DisableUDPGRO disables UDP GRO if it is enabled. Certain device drivers // DisableUDPGRO disables UDP GRO if it is enabled.
// (e.g. vxlan, geneve) do not properly handle coalesced UDP packets later
// in the stack, resulting in packet loss.
DisableUDPGRO() DisableUDPGRO()
// DisableTCPGRO disables TCP GRO if it is enabled.
DisableTCPGRO()
} }

45
tun/tun_linux.go
View file

@ -38,7 +38,6 @@ type NativeTun struct {
statusListenersShutdown chan struct{} statusListenersShutdown chan struct{}
batchSize int batchSize int
vnetHdr bool vnetHdr bool
udpGSO bool
closeOnce sync.Once closeOnce sync.Once
@ -53,7 +52,30 @@ type NativeTun struct {
toWrite []int toWrite []int
tcpGROTable *tcpGROTable tcpGROTable *tcpGROTable
udpGROTable *udpGROTable udpGROTable *udpGROTable
udpGRO bool gro groDisablementFlags
}
type groDisablementFlags int
const (
tcpGRODisabled groDisablementFlags = 1 << iota
udpGRODisabled
)
func (g *groDisablementFlags) disableTCPGRO() {
*g |= tcpGRODisabled
}
func (g *groDisablementFlags) canTCPGRO() bool {
return (*g)&tcpGRODisabled == 0
}
func (g *groDisablementFlags) disableUDPGRO() {
*g |= udpGRODisabled
}
func (g *groDisablementFlags) canUDPGRO() bool {
return (*g)&udpGRODisabled == 0
} }
func (tun *NativeTun) File() *os.File { func (tun *NativeTun) File() *os.File {
@ -346,7 +368,7 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) {
) )
tun.toWrite = tun.toWrite[:0] tun.toWrite = tun.toWrite[:0]
if tun.vnetHdr { if tun.vnetHdr {
err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.udpGRO, &tun.toWrite) err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.gro, &tun.toWrite)
if err != nil { if err != nil {
return 0, err return 0, err
} }
@ -462,9 +484,19 @@ func (tun *NativeTun) BatchSize() int {
return tun.batchSize return tun.batchSize
} }
// DisableUDPGRO disables UDP GRO if it is enabled. See the GRODevice interface
// for cases where it should be called.
func (tun *NativeTun) DisableUDPGRO() { func (tun *NativeTun) DisableUDPGRO() {
tun.writeOpMu.Lock() tun.writeOpMu.Lock()
tun.udpGRO = false tun.gro.disableUDPGRO()
tun.writeOpMu.Unlock()
}
// DisableTCPGRO disables TCP GRO if it is enabled. See the GRODevice interface
// for cases where it should be called.
func (tun *NativeTun) DisableTCPGRO() {
tun.writeOpMu.Lock()
tun.gro.disableTCPGRO()
tun.writeOpMu.Unlock() tun.writeOpMu.Unlock()
} }
@ -503,8 +535,9 @@ func (tun *NativeTun) initFromFlags(name string) error {
tun.batchSize = conn.IdealBatchSize tun.batchSize = conn.IdealBatchSize
// tunUDPOffloads were added in Linux v6.2. We do not return an // tunUDPOffloads were added in Linux v6.2. We do not return an
// error if they are unsupported at runtime. // error if they are unsupported at runtime.
tun.udpGSO = unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) == nil if unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) != nil {
tun.udpGRO = tun.udpGSO tun.gro.disableUDPGRO()
}
} else { } else {
tun.batchSize = 1 tun.batchSize = 1
} }