eburet-oss/wireguard-go-awg2-lx
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tun: add method for disabling TCP GRO on Linux

Browse source 
torvalds/linux@e269d79c7d broke virtio_net
TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then
resolved later in
torvalds/linux@89add40066. The offending
commit was pulled into various LTS releases.

Updates tailscale/tailscale#13041

Signed-off-by: Jordan Whited <jordan@tailscale.com>
This commit is contained in:
Jordan Whited 2024-09-04 12:17:44 -07:00 • committed by Jordan Whited
parent 71393c576b
commit 799c1978fa
4 changed files with 114 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

18
tun/offload_linux.go
View file

@ -763,7 +763,7 @@ const (
udp6GROCandidate
)
func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType {
func packetIsGROCandidate(b []byte, gro groDisablementFlags) groCandidateType {
if len(b) < 28 {
return notGROCandidate
}
@ -772,17 +772,17 @@ func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType {
// IPv4 packets w/IP options do not coalesce
return notGROCandidate
}
if b[9] == unix.IPPROTO_TCP && len(b) >= 40 {
if b[9] == unix.IPPROTO_TCP && len(b) >= 40 && gro.canTCPGRO() {
return tcp4GROCandidate
}
if b[9] == unix.IPPROTO_UDP && canUDPGRO {
if b[9] == unix.IPPROTO_UDP && gro.canUDPGRO() {
return udp4GROCandidate
}
} else if b[0]>>4 == 6 {
if b[6] == unix.IPPROTO_TCP && len(b) >= 60 {
if b[6] == unix.IPPROTO_TCP && len(b) >= 60 && gro.canTCPGRO() {
return tcp6GROCandidate
}
if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && canUDPGRO {
if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && gro.canUDPGRO() {
return udp6GROCandidate
}
}
@ -875,15 +875,15 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool)
// handleGRO evaluates bufs for GRO, and writes the indices of the resulting
// packets into toWrite. toWrite, tcpTable, and udpTable should initially be
// empty (but non-nil), and are passed in to save allocs as the caller may reset
// and recycle them across vectors of packets. canUDPGRO indicates if UDP GRO is
// supported.
func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, canUDPGRO bool, toWrite *[]int) error {
// and recycle them across vectors of packets. gro indicates if TCP and UDP GRO
// are supported/enabled.
func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, gro groDisablementFlags, toWrite *[]int) error {
for i := range bufs {
if offset < virtioNetHdrLen || offset > len(bufs[i])-1 {
return errors.New("invalid offset")
}
var result groResult
switch packetIsGROCandidate(bufs[i][offset:], canUDPGRO) {
switch packetIsGROCandidate(bufs[i][offset:], gro) {
case tcp4GROCandidate:
result = tcpGRO(bufs, offset, i, tcpTable, false)
case tcp6GROCandidate:

82
tun/offload_linux_test.go
View file

@ -286,11 +286,11 @@ func Fuzz_handleGRO(f *testing.F) {
pkt9 := udp6Packet(ip6PortA, ip6PortB, 100)
pkt10 := udp6Packet(ip6PortA, ip6PortB, 100)
pkt11 := udp6Packet(ip6PortA, ip6PortC, 100)
f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, true, offset)
f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, canUDPGRO bool, offset int) {
f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, 0, offset)
f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, gro int, offset int) {
pkts := [][]byte{pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11}
toWrite := make([]int, 0, len(pkts))
handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), canUDPGRO, &toWrite)
handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), groDisablementFlags(gro), &toWrite)
if len(toWrite) > len(pkts) {
t.Errorf("len(toWrite): %d > len(pkts): %d", len(toWrite), len(pkts))
}
@ -311,7 +311,7 @@ func Test_handleGRO(t *testing.T) {
tests := []struct {
name string
pktsIn [][]byte
canUDPGRO bool
gro groDisablementFlags
wantToWrite []int
wantLens []int
wantErr bool
@ -331,7 +331,7 @@ func Test_handleGRO(t *testing.T) {
udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1
udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1
},
true,
0,
[]int{0, 1, 2, 4, 5, 7, 9},
[]int{240, 228, 128, 140, 260, 160, 248},
false,
@ -351,7 +351,7 @@ func Test_handleGRO(t *testing.T) {
udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1
udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1
},
false,
udpGRODisabled,
[]int{0, 1, 2, 4, 5, 7, 8, 9, 10},
[]int{240, 128, 128, 140, 260, 160, 128, 148, 148},
false,
@ -368,7 +368,7 @@ func Test_handleGRO(t *testing.T) {
tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 201), // v6 flow 1
tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 301), // v6 flow 1
},
true,
0,
[]int{0, 2, 4, 6},
[]int{240, 240, 260, 260},
false,
@ -383,7 +383,7 @@ func Test_handleGRO(t *testing.T) {
udp4Packet(ip4PortA, ip4PortB, 100),
udp4Packet(ip4PortA, ip4PortB, 100),
},
true,
0,
[]int{0, 1, 3, 4},
[]int{140, 240, 128, 228},
false,
@ -395,7 +395,7 @@ func Test_handleGRO(t *testing.T) {
tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // v4 flow 1 seq 1 len 100
tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 201), // v4 flow 1 seq 201 len 100
},
true,
0,
[]int{0},
[]int{340},
false,
@ -412,7 +412,7 @@ func Test_handleGRO(t *testing.T) {
fields.TTL++
}),
},
true,
0,
[]int{0, 1, 2, 3},
[]int{140, 140, 128, 128},
false,
@ -429,7 +429,7 @@ func Test_handleGRO(t *testing.T) {
fields.TOS++
}),
},
true,
0,
[]int{0, 1, 2, 3},
[]int{140, 140, 128, 128},
false,
@ -446,7 +446,7 @@ func Test_handleGRO(t *testing.T) {
fields.Flags = 1
}),
},
true,
0,
[]int{0, 1, 2, 3},
[]int{140, 140, 128, 128},
false,
@ -463,7 +463,7 @@ func Test_handleGRO(t *testing.T) {
fields.Flags = 2
}),
},
true,
0,
[]int{0, 1, 2, 3},
[]int{140, 140, 128, 128},
false,
@ -480,7 +480,7 @@ func Test_handleGRO(t *testing.T) {
fields.HopLimit++
}),
},
true,
0,
[]int{0, 1, 2, 3},
[]int{160, 160, 148, 148},
false,
@ -497,7 +497,7 @@ func Test_handleGRO(t *testing.T) {
fields.TrafficClass++
}),
},
true,
0,
[]int{0, 1, 2, 3},
[]int{160, 160, 148, 148},
false,
@ -507,7 +507,7 @@ func Test_handleGRO(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
toWrite := make([]int, 0, len(tt.pktsIn))
err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), tt.canUDPGRO, &toWrite)
err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), tt.gro, &toWrite)
if err != nil {
if tt.wantErr {
return
@ -552,99 +552,111 @@ func Test_packetIsGROCandidate(t *testing.T) {
udp6TooShort := udp6[:47]
tests := []struct {
name string
b []byte
canUDPGRO bool
want groCandidateType
name string
b []byte
gro groDisablementFlags
want groCandidateType
}{
{
"tcp4",
tcp4,
true,
0,
tcp4GROCandidate,
},
{
"tcp4 no support",
tcp4,
tcpGRODisabled,
notGROCandidate,
},
{
"tcp6",
tcp6,
true,
0,
tcp6GROCandidate,
},
{
"tcp6 no support",
tcp6,
tcpGRODisabled,
notGROCandidate,
},
{
"udp4",
udp4,
true,
0,
udp4GROCandidate,
},
{
"udp4 no support",
udp4,
false,
udpGRODisabled,
notGROCandidate,
},
{
"udp6",
udp6,
true,
0,
udp6GROCandidate,
},
{
"udp6 no support",
udp6,
false,
udpGRODisabled,
notGROCandidate,
},
{
"udp4 too short",
udp4TooShort,
true,
0,
notGROCandidate,
},
{
"udp6 too short",
udp6TooShort,
true,
0,
notGROCandidate,
},
{
"tcp4 too short",
tcp4TooShort,
true,
0,
notGROCandidate,
},
{
"tcp6 too short",
tcp6TooShort,
true,
0,
notGROCandidate,
},
{
"invalid IP version",
[]byte{0x00},
true,
0,
notGROCandidate,
},
{
"invalid IP header len",
ip4InvalidHeaderLen,
true,
0,
notGROCandidate,
},
{
"ip4 invalid protocol",
ip4InvalidProtocol,
true,
0,
notGROCandidate,
},
{
"ip6 invalid protocol",
ip6InvalidProtocol,
true,
0,
notGROCandidate,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := packetIsGROCandidate(tt.b, tt.canUDPGRO); got != tt.want {
if got := packetIsGROCandidate(tt.b, tt.gro); got != tt.want {
t.Errorf("packetIsGROCandidate() = %v, want %v", got, tt.want)
}
})

23
tun/tun.go
View file

@ -52,10 +52,25 @@ type Device interface {
BatchSize() int
}
type LinuxDevice interface {
// GRODevice is a Device extended with methods for disabling GRO. Certain OS
// versions may have offload bugs. Where these bugs negatively impact throughput
// or break connectivity entirely we can use these methods to disable the
// related offload.
//
// Linux has the following known, GRO bugs.
//
// torvalds/linux@e269d79c7d35aa3808b1f3c1737d63dab504ddc8 broke virtio_net
// TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then
// resolved later in
// torvalds/linux@89add40066f9ed9abe5f7f886fe5789ff7e0c50e. The offending
// commit was pulled into various LTS releases.
//
// UDP GRO writes end up blackholing/dropping packets destined for a
// vxlan/geneve interface on kernel versions prior to 6.8.5.
type GRODevice interface {
Device
// DisableUDPGRO disables UDP GRO if it is enabled. Certain device drivers
// (e.g. vxlan, geneve) do not properly handle coalesced UDP packets later
// in the stack, resulting in packet loss.
// DisableUDPGRO disables UDP GRO if it is enabled.
DisableUDPGRO()
// DisableTCPGRO disables TCP GRO if it is enabled.
DisableTCPGRO()
}

45
tun/tun_linux.go
View file

@ -38,7 +38,6 @@ type NativeTun struct {
statusListenersShutdown chan struct{}
batchSize int
vnetHdr bool
udpGSO bool
closeOnce sync.Once
@ -53,7 +52,30 @@ type NativeTun struct {
toWrite []int
tcpGROTable *tcpGROTable
udpGROTable *udpGROTable
udpGRO bool
gro groDisablementFlags
}
type groDisablementFlags int
const (
tcpGRODisabled groDisablementFlags = 1 << iota
udpGRODisabled
)
func (g *groDisablementFlags) disableTCPGRO() {
*g |= tcpGRODisabled
}
func (g *groDisablementFlags) canTCPGRO() bool {
return (*g)&tcpGRODisabled == 0
}
func (g *groDisablementFlags) disableUDPGRO() {
*g |= udpGRODisabled
}
func (g *groDisablementFlags) canUDPGRO() bool {
return (*g)&udpGRODisabled == 0
}
func (tun *NativeTun) File() *os.File {
@ -346,7 +368,7 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) {
)
tun.toWrite = tun.toWrite[:0]
if tun.vnetHdr {
err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.udpGRO, &tun.toWrite)
err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.gro, &tun.toWrite)
if err != nil {
return 0, err
}
@ -462,9 +484,19 @@ func (tun *NativeTun) BatchSize() int {
return tun.batchSize
}
// DisableUDPGRO disables UDP GRO if it is enabled. See the GRODevice interface
// for cases where it should be called.
func (tun *NativeTun) DisableUDPGRO() {
tun.writeOpMu.Lock()
tun.udpGRO = false
tun.gro.disableUDPGRO()
tun.writeOpMu.Unlock()
}
// DisableTCPGRO disables TCP GRO if it is enabled. See the GRODevice interface
// for cases where it should be called.
func (tun *NativeTun) DisableTCPGRO() {
tun.writeOpMu.Lock()
tun.gro.disableTCPGRO()
tun.writeOpMu.Unlock()
}
@ -503,8 +535,9 @@ func (tun *NativeTun) initFromFlags(name string) error {
tun.batchSize = conn.IdealBatchSize
// tunUDPOffloads were added in Linux v6.2. We do not return an
// error if they are unsupported at runtime.
tun.udpGSO = unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) == nil
tun.udpGRO = tun.udpGSO
if unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) != nil {
tun.gro.disableUDPGRO()
}
} else {
tun.batchSize = 1
}