diff --git a/tun/checksum.go b/tun/checksum.go index b489c56..ac16569 100644 --- a/tun/checksum.go +++ b/tun/checksum.go @@ -92,6 +92,11 @@ func checksum(b []byte, initial uint64) uint16 { return uint16(ac) } +// Checksum computes an IP checksum starting with the provided initial value. +func Checksum(data []byte, initial uint16) uint16 { + return checksum(data, uint64(initial)) +} + func pseudoHeaderChecksumNoFold(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint64 { sum := checksumNoFold(srcAddr, 0) sum = checksumNoFold(dstAddr, sum) @@ -100,3 +105,9 @@ func pseudoHeaderChecksumNoFold(protocol uint8, srcAddr, dstAddr []byte, totalLe binary.BigEndian.PutUint16(tmp, totalLen) return checksumNoFold(tmp, sum) } + +// PseudoHeaderChecksum computes an IP pseudo-header checksum. srcAddr and +// dstAddr must be 4 or 16 bytes in length. +func PseudoHeaderChecksum(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { + return checksum([]byte{}, pseudoHeaderChecksumNoFold(protocol, srcAddr, dstAddr, totalLen)) +} diff --git a/tun/offload.go b/tun/offload.go new file mode 100644 index 0000000..4e84db4 --- /dev/null +++ b/tun/offload.go @@ -0,0 +1,200 @@ +package tun + +import ( + "encoding/binary" + "fmt" +) + +// GSOType represents the type of segmentation offload. +type GSOType int + +const ( + GSONone GSOType = iota + GSOTCPv4 + GSOTCPv6 + GSOUDPL4 +) + +func (g GSOType) String() string { + switch g { + case GSONone: + return "GSONone" + case GSOTCPv4: + return "GSOTCPv4" + case GSOTCPv6: + return "GSOTCPv6" + case GSOUDPL4: + return "GSOUDPL4" + default: + return "unknown" + } +} + +// GSOOptions is loosely modeled after struct virtio_net_hdr from the VIRTIO +// specification. It is a common representation of GSO metadata that can be +// applied to support packet GSO across tun.Device implementations. +type GSOOptions struct { + // GSOType represents the type of segmentation offload. + GSOType GSOType + // HdrLen is the sum of the layer 3 and 4 header lengths. This field may be + // zero when GSOType == GSONone. + HdrLen uint16 + // CsumStart is the head byte index of the packet data to be checksummed, + // i.e. the start of the TCP or UDP header. + CsumStart uint16 + // CsumOffset is the offset from CsumStart where the 2-byte checksum value + // should be placed. + CsumOffset uint16 + // GSOSize is the size of each segment exclusive of HdrLen. The tail segment + // may be smaller than this value. + GSOSize uint16 + // NeedsCsum may be set where GSOType == GSONone. When set, the checksum + // at CsumStart + CsumOffset must be a partial checksum, i.e. the + // pseudo-header sum. + NeedsCsum bool +} + +const ( + gsoIPv4SrcAddrOffset = 12 + gsoIPv6SrcAddrOffset = 8 + gsoTCPFlagsOffset = 13 + gsoIPProtoTCP = 6 + gsoIPProtoUDP = 17 +) + +const ( + gsoTCPFlagFIN uint8 = 0x01 + gsoTCPFlagPSH uint8 = 0x08 +) + +// GSOSplit splits packets from in into outBufs[][outOffset:], writing +// the size of each element into sizes. It returns the number of buffers +// populated, and/or an error. Callers may pass an in slice that overlaps with +// the first element of outBufs, i.e. &in[0] may be equal to +// &outBufs[0][outOffset]. GSONone is a valid options.GSOType regardless of the +// value of options.NeedsCsum. Length of each outBufs element must be greater +// than or equal to the length of in, otherwise output may be silently truncated. +func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outOffset int) (int, error) { + cSumAt := int(options.CsumStart) + int(options.CsumOffset) + if cSumAt+1 >= len(in) { + return 0, fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(in)) + } + + if len(in) < int(options.HdrLen) { + return 0, fmt.Errorf("length of packet (%d) < GSO HdrLen (%d)", len(in), options.HdrLen) + } + + payloadLen := len(in) - int(options.HdrLen) + if options.GSOType == GSONone || payloadLen < int(options.GSOSize) { + if len(in) > len(outBufs[0][outOffset:]) { + return 0, fmt.Errorf("length of packet (%d) exceeds output element length (%d)", len(in), len(outBufs[0][outOffset:])) + } + if options.NeedsCsum { + initial := binary.BigEndian.Uint16(in[cSumAt:]) + in[cSumAt], in[cSumAt+1] = 0, 0 + binary.BigEndian.PutUint16(in[cSumAt:], ^Checksum(in[options.CsumStart:], initial)) + } + sizes[0] = copy(outBufs[0][outOffset:], in) + return 1, nil + } + + if options.HdrLen < options.CsumStart { + return 0, fmt.Errorf("GSO HdrLen (%d) < GSO CsumStart (%d)", options.HdrLen, options.CsumStart) + } + + ipVersion := in[0] >> 4 + switch ipVersion { + case 4: + if options.GSOType != GSOTCPv4 && options.GSOType != GSOUDPL4 { + return 0, fmt.Errorf("ip header version: %d, GSO type: %s", ipVersion, options.GSOType) + } + if len(in) < 20 { + return 0, fmt.Errorf("length of packet (%d) < minimum ipv4 header size (%d)", len(in), 20) + } + case 6: + if options.GSOType != GSOTCPv6 && options.GSOType != GSOUDPL4 { + return 0, fmt.Errorf("ip header version: %d, GSO type: %s", ipVersion, options.GSOType) + } + if len(in) < 40 { + return 0, fmt.Errorf("length of packet (%d) < minimum ipv6 header size (%d)", len(in), 40) + } + default: + return 0, fmt.Errorf("invalid ip header version: %d", ipVersion) + } + + iphLen := int(options.CsumStart) + srcAddrOffset := gsoIPv6SrcAddrOffset + addrLen := 16 + if ipVersion == 4 { + srcAddrOffset = gsoIPv4SrcAddrOffset + addrLen = 4 + } + transportCsumAt := int(options.CsumStart + options.CsumOffset) + var firstTCPSeqNum uint32 + var protocol uint8 + if options.GSOType == GSOTCPv4 || options.GSOType == GSOTCPv6 { + protocol = gsoIPProtoTCP + if len(in) < int(options.CsumStart)+20 { + return 0, fmt.Errorf("length of packet (%d) < GSO CsumStart (%d) + minimum TCP header size (%d)", + len(in), options.CsumStart, 20) + } + firstTCPSeqNum = binary.BigEndian.Uint32(in[options.CsumStart+4:]) + } else { + protocol = gsoIPProtoUDP + } + nextSegmentDataAt := int(options.HdrLen) + i := 0 + for ; nextSegmentDataAt < len(in); i++ { + if i == len(outBufs) { + return i - 1, ErrTooManySegments + } + nextSegmentEnd := nextSegmentDataAt + int(options.GSOSize) + if nextSegmentEnd > len(in) { + nextSegmentEnd = len(in) + } + segmentDataLen := nextSegmentEnd - nextSegmentDataAt + totalLen := int(options.HdrLen) + segmentDataLen + sizes[i] = totalLen + out := outBufs[i][outOffset:] + + copy(out, in[:iphLen]) + if ipVersion == 4 { + if i > 0 { + id := binary.BigEndian.Uint16(out[4:]) + id += uint16(i) + binary.BigEndian.PutUint16(out[4:], id) + } + out[10], out[11] = 0, 0 + binary.BigEndian.PutUint16(out[2:], uint16(totalLen)) + ipv4CSum := ^Checksum(out[:iphLen], 0) + binary.BigEndian.PutUint16(out[10:], ipv4CSum) + } else { + binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen)) + } + + copy(out[options.CsumStart:options.HdrLen], in[options.CsumStart:options.HdrLen]) + + if protocol == gsoIPProtoTCP { + tcpSeq := firstTCPSeqNum + uint32(options.GSOSize*uint16(i)) + binary.BigEndian.PutUint32(out[options.CsumStart+4:], tcpSeq) + if nextSegmentEnd != len(in) { + clearFlags := gsoTCPFlagFIN | gsoTCPFlagPSH + out[options.CsumStart+gsoTCPFlagsOffset] &^= clearFlags + } + } else { + binary.BigEndian.PutUint16(out[options.CsumStart+4:], uint16(segmentDataLen)+(options.HdrLen-options.CsumStart)) + } + + copy(out[options.HdrLen:], in[nextSegmentDataAt:nextSegmentEnd]) + + out[transportCsumAt], out[transportCsumAt+1] = 0, 0 + transportHeaderLen := int(options.HdrLen - options.CsumStart) + lenForPseudo := uint16(transportHeaderLen + segmentDataLen) + transportCSum := PseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) + transportCSum = ^Checksum(out[options.CsumStart:totalLen], transportCSum) + binary.BigEndian.PutUint16(out[options.CsumStart+options.CsumOffset:], transportCSum) + + nextSegmentDataAt += int(options.GSOSize) + } + return i, nil +} diff --git a/tun/offload_linux.go b/tun/offload_linux.go index d360d40..6825bbc 100644 --- a/tun/offload_linux.go +++ b/tun/offload_linux.go @@ -748,7 +748,7 @@ const ( udp6GROCandidate ) -func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType { +func packetIsGROCandidate(b []byte, gro groDisablementFlags) groCandidateType { if len(b) < 28 { return notGROCandidate } @@ -757,17 +757,17 @@ func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType { // IPv4 packets w/IP options do not coalesce return notGROCandidate } - if b[9] == unix.IPPROTO_TCP && len(b) >= 40 { + if b[9] == unix.IPPROTO_TCP && len(b) >= 40 && gro.canTCPGRO() { return tcp4GROCandidate } - if b[9] == unix.IPPROTO_UDP && canUDPGRO { + if b[9] == unix.IPPROTO_UDP && gro.canUDPGRO() { return udp4GROCandidate } } else if b[0]>>4 == 6 { - if b[6] == unix.IPPROTO_TCP && len(b) >= 60 { + if b[6] == unix.IPPROTO_TCP && len(b) >= 60 && gro.canTCPGRO() { return tcp6GROCandidate } - if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && canUDPGRO { + if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && gro.canUDPGRO() { return udp6GROCandidate } } @@ -860,15 +860,15 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool) // handleGRO evaluates bufs for GRO, and writes the indices of the resulting // packets into toWrite. toWrite, tcpTable, and udpTable should initially be // empty (but non-nil), and are passed in to save allocs as the caller may reset -// and recycle them across vectors of packets. canUDPGRO indicates if UDP GRO is -// supported. -func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, canUDPGRO bool, toWrite *[]int) error { +// and recycle them across vectors of packets. gro indicates if TCP and UDP GRO +// are supported/enabled. +func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, gro groDisablementFlags, toWrite *[]int) error { for i := range bufs { if offset < virtioNetHdrLen || offset > len(bufs[i])-1 { return errors.New("invalid offset") } var result groResult - switch packetIsGROCandidate(bufs[i][offset:], canUDPGRO) { + switch packetIsGROCandidate(bufs[i][offset:], gro) { case tcp4GROCandidate: result = tcpGRO(bufs, offset, i, tcpTable, false) case tcp6GROCandidate: diff --git a/tun/tun.go b/tun/tun.go index 336d642..6fb5c56 100644 --- a/tun/tun.go +++ b/tun/tun.go @@ -51,3 +51,17 @@ type Device interface { // lifetime of a Device. BatchSize() int } + +// GRODevice is a Device extended with methods for disabling GRO. Certain OS +// versions may have offload bugs. Where these bugs negatively impact throughput +// or break connectivity entirely we can use these methods to disable the +// related offload. +type GRODevice interface { + Device + + // DisableUDPGRO disables UDP GRO if it is enabled. + DisableUDPGRO() + + // DisableTCPGRO disables TCP GRO if it is enabled. + DisableTCPGRO() +} diff --git a/tun/tun_linux.go b/tun/tun_linux.go index b10ec3c..4b7866d 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -38,7 +38,6 @@ type NativeTun struct { statusListenersShutdown chan struct{} batchSize int vnetHdr bool - udpGSO bool closeOnce sync.Once @@ -49,10 +48,34 @@ type NativeTun struct { readOpMu sync.Mutex // readOpMu guards readBuff readBuff [virtioNetHdrLen + 65535]byte // if vnetHdr every read() is prefixed by virtioNetHdr - writeOpMu sync.Mutex // writeOpMu guards toWrite, tcpGROTable + writeOpMu sync.Mutex // writeOpMu guards toWrite, tcpGROTable, udpGROTable, gro toWrite []int tcpGROTable *tcpGROTable udpGROTable *udpGROTable + gro groDisablementFlags +} + +type groDisablementFlags int + +const ( + tcpGRODisabled groDisablementFlags = 1 << iota + udpGRODisabled +) + +func (g *groDisablementFlags) disableTCPGRO() { + *g |= tcpGRODisabled +} + +func (g *groDisablementFlags) canTCPGRO() bool { + return (*g)&tcpGRODisabled == 0 +} + +func (g *groDisablementFlags) disableUDPGRO() { + *g |= udpGRODisabled +} + +func (g *groDisablementFlags) canUDPGRO() bool { + return (*g)&udpGRODisabled == 0 } func (tun *NativeTun) File() *os.File { @@ -345,7 +368,7 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) { ) tun.toWrite = tun.toWrite[:0] if tun.vnetHdr { - err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.udpGSO, &tun.toWrite) + err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.gro, &tun.toWrite) if err != nil { return 0, err } @@ -502,6 +525,20 @@ func (tun *NativeTun) BatchSize() int { return tun.batchSize } +// DisableUDPGRO disables UDP GRO if it is enabled. +func (tun *NativeTun) DisableUDPGRO() { + tun.writeOpMu.Lock() + tun.gro.disableUDPGRO() + tun.writeOpMu.Unlock() +} + +// DisableTCPGRO disables TCP GRO if it is enabled. +func (tun *NativeTun) DisableTCPGRO() { + tun.writeOpMu.Lock() + tun.gro.disableTCPGRO() + tun.writeOpMu.Unlock() +} + const ( // TODO: support TSO with ECN bits tunTCPOffloads = unix.TUN_F_CSUM | unix.TUN_F_TSO4 | unix.TUN_F_TSO6 @@ -535,7 +572,9 @@ func (tun *NativeTun) initFromFlags(name string) error { tun.batchSize = conn.IdealBatchSize // tunUDPOffloads were added in Linux v6.2. We do not return an // error if they are unsupported at runtime. - tun.udpGSO = unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) == nil + if unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) != nil { + tun.gro.disableUDPGRO() + } } else { tun.batchSize = 1 }