Apply Tailscale TUN offload APIs
This commit is contained in:
世界
parent
414291f6d6
commit
824e7573c0
5 changed files with 277 additions and 13 deletions
|
|
@ -92,6 +92,11 @@ func checksum(b []byte, initial uint64) uint16 {
|
|||
return uint16(ac)
|
||||
}
|
||||
|
||||
// Checksum computes an IP checksum starting with the provided initial value.
|
||||
func Checksum(data []byte, initial uint16) uint16 {
|
||||
return checksum(data, uint64(initial))
|
||||
}
|
||||
|
||||
func pseudoHeaderChecksumNoFold(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint64 {
|
||||
sum := checksumNoFold(srcAddr, 0)
|
||||
sum = checksumNoFold(dstAddr, sum)
|
||||
|
|
@ -100,3 +105,9 @@ func pseudoHeaderChecksumNoFold(protocol uint8, srcAddr, dstAddr []byte, totalLe
|
|||
binary.BigEndian.PutUint16(tmp, totalLen)
|
||||
return checksumNoFold(tmp, sum)
|
||||
}
|
||||
|
||||
// PseudoHeaderChecksum computes an IP pseudo-header checksum. srcAddr and
|
||||
// dstAddr must be 4 or 16 bytes in length.
|
||||
func PseudoHeaderChecksum(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 {
|
||||
return checksum([]byte{}, pseudoHeaderChecksumNoFold(protocol, srcAddr, dstAddr, totalLen))
|
||||
}
|
||||
|
|
|
|||
200
tun/offload.go
Normal file
200
tun/offload.go
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
package tun
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// GSOType represents the type of segmentation offload.
|
||||
type GSOType int
|
||||
|
||||
const (
|
||||
GSONone GSOType = iota
|
||||
GSOTCPv4
|
||||
GSOTCPv6
|
||||
GSOUDPL4
|
||||
)
|
||||
|
||||
func (g GSOType) String() string {
|
||||
switch g {
|
||||
case GSONone:
|
||||
return "GSONone"
|
||||
case GSOTCPv4:
|
||||
return "GSOTCPv4"
|
||||
case GSOTCPv6:
|
||||
return "GSOTCPv6"
|
||||
case GSOUDPL4:
|
||||
return "GSOUDPL4"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// GSOOptions is loosely modeled after struct virtio_net_hdr from the VIRTIO
|
||||
// specification. It is a common representation of GSO metadata that can be
|
||||
// applied to support packet GSO across tun.Device implementations.
|
||||
type GSOOptions struct {
|
||||
// GSOType represents the type of segmentation offload.
|
||||
GSOType GSOType
|
||||
// HdrLen is the sum of the layer 3 and 4 header lengths. This field may be
|
||||
// zero when GSOType == GSONone.
|
||||
HdrLen uint16
|
||||
// CsumStart is the head byte index of the packet data to be checksummed,
|
||||
// i.e. the start of the TCP or UDP header.
|
||||
CsumStart uint16
|
||||
// CsumOffset is the offset from CsumStart where the 2-byte checksum value
|
||||
// should be placed.
|
||||
CsumOffset uint16
|
||||
// GSOSize is the size of each segment exclusive of HdrLen. The tail segment
|
||||
// may be smaller than this value.
|
||||
GSOSize uint16
|
||||
// NeedsCsum may be set where GSOType == GSONone. When set, the checksum
|
||||
// at CsumStart + CsumOffset must be a partial checksum, i.e. the
|
||||
// pseudo-header sum.
|
||||
NeedsCsum bool
|
||||
}
|
||||
|
||||
const (
|
||||
gsoIPv4SrcAddrOffset = 12
|
||||
gsoIPv6SrcAddrOffset = 8
|
||||
gsoTCPFlagsOffset = 13
|
||||
gsoIPProtoTCP = 6
|
||||
gsoIPProtoUDP = 17
|
||||
)
|
||||
|
||||
const (
|
||||
gsoTCPFlagFIN uint8 = 0x01
|
||||
gsoTCPFlagPSH uint8 = 0x08
|
||||
)
|
||||
|
||||
// GSOSplit splits packets from in into outBufs[<index>][outOffset:], writing
|
||||
// the size of each element into sizes. It returns the number of buffers
|
||||
// populated, and/or an error. Callers may pass an in slice that overlaps with
|
||||
// the first element of outBufs, i.e. &in[0] may be equal to
|
||||
// &outBufs[0][outOffset]. GSONone is a valid options.GSOType regardless of the
|
||||
// value of options.NeedsCsum. Length of each outBufs element must be greater
|
||||
// than or equal to the length of in, otherwise output may be silently truncated.
|
||||
func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outOffset int) (int, error) {
|
||||
cSumAt := int(options.CsumStart) + int(options.CsumOffset)
|
||||
if cSumAt+1 >= len(in) {
|
||||
return 0, fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(in))
|
||||
}
|
||||
|
||||
if len(in) < int(options.HdrLen) {
|
||||
return 0, fmt.Errorf("length of packet (%d) < GSO HdrLen (%d)", len(in), options.HdrLen)
|
||||
}
|
||||
|
||||
payloadLen := len(in) - int(options.HdrLen)
|
||||
if options.GSOType == GSONone || payloadLen < int(options.GSOSize) {
|
||||
if len(in) > len(outBufs[0][outOffset:]) {
|
||||
return 0, fmt.Errorf("length of packet (%d) exceeds output element length (%d)", len(in), len(outBufs[0][outOffset:]))
|
||||
}
|
||||
if options.NeedsCsum {
|
||||
initial := binary.BigEndian.Uint16(in[cSumAt:])
|
||||
in[cSumAt], in[cSumAt+1] = 0, 0
|
||||
binary.BigEndian.PutUint16(in[cSumAt:], ^Checksum(in[options.CsumStart:], initial))
|
||||
}
|
||||
sizes[0] = copy(outBufs[0][outOffset:], in)
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
if options.HdrLen < options.CsumStart {
|
||||
return 0, fmt.Errorf("GSO HdrLen (%d) < GSO CsumStart (%d)", options.HdrLen, options.CsumStart)
|
||||
}
|
||||
|
||||
ipVersion := in[0] >> 4
|
||||
switch ipVersion {
|
||||
case 4:
|
||||
if options.GSOType != GSOTCPv4 && options.GSOType != GSOUDPL4 {
|
||||
return 0, fmt.Errorf("ip header version: %d, GSO type: %s", ipVersion, options.GSOType)
|
||||
}
|
||||
if len(in) < 20 {
|
||||
return 0, fmt.Errorf("length of packet (%d) < minimum ipv4 header size (%d)", len(in), 20)
|
||||
}
|
||||
case 6:
|
||||
if options.GSOType != GSOTCPv6 && options.GSOType != GSOUDPL4 {
|
||||
return 0, fmt.Errorf("ip header version: %d, GSO type: %s", ipVersion, options.GSOType)
|
||||
}
|
||||
if len(in) < 40 {
|
||||
return 0, fmt.Errorf("length of packet (%d) < minimum ipv6 header size (%d)", len(in), 40)
|
||||
}
|
||||
default:
|
||||
return 0, fmt.Errorf("invalid ip header version: %d", ipVersion)
|
||||
}
|
||||
|
||||
iphLen := int(options.CsumStart)
|
||||
srcAddrOffset := gsoIPv6SrcAddrOffset
|
||||
addrLen := 16
|
||||
if ipVersion == 4 {
|
||||
srcAddrOffset = gsoIPv4SrcAddrOffset
|
||||
addrLen = 4
|
||||
}
|
||||
transportCsumAt := int(options.CsumStart + options.CsumOffset)
|
||||
var firstTCPSeqNum uint32
|
||||
var protocol uint8
|
||||
if options.GSOType == GSOTCPv4 || options.GSOType == GSOTCPv6 {
|
||||
protocol = gsoIPProtoTCP
|
||||
if len(in) < int(options.CsumStart)+20 {
|
||||
return 0, fmt.Errorf("length of packet (%d) < GSO CsumStart (%d) + minimum TCP header size (%d)",
|
||||
len(in), options.CsumStart, 20)
|
||||
}
|
||||
firstTCPSeqNum = binary.BigEndian.Uint32(in[options.CsumStart+4:])
|
||||
} else {
|
||||
protocol = gsoIPProtoUDP
|
||||
}
|
||||
nextSegmentDataAt := int(options.HdrLen)
|
||||
i := 0
|
||||
for ; nextSegmentDataAt < len(in); i++ {
|
||||
if i == len(outBufs) {
|
||||
return i - 1, ErrTooManySegments
|
||||
}
|
||||
nextSegmentEnd := nextSegmentDataAt + int(options.GSOSize)
|
||||
if nextSegmentEnd > len(in) {
|
||||
nextSegmentEnd = len(in)
|
||||
}
|
||||
segmentDataLen := nextSegmentEnd - nextSegmentDataAt
|
||||
totalLen := int(options.HdrLen) + segmentDataLen
|
||||
sizes[i] = totalLen
|
||||
out := outBufs[i][outOffset:]
|
||||
|
||||
copy(out, in[:iphLen])
|
||||
if ipVersion == 4 {
|
||||
if i > 0 {
|
||||
id := binary.BigEndian.Uint16(out[4:])
|
||||
id += uint16(i)
|
||||
binary.BigEndian.PutUint16(out[4:], id)
|
||||
}
|
||||
out[10], out[11] = 0, 0
|
||||
binary.BigEndian.PutUint16(out[2:], uint16(totalLen))
|
||||
ipv4CSum := ^Checksum(out[:iphLen], 0)
|
||||
binary.BigEndian.PutUint16(out[10:], ipv4CSum)
|
||||
} else {
|
||||
binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen))
|
||||
}
|
||||
|
||||
copy(out[options.CsumStart:options.HdrLen], in[options.CsumStart:options.HdrLen])
|
||||
|
||||
if protocol == gsoIPProtoTCP {
|
||||
tcpSeq := firstTCPSeqNum + uint32(options.GSOSize*uint16(i))
|
||||
binary.BigEndian.PutUint32(out[options.CsumStart+4:], tcpSeq)
|
||||
if nextSegmentEnd != len(in) {
|
||||
clearFlags := gsoTCPFlagFIN | gsoTCPFlagPSH
|
||||
out[options.CsumStart+gsoTCPFlagsOffset] &^= clearFlags
|
||||
}
|
||||
} else {
|
||||
binary.BigEndian.PutUint16(out[options.CsumStart+4:], uint16(segmentDataLen)+(options.HdrLen-options.CsumStart))
|
||||
}
|
||||
|
||||
copy(out[options.HdrLen:], in[nextSegmentDataAt:nextSegmentEnd])
|
||||
|
||||
out[transportCsumAt], out[transportCsumAt+1] = 0, 0
|
||||
transportHeaderLen := int(options.HdrLen - options.CsumStart)
|
||||
lenForPseudo := uint16(transportHeaderLen + segmentDataLen)
|
||||
transportCSum := PseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo)
|
||||
transportCSum = ^Checksum(out[options.CsumStart:totalLen], transportCSum)
|
||||
binary.BigEndian.PutUint16(out[options.CsumStart+options.CsumOffset:], transportCSum)
|
||||
|
||||
nextSegmentDataAt += int(options.GSOSize)
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
|
@ -748,7 +748,7 @@ const (
|
|||
udp6GROCandidate
|
||||
)
|
||||
|
||||
func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType {
|
||||
func packetIsGROCandidate(b []byte, gro groDisablementFlags) groCandidateType {
|
||||
if len(b) < 28 {
|
||||
return notGROCandidate
|
||||
}
|
||||
|
|
@ -757,17 +757,17 @@ func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType {
|
|||
// IPv4 packets w/IP options do not coalesce
|
||||
return notGROCandidate
|
||||
}
|
||||
if b[9] == unix.IPPROTO_TCP && len(b) >= 40 {
|
||||
if b[9] == unix.IPPROTO_TCP && len(b) >= 40 && gro.canTCPGRO() {
|
||||
return tcp4GROCandidate
|
||||
}
|
||||
if b[9] == unix.IPPROTO_UDP && canUDPGRO {
|
||||
if b[9] == unix.IPPROTO_UDP && gro.canUDPGRO() {
|
||||
return udp4GROCandidate
|
||||
}
|
||||
} else if b[0]>>4 == 6 {
|
||||
if b[6] == unix.IPPROTO_TCP && len(b) >= 60 {
|
||||
if b[6] == unix.IPPROTO_TCP && len(b) >= 60 && gro.canTCPGRO() {
|
||||
return tcp6GROCandidate
|
||||
}
|
||||
if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && canUDPGRO {
|
||||
if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && gro.canUDPGRO() {
|
||||
return udp6GROCandidate
|
||||
}
|
||||
}
|
||||
|
|
@ -860,15 +860,15 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool)
|
|||
// handleGRO evaluates bufs for GRO, and writes the indices of the resulting
|
||||
// packets into toWrite. toWrite, tcpTable, and udpTable should initially be
|
||||
// empty (but non-nil), and are passed in to save allocs as the caller may reset
|
||||
// and recycle them across vectors of packets. canUDPGRO indicates if UDP GRO is
|
||||
// supported.
|
||||
func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, canUDPGRO bool, toWrite *[]int) error {
|
||||
// and recycle them across vectors of packets. gro indicates if TCP and UDP GRO
|
||||
// are supported/enabled.
|
||||
func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, gro groDisablementFlags, toWrite *[]int) error {
|
||||
for i := range bufs {
|
||||
if offset < virtioNetHdrLen || offset > len(bufs[i])-1 {
|
||||
return errors.New("invalid offset")
|
||||
}
|
||||
var result groResult
|
||||
switch packetIsGROCandidate(bufs[i][offset:], canUDPGRO) {
|
||||
switch packetIsGROCandidate(bufs[i][offset:], gro) {
|
||||
case tcp4GROCandidate:
|
||||
result = tcpGRO(bufs, offset, i, tcpTable, false)
|
||||
case tcp6GROCandidate:
|
||||
|
|
|
|||
14
tun/tun.go
14
tun/tun.go
|
|
@ -51,3 +51,17 @@ type Device interface {
|
|||
// lifetime of a Device.
|
||||
BatchSize() int
|
||||
}
|
||||
|
||||
// GRODevice is a Device extended with methods for disabling GRO. Certain OS
|
||||
// versions may have offload bugs. Where these bugs negatively impact throughput
|
||||
// or break connectivity entirely we can use these methods to disable the
|
||||
// related offload.
|
||||
type GRODevice interface {
|
||||
Device
|
||||
|
||||
// DisableUDPGRO disables UDP GRO if it is enabled.
|
||||
DisableUDPGRO()
|
||||
|
||||
// DisableTCPGRO disables TCP GRO if it is enabled.
|
||||
DisableTCPGRO()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ type NativeTun struct {
|
|||
statusListenersShutdown chan struct{}
|
||||
batchSize int
|
||||
vnetHdr bool
|
||||
udpGSO bool
|
||||
|
||||
closeOnce sync.Once
|
||||
|
||||
|
|
@ -49,10 +48,34 @@ type NativeTun struct {
|
|||
readOpMu sync.Mutex // readOpMu guards readBuff
|
||||
readBuff [virtioNetHdrLen + 65535]byte // if vnetHdr every read() is prefixed by virtioNetHdr
|
||||
|
||||
writeOpMu sync.Mutex // writeOpMu guards toWrite, tcpGROTable
|
||||
writeOpMu sync.Mutex // writeOpMu guards toWrite, tcpGROTable, udpGROTable, gro
|
||||
toWrite []int
|
||||
tcpGROTable *tcpGROTable
|
||||
udpGROTable *udpGROTable
|
||||
gro groDisablementFlags
|
||||
}
|
||||
|
||||
type groDisablementFlags int
|
||||
|
||||
const (
|
||||
tcpGRODisabled groDisablementFlags = 1 << iota
|
||||
udpGRODisabled
|
||||
)
|
||||
|
||||
func (g *groDisablementFlags) disableTCPGRO() {
|
||||
*g |= tcpGRODisabled
|
||||
}
|
||||
|
||||
func (g *groDisablementFlags) canTCPGRO() bool {
|
||||
return (*g)&tcpGRODisabled == 0
|
||||
}
|
||||
|
||||
func (g *groDisablementFlags) disableUDPGRO() {
|
||||
*g |= udpGRODisabled
|
||||
}
|
||||
|
||||
func (g *groDisablementFlags) canUDPGRO() bool {
|
||||
return (*g)&udpGRODisabled == 0
|
||||
}
|
||||
|
||||
func (tun *NativeTun) File() *os.File {
|
||||
|
|
@ -345,7 +368,7 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) {
|
|||
)
|
||||
tun.toWrite = tun.toWrite[:0]
|
||||
if tun.vnetHdr {
|
||||
err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.udpGSO, &tun.toWrite)
|
||||
err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.gro, &tun.toWrite)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
|
@ -502,6 +525,20 @@ func (tun *NativeTun) BatchSize() int {
|
|||
return tun.batchSize
|
||||
}
|
||||
|
||||
// DisableUDPGRO disables UDP GRO if it is enabled.
|
||||
func (tun *NativeTun) DisableUDPGRO() {
|
||||
tun.writeOpMu.Lock()
|
||||
tun.gro.disableUDPGRO()
|
||||
tun.writeOpMu.Unlock()
|
||||
}
|
||||
|
||||
// DisableTCPGRO disables TCP GRO if it is enabled.
|
||||
func (tun *NativeTun) DisableTCPGRO() {
|
||||
tun.writeOpMu.Lock()
|
||||
tun.gro.disableTCPGRO()
|
||||
tun.writeOpMu.Unlock()
|
||||
}
|
||||
|
||||
const (
|
||||
// TODO: support TSO with ECN bits
|
||||
tunTCPOffloads = unix.TUN_F_CSUM | unix.TUN_F_TSO4 | unix.TUN_F_TSO6
|
||||
|
|
@ -535,7 +572,9 @@ func (tun *NativeTun) initFromFlags(name string) error {
|
|||
tun.batchSize = conn.IdealBatchSize
|
||||
// tunUDPOffloads were added in Linux v6.2. We do not return an
|
||||
// error if they are unsupported at runtime.
|
||||
tun.udpGSO = unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) == nil
|
||||
if unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) != nil {
|
||||
tun.gro.disableUDPGRO()
|
||||
}
|
||||
} else {
|
||||
tun.batchSize = 1
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue