eburet-oss/wireguard-go-awg2-lx
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tun: export optimized IP checksum funcs

Browse source 
External implementers of tun.Device may support GRO, requiring checksum
offload.

Signed-off-by: Jordan Whited <jordan@tailscale.com>
This commit is contained in:
Jordan Whited 2024-07-23 13:34:34 -07:00 • committed by Jordan Whited
parent 60eeedfd62
commit 6c039a188c
5 changed files with 23 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

4
tun/checksum.go
View file

@ -702,7 +702,9 @@ func pseudoHeaderChecksum32(protocol uint8, srcAddr, dstAddr []byte, totalLen ui
return foldedSum return foldedSum
} }
func pseudoHeaderChecksum(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { // PseudoHeaderChecksum computes an IP pseudo-header checksum. srcAddr and
// dstAddr must be 4 or 16 bytes in length.
func PseudoHeaderChecksum(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 {
if strconv.IntSize < 64 { if strconv.IntSize < 64 {
return pseudoHeaderChecksum32(protocol, srcAddr, dstAddr, totalLen) return pseudoHeaderChecksum32(protocol, srcAddr, dstAddr, totalLen)
} }

11
tun/checksum_amd64.go
View file

@ -2,12 +2,15 @@ package tun
import "golang.org/x/sys/cpu" import "golang.org/x/sys/cpu"
// checksum computes an IP checksum starting with the provided initial value.
// The length of data should be at least 128 bytes for best performance. Smaller
// buffers will still compute a correct result. For best performance with
// smaller buffers, use shortChecksum().
var checksum = checksumAMD64 var checksum = checksumAMD64
// Checksum computes an IP checksum starting with the provided initial value.
// The length of data should be at least 128 bytes for best performance. Smaller
// buffers will still compute a correct result.
func Checksum(data []byte, initial uint16) uint16 {
return checksum(data, initial)
}
func init() { func init() {
if cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 { if cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 {
checksum = checksumAVX2 checksum = checksumAVX2

2
tun/checksum_generic.go
View file

@ -7,7 +7,7 @@ package tun
import "strconv" import "strconv"
func checksum(data []byte, initial uint16) uint16 { func Checksum(data []byte, initial uint16) uint16 {
if strconv.IntSize < 64 { if strconv.IntSize < 64 {
return checksumGeneric32(data, initial) return checksumGeneric32(data, initial)
} }

8
tun/offload.go
View file

@ -102,7 +102,7 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
// the checksum we compute. This is typically the pseudo-header sum. // the checksum we compute. This is typically the pseudo-header sum.
initial := binary.BigEndian.Uint16(in[cSumAt:]) initial := binary.BigEndian.Uint16(in[cSumAt:])
in[cSumAt], in[cSumAt+1] = 0, 0 in[cSumAt], in[cSumAt+1] = 0, 0
binary.BigEndian.PutUint16(in[cSumAt:], ^checksum(in[options.CsumStart:], initial)) binary.BigEndian.PutUint16(in[cSumAt:], ^Checksum(in[options.CsumStart:], initial))
} }
sizes[0] = copy(outBufs[0][outOffset:], in) sizes[0] = copy(outBufs[0][outOffset:], in)
return 1, nil return 1, nil
@ -179,7 +179,7 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
} }
out[10], out[11] = 0, 0 // clear ipv4 header checksum out[10], out[11] = 0, 0 // clear ipv4 header checksum
binary.BigEndian.PutUint16(out[2:], uint16(totalLen)) binary.BigEndian.PutUint16(out[2:], uint16(totalLen))
ipv4CSum := ^checksum(out[:iphLen], 0) ipv4CSum := ^Checksum(out[:iphLen], 0)
binary.BigEndian.PutUint16(out[10:], ipv4CSum) binary.BigEndian.PutUint16(out[10:], ipv4CSum)
} else { } else {
// For IPv6 we are responsible for updating the payload length field. // For IPv6 we are responsible for updating the payload length field.
@ -210,8 +210,8 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
out[transportCsumAt], out[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum out[transportCsumAt], out[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum
transportHeaderLen := int(options.HdrLen - options.CsumStart) transportHeaderLen := int(options.HdrLen - options.CsumStart)
lenForPseudo := uint16(transportHeaderLen + segmentDataLen) lenForPseudo := uint16(transportHeaderLen + segmentDataLen)
transportCSum := pseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) transportCSum := PseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo)
transportCSum = ^checksum(out[options.CsumStart:totalLen], transportCSum) transportCSum = ^Checksum(out[options.CsumStart:totalLen], transportCSum)
binary.BigEndian.PutUint16(out[options.CsumStart+options.CsumOffset:], transportCSum) binary.BigEndian.PutUint16(out[options.CsumStart+options.CsumOffset:], transportCSum)
nextSegmentDataAt += int(options.GSOSize) nextSegmentDataAt += int(options.GSOSize)

16
tun/offload_linux.go
View file

@ -410,8 +410,8 @@ func checksumValid(pkt []byte, iphLen, proto uint8, isV6 bool) bool {
addrSize = 16 addrSize = 16
} }
lenForPseudo := uint16(len(pkt) - int(iphLen)) lenForPseudo := uint16(len(pkt) - int(iphLen))
cSum := pseudoHeaderChecksum(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo) cSum := PseudoHeaderChecksum(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo)
return ^checksum(pkt[iphLen:], cSum) == 0 return ^Checksum(pkt[iphLen:], cSum) == 0
} }
// coalesceResult represents the result of attempting to coalesce two TCP // coalesceResult represents the result of attempting to coalesce two TCP
@ -659,7 +659,7 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e
hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4 hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4
pkt[10], pkt[11] = 0, 0 pkt[10], pkt[11] = 0, 0
binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length
iphCSum := ^checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum iphCSum := ^Checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum
binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field
} }
err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
@ -679,8 +679,8 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e
srcAddrAt := offset + addrOffset srcAddrAt := offset + addrOffset
srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen]
dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2]
psum := pseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) psum := PseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen)))
binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum))
} else { } else {
hdr := virtioNetHdr{} hdr := virtioNetHdr{}
err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
@ -716,7 +716,7 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e
} else { } else {
pkt[10], pkt[11] = 0, 0 pkt[10], pkt[11] = 0, 0
binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length
iphCSum := ^checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum iphCSum := ^Checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum
binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field
} }
err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
@ -739,8 +739,8 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e
srcAddrAt := offset + addrOffset srcAddrAt := offset + addrOffset
srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen]
dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2]
psum := pseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) psum := PseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen)))
binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum))
} else { } else {
hdr := virtioNetHdr{} hdr := virtioNetHdr{}
err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])