From bd7e9d35d114da345d0aa845e61686df8cfc5e1e Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 28 Mar 2023 13:40:31 -0700 Subject: [PATCH 01/60] all: rename module (#7) Signed-off-by: Jordan Whited --- conn/bind_windows.go | 2 +- conn/bindtest/bindtest.go | 2 +- device/bind_test.go | 2 +- device/device.go | 8 ++++---- device/device_test.go | 8 ++++---- device/keypair.go | 2 +- device/noise-protocol.go | 2 +- device/noise_test.go | 4 ++-- device/peer.go | 2 +- device/queueconstants_android.go | 2 +- device/queueconstants_default.go | 2 +- device/receive.go | 2 +- device/send.go | 2 +- device/sticky_default.go | 4 ++-- device/sticky_linux.go | 4 ++-- device/tun.go | 2 +- device/uapi.go | 2 +- go.mod | 2 +- ipc/namedpipe/namedpipe_test.go | 2 +- ipc/uapi_linux.go | 2 +- ipc/uapi_windows.go | 2 +- main.go | 8 ++++---- main_windows.go | 8 ++++---- tun/netstack/examples/http_client.go | 6 +++--- tun/netstack/examples/http_server.go | 6 +++--- tun/netstack/examples/ping_client.go | 6 +++--- tun/netstack/tun.go | 2 +- tun/tcp_offload_linux.go | 2 +- tun/tcp_offload_linux_test.go | 2 +- tun/tun_linux.go | 4 ++-- tun/tuntest/tuntest.go | 2 +- 31 files changed, 53 insertions(+), 53 deletions(-) diff --git a/conn/bind_windows.go b/conn/bind_windows.go index d5095e0..9638b30 100644 --- a/conn/bind_windows.go +++ b/conn/bind_windows.go @@ -17,7 +17,7 @@ import ( "golang.org/x/sys/windows" - "golang.zx2c4.com/wireguard/conn/winrio" + "github.com/tailscale/wireguard-go/conn/winrio" ) const ( diff --git a/conn/bindtest/bindtest.go b/conn/bindtest/bindtest.go index 74e7add..836d983 100644 --- a/conn/bindtest/bindtest.go +++ b/conn/bindtest/bindtest.go @@ -12,7 +12,7 @@ import ( "net/netip" "os" - "golang.zx2c4.com/wireguard/conn" + "github.com/tailscale/wireguard-go/conn" ) type ChannelBind struct { diff --git a/device/bind_test.go b/device/bind_test.go index 302a521..d64ca09 100644 --- a/device/bind_test.go +++ b/device/bind_test.go @@ -8,7 +8,7 @@ package device import ( "errors" - "golang.zx2c4.com/wireguard/conn" + "github.com/tailscale/wireguard-go/conn" ) type DummyDatagram struct { diff --git a/device/device.go b/device/device.go index 1af9fe0..7482d9b 100644 --- a/device/device.go +++ b/device/device.go @@ -11,10 +11,10 @@ import ( "sync/atomic" "time" - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/ratelimiter" - "golang.zx2c4.com/wireguard/rwcancel" - "golang.zx2c4.com/wireguard/tun" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/ratelimiter" + "github.com/tailscale/wireguard-go/rwcancel" + "github.com/tailscale/wireguard-go/tun" ) type Device struct { diff --git a/device/device_test.go b/device/device_test.go index fff172b..4088b9f 100644 --- a/device/device_test.go +++ b/device/device_test.go @@ -20,10 +20,10 @@ import ( "testing" "time" - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/conn/bindtest" - "golang.zx2c4.com/wireguard/tun" - "golang.zx2c4.com/wireguard/tun/tuntest" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/conn/bindtest" + "github.com/tailscale/wireguard-go/tun" + "github.com/tailscale/wireguard-go/tun/tuntest" ) // uapiCfg returns a string that contains cfg formatted use with IpcSet. diff --git a/device/keypair.go b/device/keypair.go index e3540d7..2689ee2 100644 --- a/device/keypair.go +++ b/device/keypair.go @@ -11,7 +11,7 @@ import ( "sync/atomic" "time" - "golang.zx2c4.com/wireguard/replay" + "github.com/tailscale/wireguard-go/replay" ) /* Due to limitations in Go and /x/crypto there is currently diff --git a/device/noise-protocol.go b/device/noise-protocol.go index e8f6145..9f2ba50 100644 --- a/device/noise-protocol.go +++ b/device/noise-protocol.go @@ -15,7 +15,7 @@ import ( "golang.org/x/crypto/chacha20poly1305" "golang.org/x/crypto/poly1305" - "golang.zx2c4.com/wireguard/tai64n" + "github.com/tailscale/wireguard-go/tai64n" ) type handshakeState int diff --git a/device/noise_test.go b/device/noise_test.go index 2dd5324..7d6af1d 100644 --- a/device/noise_test.go +++ b/device/noise_test.go @@ -10,8 +10,8 @@ import ( "encoding/binary" "testing" - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/tun/tuntest" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/tun/tuntest" ) func TestCurveWrappers(t *testing.T) { diff --git a/device/peer.go b/device/peer.go index 0ac4896..c7163ac 100644 --- a/device/peer.go +++ b/device/peer.go @@ -12,7 +12,7 @@ import ( "sync/atomic" "time" - "golang.zx2c4.com/wireguard/conn" + "github.com/tailscale/wireguard-go/conn" ) type Peer struct { diff --git a/device/queueconstants_android.go b/device/queueconstants_android.go index 3d80ead..bab9625 100644 --- a/device/queueconstants_android.go +++ b/device/queueconstants_android.go @@ -5,7 +5,7 @@ package device -import "golang.zx2c4.com/wireguard/conn" +import "github.com/tailscale/wireguard-go/conn" /* Reduce memory consumption for Android */ diff --git a/device/queueconstants_default.go b/device/queueconstants_default.go index ea763d0..9749cb7 100644 --- a/device/queueconstants_default.go +++ b/device/queueconstants_default.go @@ -7,7 +7,7 @@ package device -import "golang.zx2c4.com/wireguard/conn" +import "github.com/tailscale/wireguard-go/conn" const ( QueueStagedSize = conn.IdealBatchSize diff --git a/device/receive.go b/device/receive.go index e24d29f..c0cf747 100644 --- a/device/receive.go +++ b/device/receive.go @@ -13,10 +13,10 @@ import ( "sync" "time" + "github.com/tailscale/wireguard-go/conn" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" - "golang.zx2c4.com/wireguard/conn" ) type QueueHandshakeElement struct { diff --git a/device/send.go b/device/send.go index d22bf26..a95a46f 100644 --- a/device/send.go +++ b/device/send.go @@ -14,10 +14,10 @@ import ( "sync" "time" + "github.com/tailscale/wireguard-go/tun" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" - "golang.zx2c4.com/wireguard/tun" ) /* Outbound flow diff --git a/device/sticky_default.go b/device/sticky_default.go index 1038256..732f84c 100644 --- a/device/sticky_default.go +++ b/device/sticky_default.go @@ -3,8 +3,8 @@ package device import ( - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/rwcancel" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/rwcancel" ) func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, error) { diff --git a/device/sticky_linux.go b/device/sticky_linux.go index f9230f8..7a519c1 100644 --- a/device/sticky_linux.go +++ b/device/sticky_linux.go @@ -20,8 +20,8 @@ import ( "golang.org/x/sys/unix" - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/rwcancel" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/rwcancel" ) func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, error) { diff --git a/device/tun.go b/device/tun.go index 2a2ace9..960ecca 100644 --- a/device/tun.go +++ b/device/tun.go @@ -8,7 +8,7 @@ package device import ( "fmt" - "golang.zx2c4.com/wireguard/tun" + "github.com/tailscale/wireguard-go/tun" ) const DefaultMTU = 1420 diff --git a/device/uapi.go b/device/uapi.go index 617dcd3..2a91a93 100644 --- a/device/uapi.go +++ b/device/uapi.go @@ -18,7 +18,7 @@ import ( "sync" "time" - "golang.zx2c4.com/wireguard/ipc" + "github.com/tailscale/wireguard-go/ipc" ) type IPCError struct { diff --git a/go.mod b/go.mod index c04e1bb..0d60c9a 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module golang.zx2c4.com/wireguard +module github.com/tailscale/wireguard-go go 1.20 diff --git a/ipc/namedpipe/namedpipe_test.go b/ipc/namedpipe/namedpipe_test.go index 998453b..de7d0f6 100644 --- a/ipc/namedpipe/namedpipe_test.go +++ b/ipc/namedpipe/namedpipe_test.go @@ -20,8 +20,8 @@ import ( "testing" "time" + "github.com/tailscale/wireguard-go/ipc/namedpipe" "golang.org/x/sys/windows" - "golang.zx2c4.com/wireguard/ipc/namedpipe" ) func randomPipePath() string { diff --git a/ipc/uapi_linux.go b/ipc/uapi_linux.go index 1562a18..bfdf1bf 100644 --- a/ipc/uapi_linux.go +++ b/ipc/uapi_linux.go @@ -9,8 +9,8 @@ import ( "net" "os" + "github.com/tailscale/wireguard-go/rwcancel" "golang.org/x/sys/unix" - "golang.zx2c4.com/wireguard/rwcancel" ) type UAPIListener struct { diff --git a/ipc/uapi_windows.go b/ipc/uapi_windows.go index aa023c9..bc30ae0 100644 --- a/ipc/uapi_windows.go +++ b/ipc/uapi_windows.go @@ -8,8 +8,8 @@ package ipc import ( "net" + "github.com/tailscale/wireguard-go/ipc/namedpipe" "golang.org/x/sys/windows" - "golang.zx2c4.com/wireguard/ipc/namedpipe" ) // TODO: replace these with actual standard windows error numbers from the win package diff --git a/main.go b/main.go index e016116..55000e9 100644 --- a/main.go +++ b/main.go @@ -14,11 +14,11 @@ import ( "runtime" "strconv" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/device" + "github.com/tailscale/wireguard-go/ipc" + "github.com/tailscale/wireguard-go/tun" "golang.org/x/sys/unix" - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/device" - "golang.zx2c4.com/wireguard/ipc" - "golang.zx2c4.com/wireguard/tun" ) const ( diff --git a/main_windows.go b/main_windows.go index a4dc46f..689a9a7 100644 --- a/main_windows.go +++ b/main_windows.go @@ -12,11 +12,11 @@ import ( "golang.org/x/sys/windows" - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/device" - "golang.zx2c4.com/wireguard/ipc" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/device" + "github.com/tailscale/wireguard-go/ipc" - "golang.zx2c4.com/wireguard/tun" + "github.com/tailscale/wireguard-go/tun" ) const ( diff --git a/tun/netstack/examples/http_client.go b/tun/netstack/examples/http_client.go index ccd32ed..81f4d31 100644 --- a/tun/netstack/examples/http_client.go +++ b/tun/netstack/examples/http_client.go @@ -13,9 +13,9 @@ import ( "net/http" "net/netip" - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/device" - "golang.zx2c4.com/wireguard/tun/netstack" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/device" + "github.com/tailscale/wireguard-go/tun/netstack" ) func main() { diff --git a/tun/netstack/examples/http_server.go b/tun/netstack/examples/http_server.go index f5b7a8f..30f4544 100644 --- a/tun/netstack/examples/http_server.go +++ b/tun/netstack/examples/http_server.go @@ -14,9 +14,9 @@ import ( "net/http" "net/netip" - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/device" - "golang.zx2c4.com/wireguard/tun/netstack" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/device" + "github.com/tailscale/wireguard-go/tun/netstack" ) func main() { diff --git a/tun/netstack/examples/ping_client.go b/tun/netstack/examples/ping_client.go index 2eef0fb..fc991b1 100644 --- a/tun/netstack/examples/ping_client.go +++ b/tun/netstack/examples/ping_client.go @@ -17,9 +17,9 @@ import ( "golang.org/x/net/icmp" "golang.org/x/net/ipv4" - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/device" - "golang.zx2c4.com/wireguard/tun/netstack" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/device" + "github.com/tailscale/wireguard-go/tun/netstack" ) func main() { diff --git a/tun/netstack/tun.go b/tun/netstack/tun.go index 596cfcd..2402842 100644 --- a/tun/netstack/tun.go +++ b/tun/netstack/tun.go @@ -22,7 +22,7 @@ import ( "syscall" "time" - "golang.zx2c4.com/wireguard/tun" + "github.com/tailscale/wireguard-go/tun" "golang.org/x/net/dns/dnsmessage" "gvisor.dev/gvisor/pkg/bufferv2" diff --git a/tun/tcp_offload_linux.go b/tun/tcp_offload_linux.go index 39a7180..d64010d 100644 --- a/tun/tcp_offload_linux.go +++ b/tun/tcp_offload_linux.go @@ -12,8 +12,8 @@ import ( "io" "unsafe" + "github.com/tailscale/wireguard-go/conn" "golang.org/x/sys/unix" - "golang.zx2c4.com/wireguard/conn" ) const tcpFlagsOffset = 13 diff --git a/tun/tcp_offload_linux_test.go b/tun/tcp_offload_linux_test.go index 9160e18..e828642 100644 --- a/tun/tcp_offload_linux_test.go +++ b/tun/tcp_offload_linux_test.go @@ -9,8 +9,8 @@ import ( "net/netip" "testing" + "github.com/tailscale/wireguard-go/conn" "golang.org/x/sys/unix" - "golang.zx2c4.com/wireguard/conn" "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/header" ) diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 12cd49f..eb5051e 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -17,9 +17,9 @@ import ( "time" "unsafe" + "github.com/tailscale/wireguard-go/conn" + "github.com/tailscale/wireguard-go/rwcancel" "golang.org/x/sys/unix" - "golang.zx2c4.com/wireguard/conn" - "golang.zx2c4.com/wireguard/rwcancel" ) const ( diff --git a/tun/tuntest/tuntest.go b/tun/tuntest/tuntest.go index d07e860..e7507c2 100644 --- a/tun/tuntest/tuntest.go +++ b/tun/tuntest/tuntest.go @@ -11,7 +11,7 @@ import ( "net/netip" "os" - "golang.zx2c4.com/wireguard/tun" + "github.com/tailscale/wireguard-go/tun" ) func Ping(dst, src netip.Addr) []byte { From e26adb828d950319d0d0f17178035597e70904a7 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Wed, 27 Sep 2023 16:15:09 -0700 Subject: [PATCH 02/60] go.mod,tun/netstack: bump gvisor Signed-off-by: James Tucker --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- tun/netstack/tun.go | 14 +++++++------- tun/tcp_offload_linux_test.go | 8 ++++---- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 0d60c9a..9c9b02a 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module github.com/tailscale/wireguard-go go 1.20 require ( - golang.org/x/crypto v0.6.0 - golang.org/x/net v0.7.0 - golang.org/x/sys v0.5.1-0.20230222185716-a3b23cc77e89 + golang.org/x/crypto v0.13.0 + golang.org/x/net v0.15.0 + golang.org/x/sys v0.12.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 - gvisor.dev/gvisor v0.0.0-20221203005347-703fd9b7fbc0 + gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 ) require ( github.com/google/btree v1.0.1 // indirect - golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect ) diff --git a/go.sum b/go.sum index cfeaee6..6bcecea 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,14 @@ github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/sys v0.5.1-0.20230222185716-a3b23cc77e89 h1:260HNjMTPDya+jq5AM1zZLgG9pv9GASPAGiEEJUbRg4= -golang.org/x/sys v0.5.1-0.20230222185716-a3b23cc77e89/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= -gvisor.dev/gvisor v0.0.0-20221203005347-703fd9b7fbc0 h1:Wobr37noukisGxpKo5jAsLREcpj61RxrWYzD8uwveOY= -gvisor.dev/gvisor v0.0.0-20221203005347-703fd9b7fbc0/go.mod h1:Dn5idtptoW1dIos9U6A2rpebLs/MtTwFacjKb8jLdQA= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 h1:TbRPT0HtzFP3Cno1zZo7yPzEEnfu8EjLfl6IU9VfqkQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259/go.mod h1:AVgIgHMwK63XvmAzWG9vLQ41YnVHN0du0tEC46fI7yY= diff --git a/tun/netstack/tun.go b/tun/netstack/tun.go index 2402842..d8e70bb 100644 --- a/tun/netstack/tun.go +++ b/tun/netstack/tun.go @@ -25,7 +25,7 @@ import ( "github.com/tailscale/wireguard-go/tun" "golang.org/x/net/dns/dnsmessage" - "gvisor.dev/gvisor/pkg/bufferv2" + "gvisor.dev/gvisor/pkg/buffer" "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" "gvisor.dev/gvisor/pkg/tcpip/header" @@ -43,7 +43,7 @@ type netTun struct { ep *channel.Endpoint stack *stack.Stack events chan tun.Event - incomingPacket chan *bufferv2.View + incomingPacket chan *buffer.View mtu int dnsServers []netip.Addr hasV4, hasV6 bool @@ -61,7 +61,7 @@ func CreateNetTUN(localAddresses, dnsServers []netip.Addr, mtu int) (tun.Device, ep: channel.New(1024, uint32(mtu), ""), stack: stack.New(opts), events: make(chan tun.Event, 10), - incomingPacket: make(chan *bufferv2.View), + incomingPacket: make(chan *buffer.View), dnsServers: dnsServers, mtu: mtu, } @@ -84,7 +84,7 @@ func CreateNetTUN(localAddresses, dnsServers []netip.Addr, mtu int) (tun.Device, } protoAddr := tcpip.ProtocolAddress{ Protocol: protoNumber, - AddressWithPrefix: tcpip.Address(ip.AsSlice()).WithPrefix(), + AddressWithPrefix: tcpip.AddrFromSlice(ip.AsSlice()).WithPrefix(), } tcpipErr := dev.stack.AddProtocolAddress(1, protoAddr, stack.AddressProperties{}) if tcpipErr != nil { @@ -140,7 +140,7 @@ func (tun *netTun) Write(buf [][]byte, offset int) (int, error) { continue } - pkb := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: bufferv2.MakeWithData(packet)}) + pkb := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buffer.MakeWithData(packet)}) switch packet[0] >> 4 { case 4: tun.ep.InjectInbound(header.IPv4ProtocolNumber, pkb) @@ -198,7 +198,7 @@ func convertToFullAddr(endpoint netip.AddrPort) (tcpip.FullAddress, tcpip.Networ } return tcpip.FullAddress{ NIC: 1, - Addr: tcpip.Address(endpoint.Addr().AsSlice()), + Addr: tcpip.AddrFromSlice(endpoint.Addr().AsSlice()), Port: endpoint.Port(), }, protoNumber } @@ -453,7 +453,7 @@ func (pc *PingConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) { return 0, nil, fmt.Errorf("ping read: %s", tcpipErr) } - remoteAddr, _ := netip.AddrFromSlice([]byte(res.RemoteAddr.Addr)) + remoteAddr, _ := netip.AddrFromSlice(res.RemoteAddr.Addr.AsSlice()) return res.Count, &PingAddr{remoteAddr}, nil } diff --git a/tun/tcp_offload_linux_test.go b/tun/tcp_offload_linux_test.go index e828642..41fba70 100644 --- a/tun/tcp_offload_linux_test.go +++ b/tun/tcp_offload_linux_test.go @@ -35,8 +35,8 @@ func tcp4PacketMutateIPFields(srcIPPort, dstIPPort netip.AddrPort, flags header. srcAs4 := srcIPPort.Addr().As4() dstAs4 := dstIPPort.Addr().As4() ipFields := &header.IPv4Fields{ - SrcAddr: tcpip.Address(srcAs4[:]), - DstAddr: tcpip.Address(dstAs4[:]), + SrcAddr: tcpip.AddrFromSlice(srcAs4[:]), + DstAddr: tcpip.AddrFromSlice(dstAs4[:]), Protocol: unix.IPPROTO_TCP, TTL: 64, TotalLength: uint16(totalLen), @@ -72,8 +72,8 @@ func tcp6PacketMutateIPFields(srcIPPort, dstIPPort netip.AddrPort, flags header. srcAs16 := srcIPPort.Addr().As16() dstAs16 := dstIPPort.Addr().As16() ipFields := &header.IPv6Fields{ - SrcAddr: tcpip.Address(srcAs16[:]), - DstAddr: tcpip.Address(dstAs16[:]), + SrcAddr: tcpip.AddrFromSlice(srcAs16[:]), + DstAddr: tcpip.AddrFromSlice(dstAs16[:]), TransportProtocol: unix.IPPROTO_TCP, HopLimit: 64, PayloadLength: uint16(segmentSize + 20), From e06231b8611133d249b8a1d5eaf1588c27800f05 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 4 Apr 2023 13:04:30 -0700 Subject: [PATCH 03/60] conn, device: use UDP GSO and GRO on Linux StdNetBind probes for UDP GSO and GRO support at runtime. UDP GSO is dependent on checksum offload support on the egress netdev. UDP GSO will be disabled in the event sendmmsg() returns EIO, which is a strong signal that the egress netdev does not support checksum offload. The iperf3 results below demonstrate the effect of this commit between two Linux computers with i5-12400 CPUs. There is roughly ~13us of round trip latency between them. The first result is from commit 052af4a without UDP GSO or GRO. Starting Test: protocol: TCP, 1 streams, 131072 byte blocks [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-10.00 sec 9.85 GBytes 8.46 Gbits/sec 1139 3.01 MBytes - - - - - - - - - - - - - - - - - - - - - - - - - Test Complete. Summary Results: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 9.85 GBytes 8.46 Gbits/sec 1139 sender [ 5] 0.00-10.04 sec 9.85 GBytes 8.42 Gbits/sec receiver The second result is with UDP GSO and GRO. Starting Test: protocol: TCP, 1 streams, 131072 byte blocks [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-10.00 sec 12.3 GBytes 10.6 Gbits/sec 232 3.15 MBytes - - - - - - - - - - - - - - - - - - - - - - - - - Test Complete. Summary Results: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 12.3 GBytes 10.6 Gbits/sec 232 sender [ 5] 0.00-10.04 sec 12.3 GBytes 10.6 Gbits/sec receiver Reviewed-by: Adrian Dewhurst Signed-off-by: Jordan Whited --- conn/bind_std.go | 396 ++++++++++++------ conn/bind_std_test.go | 230 +++++++++- .../{sticky_default.go => control_default.go} | 22 +- conn/{sticky_linux.go => control_linux.go} | 51 ++- ...ky_linux_test.go => control_linux_test.go} | 4 +- conn/controlfns_linux.go | 8 + conn/errors_default.go | 12 + conn/errors_linux.go | 26 ++ conn/features_default.go | 15 + conn/features_linux.go | 42 ++ device/send.go | 8 + 11 files changed, 673 insertions(+), 141 deletions(-) rename conn/{sticky_default.go => control_default.go} (54%) rename conn/{sticky_linux.go => control_linux.go} (65%) rename conn/{sticky_linux_test.go => control_linux_test.go} (98%) create mode 100644 conn/errors_default.go create mode 100644 conn/errors_linux.go create mode 100644 conn/features_default.go create mode 100644 conn/features_linux.go diff --git a/conn/bind_std.go b/conn/bind_std.go index c701ef8..cc5cf23 100644 --- a/conn/bind_std.go +++ b/conn/bind_std.go @@ -8,6 +8,7 @@ package conn import ( "context" "errors" + "fmt" "net" "net/netip" "runtime" @@ -20,7 +21,8 @@ import ( ) var ( - _ Bind = (*StdNetBind)(nil) + _ Bind = (*StdNetBind)(nil) + _ Endpoint = (*StdNetEndpoint)(nil) ) // StdNetBind implements Bind for all platforms. While Windows has its own Bind @@ -29,16 +31,19 @@ var ( // methods for sending and receiving multiple datagrams per-syscall. See the // proposal in https://github.com/golang/go/issues/45886#issuecomment-1218301564. type StdNetBind struct { - mu sync.Mutex // protects all fields except as specified - ipv4 *net.UDPConn - ipv6 *net.UDPConn - ipv4PC *ipv4.PacketConn // will be nil on non-Linux - ipv6PC *ipv6.PacketConn // will be nil on non-Linux + mu sync.Mutex // protects all fields except as specified + ipv4 *net.UDPConn + ipv6 *net.UDPConn + ipv4PC *ipv4.PacketConn // will be nil on non-Linux + ipv6PC *ipv6.PacketConn // will be nil on non-Linux + ipv4TxOffload bool + ipv4RxOffload bool + ipv6TxOffload bool + ipv6RxOffload bool - // these three fields are not guarded by mu - udpAddrPool sync.Pool - ipv4MsgsPool sync.Pool - ipv6MsgsPool sync.Pool + // these two fields are not guarded by mu + udpAddrPool sync.Pool + msgsPool sync.Pool blackhole4 bool blackhole6 bool @@ -54,23 +59,12 @@ func NewStdNetBind() Bind { }, }, - ipv4MsgsPool: sync.Pool{ - New: func() any { - msgs := make([]ipv4.Message, IdealBatchSize) - for i := range msgs { - msgs[i].Buffers = make(net.Buffers, 1) - msgs[i].OOB = make([]byte, srcControlSize) - } - return &msgs - }, - }, - - ipv6MsgsPool: sync.Pool{ + msgsPool: sync.Pool{ New: func() any { msgs := make([]ipv6.Message, IdealBatchSize) for i := range msgs { msgs[i].Buffers = make(net.Buffers, 1) - msgs[i].OOB = make([]byte, srcControlSize) + msgs[i].OOB = make([]byte, controlSize) } return &msgs }, @@ -179,19 +173,21 @@ again: } var fns []ReceiveFunc if v4conn != nil { + s.ipv4TxOffload, s.ipv4RxOffload = supportsUDPOffload(v4conn) if runtime.GOOS == "linux" { v4pc = ipv4.NewPacketConn(v4conn) s.ipv4PC = v4pc } - fns = append(fns, s.makeReceiveIPv4(v4pc, v4conn)) + fns = append(fns, s.makeReceiveIPv4(v4pc, v4conn, s.ipv4RxOffload)) s.ipv4 = v4conn } if v6conn != nil { + s.ipv6TxOffload, s.ipv6RxOffload = supportsUDPOffload(v6conn) if runtime.GOOS == "linux" { v6pc = ipv6.NewPacketConn(v6conn) s.ipv6PC = v6pc } - fns = append(fns, s.makeReceiveIPv6(v6pc, v6conn)) + fns = append(fns, s.makeReceiveIPv6(v6pc, v6conn, s.ipv6RxOffload)) s.ipv6 = v6conn } if len(fns) == 0 { @@ -201,69 +197,93 @@ again: return fns, uint16(port), nil } -func (s *StdNetBind) makeReceiveIPv4(pc *ipv4.PacketConn, conn *net.UDPConn) ReceiveFunc { - return func(bufs [][]byte, sizes []int, eps []Endpoint) (n int, err error) { - msgs := s.ipv4MsgsPool.Get().(*[]ipv4.Message) - defer s.ipv4MsgsPool.Put(msgs) - for i := range bufs { - (*msgs)[i].Buffers[0] = bufs[i] - } - var numMsgs int - if runtime.GOOS == "linux" { - numMsgs, err = pc.ReadBatch(*msgs, 0) +func (s *StdNetBind) putMessages(msgs *[]ipv6.Message) { + for i := range *msgs { + (*msgs)[i] = ipv6.Message{Buffers: (*msgs)[i].Buffers, OOB: (*msgs)[i].OOB} + } + s.msgsPool.Put(msgs) +} + +func (s *StdNetBind) getMessages() *[]ipv6.Message { + return s.msgsPool.Get().(*[]ipv6.Message) +} + +var ( + // If compilation fails here these are no longer the same underlying type. + _ ipv6.Message = ipv4.Message{} +) + +type batchReader interface { + ReadBatch([]ipv6.Message, int) (int, error) +} + +type batchWriter interface { + WriteBatch([]ipv6.Message, int) (int, error) +} + +func (s *StdNetBind) receiveIP( + br batchReader, + conn *net.UDPConn, + rxOffload bool, + bufs [][]byte, + sizes []int, + eps []Endpoint, +) (n int, err error) { + msgs := s.getMessages() + for i := range bufs { + (*msgs)[i].Buffers[0] = bufs[i] + (*msgs)[i].OOB = (*msgs)[i].OOB[:cap((*msgs)[i].OOB)] + } + defer s.putMessages(msgs) + var numMsgs int + if runtime.GOOS == "linux" { + if rxOffload { + readAt := len(*msgs) - 2 + numMsgs, err = br.ReadBatch((*msgs)[readAt:], 0) + if err != nil { + return 0, err + } + numMsgs, err = splitCoalescedMessages(*msgs, readAt, getGSOSize) if err != nil { return 0, err } } else { - msg := &(*msgs)[0] - msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB) + numMsgs, err = br.ReadBatch(*msgs, 0) if err != nil { return 0, err } - numMsgs = 1 } - for i := 0; i < numMsgs; i++ { - msg := &(*msgs)[i] - sizes[i] = msg.N - addrPort := msg.Addr.(*net.UDPAddr).AddrPort() - ep := &StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation - getSrcFromControl(msg.OOB[:msg.NN], ep) - eps[i] = ep + } else { + msg := &(*msgs)[0] + msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB) + if err != nil { + return 0, err } - return numMsgs, nil + numMsgs = 1 + } + for i := 0; i < numMsgs; i++ { + msg := &(*msgs)[i] + sizes[i] = msg.N + if sizes[i] == 0 { + continue + } + addrPort := msg.Addr.(*net.UDPAddr).AddrPort() + ep := &StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation + getSrcFromControl(msg.OOB[:msg.NN], ep) + eps[i] = ep + } + return numMsgs, nil +} + +func (s *StdNetBind) makeReceiveIPv4(pc *ipv4.PacketConn, conn *net.UDPConn, rxOffload bool) ReceiveFunc { + return func(bufs [][]byte, sizes []int, eps []Endpoint) (n int, err error) { + return s.receiveIP(pc, conn, rxOffload, bufs, sizes, eps) } } -func (s *StdNetBind) makeReceiveIPv6(pc *ipv6.PacketConn, conn *net.UDPConn) ReceiveFunc { +func (s *StdNetBind) makeReceiveIPv6(pc *ipv6.PacketConn, conn *net.UDPConn, rxOffload bool) ReceiveFunc { return func(bufs [][]byte, sizes []int, eps []Endpoint) (n int, err error) { - msgs := s.ipv6MsgsPool.Get().(*[]ipv6.Message) - defer s.ipv6MsgsPool.Put(msgs) - for i := range bufs { - (*msgs)[i].Buffers[0] = bufs[i] - } - var numMsgs int - if runtime.GOOS == "linux" { - numMsgs, err = pc.ReadBatch(*msgs, 0) - if err != nil { - return 0, err - } - } else { - msg := &(*msgs)[0] - msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB) - if err != nil { - return 0, err - } - numMsgs = 1 - } - for i := 0; i < numMsgs; i++ { - msg := &(*msgs)[i] - sizes[i] = msg.N - addrPort := msg.Addr.(*net.UDPAddr).AddrPort() - ep := &StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation - getSrcFromControl(msg.OOB[:msg.NN], ep) - eps[i] = ep - } - return numMsgs, nil + return s.receiveIP(pc, conn, rxOffload, bufs, sizes, eps) } } @@ -293,28 +313,42 @@ func (s *StdNetBind) Close() error { } s.blackhole4 = false s.blackhole6 = false + s.ipv4TxOffload = false + s.ipv4RxOffload = false + s.ipv6TxOffload = false + s.ipv6RxOffload = false if err1 != nil { return err1 } return err2 } +type ErrUDPGSODisabled struct { + onLaddr string + RetryErr error +} + +func (e ErrUDPGSODisabled) Error() string { + return fmt.Sprintf("disabled UDP GSO on %s, NIC(s) may not support checksum offload", e.onLaddr) +} + +func (e ErrUDPGSODisabled) Unwrap() error { + return e.RetryErr +} + func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint) error { s.mu.Lock() blackhole := s.blackhole4 conn := s.ipv4 - var ( - pc4 *ipv4.PacketConn - pc6 *ipv6.PacketConn - ) + offload := s.ipv4TxOffload + br := batchWriter(s.ipv4PC) is6 := false if endpoint.DstIP().Is6() { blackhole = s.blackhole6 conn = s.ipv6 - pc6 = s.ipv6PC + br = s.ipv6PC is6 = true - } else { - pc4 = s.ipv4PC + offload = s.ipv6TxOffload } s.mu.Unlock() @@ -324,25 +358,56 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint) error { if conn == nil { return syscall.EAFNOSUPPORT } + + msgs := s.getMessages() + defer s.putMessages(msgs) + ua := s.udpAddrPool.Get().(*net.UDPAddr) + defer s.udpAddrPool.Put(ua) if is6 { - return s.send6(conn, pc6, endpoint, bufs) + as16 := endpoint.DstIP().As16() + copy(ua.IP, as16[:]) + ua.IP = ua.IP[:16] } else { - return s.send4(conn, pc4, endpoint, bufs) + as4 := endpoint.DstIP().As4() + copy(ua.IP, as4[:]) + ua.IP = ua.IP[:4] } + ua.Port = int(endpoint.(*StdNetEndpoint).Port()) + var ( + retried bool + err error + ) +retry: + if offload { + n := coalesceMessages(ua, endpoint.(*StdNetEndpoint), bufs, *msgs, setGSOSize) + err = s.send(conn, br, (*msgs)[:n]) + if err != nil && offload && errShouldDisableUDPGSO(err) { + offload = false + s.mu.Lock() + if is6 { + s.ipv6TxOffload = false + } else { + s.ipv4TxOffload = false + } + s.mu.Unlock() + retried = true + goto retry + } + } else { + for i := range bufs { + (*msgs)[i].Addr = ua + (*msgs)[i].Buffers[0] = bufs[i] + setSrcControl(&(*msgs)[i].OOB, endpoint.(*StdNetEndpoint)) + } + err = s.send(conn, br, (*msgs)[:len(bufs)]) + } + if retried { + return ErrUDPGSODisabled{onLaddr: conn.LocalAddr().String(), RetryErr: err} + } + return err } -func (s *StdNetBind) send4(conn *net.UDPConn, pc *ipv4.PacketConn, ep Endpoint, bufs [][]byte) error { - ua := s.udpAddrPool.Get().(*net.UDPAddr) - as4 := ep.DstIP().As4() - copy(ua.IP, as4[:]) - ua.IP = ua.IP[:4] - ua.Port = int(ep.(*StdNetEndpoint).Port()) - msgs := s.ipv4MsgsPool.Get().(*[]ipv4.Message) - for i, buf := range bufs { - (*msgs)[i].Buffers[0] = buf - (*msgs)[i].Addr = ua - setSrcControl(&(*msgs)[i].OOB, ep.(*StdNetEndpoint)) - } +func (s *StdNetBind) send(conn *net.UDPConn, pc batchWriter, msgs []ipv6.Message) error { var ( n int err error @@ -350,59 +415,128 @@ func (s *StdNetBind) send4(conn *net.UDPConn, pc *ipv4.PacketConn, ep Endpoint, ) if runtime.GOOS == "linux" { for { - n, err = pc.WriteBatch((*msgs)[start:len(bufs)], 0) - if err != nil || n == len((*msgs)[start:len(bufs)]) { + n, err = pc.WriteBatch(msgs[start:], 0) + if err != nil || n == len(msgs[start:]) { break } start += n } } else { - for i, buf := range bufs { - _, _, err = conn.WriteMsgUDP(buf, (*msgs)[i].OOB, ua) + for _, msg := range msgs { + _, _, err = conn.WriteMsgUDP(msg.Buffers[0], msg.OOB, msg.Addr.(*net.UDPAddr)) if err != nil { break } } } - s.udpAddrPool.Put(ua) - s.ipv4MsgsPool.Put(msgs) return err } -func (s *StdNetBind) send6(conn *net.UDPConn, pc *ipv6.PacketConn, ep Endpoint, bufs [][]byte) error { - ua := s.udpAddrPool.Get().(*net.UDPAddr) - as16 := ep.DstIP().As16() - copy(ua.IP, as16[:]) - ua.IP = ua.IP[:16] - ua.Port = int(ep.(*StdNetEndpoint).Port()) - msgs := s.ipv6MsgsPool.Get().(*[]ipv6.Message) - for i, buf := range bufs { - (*msgs)[i].Buffers[0] = buf - (*msgs)[i].Addr = ua - setSrcControl(&(*msgs)[i].OOB, ep.(*StdNetEndpoint)) - } +const ( + // Exceeding these values results in EMSGSIZE. They account for layer3 and + // layer4 headers. IPv6 does not need to account for itself as the payload + // length field is self excluding. + maxIPv4PayloadLen = 1<<16 - 1 - 20 - 8 + maxIPv6PayloadLen = 1<<16 - 1 - 8 + + // This is a hard limit imposed by the kernel. + udpSegmentMaxDatagrams = 64 +) + +type setGSOFunc func(control *[]byte, gsoSize uint16) + +func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, msgs []ipv6.Message, setGSO setGSOFunc) int { var ( - n int - err error - start int + base = -1 // index of msg we are currently coalescing into + gsoSize int // segmentation size of msgs[base] + dgramCnt int // number of dgrams coalesced into msgs[base] + endBatch bool // tracking flag to start a new batch on next iteration of bufs ) - if runtime.GOOS == "linux" { - for { - n, err = pc.WriteBatch((*msgs)[start:len(bufs)], 0) - if err != nil || n == len((*msgs)[start:len(bufs)]) { - break + maxPayloadLen := maxIPv4PayloadLen + if ep.DstIP().Is6() { + maxPayloadLen = maxIPv6PayloadLen + } + for i, buf := range bufs { + if i > 0 { + msgLen := len(buf) + baseLenBefore := len(msgs[base].Buffers[0]) + freeBaseCap := cap(msgs[base].Buffers[0]) - baseLenBefore + if msgLen+baseLenBefore <= maxPayloadLen && + msgLen <= gsoSize && + msgLen <= freeBaseCap && + dgramCnt < udpSegmentMaxDatagrams && + !endBatch { + msgs[base].Buffers[0] = append(msgs[base].Buffers[0], buf...) + if i == len(bufs)-1 { + setGSO(&msgs[base].OOB, uint16(gsoSize)) + } + dgramCnt++ + if msgLen < gsoSize { + // A smaller than gsoSize packet on the tail is legal, but + // it must end the batch. + endBatch = true + } + continue } - start += n } - } else { - for i, buf := range bufs { - _, _, err = conn.WriteMsgUDP(buf, (*msgs)[i].OOB, ua) - if err != nil { - break + if dgramCnt > 1 { + setGSO(&msgs[base].OOB, uint16(gsoSize)) + } + // Reset prior to incrementing base since we are preparing to start a + // new potential batch. + endBatch = false + base++ + gsoSize = len(buf) + setSrcControl(&msgs[base].OOB, ep) + msgs[base].Buffers[0] = buf + msgs[base].Addr = addr + dgramCnt = 1 + } + return base + 1 +} + +type getGSOFunc func(control []byte) (int, error) + +func splitCoalescedMessages(msgs []ipv6.Message, firstMsgAt int, getGSO getGSOFunc) (n int, err error) { + for i := firstMsgAt; i < len(msgs); i++ { + msg := &msgs[i] + if msg.N == 0 { + return n, err + } + var ( + gsoSize int + start int + end = msg.N + numToSplit = 1 + ) + gsoSize, err = getGSO(msg.OOB[:msg.NN]) + if err != nil { + return n, err + } + if gsoSize > 0 { + numToSplit = (msg.N + gsoSize - 1) / gsoSize + end = gsoSize + } + for j := 0; j < numToSplit; j++ { + if n > i { + return n, errors.New("splitting coalesced packet resulted in overflow") } + copied := copy(msgs[n].Buffers[0], msg.Buffers[0][start:end]) + msgs[n].N = copied + msgs[n].Addr = msg.Addr + start = end + end += gsoSize + if end > msg.N { + end = msg.N + } + n++ + } + if i != n-1 { + // It is legal for bytes to move within msg.Buffers[0] as a result + // of splitting, so we only zero the source msg len when it is not + // the destination of the last split operation above. + msg.N = 0 } } - s.udpAddrPool.Put(ua) - s.ipv6MsgsPool.Put(msgs) - return err + return n, nil } diff --git a/conn/bind_std_test.go b/conn/bind_std_test.go index 1e46776..34a3c9a 100644 --- a/conn/bind_std_test.go +++ b/conn/bind_std_test.go @@ -1,6 +1,12 @@ package conn -import "testing" +import ( + "encoding/binary" + "net" + "testing" + + "golang.org/x/net/ipv6" +) func TestStdNetBindReceiveFuncAfterClose(t *testing.T) { bind := NewStdNetBind().(*StdNetBind) @@ -20,3 +26,225 @@ func TestStdNetBindReceiveFuncAfterClose(t *testing.T) { fn(bufs, sizes, eps) } } + +func mockSetGSOSize(control *[]byte, gsoSize uint16) { + *control = (*control)[:cap(*control)] + binary.LittleEndian.PutUint16(*control, gsoSize) +} + +func Test_coalesceMessages(t *testing.T) { + cases := []struct { + name string + buffs [][]byte + wantLens []int + wantGSO []int + }{ + { + name: "one message no coalesce", + buffs: [][]byte{ + make([]byte, 1, 1), + }, + wantLens: []int{1}, + wantGSO: []int{0}, + }, + { + name: "two messages equal len coalesce", + buffs: [][]byte{ + make([]byte, 1, 2), + make([]byte, 1, 1), + }, + wantLens: []int{2}, + wantGSO: []int{1}, + }, + { + name: "two messages unequal len coalesce", + buffs: [][]byte{ + make([]byte, 2, 3), + make([]byte, 1, 1), + }, + wantLens: []int{3}, + wantGSO: []int{2}, + }, + { + name: "three messages second unequal len coalesce", + buffs: [][]byte{ + make([]byte, 2, 3), + make([]byte, 1, 1), + make([]byte, 2, 2), + }, + wantLens: []int{3, 2}, + wantGSO: []int{2, 0}, + }, + { + name: "three messages limited cap coalesce", + buffs: [][]byte{ + make([]byte, 2, 4), + make([]byte, 2, 2), + make([]byte, 2, 2), + }, + wantLens: []int{4, 2}, + wantGSO: []int{2, 0}, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + addr := &net.UDPAddr{ + IP: net.ParseIP("127.0.0.1").To4(), + Port: 1, + } + msgs := make([]ipv6.Message, len(tt.buffs)) + for i := range msgs { + msgs[i].Buffers = make([][]byte, 1) + msgs[i].OOB = make([]byte, 0, 2) + } + got := coalesceMessages(addr, &StdNetEndpoint{AddrPort: addr.AddrPort()}, tt.buffs, msgs, mockSetGSOSize) + if got != len(tt.wantLens) { + t.Fatalf("got len %d want: %d", got, len(tt.wantLens)) + } + for i := 0; i < got; i++ { + if msgs[i].Addr != addr { + t.Errorf("msgs[%d].Addr != passed addr", i) + } + gotLen := len(msgs[i].Buffers[0]) + if gotLen != tt.wantLens[i] { + t.Errorf("len(msgs[%d].Buffers[0]) %d != %d", i, gotLen, tt.wantLens[i]) + } + gotGSO, err := mockGetGSOSize(msgs[i].OOB) + if err != nil { + t.Fatalf("msgs[%d] getGSOSize err: %v", i, err) + } + if gotGSO != tt.wantGSO[i] { + t.Errorf("msgs[%d] gsoSize %d != %d", i, gotGSO, tt.wantGSO[i]) + } + } + }) + } +} + +func mockGetGSOSize(control []byte) (int, error) { + if len(control) < 2 { + return 0, nil + } + return int(binary.LittleEndian.Uint16(control)), nil +} + +func Test_splitCoalescedMessages(t *testing.T) { + newMsg := func(n, gso int) ipv6.Message { + msg := ipv6.Message{ + Buffers: [][]byte{make([]byte, 1<<16-1)}, + N: n, + OOB: make([]byte, 2), + } + binary.LittleEndian.PutUint16(msg.OOB, uint16(gso)) + if gso > 0 { + msg.NN = 2 + } + return msg + } + + cases := []struct { + name string + msgs []ipv6.Message + firstMsgAt int + wantNumEval int + wantMsgLens []int + wantErr bool + }{ + { + name: "second last split last empty", + msgs: []ipv6.Message{ + newMsg(0, 0), + newMsg(0, 0), + newMsg(3, 1), + newMsg(0, 0), + }, + firstMsgAt: 2, + wantNumEval: 3, + wantMsgLens: []int{1, 1, 1, 0}, + wantErr: false, + }, + { + name: "second last no split last empty", + msgs: []ipv6.Message{ + newMsg(0, 0), + newMsg(0, 0), + newMsg(1, 0), + newMsg(0, 0), + }, + firstMsgAt: 2, + wantNumEval: 1, + wantMsgLens: []int{1, 0, 0, 0}, + wantErr: false, + }, + { + name: "second last no split last no split", + msgs: []ipv6.Message{ + newMsg(0, 0), + newMsg(0, 0), + newMsg(1, 0), + newMsg(1, 0), + }, + firstMsgAt: 2, + wantNumEval: 2, + wantMsgLens: []int{1, 1, 0, 0}, + wantErr: false, + }, + { + name: "second last no split last split", + msgs: []ipv6.Message{ + newMsg(0, 0), + newMsg(0, 0), + newMsg(1, 0), + newMsg(3, 1), + }, + firstMsgAt: 2, + wantNumEval: 4, + wantMsgLens: []int{1, 1, 1, 1}, + wantErr: false, + }, + { + name: "second last split last split", + msgs: []ipv6.Message{ + newMsg(0, 0), + newMsg(0, 0), + newMsg(2, 1), + newMsg(2, 1), + }, + firstMsgAt: 2, + wantNumEval: 4, + wantMsgLens: []int{1, 1, 1, 1}, + wantErr: false, + }, + { + name: "second last no split last split overflow", + msgs: []ipv6.Message{ + newMsg(0, 0), + newMsg(0, 0), + newMsg(1, 0), + newMsg(4, 1), + }, + firstMsgAt: 2, + wantNumEval: 4, + wantMsgLens: []int{1, 1, 1, 1}, + wantErr: true, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + got, err := splitCoalescedMessages(tt.msgs, 2, mockGetGSOSize) + if err != nil && !tt.wantErr { + t.Fatalf("err: %v", err) + } + if got != tt.wantNumEval { + t.Fatalf("got to eval: %d want: %d", got, tt.wantNumEval) + } + for i, msg := range tt.msgs { + if msg.N != tt.wantMsgLens[i] { + t.Fatalf("msg[%d].N: %d want: %d", i, msg.N, tt.wantMsgLens[i]) + } + } + }) + } +} diff --git a/conn/sticky_default.go b/conn/control_default.go similarity index 54% rename from conn/sticky_default.go rename to conn/control_default.go index 1fa8a0c..a8bc06a 100644 --- a/conn/sticky_default.go +++ b/conn/control_default.go @@ -1,4 +1,4 @@ -//go:build !linux || android +//go:build !(linux && !android) /* SPDX-License-Identifier: MIT * @@ -21,8 +21,9 @@ func (e *StdNetEndpoint) SrcToString() string { return "" } -// TODO: macOS, FreeBSD and other BSDs likely do support this feature set, but -// use alternatively named flags and need ports and require testing. +// TODO: macOS, FreeBSD and other BSDs likely do support the sticky sockets +// ({get,set}srcControl feature set, but use alternatively named flags and need +// ports and require testing. // getSrcFromControl parses the control for PKTINFO and if found updates ep with // the source information found. @@ -34,8 +35,17 @@ func getSrcFromControl(control []byte, ep *StdNetEndpoint) { func setSrcControl(control *[]byte, ep *StdNetEndpoint) { } -// srcControlSize returns the recommended buffer size for pooling sticky control -// data. -const srcControlSize = 0 +// getGSOSize parses control for UDP_GRO and if found returns its GSO size data. +func getGSOSize(control []byte) (int, error) { + return 0, nil +} + +// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. +func setGSOSize(control *[]byte, gsoSize uint16) { +} + +// controlSize returns the recommended buffer size for pooling sticky and UDP +// offloading control data. +const controlSize = 0 const StdNetSupportsStickySockets = false diff --git a/conn/sticky_linux.go b/conn/control_linux.go similarity index 65% rename from conn/sticky_linux.go rename to conn/control_linux.go index a30ccc7..f32f26a 100644 --- a/conn/sticky_linux.go +++ b/conn/control_linux.go @@ -8,6 +8,7 @@ package conn import ( + "fmt" "net/netip" "unsafe" @@ -105,6 +106,54 @@ func setSrcControl(control *[]byte, ep *StdNetEndpoint) { *control = append(*control, ep.src...) } -var srcControlSize = unix.CmsgSpace(unix.SizeofInet6Pktinfo) +const ( + sizeOfGSOData = 2 +) + +// getGSOSize parses control for UDP_GRO and if found returns its GSO size data. +func getGSOSize(control []byte) (int, error) { + var ( + hdr unix.Cmsghdr + data []byte + rem = control + err error + ) + + for len(rem) > unix.SizeofCmsghdr { + hdr, data, rem, err = unix.ParseOneSocketControlMessage(rem) + if err != nil { + return 0, fmt.Errorf("error parsing socket control message: %w", err) + } + if hdr.Level == socketOptionLevelUDP && hdr.Type == socketOptionUDPGRO && len(data) >= sizeOfGSOData { + var gso uint16 + copy(unsafe.Slice((*byte)(unsafe.Pointer(&gso)), sizeOfGSOData), data[:sizeOfGSOData]) + return int(gso), nil + } + } + return 0, nil +} + +// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. It leaves existing +// data in control untouched. +func setGSOSize(control *[]byte, gsoSize uint16) { + existingLen := len(*control) + avail := cap(*control) - existingLen + space := unix.CmsgSpace(sizeOfGSOData) + if avail < space { + return + } + *control = (*control)[:cap(*control)] + gsoControl := (*control)[existingLen:] + hdr := (*unix.Cmsghdr)(unsafe.Pointer(&(gsoControl)[0])) + hdr.Level = socketOptionLevelUDP + hdr.Type = socketOptionUDPSegment + hdr.SetLen(unix.CmsgLen(sizeOfGSOData)) + copy((gsoControl)[unix.SizeofCmsghdr:], unsafe.Slice((*byte)(unsafe.Pointer(&gsoSize)), sizeOfGSOData)) + *control = (*control)[:existingLen+space] +} + +// controlSize returns the recommended buffer size for pooling sticky and UDP +// offloading control data. +var controlSize = unix.CmsgSpace(unix.SizeofInet6Pktinfo) + unix.CmsgSpace(sizeOfGSOData) const StdNetSupportsStickySockets = true diff --git a/conn/sticky_linux_test.go b/conn/control_linux_test.go similarity index 98% rename from conn/sticky_linux_test.go rename to conn/control_linux_test.go index 679213a..3ca7d37 100644 --- a/conn/sticky_linux_test.go +++ b/conn/control_linux_test.go @@ -60,7 +60,7 @@ func Test_setSrcControl(t *testing.T) { } setSrc(ep, netip.MustParseAddr("127.0.0.1"), 5) - control := make([]byte, srcControlSize) + control := make([]byte, controlSize) setSrcControl(&control, ep) @@ -89,7 +89,7 @@ func Test_setSrcControl(t *testing.T) { } setSrc(ep, netip.MustParseAddr("::1"), 5) - control := make([]byte, srcControlSize) + control := make([]byte, controlSize) setSrcControl(&control, ep) diff --git a/conn/controlfns_linux.go b/conn/controlfns_linux.go index a2396fe..752fbca 100644 --- a/conn/controlfns_linux.go +++ b/conn/controlfns_linux.go @@ -57,5 +57,13 @@ func init() { } return err }, + + // Attempt to enable UDP_GRO + func(network, address string, c syscall.RawConn) error { + c.Control(func(fd uintptr) { + _ = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO, 1) + }) + return nil + }, ) } diff --git a/conn/errors_default.go b/conn/errors_default.go new file mode 100644 index 0000000..f1e5b90 --- /dev/null +++ b/conn/errors_default.go @@ -0,0 +1,12 @@ +//go:build !linux + +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + */ + +package conn + +func errShouldDisableUDPGSO(err error) bool { + return false +} diff --git a/conn/errors_linux.go b/conn/errors_linux.go new file mode 100644 index 0000000..8e61000 --- /dev/null +++ b/conn/errors_linux.go @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + */ + +package conn + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func errShouldDisableUDPGSO(err error) bool { + var serr *os.SyscallError + if errors.As(err, &serr) { + // EIO is returned by udp_send_skb() if the device driver does not have + // tx checksumming enabled, which is a hard requirement of UDP_SEGMENT. + // See: + // https://git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man7/udp.7?id=806eabd74910447f21005160e90957bde4db0183#n228 + // https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/ipv4/udp.c?h=v6.2&id=c9c3395d5e3dcc6daee66c6908354d47bf98cb0c#n942 + return serr.Err == unix.EIO + } + return false +} diff --git a/conn/features_default.go b/conn/features_default.go new file mode 100644 index 0000000..d53ff5f --- /dev/null +++ b/conn/features_default.go @@ -0,0 +1,15 @@ +//go:build !linux +// +build !linux + +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + */ + +package conn + +import "net" + +func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) { + return +} diff --git a/conn/features_linux.go b/conn/features_linux.go new file mode 100644 index 0000000..513202e --- /dev/null +++ b/conn/features_linux.go @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + */ + +package conn + +import ( + "net" + + "golang.org/x/sys/unix" +) + +const ( + // TODO: upstream to x/sys/unix + socketOptionLevelUDP = 17 + socketOptionUDPSegment = 103 + socketOptionUDPGRO = 104 +) + +func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) { + rc, err := conn.SyscallConn() + if err != nil { + return + } + err = rc.Control(func(fd uintptr) { + _, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPSegment) + if errSyscall != nil { + return + } + txOffload = true + opt, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO) + if errSyscall != nil { + return + } + rxOffload = opt == 1 + }) + if err != nil { + return false, false + } + return txOffload, rxOffload +} diff --git a/device/send.go b/device/send.go index a95a46f..ea34997 100644 --- a/device/send.go +++ b/device/send.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "github.com/tailscale/wireguard-go/conn" "github.com/tailscale/wireguard-go/tun" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/net/ipv4" @@ -525,6 +526,13 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) { device.PutOutboundElement(elem) } device.PutOutboundElementsSlice(elems) + if err != nil { + var errGSO conn.ErrUDPGSODisabled + if errors.As(err, &errGSO) { + device.log.Verbosef(err.Error()) + err = errGSO.RetryErr + } + } if err != nil { device.log.Errorf("%v - Failed to send data packets: %v", peer, err) continue From d831fef379ddf6876304385621dd61995e7e32f5 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 4 Apr 2023 13:06:08 -0700 Subject: [PATCH 04/60] device: distribute crypto work as slice of elements After reducing UDP stack traversal overhead via GSO and GRO, runtime.chanrecv() began to account for a high percentage (20% in one environment) of perf samples during a throughput benchmark. The individual packet channel ops with the crypto goroutines was the primary contributor to this overhead. Updating these channels to pass vectors, which the device package already handles at its ends, reduced this overhead substantially, and improved throughput. The iperf3 results below demonstrate the effect of this commit between two Linux computers with i5-12400 CPUs. There is roughly ~13us of round trip latency between them. The first result is with UDP GSO and GRO, and with single element channels. Starting Test: protocol: TCP, 1 streams, 131072 byte blocks [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-10.00 sec 12.3 GBytes 10.6 Gbits/sec 232 3.15 MBytes - - - - - - - - - - - - - - - - - - - - - - - - - Test Complete. Summary Results: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 12.3 GBytes 10.6 Gbits/sec 232 sender [ 5] 0.00-10.04 sec 12.3 GBytes 10.6 Gbits/sec receiver The second result is with channels updated to pass a slice of elements. Starting Test: protocol: TCP, 1 streams, 131072 byte blocks [ ID] Interval Transfer Bitrate Retr Cwnd [ 5] 0.00-10.00 sec 13.2 GBytes 11.3 Gbits/sec 182 3.15 MBytes - - - - - - - - - - - - - - - - - - - - - - - - - Test Complete. Summary Results: [ ID] Interval Transfer Bitrate Retr [ 5] 0.00-10.00 sec 13.2 GBytes 11.3 Gbits/sec 182 sender [ 5] 0.00-10.04 sec 13.2 GBytes 11.3 Gbits/sec receiver Reviewed-by: Adrian Dewhurst Signed-off-by: Jordan Whited --- device/channels.go | 8 ++++---- device/receive.go | 42 ++++++++++++++++++++-------------------- device/send.go | 48 +++++++++++++++++++++++----------------------- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/device/channels.go b/device/channels.go index 039d8df..40ee5c9 100644 --- a/device/channels.go +++ b/device/channels.go @@ -19,13 +19,13 @@ import ( // call wg.Done to remove the initial reference. // When the refcount hits 0, the queue's channel is closed. type outboundQueue struct { - c chan *QueueOutboundElement + c chan *[]*QueueOutboundElement wg sync.WaitGroup } func newOutboundQueue() *outboundQueue { q := &outboundQueue{ - c: make(chan *QueueOutboundElement, QueueOutboundSize), + c: make(chan *[]*QueueOutboundElement, QueueOutboundSize), } q.wg.Add(1) go func() { @@ -37,13 +37,13 @@ func newOutboundQueue() *outboundQueue { // A inboundQueue is similar to an outboundQueue; see those docs. type inboundQueue struct { - c chan *QueueInboundElement + c chan *[]*QueueInboundElement wg sync.WaitGroup } func newInboundQueue() *inboundQueue { q := &inboundQueue{ - c: make(chan *QueueInboundElement, QueueInboundSize), + c: make(chan *[]*QueueInboundElement, QueueInboundSize), } q.wg.Add(1) go func() { diff --git a/device/receive.go b/device/receive.go index c0cf747..744bf18 100644 --- a/device/receive.go +++ b/device/receive.go @@ -220,9 +220,7 @@ func (device *Device) RoutineReceiveIncoming(maxBatchSize int, recv conn.Receive for peer, elems := range elemsByPeer { if peer.isRunning.Load() { peer.queue.inbound.c <- elems - for _, elem := range *elems { - device.queue.decryption.c <- elem - } + device.queue.decryption.c <- elems } else { for _, elem := range *elems { device.PutMessageBuffer(elem.buffer) @@ -241,26 +239,28 @@ func (device *Device) RoutineDecryption(id int) { defer device.log.Verbosef("Routine: decryption worker %d - stopped", id) device.log.Verbosef("Routine: decryption worker %d - started", id) - for elem := range device.queue.decryption.c { - // split message into fields - counter := elem.packet[MessageTransportOffsetCounter:MessageTransportOffsetContent] - content := elem.packet[MessageTransportOffsetContent:] + for elems := range device.queue.decryption.c { + for _, elem := range *elems { + // split message into fields + counter := elem.packet[MessageTransportOffsetCounter:MessageTransportOffsetContent] + content := elem.packet[MessageTransportOffsetContent:] - // decrypt and release to consumer - var err error - elem.counter = binary.LittleEndian.Uint64(counter) - // copy counter to nonce - binary.LittleEndian.PutUint64(nonce[0x4:0xc], elem.counter) - elem.packet, err = elem.keypair.receive.Open( - content[:0], - nonce[:], - content, - nil, - ) - if err != nil { - elem.packet = nil + // decrypt and release to consumer + var err error + elem.counter = binary.LittleEndian.Uint64(counter) + // copy counter to nonce + binary.LittleEndian.PutUint64(nonce[0x4:0xc], elem.counter) + elem.packet, err = elem.keypair.receive.Open( + content[:0], + nonce[:], + content, + nil, + ) + if err != nil { + elem.packet = nil + } + elem.Unlock() } - elem.Unlock() } } diff --git a/device/send.go b/device/send.go index ea34997..7adfacf 100644 --- a/device/send.go +++ b/device/send.go @@ -385,9 +385,7 @@ top: // add to parallel and sequential queue if peer.isRunning.Load() { peer.queue.outbound.c <- elems - for _, elem := range *elems { - peer.device.queue.encryption.c <- elem - } + peer.device.queue.encryption.c <- elems } else { for _, elem := range *elems { peer.device.PutMessageBuffer(elem.buffer) @@ -447,32 +445,34 @@ func (device *Device) RoutineEncryption(id int) { defer device.log.Verbosef("Routine: encryption worker %d - stopped", id) device.log.Verbosef("Routine: encryption worker %d - started", id) - for elem := range device.queue.encryption.c { - // populate header fields - header := elem.buffer[:MessageTransportHeaderSize] + for elems := range device.queue.encryption.c { + for _, elem := range *elems { + // populate header fields + header := elem.buffer[:MessageTransportHeaderSize] - fieldType := header[0:4] - fieldReceiver := header[4:8] - fieldNonce := header[8:16] + fieldType := header[0:4] + fieldReceiver := header[4:8] + fieldNonce := header[8:16] - binary.LittleEndian.PutUint32(fieldType, MessageTransportType) - binary.LittleEndian.PutUint32(fieldReceiver, elem.keypair.remoteIndex) - binary.LittleEndian.PutUint64(fieldNonce, elem.nonce) + binary.LittleEndian.PutUint32(fieldType, MessageTransportType) + binary.LittleEndian.PutUint32(fieldReceiver, elem.keypair.remoteIndex) + binary.LittleEndian.PutUint64(fieldNonce, elem.nonce) - // pad content to multiple of 16 - paddingSize := calculatePaddingSize(len(elem.packet), int(device.tun.mtu.Load())) - elem.packet = append(elem.packet, paddingZeros[:paddingSize]...) + // pad content to multiple of 16 + paddingSize := calculatePaddingSize(len(elem.packet), int(device.tun.mtu.Load())) + elem.packet = append(elem.packet, paddingZeros[:paddingSize]...) - // encrypt content and release to consumer + // encrypt content and release to consumer - binary.LittleEndian.PutUint64(nonce[4:], elem.nonce) - elem.packet = elem.keypair.send.Seal( - header, - nonce[:], - elem.packet, - nil, - ) - elem.Unlock() + binary.LittleEndian.PutUint64(nonce[4:], elem.nonce) + elem.packet = elem.keypair.send.Seal( + header, + nonce[:], + elem.packet, + nil, + ) + elem.Unlock() + } } } From 915962ded2318b41ddc7d2664bad3ffb12a8998f Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 4 Apr 2023 13:07:11 -0700 Subject: [PATCH 05/60] tun: unwind summing loop in checksumNoFold() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit $ benchstat old.txt new.txt goos: linux goarch: amd64 pkg: golang.zx2c4.com/wireguard/tun cpu: 12th Gen Intel(R) Core(TM) i5-12400 │ old.txt │ new.txt │ │ sec/op │ sec/op vs base │ Checksum/64-12 10.670n ± 2% 4.769n ± 0% -55.30% (p=0.000 n=10) Checksum/128-12 19.665n ± 2% 8.032n ± 0% -59.16% (p=0.000 n=10) Checksum/256-12 37.68n ± 1% 16.06n ± 0% -57.37% (p=0.000 n=10) Checksum/512-12 76.61n ± 3% 32.13n ± 0% -58.06% (p=0.000 n=10) Checksum/1024-12 160.55n ± 4% 64.25n ± 0% -59.98% (p=0.000 n=10) Checksum/1500-12 231.05n ± 7% 94.12n ± 0% -59.26% (p=0.000 n=10) Checksum/2048-12 309.5n ± 3% 128.5n ± 0% -58.48% (p=0.000 n=10) Checksum/4096-12 603.8n ± 4% 257.2n ± 0% -57.41% (p=0.000 n=10) Checksum/8192-12 1185.0n ± 3% 515.5n ± 0% -56.50% (p=0.000 n=10) Checksum/9000-12 1328.5n ± 5% 564.8n ± 0% -57.49% (p=0.000 n=10) Checksum/9001-12 1340.5n ± 3% 564.8n ± 0% -57.87% (p=0.000 n=10) geomean 185.3n 77.99n -57.92% Reviewed-by: Adrian Dewhurst Signed-off-by: Jordan Whited --- tun/checksum.go | 100 +++++++++++++++++++++++++++++++++++++------ tun/checksum_test.go | 35 +++++++++++++++ 2 files changed, 123 insertions(+), 12 deletions(-) create mode 100644 tun/checksum_test.go diff --git a/tun/checksum.go b/tun/checksum.go index f4f8471..29a8fc8 100644 --- a/tun/checksum.go +++ b/tun/checksum.go @@ -3,23 +3,99 @@ package tun import "encoding/binary" // TODO: Explore SIMD and/or other assembly optimizations. +// TODO: Test native endian loads. See RFC 1071 section 2 part B. func checksumNoFold(b []byte, initial uint64) uint64 { ac := initial - i := 0 - n := len(b) - for n >= 4 { - ac += uint64(binary.BigEndian.Uint32(b[i : i+4])) - n -= 4 - i += 4 + + for len(b) >= 128 { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + ac += uint64(binary.BigEndian.Uint32(b[8:12])) + ac += uint64(binary.BigEndian.Uint32(b[12:16])) + ac += uint64(binary.BigEndian.Uint32(b[16:20])) + ac += uint64(binary.BigEndian.Uint32(b[20:24])) + ac += uint64(binary.BigEndian.Uint32(b[24:28])) + ac += uint64(binary.BigEndian.Uint32(b[28:32])) + ac += uint64(binary.BigEndian.Uint32(b[32:36])) + ac += uint64(binary.BigEndian.Uint32(b[36:40])) + ac += uint64(binary.BigEndian.Uint32(b[40:44])) + ac += uint64(binary.BigEndian.Uint32(b[44:48])) + ac += uint64(binary.BigEndian.Uint32(b[48:52])) + ac += uint64(binary.BigEndian.Uint32(b[52:56])) + ac += uint64(binary.BigEndian.Uint32(b[56:60])) + ac += uint64(binary.BigEndian.Uint32(b[60:64])) + ac += uint64(binary.BigEndian.Uint32(b[64:68])) + ac += uint64(binary.BigEndian.Uint32(b[68:72])) + ac += uint64(binary.BigEndian.Uint32(b[72:76])) + ac += uint64(binary.BigEndian.Uint32(b[76:80])) + ac += uint64(binary.BigEndian.Uint32(b[80:84])) + ac += uint64(binary.BigEndian.Uint32(b[84:88])) + ac += uint64(binary.BigEndian.Uint32(b[88:92])) + ac += uint64(binary.BigEndian.Uint32(b[92:96])) + ac += uint64(binary.BigEndian.Uint32(b[96:100])) + ac += uint64(binary.BigEndian.Uint32(b[100:104])) + ac += uint64(binary.BigEndian.Uint32(b[104:108])) + ac += uint64(binary.BigEndian.Uint32(b[108:112])) + ac += uint64(binary.BigEndian.Uint32(b[112:116])) + ac += uint64(binary.BigEndian.Uint32(b[116:120])) + ac += uint64(binary.BigEndian.Uint32(b[120:124])) + ac += uint64(binary.BigEndian.Uint32(b[124:128])) + b = b[128:] } - for n >= 2 { - ac += uint64(binary.BigEndian.Uint16(b[i : i+2])) - n -= 2 - i += 2 + if len(b) >= 64 { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + ac += uint64(binary.BigEndian.Uint32(b[8:12])) + ac += uint64(binary.BigEndian.Uint32(b[12:16])) + ac += uint64(binary.BigEndian.Uint32(b[16:20])) + ac += uint64(binary.BigEndian.Uint32(b[20:24])) + ac += uint64(binary.BigEndian.Uint32(b[24:28])) + ac += uint64(binary.BigEndian.Uint32(b[28:32])) + ac += uint64(binary.BigEndian.Uint32(b[32:36])) + ac += uint64(binary.BigEndian.Uint32(b[36:40])) + ac += uint64(binary.BigEndian.Uint32(b[40:44])) + ac += uint64(binary.BigEndian.Uint32(b[44:48])) + ac += uint64(binary.BigEndian.Uint32(b[48:52])) + ac += uint64(binary.BigEndian.Uint32(b[52:56])) + ac += uint64(binary.BigEndian.Uint32(b[56:60])) + ac += uint64(binary.BigEndian.Uint32(b[60:64])) + b = b[64:] } - if n == 1 { - ac += uint64(b[i]) << 8 + if len(b) >= 32 { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + ac += uint64(binary.BigEndian.Uint32(b[8:12])) + ac += uint64(binary.BigEndian.Uint32(b[12:16])) + ac += uint64(binary.BigEndian.Uint32(b[16:20])) + ac += uint64(binary.BigEndian.Uint32(b[20:24])) + ac += uint64(binary.BigEndian.Uint32(b[24:28])) + ac += uint64(binary.BigEndian.Uint32(b[28:32])) + b = b[32:] } + if len(b) >= 16 { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + ac += uint64(binary.BigEndian.Uint32(b[8:12])) + ac += uint64(binary.BigEndian.Uint32(b[12:16])) + b = b[16:] + } + if len(b) >= 8 { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + b = b[8:] + } + if len(b) >= 4 { + ac += uint64(binary.BigEndian.Uint32(b)) + b = b[4:] + } + if len(b) >= 2 { + ac += uint64(binary.BigEndian.Uint16(b)) + b = b[2:] + } + if len(b) == 1 { + ac += uint64(b[0]) << 8 + } + return ac } diff --git a/tun/checksum_test.go b/tun/checksum_test.go new file mode 100644 index 0000000..c1ccff5 --- /dev/null +++ b/tun/checksum_test.go @@ -0,0 +1,35 @@ +package tun + +import ( + "fmt" + "math/rand" + "testing" +) + +func BenchmarkChecksum(b *testing.B) { + lengths := []int{ + 64, + 128, + 256, + 512, + 1024, + 1500, + 2048, + 4096, + 8192, + 9000, + 9001, + } + + for _, length := range lengths { + b.Run(fmt.Sprintf("%d", length), func(b *testing.B) { + buf := make([]byte, length) + rng := rand.New(rand.NewSource(1)) + rng.Read(buf) + b.ResetTimer() + for i := 0; i < b.N; i++ { + checksum(buf, 0) + } + }) + } +} From ceb9a09d035f373d6ca9617b5e25195c100017fb Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 4 Apr 2023 13:07:35 -0700 Subject: [PATCH 06/60] tun: reduce redundant checksumming in tcpGRO() IPv4 header and pseudo header checksums were being computed on every merge operation. Additionally, virtioNetHdr was being written at the same time. This delays those operations until after all coalescing has occurred. Reviewed-by: Adrian Dewhurst Signed-off-by: Jordan Whited --- tun/tcp_offload_linux.go | 162 ++++++++++++++++++++++++--------------- 1 file changed, 99 insertions(+), 63 deletions(-) diff --git a/tun/tcp_offload_linux.go b/tun/tcp_offload_linux.go index d64010d..6728823 100644 --- a/tun/tcp_offload_linux.go +++ b/tun/tcp_offload_linux.go @@ -269,11 +269,11 @@ func tcpChecksumValid(pkt []byte, iphLen uint8, isV6 bool) bool { type coalesceResult int const ( - coalesceInsufficientCap coalesceResult = 0 - coalescePSHEnding coalesceResult = 1 - coalesceItemInvalidCSum coalesceResult = 2 - coalescePktInvalidCSum coalesceResult = 3 - coalesceSuccess coalesceResult = 4 + coalesceInsufficientCap coalesceResult = iota + coalescePSHEnding + coalesceItemInvalidCSum + coalescePktInvalidCSum + coalesceSuccess ) // coalesceTCPPackets attempts to coalesce pkt with the packet described by @@ -339,42 +339,6 @@ func coalesceTCPPackets(mode canCoalesce, pkt []byte, pktBuffsIndex int, gsoSize if gsoSize > item.gsoSize { item.gsoSize = gsoSize } - hdr := virtioNetHdr{ - flags: unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, // this turns into CHECKSUM_PARTIAL in the skb - hdrLen: uint16(headersLen), - gsoSize: uint16(item.gsoSize), - csumStart: uint16(item.iphLen), - csumOffset: 16, - } - - // Recalculate the total len (IPv4) or payload len (IPv6). Recalculate the - // (IPv4) header checksum. - if isV6 { - hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV6 - binary.BigEndian.PutUint16(pktHead[4:], uint16(coalescedLen)-uint16(item.iphLen)) // set new payload len - } else { - hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4 - pktHead[10], pktHead[11] = 0, 0 // clear checksum field - binary.BigEndian.PutUint16(pktHead[2:], uint16(coalescedLen)) // set new total length - iphCSum := ^checksum(pktHead[:item.iphLen], 0) // compute checksum - binary.BigEndian.PutUint16(pktHead[10:], iphCSum) // set checksum field - } - hdr.encode(bufs[item.bufsIndex][bufsOffset-virtioNetHdrLen:]) - - // Calculate the pseudo header checksum and place it at the TCP checksum - // offset. Downstream checksum offloading will combine this with computation - // of the tcp header and payload checksum. - addrLen := 4 - addrOffset := ipv4SrcAddrOffset - if isV6 { - addrLen = 16 - addrOffset = ipv6SrcAddrOffset - } - srcAddrAt := bufsOffset + addrOffset - srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] - dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] - psum := pseudoHeaderChecksumNoFold(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(coalescedLen-int(item.iphLen))) - binary.BigEndian.PutUint16(pktHead[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) item.numMerged++ return coalesceSuccess @@ -390,43 +354,52 @@ const ( maxUint16 = 1<<16 - 1 ) +type tcpGROResult int + +const ( + tcpGROResultNoop tcpGROResult = iota + tcpGROResultTableInsert + tcpGROResultCoalesced +) + // tcpGRO evaluates the TCP packet at pktI in bufs for coalescing with -// existing packets tracked in table. It will return false when pktI is not -// coalesced, otherwise true. This indicates to the caller if bufs[pktI] -// should be written to the Device. -func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) (pktCoalesced bool) { +// existing packets tracked in table. It returns a tcpGROResultNoop when no +// action was taken, tcpGROResultTableInsert when the evaluated packet was +// inserted into table, and tcpGROResultCoalesced when the evaluated packet was +// coalesced with another packet in table. +func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) tcpGROResult { pkt := bufs[pktI][offset:] if len(pkt) > maxUint16 { // A valid IPv4 or IPv6 packet will never exceed this. - return false + return tcpGROResultNoop } iphLen := int((pkt[0] & 0x0F) * 4) if isV6 { iphLen = 40 ipv6HPayloadLen := int(binary.BigEndian.Uint16(pkt[4:])) if ipv6HPayloadLen != len(pkt)-iphLen { - return false + return tcpGROResultNoop } } else { totalLen := int(binary.BigEndian.Uint16(pkt[2:])) if totalLen != len(pkt) { - return false + return tcpGROResultNoop } } if len(pkt) < iphLen { - return false + return tcpGROResultNoop } tcphLen := int((pkt[iphLen+12] >> 4) * 4) if tcphLen < 20 || tcphLen > 60 { - return false + return tcpGROResultNoop } if len(pkt) < iphLen+tcphLen { - return false + return tcpGROResultNoop } if !isV6 { if pkt[6]&ipv4FlagMoreFragments != 0 || pkt[6]<<3 != 0 || pkt[7] != 0 { // no GRO support for fragmented segments for now - return false + return tcpGROResultNoop } } tcpFlags := pkt[iphLen+tcpFlagsOffset] @@ -434,14 +407,14 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) // not a candidate if any non-ACK flags (except PSH+ACK) are set if tcpFlags != tcpFlagACK { if pkt[iphLen+tcpFlagsOffset] != tcpFlagACK|tcpFlagPSH { - return false + return tcpGROResultNoop } pshSet = true } gsoSize := uint16(len(pkt) - tcphLen - iphLen) // not a candidate if payload len is 0 if gsoSize < 1 { - return false + return tcpGROResultNoop } seq := binary.BigEndian.Uint32(pkt[iphLen+4:]) srcAddrOffset := ipv4SrcAddrOffset @@ -452,7 +425,7 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) } items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, pktI) if !existing { - return false + return tcpGROResultNoop } for i := len(items) - 1; i >= 0; i-- { // In the best case of packets arriving in order iterating in reverse is @@ -470,20 +443,20 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) switch result { case coalesceSuccess: table.updateAt(item, i) - return true + return tcpGROResultCoalesced case coalesceItemInvalidCSum: // delete the item with an invalid csum table.deleteAt(item.key, i) case coalescePktInvalidCSum: // no point in inserting an item that we can't coalesce - return false + return tcpGROResultNoop default: } } } // failed to coalesce with any other packets; store the item in the flow table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, pktI) - return false + return tcpGROResultTableInsert } func isTCP4NoIPOptions(b []byte) bool { @@ -515,6 +488,64 @@ func isTCP6NoEH(b []byte) bool { return true } +// applyCoalesceAccounting updates bufs to account for coalescing based on the +// metadata found in table. +func applyCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable, isV6 bool) error { + for _, items := range table.itemsByFlow { + for _, item := range items { + if item.numMerged > 0 { + hdr := virtioNetHdr{ + flags: unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, // this turns into CHECKSUM_PARTIAL in the skb + hdrLen: uint16(item.iphLen + item.tcphLen), + gsoSize: item.gsoSize, + csumStart: uint16(item.iphLen), + csumOffset: 16, + } + pkt := bufs[item.bufsIndex][offset:] + + // Recalculate the total len (IPv4) or payload len (IPv6). + // Recalculate the (IPv4) header checksum. + if isV6 { + hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV6 + binary.BigEndian.PutUint16(pkt[4:], uint16(len(pkt))-uint16(item.iphLen)) // set new IPv6 header payload len + } else { + hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4 + pkt[10], pkt[11] = 0, 0 + binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length + iphCSum := ^checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum + binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field + } + err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) + if err != nil { + return err + } + + // Calculate the pseudo header checksum and place it at the TCP + // checksum offset. Downstream checksum offloading will combine + // this with computation of the tcp header and payload checksum. + addrLen := 4 + addrOffset := ipv4SrcAddrOffset + if isV6 { + addrLen = 16 + addrOffset = ipv6SrcAddrOffset + } + srcAddrAt := offset + addrOffset + srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] + dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] + psum := pseudoHeaderChecksumNoFold(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) + binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) + } else { + hdr := virtioNetHdr{} + err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) + if err != nil { + return err + } + } + } + } + return nil +} + // handleGRO evaluates bufs for GRO, and writes the indices of the resulting // packets into toWrite. toWrite, tcp4Table, and tcp6Table should initially be // empty (but non-nil), and are passed in to save allocs as the caller may reset @@ -524,23 +555,28 @@ func handleGRO(bufs [][]byte, offset int, tcp4Table, tcp6Table *tcpGROTable, toW if offset < virtioNetHdrLen || offset > len(bufs[i])-1 { return errors.New("invalid offset") } - var coalesced bool + var result tcpGROResult switch { case isTCP4NoIPOptions(bufs[i][offset:]): // ipv4 packets w/IP options do not coalesce - coalesced = tcpGRO(bufs, offset, i, tcp4Table, false) + result = tcpGRO(bufs, offset, i, tcp4Table, false) case isTCP6NoEH(bufs[i][offset:]): // ipv6 packets w/extension headers do not coalesce - coalesced = tcpGRO(bufs, offset, i, tcp6Table, true) + result = tcpGRO(bufs, offset, i, tcp6Table, true) } - if !coalesced { + switch result { + case tcpGROResultNoop: hdr := virtioNetHdr{} err := hdr.encode(bufs[i][offset-virtioNetHdrLen:]) if err != nil { return err } + fallthrough + case tcpGROResultTableInsert: *toWrite = append(*toWrite, i) } } - return nil + err4 := applyCoalesceAccounting(bufs, offset, tcp4Table, false) + err6 := applyCoalesceAccounting(bufs, offset, tcp6Table, true) + return errors.Join(err4, err6) } // tcpTSO splits packets from in into outBuffs, writing the size of each From cc7b29b8c60406713a50faf0175d6a0481a182f6 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 5 Apr 2023 15:40:40 -0700 Subject: [PATCH 07/60] device: move Queue{In,Out}boundElement Mutex to container type Queue{In,Out}boundElement locking can contribute to significant overhead via sync.Mutex.lockSlow() in some environments. These types are passed throughout the device package as elements in a slice, so move the per-element Mutex to a container around the slice. Signed-off-by: Jordan Whited --- device/channels.go | 32 +++++++-------- device/device.go | 10 ++--- device/peer.go | 8 ++-- device/pools.go | 44 +++++++++++---------- device/receive.go | 43 ++++++++++---------- device/send.go | 99 ++++++++++++++++++++++++---------------------- 6 files changed, 123 insertions(+), 113 deletions(-) diff --git a/device/channels.go b/device/channels.go index 40ee5c9..e526f6b 100644 --- a/device/channels.go +++ b/device/channels.go @@ -19,13 +19,13 @@ import ( // call wg.Done to remove the initial reference. // When the refcount hits 0, the queue's channel is closed. type outboundQueue struct { - c chan *[]*QueueOutboundElement + c chan *QueueOutboundElementsContainer wg sync.WaitGroup } func newOutboundQueue() *outboundQueue { q := &outboundQueue{ - c: make(chan *[]*QueueOutboundElement, QueueOutboundSize), + c: make(chan *QueueOutboundElementsContainer, QueueOutboundSize), } q.wg.Add(1) go func() { @@ -37,13 +37,13 @@ func newOutboundQueue() *outboundQueue { // A inboundQueue is similar to an outboundQueue; see those docs. type inboundQueue struct { - c chan *[]*QueueInboundElement + c chan *QueueInboundElementsContainer wg sync.WaitGroup } func newInboundQueue() *inboundQueue { q := &inboundQueue{ - c: make(chan *[]*QueueInboundElement, QueueInboundSize), + c: make(chan *QueueInboundElementsContainer, QueueInboundSize), } q.wg.Add(1) go func() { @@ -72,7 +72,7 @@ func newHandshakeQueue() *handshakeQueue { } type autodrainingInboundQueue struct { - c chan *[]*QueueInboundElement + c chan *QueueInboundElementsContainer } // newAutodrainingInboundQueue returns a channel that will be drained when it gets GC'd. @@ -81,7 +81,7 @@ type autodrainingInboundQueue struct { // some other means, such as sending a sentinel nil values. func newAutodrainingInboundQueue(device *Device) *autodrainingInboundQueue { q := &autodrainingInboundQueue{ - c: make(chan *[]*QueueInboundElement, QueueInboundSize), + c: make(chan *QueueInboundElementsContainer, QueueInboundSize), } runtime.SetFinalizer(q, device.flushInboundQueue) return q @@ -90,13 +90,13 @@ func newAutodrainingInboundQueue(device *Device) *autodrainingInboundQueue { func (device *Device) flushInboundQueue(q *autodrainingInboundQueue) { for { select { - case elems := <-q.c: - for _, elem := range *elems { - elem.Lock() + case elemsContainer := <-q.c: + elemsContainer.Lock() + for _, elem := range elemsContainer.elems { device.PutMessageBuffer(elem.buffer) device.PutInboundElement(elem) } - device.PutInboundElementsSlice(elems) + device.PutInboundElementsContainer(elemsContainer) default: return } @@ -104,7 +104,7 @@ func (device *Device) flushInboundQueue(q *autodrainingInboundQueue) { } type autodrainingOutboundQueue struct { - c chan *[]*QueueOutboundElement + c chan *QueueOutboundElementsContainer } // newAutodrainingOutboundQueue returns a channel that will be drained when it gets GC'd. @@ -114,7 +114,7 @@ type autodrainingOutboundQueue struct { // All sends to the channel must be best-effort, because there may be no receivers. func newAutodrainingOutboundQueue(device *Device) *autodrainingOutboundQueue { q := &autodrainingOutboundQueue{ - c: make(chan *[]*QueueOutboundElement, QueueOutboundSize), + c: make(chan *QueueOutboundElementsContainer, QueueOutboundSize), } runtime.SetFinalizer(q, device.flushOutboundQueue) return q @@ -123,13 +123,13 @@ func newAutodrainingOutboundQueue(device *Device) *autodrainingOutboundQueue { func (device *Device) flushOutboundQueue(q *autodrainingOutboundQueue) { for { select { - case elems := <-q.c: - for _, elem := range *elems { - elem.Lock() + case elemsContainer := <-q.c: + elemsContainer.Lock() + for _, elem := range elemsContainer.elems { device.PutMessageBuffer(elem.buffer) device.PutOutboundElement(elem) } - device.PutOutboundElementsSlice(elems) + device.PutOutboundElementsContainer(elemsContainer) default: return } diff --git a/device/device.go b/device/device.go index 7482d9b..5c666ac 100644 --- a/device/device.go +++ b/device/device.go @@ -68,11 +68,11 @@ type Device struct { cookieChecker CookieChecker pool struct { - outboundElementsSlice *WaitPool - inboundElementsSlice *WaitPool - messageBuffers *WaitPool - inboundElements *WaitPool - outboundElements *WaitPool + inboundElementsContainer *WaitPool + outboundElementsContainer *WaitPool + messageBuffers *WaitPool + inboundElements *WaitPool + outboundElements *WaitPool } queue struct { diff --git a/device/peer.go b/device/peer.go index c7163ac..22757d4 100644 --- a/device/peer.go +++ b/device/peer.go @@ -45,9 +45,9 @@ type Peer struct { } queue struct { - staged chan *[]*QueueOutboundElement // staged packets before a handshake is available - outbound *autodrainingOutboundQueue // sequential ordering of udp transmission - inbound *autodrainingInboundQueue // sequential ordering of tun writing + staged chan *QueueOutboundElementsContainer // staged packets before a handshake is available + outbound *autodrainingOutboundQueue // sequential ordering of udp transmission + inbound *autodrainingInboundQueue // sequential ordering of tun writing } cookieGenerator CookieGenerator @@ -81,7 +81,7 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) { peer.device = device peer.queue.outbound = newAutodrainingOutboundQueue(device) peer.queue.inbound = newAutodrainingInboundQueue(device) - peer.queue.staged = make(chan *[]*QueueOutboundElement, QueueStagedSize) + peer.queue.staged = make(chan *QueueOutboundElementsContainer, QueueStagedSize) // map public key _, ok := device.peers.keyMap[pk] diff --git a/device/pools.go b/device/pools.go index 02a5d6a..94f3dc7 100644 --- a/device/pools.go +++ b/device/pools.go @@ -46,13 +46,13 @@ func (p *WaitPool) Put(x any) { } func (device *Device) PopulatePools() { - device.pool.outboundElementsSlice = NewWaitPool(PreallocatedBuffersPerPool, func() any { - s := make([]*QueueOutboundElement, 0, device.BatchSize()) - return &s - }) - device.pool.inboundElementsSlice = NewWaitPool(PreallocatedBuffersPerPool, func() any { + device.pool.inboundElementsContainer = NewWaitPool(PreallocatedBuffersPerPool, func() any { s := make([]*QueueInboundElement, 0, device.BatchSize()) - return &s + return &QueueInboundElementsContainer{elems: s} + }) + device.pool.outboundElementsContainer = NewWaitPool(PreallocatedBuffersPerPool, func() any { + s := make([]*QueueOutboundElement, 0, device.BatchSize()) + return &QueueOutboundElementsContainer{elems: s} }) device.pool.messageBuffers = NewWaitPool(PreallocatedBuffersPerPool, func() any { return new([MaxMessageSize]byte) @@ -65,28 +65,32 @@ func (device *Device) PopulatePools() { }) } -func (device *Device) GetOutboundElementsSlice() *[]*QueueOutboundElement { - return device.pool.outboundElementsSlice.Get().(*[]*QueueOutboundElement) +func (device *Device) GetInboundElementsContainer() *QueueInboundElementsContainer { + c := device.pool.inboundElementsContainer.Get().(*QueueInboundElementsContainer) + c.Mutex = sync.Mutex{} + return c } -func (device *Device) PutOutboundElementsSlice(s *[]*QueueOutboundElement) { - for i := range *s { - (*s)[i] = nil +func (device *Device) PutInboundElementsContainer(c *QueueInboundElementsContainer) { + for i := range c.elems { + c.elems[i] = nil } - *s = (*s)[:0] - device.pool.outboundElementsSlice.Put(s) + c.elems = c.elems[:0] + device.pool.inboundElementsContainer.Put(c) } -func (device *Device) GetInboundElementsSlice() *[]*QueueInboundElement { - return device.pool.inboundElementsSlice.Get().(*[]*QueueInboundElement) +func (device *Device) GetOutboundElementsContainer() *QueueOutboundElementsContainer { + c := device.pool.outboundElementsContainer.Get().(*QueueOutboundElementsContainer) + c.Mutex = sync.Mutex{} + return c } -func (device *Device) PutInboundElementsSlice(s *[]*QueueInboundElement) { - for i := range *s { - (*s)[i] = nil +func (device *Device) PutOutboundElementsContainer(c *QueueOutboundElementsContainer) { + for i := range c.elems { + c.elems[i] = nil } - *s = (*s)[:0] - device.pool.inboundElementsSlice.Put(s) + c.elems = c.elems[:0] + device.pool.outboundElementsContainer.Put(c) } func (device *Device) GetMessageBuffer() *[MaxMessageSize]byte { diff --git a/device/receive.go b/device/receive.go index 744bf18..da663e9 100644 --- a/device/receive.go +++ b/device/receive.go @@ -27,7 +27,6 @@ type QueueHandshakeElement struct { } type QueueInboundElement struct { - sync.Mutex buffer *[MaxMessageSize]byte packet []byte counter uint64 @@ -35,6 +34,11 @@ type QueueInboundElement struct { endpoint conn.Endpoint } +type QueueInboundElementsContainer struct { + sync.Mutex + elems []*QueueInboundElement +} + // clearPointers clears elem fields that contain pointers. // This makes the garbage collector's life easier and // avoids accidentally keeping other objects around unnecessarily. @@ -87,7 +91,7 @@ func (device *Device) RoutineReceiveIncoming(maxBatchSize int, recv conn.Receive count int endpoints = make([]conn.Endpoint, maxBatchSize) deathSpiral int - elemsByPeer = make(map[*Peer]*[]*QueueInboundElement, maxBatchSize) + elemsByPeer = make(map[*Peer]*QueueInboundElementsContainer, maxBatchSize) ) for i := range bufsArrs { @@ -170,15 +174,14 @@ func (device *Device) RoutineReceiveIncoming(maxBatchSize int, recv conn.Receive elem.keypair = keypair elem.endpoint = endpoints[i] elem.counter = 0 - elem.Mutex = sync.Mutex{} - elem.Lock() elemsForPeer, ok := elemsByPeer[peer] if !ok { - elemsForPeer = device.GetInboundElementsSlice() + elemsForPeer = device.GetInboundElementsContainer() + elemsForPeer.Lock() elemsByPeer[peer] = elemsForPeer } - *elemsForPeer = append(*elemsForPeer, elem) + elemsForPeer.elems = append(elemsForPeer.elems, elem) bufsArrs[i] = device.GetMessageBuffer() bufs[i] = bufsArrs[i][:] continue @@ -217,16 +220,16 @@ func (device *Device) RoutineReceiveIncoming(maxBatchSize int, recv conn.Receive default: } } - for peer, elems := range elemsByPeer { + for peer, elemsContainer := range elemsByPeer { if peer.isRunning.Load() { - peer.queue.inbound.c <- elems - device.queue.decryption.c <- elems + peer.queue.inbound.c <- elemsContainer + device.queue.decryption.c <- elemsContainer } else { - for _, elem := range *elems { + for _, elem := range elemsContainer.elems { device.PutMessageBuffer(elem.buffer) device.PutInboundElement(elem) } - device.PutInboundElementsSlice(elems) + device.PutInboundElementsContainer(elemsContainer) } delete(elemsByPeer, peer) } @@ -239,8 +242,8 @@ func (device *Device) RoutineDecryption(id int) { defer device.log.Verbosef("Routine: decryption worker %d - stopped", id) device.log.Verbosef("Routine: decryption worker %d - started", id) - for elems := range device.queue.decryption.c { - for _, elem := range *elems { + for elemsContainer := range device.queue.decryption.c { + for _, elem := range elemsContainer.elems { // split message into fields counter := elem.packet[MessageTransportOffsetCounter:MessageTransportOffsetContent] content := elem.packet[MessageTransportOffsetContent:] @@ -259,8 +262,8 @@ func (device *Device) RoutineDecryption(id int) { if err != nil { elem.packet = nil } - elem.Unlock() } + elemsContainer.Unlock() } } @@ -437,12 +440,12 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { bufs := make([][]byte, 0, maxBatchSize) - for elems := range peer.queue.inbound.c { - if elems == nil { + for elemsContainer := range peer.queue.inbound.c { + if elemsContainer == nil { return } - for _, elem := range *elems { - elem.Lock() + elemsContainer.Lock() + for _, elem := range elemsContainer.elems { if elem.packet == nil { // decryption failed continue @@ -515,11 +518,11 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { device.log.Errorf("Failed to write packets to TUN device: %v", err) } } - for _, elem := range *elems { + for _, elem := range elemsContainer.elems { device.PutMessageBuffer(elem.buffer) device.PutInboundElement(elem) } bufs = bufs[:0] - device.PutInboundElementsSlice(elems) + device.PutInboundElementsContainer(elemsContainer) } } diff --git a/device/send.go b/device/send.go index 7adfacf..95a5cbe 100644 --- a/device/send.go +++ b/device/send.go @@ -46,7 +46,6 @@ import ( */ type QueueOutboundElement struct { - sync.Mutex buffer *[MaxMessageSize]byte // slice holding the packet data packet []byte // slice of "buffer" (always!) nonce uint64 // nonce for encryption @@ -54,10 +53,14 @@ type QueueOutboundElement struct { peer *Peer // related peer } +type QueueOutboundElementsContainer struct { + sync.Mutex + elems []*QueueOutboundElement +} + func (device *Device) NewOutboundElement() *QueueOutboundElement { elem := device.GetOutboundElement() elem.buffer = device.GetMessageBuffer() - elem.Mutex = sync.Mutex{} elem.nonce = 0 // keypair and peer were cleared (if necessary) by clearPointers. return elem @@ -79,15 +82,15 @@ func (elem *QueueOutboundElement) clearPointers() { func (peer *Peer) SendKeepalive() { if len(peer.queue.staged) == 0 && peer.isRunning.Load() { elem := peer.device.NewOutboundElement() - elems := peer.device.GetOutboundElementsSlice() - *elems = append(*elems, elem) + elemsContainer := peer.device.GetOutboundElementsContainer() + elemsContainer.elems = append(elemsContainer.elems, elem) select { - case peer.queue.staged <- elems: + case peer.queue.staged <- elemsContainer: peer.device.log.Verbosef("%v - Sending keepalive packet", peer) default: peer.device.PutMessageBuffer(elem.buffer) peer.device.PutOutboundElement(elem) - peer.device.PutOutboundElementsSlice(elems) + peer.device.PutOutboundElementsContainer(elemsContainer) } } peer.SendStagedPackets() @@ -219,7 +222,7 @@ func (device *Device) RoutineReadFromTUN() { readErr error elems = make([]*QueueOutboundElement, batchSize) bufs = make([][]byte, batchSize) - elemsByPeer = make(map[*Peer]*[]*QueueOutboundElement, batchSize) + elemsByPeer = make(map[*Peer]*QueueOutboundElementsContainer, batchSize) count = 0 sizes = make([]int, batchSize) offset = MessageTransportHeaderSize @@ -276,10 +279,10 @@ func (device *Device) RoutineReadFromTUN() { } elemsForPeer, ok := elemsByPeer[peer] if !ok { - elemsForPeer = device.GetOutboundElementsSlice() + elemsForPeer = device.GetOutboundElementsContainer() elemsByPeer[peer] = elemsForPeer } - *elemsForPeer = append(*elemsForPeer, elem) + elemsForPeer.elems = append(elemsForPeer.elems, elem) elems[i] = device.NewOutboundElement() bufs[i] = elems[i].buffer[:] } @@ -289,11 +292,11 @@ func (device *Device) RoutineReadFromTUN() { peer.StagePackets(elemsForPeer) peer.SendStagedPackets() } else { - for _, elem := range *elemsForPeer { + for _, elem := range elemsForPeer.elems { device.PutMessageBuffer(elem.buffer) device.PutOutboundElement(elem) } - device.PutOutboundElementsSlice(elemsForPeer) + device.PutOutboundElementsContainer(elemsForPeer) } delete(elemsByPeer, peer) } @@ -317,7 +320,7 @@ func (device *Device) RoutineReadFromTUN() { } } -func (peer *Peer) StagePackets(elems *[]*QueueOutboundElement) { +func (peer *Peer) StagePackets(elems *QueueOutboundElementsContainer) { for { select { case peer.queue.staged <- elems: @@ -326,11 +329,11 @@ func (peer *Peer) StagePackets(elems *[]*QueueOutboundElement) { } select { case tooOld := <-peer.queue.staged: - for _, elem := range *tooOld { + for _, elem := range tooOld.elems { peer.device.PutMessageBuffer(elem.buffer) peer.device.PutOutboundElement(elem) } - peer.device.PutOutboundElementsSlice(tooOld) + peer.device.PutOutboundElementsContainer(tooOld) default: } } @@ -349,52 +352,52 @@ top: } for { - var elemsOOO *[]*QueueOutboundElement + var elemsContainerOOO *QueueOutboundElementsContainer select { - case elems := <-peer.queue.staged: + case elemsContainer := <-peer.queue.staged: i := 0 - for _, elem := range *elems { + for _, elem := range elemsContainer.elems { elem.peer = peer elem.nonce = keypair.sendNonce.Add(1) - 1 if elem.nonce >= RejectAfterMessages { keypair.sendNonce.Store(RejectAfterMessages) - if elemsOOO == nil { - elemsOOO = peer.device.GetOutboundElementsSlice() + if elemsContainerOOO == nil { + elemsContainerOOO = peer.device.GetOutboundElementsContainer() } - *elemsOOO = append(*elemsOOO, elem) + elemsContainerOOO.elems = append(elemsContainerOOO.elems, elem) continue } else { - (*elems)[i] = elem + elemsContainer.elems[i] = elem i++ } elem.keypair = keypair - elem.Lock() } - *elems = (*elems)[:i] + elemsContainer.Lock() + elemsContainer.elems = elemsContainer.elems[:i] - if elemsOOO != nil { - peer.StagePackets(elemsOOO) // XXX: Out of order, but we can't front-load go chans + if elemsContainerOOO != nil { + peer.StagePackets(elemsContainerOOO) // XXX: Out of order, but we can't front-load go chans } - if len(*elems) == 0 { - peer.device.PutOutboundElementsSlice(elems) + if len(elemsContainer.elems) == 0 { + peer.device.PutOutboundElementsContainer(elemsContainer) goto top } // add to parallel and sequential queue if peer.isRunning.Load() { - peer.queue.outbound.c <- elems - peer.device.queue.encryption.c <- elems + peer.queue.outbound.c <- elemsContainer + peer.device.queue.encryption.c <- elemsContainer } else { - for _, elem := range *elems { + for _, elem := range elemsContainer.elems { peer.device.PutMessageBuffer(elem.buffer) peer.device.PutOutboundElement(elem) } - peer.device.PutOutboundElementsSlice(elems) + peer.device.PutOutboundElementsContainer(elemsContainer) } - if elemsOOO != nil { + if elemsContainerOOO != nil { goto top } default: @@ -406,12 +409,12 @@ top: func (peer *Peer) FlushStagedPackets() { for { select { - case elems := <-peer.queue.staged: - for _, elem := range *elems { + case elemsContainer := <-peer.queue.staged: + for _, elem := range elemsContainer.elems { peer.device.PutMessageBuffer(elem.buffer) peer.device.PutOutboundElement(elem) } - peer.device.PutOutboundElementsSlice(elems) + peer.device.PutOutboundElementsContainer(elemsContainer) default: return } @@ -433,7 +436,7 @@ func calculatePaddingSize(packetSize, mtu int) int { return paddedSize - lastUnit } -/* Encrypts the elements in the queue +/* Encrypts the elems in the queue * and marks them for sequential consumption (by releasing the mutex) * * Obs. One instance per core @@ -445,8 +448,8 @@ func (device *Device) RoutineEncryption(id int) { defer device.log.Verbosef("Routine: encryption worker %d - stopped", id) device.log.Verbosef("Routine: encryption worker %d - started", id) - for elems := range device.queue.encryption.c { - for _, elem := range *elems { + for elemsContainer := range device.queue.encryption.c { + for _, elem := range elemsContainer.elems { // populate header fields header := elem.buffer[:MessageTransportHeaderSize] @@ -471,8 +474,8 @@ func (device *Device) RoutineEncryption(id int) { elem.packet, nil, ) - elem.Unlock() } + elemsContainer.Unlock() } } @@ -486,28 +489,28 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) { bufs := make([][]byte, 0, maxBatchSize) - for elems := range peer.queue.outbound.c { + for elemsContainer := range peer.queue.outbound.c { bufs = bufs[:0] - if elems == nil { + if elemsContainer == nil { return } if !peer.isRunning.Load() { - // peer has been stopped; return re-usable elems to the shared pool. + // peer has been stopped; return re-usable elemsContainer to the shared pool. // This is an optimization only. It is possible for the peer to be stopped // immediately after this check, in which case, elem will get processed. // The timers and SendBuffers code are resilient to a few stragglers. // TODO: rework peer shutdown order to ensure // that we never accidentally keep timers alive longer than necessary. - for _, elem := range *elems { - elem.Lock() + elemsContainer.Lock() + for _, elem := range elemsContainer.elems { device.PutMessageBuffer(elem.buffer) device.PutOutboundElement(elem) } continue } dataSent := false - for _, elem := range *elems { - elem.Lock() + elemsContainer.Lock() + for _, elem := range elemsContainer.elems { if len(elem.packet) != MessageKeepaliveSize { dataSent = true } @@ -521,11 +524,11 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) { if dataSent { peer.timersDataSent() } - for _, elem := range *elems { + for _, elem := range elemsContainer.elems { device.PutMessageBuffer(elem.buffer) device.PutOutboundElement(elem) } - device.PutOutboundElementsSlice(elems) + device.PutOutboundElementsContainer(elemsContainer) if err != nil { var errGSO conn.ErrUDPGSODisabled if errors.As(err, &errGSO) { From ec6f23b33e03f304681b0a440f318636d002a380 Mon Sep 17 00:00:00 2001 From: Adrian Dewhurst Date: Thu, 18 May 2023 10:09:53 -0400 Subject: [PATCH 08/60] tun: checksum tests and benchmarks Signed-off-by: Adrian Dewhurst --- tun/checksum_generic_test.go | 9 + tun/checksum_test.go | 577 ++++++++++++++++++++++++++++++++++- 2 files changed, 579 insertions(+), 7 deletions(-) create mode 100644 tun/checksum_generic_test.go diff --git a/tun/checksum_generic_test.go b/tun/checksum_generic_test.go new file mode 100644 index 0000000..a0c9457 --- /dev/null +++ b/tun/checksum_generic_test.go @@ -0,0 +1,9 @@ +package tun + +var archChecksumFuncs = []archChecksumDetails{ + { + name: "generic", + available: true, + f: checksum, + }, +} diff --git a/tun/checksum_test.go b/tun/checksum_test.go index c1ccff5..b3a3583 100644 --- a/tun/checksum_test.go +++ b/tun/checksum_test.go @@ -2,33 +2,596 @@ package tun import ( "fmt" + "math" "math/rand" + "net/netip" + "sort" + "syscall" "testing" + "unsafe" + + "gvisor.dev/gvisor/pkg/tcpip" + gvisorChecksum "gvisor.dev/gvisor/pkg/tcpip/checksum" + "gvisor.dev/gvisor/pkg/tcpip/header" ) +type archChecksumDetails struct { + name string + available bool + f func([]byte, uint64) uint16 +} + +func deterministicRandomBytes(seed int64, length int) []byte { + rng := rand.New(rand.NewSource(seed)) + buf := make([]byte, length) + n, err := rng.Read(buf) + if err != nil { + panic(err) + } + if n != length { + panic("incomplete random buffer") + } + return buf +} + +func getPageAlignedRandomBytes(seed int64, length int) []byte { + alignment := syscall.Getpagesize() + buf := deterministicRandomBytes(seed, length+(alignment-1)) + bufPtr := uintptr(unsafe.Pointer(&buf[0])) + alignedBufPtr := (bufPtr + uintptr(alignment-1)) & ^uintptr(alignment-1) + alignedStart := int(alignedBufPtr - bufPtr) + return buf[alignedStart:] +} + +func TestChecksum(t *testing.T) { + alignedBuf := getPageAlignedRandomBytes(10, 8192) + allOnes := make([]byte, 65535) + for i := range allOnes { + allOnes[i] = 0xff + } + allFE := make([]byte, 65535) + for i := range allFE { + allFE[i] = 0xfe + } + + tests := []struct { + name string + data []byte + initial uint16 + want uint16 + }{ + { + name: "empty", + data: []byte{}, + initial: 0, + want: 0, + }, + { + name: "max initial", + data: []byte{}, + initial: math.MaxUint16, + want: 0xffff, + }, + { + name: "odd length", + data: []byte{0x01, 0x02, 0x01}, + initial: 0, + want: 0x0202, + }, + { + name: "tiny", + data: []byte{0x01, 0x02, 0x01, 0x02, 0x01, 0x02}, + initial: 0, + want: 0x0306, + }, + { + name: "initial", + data: []byte{0x01, 0x02, 0x01, 0x02, 0x01, 0x02}, + initial: 0x1000, + want: 0x1306, + }, + // cleanup0 through cleanup15 is 1024 (handled by large SIMD loops) + + // 32 (handled by small SIMD loops) + n, where n ranges from 0 to 15 + // to cover all of the leftover byte sizes that are possible after small + // SIMD loops that handle 16 bytes. + { + name: "cleanup0", + data: deterministicRandomBytes(1, 1056), + initial: 0, + want: 0x11ec, + }, + { + name: "cleanup1", + data: deterministicRandomBytes(1, 1057), + initial: 0, + want: 0xc5ec, + }, + { + name: "cleanup2", + data: deterministicRandomBytes(1, 1058), + initial: 0, + want: 0xc6ad, + }, + { + name: "cleanup3", + data: deterministicRandomBytes(1, 1059), + initial: 0, + want: 0x86ae, + }, + { + name: "cleanup4", + data: deterministicRandomBytes(1, 1060), + initial: 0, + want: 0x878e, + }, + { + name: "cleanup5", + data: deterministicRandomBytes(1, 1061), + initial: 0, + want: 0xdb8e, + }, + { + name: "cleanup6", + data: deterministicRandomBytes(1, 1062), + initial: 0, + want: 0xdbd5, + }, + { + name: "cleanup7", + data: deterministicRandomBytes(1, 1063), + initial: 0, + want: 0xcfd6, + }, + { + name: "cleanup8", + data: deterministicRandomBytes(1, 1064), + initial: 0, + want: 0xd090, + }, + { + name: "cleanup9", + data: deterministicRandomBytes(1, 1065), + initial: 0, + want: 0x0791, + }, + { + name: "cleanup10", + data: deterministicRandomBytes(1, 1066), + initial: 0, + want: 0x079f, + }, + { + name: "cleanup11", + data: deterministicRandomBytes(1, 1067), + initial: 0, + want: 0xba9f, + }, + { + name: "cleanup12", + data: deterministicRandomBytes(1, 1068), + initial: 0, + want: 0xbb0c, + }, + { + name: "cleanup13", + data: deterministicRandomBytes(1, 1069), + initial: 0, + want: 0x770d, + }, + { + name: "cleanup14", + data: deterministicRandomBytes(1, 1070), + initial: 0, + want: 0x780a, + }, + { + name: "cleanup15", + data: deterministicRandomBytes(1, 1071), + initial: 0, + want: 0x640b, + }, + // small1 through small15 covers small sizes that are not large enough + // to do overlapped reads. + { + name: "small1", + data: deterministicRandomBytes(2, 1), + initial: 0x1122, + want: 0x4022, + }, + { + name: "small2", + data: deterministicRandomBytes(2, 2), + initial: 0x1122, + want: 0x40a4, + }, + { + name: "small3", + data: deterministicRandomBytes(2, 3), + initial: 0x1122, + want: 0xc2a4, + }, + { + name: "small4", + data: deterministicRandomBytes(2, 4), + initial: 0x1122, + want: 0xc36f, + }, + { + name: "small5", + data: deterministicRandomBytes(2, 5), + initial: 0x1122, + want: 0xa570, + }, + { + name: "small6", + data: deterministicRandomBytes(2, 6), + initial: 0x1122, + want: 0xa669, + }, + { + name: "small7", + data: deterministicRandomBytes(2, 7), + initial: 0x1122, + want: 0x0f6a, + }, + { + name: "small8", + data: deterministicRandomBytes(2, 8), + initial: 0x1122, + want: 0x0fd9, + }, + { + name: "small9", + data: deterministicRandomBytes(2, 9), + initial: 0x1122, + want: 0x40d9, + }, + { + name: "small10", + data: deterministicRandomBytes(2, 10), + initial: 0x1122, + want: 0x411d, + }, + { + name: "small11", + data: deterministicRandomBytes(2, 11), + initial: 0x1122, + want: 0x011e, + }, + { + name: "small12", + data: deterministicRandomBytes(2, 12), + initial: 0x1122, + want: 0x01c8, + }, + { + name: "small13", + data: deterministicRandomBytes(2, 13), + initial: 0x1122, + want: 0x4dc8, + }, + { + name: "small14", + data: deterministicRandomBytes(2, 14), + initial: 0x1122, + want: 0x4eb5, + }, + { + name: "small15", + data: deterministicRandomBytes(2, 15), + initial: 0x1122, + want: 0xa4b5, + }, + // other small-ish sizes + { + name: "small16", + data: deterministicRandomBytes(1, 16), + initial: 0, + want: 0x02fa, + }, + { + name: "small32", + data: deterministicRandomBytes(1, 32), + initial: 0, + want: 0x03ee, + }, + { + name: "small64", + data: deterministicRandomBytes(1, 64), + initial: 0, + want: 0x3f85, + }, + { + name: "medium", + data: deterministicRandomBytes(1, 1400), + initial: 0, + want: 0xbea5, + }, + { + name: "big", + data: deterministicRandomBytes(2, 65000), + initial: 0, + want: 0x3ba7, + }, + { + name: "big-initial", + data: deterministicRandomBytes(2, 65000), + initial: 0x1234, + want: 0x4ddb, + }, + { + // big-small-loop is intended to exercise a few iterations of a big + // initial loop of 128 bytes or larger + a smaller loop of 16 bytes + // + some leftover + name: "big-small-loop", + data: deterministicRandomBytes(3, 1094), + initial: 0x9999, + want: 0xe65b, + }, + { + name: "page-aligned", + data: alignedBuf[:4096], + initial: 0, + want: 0x963b, + }, + { + name: "32-aligned", + data: alignedBuf[32:4128], + initial: 0, + want: 0x30c4, + }, + { + name: "16-aligned", + data: alignedBuf[16:4112], + initial: 0, + want: 0xaeff, + }, + { + name: "8-aligned", + data: alignedBuf[8:4104], + initial: 0, + want: 0x6c3b, + }, + { + name: "4-aligned", + data: alignedBuf[4:4100], + initial: 0, + want: 0x2e4a, + }, + { + name: "2-aligned", + data: alignedBuf[2:4098], + initial: 0, + want: 0xc702, + }, + { + name: "unaligned", + data: alignedBuf[1:4097], + initial: 0, + want: 0x3bc7, + }, + { + name: "unalignedAndOdd", + data: alignedBuf[1:4096], + initial: 0, + want: 0x3b13, + }, + { + name: "fe1282", + data: allFE[:1282], + initial: 0, + want: 0x7c7c, + }, + { + name: "fe", + data: allFE, + initial: 0, + want: 0x7e81, + }, + { + name: "maximum", + data: allOnes, + initial: 0, + want: 0xff00, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for _, fd := range archChecksumFuncs { + t.Run(fd.name, func(t *testing.T) { + if !fd.available { + t.Skip("can not run on this system") + } + if got := fd.f(tt.data, uint64(tt.initial)); got != tt.want { + t.Errorf("%s checksum = %04x, want %04x", fd.name, got, tt.want) + } + }) + } + t.Run("reference", func(t *testing.T) { + if got := gvisorChecksum.Checksum(tt.data, tt.initial); got != tt.want { + t.Errorf("reference checksum = %04x, want %04x", got, tt.want) + } + }) + }) + } +} + +func TestPseudoHeaderChecksumNoFold(t *testing.T) { + tests := []struct { + name string + protocol uint8 + srcAddr []byte + dstAddr []byte + totalLen uint16 + want uint16 + }{ + { + name: "ipv4", + protocol: syscall.IPPROTO_TCP, + srcAddr: netip.MustParseAddr("192.168.1.1").AsSlice(), + dstAddr: netip.MustParseAddr("192.168.1.2").AsSlice(), + totalLen: 1492, + want: 0x892e, + }, + { + name: "ipv6", + protocol: syscall.IPPROTO_TCP, + srcAddr: netip.MustParseAddr("2001:db8:3333:4444:5555:6666:7777:8888").AsSlice(), + dstAddr: netip.MustParseAddr("2001:db8:aaaa:bbbb:cccc:dddd:eeee:ffff").AsSlice(), + totalLen: 1492, + want: 0x947f, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotNoFold := pseudoHeaderChecksumNoFold(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) + got := checksum([]byte{}, gotNoFold) + if got != tt.want { + t.Errorf("pseudoHeaderChecksumNoFold() = %x, folds to %04x, want %04x", gotNoFold, got, tt.want) + } + + got = header.PseudoHeaderChecksum( + tcpip.TransportProtocolNumber(tt.protocol), + tcpip.AddrFromSlice(tt.srcAddr), + tcpip.AddrFromSlice(tt.dstAddr), + tt.totalLen) + if got != tt.want { + t.Errorf("header.PseudoHeaderChecksum() = %04x, want %04x", got, tt.want) + } + }) + } +} + +func FuzzChecksum(f *testing.F) { + buf := getPageAlignedRandomBytes(1234, 65536) + + f.Add([]byte{}, uint16(0)) + f.Add([]byte{}, uint16(0x1234)) + f.Add([]byte{}, uint16(0)) + f.Add(buf[:15], uint16(0x1234)) + f.Add(buf[:256], uint16(0x1234)) + f.Add(buf[:1280], uint16(0x1234)) + f.Add(buf[:1288], uint16(0x1234)) + f.Add(buf[1:1050], uint16(0x1234)) + + f.Fuzz(func(t *testing.T, data []byte, initial uint16) { + want := gvisorChecksum.Checksum(data, initial) + + for _, fd := range archChecksumFuncs { + t.Run(fd.name, func(t *testing.T) { + if !fd.available { + t.Skip("can not run on this system") + } + if got := fd.f(data, uint64(initial)); got != want { + t.Errorf("%s checksum = %04x, want %04x", fd.name, got, want) + } + }) + } + }) +} + +var result uint16 +var result64 uint64 + func BenchmarkChecksum(b *testing.B) { + offsets := []int{ // offsets from page alignment + 0, + 1, + 2, + 4, + 8, + 16, + } lengths := []int{ + 0, + 7, + 15, + 16, + 31, 64, + 90, + 95, 128, 256, 512, 1024, + 1240, 1500, 2048, 4096, 8192, 9000, 9001, + 16384, + 65536, } + if !sort.IntsAreSorted(offsets) { + b.Fatal("offsets are not sorted") + } + largestLength := lengths[len(lengths)-1] + if !sort.IntsAreSorted(lengths) { + b.Fatal("lengths are not sorted") + } + largestOffset := lengths[len(offsets)-1] + alignedBuf := getPageAlignedRandomBytes(1, largestOffset+largestLength) + var r uint16 + for _, offset := range offsets { + name := fmt.Sprintf("%vAligned", offset) + if offset == 0 { + name = "pageAligned" + } + offsetBuf := alignedBuf[offset:] + b.Run(name, func(b *testing.B) { + for _, length := range lengths { + b.Run(fmt.Sprintf("%d", length), func(b *testing.B) { + for _, fd := range archChecksumFuncs { + b.Run(fd.name, func(b *testing.B) { + if !fd.available { + b.Skip("can not run on this system") + } + b.SetBytes(int64(length)) + for i := 0; i < b.N; i++ { + r += fd.f(offsetBuf[:length], 0) + } + }) + } + }) + } + }) + } + result = r +} - for _, length := range lengths { - b.Run(fmt.Sprintf("%d", length), func(b *testing.B) { - buf := make([]byte, length) - rng := rand.New(rand.NewSource(1)) - rng.Read(buf) - b.ResetTimer() +func BenchmarkPseudoHeaderChecksum(b *testing.B) { + tests := []struct { + name string + protocol uint8 + srcAddr []byte + dstAddr []byte + totalLen uint16 + want uint16 + }{ + { + name: "ipv4", + protocol: syscall.IPPROTO_TCP, + srcAddr: []byte{192, 168, 1, 1}, + dstAddr: []byte{192, 168, 1, 2}, + totalLen: 1492, + want: 0x892e, + }, + { + name: "ipv6", + protocol: syscall.IPPROTO_TCP, + srcAddr: netip.MustParseAddr("2001:db8:3333:4444:5555:6666:7777:8888").AsSlice(), + dstAddr: netip.MustParseAddr("2001:db8:aaaa:bbbb:cccc:dddd:eeee:ffff").AsSlice(), + totalLen: 1492, + want: 0x892e, + }, + } + for _, tt := range tests { + b.Run(tt.name, func(b *testing.B) { for i := 0; i < b.N; i++ { - checksum(buf, 0) + result64 += pseudoHeaderChecksumNoFold(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) } }) } From 88b11b4a0dde05ecd823e896c198014b1652e7c9 Mon Sep 17 00:00:00 2001 From: Adrian Dewhurst Date: Thu, 30 Mar 2023 21:48:43 -0400 Subject: [PATCH 09/60] tun: AMD64 optimized checksum This adds AMD64 assembly implementations of IP checksum computation, one for baseline AMD64 and the other for v3 AMD64 (AVX2 and BMI2). All performance numbers reported are from a Ryzen 7 4750U but similar improvements are expected for a wide range of processors. The generic IP checksum implementation has also been further improved to be significantly faster using bits.AddUint64 (for a 64KiB buffer the throughput improves from 15,000MiB/s to 27,600MiB/s; similar gains are also reported on ARM64 but I do not have specific numbers). The baseline AMD64 implementation for a 64KiB buffer reports 32,700MiB/s and the AVX2 implementation is slightly over 107,000MiB/s. Unfortunately, for very small sizes (e.g. the expected size for an IPv4 header) setting up SIMD computation involves some overhead that makes computing a checksum for small buffers slower than a non-SIMD implementation. Even more unfortunately, testing for this at runtimen in Go and calling a func optimized for small buffers mitigates most of the improvement due to call overhead. The break even point is around 256 byte buffers; IPv4 headers are no more than 60 bytes including extensions. IPv6 headers do not have a checksum but are a fixed size of 40 bytes. As a result, the generated assembly code uses an alternate approach for buffers of less than 256 bytes. Additionally, buffers of less than 32 bytes need to be handled specially because the strategy for reading buffers that are not a multiple of 8 bytes fails when the buffer is too small. As suggested by additional benchmarking, pseudo header computation has been rewritten to be faster (benchmark time reduced by 1/2 to 1/4). Updates tailscale/corp#9755 Signed-off-by: Adrian Dewhurst --- tun/checksum.go | 764 ++++++++++++++++++++++++---- tun/checksum_amd64.go | 20 + tun/checksum_amd64_test.go | 45 ++ tun/checksum_generated_amd64.go | 18 + tun/checksum_generated_amd64.s | 851 ++++++++++++++++++++++++++++++++ tun/checksum_generic.go | 15 + tun/checksum_generic_test.go | 21 +- tun/checksum_test.go | 56 ++- tun/generate_amd64.go | 579 ++++++++++++++++++++++ tun/tcp_offload_linux.go | 12 +- 10 files changed, 2266 insertions(+), 115 deletions(-) create mode 100644 tun/checksum_amd64.go create mode 100644 tun/checksum_amd64_test.go create mode 100644 tun/checksum_generated_amd64.go create mode 100644 tun/checksum_generated_amd64.s create mode 100644 tun/checksum_generic.go create mode 100644 tun/generate_amd64.go diff --git a/tun/checksum.go b/tun/checksum.go index 29a8fc8..ee3f359 100644 --- a/tun/checksum.go +++ b/tun/checksum.go @@ -1,118 +1,710 @@ package tun -import "encoding/binary" +import ( + "encoding/binary" + "math/bits" + "strconv" -// TODO: Explore SIMD and/or other assembly optimizations. -// TODO: Test native endian loads. See RFC 1071 section 2 part B. -func checksumNoFold(b []byte, initial uint64) uint64 { - ac := initial + "golang.org/x/sys/cpu" +) + +// checksumGeneric64 is a reference implementation of checksum using 64 bit +// arithmetic for use in testing or when an architecture-specific implementation +// is not available. +func checksumGeneric64(b []byte, initial uint16) uint16 { + var ac uint64 + var carry uint64 + + if cpu.IsBigEndian { + ac = uint64(initial) + } else { + ac = uint64(bits.ReverseBytes16(initial)) + } for len(b) >= 128 { - ac += uint64(binary.BigEndian.Uint32(b[:4])) - ac += uint64(binary.BigEndian.Uint32(b[4:8])) - ac += uint64(binary.BigEndian.Uint32(b[8:12])) - ac += uint64(binary.BigEndian.Uint32(b[12:16])) - ac += uint64(binary.BigEndian.Uint32(b[16:20])) - ac += uint64(binary.BigEndian.Uint32(b[20:24])) - ac += uint64(binary.BigEndian.Uint32(b[24:28])) - ac += uint64(binary.BigEndian.Uint32(b[28:32])) - ac += uint64(binary.BigEndian.Uint32(b[32:36])) - ac += uint64(binary.BigEndian.Uint32(b[36:40])) - ac += uint64(binary.BigEndian.Uint32(b[40:44])) - ac += uint64(binary.BigEndian.Uint32(b[44:48])) - ac += uint64(binary.BigEndian.Uint32(b[48:52])) - ac += uint64(binary.BigEndian.Uint32(b[52:56])) - ac += uint64(binary.BigEndian.Uint32(b[56:60])) - ac += uint64(binary.BigEndian.Uint32(b[60:64])) - ac += uint64(binary.BigEndian.Uint32(b[64:68])) - ac += uint64(binary.BigEndian.Uint32(b[68:72])) - ac += uint64(binary.BigEndian.Uint32(b[72:76])) - ac += uint64(binary.BigEndian.Uint32(b[76:80])) - ac += uint64(binary.BigEndian.Uint32(b[80:84])) - ac += uint64(binary.BigEndian.Uint32(b[84:88])) - ac += uint64(binary.BigEndian.Uint32(b[88:92])) - ac += uint64(binary.BigEndian.Uint32(b[92:96])) - ac += uint64(binary.BigEndian.Uint32(b[96:100])) - ac += uint64(binary.BigEndian.Uint32(b[100:104])) - ac += uint64(binary.BigEndian.Uint32(b[104:108])) - ac += uint64(binary.BigEndian.Uint32(b[108:112])) - ac += uint64(binary.BigEndian.Uint32(b[112:116])) - ac += uint64(binary.BigEndian.Uint32(b[116:120])) - ac += uint64(binary.BigEndian.Uint32(b[120:124])) - ac += uint64(binary.BigEndian.Uint32(b[124:128])) + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[32:40]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[40:48]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[48:56]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[56:64]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[64:72]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[72:80]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[80:88]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[88:96]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[96:104]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[104:112]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[112:120]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[120:128]), carry) + } else { + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[32:40]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[40:48]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[48:56]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[56:64]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[64:72]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[72:80]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[80:88]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[88:96]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[96:104]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[104:112]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[112:120]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[120:128]), carry) + } b = b[128:] } if len(b) >= 64 { - ac += uint64(binary.BigEndian.Uint32(b[:4])) - ac += uint64(binary.BigEndian.Uint32(b[4:8])) - ac += uint64(binary.BigEndian.Uint32(b[8:12])) - ac += uint64(binary.BigEndian.Uint32(b[12:16])) - ac += uint64(binary.BigEndian.Uint32(b[16:20])) - ac += uint64(binary.BigEndian.Uint32(b[20:24])) - ac += uint64(binary.BigEndian.Uint32(b[24:28])) - ac += uint64(binary.BigEndian.Uint32(b[28:32])) - ac += uint64(binary.BigEndian.Uint32(b[32:36])) - ac += uint64(binary.BigEndian.Uint32(b[36:40])) - ac += uint64(binary.BigEndian.Uint32(b[40:44])) - ac += uint64(binary.BigEndian.Uint32(b[44:48])) - ac += uint64(binary.BigEndian.Uint32(b[48:52])) - ac += uint64(binary.BigEndian.Uint32(b[52:56])) - ac += uint64(binary.BigEndian.Uint32(b[56:60])) - ac += uint64(binary.BigEndian.Uint32(b[60:64])) + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[32:40]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[40:48]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[48:56]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[56:64]), carry) + } else { + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[32:40]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[40:48]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[48:56]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[56:64]), carry) + } b = b[64:] } if len(b) >= 32 { - ac += uint64(binary.BigEndian.Uint32(b[:4])) - ac += uint64(binary.BigEndian.Uint32(b[4:8])) - ac += uint64(binary.BigEndian.Uint32(b[8:12])) - ac += uint64(binary.BigEndian.Uint32(b[12:16])) - ac += uint64(binary.BigEndian.Uint32(b[16:20])) - ac += uint64(binary.BigEndian.Uint32(b[20:24])) - ac += uint64(binary.BigEndian.Uint32(b[24:28])) - ac += uint64(binary.BigEndian.Uint32(b[28:32])) + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry) + } else { + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry) + } b = b[32:] } if len(b) >= 16 { - ac += uint64(binary.BigEndian.Uint32(b[:4])) - ac += uint64(binary.BigEndian.Uint32(b[4:8])) - ac += uint64(binary.BigEndian.Uint32(b[8:12])) - ac += uint64(binary.BigEndian.Uint32(b[12:16])) + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) + } else { + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) + } b = b[16:] } if len(b) >= 8 { - ac += uint64(binary.BigEndian.Uint32(b[:4])) - ac += uint64(binary.BigEndian.Uint32(b[4:8])) + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b), carry) + } else { + ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b), carry) + } b = b[8:] } if len(b) >= 4 { - ac += uint64(binary.BigEndian.Uint32(b)) + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, uint64(binary.BigEndian.Uint32(b)), carry) + } else { + ac, carry = bits.Add64(ac, uint64(binary.LittleEndian.Uint32(b)), carry) + } b = b[4:] } if len(b) >= 2 { - ac += uint64(binary.BigEndian.Uint16(b)) + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, uint64(binary.BigEndian.Uint16(b)), carry) + } else { + ac, carry = bits.Add64(ac, uint64(binary.LittleEndian.Uint16(b)), carry) + } b = b[2:] } - if len(b) == 1 { - ac += uint64(b[0]) << 8 + if len(b) >= 1 { + if cpu.IsBigEndian { + ac, carry = bits.Add64(ac, uint64(b[0])<<8, carry) + } else { + ac, carry = bits.Add64(ac, uint64(b[0]), carry) + } } - return ac + folded := ipChecksumFold64(ac, carry) + if !cpu.IsBigEndian { + folded = bits.ReverseBytes16(folded) + } + return folded } -func checksum(b []byte, initial uint64) uint16 { - ac := checksumNoFold(b, initial) - ac = (ac >> 16) + (ac & 0xffff) - ac = (ac >> 16) + (ac & 0xffff) - ac = (ac >> 16) + (ac & 0xffff) - ac = (ac >> 16) + (ac & 0xffff) - return uint16(ac) +// checksumGeneric32 is a reference implementation of checksum using 32 bit +// arithmetic for use in testing or when an architecture-specific implementation +// is not available. +func checksumGeneric32(b []byte, initial uint16) uint16 { + var ac uint32 + var carry uint32 + + if cpu.IsBigEndian { + ac = uint32(initial) + } else { + ac = uint32(bits.ReverseBytes16(initial)) + } + + for len(b) >= 64 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:8]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[16:20]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[20:24]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[24:28]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[28:32]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[32:36]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[36:40]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[40:44]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[44:48]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[48:52]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[52:56]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[56:60]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[60:64]), carry) + } else { + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:8]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[16:20]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[20:24]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[24:28]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[28:32]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[32:36]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[36:40]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[40:44]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[44:48]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[48:52]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[52:56]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[56:60]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[60:64]), carry) + } + b = b[64:] + } + if len(b) >= 32 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[16:20]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[20:24]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[24:28]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[28:32]), carry) + } else { + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[16:20]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[20:24]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[24:28]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[28:32]), carry) + } + b = b[32:] + } + if len(b) >= 16 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry) + } else { + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry) + } + b = b[16:] + } + if len(b) >= 8 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) + } else { + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry) + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) + } + b = b[8:] + } + if len(b) >= 4 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b), carry) + } else { + ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b), carry) + } + b = b[4:] + } + if len(b) >= 2 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, uint32(binary.BigEndian.Uint16(b)), carry) + } else { + ac, carry = bits.Add32(ac, uint32(binary.LittleEndian.Uint16(b)), carry) + } + b = b[2:] + } + if len(b) >= 1 { + if cpu.IsBigEndian { + ac, carry = bits.Add32(ac, uint32(b[0])<<8, carry) + } else { + ac, carry = bits.Add32(ac, uint32(b[0]), carry) + } + } + + folded := ipChecksumFold32(ac, carry) + if !cpu.IsBigEndian { + folded = bits.ReverseBytes16(folded) + } + return folded } -func pseudoHeaderChecksumNoFold(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint64 { - sum := checksumNoFold(srcAddr, 0) - sum = checksumNoFold(dstAddr, sum) - sum = checksumNoFold([]byte{0, protocol}, sum) - tmp := make([]byte, 2) - binary.BigEndian.PutUint16(tmp, totalLen) - return checksumNoFold(tmp, sum) +// checksumGeneric32Alternate is an alternate reference implementation of +// checksum using 32 bit arithmetic for use in testing or when an +// architecture-specific implementation is not available. +func checksumGeneric32Alternate(b []byte, initial uint16) uint16 { + var ac uint32 + + if cpu.IsBigEndian { + ac = uint32(initial) + } else { + ac = uint32(bits.ReverseBytes16(initial)) + } + + for len(b) >= 64 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b[:2])) + ac += uint32(binary.BigEndian.Uint16(b[2:4])) + ac += uint32(binary.BigEndian.Uint16(b[4:6])) + ac += uint32(binary.BigEndian.Uint16(b[6:8])) + ac += uint32(binary.BigEndian.Uint16(b[8:10])) + ac += uint32(binary.BigEndian.Uint16(b[10:12])) + ac += uint32(binary.BigEndian.Uint16(b[12:14])) + ac += uint32(binary.BigEndian.Uint16(b[14:16])) + ac += uint32(binary.BigEndian.Uint16(b[16:18])) + ac += uint32(binary.BigEndian.Uint16(b[18:20])) + ac += uint32(binary.BigEndian.Uint16(b[20:22])) + ac += uint32(binary.BigEndian.Uint16(b[22:24])) + ac += uint32(binary.BigEndian.Uint16(b[24:26])) + ac += uint32(binary.BigEndian.Uint16(b[26:28])) + ac += uint32(binary.BigEndian.Uint16(b[28:30])) + ac += uint32(binary.BigEndian.Uint16(b[30:32])) + ac += uint32(binary.BigEndian.Uint16(b[32:34])) + ac += uint32(binary.BigEndian.Uint16(b[34:36])) + ac += uint32(binary.BigEndian.Uint16(b[36:38])) + ac += uint32(binary.BigEndian.Uint16(b[38:40])) + ac += uint32(binary.BigEndian.Uint16(b[40:42])) + ac += uint32(binary.BigEndian.Uint16(b[42:44])) + ac += uint32(binary.BigEndian.Uint16(b[44:46])) + ac += uint32(binary.BigEndian.Uint16(b[46:48])) + ac += uint32(binary.BigEndian.Uint16(b[48:50])) + ac += uint32(binary.BigEndian.Uint16(b[50:52])) + ac += uint32(binary.BigEndian.Uint16(b[52:54])) + ac += uint32(binary.BigEndian.Uint16(b[54:56])) + ac += uint32(binary.BigEndian.Uint16(b[56:58])) + ac += uint32(binary.BigEndian.Uint16(b[58:60])) + ac += uint32(binary.BigEndian.Uint16(b[60:62])) + ac += uint32(binary.BigEndian.Uint16(b[62:64])) + } else { + ac += uint32(binary.LittleEndian.Uint16(b[:2])) + ac += uint32(binary.LittleEndian.Uint16(b[2:4])) + ac += uint32(binary.LittleEndian.Uint16(b[4:6])) + ac += uint32(binary.LittleEndian.Uint16(b[6:8])) + ac += uint32(binary.LittleEndian.Uint16(b[8:10])) + ac += uint32(binary.LittleEndian.Uint16(b[10:12])) + ac += uint32(binary.LittleEndian.Uint16(b[12:14])) + ac += uint32(binary.LittleEndian.Uint16(b[14:16])) + ac += uint32(binary.LittleEndian.Uint16(b[16:18])) + ac += uint32(binary.LittleEndian.Uint16(b[18:20])) + ac += uint32(binary.LittleEndian.Uint16(b[20:22])) + ac += uint32(binary.LittleEndian.Uint16(b[22:24])) + ac += uint32(binary.LittleEndian.Uint16(b[24:26])) + ac += uint32(binary.LittleEndian.Uint16(b[26:28])) + ac += uint32(binary.LittleEndian.Uint16(b[28:30])) + ac += uint32(binary.LittleEndian.Uint16(b[30:32])) + ac += uint32(binary.LittleEndian.Uint16(b[32:34])) + ac += uint32(binary.LittleEndian.Uint16(b[34:36])) + ac += uint32(binary.LittleEndian.Uint16(b[36:38])) + ac += uint32(binary.LittleEndian.Uint16(b[38:40])) + ac += uint32(binary.LittleEndian.Uint16(b[40:42])) + ac += uint32(binary.LittleEndian.Uint16(b[42:44])) + ac += uint32(binary.LittleEndian.Uint16(b[44:46])) + ac += uint32(binary.LittleEndian.Uint16(b[46:48])) + ac += uint32(binary.LittleEndian.Uint16(b[48:50])) + ac += uint32(binary.LittleEndian.Uint16(b[50:52])) + ac += uint32(binary.LittleEndian.Uint16(b[52:54])) + ac += uint32(binary.LittleEndian.Uint16(b[54:56])) + ac += uint32(binary.LittleEndian.Uint16(b[56:58])) + ac += uint32(binary.LittleEndian.Uint16(b[58:60])) + ac += uint32(binary.LittleEndian.Uint16(b[60:62])) + ac += uint32(binary.LittleEndian.Uint16(b[62:64])) + } + b = b[64:] + } + if len(b) >= 32 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b[:2])) + ac += uint32(binary.BigEndian.Uint16(b[2:4])) + ac += uint32(binary.BigEndian.Uint16(b[4:6])) + ac += uint32(binary.BigEndian.Uint16(b[6:8])) + ac += uint32(binary.BigEndian.Uint16(b[8:10])) + ac += uint32(binary.BigEndian.Uint16(b[10:12])) + ac += uint32(binary.BigEndian.Uint16(b[12:14])) + ac += uint32(binary.BigEndian.Uint16(b[14:16])) + ac += uint32(binary.BigEndian.Uint16(b[16:18])) + ac += uint32(binary.BigEndian.Uint16(b[18:20])) + ac += uint32(binary.BigEndian.Uint16(b[20:22])) + ac += uint32(binary.BigEndian.Uint16(b[22:24])) + ac += uint32(binary.BigEndian.Uint16(b[24:26])) + ac += uint32(binary.BigEndian.Uint16(b[26:28])) + ac += uint32(binary.BigEndian.Uint16(b[28:30])) + ac += uint32(binary.BigEndian.Uint16(b[30:32])) + } else { + ac += uint32(binary.LittleEndian.Uint16(b[:2])) + ac += uint32(binary.LittleEndian.Uint16(b[2:4])) + ac += uint32(binary.LittleEndian.Uint16(b[4:6])) + ac += uint32(binary.LittleEndian.Uint16(b[6:8])) + ac += uint32(binary.LittleEndian.Uint16(b[8:10])) + ac += uint32(binary.LittleEndian.Uint16(b[10:12])) + ac += uint32(binary.LittleEndian.Uint16(b[12:14])) + ac += uint32(binary.LittleEndian.Uint16(b[14:16])) + ac += uint32(binary.LittleEndian.Uint16(b[16:18])) + ac += uint32(binary.LittleEndian.Uint16(b[18:20])) + ac += uint32(binary.LittleEndian.Uint16(b[20:22])) + ac += uint32(binary.LittleEndian.Uint16(b[22:24])) + ac += uint32(binary.LittleEndian.Uint16(b[24:26])) + ac += uint32(binary.LittleEndian.Uint16(b[26:28])) + ac += uint32(binary.LittleEndian.Uint16(b[28:30])) + ac += uint32(binary.LittleEndian.Uint16(b[30:32])) + } + b = b[32:] + } + if len(b) >= 16 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b[:2])) + ac += uint32(binary.BigEndian.Uint16(b[2:4])) + ac += uint32(binary.BigEndian.Uint16(b[4:6])) + ac += uint32(binary.BigEndian.Uint16(b[6:8])) + ac += uint32(binary.BigEndian.Uint16(b[8:10])) + ac += uint32(binary.BigEndian.Uint16(b[10:12])) + ac += uint32(binary.BigEndian.Uint16(b[12:14])) + ac += uint32(binary.BigEndian.Uint16(b[14:16])) + } else { + ac += uint32(binary.LittleEndian.Uint16(b[:2])) + ac += uint32(binary.LittleEndian.Uint16(b[2:4])) + ac += uint32(binary.LittleEndian.Uint16(b[4:6])) + ac += uint32(binary.LittleEndian.Uint16(b[6:8])) + ac += uint32(binary.LittleEndian.Uint16(b[8:10])) + ac += uint32(binary.LittleEndian.Uint16(b[10:12])) + ac += uint32(binary.LittleEndian.Uint16(b[12:14])) + ac += uint32(binary.LittleEndian.Uint16(b[14:16])) + } + b = b[16:] + } + if len(b) >= 8 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b[:2])) + ac += uint32(binary.BigEndian.Uint16(b[2:4])) + ac += uint32(binary.BigEndian.Uint16(b[4:6])) + ac += uint32(binary.BigEndian.Uint16(b[6:8])) + } else { + ac += uint32(binary.LittleEndian.Uint16(b[:2])) + ac += uint32(binary.LittleEndian.Uint16(b[2:4])) + ac += uint32(binary.LittleEndian.Uint16(b[4:6])) + ac += uint32(binary.LittleEndian.Uint16(b[6:8])) + } + b = b[8:] + } + if len(b) >= 4 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b[:2])) + ac += uint32(binary.BigEndian.Uint16(b[2:4])) + } else { + ac += uint32(binary.LittleEndian.Uint16(b[:2])) + ac += uint32(binary.LittleEndian.Uint16(b[2:4])) + } + b = b[4:] + } + if len(b) >= 2 { + if cpu.IsBigEndian { + ac += uint32(binary.BigEndian.Uint16(b)) + } else { + ac += uint32(binary.LittleEndian.Uint16(b)) + } + b = b[2:] + } + if len(b) >= 1 { + if cpu.IsBigEndian { + ac += uint32(b[0]) << 8 + } else { + ac += uint32(b[0]) + } + } + + folded := ipChecksumFold32(ac, 0) + if !cpu.IsBigEndian { + folded = bits.ReverseBytes16(folded) + } + return folded +} + +// checksumGeneric64Alternate is an alternate reference implementation of +// checksum using 64 bit arithmetic for use in testing or when an +// architecture-specific implementation is not available. +func checksumGeneric64Alternate(b []byte, initial uint16) uint16 { + var ac uint64 + + if cpu.IsBigEndian { + ac = uint64(initial) + } else { + ac = uint64(bits.ReverseBytes16(initial)) + } + + for len(b) >= 64 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + ac += uint64(binary.BigEndian.Uint32(b[8:12])) + ac += uint64(binary.BigEndian.Uint32(b[12:16])) + ac += uint64(binary.BigEndian.Uint32(b[16:20])) + ac += uint64(binary.BigEndian.Uint32(b[20:24])) + ac += uint64(binary.BigEndian.Uint32(b[24:28])) + ac += uint64(binary.BigEndian.Uint32(b[28:32])) + ac += uint64(binary.BigEndian.Uint32(b[32:36])) + ac += uint64(binary.BigEndian.Uint32(b[36:40])) + ac += uint64(binary.BigEndian.Uint32(b[40:44])) + ac += uint64(binary.BigEndian.Uint32(b[44:48])) + ac += uint64(binary.BigEndian.Uint32(b[48:52])) + ac += uint64(binary.BigEndian.Uint32(b[52:56])) + ac += uint64(binary.BigEndian.Uint32(b[56:60])) + ac += uint64(binary.BigEndian.Uint32(b[60:64])) + } else { + ac += uint64(binary.LittleEndian.Uint32(b[:4])) + ac += uint64(binary.LittleEndian.Uint32(b[4:8])) + ac += uint64(binary.LittleEndian.Uint32(b[8:12])) + ac += uint64(binary.LittleEndian.Uint32(b[12:16])) + ac += uint64(binary.LittleEndian.Uint32(b[16:20])) + ac += uint64(binary.LittleEndian.Uint32(b[20:24])) + ac += uint64(binary.LittleEndian.Uint32(b[24:28])) + ac += uint64(binary.LittleEndian.Uint32(b[28:32])) + ac += uint64(binary.LittleEndian.Uint32(b[32:36])) + ac += uint64(binary.LittleEndian.Uint32(b[36:40])) + ac += uint64(binary.LittleEndian.Uint32(b[40:44])) + ac += uint64(binary.LittleEndian.Uint32(b[44:48])) + ac += uint64(binary.LittleEndian.Uint32(b[48:52])) + ac += uint64(binary.LittleEndian.Uint32(b[52:56])) + ac += uint64(binary.LittleEndian.Uint32(b[56:60])) + ac += uint64(binary.LittleEndian.Uint32(b[60:64])) + } + b = b[64:] + } + if len(b) >= 32 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + ac += uint64(binary.BigEndian.Uint32(b[8:12])) + ac += uint64(binary.BigEndian.Uint32(b[12:16])) + ac += uint64(binary.BigEndian.Uint32(b[16:20])) + ac += uint64(binary.BigEndian.Uint32(b[20:24])) + ac += uint64(binary.BigEndian.Uint32(b[24:28])) + ac += uint64(binary.BigEndian.Uint32(b[28:32])) + } else { + ac += uint64(binary.LittleEndian.Uint32(b[:4])) + ac += uint64(binary.LittleEndian.Uint32(b[4:8])) + ac += uint64(binary.LittleEndian.Uint32(b[8:12])) + ac += uint64(binary.LittleEndian.Uint32(b[12:16])) + ac += uint64(binary.LittleEndian.Uint32(b[16:20])) + ac += uint64(binary.LittleEndian.Uint32(b[20:24])) + ac += uint64(binary.LittleEndian.Uint32(b[24:28])) + ac += uint64(binary.LittleEndian.Uint32(b[28:32])) + } + b = b[32:] + } + if len(b) >= 16 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + ac += uint64(binary.BigEndian.Uint32(b[8:12])) + ac += uint64(binary.BigEndian.Uint32(b[12:16])) + } else { + ac += uint64(binary.LittleEndian.Uint32(b[:4])) + ac += uint64(binary.LittleEndian.Uint32(b[4:8])) + ac += uint64(binary.LittleEndian.Uint32(b[8:12])) + ac += uint64(binary.LittleEndian.Uint32(b[12:16])) + } + b = b[16:] + } + if len(b) >= 8 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint32(b[:4])) + ac += uint64(binary.BigEndian.Uint32(b[4:8])) + } else { + ac += uint64(binary.LittleEndian.Uint32(b[:4])) + ac += uint64(binary.LittleEndian.Uint32(b[4:8])) + } + b = b[8:] + } + if len(b) >= 4 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint32(b)) + } else { + ac += uint64(binary.LittleEndian.Uint32(b)) + } + b = b[4:] + } + if len(b) >= 2 { + if cpu.IsBigEndian { + ac += uint64(binary.BigEndian.Uint16(b)) + } else { + ac += uint64(binary.LittleEndian.Uint16(b)) + } + b = b[2:] + } + if len(b) >= 1 { + if cpu.IsBigEndian { + ac += uint64(b[0]) << 8 + } else { + ac += uint64(b[0]) + } + } + + folded := ipChecksumFold64(ac, 0) + if !cpu.IsBigEndian { + folded = bits.ReverseBytes16(folded) + } + return folded +} + +func ipChecksumFold64(unfolded uint64, initialCarry uint64) uint16 { + sum, carry := bits.Add32(uint32(unfolded>>32), uint32(unfolded&0xffff_ffff), uint32(initialCarry)) + // if carry != 0, sum <= 0xffff_fffe, otherwise sum <= 0xffff_ffff + // therefore (sum >> 16) + (sum & 0xffff) + carry <= 0x1_fffe; so there is + // no need to save the carry flag + sum = (sum >> 16) + (sum & 0xffff) + carry + // sum <= 0x1_fffe therefore this is the last fold needed: + // if (sum >> 16) > 0 then + // (sum >> 16) == 1 && (sum & 0xffff) <= 0xfffe and therefore + // the addition will not overflow + // otherwise (sum >> 16) == 0 and sum will be unchanged + sum = (sum >> 16) + (sum & 0xffff) + return uint16(sum) +} + +func ipChecksumFold32(unfolded uint32, initialCarry uint32) uint16 { + sum := (unfolded >> 16) + (unfolded & 0xffff) + initialCarry + // sum <= 0x1_ffff: + // 0xffff + 0xffff = 0x1_fffe + // initialCarry is 0 or 1, for a combined maximum of 0x1_ffff + sum = (sum >> 16) + (sum & 0xffff) + // sum <= 0x1_0000 therefore this is the last fold needed: + // if (sum >> 16) > 0 then + // (sum >> 16) == 1 && (sum & 0xffff) == 0 and therefore + // the addition will not overflow + // otherwise (sum >> 16) == 0 and sum will be unchanged + sum = (sum >> 16) + (sum & 0xffff) + return uint16(sum) +} + +func addrPartialChecksum64(addr []byte, initial, carryIn uint64) (sum, carry uint64) { + sum, carry = initial, carryIn + switch len(addr) { + case 4: // IPv4 + if cpu.IsBigEndian { + sum, carry = bits.Add64(sum, uint64(binary.BigEndian.Uint32(addr)), carry) + } else { + sum, carry = bits.Add64(sum, uint64(binary.LittleEndian.Uint32(addr)), carry) + } + case 16: // IPv6 + if cpu.IsBigEndian { + sum, carry = bits.Add64(sum, binary.BigEndian.Uint64(addr), carry) + sum, carry = bits.Add64(sum, binary.BigEndian.Uint64(addr[8:]), carry) + } else { + sum, carry = bits.Add64(sum, binary.LittleEndian.Uint64(addr), carry) + sum, carry = bits.Add64(sum, binary.LittleEndian.Uint64(addr[8:]), carry) + } + default: + panic("bad addr length") + } + return sum, carry +} + +func addrPartialChecksum32(addr []byte, initial, carryIn uint32) (sum, carry uint32) { + sum, carry = initial, carryIn + switch len(addr) { + case 4: // IPv4 + if cpu.IsBigEndian { + sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr), carry) + } else { + sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr), carry) + } + case 16: // IPv6 + if cpu.IsBigEndian { + sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr), carry) + sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[4:8]), carry) + sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[8:12]), carry) + sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[12:16]), carry) + } else { + sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr), carry) + sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[4:8]), carry) + sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[8:12]), carry) + sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[12:16]), carry) + } + default: + panic("bad addr length") + } + return sum, carry +} + +func pseudoHeaderChecksum64(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { + var sum uint64 + if cpu.IsBigEndian { + sum = uint64(totalLen) + uint64(protocol) + } else { + sum = uint64(bits.ReverseBytes16(totalLen)) + uint64(protocol)<<8 + } + sum, carry := addrPartialChecksum64(srcAddr, sum, 0) + sum, carry = addrPartialChecksum64(dstAddr, sum, carry) + + foldedSum := ipChecksumFold64(sum, carry) + if !cpu.IsBigEndian { + foldedSum = bits.ReverseBytes16(foldedSum) + } + return foldedSum +} + +func pseudoHeaderChecksum32(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { + var sum uint32 + if cpu.IsBigEndian { + sum = uint32(totalLen) + uint32(protocol) + } else { + sum = uint32(bits.ReverseBytes16(totalLen)) + uint32(protocol)<<8 + } + sum, carry := addrPartialChecksum32(srcAddr, sum, 0) + sum, carry = addrPartialChecksum32(dstAddr, sum, carry) + + foldedSum := ipChecksumFold32(sum, carry) + if !cpu.IsBigEndian { + foldedSum = bits.ReverseBytes16(foldedSum) + } + return foldedSum +} + +func pseudoHeaderChecksum(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { + if strconv.IntSize < 64 { + return pseudoHeaderChecksum32(protocol, srcAddr, dstAddr, totalLen) + } + return pseudoHeaderChecksum64(protocol, srcAddr, dstAddr, totalLen) } diff --git a/tun/checksum_amd64.go b/tun/checksum_amd64.go new file mode 100644 index 0000000..5e87693 --- /dev/null +++ b/tun/checksum_amd64.go @@ -0,0 +1,20 @@ +package tun + +import "golang.org/x/sys/cpu" + +// checksum computes an IP checksum starting with the provided initial value. +// The length of data should be at least 128 bytes for best performance. Smaller +// buffers will still compute a correct result. For best performance with +// smaller buffers, use shortChecksum(). +var checksum = checksumAMD64 + +func init() { + if cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 { + checksum = checksumAVX2 + return + } + if cpu.X86.HasSSE2 { + checksum = checksumSSE2 + return + } +} diff --git a/tun/checksum_amd64_test.go b/tun/checksum_amd64_test.go new file mode 100644 index 0000000..7a0b681 --- /dev/null +++ b/tun/checksum_amd64_test.go @@ -0,0 +1,45 @@ +//go:build amd64 + +package tun + +import ( + "golang.org/x/sys/cpu" +) + +var archChecksumFuncs = []archChecksumDetails{ + { + name: "generic32", + available: true, + f: checksumGeneric32, + }, + { + name: "generic64", + available: true, + f: checksumGeneric64, + }, + { + name: "generic32Alternate", + available: true, + f: checksumGeneric32Alternate, + }, + { + name: "generic64Alternate", + available: true, + f: checksumGeneric64Alternate, + }, + { + name: "AMD64", + available: true, + f: checksumAMD64, + }, + { + name: "SSE2", + available: cpu.X86.HasSSE2, + f: checksumSSE2, + }, + { + name: "AVX2", + available: cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2, + f: checksumAVX2, + }, +} diff --git a/tun/checksum_generated_amd64.go b/tun/checksum_generated_amd64.go new file mode 100644 index 0000000..b4a2941 --- /dev/null +++ b/tun/checksum_generated_amd64.go @@ -0,0 +1,18 @@ +// Code generated by command: go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go. DO NOT EDIT. + +package tun + +// checksumAVX2 computes an IP checksum using amd64 v3 instructions (AVX2, BMI2) +// +//go:noescape +func checksumAVX2(b []byte, initial uint16) uint16 + +// checksumSSE2 computes an IP checksum using amd64 baseline instructions (SSE2) +// +//go:noescape +func checksumSSE2(b []byte, initial uint16) uint16 + +// checksumAMD64 computes an IP checksum using amd64 baseline instructions +// +//go:noescape +func checksumAMD64(b []byte, initial uint16) uint16 diff --git a/tun/checksum_generated_amd64.s b/tun/checksum_generated_amd64.s new file mode 100644 index 0000000..5f2e4c5 --- /dev/null +++ b/tun/checksum_generated_amd64.s @@ -0,0 +1,851 @@ +// Code generated by command: go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go. DO NOT EDIT. + +#include "textflag.h" + +DATA xmmLoadMasks<>+0(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff" +DATA xmmLoadMasks<>+16(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff" +DATA xmmLoadMasks<>+32(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff" +DATA xmmLoadMasks<>+48(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff" +DATA xmmLoadMasks<>+64(SB)/16, $"\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +DATA xmmLoadMasks<>+80(SB)/16, $"\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +DATA xmmLoadMasks<>+96(SB)/16, $"\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +GLOBL xmmLoadMasks<>(SB), RODATA|NOPTR, $112 + +// func checksumAVX2(b []byte, initial uint16) uint16 +// Requires: AVX, AVX2, BMI2 +TEXT ·checksumAVX2(SB), NOSPLIT|NOFRAME, $0-34 + MOVWQZX initial+24(FP), AX + XCHGB AH, AL + MOVQ b_base+0(FP), DX + MOVQ b_len+8(FP), BX + + // handle odd length buffers; they are difficult to handle in general + TESTQ $0x00000001, BX + JZ lengthIsEven + MOVBQZX -1(DX)(BX*1), CX + DECQ BX + ADDQ CX, AX + +lengthIsEven: + // handle tiny buffers (<=31 bytes) specially + CMPQ BX, $0x1f + JGT bufferIsNotTiny + XORQ CX, CX + XORQ SI, SI + XORQ DI, DI + + // shift twice to start because length is guaranteed to be even + // n = n >> 2; CF = originalN & 2 + SHRQ $0x02, BX + JNC handleTiny4 + + // tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:] + MOVWQZX (DX), CX + ADDQ $0x02, DX + +handleTiny4: + // n = n >> 1; CF = originalN & 4 + SHRQ $0x01, BX + JNC handleTiny8 + + // tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:] + MOVLQZX (DX), SI + ADDQ $0x04, DX + +handleTiny8: + // n = n >> 1; CF = originalN & 8 + SHRQ $0x01, BX + JNC handleTiny16 + + // tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:] + MOVQ (DX), DI + ADDQ $0x08, DX + +handleTiny16: + // n = n >> 1; CF = originalN & 16 + // n == 0 now, otherwise we would have branched after comparing with tinyBufferSize + SHRQ $0x01, BX + JNC handleTinyFinish + ADDQ (DX), AX + ADCQ 8(DX), AX + +handleTinyFinish: + // CF should be included from the previous add, so we use ADCQ. + // If we arrived via the JNC above, then CF=0 due to the branch condition, + // so ADCQ will still produce the correct result. + ADCQ CX, AX + ADCQ SI, AX + ADCQ DI, AX + JMP foldAndReturn + +bufferIsNotTiny: + // skip all SIMD for small buffers + CMPQ BX, $0x00000100 + JGE startSIMD + + // Accumulate carries in this register. It is never expected to overflow. + XORQ SI, SI + + // We will perform an overlapped read for buffers with length not a multiple of 8. + // Overlapped in this context means some memory will be read twice, but a shift will + // eliminate the duplicated data. This extra read is performed at the end of the buffer to + // preserve any alignment that may exist for the start of the buffer. + MOVQ BX, CX + SHRQ $0x03, BX + ANDQ $0x07, CX + JZ handleRemaining8 + LEAQ (DX)(BX*8), DI + MOVQ -8(DI)(CX*1), DI + + // Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8) + SHLQ $0x03, CX + NEGQ CX + ADDQ $0x40, CX + SHRQ CL, DI + ADDQ DI, AX + ADCQ $0x00, SI + +handleRemaining8: + SHRQ $0x01, BX + JNC handleRemaining16 + ADDQ (DX), AX + ADCQ $0x00, SI + ADDQ $0x08, DX + +handleRemaining16: + SHRQ $0x01, BX + JNC handleRemaining32 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ $0x00, SI + ADDQ $0x10, DX + +handleRemaining32: + SHRQ $0x01, BX + JNC handleRemaining64 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ $0x00, SI + ADDQ $0x20, DX + +handleRemaining64: + SHRQ $0x01, BX + JNC handleRemaining128 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ $0x00, SI + ADDQ $0x40, DX + +handleRemaining128: + SHRQ $0x01, BX + JNC handleRemainingComplete + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ 64(DX), AX + ADCQ 72(DX), AX + ADCQ 80(DX), AX + ADCQ 88(DX), AX + ADCQ 96(DX), AX + ADCQ 104(DX), AX + ADCQ 112(DX), AX + ADCQ 120(DX), AX + ADCQ $0x00, SI + ADDQ $0x80, DX + +handleRemainingComplete: + ADDQ SI, AX + JMP foldAndReturn + +startSIMD: + VPXOR Y0, Y0, Y0 + VPXOR Y1, Y1, Y1 + VPXOR Y2, Y2, Y2 + VPXOR Y3, Y3, Y3 + MOVQ BX, CX + + // Update number of bytes remaining after the loop completes + ANDQ $0xff, BX + + // Number of 256 byte iterations + SHRQ $0x08, CX + JZ smallLoop + +bigLoop: + VPMOVZXWD (DX), Y4 + VPADDD Y4, Y0, Y0 + VPMOVZXWD 16(DX), Y4 + VPADDD Y4, Y1, Y1 + VPMOVZXWD 32(DX), Y4 + VPADDD Y4, Y2, Y2 + VPMOVZXWD 48(DX), Y4 + VPADDD Y4, Y3, Y3 + VPMOVZXWD 64(DX), Y4 + VPADDD Y4, Y0, Y0 + VPMOVZXWD 80(DX), Y4 + VPADDD Y4, Y1, Y1 + VPMOVZXWD 96(DX), Y4 + VPADDD Y4, Y2, Y2 + VPMOVZXWD 112(DX), Y4 + VPADDD Y4, Y3, Y3 + VPMOVZXWD 128(DX), Y4 + VPADDD Y4, Y0, Y0 + VPMOVZXWD 144(DX), Y4 + VPADDD Y4, Y1, Y1 + VPMOVZXWD 160(DX), Y4 + VPADDD Y4, Y2, Y2 + VPMOVZXWD 176(DX), Y4 + VPADDD Y4, Y3, Y3 + VPMOVZXWD 192(DX), Y4 + VPADDD Y4, Y0, Y0 + VPMOVZXWD 208(DX), Y4 + VPADDD Y4, Y1, Y1 + VPMOVZXWD 224(DX), Y4 + VPADDD Y4, Y2, Y2 + VPMOVZXWD 240(DX), Y4 + VPADDD Y4, Y3, Y3 + ADDQ $0x00000100, DX + DECQ CX + JNZ bigLoop + CMPQ BX, $0x10 + JLT doneSmallLoop + + // now read a single 16 byte unit of data at a time +smallLoop: + VPMOVZXWD (DX), Y4 + VPADDD Y4, Y0, Y0 + ADDQ $0x10, DX + SUBQ $0x10, BX + CMPQ BX, $0x10 + JGE smallLoop + +doneSmallLoop: + CMPQ BX, $0x00 + JE doneSIMD + + // There are between 1 and 15 bytes remaining. Perform an overlapped read. + LEAQ xmmLoadMasks<>+0(SB), CX + VMOVDQU -16(DX)(BX*1), X4 + VPAND -16(CX)(BX*8), X4, X4 + VPMOVZXWD X4, Y4 + VPADDD Y4, Y0, Y0 + +doneSIMD: + // Multi-chain loop is done, combine the accumulators + VPADDD Y1, Y0, Y0 + VPADDD Y2, Y0, Y0 + VPADDD Y3, Y0, Y0 + + // extract the YMM into a pair of XMM and sum them + VEXTRACTI128 $0x01, Y0, X1 + VPADDD X0, X1, X0 + + // extract the XMM into GP64 + VPEXTRQ $0x00, X0, CX + VPEXTRQ $0x01, X0, DX + + // no more AVX code, clear upper registers to avoid SSE slowdowns + VZEROUPPER + ADDQ CX, AX + ADCQ DX, AX + +foldAndReturn: + // add CF and fold + RORXQ $0x20, AX, CX + ADCL CX, AX + RORXL $0x10, AX, CX + ADCW CX, AX + ADCW $0x00, AX + XCHGB AH, AL + MOVW AX, ret+32(FP) + RET + +// func checksumSSE2(b []byte, initial uint16) uint16 +// Requires: SSE2 +TEXT ·checksumSSE2(SB), NOSPLIT|NOFRAME, $0-34 + MOVWQZX initial+24(FP), AX + XCHGB AH, AL + MOVQ b_base+0(FP), DX + MOVQ b_len+8(FP), BX + + // handle odd length buffers; they are difficult to handle in general + TESTQ $0x00000001, BX + JZ lengthIsEven + MOVBQZX -1(DX)(BX*1), CX + DECQ BX + ADDQ CX, AX + +lengthIsEven: + // handle tiny buffers (<=31 bytes) specially + CMPQ BX, $0x1f + JGT bufferIsNotTiny + XORQ CX, CX + XORQ SI, SI + XORQ DI, DI + + // shift twice to start because length is guaranteed to be even + // n = n >> 2; CF = originalN & 2 + SHRQ $0x02, BX + JNC handleTiny4 + + // tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:] + MOVWQZX (DX), CX + ADDQ $0x02, DX + +handleTiny4: + // n = n >> 1; CF = originalN & 4 + SHRQ $0x01, BX + JNC handleTiny8 + + // tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:] + MOVLQZX (DX), SI + ADDQ $0x04, DX + +handleTiny8: + // n = n >> 1; CF = originalN & 8 + SHRQ $0x01, BX + JNC handleTiny16 + + // tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:] + MOVQ (DX), DI + ADDQ $0x08, DX + +handleTiny16: + // n = n >> 1; CF = originalN & 16 + // n == 0 now, otherwise we would have branched after comparing with tinyBufferSize + SHRQ $0x01, BX + JNC handleTinyFinish + ADDQ (DX), AX + ADCQ 8(DX), AX + +handleTinyFinish: + // CF should be included from the previous add, so we use ADCQ. + // If we arrived via the JNC above, then CF=0 due to the branch condition, + // so ADCQ will still produce the correct result. + ADCQ CX, AX + ADCQ SI, AX + ADCQ DI, AX + JMP foldAndReturn + +bufferIsNotTiny: + // skip all SIMD for small buffers + CMPQ BX, $0x00000100 + JGE startSIMD + + // Accumulate carries in this register. It is never expected to overflow. + XORQ SI, SI + + // We will perform an overlapped read for buffers with length not a multiple of 8. + // Overlapped in this context means some memory will be read twice, but a shift will + // eliminate the duplicated data. This extra read is performed at the end of the buffer to + // preserve any alignment that may exist for the start of the buffer. + MOVQ BX, CX + SHRQ $0x03, BX + ANDQ $0x07, CX + JZ handleRemaining8 + LEAQ (DX)(BX*8), DI + MOVQ -8(DI)(CX*1), DI + + // Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8) + SHLQ $0x03, CX + NEGQ CX + ADDQ $0x40, CX + SHRQ CL, DI + ADDQ DI, AX + ADCQ $0x00, SI + +handleRemaining8: + SHRQ $0x01, BX + JNC handleRemaining16 + ADDQ (DX), AX + ADCQ $0x00, SI + ADDQ $0x08, DX + +handleRemaining16: + SHRQ $0x01, BX + JNC handleRemaining32 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ $0x00, SI + ADDQ $0x10, DX + +handleRemaining32: + SHRQ $0x01, BX + JNC handleRemaining64 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ $0x00, SI + ADDQ $0x20, DX + +handleRemaining64: + SHRQ $0x01, BX + JNC handleRemaining128 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ $0x00, SI + ADDQ $0x40, DX + +handleRemaining128: + SHRQ $0x01, BX + JNC handleRemainingComplete + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ 64(DX), AX + ADCQ 72(DX), AX + ADCQ 80(DX), AX + ADCQ 88(DX), AX + ADCQ 96(DX), AX + ADCQ 104(DX), AX + ADCQ 112(DX), AX + ADCQ 120(DX), AX + ADCQ $0x00, SI + ADDQ $0x80, DX + +handleRemainingComplete: + ADDQ SI, AX + JMP foldAndReturn + +startSIMD: + PXOR X0, X0 + PXOR X1, X1 + PXOR X2, X2 + PXOR X3, X3 + PXOR X4, X4 + MOVQ BX, CX + + // Update number of bytes remaining after the loop completes + ANDQ $0xff, BX + + // Number of 256 byte iterations + SHRQ $0x08, CX + JZ smallLoop + +bigLoop: + MOVOU (DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X2 + MOVOU 16(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X1 + PADDD X6, X3 + MOVOU 32(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X2 + PADDD X6, X0 + MOVOU 48(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X3 + PADDD X6, X1 + MOVOU 64(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X2 + MOVOU 80(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X1 + PADDD X6, X3 + MOVOU 96(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X2 + PADDD X6, X0 + MOVOU 112(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X3 + PADDD X6, X1 + MOVOU 128(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X2 + MOVOU 144(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X1 + PADDD X6, X3 + MOVOU 160(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X2 + PADDD X6, X0 + MOVOU 176(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X3 + PADDD X6, X1 + MOVOU 192(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X2 + MOVOU 208(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X1 + PADDD X6, X3 + MOVOU 224(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X2 + PADDD X6, X0 + MOVOU 240(DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X3 + PADDD X6, X1 + ADDQ $0x00000100, DX + DECQ CX + JNZ bigLoop + CMPQ BX, $0x10 + JLT doneSmallLoop + + // now read a single 16 byte unit of data at a time +smallLoop: + MOVOU (DX), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X1 + ADDQ $0x10, DX + SUBQ $0x10, BX + CMPQ BX, $0x10 + JGE smallLoop + +doneSmallLoop: + CMPQ BX, $0x00 + JE doneSIMD + + // There are between 1 and 15 bytes remaining. Perform an overlapped read. + LEAQ xmmLoadMasks<>+0(SB), CX + MOVOU -16(DX)(BX*1), X5 + PAND -16(CX)(BX*8), X5 + MOVOA X5, X6 + PUNPCKHWL X4, X5 + PUNPCKLWL X4, X6 + PADDD X5, X0 + PADDD X6, X1 + +doneSIMD: + // Multi-chain loop is done, combine the accumulators + PADDD X1, X0 + PADDD X2, X0 + PADDD X3, X0 + + // extract the XMM into GP64 + MOVQ X0, CX + PSRLDQ $0x08, X0 + MOVQ X0, DX + ADDQ CX, AX + ADCQ DX, AX + +foldAndReturn: + // add CF and fold + MOVL AX, CX + ADCQ $0x00, CX + SHRQ $0x20, AX + ADDQ CX, AX + MOVWQZX AX, CX + SHRQ $0x10, AX + ADDQ CX, AX + MOVW AX, CX + SHRQ $0x10, AX + ADDW CX, AX + ADCW $0x00, AX + XCHGB AH, AL + MOVW AX, ret+32(FP) + RET + +// func checksumAMD64(b []byte, initial uint16) uint16 +TEXT ·checksumAMD64(SB), NOSPLIT|NOFRAME, $0-34 + MOVWQZX initial+24(FP), AX + XCHGB AH, AL + MOVQ b_base+0(FP), DX + MOVQ b_len+8(FP), BX + + // handle odd length buffers; they are difficult to handle in general + TESTQ $0x00000001, BX + JZ lengthIsEven + MOVBQZX -1(DX)(BX*1), CX + DECQ BX + ADDQ CX, AX + +lengthIsEven: + // handle tiny buffers (<=31 bytes) specially + CMPQ BX, $0x1f + JGT bufferIsNotTiny + XORQ CX, CX + XORQ SI, SI + XORQ DI, DI + + // shift twice to start because length is guaranteed to be even + // n = n >> 2; CF = originalN & 2 + SHRQ $0x02, BX + JNC handleTiny4 + + // tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:] + MOVWQZX (DX), CX + ADDQ $0x02, DX + +handleTiny4: + // n = n >> 1; CF = originalN & 4 + SHRQ $0x01, BX + JNC handleTiny8 + + // tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:] + MOVLQZX (DX), SI + ADDQ $0x04, DX + +handleTiny8: + // n = n >> 1; CF = originalN & 8 + SHRQ $0x01, BX + JNC handleTiny16 + + // tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:] + MOVQ (DX), DI + ADDQ $0x08, DX + +handleTiny16: + // n = n >> 1; CF = originalN & 16 + // n == 0 now, otherwise we would have branched after comparing with tinyBufferSize + SHRQ $0x01, BX + JNC handleTinyFinish + ADDQ (DX), AX + ADCQ 8(DX), AX + +handleTinyFinish: + // CF should be included from the previous add, so we use ADCQ. + // If we arrived via the JNC above, then CF=0 due to the branch condition, + // so ADCQ will still produce the correct result. + ADCQ CX, AX + ADCQ SI, AX + ADCQ DI, AX + JMP foldAndReturn + +bufferIsNotTiny: + // Number of 256 byte iterations into loop counter + MOVQ BX, CX + + // Update number of bytes remaining after the loop completes + ANDQ $0xff, BX + SHRQ $0x08, CX + JZ startCleanup + CLC + XORQ SI, SI + XORQ DI, DI + XORQ R8, R8 + XORQ R9, R9 + XORQ R10, R10 + XORQ R11, R11 + XORQ R12, R12 + +bigLoop: + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ $0x00, SI + ADDQ 32(DX), DI + ADCQ 40(DX), DI + ADCQ 48(DX), DI + ADCQ 56(DX), DI + ADCQ $0x00, R8 + ADDQ 64(DX), R9 + ADCQ 72(DX), R9 + ADCQ 80(DX), R9 + ADCQ 88(DX), R9 + ADCQ $0x00, R10 + ADDQ 96(DX), R11 + ADCQ 104(DX), R11 + ADCQ 112(DX), R11 + ADCQ 120(DX), R11 + ADCQ $0x00, R12 + ADDQ 128(DX), AX + ADCQ 136(DX), AX + ADCQ 144(DX), AX + ADCQ 152(DX), AX + ADCQ $0x00, SI + ADDQ 160(DX), DI + ADCQ 168(DX), DI + ADCQ 176(DX), DI + ADCQ 184(DX), DI + ADCQ $0x00, R8 + ADDQ 192(DX), R9 + ADCQ 200(DX), R9 + ADCQ 208(DX), R9 + ADCQ 216(DX), R9 + ADCQ $0x00, R10 + ADDQ 224(DX), R11 + ADCQ 232(DX), R11 + ADCQ 240(DX), R11 + ADCQ 248(DX), R11 + ADCQ $0x00, R12 + ADDQ $0x00000100, DX + SUBQ $0x01, CX + JNZ bigLoop + ADDQ SI, AX + ADCQ DI, AX + ADCQ R8, AX + ADCQ R9, AX + ADCQ R10, AX + ADCQ R11, AX + ADCQ R12, AX + + // accumulate CF (twice, in case the first time overflows) + ADCQ $0x00, AX + ADCQ $0x00, AX + +startCleanup: + // Accumulate carries in this register. It is never expected to overflow. + XORQ SI, SI + + // We will perform an overlapped read for buffers with length not a multiple of 8. + // Overlapped in this context means some memory will be read twice, but a shift will + // eliminate the duplicated data. This extra read is performed at the end of the buffer to + // preserve any alignment that may exist for the start of the buffer. + MOVQ BX, CX + SHRQ $0x03, BX + ANDQ $0x07, CX + JZ handleRemaining8 + LEAQ (DX)(BX*8), DI + MOVQ -8(DI)(CX*1), DI + + // Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8) + SHLQ $0x03, CX + NEGQ CX + ADDQ $0x40, CX + SHRQ CL, DI + ADDQ DI, AX + ADCQ $0x00, SI + +handleRemaining8: + SHRQ $0x01, BX + JNC handleRemaining16 + ADDQ (DX), AX + ADCQ $0x00, SI + ADDQ $0x08, DX + +handleRemaining16: + SHRQ $0x01, BX + JNC handleRemaining32 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ $0x00, SI + ADDQ $0x10, DX + +handleRemaining32: + SHRQ $0x01, BX + JNC handleRemaining64 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ $0x00, SI + ADDQ $0x20, DX + +handleRemaining64: + SHRQ $0x01, BX + JNC handleRemaining128 + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ $0x00, SI + ADDQ $0x40, DX + +handleRemaining128: + SHRQ $0x01, BX + JNC handleRemainingComplete + ADDQ (DX), AX + ADCQ 8(DX), AX + ADCQ 16(DX), AX + ADCQ 24(DX), AX + ADCQ 32(DX), AX + ADCQ 40(DX), AX + ADCQ 48(DX), AX + ADCQ 56(DX), AX + ADCQ 64(DX), AX + ADCQ 72(DX), AX + ADCQ 80(DX), AX + ADCQ 88(DX), AX + ADCQ 96(DX), AX + ADCQ 104(DX), AX + ADCQ 112(DX), AX + ADCQ 120(DX), AX + ADCQ $0x00, SI + ADDQ $0x80, DX + +handleRemainingComplete: + ADDQ SI, AX + +foldAndReturn: + // add CF and fold + MOVL AX, CX + ADCQ $0x00, CX + SHRQ $0x20, AX + ADDQ CX, AX + MOVWQZX AX, CX + SHRQ $0x10, AX + ADDQ CX, AX + MOVW AX, CX + SHRQ $0x10, AX + ADDW CX, AX + ADCW $0x00, AX + XCHGB AH, AL + MOVW AX, ret+32(FP) + RET diff --git a/tun/checksum_generic.go b/tun/checksum_generic.go new file mode 100644 index 0000000..d0bfb69 --- /dev/null +++ b/tun/checksum_generic.go @@ -0,0 +1,15 @@ +// This file contains IP checksum algorithms that are not specific to any +// architecture and don't use hardware acceleration. + +//go:build !amd64 + +package tun + +import "strconv" + +func checksum(data []byte, initial uint16) uint16 { + if strconv.IntSize < 64 { + return checksumGeneric32(data, initial) + } + return checksumGeneric64(data, initial) +} diff --git a/tun/checksum_generic_test.go b/tun/checksum_generic_test.go index a0c9457..401a7bb 100644 --- a/tun/checksum_generic_test.go +++ b/tun/checksum_generic_test.go @@ -1,9 +1,26 @@ +//go:build !amd64 + package tun var archChecksumFuncs = []archChecksumDetails{ { - name: "generic", + name: "generic32", available: true, - f: checksum, + f: checksumGeneric32, + }, + { + name: "generic32Alternate", + available: true, + f: checksumGeneric32Alternate, + }, + { + name: "generic64", + available: true, + f: checksumGeneric64, + }, + { + name: "generic64Alternate", + available: true, + f: checksumGeneric64Alternate, }, } diff --git a/tun/checksum_test.go b/tun/checksum_test.go index b3a3583..c40efc9 100644 --- a/tun/checksum_test.go +++ b/tun/checksum_test.go @@ -18,7 +18,7 @@ import ( type archChecksumDetails struct { name string available bool - f func([]byte, uint64) uint16 + f func([]byte, uint16) uint16 } func deterministicRandomBytes(seed int64, length int) []byte { @@ -402,7 +402,7 @@ func TestChecksum(t *testing.T) { if !fd.available { t.Skip("can not run on this system") } - if got := fd.f(tt.data, uint64(tt.initial)); got != tt.want { + if got := fd.f(tt.data, tt.initial); got != tt.want { t.Errorf("%s checksum = %04x, want %04x", fd.name, got, tt.want) } }) @@ -444,20 +444,28 @@ func TestPseudoHeaderChecksumNoFold(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotNoFold := pseudoHeaderChecksumNoFold(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) - got := checksum([]byte{}, gotNoFold) - if got != tt.want { - t.Errorf("pseudoHeaderChecksumNoFold() = %x, folds to %04x, want %04x", gotNoFold, got, tt.want) - } - - got = header.PseudoHeaderChecksum( - tcpip.TransportProtocolNumber(tt.protocol), - tcpip.AddrFromSlice(tt.srcAddr), - tcpip.AddrFromSlice(tt.dstAddr), - tt.totalLen) - if got != tt.want { - t.Errorf("header.PseudoHeaderChecksum() = %04x, want %04x", got, tt.want) - } + t.Run("pseudoHeaderChecksum32", func(t *testing.T) { + got := pseudoHeaderChecksum32(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) + if got != tt.want { + t.Errorf("got %04x, want %04x", got, tt.want) + } + }) + t.Run("pseudoHeaderChecksum64", func(t *testing.T) { + got := pseudoHeaderChecksum64(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) + if got != tt.want { + t.Errorf("got %04x, want %04x", got, tt.want) + } + }) + t.Run("reference", func(t *testing.T) { + got := header.PseudoHeaderChecksum( + tcpip.TransportProtocolNumber(tt.protocol), + tcpip.AddrFromSlice(tt.srcAddr), + tcpip.AddrFromSlice(tt.dstAddr), + tt.totalLen) + if got != tt.want { + t.Errorf("got %04x, want %04x", got, tt.want) + } + }) }) } } @@ -482,7 +490,7 @@ func FuzzChecksum(f *testing.F) { if !fd.available { t.Skip("can not run on this system") } - if got := fd.f(data, uint64(initial)); got != want { + if got := fd.f(data, initial); got != want { t.Errorf("%s checksum = %04x, want %04x", fd.name, got, want) } }) @@ -491,7 +499,6 @@ func FuzzChecksum(f *testing.F) { } var result uint16 -var result64 uint64 func BenchmarkChecksum(b *testing.B) { offsets := []int{ // offsets from page alignment @@ -590,9 +597,16 @@ func BenchmarkPseudoHeaderChecksum(b *testing.B) { } for _, tt := range tests { b.Run(tt.name, func(b *testing.B) { - for i := 0; i < b.N; i++ { - result64 += pseudoHeaderChecksumNoFold(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) - } + b.Run("pseudoHeaderChecksum32", func(b *testing.B) { + for i := 0; i < b.N; i++ { + result += pseudoHeaderChecksum32(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) + } + }) + b.Run("pseudoHeaderChecksum64", func(b *testing.B) { + for i := 0; i < b.N; i++ { + result += pseudoHeaderChecksum64(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) + } + }) }) } } diff --git a/tun/generate_amd64.go b/tun/generate_amd64.go new file mode 100644 index 0000000..543e211 --- /dev/null +++ b/tun/generate_amd64.go @@ -0,0 +1,579 @@ +//go:build ignore + +//go:generate go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go + +package main + +import ( + "fmt" + "math" + "math/bits" + + . "github.com/mmcloughlin/avo/build" + "github.com/mmcloughlin/avo/operand" + "github.com/mmcloughlin/avo/reg" +) + +const checksumSignature = "func(b []byte, initial uint16) uint16" + +func loadParams() (accum, buf, n reg.GPVirtual) { + accum, buf, n = GP64(), GP64(), GP64() + Load(Param("initial"), accum) + XCHGB(accum.As8H(), accum.As8L()) + Load(Param("b").Base(), buf) + Load(Param("b").Len(), n) + return +} + +type simdStrategy int + +const ( + sse2 = iota + avx2 +) + +const tinyBufferSize = 31 // A buffer is tiny if it has at most 31 bytes. + +func generateSIMDChecksum(name, doc string, minSIMDSize, chains int, strategy simdStrategy) { + TEXT(name, NOSPLIT|NOFRAME, checksumSignature) + Pragma("noescape") + Doc(doc) + + accum64, buf, n := loadParams() + + handleOddLength(n, buf, accum64) + // no chance of overflow because accum64 was initialized by a uint16 and + // handleOddLength adds at most a uint8 + handleTinyBuffers(n, buf, accum64, operand.LabelRef("foldAndReturn"), operand.LabelRef("bufferIsNotTiny")) + Label("bufferIsNotTiny") + + const simdReadSize = 16 + + if minSIMDSize > tinyBufferSize { + Comment("skip all SIMD for small buffers") + if minSIMDSize <= math.MaxUint8 { + CMPQ(n, operand.U8(minSIMDSize)) + } else { + CMPQ(n, operand.U32(minSIMDSize)) + } + JGE(operand.LabelRef("startSIMD")) + + handleRemaining(n, buf, accum64, minSIMDSize-1) + JMP(operand.LabelRef("foldAndReturn")) + } + + Label("startSIMD") + + // chains is the number of accumulators to use. This improves speed via + // reduced data dependency. We combine the accumulators once when the big + // loop is complete. + simdAccumulate := make([]reg.VecVirtual, chains) + for i := range simdAccumulate { + switch strategy { + case sse2: + simdAccumulate[i] = XMM() + PXOR(simdAccumulate[i], simdAccumulate[i]) + case avx2: + simdAccumulate[i] = YMM() + VPXOR(simdAccumulate[i], simdAccumulate[i], simdAccumulate[i]) + } + } + var zero reg.VecVirtual + if strategy == sse2 { + zero = XMM() + PXOR(zero, zero) + } + + // Number of loads per big loop + const unroll = 16 + // Number of bytes + loopSize := uint64(simdReadSize * unroll) + if bits.Len64(loopSize) != bits.Len64(loopSize-1)+1 { + panic("loopSize is not a power of 2") + } + loopCount := GP64() + + MOVQ(n, loopCount) + Comment("Update number of bytes remaining after the loop completes") + ANDQ(operand.Imm(loopSize-1), n) + Comment(fmt.Sprintf("Number of %d byte iterations", loopSize)) + SHRQ(operand.Imm(uint64(bits.Len64(loopSize-1))), loopCount) + JZ(operand.LabelRef("smallLoop")) + Label("bigLoop") + for i := 0; i < unroll; i++ { + chain := i % chains + switch strategy { + case sse2: + sse2AccumulateStep(i*simdReadSize, buf, zero, simdAccumulate[chain], simdAccumulate[(chain+chains/2)%chains]) + case avx2: + avx2AccumulateStep(i*simdReadSize, buf, simdAccumulate[chain]) + } + } + ADDQ(operand.U32(loopSize), buf) + DECQ(loopCount) + JNZ(operand.LabelRef("bigLoop")) + + Label("bigCleanup") + + CMPQ(n, operand.Imm(uint64(simdReadSize))) + JLT(operand.LabelRef("doneSmallLoop")) + + Commentf("now read a single %d byte unit of data at a time", simdReadSize) + Label("smallLoop") + + switch strategy { + case sse2: + sse2AccumulateStep(0, buf, zero, simdAccumulate[0], simdAccumulate[1]) + case avx2: + avx2AccumulateStep(0, buf, simdAccumulate[0]) + } + ADDQ(operand.Imm(uint64(simdReadSize)), buf) + SUBQ(operand.Imm(uint64(simdReadSize)), n) + CMPQ(n, operand.Imm(uint64(simdReadSize))) + JGE(operand.LabelRef("smallLoop")) + + Label("doneSmallLoop") + CMPQ(n, operand.Imm(0)) + JE(operand.LabelRef("doneSIMD")) + + Commentf("There are between 1 and %d bytes remaining. Perform an overlapped read.", simdReadSize-1) + + maskDataPtr := GP64() + LEAQ(operand.NewDataAddr(operand.NewStaticSymbol("xmmLoadMasks"), 0), maskDataPtr) + dataAddr := operand.Mem{Index: n, Scale: 1, Base: buf, Disp: -simdReadSize} + // scale 8 is only correct here because n is guaranteed to be even and we + // do not generate masks for odd lengths + maskAddr := operand.Mem{Base: maskDataPtr, Index: n, Scale: 8, Disp: -16} + remainder := XMM() + + switch strategy { + case sse2: + MOVOU(dataAddr, remainder) + PAND(maskAddr, remainder) + low := XMM() + MOVOA(remainder, low) + PUNPCKHWL(zero, remainder) + PUNPCKLWL(zero, low) + PADDD(remainder, simdAccumulate[0]) + PADDD(low, simdAccumulate[1]) + case avx2: + // Note: this is very similar to the sse2 path but MOVOU has a massive + // performance hit if used here, presumably due to switching between SSE + // and AVX2 modes. + VMOVDQU(dataAddr, remainder) + VPAND(maskAddr, remainder, remainder) + + temp := YMM() + VPMOVZXWD(remainder, temp) + VPADDD(temp, simdAccumulate[0], simdAccumulate[0]) + } + + Label("doneSIMD") + + Comment("Multi-chain loop is done, combine the accumulators") + for i := range simdAccumulate { + if i == 0 { + continue + } + switch strategy { + case sse2: + PADDD(simdAccumulate[i], simdAccumulate[0]) + case avx2: + VPADDD(simdAccumulate[i], simdAccumulate[0], simdAccumulate[0]) + } + } + + if strategy == avx2 { + Comment("extract the YMM into a pair of XMM and sum them") + tmp := YMM() + VEXTRACTI128(operand.Imm(1), simdAccumulate[0], tmp.AsX()) + + xAccumulate := XMM() + VPADDD(simdAccumulate[0].AsX(), tmp.AsX(), xAccumulate) + simdAccumulate = []reg.VecVirtual{xAccumulate} + } + + Comment("extract the XMM into GP64") + low, high := GP64(), GP64() + switch strategy { + case sse2: + MOVQ(simdAccumulate[0], low) + PSRLDQ(operand.Imm(8), simdAccumulate[0]) + MOVQ(simdAccumulate[0], high) + case avx2: + VPEXTRQ(operand.Imm(0), simdAccumulate[0], low) + VPEXTRQ(operand.Imm(1), simdAccumulate[0], high) + + Comment("no more AVX code, clear upper registers to avoid SSE slowdowns") + VZEROUPPER() + } + ADDQ(low, accum64) + ADCQ(high, accum64) + Label("foldAndReturn") + foldWithCF(accum64, strategy == avx2) + XCHGB(accum64.As8H(), accum64.As8L()) + Store(accum64.As16(), ReturnIndex(0)) + RET() +} + +// handleOddLength generates instructions to incorporate the last byte into +// accum64 if the length is odd. CF may be set if accum64 overflows; be sure to +// handle that if overflow is possible. +func handleOddLength(n, buf, accum64 reg.GPVirtual) { + Comment("handle odd length buffers; they are difficult to handle in general") + TESTQ(operand.U32(1), n) + JZ(operand.LabelRef("lengthIsEven")) + + tmp := GP64() + MOVBQZX(operand.Mem{Base: buf, Index: n, Scale: 1, Disp: -1}, tmp) + DECQ(n) + ADDQ(tmp, accum64) + + Label("lengthIsEven") +} + +func sse2AccumulateStep(offset int, buf reg.GPVirtual, zero, accumulate1, accumulate2 reg.VecVirtual) { + high, low := XMM(), XMM() + MOVOU(operand.Mem{Disp: offset, Base: buf}, high) + MOVOA(high, low) + PUNPCKHWL(zero, high) + PUNPCKLWL(zero, low) + PADDD(high, accumulate1) + PADDD(low, accumulate2) +} + +func avx2AccumulateStep(offset int, buf reg.GPVirtual, accumulate reg.VecVirtual) { + tmp := YMM() + VPMOVZXWD(operand.Mem{Disp: offset, Base: buf}, tmp) + VPADDD(tmp, accumulate, accumulate) +} + +func generateAMD64Checksum(name, doc string) { + TEXT(name, NOSPLIT|NOFRAME, checksumSignature) + Pragma("noescape") + Doc(doc) + + accum64, buf, n := loadParams() + + handleOddLength(n, buf, accum64) + // no chance of overflow because accum64 was initialized by a uint16 and + // handleOddLength adds at most a uint8 + handleTinyBuffers(n, buf, accum64, operand.LabelRef("foldAndReturn"), operand.LabelRef("bufferIsNotTiny")) + Label("bufferIsNotTiny") + + const ( + // numChains is the number of accumulators and carry counters to use. + // This improves speed via reduced data dependency. We combine the + // accumulators and carry counters once when the loop is complete. + numChains = 4 + unroll = 32 // The number of 64-bit reads to perform per iteration of the loop. + loopSize = 8 * unroll // The number of bytes read per iteration of the loop. + ) + if bits.Len(loopSize) != bits.Len(loopSize-1)+1 { + panic("loopSize is not a power of 2") + } + loopCount := GP64() + + Comment(fmt.Sprintf("Number of %d byte iterations into loop counter", loopSize)) + MOVQ(n, loopCount) + Comment("Update number of bytes remaining after the loop completes") + ANDQ(operand.Imm(loopSize-1), n) + SHRQ(operand.Imm(uint64(bits.Len(loopSize-1))), loopCount) + JZ(operand.LabelRef("startCleanup")) + CLC() + + chains := make([]struct { + accum reg.GPVirtual + carries reg.GPVirtual + }, numChains) + for i := range chains { + if i == 0 { + chains[i].accum = accum64 + } else { + chains[i].accum = GP64() + XORQ(chains[i].accum, chains[i].accum) + } + chains[i].carries = GP64() + XORQ(chains[i].carries, chains[i].carries) + } + + Label("bigLoop") + + var curChain int + for i := 0; i < unroll; i++ { + // It is significantly faster to use a ADCX/ADOX pair instead of plain + // ADC, which results in two dependency chains, however those require + // ADX support, which was added after AVX2. If AVX2 is available, that's + // even better than ADCX/ADOX. + // + // However, multiple dependency chains using multiple accumulators and + // occasionally storing CF into temporary counters seems to work almost + // as well. + addr := operand.Mem{Disp: i * 8, Base: buf} + + if i%4 == 0 { + if i > 0 { + ADCQ(operand.Imm(0), chains[curChain].carries) + curChain = (curChain + 1) % len(chains) + } + ADDQ(addr, chains[curChain].accum) + } else { + ADCQ(addr, chains[curChain].accum) + } + } + ADCQ(operand.Imm(0), chains[curChain].carries) + ADDQ(operand.U32(loopSize), buf) + SUBQ(operand.Imm(1), loopCount) + JNZ(operand.LabelRef("bigLoop")) + for i := range chains { + if i == 0 { + ADDQ(chains[i].carries, accum64) + continue + } + ADCQ(chains[i].accum, accum64) + ADCQ(chains[i].carries, accum64) + } + + accumulateCF(accum64) + + Label("startCleanup") + handleRemaining(n, buf, accum64, loopSize-1) + Label("foldAndReturn") + foldWithCF(accum64, false) + + XCHGB(accum64.As8H(), accum64.As8L()) + Store(accum64.As16(), ReturnIndex(0)) + RET() +} + +// handleTinyBuffers computes checksums if the buffer length (the n parameter) +// is less than 32. After computing the checksum, a jump to returnLabel will +// be executed. Otherwise, if the buffer length is at least 32, nothing will be +// modified; a jump to continueLabel will be executed instead. +// +// When jumping to returnLabel, CF may be set and must be accommodated e.g. +// using foldWithCF or accumulateCF. +// +// Anecdotally, this appears to be faster than attempting to coordinate an +// overlapped read (which would also require special handling for buffers +// smaller than 8). +func handleTinyBuffers(n, buf, accum reg.GPVirtual, returnLabel, continueLabel operand.LabelRef) { + Comment("handle tiny buffers (<=31 bytes) specially") + CMPQ(n, operand.Imm(tinyBufferSize)) + JGT(continueLabel) + + tmp2, tmp4, tmp8 := GP64(), GP64(), GP64() + XORQ(tmp2, tmp2) + XORQ(tmp4, tmp4) + XORQ(tmp8, tmp8) + + Comment("shift twice to start because length is guaranteed to be even", + "n = n >> 2; CF = originalN & 2") + SHRQ(operand.Imm(2), n) + JNC(operand.LabelRef("handleTiny4")) + Comment("tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:]") + MOVWQZX(operand.Mem{Base: buf}, tmp2) + ADDQ(operand.Imm(2), buf) + + Label("handleTiny4") + Comment("n = n >> 1; CF = originalN & 4") + SHRQ(operand.Imm(1), n) + JNC(operand.LabelRef("handleTiny8")) + Comment("tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:]") + MOVLQZX(operand.Mem{Base: buf}, tmp4) + ADDQ(operand.Imm(4), buf) + + Label("handleTiny8") + Comment("n = n >> 1; CF = originalN & 8") + SHRQ(operand.Imm(1), n) + JNC(operand.LabelRef("handleTiny16")) + Comment("tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:]") + MOVQ(operand.Mem{Base: buf}, tmp8) + ADDQ(operand.Imm(8), buf) + + Label("handleTiny16") + Comment("n = n >> 1; CF = originalN & 16", + "n == 0 now, otherwise we would have branched after comparing with tinyBufferSize") + SHRQ(operand.Imm(1), n) + JNC(operand.LabelRef("handleTinyFinish")) + ADDQ(operand.Mem{Base: buf}, accum) + ADCQ(operand.Mem{Base: buf, Disp: 8}, accum) + + Label("handleTinyFinish") + Comment("CF should be included from the previous add, so we use ADCQ.", + "If we arrived via the JNC above, then CF=0 due to the branch condition,", + "so ADCQ will still produce the correct result.") + ADCQ(tmp2, accum) + ADCQ(tmp4, accum) + ADCQ(tmp8, accum) + + JMP(returnLabel) +} + +// handleRemaining generates a series of conditional unrolled additions, +// starting with 8 bytes long and doubling each time until the length reaches +// max. This is the reverse order of what may be intuitive, but makes the branch +// conditions convenient to compute: perform one right shift each time and test +// against CF. +// +// When done, CF may be set and must be accommodated e.g., using foldWithCF or +// accumulateCF. +// +// If n is not a multiple of 8, an extra 64 bit read at the end of the buffer +// will be performed, overlapping with data that will be read later. The +// duplicate data will be shifted off. +// +// The original buffer length must have been at least 8 bytes long, even if +// n < 8, otherwise this will access memory before the start of the buffer, +// which may be unsafe. +func handleRemaining(n, buf, accum64 reg.GPVirtual, max int) { + Comment("Accumulate carries in this register. It is never expected to overflow.") + carries := GP64() + XORQ(carries, carries) + + Comment("We will perform an overlapped read for buffers with length not a multiple of 8.", + "Overlapped in this context means some memory will be read twice, but a shift will", + "eliminate the duplicated data. This extra read is performed at the end of the buffer to", + "preserve any alignment that may exist for the start of the buffer.") + leftover := reg.RCX + MOVQ(n, leftover) + SHRQ(operand.Imm(3), n) // n is now the number of 64 bit reads remaining + ANDQ(operand.Imm(0x7), leftover) // leftover is now the number of bytes to read from the end + JZ(operand.LabelRef("handleRemaining8")) + endBuf := GP64() + // endBuf is the position near the end of the buffer that is just past the + // last multiple of 8: (buf + len(buf)) & ^0x7 + LEAQ(operand.Mem{Base: buf, Index: n, Scale: 8}, endBuf) + + overlapRead := GP64() + // equivalent to overlapRead = binary.LittleEndian.Uint64(buf[len(buf)-8:len(buf)]) + MOVQ(operand.Mem{Base: endBuf, Index: leftover, Scale: 1, Disp: -8}, overlapRead) + + Comment("Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8)") + SHLQ(operand.Imm(3), leftover) // leftover = leftover * 8 + NEGQ(leftover) // leftover = -leftover; this completes the (-leftoverBytes*8) part of the expression + ADDQ(operand.Imm(64), leftover) // now we have (64 - leftoverBytes*8) + SHRQ(reg.CL, overlapRead) // shift right by (64 - leftoverBytes*8); CL is the low 8 bits of leftover (set to RCX above) and variable shift only accepts CL + + ADDQ(overlapRead, accum64) + ADCQ(operand.Imm(0), carries) + + for curBytes := 8; curBytes <= max; curBytes *= 2 { + Label(fmt.Sprintf("handleRemaining%d", curBytes)) + SHRQ(operand.Imm(1), n) + if curBytes*2 <= max { + JNC(operand.LabelRef(fmt.Sprintf("handleRemaining%d", curBytes*2))) + } else { + JNC(operand.LabelRef("handleRemainingComplete")) + } + + numLoads := curBytes / 8 + for i := 0; i < numLoads; i++ { + addr := operand.Mem{Base: buf, Disp: i * 8} + // It is possible to add the multiple dependency chains trick here + // that generateAMD64Checksum uses but anecdotally it does not + // appear to outweigh the cost. + if i == 0 { + ADDQ(addr, accum64) + continue + } + ADCQ(addr, accum64) + } + ADCQ(operand.Imm(0), carries) + + if curBytes > math.MaxUint8 { + ADDQ(operand.U32(uint64(curBytes)), buf) + } else { + ADDQ(operand.U8(uint64(curBytes)), buf) + } + if curBytes*2 >= max { + continue + } + JMP(operand.LabelRef(fmt.Sprintf("handleRemaining%d", curBytes*2))) + } + Label("handleRemainingComplete") + ADDQ(carries, accum64) +} + +func accumulateCF(accum64 reg.GPVirtual) { + Comment("accumulate CF (twice, in case the first time overflows)") + // accum64 += CF + ADCQ(operand.Imm(0), accum64) + // accum64 += CF again if the previous add overflowed. The previous add was + // 0 or 1. If it overflowed, then accum64 == 0, so adding another 1 can + // never overflow. + ADCQ(operand.Imm(0), accum64) +} + +// foldWithCF generates instructions to fold accum (a GP64) into a 16-bit value +// according to ones-complement arithmetic. BMI2 instructions will be used if +// allowBMI2 is true (requires fewer instructions). +func foldWithCF(accum reg.GPVirtual, allowBMI2 bool) { + Comment("add CF and fold") + + // CF|accum max value starts as 0x1_ffff_ffff_ffff_ffff + + tmp := GP64() + if allowBMI2 { + // effectively, tmp = accum >> 32 (technically, this is a rotate) + RORXQ(operand.Imm(32), accum, tmp) + // accum as uint32 = uint32(accum) + uint32(tmp64) + CF; max value 0xffff_ffff + CF set + ADCL(tmp.As32(), accum.As32()) + // effectively, tmp64 as uint32 = uint32(accum) >> 16 (also a rotate) + RORXL(operand.Imm(16), accum.As32(), tmp.As32()) + // accum as uint16 = uint16(accum) + uint16(tmp) + CF; max value 0xffff + CF unset or 0xfffe + CF set + ADCW(tmp.As16(), accum.As16()) + } else { + // tmp = uint32(accum); max value 0xffff_ffff + // MOVL clears the upper 32 bits of a GP64 so this is equivalent to the + // non-existent MOVLQZX. + MOVL(accum.As32(), tmp.As32()) + // tmp += CF; max value 0x1_0000_0000, CF unset + ADCQ(operand.Imm(0), tmp) + // accum = accum >> 32; max value 0xffff_ffff + SHRQ(operand.Imm(32), accum) + // accum = accum + tmp; max value 0x1_ffff_ffff + CF unset + ADDQ(tmp, accum) + // tmp = uint16(accum); max value 0xffff + MOVWQZX(accum.As16(), tmp) + // accum = accum >> 16; max value 0x1_ffff + SHRQ(operand.Imm(16), accum) + // accum = accum + tmp; max value 0x2_fffe + CF unset + ADDQ(tmp, accum) + // tmp as uint16 = uint16(accum); max value 0xffff + MOVW(accum.As16(), tmp.As16()) + // accum = accum >> 16; max value 0x2 + SHRQ(operand.Imm(16), accum) + // accum as uint16 = uint16(accum) + uint16(tmp); max value 0xffff + CF unset or 0x2 + CF set + ADDW(tmp.As16(), accum.As16()) + } + // accum as uint16 += CF; will not overflow: either CF was 0 or accum <= 0xfffe + ADCW(operand.Imm(0), accum.As16()) +} + +func generateLoadMasks() { + var offset int + // xmmLoadMasks is a table of masks that can be used with PAND to zero all but the last N bytes in an XMM, N=2,4,6,8,10,12,14 + GLOBL("xmmLoadMasks", RODATA|NOPTR) + + for n := 2; n < 16; n += 2 { + var pattern [16]byte + for i := 0; i < len(pattern); i++ { + if i < len(pattern)-n { + pattern[i] = 0 + continue + } + pattern[i] = 0xff + } + DATA(offset, operand.String(pattern[:])) + offset += len(pattern) + } +} + +func main() { + generateLoadMasks() + generateSIMDChecksum("checksumAVX2", "checksumAVX2 computes an IP checksum using amd64 v3 instructions (AVX2, BMI2)", 256, 4, avx2) + generateSIMDChecksum("checksumSSE2", "checksumSSE2 computes an IP checksum using amd64 baseline instructions (SSE2)", 256, 4, sse2) + generateAMD64Checksum("checksumAMD64", "checksumAMD64 computes an IP checksum using amd64 baseline instructions") + Generate() +} diff --git a/tun/tcp_offload_linux.go b/tun/tcp_offload_linux.go index 6728823..b023bbd 100644 --- a/tun/tcp_offload_linux.go +++ b/tun/tcp_offload_linux.go @@ -260,8 +260,8 @@ func tcpChecksumValid(pkt []byte, iphLen uint8, isV6 bool) bool { addrSize = 16 } tcpTotalLen := uint16(len(pkt) - int(iphLen)) - tcpCSumNoFold := pseudoHeaderChecksumNoFold(unix.IPPROTO_TCP, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], tcpTotalLen) - return ^checksum(pkt[iphLen:], tcpCSumNoFold) == 0 + tcpCSum := pseudoHeaderChecksum(unix.IPPROTO_TCP, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], tcpTotalLen) + return ^checksum(pkt[iphLen:], tcpCSum) == 0 } // coalesceResult represents the result of attempting to coalesce two TCP @@ -532,7 +532,7 @@ func applyCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable, isV6 srcAddrAt := offset + addrOffset srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] - psum := pseudoHeaderChecksumNoFold(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) + psum := pseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) } else { hdr := virtioNetHdr{} @@ -643,8 +643,8 @@ func tcpTSO(in []byte, hdr virtioNetHdr, outBuffs [][]byte, sizes []int, outOffs // TCP checksum tcpHLen := int(hdr.hdrLen - hdr.csumStart) tcpLenForPseudo := uint16(tcpHLen + segmentDataLen) - tcpCSumNoFold := pseudoHeaderChecksumNoFold(unix.IPPROTO_TCP, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], tcpLenForPseudo) - tcpCSum := ^checksum(out[hdr.csumStart:totalLen], tcpCSumNoFold) + tcpCSum := pseudoHeaderChecksum(unix.IPPROTO_TCP, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], tcpLenForPseudo) + tcpCSum = ^checksum(out[hdr.csumStart:totalLen], tcpCSum) binary.BigEndian.PutUint16(out[hdr.csumStart+hdr.csumOffset:], tcpCSum) nextSegmentDataAt += int(hdr.gsoSize) @@ -658,6 +658,6 @@ func gsoNoneChecksum(in []byte, cSumStart, cSumOffset uint16) error { // checksum we compute. This is typically the pseudo-header checksum. initial := binary.BigEndian.Uint16(in[cSumAt:]) in[cSumAt], in[cSumAt+1] = 0, 0 - binary.BigEndian.PutUint16(in[cSumAt:], ^checksum(in[cSumStart:], uint64(initial))) + binary.BigEndian.PutUint16(in[cSumAt:], ^checksum(in[cSumStart:], initial)) return nil } From 6cd5922a04de9370d48cf742751d4cdcc6ce65e1 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 24 Aug 2023 14:39:23 -0700 Subject: [PATCH 10/60] all: adjust build tags for plan9 Signed-off-by: Brad Fitzpatrick --- conn/bind_std.go | 7 ++++++- conn/controlfns_unix.go | 2 +- conn/erraddrinuse.go | 14 ++++++++++++++ ipc/{uapi_wasm.go => uapi_fake.go} | 4 +++- rwcancel/rwcancel.go | 2 +- rwcancel/rwcancel_stub.go | 2 +- 6 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 conn/erraddrinuse.go rename ipc/{uapi_wasm.go => uapi_fake.go} (72%) diff --git a/conn/bind_std.go b/conn/bind_std.go index cc5cf23..428e528 100644 --- a/conn/bind_std.go +++ b/conn/bind_std.go @@ -136,6 +136,11 @@ func listenNet(network string, port int) (*net.UDPConn, int, error) { return conn.(*net.UDPConn), uaddr.Port, nil } +// errEADDRINUSE is syscall.EADDRINUSE, boxed into an interface once +// in erraddrinuse.go on almost all platforms. For other platforms, +// it's at least non-nil. +var errEADDRINUSE error = errors.New("") + func (s *StdNetBind) Open(uport uint16) ([]ReceiveFunc, uint16, error) { s.mu.Lock() defer s.mu.Unlock() @@ -162,7 +167,7 @@ again: // Listen on the same port as we're using for ipv4. v6conn, port, err = listenNet("udp6", port) - if uport == 0 && errors.Is(err, syscall.EADDRINUSE) && tries < 100 { + if uport == 0 && errors.Is(err, errEADDRINUSE) && tries < 100 { v4conn.Close() tries++ goto again diff --git a/conn/controlfns_unix.go b/conn/controlfns_unix.go index 91692c0..5cc4d98 100644 --- a/conn/controlfns_unix.go +++ b/conn/controlfns_unix.go @@ -1,4 +1,4 @@ -//go:build !windows && !linux && !wasm +//go:build !windows && !linux && !wasm && !plan9 /* SPDX-License-Identifier: MIT * diff --git a/conn/erraddrinuse.go b/conn/erraddrinuse.go new file mode 100644 index 0000000..751660e --- /dev/null +++ b/conn/erraddrinuse.go @@ -0,0 +1,14 @@ +//go:build !plan9 + +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + */ + +package conn + +import "syscall" + +func init() { + errEADDRINUSE = syscall.EADDRINUSE +} diff --git a/ipc/uapi_wasm.go b/ipc/uapi_fake.go similarity index 72% rename from ipc/uapi_wasm.go rename to ipc/uapi_fake.go index fa84684..e68863d 100644 --- a/ipc/uapi_wasm.go +++ b/ipc/uapi_fake.go @@ -1,3 +1,5 @@ +//go:build wasm || plan9 + /* SPDX-License-Identifier: MIT * * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. @@ -5,7 +7,7 @@ package ipc -// Made up sentinel error codes for {js,wasip1}/wasm. +// Made up sentinel error codes for {js,wasip1}/wasm, and plan9. const ( IpcErrorIO = 1 IpcErrorInvalid = 2 diff --git a/rwcancel/rwcancel.go b/rwcancel/rwcancel.go index e397c0e..dd649d4 100644 --- a/rwcancel/rwcancel.go +++ b/rwcancel/rwcancel.go @@ -1,4 +1,4 @@ -//go:build !windows && !wasm +//go:build !windows && !wasm && !plan9 /* SPDX-License-Identifier: MIT * diff --git a/rwcancel/rwcancel_stub.go b/rwcancel/rwcancel_stub.go index 2a98b2b..4623801 100644 --- a/rwcancel/rwcancel_stub.go +++ b/rwcancel/rwcancel_stub.go @@ -1,4 +1,4 @@ -//go:build windows || wasm +//go:build windows || wasm || plan9 // SPDX-License-Identifier: MIT From 202a3401e7f83a2f809fefaa8231d79e0e97edae Mon Sep 17 00:00:00 2001 From: Andrea Barisani Date: Wed, 30 Aug 2023 14:05:01 +0200 Subject: [PATCH 11/60] adjust build tags for tamago --- conn/controlfns_unix.go | 2 +- ipc/uapi_tamago.go | 17 +++++++++++++++++ rwcancel/rwcancel.go | 2 +- rwcancel/rwcancel_stub.go | 2 +- 4 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 ipc/uapi_tamago.go diff --git a/conn/controlfns_unix.go b/conn/controlfns_unix.go index 5cc4d98..144a880 100644 --- a/conn/controlfns_unix.go +++ b/conn/controlfns_unix.go @@ -1,4 +1,4 @@ -//go:build !windows && !linux && !wasm && !plan9 +//go:build !windows && !linux && !wasm && !plan9 && !tamago /* SPDX-License-Identifier: MIT * diff --git a/ipc/uapi_tamago.go b/ipc/uapi_tamago.go new file mode 100644 index 0000000..85a725a --- /dev/null +++ b/ipc/uapi_tamago.go @@ -0,0 +1,17 @@ +//go:build tamago + +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + */ + +package ipc + +// Made up sentinel error codes for tamago platform. +const ( + IpcErrorIO = 1 + IpcErrorInvalid = 2 + IpcErrorPortInUse = 3 + IpcErrorUnknown = 4 + IpcErrorProtocol = 5 +) diff --git a/rwcancel/rwcancel.go b/rwcancel/rwcancel.go index dd649d4..ceb87e4 100644 --- a/rwcancel/rwcancel.go +++ b/rwcancel/rwcancel.go @@ -1,4 +1,4 @@ -//go:build !windows && !wasm && !plan9 +//go:build !windows && !wasm && !plan9 && !tamago /* SPDX-License-Identifier: MIT * diff --git a/rwcancel/rwcancel_stub.go b/rwcancel/rwcancel_stub.go index 4623801..60ae9af 100644 --- a/rwcancel/rwcancel_stub.go +++ b/rwcancel/rwcancel_stub.go @@ -1,4 +1,4 @@ -//go:build windows || wasm || plan9 +//go:build windows || wasm || plan9 || tamago // SPDX-License-Identifier: MIT From 2f6748dc88e777ff6eed22f5ce5d7658c6bb9410 Mon Sep 17 00:00:00 2001 From: James Tucker Date: Wed, 27 Sep 2023 14:52:21 -0700 Subject: [PATCH 12/60] tun: fix crash when ForceMTU is called after close Close closes the events channel, resulting in a panic from send on closed channel. Reported-By: Brad Fitzpatrick Link: https://github.com/tailscale/tailscale/issues/9555 Signed-off-by: James Tucker --- tun/tun_windows.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tun/tun_windows.go b/tun/tun_windows.go index 0cb4ce1..34f2980 100644 --- a/tun/tun_windows.go +++ b/tun/tun_windows.go @@ -127,6 +127,9 @@ func (tun *NativeTun) MTU() (int, error) { // TODO: This is a temporary hack. We really need to be monitoring the interface in real time and adapting to MTU changes. func (tun *NativeTun) ForceMTU(mtu int) { + if tun.close.Load() { + return + } update := tun.forcedMTU != mtu tun.forcedMTU = mtu if update { From 24f8d7c9e7312590e1f2d8e0ca02c30b9d88e91a Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Fri, 6 Oct 2023 17:16:49 -0700 Subject: [PATCH 13/60] tun: implement UDP GSO/GRO for Linux Signed-off-by: Jordan Whited --- ...{tcp_offload_linux.go => offload_linux.go} | 595 ++++++++++++++---- ...ad_linux_test.go => offload_linux_test.go} | 405 ++++++++++-- ...65e4830d6dc087cab24cd1e154c2e790589a309b77 | 8 - ...6784411a8ce2e8e03aa3384105e581f2c67494700d | 8 - tun/tun_linux.go | 71 ++- 5 files changed, 849 insertions(+), 238 deletions(-) rename tun/{tcp_offload_linux.go => offload_linux.go} (50%) rename tun/{tcp_offload_linux_test.go => offload_linux_test.go} (52%) delete mode 100644 tun/testdata/fuzz/Fuzz_handleGRO/032aec0105f26f709c118365e4830d6dc087cab24cd1e154c2e790589a309b77 delete mode 100644 tun/testdata/fuzz/Fuzz_handleGRO/0da283f9a2098dec30d1c86784411a8ce2e8e03aa3384105e581f2c67494700d diff --git a/tun/tcp_offload_linux.go b/tun/offload_linux.go similarity index 50% rename from tun/tcp_offload_linux.go rename to tun/offload_linux.go index b023bbd..54461cc 100644 --- a/tun/tcp_offload_linux.go +++ b/tun/offload_linux.go @@ -57,22 +57,23 @@ const ( virtioNetHdrLen = int(unsafe.Sizeof(virtioNetHdr{})) ) -// flowKey represents the key for a flow. -type flowKey struct { +// tcpFlowKey represents the key for a TCP flow. +type tcpFlowKey struct { srcAddr, dstAddr [16]byte srcPort, dstPort uint16 rxAck uint32 // varying ack values should not be coalesced. Treat them as separate flows. + isV6 bool } -// tcpGROTable holds flow and coalescing information for the purposes of GRO. +// tcpGROTable holds flow and coalescing information for the purposes of TCP GRO. type tcpGROTable struct { - itemsByFlow map[flowKey][]tcpGROItem + itemsByFlow map[tcpFlowKey][]tcpGROItem itemsPool [][]tcpGROItem } func newTCPGROTable() *tcpGROTable { t := &tcpGROTable{ - itemsByFlow: make(map[flowKey][]tcpGROItem, conn.IdealBatchSize), + itemsByFlow: make(map[tcpFlowKey][]tcpGROItem, conn.IdealBatchSize), itemsPool: make([][]tcpGROItem, conn.IdealBatchSize), } for i := range t.itemsPool { @@ -81,14 +82,15 @@ func newTCPGROTable() *tcpGROTable { return t } -func newFlowKey(pkt []byte, srcAddr, dstAddr, tcphOffset int) flowKey { - key := flowKey{} - addrSize := dstAddr - srcAddr - copy(key.srcAddr[:], pkt[srcAddr:dstAddr]) - copy(key.dstAddr[:], pkt[dstAddr:dstAddr+addrSize]) +func newTCPFlowKey(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset int) tcpFlowKey { + key := tcpFlowKey{} + addrSize := dstAddrOffset - srcAddrOffset + copy(key.srcAddr[:], pkt[srcAddrOffset:dstAddrOffset]) + copy(key.dstAddr[:], pkt[dstAddrOffset:dstAddrOffset+addrSize]) key.srcPort = binary.BigEndian.Uint16(pkt[tcphOffset:]) key.dstPort = binary.BigEndian.Uint16(pkt[tcphOffset+2:]) key.rxAck = binary.BigEndian.Uint32(pkt[tcphOffset+8:]) + key.isV6 = addrSize == 16 return key } @@ -96,7 +98,7 @@ func newFlowKey(pkt []byte, srcAddr, dstAddr, tcphOffset int) flowKey { // returning the packets found for the flow, or inserting a new one if none // is found. func (t *tcpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen, bufsIndex int) ([]tcpGROItem, bool) { - key := newFlowKey(pkt, srcAddrOffset, dstAddrOffset, tcphOffset) + key := newTCPFlowKey(pkt, srcAddrOffset, dstAddrOffset, tcphOffset) items, ok := t.itemsByFlow[key] if ok { return items, ok @@ -108,7 +110,7 @@ func (t *tcpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, t // insert an item in the table for the provided packet and packet metadata. func (t *tcpGROTable) insert(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen, bufsIndex int) { - key := newFlowKey(pkt, srcAddrOffset, dstAddrOffset, tcphOffset) + key := newTCPFlowKey(pkt, srcAddrOffset, dstAddrOffset, tcphOffset) item := tcpGROItem{ key: key, bufsIndex: uint16(bufsIndex), @@ -131,7 +133,7 @@ func (t *tcpGROTable) updateAt(item tcpGROItem, i int) { items[i] = item } -func (t *tcpGROTable) deleteAt(key flowKey, i int) { +func (t *tcpGROTable) deleteAt(key tcpFlowKey, i int) { items, _ := t.itemsByFlow[key] items = append(items[:i], items[i+1:]...) t.itemsByFlow[key] = items @@ -140,7 +142,7 @@ func (t *tcpGROTable) deleteAt(key flowKey, i int) { // tcpGROItem represents bookkeeping data for a TCP packet during the lifetime // of a GRO evaluation across a vector of packets. type tcpGROItem struct { - key flowKey + key tcpFlowKey sentSeq uint32 // the sequence number bufsIndex uint16 // the index into the original bufs slice numMerged uint16 // the number of packets merged into this item @@ -164,6 +166,103 @@ func (t *tcpGROTable) reset() { } } +// udpFlowKey represents the key for a UDP flow. +type udpFlowKey struct { + srcAddr, dstAddr [16]byte + srcPort, dstPort uint16 + isV6 bool +} + +// udpGROTable holds flow and coalescing information for the purposes of UDP GRO. +type udpGROTable struct { + itemsByFlow map[udpFlowKey][]udpGROItem + itemsPool [][]udpGROItem +} + +func newUDPGROTable() *udpGROTable { + u := &udpGROTable{ + itemsByFlow: make(map[udpFlowKey][]udpGROItem, conn.IdealBatchSize), + itemsPool: make([][]udpGROItem, conn.IdealBatchSize), + } + for i := range u.itemsPool { + u.itemsPool[i] = make([]udpGROItem, 0, conn.IdealBatchSize) + } + return u +} + +func newUDPFlowKey(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset int) udpFlowKey { + key := udpFlowKey{} + addrSize := dstAddrOffset - srcAddrOffset + copy(key.srcAddr[:], pkt[srcAddrOffset:dstAddrOffset]) + copy(key.dstAddr[:], pkt[dstAddrOffset:dstAddrOffset+addrSize]) + key.srcPort = binary.BigEndian.Uint16(pkt[udphOffset:]) + key.dstPort = binary.BigEndian.Uint16(pkt[udphOffset+2:]) + key.isV6 = addrSize == 16 + return key +} + +// lookupOrInsert looks up a flow for the provided packet and metadata, +// returning the packets found for the flow, or inserting a new one if none +// is found. +func (u *udpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset, bufsIndex int) ([]udpGROItem, bool) { + key := newUDPFlowKey(pkt, srcAddrOffset, dstAddrOffset, udphOffset) + items, ok := u.itemsByFlow[key] + if ok { + return items, ok + } + // TODO: insert() performs another map lookup. This could be rearranged to avoid. + u.insert(pkt, srcAddrOffset, dstAddrOffset, udphOffset, bufsIndex, false) + return nil, false +} + +// insert an item in the table for the provided packet and packet metadata. +func (u *udpGROTable) insert(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset, bufsIndex int, cSumKnownInvalid bool) { + key := newUDPFlowKey(pkt, srcAddrOffset, dstAddrOffset, udphOffset) + item := udpGROItem{ + key: key, + bufsIndex: uint16(bufsIndex), + gsoSize: uint16(len(pkt[udphOffset+udphLen:])), + iphLen: uint8(udphOffset), + cSumKnownInvalid: cSumKnownInvalid, + } + items, ok := u.itemsByFlow[key] + if !ok { + items = u.newItems() + } + items = append(items, item) + u.itemsByFlow[key] = items +} + +func (u *udpGROTable) updateAt(item udpGROItem, i int) { + items, _ := u.itemsByFlow[item.key] + items[i] = item +} + +// udpGROItem represents bookkeeping data for a UDP packet during the lifetime +// of a GRO evaluation across a vector of packets. +type udpGROItem struct { + key udpFlowKey + bufsIndex uint16 // the index into the original bufs slice + numMerged uint16 // the number of packets merged into this item + gsoSize uint16 // payload size + iphLen uint8 // ip header len + cSumKnownInvalid bool // UDP header checksum validity; a false value DOES NOT imply valid, just unknown. +} + +func (u *udpGROTable) newItems() []udpGROItem { + var items []udpGROItem + items, u.itemsPool = u.itemsPool[len(u.itemsPool)-1], u.itemsPool[:len(u.itemsPool)-1] + return items +} + +func (u *udpGROTable) reset() { + for k, items := range u.itemsByFlow { + items = items[:0] + u.itemsPool = append(u.itemsPool, items) + delete(u.itemsByFlow, k) + } +} + // canCoalesce represents the outcome of checking if two TCP packets are // candidates for coalescing. type canCoalesce int @@ -174,6 +273,61 @@ const ( coalesceAppend canCoalesce = 1 ) +// ipHeadersCanCoalesce returns true if the IP headers found in pktA and pktB +// meet all requirements to be merged as part of a GRO operation, otherwise it +// returns false. +func ipHeadersCanCoalesce(pktA, pktB []byte) bool { + if len(pktA) < 9 || len(pktB) < 9 { + return false + } + if pktA[0]>>4 == 6 { + if pktA[0] != pktB[0] || pktA[1]>>4 != pktB[1]>>4 { + // cannot coalesce with unequal Traffic class values + return false + } + if pktA[7] != pktB[7] { + // cannot coalesce with unequal Hop limit values + return false + } + } else { + if pktA[1] != pktB[1] { + // cannot coalesce with unequal ToS values + return false + } + if pktA[6]>>5 != pktB[6]>>5 { + // cannot coalesce with unequal DF or reserved bits. MF is checked + // further up the stack. + return false + } + if pktA[8] != pktB[8] { + // cannot coalesce with unequal TTL values + return false + } + } + return true +} + +// udpPacketsCanCoalesce evaluates if pkt can be coalesced with the packet +// described by item. iphLen and gsoSize describe pkt. bufs is the vector of +// packets involved in the current GRO evaluation. bufsOffset is the offset at +// which packet data begins within bufs. +func udpPacketsCanCoalesce(pkt []byte, iphLen uint8, gsoSize uint16, item udpGROItem, bufs [][]byte, bufsOffset int) canCoalesce { + pktTarget := bufs[item.bufsIndex][bufsOffset:] + if !ipHeadersCanCoalesce(pkt, pktTarget) { + return coalesceUnavailable + } + if len(pktTarget[iphLen+udphLen:])%int(item.gsoSize) != 0 { + // A smaller than gsoSize packet has been appended previously. + // Nothing can come after a smaller packet on the end. + return coalesceUnavailable + } + if gsoSize > item.gsoSize { + // We cannot have a larger packet following a smaller one. + return coalesceUnavailable + } + return coalesceAppend +} + // tcpPacketsCanCoalesce evaluates if pkt can be coalesced with the packet // described by item. This function makes considerations that match the kernel's // GRO self tests, which can be found in tools/testing/selftests/net/gro.c. @@ -189,29 +343,8 @@ func tcpPacketsCanCoalesce(pkt []byte, iphLen, tcphLen uint8, seq uint32, pshSet return coalesceUnavailable } } - if pkt[0]>>4 == 6 { - if pkt[0] != pktTarget[0] || pkt[1]>>4 != pktTarget[1]>>4 { - // cannot coalesce with unequal Traffic class values - return coalesceUnavailable - } - if pkt[7] != pktTarget[7] { - // cannot coalesce with unequal Hop limit values - return coalesceUnavailable - } - } else { - if pkt[1] != pktTarget[1] { - // cannot coalesce with unequal ToS values - return coalesceUnavailable - } - if pkt[6]>>5 != pktTarget[6]>>5 { - // cannot coalesce with unequal DF or reserved bits. MF is checked - // further up the stack. - return coalesceUnavailable - } - if pkt[8] != pktTarget[8] { - // cannot coalesce with unequal TTL values - return coalesceUnavailable - } + if !ipHeadersCanCoalesce(pkt, pktTarget) { + return coalesceUnavailable } // seq adjacency lhsLen := item.gsoSize @@ -252,16 +385,16 @@ func tcpPacketsCanCoalesce(pkt []byte, iphLen, tcphLen uint8, seq uint32, pshSet return coalesceUnavailable } -func tcpChecksumValid(pkt []byte, iphLen uint8, isV6 bool) bool { +func checksumValid(pkt []byte, iphLen, proto uint8, isV6 bool) bool { srcAddrAt := ipv4SrcAddrOffset addrSize := 4 if isV6 { srcAddrAt = ipv6SrcAddrOffset addrSize = 16 } - tcpTotalLen := uint16(len(pkt) - int(iphLen)) - tcpCSum := pseudoHeaderChecksum(unix.IPPROTO_TCP, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], tcpTotalLen) - return ^checksum(pkt[iphLen:], tcpCSum) == 0 + lenForPseudo := uint16(len(pkt) - int(iphLen)) + cSum := pseudoHeaderChecksum(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo) + return ^checksum(pkt[iphLen:], cSum) == 0 } // coalesceResult represents the result of attempting to coalesce two TCP @@ -276,8 +409,36 @@ const ( coalesceSuccess ) +// coalesceUDPPackets attempts to coalesce pkt with the packet described by +// item, and returns the outcome. +func coalesceUDPPackets(pkt []byte, item *udpGROItem, bufs [][]byte, bufsOffset int, isV6 bool) coalesceResult { + pktHead := bufs[item.bufsIndex][bufsOffset:] // the packet that will end up at the front + headersLen := item.iphLen + udphLen + coalescedLen := len(bufs[item.bufsIndex][bufsOffset:]) + len(pkt) - int(headersLen) + + if cap(pktHead)-bufsOffset < coalescedLen { + // We don't want to allocate a new underlying array if capacity is + // too small. + return coalesceInsufficientCap + } + if item.numMerged == 0 { + if item.cSumKnownInvalid || !checksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, unix.IPPROTO_UDP, isV6) { + return coalesceItemInvalidCSum + } + } + if !checksumValid(pkt, item.iphLen, unix.IPPROTO_UDP, isV6) { + return coalescePktInvalidCSum + } + extendBy := len(pkt) - int(headersLen) + bufs[item.bufsIndex] = append(bufs[item.bufsIndex], make([]byte, extendBy)...) + copy(bufs[item.bufsIndex][bufsOffset+len(pktHead):], pkt[headersLen:]) + + item.numMerged++ + return coalesceSuccess +} + // coalesceTCPPackets attempts to coalesce pkt with the packet described by -// item, returning the outcome. This function may swap bufs elements in the +// item, and returns the outcome. This function may swap bufs elements in the // event of a prepend as item's bufs index is already being tracked for writing // to a Device. func coalesceTCPPackets(mode canCoalesce, pkt []byte, pktBuffsIndex int, gsoSize uint16, seq uint32, pshSet bool, item *tcpGROItem, bufs [][]byte, bufsOffset int, isV6 bool) coalesceResult { @@ -297,11 +458,11 @@ func coalesceTCPPackets(mode canCoalesce, pkt []byte, pktBuffsIndex int, gsoSize return coalescePSHEnding } if item.numMerged == 0 { - if !tcpChecksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, isV6) { + if !checksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, unix.IPPROTO_TCP, isV6) { return coalesceItemInvalidCSum } } - if !tcpChecksumValid(pkt, item.iphLen, isV6) { + if !checksumValid(pkt, item.iphLen, unix.IPPROTO_TCP, isV6) { return coalescePktInvalidCSum } item.sentSeq = seq @@ -319,11 +480,11 @@ func coalesceTCPPackets(mode canCoalesce, pkt []byte, pktBuffsIndex int, gsoSize return coalesceInsufficientCap } if item.numMerged == 0 { - if !tcpChecksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, isV6) { + if !checksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, unix.IPPROTO_TCP, isV6) { return coalesceItemInvalidCSum } } - if !tcpChecksumValid(pkt, item.iphLen, isV6) { + if !checksumValid(pkt, item.iphLen, unix.IPPROTO_TCP, isV6) { return coalescePktInvalidCSum } if pshSet { @@ -354,52 +515,52 @@ const ( maxUint16 = 1<<16 - 1 ) -type tcpGROResult int +type groResult int const ( - tcpGROResultNoop tcpGROResult = iota - tcpGROResultTableInsert - tcpGROResultCoalesced + groResultNoop groResult = iota + groResultTableInsert + groResultCoalesced ) // tcpGRO evaluates the TCP packet at pktI in bufs for coalescing with -// existing packets tracked in table. It returns a tcpGROResultNoop when no -// action was taken, tcpGROResultTableInsert when the evaluated packet was -// inserted into table, and tcpGROResultCoalesced when the evaluated packet was +// existing packets tracked in table. It returns a groResultNoop when no +// action was taken, groResultTableInsert when the evaluated packet was +// inserted into table, and groResultCoalesced when the evaluated packet was // coalesced with another packet in table. -func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) tcpGROResult { +func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) groResult { pkt := bufs[pktI][offset:] if len(pkt) > maxUint16 { // A valid IPv4 or IPv6 packet will never exceed this. - return tcpGROResultNoop + return groResultNoop } iphLen := int((pkt[0] & 0x0F) * 4) if isV6 { iphLen = 40 ipv6HPayloadLen := int(binary.BigEndian.Uint16(pkt[4:])) if ipv6HPayloadLen != len(pkt)-iphLen { - return tcpGROResultNoop + return groResultNoop } } else { totalLen := int(binary.BigEndian.Uint16(pkt[2:])) if totalLen != len(pkt) { - return tcpGROResultNoop + return groResultNoop } } if len(pkt) < iphLen { - return tcpGROResultNoop + return groResultNoop } tcphLen := int((pkt[iphLen+12] >> 4) * 4) if tcphLen < 20 || tcphLen > 60 { - return tcpGROResultNoop + return groResultNoop } if len(pkt) < iphLen+tcphLen { - return tcpGROResultNoop + return groResultNoop } if !isV6 { if pkt[6]&ipv4FlagMoreFragments != 0 || pkt[6]<<3 != 0 || pkt[7] != 0 { // no GRO support for fragmented segments for now - return tcpGROResultNoop + return groResultNoop } } tcpFlags := pkt[iphLen+tcpFlagsOffset] @@ -407,14 +568,14 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) // not a candidate if any non-ACK flags (except PSH+ACK) are set if tcpFlags != tcpFlagACK { if pkt[iphLen+tcpFlagsOffset] != tcpFlagACK|tcpFlagPSH { - return tcpGROResultNoop + return groResultNoop } pshSet = true } gsoSize := uint16(len(pkt) - tcphLen - iphLen) // not a candidate if payload len is 0 if gsoSize < 1 { - return tcpGROResultNoop + return groResultNoop } seq := binary.BigEndian.Uint32(pkt[iphLen+4:]) srcAddrOffset := ipv4SrcAddrOffset @@ -425,7 +586,7 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) } items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, pktI) if !existing { - return tcpGROResultNoop + return groResultTableInsert } for i := len(items) - 1; i >= 0; i-- { // In the best case of packets arriving in order iterating in reverse is @@ -443,54 +604,25 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) switch result { case coalesceSuccess: table.updateAt(item, i) - return tcpGROResultCoalesced + return groResultCoalesced case coalesceItemInvalidCSum: // delete the item with an invalid csum table.deleteAt(item.key, i) case coalescePktInvalidCSum: // no point in inserting an item that we can't coalesce - return tcpGROResultNoop + return groResultNoop default: } } } // failed to coalesce with any other packets; store the item in the flow table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, pktI) - return tcpGROResultTableInsert + return groResultTableInsert } -func isTCP4NoIPOptions(b []byte) bool { - if len(b) < 40 { - return false - } - if b[0]>>4 != 4 { - return false - } - if b[0]&0x0F != 5 { - return false - } - if b[9] != unix.IPPROTO_TCP { - return false - } - return true -} - -func isTCP6NoEH(b []byte) bool { - if len(b) < 60 { - return false - } - if b[0]>>4 != 6 { - return false - } - if b[6] != unix.IPPROTO_TCP { - return false - } - return true -} - -// applyCoalesceAccounting updates bufs to account for coalescing based on the +// applyTCPCoalesceAccounting updates bufs to account for coalescing based on the // metadata found in table. -func applyCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable, isV6 bool) error { +func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) error { for _, items := range table.itemsByFlow { for _, item := range items { if item.numMerged > 0 { @@ -505,7 +637,7 @@ func applyCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable, isV6 // Recalculate the total len (IPv4) or payload len (IPv6). // Recalculate the (IPv4) header checksum. - if isV6 { + if item.key.isV6 { hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV6 binary.BigEndian.PutUint16(pkt[4:], uint16(len(pkt))-uint16(item.iphLen)) // set new IPv6 header payload len } else { @@ -525,7 +657,7 @@ func applyCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable, isV6 // this with computation of the tcp header and payload checksum. addrLen := 4 addrOffset := ipv4SrcAddrOffset - if isV6 { + if item.key.isV6 { addrLen = 16 addrOffset = ipv6SrcAddrOffset } @@ -546,54 +678,244 @@ func applyCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable, isV6 return nil } +// applyUDPCoalesceAccounting updates bufs to account for coalescing based on the +// metadata found in table. +func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) error { + for _, items := range table.itemsByFlow { + for _, item := range items { + if item.numMerged > 0 { + hdr := virtioNetHdr{ + flags: unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, // this turns into CHECKSUM_PARTIAL in the skb + hdrLen: uint16(item.iphLen + udphLen), + gsoSize: item.gsoSize, + csumStart: uint16(item.iphLen), + csumOffset: 6, + } + pkt := bufs[item.bufsIndex][offset:] + + // Recalculate the total len (IPv4) or payload len (IPv6). + // Recalculate the (IPv4) header checksum. + hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_UDP_L4 + if item.key.isV6 { + binary.BigEndian.PutUint16(pkt[4:], uint16(len(pkt))-uint16(item.iphLen)) // set new IPv6 header payload len + } else { + pkt[10], pkt[11] = 0, 0 + binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length + iphCSum := ^checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum + binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field + } + err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) + if err != nil { + return err + } + + // Recalculate the UDP len field value + binary.BigEndian.PutUint16(pkt[item.iphLen+4:], uint16(len(pkt[item.iphLen:]))) + + // Calculate the pseudo header checksum and place it at the UDP + // checksum offset. Downstream checksum offloading will combine + // this with computation of the udp header and payload checksum. + addrLen := 4 + addrOffset := ipv4SrcAddrOffset + if item.key.isV6 { + addrLen = 16 + addrOffset = ipv6SrcAddrOffset + } + srcAddrAt := offset + addrOffset + srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] + dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] + psum := pseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) + binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) + } else { + hdr := virtioNetHdr{} + err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) + if err != nil { + return err + } + } + } + } + return nil +} + +type groCandidateType uint8 + +const ( + notGROCandidate groCandidateType = iota + tcp4GROCandidate + tcp6GROCandidate + udp4GROCandidate + udp6GROCandidate +) + +func packetIsGROCandidate(b []byte) groCandidateType { + if len(b) < 28 { + return notGROCandidate + } + if b[0]>>4 == 4 { + if b[0]&0x0F != 5 { + // IPv4 packets w/IP options do not coalesce + return notGROCandidate + } + if b[9] == unix.IPPROTO_TCP && len(b) >= 40 { + return tcp4GROCandidate + } + if b[9] == unix.IPPROTO_UDP { + return udp4GROCandidate + } + } else if b[0]>>4 == 6 { + if b[6] == unix.IPPROTO_TCP && len(b) >= 60 { + return tcp6GROCandidate + } + if b[6] == unix.IPPROTO_UDP && len(b) >= 48 { + return udp6GROCandidate + } + } + return notGROCandidate +} + +const ( + udphLen = 8 +) + +// udpGRO evaluates the UDP packet at pktI in bufs for coalescing with +// existing packets tracked in table. It returns a groResultNoop when no +// action was taken, groResultTableInsert when the evaluated packet was +// inserted into table, and groResultCoalesced when the evaluated packet was +// coalesced with another packet in table. +func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool) groResult { + pkt := bufs[pktI][offset:] + if len(pkt) > maxUint16 { + // A valid IPv4 or IPv6 packet will never exceed this. + return groResultNoop + } + iphLen := int((pkt[0] & 0x0F) * 4) + if isV6 { + iphLen = 40 + ipv6HPayloadLen := int(binary.BigEndian.Uint16(pkt[4:])) + if ipv6HPayloadLen != len(pkt)-iphLen { + return groResultNoop + } + } else { + totalLen := int(binary.BigEndian.Uint16(pkt[2:])) + if totalLen != len(pkt) { + return groResultNoop + } + } + if len(pkt) < iphLen { + return groResultNoop + } + if len(pkt) < iphLen+udphLen { + return groResultNoop + } + if !isV6 { + if pkt[6]&ipv4FlagMoreFragments != 0 || pkt[6]<<3 != 0 || pkt[7] != 0 { + // no GRO support for fragmented segments for now + return groResultNoop + } + } + gsoSize := uint16(len(pkt) - udphLen - iphLen) + // not a candidate if payload len is 0 + if gsoSize < 1 { + return groResultNoop + } + srcAddrOffset := ipv4SrcAddrOffset + addrLen := 4 + if isV6 { + srcAddrOffset = ipv6SrcAddrOffset + addrLen = 16 + } + items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, pktI) + if !existing { + return groResultTableInsert + } + // With UDP we only check the last item, otherwise we could reorder packets + // for a given flow. We must also always insert a new item, or successfully + // coalesce with an existing item, for the same reason. + item := items[len(items)-1] + can := udpPacketsCanCoalesce(pkt, uint8(iphLen), gsoSize, item, bufs, offset) + var pktCSumKnownInvalid bool + if can == coalesceAppend { + result := coalesceUDPPackets(pkt, &item, bufs, offset, isV6) + switch result { + case coalesceSuccess: + table.updateAt(item, len(items)-1) + return groResultCoalesced + case coalesceItemInvalidCSum: + // If the existing item has an invalid csum we take no action. A new + // item will be stored after it, and the existing item will never be + // revisited as part of future coalescing candidacy checks. + case coalescePktInvalidCSum: + // We must insert a new item, but we also mark it as invalid csum + // to prevent a repeat checksum validation. + pktCSumKnownInvalid = true + default: + } + } + // failed to coalesce with any other packets; store the item in the flow + table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, pktI, pktCSumKnownInvalid) + return groResultTableInsert +} + // handleGRO evaluates bufs for GRO, and writes the indices of the resulting -// packets into toWrite. toWrite, tcp4Table, and tcp6Table should initially be +// packets into toWrite. toWrite, tcpTable, and udpTable should initially be // empty (but non-nil), and are passed in to save allocs as the caller may reset // and recycle them across vectors of packets. -func handleGRO(bufs [][]byte, offset int, tcp4Table, tcp6Table *tcpGROTable, toWrite *[]int) error { +func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, toWrite *[]int) error { for i := range bufs { if offset < virtioNetHdrLen || offset > len(bufs[i])-1 { return errors.New("invalid offset") } - var result tcpGROResult - switch { - case isTCP4NoIPOptions(bufs[i][offset:]): // ipv4 packets w/IP options do not coalesce - result = tcpGRO(bufs, offset, i, tcp4Table, false) - case isTCP6NoEH(bufs[i][offset:]): // ipv6 packets w/extension headers do not coalesce - result = tcpGRO(bufs, offset, i, tcp6Table, true) + var result groResult + switch packetIsGROCandidate(bufs[i][offset:]) { + case tcp4GROCandidate: + result = tcpGRO(bufs, offset, i, tcpTable, false) + case tcp6GROCandidate: + result = tcpGRO(bufs, offset, i, tcpTable, true) + case udp4GROCandidate: + result = udpGRO(bufs, offset, i, udpTable, false) + case udp6GROCandidate: + result = udpGRO(bufs, offset, i, udpTable, true) } switch result { - case tcpGROResultNoop: + case groResultNoop: hdr := virtioNetHdr{} err := hdr.encode(bufs[i][offset-virtioNetHdrLen:]) if err != nil { return err } fallthrough - case tcpGROResultTableInsert: + case groResultTableInsert: *toWrite = append(*toWrite, i) } } - err4 := applyCoalesceAccounting(bufs, offset, tcp4Table, false) - err6 := applyCoalesceAccounting(bufs, offset, tcp6Table, true) - return errors.Join(err4, err6) + errTCP := applyTCPCoalesceAccounting(bufs, offset, tcpTable) + errUDP := applyUDPCoalesceAccounting(bufs, offset, udpTable) + return errors.Join(errTCP, errUDP) } -// tcpTSO splits packets from in into outBuffs, writing the size of each +// gsoSplit splits packets from in into outBuffs, writing the size of each // element into sizes. It returns the number of buffers populated, and/or an // error. -func tcpTSO(in []byte, hdr virtioNetHdr, outBuffs [][]byte, sizes []int, outOffset int) (int, error) { +func gsoSplit(in []byte, hdr virtioNetHdr, outBuffs [][]byte, sizes []int, outOffset int, isV6 bool) (int, error) { iphLen := int(hdr.csumStart) srcAddrOffset := ipv6SrcAddrOffset addrLen := 16 - if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV4 { + if !isV6 { in[10], in[11] = 0, 0 // clear ipv4 header checksum srcAddrOffset = ipv4SrcAddrOffset addrLen = 4 } - tcpCSumAt := int(hdr.csumStart + hdr.csumOffset) - in[tcpCSumAt], in[tcpCSumAt+1] = 0, 0 // clear tcp checksum - firstTCPSeqNum := binary.BigEndian.Uint32(in[hdr.csumStart+4:]) + transportCsumAt := int(hdr.csumStart + hdr.csumOffset) + in[transportCsumAt], in[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum + var firstTCPSeqNum uint32 + var protocol uint8 + if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV4 || hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV6 { + protocol = unix.IPPROTO_TCP + firstTCPSeqNum = binary.BigEndian.Uint32(in[hdr.csumStart+4:]) + } else { + protocol = unix.IPPROTO_UDP + } nextSegmentDataAt := int(hdr.hdrLen) i := 0 for ; nextSegmentDataAt < len(in); i++ { @@ -610,7 +932,7 @@ func tcpTSO(in []byte, hdr virtioNetHdr, outBuffs [][]byte, sizes []int, outOffs out := outBuffs[i][outOffset:] copy(out, in[:iphLen]) - if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV4 { + if !isV6 { // For IPv4 we are responsible for incrementing the ID field, // updating the total len field, and recalculating the header // checksum. @@ -627,25 +949,32 @@ func tcpTSO(in []byte, hdr virtioNetHdr, outBuffs [][]byte, sizes []int, outOffs binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen)) } - // TCP header + // copy transport header copy(out[hdr.csumStart:hdr.hdrLen], in[hdr.csumStart:hdr.hdrLen]) - tcpSeq := firstTCPSeqNum + uint32(hdr.gsoSize*uint16(i)) - binary.BigEndian.PutUint32(out[hdr.csumStart+4:], tcpSeq) - if nextSegmentEnd != len(in) { - // FIN and PSH should only be set on last segment - clearFlags := tcpFlagFIN | tcpFlagPSH - out[hdr.csumStart+tcpFlagsOffset] &^= clearFlags + + if protocol == unix.IPPROTO_TCP { + // set TCP seq and adjust TCP flags + tcpSeq := firstTCPSeqNum + uint32(hdr.gsoSize*uint16(i)) + binary.BigEndian.PutUint32(out[hdr.csumStart+4:], tcpSeq) + if nextSegmentEnd != len(in) { + // FIN and PSH should only be set on last segment + clearFlags := tcpFlagFIN | tcpFlagPSH + out[hdr.csumStart+tcpFlagsOffset] &^= clearFlags + } + } else { + // set UDP header len + binary.BigEndian.PutUint16(out[hdr.csumStart+4:], uint16(segmentDataLen)+(hdr.hdrLen-hdr.csumStart)) } // payload copy(out[hdr.hdrLen:], in[nextSegmentDataAt:nextSegmentEnd]) - // TCP checksum - tcpHLen := int(hdr.hdrLen - hdr.csumStart) - tcpLenForPseudo := uint16(tcpHLen + segmentDataLen) - tcpCSum := pseudoHeaderChecksum(unix.IPPROTO_TCP, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], tcpLenForPseudo) - tcpCSum = ^checksum(out[hdr.csumStart:totalLen], tcpCSum) - binary.BigEndian.PutUint16(out[hdr.csumStart+hdr.csumOffset:], tcpCSum) + // transport checksum + transportHeaderLen := int(hdr.hdrLen - hdr.csumStart) + lenForPseudo := uint16(transportHeaderLen + segmentDataLen) + transportCSum := pseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) + transportCSum = ^checksum(out[hdr.csumStart:totalLen], transportCSum) + binary.BigEndian.PutUint16(out[hdr.csumStart+hdr.csumOffset:], transportCSum) nextSegmentDataAt += int(hdr.gsoSize) } diff --git a/tun/tcp_offload_linux_test.go b/tun/offload_linux_test.go similarity index 52% rename from tun/tcp_offload_linux_test.go rename to tun/offload_linux_test.go index 41fba70..192232c 100644 --- a/tun/tcp_offload_linux_test.go +++ b/tun/offload_linux_test.go @@ -28,6 +28,71 @@ var ( ip6PortC = netip.MustParseAddrPort("[2001:db8::3]:1") ) +func udp4PacketMutateIPFields(srcIPPort, dstIPPort netip.AddrPort, payloadLen int, ipFn func(*header.IPv4Fields)) []byte { + totalLen := 28 + payloadLen + b := make([]byte, offset+int(totalLen), 65535) + ipv4H := header.IPv4(b[offset:]) + srcAs4 := srcIPPort.Addr().As4() + dstAs4 := dstIPPort.Addr().As4() + ipFields := &header.IPv4Fields{ + SrcAddr: tcpip.AddrFromSlice(srcAs4[:]), + DstAddr: tcpip.AddrFromSlice(dstAs4[:]), + Protocol: unix.IPPROTO_UDP, + TTL: 64, + TotalLength: uint16(totalLen), + } + if ipFn != nil { + ipFn(ipFields) + } + ipv4H.Encode(ipFields) + udpH := header.UDP(b[offset+20:]) + udpH.Encode(&header.UDPFields{ + SrcPort: srcIPPort.Port(), + DstPort: dstIPPort.Port(), + Length: uint16(payloadLen + udphLen), + }) + ipv4H.SetChecksum(^ipv4H.CalculateChecksum()) + pseudoCsum := header.PseudoHeaderChecksum(unix.IPPROTO_UDP, ipv4H.SourceAddress(), ipv4H.DestinationAddress(), uint16(udphLen+payloadLen)) + udpH.SetChecksum(^udpH.CalculateChecksum(pseudoCsum)) + return b +} + +func udp6Packet(srcIPPort, dstIPPort netip.AddrPort, payloadLen int) []byte { + return udp6PacketMutateIPFields(srcIPPort, dstIPPort, payloadLen, nil) +} + +func udp6PacketMutateIPFields(srcIPPort, dstIPPort netip.AddrPort, payloadLen int, ipFn func(*header.IPv6Fields)) []byte { + totalLen := 48 + payloadLen + b := make([]byte, offset+int(totalLen), 65535) + ipv6H := header.IPv6(b[offset:]) + srcAs16 := srcIPPort.Addr().As16() + dstAs16 := dstIPPort.Addr().As16() + ipFields := &header.IPv6Fields{ + SrcAddr: tcpip.AddrFromSlice(srcAs16[:]), + DstAddr: tcpip.AddrFromSlice(dstAs16[:]), + TransportProtocol: unix.IPPROTO_UDP, + HopLimit: 64, + PayloadLength: uint16(payloadLen + udphLen), + } + if ipFn != nil { + ipFn(ipFields) + } + ipv6H.Encode(ipFields) + udpH := header.UDP(b[offset+40:]) + udpH.Encode(&header.UDPFields{ + SrcPort: srcIPPort.Port(), + DstPort: dstIPPort.Port(), + Length: uint16(payloadLen + udphLen), + }) + pseudoCsum := header.PseudoHeaderChecksum(unix.IPPROTO_UDP, ipv6H.SourceAddress(), ipv6H.DestinationAddress(), uint16(udphLen+payloadLen)) + udpH.SetChecksum(^udpH.CalculateChecksum(pseudoCsum)) + return b +} + +func udp4Packet(srcIPPort, dstIPPort netip.AddrPort, payloadLen int) []byte { + return udp4PacketMutateIPFields(srcIPPort, dstIPPort, payloadLen, nil) +} + func tcp4PacketMutateIPFields(srcIPPort, dstIPPort netip.AddrPort, flags header.TCPFlags, segmentSize, seq uint32, ipFn func(*header.IPv4Fields)) []byte { totalLen := 40 + segmentSize b := make([]byte, offset+int(totalLen), 65535) @@ -137,6 +202,34 @@ func Test_handleVirtioRead(t *testing.T) { []int{160, 160}, false, }, + { + "udp4", + virtioNetHdr{ + flags: unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, + gsoType: unix.VIRTIO_NET_HDR_GSO_UDP_L4, + gsoSize: 100, + hdrLen: 28, + csumStart: 20, + csumOffset: 6, + }, + udp4Packet(ip4PortA, ip4PortB, 200), + []int{128, 128}, + false, + }, + { + "udp6", + virtioNetHdr{ + flags: unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, + gsoType: unix.VIRTIO_NET_HDR_GSO_UDP_L4, + gsoSize: 100, + hdrLen: 48, + csumStart: 40, + csumOffset: 6, + }, + udp6Packet(ip6PortA, ip6PortB, 200), + []int{148, 148}, + false, + }, } for _, tt := range tests { @@ -173,6 +266,13 @@ func flipTCP4Checksum(b []byte) []byte { return b } +func flipUDP4Checksum(b []byte) []byte { + at := virtioNetHdrLen + 20 + 6 // 20 byte ipv4 header; udp csum offset is 6 + b[at] ^= 0xFF + b[at+1] ^= 0xFF + return b +} + func Fuzz_handleGRO(f *testing.F) { pkt0 := tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1) pkt1 := tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101) @@ -180,11 +280,17 @@ func Fuzz_handleGRO(f *testing.F) { pkt3 := tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1) pkt4 := tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101) pkt5 := tcp6Packet(ip6PortA, ip6PortC, header.TCPFlagAck, 100, 201) - f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, offset) - f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5 []byte, offset int) { - pkts := [][]byte{pkt0, pkt1, pkt2, pkt3, pkt4, pkt5} + pkt6 := udp4Packet(ip4PortA, ip4PortB, 100) + pkt7 := udp4Packet(ip4PortA, ip4PortB, 100) + pkt8 := udp4Packet(ip4PortA, ip4PortC, 100) + pkt9 := udp6Packet(ip6PortA, ip6PortB, 100) + pkt10 := udp6Packet(ip6PortA, ip6PortB, 100) + pkt11 := udp6Packet(ip6PortA, ip6PortC, 100) + f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, offset) + f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, offset int) { + pkts := [][]byte{pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11} toWrite := make([]int, 0, len(pkts)) - handleGRO(pkts, offset, newTCPGROTable(), newTCPGROTable(), &toWrite) + handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), &toWrite) if len(toWrite) > len(pkts) { t.Errorf("len(toWrite): %d > len(pkts): %d", len(toWrite), len(pkts)) } @@ -210,17 +316,22 @@ func Test_handleGRO(t *testing.T) { wantErr bool }{ { - "multiple flows", + "multiple protocols and flows", [][]byte{ - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // v4 flow 1 - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101), // v4 flow 1 - tcp4Packet(ip4PortA, ip4PortC, header.TCPFlagAck, 100, 201), // v4 flow 2 - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1), // v6 flow 1 - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101), // v6 flow 1 - tcp6Packet(ip6PortA, ip6PortC, header.TCPFlagAck, 100, 201), // v6 flow 2 + tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // tcp4 flow 1 + udp4Packet(ip4PortA, ip4PortB, 100), // udp4 flow 1 + udp4Packet(ip4PortA, ip4PortC, 100), // udp4 flow 2 + tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101), // tcp4 flow 1 + tcp4Packet(ip4PortA, ip4PortC, header.TCPFlagAck, 100, 201), // tcp4 flow 2 + tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1), // tcp6 flow 1 + tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101), // tcp6 flow 1 + tcp6Packet(ip6PortA, ip6PortC, header.TCPFlagAck, 100, 201), // tcp6 flow 2 + udp4Packet(ip4PortA, ip4PortB, 100), // udp4 flow 1 + udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 + udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 }, - []int{0, 2, 3, 5}, - []int{240, 140, 260, 160}, + []int{0, 1, 2, 4, 5, 7, 9}, + []int{240, 228, 128, 140, 260, 160, 248}, false, }, { @@ -245,9 +356,12 @@ func Test_handleGRO(t *testing.T) { flipTCP4Checksum(tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1)), // v4 flow 1 seq 1 len 100 tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101), // v4 flow 1 seq 101 len 100 tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 201), // v4 flow 1 seq 201 len 100 + flipUDP4Checksum(udp4Packet(ip4PortA, ip4PortB, 100)), + udp4Packet(ip4PortA, ip4PortB, 100), + udp4Packet(ip4PortA, ip4PortB, 100), }, - []int{0, 1}, - []int{140, 240}, + []int{0, 1, 3, 4}, + []int{140, 240, 128, 228}, false, }, { @@ -262,75 +376,99 @@ func Test_handleGRO(t *testing.T) { false, }, { - "tcp4 unequal TTL", + "unequal TTL", [][]byte{ tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), tcp4PacketMutateIPFields(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv4Fields) { fields.TTL++ }), + udp4Packet(ip4PortA, ip4PortB, 100), + udp4PacketMutateIPFields(ip4PortA, ip4PortB, 100, func(fields *header.IPv4Fields) { + fields.TTL++ + }), }, - []int{0, 1}, - []int{140, 140}, + []int{0, 1, 2, 3}, + []int{140, 140, 128, 128}, false, }, { - "tcp4 unequal ToS", + "unequal ToS", [][]byte{ tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), tcp4PacketMutateIPFields(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv4Fields) { fields.TOS++ }), + udp4Packet(ip4PortA, ip4PortB, 100), + udp4PacketMutateIPFields(ip4PortA, ip4PortB, 100, func(fields *header.IPv4Fields) { + fields.TOS++ + }), }, - []int{0, 1}, - []int{140, 140}, + []int{0, 1, 2, 3}, + []int{140, 140, 128, 128}, false, }, { - "tcp4 unequal flags more fragments set", + "unequal flags more fragments set", [][]byte{ tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), tcp4PacketMutateIPFields(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv4Fields) { fields.Flags = 1 }), + udp4Packet(ip4PortA, ip4PortB, 100), + udp4PacketMutateIPFields(ip4PortA, ip4PortB, 100, func(fields *header.IPv4Fields) { + fields.Flags = 1 + }), }, - []int{0, 1}, - []int{140, 140}, + []int{0, 1, 2, 3}, + []int{140, 140, 128, 128}, false, }, { - "tcp4 unequal flags DF set", + "unequal flags DF set", [][]byte{ tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), tcp4PacketMutateIPFields(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv4Fields) { fields.Flags = 2 }), + udp4Packet(ip4PortA, ip4PortB, 100), + udp4PacketMutateIPFields(ip4PortA, ip4PortB, 100, func(fields *header.IPv4Fields) { + fields.Flags = 2 + }), }, - []int{0, 1}, - []int{140, 140}, + []int{0, 1, 2, 3}, + []int{140, 140, 128, 128}, false, }, { - "tcp6 unequal hop limit", + "ipv6 unequal hop limit", [][]byte{ tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1), tcp6PacketMutateIPFields(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv6Fields) { fields.HopLimit++ }), + udp6Packet(ip6PortA, ip6PortB, 100), + udp6PacketMutateIPFields(ip6PortA, ip6PortB, 100, func(fields *header.IPv6Fields) { + fields.HopLimit++ + }), }, - []int{0, 1}, - []int{160, 160}, + []int{0, 1, 2, 3}, + []int{160, 160, 148, 148}, false, }, { - "tcp6 unequal traffic class", + "ipv6 unequal traffic class", [][]byte{ tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1), tcp6PacketMutateIPFields(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv6Fields) { fields.TrafficClass++ }), + udp6Packet(ip6PortA, ip6PortB, 100), + udp6PacketMutateIPFields(ip6PortA, ip6PortB, 100, func(fields *header.IPv6Fields) { + fields.TrafficClass++ + }), }, - []int{0, 1}, - []int{160, 160}, + []int{0, 1, 2, 3}, + []int{160, 160, 148, 148}, false, }, } @@ -338,7 +476,7 @@ func Test_handleGRO(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { toWrite := make([]int, 0, len(tt.pktsIn)) - err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newTCPGROTable(), &toWrite) + err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), &toWrite) if err != nil { if tt.wantErr { return @@ -360,51 +498,198 @@ func Test_handleGRO(t *testing.T) { } } -func Test_isTCP4NoIPOptions(t *testing.T) { - valid := tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1)[virtioNetHdrLen:] - invalidLen := valid[:39] - invalidHeaderLen := make([]byte, len(valid)) - copy(invalidHeaderLen, valid) - invalidHeaderLen[0] = 0x46 - invalidProtocol := make([]byte, len(valid)) - copy(invalidProtocol, valid) - invalidProtocol[9] = unix.IPPROTO_TCP + 1 +func Test_packetIsGROCandidate(t *testing.T) { + tcp4 := tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1)[virtioNetHdrLen:] + tcp4TooShort := tcp4[:39] + ip4InvalidHeaderLen := make([]byte, len(tcp4)) + copy(ip4InvalidHeaderLen, tcp4) + ip4InvalidHeaderLen[0] = 0x46 + ip4InvalidProtocol := make([]byte, len(tcp4)) + copy(ip4InvalidProtocol, tcp4) + ip4InvalidProtocol[9] = unix.IPPROTO_GRE + + tcp6 := tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1)[virtioNetHdrLen:] + tcp6TooShort := tcp6[:59] + ip6InvalidProtocol := make([]byte, len(tcp6)) + copy(ip6InvalidProtocol, tcp6) + ip6InvalidProtocol[6] = unix.IPPROTO_GRE + + udp4 := udp4Packet(ip4PortA, ip4PortB, 100)[virtioNetHdrLen:] + udp4TooShort := udp4[:27] + + udp6 := udp6Packet(ip6PortA, ip6PortB, 100)[virtioNetHdrLen:] + udp6TooShort := udp6[:47] tests := []struct { name string b []byte - want bool + want groCandidateType }{ { - "valid", - valid, - true, + "tcp4", + tcp4, + tcp4GROCandidate, }, { - "invalid length", - invalidLen, - false, + "tcp6", + tcp6, + tcp6GROCandidate, }, { - "invalid version", + "udp4", + udp4, + udp4GROCandidate, + }, + { + "udp6", + udp6, + udp6GROCandidate, + }, + { + "udp4 too short", + udp4TooShort, + notGROCandidate, + }, + { + "udp6 too short", + udp6TooShort, + notGROCandidate, + }, + { + "tcp4 too short", + tcp4TooShort, + notGROCandidate, + }, + { + "tcp6 too short", + tcp6TooShort, + notGROCandidate, + }, + { + "invalid IP version", []byte{0x00}, - false, + notGROCandidate, }, { - "invalid header len", - invalidHeaderLen, - false, + "invalid IP header len", + ip4InvalidHeaderLen, + notGROCandidate, }, { - "invalid protocol", - invalidProtocol, - false, + "ip4 invalid protocol", + ip4InvalidProtocol, + notGROCandidate, + }, + { + "ip6 invalid protocol", + ip6InvalidProtocol, + notGROCandidate, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := isTCP4NoIPOptions(tt.b); got != tt.want { - t.Errorf("isTCP4NoIPOptions() = %v, want %v", got, tt.want) + if got := packetIsGROCandidate(tt.b); got != tt.want { + t.Errorf("packetIsGROCandidate() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_udpPacketsCanCoalesce(t *testing.T) { + udp4a := udp4Packet(ip4PortA, ip4PortB, 100) + udp4b := udp4Packet(ip4PortA, ip4PortB, 100) + udp4c := udp4Packet(ip4PortA, ip4PortB, 110) + + type args struct { + pkt []byte + iphLen uint8 + gsoSize uint16 + item udpGROItem + bufs [][]byte + bufsOffset int + } + tests := []struct { + name string + args args + want canCoalesce + }{ + { + "coalesceAppend equal gso", + args{ + pkt: udp4a[offset:], + iphLen: 20, + gsoSize: 100, + item: udpGROItem{ + gsoSize: 100, + iphLen: 20, + }, + bufs: [][]byte{ + udp4a, + udp4b, + }, + bufsOffset: offset, + }, + coalesceAppend, + }, + { + "coalesceAppend smaller gso", + args{ + pkt: udp4a[offset : len(udp4a)-90], + iphLen: 20, + gsoSize: 10, + item: udpGROItem{ + gsoSize: 100, + iphLen: 20, + }, + bufs: [][]byte{ + udp4a, + udp4b, + }, + bufsOffset: offset, + }, + coalesceAppend, + }, + { + "coalesceUnavailable smaller gso previously appended", + args{ + pkt: udp4a[offset:], + iphLen: 20, + gsoSize: 100, + item: udpGROItem{ + gsoSize: 100, + iphLen: 20, + }, + bufs: [][]byte{ + udp4c, + udp4b, + }, + bufsOffset: offset, + }, + coalesceUnavailable, + }, + { + "coalesceUnavailable larger following smaller", + args{ + pkt: udp4c[offset:], + iphLen: 20, + gsoSize: 110, + item: udpGROItem{ + gsoSize: 100, + iphLen: 20, + }, + bufs: [][]byte{ + udp4a, + udp4c, + }, + bufsOffset: offset, + }, + coalesceUnavailable, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := udpPacketsCanCoalesce(tt.args.pkt, tt.args.iphLen, tt.args.gsoSize, tt.args.item, tt.args.bufs, tt.args.bufsOffset); got != tt.want { + t.Errorf("udpPacketsCanCoalesce() = %v, want %v", got, tt.want) } }) } diff --git a/tun/testdata/fuzz/Fuzz_handleGRO/032aec0105f26f709c118365e4830d6dc087cab24cd1e154c2e790589a309b77 b/tun/testdata/fuzz/Fuzz_handleGRO/032aec0105f26f709c118365e4830d6dc087cab24cd1e154c2e790589a309b77 deleted file mode 100644 index 5461e79..0000000 --- a/tun/testdata/fuzz/Fuzz_handleGRO/032aec0105f26f709c118365e4830d6dc087cab24cd1e154c2e790589a309b77 +++ /dev/null @@ -1,8 +0,0 @@ -go test fuzz v1 -[]byte("0") -[]byte("0") -[]byte("0") -[]byte("0") -[]byte("0") -[]byte("0") -int(34) diff --git a/tun/testdata/fuzz/Fuzz_handleGRO/0da283f9a2098dec30d1c86784411a8ce2e8e03aa3384105e581f2c67494700d b/tun/testdata/fuzz/Fuzz_handleGRO/0da283f9a2098dec30d1c86784411a8ce2e8e03aa3384105e581f2c67494700d deleted file mode 100644 index b441819..0000000 --- a/tun/testdata/fuzz/Fuzz_handleGRO/0da283f9a2098dec30d1c86784411a8ce2e8e03aa3384105e581f2c67494700d +++ /dev/null @@ -1,8 +0,0 @@ -go test fuzz v1 -[]byte("0") -[]byte("0") -[]byte("0") -[]byte("0") -[]byte("0") -[]byte("0") -int(-48) diff --git a/tun/tun_linux.go b/tun/tun_linux.go index eb5051e..94bffcc 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -38,6 +38,7 @@ type NativeTun struct { statusListenersShutdown chan struct{} batchSize int vnetHdr bool + udpGSO bool closeOnce sync.Once @@ -48,9 +49,10 @@ type NativeTun struct { readOpMu sync.Mutex // readOpMu guards readBuff readBuff [virtioNetHdrLen + 65535]byte // if vnetHdr every read() is prefixed by virtioNetHdr - writeOpMu sync.Mutex // writeOpMu guards toWrite, tcp4GROTable, tcp6GROTable - toWrite []int - tcp4GROTable, tcp6GROTable *tcpGROTable + writeOpMu sync.Mutex // writeOpMu guards toWrite, tcpGROTable + toWrite []int + tcpGROTable *tcpGROTable + udpGROTable *udpGROTable } func (tun *NativeTun) File() *os.File { @@ -333,8 +335,8 @@ func (tun *NativeTun) nameSlow() (string, error) { func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) { tun.writeOpMu.Lock() defer func() { - tun.tcp4GROTable.reset() - tun.tcp6GROTable.reset() + tun.tcpGROTable.reset() + tun.udpGROTable.reset() tun.writeOpMu.Unlock() }() var ( @@ -343,7 +345,7 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) { ) tun.toWrite = tun.toWrite[:0] if tun.vnetHdr { - err := handleGRO(bufs, offset, tun.tcp4GROTable, tun.tcp6GROTable, &tun.toWrite) + err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, &tun.toWrite) if err != nil { return 0, err } @@ -394,37 +396,42 @@ func handleVirtioRead(in []byte, bufs [][]byte, sizes []int, offset int) (int, e sizes[0] = n return 1, nil } - if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 { + if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { return 0, fmt.Errorf("unsupported virtio GSO type: %d", hdr.gsoType) } ipVersion := in[0] >> 4 switch ipVersion { case 4: - if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 { + if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { return 0, fmt.Errorf("ip header version: %d, GSO type: %d", ipVersion, hdr.gsoType) } case 6: - if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 { + if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { return 0, fmt.Errorf("ip header version: %d, GSO type: %d", ipVersion, hdr.gsoType) } default: return 0, fmt.Errorf("invalid ip header version: %d", ipVersion) } - if len(in) <= int(hdr.csumStart+12) { - return 0, errors.New("packet is too short") - } // Don't trust hdr.hdrLen from the kernel as it can be equal to the length // of the entire first packet when the kernel is handling it as part of a - // FORWARD path. Instead, parse the TCP header length and add it onto + // FORWARD path. Instead, parse the transport header length and add it onto // csumStart, which is synonymous for IP header length. - tcpHLen := uint16(in[hdr.csumStart+12] >> 4 * 4) - if tcpHLen < 20 || tcpHLen > 60 { - // A TCP header must be between 20 and 60 bytes in length. - return 0, fmt.Errorf("tcp header len is invalid: %d", tcpHLen) + if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_UDP_L4 { + hdr.hdrLen = hdr.csumStart + 8 + } else { + if len(in) <= int(hdr.csumStart+12) { + return 0, errors.New("packet is too short") + } + + tcpHLen := uint16(in[hdr.csumStart+12] >> 4 * 4) + if tcpHLen < 20 || tcpHLen > 60 { + // A TCP header must be between 20 and 60 bytes in length. + return 0, fmt.Errorf("tcp header len is invalid: %d", tcpHLen) + } + hdr.hdrLen = hdr.csumStart + tcpHLen } - hdr.hdrLen = hdr.csumStart + tcpHLen if len(in) < int(hdr.hdrLen) { return 0, fmt.Errorf("length of packet (%d) < virtioNetHdr.hdrLen (%d)", len(in), hdr.hdrLen) @@ -438,7 +445,7 @@ func handleVirtioRead(in []byte, bufs [][]byte, sizes []int, offset int) (int, e return 0, fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(in)) } - return tcpTSO(in, hdr, bufs, sizes, offset) + return gsoSplit(in, hdr, bufs, sizes, offset, ipVersion == 6) } func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) { @@ -497,7 +504,8 @@ func (tun *NativeTun) BatchSize() int { const ( // TODO: support TSO with ECN bits - tunOffloads = unix.TUN_F_CSUM | unix.TUN_F_TSO4 | unix.TUN_F_TSO6 + tunTCPOffloads = unix.TUN_F_CSUM | unix.TUN_F_TSO4 | unix.TUN_F_TSO6 + tunUDPOffloads = unix.TUN_F_USO4 | unix.TUN_F_USO6 ) func (tun *NativeTun) initFromFlags(name string) error { @@ -519,12 +527,17 @@ func (tun *NativeTun) initFromFlags(name string) error { } got := ifr.Uint16() if got&unix.IFF_VNET_HDR != 0 { - err = unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunOffloads) + // tunTCPOffloads were added in Linux v2.6. We require their support + // if IFF_VNET_HDR is set. + err = unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads) if err != nil { return } tun.vnetHdr = true tun.batchSize = conn.IdealBatchSize + // tunUDPOffloads were added in Linux v6.2. We do not return an + // error if they are unsupported at runtime. + tun.udpGSO = unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) == nil } else { tun.batchSize = 1 } @@ -575,8 +588,8 @@ func CreateTUNFromFile(file *os.File, mtu int) (Device, error) { events: make(chan Event, 5), errors: make(chan error, 5), statusListenersShutdown: make(chan struct{}), - tcp4GROTable: newTCPGROTable(), - tcp6GROTable: newTCPGROTable(), + tcpGROTable: newTCPGROTable(), + udpGROTable: newUDPGROTable(), toWrite: make([]int, 0, conn.IdealBatchSize), } @@ -628,12 +641,12 @@ func CreateUnmonitoredTUNFromFD(fd int) (Device, string, error) { } file := os.NewFile(uintptr(fd), "/dev/tun") tun := &NativeTun{ - tunFile: file, - events: make(chan Event, 5), - errors: make(chan error, 5), - tcp4GROTable: newTCPGROTable(), - tcp6GROTable: newTCPGROTable(), - toWrite: make([]int, 0, conn.IdealBatchSize), + tunFile: file, + events: make(chan Event, 5), + errors: make(chan error, 5), + tcpGROTable: newTCPGROTable(), + udpGROTable: newUDPGROTable(), + toWrite: make([]int, 0, conn.IdealBatchSize), } name, err := tun.Name() if err != nil { From db7604d1aa907a37c91164982d2a881fe6edc3c1 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 31 Oct 2023 18:08:04 -0700 Subject: [PATCH 14/60] tun: don't assume UDP GRO is supported Signed-off-by: Jordan Whited --- tun/offload_linux.go | 13 +++---- tun/offload_linux_test.go | 72 ++++++++++++++++++++++++++++++++++----- tun/tun_linux.go | 2 +- 3 files changed, 72 insertions(+), 15 deletions(-) diff --git a/tun/offload_linux.go b/tun/offload_linux.go index 54461cc..9a9d38e 100644 --- a/tun/offload_linux.go +++ b/tun/offload_linux.go @@ -748,7 +748,7 @@ const ( udp6GROCandidate ) -func packetIsGROCandidate(b []byte) groCandidateType { +func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType { if len(b) < 28 { return notGROCandidate } @@ -760,14 +760,14 @@ func packetIsGROCandidate(b []byte) groCandidateType { if b[9] == unix.IPPROTO_TCP && len(b) >= 40 { return tcp4GROCandidate } - if b[9] == unix.IPPROTO_UDP { + if b[9] == unix.IPPROTO_UDP && canUDPGRO { return udp4GROCandidate } } else if b[0]>>4 == 6 { if b[6] == unix.IPPROTO_TCP && len(b) >= 60 { return tcp6GROCandidate } - if b[6] == unix.IPPROTO_UDP && len(b) >= 48 { + if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && canUDPGRO { return udp6GROCandidate } } @@ -860,14 +860,15 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool) // handleGRO evaluates bufs for GRO, and writes the indices of the resulting // packets into toWrite. toWrite, tcpTable, and udpTable should initially be // empty (but non-nil), and are passed in to save allocs as the caller may reset -// and recycle them across vectors of packets. -func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, toWrite *[]int) error { +// and recycle them across vectors of packets. canUDPGRO indicates if UDP GRO is +// supported. +func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, canUDPGRO bool, toWrite *[]int) error { for i := range bufs { if offset < virtioNetHdrLen || offset > len(bufs[i])-1 { return errors.New("invalid offset") } var result groResult - switch packetIsGROCandidate(bufs[i][offset:]) { + switch packetIsGROCandidate(bufs[i][offset:], canUDPGRO) { case tcp4GROCandidate: result = tcpGRO(bufs, offset, i, tcpTable, false) case tcp6GROCandidate: diff --git a/tun/offload_linux_test.go b/tun/offload_linux_test.go index 192232c..91f3941 100644 --- a/tun/offload_linux_test.go +++ b/tun/offload_linux_test.go @@ -286,11 +286,11 @@ func Fuzz_handleGRO(f *testing.F) { pkt9 := udp6Packet(ip6PortA, ip6PortB, 100) pkt10 := udp6Packet(ip6PortA, ip6PortB, 100) pkt11 := udp6Packet(ip6PortA, ip6PortC, 100) - f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, offset) - f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, offset int) { + f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, true, offset) + f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, canUDPGRO bool, offset int) { pkts := [][]byte{pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11} toWrite := make([]int, 0, len(pkts)) - handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), &toWrite) + handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), canUDPGRO, &toWrite) if len(toWrite) > len(pkts) { t.Errorf("len(toWrite): %d > len(pkts): %d", len(toWrite), len(pkts)) } @@ -311,6 +311,7 @@ func Test_handleGRO(t *testing.T) { tests := []struct { name string pktsIn [][]byte + canUDPGRO bool wantToWrite []int wantLens []int wantErr bool @@ -330,10 +331,31 @@ func Test_handleGRO(t *testing.T) { udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 }, + true, []int{0, 1, 2, 4, 5, 7, 9}, []int{240, 228, 128, 140, 260, 160, 248}, false, }, + { + "multiple protocols and flows no UDP GRO", + [][]byte{ + tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // tcp4 flow 1 + udp4Packet(ip4PortA, ip4PortB, 100), // udp4 flow 1 + udp4Packet(ip4PortA, ip4PortC, 100), // udp4 flow 2 + tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101), // tcp4 flow 1 + tcp4Packet(ip4PortA, ip4PortC, header.TCPFlagAck, 100, 201), // tcp4 flow 2 + tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1), // tcp6 flow 1 + tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101), // tcp6 flow 1 + tcp6Packet(ip6PortA, ip6PortC, header.TCPFlagAck, 100, 201), // tcp6 flow 2 + udp4Packet(ip4PortA, ip4PortB, 100), // udp4 flow 1 + udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 + udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 + }, + false, + []int{0, 1, 2, 4, 5, 7, 8, 9, 10}, + []int{240, 128, 128, 140, 260, 160, 128, 148, 148}, + false, + }, { "PSH interleaved", [][]byte{ @@ -346,6 +368,7 @@ func Test_handleGRO(t *testing.T) { tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 201), // v6 flow 1 tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 301), // v6 flow 1 }, + true, []int{0, 2, 4, 6}, []int{240, 240, 260, 260}, false, @@ -360,6 +383,7 @@ func Test_handleGRO(t *testing.T) { udp4Packet(ip4PortA, ip4PortB, 100), udp4Packet(ip4PortA, ip4PortB, 100), }, + true, []int{0, 1, 3, 4}, []int{140, 240, 128, 228}, false, @@ -371,6 +395,7 @@ func Test_handleGRO(t *testing.T) { tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // v4 flow 1 seq 1 len 100 tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 201), // v4 flow 1 seq 201 len 100 }, + true, []int{0}, []int{340}, false, @@ -387,6 +412,7 @@ func Test_handleGRO(t *testing.T) { fields.TTL++ }), }, + true, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -403,6 +429,7 @@ func Test_handleGRO(t *testing.T) { fields.TOS++ }), }, + true, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -419,6 +446,7 @@ func Test_handleGRO(t *testing.T) { fields.Flags = 1 }), }, + true, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -435,6 +463,7 @@ func Test_handleGRO(t *testing.T) { fields.Flags = 2 }), }, + true, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -451,6 +480,7 @@ func Test_handleGRO(t *testing.T) { fields.HopLimit++ }), }, + true, []int{0, 1, 2, 3}, []int{160, 160, 148, 148}, false, @@ -467,6 +497,7 @@ func Test_handleGRO(t *testing.T) { fields.TrafficClass++ }), }, + true, []int{0, 1, 2, 3}, []int{160, 160, 148, 148}, false, @@ -476,7 +507,7 @@ func Test_handleGRO(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { toWrite := make([]int, 0, len(tt.pktsIn)) - err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), &toWrite) + err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), tt.canUDPGRO, &toWrite) if err != nil { if tt.wantErr { return @@ -521,74 +552,99 @@ func Test_packetIsGROCandidate(t *testing.T) { udp6TooShort := udp6[:47] tests := []struct { - name string - b []byte - want groCandidateType + name string + b []byte + canUDPGRO bool + want groCandidateType }{ { "tcp4", tcp4, + true, tcp4GROCandidate, }, { "tcp6", tcp6, + true, tcp6GROCandidate, }, { "udp4", udp4, + true, udp4GROCandidate, }, + { + "udp4 no support", + udp4, + false, + notGROCandidate, + }, { "udp6", udp6, + true, udp6GROCandidate, }, + { + "udp6 no support", + udp6, + false, + notGROCandidate, + }, { "udp4 too short", udp4TooShort, + true, notGROCandidate, }, { "udp6 too short", udp6TooShort, + true, notGROCandidate, }, { "tcp4 too short", tcp4TooShort, + true, notGROCandidate, }, { "tcp6 too short", tcp6TooShort, + true, notGROCandidate, }, { "invalid IP version", []byte{0x00}, + true, notGROCandidate, }, { "invalid IP header len", ip4InvalidHeaderLen, + true, notGROCandidate, }, { "ip4 invalid protocol", ip4InvalidProtocol, + true, notGROCandidate, }, { "ip6 invalid protocol", ip6InvalidProtocol, + true, notGROCandidate, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := packetIsGROCandidate(tt.b); got != tt.want { + if got := packetIsGROCandidate(tt.b, tt.canUDPGRO); got != tt.want { t.Errorf("packetIsGROCandidate() = %v, want %v", got, tt.want) } }) diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 94bffcc..9313ebf 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -345,7 +345,7 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) { ) tun.toWrite = tun.toWrite[:0] if tun.vnetHdr { - err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, &tun.toWrite) + err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.udpGSO, &tun.toWrite) if err != nil { return 0, err } From 8cc8b8b11b1f7189f3e19616d2e233fce6ee7eda Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 20 Nov 2023 16:49:06 -0800 Subject: [PATCH 15/60] device: change Peer.endpoint locking to reduce contention Access to Peer.endpoint was previously synchronized by Peer.RWMutex. This has now moved to Peer.endpoint.Mutex. Peer.SendBuffers() is now the sole caller of Endpoint.ClearSrc(), which is signaled via a new bool, Peer.endpoint.clearSrcOnTx. Previous Callers of Endpoint.ClearSrc() now set this bool, primarily via peer.markEndpointSrcForClearing(). Peer.SetEndpointFromPacket() clears Peer.endpoint.clearSrcOnTx when an updated conn.Endpoint is stored. This maintains the same event order as before, i.e. a conn.Endpoint received after peer.endpoint.clearSrcOnTx is set, but before the next Peer.SendBuffers() call results in the latest conn.Endpoint source being used for the next packet transmission. These changes result in throughput improvements for single flow, parallel (-P n) flow, and bidirectional (--bidir) flow iperf3 TCP/UDP tests as measured on both Linux and Windows. Latency under load improves especially for high throughput Linux scenarios. These improvements are likely realized on all platforms to some degree, as the changes are not platform-specific. Co-authored-by: James Tucker Signed-off-by: Jordan Whited --- device/device.go | 12 ++-------- device/mobilequirks.go | 6 ++--- device/peer.go | 50 ++++++++++++++++++++++++++------------ device/sticky_linux.go | 30 +++++++++++------------ device/timers.go | 12 ++-------- device/uapi.go | 54 ++++++++++++++++++++---------------------- 6 files changed, 83 insertions(+), 81 deletions(-) diff --git a/device/device.go b/device/device.go index 5c666ac..86dff0d 100644 --- a/device/device.go +++ b/device/device.go @@ -461,11 +461,7 @@ func (device *Device) BindSetMark(mark uint32) error { // clear cached source addresses device.peers.RLock() for _, peer := range device.peers.keyMap { - peer.Lock() - defer peer.Unlock() - if peer.endpoint != nil { - peer.endpoint.ClearSrc() - } + peer.markEndpointSrcForClearing() } device.peers.RUnlock() @@ -515,11 +511,7 @@ func (device *Device) BindUpdate() error { // clear cached source addresses device.peers.RLock() for _, peer := range device.peers.keyMap { - peer.Lock() - defer peer.Unlock() - if peer.endpoint != nil { - peer.endpoint.ClearSrc() - } + peer.markEndpointSrcForClearing() } device.peers.RUnlock() diff --git a/device/mobilequirks.go b/device/mobilequirks.go index 4e5051d..0a0080e 100644 --- a/device/mobilequirks.go +++ b/device/mobilequirks.go @@ -11,9 +11,9 @@ func (device *Device) DisableSomeRoamingForBrokenMobileSemantics() { device.net.brokenRoaming = true device.peers.RLock() for _, peer := range device.peers.keyMap { - peer.Lock() - peer.disableRoaming = peer.endpoint != nil - peer.Unlock() + peer.endpoint.Lock() + peer.endpoint.disableRoaming = peer.endpoint.val != nil + peer.endpoint.Unlock() } device.peers.RUnlock() } diff --git a/device/peer.go b/device/peer.go index 22757d4..89b719b 100644 --- a/device/peer.go +++ b/device/peer.go @@ -17,17 +17,20 @@ import ( type Peer struct { isRunning atomic.Bool - sync.RWMutex // Mostly protects endpoint, but is generally taken whenever we modify peer keypairs Keypairs handshake Handshake device *Device - endpoint conn.Endpoint stopping sync.WaitGroup // routines pending stop txBytes atomic.Uint64 // bytes send to peer (endpoint) rxBytes atomic.Uint64 // bytes received from peer lastHandshakeNano atomic.Int64 // nano seconds since epoch - disableRoaming bool + endpoint struct { + sync.Mutex + val conn.Endpoint + clearSrcOnTx bool // signal to val.ClearSrc() prior to next packet transmission + disableRoaming bool + } timers struct { retransmitHandshake *Timer @@ -74,8 +77,6 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) { // create peer peer := new(Peer) - peer.Lock() - defer peer.Unlock() peer.cookieGenerator.Init(pk) peer.device = device @@ -97,7 +98,11 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) { handshake.mutex.Unlock() // reset endpoint - peer.endpoint = nil + peer.endpoint.Lock() + peer.endpoint.val = nil + peer.endpoint.disableRoaming = false + peer.endpoint.clearSrcOnTx = false + peer.endpoint.Unlock() // init timers peer.timersInit() @@ -116,14 +121,19 @@ func (peer *Peer) SendBuffers(buffers [][]byte) error { return nil } - peer.RLock() - defer peer.RUnlock() - - if peer.endpoint == nil { + peer.endpoint.Lock() + endpoint := peer.endpoint.val + if endpoint == nil { + peer.endpoint.Unlock() return errors.New("no known endpoint for peer") } + if peer.endpoint.clearSrcOnTx { + endpoint.ClearSrc() + peer.endpoint.clearSrcOnTx = false + } + peer.endpoint.Unlock() - err := peer.device.net.bind.Send(buffers, peer.endpoint) + err := peer.device.net.bind.Send(buffers, endpoint) if err == nil { var totalLen uint64 for _, b := range buffers { @@ -267,10 +277,20 @@ func (peer *Peer) Stop() { } func (peer *Peer) SetEndpointFromPacket(endpoint conn.Endpoint) { - if peer.disableRoaming { + peer.endpoint.Lock() + defer peer.endpoint.Unlock() + if peer.endpoint.disableRoaming { return } - peer.Lock() - peer.endpoint = endpoint - peer.Unlock() + peer.endpoint.clearSrcOnTx = false + peer.endpoint.val = endpoint +} + +func (peer *Peer) markEndpointSrcForClearing() { + peer.endpoint.Lock() + defer peer.endpoint.Unlock() + if peer.endpoint.val == nil { + return + } + peer.endpoint.clearSrcOnTx = true } diff --git a/device/sticky_linux.go b/device/sticky_linux.go index 7a519c1..6eeced2 100644 --- a/device/sticky_linux.go +++ b/device/sticky_linux.go @@ -110,17 +110,17 @@ func (device *Device) routineRouteListener(bind conn.Bind, netlinkSock int, netl if !ok { break } - pePtr.peer.Lock() - if &pePtr.peer.endpoint != pePtr.endpoint { - pePtr.peer.Unlock() + pePtr.peer.endpoint.Lock() + if &pePtr.peer.endpoint.val != pePtr.endpoint { + pePtr.peer.endpoint.Unlock() break } - if uint32(pePtr.peer.endpoint.(*conn.StdNetEndpoint).SrcIfidx()) == ifidx { - pePtr.peer.Unlock() + if uint32(pePtr.peer.endpoint.val.(*conn.StdNetEndpoint).SrcIfidx()) == ifidx { + pePtr.peer.endpoint.Unlock() break } - pePtr.peer.endpoint.(*conn.StdNetEndpoint).ClearSrc() - pePtr.peer.Unlock() + pePtr.peer.endpoint.clearSrcOnTx = true + pePtr.peer.endpoint.Unlock() } attr = attr[attrhdr.Len:] } @@ -134,18 +134,18 @@ func (device *Device) routineRouteListener(bind conn.Bind, netlinkSock int, netl device.peers.RLock() i := uint32(1) for _, peer := range device.peers.keyMap { - peer.RLock() - if peer.endpoint == nil { - peer.RUnlock() + peer.endpoint.Lock() + if peer.endpoint.val == nil { + peer.endpoint.Unlock() continue } - nativeEP, _ := peer.endpoint.(*conn.StdNetEndpoint) + nativeEP, _ := peer.endpoint.val.(*conn.StdNetEndpoint) if nativeEP == nil { - peer.RUnlock() + peer.endpoint.Unlock() continue } if nativeEP.DstIP().Is6() || nativeEP.SrcIfidx() == 0 { - peer.RUnlock() + peer.endpoint.Unlock() break } nlmsg := struct { @@ -188,10 +188,10 @@ func (device *Device) routineRouteListener(bind conn.Bind, netlinkSock int, netl reqPeerLock.Lock() reqPeer[i] = peerEndpointPtr{ peer: peer, - endpoint: &peer.endpoint, + endpoint: &peer.endpoint.val, } reqPeerLock.Unlock() - peer.RUnlock() + peer.endpoint.Unlock() i++ _, err := netlinkCancel.Write((*[unsafe.Sizeof(nlmsg)]byte)(unsafe.Pointer(&nlmsg))[:]) if err != nil { diff --git a/device/timers.go b/device/timers.go index e28732c..d4a4ed4 100644 --- a/device/timers.go +++ b/device/timers.go @@ -100,11 +100,7 @@ func expiredRetransmitHandshake(peer *Peer) { peer.device.log.Verbosef("%s - Handshake did not complete after %d seconds, retrying (try %d)", peer, int(RekeyTimeout.Seconds()), peer.timers.handshakeAttempts.Load()+1) /* We clear the endpoint address src address, in case this is the cause of trouble. */ - peer.Lock() - if peer.endpoint != nil { - peer.endpoint.ClearSrc() - } - peer.Unlock() + peer.markEndpointSrcForClearing() peer.SendHandshakeInitiation(true) } @@ -123,11 +119,7 @@ func expiredSendKeepalive(peer *Peer) { func expiredNewHandshake(peer *Peer) { peer.device.log.Verbosef("%s - Retrying handshake because we stopped hearing back after %d seconds", peer, int((KeepaliveTimeout + RekeyTimeout).Seconds())) /* We clear the endpoint address src address, in case this is the cause of trouble. */ - peer.Lock() - if peer.endpoint != nil { - peer.endpoint.ClearSrc() - } - peer.Unlock() + peer.markEndpointSrcForClearing() peer.SendHandshakeInitiation(false) } diff --git a/device/uapi.go b/device/uapi.go index 2a91a93..4987cda 100644 --- a/device/uapi.go +++ b/device/uapi.go @@ -99,33 +99,31 @@ func (device *Device) IpcGetOperation(w io.Writer) error { for _, peer := range device.peers.keyMap { // Serialize peer state. - // Do the work in an anonymous function so that we can use defer. - func() { - peer.RLock() - defer peer.RUnlock() + peer.handshake.mutex.RLock() + keyf("public_key", (*[32]byte)(&peer.handshake.remoteStatic)) + keyf("preshared_key", (*[32]byte)(&peer.handshake.presharedKey)) + peer.handshake.mutex.RUnlock() + sendf("protocol_version=1") + peer.endpoint.Lock() + if peer.endpoint.val != nil { + sendf("endpoint=%s", peer.endpoint.val.DstToString()) + } + peer.endpoint.Unlock() - keyf("public_key", (*[32]byte)(&peer.handshake.remoteStatic)) - keyf("preshared_key", (*[32]byte)(&peer.handshake.presharedKey)) - sendf("protocol_version=1") - if peer.endpoint != nil { - sendf("endpoint=%s", peer.endpoint.DstToString()) - } + nano := peer.lastHandshakeNano.Load() + secs := nano / time.Second.Nanoseconds() + nano %= time.Second.Nanoseconds() - nano := peer.lastHandshakeNano.Load() - secs := nano / time.Second.Nanoseconds() - nano %= time.Second.Nanoseconds() + sendf("last_handshake_time_sec=%d", secs) + sendf("last_handshake_time_nsec=%d", nano) + sendf("tx_bytes=%d", peer.txBytes.Load()) + sendf("rx_bytes=%d", peer.rxBytes.Load()) + sendf("persistent_keepalive_interval=%d", peer.persistentKeepaliveInterval.Load()) - sendf("last_handshake_time_sec=%d", secs) - sendf("last_handshake_time_nsec=%d", nano) - sendf("tx_bytes=%d", peer.txBytes.Load()) - sendf("rx_bytes=%d", peer.rxBytes.Load()) - sendf("persistent_keepalive_interval=%d", peer.persistentKeepaliveInterval.Load()) - - device.allowedips.EntriesForPeer(peer, func(prefix netip.Prefix) bool { - sendf("allowed_ip=%s", prefix.String()) - return true - }) - }() + device.allowedips.EntriesForPeer(peer, func(prefix netip.Prefix) bool { + sendf("allowed_ip=%s", prefix.String()) + return true + }) } }() @@ -262,7 +260,7 @@ func (peer *ipcSetPeer) handlePostConfig() { return } if peer.created { - peer.disableRoaming = peer.device.net.brokenRoaming && peer.endpoint != nil + peer.endpoint.disableRoaming = peer.device.net.brokenRoaming && peer.endpoint.val != nil } if peer.device.isUp() { peer.Start() @@ -345,9 +343,9 @@ func (device *Device) handlePeerLine(peer *ipcSetPeer, key, value string) error if err != nil { return ipcErrorf(ipc.IpcErrorInvalid, "failed to set endpoint %v: %w", value, err) } - peer.Lock() - defer peer.Unlock() - peer.endpoint = endpoint + peer.endpoint.Lock() + defer peer.endpoint.Unlock() + peer.endpoint.val = endpoint case "persistent_keepalive_interval": device.log.Verbosef("%v - UAPI: Updating persistent keepalive interval", peer.Peer) From cc193a0b327276d902b3b688d3a06a857b17fcb7 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 7 Nov 2023 15:24:21 -0800 Subject: [PATCH 16/60] device: reduce redundant per-packet overhead in RX path Peer.RoutineSequentialReceiver() deals with packet vectors and does not need to perform timer and endpoint operations for every packet in a given vector. Changing these per-packet operations to per-vector improves throughput by as much as 10% in some environments. Signed-off-by: Jordan Whited --- device/receive.go | 21 +++++++++++++++------ device/send.go | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/device/receive.go b/device/receive.go index da663e9..af2db44 100644 --- a/device/receive.go +++ b/device/receive.go @@ -445,7 +445,9 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { return } elemsContainer.Lock() - for _, elem := range elemsContainer.elems { + validTailPacket := -1 + dataPacketReceived := false + for i, elem := range elemsContainer.elems { if elem.packet == nil { // decryption failed continue @@ -455,21 +457,19 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { continue } - peer.SetEndpointFromPacket(elem.endpoint) + validTailPacket = i if peer.ReceivedWithKeypair(elem.keypair) { + peer.SetEndpointFromPacket(elem.endpoint) peer.timersHandshakeComplete() peer.SendStagedPackets() } - peer.keepKeyFreshReceiving() - peer.timersAnyAuthenticatedPacketTraversal() - peer.timersAnyAuthenticatedPacketReceived() peer.rxBytes.Add(uint64(len(elem.packet) + MinMessageSize)) if len(elem.packet) == 0 { device.log.Verbosef("%v - Receiving keepalive packet", peer) continue } - peer.timersDataReceived() + dataPacketReceived = true switch elem.packet[0] >> 4 { case 4: @@ -512,6 +512,15 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { bufs = append(bufs, elem.buffer[:MessageTransportOffsetContent+len(elem.packet)]) } + if validTailPacket >= 0 { + peer.SetEndpointFromPacket(elemsContainer.elems[validTailPacket].endpoint) + peer.keepKeyFreshReceiving() + peer.timersAnyAuthenticatedPacketTraversal() + peer.timersAnyAuthenticatedPacketReceived() + } + if dataPacketReceived { + peer.timersDataReceived() + } if len(bufs) > 0 { _, err := device.tun.device.Write(bufs, MessageTransportOffsetContent) if err != nil && !device.isClosed() { diff --git a/device/send.go b/device/send.go index 95a5cbe..2701f4e 100644 --- a/device/send.go +++ b/device/send.go @@ -436,7 +436,7 @@ func calculatePaddingSize(packetSize, mtu int) int { return paddedSize - lastUnit } -/* Encrypts the elems in the queue +/* Encrypts the elements in the queue * and marks them for sequential consumption (by releasing the mutex) * * Obs. One instance per core @@ -495,7 +495,7 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) { return } if !peer.isRunning.Load() { - // peer has been stopped; return re-usable elemsContainer to the shared pool. + // peer has been stopped; return re-usable elems to the shared pool. // This is an optimization only. It is possible for the peer to be stopped // immediately after this check, in which case, elem will get processed. // The timers and SendBuffers code are resilient to a few stragglers. From 64040e66467d89c9312cf442866185404b7ec60c Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sat, 13 Apr 2024 10:55:05 -0700 Subject: [PATCH 17/60] ipc: build on aix Updates tailscale/tailscale#11361 --- ipc/uapi_fake.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipc/uapi_fake.go b/ipc/uapi_fake.go index e68863d..3e3dcdf 100644 --- a/ipc/uapi_fake.go +++ b/ipc/uapi_fake.go @@ -1,4 +1,4 @@ -//go:build wasm || plan9 +//go:build wasm || plan9 || aix /* SPDX-License-Identifier: MIT * From 03c5a0ccf7546055344017505460bdb9c0425b17 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Mon, 29 Apr 2024 09:15:56 -0700 Subject: [PATCH 18/60] tun: implement API for disabling UDP GRO on Linux Certain device drivers (e.g. vxlan, geneve) do not properly handle coalesced UDP packets later in the stack, resulting in packet loss. Signed-off-by: Jordan Whited --- tun/tun.go | 8 ++++++++ tun/tun_linux.go | 12 ++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tun/tun.go b/tun/tun.go index 0ae53d0..d3c5012 100644 --- a/tun/tun.go +++ b/tun/tun.go @@ -51,3 +51,11 @@ type Device interface { // lifetime of a Device. BatchSize() int } + +type LinuxDevice interface { + Device + // DisableUDPGRO disables UDP GRO if it is enabled. Certain device drivers + // (e.g. vxlan, geneve) do not properly handle coalesced UDP packets later + // in the stack, resulting in packet loss. + DisableUDPGRO() +} diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 9313ebf..6aa03d4 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -49,10 +49,11 @@ type NativeTun struct { readOpMu sync.Mutex // readOpMu guards readBuff readBuff [virtioNetHdrLen + 65535]byte // if vnetHdr every read() is prefixed by virtioNetHdr - writeOpMu sync.Mutex // writeOpMu guards toWrite, tcpGROTable + writeOpMu sync.Mutex // writeOpMu guards the following fields toWrite []int tcpGROTable *tcpGROTable udpGROTable *udpGROTable + udpGRO bool } func (tun *NativeTun) File() *os.File { @@ -345,7 +346,7 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) { ) tun.toWrite = tun.toWrite[:0] if tun.vnetHdr { - err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.udpGSO, &tun.toWrite) + err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.udpGRO, &tun.toWrite) if err != nil { return 0, err } @@ -502,6 +503,12 @@ func (tun *NativeTun) BatchSize() int { return tun.batchSize } +func (tun *NativeTun) DisableUDPGRO() { + tun.writeOpMu.Lock() + tun.udpGRO = false + tun.writeOpMu.Unlock() +} + const ( // TODO: support TSO with ECN bits tunTCPOffloads = unix.TUN_F_CSUM | unix.TUN_F_TSO4 | unix.TUN_F_TSO6 @@ -538,6 +545,7 @@ func (tun *NativeTun) initFromFlags(name string) error { // tunUDPOffloads were added in Linux v6.2. We do not return an // error if they are unsupported at runtime. tun.udpGSO = unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) == nil + tun.udpGRO = tun.udpGSO } else { tun.batchSize = 1 } From 1e088837d114a74a2c335968e7d40537447b9b4f Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 27 Jun 2024 08:43:41 -0700 Subject: [PATCH 19/60] device: fix WaitPool sync.Cond usage The sync.Locker used with a sync.Cond must be acquired when changing the associated condition, otherwise there is a window within sync.Cond.Wait() where a wake-up may be missed. Fixes: 4846070 ("device: use a waiting sync.Pool instead of a channel") Signed-off-by: Jordan Whited --- device/pools.go | 11 ++++++----- device/pools_test.go | 5 +++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/device/pools.go b/device/pools.go index 94f3dc7..55d2be7 100644 --- a/device/pools.go +++ b/device/pools.go @@ -7,14 +7,13 @@ package device import ( "sync" - "sync/atomic" ) type WaitPool struct { pool sync.Pool cond sync.Cond lock sync.Mutex - count atomic.Uint32 + count uint32 // Get calls not yet Put back max uint32 } @@ -27,10 +26,10 @@ func NewWaitPool(max uint32, new func() any) *WaitPool { func (p *WaitPool) Get() any { if p.max != 0 { p.lock.Lock() - for p.count.Load() >= p.max { + for p.count >= p.max { p.cond.Wait() } - p.count.Add(1) + p.count++ p.lock.Unlock() } return p.pool.Get() @@ -41,7 +40,9 @@ func (p *WaitPool) Put(x any) { if p.max == 0 { return } - p.count.Add(^uint32(0)) + p.lock.Lock() + defer p.lock.Unlock() + p.count-- p.cond.Signal() } diff --git a/device/pools_test.go b/device/pools_test.go index 82d7493..2b16f39 100644 --- a/device/pools_test.go +++ b/device/pools_test.go @@ -15,7 +15,6 @@ import ( ) func TestWaitPool(t *testing.T) { - t.Skip("Currently disabled") var wg sync.WaitGroup var trials atomic.Int32 startTrials := int32(100000) @@ -32,7 +31,9 @@ func TestWaitPool(t *testing.T) { wg.Add(workers) var max atomic.Uint32 updateMax := func() { - count := p.count.Load() + p.lock.Lock() + count := p.count + p.lock.Unlock() if count > p.max { t.Errorf("count (%d) > max (%d)", count, p.max) } From cfa45674af86ac5074d879257ca513438bb291eb Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 27 Jun 2024 09:06:40 -0700 Subject: [PATCH 20/60] device: fix missed return of QueueOutboundElementsContainer to its WaitPool Fixes: 3bb8fec ("conn, device, tun: implement vectorized I/O plumbing") Signed-off-by: Jordan Whited --- device/send.go | 1 + 1 file changed, 1 insertion(+) diff --git a/device/send.go b/device/send.go index 2701f4e..8ed2e5f 100644 --- a/device/send.go +++ b/device/send.go @@ -506,6 +506,7 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) { device.PutMessageBuffer(elem.buffer) device.PutOutboundElement(elem) } + device.PutOutboundElementsContainer(elemsContainer) continue } dataSent := false From 2f5d148bcfe13a65d0f85dca8698200543c226da Mon Sep 17 00:00:00 2001 From: James Tucker Date: Fri, 7 Jun 2024 16:57:40 -0700 Subject: [PATCH 21/60] conn,device: enable cryptorouting via PeerAwareEndpoint Introduce an optional extension point for Endpoint that enables a path for WireGuard to inform an integration about the peer public key that is associated with an Endpoint. The API is expected to return either the same or a new Endpoint in response to this function. A future version of this patch could potentially remove the returned Endpoint, but would require larger integrator changes downstream. This adds a small per-packet cost that could later be removed with a larger refactor of the wireguard-go interface and Tailscale magicsock code, as well as introducing a generic bound for Endpoint in a device & bind instance. Updates tailscale/corp#20732 --- conn/conn.go | 14 ++++++++++++++ device/noise-protocol.go | 2 +- device/peer.go | 3 +++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/conn/conn.go b/conn/conn.go index a1f57d2..8df5aaa 100644 --- a/conn/conn.go +++ b/conn/conn.go @@ -84,6 +84,20 @@ type Endpoint interface { SrcIP() netip.Addr } +// PeerAwareEndpoint is an optional Endpoint specialization for +// integrations that want to know about the outcome of cryptorouting +// identification. +// +// If they receive a packet from a source they had not pre-identified, +// to learn the identification WireGuard can derive from the session +// or handshake. +// +// If GetPeerEndpoint returns nil, WireGuard will be unable to respond +// to the peer until a new endpoint is written by a later packet. +type PeerAwareEndpoint interface { + GetPeerEndpoint(peerPublicKey [32]byte) Endpoint +} + var ( ErrBindAlreadyOpen = errors.New("bind is already open") ErrWrongEndpointType = errors.New("endpoint type does not correspond with bind type") diff --git a/device/noise-protocol.go b/device/noise-protocol.go index 9f2ba50..2d8f984 100644 --- a/device/noise-protocol.go +++ b/device/noise-protocol.go @@ -124,7 +124,7 @@ type Handshake struct { localEphemeral NoisePrivateKey // ephemeral secret key localIndex uint32 // used to clear hash-table remoteIndex uint32 // index for sending - remoteStatic NoisePublicKey // long term key + remoteStatic NoisePublicKey // long term key, never changes, can be accessed without mutex remoteEphemeral NoisePublicKey // ephemeral public key precomputedStaticStatic [NoisePublicKeySize]byte // precomputed shared secret lastTimestamp tai64n.Timestamp diff --git a/device/peer.go b/device/peer.go index 89b719b..876e5da 100644 --- a/device/peer.go +++ b/device/peer.go @@ -283,6 +283,9 @@ func (peer *Peer) SetEndpointFromPacket(endpoint conn.Endpoint) { return } peer.endpoint.clearSrcOnTx = false + if ep, ok := endpoint.(conn.PeerAwareEndpoint); ok { + endpoint = ep.GetPeerEndpoint(peer.handshake.remoteStatic) + } peer.endpoint.val = endpoint } From 60eeedfd624bb0ee4d173c1aaac1ed0ddedf13fe Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 16 Jul 2024 15:36:20 -0700 Subject: [PATCH 22/60] tun: export GSOSplit() for external Device implementers External implementers of tun.Device may support GSO, and may also be platform-agnostic, e.g. gVisor. Signed-off-by: Jordan Whited --- tun/offload.go | 220 +++++++++++++++++++++++++++++++++++++++++++ tun/offload_linux.go | 134 +++++--------------------- tun/offload_test.go | 95 +++++++++++++++++++ tun/tun_linux.go | 67 +++---------- 4 files changed, 354 insertions(+), 162 deletions(-) create mode 100644 tun/offload.go create mode 100644 tun/offload_test.go diff --git a/tun/offload.go b/tun/offload.go new file mode 100644 index 0000000..6627e46 --- /dev/null +++ b/tun/offload.go @@ -0,0 +1,220 @@ +package tun + +import ( + "encoding/binary" + "fmt" +) + +// GSOType represents the type of segmentation offload. +type GSOType int + +const ( + GSONone GSOType = iota + GSOTCPv4 + GSOTCPv6 + GSOUDPL4 +) + +func (g GSOType) String() string { + switch g { + case GSONone: + return "GSONone" + case GSOTCPv4: + return "GSOTCPv4" + case GSOTCPv6: + return "GSOTCPv6" + case GSOUDPL4: + return "GSOUDPL4" + default: + return "unknown" + } +} + +// GSOOptions is loosely modeled after struct virtio_net_hdr from the VIRTIO +// specification. It is a common representation of GSO metadata that can be +// applied to support packet GSO across tun.Device implementations. +type GSOOptions struct { + // GSOType represents the type of segmentation offload. + GSOType GSOType + // HdrLen is the sum of the layer 3 and 4 header lengths. This field may be + // zero when GSOType == GSONone. + HdrLen uint16 + // CsumStart is the head byte index of the packet data to be checksummed, + // i.e. the start of the TCP or UDP header. + CsumStart uint16 + // CsumOffset is the offset from CsumStart where the 2-byte checksum value + // should be placed. + CsumOffset uint16 + // GSOSize is the size of each segment exclusive of HdrLen. The tail segment + // may be smaller than this value. + GSOSize uint16 + // NeedsCsum may be set where GSOType == GSONone. When set, the checksum + // at CsumStart + CsumOffset must be a partial checksum, i.e. the + // pseudo-header sum. + NeedsCsum bool +} + +const ( + ipv4SrcAddrOffset = 12 + ipv6SrcAddrOffset = 8 +) + +const tcpFlagsOffset = 13 + +const ( + tcpFlagFIN uint8 = 0x01 + tcpFlagPSH uint8 = 0x08 + tcpFlagACK uint8 = 0x10 +) + +const ( + // defined here in order to avoid importation of any platform-specific pkgs + ipProtoTCP = 6 + ipProtoUDP = 17 +) + +// GSOSplit splits packets from 'in' into outBufs[][outOffset:], writing +// the size of each element into sizes. It returns the number of buffers +// populated, and/or an error. Callers may pass an 'in' slice that overlaps with +// the first element of outBuffers, i.e. &in[0] may be equal to +// &outBufs[0][outOffset]. GSONone is a valid options.GSOType regardless of the +// value of options.NeedsCsum. Length of each outBufs element must be greater +// than or equal to the length of 'in', otherwise output may be silently +// truncated. +func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outOffset int) (int, error) { + cSumAt := int(options.CsumStart) + int(options.CsumOffset) + if cSumAt+1 >= len(in) { + return 0, fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(in)) + } + + if len(in) < int(options.HdrLen) { + return 0, fmt.Errorf("length of packet (%d) < GSO HdrLen (%d)", len(in), options.HdrLen) + } + + // Handle the conditions where we are copying a single element to outBuffs. + payloadLen := len(in) - int(options.HdrLen) + if options.GSOType == GSONone || payloadLen < int(options.GSOSize) { + if len(in) > len(outBufs[0][outOffset:]) { + return 0, fmt.Errorf("length of packet (%d) exceeds output element length (%d)", len(in), len(outBufs[0][outOffset:])) + } + if options.NeedsCsum { + // The initial value at the checksum offset should be summed with + // the checksum we compute. This is typically the pseudo-header sum. + initial := binary.BigEndian.Uint16(in[cSumAt:]) + in[cSumAt], in[cSumAt+1] = 0, 0 + binary.BigEndian.PutUint16(in[cSumAt:], ^checksum(in[options.CsumStart:], initial)) + } + sizes[0] = copy(outBufs[0][outOffset:], in) + return 1, nil + } + + if options.HdrLen < options.CsumStart { + return 0, fmt.Errorf("GSO HdrLen (%d) < GSO CsumStart (%d)", options.HdrLen, options.CsumStart) + } + + ipVersion := in[0] >> 4 + switch ipVersion { + case 4: + if options.GSOType != GSOTCPv4 && options.GSOType != GSOUDPL4 { + return 0, fmt.Errorf("ip header version: %d, GSO type: %s", ipVersion, options.GSOType) + } + if len(in) < 20 { + return 0, fmt.Errorf("length of packet (%d) < minimum ipv4 header size (%d)", len(in), 20) + } + case 6: + if options.GSOType != GSOTCPv6 && options.GSOType != GSOUDPL4 { + return 0, fmt.Errorf("ip header version: %d, GSO type: %s", ipVersion, options.GSOType) + } + if len(in) < 40 { + return 0, fmt.Errorf("length of packet (%d) < minimum ipv6 header size (%d)", len(in), 40) + } + default: + return 0, fmt.Errorf("invalid ip header version: %d", ipVersion) + } + + iphLen := int(options.CsumStart) + srcAddrOffset := ipv6SrcAddrOffset + addrLen := 16 + if ipVersion == 4 { + srcAddrOffset = ipv4SrcAddrOffset + addrLen = 4 + } + transportCsumAt := int(options.CsumStart + options.CsumOffset) + var firstTCPSeqNum uint32 + var protocol uint8 + if options.GSOType == GSOTCPv4 || options.GSOType == GSOTCPv6 { + protocol = ipProtoTCP + if len(in) < int(options.CsumStart)+20 { + return 0, fmt.Errorf("length of packet (%d) < GSO CsumStart (%d) + minimum TCP header size (%d)", + len(in), options.CsumStart, 20) + } + firstTCPSeqNum = binary.BigEndian.Uint32(in[options.CsumStart+4:]) + } else { + protocol = ipProtoUDP + } + nextSegmentDataAt := int(options.HdrLen) + i := 0 + for ; nextSegmentDataAt < len(in); i++ { + if i == len(outBufs) { + return i - 1, ErrTooManySegments + } + nextSegmentEnd := nextSegmentDataAt + int(options.GSOSize) + if nextSegmentEnd > len(in) { + nextSegmentEnd = len(in) + } + segmentDataLen := nextSegmentEnd - nextSegmentDataAt + totalLen := int(options.HdrLen) + segmentDataLen + sizes[i] = totalLen + out := outBufs[i][outOffset:] + + copy(out, in[:iphLen]) + if ipVersion == 4 { + // For IPv4 we are responsible for incrementing the ID field, + // updating the total len field, and recalculating the header + // checksum. + if i > 0 { + id := binary.BigEndian.Uint16(out[4:]) + id += uint16(i) + binary.BigEndian.PutUint16(out[4:], id) + } + out[10], out[11] = 0, 0 // clear ipv4 header checksum + binary.BigEndian.PutUint16(out[2:], uint16(totalLen)) + ipv4CSum := ^checksum(out[:iphLen], 0) + binary.BigEndian.PutUint16(out[10:], ipv4CSum) + } else { + // For IPv6 we are responsible for updating the payload length field. + binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen)) + } + + // copy transport header + copy(out[options.CsumStart:options.HdrLen], in[options.CsumStart:options.HdrLen]) + + if protocol == ipProtoTCP { + // set TCP seq and adjust TCP flags + tcpSeq := firstTCPSeqNum + uint32(options.GSOSize*uint16(i)) + binary.BigEndian.PutUint32(out[options.CsumStart+4:], tcpSeq) + if nextSegmentEnd != len(in) { + // FIN and PSH should only be set on last segment + clearFlags := tcpFlagFIN | tcpFlagPSH + out[options.CsumStart+tcpFlagsOffset] &^= clearFlags + } + } else { + // set UDP header len + binary.BigEndian.PutUint16(out[options.CsumStart+4:], uint16(segmentDataLen)+(options.HdrLen-options.CsumStart)) + } + + // payload + copy(out[options.HdrLen:], in[nextSegmentDataAt:nextSegmentEnd]) + + // transport checksum + out[transportCsumAt], out[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum + transportHeaderLen := int(options.HdrLen - options.CsumStart) + lenForPseudo := uint16(transportHeaderLen + segmentDataLen) + transportCSum := pseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) + transportCSum = ^checksum(out[options.CsumStart:totalLen], transportCSum) + binary.BigEndian.PutUint16(out[options.CsumStart+options.CsumOffset:], transportCSum) + + nextSegmentDataAt += int(options.GSOSize) + } + return i, nil +} diff --git a/tun/offload_linux.go b/tun/offload_linux.go index 9a9d38e..3f0dc53 100644 --- a/tun/offload_linux.go +++ b/tun/offload_linux.go @@ -9,6 +9,7 @@ import ( "bytes" "encoding/binary" "errors" + "fmt" "io" "unsafe" @@ -16,14 +17,6 @@ import ( "golang.org/x/sys/unix" ) -const tcpFlagsOffset = 13 - -const ( - tcpFlagFIN uint8 = 0x01 - tcpFlagPSH uint8 = 0x08 - tcpFlagACK uint8 = 0x10 -) - // virtioNetHdr is defined in the kernel in include/uapi/linux/virtio_net.h. The // kernel symbol is virtio_net_hdr. type virtioNetHdr struct { @@ -35,6 +28,30 @@ type virtioNetHdr struct { csumOffset uint16 } +func (v *virtioNetHdr) toGSOOptions() (GSOOptions, error) { + var gsoType GSOType + switch v.gsoType { + case unix.VIRTIO_NET_HDR_GSO_NONE: + gsoType = GSONone + case unix.VIRTIO_NET_HDR_GSO_TCPV4: + gsoType = GSOTCPv4 + case unix.VIRTIO_NET_HDR_GSO_TCPV6: + gsoType = GSOTCPv6 + case unix.VIRTIO_NET_HDR_GSO_UDP_L4: + gsoType = GSOUDPL4 + default: + return GSOOptions{}, fmt.Errorf("unsupported virtio gsoType: %d", v.gsoType) + } + return GSOOptions{ + GSOType: gsoType, + HdrLen: v.hdrLen, + CsumStart: v.csumStart, + CsumOffset: v.csumOffset, + GSOSize: v.gsoSize, + NeedsCsum: v.flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0, + }, nil +} + func (v *virtioNetHdr) decode(b []byte) error { if len(b) < virtioNetHdrLen { return io.ErrShortBuffer @@ -510,9 +527,7 @@ const ( ) const ( - ipv4SrcAddrOffset = 12 - ipv6SrcAddrOffset = 8 - maxUint16 = 1<<16 - 1 + maxUint16 = 1<<16 - 1 ) type groResult int @@ -894,100 +909,3 @@ func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGR errUDP := applyUDPCoalesceAccounting(bufs, offset, udpTable) return errors.Join(errTCP, errUDP) } - -// gsoSplit splits packets from in into outBuffs, writing the size of each -// element into sizes. It returns the number of buffers populated, and/or an -// error. -func gsoSplit(in []byte, hdr virtioNetHdr, outBuffs [][]byte, sizes []int, outOffset int, isV6 bool) (int, error) { - iphLen := int(hdr.csumStart) - srcAddrOffset := ipv6SrcAddrOffset - addrLen := 16 - if !isV6 { - in[10], in[11] = 0, 0 // clear ipv4 header checksum - srcAddrOffset = ipv4SrcAddrOffset - addrLen = 4 - } - transportCsumAt := int(hdr.csumStart + hdr.csumOffset) - in[transportCsumAt], in[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum - var firstTCPSeqNum uint32 - var protocol uint8 - if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV4 || hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV6 { - protocol = unix.IPPROTO_TCP - firstTCPSeqNum = binary.BigEndian.Uint32(in[hdr.csumStart+4:]) - } else { - protocol = unix.IPPROTO_UDP - } - nextSegmentDataAt := int(hdr.hdrLen) - i := 0 - for ; nextSegmentDataAt < len(in); i++ { - if i == len(outBuffs) { - return i - 1, ErrTooManySegments - } - nextSegmentEnd := nextSegmentDataAt + int(hdr.gsoSize) - if nextSegmentEnd > len(in) { - nextSegmentEnd = len(in) - } - segmentDataLen := nextSegmentEnd - nextSegmentDataAt - totalLen := int(hdr.hdrLen) + segmentDataLen - sizes[i] = totalLen - out := outBuffs[i][outOffset:] - - copy(out, in[:iphLen]) - if !isV6 { - // For IPv4 we are responsible for incrementing the ID field, - // updating the total len field, and recalculating the header - // checksum. - if i > 0 { - id := binary.BigEndian.Uint16(out[4:]) - id += uint16(i) - binary.BigEndian.PutUint16(out[4:], id) - } - binary.BigEndian.PutUint16(out[2:], uint16(totalLen)) - ipv4CSum := ^checksum(out[:iphLen], 0) - binary.BigEndian.PutUint16(out[10:], ipv4CSum) - } else { - // For IPv6 we are responsible for updating the payload length field. - binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen)) - } - - // copy transport header - copy(out[hdr.csumStart:hdr.hdrLen], in[hdr.csumStart:hdr.hdrLen]) - - if protocol == unix.IPPROTO_TCP { - // set TCP seq and adjust TCP flags - tcpSeq := firstTCPSeqNum + uint32(hdr.gsoSize*uint16(i)) - binary.BigEndian.PutUint32(out[hdr.csumStart+4:], tcpSeq) - if nextSegmentEnd != len(in) { - // FIN and PSH should only be set on last segment - clearFlags := tcpFlagFIN | tcpFlagPSH - out[hdr.csumStart+tcpFlagsOffset] &^= clearFlags - } - } else { - // set UDP header len - binary.BigEndian.PutUint16(out[hdr.csumStart+4:], uint16(segmentDataLen)+(hdr.hdrLen-hdr.csumStart)) - } - - // payload - copy(out[hdr.hdrLen:], in[nextSegmentDataAt:nextSegmentEnd]) - - // transport checksum - transportHeaderLen := int(hdr.hdrLen - hdr.csumStart) - lenForPseudo := uint16(transportHeaderLen + segmentDataLen) - transportCSum := pseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) - transportCSum = ^checksum(out[hdr.csumStart:totalLen], transportCSum) - binary.BigEndian.PutUint16(out[hdr.csumStart+hdr.csumOffset:], transportCSum) - - nextSegmentDataAt += int(hdr.gsoSize) - } - return i, nil -} - -func gsoNoneChecksum(in []byte, cSumStart, cSumOffset uint16) error { - cSumAt := cSumStart + cSumOffset - // The initial value at the checksum offset should be summed with the - // checksum we compute. This is typically the pseudo-header checksum. - initial := binary.BigEndian.Uint16(in[cSumAt:]) - in[cSumAt], in[cSumAt+1] = 0, 0 - binary.BigEndian.PutUint16(in[cSumAt:], ^checksum(in[cSumStart:], initial)) - return nil -} diff --git a/tun/offload_test.go b/tun/offload_test.go new file mode 100644 index 0000000..82a37b9 --- /dev/null +++ b/tun/offload_test.go @@ -0,0 +1,95 @@ +package tun + +import ( + "net/netip" + "testing" + + "github.com/tailscale/wireguard-go/conn" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/header" +) + +func Fuzz_GSOSplit(f *testing.F) { + const segmentSize = 100 + + tcpFields := &header.TCPFields{ + SrcPort: 1, + DstPort: 1, + SeqNum: 1, + AckNum: 1, + DataOffset: 20, + Flags: header.TCPFlagAck | header.TCPFlagPsh, + WindowSize: 3000, + } + udpFields := &header.UDPFields{ + SrcPort: 1, + DstPort: 1, + Length: 8 + segmentSize, + } + + gsoTCPv4 := make([]byte, 20+20+segmentSize) + header.IPv4(gsoTCPv4).Encode(&header.IPv4Fields{ + SrcAddr: tcpip.AddrFromSlice(netip.MustParseAddr("192.0.2.1").AsSlice()), + DstAddr: tcpip.AddrFromSlice(netip.MustParseAddr("192.0.2.2").AsSlice()), + Protocol: ipProtoTCP, + TTL: 64, + TotalLength: uint16(len(gsoTCPv4)), + }) + header.TCP(gsoTCPv4[20:]).Encode(tcpFields) + + gsoUDPv4 := make([]byte, 20+8+segmentSize) + header.IPv4(gsoUDPv4).Encode(&header.IPv4Fields{ + SrcAddr: tcpip.AddrFromSlice(netip.MustParseAddr("192.0.2.1").AsSlice()), + DstAddr: tcpip.AddrFromSlice(netip.MustParseAddr("192.0.2.2").AsSlice()), + Protocol: ipProtoUDP, + TTL: 64, + TotalLength: uint16(len(gsoUDPv4)), + }) + header.UDP(gsoTCPv4[20:]).Encode(udpFields) + + gsoTCPv6 := make([]byte, 40+20+segmentSize) + header.IPv6(gsoTCPv6).Encode(&header.IPv6Fields{ + SrcAddr: tcpip.AddrFromSlice(netip.MustParseAddr("2001:db8::1").AsSlice()), + DstAddr: tcpip.AddrFromSlice(netip.MustParseAddr("2001:db8::2").AsSlice()), + TransportProtocol: ipProtoTCP, + HopLimit: 64, + PayloadLength: uint16(20 + segmentSize), + }) + header.TCP(gsoTCPv6[40:]).Encode(tcpFields) + + gsoUDPv6 := make([]byte, 40+8+segmentSize) + header.IPv6(gsoUDPv6).Encode(&header.IPv6Fields{ + SrcAddr: tcpip.AddrFromSlice(netip.MustParseAddr("2001:db8::1").AsSlice()), + DstAddr: tcpip.AddrFromSlice(netip.MustParseAddr("2001:db8::2").AsSlice()), + TransportProtocol: ipProtoUDP, + HopLimit: 64, + PayloadLength: uint16(8 + segmentSize), + }) + header.UDP(gsoUDPv6[20:]).Encode(udpFields) + + out := make([][]byte, conn.IdealBatchSize) + for i := range out { + out[i] = make([]byte, 65535) + } + sizes := make([]int, conn.IdealBatchSize) + + f.Add(gsoTCPv4, int(GSOTCPv4), uint16(40), uint16(20), uint16(16), uint16(100), false) + f.Add(gsoUDPv4, int(GSOUDPL4), uint16(28), uint16(20), uint16(6), uint16(100), false) + f.Add(gsoTCPv6, int(GSOTCPv6), uint16(60), uint16(40), uint16(16), uint16(100), false) + f.Add(gsoUDPv6, int(GSOUDPL4), uint16(48), uint16(40), uint16(6), uint16(100), false) + + f.Fuzz(func(t *testing.T, pkt []byte, gsoType int, hdrLen, csumStart, csumOffset, gsoSize uint16, needsCsum bool) { + options := GSOOptions{ + GSOType: GSOType(gsoType), + HdrLen: hdrLen, + CsumStart: csumStart, + CsumOffset: csumOffset, + GSOSize: gsoSize, + NeedsCsum: needsCsum, + } + n, _ := GSOSplit(pkt, options, out, sizes, 0) + if n > len(sizes) { + t.Errorf("n (%d) > len(sizes): %d", n, len(sizes)) + } + }) +} diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 6aa03d4..664eecc 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -380,73 +380,32 @@ func handleVirtioRead(in []byte, bufs [][]byte, sizes []int, offset int) (int, e return 0, err } in = in[virtioNetHdrLen:] - if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_NONE { - if hdr.flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0 { - // This means CHECKSUM_PARTIAL in skb context. We are responsible - // for computing the checksum starting at hdr.csumStart and placing - // at hdr.csumOffset. - err = gsoNoneChecksum(in, hdr.csumStart, hdr.csumOffset) - if err != nil { - return 0, err - } - } - if len(in) > len(bufs[0][offset:]) { - return 0, fmt.Errorf("read len %d overflows bufs element len %d", len(in), len(bufs[0][offset:])) - } - n := copy(bufs[0][offset:], in) - sizes[0] = n - return 1, nil - } - if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { - return 0, fmt.Errorf("unsupported virtio GSO type: %d", hdr.gsoType) + + options, err := hdr.toGSOOptions() + if err != nil { + return 0, err } - ipVersion := in[0] >> 4 - switch ipVersion { - case 4: - if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { - return 0, fmt.Errorf("ip header version: %d, GSO type: %d", ipVersion, hdr.gsoType) - } - case 6: - if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { - return 0, fmt.Errorf("ip header version: %d, GSO type: %d", ipVersion, hdr.gsoType) - } - default: - return 0, fmt.Errorf("invalid ip header version: %d", ipVersion) - } - - // Don't trust hdr.hdrLen from the kernel as it can be equal to the length + // Don't trust HdrLen from the kernel as it can be equal to the length // of the entire first packet when the kernel is handling it as part of a // FORWARD path. Instead, parse the transport header length and add it onto - // csumStart, which is synonymous for IP header length. - if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_UDP_L4 { - hdr.hdrLen = hdr.csumStart + 8 - } else { - if len(in) <= int(hdr.csumStart+12) { + // CsumStart, which is synonymous for IP header length. + if options.GSOType == GSOUDPL4 { + options.HdrLen = options.CsumStart + 8 + } else if options.GSOType != GSONone { + if len(in) <= int(options.CsumStart+12) { return 0, errors.New("packet is too short") } - tcpHLen := uint16(in[hdr.csumStart+12] >> 4 * 4) + tcpHLen := uint16(in[options.CsumStart+12] >> 4 * 4) if tcpHLen < 20 || tcpHLen > 60 { // A TCP header must be between 20 and 60 bytes in length. return 0, fmt.Errorf("tcp header len is invalid: %d", tcpHLen) } - hdr.hdrLen = hdr.csumStart + tcpHLen + options.HdrLen = options.CsumStart + tcpHLen } - if len(in) < int(hdr.hdrLen) { - return 0, fmt.Errorf("length of packet (%d) < virtioNetHdr.hdrLen (%d)", len(in), hdr.hdrLen) - } - - if hdr.hdrLen < hdr.csumStart { - return 0, fmt.Errorf("virtioNetHdr.hdrLen (%d) < virtioNetHdr.csumStart (%d)", hdr.hdrLen, hdr.csumStart) - } - cSumAt := int(hdr.csumStart + hdr.csumOffset) - if cSumAt+1 >= len(in) { - return 0, fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(in)) - } - - return gsoSplit(in, hdr, bufs, sizes, offset, ipVersion == 6) + return GSOSplit(in, options, bufs, sizes, offset) } func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) { From 6c039a188c2d15592023f851731cd03214af32ec Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 23 Jul 2024 13:34:34 -0700 Subject: [PATCH 23/60] tun: export optimized IP checksum funcs External implementers of tun.Device may support GRO, requiring checksum offload. Signed-off-by: Jordan Whited --- tun/checksum.go | 4 +++- tun/checksum_amd64.go | 11 +++++++---- tun/checksum_generic.go | 2 +- tun/offload.go | 8 ++++---- tun/offload_linux.go | 16 ++++++++-------- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/tun/checksum.go b/tun/checksum.go index ee3f359..6634050 100644 --- a/tun/checksum.go +++ b/tun/checksum.go @@ -702,7 +702,9 @@ func pseudoHeaderChecksum32(protocol uint8, srcAddr, dstAddr []byte, totalLen ui return foldedSum } -func pseudoHeaderChecksum(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { +// PseudoHeaderChecksum computes an IP pseudo-header checksum. srcAddr and +// dstAddr must be 4 or 16 bytes in length. +func PseudoHeaderChecksum(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { if strconv.IntSize < 64 { return pseudoHeaderChecksum32(protocol, srcAddr, dstAddr, totalLen) } diff --git a/tun/checksum_amd64.go b/tun/checksum_amd64.go index 5e87693..4fb684e 100644 --- a/tun/checksum_amd64.go +++ b/tun/checksum_amd64.go @@ -2,12 +2,15 @@ package tun import "golang.org/x/sys/cpu" -// checksum computes an IP checksum starting with the provided initial value. -// The length of data should be at least 128 bytes for best performance. Smaller -// buffers will still compute a correct result. For best performance with -// smaller buffers, use shortChecksum(). var checksum = checksumAMD64 +// Checksum computes an IP checksum starting with the provided initial value. +// The length of data should be at least 128 bytes for best performance. Smaller +// buffers will still compute a correct result. +func Checksum(data []byte, initial uint16) uint16 { + return checksum(data, initial) +} + func init() { if cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 { checksum = checksumAVX2 diff --git a/tun/checksum_generic.go b/tun/checksum_generic.go index d0bfb69..2ef201a 100644 --- a/tun/checksum_generic.go +++ b/tun/checksum_generic.go @@ -7,7 +7,7 @@ package tun import "strconv" -func checksum(data []byte, initial uint16) uint16 { +func Checksum(data []byte, initial uint16) uint16 { if strconv.IntSize < 64 { return checksumGeneric32(data, initial) } diff --git a/tun/offload.go b/tun/offload.go index 6627e46..6db437c 100644 --- a/tun/offload.go +++ b/tun/offload.go @@ -102,7 +102,7 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO // the checksum we compute. This is typically the pseudo-header sum. initial := binary.BigEndian.Uint16(in[cSumAt:]) in[cSumAt], in[cSumAt+1] = 0, 0 - binary.BigEndian.PutUint16(in[cSumAt:], ^checksum(in[options.CsumStart:], initial)) + binary.BigEndian.PutUint16(in[cSumAt:], ^Checksum(in[options.CsumStart:], initial)) } sizes[0] = copy(outBufs[0][outOffset:], in) return 1, nil @@ -179,7 +179,7 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO } out[10], out[11] = 0, 0 // clear ipv4 header checksum binary.BigEndian.PutUint16(out[2:], uint16(totalLen)) - ipv4CSum := ^checksum(out[:iphLen], 0) + ipv4CSum := ^Checksum(out[:iphLen], 0) binary.BigEndian.PutUint16(out[10:], ipv4CSum) } else { // For IPv6 we are responsible for updating the payload length field. @@ -210,8 +210,8 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO out[transportCsumAt], out[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum transportHeaderLen := int(options.HdrLen - options.CsumStart) lenForPseudo := uint16(transportHeaderLen + segmentDataLen) - transportCSum := pseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) - transportCSum = ^checksum(out[options.CsumStart:totalLen], transportCSum) + transportCSum := PseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) + transportCSum = ^Checksum(out[options.CsumStart:totalLen], transportCSum) binary.BigEndian.PutUint16(out[options.CsumStart+options.CsumOffset:], transportCSum) nextSegmentDataAt += int(options.GSOSize) diff --git a/tun/offload_linux.go b/tun/offload_linux.go index 3f0dc53..fe34401 100644 --- a/tun/offload_linux.go +++ b/tun/offload_linux.go @@ -410,8 +410,8 @@ func checksumValid(pkt []byte, iphLen, proto uint8, isV6 bool) bool { addrSize = 16 } lenForPseudo := uint16(len(pkt) - int(iphLen)) - cSum := pseudoHeaderChecksum(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo) - return ^checksum(pkt[iphLen:], cSum) == 0 + cSum := PseudoHeaderChecksum(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo) + return ^Checksum(pkt[iphLen:], cSum) == 0 } // coalesceResult represents the result of attempting to coalesce two TCP @@ -659,7 +659,7 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4 pkt[10], pkt[11] = 0, 0 binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length - iphCSum := ^checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum + iphCSum := ^Checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field } err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -679,8 +679,8 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e srcAddrAt := offset + addrOffset srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] - psum := pseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) - binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) + psum := PseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) + binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum)) } else { hdr := virtioNetHdr{} err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -716,7 +716,7 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e } else { pkt[10], pkt[11] = 0, 0 binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length - iphCSum := ^checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum + iphCSum := ^Checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field } err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -739,8 +739,8 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e srcAddrAt := offset + addrOffset srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] - psum := pseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) - binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) + psum := PseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) + binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum)) } else { hdr := virtioNetHdr{} err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) From 71393c576b98c1903cd8d31450b603535586f81e Mon Sep 17 00:00:00 2001 From: Adrian Dewhurst Date: Wed, 31 Jul 2024 16:08:04 -0400 Subject: [PATCH 24/60] tun: fix checksum test failures on non-4KiB page sizes When generating page-aligned random bytes, random data started at the beginning of the buffer that will be chopped off. When the page size differs, the start of the returned slice is different than expected for the expected checksums, causing the tests to fail. --- tun/checksum_test.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tun/checksum_test.go b/tun/checksum_test.go index c40efc9..f5b8f18 100644 --- a/tun/checksum_test.go +++ b/tun/checksum_test.go @@ -21,26 +21,33 @@ type archChecksumDetails struct { f func([]byte, uint16) uint16 } -func deterministicRandomBytes(seed int64, length int) []byte { +func fillRandomBuffer(seed int64, buf []byte) { rng := rand.New(rand.NewSource(seed)) - buf := make([]byte, length) n, err := rng.Read(buf) if err != nil { panic(err) } - if n != length { + if n != len(buf) { panic("incomplete random buffer") } +} + +func deterministicRandomBytes(seed int64, length int) []byte { + buf := make([]byte, length) + fillRandomBuffer(seed, buf) return buf } func getPageAlignedRandomBytes(seed int64, length int) []byte { alignment := syscall.Getpagesize() - buf := deterministicRandomBytes(seed, length+(alignment-1)) + buf := make([]byte, length+(alignment-1)) bufPtr := uintptr(unsafe.Pointer(&buf[0])) alignedBufPtr := (bufPtr + uintptr(alignment-1)) & ^uintptr(alignment-1) alignedStart := int(alignedBufPtr - bufPtr) - return buf[alignedStart:] + + buf = buf[alignedStart : alignedStart+length] + fillRandomBuffer(seed, buf) + return buf } func TestChecksum(t *testing.T) { From 799c1978fafc07fae8a15a42c8535b70e6c69e6e Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 4 Sep 2024 12:17:44 -0700 Subject: [PATCH 25/60] tun: add method for disabling TCP GRO on Linux torvalds/linux@e269d79c7d35aa3808b1f3c1737d63dab504ddc8 broke virtio_net TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then resolved later in torvalds/linux@89add40066f9ed9abe5f7f886fe5789ff7e0c50e. The offending commit was pulled into various LTS releases. Updates tailscale/tailscale#13041 Signed-off-by: Jordan Whited --- tun/offload_linux.go | 18 ++++----- tun/offload_linux_test.go | 82 ++++++++++++++++++++++----------------- tun/tun.go | 23 +++++++++-- tun/tun_linux.go | 45 ++++++++++++++++++--- 4 files changed, 114 insertions(+), 54 deletions(-) diff --git a/tun/offload_linux.go b/tun/offload_linux.go index fe34401..fb6ac5b 100644 --- a/tun/offload_linux.go +++ b/tun/offload_linux.go @@ -763,7 +763,7 @@ const ( udp6GROCandidate ) -func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType { +func packetIsGROCandidate(b []byte, gro groDisablementFlags) groCandidateType { if len(b) < 28 { return notGROCandidate } @@ -772,17 +772,17 @@ func packetIsGROCandidate(b []byte, canUDPGRO bool) groCandidateType { // IPv4 packets w/IP options do not coalesce return notGROCandidate } - if b[9] == unix.IPPROTO_TCP && len(b) >= 40 { + if b[9] == unix.IPPROTO_TCP && len(b) >= 40 && gro.canTCPGRO() { return tcp4GROCandidate } - if b[9] == unix.IPPROTO_UDP && canUDPGRO { + if b[9] == unix.IPPROTO_UDP && gro.canUDPGRO() { return udp4GROCandidate } } else if b[0]>>4 == 6 { - if b[6] == unix.IPPROTO_TCP && len(b) >= 60 { + if b[6] == unix.IPPROTO_TCP && len(b) >= 60 && gro.canTCPGRO() { return tcp6GROCandidate } - if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && canUDPGRO { + if b[6] == unix.IPPROTO_UDP && len(b) >= 48 && gro.canUDPGRO() { return udp6GROCandidate } } @@ -875,15 +875,15 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool) // handleGRO evaluates bufs for GRO, and writes the indices of the resulting // packets into toWrite. toWrite, tcpTable, and udpTable should initially be // empty (but non-nil), and are passed in to save allocs as the caller may reset -// and recycle them across vectors of packets. canUDPGRO indicates if UDP GRO is -// supported. -func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, canUDPGRO bool, toWrite *[]int) error { +// and recycle them across vectors of packets. gro indicates if TCP and UDP GRO +// are supported/enabled. +func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, gro groDisablementFlags, toWrite *[]int) error { for i := range bufs { if offset < virtioNetHdrLen || offset > len(bufs[i])-1 { return errors.New("invalid offset") } var result groResult - switch packetIsGROCandidate(bufs[i][offset:], canUDPGRO) { + switch packetIsGROCandidate(bufs[i][offset:], gro) { case tcp4GROCandidate: result = tcpGRO(bufs, offset, i, tcpTable, false) case tcp6GROCandidate: diff --git a/tun/offload_linux_test.go b/tun/offload_linux_test.go index 91f3941..4070378 100644 --- a/tun/offload_linux_test.go +++ b/tun/offload_linux_test.go @@ -286,11 +286,11 @@ func Fuzz_handleGRO(f *testing.F) { pkt9 := udp6Packet(ip6PortA, ip6PortB, 100) pkt10 := udp6Packet(ip6PortA, ip6PortB, 100) pkt11 := udp6Packet(ip6PortA, ip6PortC, 100) - f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, true, offset) - f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, canUDPGRO bool, offset int) { + f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, 0, offset) + f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, gro int, offset int) { pkts := [][]byte{pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11} toWrite := make([]int, 0, len(pkts)) - handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), canUDPGRO, &toWrite) + handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), groDisablementFlags(gro), &toWrite) if len(toWrite) > len(pkts) { t.Errorf("len(toWrite): %d > len(pkts): %d", len(toWrite), len(pkts)) } @@ -311,7 +311,7 @@ func Test_handleGRO(t *testing.T) { tests := []struct { name string pktsIn [][]byte - canUDPGRO bool + gro groDisablementFlags wantToWrite []int wantLens []int wantErr bool @@ -331,7 +331,7 @@ func Test_handleGRO(t *testing.T) { udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 }, - true, + 0, []int{0, 1, 2, 4, 5, 7, 9}, []int{240, 228, 128, 140, 260, 160, 248}, false, @@ -351,7 +351,7 @@ func Test_handleGRO(t *testing.T) { udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 }, - false, + udpGRODisabled, []int{0, 1, 2, 4, 5, 7, 8, 9, 10}, []int{240, 128, 128, 140, 260, 160, 128, 148, 148}, false, @@ -368,7 +368,7 @@ func Test_handleGRO(t *testing.T) { tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 201), // v6 flow 1 tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 301), // v6 flow 1 }, - true, + 0, []int{0, 2, 4, 6}, []int{240, 240, 260, 260}, false, @@ -383,7 +383,7 @@ func Test_handleGRO(t *testing.T) { udp4Packet(ip4PortA, ip4PortB, 100), udp4Packet(ip4PortA, ip4PortB, 100), }, - true, + 0, []int{0, 1, 3, 4}, []int{140, 240, 128, 228}, false, @@ -395,7 +395,7 @@ func Test_handleGRO(t *testing.T) { tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // v4 flow 1 seq 1 len 100 tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 201), // v4 flow 1 seq 201 len 100 }, - true, + 0, []int{0}, []int{340}, false, @@ -412,7 +412,7 @@ func Test_handleGRO(t *testing.T) { fields.TTL++ }), }, - true, + 0, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -429,7 +429,7 @@ func Test_handleGRO(t *testing.T) { fields.TOS++ }), }, - true, + 0, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -446,7 +446,7 @@ func Test_handleGRO(t *testing.T) { fields.Flags = 1 }), }, - true, + 0, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -463,7 +463,7 @@ func Test_handleGRO(t *testing.T) { fields.Flags = 2 }), }, - true, + 0, []int{0, 1, 2, 3}, []int{140, 140, 128, 128}, false, @@ -480,7 +480,7 @@ func Test_handleGRO(t *testing.T) { fields.HopLimit++ }), }, - true, + 0, []int{0, 1, 2, 3}, []int{160, 160, 148, 148}, false, @@ -497,7 +497,7 @@ func Test_handleGRO(t *testing.T) { fields.TrafficClass++ }), }, - true, + 0, []int{0, 1, 2, 3}, []int{160, 160, 148, 148}, false, @@ -507,7 +507,7 @@ func Test_handleGRO(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { toWrite := make([]int, 0, len(tt.pktsIn)) - err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), tt.canUDPGRO, &toWrite) + err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), tt.gro, &toWrite) if err != nil { if tt.wantErr { return @@ -552,99 +552,111 @@ func Test_packetIsGROCandidate(t *testing.T) { udp6TooShort := udp6[:47] tests := []struct { - name string - b []byte - canUDPGRO bool - want groCandidateType + name string + b []byte + gro groDisablementFlags + want groCandidateType }{ { "tcp4", tcp4, - true, + 0, tcp4GROCandidate, }, + { + "tcp4 no support", + tcp4, + tcpGRODisabled, + notGROCandidate, + }, { "tcp6", tcp6, - true, + 0, tcp6GROCandidate, }, + { + "tcp6 no support", + tcp6, + tcpGRODisabled, + notGROCandidate, + }, { "udp4", udp4, - true, + 0, udp4GROCandidate, }, { "udp4 no support", udp4, - false, + udpGRODisabled, notGROCandidate, }, { "udp6", udp6, - true, + 0, udp6GROCandidate, }, { "udp6 no support", udp6, - false, + udpGRODisabled, notGROCandidate, }, { "udp4 too short", udp4TooShort, - true, + 0, notGROCandidate, }, { "udp6 too short", udp6TooShort, - true, + 0, notGROCandidate, }, { "tcp4 too short", tcp4TooShort, - true, + 0, notGROCandidate, }, { "tcp6 too short", tcp6TooShort, - true, + 0, notGROCandidate, }, { "invalid IP version", []byte{0x00}, - true, + 0, notGROCandidate, }, { "invalid IP header len", ip4InvalidHeaderLen, - true, + 0, notGROCandidate, }, { "ip4 invalid protocol", ip4InvalidProtocol, - true, + 0, notGROCandidate, }, { "ip6 invalid protocol", ip6InvalidProtocol, - true, + 0, notGROCandidate, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := packetIsGROCandidate(tt.b, tt.canUDPGRO); got != tt.want { + if got := packetIsGROCandidate(tt.b, tt.gro); got != tt.want { t.Errorf("packetIsGROCandidate() = %v, want %v", got, tt.want) } }) diff --git a/tun/tun.go b/tun/tun.go index d3c5012..719a606 100644 --- a/tun/tun.go +++ b/tun/tun.go @@ -52,10 +52,25 @@ type Device interface { BatchSize() int } -type LinuxDevice interface { +// GRODevice is a Device extended with methods for disabling GRO. Certain OS +// versions may have offload bugs. Where these bugs negatively impact throughput +// or break connectivity entirely we can use these methods to disable the +// related offload. +// +// Linux has the following known, GRO bugs. +// +// torvalds/linux@e269d79c7d35aa3808b1f3c1737d63dab504ddc8 broke virtio_net +// TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then +// resolved later in +// torvalds/linux@89add40066f9ed9abe5f7f886fe5789ff7e0c50e. The offending +// commit was pulled into various LTS releases. +// +// UDP GRO writes end up blackholing/dropping packets destined for a +// vxlan/geneve interface on kernel versions prior to 6.8.5. +type GRODevice interface { Device - // DisableUDPGRO disables UDP GRO if it is enabled. Certain device drivers - // (e.g. vxlan, geneve) do not properly handle coalesced UDP packets later - // in the stack, resulting in packet loss. + // DisableUDPGRO disables UDP GRO if it is enabled. DisableUDPGRO() + // DisableTCPGRO disables TCP GRO if it is enabled. + DisableTCPGRO() } diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 664eecc..4a03387 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -38,7 +38,6 @@ type NativeTun struct { statusListenersShutdown chan struct{} batchSize int vnetHdr bool - udpGSO bool closeOnce sync.Once @@ -53,7 +52,30 @@ type NativeTun struct { toWrite []int tcpGROTable *tcpGROTable udpGROTable *udpGROTable - udpGRO bool + gro groDisablementFlags +} + +type groDisablementFlags int + +const ( + tcpGRODisabled groDisablementFlags = 1 << iota + udpGRODisabled +) + +func (g *groDisablementFlags) disableTCPGRO() { + *g |= tcpGRODisabled +} + +func (g *groDisablementFlags) canTCPGRO() bool { + return (*g)&tcpGRODisabled == 0 +} + +func (g *groDisablementFlags) disableUDPGRO() { + *g |= udpGRODisabled +} + +func (g *groDisablementFlags) canUDPGRO() bool { + return (*g)&udpGRODisabled == 0 } func (tun *NativeTun) File() *os.File { @@ -346,7 +368,7 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) { ) tun.toWrite = tun.toWrite[:0] if tun.vnetHdr { - err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.udpGRO, &tun.toWrite) + err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.gro, &tun.toWrite) if err != nil { return 0, err } @@ -462,9 +484,19 @@ func (tun *NativeTun) BatchSize() int { return tun.batchSize } +// DisableUDPGRO disables UDP GRO if it is enabled. See the GRODevice interface +// for cases where it should be called. func (tun *NativeTun) DisableUDPGRO() { tun.writeOpMu.Lock() - tun.udpGRO = false + tun.gro.disableUDPGRO() + tun.writeOpMu.Unlock() +} + +// DisableTCPGRO disables TCP GRO if it is enabled. See the GRODevice interface +// for cases where it should be called. +func (tun *NativeTun) DisableTCPGRO() { + tun.writeOpMu.Lock() + tun.gro.disableTCPGRO() tun.writeOpMu.Unlock() } @@ -503,8 +535,9 @@ func (tun *NativeTun) initFromFlags(name string) error { tun.batchSize = conn.IdealBatchSize // tunUDPOffloads were added in Linux v6.2. We do not return an // error if they are unsupported at runtime. - tun.udpGSO = unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) == nil - tun.udpGRO = tun.udpGSO + if unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) != nil { + tun.gro.disableUDPGRO() + } } else { tun.batchSize = 1 } From 4e883d38c8d363e92e02445e4c664f2c27ffdb88 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 12 Nov 2024 16:53:55 -0800 Subject: [PATCH 26/60] tun: use x/sys/unix IoctlIfreq/NewIfreq to set MTU on Linux The manual struct packing was suspect: https://github.com/tailscale/tailscale/issues/11899 And no need for doing it manually if there's API for it already. Updates tailscale/tailscale#11899 Signed-off-by: Brad Fitzpatrick --- tun/tun_linux.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 4a03387..7cdbf88 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -269,21 +269,15 @@ func (tun *NativeTun) setMTU(n int) error { defer unix.Close(fd) - // do ioctl call - var ifr [ifReqSize]byte - copy(ifr[:], name) - *(*uint32)(unsafe.Pointer(&ifr[unix.IFNAMSIZ])) = uint32(n) - _, _, errno := unix.Syscall( - unix.SYS_IOCTL, - uintptr(fd), - uintptr(unix.SIOCSIFMTU), - uintptr(unsafe.Pointer(&ifr[0])), - ) - - if errno != 0 { - return fmt.Errorf("failed to set MTU of TUN device: %w", errno) + req, err := unix.NewIfreq(name) + if err != nil { + return fmt.Errorf("unix.NewIfreq(%q): %w", name, err) + } + req.SetUint32(uint32(n)) + err = unix.IoctlIfreq(fd, unix.SIOCSIFMTU, req) + if err != nil { + return fmt.Errorf("failed to set MTU of TUN device %q: %w", name, err) } - return nil } From 0b8b35511f19726b3dc1ff25e57061c326ad0e5b Mon Sep 17 00:00:00 2001 From: Nahum Shalman Date: Mon, 2 Dec 2024 14:30:52 +0000 Subject: [PATCH 27/60] ipc: build on solaris/illumos --- ipc/uapi_fake.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipc/uapi_fake.go b/ipc/uapi_fake.go index 3e3dcdf..a2e0f85 100644 --- a/ipc/uapi_fake.go +++ b/ipc/uapi_fake.go @@ -1,4 +1,4 @@ -//go:build wasm || plan9 || aix +//go:build wasm || plan9 || aix || solaris || illumos /* SPDX-License-Identifier: MIT * From 91a0587fb251a72c28724ee111fe04cf1436ca4c Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Mon, 3 Mar 2025 16:01:00 -0800 Subject: [PATCH 28/60] tun: add plan9 support Reviewed-by: James Tucker --- tun/tun_plan9.go | 147 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 tun/tun_plan9.go diff --git a/tun/tun_plan9.go b/tun/tun_plan9.go new file mode 100644 index 0000000..7b66ead --- /dev/null +++ b/tun/tun_plan9.go @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + */ + +package tun + +import ( + "fmt" + "io" + "os" + "strconv" + "strings" + "sync" +) + +type NativeTun struct { + name string // "/net/ipifc/2" + ctlFile *os.File + dataFile *os.File + events chan Event + errors chan error + closeOnce sync.Once +} + +func CreateTUN(_ string, mtu int) (Device, error) { + ctl, err := os.OpenFile("/net/ipifc/clone", os.O_RDWR, 0) + if err != nil { + return nil, err + } + nbuf := make([]byte, 5) + n, err := ctl.Read(nbuf) + if err != nil { + ctl.Close() + return nil, fmt.Errorf("error reading from clone file: %w", err) + } + ifn, err := strconv.Atoi(strings.TrimSpace(string(nbuf[:n]))) + if err != nil { + ctl.Close() + return nil, fmt.Errorf("error converting clone result %q to int: %w", nbuf[:n], err) + } + + if _, err := fmt.Fprintf(ctl, "bind pkt\n"); err != nil { + ctl.Close() + return nil, fmt.Errorf("error binding to pkt: %w", err) + } + if mtu > 0 { + if _, err := fmt.Fprintf(ctl, "mtu %d\n", mtu); err != nil { + ctl.Close() + return nil, fmt.Errorf("error setting MTU: %w", err) + } + } + + dataFile, err := os.OpenFile(fmt.Sprintf("/net/ipifc/%d/data", ifn), os.O_RDWR, 0) + if err != nil { + ctl.Close() + return nil, err + } + + tun := &NativeTun{ + ctlFile: ctl, + dataFile: dataFile, + name: fmt.Sprintf("/net/ipifc/%d", ifn), + events: make(chan Event, 10), + errors: make(chan error, 5), + } + tun.events <- EventUp + + return tun, nil +} + +func (tun *NativeTun) Name() (string, error) { + return tun.name, nil +} + +func (tun *NativeTun) File() *os.File { + return tun.ctlFile +} + +func (tun *NativeTun) Events() <-chan Event { + return tun.events +} + +func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) { + select { + case err := <-tun.errors: + return 0, err + default: + n, err := tun.dataFile.Read(bufs[0][offset:]) + if n == 1 && bufs[0][offset] == 0 { + // EOF + err = io.EOF + n = 0 + } + sizes[0] = n + return 1, err + } +} + +func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) { + for i, buf := range bufs { + if _, err := tun.dataFile.Write(buf[offset:]); err != nil { + return i, err + } + } + return len(bufs), nil +} + +func (tun *NativeTun) Close() error { + var err1, err2 error + tun.closeOnce.Do(func() { + _, err1 := fmt.Fprintf(tun.ctlFile, "unbind\n") + if err := tun.ctlFile.Close(); err != nil && err1 == nil { + err1 = err + } + err2 = tun.dataFile.Close() + }) + if err1 != nil { + return err1 + } + return err2 +} + +func (tun *NativeTun) MTU() (int, error) { + var buf [100]byte + f, err := os.Open(tun.name + "/status") + if err != nil { + return 0, err + } + defer f.Close() + n, err := f.Read(buf[:]) + _, res, ok := strings.Cut(string(buf[:n]), " maxtu ") + if ok { + if mtus, _, ok := strings.Cut(res, " "); ok { + mtu, err := strconv.Atoi(mtus) + if err != nil { + return 0, fmt.Errorf("error converting mtu %q to int: %w", mtus, err) + } + return mtu, nil + } + } + return 0, fmt.Errorf("no 'maxtu' field found in %s/status", tun.name) +} + +func (tun *NativeTun) BatchSize() int { + return 1 +} From 6413b491d4dc3c5e5b295907b0b1242c6272427e Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 28 May 2025 15:05:06 -0700 Subject: [PATCH 29/60] .github/workflows: establish basic build and test actions jobs Upstream doesn't use GitHub actions for CI as GitHub is simply a mirror. Our workflows involve GitHub, so establish some basic CI jobs. Updates tailscale/corp#28877 Signed-off-by: Jordan Whited --- .github/workflows/test.yml | 68 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a370995 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,68 @@ +name: CI + +on: + push: + branches: ["tailscale"] + pull_request: + branches: ["tailscale"] + +jobs: + build: + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - goos: linux + goarch: amd64 + - goos: linux + goarch: arm64 + - goos: linux + goarch: "386" + - goos: linux + goarch: loong64 + - goos: linux + goarch: arm + goarm: "5" + - goos: linux + goarch: arm + goarm: "7" + # macOS + - goos: darwin + goarch: amd64 + - goos: darwin + goarch: arm64 + # Windows + - goos: windows + goarch: amd64 + - goos: windows + goarch: arm64 + # BSDs + - goos: freebsd + goarch: amd64 + - goos: openbsd + goarch: amd64 + steps: + - name: checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: setup go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version-file: go.mod + - name: build + run: go build ./... + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + CGO_ENABLED: "0" + + test: + runs-on: ubuntu-22.04 + steps: + - name: checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: setup go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version-file: go.mod + - name: test + run: go test -race -v ./... From 2b555120c89de2b197f2cfed2787ccda28616c8c Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 11 Dec 2023 16:35:57 +0100 Subject: [PATCH 30/60] device: do atomic 64-bit add outside of vector loop Only bother updating the rxBytes counter once we've processed a whole vector, since additions are atomic. cherry picked from commit WireGuard/wireguard-go@542e565baa776ed4c5c55b73ef9aa38d33d55197 Updates tailscale/corp#28879 Signed-off-by: Jason A. Donenfeld --- device/receive.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/device/receive.go b/device/receive.go index af2db44..01804b7 100644 --- a/device/receive.go +++ b/device/receive.go @@ -447,6 +447,7 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { elemsContainer.Lock() validTailPacket := -1 dataPacketReceived := false + rxBytesLen := uint64(0) for i, elem := range elemsContainer.elems { if elem.packet == nil { // decryption failed @@ -463,7 +464,7 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { peer.timersHandshakeComplete() peer.SendStagedPackets() } - peer.rxBytes.Add(uint64(len(elem.packet) + MinMessageSize)) + rxBytesLen += uint64(len(elem.packet) + MinMessageSize) if len(elem.packet) == 0 { device.log.Verbosef("%v - Receiving keepalive packet", peer) @@ -512,6 +513,8 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { bufs = append(bufs, elem.buffer[:MessageTransportOffsetContent+len(elem.packet)]) } + + peer.rxBytes.Add(rxBytesLen) if validTailPacket >= 0 { peer.SetEndpointFromPacket(elemsContainer.elems[validTailPacket].endpoint) peer.keepKeyFreshReceiving() From f74ff38c79b272da9b154a0a324cd51ec6a1fee8 Mon Sep 17 00:00:00 2001 From: Martin Basovnik Date: Fri, 10 Nov 2023 11:10:12 +0100 Subject: [PATCH 31/60] device: fix possible deadlock in close method There is a possible deadlock in `device.Close()` when you try to close the device very soon after its start. The problem is that two different methods acquire the same locks in different order: 1. device.Close() - device.ipcMutex.Lock() - device.state.Lock() 2. device.changeState(deviceState) - device.state.Lock() - device.ipcMutex.Lock() Reproducer: func TestDevice_deadlock(t *testing.T) { d := randDevice(t) d.Close() } Problem: $ go clean -testcache && go test -race -timeout 3s -run TestDevice_deadlock ./device | grep -A 10 sync.runtime_SemacquireMutex sync.runtime_SemacquireMutex(0xc000117d20?, 0x94?, 0x0?) /usr/local/opt/go/libexec/src/runtime/sema.go:77 +0x25 sync.(*Mutex).lockSlow(0xc000130518) /usr/local/opt/go/libexec/src/sync/mutex.go:171 +0x213 sync.(*Mutex).Lock(0xc000130518) /usr/local/opt/go/libexec/src/sync/mutex.go:90 +0x55 golang.zx2c4.com/wireguard/device.(*Device).Close(0xc000130500) /Users/martin.basovnik/git/basovnik/wireguard-go/device/device.go:373 +0xb6 golang.zx2c4.com/wireguard/device.TestDevice_deadlock(0x0?) /Users/martin.basovnik/git/basovnik/wireguard-go/device/device_test.go:480 +0x2c testing.tRunner(0xc00014c000, 0x131d7b0) -- sync.runtime_SemacquireMutex(0xc000130564?, 0x60?, 0xc000130548?) /usr/local/opt/go/libexec/src/runtime/sema.go:77 +0x25 sync.(*Mutex).lockSlow(0xc000130750) /usr/local/opt/go/libexec/src/sync/mutex.go:171 +0x213 sync.(*Mutex).Lock(0xc000130750) /usr/local/opt/go/libexec/src/sync/mutex.go:90 +0x55 sync.(*RWMutex).Lock(0xc000130750) /usr/local/opt/go/libexec/src/sync/rwmutex.go:147 +0x45 golang.zx2c4.com/wireguard/device.(*Device).upLocked(0xc000130500) /Users/martin.basovnik/git/basovnik/wireguard-go/device/device.go:179 +0x72 golang.zx2c4.com/wireguard/device.(*Device).changeState(0xc000130500, 0x1) cherry picked from commit WireGuard/wireguard-go@12269c2761734b15625017d8565745096325392f Updates tailscale/corp#28879 Signed-off-by: Martin Basovnik Signed-off-by: Jason A. Donenfeld --- device/device.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/device/device.go b/device/device.go index 86dff0d..5b23485 100644 --- a/device/device.go +++ b/device/device.go @@ -368,10 +368,10 @@ func (device *Device) RemoveAllPeers() { } func (device *Device) Close() { - device.ipcMutex.Lock() - defer device.ipcMutex.Unlock() device.state.Lock() defer device.state.Unlock() + device.ipcMutex.Lock() + defer device.ipcMutex.Unlock() if device.isClosed() { return } From 19f7e298052c939c07a5f9eb2b60fb62ff0d10c1 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Thu, 26 Dec 2024 20:36:53 +0100 Subject: [PATCH 32/60] device: reduce RoutineHandshake allocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce allocations by eliminating byte reader, hand-rolled decoding and reusing message structs. Synthetic benchmark: var msgSink MessageInitiation func BenchmarkMessageInitiationUnmarshal(b *testing.B) { packet := make([]byte, MessageInitiationSize) reader := bytes.NewReader(packet) err := binary.Read(reader, binary.LittleEndian, &msgSink) if err != nil { b.Fatal(err) } b.Run("binary.Read", func(b *testing.B) { b.ReportAllocs() for range b.N { reader := bytes.NewReader(packet) _ = binary.Read(reader, binary.LittleEndian, &msgSink) } }) b.Run("unmarshal", func(b *testing.B) { b.ReportAllocs() for range b.N { _ = msgSink.unmarshal(packet) } }) } Results: │ - │ │ sec/op │ MessageInitiationUnmarshal/binary.Read-8 1.508µ ± 2% MessageInitiationUnmarshal/unmarshal-8 12.66n ± 2% │ - │ │ B/op │ MessageInitiationUnmarshal/binary.Read-8 208.0 ± 0% MessageInitiationUnmarshal/unmarshal-8 0.000 ± 0% │ - │ │ allocs/op │ MessageInitiationUnmarshal/binary.Read-8 2.000 ± 0% MessageInitiationUnmarshal/unmarshal-8 0.000 ± 0% cherry picked from commit WireGuard/wireguard-go@9e7529c3d2d0c54f4d5384c01645a9279e4740ae Updates tailscale/corp#28879 Signed-off-by: Alexander Yastrebov Signed-off-by: Jason A. Donenfeld --- device/noise-protocol.go | 48 ++++++++++++++++++++++++++++++++++++++++ device/receive.go | 10 +++------ 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/device/noise-protocol.go b/device/noise-protocol.go index 2d8f984..2e7e9ae 100644 --- a/device/noise-protocol.go +++ b/device/noise-protocol.go @@ -6,6 +6,7 @@ package device import ( + "encoding/binary" "errors" "fmt" "sync" @@ -115,6 +116,53 @@ type MessageCookieReply struct { Cookie [blake2s.Size128 + poly1305.TagSize]byte } +var errMessageTooShort = errors.New("message too short") + +func (msg *MessageInitiation) unmarshal(b []byte) error { + if len(b) < MessageInitiationSize { + return errMessageTooShort + } + + msg.Type = binary.LittleEndian.Uint32(b) + msg.Sender = binary.LittleEndian.Uint32(b[4:]) + copy(msg.Ephemeral[:], b[8:]) + copy(msg.Static[:], b[8+len(msg.Ephemeral):]) + copy(msg.Timestamp[:], b[8+len(msg.Ephemeral)+len(msg.Static):]) + copy(msg.MAC1[:], b[8+len(msg.Ephemeral)+len(msg.Static)+len(msg.Timestamp):]) + copy(msg.MAC2[:], b[8+len(msg.Ephemeral)+len(msg.Static)+len(msg.Timestamp)+len(msg.MAC1):]) + + return nil +} + +func (msg *MessageResponse) unmarshal(b []byte) error { + if len(b) < MessageResponseSize { + return errMessageTooShort + } + + msg.Type = binary.LittleEndian.Uint32(b) + msg.Sender = binary.LittleEndian.Uint32(b[4:]) + msg.Receiver = binary.LittleEndian.Uint32(b[8:]) + copy(msg.Ephemeral[:], b[12:]) + copy(msg.Empty[:], b[12+len(msg.Ephemeral):]) + copy(msg.MAC1[:], b[12+len(msg.Ephemeral)+len(msg.Empty):]) + copy(msg.MAC2[:], b[12+len(msg.Ephemeral)+len(msg.Empty)+len(msg.MAC1):]) + + return nil +} + +func (msg *MessageCookieReply) unmarshal(b []byte) error { + if len(b) < MessageCookieReplySize { + return errMessageTooShort + } + + msg.Type = binary.LittleEndian.Uint32(b) + msg.Receiver = binary.LittleEndian.Uint32(b[4:]) + copy(msg.Nonce[:], b[8:]) + copy(msg.Cookie[:], b[8+len(msg.Nonce):]) + + return nil +} + type Handshake struct { state handshakeState mutex sync.RWMutex diff --git a/device/receive.go b/device/receive.go index 01804b7..bc37f91 100644 --- a/device/receive.go +++ b/device/receive.go @@ -6,7 +6,6 @@ package device import ( - "bytes" "encoding/binary" "errors" "net" @@ -287,8 +286,7 @@ func (device *Device) RoutineHandshake(id int) { // unmarshal packet var reply MessageCookieReply - reader := bytes.NewReader(elem.packet) - err := binary.Read(reader, binary.LittleEndian, &reply) + err := reply.unmarshal(elem.packet) if err != nil { device.log.Verbosef("Failed to decode cookie reply") goto skip @@ -353,8 +351,7 @@ func (device *Device) RoutineHandshake(id int) { // unmarshal var msg MessageInitiation - reader := bytes.NewReader(elem.packet) - err := binary.Read(reader, binary.LittleEndian, &msg) + err := msg.unmarshal(elem.packet) if err != nil { device.log.Errorf("Failed to decode initiation message") goto skip @@ -386,8 +383,7 @@ func (device *Device) RoutineHandshake(id int) { // unmarshal var msg MessageResponse - reader := bytes.NewReader(elem.packet) - err := binary.Read(reader, binary.LittleEndian, &msg) + err := msg.unmarshal(elem.packet) if err != nil { device.log.Errorf("Failed to decode response message") goto skip From ae0636254c5f6232d78b7d9a9908eee434639406 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Thu, 15 May 2025 16:48:14 +0200 Subject: [PATCH 33/60] device: make unmarshall length checks exact This is already enforced in receive.go, but if these unmarshallers are to have error return values anyway, make them as explicit as possible. cherry picked from commit WireGuard/wireguard-go@842888ac5c93ccc5ee6344eceaadf783fcf1e243 Updates tailscale/corp#28879 Signed-off-by: Jason A. Donenfeld --- device/noise-protocol.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/device/noise-protocol.go b/device/noise-protocol.go index 2e7e9ae..1e99f68 100644 --- a/device/noise-protocol.go +++ b/device/noise-protocol.go @@ -116,11 +116,11 @@ type MessageCookieReply struct { Cookie [blake2s.Size128 + poly1305.TagSize]byte } -var errMessageTooShort = errors.New("message too short") +var errMessageLengthMismatch = errors.New("message length mismatch") func (msg *MessageInitiation) unmarshal(b []byte) error { - if len(b) < MessageInitiationSize { - return errMessageTooShort + if len(b) != MessageInitiationSize { + return errMessageLengthMismatch } msg.Type = binary.LittleEndian.Uint32(b) @@ -135,8 +135,8 @@ func (msg *MessageInitiation) unmarshal(b []byte) error { } func (msg *MessageResponse) unmarshal(b []byte) error { - if len(b) < MessageResponseSize { - return errMessageTooShort + if len(b) != MessageResponseSize { + return errMessageLengthMismatch } msg.Type = binary.LittleEndian.Uint32(b) @@ -151,8 +151,8 @@ func (msg *MessageResponse) unmarshal(b []byte) error { } func (msg *MessageCookieReply) unmarshal(b []byte) error { - if len(b) < MessageCookieReplySize { - return errMessageTooShort + if len(b) != MessageCookieReplySize { + return errMessageLengthMismatch } msg.Type = binary.LittleEndian.Uint32(b) From 022546570f38af621ea050600e3b1eb43d430a88 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Sat, 17 May 2025 11:34:30 +0200 Subject: [PATCH 34/60] device: optimize message encoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Optimize message encoding by eliminating binary.Write (which internally uses reflection) in favour of hand-rolled encoding. This is companion to 9e7529c3d2d0c54f4d5384c01645a9279e4740ae. Synthetic benchmark: var packetSink []byte func BenchmarkMessageInitiationMarshal(b *testing.B) { var msg MessageInitiation b.Run("binary.Write", func(b *testing.B) { b.ReportAllocs() for range b.N { var buf [MessageInitiationSize]byte writer := bytes.NewBuffer(buf[:0]) _ = binary.Write(writer, binary.LittleEndian, msg) packetSink = writer.Bytes() } }) b.Run("binary.Encode", func(b *testing.B) { b.ReportAllocs() for range b.N { packet := make([]byte, MessageInitiationSize) _, _ = binary.Encode(packet, binary.LittleEndian, msg) packetSink = packet } }) b.Run("marshal", func(b *testing.B) { b.ReportAllocs() for range b.N { packet := make([]byte, MessageInitiationSize) _ = msg.marshal(packet) packetSink = packet } }) } Results: │ - │ │ sec/op │ MessageInitiationMarshal/binary.Write-8 1.337µ ± 0% MessageInitiationMarshal/binary.Encode-8 1.242µ ± 0% MessageInitiationMarshal/marshal-8 53.05n ± 1% │ - │ │ B/op │ MessageInitiationMarshal/binary.Write-8 368.0 ± 0% MessageInitiationMarshal/binary.Encode-8 160.0 ± 0% MessageInitiationMarshal/marshal-8 160.0 ± 0% │ - │ │ allocs/op │ MessageInitiationMarshal/binary.Write-8 3.000 ± 0% MessageInitiationMarshal/binary.Encode-8 1.000 ± 0% MessageInitiationMarshal/marshal-8 1.000 ± 0% cherry picked from commit WireGuard/wireguard-go@264889f0bbdf9250bb8389a637dd5f38389bfe0b Updates tailscale/corp#28879 Signed-off-by: Alexander Yastrebov Signed-off-by: Jason A. Donenfeld --- device/noise-protocol.go | 45 ++++++++++++++++++++++++++++++++++++++++ device/send.go | 21 +++++++------------ 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/device/noise-protocol.go b/device/noise-protocol.go index 1e99f68..cb4dedb 100644 --- a/device/noise-protocol.go +++ b/device/noise-protocol.go @@ -134,6 +134,22 @@ func (msg *MessageInitiation) unmarshal(b []byte) error { return nil } +func (msg *MessageInitiation) marshal(b []byte) error { + if len(b) != MessageInitiationSize { + return errMessageLengthMismatch + } + + binary.LittleEndian.PutUint32(b, msg.Type) + binary.LittleEndian.PutUint32(b[4:], msg.Sender) + copy(b[8:], msg.Ephemeral[:]) + copy(b[8+len(msg.Ephemeral):], msg.Static[:]) + copy(b[8+len(msg.Ephemeral)+len(msg.Static):], msg.Timestamp[:]) + copy(b[8+len(msg.Ephemeral)+len(msg.Static)+len(msg.Timestamp):], msg.MAC1[:]) + copy(b[8+len(msg.Ephemeral)+len(msg.Static)+len(msg.Timestamp)+len(msg.MAC1):], msg.MAC2[:]) + + return nil +} + func (msg *MessageResponse) unmarshal(b []byte) error { if len(b) != MessageResponseSize { return errMessageLengthMismatch @@ -150,6 +166,22 @@ func (msg *MessageResponse) unmarshal(b []byte) error { return nil } +func (msg *MessageResponse) marshal(b []byte) error { + if len(b) != MessageResponseSize { + return errMessageLengthMismatch + } + + binary.LittleEndian.PutUint32(b, msg.Type) + binary.LittleEndian.PutUint32(b[4:], msg.Sender) + binary.LittleEndian.PutUint32(b[8:], msg.Receiver) + copy(b[12:], msg.Ephemeral[:]) + copy(b[12+len(msg.Ephemeral):], msg.Empty[:]) + copy(b[12+len(msg.Ephemeral)+len(msg.Empty):], msg.MAC1[:]) + copy(b[12+len(msg.Ephemeral)+len(msg.Empty)+len(msg.MAC1):], msg.MAC2[:]) + + return nil +} + func (msg *MessageCookieReply) unmarshal(b []byte) error { if len(b) != MessageCookieReplySize { return errMessageLengthMismatch @@ -163,6 +195,19 @@ func (msg *MessageCookieReply) unmarshal(b []byte) error { return nil } +func (msg *MessageCookieReply) marshal(b []byte) error { + if len(b) != MessageCookieReplySize { + return errMessageLengthMismatch + } + + binary.LittleEndian.PutUint32(b, msg.Type) + binary.LittleEndian.PutUint32(b[4:], msg.Receiver) + copy(b[8:], msg.Nonce[:]) + copy(b[8+len(msg.Nonce):], msg.Cookie[:]) + + return nil +} + type Handshake struct { state handshakeState mutex sync.RWMutex diff --git a/device/send.go b/device/send.go index 8ed2e5f..7900f57 100644 --- a/device/send.go +++ b/device/send.go @@ -6,7 +6,6 @@ package device import ( - "bytes" "encoding/binary" "errors" "net" @@ -124,10 +123,8 @@ func (peer *Peer) SendHandshakeInitiation(isRetry bool) error { return err } - var buf [MessageInitiationSize]byte - writer := bytes.NewBuffer(buf[:0]) - binary.Write(writer, binary.LittleEndian, msg) - packet := writer.Bytes() + packet := make([]byte, MessageInitiationSize) + _ = msg.marshal(packet) peer.cookieGenerator.AddMacs(packet) peer.timersAnyAuthenticatedPacketTraversal() @@ -155,10 +152,8 @@ func (peer *Peer) SendHandshakeResponse() error { return err } - var buf [MessageResponseSize]byte - writer := bytes.NewBuffer(buf[:0]) - binary.Write(writer, binary.LittleEndian, response) - packet := writer.Bytes() + packet := make([]byte, MessageResponseSize) + _ = response.marshal(packet) peer.cookieGenerator.AddMacs(packet) err = peer.BeginSymmetricSession() @@ -189,11 +184,11 @@ func (device *Device) SendHandshakeCookie(initiatingElem *QueueHandshakeElement) return err } - var buf [MessageCookieReplySize]byte - writer := bytes.NewBuffer(buf[:0]) - binary.Write(writer, binary.LittleEndian, reply) + packet := make([]byte, MessageCookieReplySize) + _ = reply.marshal(packet) // TODO: allocation could be avoided - device.net.bind.Send([][]byte{writer.Bytes()}, initiatingElem.endpoint) + device.net.bind.Send([][]byte{packet}, initiatingElem.endpoint) + return nil } From 65cd6eed7d7f688acea0b24ba64a781a9c58248e Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Fri, 30 May 2025 13:14:43 -0700 Subject: [PATCH 35/60] conn,device: provide 8 free bytes at packet head to conn.Bind.Send() This enables a conn.Bind to bring its own encapsulating transport, e.g. VXLAN/Geneve. Updates tailscale/corp#27502 Signed-off-by: Jordan Whited --- conn/bind_std.go | 9 +++++---- conn/bind_std_test.go | 2 +- conn/bind_windows.go | 3 ++- conn/bindtest/bindtest.go | 3 ++- conn/conn.go | 8 +++++--- device/constants.go | 6 +++--- device/device_test.go | 10 +++++----- device/noise-protocol.go | 15 ++++++++------- device/peer.go | 5 ++++- device/send.go | 36 +++++++++++++++++++++++------------- 10 files changed, 58 insertions(+), 39 deletions(-) diff --git a/conn/bind_std.go b/conn/bind_std.go index 428e528..fc05634 100644 --- a/conn/bind_std.go +++ b/conn/bind_std.go @@ -341,7 +341,7 @@ func (e ErrUDPGSODisabled) Unwrap() error { return e.RetryErr } -func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint) error { +func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error { s.mu.Lock() blackhole := s.blackhole4 conn := s.ipv4 @@ -384,7 +384,7 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint) error { ) retry: if offload { - n := coalesceMessages(ua, endpoint.(*StdNetEndpoint), bufs, *msgs, setGSOSize) + n := coalesceMessages(ua, endpoint.(*StdNetEndpoint), bufs, offset, *msgs, setGSOSize) err = s.send(conn, br, (*msgs)[:n]) if err != nil && offload && errShouldDisableUDPGSO(err) { offload = false @@ -401,7 +401,7 @@ retry: } else { for i := range bufs { (*msgs)[i].Addr = ua - (*msgs)[i].Buffers[0] = bufs[i] + (*msgs)[i].Buffers[0] = bufs[i][offset:] setSrcControl(&(*msgs)[i].OOB, endpoint.(*StdNetEndpoint)) } err = s.send(conn, br, (*msgs)[:len(bufs)]) @@ -450,7 +450,7 @@ const ( type setGSOFunc func(control *[]byte, gsoSize uint16) -func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, msgs []ipv6.Message, setGSO setGSOFunc) int { +func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offset int, msgs []ipv6.Message, setGSO setGSOFunc) int { var ( base = -1 // index of msg we are currently coalescing into gsoSize int // segmentation size of msgs[base] @@ -462,6 +462,7 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, msgs maxPayloadLen = maxIPv6PayloadLen } for i, buf := range bufs { + buf = buf[offset:] if i > 0 { msgLen := len(buf) baseLenBefore := len(msgs[base].Buffers[0]) diff --git a/conn/bind_std_test.go b/conn/bind_std_test.go index 34a3c9a..77af0d9 100644 --- a/conn/bind_std_test.go +++ b/conn/bind_std_test.go @@ -98,7 +98,7 @@ func Test_coalesceMessages(t *testing.T) { msgs[i].Buffers = make([][]byte, 1) msgs[i].OOB = make([]byte, 0, 2) } - got := coalesceMessages(addr, &StdNetEndpoint{AddrPort: addr.AddrPort()}, tt.buffs, msgs, mockSetGSOSize) + got := coalesceMessages(addr, &StdNetEndpoint{AddrPort: addr.AddrPort()}, tt.buffs, 0, msgs, mockSetGSOSize) if got != len(tt.wantLens) { t.Fatalf("got len %d want: %d", got, len(tt.wantLens)) } diff --git a/conn/bind_windows.go b/conn/bind_windows.go index 9638b30..737b475 100644 --- a/conn/bind_windows.go +++ b/conn/bind_windows.go @@ -486,7 +486,7 @@ func (bind *afWinRingBind) Send(buf []byte, nend *WinRingEndpoint, isOpen *atomi return winrio.SendEx(bind.rq, dataBuffer, 1, nil, addressBuffer, nil, nil, 0, 0) } -func (bind *WinRingBind) Send(bufs [][]byte, endpoint Endpoint) error { +func (bind *WinRingBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error { nend, ok := endpoint.(*WinRingEndpoint) if !ok { return ErrWrongEndpointType @@ -494,6 +494,7 @@ func (bind *WinRingBind) Send(bufs [][]byte, endpoint Endpoint) error { bind.mu.RLock() defer bind.mu.RUnlock() for _, buf := range bufs { + buf = buf[offset:] switch nend.family { case windows.AF_INET: if bind.v4.blackhole { diff --git a/conn/bindtest/bindtest.go b/conn/bindtest/bindtest.go index 836d983..741b776 100644 --- a/conn/bindtest/bindtest.go +++ b/conn/bindtest/bindtest.go @@ -107,8 +107,9 @@ func (c *ChannelBind) makeReceiveFunc(ch chan []byte) conn.ReceiveFunc { } } -func (c *ChannelBind) Send(bufs [][]byte, ep conn.Endpoint) error { +func (c *ChannelBind) Send(bufs [][]byte, ep conn.Endpoint, offset int) error { for _, b := range bufs { + b = b[offset:] select { case <-c.closeSignal: return net.ErrClosed diff --git a/conn/conn.go b/conn/conn.go index 8df5aaa..5083648 100644 --- a/conn/conn.go +++ b/conn/conn.go @@ -45,9 +45,11 @@ type Bind interface { // This mark is passed to the kernel as the socket option SO_MARK. SetMark(mark uint32) error - // Send writes one or more packets in bufs to address ep. The length of - // bufs must not exceed BatchSize(). - Send(bufs [][]byte, ep Endpoint) error + // Send writes one or more packets in bufs to address ep. A nonzero offset + // can be used to instruct the Bind on where packet data begins in each + // element of the bufs slice. Space preceding offset is free to use for + // additional encapsulation. The length of bufs must not exceed BatchSize(). + Send(bufs [][]byte, ep Endpoint, offset int) error // ParseEndpoint creates a new endpoint from a string. ParseEndpoint(s string) (Endpoint, error) diff --git a/device/constants.go b/device/constants.go index 59854a1..92c3bde 100644 --- a/device/constants.go +++ b/device/constants.go @@ -27,9 +27,9 @@ const ( ) const ( - MinMessageSize = MessageKeepaliveSize // minimum size of transport message (keepalive) - MaxMessageSize = MaxSegmentSize // maximum size of transport message - MaxContentSize = MaxSegmentSize - MessageTransportSize // maximum size of transport message content + MinMessageSize = MessageKeepaliveSize // minimum size of transport message (keepalive) + MaxMessageSize = MaxSegmentSize // maximum size of transport message + MaxContentSize = MaxSegmentSize - MessageTransportSize - MessageEncapsulatingTransportSize // maximum size of transport message content ) /* Implementation constants */ diff --git a/device/device_test.go b/device/device_test.go index 4088b9f..e443421 100644 --- a/device/device_test.go +++ b/device/device_test.go @@ -426,11 +426,11 @@ type fakeBindSized struct { func (b *fakeBindSized) Open(port uint16) (fns []conn.ReceiveFunc, actualPort uint16, err error) { return nil, 0, nil } -func (b *fakeBindSized) Close() error { return nil } -func (b *fakeBindSized) SetMark(mark uint32) error { return nil } -func (b *fakeBindSized) Send(bufs [][]byte, ep conn.Endpoint) error { return nil } -func (b *fakeBindSized) ParseEndpoint(s string) (conn.Endpoint, error) { return nil, nil } -func (b *fakeBindSized) BatchSize() int { return b.size } +func (b *fakeBindSized) Close() error { return nil } +func (b *fakeBindSized) SetMark(mark uint32) error { return nil } +func (b *fakeBindSized) Send(bufs [][]byte, ep conn.Endpoint, offset int) error { return nil } +func (b *fakeBindSized) ParseEndpoint(s string) (conn.Endpoint, error) { return nil, nil } +func (b *fakeBindSized) BatchSize() int { return b.size } type fakeTUNDeviceSized struct { size int diff --git a/device/noise-protocol.go b/device/noise-protocol.go index cb4dedb..555ce91 100644 --- a/device/noise-protocol.go +++ b/device/noise-protocol.go @@ -61,13 +61,14 @@ const ( ) const ( - MessageInitiationSize = 148 // size of handshake initiation message - MessageResponseSize = 92 // size of response message - MessageCookieReplySize = 64 // size of cookie reply message - MessageTransportHeaderSize = 16 // size of data preceding content in transport message - MessageTransportSize = MessageTransportHeaderSize + poly1305.TagSize // size of empty transport - MessageKeepaliveSize = MessageTransportSize // size of keepalive - MessageHandshakeSize = MessageInitiationSize // size of largest handshake related message + MessageInitiationSize = 148 // size of handshake initiation message + MessageResponseSize = 92 // size of response message + MessageCookieReplySize = 64 // size of cookie reply message + MessageTransportHeaderSize = 16 // size of data preceding content in transport message + MessageEncapsulatingTransportSize = 8 // size of optional, free (for use by conn.Bind.Send()) space preceding the transport header + MessageTransportSize = MessageTransportHeaderSize + poly1305.TagSize // size of empty transport + MessageKeepaliveSize = MessageTransportSize // size of keepalive + MessageHandshakeSize = MessageInitiationSize // size of largest handshake related message ) const ( diff --git a/device/peer.go b/device/peer.go index 876e5da..f79a0af 100644 --- a/device/peer.go +++ b/device/peer.go @@ -113,6 +113,9 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) { return peer, nil } +// SendBuffers sends buffers to peer. WireGuard packet data in each element of +// buffers must be preceded by MessageEncapsulatingTransportSize number of +// bytes. func (peer *Peer) SendBuffers(buffers [][]byte) error { peer.device.net.RLock() defer peer.device.net.RUnlock() @@ -133,7 +136,7 @@ func (peer *Peer) SendBuffers(buffers [][]byte) error { } peer.endpoint.Unlock() - err := peer.device.net.bind.Send(buffers, endpoint) + err := peer.device.net.bind.Send(buffers, endpoint, MessageEncapsulatingTransportSize) if err == nil { var totalLen uint64 for _, b := range buffers { diff --git a/device/send.go b/device/send.go index 7900f57..bf854b7 100644 --- a/device/send.go +++ b/device/send.go @@ -45,11 +45,15 @@ import ( */ type QueueOutboundElement struct { - buffer *[MaxMessageSize]byte // slice holding the packet data - packet []byte // slice of "buffer" (always!) - nonce uint64 // nonce for encryption - keypair *Keypair // keypair for encryption - peer *Peer // related peer + buffer *[MaxMessageSize]byte // slice holding the packet data + // packet is always a slice of "buffer". The starting offset in buffer + // is either: + // a) MessageEncapsulatingTransportSize+MessageTransportHeaderSize (plaintext) + // b) 0 (post-encryption) + packet []byte + nonce uint64 // nonce for encryption + keypair *Keypair // keypair for encryption + peer *Peer // related peer } type QueueOutboundElementsContainer struct { @@ -123,14 +127,15 @@ func (peer *Peer) SendHandshakeInitiation(isRetry bool) error { return err } - packet := make([]byte, MessageInitiationSize) + buf := make([]byte, MessageEncapsulatingTransportSize+MessageInitiationSize) + packet := buf[MessageEncapsulatingTransportSize:] _ = msg.marshal(packet) peer.cookieGenerator.AddMacs(packet) peer.timersAnyAuthenticatedPacketTraversal() peer.timersAnyAuthenticatedPacketSent() - err = peer.SendBuffers([][]byte{packet}) + err = peer.SendBuffers([][]byte{buf}) if err != nil { peer.device.log.Errorf("%v - Failed to send handshake initiation: %v", peer, err) } @@ -152,7 +157,8 @@ func (peer *Peer) SendHandshakeResponse() error { return err } - packet := make([]byte, MessageResponseSize) + buf := make([]byte, MessageEncapsulatingTransportSize+MessageResponseSize) + packet := buf[MessageEncapsulatingTransportSize:] _ = response.marshal(packet) peer.cookieGenerator.AddMacs(packet) @@ -167,7 +173,7 @@ func (peer *Peer) SendHandshakeResponse() error { peer.timersAnyAuthenticatedPacketSent() // TODO: allocation could be avoided - err = peer.SendBuffers([][]byte{packet}) + err = peer.SendBuffers([][]byte{buf}) if err != nil { peer.device.log.Errorf("%v - Failed to send handshake response: %v", peer, err) } @@ -184,10 +190,11 @@ func (device *Device) SendHandshakeCookie(initiatingElem *QueueHandshakeElement) return err } - packet := make([]byte, MessageCookieReplySize) + buf := make([]byte, MessageEncapsulatingTransportSize+MessageCookieReplySize) + packet := buf[MessageEncapsulatingTransportSize:] _ = reply.marshal(packet) // TODO: allocation could be avoided - device.net.bind.Send([][]byte{packet}, initiatingElem.endpoint) + device.net.bind.Send([][]byte{buf}, initiatingElem.endpoint, MessageEncapsulatingTransportSize) return nil } @@ -220,7 +227,7 @@ func (device *Device) RoutineReadFromTUN() { elemsByPeer = make(map[*Peer]*QueueOutboundElementsContainer, batchSize) count = 0 sizes = make([]int, batchSize) - offset = MessageTransportHeaderSize + offset = MessageEncapsulatingTransportSize + MessageTransportHeaderSize ) for i := range elems { @@ -446,7 +453,7 @@ func (device *Device) RoutineEncryption(id int) { for elemsContainer := range device.queue.encryption.c { for _, elem := range elemsContainer.elems { // populate header fields - header := elem.buffer[:MessageTransportHeaderSize] + header := elem.buffer[MessageEncapsulatingTransportSize : MessageEncapsulatingTransportSize+MessageTransportHeaderSize] fieldType := header[0:4] fieldReceiver := header[4:8] @@ -469,6 +476,9 @@ func (device *Device) RoutineEncryption(id int) { elem.packet, nil, ) + + // re-slice packet to include encapsulating transport space + elem.packet = elem.buffer[:MessageEncapsulatingTransportSize+len(elem.packet)] } elemsContainer.Unlock() } From 24483d7a00033707e47a0322de6970ddc504e454 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Tue, 1 Jul 2025 14:44:37 -0700 Subject: [PATCH 36/60] conn,device: always perform PeerAwareEndpoint check It was previously suppressed if roaming was disabled for the peer. Tailscale always disables roaming as we explicitly configure conn.Endpoint's for all peers. This commit also modifies PeerAwareEndpoint usage such that wireguard-go never uses/sets it as a Peer Endpoint value. In theory we (Tailscale) always disable roaming, so we should always return early from SetEndpointFromPacket(), but this acts as an extra footgun guard and improves clarity around intended usage. Updates tailscale/corp#27502 Updates tailscale/corp#29422 Updates tailscale/corp#30042 Signed-off-by: Jordan Whited --- conn/conn.go | 12 ++++++++---- device/peer.go | 7 ++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/conn/conn.go b/conn/conn.go index 5083648..2a04e6d 100644 --- a/conn/conn.go +++ b/conn/conn.go @@ -87,17 +87,21 @@ type Endpoint interface { } // PeerAwareEndpoint is an optional Endpoint specialization for -// integrations that want to know about the outcome of cryptorouting +// integrations that want to know about the outcome of Cryptokey Routing // identification. // // If they receive a packet from a source they had not pre-identified, // to learn the identification WireGuard can derive from the session // or handshake. // -// If GetPeerEndpoint returns nil, WireGuard will be unable to respond -// to the peer until a new endpoint is written by a later packet. +// wireguard-go never installs a [PeerAwareEndpoint] as the [Endpoint] for a +// [Peer]. type PeerAwareEndpoint interface { - GetPeerEndpoint(peerPublicKey [32]byte) Endpoint + // FromPeer is called at least once per successfully Cryptokey Routing ID'd + // [ReceiveFunc] packets batch for a given node key. wireguard-go will + // always call it for the latest/tail packet in the batch, only ever + // suppressing calls for older packets. + FromPeer(peerPublicKey [32]byte) } var ( diff --git a/device/peer.go b/device/peer.go index f79a0af..c188c31 100644 --- a/device/peer.go +++ b/device/peer.go @@ -282,13 +282,14 @@ func (peer *Peer) Stop() { func (peer *Peer) SetEndpointFromPacket(endpoint conn.Endpoint) { peer.endpoint.Lock() defer peer.endpoint.Unlock() + if ep, ok := endpoint.(conn.PeerAwareEndpoint); ok { + ep.FromPeer(peer.handshake.remoteStatic) + return + } if peer.endpoint.disableRoaming { return } peer.endpoint.clearSrcOnTx = false - if ep, ok := endpoint.(conn.PeerAwareEndpoint); ok { - endpoint = ep.GetPeerEndpoint(peer.handshake.remoteStatic) - } peer.endpoint.val = endpoint } From 1f398ae148a8a8183e855289b4a4492dde0490dd Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 3 Jul 2025 20:54:49 -0700 Subject: [PATCH 37/60] conn,device: implement InitiationAwareEndpoint To be implemented by [magicsock.lazyEndpoint], which is responsible for triggering JIT peer configuration. Updates tailscale/corp#20732 Updates tailscale/corp#30042 Signed-off-by: Jordan Whited --- conn/conn.go | 15 +++++++++++++++ device/noise-protocol.go | 8 +++++++- device/noise_test.go | 30 +++++++++++++++++++++++++++++- device/receive.go | 2 +- 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/conn/conn.go b/conn/conn.go index 2a04e6d..b949641 100644 --- a/conn/conn.go +++ b/conn/conn.go @@ -86,6 +86,21 @@ type Endpoint interface { SrcIP() netip.Addr } +// InitiationAwareEndpoint is an optional [Endpoint] specialization for +// integrations that want to know when a WireGuard handshake initiation +// message has been received, enabling just-in-time peer configuration before +// attempted decryption. +// +// It's most useful when used in combination with [PeerAwareEndpoint], enabling +// JIT peer configuration and post-decryption peer verification from a single +// implementer. +type InitiationAwareEndpoint interface { + // InitiationMessagePublicKey is called when a handshake initiation message + // has been received, and the sender's public key has been identified, but + // BEFORE an attempt has been made to verify it. + InitiationMessagePublicKey(peerPublicKey [32]byte) +} + // PeerAwareEndpoint is an optional Endpoint specialization for // integrations that want to know about the outcome of Cryptokey Routing // identification. diff --git a/device/noise-protocol.go b/device/noise-protocol.go index 555ce91..ad5838e 100644 --- a/device/noise-protocol.go +++ b/device/noise-protocol.go @@ -16,6 +16,7 @@ import ( "golang.org/x/crypto/chacha20poly1305" "golang.org/x/crypto/poly1305" + "github.com/tailscale/wireguard-go/conn" "github.com/tailscale/wireguard-go/tai64n" ) @@ -338,7 +339,7 @@ func (device *Device) CreateMessageInitiation(peer *Peer) (*MessageInitiation, e return &msg, nil } -func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation) *Peer { +func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation, endpoint conn.Endpoint) *Peer { var ( hash [blake2s.Size]byte chainKey [blake2s.Size]byte @@ -372,6 +373,11 @@ func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation) *Peer { // lookup peer + initEP, ok := endpoint.(conn.InitiationAwareEndpoint) + if ok { + initEP.InitiationMessagePublicKey(peerPK) + } + peer := device.LookupPeer(peerPK) if peer == nil || !peer.isRunning.Load() { return nil diff --git a/device/noise_test.go b/device/noise_test.go index 7d6af1d..160bee5 100644 --- a/device/noise_test.go +++ b/device/noise_test.go @@ -8,6 +8,7 @@ package device import ( "bytes" "encoding/binary" + "net/netip" "testing" "github.com/tailscale/wireguard-go/conn" @@ -56,6 +57,26 @@ func assertEqual(t *testing.T, a, b []byte) { } } +type initAwareEP struct { + calledWith *[32]byte +} + +var _ conn.Endpoint = (*initAwareEP)(nil) +var _ conn.InitiationAwareEndpoint = (*initAwareEP)(nil) + +func (i *initAwareEP) ClearSrc() {} +func (i *initAwareEP) SrcToString() string { return "" } +func (i *initAwareEP) DstToString() string { return "" } +func (i *initAwareEP) DstToBytes() []byte { return nil } +func (i *initAwareEP) DstIP() netip.Addr { return netip.Addr{} } +func (i *initAwareEP) SrcIP() netip.Addr { return netip.Addr{} } + +func (i *initAwareEP) InitiationMessagePublicKey(peerPublicKey [32]byte) { + calledWith := [32]byte{} + copy(calledWith[:], peerPublicKey[:]) + i.calledWith = &calledWith +} + func TestNoiseHandshake(t *testing.T) { dev1 := randDevice(t) dev2 := randDevice(t) @@ -93,10 +114,17 @@ func TestNoiseHandshake(t *testing.T) { writer := bytes.NewBuffer(packet) err = binary.Write(writer, binary.LittleEndian, msg1) assertNil(t, err) - peer := dev2.ConsumeMessageInitiation(msg1) + initEP := &initAwareEP{} + peer := dev2.ConsumeMessageInitiation(msg1, initEP) if peer == nil { t.Fatal("handshake failed at initiation message") } + if initEP.calledWith == nil { + t.Fatal("initAwareEP never called") + } + if *initEP.calledWith != dev1.staticIdentity.publicKey { + t.Fatal("initAwareEP called with unexpected public key") + } assertEqual( t, diff --git a/device/receive.go b/device/receive.go index bc37f91..e74de1a 100644 --- a/device/receive.go +++ b/device/receive.go @@ -359,7 +359,7 @@ func (device *Device) RoutineHandshake(id int) { // consume initiation - peer := device.ConsumeMessageInitiation(&msg) + peer := device.ConsumeMessageInitiation(&msg, elem.endpoint) if peer == nil { device.log.Verbosef("Received invalid initiation message from %s", elem.endpoint.DstToString()) goto skip From 4064566ecaf999e6d097f16f47c16453fbc67d8e Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Thu, 10 Jul 2025 21:36:38 -0700 Subject: [PATCH 38/60] device: fix keepalive detection in TX path Updates tailscale/corp#30364 Signed-off-by: Jordan Whited --- device/send.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/device/send.go b/device/send.go index bf854b7..c8bb079 100644 --- a/device/send.go +++ b/device/send.go @@ -517,7 +517,7 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) { dataSent := false elemsContainer.Lock() for _, elem := range elemsContainer.elems { - if len(elem.packet) != MessageKeepaliveSize { + if len(elem.packet[MessageEncapsulatingTransportSize:]) != MessageKeepaliveSize { dataSent = true } bufs = append(bufs, elem.packet) From 1d0488a3d7da6b6ed79202519f30e7a286e0d4e6 Mon Sep 17 00:00:00 2001 From: Jordan Whited Date: Wed, 16 Jul 2025 08:43:20 -0700 Subject: [PATCH 39/60] conn,device: eval conn.PeerAwareEndpoint per-packet Peer.SetEndpointFromPacket is not called per-packet. It is guaranteed to be called at least once per packet batch. Updates tailscale/corp#30042 Updates tailscale/corp#20732 Signed-off-by: Jordan Whited --- conn/conn.go | 5 +++-- device/peer.go | 4 ---- device/receive.go | 3 +++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conn/conn.go b/conn/conn.go index b949641..f178161 100644 --- a/conn/conn.go +++ b/conn/conn.go @@ -109,8 +109,9 @@ type InitiationAwareEndpoint interface { // to learn the identification WireGuard can derive from the session // or handshake. // -// wireguard-go never installs a [PeerAwareEndpoint] as the [Endpoint] for a -// [Peer]. +// A [PeerAwareEndpoint] may be installed as the [conn.Endpoint] following +// successful decryption unless endpoint roaming has been disabled for +// the peer. type PeerAwareEndpoint interface { // FromPeer is called at least once per successfully Cryptokey Routing ID'd // [ReceiveFunc] packets batch for a given node key. wireguard-go will diff --git a/device/peer.go b/device/peer.go index c188c31..064feb2 100644 --- a/device/peer.go +++ b/device/peer.go @@ -282,10 +282,6 @@ func (peer *Peer) Stop() { func (peer *Peer) SetEndpointFromPacket(endpoint conn.Endpoint) { peer.endpoint.Lock() defer peer.endpoint.Unlock() - if ep, ok := endpoint.(conn.PeerAwareEndpoint); ok { - ep.FromPeer(peer.handshake.remoteStatic) - return - } if peer.endpoint.disableRoaming { return } diff --git a/device/receive.go b/device/receive.go index e74de1a..02c8f21 100644 --- a/device/receive.go +++ b/device/receive.go @@ -460,6 +460,9 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { peer.timersHandshakeComplete() peer.SendStagedPackets() } + if ep, ok := elem.endpoint.(conn.PeerAwareEndpoint); ok { + ep.FromPeer(peer.handshake.remoteStatic) + } rxBytesLen += uint64(len(elem.packet) + MinMessageSize) if len(elem.packet) == 0 { From f8eafb781b99db38ef49be66bb4353e94cccd43a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 7 Nov 2022 15:29:08 +0800 Subject: [PATCH 40/60] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e460293..c7b1915 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ wireguard-go +/.idea/ +.DS_Store \ No newline at end of file From e4aedc6f6e59e1827c7146cc8dc6947b91aff55c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Sat, 9 Dec 2023 16:59:55 +0800 Subject: [PATCH 41/60] Add remove unused script --- remove-unused.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 remove-unused.sh diff --git a/remove-unused.sh b/remove-unused.sh new file mode 100755 index 0000000..43f3f1b --- /dev/null +++ b/remove-unused.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +function remove_unused() { + git rm -rf --ignore-unmatch \ + .github \ + tests \ + *_test.go \ + **/*_test.go \ + conn/bindtest \ + tun/netstack \ + tun/tuntest \ + tun/testdata \ + main*.go \ + *.md +} + +remove_unused +remove_unused + +go mod tidy +git commit -a -m "Remove unused" From 177dea98065e796f9b0a0c96cdf012e5082813bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 15 Sep 2025 18:12:04 +0800 Subject: [PATCH 42/60] Remove unused --- .github/workflows/test.yml | 68 -- README.md | 77 -- conn/bind_std_test.go | 250 ------ conn/bindtest/bindtest.go | 137 ---- conn/conn_test.go | 24 - conn/control_linux_test.go | 266 ------- device/allowedips_rand_test.go | 141 ---- device/allowedips_test.go | 247 ------ device/bind_test.go | 56 -- device/cookie_test.go | 190 ----- device/device_test.go | 476 ------------ device/endpoint_test.go | 49 -- device/kdf_test.go | 85 --- device/noise_test.go | 207 ----- device/pools_test.go | 140 ---- device/race_disabled_test.go | 10 - device/race_enabled_test.go | 10 - format_test.go | 51 -- go.mod | 6 - go.sum | 6 - ipc/namedpipe/namedpipe_test.go | 674 ---------------- main.go | 268 ------- main_windows.go | 99 --- ratelimiter/ratelimiter_test.go | 119 --- replay/replay_test.go | 119 --- tai64n/tai64n_test.go | 40 - tests/netns.sh | 425 ----------- tun/alignment_windows_test.go | 67 -- tun/checksum_amd64_test.go | 45 -- tun/checksum_generic_test.go | 26 - tun/checksum_test.go | 619 --------------- tun/netstack/examples/http_client.go | 54 -- tun/netstack/examples/http_server.go | 51 -- tun/netstack/examples/ping_client.go | 75 -- tun/netstack/tun.go | 1055 -------------------------- tun/offload_linux_test.go | 764 ------------------- tun/offload_test.go | 95 --- tun/tuntest/tuntest.go | 155 ---- 38 files changed, 7246 deletions(-) delete mode 100644 .github/workflows/test.yml delete mode 100644 README.md delete mode 100644 conn/bind_std_test.go delete mode 100644 conn/bindtest/bindtest.go delete mode 100644 conn/conn_test.go delete mode 100644 conn/control_linux_test.go delete mode 100644 device/allowedips_rand_test.go delete mode 100644 device/allowedips_test.go delete mode 100644 device/bind_test.go delete mode 100644 device/cookie_test.go delete mode 100644 device/device_test.go delete mode 100644 device/endpoint_test.go delete mode 100644 device/kdf_test.go delete mode 100644 device/noise_test.go delete mode 100644 device/pools_test.go delete mode 100644 device/race_disabled_test.go delete mode 100644 device/race_enabled_test.go delete mode 100644 format_test.go delete mode 100644 ipc/namedpipe/namedpipe_test.go delete mode 100644 main.go delete mode 100644 main_windows.go delete mode 100644 ratelimiter/ratelimiter_test.go delete mode 100644 replay/replay_test.go delete mode 100644 tai64n/tai64n_test.go delete mode 100755 tests/netns.sh delete mode 100644 tun/alignment_windows_test.go delete mode 100644 tun/checksum_amd64_test.go delete mode 100644 tun/checksum_generic_test.go delete mode 100644 tun/checksum_test.go delete mode 100644 tun/netstack/examples/http_client.go delete mode 100644 tun/netstack/examples/http_server.go delete mode 100644 tun/netstack/examples/ping_client.go delete mode 100644 tun/netstack/tun.go delete mode 100644 tun/offload_linux_test.go delete mode 100644 tun/offload_test.go delete mode 100644 tun/tuntest/tuntest.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index a370995..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: CI - -on: - push: - branches: ["tailscale"] - pull_request: - branches: ["tailscale"] - -jobs: - build: - runs-on: ubuntu-22.04 - strategy: - matrix: - include: - - goos: linux - goarch: amd64 - - goos: linux - goarch: arm64 - - goos: linux - goarch: "386" - - goos: linux - goarch: loong64 - - goos: linux - goarch: arm - goarm: "5" - - goos: linux - goarch: arm - goarm: "7" - # macOS - - goos: darwin - goarch: amd64 - - goos: darwin - goarch: arm64 - # Windows - - goos: windows - goarch: amd64 - - goos: windows - goarch: arm64 - # BSDs - - goos: freebsd - goarch: amd64 - - goos: openbsd - goarch: amd64 - steps: - - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: setup go - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version-file: go.mod - - name: build - run: go build ./... - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: "0" - - test: - runs-on: ubuntu-22.04 - steps: - - name: checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: setup go - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 - with: - go-version-file: go.mod - - name: test - run: go test -race -v ./... diff --git a/README.md b/README.md deleted file mode 100644 index 074f7ec..0000000 --- a/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Go Implementation of [WireGuard](https://www.wireguard.com/) - -This is an implementation of WireGuard in Go. - -## Usage - -Most Linux kernel WireGuard users are used to adding an interface with `ip link add wg0 type wireguard`. With wireguard-go, instead simply run: - -``` -$ wireguard-go wg0 -``` - -This will create an interface and fork into the background. To remove the interface, use the usual `ip link del wg0`, or if your system does not support removing interfaces directly, you may instead remove the control socket via `rm -f /var/run/wireguard/wg0.sock`, which will result in wireguard-go shutting down. - -To run wireguard-go without forking to the background, pass `-f` or `--foreground`: - -``` -$ wireguard-go -f wg0 -``` - -When an interface is running, you may use [`wg(8)`](https://git.zx2c4.com/wireguard-tools/about/src/man/wg.8) to configure it, as well as the usual `ip(8)` and `ifconfig(8)` commands. - -To run with more logging you may set the environment variable `LOG_LEVEL=debug`. - -## Platforms - -### Linux - -This will run on Linux; however you should instead use the kernel module, which is faster and better integrated into the OS. See the [installation page](https://www.wireguard.com/install/) for instructions. - -### macOS - -This runs on macOS using the utun driver. It does not yet support sticky sockets, and won't support fwmarks because of Darwin limitations. Since the utun driver cannot have arbitrary interface names, you must either use `utun[0-9]+` for an explicit interface name or `utun` to have the kernel select one for you. If you choose `utun` as the interface name, and the environment variable `WG_TUN_NAME_FILE` is defined, then the actual name of the interface chosen by the kernel is written to the file specified by that variable. - -### Windows - -This runs on Windows, but you should instead use it from the more [fully featured Windows app](https://git.zx2c4.com/wireguard-windows/about/), which uses this as a module. - -### FreeBSD - -This will run on FreeBSD. It does not yet support sticky sockets. Fwmark is mapped to `SO_USER_COOKIE`. - -### OpenBSD - -This will run on OpenBSD. It does not yet support sticky sockets. Fwmark is mapped to `SO_RTABLE`. Since the tun driver cannot have arbitrary interface names, you must either use `tun[0-9]+` for an explicit interface name or `tun` to have the program select one for you. If you choose `tun` as the interface name, and the environment variable `WG_TUN_NAME_FILE` is defined, then the actual name of the interface chosen by the kernel is written to the file specified by that variable. - -## Building - -This requires an installation of the latest version of [Go](https://go.dev/). - -``` -$ git clone https://git.zx2c4.com/wireguard-go -$ cd wireguard-go -$ make -``` - -## License - - Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - - Permission is hereby granted, free of charge, to any person obtaining a copy of - this software and associated documentation files (the "Software"), to deal in - the Software without restriction, including without limitation the rights to - use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies - of the Software, and to permit persons to whom the Software is furnished to do - so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. diff --git a/conn/bind_std_test.go b/conn/bind_std_test.go deleted file mode 100644 index 77af0d9..0000000 --- a/conn/bind_std_test.go +++ /dev/null @@ -1,250 +0,0 @@ -package conn - -import ( - "encoding/binary" - "net" - "testing" - - "golang.org/x/net/ipv6" -) - -func TestStdNetBindReceiveFuncAfterClose(t *testing.T) { - bind := NewStdNetBind().(*StdNetBind) - fns, _, err := bind.Open(0) - if err != nil { - t.Fatal(err) - } - bind.Close() - bufs := make([][]byte, 1) - bufs[0] = make([]byte, 1) - sizes := make([]int, 1) - eps := make([]Endpoint, 1) - for _, fn := range fns { - // The ReceiveFuncs must not access conn-related fields on StdNetBind - // unguarded. Close() nils the conn-related fields resulting in a panic - // if they violate the mutex. - fn(bufs, sizes, eps) - } -} - -func mockSetGSOSize(control *[]byte, gsoSize uint16) { - *control = (*control)[:cap(*control)] - binary.LittleEndian.PutUint16(*control, gsoSize) -} - -func Test_coalesceMessages(t *testing.T) { - cases := []struct { - name string - buffs [][]byte - wantLens []int - wantGSO []int - }{ - { - name: "one message no coalesce", - buffs: [][]byte{ - make([]byte, 1, 1), - }, - wantLens: []int{1}, - wantGSO: []int{0}, - }, - { - name: "two messages equal len coalesce", - buffs: [][]byte{ - make([]byte, 1, 2), - make([]byte, 1, 1), - }, - wantLens: []int{2}, - wantGSO: []int{1}, - }, - { - name: "two messages unequal len coalesce", - buffs: [][]byte{ - make([]byte, 2, 3), - make([]byte, 1, 1), - }, - wantLens: []int{3}, - wantGSO: []int{2}, - }, - { - name: "three messages second unequal len coalesce", - buffs: [][]byte{ - make([]byte, 2, 3), - make([]byte, 1, 1), - make([]byte, 2, 2), - }, - wantLens: []int{3, 2}, - wantGSO: []int{2, 0}, - }, - { - name: "three messages limited cap coalesce", - buffs: [][]byte{ - make([]byte, 2, 4), - make([]byte, 2, 2), - make([]byte, 2, 2), - }, - wantLens: []int{4, 2}, - wantGSO: []int{2, 0}, - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - addr := &net.UDPAddr{ - IP: net.ParseIP("127.0.0.1").To4(), - Port: 1, - } - msgs := make([]ipv6.Message, len(tt.buffs)) - for i := range msgs { - msgs[i].Buffers = make([][]byte, 1) - msgs[i].OOB = make([]byte, 0, 2) - } - got := coalesceMessages(addr, &StdNetEndpoint{AddrPort: addr.AddrPort()}, tt.buffs, 0, msgs, mockSetGSOSize) - if got != len(tt.wantLens) { - t.Fatalf("got len %d want: %d", got, len(tt.wantLens)) - } - for i := 0; i < got; i++ { - if msgs[i].Addr != addr { - t.Errorf("msgs[%d].Addr != passed addr", i) - } - gotLen := len(msgs[i].Buffers[0]) - if gotLen != tt.wantLens[i] { - t.Errorf("len(msgs[%d].Buffers[0]) %d != %d", i, gotLen, tt.wantLens[i]) - } - gotGSO, err := mockGetGSOSize(msgs[i].OOB) - if err != nil { - t.Fatalf("msgs[%d] getGSOSize err: %v", i, err) - } - if gotGSO != tt.wantGSO[i] { - t.Errorf("msgs[%d] gsoSize %d != %d", i, gotGSO, tt.wantGSO[i]) - } - } - }) - } -} - -func mockGetGSOSize(control []byte) (int, error) { - if len(control) < 2 { - return 0, nil - } - return int(binary.LittleEndian.Uint16(control)), nil -} - -func Test_splitCoalescedMessages(t *testing.T) { - newMsg := func(n, gso int) ipv6.Message { - msg := ipv6.Message{ - Buffers: [][]byte{make([]byte, 1<<16-1)}, - N: n, - OOB: make([]byte, 2), - } - binary.LittleEndian.PutUint16(msg.OOB, uint16(gso)) - if gso > 0 { - msg.NN = 2 - } - return msg - } - - cases := []struct { - name string - msgs []ipv6.Message - firstMsgAt int - wantNumEval int - wantMsgLens []int - wantErr bool - }{ - { - name: "second last split last empty", - msgs: []ipv6.Message{ - newMsg(0, 0), - newMsg(0, 0), - newMsg(3, 1), - newMsg(0, 0), - }, - firstMsgAt: 2, - wantNumEval: 3, - wantMsgLens: []int{1, 1, 1, 0}, - wantErr: false, - }, - { - name: "second last no split last empty", - msgs: []ipv6.Message{ - newMsg(0, 0), - newMsg(0, 0), - newMsg(1, 0), - newMsg(0, 0), - }, - firstMsgAt: 2, - wantNumEval: 1, - wantMsgLens: []int{1, 0, 0, 0}, - wantErr: false, - }, - { - name: "second last no split last no split", - msgs: []ipv6.Message{ - newMsg(0, 0), - newMsg(0, 0), - newMsg(1, 0), - newMsg(1, 0), - }, - firstMsgAt: 2, - wantNumEval: 2, - wantMsgLens: []int{1, 1, 0, 0}, - wantErr: false, - }, - { - name: "second last no split last split", - msgs: []ipv6.Message{ - newMsg(0, 0), - newMsg(0, 0), - newMsg(1, 0), - newMsg(3, 1), - }, - firstMsgAt: 2, - wantNumEval: 4, - wantMsgLens: []int{1, 1, 1, 1}, - wantErr: false, - }, - { - name: "second last split last split", - msgs: []ipv6.Message{ - newMsg(0, 0), - newMsg(0, 0), - newMsg(2, 1), - newMsg(2, 1), - }, - firstMsgAt: 2, - wantNumEval: 4, - wantMsgLens: []int{1, 1, 1, 1}, - wantErr: false, - }, - { - name: "second last no split last split overflow", - msgs: []ipv6.Message{ - newMsg(0, 0), - newMsg(0, 0), - newMsg(1, 0), - newMsg(4, 1), - }, - firstMsgAt: 2, - wantNumEval: 4, - wantMsgLens: []int{1, 1, 1, 1}, - wantErr: true, - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - got, err := splitCoalescedMessages(tt.msgs, 2, mockGetGSOSize) - if err != nil && !tt.wantErr { - t.Fatalf("err: %v", err) - } - if got != tt.wantNumEval { - t.Fatalf("got to eval: %d want: %d", got, tt.wantNumEval) - } - for i, msg := range tt.msgs { - if msg.N != tt.wantMsgLens[i] { - t.Fatalf("msg[%d].N: %d want: %d", i, msg.N, tt.wantMsgLens[i]) - } - } - }) - } -} diff --git a/conn/bindtest/bindtest.go b/conn/bindtest/bindtest.go deleted file mode 100644 index 741b776..0000000 --- a/conn/bindtest/bindtest.go +++ /dev/null @@ -1,137 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package bindtest - -import ( - "fmt" - "math/rand" - "net" - "net/netip" - "os" - - "github.com/tailscale/wireguard-go/conn" -) - -type ChannelBind struct { - rx4, tx4 *chan []byte - rx6, tx6 *chan []byte - closeSignal chan bool - source4, source6 ChannelEndpoint - target4, target6 ChannelEndpoint -} - -type ChannelEndpoint uint16 - -var ( - _ conn.Bind = (*ChannelBind)(nil) - _ conn.Endpoint = (*ChannelEndpoint)(nil) -) - -func NewChannelBinds() [2]conn.Bind { - arx4 := make(chan []byte, 8192) - brx4 := make(chan []byte, 8192) - arx6 := make(chan []byte, 8192) - brx6 := make(chan []byte, 8192) - var binds [2]ChannelBind - binds[0].rx4 = &arx4 - binds[0].tx4 = &brx4 - binds[1].rx4 = &brx4 - binds[1].tx4 = &arx4 - binds[0].rx6 = &arx6 - binds[0].tx6 = &brx6 - binds[1].rx6 = &brx6 - binds[1].tx6 = &arx6 - binds[0].target4 = ChannelEndpoint(1) - binds[1].target4 = ChannelEndpoint(2) - binds[0].target6 = ChannelEndpoint(3) - binds[1].target6 = ChannelEndpoint(4) - binds[0].source4 = binds[1].target4 - binds[0].source6 = binds[1].target6 - binds[1].source4 = binds[0].target4 - binds[1].source6 = binds[0].target6 - return [2]conn.Bind{&binds[0], &binds[1]} -} - -func (c ChannelEndpoint) ClearSrc() {} - -func (c ChannelEndpoint) SrcToString() string { return "" } - -func (c ChannelEndpoint) DstToString() string { return fmt.Sprintf("127.0.0.1:%d", c) } - -func (c ChannelEndpoint) DstToBytes() []byte { return []byte{byte(c)} } - -func (c ChannelEndpoint) DstIP() netip.Addr { return netip.AddrFrom4([4]byte{127, 0, 0, 1}) } - -func (c ChannelEndpoint) SrcIP() netip.Addr { return netip.Addr{} } - -func (c *ChannelBind) Open(port uint16) (fns []conn.ReceiveFunc, actualPort uint16, err error) { - c.closeSignal = make(chan bool) - fns = append(fns, c.makeReceiveFunc(*c.rx4)) - fns = append(fns, c.makeReceiveFunc(*c.rx6)) - if rand.Uint32()&1 == 0 { - return fns, uint16(c.source4), nil - } else { - return fns, uint16(c.source6), nil - } -} - -func (c *ChannelBind) Close() error { - if c.closeSignal != nil { - select { - case <-c.closeSignal: - default: - close(c.closeSignal) - } - } - return nil -} - -func (c *ChannelBind) BatchSize() int { return 1 } - -func (c *ChannelBind) SetMark(mark uint32) error { return nil } - -func (c *ChannelBind) makeReceiveFunc(ch chan []byte) conn.ReceiveFunc { - return func(bufs [][]byte, sizes []int, eps []conn.Endpoint) (n int, err error) { - select { - case <-c.closeSignal: - return 0, net.ErrClosed - case rx := <-ch: - copied := copy(bufs[0], rx) - sizes[0] = copied - eps[0] = c.target6 - return 1, nil - } - } -} - -func (c *ChannelBind) Send(bufs [][]byte, ep conn.Endpoint, offset int) error { - for _, b := range bufs { - b = b[offset:] - select { - case <-c.closeSignal: - return net.ErrClosed - default: - bc := make([]byte, len(b)) - copy(bc, b) - if ep.(ChannelEndpoint) == c.target4 { - *c.tx4 <- bc - } else if ep.(ChannelEndpoint) == c.target6 { - *c.tx6 <- bc - } else { - return os.ErrInvalid - } - } - } - return nil -} - -func (c *ChannelBind) ParseEndpoint(s string) (conn.Endpoint, error) { - addr, err := netip.ParseAddrPort(s) - if err != nil { - return nil, err - } - return ChannelEndpoint(addr.Port()), nil -} diff --git a/conn/conn_test.go b/conn/conn_test.go deleted file mode 100644 index c6194ee..0000000 --- a/conn/conn_test.go +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package conn - -import ( - "testing" -) - -func TestPrettyName(t *testing.T) { - var ( - recvFunc ReceiveFunc = func(bufs [][]byte, sizes []int, eps []Endpoint) (n int, err error) { return } - ) - - const want = "TestPrettyName" - - t.Run("ReceiveFunc.PrettyName", func(t *testing.T) { - if got := recvFunc.PrettyName(); got != want { - t.Errorf("PrettyName() = %v, want %v", got, want) - } - }) -} diff --git a/conn/control_linux_test.go b/conn/control_linux_test.go deleted file mode 100644 index 3ca7d37..0000000 --- a/conn/control_linux_test.go +++ /dev/null @@ -1,266 +0,0 @@ -//go:build linux && !android - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package conn - -import ( - "context" - "net" - "net/netip" - "runtime" - "testing" - "unsafe" - - "golang.org/x/sys/unix" -) - -func setSrc(ep *StdNetEndpoint, addr netip.Addr, ifidx int32) { - var buf []byte - if addr.Is4() { - buf = make([]byte, unix.CmsgSpace(unix.SizeofInet4Pktinfo)) - hdr := unix.Cmsghdr{ - Level: unix.IPPROTO_IP, - Type: unix.IP_PKTINFO, - } - hdr.SetLen(unix.CmsgLen(unix.SizeofInet4Pktinfo)) - copy(buf, unsafe.Slice((*byte)(unsafe.Pointer(&hdr)), int(unsafe.Sizeof(hdr)))) - - info := unix.Inet4Pktinfo{ - Ifindex: ifidx, - Spec_dst: addr.As4(), - } - copy(buf[unix.CmsgLen(0):], unsafe.Slice((*byte)(unsafe.Pointer(&info)), unix.SizeofInet4Pktinfo)) - } else { - buf = make([]byte, unix.CmsgSpace(unix.SizeofInet6Pktinfo)) - hdr := unix.Cmsghdr{ - Level: unix.IPPROTO_IPV6, - Type: unix.IPV6_PKTINFO, - } - hdr.SetLen(unix.CmsgLen(unix.SizeofInet6Pktinfo)) - copy(buf, unsafe.Slice((*byte)(unsafe.Pointer(&hdr)), int(unsafe.Sizeof(hdr)))) - - info := unix.Inet6Pktinfo{ - Ifindex: uint32(ifidx), - Addr: addr.As16(), - } - copy(buf[unix.CmsgLen(0):], unsafe.Slice((*byte)(unsafe.Pointer(&info)), unix.SizeofInet6Pktinfo)) - } - - ep.src = buf -} - -func Test_setSrcControl(t *testing.T) { - t.Run("IPv4", func(t *testing.T) { - ep := &StdNetEndpoint{ - AddrPort: netip.MustParseAddrPort("127.0.0.1:1234"), - } - setSrc(ep, netip.MustParseAddr("127.0.0.1"), 5) - - control := make([]byte, controlSize) - - setSrcControl(&control, ep) - - hdr := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) - if hdr.Level != unix.IPPROTO_IP { - t.Errorf("unexpected level: %d", hdr.Level) - } - if hdr.Type != unix.IP_PKTINFO { - t.Errorf("unexpected type: %d", hdr.Type) - } - if uint(hdr.Len) != uint(unix.CmsgLen(int(unsafe.Sizeof(unix.Inet4Pktinfo{})))) { - t.Errorf("unexpected length: %d", hdr.Len) - } - info := (*unix.Inet4Pktinfo)(unsafe.Pointer(&control[unix.CmsgLen(0)])) - if info.Spec_dst[0] != 127 || info.Spec_dst[1] != 0 || info.Spec_dst[2] != 0 || info.Spec_dst[3] != 1 { - t.Errorf("unexpected address: %v", info.Spec_dst) - } - if info.Ifindex != 5 { - t.Errorf("unexpected ifindex: %d", info.Ifindex) - } - }) - - t.Run("IPv6", func(t *testing.T) { - ep := &StdNetEndpoint{ - AddrPort: netip.MustParseAddrPort("[::1]:1234"), - } - setSrc(ep, netip.MustParseAddr("::1"), 5) - - control := make([]byte, controlSize) - - setSrcControl(&control, ep) - - hdr := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) - if hdr.Level != unix.IPPROTO_IPV6 { - t.Errorf("unexpected level: %d", hdr.Level) - } - if hdr.Type != unix.IPV6_PKTINFO { - t.Errorf("unexpected type: %d", hdr.Type) - } - if uint(hdr.Len) != uint(unix.CmsgLen(int(unsafe.Sizeof(unix.Inet6Pktinfo{})))) { - t.Errorf("unexpected length: %d", hdr.Len) - } - info := (*unix.Inet6Pktinfo)(unsafe.Pointer(&control[unix.CmsgLen(0)])) - if info.Addr != ep.SrcIP().As16() { - t.Errorf("unexpected address: %v", info.Addr) - } - if info.Ifindex != 5 { - t.Errorf("unexpected ifindex: %d", info.Ifindex) - } - }) - - t.Run("ClearOnNoSrc", func(t *testing.T) { - control := make([]byte, unix.CmsgLen(0)) - hdr := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) - hdr.Level = 1 - hdr.Type = 2 - hdr.Len = 3 - - setSrcControl(&control, &StdNetEndpoint{}) - - if len(control) != 0 { - t.Errorf("unexpected control: %v", control) - } - }) -} - -func Test_getSrcFromControl(t *testing.T) { - t.Run("IPv4", func(t *testing.T) { - control := make([]byte, unix.CmsgSpace(unix.SizeofInet4Pktinfo)) - hdr := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) - hdr.Level = unix.IPPROTO_IP - hdr.Type = unix.IP_PKTINFO - hdr.SetLen(unix.CmsgLen(int(unsafe.Sizeof(unix.Inet4Pktinfo{})))) - info := (*unix.Inet4Pktinfo)(unsafe.Pointer(&control[unix.CmsgLen(0)])) - info.Spec_dst = [4]byte{127, 0, 0, 1} - info.Ifindex = 5 - - ep := &StdNetEndpoint{} - getSrcFromControl(control, ep) - - if ep.SrcIP() != netip.MustParseAddr("127.0.0.1") { - t.Errorf("unexpected address: %v", ep.SrcIP()) - } - if ep.SrcIfidx() != 5 { - t.Errorf("unexpected ifindex: %d", ep.SrcIfidx()) - } - }) - t.Run("IPv6", func(t *testing.T) { - control := make([]byte, unix.CmsgSpace(unix.SizeofInet6Pktinfo)) - hdr := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) - hdr.Level = unix.IPPROTO_IPV6 - hdr.Type = unix.IPV6_PKTINFO - hdr.SetLen(unix.CmsgLen(int(unsafe.Sizeof(unix.Inet6Pktinfo{})))) - info := (*unix.Inet6Pktinfo)(unsafe.Pointer(&control[unix.CmsgLen(0)])) - info.Addr = [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} - info.Ifindex = 5 - - ep := &StdNetEndpoint{} - getSrcFromControl(control, ep) - - if ep.SrcIP() != netip.MustParseAddr("::1") { - t.Errorf("unexpected address: %v", ep.SrcIP()) - } - if ep.SrcIfidx() != 5 { - t.Errorf("unexpected ifindex: %d", ep.SrcIfidx()) - } - }) - t.Run("ClearOnEmpty", func(t *testing.T) { - var control []byte - ep := &StdNetEndpoint{} - setSrc(ep, netip.MustParseAddr("::1"), 5) - - getSrcFromControl(control, ep) - if ep.SrcIP().IsValid() { - t.Errorf("unexpected address: %v", ep.SrcIP()) - } - if ep.SrcIfidx() != 0 { - t.Errorf("unexpected ifindex: %d", ep.SrcIfidx()) - } - }) - t.Run("Multiple", func(t *testing.T) { - zeroControl := make([]byte, unix.CmsgSpace(0)) - zeroHdr := (*unix.Cmsghdr)(unsafe.Pointer(&zeroControl[0])) - zeroHdr.SetLen(unix.CmsgLen(0)) - - control := make([]byte, unix.CmsgSpace(unix.SizeofInet4Pktinfo)) - hdr := (*unix.Cmsghdr)(unsafe.Pointer(&control[0])) - hdr.Level = unix.IPPROTO_IP - hdr.Type = unix.IP_PKTINFO - hdr.SetLen(unix.CmsgLen(int(unsafe.Sizeof(unix.Inet4Pktinfo{})))) - info := (*unix.Inet4Pktinfo)(unsafe.Pointer(&control[unix.CmsgLen(0)])) - info.Spec_dst = [4]byte{127, 0, 0, 1} - info.Ifindex = 5 - - combined := make([]byte, 0) - combined = append(combined, zeroControl...) - combined = append(combined, control...) - - ep := &StdNetEndpoint{} - getSrcFromControl(combined, ep) - - if ep.SrcIP() != netip.MustParseAddr("127.0.0.1") { - t.Errorf("unexpected address: %v", ep.SrcIP()) - } - if ep.SrcIfidx() != 5 { - t.Errorf("unexpected ifindex: %d", ep.SrcIfidx()) - } - }) -} - -func Test_listenConfig(t *testing.T) { - t.Run("IPv4", func(t *testing.T) { - conn, err := listenConfig().ListenPacket(context.Background(), "udp4", ":0") - if err != nil { - t.Fatal(err) - } - defer conn.Close() - sc, err := conn.(*net.UDPConn).SyscallConn() - if err != nil { - t.Fatal(err) - } - - if runtime.GOOS == "linux" { - var i int - sc.Control(func(fd uintptr) { - i, err = unix.GetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_PKTINFO) - }) - if err != nil { - t.Fatal(err) - } - if i != 1 { - t.Error("IP_PKTINFO not set!") - } - } else { - t.Logf("listenConfig() does not set IPV6_RECVPKTINFO on %s", runtime.GOOS) - } - }) - t.Run("IPv6", func(t *testing.T) { - conn, err := listenConfig().ListenPacket(context.Background(), "udp6", ":0") - if err != nil { - t.Fatal(err) - } - sc, err := conn.(*net.UDPConn).SyscallConn() - if err != nil { - t.Fatal(err) - } - - if runtime.GOOS == "linux" { - var i int - sc.Control(func(fd uintptr) { - i, err = unix.GetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_RECVPKTINFO) - }) - if err != nil { - t.Fatal(err) - } - if i != 1 { - t.Error("IPV6_PKTINFO not set!") - } - } else { - t.Logf("listenConfig() does not set IPV6_RECVPKTINFO on %s", runtime.GOOS) - } - }) -} diff --git a/device/allowedips_rand_test.go b/device/allowedips_rand_test.go deleted file mode 100644 index 07065c3..0000000 --- a/device/allowedips_rand_test.go +++ /dev/null @@ -1,141 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -import ( - "math/rand" - "net" - "net/netip" - "sort" - "testing" -) - -const ( - NumberOfPeers = 100 - NumberOfPeerRemovals = 4 - NumberOfAddresses = 250 - NumberOfTests = 10000 -) - -type SlowNode struct { - peer *Peer - cidr uint8 - bits []byte -} - -type SlowRouter []*SlowNode - -func (r SlowRouter) Len() int { - return len(r) -} - -func (r SlowRouter) Less(i, j int) bool { - return r[i].cidr > r[j].cidr -} - -func (r SlowRouter) Swap(i, j int) { - r[i], r[j] = r[j], r[i] -} - -func (r SlowRouter) Insert(addr []byte, cidr uint8, peer *Peer) SlowRouter { - for _, t := range r { - if t.cidr == cidr && commonBits(t.bits, addr) >= cidr { - t.peer = peer - t.bits = addr - return r - } - } - r = append(r, &SlowNode{ - cidr: cidr, - bits: addr, - peer: peer, - }) - sort.Sort(r) - return r -} - -func (r SlowRouter) Lookup(addr []byte) *Peer { - for _, t := range r { - common := commonBits(t.bits, addr) - if common >= t.cidr { - return t.peer - } - } - return nil -} - -func (r SlowRouter) RemoveByPeer(peer *Peer) SlowRouter { - n := 0 - for _, x := range r { - if x.peer != peer { - r[n] = x - n++ - } - } - return r[:n] -} - -func TestTrieRandom(t *testing.T) { - var slow4, slow6 SlowRouter - var peers []*Peer - var allowedIPs AllowedIPs - - rand.Seed(1) - - for n := 0; n < NumberOfPeers; n++ { - peers = append(peers, &Peer{}) - } - - for n := 0; n < NumberOfAddresses; n++ { - var addr4 [4]byte - rand.Read(addr4[:]) - cidr := uint8(rand.Intn(32) + 1) - index := rand.Intn(NumberOfPeers) - allowedIPs.Insert(netip.PrefixFrom(netip.AddrFrom4(addr4), int(cidr)), peers[index]) - slow4 = slow4.Insert(addr4[:], cidr, peers[index]) - - var addr6 [16]byte - rand.Read(addr6[:]) - cidr = uint8(rand.Intn(128) + 1) - index = rand.Intn(NumberOfPeers) - allowedIPs.Insert(netip.PrefixFrom(netip.AddrFrom16(addr6), int(cidr)), peers[index]) - slow6 = slow6.Insert(addr6[:], cidr, peers[index]) - } - - var p int - for p = 0; ; p++ { - for n := 0; n < NumberOfTests; n++ { - var addr4 [4]byte - rand.Read(addr4[:]) - peer1 := slow4.Lookup(addr4[:]) - peer2 := allowedIPs.Lookup(addr4[:]) - if peer1 != peer2 { - t.Errorf("Trie did not match naive implementation, for %v: want %p, got %p", net.IP(addr4[:]), peer1, peer2) - } - - var addr6 [16]byte - rand.Read(addr6[:]) - peer1 = slow6.Lookup(addr6[:]) - peer2 = allowedIPs.Lookup(addr6[:]) - if peer1 != peer2 { - t.Errorf("Trie did not match naive implementation, for %v: want %p, got %p", net.IP(addr6[:]), peer1, peer2) - } - } - if p >= len(peers) || p >= NumberOfPeerRemovals { - break - } - allowedIPs.RemoveByPeer(peers[p]) - slow4 = slow4.RemoveByPeer(peers[p]) - slow6 = slow6.RemoveByPeer(peers[p]) - } - for ; p < len(peers); p++ { - allowedIPs.RemoveByPeer(peers[p]) - } - - if allowedIPs.IPv4 != nil || allowedIPs.IPv6 != nil { - t.Error("Failed to remove all nodes from trie by peer") - } -} diff --git a/device/allowedips_test.go b/device/allowedips_test.go deleted file mode 100644 index cde068e..0000000 --- a/device/allowedips_test.go +++ /dev/null @@ -1,247 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -import ( - "math/rand" - "net" - "net/netip" - "testing" -) - -type testPairCommonBits struct { - s1 []byte - s2 []byte - match uint8 -} - -func TestCommonBits(t *testing.T) { - tests := []testPairCommonBits{ - {s1: []byte{1, 4, 53, 128}, s2: []byte{0, 0, 0, 0}, match: 7}, - {s1: []byte{0, 4, 53, 128}, s2: []byte{0, 0, 0, 0}, match: 13}, - {s1: []byte{0, 4, 53, 253}, s2: []byte{0, 4, 53, 252}, match: 31}, - {s1: []byte{192, 168, 1, 1}, s2: []byte{192, 169, 1, 1}, match: 15}, - {s1: []byte{65, 168, 1, 1}, s2: []byte{192, 169, 1, 1}, match: 0}, - } - - for _, p := range tests { - v := commonBits(p.s1, p.s2) - if v != p.match { - t.Error( - "For slice", p.s1, p.s2, - "expected match", p.match, - ",but got", v, - ) - } - } -} - -func benchmarkTrie(peerNumber, addressNumber, addressLength int, b *testing.B) { - var trie *trieEntry - var peers []*Peer - root := parentIndirection{&trie, 2} - - rand.Seed(1) - - const AddressLength = 4 - - for n := 0; n < peerNumber; n++ { - peers = append(peers, &Peer{}) - } - - for n := 0; n < addressNumber; n++ { - var addr [AddressLength]byte - rand.Read(addr[:]) - cidr := uint8(rand.Uint32() % (AddressLength * 8)) - index := rand.Int() % peerNumber - root.insert(addr[:], cidr, peers[index]) - } - - for n := 0; n < b.N; n++ { - var addr [AddressLength]byte - rand.Read(addr[:]) - trie.lookup(addr[:]) - } -} - -func BenchmarkTrieIPv4Peers100Addresses1000(b *testing.B) { - benchmarkTrie(100, 1000, net.IPv4len, b) -} - -func BenchmarkTrieIPv4Peers10Addresses10(b *testing.B) { - benchmarkTrie(10, 10, net.IPv4len, b) -} - -func BenchmarkTrieIPv6Peers100Addresses1000(b *testing.B) { - benchmarkTrie(100, 1000, net.IPv6len, b) -} - -func BenchmarkTrieIPv6Peers10Addresses10(b *testing.B) { - benchmarkTrie(10, 10, net.IPv6len, b) -} - -/* Test ported from kernel implementation: - * selftest/allowedips.h - */ -func TestTrieIPv4(t *testing.T) { - a := &Peer{} - b := &Peer{} - c := &Peer{} - d := &Peer{} - e := &Peer{} - g := &Peer{} - h := &Peer{} - - var allowedIPs AllowedIPs - - insert := func(peer *Peer, a, b, c, d byte, cidr uint8) { - allowedIPs.Insert(netip.PrefixFrom(netip.AddrFrom4([4]byte{a, b, c, d}), int(cidr)), peer) - } - - assertEQ := func(peer *Peer, a, b, c, d byte) { - p := allowedIPs.Lookup([]byte{a, b, c, d}) - if p != peer { - t.Error("Assert EQ failed") - } - } - - assertNEQ := func(peer *Peer, a, b, c, d byte) { - p := allowedIPs.Lookup([]byte{a, b, c, d}) - if p == peer { - t.Error("Assert NEQ failed") - } - } - - insert(a, 192, 168, 4, 0, 24) - insert(b, 192, 168, 4, 4, 32) - insert(c, 192, 168, 0, 0, 16) - insert(d, 192, 95, 5, 64, 27) - insert(c, 192, 95, 5, 65, 27) - insert(e, 0, 0, 0, 0, 0) - insert(g, 64, 15, 112, 0, 20) - insert(h, 64, 15, 123, 211, 25) - insert(a, 10, 0, 0, 0, 25) - insert(b, 10, 0, 0, 128, 25) - insert(a, 10, 1, 0, 0, 30) - insert(b, 10, 1, 0, 4, 30) - insert(c, 10, 1, 0, 8, 29) - insert(d, 10, 1, 0, 16, 29) - - assertEQ(a, 192, 168, 4, 20) - assertEQ(a, 192, 168, 4, 0) - assertEQ(b, 192, 168, 4, 4) - assertEQ(c, 192, 168, 200, 182) - assertEQ(c, 192, 95, 5, 68) - assertEQ(e, 192, 95, 5, 96) - assertEQ(g, 64, 15, 116, 26) - assertEQ(g, 64, 15, 127, 3) - - insert(a, 1, 0, 0, 0, 32) - insert(a, 64, 0, 0, 0, 32) - insert(a, 128, 0, 0, 0, 32) - insert(a, 192, 0, 0, 0, 32) - insert(a, 255, 0, 0, 0, 32) - - assertEQ(a, 1, 0, 0, 0) - assertEQ(a, 64, 0, 0, 0) - assertEQ(a, 128, 0, 0, 0) - assertEQ(a, 192, 0, 0, 0) - assertEQ(a, 255, 0, 0, 0) - - allowedIPs.RemoveByPeer(a) - - assertNEQ(a, 1, 0, 0, 0) - assertNEQ(a, 64, 0, 0, 0) - assertNEQ(a, 128, 0, 0, 0) - assertNEQ(a, 192, 0, 0, 0) - assertNEQ(a, 255, 0, 0, 0) - - allowedIPs.RemoveByPeer(a) - allowedIPs.RemoveByPeer(b) - allowedIPs.RemoveByPeer(c) - allowedIPs.RemoveByPeer(d) - allowedIPs.RemoveByPeer(e) - allowedIPs.RemoveByPeer(g) - allowedIPs.RemoveByPeer(h) - if allowedIPs.IPv4 != nil || allowedIPs.IPv6 != nil { - t.Error("Expected removing all the peers to empty trie, but it did not") - } - - insert(a, 192, 168, 0, 0, 16) - insert(a, 192, 168, 0, 0, 24) - - allowedIPs.RemoveByPeer(a) - - assertNEQ(a, 192, 168, 0, 1) -} - -/* Test ported from kernel implementation: - * selftest/allowedips.h - */ -func TestTrieIPv6(t *testing.T) { - a := &Peer{} - b := &Peer{} - c := &Peer{} - d := &Peer{} - e := &Peer{} - f := &Peer{} - g := &Peer{} - h := &Peer{} - - var allowedIPs AllowedIPs - - expand := func(a uint32) []byte { - var out [4]byte - out[0] = byte(a >> 24 & 0xff) - out[1] = byte(a >> 16 & 0xff) - out[2] = byte(a >> 8 & 0xff) - out[3] = byte(a & 0xff) - return out[:] - } - - insert := func(peer *Peer, a, b, c, d uint32, cidr uint8) { - var addr []byte - addr = append(addr, expand(a)...) - addr = append(addr, expand(b)...) - addr = append(addr, expand(c)...) - addr = append(addr, expand(d)...) - allowedIPs.Insert(netip.PrefixFrom(netip.AddrFrom16(*(*[16]byte)(addr)), int(cidr)), peer) - } - - assertEQ := func(peer *Peer, a, b, c, d uint32) { - var addr []byte - addr = append(addr, expand(a)...) - addr = append(addr, expand(b)...) - addr = append(addr, expand(c)...) - addr = append(addr, expand(d)...) - p := allowedIPs.Lookup(addr) - if p != peer { - t.Error("Assert EQ failed") - } - } - - insert(d, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128) - insert(c, 0x26075300, 0x60006b00, 0, 0, 64) - insert(e, 0, 0, 0, 0, 0) - insert(f, 0, 0, 0, 0, 0) - insert(g, 0x24046800, 0, 0, 0, 32) - insert(h, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 64) - insert(a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 128) - insert(c, 0x24446800, 0x40e40800, 0xdeaebeef, 0xdefbeef, 128) - insert(b, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98) - - assertEQ(d, 0x26075300, 0x60006b00, 0, 0xc05f0543) - assertEQ(c, 0x26075300, 0x60006b00, 0, 0xc02e01ee) - assertEQ(f, 0x26075300, 0x60006b01, 0, 0) - assertEQ(g, 0x24046800, 0x40040806, 0, 0x1006) - assertEQ(g, 0x24046800, 0x40040806, 0x1234, 0x5678) - assertEQ(f, 0x240467ff, 0x40040806, 0x1234, 0x5678) - assertEQ(f, 0x24046801, 0x40040806, 0x1234, 0x5678) - assertEQ(h, 0x24046800, 0x40040800, 0x1234, 0x5678) - assertEQ(h, 0x24046800, 0x40040800, 0, 0) - assertEQ(h, 0x24046800, 0x40040800, 0x10101010, 0x10101010) - assertEQ(a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef) -} diff --git a/device/bind_test.go b/device/bind_test.go deleted file mode 100644 index d64ca09..0000000 --- a/device/bind_test.go +++ /dev/null @@ -1,56 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -import ( - "errors" - - "github.com/tailscale/wireguard-go/conn" -) - -type DummyDatagram struct { - msg []byte - endpoint conn.Endpoint -} - -type DummyBind struct { - in6 chan DummyDatagram - in4 chan DummyDatagram - closed bool -} - -func (b *DummyBind) SetMark(v uint32) error { - return nil -} - -func (b *DummyBind) ReceiveIPv6(buf []byte) (int, conn.Endpoint, error) { - datagram, ok := <-b.in6 - if !ok { - return 0, nil, errors.New("closed") - } - copy(buf, datagram.msg) - return len(datagram.msg), datagram.endpoint, nil -} - -func (b *DummyBind) ReceiveIPv4(buf []byte) (int, conn.Endpoint, error) { - datagram, ok := <-b.in4 - if !ok { - return 0, nil, errors.New("closed") - } - copy(buf, datagram.msg) - return len(datagram.msg), datagram.endpoint, nil -} - -func (b *DummyBind) Close() error { - close(b.in6) - close(b.in4) - b.closed = true - return nil -} - -func (b *DummyBind) Send(buf []byte, end conn.Endpoint) error { - return nil -} diff --git a/device/cookie_test.go b/device/cookie_test.go deleted file mode 100644 index 4f1e50a..0000000 --- a/device/cookie_test.go +++ /dev/null @@ -1,190 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -import ( - "testing" -) - -func TestCookieMAC1(t *testing.T) { - // setup generator / checker - - var ( - generator CookieGenerator - checker CookieChecker - ) - - sk, err := newPrivateKey() - if err != nil { - t.Fatal(err) - } - pk := sk.publicKey() - - generator.Init(pk) - checker.Init(pk) - - // check mac1 - - src := []byte{192, 168, 13, 37, 10, 10, 10} - - checkMAC1 := func(msg []byte) { - generator.AddMacs(msg) - if !checker.CheckMAC1(msg) { - t.Fatal("MAC1 generation/verification failed") - } - if checker.CheckMAC2(msg, src) { - t.Fatal("MAC2 generation/verification failed") - } - } - - checkMAC1([]byte{ - 0x99, 0xbb, 0xa5, 0xfc, 0x99, 0xaa, 0x83, 0xbd, - 0x7b, 0x00, 0xc5, 0x9a, 0x4c, 0xb9, 0xcf, 0x62, - 0x40, 0x23, 0xf3, 0x8e, 0xd8, 0xd0, 0x62, 0x64, - 0x5d, 0xb2, 0x80, 0x13, 0xda, 0xce, 0xc6, 0x91, - 0x61, 0xd6, 0x30, 0xf1, 0x32, 0xb3, 0xa2, 0xf4, - 0x7b, 0x43, 0xb5, 0xa7, 0xe2, 0xb1, 0xf5, 0x6c, - 0x74, 0x6b, 0xb0, 0xcd, 0x1f, 0x94, 0x86, 0x7b, - 0xc8, 0xfb, 0x92, 0xed, 0x54, 0x9b, 0x44, 0xf5, - 0xc8, 0x7d, 0xb7, 0x8e, 0xff, 0x49, 0xc4, 0xe8, - 0x39, 0x7c, 0x19, 0xe0, 0x60, 0x19, 0x51, 0xf8, - 0xe4, 0x8e, 0x02, 0xf1, 0x7f, 0x1d, 0xcc, 0x8e, - 0xb0, 0x07, 0xff, 0xf8, 0xaf, 0x7f, 0x66, 0x82, - 0x83, 0xcc, 0x7c, 0xfa, 0x80, 0xdb, 0x81, 0x53, - 0xad, 0xf7, 0xd8, 0x0c, 0x10, 0xe0, 0x20, 0xfd, - 0xe8, 0x0b, 0x3f, 0x90, 0x15, 0xcd, 0x93, 0xad, - 0x0b, 0xd5, 0x0c, 0xcc, 0x88, 0x56, 0xe4, 0x3f, - }) - - checkMAC1([]byte{ - 0x33, 0xe7, 0x2a, 0x84, 0x9f, 0xff, 0x57, 0x6c, - 0x2d, 0xc3, 0x2d, 0xe1, 0xf5, 0x5c, 0x97, 0x56, - 0xb8, 0x93, 0xc2, 0x7d, 0xd4, 0x41, 0xdd, 0x7a, - 0x4a, 0x59, 0x3b, 0x50, 0xdd, 0x7a, 0x7a, 0x8c, - 0x9b, 0x96, 0xaf, 0x55, 0x3c, 0xeb, 0x6d, 0x0b, - 0x13, 0x0b, 0x97, 0x98, 0xb3, 0x40, 0xc3, 0xcc, - 0xb8, 0x57, 0x33, 0x45, 0x6e, 0x8b, 0x09, 0x2b, - 0x81, 0x2e, 0xd2, 0xb9, 0x66, 0x0b, 0x93, 0x05, - }) - - checkMAC1([]byte{ - 0x9b, 0x96, 0xaf, 0x55, 0x3c, 0xeb, 0x6d, 0x0b, - 0x13, 0x0b, 0x97, 0x98, 0xb3, 0x40, 0xc3, 0xcc, - 0xb8, 0x57, 0x33, 0x45, 0x6e, 0x8b, 0x09, 0x2b, - 0x81, 0x2e, 0xd2, 0xb9, 0x66, 0x0b, 0x93, 0x05, - }) - - // exchange cookie reply - - func() { - msg := []byte{ - 0x6d, 0xd7, 0xc3, 0x2e, 0xb0, 0x76, 0xd8, 0xdf, - 0x30, 0x65, 0x7d, 0x62, 0x3e, 0xf8, 0x9a, 0xe8, - 0xe7, 0x3c, 0x64, 0xa3, 0x78, 0x48, 0xda, 0xf5, - 0x25, 0x61, 0x28, 0x53, 0x79, 0x32, 0x86, 0x9f, - 0xa0, 0x27, 0x95, 0x69, 0xb6, 0xba, 0xd0, 0xa2, - 0xf8, 0x68, 0xea, 0xa8, 0x62, 0xf2, 0xfd, 0x1b, - 0xe0, 0xb4, 0x80, 0xe5, 0x6b, 0x3a, 0x16, 0x9e, - 0x35, 0xf6, 0xa8, 0xf2, 0x4f, 0x9a, 0x7b, 0xe9, - 0x77, 0x0b, 0xc2, 0xb4, 0xed, 0xba, 0xf9, 0x22, - 0xc3, 0x03, 0x97, 0x42, 0x9f, 0x79, 0x74, 0x27, - 0xfe, 0xf9, 0x06, 0x6e, 0x97, 0x3a, 0xa6, 0x8f, - 0xc9, 0x57, 0x0a, 0x54, 0x4c, 0x64, 0x4a, 0xe2, - 0x4f, 0xa1, 0xce, 0x95, 0x9b, 0x23, 0xa9, 0x2b, - 0x85, 0x93, 0x42, 0xb0, 0xa5, 0x53, 0xed, 0xeb, - 0x63, 0x2a, 0xf1, 0x6d, 0x46, 0xcb, 0x2f, 0x61, - 0x8c, 0xe1, 0xe8, 0xfa, 0x67, 0x20, 0x80, 0x6d, - } - generator.AddMacs(msg) - reply, err := checker.CreateReply(msg, 1377, src) - if err != nil { - t.Fatal("Failed to create cookie reply:", err) - } - if !generator.ConsumeReply(reply) { - t.Fatal("Failed to consume cookie reply") - } - }() - - // check mac2 - - checkMAC2 := func(msg []byte) { - generator.AddMacs(msg) - - if !checker.CheckMAC1(msg) { - t.Fatal("MAC1 generation/verification failed") - } - if !checker.CheckMAC2(msg, src) { - t.Fatal("MAC2 generation/verification failed") - } - - msg[5] ^= 0x20 - - if checker.CheckMAC1(msg) { - t.Fatal("MAC1 generation/verification failed") - } - if checker.CheckMAC2(msg, src) { - t.Fatal("MAC2 generation/verification failed") - } - - msg[5] ^= 0x20 - - srcBad1 := []byte{192, 168, 13, 37, 40, 1} - if checker.CheckMAC2(msg, srcBad1) { - t.Fatal("MAC2 generation/verification failed") - } - - srcBad2 := []byte{192, 168, 13, 38, 40, 1} - if checker.CheckMAC2(msg, srcBad2) { - t.Fatal("MAC2 generation/verification failed") - } - } - - checkMAC2([]byte{ - 0x03, 0x31, 0xb9, 0x9e, 0xb0, 0x2a, 0x54, 0xa3, - 0xc1, 0x3f, 0xb4, 0x96, 0x16, 0xb9, 0x25, 0x15, - 0x3d, 0x3a, 0x82, 0xf9, 0x58, 0x36, 0x86, 0x3f, - 0x13, 0x2f, 0xfe, 0xb2, 0x53, 0x20, 0x8c, 0x3f, - 0xba, 0xeb, 0xfb, 0x4b, 0x1b, 0x22, 0x02, 0x69, - 0x2c, 0x90, 0xbc, 0xdc, 0xcf, 0xcf, 0x85, 0xeb, - 0x62, 0x66, 0x6f, 0xe8, 0xe1, 0xa6, 0xa8, 0x4c, - 0xa0, 0x04, 0x23, 0x15, 0x42, 0xac, 0xfa, 0x38, - }) - - checkMAC2([]byte{ - 0x0e, 0x2f, 0x0e, 0xa9, 0x29, 0x03, 0xe1, 0xf3, - 0x24, 0x01, 0x75, 0xad, 0x16, 0xa5, 0x66, 0x85, - 0xca, 0x66, 0xe0, 0xbd, 0xc6, 0x34, 0xd8, 0x84, - 0x09, 0x9a, 0x58, 0x14, 0xfb, 0x05, 0xda, 0xf5, - 0x90, 0xf5, 0x0c, 0x4e, 0x22, 0x10, 0xc9, 0x85, - 0x0f, 0xe3, 0x77, 0x35, 0xe9, 0x6b, 0xc2, 0x55, - 0x32, 0x46, 0xae, 0x25, 0xe0, 0xe3, 0x37, 0x7a, - 0x4b, 0x71, 0xcc, 0xfc, 0x91, 0xdf, 0xd6, 0xca, - 0xfe, 0xee, 0xce, 0x3f, 0x77, 0xa2, 0xfd, 0x59, - 0x8e, 0x73, 0x0a, 0x8d, 0x5c, 0x24, 0x14, 0xca, - 0x38, 0x91, 0xb8, 0x2c, 0x8c, 0xa2, 0x65, 0x7b, - 0xbc, 0x49, 0xbc, 0xb5, 0x58, 0xfc, 0xe3, 0xd7, - 0x02, 0xcf, 0xf7, 0x4c, 0x60, 0x91, 0xed, 0x55, - 0xe9, 0xf9, 0xfe, 0xd1, 0x44, 0x2c, 0x75, 0xf2, - 0xb3, 0x5d, 0x7b, 0x27, 0x56, 0xc0, 0x48, 0x4f, - 0xb0, 0xba, 0xe4, 0x7d, 0xd0, 0xaa, 0xcd, 0x3d, - 0xe3, 0x50, 0xd2, 0xcf, 0xb9, 0xfa, 0x4b, 0x2d, - 0xc6, 0xdf, 0x3b, 0x32, 0x98, 0x45, 0xe6, 0x8f, - 0x1c, 0x5c, 0xa2, 0x20, 0x7d, 0x1c, 0x28, 0xc2, - 0xd4, 0xa1, 0xe0, 0x21, 0x52, 0x8f, 0x1c, 0xd0, - 0x62, 0x97, 0x48, 0xbb, 0xf4, 0xa9, 0xcb, 0x35, - 0xf2, 0x07, 0xd3, 0x50, 0xd8, 0xa9, 0xc5, 0x9a, - 0x0f, 0xbd, 0x37, 0xaf, 0xe1, 0x45, 0x19, 0xee, - 0x41, 0xf3, 0xf7, 0xe5, 0xe0, 0x30, 0x3f, 0xbe, - 0x3d, 0x39, 0x64, 0x00, 0x7a, 0x1a, 0x51, 0x5e, - 0xe1, 0x70, 0x0b, 0xb9, 0x77, 0x5a, 0xf0, 0xc4, - 0x8a, 0xa1, 0x3a, 0x77, 0x1a, 0xe0, 0xc2, 0x06, - 0x91, 0xd5, 0xe9, 0x1c, 0xd3, 0xfe, 0xab, 0x93, - 0x1a, 0x0a, 0x4c, 0xbb, 0xf0, 0xff, 0xdc, 0xaa, - 0x61, 0x73, 0xcb, 0x03, 0x4b, 0x71, 0x68, 0x64, - 0x3d, 0x82, 0x31, 0x41, 0xd7, 0x8b, 0x22, 0x7b, - 0x7d, 0xa1, 0xd5, 0x85, 0x6d, 0xf0, 0x1b, 0xaa, - }) -} diff --git a/device/device_test.go b/device/device_test.go deleted file mode 100644 index e443421..0000000 --- a/device/device_test.go +++ /dev/null @@ -1,476 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -import ( - "bytes" - "encoding/hex" - "fmt" - "io" - "math/rand" - "net/netip" - "os" - "runtime" - "runtime/pprof" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/conn/bindtest" - "github.com/tailscale/wireguard-go/tun" - "github.com/tailscale/wireguard-go/tun/tuntest" -) - -// uapiCfg returns a string that contains cfg formatted use with IpcSet. -// cfg is a series of alternating key/value strings. -// uapiCfg exists because editors and humans like to insert -// whitespace into configs, which can cause failures, some of which are silent. -// For example, a leading blank newline causes the remainder -// of the config to be silently ignored. -func uapiCfg(cfg ...string) string { - if len(cfg)%2 != 0 { - panic("odd number of args to uapiReader") - } - buf := new(bytes.Buffer) - for i, s := range cfg { - buf.WriteString(s) - sep := byte('\n') - if i%2 == 0 { - sep = '=' - } - buf.WriteByte(sep) - } - return buf.String() -} - -// genConfigs generates a pair of configs that connect to each other. -// The configs use distinct, probably-usable ports. -func genConfigs(tb testing.TB) (cfgs, endpointCfgs [2]string) { - var key1, key2 NoisePrivateKey - _, err := rand.Read(key1[:]) - if err != nil { - tb.Errorf("unable to generate private key random bytes: %v", err) - } - _, err = rand.Read(key2[:]) - if err != nil { - tb.Errorf("unable to generate private key random bytes: %v", err) - } - pub1, pub2 := key1.publicKey(), key2.publicKey() - - cfgs[0] = uapiCfg( - "private_key", hex.EncodeToString(key1[:]), - "listen_port", "0", - "replace_peers", "true", - "public_key", hex.EncodeToString(pub2[:]), - "protocol_version", "1", - "replace_allowed_ips", "true", - "allowed_ip", "1.0.0.2/32", - ) - endpointCfgs[0] = uapiCfg( - "public_key", hex.EncodeToString(pub2[:]), - "endpoint", "127.0.0.1:%d", - ) - cfgs[1] = uapiCfg( - "private_key", hex.EncodeToString(key2[:]), - "listen_port", "0", - "replace_peers", "true", - "public_key", hex.EncodeToString(pub1[:]), - "protocol_version", "1", - "replace_allowed_ips", "true", - "allowed_ip", "1.0.0.1/32", - ) - endpointCfgs[1] = uapiCfg( - "public_key", hex.EncodeToString(pub1[:]), - "endpoint", "127.0.0.1:%d", - ) - return -} - -// A testPair is a pair of testPeers. -type testPair [2]testPeer - -// A testPeer is a peer used for testing. -type testPeer struct { - tun *tuntest.ChannelTUN - dev *Device - ip netip.Addr -} - -type SendDirection bool - -const ( - Ping SendDirection = true - Pong SendDirection = false -) - -func (d SendDirection) String() string { - if d == Ping { - return "ping" - } - return "pong" -} - -func (pair *testPair) Send(tb testing.TB, ping SendDirection, done chan struct{}) { - tb.Helper() - p0, p1 := pair[0], pair[1] - if !ping { - // pong is the new ping - p0, p1 = p1, p0 - } - msg := tuntest.Ping(p0.ip, p1.ip) - p1.tun.Outbound <- msg - timer := time.NewTimer(5 * time.Second) - defer timer.Stop() - var err error - select { - case msgRecv := <-p0.tun.Inbound: - if !bytes.Equal(msg, msgRecv) { - err = fmt.Errorf("%s did not transit correctly", ping) - } - case <-timer.C: - err = fmt.Errorf("%s did not transit", ping) - case <-done: - } - if err != nil { - // The error may have occurred because the test is done. - select { - case <-done: - return - default: - } - // Real error. - tb.Error(err) - } -} - -// genTestPair creates a testPair. -func genTestPair(tb testing.TB, realSocket bool) (pair testPair) { - cfg, endpointCfg := genConfigs(tb) - var binds [2]conn.Bind - if realSocket { - binds[0], binds[1] = conn.NewDefaultBind(), conn.NewDefaultBind() - } else { - binds = bindtest.NewChannelBinds() - } - // Bring up a ChannelTun for each config. - for i := range pair { - p := &pair[i] - p.tun = tuntest.NewChannelTUN() - p.ip = netip.AddrFrom4([4]byte{1, 0, 0, byte(i + 1)}) - level := LogLevelVerbose - if _, ok := tb.(*testing.B); ok && !testing.Verbose() { - level = LogLevelError - } - p.dev = NewDevice(p.tun.TUN(), binds[i], NewLogger(level, fmt.Sprintf("dev%d: ", i))) - if err := p.dev.IpcSet(cfg[i]); err != nil { - tb.Errorf("failed to configure device %d: %v", i, err) - p.dev.Close() - continue - } - if err := p.dev.Up(); err != nil { - tb.Errorf("failed to bring up device %d: %v", i, err) - p.dev.Close() - continue - } - endpointCfg[i^1] = fmt.Sprintf(endpointCfg[i^1], p.dev.net.port) - } - for i := range pair { - p := &pair[i] - if err := p.dev.IpcSet(endpointCfg[i]); err != nil { - tb.Errorf("failed to configure device endpoint %d: %v", i, err) - p.dev.Close() - continue - } - // The device is ready. Close it when the test completes. - tb.Cleanup(p.dev.Close) - } - return -} - -func TestTwoDevicePing(t *testing.T) { - goroutineLeakCheck(t) - pair := genTestPair(t, true) - t.Run("ping 1.0.0.1", func(t *testing.T) { - pair.Send(t, Ping, nil) - }) - t.Run("ping 1.0.0.2", func(t *testing.T) { - pair.Send(t, Pong, nil) - }) -} - -func TestUpDown(t *testing.T) { - goroutineLeakCheck(t) - const itrials = 50 - const otrials = 10 - - for n := 0; n < otrials; n++ { - pair := genTestPair(t, false) - for i := range pair { - for k := range pair[i].dev.peers.keyMap { - pair[i].dev.IpcSet(fmt.Sprintf("public_key=%s\npersistent_keepalive_interval=1\n", hex.EncodeToString(k[:]))) - } - } - var wg sync.WaitGroup - wg.Add(len(pair)) - for i := range pair { - go func(d *Device) { - defer wg.Done() - for i := 0; i < itrials; i++ { - if err := d.Up(); err != nil { - t.Errorf("failed up bring up device: %v", err) - } - time.Sleep(time.Duration(rand.Intn(int(time.Nanosecond * (0x10000 - 1))))) - if err := d.Down(); err != nil { - t.Errorf("failed to bring down device: %v", err) - } - time.Sleep(time.Duration(rand.Intn(int(time.Nanosecond * (0x10000 - 1))))) - } - }(pair[i].dev) - } - wg.Wait() - for i := range pair { - pair[i].dev.Up() - pair[i].dev.Close() - } - } -} - -// TestConcurrencySafety does other things concurrently with tunnel use. -// It is intended to be used with the race detector to catch data races. -func TestConcurrencySafety(t *testing.T) { - pair := genTestPair(t, true) - done := make(chan struct{}) - - const warmupIters = 10 - var warmup sync.WaitGroup - warmup.Add(warmupIters) - go func() { - // Send data continuously back and forth until we're done. - // Note that we may continue to attempt to send data - // even after done is closed. - i := warmupIters - for ping := Ping; ; ping = !ping { - pair.Send(t, ping, done) - select { - case <-done: - return - default: - } - if i > 0 { - warmup.Done() - i-- - } - } - }() - warmup.Wait() - - applyCfg := func(cfg string) { - err := pair[0].dev.IpcSet(cfg) - if err != nil { - t.Fatal(err) - } - } - - // Change persistent_keepalive_interval concurrently with tunnel use. - t.Run("persistentKeepaliveInterval", func(t *testing.T) { - var pub NoisePublicKey - for key := range pair[0].dev.peers.keyMap { - pub = key - break - } - cfg := uapiCfg( - "public_key", hex.EncodeToString(pub[:]), - "persistent_keepalive_interval", "1", - ) - for i := 0; i < 1000; i++ { - applyCfg(cfg) - } - }) - - // Change private keys concurrently with tunnel use. - t.Run("privateKey", func(t *testing.T) { - bad := uapiCfg("private_key", "7777777777777777777777777777777777777777777777777777777777777777") - good := uapiCfg("private_key", hex.EncodeToString(pair[0].dev.staticIdentity.privateKey[:])) - // Set iters to a large number like 1000 to flush out data races quickly. - // Don't leave it large. That can cause logical races - // in which the handshake is interleaved with key changes - // such that the private key appears to be unchanging but - // other state gets reset, which can cause handshake failures like - // "Received packet with invalid mac1". - const iters = 1 - for i := 0; i < iters; i++ { - applyCfg(bad) - applyCfg(good) - } - }) - - // Perform bind updates and keepalive sends concurrently with tunnel use. - t.Run("bindUpdate and keepalive", func(t *testing.T) { - const iters = 10 - for i := 0; i < iters; i++ { - for _, peer := range pair { - peer.dev.BindUpdate() - peer.dev.SendKeepalivesToPeersWithCurrentKeypair() - } - } - }) - - close(done) -} - -func BenchmarkLatency(b *testing.B) { - pair := genTestPair(b, true) - - // Establish a connection. - pair.Send(b, Ping, nil) - pair.Send(b, Pong, nil) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - pair.Send(b, Ping, nil) - pair.Send(b, Pong, nil) - } -} - -func BenchmarkThroughput(b *testing.B) { - pair := genTestPair(b, true) - - // Establish a connection. - pair.Send(b, Ping, nil) - pair.Send(b, Pong, nil) - - // Measure how long it takes to receive b.N packets, - // starting when we receive the first packet. - var recv atomic.Uint64 - var elapsed time.Duration - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - var start time.Time - for { - <-pair[0].tun.Inbound - new := recv.Add(1) - if new == 1 { - start = time.Now() - } - // Careful! Don't change this to else if; b.N can be equal to 1. - if new == uint64(b.N) { - elapsed = time.Since(start) - return - } - } - }() - - // Send packets as fast as we can until we've received enough. - ping := tuntest.Ping(pair[0].ip, pair[1].ip) - pingc := pair[1].tun.Outbound - var sent uint64 - for recv.Load() != uint64(b.N) { - sent++ - pingc <- ping - } - wg.Wait() - - b.ReportMetric(float64(elapsed)/float64(b.N), "ns/op") - b.ReportMetric(1-float64(b.N)/float64(sent), "packet-loss") -} - -func BenchmarkUAPIGet(b *testing.B) { - pair := genTestPair(b, true) - pair.Send(b, Ping, nil) - pair.Send(b, Pong, nil) - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - pair[0].dev.IpcGetOperation(io.Discard) - } -} - -func goroutineLeakCheck(t *testing.T) { - goroutines := func() (int, []byte) { - p := pprof.Lookup("goroutine") - b := new(bytes.Buffer) - p.WriteTo(b, 1) - return p.Count(), b.Bytes() - } - - startGoroutines, startStacks := goroutines() - t.Cleanup(func() { - if t.Failed() { - return - } - // Give goroutines time to exit, if they need it. - for i := 0; i < 10000; i++ { - if runtime.NumGoroutine() <= startGoroutines { - return - } - time.Sleep(1 * time.Millisecond) - } - endGoroutines, endStacks := goroutines() - t.Logf("starting stacks:\n%s\n", startStacks) - t.Logf("ending stacks:\n%s\n", endStacks) - t.Fatalf("expected %d goroutines, got %d, leak?", startGoroutines, endGoroutines) - }) -} - -type fakeBindSized struct { - size int -} - -func (b *fakeBindSized) Open(port uint16) (fns []conn.ReceiveFunc, actualPort uint16, err error) { - return nil, 0, nil -} -func (b *fakeBindSized) Close() error { return nil } -func (b *fakeBindSized) SetMark(mark uint32) error { return nil } -func (b *fakeBindSized) Send(bufs [][]byte, ep conn.Endpoint, offset int) error { return nil } -func (b *fakeBindSized) ParseEndpoint(s string) (conn.Endpoint, error) { return nil, nil } -func (b *fakeBindSized) BatchSize() int { return b.size } - -type fakeTUNDeviceSized struct { - size int -} - -func (t *fakeTUNDeviceSized) File() *os.File { return nil } -func (t *fakeTUNDeviceSized) Read(bufs [][]byte, sizes []int, offset int) (n int, err error) { - return 0, nil -} -func (t *fakeTUNDeviceSized) Write(bufs [][]byte, offset int) (int, error) { return 0, nil } -func (t *fakeTUNDeviceSized) MTU() (int, error) { return 0, nil } -func (t *fakeTUNDeviceSized) Name() (string, error) { return "", nil } -func (t *fakeTUNDeviceSized) Events() <-chan tun.Event { return nil } -func (t *fakeTUNDeviceSized) Close() error { return nil } -func (t *fakeTUNDeviceSized) BatchSize() int { return t.size } - -func TestBatchSize(t *testing.T) { - d := Device{} - - d.net.bind = &fakeBindSized{1} - d.tun.device = &fakeTUNDeviceSized{1} - if want, got := 1, d.BatchSize(); got != want { - t.Errorf("expected batch size %d, got %d", want, got) - } - - d.net.bind = &fakeBindSized{1} - d.tun.device = &fakeTUNDeviceSized{128} - if want, got := 128, d.BatchSize(); got != want { - t.Errorf("expected batch size %d, got %d", want, got) - } - - d.net.bind = &fakeBindSized{128} - d.tun.device = &fakeTUNDeviceSized{1} - if want, got := 128, d.BatchSize(); got != want { - t.Errorf("expected batch size %d, got %d", want, got) - } - - d.net.bind = &fakeBindSized{128} - d.tun.device = &fakeTUNDeviceSized{128} - if want, got := 128, d.BatchSize(); got != want { - t.Errorf("expected batch size %d, got %d", want, got) - } -} diff --git a/device/endpoint_test.go b/device/endpoint_test.go deleted file mode 100644 index 93a4998..0000000 --- a/device/endpoint_test.go +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -import ( - "math/rand" - "net/netip" -) - -type DummyEndpoint struct { - src, dst netip.Addr -} - -func CreateDummyEndpoint() (*DummyEndpoint, error) { - var src, dst [16]byte - if _, err := rand.Read(src[:]); err != nil { - return nil, err - } - _, err := rand.Read(dst[:]) - return &DummyEndpoint{netip.AddrFrom16(src), netip.AddrFrom16(dst)}, err -} - -func (e *DummyEndpoint) ClearSrc() {} - -func (e *DummyEndpoint) SrcToString() string { - return netip.AddrPortFrom(e.SrcIP(), 1000).String() -} - -func (e *DummyEndpoint) DstToString() string { - return netip.AddrPortFrom(e.DstIP(), 1000).String() -} - -func (e *DummyEndpoint) DstToBytes() []byte { - out := e.DstIP().AsSlice() - out = append(out, byte(1000&0xff)) - out = append(out, byte((1000>>8)&0xff)) - return out -} - -func (e *DummyEndpoint) DstIP() netip.Addr { - return e.dst -} - -func (e *DummyEndpoint) SrcIP() netip.Addr { - return e.src -} diff --git a/device/kdf_test.go b/device/kdf_test.go deleted file mode 100644 index f9c76d6..0000000 --- a/device/kdf_test.go +++ /dev/null @@ -1,85 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -import ( - "encoding/hex" - "testing" - - "golang.org/x/crypto/blake2s" -) - -type KDFTest struct { - key string - input string - t0 string - t1 string - t2 string -} - -func assertEquals(t *testing.T, a, b string) { - if a != b { - t.Fatal("expected", a, "=", b) - } -} - -func TestKDF(t *testing.T) { - tests := []KDFTest{ - { - key: "746573742d6b6579", - input: "746573742d696e707574", - t0: "6f0e5ad38daba1bea8a0d213688736f19763239305e0f58aba697f9ffc41c633", - t1: "df1194df20802a4fe594cde27e92991c8cae66c366e8106aaa937a55fa371e8a", - t2: "fac6e2745a325f5dc5d11a5b165aad08b0ada28e7b4e666b7c077934a4d76c24", - }, - { - key: "776972656775617264", - input: "776972656775617264", - t0: "491d43bbfdaa8750aaf535e334ecbfe5129967cd64635101c566d4caefda96e8", - t1: "1e71a379baefd8a79aa4662212fcafe19a23e2b609a3db7d6bcba8f560e3d25f", - t2: "31e1ae48bddfbe5de38f295e5452b1909a1b4e38e183926af3780b0c1e1f0160", - }, - { - key: "", - input: "", - t0: "8387b46bf43eccfcf349552a095d8315c4055beb90208fb1be23b894bc2ed5d0", - t1: "58a0e5f6faefccf4807bff1f05fa8a9217945762040bcec2f4b4a62bdfe0e86e", - t2: "0ce6ea98ec548f8e281e93e32db65621c45eb18dc6f0a7ad94178610a2f7338e", - }, - } - - var t0, t1, t2 [blake2s.Size]byte - - for _, test := range tests { - key, _ := hex.DecodeString(test.key) - input, _ := hex.DecodeString(test.input) - KDF3(&t0, &t1, &t2, key, input) - t0s := hex.EncodeToString(t0[:]) - t1s := hex.EncodeToString(t1[:]) - t2s := hex.EncodeToString(t2[:]) - assertEquals(t, t0s, test.t0) - assertEquals(t, t1s, test.t1) - assertEquals(t, t2s, test.t2) - } - - for _, test := range tests { - key, _ := hex.DecodeString(test.key) - input, _ := hex.DecodeString(test.input) - KDF2(&t0, &t1, key, input) - t0s := hex.EncodeToString(t0[:]) - t1s := hex.EncodeToString(t1[:]) - assertEquals(t, t0s, test.t0) - assertEquals(t, t1s, test.t1) - } - - for _, test := range tests { - key, _ := hex.DecodeString(test.key) - input, _ := hex.DecodeString(test.input) - KDF1(&t0, key, input) - t0s := hex.EncodeToString(t0[:]) - assertEquals(t, t0s, test.t0) - } -} diff --git a/device/noise_test.go b/device/noise_test.go deleted file mode 100644 index 160bee5..0000000 --- a/device/noise_test.go +++ /dev/null @@ -1,207 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -import ( - "bytes" - "encoding/binary" - "net/netip" - "testing" - - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/tun/tuntest" -) - -func TestCurveWrappers(t *testing.T) { - sk1, err := newPrivateKey() - assertNil(t, err) - - sk2, err := newPrivateKey() - assertNil(t, err) - - pk1 := sk1.publicKey() - pk2 := sk2.publicKey() - - ss1, err1 := sk1.sharedSecret(pk2) - ss2, err2 := sk2.sharedSecret(pk1) - - if ss1 != ss2 || err1 != nil || err2 != nil { - t.Fatal("Failed to compute shared secet") - } -} - -func randDevice(t *testing.T) *Device { - sk, err := newPrivateKey() - if err != nil { - t.Fatal(err) - } - tun := tuntest.NewChannelTUN() - logger := NewLogger(LogLevelError, "") - device := NewDevice(tun.TUN(), conn.NewDefaultBind(), logger) - device.SetPrivateKey(sk) - return device -} - -func assertNil(t *testing.T, err error) { - if err != nil { - t.Fatal(err) - } -} - -func assertEqual(t *testing.T, a, b []byte) { - if !bytes.Equal(a, b) { - t.Fatal(a, "!=", b) - } -} - -type initAwareEP struct { - calledWith *[32]byte -} - -var _ conn.Endpoint = (*initAwareEP)(nil) -var _ conn.InitiationAwareEndpoint = (*initAwareEP)(nil) - -func (i *initAwareEP) ClearSrc() {} -func (i *initAwareEP) SrcToString() string { return "" } -func (i *initAwareEP) DstToString() string { return "" } -func (i *initAwareEP) DstToBytes() []byte { return nil } -func (i *initAwareEP) DstIP() netip.Addr { return netip.Addr{} } -func (i *initAwareEP) SrcIP() netip.Addr { return netip.Addr{} } - -func (i *initAwareEP) InitiationMessagePublicKey(peerPublicKey [32]byte) { - calledWith := [32]byte{} - copy(calledWith[:], peerPublicKey[:]) - i.calledWith = &calledWith -} - -func TestNoiseHandshake(t *testing.T) { - dev1 := randDevice(t) - dev2 := randDevice(t) - - defer dev1.Close() - defer dev2.Close() - - peer1, err := dev2.NewPeer(dev1.staticIdentity.privateKey.publicKey()) - if err != nil { - t.Fatal(err) - } - peer2, err := dev1.NewPeer(dev2.staticIdentity.privateKey.publicKey()) - if err != nil { - t.Fatal(err) - } - peer1.Start() - peer2.Start() - - assertEqual( - t, - peer1.handshake.precomputedStaticStatic[:], - peer2.handshake.precomputedStaticStatic[:], - ) - - /* simulate handshake */ - - // initiation message - - t.Log("exchange initiation message") - - msg1, err := dev1.CreateMessageInitiation(peer2) - assertNil(t, err) - - packet := make([]byte, 0, 256) - writer := bytes.NewBuffer(packet) - err = binary.Write(writer, binary.LittleEndian, msg1) - assertNil(t, err) - initEP := &initAwareEP{} - peer := dev2.ConsumeMessageInitiation(msg1, initEP) - if peer == nil { - t.Fatal("handshake failed at initiation message") - } - if initEP.calledWith == nil { - t.Fatal("initAwareEP never called") - } - if *initEP.calledWith != dev1.staticIdentity.publicKey { - t.Fatal("initAwareEP called with unexpected public key") - } - - assertEqual( - t, - peer1.handshake.chainKey[:], - peer2.handshake.chainKey[:], - ) - - assertEqual( - t, - peer1.handshake.hash[:], - peer2.handshake.hash[:], - ) - - // response message - - t.Log("exchange response message") - - msg2, err := dev2.CreateMessageResponse(peer1) - assertNil(t, err) - - peer = dev1.ConsumeMessageResponse(msg2) - if peer == nil { - t.Fatal("handshake failed at response message") - } - - assertEqual( - t, - peer1.handshake.chainKey[:], - peer2.handshake.chainKey[:], - ) - - assertEqual( - t, - peer1.handshake.hash[:], - peer2.handshake.hash[:], - ) - - // key pairs - - t.Log("deriving keys") - - err = peer1.BeginSymmetricSession() - if err != nil { - t.Fatal("failed to derive keypair for peer 1", err) - } - - err = peer2.BeginSymmetricSession() - if err != nil { - t.Fatal("failed to derive keypair for peer 2", err) - } - - key1 := peer1.keypairs.next.Load() - key2 := peer2.keypairs.current - - // encrypting / decryption test - - t.Log("test key pairs") - - func() { - testMsg := []byte("wireguard test message 1") - var err error - var out []byte - var nonce [12]byte - out = key1.send.Seal(out, nonce[:], testMsg, nil) - out, err = key2.receive.Open(out[:0], nonce[:], out, nil) - assertNil(t, err) - assertEqual(t, out, testMsg) - }() - - func() { - testMsg := []byte("wireguard test message 2") - var err error - var out []byte - var nonce [12]byte - out = key2.send.Seal(out, nonce[:], testMsg, nil) - out, err = key1.receive.Open(out[:0], nonce[:], out, nil) - assertNil(t, err) - assertEqual(t, out, testMsg) - }() -} diff --git a/device/pools_test.go b/device/pools_test.go deleted file mode 100644 index 2b16f39..0000000 --- a/device/pools_test.go +++ /dev/null @@ -1,140 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -import ( - "math/rand" - "runtime" - "sync" - "sync/atomic" - "testing" - "time" -) - -func TestWaitPool(t *testing.T) { - var wg sync.WaitGroup - var trials atomic.Int32 - startTrials := int32(100000) - if raceEnabled { - // This test can be very slow with -race. - startTrials /= 10 - } - trials.Store(startTrials) - workers := runtime.NumCPU() + 2 - if workers-4 <= 0 { - t.Skip("Not enough cores") - } - p := NewWaitPool(uint32(workers-4), func() any { return make([]byte, 16) }) - wg.Add(workers) - var max atomic.Uint32 - updateMax := func() { - p.lock.Lock() - count := p.count - p.lock.Unlock() - if count > p.max { - t.Errorf("count (%d) > max (%d)", count, p.max) - } - for { - old := max.Load() - if count <= old { - break - } - if max.CompareAndSwap(old, count) { - break - } - } - } - for i := 0; i < workers; i++ { - go func() { - defer wg.Done() - for trials.Add(-1) > 0 { - updateMax() - x := p.Get() - updateMax() - time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) - updateMax() - p.Put(x) - updateMax() - } - }() - } - wg.Wait() - if max.Load() != p.max { - t.Errorf("Actual maximum count (%d) != ideal maximum count (%d)", max, p.max) - } -} - -func BenchmarkWaitPool(b *testing.B) { - var wg sync.WaitGroup - var trials atomic.Int32 - trials.Store(int32(b.N)) - workers := runtime.NumCPU() + 2 - if workers-4 <= 0 { - b.Skip("Not enough cores") - } - p := NewWaitPool(uint32(workers-4), func() any { return make([]byte, 16) }) - wg.Add(workers) - b.ResetTimer() - for i := 0; i < workers; i++ { - go func() { - defer wg.Done() - for trials.Add(-1) > 0 { - x := p.Get() - time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) - p.Put(x) - } - }() - } - wg.Wait() -} - -func BenchmarkWaitPoolEmpty(b *testing.B) { - var wg sync.WaitGroup - var trials atomic.Int32 - trials.Store(int32(b.N)) - workers := runtime.NumCPU() + 2 - if workers-4 <= 0 { - b.Skip("Not enough cores") - } - p := NewWaitPool(0, func() any { return make([]byte, 16) }) - wg.Add(workers) - b.ResetTimer() - for i := 0; i < workers; i++ { - go func() { - defer wg.Done() - for trials.Add(-1) > 0 { - x := p.Get() - time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) - p.Put(x) - } - }() - } - wg.Wait() -} - -func BenchmarkSyncPool(b *testing.B) { - var wg sync.WaitGroup - var trials atomic.Int32 - trials.Store(int32(b.N)) - workers := runtime.NumCPU() + 2 - if workers-4 <= 0 { - b.Skip("Not enough cores") - } - p := sync.Pool{New: func() any { return make([]byte, 16) }} - wg.Add(workers) - b.ResetTimer() - for i := 0; i < workers; i++ { - go func() { - defer wg.Done() - for trials.Add(-1) > 0 { - x := p.Get() - time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) - p.Put(x) - } - }() - } - wg.Wait() -} diff --git a/device/race_disabled_test.go b/device/race_disabled_test.go deleted file mode 100644 index bb5c450..0000000 --- a/device/race_disabled_test.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !race - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -const raceEnabled = false diff --git a/device/race_enabled_test.go b/device/race_enabled_test.go deleted file mode 100644 index 4e9daea..0000000 --- a/device/race_enabled_test.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build race - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package device - -const raceEnabled = true diff --git a/format_test.go b/format_test.go deleted file mode 100644 index 6f6cab7..0000000 --- a/format_test.go +++ /dev/null @@ -1,51 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ -package main - -import ( - "bytes" - "go/format" - "io/fs" - "os" - "path/filepath" - "runtime" - "sync" - "testing" -) - -func TestFormatting(t *testing.T) { - var wg sync.WaitGroup - filepath.WalkDir(".", func(path string, d fs.DirEntry, err error) error { - if err != nil { - t.Errorf("unable to walk %s: %v", path, err) - return nil - } - if d.IsDir() || filepath.Ext(path) != ".go" { - return nil - } - wg.Add(1) - go func(path string) { - defer wg.Done() - src, err := os.ReadFile(path) - if err != nil { - t.Errorf("unable to read %s: %v", path, err) - return - } - if runtime.GOOS == "windows" { - src = bytes.ReplaceAll(src, []byte{'\r', '\n'}, []byte{'\n'}) - } - formatted, err := format.Source(src) - if err != nil { - t.Errorf("unable to format %s: %v", path, err) - return - } - if !bytes.Equal(src, formatted) { - t.Errorf("unformatted code: %s", path) - } - }(path) - return nil - }) - wg.Wait() -} diff --git a/go.mod b/go.mod index 9c9b02a..7476734 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,4 @@ require ( golang.org/x/net v0.15.0 golang.org/x/sys v0.12.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 - gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 -) - -require ( - github.com/google/btree v1.0.1 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect ) diff --git a/go.sum b/go.sum index 6bcecea..ec4169f 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,8 @@ -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= -gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 h1:TbRPT0HtzFP3Cno1zZo7yPzEEnfu8EjLfl6IU9VfqkQ= -gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259/go.mod h1:AVgIgHMwK63XvmAzWG9vLQ41YnVHN0du0tEC46fI7yY= diff --git a/ipc/namedpipe/namedpipe_test.go b/ipc/namedpipe/namedpipe_test.go deleted file mode 100644 index de7d0f6..0000000 --- a/ipc/namedpipe/namedpipe_test.go +++ /dev/null @@ -1,674 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Copyright 2015 Microsoft -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build windows - -package namedpipe_test - -import ( - "bufio" - "bytes" - "context" - "errors" - "io" - "net" - "os" - "sync" - "syscall" - "testing" - "time" - - "github.com/tailscale/wireguard-go/ipc/namedpipe" - "golang.org/x/sys/windows" -) - -func randomPipePath() string { - guid, err := windows.GenerateGUID() - if err != nil { - panic(err) - } - return `\\.\PIPE\go-namedpipe-test-` + guid.String() -} - -func TestPingPong(t *testing.T) { - const ( - ping = 42 - pong = 24 - ) - pipePath := randomPipePath() - listener, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatalf("unable to listen on pipe: %v", err) - } - defer listener.Close() - go func() { - incoming, err := listener.Accept() - if err != nil { - t.Fatalf("unable to accept pipe connection: %v", err) - } - defer incoming.Close() - var data [1]byte - _, err = incoming.Read(data[:]) - if err != nil { - t.Fatalf("unable to read ping from pipe: %v", err) - } - if data[0] != ping { - t.Fatalf("expected ping, got %d", data[0]) - } - data[0] = pong - _, err = incoming.Write(data[:]) - if err != nil { - t.Fatalf("unable to write pong to pipe: %v", err) - } - }() - client, err := namedpipe.DialTimeout(pipePath, time.Duration(0)) - if err != nil { - t.Fatalf("unable to dial pipe: %v", err) - } - defer client.Close() - client.SetDeadline(time.Now().Add(time.Second * 5)) - var data [1]byte - data[0] = ping - _, err = client.Write(data[:]) - if err != nil { - t.Fatalf("unable to write ping to pipe: %v", err) - } - _, err = client.Read(data[:]) - if err != nil { - t.Fatalf("unable to read pong from pipe: %v", err) - } - if data[0] != pong { - t.Fatalf("expected pong, got %d", data[0]) - } -} - -func TestDialUnknownFailsImmediately(t *testing.T) { - _, err := namedpipe.DialTimeout(randomPipePath(), time.Duration(0)) - if !errors.Is(err, syscall.ENOENT) { - t.Fatalf("expected ENOENT got %v", err) - } -} - -func TestDialListenerTimesOut(t *testing.T) { - pipePath := randomPipePath() - l, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - pipe, err := namedpipe.DialTimeout(pipePath, 10*time.Millisecond) - if err == nil { - pipe.Close() - } - if err != os.ErrDeadlineExceeded { - t.Fatalf("expected os.ErrDeadlineExceeded, got %v", err) - } -} - -func TestDialContextListenerTimesOut(t *testing.T) { - pipePath := randomPipePath() - l, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - d := 10 * time.Millisecond - ctx, _ := context.WithTimeout(context.Background(), d) - pipe, err := namedpipe.DialContext(ctx, pipePath) - if err == nil { - pipe.Close() - } - if err != context.DeadlineExceeded { - t.Fatalf("expected context.DeadlineExceeded, got %v", err) - } -} - -func TestDialListenerGetsCancelled(t *testing.T) { - pipePath := randomPipePath() - ctx, cancel := context.WithCancel(context.Background()) - l, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - ch := make(chan error) - go func(ctx context.Context, ch chan error) { - _, err := namedpipe.DialContext(ctx, pipePath) - ch <- err - }(ctx, ch) - time.Sleep(time.Millisecond * 30) - cancel() - err = <-ch - if err != context.Canceled { - t.Fatalf("expected context.Canceled, got %v", err) - } -} - -func TestDialAccessDeniedWithRestrictedSD(t *testing.T) { - if windows.NewLazySystemDLL("ntdll.dll").NewProc("wine_get_version").Find() == nil { - t.Skip("dacls on named pipes are broken on wine") - } - pipePath := randomPipePath() - sd, _ := windows.SecurityDescriptorFromString("D:") - l, err := (&namedpipe.ListenConfig{ - SecurityDescriptor: sd, - }).Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - pipe, err := namedpipe.DialTimeout(pipePath, time.Duration(0)) - if err == nil { - pipe.Close() - } - if !errors.Is(err, windows.ERROR_ACCESS_DENIED) { - t.Fatalf("expected ERROR_ACCESS_DENIED, got %v", err) - } -} - -func getConnection(cfg *namedpipe.ListenConfig) (client, server net.Conn, err error) { - pipePath := randomPipePath() - if cfg == nil { - cfg = &namedpipe.ListenConfig{} - } - l, err := cfg.Listen(pipePath) - if err != nil { - return - } - defer l.Close() - - type response struct { - c net.Conn - err error - } - ch := make(chan response) - go func() { - c, err := l.Accept() - ch <- response{c, err} - }() - - c, err := namedpipe.DialTimeout(pipePath, time.Duration(0)) - if err != nil { - return - } - - r := <-ch - if err = r.err; err != nil { - c.Close() - return - } - - client = c - server = r.c - return -} - -func TestReadTimeout(t *testing.T) { - c, s, err := getConnection(nil) - if err != nil { - t.Fatal(err) - } - defer c.Close() - defer s.Close() - - c.SetReadDeadline(time.Now().Add(10 * time.Millisecond)) - - buf := make([]byte, 10) - _, err = c.Read(buf) - if err != os.ErrDeadlineExceeded { - t.Fatalf("expected os.ErrDeadlineExceeded, got %v", err) - } -} - -func server(l net.Listener, ch chan int) { - c, err := l.Accept() - if err != nil { - panic(err) - } - rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c)) - s, err := rw.ReadString('\n') - if err != nil { - panic(err) - } - _, err = rw.WriteString("got " + s) - if err != nil { - panic(err) - } - err = rw.Flush() - if err != nil { - panic(err) - } - c.Close() - ch <- 1 -} - -func TestFullListenDialReadWrite(t *testing.T) { - pipePath := randomPipePath() - l, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - - ch := make(chan int) - go server(l, ch) - - c, err := namedpipe.DialTimeout(pipePath, time.Duration(0)) - if err != nil { - t.Fatal(err) - } - defer c.Close() - - rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c)) - _, err = rw.WriteString("hello world\n") - if err != nil { - t.Fatal(err) - } - err = rw.Flush() - if err != nil { - t.Fatal(err) - } - - s, err := rw.ReadString('\n') - if err != nil { - t.Fatal(err) - } - ms := "got hello world\n" - if s != ms { - t.Errorf("expected '%s', got '%s'", ms, s) - } - - <-ch -} - -func TestCloseAbortsListen(t *testing.T) { - pipePath := randomPipePath() - l, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatal(err) - } - - ch := make(chan error) - go func() { - _, err := l.Accept() - ch <- err - }() - - time.Sleep(30 * time.Millisecond) - l.Close() - - err = <-ch - if err != net.ErrClosed { - t.Fatalf("expected net.ErrClosed, got %v", err) - } -} - -func ensureEOFOnClose(t *testing.T, r io.Reader, w io.Closer) { - b := make([]byte, 10) - w.Close() - n, err := r.Read(b) - if n > 0 { - t.Errorf("unexpected byte count %d", n) - } - if err != io.EOF { - t.Errorf("expected EOF: %v", err) - } -} - -func TestCloseClientEOFServer(t *testing.T) { - c, s, err := getConnection(nil) - if err != nil { - t.Fatal(err) - } - defer c.Close() - defer s.Close() - ensureEOFOnClose(t, c, s) -} - -func TestCloseServerEOFClient(t *testing.T) { - c, s, err := getConnection(nil) - if err != nil { - t.Fatal(err) - } - defer c.Close() - defer s.Close() - ensureEOFOnClose(t, s, c) -} - -func TestCloseWriteEOF(t *testing.T) { - cfg := &namedpipe.ListenConfig{ - MessageMode: true, - } - c, s, err := getConnection(cfg) - if err != nil { - t.Fatal(err) - } - defer c.Close() - defer s.Close() - - type closeWriter interface { - CloseWrite() error - } - - err = c.(closeWriter).CloseWrite() - if err != nil { - t.Fatal(err) - } - - b := make([]byte, 10) - _, err = s.Read(b) - if err != io.EOF { - t.Fatal(err) - } -} - -func TestAcceptAfterCloseFails(t *testing.T) { - pipePath := randomPipePath() - l, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatal(err) - } - l.Close() - _, err = l.Accept() - if err != net.ErrClosed { - t.Fatalf("expected net.ErrClosed, got %v", err) - } -} - -func TestDialTimesOutByDefault(t *testing.T) { - pipePath := randomPipePath() - l, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - pipe, err := namedpipe.DialTimeout(pipePath, time.Duration(0)) // Should timeout after 2 seconds. - if err == nil { - pipe.Close() - } - if err != os.ErrDeadlineExceeded { - t.Fatalf("expected os.ErrDeadlineExceeded, got %v", err) - } -} - -func TestTimeoutPendingRead(t *testing.T) { - pipePath := randomPipePath() - l, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - - serverDone := make(chan struct{}) - - go func() { - s, err := l.Accept() - if err != nil { - t.Fatal(err) - } - time.Sleep(1 * time.Second) - s.Close() - close(serverDone) - }() - - client, err := namedpipe.DialTimeout(pipePath, time.Duration(0)) - if err != nil { - t.Fatal(err) - } - defer client.Close() - - clientErr := make(chan error) - go func() { - buf := make([]byte, 10) - _, err = client.Read(buf) - clientErr <- err - }() - - time.Sleep(100 * time.Millisecond) // make *sure* the pipe is reading before we set the deadline - client.SetReadDeadline(time.Unix(1, 0)) - - select { - case err = <-clientErr: - if err != os.ErrDeadlineExceeded { - t.Fatalf("expected os.ErrDeadlineExceeded, got %v", err) - } - case <-time.After(100 * time.Millisecond): - t.Fatalf("timed out while waiting for read to cancel") - <-clientErr - } - <-serverDone -} - -func TestTimeoutPendingWrite(t *testing.T) { - pipePath := randomPipePath() - l, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - - serverDone := make(chan struct{}) - - go func() { - s, err := l.Accept() - if err != nil { - t.Fatal(err) - } - time.Sleep(1 * time.Second) - s.Close() - close(serverDone) - }() - - client, err := namedpipe.DialTimeout(pipePath, time.Duration(0)) - if err != nil { - t.Fatal(err) - } - defer client.Close() - - clientErr := make(chan error) - go func() { - _, err = client.Write([]byte("this should timeout")) - clientErr <- err - }() - - time.Sleep(100 * time.Millisecond) // make *sure* the pipe is writing before we set the deadline - client.SetWriteDeadline(time.Unix(1, 0)) - - select { - case err = <-clientErr: - if err != os.ErrDeadlineExceeded { - t.Fatalf("expected os.ErrDeadlineExceeded, got %v", err) - } - case <-time.After(100 * time.Millisecond): - t.Fatalf("timed out while waiting for write to cancel") - <-clientErr - } - <-serverDone -} - -type CloseWriter interface { - CloseWrite() error -} - -func TestEchoWithMessaging(t *testing.T) { - pipePath := randomPipePath() - l, err := (&namedpipe.ListenConfig{ - MessageMode: true, // Use message mode so that CloseWrite() is supported - InputBufferSize: 65536, // Use 64KB buffers to improve performance - OutputBufferSize: 65536, - }).Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - - listenerDone := make(chan bool) - clientDone := make(chan bool) - go func() { - // server echo - conn, err := l.Accept() - if err != nil { - t.Fatal(err) - } - defer conn.Close() - - time.Sleep(500 * time.Millisecond) // make *sure* we don't begin to read before eof signal is sent - _, err = io.Copy(conn, conn) - if err != nil { - t.Fatal(err) - } - conn.(CloseWriter).CloseWrite() - close(listenerDone) - }() - client, err := namedpipe.DialTimeout(pipePath, time.Second) - if err != nil { - t.Fatal(err) - } - defer client.Close() - - go func() { - // client read back - bytes := make([]byte, 2) - n, e := client.Read(bytes) - if e != nil { - t.Fatal(e) - } - if n != 2 || bytes[0] != 0 || bytes[1] != 1 { - t.Fatalf("expected 2 bytes, got %v", n) - } - close(clientDone) - }() - - payload := make([]byte, 2) - payload[0] = 0 - payload[1] = 1 - - n, err := client.Write(payload) - if err != nil { - t.Fatal(err) - } - if n != 2 { - t.Fatalf("expected 2 bytes, got %v", n) - } - client.(CloseWriter).CloseWrite() - <-listenerDone - <-clientDone -} - -func TestConnectRace(t *testing.T) { - pipePath := randomPipePath() - l, err := namedpipe.Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - go func() { - for { - s, err := l.Accept() - if err == net.ErrClosed { - return - } - - if err != nil { - t.Fatal(err) - } - s.Close() - } - }() - - for i := 0; i < 1000; i++ { - c, err := namedpipe.DialTimeout(pipePath, time.Duration(0)) - if err != nil { - t.Fatal(err) - } - c.Close() - } -} - -func TestMessageReadMode(t *testing.T) { - if maj, _, _ := windows.RtlGetNtVersionNumbers(); maj <= 8 { - t.Skipf("Skipping on Windows %d", maj) - } - var wg sync.WaitGroup - defer wg.Wait() - pipePath := randomPipePath() - l, err := (&namedpipe.ListenConfig{MessageMode: true}).Listen(pipePath) - if err != nil { - t.Fatal(err) - } - defer l.Close() - - msg := ([]byte)("hello world") - - wg.Add(1) - go func() { - defer wg.Done() - s, err := l.Accept() - if err != nil { - t.Fatal(err) - } - _, err = s.Write(msg) - if err != nil { - t.Fatal(err) - } - s.Close() - }() - - c, err := namedpipe.DialTimeout(pipePath, time.Duration(0)) - if err != nil { - t.Fatal(err) - } - defer c.Close() - - mode := uint32(windows.PIPE_READMODE_MESSAGE) - err = windows.SetNamedPipeHandleState(c.(interface{ Handle() windows.Handle }).Handle(), &mode, nil, nil) - if err != nil { - t.Fatal(err) - } - - ch := make([]byte, 1) - var vmsg []byte - for { - n, err := c.Read(ch) - if err == io.EOF { - break - } - if err != nil { - t.Fatal(err) - } - if n != 1 { - t.Fatalf("expected 1, got %d", n) - } - vmsg = append(vmsg, ch[0]) - } - if !bytes.Equal(msg, vmsg) { - t.Fatalf("expected %s, got %s", msg, vmsg) - } -} - -func TestListenConnectRace(t *testing.T) { - if testing.Short() { - t.Skip("Skipping long race test") - } - pipePath := randomPipePath() - for i := 0; i < 50 && !t.Failed(); i++ { - var wg sync.WaitGroup - wg.Add(1) - go func() { - c, err := namedpipe.DialTimeout(pipePath, time.Duration(0)) - if err == nil { - c.Close() - } - wg.Done() - }() - s, err := namedpipe.Listen(pipePath) - if err != nil { - t.Error(i, err) - } else { - s.Close() - } - wg.Wait() - } -} diff --git a/main.go b/main.go deleted file mode 100644 index 55000e9..0000000 --- a/main.go +++ /dev/null @@ -1,268 +0,0 @@ -//go:build !windows - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package main - -import ( - "fmt" - "os" - "os/signal" - "runtime" - "strconv" - - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/device" - "github.com/tailscale/wireguard-go/ipc" - "github.com/tailscale/wireguard-go/tun" - "golang.org/x/sys/unix" -) - -const ( - ExitSetupSuccess = 0 - ExitSetupFailed = 1 -) - -const ( - ENV_WG_TUN_FD = "WG_TUN_FD" - ENV_WG_UAPI_FD = "WG_UAPI_FD" - ENV_WG_PROCESS_FOREGROUND = "WG_PROCESS_FOREGROUND" -) - -func printUsage() { - fmt.Printf("Usage: %s [-f/--foreground] INTERFACE-NAME\n", os.Args[0]) -} - -func warning() { - switch runtime.GOOS { - case "linux", "freebsd", "openbsd": - if os.Getenv(ENV_WG_PROCESS_FOREGROUND) == "1" { - return - } - default: - return - } - - fmt.Fprintln(os.Stderr, "┌──────────────────────────────────────────────────────┐") - fmt.Fprintln(os.Stderr, "│ │") - fmt.Fprintln(os.Stderr, "│ Running wireguard-go is not required because this │") - fmt.Fprintln(os.Stderr, "│ kernel has first class support for WireGuard. For │") - fmt.Fprintln(os.Stderr, "│ information on installing the kernel module, │") - fmt.Fprintln(os.Stderr, "│ please visit: │") - fmt.Fprintln(os.Stderr, "│ https://www.wireguard.com/install/ │") - fmt.Fprintln(os.Stderr, "│ │") - fmt.Fprintln(os.Stderr, "└──────────────────────────────────────────────────────┘") -} - -func main() { - if len(os.Args) == 2 && os.Args[1] == "--version" { - fmt.Printf("wireguard-go v%s\n\nUserspace WireGuard daemon for %s-%s.\nInformation available at https://www.wireguard.com.\nCopyright (C) Jason A. Donenfeld .\n", Version, runtime.GOOS, runtime.GOARCH) - return - } - - warning() - - var foreground bool - var interfaceName string - if len(os.Args) < 2 || len(os.Args) > 3 { - printUsage() - return - } - - switch os.Args[1] { - - case "-f", "--foreground": - foreground = true - if len(os.Args) != 3 { - printUsage() - return - } - interfaceName = os.Args[2] - - default: - foreground = false - if len(os.Args) != 2 { - printUsage() - return - } - interfaceName = os.Args[1] - } - - if !foreground { - foreground = os.Getenv(ENV_WG_PROCESS_FOREGROUND) == "1" - } - - // get log level (default: info) - - logLevel := func() int { - switch os.Getenv("LOG_LEVEL") { - case "verbose", "debug": - return device.LogLevelVerbose - case "error": - return device.LogLevelError - case "silent": - return device.LogLevelSilent - } - return device.LogLevelError - }() - - // open TUN device (or use supplied fd) - - tdev, err := func() (tun.Device, error) { - tunFdStr := os.Getenv(ENV_WG_TUN_FD) - if tunFdStr == "" { - return tun.CreateTUN(interfaceName, device.DefaultMTU) - } - - // construct tun device from supplied fd - - fd, err := strconv.ParseUint(tunFdStr, 10, 32) - if err != nil { - return nil, err - } - - err = unix.SetNonblock(int(fd), true) - if err != nil { - return nil, err - } - - file := os.NewFile(uintptr(fd), "") - return tun.CreateTUNFromFile(file, device.DefaultMTU) - }() - - if err == nil { - realInterfaceName, err2 := tdev.Name() - if err2 == nil { - interfaceName = realInterfaceName - } - } - - logger := device.NewLogger( - logLevel, - fmt.Sprintf("(%s) ", interfaceName), - ) - - logger.Verbosef("Starting wireguard-go version %s", Version) - - if err != nil { - logger.Errorf("Failed to create TUN device: %v", err) - os.Exit(ExitSetupFailed) - } - - // open UAPI file (or use supplied fd) - - fileUAPI, err := func() (*os.File, error) { - uapiFdStr := os.Getenv(ENV_WG_UAPI_FD) - if uapiFdStr == "" { - return ipc.UAPIOpen(interfaceName) - } - - // use supplied fd - - fd, err := strconv.ParseUint(uapiFdStr, 10, 32) - if err != nil { - return nil, err - } - - return os.NewFile(uintptr(fd), ""), nil - }() - if err != nil { - logger.Errorf("UAPI listen error: %v", err) - os.Exit(ExitSetupFailed) - return - } - // daemonize the process - - if !foreground { - env := os.Environ() - env = append(env, fmt.Sprintf("%s=3", ENV_WG_TUN_FD)) - env = append(env, fmt.Sprintf("%s=4", ENV_WG_UAPI_FD)) - env = append(env, fmt.Sprintf("%s=1", ENV_WG_PROCESS_FOREGROUND)) - files := [3]*os.File{} - if os.Getenv("LOG_LEVEL") != "" && logLevel != device.LogLevelSilent { - files[0], _ = os.Open(os.DevNull) - files[1] = os.Stdout - files[2] = os.Stderr - } else { - files[0], _ = os.Open(os.DevNull) - files[1], _ = os.Open(os.DevNull) - files[2], _ = os.Open(os.DevNull) - } - attr := &os.ProcAttr{ - Files: []*os.File{ - files[0], // stdin - files[1], // stdout - files[2], // stderr - tdev.File(), - fileUAPI, - }, - Dir: ".", - Env: env, - } - - path, err := os.Executable() - if err != nil { - logger.Errorf("Failed to determine executable: %v", err) - os.Exit(ExitSetupFailed) - } - - process, err := os.StartProcess( - path, - os.Args, - attr, - ) - if err != nil { - logger.Errorf("Failed to daemonize: %v", err) - os.Exit(ExitSetupFailed) - } - process.Release() - return - } - - device := device.NewDevice(tdev, conn.NewDefaultBind(), logger) - - logger.Verbosef("Device started") - - errs := make(chan error) - term := make(chan os.Signal, 1) - - uapi, err := ipc.UAPIListen(interfaceName, fileUAPI) - if err != nil { - logger.Errorf("Failed to listen on uapi socket: %v", err) - os.Exit(ExitSetupFailed) - } - - go func() { - for { - conn, err := uapi.Accept() - if err != nil { - errs <- err - return - } - go device.IpcHandle(conn) - } - }() - - logger.Verbosef("UAPI listener started") - - // wait for program to terminate - - signal.Notify(term, unix.SIGTERM) - signal.Notify(term, os.Interrupt) - - select { - case <-term: - case <-errs: - case <-device.Wait(): - } - - // clean up - - uapi.Close() - device.Close() - - logger.Verbosef("Shutting down") -} diff --git a/main_windows.go b/main_windows.go deleted file mode 100644 index 689a9a7..0000000 --- a/main_windows.go +++ /dev/null @@ -1,99 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package main - -import ( - "fmt" - "os" - "os/signal" - - "golang.org/x/sys/windows" - - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/device" - "github.com/tailscale/wireguard-go/ipc" - - "github.com/tailscale/wireguard-go/tun" -) - -const ( - ExitSetupSuccess = 0 - ExitSetupFailed = 1 -) - -func main() { - if len(os.Args) != 2 { - os.Exit(ExitSetupFailed) - } - interfaceName := os.Args[1] - - fmt.Fprintln(os.Stderr, "Warning: this is a test program for Windows, mainly used for debugging this Go package. For a real WireGuard for Windows client, the repo you want is , which includes this code as a module.") - - logger := device.NewLogger( - device.LogLevelVerbose, - fmt.Sprintf("(%s) ", interfaceName), - ) - logger.Verbosef("Starting wireguard-go version %s", Version) - - tun, err := tun.CreateTUN(interfaceName, 0) - if err == nil { - realInterfaceName, err2 := tun.Name() - if err2 == nil { - interfaceName = realInterfaceName - } - } else { - logger.Errorf("Failed to create TUN device: %v", err) - os.Exit(ExitSetupFailed) - } - - device := device.NewDevice(tun, conn.NewDefaultBind(), logger) - err = device.Up() - if err != nil { - logger.Errorf("Failed to bring up device: %v", err) - os.Exit(ExitSetupFailed) - } - logger.Verbosef("Device started") - - uapi, err := ipc.UAPIListen(interfaceName) - if err != nil { - logger.Errorf("Failed to listen on uapi socket: %v", err) - os.Exit(ExitSetupFailed) - } - - errs := make(chan error) - term := make(chan os.Signal, 1) - - go func() { - for { - conn, err := uapi.Accept() - if err != nil { - errs <- err - return - } - go device.IpcHandle(conn) - } - }() - logger.Verbosef("UAPI listener started") - - // wait for program to terminate - - signal.Notify(term, os.Interrupt) - signal.Notify(term, os.Kill) - signal.Notify(term, windows.SIGTERM) - - select { - case <-term: - case <-errs: - case <-device.Wait(): - } - - // clean up - - uapi.Close() - device.Close() - - logger.Verbosef("Shutting down") -} diff --git a/ratelimiter/ratelimiter_test.go b/ratelimiter/ratelimiter_test.go deleted file mode 100644 index 0bfa3af..0000000 --- a/ratelimiter/ratelimiter_test.go +++ /dev/null @@ -1,119 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package ratelimiter - -import ( - "net/netip" - "testing" - "time" -) - -type result struct { - allowed bool - text string - wait time.Duration -} - -func TestRatelimiter(t *testing.T) { - var rate Ratelimiter - var expectedResults []result - - nano := func(nano int64) time.Duration { - return time.Nanosecond * time.Duration(nano) - } - - add := func(res result) { - expectedResults = append( - expectedResults, - res, - ) - } - - for i := 0; i < packetsBurstable; i++ { - add(result{ - allowed: true, - text: "initial burst", - }) - } - - add(result{ - allowed: false, - text: "after burst", - }) - - add(result{ - allowed: true, - wait: nano(time.Second.Nanoseconds() / packetsPerSecond), - text: "filling tokens for single packet", - }) - - add(result{ - allowed: false, - text: "not having refilled enough", - }) - - add(result{ - allowed: true, - wait: 2 * (nano(time.Second.Nanoseconds() / packetsPerSecond)), - text: "filling tokens for two packet burst", - }) - - add(result{ - allowed: true, - text: "second packet in 2 packet burst", - }) - - add(result{ - allowed: false, - text: "packet following 2 packet burst", - }) - - ips := []netip.Addr{ - netip.MustParseAddr("127.0.0.1"), - netip.MustParseAddr("192.168.1.1"), - netip.MustParseAddr("172.167.2.3"), - netip.MustParseAddr("97.231.252.215"), - netip.MustParseAddr("248.97.91.167"), - netip.MustParseAddr("188.208.233.47"), - netip.MustParseAddr("104.2.183.179"), - netip.MustParseAddr("72.129.46.120"), - netip.MustParseAddr("2001:0db8:0a0b:12f0:0000:0000:0000:0001"), - netip.MustParseAddr("f5c2:818f:c052:655a:9860:b136:6894:25f0"), - netip.MustParseAddr("b2d7:15ab:48a7:b07c:a541:f144:a9fe:54fc"), - netip.MustParseAddr("a47b:786e:1671:a22b:d6f9:4ab0:abc7:c918"), - netip.MustParseAddr("ea1e:d155:7f7a:98fb:2bf5:9483:80f6:5445"), - netip.MustParseAddr("3f0e:54a2:f5b4:cd19:a21d:58e1:3746:84c4"), - } - - now := time.Now() - rate.timeNow = func() time.Time { - return now - } - defer func() { - // Lock to avoid data race with cleanup goroutine from Init. - rate.mu.Lock() - defer rate.mu.Unlock() - - rate.timeNow = time.Now - }() - timeSleep := func(d time.Duration) { - now = now.Add(d + 1) - rate.cleanup() - } - - rate.Init() - defer rate.Close() - - for i, res := range expectedResults { - timeSleep(res.wait) - for _, ip := range ips { - allowed := rate.Allow(ip) - if allowed != res.allowed { - t.Fatalf("%d: %s: rate.Allow(%q)=%v, want %v", i, res.text, ip, allowed, res.allowed) - } - } - } -} diff --git a/replay/replay_test.go b/replay/replay_test.go deleted file mode 100644 index 9a9e4a8..0000000 --- a/replay/replay_test.go +++ /dev/null @@ -1,119 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package replay - -import ( - "testing" -) - -/* Ported from the linux kernel implementation - * - * - */ - -const RejectAfterMessages = 1<<64 - 1<<13 - 1 - -func TestReplay(t *testing.T) { - var filter Filter - - const T_LIM = windowSize + 1 - - testNumber := 0 - T := func(n uint64, expected bool) { - testNumber++ - if filter.ValidateCounter(n, RejectAfterMessages) != expected { - t.Fatal("Test", testNumber, "failed", n, expected) - } - } - - filter.Reset() - - T(0, true) /* 1 */ - T(1, true) /* 2 */ - T(1, false) /* 3 */ - T(9, true) /* 4 */ - T(8, true) /* 5 */ - T(7, true) /* 6 */ - T(7, false) /* 7 */ - T(T_LIM, true) /* 8 */ - T(T_LIM-1, true) /* 9 */ - T(T_LIM-1, false) /* 10 */ - T(T_LIM-2, true) /* 11 */ - T(2, true) /* 12 */ - T(2, false) /* 13 */ - T(T_LIM+16, true) /* 14 */ - T(3, false) /* 15 */ - T(T_LIM+16, false) /* 16 */ - T(T_LIM*4, true) /* 17 */ - T(T_LIM*4-(T_LIM-1), true) /* 18 */ - T(10, false) /* 19 */ - T(T_LIM*4-T_LIM, false) /* 20 */ - T(T_LIM*4-(T_LIM+1), false) /* 21 */ - T(T_LIM*4-(T_LIM-2), true) /* 22 */ - T(T_LIM*4+1-T_LIM, false) /* 23 */ - T(0, false) /* 24 */ - T(RejectAfterMessages, false) /* 25 */ - T(RejectAfterMessages-1, true) /* 26 */ - T(RejectAfterMessages, false) /* 27 */ - T(RejectAfterMessages-1, false) /* 28 */ - T(RejectAfterMessages-2, true) /* 29 */ - T(RejectAfterMessages+1, false) /* 30 */ - T(RejectAfterMessages+2, false) /* 31 */ - T(RejectAfterMessages-2, false) /* 32 */ - T(RejectAfterMessages-3, true) /* 33 */ - T(0, false) /* 34 */ - - t.Log("Bulk test 1") - filter.Reset() - testNumber = 0 - for i := uint64(1); i <= windowSize; i++ { - T(i, true) - } - T(0, true) - T(0, false) - - t.Log("Bulk test 2") - filter.Reset() - testNumber = 0 - for i := uint64(2); i <= windowSize+1; i++ { - T(i, true) - } - T(1, true) - T(0, false) - - t.Log("Bulk test 3") - filter.Reset() - testNumber = 0 - for i := uint64(windowSize + 1); i > 0; i-- { - T(i, true) - } - - t.Log("Bulk test 4") - filter.Reset() - testNumber = 0 - for i := uint64(windowSize + 2); i > 1; i-- { - T(i, true) - } - T(0, false) - - t.Log("Bulk test 5") - filter.Reset() - testNumber = 0 - for i := uint64(windowSize); i > 0; i-- { - T(i, true) - } - T(windowSize+1, true) - T(0, false) - - t.Log("Bulk test 6") - filter.Reset() - testNumber = 0 - for i := uint64(windowSize); i > 0; i-- { - T(i, true) - } - T(0, true) - T(windowSize+1, true) -} diff --git a/tai64n/tai64n_test.go b/tai64n/tai64n_test.go deleted file mode 100644 index c70fc1a..0000000 --- a/tai64n/tai64n_test.go +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package tai64n - -import ( - "testing" - "time" -) - -// Test that timestamps are monotonic as required by Wireguard and that -// nanosecond-level information is whitened to prevent side channel attacks. -func TestMonotonic(t *testing.T) { - startTime := time.Unix(0, 123456789) // a nontrivial bit pattern - // Whitening should reduce timestamp granularity - // to more than 10 but fewer than 20 milliseconds. - tests := []struct { - name string - t1, t2 time.Time - wantAfter bool - }{ - {"after_10_ns", startTime, startTime.Add(10 * time.Nanosecond), false}, - {"after_10_us", startTime, startTime.Add(10 * time.Microsecond), false}, - {"after_1_ms", startTime, startTime.Add(time.Millisecond), false}, - {"after_10_ms", startTime, startTime.Add(10 * time.Millisecond), false}, - {"after_20_ms", startTime, startTime.Add(20 * time.Millisecond), true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ts1, ts2 := stamp(tt.t1), stamp(tt.t2) - got := ts2.After(ts1) - if got != tt.wantAfter { - t.Errorf("after = %v; want %v", got, tt.wantAfter) - } - }) - } -} diff --git a/tests/netns.sh b/tests/netns.sh deleted file mode 100755 index 2f2a2cd..0000000 --- a/tests/netns.sh +++ /dev/null @@ -1,425 +0,0 @@ -#!/bin/bash - -# Copyright (C) 2015-2017 Jason A. Donenfeld . All Rights Reserved. - -# This script tests the below topology: -# -# ┌─────────────────────┐ ┌──────────────────────────────────┐ ┌─────────────────────┐ -# │ $ns1 namespace │ │ $ns0 namespace │ │ $ns2 namespace │ -# │ │ │ │ │ │ -# │┌────────┐ │ │ ┌────────┐ │ │ ┌────────┐│ -# ││ wg1 │───────────┼───┼────────────│ lo │────────────┼───┼───────────│ wg2 ││ -# │├────────┴──────────┐│ │ ┌───────┴────────┴────────┐ │ │┌──────────┴────────┤│ -# ││192.168.241.1/24 ││ │ │(ns1) (ns2) │ │ ││192.168.241.2/24 ││ -# ││fd00::1/24 ││ │ │127.0.0.1:1 127.0.0.1:2│ │ ││fd00::2/24 ││ -# │└───────────────────┘│ │ │[::]:1 [::]:2 │ │ │└───────────────────┘│ -# └─────────────────────┘ │ └─────────────────────────┘ │ └─────────────────────┘ -# └──────────────────────────────────┘ -# -# After the topology is prepared we run a series of TCP/UDP iperf3 tests between the -# wireguard peers in $ns1 and $ns2. Note that $ns0 is the endpoint for the wg1 -# interfaces in $ns1 and $ns2. See https://www.wireguard.com/netns/ for further -# details on how this is accomplished. - -# This code is ported to the WireGuard-Go directly from the kernel project. -# -# Please ensure that you have installed the newest version of the WireGuard -# tools from the WireGuard project and before running these tests as: -# -# ./netns.sh - -set -e - -exec 3>&1 -export WG_HIDE_KEYS=never -netns0="wg-test-$$-0" -netns1="wg-test-$$-1" -netns2="wg-test-$$-2" -program=$1 -export LOG_LEVEL="verbose" - -pretty() { echo -e "\x1b[32m\x1b[1m[+] ${1:+NS$1: }${2}\x1b[0m" >&3; } -pp() { pretty "" "$*"; "$@"; } -maybe_exec() { if [[ $BASHPID -eq $$ ]]; then "$@"; else exec "$@"; fi; } -n0() { pretty 0 "$*"; maybe_exec ip netns exec $netns0 "$@"; } -n1() { pretty 1 "$*"; maybe_exec ip netns exec $netns1 "$@"; } -n2() { pretty 2 "$*"; maybe_exec ip netns exec $netns2 "$@"; } -ip0() { pretty 0 "ip $*"; ip -n $netns0 "$@"; } -ip1() { pretty 1 "ip $*"; ip -n $netns1 "$@"; } -ip2() { pretty 2 "ip $*"; ip -n $netns2 "$@"; } -sleep() { read -t "$1" -N 0 || true; } -waitiperf() { pretty "${1//*-}" "wait for iperf:5201"; while [[ $(ss -N "$1" -tlp 'sport = 5201') != *iperf3* ]]; do sleep 0.1; done; } -waitncatudp() { pretty "${1//*-}" "wait for udp:1111"; while [[ $(ss -N "$1" -ulp 'sport = 1111') != *ncat* ]]; do sleep 0.1; done; } -waitiface() { pretty "${1//*-}" "wait for $2 to come up"; ip netns exec "$1" bash -c "while [[ \$(< \"/sys/class/net/$2/operstate\") != up ]]; do read -t .1 -N 0 || true; done;"; } - -cleanup() { - set +e - exec 2>/dev/null - printf "$orig_message_cost" > /proc/sys/net/core/message_cost - ip0 link del dev wg1 - ip1 link del dev wg1 - ip2 link del dev wg1 - local to_kill="$(ip netns pids $netns0) $(ip netns pids $netns1) $(ip netns pids $netns2)" - [[ -n $to_kill ]] && kill $to_kill - pp ip netns del $netns1 - pp ip netns del $netns2 - pp ip netns del $netns0 - exit -} - -orig_message_cost="$(< /proc/sys/net/core/message_cost)" -trap cleanup EXIT -printf 0 > /proc/sys/net/core/message_cost - -ip netns del $netns0 2>/dev/null || true -ip netns del $netns1 2>/dev/null || true -ip netns del $netns2 2>/dev/null || true -pp ip netns add $netns0 -pp ip netns add $netns1 -pp ip netns add $netns2 -ip0 link set up dev lo - -# ip0 link add dev wg1 type wireguard -n0 $program wg1 -ip0 link set wg1 netns $netns1 - -# ip0 link add dev wg1 type wireguard -n0 $program wg2 -ip0 link set wg2 netns $netns2 - -key1="$(pp wg genkey)" -key2="$(pp wg genkey)" -pub1="$(pp wg pubkey <<<"$key1")" -pub2="$(pp wg pubkey <<<"$key2")" -psk="$(pp wg genpsk)" -[[ -n $key1 && -n $key2 && -n $psk ]] - -configure_peers() { - - ip1 addr add 192.168.241.1/24 dev wg1 - ip1 addr add fd00::1/24 dev wg1 - - ip2 addr add 192.168.241.2/24 dev wg2 - ip2 addr add fd00::2/24 dev wg2 - - n0 wg set wg1 \ - private-key <(echo "$key1") \ - listen-port 10000 \ - peer "$pub2" \ - preshared-key <(echo "$psk") \ - allowed-ips 192.168.241.2/32,fd00::2/128 - n0 wg set wg2 \ - private-key <(echo "$key2") \ - listen-port 20000 \ - peer "$pub1" \ - preshared-key <(echo "$psk") \ - allowed-ips 192.168.241.1/32,fd00::1/128 - - n0 wg showconf wg1 - n0 wg showconf wg2 - - ip1 link set up dev wg1 - ip2 link set up dev wg2 - sleep 1 -} -configure_peers - -tests() { - # Ping over IPv4 - n2 ping -c 10 -f -W 1 192.168.241.1 - n1 ping -c 10 -f -W 1 192.168.241.2 - - # Ping over IPv6 - n2 ping6 -c 10 -f -W 1 fd00::1 - n1 ping6 -c 10 -f -W 1 fd00::2 - - # TCP over IPv4 - n2 iperf3 -s -1 -B 192.168.241.2 & - waitiperf $netns2 - n1 iperf3 -Z -n 1G -c 192.168.241.2 - - # TCP over IPv6 - n1 iperf3 -s -1 -B fd00::1 & - waitiperf $netns1 - n2 iperf3 -Z -n 1G -c fd00::1 - - # UDP over IPv4 - n1 iperf3 -s -1 -B 192.168.241.1 & - waitiperf $netns1 - n2 iperf3 -Z -n 1G -b 0 -u -c 192.168.241.1 - - # UDP over IPv6 - n2 iperf3 -s -1 -B fd00::2 & - waitiperf $netns2 - n1 iperf3 -Z -n 1G -b 0 -u -c fd00::2 -} - -[[ $(ip1 link show dev wg1) =~ mtu\ ([0-9]+) ]] && orig_mtu="${BASH_REMATCH[1]}" -big_mtu=$(( 34816 - 1500 + $orig_mtu )) - -# Test using IPv4 as outer transport -n0 wg set wg1 peer "$pub2" endpoint 127.0.0.1:20000 -n0 wg set wg2 peer "$pub1" endpoint 127.0.0.1:10000 - -# Before calling tests, we first make sure that the stats counters are working -n2 ping -c 10 -f -W 1 192.168.241.1 -{ read _; read _; read _; read rx_bytes _; read _; read tx_bytes _; } < <(ip2 -stats link show dev wg2) -ip2 -stats link show dev wg2 -n0 wg show -[[ $rx_bytes -ge 840 && $tx_bytes -ge 880 && $rx_bytes -lt 2500 && $rx_bytes -lt 2500 ]] -echo "counters working" -tests -ip1 link set wg1 mtu $big_mtu -ip2 link set wg2 mtu $big_mtu -tests - -ip1 link set wg1 mtu $orig_mtu -ip2 link set wg2 mtu $orig_mtu - -# Test using IPv6 as outer transport -n0 wg set wg1 peer "$pub2" endpoint [::1]:20000 -n0 wg set wg2 peer "$pub1" endpoint [::1]:10000 -tests -ip1 link set wg1 mtu $big_mtu -ip2 link set wg2 mtu $big_mtu -tests - -ip1 link set wg1 mtu $orig_mtu -ip2 link set wg2 mtu $orig_mtu - -# Test using IPv4 that roaming works -ip0 -4 addr del 127.0.0.1/8 dev lo -ip0 -4 addr add 127.212.121.99/8 dev lo -n0 wg set wg1 listen-port 9999 -n0 wg set wg1 peer "$pub2" endpoint 127.0.0.1:20000 -n1 ping6 -W 1 -c 1 fd00::2 -[[ $(n2 wg show wg2 endpoints) == "$pub1 127.212.121.99:9999" ]] - -# Test using IPv6 that roaming works -n1 wg set wg1 listen-port 9998 -n1 wg set wg1 peer "$pub2" endpoint [::1]:20000 -n1 ping -W 1 -c 1 192.168.241.2 -[[ $(n2 wg show wg2 endpoints) == "$pub1 [::1]:9998" ]] - -# Test that crypto-RP filter works -n1 wg set wg1 peer "$pub2" allowed-ips 192.168.241.0/24 -exec 4< <(n1 ncat -l -u -p 1111) -nmap_pid=$! -waitncatudp $netns1 -n2 ncat -u 192.168.241.1 1111 <<<"X" -read -r -N 1 -t 1 out <&4 && [[ $out == "X" ]] -kill $nmap_pid -more_specific_key="$(pp wg genkey | pp wg pubkey)" -n0 wg set wg1 peer "$more_specific_key" allowed-ips 192.168.241.2/32 -n0 wg set wg2 listen-port 9997 -exec 4< <(n1 ncat -l -u -p 1111) -nmap_pid=$! -waitncatudp $netns1 -n2 ncat -u 192.168.241.1 1111 <<<"X" -! read -r -N 1 -t 1 out <&4 -kill $nmap_pid -n0 wg set wg1 peer "$more_specific_key" remove -[[ $(n1 wg show wg1 endpoints) == "$pub2 [::1]:9997" ]] - -ip1 link del wg1 -ip2 link del wg2 - -# Test using NAT. We now change the topology to this: -# ┌────────────────────────────────────────┐ ┌────────────────────────────────────────────────┐ ┌────────────────────────────────────────┐ -# │ $ns1 namespace │ │ $ns0 namespace │ │ $ns2 namespace │ -# │ │ │ │ │ │ -# │ ┌─────┐ ┌─────┐ │ │ ┌──────┐ ┌──────┐ │ │ ┌─────┐ ┌─────┐ │ -# │ │ wg1 │─────────────│vethc│───────────┼────┼────│vethrc│ │vethrs│──────────────┼─────┼──│veths│────────────│ wg2 │ │ -# │ ├─────┴──────────┐ ├─────┴──────────┐│ │ ├──────┴─────────┐ ├──────┴────────────┐ │ │ ├─────┴──────────┐ ├─────┴──────────┐ │ -# │ │192.168.241.1/24│ │192.168.1.100/24││ │ │192.168.1.100/24│ │10.0.0.1/24 │ │ │ │10.0.0.100/24 │ │192.168.241.2/24│ │ -# │ │fd00::1/24 │ │ ││ │ │ │ │SNAT:192.168.1.0/24│ │ │ │ │ │fd00::2/24 │ │ -# │ └────────────────┘ └────────────────┘│ │ └────────────────┘ └───────────────────┘ │ │ └────────────────┘ └────────────────┘ │ -# └────────────────────────────────────────┘ └────────────────────────────────────────────────┘ └────────────────────────────────────────┘ - -# ip1 link add dev wg1 type wireguard -# ip2 link add dev wg1 type wireguard - -n1 $program wg1 -n2 $program wg2 - -configure_peers - -ip0 link add vethrc type veth peer name vethc -ip0 link add vethrs type veth peer name veths -ip0 link set vethc netns $netns1 -ip0 link set veths netns $netns2 -ip0 link set vethrc up -ip0 link set vethrs up -ip0 addr add 192.168.1.1/24 dev vethrc -ip0 addr add 10.0.0.1/24 dev vethrs -ip1 addr add 192.168.1.100/24 dev vethc -ip1 link set vethc up -ip1 route add default via 192.168.1.1 -ip2 addr add 10.0.0.100/24 dev veths -ip2 link set veths up -waitiface $netns0 vethrc -waitiface $netns0 vethrs -waitiface $netns1 vethc -waitiface $netns2 veths - -n0 bash -c 'printf 1 > /proc/sys/net/ipv4/ip_forward' -n0 bash -c 'printf 2 > /proc/sys/net/netfilter/nf_conntrack_udp_timeout' -n0 bash -c 'printf 2 > /proc/sys/net/netfilter/nf_conntrack_udp_timeout_stream' -n0 iptables -t nat -A POSTROUTING -s 192.168.1.0/24 -d 10.0.0.0/24 -j SNAT --to 10.0.0.1 - -n0 wg set wg1 peer "$pub2" endpoint 10.0.0.100:20000 persistent-keepalive 1 -n1 ping -W 1 -c 1 192.168.241.2 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n2 wg show wg2 endpoints) == "$pub1 10.0.0.1:10000" ]] -# Demonstrate n2 can still send packets to n1, since persistent-keepalive will prevent connection tracking entry from expiring (to see entries: `n0 conntrack -L`). -pp sleep 3 -n2 ping -W 1 -c 1 192.168.241.1 - -n0 iptables -t nat -F -ip0 link del vethrc -ip0 link del vethrs -ip1 link del wg1 -ip2 link del wg2 - -# Test that saddr routing is sticky but not too sticky, changing to this topology: -# ┌────────────────────────────────────────┐ ┌────────────────────────────────────────┐ -# │ $ns1 namespace │ │ $ns2 namespace │ -# │ │ │ │ -# │ ┌─────┐ ┌─────┐ │ │ ┌─────┐ ┌─────┐ │ -# │ │ wg1 │─────────────│veth1│───────────┼────┼──│veth2│────────────│ wg2 │ │ -# │ ├─────┴──────────┐ ├─────┴──────────┐│ │ ├─────┴──────────┐ ├─────┴──────────┐ │ -# │ │192.168.241.1/24│ │10.0.0.1/24 ││ │ │10.0.0.2/24 │ │192.168.241.2/24│ │ -# │ │fd00::1/24 │ │fd00:aa::1/96 ││ │ │fd00:aa::2/96 │ │fd00::2/24 │ │ -# │ └────────────────┘ └────────────────┘│ │ └────────────────┘ └────────────────┘ │ -# └────────────────────────────────────────┘ └────────────────────────────────────────┘ - -# ip1 link add dev wg1 type wireguard -# ip2 link add dev wg1 type wireguard -n1 $program wg1 -n2 $program wg2 - -configure_peers - -ip1 link add veth1 type veth peer name veth2 -ip1 link set veth2 netns $netns2 -n1 bash -c 'printf 0 > /proc/sys/net/ipv6/conf/veth1/accept_dad' -n2 bash -c 'printf 0 > /proc/sys/net/ipv6/conf/veth2/accept_dad' -n1 bash -c 'printf 1 > /proc/sys/net/ipv4/conf/veth1/promote_secondaries' - -# First we check that we aren't overly sticky and can fall over to new IPs when old ones are removed -ip1 addr add 10.0.0.1/24 dev veth1 -ip1 addr add fd00:aa::1/96 dev veth1 -ip2 addr add 10.0.0.2/24 dev veth2 -ip2 addr add fd00:aa::2/96 dev veth2 -ip1 link set veth1 up -ip2 link set veth2 up -waitiface $netns1 veth1 -waitiface $netns2 veth2 -n0 wg set wg1 peer "$pub2" endpoint 10.0.0.2:20000 -n1 ping -W 1 -c 1 192.168.241.2 -ip1 addr add 10.0.0.10/24 dev veth1 -ip1 addr del 10.0.0.1/24 dev veth1 -n1 ping -W 1 -c 1 192.168.241.2 -n0 wg set wg1 peer "$pub2" endpoint [fd00:aa::2]:20000 -n1 ping -W 1 -c 1 192.168.241.2 -ip1 addr add fd00:aa::10/96 dev veth1 -ip1 addr del fd00:aa::1/96 dev veth1 -n1 ping -W 1 -c 1 192.168.241.2 - -# Now we show that we can successfully do reply to sender routing -ip1 link set veth1 down -ip2 link set veth2 down -ip1 addr flush dev veth1 -ip2 addr flush dev veth2 -ip1 addr add 10.0.0.1/24 dev veth1 -ip1 addr add 10.0.0.2/24 dev veth1 -ip1 addr add fd00:aa::1/96 dev veth1 -ip1 addr add fd00:aa::2/96 dev veth1 -ip2 addr add 10.0.0.3/24 dev veth2 -ip2 addr add fd00:aa::3/96 dev veth2 -ip1 link set veth1 up -ip2 link set veth2 up -waitiface $netns1 veth1 -waitiface $netns2 veth2 -n0 wg set wg2 peer "$pub1" endpoint 10.0.0.1:10000 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n0 wg show wg2 endpoints) == "$pub1 10.0.0.1:10000" ]] -n0 wg set wg2 peer "$pub1" endpoint [fd00:aa::1]:10000 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n0 wg show wg2 endpoints) == "$pub1 [fd00:aa::1]:10000" ]] -n0 wg set wg2 peer "$pub1" endpoint 10.0.0.2:10000 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n0 wg show wg2 endpoints) == "$pub1 10.0.0.2:10000" ]] -n0 wg set wg2 peer "$pub1" endpoint [fd00:aa::2]:10000 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n0 wg show wg2 endpoints) == "$pub1 [fd00:aa::2]:10000" ]] - -ip1 link del veth1 -ip1 link del wg1 -ip2 link del wg2 - -# Test that Netlink/IPC is working properly by doing things that usually cause split responses - -n0 $program wg0 -sleep 5 -config=( "[Interface]" "PrivateKey=$(wg genkey)" "[Peer]" "PublicKey=$(wg genkey)" ) -for a in {1..255}; do - for b in {0..255}; do - config+=( "AllowedIPs=$a.$b.0.0/16,$a::$b/128" ) - done -done -n0 wg setconf wg0 <(printf '%s\n' "${config[@]}") -i=0 -for ip in $(n0 wg show wg0 allowed-ips); do - ((++i)) -done -((i == 255*256*2+1)) -ip0 link del wg0 - -n0 $program wg0 -config=( "[Interface]" "PrivateKey=$(wg genkey)" ) -for a in {1..40}; do - config+=( "[Peer]" "PublicKey=$(wg genkey)" ) - for b in {1..52}; do - config+=( "AllowedIPs=$a.$b.0.0/16" ) - done -done -n0 wg setconf wg0 <(printf '%s\n' "${config[@]}") -i=0 -while read -r line; do - j=0 - for ip in $line; do - ((++j)) - done - ((j == 53)) - ((++i)) -done < <(n0 wg show wg0 allowed-ips) -((i == 40)) -ip0 link del wg0 - -n0 $program wg0 -config=( ) -for i in {1..29}; do - config+=( "[Peer]" "PublicKey=$(wg genkey)" ) -done -config+=( "[Peer]" "PublicKey=$(wg genkey)" "AllowedIPs=255.2.3.4/32,abcd::255/128" ) -n0 wg setconf wg0 <(printf '%s\n' "${config[@]}") -n0 wg showconf wg0 > /dev/null -ip0 link del wg0 - -! n0 wg show doesnotexist || false - -declare -A objects -while read -t 0.1 -r line 2>/dev/null || [[ $? -ne 142 ]]; do - [[ $line =~ .*(wg[0-9]+:\ [A-Z][a-z]+\ [0-9]+)\ .*(created|destroyed).* ]] || continue - objects["${BASH_REMATCH[1]}"]+="${BASH_REMATCH[2]}" -done < /dev/kmsg -alldeleted=1 -for object in "${!objects[@]}"; do - if [[ ${objects["$object"]} != *createddestroyed ]]; then - echo "Error: $object: merely ${objects["$object"]}" >&3 - alldeleted=0 - fi -done -[[ $alldeleted -eq 1 ]] -pretty "" "Objects that were created were also destroyed." diff --git a/tun/alignment_windows_test.go b/tun/alignment_windows_test.go deleted file mode 100644 index 67a785e..0000000 --- a/tun/alignment_windows_test.go +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package tun - -import ( - "reflect" - "testing" - "unsafe" -) - -func checkAlignment(t *testing.T, name string, offset uintptr) { - t.Helper() - if offset%8 != 0 { - t.Errorf("offset of %q within struct is %d bytes, which does not align to 64-bit word boundaries (missing %d bytes). Atomic operations will crash on 32-bit systems.", name, offset, 8-(offset%8)) - } -} - -// TestRateJugglerAlignment checks that atomically-accessed fields are -// aligned to 64-bit boundaries, as required by the atomic package. -// -// Unfortunately, violating this rule on 32-bit platforms results in a -// hard segfault at runtime. -func TestRateJugglerAlignment(t *testing.T) { - var r rateJuggler - - typ := reflect.TypeOf(&r).Elem() - t.Logf("Peer type size: %d, with fields:", typ.Size()) - for i := 0; i < typ.NumField(); i++ { - field := typ.Field(i) - t.Logf("\t%30s\toffset=%3v\t(type size=%3d, align=%d)", - field.Name, - field.Offset, - field.Type.Size(), - field.Type.Align(), - ) - } - - checkAlignment(t, "rateJuggler.current", unsafe.Offsetof(r.current)) - checkAlignment(t, "rateJuggler.nextByteCount", unsafe.Offsetof(r.nextByteCount)) - checkAlignment(t, "rateJuggler.nextStartTime", unsafe.Offsetof(r.nextStartTime)) -} - -// TestNativeTunAlignment checks that atomically-accessed fields are -// aligned to 64-bit boundaries, as required by the atomic package. -// -// Unfortunately, violating this rule on 32-bit platforms results in a -// hard segfault at runtime. -func TestNativeTunAlignment(t *testing.T) { - var tun NativeTun - - typ := reflect.TypeOf(&tun).Elem() - t.Logf("Peer type size: %d, with fields:", typ.Size()) - for i := 0; i < typ.NumField(); i++ { - field := typ.Field(i) - t.Logf("\t%30s\toffset=%3v\t(type size=%3d, align=%d)", - field.Name, - field.Offset, - field.Type.Size(), - field.Type.Align(), - ) - } - - checkAlignment(t, "NativeTun.rate", unsafe.Offsetof(tun.rate)) -} diff --git a/tun/checksum_amd64_test.go b/tun/checksum_amd64_test.go deleted file mode 100644 index 7a0b681..0000000 --- a/tun/checksum_amd64_test.go +++ /dev/null @@ -1,45 +0,0 @@ -//go:build amd64 - -package tun - -import ( - "golang.org/x/sys/cpu" -) - -var archChecksumFuncs = []archChecksumDetails{ - { - name: "generic32", - available: true, - f: checksumGeneric32, - }, - { - name: "generic64", - available: true, - f: checksumGeneric64, - }, - { - name: "generic32Alternate", - available: true, - f: checksumGeneric32Alternate, - }, - { - name: "generic64Alternate", - available: true, - f: checksumGeneric64Alternate, - }, - { - name: "AMD64", - available: true, - f: checksumAMD64, - }, - { - name: "SSE2", - available: cpu.X86.HasSSE2, - f: checksumSSE2, - }, - { - name: "AVX2", - available: cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2, - f: checksumAVX2, - }, -} diff --git a/tun/checksum_generic_test.go b/tun/checksum_generic_test.go deleted file mode 100644 index 401a7bb..0000000 --- a/tun/checksum_generic_test.go +++ /dev/null @@ -1,26 +0,0 @@ -//go:build !amd64 - -package tun - -var archChecksumFuncs = []archChecksumDetails{ - { - name: "generic32", - available: true, - f: checksumGeneric32, - }, - { - name: "generic32Alternate", - available: true, - f: checksumGeneric32Alternate, - }, - { - name: "generic64", - available: true, - f: checksumGeneric64, - }, - { - name: "generic64Alternate", - available: true, - f: checksumGeneric64Alternate, - }, -} diff --git a/tun/checksum_test.go b/tun/checksum_test.go deleted file mode 100644 index f5b8f18..0000000 --- a/tun/checksum_test.go +++ /dev/null @@ -1,619 +0,0 @@ -package tun - -import ( - "fmt" - "math" - "math/rand" - "net/netip" - "sort" - "syscall" - "testing" - "unsafe" - - "gvisor.dev/gvisor/pkg/tcpip" - gvisorChecksum "gvisor.dev/gvisor/pkg/tcpip/checksum" - "gvisor.dev/gvisor/pkg/tcpip/header" -) - -type archChecksumDetails struct { - name string - available bool - f func([]byte, uint16) uint16 -} - -func fillRandomBuffer(seed int64, buf []byte) { - rng := rand.New(rand.NewSource(seed)) - n, err := rng.Read(buf) - if err != nil { - panic(err) - } - if n != len(buf) { - panic("incomplete random buffer") - } -} - -func deterministicRandomBytes(seed int64, length int) []byte { - buf := make([]byte, length) - fillRandomBuffer(seed, buf) - return buf -} - -func getPageAlignedRandomBytes(seed int64, length int) []byte { - alignment := syscall.Getpagesize() - buf := make([]byte, length+(alignment-1)) - bufPtr := uintptr(unsafe.Pointer(&buf[0])) - alignedBufPtr := (bufPtr + uintptr(alignment-1)) & ^uintptr(alignment-1) - alignedStart := int(alignedBufPtr - bufPtr) - - buf = buf[alignedStart : alignedStart+length] - fillRandomBuffer(seed, buf) - return buf -} - -func TestChecksum(t *testing.T) { - alignedBuf := getPageAlignedRandomBytes(10, 8192) - allOnes := make([]byte, 65535) - for i := range allOnes { - allOnes[i] = 0xff - } - allFE := make([]byte, 65535) - for i := range allFE { - allFE[i] = 0xfe - } - - tests := []struct { - name string - data []byte - initial uint16 - want uint16 - }{ - { - name: "empty", - data: []byte{}, - initial: 0, - want: 0, - }, - { - name: "max initial", - data: []byte{}, - initial: math.MaxUint16, - want: 0xffff, - }, - { - name: "odd length", - data: []byte{0x01, 0x02, 0x01}, - initial: 0, - want: 0x0202, - }, - { - name: "tiny", - data: []byte{0x01, 0x02, 0x01, 0x02, 0x01, 0x02}, - initial: 0, - want: 0x0306, - }, - { - name: "initial", - data: []byte{0x01, 0x02, 0x01, 0x02, 0x01, 0x02}, - initial: 0x1000, - want: 0x1306, - }, - // cleanup0 through cleanup15 is 1024 (handled by large SIMD loops) + - // 32 (handled by small SIMD loops) + n, where n ranges from 0 to 15 - // to cover all of the leftover byte sizes that are possible after small - // SIMD loops that handle 16 bytes. - { - name: "cleanup0", - data: deterministicRandomBytes(1, 1056), - initial: 0, - want: 0x11ec, - }, - { - name: "cleanup1", - data: deterministicRandomBytes(1, 1057), - initial: 0, - want: 0xc5ec, - }, - { - name: "cleanup2", - data: deterministicRandomBytes(1, 1058), - initial: 0, - want: 0xc6ad, - }, - { - name: "cleanup3", - data: deterministicRandomBytes(1, 1059), - initial: 0, - want: 0x86ae, - }, - { - name: "cleanup4", - data: deterministicRandomBytes(1, 1060), - initial: 0, - want: 0x878e, - }, - { - name: "cleanup5", - data: deterministicRandomBytes(1, 1061), - initial: 0, - want: 0xdb8e, - }, - { - name: "cleanup6", - data: deterministicRandomBytes(1, 1062), - initial: 0, - want: 0xdbd5, - }, - { - name: "cleanup7", - data: deterministicRandomBytes(1, 1063), - initial: 0, - want: 0xcfd6, - }, - { - name: "cleanup8", - data: deterministicRandomBytes(1, 1064), - initial: 0, - want: 0xd090, - }, - { - name: "cleanup9", - data: deterministicRandomBytes(1, 1065), - initial: 0, - want: 0x0791, - }, - { - name: "cleanup10", - data: deterministicRandomBytes(1, 1066), - initial: 0, - want: 0x079f, - }, - { - name: "cleanup11", - data: deterministicRandomBytes(1, 1067), - initial: 0, - want: 0xba9f, - }, - { - name: "cleanup12", - data: deterministicRandomBytes(1, 1068), - initial: 0, - want: 0xbb0c, - }, - { - name: "cleanup13", - data: deterministicRandomBytes(1, 1069), - initial: 0, - want: 0x770d, - }, - { - name: "cleanup14", - data: deterministicRandomBytes(1, 1070), - initial: 0, - want: 0x780a, - }, - { - name: "cleanup15", - data: deterministicRandomBytes(1, 1071), - initial: 0, - want: 0x640b, - }, - // small1 through small15 covers small sizes that are not large enough - // to do overlapped reads. - { - name: "small1", - data: deterministicRandomBytes(2, 1), - initial: 0x1122, - want: 0x4022, - }, - { - name: "small2", - data: deterministicRandomBytes(2, 2), - initial: 0x1122, - want: 0x40a4, - }, - { - name: "small3", - data: deterministicRandomBytes(2, 3), - initial: 0x1122, - want: 0xc2a4, - }, - { - name: "small4", - data: deterministicRandomBytes(2, 4), - initial: 0x1122, - want: 0xc36f, - }, - { - name: "small5", - data: deterministicRandomBytes(2, 5), - initial: 0x1122, - want: 0xa570, - }, - { - name: "small6", - data: deterministicRandomBytes(2, 6), - initial: 0x1122, - want: 0xa669, - }, - { - name: "small7", - data: deterministicRandomBytes(2, 7), - initial: 0x1122, - want: 0x0f6a, - }, - { - name: "small8", - data: deterministicRandomBytes(2, 8), - initial: 0x1122, - want: 0x0fd9, - }, - { - name: "small9", - data: deterministicRandomBytes(2, 9), - initial: 0x1122, - want: 0x40d9, - }, - { - name: "small10", - data: deterministicRandomBytes(2, 10), - initial: 0x1122, - want: 0x411d, - }, - { - name: "small11", - data: deterministicRandomBytes(2, 11), - initial: 0x1122, - want: 0x011e, - }, - { - name: "small12", - data: deterministicRandomBytes(2, 12), - initial: 0x1122, - want: 0x01c8, - }, - { - name: "small13", - data: deterministicRandomBytes(2, 13), - initial: 0x1122, - want: 0x4dc8, - }, - { - name: "small14", - data: deterministicRandomBytes(2, 14), - initial: 0x1122, - want: 0x4eb5, - }, - { - name: "small15", - data: deterministicRandomBytes(2, 15), - initial: 0x1122, - want: 0xa4b5, - }, - // other small-ish sizes - { - name: "small16", - data: deterministicRandomBytes(1, 16), - initial: 0, - want: 0x02fa, - }, - { - name: "small32", - data: deterministicRandomBytes(1, 32), - initial: 0, - want: 0x03ee, - }, - { - name: "small64", - data: deterministicRandomBytes(1, 64), - initial: 0, - want: 0x3f85, - }, - { - name: "medium", - data: deterministicRandomBytes(1, 1400), - initial: 0, - want: 0xbea5, - }, - { - name: "big", - data: deterministicRandomBytes(2, 65000), - initial: 0, - want: 0x3ba7, - }, - { - name: "big-initial", - data: deterministicRandomBytes(2, 65000), - initial: 0x1234, - want: 0x4ddb, - }, - { - // big-small-loop is intended to exercise a few iterations of a big - // initial loop of 128 bytes or larger + a smaller loop of 16 bytes - // + some leftover - name: "big-small-loop", - data: deterministicRandomBytes(3, 1094), - initial: 0x9999, - want: 0xe65b, - }, - { - name: "page-aligned", - data: alignedBuf[:4096], - initial: 0, - want: 0x963b, - }, - { - name: "32-aligned", - data: alignedBuf[32:4128], - initial: 0, - want: 0x30c4, - }, - { - name: "16-aligned", - data: alignedBuf[16:4112], - initial: 0, - want: 0xaeff, - }, - { - name: "8-aligned", - data: alignedBuf[8:4104], - initial: 0, - want: 0x6c3b, - }, - { - name: "4-aligned", - data: alignedBuf[4:4100], - initial: 0, - want: 0x2e4a, - }, - { - name: "2-aligned", - data: alignedBuf[2:4098], - initial: 0, - want: 0xc702, - }, - { - name: "unaligned", - data: alignedBuf[1:4097], - initial: 0, - want: 0x3bc7, - }, - { - name: "unalignedAndOdd", - data: alignedBuf[1:4096], - initial: 0, - want: 0x3b13, - }, - { - name: "fe1282", - data: allFE[:1282], - initial: 0, - want: 0x7c7c, - }, - { - name: "fe", - data: allFE, - initial: 0, - want: 0x7e81, - }, - { - name: "maximum", - data: allOnes, - initial: 0, - want: 0xff00, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - for _, fd := range archChecksumFuncs { - t.Run(fd.name, func(t *testing.T) { - if !fd.available { - t.Skip("can not run on this system") - } - if got := fd.f(tt.data, tt.initial); got != tt.want { - t.Errorf("%s checksum = %04x, want %04x", fd.name, got, tt.want) - } - }) - } - t.Run("reference", func(t *testing.T) { - if got := gvisorChecksum.Checksum(tt.data, tt.initial); got != tt.want { - t.Errorf("reference checksum = %04x, want %04x", got, tt.want) - } - }) - }) - } -} - -func TestPseudoHeaderChecksumNoFold(t *testing.T) { - tests := []struct { - name string - protocol uint8 - srcAddr []byte - dstAddr []byte - totalLen uint16 - want uint16 - }{ - { - name: "ipv4", - protocol: syscall.IPPROTO_TCP, - srcAddr: netip.MustParseAddr("192.168.1.1").AsSlice(), - dstAddr: netip.MustParseAddr("192.168.1.2").AsSlice(), - totalLen: 1492, - want: 0x892e, - }, - { - name: "ipv6", - protocol: syscall.IPPROTO_TCP, - srcAddr: netip.MustParseAddr("2001:db8:3333:4444:5555:6666:7777:8888").AsSlice(), - dstAddr: netip.MustParseAddr("2001:db8:aaaa:bbbb:cccc:dddd:eeee:ffff").AsSlice(), - totalLen: 1492, - want: 0x947f, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Run("pseudoHeaderChecksum32", func(t *testing.T) { - got := pseudoHeaderChecksum32(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) - if got != tt.want { - t.Errorf("got %04x, want %04x", got, tt.want) - } - }) - t.Run("pseudoHeaderChecksum64", func(t *testing.T) { - got := pseudoHeaderChecksum64(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) - if got != tt.want { - t.Errorf("got %04x, want %04x", got, tt.want) - } - }) - t.Run("reference", func(t *testing.T) { - got := header.PseudoHeaderChecksum( - tcpip.TransportProtocolNumber(tt.protocol), - tcpip.AddrFromSlice(tt.srcAddr), - tcpip.AddrFromSlice(tt.dstAddr), - tt.totalLen) - if got != tt.want { - t.Errorf("got %04x, want %04x", got, tt.want) - } - }) - }) - } -} - -func FuzzChecksum(f *testing.F) { - buf := getPageAlignedRandomBytes(1234, 65536) - - f.Add([]byte{}, uint16(0)) - f.Add([]byte{}, uint16(0x1234)) - f.Add([]byte{}, uint16(0)) - f.Add(buf[:15], uint16(0x1234)) - f.Add(buf[:256], uint16(0x1234)) - f.Add(buf[:1280], uint16(0x1234)) - f.Add(buf[:1288], uint16(0x1234)) - f.Add(buf[1:1050], uint16(0x1234)) - - f.Fuzz(func(t *testing.T, data []byte, initial uint16) { - want := gvisorChecksum.Checksum(data, initial) - - for _, fd := range archChecksumFuncs { - t.Run(fd.name, func(t *testing.T) { - if !fd.available { - t.Skip("can not run on this system") - } - if got := fd.f(data, initial); got != want { - t.Errorf("%s checksum = %04x, want %04x", fd.name, got, want) - } - }) - } - }) -} - -var result uint16 - -func BenchmarkChecksum(b *testing.B) { - offsets := []int{ // offsets from page alignment - 0, - 1, - 2, - 4, - 8, - 16, - } - lengths := []int{ - 0, - 7, - 15, - 16, - 31, - 64, - 90, - 95, - 128, - 256, - 512, - 1024, - 1240, - 1500, - 2048, - 4096, - 8192, - 9000, - 9001, - 16384, - 65536, - } - if !sort.IntsAreSorted(offsets) { - b.Fatal("offsets are not sorted") - } - largestLength := lengths[len(lengths)-1] - if !sort.IntsAreSorted(lengths) { - b.Fatal("lengths are not sorted") - } - largestOffset := lengths[len(offsets)-1] - alignedBuf := getPageAlignedRandomBytes(1, largestOffset+largestLength) - var r uint16 - for _, offset := range offsets { - name := fmt.Sprintf("%vAligned", offset) - if offset == 0 { - name = "pageAligned" - } - offsetBuf := alignedBuf[offset:] - b.Run(name, func(b *testing.B) { - for _, length := range lengths { - b.Run(fmt.Sprintf("%d", length), func(b *testing.B) { - for _, fd := range archChecksumFuncs { - b.Run(fd.name, func(b *testing.B) { - if !fd.available { - b.Skip("can not run on this system") - } - b.SetBytes(int64(length)) - for i := 0; i < b.N; i++ { - r += fd.f(offsetBuf[:length], 0) - } - }) - } - }) - } - }) - } - result = r -} - -func BenchmarkPseudoHeaderChecksum(b *testing.B) { - tests := []struct { - name string - protocol uint8 - srcAddr []byte - dstAddr []byte - totalLen uint16 - want uint16 - }{ - { - name: "ipv4", - protocol: syscall.IPPROTO_TCP, - srcAddr: []byte{192, 168, 1, 1}, - dstAddr: []byte{192, 168, 1, 2}, - totalLen: 1492, - want: 0x892e, - }, - { - name: "ipv6", - protocol: syscall.IPPROTO_TCP, - srcAddr: netip.MustParseAddr("2001:db8:3333:4444:5555:6666:7777:8888").AsSlice(), - dstAddr: netip.MustParseAddr("2001:db8:aaaa:bbbb:cccc:dddd:eeee:ffff").AsSlice(), - totalLen: 1492, - want: 0x892e, - }, - } - for _, tt := range tests { - b.Run(tt.name, func(b *testing.B) { - b.Run("pseudoHeaderChecksum32", func(b *testing.B) { - for i := 0; i < b.N; i++ { - result += pseudoHeaderChecksum32(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) - } - }) - b.Run("pseudoHeaderChecksum64", func(b *testing.B) { - for i := 0; i < b.N; i++ { - result += pseudoHeaderChecksum64(tt.protocol, tt.srcAddr, tt.dstAddr, tt.totalLen) - } - }) - }) - } -} diff --git a/tun/netstack/examples/http_client.go b/tun/netstack/examples/http_client.go deleted file mode 100644 index 81f4d31..0000000 --- a/tun/netstack/examples/http_client.go +++ /dev/null @@ -1,54 +0,0 @@ -//go:build ignore - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package main - -import ( - "io" - "log" - "net/http" - "net/netip" - - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/device" - "github.com/tailscale/wireguard-go/tun/netstack" -) - -func main() { - tun, tnet, err := netstack.CreateNetTUN( - []netip.Addr{netip.MustParseAddr("192.168.4.28")}, - []netip.Addr{netip.MustParseAddr("8.8.8.8")}, - 1420) - if err != nil { - log.Panic(err) - } - dev := device.NewDevice(tun, conn.NewDefaultBind(), device.NewLogger(device.LogLevelVerbose, "")) - err = dev.IpcSet(`private_key=087ec6e14bbed210e7215cdc73468dfa23f080a1bfb8665b2fd809bd99d28379 -public_key=c4c8e984c5322c8184c72265b92b250fdb63688705f504ba003c88f03393cf28 -allowed_ip=0.0.0.0/0 -endpoint=127.0.0.1:58120 -`) - err = dev.Up() - if err != nil { - log.Panic(err) - } - - client := http.Client{ - Transport: &http.Transport{ - DialContext: tnet.DialContext, - }, - } - resp, err := client.Get("http://192.168.4.29/") - if err != nil { - log.Panic(err) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - log.Panic(err) - } - log.Println(string(body)) -} diff --git a/tun/netstack/examples/http_server.go b/tun/netstack/examples/http_server.go deleted file mode 100644 index 30f4544..0000000 --- a/tun/netstack/examples/http_server.go +++ /dev/null @@ -1,51 +0,0 @@ -//go:build ignore - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package main - -import ( - "io" - "log" - "net" - "net/http" - "net/netip" - - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/device" - "github.com/tailscale/wireguard-go/tun/netstack" -) - -func main() { - tun, tnet, err := netstack.CreateNetTUN( - []netip.Addr{netip.MustParseAddr("192.168.4.29")}, - []netip.Addr{netip.MustParseAddr("8.8.8.8"), netip.MustParseAddr("8.8.4.4")}, - 1420, - ) - if err != nil { - log.Panic(err) - } - dev := device.NewDevice(tun, conn.NewDefaultBind(), device.NewLogger(device.LogLevelVerbose, "")) - dev.IpcSet(`private_key=003ed5d73b55806c30de3f8a7bdab38af13539220533055e635690b8b87ad641 -listen_port=58120 -public_key=f928d4f6c1b86c12f2562c10b07c555c5c57fd00f59e90c8d8d88767271cbf7c -allowed_ip=192.168.4.28/32 -persistent_keepalive_interval=25 -`) - dev.Up() - listener, err := tnet.ListenTCP(&net.TCPAddr{Port: 80}) - if err != nil { - log.Panicln(err) - } - http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { - log.Printf("> %s - %s - %s", request.RemoteAddr, request.URL.String(), request.UserAgent()) - io.WriteString(writer, "Hello from userspace TCP!") - }) - err = http.Serve(listener, nil) - if err != nil { - log.Panicln(err) - } -} diff --git a/tun/netstack/examples/ping_client.go b/tun/netstack/examples/ping_client.go deleted file mode 100644 index fc991b1..0000000 --- a/tun/netstack/examples/ping_client.go +++ /dev/null @@ -1,75 +0,0 @@ -//go:build ignore - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package main - -import ( - "bytes" - "log" - "math/rand" - "net/netip" - "time" - - "golang.org/x/net/icmp" - "golang.org/x/net/ipv4" - - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/device" - "github.com/tailscale/wireguard-go/tun/netstack" -) - -func main() { - tun, tnet, err := netstack.CreateNetTUN( - []netip.Addr{netip.MustParseAddr("192.168.4.29")}, - []netip.Addr{netip.MustParseAddr("8.8.8.8")}, - 1420) - if err != nil { - log.Panic(err) - } - dev := device.NewDevice(tun, conn.NewDefaultBind(), device.NewLogger(device.LogLevelVerbose, "")) - dev.IpcSet(`private_key=a8dac1d8a70a751f0f699fb14ba1cff7b79cf4fbd8f09f44c6e6a90d0369604f -public_key=25123c5dcd3328ff645e4f2a3fce0d754400d3887a0cb7c56f0267e20fbf3c5b -endpoint=163.172.161.0:12912 -allowed_ip=0.0.0.0/0 -`) - err = dev.Up() - if err != nil { - log.Panic(err) - } - - socket, err := tnet.Dial("ping4", "zx2c4.com") - if err != nil { - log.Panic(err) - } - requestPing := icmp.Echo{ - Seq: rand.Intn(1 << 16), - Data: []byte("gopher burrow"), - } - icmpBytes, _ := (&icmp.Message{Type: ipv4.ICMPTypeEcho, Code: 0, Body: &requestPing}).Marshal(nil) - socket.SetReadDeadline(time.Now().Add(time.Second * 10)) - start := time.Now() - _, err = socket.Write(icmpBytes) - if err != nil { - log.Panic(err) - } - n, err := socket.Read(icmpBytes[:]) - if err != nil { - log.Panic(err) - } - replyPacket, err := icmp.ParseMessage(1, icmpBytes[:n]) - if err != nil { - log.Panic(err) - } - replyPing, ok := replyPacket.Body.(*icmp.Echo) - if !ok { - log.Panicf("invalid reply type: %v", replyPacket) - } - if !bytes.Equal(replyPing.Data, requestPing.Data) || replyPing.Seq != requestPing.Seq { - log.Panicf("invalid ping reply: %v", replyPing) - } - log.Printf("Ping latency: %v", time.Since(start)) -} diff --git a/tun/netstack/tun.go b/tun/netstack/tun.go deleted file mode 100644 index d8e70bb..0000000 --- a/tun/netstack/tun.go +++ /dev/null @@ -1,1055 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package netstack - -import ( - "bytes" - "context" - "crypto/rand" - "encoding/binary" - "errors" - "fmt" - "io" - "net" - "net/netip" - "os" - "regexp" - "strconv" - "strings" - "syscall" - "time" - - "github.com/tailscale/wireguard-go/tun" - - "golang.org/x/net/dns/dnsmessage" - "gvisor.dev/gvisor/pkg/buffer" - "gvisor.dev/gvisor/pkg/tcpip" - "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" - "gvisor.dev/gvisor/pkg/tcpip/header" - "gvisor.dev/gvisor/pkg/tcpip/link/channel" - "gvisor.dev/gvisor/pkg/tcpip/network/ipv4" - "gvisor.dev/gvisor/pkg/tcpip/network/ipv6" - "gvisor.dev/gvisor/pkg/tcpip/stack" - "gvisor.dev/gvisor/pkg/tcpip/transport/icmp" - "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" - "gvisor.dev/gvisor/pkg/tcpip/transport/udp" - "gvisor.dev/gvisor/pkg/waiter" -) - -type netTun struct { - ep *channel.Endpoint - stack *stack.Stack - events chan tun.Event - incomingPacket chan *buffer.View - mtu int - dnsServers []netip.Addr - hasV4, hasV6 bool -} - -type Net netTun - -func CreateNetTUN(localAddresses, dnsServers []netip.Addr, mtu int) (tun.Device, *Net, error) { - opts := stack.Options{ - NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol}, - TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol6, icmp.NewProtocol4}, - HandleLocal: true, - } - dev := &netTun{ - ep: channel.New(1024, uint32(mtu), ""), - stack: stack.New(opts), - events: make(chan tun.Event, 10), - incomingPacket: make(chan *buffer.View), - dnsServers: dnsServers, - mtu: mtu, - } - sackEnabledOpt := tcpip.TCPSACKEnabled(true) // TCP SACK is disabled by default - tcpipErr := dev.stack.SetTransportProtocolOption(tcp.ProtocolNumber, &sackEnabledOpt) - if tcpipErr != nil { - return nil, nil, fmt.Errorf("could not enable TCP SACK: %v", tcpipErr) - } - dev.ep.AddNotify(dev) - tcpipErr = dev.stack.CreateNIC(1, dev.ep) - if tcpipErr != nil { - return nil, nil, fmt.Errorf("CreateNIC: %v", tcpipErr) - } - for _, ip := range localAddresses { - var protoNumber tcpip.NetworkProtocolNumber - if ip.Is4() { - protoNumber = ipv4.ProtocolNumber - } else if ip.Is6() { - protoNumber = ipv6.ProtocolNumber - } - protoAddr := tcpip.ProtocolAddress{ - Protocol: protoNumber, - AddressWithPrefix: tcpip.AddrFromSlice(ip.AsSlice()).WithPrefix(), - } - tcpipErr := dev.stack.AddProtocolAddress(1, protoAddr, stack.AddressProperties{}) - if tcpipErr != nil { - return nil, nil, fmt.Errorf("AddProtocolAddress(%v): %v", ip, tcpipErr) - } - if ip.Is4() { - dev.hasV4 = true - } else if ip.Is6() { - dev.hasV6 = true - } - } - if dev.hasV4 { - dev.stack.AddRoute(tcpip.Route{Destination: header.IPv4EmptySubnet, NIC: 1}) - } - if dev.hasV6 { - dev.stack.AddRoute(tcpip.Route{Destination: header.IPv6EmptySubnet, NIC: 1}) - } - - dev.events <- tun.EventUp - return dev, (*Net)(dev), nil -} - -func (tun *netTun) Name() (string, error) { - return "go", nil -} - -func (tun *netTun) File() *os.File { - return nil -} - -func (tun *netTun) Events() <-chan tun.Event { - return tun.events -} - -func (tun *netTun) Read(buf [][]byte, sizes []int, offset int) (int, error) { - view, ok := <-tun.incomingPacket - if !ok { - return 0, os.ErrClosed - } - - n, err := view.Read(buf[0][offset:]) - if err != nil { - return 0, err - } - sizes[0] = n - return 1, nil -} - -func (tun *netTun) Write(buf [][]byte, offset int) (int, error) { - for _, buf := range buf { - packet := buf[offset:] - if len(packet) == 0 { - continue - } - - pkb := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buffer.MakeWithData(packet)}) - switch packet[0] >> 4 { - case 4: - tun.ep.InjectInbound(header.IPv4ProtocolNumber, pkb) - case 6: - tun.ep.InjectInbound(header.IPv6ProtocolNumber, pkb) - default: - return 0, syscall.EAFNOSUPPORT - } - } - return len(buf), nil -} - -func (tun *netTun) WriteNotify() { - pkt := tun.ep.Read() - if pkt.IsNil() { - return - } - - view := pkt.ToView() - pkt.DecRef() - - tun.incomingPacket <- view -} - -func (tun *netTun) Close() error { - tun.stack.RemoveNIC(1) - - if tun.events != nil { - close(tun.events) - } - - tun.ep.Close() - - if tun.incomingPacket != nil { - close(tun.incomingPacket) - } - - return nil -} - -func (tun *netTun) MTU() (int, error) { - return tun.mtu, nil -} - -func (tun *netTun) BatchSize() int { - return 1 -} - -func convertToFullAddr(endpoint netip.AddrPort) (tcpip.FullAddress, tcpip.NetworkProtocolNumber) { - var protoNumber tcpip.NetworkProtocolNumber - if endpoint.Addr().Is4() { - protoNumber = ipv4.ProtocolNumber - } else { - protoNumber = ipv6.ProtocolNumber - } - return tcpip.FullAddress{ - NIC: 1, - Addr: tcpip.AddrFromSlice(endpoint.Addr().AsSlice()), - Port: endpoint.Port(), - }, protoNumber -} - -func (net *Net) DialContextTCPAddrPort(ctx context.Context, addr netip.AddrPort) (*gonet.TCPConn, error) { - fa, pn := convertToFullAddr(addr) - return gonet.DialContextTCP(ctx, net.stack, fa, pn) -} - -func (net *Net) DialContextTCP(ctx context.Context, addr *net.TCPAddr) (*gonet.TCPConn, error) { - if addr == nil { - return net.DialContextTCPAddrPort(ctx, netip.AddrPort{}) - } - ip, _ := netip.AddrFromSlice(addr.IP) - return net.DialContextTCPAddrPort(ctx, netip.AddrPortFrom(ip, uint16(addr.Port))) -} - -func (net *Net) DialTCPAddrPort(addr netip.AddrPort) (*gonet.TCPConn, error) { - fa, pn := convertToFullAddr(addr) - return gonet.DialTCP(net.stack, fa, pn) -} - -func (net *Net) DialTCP(addr *net.TCPAddr) (*gonet.TCPConn, error) { - if addr == nil { - return net.DialTCPAddrPort(netip.AddrPort{}) - } - ip, _ := netip.AddrFromSlice(addr.IP) - return net.DialTCPAddrPort(netip.AddrPortFrom(ip, uint16(addr.Port))) -} - -func (net *Net) ListenTCPAddrPort(addr netip.AddrPort) (*gonet.TCPListener, error) { - fa, pn := convertToFullAddr(addr) - return gonet.ListenTCP(net.stack, fa, pn) -} - -func (net *Net) ListenTCP(addr *net.TCPAddr) (*gonet.TCPListener, error) { - if addr == nil { - return net.ListenTCPAddrPort(netip.AddrPort{}) - } - ip, _ := netip.AddrFromSlice(addr.IP) - return net.ListenTCPAddrPort(netip.AddrPortFrom(ip, uint16(addr.Port))) -} - -func (net *Net) DialUDPAddrPort(laddr, raddr netip.AddrPort) (*gonet.UDPConn, error) { - var lfa, rfa *tcpip.FullAddress - var pn tcpip.NetworkProtocolNumber - if laddr.IsValid() || laddr.Port() > 0 { - var addr tcpip.FullAddress - addr, pn = convertToFullAddr(laddr) - lfa = &addr - } - if raddr.IsValid() || raddr.Port() > 0 { - var addr tcpip.FullAddress - addr, pn = convertToFullAddr(raddr) - rfa = &addr - } - return gonet.DialUDP(net.stack, lfa, rfa, pn) -} - -func (net *Net) ListenUDPAddrPort(laddr netip.AddrPort) (*gonet.UDPConn, error) { - return net.DialUDPAddrPort(laddr, netip.AddrPort{}) -} - -func (net *Net) DialUDP(laddr, raddr *net.UDPAddr) (*gonet.UDPConn, error) { - var la, ra netip.AddrPort - if laddr != nil { - ip, _ := netip.AddrFromSlice(laddr.IP) - la = netip.AddrPortFrom(ip, uint16(laddr.Port)) - } - if raddr != nil { - ip, _ := netip.AddrFromSlice(raddr.IP) - ra = netip.AddrPortFrom(ip, uint16(raddr.Port)) - } - return net.DialUDPAddrPort(la, ra) -} - -func (net *Net) ListenUDP(laddr *net.UDPAddr) (*gonet.UDPConn, error) { - return net.DialUDP(laddr, nil) -} - -type PingConn struct { - laddr PingAddr - raddr PingAddr - wq waiter.Queue - ep tcpip.Endpoint - deadline *time.Timer -} - -type PingAddr struct{ addr netip.Addr } - -func (ia PingAddr) String() string { - return ia.addr.String() -} - -func (ia PingAddr) Network() string { - if ia.addr.Is4() { - return "ping4" - } else if ia.addr.Is6() { - return "ping6" - } - return "ping" -} - -func (ia PingAddr) Addr() netip.Addr { - return ia.addr -} - -func PingAddrFromAddr(addr netip.Addr) *PingAddr { - return &PingAddr{addr} -} - -func (net *Net) DialPingAddr(laddr, raddr netip.Addr) (*PingConn, error) { - if !laddr.IsValid() && !raddr.IsValid() { - return nil, errors.New("ping dial: invalid address") - } - v6 := laddr.Is6() || raddr.Is6() - bind := laddr.IsValid() - if !bind { - if v6 { - laddr = netip.IPv6Unspecified() - } else { - laddr = netip.IPv4Unspecified() - } - } - - tn := icmp.ProtocolNumber4 - pn := ipv4.ProtocolNumber - if v6 { - tn = icmp.ProtocolNumber6 - pn = ipv6.ProtocolNumber - } - - pc := &PingConn{ - laddr: PingAddr{laddr}, - deadline: time.NewTimer(time.Hour << 10), - } - pc.deadline.Stop() - - ep, tcpipErr := net.stack.NewEndpoint(tn, pn, &pc.wq) - if tcpipErr != nil { - return nil, fmt.Errorf("ping socket: endpoint: %s", tcpipErr) - } - pc.ep = ep - - if bind { - fa, _ := convertToFullAddr(netip.AddrPortFrom(laddr, 0)) - if tcpipErr = pc.ep.Bind(fa); tcpipErr != nil { - return nil, fmt.Errorf("ping bind: %s", tcpipErr) - } - } - - if raddr.IsValid() { - pc.raddr = PingAddr{raddr} - fa, _ := convertToFullAddr(netip.AddrPortFrom(raddr, 0)) - if tcpipErr = pc.ep.Connect(fa); tcpipErr != nil { - return nil, fmt.Errorf("ping connect: %s", tcpipErr) - } - } - - return pc, nil -} - -func (net *Net) ListenPingAddr(laddr netip.Addr) (*PingConn, error) { - return net.DialPingAddr(laddr, netip.Addr{}) -} - -func (net *Net) DialPing(laddr, raddr *PingAddr) (*PingConn, error) { - var la, ra netip.Addr - if laddr != nil { - la = laddr.addr - } - if raddr != nil { - ra = raddr.addr - } - return net.DialPingAddr(la, ra) -} - -func (net *Net) ListenPing(laddr *PingAddr) (*PingConn, error) { - var la netip.Addr - if laddr != nil { - la = laddr.addr - } - return net.ListenPingAddr(la) -} - -func (pc *PingConn) LocalAddr() net.Addr { - return pc.laddr -} - -func (pc *PingConn) RemoteAddr() net.Addr { - return pc.raddr -} - -func (pc *PingConn) Close() error { - pc.deadline.Reset(0) - pc.ep.Close() - return nil -} - -func (pc *PingConn) SetWriteDeadline(t time.Time) error { - return errors.New("not implemented") -} - -func (pc *PingConn) WriteTo(p []byte, addr net.Addr) (n int, err error) { - var na netip.Addr - switch v := addr.(type) { - case *PingAddr: - na = v.addr - case *net.IPAddr: - na, _ = netip.AddrFromSlice(v.IP) - default: - return 0, fmt.Errorf("ping write: wrong net.Addr type") - } - if !((na.Is4() && pc.laddr.addr.Is4()) || (na.Is6() && pc.laddr.addr.Is6())) { - return 0, fmt.Errorf("ping write: mismatched protocols") - } - - buf := bytes.NewReader(p) - rfa, _ := convertToFullAddr(netip.AddrPortFrom(na, 0)) - // won't block, no deadlines - n64, tcpipErr := pc.ep.Write(buf, tcpip.WriteOptions{ - To: &rfa, - }) - if tcpipErr != nil { - return int(n64), fmt.Errorf("ping write: %s", tcpipErr) - } - - return int(n64), nil -} - -func (pc *PingConn) Write(p []byte) (n int, err error) { - return pc.WriteTo(p, &pc.raddr) -} - -func (pc *PingConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) { - e, notifyCh := waiter.NewChannelEntry(waiter.EventIn) - pc.wq.EventRegister(&e) - defer pc.wq.EventUnregister(&e) - - select { - case <-pc.deadline.C: - return 0, nil, os.ErrDeadlineExceeded - case <-notifyCh: - } - - w := tcpip.SliceWriter(p) - - res, tcpipErr := pc.ep.Read(&w, tcpip.ReadOptions{ - NeedRemoteAddr: true, - }) - if tcpipErr != nil { - return 0, nil, fmt.Errorf("ping read: %s", tcpipErr) - } - - remoteAddr, _ := netip.AddrFromSlice(res.RemoteAddr.Addr.AsSlice()) - return res.Count, &PingAddr{remoteAddr}, nil -} - -func (pc *PingConn) Read(p []byte) (n int, err error) { - n, _, err = pc.ReadFrom(p) - return -} - -func (pc *PingConn) SetDeadline(t time.Time) error { - // pc.SetWriteDeadline is unimplemented - - return pc.SetReadDeadline(t) -} - -func (pc *PingConn) SetReadDeadline(t time.Time) error { - pc.deadline.Reset(time.Until(t)) - return nil -} - -var ( - errNoSuchHost = errors.New("no such host") - errLameReferral = errors.New("lame referral") - errCannotUnmarshalDNSMessage = errors.New("cannot unmarshal DNS message") - errCannotMarshalDNSMessage = errors.New("cannot marshal DNS message") - errServerMisbehaving = errors.New("server misbehaving") - errInvalidDNSResponse = errors.New("invalid DNS response") - errNoAnswerFromDNSServer = errors.New("no answer from DNS server") - errServerTemporarilyMisbehaving = errors.New("server misbehaving") - errCanceled = errors.New("operation was canceled") - errTimeout = errors.New("i/o timeout") - errNumericPort = errors.New("port must be numeric") - errNoSuitableAddress = errors.New("no suitable address found") - errMissingAddress = errors.New("missing address") -) - -func (net *Net) LookupHost(host string) (addrs []string, err error) { - return net.LookupContextHost(context.Background(), host) -} - -func isDomainName(s string) bool { - l := len(s) - if l == 0 || l > 254 || l == 254 && s[l-1] != '.' { - return false - } - last := byte('.') - nonNumeric := false - partlen := 0 - for i := 0; i < len(s); i++ { - c := s[i] - switch { - default: - return false - case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_': - nonNumeric = true - partlen++ - case '0' <= c && c <= '9': - partlen++ - case c == '-': - if last == '.' { - return false - } - partlen++ - nonNumeric = true - case c == '.': - if last == '.' || last == '-' { - return false - } - if partlen > 63 || partlen == 0 { - return false - } - partlen = 0 - } - last = c - } - if last == '-' || partlen > 63 { - return false - } - return nonNumeric -} - -func randU16() uint16 { - var b [2]byte - _, err := rand.Read(b[:]) - if err != nil { - panic(err) - } - return binary.LittleEndian.Uint16(b[:]) -} - -func newRequest(q dnsmessage.Question) (id uint16, udpReq, tcpReq []byte, err error) { - id = randU16() - b := dnsmessage.NewBuilder(make([]byte, 2, 514), dnsmessage.Header{ID: id, RecursionDesired: true}) - b.EnableCompression() - if err := b.StartQuestions(); err != nil { - return 0, nil, nil, err - } - if err := b.Question(q); err != nil { - return 0, nil, nil, err - } - tcpReq, err = b.Finish() - udpReq = tcpReq[2:] - l := len(tcpReq) - 2 - tcpReq[0] = byte(l >> 8) - tcpReq[1] = byte(l) - return id, udpReq, tcpReq, err -} - -func equalASCIIName(x, y dnsmessage.Name) bool { - if x.Length != y.Length { - return false - } - for i := 0; i < int(x.Length); i++ { - a := x.Data[i] - b := y.Data[i] - if 'A' <= a && a <= 'Z' { - a += 0x20 - } - if 'A' <= b && b <= 'Z' { - b += 0x20 - } - if a != b { - return false - } - } - return true -} - -func checkResponse(reqID uint16, reqQues dnsmessage.Question, respHdr dnsmessage.Header, respQues dnsmessage.Question) bool { - if !respHdr.Response { - return false - } - if reqID != respHdr.ID { - return false - } - if reqQues.Type != respQues.Type || reqQues.Class != respQues.Class || !equalASCIIName(reqQues.Name, respQues.Name) { - return false - } - return true -} - -func dnsPacketRoundTrip(c net.Conn, id uint16, query dnsmessage.Question, b []byte) (dnsmessage.Parser, dnsmessage.Header, error) { - if _, err := c.Write(b); err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, err - } - b = make([]byte, 512) - for { - n, err := c.Read(b) - if err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, err - } - var p dnsmessage.Parser - h, err := p.Start(b[:n]) - if err != nil { - continue - } - q, err := p.Question() - if err != nil || !checkResponse(id, query, h, q) { - continue - } - return p, h, nil - } -} - -func dnsStreamRoundTrip(c net.Conn, id uint16, query dnsmessage.Question, b []byte) (dnsmessage.Parser, dnsmessage.Header, error) { - if _, err := c.Write(b); err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, err - } - b = make([]byte, 1280) - if _, err := io.ReadFull(c, b[:2]); err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, err - } - l := int(b[0])<<8 | int(b[1]) - if l > len(b) { - b = make([]byte, l) - } - n, err := io.ReadFull(c, b[:l]) - if err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, err - } - var p dnsmessage.Parser - h, err := p.Start(b[:n]) - if err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, errCannotUnmarshalDNSMessage - } - q, err := p.Question() - if err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, errCannotUnmarshalDNSMessage - } - if !checkResponse(id, query, h, q) { - return dnsmessage.Parser{}, dnsmessage.Header{}, errInvalidDNSResponse - } - return p, h, nil -} - -func (tnet *Net) exchange(ctx context.Context, server netip.Addr, q dnsmessage.Question, timeout time.Duration) (dnsmessage.Parser, dnsmessage.Header, error) { - q.Class = dnsmessage.ClassINET - id, udpReq, tcpReq, err := newRequest(q) - if err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, errCannotMarshalDNSMessage - } - - for _, useUDP := range []bool{true, false} { - ctx, cancel := context.WithDeadline(ctx, time.Now().Add(timeout)) - defer cancel() - - var c net.Conn - var err error - if useUDP { - c, err = tnet.DialUDPAddrPort(netip.AddrPort{}, netip.AddrPortFrom(server, 53)) - } else { - c, err = tnet.DialContextTCPAddrPort(ctx, netip.AddrPortFrom(server, 53)) - } - - if err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, err - } - if d, ok := ctx.Deadline(); ok && !d.IsZero() { - err := c.SetDeadline(d) - if err != nil { - return dnsmessage.Parser{}, dnsmessage.Header{}, err - } - } - var p dnsmessage.Parser - var h dnsmessage.Header - if useUDP { - p, h, err = dnsPacketRoundTrip(c, id, q, udpReq) - } else { - p, h, err = dnsStreamRoundTrip(c, id, q, tcpReq) - } - c.Close() - if err != nil { - if err == context.Canceled { - err = errCanceled - } else if err == context.DeadlineExceeded { - err = errTimeout - } - return dnsmessage.Parser{}, dnsmessage.Header{}, err - } - if err := p.SkipQuestion(); err != dnsmessage.ErrSectionDone { - return dnsmessage.Parser{}, dnsmessage.Header{}, errInvalidDNSResponse - } - if h.Truncated { - continue - } - return p, h, nil - } - return dnsmessage.Parser{}, dnsmessage.Header{}, errNoAnswerFromDNSServer -} - -func checkHeader(p *dnsmessage.Parser, h dnsmessage.Header) error { - if h.RCode == dnsmessage.RCodeNameError { - return errNoSuchHost - } - _, err := p.AnswerHeader() - if err != nil && err != dnsmessage.ErrSectionDone { - return errCannotUnmarshalDNSMessage - } - if h.RCode == dnsmessage.RCodeSuccess && !h.Authoritative && !h.RecursionAvailable && err == dnsmessage.ErrSectionDone { - return errLameReferral - } - if h.RCode != dnsmessage.RCodeSuccess && h.RCode != dnsmessage.RCodeNameError { - if h.RCode == dnsmessage.RCodeServerFailure { - return errServerTemporarilyMisbehaving - } - return errServerMisbehaving - } - return nil -} - -func skipToAnswer(p *dnsmessage.Parser, qtype dnsmessage.Type) error { - for { - h, err := p.AnswerHeader() - if err == dnsmessage.ErrSectionDone { - return errNoSuchHost - } - if err != nil { - return errCannotUnmarshalDNSMessage - } - if h.Type == qtype { - return nil - } - if err := p.SkipAnswer(); err != nil { - return errCannotUnmarshalDNSMessage - } - } -} - -func (tnet *Net) tryOneName(ctx context.Context, name string, qtype dnsmessage.Type) (dnsmessage.Parser, string, error) { - var lastErr error - - n, err := dnsmessage.NewName(name) - if err != nil { - return dnsmessage.Parser{}, "", errCannotMarshalDNSMessage - } - q := dnsmessage.Question{ - Name: n, - Type: qtype, - Class: dnsmessage.ClassINET, - } - - for i := 0; i < 2; i++ { - for _, server := range tnet.dnsServers { - p, h, err := tnet.exchange(ctx, server, q, time.Second*5) - if err != nil { - dnsErr := &net.DNSError{ - Err: err.Error(), - Name: name, - Server: server.String(), - } - if nerr, ok := err.(net.Error); ok && nerr.Timeout() { - dnsErr.IsTimeout = true - } - if _, ok := err.(*net.OpError); ok { - dnsErr.IsTemporary = true - } - lastErr = dnsErr - continue - } - - if err := checkHeader(&p, h); err != nil { - dnsErr := &net.DNSError{ - Err: err.Error(), - Name: name, - Server: server.String(), - } - if err == errServerTemporarilyMisbehaving { - dnsErr.IsTemporary = true - } - if err == errNoSuchHost { - dnsErr.IsNotFound = true - return p, server.String(), dnsErr - } - lastErr = dnsErr - continue - } - - err = skipToAnswer(&p, qtype) - if err == nil { - return p, server.String(), nil - } - lastErr = &net.DNSError{ - Err: err.Error(), - Name: name, - Server: server.String(), - } - if err == errNoSuchHost { - lastErr.(*net.DNSError).IsNotFound = true - return p, server.String(), lastErr - } - } - } - return dnsmessage.Parser{}, "", lastErr -} - -func (tnet *Net) LookupContextHost(ctx context.Context, host string) ([]string, error) { - if host == "" || (!tnet.hasV6 && !tnet.hasV4) { - return nil, &net.DNSError{Err: errNoSuchHost.Error(), Name: host, IsNotFound: true} - } - zlen := len(host) - if strings.IndexByte(host, ':') != -1 { - if zidx := strings.LastIndexByte(host, '%'); zidx != -1 { - zlen = zidx - } - } - if ip, err := netip.ParseAddr(host[:zlen]); err == nil { - return []string{ip.String()}, nil - } - - if !isDomainName(host) { - return nil, &net.DNSError{Err: errNoSuchHost.Error(), Name: host, IsNotFound: true} - } - type result struct { - p dnsmessage.Parser - server string - error - } - var addrsV4, addrsV6 []netip.Addr - lanes := 0 - if tnet.hasV4 { - lanes++ - } - if tnet.hasV6 { - lanes++ - } - lane := make(chan result, lanes) - var lastErr error - if tnet.hasV4 { - go func() { - p, server, err := tnet.tryOneName(ctx, host+".", dnsmessage.TypeA) - lane <- result{p, server, err} - }() - } - if tnet.hasV6 { - go func() { - p, server, err := tnet.tryOneName(ctx, host+".", dnsmessage.TypeAAAA) - lane <- result{p, server, err} - }() - } - for l := 0; l < lanes; l++ { - result := <-lane - if result.error != nil { - if lastErr == nil { - lastErr = result.error - } - continue - } - - loop: - for { - h, err := result.p.AnswerHeader() - if err != nil && err != dnsmessage.ErrSectionDone { - lastErr = &net.DNSError{ - Err: errCannotMarshalDNSMessage.Error(), - Name: host, - Server: result.server, - } - } - if err != nil { - break - } - switch h.Type { - case dnsmessage.TypeA: - a, err := result.p.AResource() - if err != nil { - lastErr = &net.DNSError{ - Err: errCannotMarshalDNSMessage.Error(), - Name: host, - Server: result.server, - } - break loop - } - addrsV4 = append(addrsV4, netip.AddrFrom4(a.A)) - - case dnsmessage.TypeAAAA: - aaaa, err := result.p.AAAAResource() - if err != nil { - lastErr = &net.DNSError{ - Err: errCannotMarshalDNSMessage.Error(), - Name: host, - Server: result.server, - } - break loop - } - addrsV6 = append(addrsV6, netip.AddrFrom16(aaaa.AAAA)) - - default: - if err := result.p.SkipAnswer(); err != nil { - lastErr = &net.DNSError{ - Err: errCannotMarshalDNSMessage.Error(), - Name: host, - Server: result.server, - } - break loop - } - continue - } - } - } - // We don't do RFC6724. Instead just put V6 addresses first if an IPv6 address is enabled - var addrs []netip.Addr - if tnet.hasV6 { - addrs = append(addrsV6, addrsV4...) - } else { - addrs = append(addrsV4, addrsV6...) - } - - if len(addrs) == 0 && lastErr != nil { - return nil, lastErr - } - saddrs := make([]string, 0, len(addrs)) - for _, ip := range addrs { - saddrs = append(saddrs, ip.String()) - } - return saddrs, nil -} - -func partialDeadline(now, deadline time.Time, addrsRemaining int) (time.Time, error) { - if deadline.IsZero() { - return deadline, nil - } - timeRemaining := deadline.Sub(now) - if timeRemaining <= 0 { - return time.Time{}, errTimeout - } - timeout := timeRemaining / time.Duration(addrsRemaining) - const saneMinimum = 2 * time.Second - if timeout < saneMinimum { - if timeRemaining < saneMinimum { - timeout = timeRemaining - } else { - timeout = saneMinimum - } - } - return now.Add(timeout), nil -} - -var protoSplitter = regexp.MustCompile(`^(tcp|udp|ping)(4|6)?$`) - -func (tnet *Net) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - if ctx == nil { - panic("nil context") - } - var acceptV4, acceptV6 bool - matches := protoSplitter.FindStringSubmatch(network) - if matches == nil { - return nil, &net.OpError{Op: "dial", Err: net.UnknownNetworkError(network)} - } else if len(matches[2]) == 0 { - acceptV4 = true - acceptV6 = true - } else { - acceptV4 = matches[2][0] == '4' - acceptV6 = !acceptV4 - } - var host string - var port int - if matches[1] == "ping" { - host = address - } else { - var sport string - var err error - host, sport, err = net.SplitHostPort(address) - if err != nil { - return nil, &net.OpError{Op: "dial", Err: err} - } - port, err = strconv.Atoi(sport) - if err != nil || port < 0 || port > 65535 { - return nil, &net.OpError{Op: "dial", Err: errNumericPort} - } - } - allAddr, err := tnet.LookupContextHost(ctx, host) - if err != nil { - return nil, &net.OpError{Op: "dial", Err: err} - } - var addrs []netip.AddrPort - for _, addr := range allAddr { - ip, err := netip.ParseAddr(addr) - if err == nil && ((ip.Is4() && acceptV4) || (ip.Is6() && acceptV6)) { - addrs = append(addrs, netip.AddrPortFrom(ip, uint16(port))) - } - } - if len(addrs) == 0 && len(allAddr) != 0 { - return nil, &net.OpError{Op: "dial", Err: errNoSuitableAddress} - } - - var firstErr error - for i, addr := range addrs { - select { - case <-ctx.Done(): - err := ctx.Err() - if err == context.Canceled { - err = errCanceled - } else if err == context.DeadlineExceeded { - err = errTimeout - } - return nil, &net.OpError{Op: "dial", Err: err} - default: - } - - dialCtx := ctx - if deadline, hasDeadline := ctx.Deadline(); hasDeadline { - partialDeadline, err := partialDeadline(time.Now(), deadline, len(addrs)-i) - if err != nil { - if firstErr == nil { - firstErr = &net.OpError{Op: "dial", Err: err} - } - break - } - if partialDeadline.Before(deadline) { - var cancel context.CancelFunc - dialCtx, cancel = context.WithDeadline(ctx, partialDeadline) - defer cancel() - } - } - - var c net.Conn - switch matches[1] { - case "tcp": - c, err = tnet.DialContextTCPAddrPort(dialCtx, addr) - case "udp": - c, err = tnet.DialUDPAddrPort(netip.AddrPort{}, addr) - case "ping": - c, err = tnet.DialPingAddr(netip.Addr{}, addr.Addr()) - } - if err == nil { - return c, nil - } - if firstErr == nil { - firstErr = err - } - } - if firstErr == nil { - firstErr = &net.OpError{Op: "dial", Err: errMissingAddress} - } - return nil, firstErr -} - -func (tnet *Net) Dial(network, address string) (net.Conn, error) { - return tnet.DialContext(context.Background(), network, address) -} diff --git a/tun/offload_linux_test.go b/tun/offload_linux_test.go deleted file mode 100644 index 4070378..0000000 --- a/tun/offload_linux_test.go +++ /dev/null @@ -1,764 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package tun - -import ( - "net/netip" - "testing" - - "github.com/tailscale/wireguard-go/conn" - "golang.org/x/sys/unix" - "gvisor.dev/gvisor/pkg/tcpip" - "gvisor.dev/gvisor/pkg/tcpip/header" -) - -const ( - offset = virtioNetHdrLen -) - -var ( - ip4PortA = netip.MustParseAddrPort("192.0.2.1:1") - ip4PortB = netip.MustParseAddrPort("192.0.2.2:1") - ip4PortC = netip.MustParseAddrPort("192.0.2.3:1") - ip6PortA = netip.MustParseAddrPort("[2001:db8::1]:1") - ip6PortB = netip.MustParseAddrPort("[2001:db8::2]:1") - ip6PortC = netip.MustParseAddrPort("[2001:db8::3]:1") -) - -func udp4PacketMutateIPFields(srcIPPort, dstIPPort netip.AddrPort, payloadLen int, ipFn func(*header.IPv4Fields)) []byte { - totalLen := 28 + payloadLen - b := make([]byte, offset+int(totalLen), 65535) - ipv4H := header.IPv4(b[offset:]) - srcAs4 := srcIPPort.Addr().As4() - dstAs4 := dstIPPort.Addr().As4() - ipFields := &header.IPv4Fields{ - SrcAddr: tcpip.AddrFromSlice(srcAs4[:]), - DstAddr: tcpip.AddrFromSlice(dstAs4[:]), - Protocol: unix.IPPROTO_UDP, - TTL: 64, - TotalLength: uint16(totalLen), - } - if ipFn != nil { - ipFn(ipFields) - } - ipv4H.Encode(ipFields) - udpH := header.UDP(b[offset+20:]) - udpH.Encode(&header.UDPFields{ - SrcPort: srcIPPort.Port(), - DstPort: dstIPPort.Port(), - Length: uint16(payloadLen + udphLen), - }) - ipv4H.SetChecksum(^ipv4H.CalculateChecksum()) - pseudoCsum := header.PseudoHeaderChecksum(unix.IPPROTO_UDP, ipv4H.SourceAddress(), ipv4H.DestinationAddress(), uint16(udphLen+payloadLen)) - udpH.SetChecksum(^udpH.CalculateChecksum(pseudoCsum)) - return b -} - -func udp6Packet(srcIPPort, dstIPPort netip.AddrPort, payloadLen int) []byte { - return udp6PacketMutateIPFields(srcIPPort, dstIPPort, payloadLen, nil) -} - -func udp6PacketMutateIPFields(srcIPPort, dstIPPort netip.AddrPort, payloadLen int, ipFn func(*header.IPv6Fields)) []byte { - totalLen := 48 + payloadLen - b := make([]byte, offset+int(totalLen), 65535) - ipv6H := header.IPv6(b[offset:]) - srcAs16 := srcIPPort.Addr().As16() - dstAs16 := dstIPPort.Addr().As16() - ipFields := &header.IPv6Fields{ - SrcAddr: tcpip.AddrFromSlice(srcAs16[:]), - DstAddr: tcpip.AddrFromSlice(dstAs16[:]), - TransportProtocol: unix.IPPROTO_UDP, - HopLimit: 64, - PayloadLength: uint16(payloadLen + udphLen), - } - if ipFn != nil { - ipFn(ipFields) - } - ipv6H.Encode(ipFields) - udpH := header.UDP(b[offset+40:]) - udpH.Encode(&header.UDPFields{ - SrcPort: srcIPPort.Port(), - DstPort: dstIPPort.Port(), - Length: uint16(payloadLen + udphLen), - }) - pseudoCsum := header.PseudoHeaderChecksum(unix.IPPROTO_UDP, ipv6H.SourceAddress(), ipv6H.DestinationAddress(), uint16(udphLen+payloadLen)) - udpH.SetChecksum(^udpH.CalculateChecksum(pseudoCsum)) - return b -} - -func udp4Packet(srcIPPort, dstIPPort netip.AddrPort, payloadLen int) []byte { - return udp4PacketMutateIPFields(srcIPPort, dstIPPort, payloadLen, nil) -} - -func tcp4PacketMutateIPFields(srcIPPort, dstIPPort netip.AddrPort, flags header.TCPFlags, segmentSize, seq uint32, ipFn func(*header.IPv4Fields)) []byte { - totalLen := 40 + segmentSize - b := make([]byte, offset+int(totalLen), 65535) - ipv4H := header.IPv4(b[offset:]) - srcAs4 := srcIPPort.Addr().As4() - dstAs4 := dstIPPort.Addr().As4() - ipFields := &header.IPv4Fields{ - SrcAddr: tcpip.AddrFromSlice(srcAs4[:]), - DstAddr: tcpip.AddrFromSlice(dstAs4[:]), - Protocol: unix.IPPROTO_TCP, - TTL: 64, - TotalLength: uint16(totalLen), - } - if ipFn != nil { - ipFn(ipFields) - } - ipv4H.Encode(ipFields) - tcpH := header.TCP(b[offset+20:]) - tcpH.Encode(&header.TCPFields{ - SrcPort: srcIPPort.Port(), - DstPort: dstIPPort.Port(), - SeqNum: seq, - AckNum: 1, - DataOffset: 20, - Flags: flags, - WindowSize: 3000, - }) - ipv4H.SetChecksum(^ipv4H.CalculateChecksum()) - pseudoCsum := header.PseudoHeaderChecksum(unix.IPPROTO_TCP, ipv4H.SourceAddress(), ipv4H.DestinationAddress(), uint16(20+segmentSize)) - tcpH.SetChecksum(^tcpH.CalculateChecksum(pseudoCsum)) - return b -} - -func tcp4Packet(srcIPPort, dstIPPort netip.AddrPort, flags header.TCPFlags, segmentSize, seq uint32) []byte { - return tcp4PacketMutateIPFields(srcIPPort, dstIPPort, flags, segmentSize, seq, nil) -} - -func tcp6PacketMutateIPFields(srcIPPort, dstIPPort netip.AddrPort, flags header.TCPFlags, segmentSize, seq uint32, ipFn func(*header.IPv6Fields)) []byte { - totalLen := 60 + segmentSize - b := make([]byte, offset+int(totalLen), 65535) - ipv6H := header.IPv6(b[offset:]) - srcAs16 := srcIPPort.Addr().As16() - dstAs16 := dstIPPort.Addr().As16() - ipFields := &header.IPv6Fields{ - SrcAddr: tcpip.AddrFromSlice(srcAs16[:]), - DstAddr: tcpip.AddrFromSlice(dstAs16[:]), - TransportProtocol: unix.IPPROTO_TCP, - HopLimit: 64, - PayloadLength: uint16(segmentSize + 20), - } - if ipFn != nil { - ipFn(ipFields) - } - ipv6H.Encode(ipFields) - tcpH := header.TCP(b[offset+40:]) - tcpH.Encode(&header.TCPFields{ - SrcPort: srcIPPort.Port(), - DstPort: dstIPPort.Port(), - SeqNum: seq, - AckNum: 1, - DataOffset: 20, - Flags: flags, - WindowSize: 3000, - }) - pseudoCsum := header.PseudoHeaderChecksum(unix.IPPROTO_TCP, ipv6H.SourceAddress(), ipv6H.DestinationAddress(), uint16(20+segmentSize)) - tcpH.SetChecksum(^tcpH.CalculateChecksum(pseudoCsum)) - return b -} - -func tcp6Packet(srcIPPort, dstIPPort netip.AddrPort, flags header.TCPFlags, segmentSize, seq uint32) []byte { - return tcp6PacketMutateIPFields(srcIPPort, dstIPPort, flags, segmentSize, seq, nil) -} - -func Test_handleVirtioRead(t *testing.T) { - tests := []struct { - name string - hdr virtioNetHdr - pktIn []byte - wantLens []int - wantErr bool - }{ - { - "tcp4", - virtioNetHdr{ - flags: unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, - gsoType: unix.VIRTIO_NET_HDR_GSO_TCPV4, - gsoSize: 100, - hdrLen: 40, - csumStart: 20, - csumOffset: 16, - }, - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck|header.TCPFlagPsh, 200, 1), - []int{140, 140}, - false, - }, - { - "tcp6", - virtioNetHdr{ - flags: unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, - gsoType: unix.VIRTIO_NET_HDR_GSO_TCPV6, - gsoSize: 100, - hdrLen: 60, - csumStart: 40, - csumOffset: 16, - }, - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck|header.TCPFlagPsh, 200, 1), - []int{160, 160}, - false, - }, - { - "udp4", - virtioNetHdr{ - flags: unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, - gsoType: unix.VIRTIO_NET_HDR_GSO_UDP_L4, - gsoSize: 100, - hdrLen: 28, - csumStart: 20, - csumOffset: 6, - }, - udp4Packet(ip4PortA, ip4PortB, 200), - []int{128, 128}, - false, - }, - { - "udp6", - virtioNetHdr{ - flags: unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, - gsoType: unix.VIRTIO_NET_HDR_GSO_UDP_L4, - gsoSize: 100, - hdrLen: 48, - csumStart: 40, - csumOffset: 6, - }, - udp6Packet(ip6PortA, ip6PortB, 200), - []int{148, 148}, - false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out := make([][]byte, conn.IdealBatchSize) - sizes := make([]int, conn.IdealBatchSize) - for i := range out { - out[i] = make([]byte, 65535) - } - tt.hdr.encode(tt.pktIn) - n, err := handleVirtioRead(tt.pktIn, out, sizes, offset) - if err != nil { - if tt.wantErr { - return - } - t.Fatalf("got err: %v", err) - } - if n != len(tt.wantLens) { - t.Fatalf("got %d packets, wanted %d", n, len(tt.wantLens)) - } - for i := range tt.wantLens { - if tt.wantLens[i] != sizes[i] { - t.Fatalf("wantLens[%d]: %d != outSizes: %d", i, tt.wantLens[i], sizes[i]) - } - } - }) - } -} - -func flipTCP4Checksum(b []byte) []byte { - at := virtioNetHdrLen + 20 + 16 // 20 byte ipv4 header; tcp csum offset is 16 - b[at] ^= 0xFF - b[at+1] ^= 0xFF - return b -} - -func flipUDP4Checksum(b []byte) []byte { - at := virtioNetHdrLen + 20 + 6 // 20 byte ipv4 header; udp csum offset is 6 - b[at] ^= 0xFF - b[at+1] ^= 0xFF - return b -} - -func Fuzz_handleGRO(f *testing.F) { - pkt0 := tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1) - pkt1 := tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101) - pkt2 := tcp4Packet(ip4PortA, ip4PortC, header.TCPFlagAck, 100, 201) - pkt3 := tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1) - pkt4 := tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101) - pkt5 := tcp6Packet(ip6PortA, ip6PortC, header.TCPFlagAck, 100, 201) - pkt6 := udp4Packet(ip4PortA, ip4PortB, 100) - pkt7 := udp4Packet(ip4PortA, ip4PortB, 100) - pkt8 := udp4Packet(ip4PortA, ip4PortC, 100) - pkt9 := udp6Packet(ip6PortA, ip6PortB, 100) - pkt10 := udp6Packet(ip6PortA, ip6PortB, 100) - pkt11 := udp6Packet(ip6PortA, ip6PortC, 100) - f.Add(pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11, 0, offset) - f.Fuzz(func(t *testing.T, pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11 []byte, gro int, offset int) { - pkts := [][]byte{pkt0, pkt1, pkt2, pkt3, pkt4, pkt5, pkt6, pkt7, pkt8, pkt9, pkt10, pkt11} - toWrite := make([]int, 0, len(pkts)) - handleGRO(pkts, offset, newTCPGROTable(), newUDPGROTable(), groDisablementFlags(gro), &toWrite) - if len(toWrite) > len(pkts) { - t.Errorf("len(toWrite): %d > len(pkts): %d", len(toWrite), len(pkts)) - } - seenWriteI := make(map[int]bool) - for _, writeI := range toWrite { - if writeI < 0 || writeI > len(pkts)-1 { - t.Errorf("toWrite value (%d) outside bounds of len(pkts): %d", writeI, len(pkts)) - } - if seenWriteI[writeI] { - t.Errorf("duplicate toWrite value: %d", writeI) - } - seenWriteI[writeI] = true - } - }) -} - -func Test_handleGRO(t *testing.T) { - tests := []struct { - name string - pktsIn [][]byte - gro groDisablementFlags - wantToWrite []int - wantLens []int - wantErr bool - }{ - { - "multiple protocols and flows", - [][]byte{ - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // tcp4 flow 1 - udp4Packet(ip4PortA, ip4PortB, 100), // udp4 flow 1 - udp4Packet(ip4PortA, ip4PortC, 100), // udp4 flow 2 - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101), // tcp4 flow 1 - tcp4Packet(ip4PortA, ip4PortC, header.TCPFlagAck, 100, 201), // tcp4 flow 2 - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1), // tcp6 flow 1 - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101), // tcp6 flow 1 - tcp6Packet(ip6PortA, ip6PortC, header.TCPFlagAck, 100, 201), // tcp6 flow 2 - udp4Packet(ip4PortA, ip4PortB, 100), // udp4 flow 1 - udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 - udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 - }, - 0, - []int{0, 1, 2, 4, 5, 7, 9}, - []int{240, 228, 128, 140, 260, 160, 248}, - false, - }, - { - "multiple protocols and flows no UDP GRO", - [][]byte{ - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // tcp4 flow 1 - udp4Packet(ip4PortA, ip4PortB, 100), // udp4 flow 1 - udp4Packet(ip4PortA, ip4PortC, 100), // udp4 flow 2 - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101), // tcp4 flow 1 - tcp4Packet(ip4PortA, ip4PortC, header.TCPFlagAck, 100, 201), // tcp4 flow 2 - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1), // tcp6 flow 1 - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101), // tcp6 flow 1 - tcp6Packet(ip6PortA, ip6PortC, header.TCPFlagAck, 100, 201), // tcp6 flow 2 - udp4Packet(ip4PortA, ip4PortB, 100), // udp4 flow 1 - udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 - udp6Packet(ip6PortA, ip6PortB, 100), // udp6 flow 1 - }, - udpGRODisabled, - []int{0, 1, 2, 4, 5, 7, 8, 9, 10}, - []int{240, 128, 128, 140, 260, 160, 128, 148, 148}, - false, - }, - { - "PSH interleaved", - [][]byte{ - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // v4 flow 1 - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck|header.TCPFlagPsh, 100, 101), // v4 flow 1 - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 201), // v4 flow 1 - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 301), // v4 flow 1 - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1), // v6 flow 1 - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck|header.TCPFlagPsh, 100, 101), // v6 flow 1 - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 201), // v6 flow 1 - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 301), // v6 flow 1 - }, - 0, - []int{0, 2, 4, 6}, - []int{240, 240, 260, 260}, - false, - }, - { - "coalesceItemInvalidCSum", - [][]byte{ - flipTCP4Checksum(tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1)), // v4 flow 1 seq 1 len 100 - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101), // v4 flow 1 seq 101 len 100 - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 201), // v4 flow 1 seq 201 len 100 - flipUDP4Checksum(udp4Packet(ip4PortA, ip4PortB, 100)), - udp4Packet(ip4PortA, ip4PortB, 100), - udp4Packet(ip4PortA, ip4PortB, 100), - }, - 0, - []int{0, 1, 3, 4}, - []int{140, 240, 128, 228}, - false, - }, - { - "out of order", - [][]byte{ - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101), // v4 flow 1 seq 101 len 100 - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), // v4 flow 1 seq 1 len 100 - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 201), // v4 flow 1 seq 201 len 100 - }, - 0, - []int{0}, - []int{340}, - false, - }, - { - "unequal TTL", - [][]byte{ - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), - tcp4PacketMutateIPFields(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv4Fields) { - fields.TTL++ - }), - udp4Packet(ip4PortA, ip4PortB, 100), - udp4PacketMutateIPFields(ip4PortA, ip4PortB, 100, func(fields *header.IPv4Fields) { - fields.TTL++ - }), - }, - 0, - []int{0, 1, 2, 3}, - []int{140, 140, 128, 128}, - false, - }, - { - "unequal ToS", - [][]byte{ - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), - tcp4PacketMutateIPFields(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv4Fields) { - fields.TOS++ - }), - udp4Packet(ip4PortA, ip4PortB, 100), - udp4PacketMutateIPFields(ip4PortA, ip4PortB, 100, func(fields *header.IPv4Fields) { - fields.TOS++ - }), - }, - 0, - []int{0, 1, 2, 3}, - []int{140, 140, 128, 128}, - false, - }, - { - "unequal flags more fragments set", - [][]byte{ - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), - tcp4PacketMutateIPFields(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv4Fields) { - fields.Flags = 1 - }), - udp4Packet(ip4PortA, ip4PortB, 100), - udp4PacketMutateIPFields(ip4PortA, ip4PortB, 100, func(fields *header.IPv4Fields) { - fields.Flags = 1 - }), - }, - 0, - []int{0, 1, 2, 3}, - []int{140, 140, 128, 128}, - false, - }, - { - "unequal flags DF set", - [][]byte{ - tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1), - tcp4PacketMutateIPFields(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv4Fields) { - fields.Flags = 2 - }), - udp4Packet(ip4PortA, ip4PortB, 100), - udp4PacketMutateIPFields(ip4PortA, ip4PortB, 100, func(fields *header.IPv4Fields) { - fields.Flags = 2 - }), - }, - 0, - []int{0, 1, 2, 3}, - []int{140, 140, 128, 128}, - false, - }, - { - "ipv6 unequal hop limit", - [][]byte{ - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1), - tcp6PacketMutateIPFields(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv6Fields) { - fields.HopLimit++ - }), - udp6Packet(ip6PortA, ip6PortB, 100), - udp6PacketMutateIPFields(ip6PortA, ip6PortB, 100, func(fields *header.IPv6Fields) { - fields.HopLimit++ - }), - }, - 0, - []int{0, 1, 2, 3}, - []int{160, 160, 148, 148}, - false, - }, - { - "ipv6 unequal traffic class", - [][]byte{ - tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1), - tcp6PacketMutateIPFields(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 101, func(fields *header.IPv6Fields) { - fields.TrafficClass++ - }), - udp6Packet(ip6PortA, ip6PortB, 100), - udp6PacketMutateIPFields(ip6PortA, ip6PortB, 100, func(fields *header.IPv6Fields) { - fields.TrafficClass++ - }), - }, - 0, - []int{0, 1, 2, 3}, - []int{160, 160, 148, 148}, - false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - toWrite := make([]int, 0, len(tt.pktsIn)) - err := handleGRO(tt.pktsIn, offset, newTCPGROTable(), newUDPGROTable(), tt.gro, &toWrite) - if err != nil { - if tt.wantErr { - return - } - t.Fatalf("got err: %v", err) - } - if len(toWrite) != len(tt.wantToWrite) { - t.Fatalf("got %d packets, wanted %d", len(toWrite), len(tt.wantToWrite)) - } - for i, pktI := range tt.wantToWrite { - if tt.wantToWrite[i] != toWrite[i] { - t.Fatalf("wantToWrite[%d]: %d != toWrite: %d", i, tt.wantToWrite[i], toWrite[i]) - } - if tt.wantLens[i] != len(tt.pktsIn[pktI][offset:]) { - t.Errorf("wanted len %d packet at %d, got: %d", tt.wantLens[i], i, len(tt.pktsIn[pktI][offset:])) - } - } - }) - } -} - -func Test_packetIsGROCandidate(t *testing.T) { - tcp4 := tcp4Packet(ip4PortA, ip4PortB, header.TCPFlagAck, 100, 1)[virtioNetHdrLen:] - tcp4TooShort := tcp4[:39] - ip4InvalidHeaderLen := make([]byte, len(tcp4)) - copy(ip4InvalidHeaderLen, tcp4) - ip4InvalidHeaderLen[0] = 0x46 - ip4InvalidProtocol := make([]byte, len(tcp4)) - copy(ip4InvalidProtocol, tcp4) - ip4InvalidProtocol[9] = unix.IPPROTO_GRE - - tcp6 := tcp6Packet(ip6PortA, ip6PortB, header.TCPFlagAck, 100, 1)[virtioNetHdrLen:] - tcp6TooShort := tcp6[:59] - ip6InvalidProtocol := make([]byte, len(tcp6)) - copy(ip6InvalidProtocol, tcp6) - ip6InvalidProtocol[6] = unix.IPPROTO_GRE - - udp4 := udp4Packet(ip4PortA, ip4PortB, 100)[virtioNetHdrLen:] - udp4TooShort := udp4[:27] - - udp6 := udp6Packet(ip6PortA, ip6PortB, 100)[virtioNetHdrLen:] - udp6TooShort := udp6[:47] - - tests := []struct { - name string - b []byte - gro groDisablementFlags - want groCandidateType - }{ - { - "tcp4", - tcp4, - 0, - tcp4GROCandidate, - }, - { - "tcp4 no support", - tcp4, - tcpGRODisabled, - notGROCandidate, - }, - { - "tcp6", - tcp6, - 0, - tcp6GROCandidate, - }, - { - "tcp6 no support", - tcp6, - tcpGRODisabled, - notGROCandidate, - }, - { - "udp4", - udp4, - 0, - udp4GROCandidate, - }, - { - "udp4 no support", - udp4, - udpGRODisabled, - notGROCandidate, - }, - { - "udp6", - udp6, - 0, - udp6GROCandidate, - }, - { - "udp6 no support", - udp6, - udpGRODisabled, - notGROCandidate, - }, - { - "udp4 too short", - udp4TooShort, - 0, - notGROCandidate, - }, - { - "udp6 too short", - udp6TooShort, - 0, - notGROCandidate, - }, - { - "tcp4 too short", - tcp4TooShort, - 0, - notGROCandidate, - }, - { - "tcp6 too short", - tcp6TooShort, - 0, - notGROCandidate, - }, - { - "invalid IP version", - []byte{0x00}, - 0, - notGROCandidate, - }, - { - "invalid IP header len", - ip4InvalidHeaderLen, - 0, - notGROCandidate, - }, - { - "ip4 invalid protocol", - ip4InvalidProtocol, - 0, - notGROCandidate, - }, - { - "ip6 invalid protocol", - ip6InvalidProtocol, - 0, - notGROCandidate, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := packetIsGROCandidate(tt.b, tt.gro); got != tt.want { - t.Errorf("packetIsGROCandidate() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_udpPacketsCanCoalesce(t *testing.T) { - udp4a := udp4Packet(ip4PortA, ip4PortB, 100) - udp4b := udp4Packet(ip4PortA, ip4PortB, 100) - udp4c := udp4Packet(ip4PortA, ip4PortB, 110) - - type args struct { - pkt []byte - iphLen uint8 - gsoSize uint16 - item udpGROItem - bufs [][]byte - bufsOffset int - } - tests := []struct { - name string - args args - want canCoalesce - }{ - { - "coalesceAppend equal gso", - args{ - pkt: udp4a[offset:], - iphLen: 20, - gsoSize: 100, - item: udpGROItem{ - gsoSize: 100, - iphLen: 20, - }, - bufs: [][]byte{ - udp4a, - udp4b, - }, - bufsOffset: offset, - }, - coalesceAppend, - }, - { - "coalesceAppend smaller gso", - args{ - pkt: udp4a[offset : len(udp4a)-90], - iphLen: 20, - gsoSize: 10, - item: udpGROItem{ - gsoSize: 100, - iphLen: 20, - }, - bufs: [][]byte{ - udp4a, - udp4b, - }, - bufsOffset: offset, - }, - coalesceAppend, - }, - { - "coalesceUnavailable smaller gso previously appended", - args{ - pkt: udp4a[offset:], - iphLen: 20, - gsoSize: 100, - item: udpGROItem{ - gsoSize: 100, - iphLen: 20, - }, - bufs: [][]byte{ - udp4c, - udp4b, - }, - bufsOffset: offset, - }, - coalesceUnavailable, - }, - { - "coalesceUnavailable larger following smaller", - args{ - pkt: udp4c[offset:], - iphLen: 20, - gsoSize: 110, - item: udpGROItem{ - gsoSize: 100, - iphLen: 20, - }, - bufs: [][]byte{ - udp4a, - udp4c, - }, - bufsOffset: offset, - }, - coalesceUnavailable, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := udpPacketsCanCoalesce(tt.args.pkt, tt.args.iphLen, tt.args.gsoSize, tt.args.item, tt.args.bufs, tt.args.bufsOffset); got != tt.want { - t.Errorf("udpPacketsCanCoalesce() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/tun/offload_test.go b/tun/offload_test.go deleted file mode 100644 index 82a37b9..0000000 --- a/tun/offload_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package tun - -import ( - "net/netip" - "testing" - - "github.com/tailscale/wireguard-go/conn" - "gvisor.dev/gvisor/pkg/tcpip" - "gvisor.dev/gvisor/pkg/tcpip/header" -) - -func Fuzz_GSOSplit(f *testing.F) { - const segmentSize = 100 - - tcpFields := &header.TCPFields{ - SrcPort: 1, - DstPort: 1, - SeqNum: 1, - AckNum: 1, - DataOffset: 20, - Flags: header.TCPFlagAck | header.TCPFlagPsh, - WindowSize: 3000, - } - udpFields := &header.UDPFields{ - SrcPort: 1, - DstPort: 1, - Length: 8 + segmentSize, - } - - gsoTCPv4 := make([]byte, 20+20+segmentSize) - header.IPv4(gsoTCPv4).Encode(&header.IPv4Fields{ - SrcAddr: tcpip.AddrFromSlice(netip.MustParseAddr("192.0.2.1").AsSlice()), - DstAddr: tcpip.AddrFromSlice(netip.MustParseAddr("192.0.2.2").AsSlice()), - Protocol: ipProtoTCP, - TTL: 64, - TotalLength: uint16(len(gsoTCPv4)), - }) - header.TCP(gsoTCPv4[20:]).Encode(tcpFields) - - gsoUDPv4 := make([]byte, 20+8+segmentSize) - header.IPv4(gsoUDPv4).Encode(&header.IPv4Fields{ - SrcAddr: tcpip.AddrFromSlice(netip.MustParseAddr("192.0.2.1").AsSlice()), - DstAddr: tcpip.AddrFromSlice(netip.MustParseAddr("192.0.2.2").AsSlice()), - Protocol: ipProtoUDP, - TTL: 64, - TotalLength: uint16(len(gsoUDPv4)), - }) - header.UDP(gsoTCPv4[20:]).Encode(udpFields) - - gsoTCPv6 := make([]byte, 40+20+segmentSize) - header.IPv6(gsoTCPv6).Encode(&header.IPv6Fields{ - SrcAddr: tcpip.AddrFromSlice(netip.MustParseAddr("2001:db8::1").AsSlice()), - DstAddr: tcpip.AddrFromSlice(netip.MustParseAddr("2001:db8::2").AsSlice()), - TransportProtocol: ipProtoTCP, - HopLimit: 64, - PayloadLength: uint16(20 + segmentSize), - }) - header.TCP(gsoTCPv6[40:]).Encode(tcpFields) - - gsoUDPv6 := make([]byte, 40+8+segmentSize) - header.IPv6(gsoUDPv6).Encode(&header.IPv6Fields{ - SrcAddr: tcpip.AddrFromSlice(netip.MustParseAddr("2001:db8::1").AsSlice()), - DstAddr: tcpip.AddrFromSlice(netip.MustParseAddr("2001:db8::2").AsSlice()), - TransportProtocol: ipProtoUDP, - HopLimit: 64, - PayloadLength: uint16(8 + segmentSize), - }) - header.UDP(gsoUDPv6[20:]).Encode(udpFields) - - out := make([][]byte, conn.IdealBatchSize) - for i := range out { - out[i] = make([]byte, 65535) - } - sizes := make([]int, conn.IdealBatchSize) - - f.Add(gsoTCPv4, int(GSOTCPv4), uint16(40), uint16(20), uint16(16), uint16(100), false) - f.Add(gsoUDPv4, int(GSOUDPL4), uint16(28), uint16(20), uint16(6), uint16(100), false) - f.Add(gsoTCPv6, int(GSOTCPv6), uint16(60), uint16(40), uint16(16), uint16(100), false) - f.Add(gsoUDPv6, int(GSOUDPL4), uint16(48), uint16(40), uint16(6), uint16(100), false) - - f.Fuzz(func(t *testing.T, pkt []byte, gsoType int, hdrLen, csumStart, csumOffset, gsoSize uint16, needsCsum bool) { - options := GSOOptions{ - GSOType: GSOType(gsoType), - HdrLen: hdrLen, - CsumStart: csumStart, - CsumOffset: csumOffset, - GSOSize: gsoSize, - NeedsCsum: needsCsum, - } - n, _ := GSOSplit(pkt, options, out, sizes, 0) - if n > len(sizes) { - t.Errorf("n (%d) > len(sizes): %d", n, len(sizes)) - } - }) -} diff --git a/tun/tuntest/tuntest.go b/tun/tuntest/tuntest.go deleted file mode 100644 index e7507c2..0000000 --- a/tun/tuntest/tuntest.go +++ /dev/null @@ -1,155 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package tuntest - -import ( - "encoding/binary" - "io" - "net/netip" - "os" - - "github.com/tailscale/wireguard-go/tun" -) - -func Ping(dst, src netip.Addr) []byte { - localPort := uint16(1337) - seq := uint16(0) - - payload := make([]byte, 4) - binary.BigEndian.PutUint16(payload[0:], localPort) - binary.BigEndian.PutUint16(payload[2:], seq) - - return genICMPv4(payload, dst, src) -} - -// Checksum is the "internet checksum" from https://tools.ietf.org/html/rfc1071. -func checksum(buf []byte, initial uint16) uint16 { - v := uint32(initial) - for i := 0; i < len(buf)-1; i += 2 { - v += uint32(binary.BigEndian.Uint16(buf[i:])) - } - if len(buf)%2 == 1 { - v += uint32(buf[len(buf)-1]) << 8 - } - for v > 0xffff { - v = (v >> 16) + (v & 0xffff) - } - return ^uint16(v) -} - -func genICMPv4(payload []byte, dst, src netip.Addr) []byte { - const ( - icmpv4ProtocolNumber = 1 - icmpv4Echo = 8 - icmpv4ChecksumOffset = 2 - icmpv4Size = 8 - ipv4Size = 20 - ipv4TotalLenOffset = 2 - ipv4ChecksumOffset = 10 - ttl = 65 - headerSize = ipv4Size + icmpv4Size - ) - - pkt := make([]byte, headerSize+len(payload)) - - ip := pkt[0:ipv4Size] - icmpv4 := pkt[ipv4Size : ipv4Size+icmpv4Size] - - // https://tools.ietf.org/html/rfc792 - icmpv4[0] = icmpv4Echo // type - icmpv4[1] = 0 // code - chksum := ^checksum(icmpv4, checksum(payload, 0)) - binary.BigEndian.PutUint16(icmpv4[icmpv4ChecksumOffset:], chksum) - - // https://tools.ietf.org/html/rfc760 section 3.1 - length := uint16(len(pkt)) - ip[0] = (4 << 4) | (ipv4Size / 4) - binary.BigEndian.PutUint16(ip[ipv4TotalLenOffset:], length) - ip[8] = ttl - ip[9] = icmpv4ProtocolNumber - copy(ip[12:], src.AsSlice()) - copy(ip[16:], dst.AsSlice()) - chksum = ^checksum(ip[:], 0) - binary.BigEndian.PutUint16(ip[ipv4ChecksumOffset:], chksum) - - copy(pkt[headerSize:], payload) - return pkt -} - -type ChannelTUN struct { - Inbound chan []byte // incoming packets, closed on TUN close - Outbound chan []byte // outbound packets, blocks forever on TUN close - - closed chan struct{} - events chan tun.Event - tun chTun -} - -func NewChannelTUN() *ChannelTUN { - c := &ChannelTUN{ - Inbound: make(chan []byte), - Outbound: make(chan []byte), - closed: make(chan struct{}), - events: make(chan tun.Event, 1), - } - c.tun.c = c - c.events <- tun.EventUp - return c -} - -func (c *ChannelTUN) TUN() tun.Device { - return &c.tun -} - -type chTun struct { - c *ChannelTUN -} - -func (t *chTun) File() *os.File { return nil } - -func (t *chTun) Read(packets [][]byte, sizes []int, offset int) (int, error) { - select { - case <-t.c.closed: - return 0, os.ErrClosed - case msg := <-t.c.Outbound: - n := copy(packets[0][offset:], msg) - sizes[0] = n - return 1, nil - } -} - -// Write is called by the wireguard device to deliver a packet for routing. -func (t *chTun) Write(packets [][]byte, offset int) (int, error) { - if offset == -1 { - close(t.c.closed) - close(t.c.events) - return 0, io.EOF - } - for i, data := range packets { - msg := make([]byte, len(data)-offset) - copy(msg, data[offset:]) - select { - case <-t.c.closed: - return i, os.ErrClosed - case t.c.Inbound <- msg: - } - } - return len(packets), nil -} - -func (t *chTun) BatchSize() int { - return 1 -} - -const DefaultMTU = 1420 - -func (t *chTun) MTU() (int, error) { return DefaultMTU, nil } -func (t *chTun) Name() (string, error) { return "loopbackTun1", nil } -func (t *chTun) Events() <-chan tun.Event { return t.c.events } -func (t *chTun) Close() error { - t.Write(nil, -1) - return nil -} From dfa4a8968f3d670a7d87bd5c58cacac90a38d82f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Sat, 9 Dec 2023 17:27:04 +0800 Subject: [PATCH 43/60] Add module_rename.py --- module_rename.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 module_rename.py diff --git a/module_rename.py b/module_rename.py new file mode 100644 index 0000000..bf03152 --- /dev/null +++ b/module_rename.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import argparse +import fileinput + + +PKG_ORIGINAL = "github.com/tailscale/wireguard-go" +PKG_NEW = "github.com/sagernet/wireguard-go" + +EXTENSIONS = [".go", ".md", ".mod", ".sh"] + +parser = argparse.ArgumentParser() +parser.add_argument("-r", "--reverse", action="store_true") +args = parser.parse_args() + + +def replace_line(line): + if args.reverse: + return line.replace(PKG_NEW, PKG_ORIGINAL) + return line.replace(PKG_ORIGINAL, PKG_NEW) + + +for dirpath, dirnames, filenames in os.walk("."): + # Skip hidden directories like .git + dirnames[:] = [d for d in dirnames if not d[0] == "."] + filenames = [f for f in filenames if os.path.splitext(f)[1] in EXTENSIONS] + for filename in filenames: + file_path = os.path.join(dirpath, filename) + with fileinput.FileInput(file_path, inplace=True) as file: + for line in file: + print(replace_line(line), end="") From 558e1e0493c5ce35f33f23527e4dd466d9a6d312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 15 Sep 2025 18:12:36 +0800 Subject: [PATCH 44/60] Rename module --- conn/bind_windows.go | 2 +- device/device.go | 8 ++++---- device/keypair.go | 2 +- device/noise-protocol.go | 4 ++-- device/peer.go | 2 +- device/queueconstants_android.go | 2 +- device/queueconstants_default.go | 2 +- device/receive.go | 2 +- device/send.go | 4 ++-- device/sticky_default.go | 4 ++-- device/sticky_linux.go | 4 ++-- device/tun.go | 2 +- device/uapi.go | 2 +- go.mod | 2 +- ipc/uapi_linux.go | 2 +- ipc/uapi_windows.go | 2 +- tun/offload_linux.go | 2 +- tun/tun_linux.go | 4 ++-- 18 files changed, 26 insertions(+), 26 deletions(-) diff --git a/conn/bind_windows.go b/conn/bind_windows.go index 737b475..d32ecd6 100644 --- a/conn/bind_windows.go +++ b/conn/bind_windows.go @@ -17,7 +17,7 @@ import ( "golang.org/x/sys/windows" - "github.com/tailscale/wireguard-go/conn/winrio" + "github.com/sagernet/wireguard-go/conn/winrio" ) const ( diff --git a/device/device.go b/device/device.go index 5b23485..7c70554 100644 --- a/device/device.go +++ b/device/device.go @@ -11,10 +11,10 @@ import ( "sync/atomic" "time" - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/ratelimiter" - "github.com/tailscale/wireguard-go/rwcancel" - "github.com/tailscale/wireguard-go/tun" + "github.com/sagernet/wireguard-go/conn" + "github.com/sagernet/wireguard-go/ratelimiter" + "github.com/sagernet/wireguard-go/rwcancel" + "github.com/sagernet/wireguard-go/tun" ) type Device struct { diff --git a/device/keypair.go b/device/keypair.go index 2689ee2..fb3d392 100644 --- a/device/keypair.go +++ b/device/keypair.go @@ -11,7 +11,7 @@ import ( "sync/atomic" "time" - "github.com/tailscale/wireguard-go/replay" + "github.com/sagernet/wireguard-go/replay" ) /* Due to limitations in Go and /x/crypto there is currently diff --git a/device/noise-protocol.go b/device/noise-protocol.go index ad5838e..50dcb60 100644 --- a/device/noise-protocol.go +++ b/device/noise-protocol.go @@ -16,8 +16,8 @@ import ( "golang.org/x/crypto/chacha20poly1305" "golang.org/x/crypto/poly1305" - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/tai64n" + "github.com/sagernet/wireguard-go/conn" + "github.com/sagernet/wireguard-go/tai64n" ) type handshakeState int diff --git a/device/peer.go b/device/peer.go index 064feb2..1307300 100644 --- a/device/peer.go +++ b/device/peer.go @@ -12,7 +12,7 @@ import ( "sync/atomic" "time" - "github.com/tailscale/wireguard-go/conn" + "github.com/sagernet/wireguard-go/conn" ) type Peer struct { diff --git a/device/queueconstants_android.go b/device/queueconstants_android.go index bab9625..d0718d2 100644 --- a/device/queueconstants_android.go +++ b/device/queueconstants_android.go @@ -5,7 +5,7 @@ package device -import "github.com/tailscale/wireguard-go/conn" +import "github.com/sagernet/wireguard-go/conn" /* Reduce memory consumption for Android */ diff --git a/device/queueconstants_default.go b/device/queueconstants_default.go index 9749cb7..79b74b0 100644 --- a/device/queueconstants_default.go +++ b/device/queueconstants_default.go @@ -7,7 +7,7 @@ package device -import "github.com/tailscale/wireguard-go/conn" +import "github.com/sagernet/wireguard-go/conn" const ( QueueStagedSize = conn.IdealBatchSize diff --git a/device/receive.go b/device/receive.go index 02c8f21..01baad8 100644 --- a/device/receive.go +++ b/device/receive.go @@ -12,7 +12,7 @@ import ( "sync" "time" - "github.com/tailscale/wireguard-go/conn" + "github.com/sagernet/wireguard-go/conn" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" diff --git a/device/send.go b/device/send.go index c8bb079..ff082a8 100644 --- a/device/send.go +++ b/device/send.go @@ -13,8 +13,8 @@ import ( "sync" "time" - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/tun" + "github.com/sagernet/wireguard-go/conn" + "github.com/sagernet/wireguard-go/tun" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" diff --git a/device/sticky_default.go b/device/sticky_default.go index 732f84c..0d02174 100644 --- a/device/sticky_default.go +++ b/device/sticky_default.go @@ -3,8 +3,8 @@ package device import ( - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/rwcancel" + "github.com/sagernet/wireguard-go/conn" + "github.com/sagernet/wireguard-go/rwcancel" ) func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, error) { diff --git a/device/sticky_linux.go b/device/sticky_linux.go index 6eeced2..79ece63 100644 --- a/device/sticky_linux.go +++ b/device/sticky_linux.go @@ -20,8 +20,8 @@ import ( "golang.org/x/sys/unix" - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/rwcancel" + "github.com/sagernet/wireguard-go/conn" + "github.com/sagernet/wireguard-go/rwcancel" ) func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, error) { diff --git a/device/tun.go b/device/tun.go index 960ecca..891575d 100644 --- a/device/tun.go +++ b/device/tun.go @@ -8,7 +8,7 @@ package device import ( "fmt" - "github.com/tailscale/wireguard-go/tun" + "github.com/sagernet/wireguard-go/tun" ) const DefaultMTU = 1420 diff --git a/device/uapi.go b/device/uapi.go index 4987cda..6bbccbd 100644 --- a/device/uapi.go +++ b/device/uapi.go @@ -18,7 +18,7 @@ import ( "sync" "time" - "github.com/tailscale/wireguard-go/ipc" + "github.com/sagernet/wireguard-go/ipc" ) type IPCError struct { diff --git a/go.mod b/go.mod index 7476734..2dfe133 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/tailscale/wireguard-go +module github.com/sagernet/wireguard-go go 1.20 diff --git a/ipc/uapi_linux.go b/ipc/uapi_linux.go index bfdf1bf..6a662d3 100644 --- a/ipc/uapi_linux.go +++ b/ipc/uapi_linux.go @@ -9,7 +9,7 @@ import ( "net" "os" - "github.com/tailscale/wireguard-go/rwcancel" + "github.com/sagernet/wireguard-go/rwcancel" "golang.org/x/sys/unix" ) diff --git a/ipc/uapi_windows.go b/ipc/uapi_windows.go index bc30ae0..5d236b3 100644 --- a/ipc/uapi_windows.go +++ b/ipc/uapi_windows.go @@ -8,7 +8,7 @@ package ipc import ( "net" - "github.com/tailscale/wireguard-go/ipc/namedpipe" + "github.com/sagernet/wireguard-go/ipc/namedpipe" "golang.org/x/sys/windows" ) diff --git a/tun/offload_linux.go b/tun/offload_linux.go index fb6ac5b..6ba8547 100644 --- a/tun/offload_linux.go +++ b/tun/offload_linux.go @@ -13,7 +13,7 @@ import ( "io" "unsafe" - "github.com/tailscale/wireguard-go/conn" + "github.com/sagernet/wireguard-go/conn" "golang.org/x/sys/unix" ) diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 7cdbf88..4e27c1b 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -17,8 +17,8 @@ import ( "time" "unsafe" - "github.com/tailscale/wireguard-go/conn" - "github.com/tailscale/wireguard-go/rwcancel" + "github.com/sagernet/wireguard-go/conn" + "github.com/sagernet/wireguard-go/rwcancel" "golang.org/x/sys/unix" ) From d79d3c88fd3f45d0a4d7b8a895eca6c8b76cdb00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 15 Sep 2025 18:12:47 +0800 Subject: [PATCH 45/60] Reformat code --- conn/bind_std.go | 6 ++---- conn/bind_windows.go | 3 +-- device/noise-protocol.go | 5 ++--- device/receive.go | 1 - device/sticky_linux.go | 3 +-- ipc/uapi_linux.go | 1 - tun/errors.go | 10 ++++------ tun/tun_linux.go | 4 +--- 8 files changed, 11 insertions(+), 22 deletions(-) diff --git a/conn/bind_std.go b/conn/bind_std.go index fc05634..249f53d 100644 --- a/conn/bind_std.go +++ b/conn/bind_std.go @@ -213,10 +213,8 @@ func (s *StdNetBind) getMessages() *[]ipv6.Message { return s.msgsPool.Get().(*[]ipv6.Message) } -var ( - // If compilation fails here these are no longer the same underlying type. - _ ipv6.Message = ipv4.Message{} -) +// If compilation fails here these are no longer the same underlying type. +var _ ipv6.Message = ipv4.Message{} type batchReader interface { ReadBatch([]ipv6.Message, int) (int, error) diff --git a/conn/bind_windows.go b/conn/bind_windows.go index d32ecd6..81a820b 100644 --- a/conn/bind_windows.go +++ b/conn/bind_windows.go @@ -15,9 +15,8 @@ import ( "sync/atomic" "unsafe" - "golang.org/x/sys/windows" - "github.com/sagernet/wireguard-go/conn/winrio" + "golang.org/x/sys/windows" ) const ( diff --git a/device/noise-protocol.go b/device/noise-protocol.go index 50dcb60..6e67d04 100644 --- a/device/noise-protocol.go +++ b/device/noise-protocol.go @@ -12,12 +12,11 @@ import ( "sync" "time" + "github.com/sagernet/wireguard-go/conn" + "github.com/sagernet/wireguard-go/tai64n" "golang.org/x/crypto/blake2s" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/crypto/poly1305" - - "github.com/sagernet/wireguard-go/conn" - "github.com/sagernet/wireguard-go/tai64n" ) type handshakeState int diff --git a/device/receive.go b/device/receive.go index 01baad8..6ab2680 100644 --- a/device/receive.go +++ b/device/receive.go @@ -411,7 +411,6 @@ func (device *Device) RoutineHandshake(id int) { // derive keypair err = peer.BeginSymmetricSession() - if err != nil { device.log.Errorf("%v - Failed to derive keypair: %v", peer, err) goto skip diff --git a/device/sticky_linux.go b/device/sticky_linux.go index 79ece63..b5b0e51 100644 --- a/device/sticky_linux.go +++ b/device/sticky_linux.go @@ -18,10 +18,9 @@ import ( "sync" "unsafe" - "golang.org/x/sys/unix" - "github.com/sagernet/wireguard-go/conn" "github.com/sagernet/wireguard-go/rwcancel" + "golang.org/x/sys/unix" ) func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, error) { diff --git a/ipc/uapi_linux.go b/ipc/uapi_linux.go index 6a662d3..c14d5d0 100644 --- a/ipc/uapi_linux.go +++ b/ipc/uapi_linux.go @@ -84,7 +84,6 @@ func UAPIListen(name string, file *os.File) (net.Listener, error) { unix.IN_DELETE| unix.IN_DELETE_SELF, ) - if err != nil { return nil, err } diff --git a/tun/errors.go b/tun/errors.go index 75ae3a4..2c49fc7 100644 --- a/tun/errors.go +++ b/tun/errors.go @@ -4,9 +4,7 @@ import ( "errors" ) -var ( - // ErrTooManySegments is returned by Device.Read() when segmentation - // overflows the length of supplied buffers. This error should not cause - // reads to cease. - ErrTooManySegments = errors.New("too many segments") -) +// ErrTooManySegments is returned by Device.Read() when segmentation +// overflows the length of supplied buffers. This error should not cause +// reads to cease. +var ErrTooManySegments = errors.New("too many segments") diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 4e27c1b..8f4aed1 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -506,9 +506,7 @@ func (tun *NativeTun) initFromFlags(name string) error { return err } if e := sc.Control(func(fd uintptr) { - var ( - ifr *unix.Ifreq - ) + var ifr *unix.Ifreq ifr, err = unix.NewIfreq(name) if err != nil { return From 2835be44111d5edbcac44401dc0c6c9cbd4eb48b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 7 Nov 2022 15:41:48 +0800 Subject: [PATCH 46/60] Add custom worker size params (cherry picked from commit 7c2acadba17cadf8a1df957c49e1333130d460ad) (cherry picked from commit a7bac1754e7717e1d4009d1ffd2d13330067d631) (cherry picked from commit 7a2f11c693b49e784318bbf987173095c67b563d) --- device/device.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/device/device.go b/device/device.go index 7c70554..17619bd 100644 --- a/device/device.go +++ b/device/device.go @@ -281,7 +281,7 @@ func (device *Device) SetPrivateKey(sk NoisePrivateKey) error { return nil } -func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger) *Device { +func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger, workers int) *Device { device := new(Device) device.state.state.Store(uint32(deviceStateDown)) device.closed = make(chan struct{}) @@ -308,10 +308,12 @@ func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger) *Device { // start workers - cpus := runtime.NumCPU() + if workers == 0 { + workers = runtime.NumCPU() + } device.state.stopping.Wait() - device.queue.encryption.wg.Add(cpus) // One for each RoutineHandshake - for i := 0; i < cpus; i++ { + device.queue.encryption.wg.Add(workers) // One for each RoutineHandshake + for i := 0; i < workers; i++ { go device.RoutineEncryption(i + 1) go device.RoutineDecryption(i + 1) go device.RoutineHandshake(i + 1) From 606102e010019d1dda58d932329f096e55d2f360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Mon, 7 Aug 2023 20:57:31 +0800 Subject: [PATCH 47/60] Add pause support --- device/device.go | 14 ++++++++++---- device/timers.go | 3 +++ go.mod | 1 + go.sum | 2 ++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/device/device.go b/device/device.go index 17619bd..e491e53 100644 --- a/device/device.go +++ b/device/device.go @@ -6,11 +6,15 @@ package device import ( + "context" "runtime" "sync" "sync/atomic" "time" + "github.com/sagernet/sing/service" + "github.com/sagernet/sing/service/pause" + "github.com/sagernet/wireguard-go/conn" "github.com/sagernet/wireguard-go/ratelimiter" "github.com/sagernet/wireguard-go/rwcancel" @@ -86,9 +90,10 @@ type Device struct { mtu atomic.Int32 } - ipcMutex sync.RWMutex - closed chan struct{} - log *Logger + ipcMutex sync.RWMutex + closed chan struct{} + log *Logger + pauseManager pause.Manager } // deviceState represents the state of a Device. @@ -281,8 +286,9 @@ func (device *Device) SetPrivateKey(sk NoisePrivateKey) error { return nil } -func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger, workers int) *Device { +func NewDevice(ctx context.Context, tunDevice tun.Device, bind conn.Bind, logger *Logger, workers int) *Device { device := new(Device) + device.pauseManager = service.FromContext[pause.Manager](ctx) device.state.state.Store(uint32(deviceStateDown)) device.closed = make(chan struct{}) device.log = logger diff --git a/device/timers.go b/device/timers.go index d4a4ed4..5839557 100644 --- a/device/timers.go +++ b/device/timers.go @@ -39,6 +39,9 @@ func (peer *Peer) NewTimer(expirationFunction func(*Peer)) *Timer { timer.isPending = false timer.modifyingLock.Unlock() + if pauseManager := peer.device.pauseManager; pauseManager != nil { + pauseManager.WaitActive() + } expirationFunction(peer) }) timer.Stop() diff --git a/go.mod b/go.mod index 2dfe133..37359e6 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/sagernet/wireguard-go go 1.20 require ( + github.com/sagernet/sing v0.7.10 golang.org/x/crypto v0.13.0 golang.org/x/net v0.15.0 golang.org/x/sys v0.12.0 diff --git a/go.sum b/go.sum index ec4169f..c405e0a 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/sagernet/sing v0.7.10 h1:2yPhZFx+EkyHPH8hXNezgyRSHyGY12CboId7CtwLROw= +github.com/sagernet/sing v0.7.10/go.mod h1:ARkL0gM13/Iv5VCZmci/NuoOlePoIsW0m7BWfln/Hak= golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= From b2a20cdd7732f28fd182ff17e1cb77f86098697c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Wed, 20 Nov 2024 18:53:51 +0800 Subject: [PATCH 48/60] Add device.InputPacket --- device/send.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/device/send.go b/device/send.go index ff082a8..2c690dc 100644 --- a/device/send.go +++ b/device/send.go @@ -322,6 +322,30 @@ func (device *Device) RoutineReadFromTUN() { } } +func (device *Device) InputPacket(destination []byte, packetSlices [][]byte) { + peer := device.allowedips.Lookup(destination) + if peer == nil { + return + } + elem := device.NewOutboundElement() + packet := elem.buffer[MessageTransportHeaderSize:] + var n int + for _, packetSlice := range packetSlices { + n += copy(packet[n:], packetSlice) + } + elem.packet = packet[:n] + elemsForPeer := device.GetOutboundElementsContainer() + if peer.isRunning.Load() { + elemsForPeer.elems = append(elemsForPeer.elems, elem) + peer.StagePackets(elemsForPeer) + peer.SendStagedPackets() + } else { + device.PutMessageBuffer(elem.buffer) + device.PutOutboundElement(elem) + device.PutOutboundElementsContainer(elemsForPeer) + } +} + func (peer *Peer) StagePackets(elems *QueueOutboundElementsContainer) { for { select { From c63bc19bc949c3de3beab1eedd55ecf634f19d7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Fri, 15 Dec 2023 18:44:05 +0800 Subject: [PATCH 49/60] Export std net bind --- conn/bind_std.go | 58 +++++++++++++++++++++++++++++---- conn/bind_windows.go | 76 ++++++++++++++++++++++++++++++++++++++++---- conn/conn.go | 2 ++ conn/controlfns.go | 26 ++------------- conn/default.go | 6 +++- device/device.go | 1 - 6 files changed, 129 insertions(+), 40 deletions(-) diff --git a/conn/bind_std.go b/conn/bind_std.go index 249f53d..a1d4d9e 100644 --- a/conn/bind_std.go +++ b/conn/bind_std.go @@ -16,6 +16,9 @@ import ( "sync" "syscall" + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/control" + M "github.com/sagernet/sing/common/metadata" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" ) @@ -31,6 +34,9 @@ var ( // methods for sending and receiving multiple datagrams per-syscall. See the // proposal in https://github.com/golang/go/issues/45886#issuecomment-1218301564. type StdNetBind struct { + externalControl control.Func + reservedForEndpoint map[netip.AddrPort][3]uint8 + mu sync.Mutex // protects all fields except as specified ipv4 *net.UDPConn ipv6 *net.UDPConn @@ -49,8 +55,11 @@ type StdNetBind struct { blackhole6 bool } -func NewStdNetBind() Bind { +func NewStdNetBind(externalControl control.Func) Bind { return &StdNetBind{ + externalControl: externalControl, + reservedForEndpoint: make(map[netip.AddrPort][3]uint8), + udpAddrPool: sync.Pool{ New: func() any { return &net.UDPAddr{ @@ -118,8 +127,29 @@ func (e *StdNetEndpoint) DstToString() string { return e.AddrPort.String() } -func listenNet(network string, port int) (*net.UDPConn, int, error) { - conn, err := listenConfig().ListenPacket(context.Background(), network, ":"+strconv.Itoa(port)) +func listenNet(externalControl control.Func, network string, port int) (*net.UDPConn, int, error) { + var listenerAddr string + if network == "udp6" { + listenerAddr = "[::]:" + strconv.Itoa(port) + } else { + listenerAddr = ":" + strconv.Itoa(port) + } + + var listener net.ListenConfig + listener.Control = func(network, address string, conn syscall.RawConn) error { + for _, wgControlFn := range controlFns { + err := wgControlFn(network, address, conn) + if err != nil { + return err + } + } + if externalControl != nil { + return externalControl(network, address, conn) + } else { + return nil + } + } + conn, err := listener.ListenPacket(context.Background(), network, listenerAddr) if err != nil { return nil, 0, err } @@ -160,13 +190,13 @@ again: var v4pc *ipv4.PacketConn var v6pc *ipv6.PacketConn - v4conn, port, err = listenNet("udp4", port) + v4conn, port, err = listenNet(s.externalControl, "udp4", port) if err != nil && !errors.Is(err, syscall.EAFNOSUPPORT) { return nil, 0, err } // Listen on the same port as we're using for ipv4. - v6conn, port, err = listenNet("udp6", port) + v6conn, port, err = listenNet(s.externalControl, "udp6", port) if uport == 0 && errors.Is(err, errEADDRINUSE) && tries < 100 { v4conn.Close() tries++ @@ -270,8 +300,10 @@ func (s *StdNetBind) receiveIP( if sizes[i] == 0 { continue } - addrPort := msg.Addr.(*net.UDPAddr).AddrPort() - ep := &StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation + if msg.N > 3 { + common.ClearArray(bufs[i][1:4]) + } + ep := &StdNetEndpoint{AddrPort: M.AddrPortFromNet(msg.Addr)} // TODO: remove allocation getSrcFromControl(msg.OOB[:msg.NN], ep) eps[i] = ep } @@ -380,6 +412,14 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error { retried bool err error ) + for _, buf := range bufs { + if len(buf) > 3 { + reserved, loaded := s.reservedForEndpoint[endpoint.(*StdNetEndpoint).AddrPort] + if loaded { + copy(buf[1:4], reserved[:]) + } + } + } retry: if offload { n := coalesceMessages(ua, endpoint.(*StdNetEndpoint), bufs, offset, *msgs, setGSOSize) @@ -410,6 +450,10 @@ retry: return err } +func (s *StdNetBind) SetReservedForEndpoint(destination netip.AddrPort, reserved [3]byte) { + s.reservedForEndpoint[destination] = reserved +} + func (s *StdNetBind) send(conn *net.UDPConn, pc batchWriter, msgs []ipv6.Message) error { var ( n int diff --git a/conn/bind_windows.go b/conn/bind_windows.go index 81a820b..51c0974 100644 --- a/conn/bind_windows.go +++ b/conn/bind_windows.go @@ -15,6 +15,10 @@ import ( "sync/atomic" "unsafe" + "github.com/sagernet/sing/common" + "github.com/sagernet/sing/common/control" + E "github.com/sagernet/sing/common/exceptions" + M "github.com/sagernet/sing/common/metadata" "github.com/sagernet/wireguard-go/conn/winrio" "golang.org/x/sys/windows" ) @@ -71,18 +75,26 @@ type afWinRingBind struct { // WinRingBind uses Windows registered I/O for fast ring buffered networking. type WinRingBind struct { + externalControl control.Func + reservedForEndpoint map[WinRingEndpoint][3]uint8 + v4, v6 afWinRingBind mu sync.RWMutex isOpen atomic.Uint32 // 0, 1, or 2 } -func NewDefaultBind() Bind { return NewWinRingBind() } +func NewDefaultBind(externalControl control.Func) Bind { + return NewWinRingBind(externalControl) +} -func NewWinRingBind() Bind { +func NewWinRingBind(externalControl control.Func) Bind { if !winrio.Initialize() { - return NewStdNetBind() + return NewStdNetBind(externalControl) + } + return &WinRingBind{ + externalControl: externalControl, + reservedForEndpoint: make(map[WinRingEndpoint][3]uint8), } - return new(WinRingBind) } type WinRingEndpoint struct { @@ -238,7 +250,7 @@ func (ring *ringBuffer) Open() error { return nil } -func (bind *afWinRingBind) Open(family int32, sa windows.Sockaddr) (windows.Sockaddr, error) { +func (bind *afWinRingBind) Open(family int32, sa windows.Sockaddr, externalControl control.Func) (windows.Sockaddr, error) { var err error bind.sock, err = winrio.Socket(family, windows.SOCK_DGRAM, windows.IPPROTO_UDP) if err != nil { @@ -256,6 +268,19 @@ func (bind *afWinRingBind) Open(family int32, sa windows.Sockaddr) (windows.Sock if err != nil { return nil, err } + var network string + switch family { + case windows.AF_INET: + network = "udp4" + case windows.AF_INET6: + network = "udp6" + } + if externalControl != nil { + err = externalControl(network, M.AddrPortFromSockaddr(sa).String(), &fakeRawConn{bind.sock}) + if err != nil { + return nil, err + } + } err = windows.Bind(bind.sock, sa) if err != nil { return nil, err @@ -267,6 +292,23 @@ func (bind *afWinRingBind) Open(family int32, sa windows.Sockaddr) (windows.Sock return sa, nil } +type fakeRawConn struct { + socket windows.Handle +} + +func (c *fakeRawConn) Control(f func(fd uintptr)) error { + f(uintptr(c.socket)) + return nil +} + +func (c *fakeRawConn) Read(f func(fd uintptr) (done bool)) error { + panic("not implemented") +} + +func (c *fakeRawConn) Write(f func(fd uintptr) (done bool)) error { + panic("not implemented") +} + func (bind *WinRingBind) Open(port uint16) (recvFns []ReceiveFunc, selectedPort uint16, err error) { bind.mu.Lock() defer bind.mu.Unlock() @@ -279,11 +321,11 @@ func (bind *WinRingBind) Open(port uint16) (recvFns []ReceiveFunc, selectedPort return nil, 0, ErrBindAlreadyOpen } var sa windows.Sockaddr - sa, err = bind.v4.Open(windows.AF_INET, &windows.SockaddrInet4{Port: int(port)}) + sa, err = bind.v4.Open(windows.AF_INET, &windows.SockaddrInet4{Port: int(port)}, bind.externalControl) if err != nil { return nil, 0, err } - sa, err = bind.v6.Open(windows.AF_INET6, &windows.SockaddrInet6{Port: sa.(*windows.SockaddrInet4).Port}) + sa, err = bind.v6.Open(windows.AF_INET6, &windows.SockaddrInet6{Port: sa.(*windows.SockaddrInet4).Port}, bind.externalControl) if err != nil { return nil, 0, err } @@ -419,6 +461,9 @@ func (bind *WinRingBind) receiveIPv4(bufs [][]byte, sizes []int, eps []Endpoint) bind.mu.RLock() defer bind.mu.RUnlock() n, ep, err := bind.v4.Receive(bufs[0], &bind.isOpen) + if n > 3 { + common.ClearArray(bufs[0][1:4]) + } sizes[0] = n eps[0] = ep return 1, err @@ -428,6 +473,9 @@ func (bind *WinRingBind) receiveIPv6(bufs [][]byte, sizes []int, eps []Endpoint) bind.mu.RLock() defer bind.mu.RUnlock() n, ep, err := bind.v6.Receive(bufs[0], &bind.isOpen) + if n > 3 { + common.ClearArray(bufs[0][1:4]) + } sizes[0] = n eps[0] = ep return 1, err @@ -494,6 +542,12 @@ func (bind *WinRingBind) Send(bufs [][]byte, endpoint Endpoint, offset int) erro defer bind.mu.RUnlock() for _, buf := range bufs { buf = buf[offset:] + if len(buf) > 3 { + reserved, loaded := bind.reservedForEndpoint[*endpoint.(*WinRingEndpoint)] + if loaded { + copy(buf[1:4], reserved[:]) + } + } switch nend.family { case windows.AF_INET: if bind.v4.blackhole { @@ -514,6 +568,14 @@ func (bind *WinRingBind) Send(bufs [][]byte, endpoint Endpoint, offset int) erro return nil } +func (bind *WinRingBind) SetReservedForEndpoint(destination netip.AddrPort, reserved [3]byte) { + endpoint, err := bind.ParseEndpoint(destination.String()) + if err != nil { + panic(E.Cause(err, "parse destination as WinRingEndpoint")) + } + bind.reservedForEndpoint[*endpoint.(*WinRingEndpoint)] = reserved +} + func (s *StdNetBind) BindSocketToInterface4(interfaceIndex uint32, blackhole bool) error { s.mu.Lock() defer s.mu.Unlock() diff --git a/conn/conn.go b/conn/conn.go index f178161..4c5c194 100644 --- a/conn/conn.go +++ b/conn/conn.go @@ -57,6 +57,8 @@ type Bind interface { // BatchSize is the number of buffers expected to be passed to // the ReceiveFuncs, and the maximum expected to be passed to SendBatch. BatchSize() int + + SetReservedForEndpoint(destination netip.AddrPort, reserved [3]byte) } // BindSocketToInterface is implemented by Bind objects that support being diff --git a/conn/controlfns.go b/conn/controlfns.go index 4f7d90f..44b52fc 100644 --- a/conn/controlfns.go +++ b/conn/controlfns.go @@ -6,8 +6,7 @@ package conn import ( - "net" - "syscall" + "github.com/sagernet/sing/common/control" ) // UDP socket read/write buffer size (7MB). The value of 7MB is chosen as it is @@ -17,27 +16,6 @@ import ( // around this limitation) const socketBufferSize = 7 << 20 -// controlFn is the callback function signature from net.ListenConfig.Control. -// It is used to apply platform specific configuration to the socket prior to -// bind. -type controlFn func(network, address string, c syscall.RawConn) error - // controlFns is a list of functions that are called from the listen config // that can apply socket options. -var controlFns = []controlFn{} - -// listenConfig returns a net.ListenConfig that applies the controlFns to the -// socket prior to bind. This is used to apply socket buffer sizing and packet -// information OOB configuration for sticky sockets. -func listenConfig() *net.ListenConfig { - return &net.ListenConfig{ - Control: func(network, address string, c syscall.RawConn) error { - for _, fn := range controlFns { - if err := fn(network, address, c); err != nil { - return err - } - } - return nil - }, - } -} +var controlFns []control.Func diff --git a/conn/default.go b/conn/default.go index b6f761b..1892c3f 100644 --- a/conn/default.go +++ b/conn/default.go @@ -7,4 +7,8 @@ package conn -func NewDefaultBind() Bind { return NewStdNetBind() } +import "github.com/sagernet/sing/common/control" + +func NewDefaultBind(externalControl control.Func) Bind { + return NewStdNetBind(externalControl) +} diff --git a/device/device.go b/device/device.go index e491e53..14cd080 100644 --- a/device/device.go +++ b/device/device.go @@ -14,7 +14,6 @@ import ( "github.com/sagernet/sing/service" "github.com/sagernet/sing/service/pause" - "github.com/sagernet/wireguard-go/conn" "github.com/sagernet/wireguard-go/ratelimiter" "github.com/sagernet/wireguard-go/rwcancel" From 16510ac47288a3f634f1b2807dab47a47de569b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Wed, 17 Sep 2025 19:03:11 +0800 Subject: [PATCH 50/60] Fix input packet --- device/send.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/device/send.go b/device/send.go index 2c690dc..cf0f693 100644 --- a/device/send.go +++ b/device/send.go @@ -328,7 +328,7 @@ func (device *Device) InputPacket(destination []byte, packetSlices [][]byte) { return } elem := device.NewOutboundElement() - packet := elem.buffer[MessageTransportHeaderSize:] + packet := elem.buffer[MessageEncapsulatingTransportSize+MessageTransportHeaderSize:] var n int for _, packetSlice := range packetSlices { n += copy(packet[n:], packetSlice) From 506b7631853cf3864335ef38f7f0b8453a285703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Tue, 24 Feb 2026 15:47:47 +0800 Subject: [PATCH 51/60] Fix reserved bytes offset in StdNetBind.Send --- conn/bind_std.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conn/bind_std.go b/conn/bind_std.go index a1d4d9e..6bf5978 100644 --- a/conn/bind_std.go +++ b/conn/bind_std.go @@ -413,10 +413,10 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error { err error ) for _, buf := range bufs { - if len(buf) > 3 { + if len(buf) > offset+3 { reserved, loaded := s.reservedForEndpoint[endpoint.(*StdNetEndpoint).AddrPort] if loaded { - copy(buf[1:4], reserved[:]) + copy(buf[offset+1:offset+4], reserved[:]) } } } From 12a012205e3c444be02aba91a840455f74c127e1 Mon Sep 17 00:00:00 2001 From: Yaroslav Gurov Date: Tue, 31 Mar 2026 15:48:57 +0000 Subject: [PATCH 52/60] readme: actualize type for H1-H4 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 301b5bf..e1c7309 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ The amount of junk packets specified in `Jc` with a random size between `Jmin` a ### Message headers -Every message in wireguard has `int32` type at the beginning of the packet. This field could be controlled by specifying the params below: +Every message in wireguard has `uint32` type at the beginning of the packet. This field could be controlled by specifying the params below: - `H1: string` - header range of handshake initial message - `H2: string` - header range of handshake initial message From f4f4c999267437c3eb909e8d0e5278fb4596d9a7 Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 31 Mar 2026 16:37:57 +0300 Subject: [PATCH 53/60] fix: apply S4 transport padding to keepalive packets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keepalive packets were excluded from S4 padding because the padding logic was nested inside the dataSent guard. The receiving side (DeterminePacketTypeAndPadding) expects S4 padding on all transport packets, so unpadded keepalives fail H4 header validation and are silently dropped. This prevents the responder from completing key confirmation — lastHandshakeNano stays 0 until real data flows through the tunnel. --- device/send.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/device/send.go b/device/send.go index 5e5cc1b..0cc57da 100644 --- a/device/send.go +++ b/device/send.go @@ -574,16 +574,15 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) { for _, elem := range elemsContainer.elems { if len(elem.packet) != MessageKeepaliveSize { dataSent = true - - if padding := device.paddings.transport; padding > 0 { - // elem.packet is stored at the start of elem.buffer - // with zero padding - for i := len(elem.packet) - 1; i >= 0; i-- { - elem.buffer[i+padding] = elem.buffer[i] - } - rand.Read(elem.buffer[:padding]) - elem.packet = elem.buffer[:padding+len(elem.packet)] + } + if padding := device.paddings.transport; padding > 0 { + // elem.packet is stored at the start of elem.buffer + // with zero padding + for i := len(elem.packet) - 1; i >= 0; i-- { + elem.buffer[i+padding] = elem.buffer[i] } + rand.Read(elem.buffer[:padding]) + elem.packet = elem.buffer[:padding+len(elem.packet)] } bufs = append(bufs, elem.packet) } From 58f534ff106c89ae30ce6b1cf16a9906aa31a83a Mon Sep 17 00:00:00 2001 From: Leadaxe <247031499+Leadaxe@users.noreply.github.com> Date: Tue, 9 Jun 2026 17:51:32 +0300 Subject: [PATCH 54/60] =?UTF-8?q?docs:=20add=20README=20=E2=80=94=20sagern?= =?UTF-8?q?et/wireguard-go=20+=20AmneziaWG=202.0=20(3-way=20merge)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explain why this fork exists (sing-box needs sagernet APIs, AmneziaWG obfuscation lives in amneziawg-go, neither alone works), how the merge is done (3-way merge + MessageEncapsulatingTransportSize=0, obfuscation isolated to device/), how sing-box-lx consumes it (submodule + replace), and the recipe to rebase onto a new sagernet tag. --- README.md | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..90b89dc --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# wireguard-go (lx fork) — sagernet + AmneziaWG 2.0 + +The WireGuard-Go runtime used by **[sing-box-lx](https://github.com/Leadaxe/sing-box-lx)**: +**[sagernet/wireguard-go](https://github.com/sagernet/wireguard-go)** (the fork sing-box builds on) **+ AmneziaWG 2.0 obfuscation**, merged together. + +This is **not** a general-purpose project. It exists for one reason — see below — and lives on the **`lx`** branch. + +--- + +## Why this fork exists + +sing-box's WireGuard endpoint needs **sagernet/wireguard-go**'s additions (the `conn.Bind.Send(…, offset)` contract, `device.InputPacket`, reserved/control). AmneziaWG's DPI-evasion obfuscation lives in **[amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go)**, which is a fork of *upstream* wireguard-go and therefore **lacks** those sagernet additions. + +So neither fork alone works for sing-box-lx: + +| | sing-box-compat API | AmneziaWG obfuscation | +|---|:---:|:---:| +| `sagernet/wireguard-go` | ✅ | ❌ | +| `amnezia-vpn/amneziawg-go` | ❌ | ✅ | +| **this fork** | ✅ | ✅ | + +The approach is **(A): keep the sagernet base, graft the obfuscation onto it.** sing-box compiles unchanged against this fork, and a config without AWG fields behaves exactly like plain WireGuard. + +## How the merge works + +Both `sagernet/wireguard-go` and `amneziawg-go` descend from the same upstream `git.zx2c4.com/wireguard-go`, so they share git history — which makes a real **3-way merge** possible (not a hand-port). + +- **Base:** `sagernet/wireguard-go` (the exact commit sing-box pins — currently `506b7631853c`). +- **Merged in:** `amnezia-vpn/amneziawg-go` (a tip with AWG2 / I1–I5 + the S4-keepalive fix). +- **Key trick:** `MessageEncapsulatingTransportSize` is set to **`0`** in `device/noise-protocol.go`. sing-box-lx does not use sagernet's 8-byte `Bind.Send` headroom, and zeroing it makes the AmneziaWG obfuscation compose cleanly with no weave conflicts in the packet send path. +- **Isolation:** the obfuscation is confined to `device/` — new files `device/obf*.go`, `device/magic-header.go`, plus grafts in `device/{send,receive,device,uapi}.go`. **`conn/`, `tun/`, `ipc/` stay pure sagernet.** +- **Module path is unchanged** (`module github.com/sagernet/wireguard-go`) so the consumer plugs it in with a `replace` directive and needs **no import edits**. + +## Consumed by + +[sing-box-lx](https://github.com/Leadaxe/sing-box-lx) wires this in as a git submodule + a `replace`: + +``` +# sing-box-lx/.gitmodules +[submodule "submodules/wireguard-go"] + url = https://github.com/Leadaxe/wireguard-go + branch = lx + +# sing-box-lx/go.mod (// lx) +replace github.com/sagernet/wireguard-go => ./submodules/wireguard-go +``` + +Built with the `with_awg` tag, it has been **live-validated** against a real AmneziaWG 2.0 server (handshake + keepalive + outbound traffic) and cross-compiles on linux/darwin/windows × amd64/arm64. + +## Maintaining it (rebase onto a new sagernet tag) + +When sing-box bumps `sagernet/wireguard-go`, redo the merge: + +```sh +git remote add origin https://github.com/sagernet/wireguard-go # base +git remote add amnezia https://github.com/amnezia-vpn/amneziawg-go # obfuscation source +git fetch --all +git checkout -b lx +git merge amnezia/master # real 3-way merge via the shared upstream ancestor +``` + +Conflict resolution recipe: + +1. New `device/obf*.go` + `device/magic-header.go` come in clean. +2. **Mechanical** conflicts (amnezia → sagernet import paths, `queueconstants*`, `sticky*`, `tun.go`) → take **ours** (sagernet). +3. Remove amnezia-added infra duplicates: `conn/gso_*.go`, `outline/*`, `tun/*_test.go`. +4. `device/device.go` → **union** (sagernet `pauseManager` + amnezia obf fields). +5. `device/send.go` / `receive.go` → take amnezia's obfuscation, set `MessageEncapsulatingTransportSize = 0`, and keep the 3-arg `bind.Send(…, 0)` calls. +6. `conn/`, `tun/`, `ipc/`, `go.mod` module path → **ours** (sagernet). + +Then in sing-box-lx: bump the submodule, `make -f Makefile.lx lx-build`, and re-test against an AWG2 server. + +## Links + +| | | +|---|---| +| Consumer | [Leadaxe/sing-box-lx](https://github.com/Leadaxe/sing-box-lx) | +| Base | [sagernet/wireguard-go](https://github.com/sagernet/wireguard-go) | +| Obfuscation source | [amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go) · [docs.amnezia.org](https://docs.amnezia.org/documentation/amnezia-wg/) | +| Original | [WireGuard/wireguard-go](https://git.zx2c4.com/wireguard-go/about/) | + +## License + +MIT, inherited from WireGuard-Go (see [`LICENSE`](LICENSE)). The AmneziaWG obfuscation is likewise MIT (from amneziawg-go). This is an unofficial fork, not affiliated with WireGuard, SagerNet, or Amnezia. From 7e50606f8014633270abe13e801fc92c4256afeb Mon Sep 17 00:00:00 2001 From: Leadaxe <247031499+Leadaxe@users.noreply.github.com> Date: Tue, 9 Jun 2026 17:58:39 +0300 Subject: [PATCH 55/60] docs: bilingual README (EN + RU); repo renamed to wireguard-go-awg2-lx --- README.md | 4 ++- README.ru.md | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 README.ru.md diff --git a/README.md b/README.md index 90b89dc..42b94d1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +**English** · [Русский](README.ru.md) + # wireguard-go (lx fork) — sagernet + AmneziaWG 2.0 The WireGuard-Go runtime used by **[sing-box-lx](https://github.com/Leadaxe/sing-box-lx)**: @@ -38,7 +40,7 @@ Both `sagernet/wireguard-go` and `amneziawg-go` descend from the same upstream ` ``` # sing-box-lx/.gitmodules [submodule "submodules/wireguard-go"] - url = https://github.com/Leadaxe/wireguard-go + url = https://github.com/Leadaxe/wireguard-go-awg2-lx branch = lx # sing-box-lx/go.mod (// lx) diff --git a/README.ru.md b/README.ru.md new file mode 100644 index 0000000..9acd5d5 --- /dev/null +++ b/README.ru.md @@ -0,0 +1,86 @@ +[English](README.md) · **Русский** + +# wireguard-go (lx-форк) — sagernet + AmneziaWG 2.0 + +Рантайм WireGuard-Go для **[sing-box-lx](https://github.com/Leadaxe/sing-box-lx)**: +**[sagernet/wireguard-go](https://github.com/sagernet/wireguard-go)** (форк, на котором собирается sing-box) **+ обфускация AmneziaWG 2.0**, слитые вместе. + +Это **не** универсальный проект. Он существует ради одной задачи (см. ниже) и живёт на ветке **`lx`**. + +--- + +## Зачем этот форк + +WireGuard-endpoint sing-box нуждается в добавках **sagernet/wireguard-go** (контракт `conn.Bind.Send(…, offset)`, `device.InputPacket`, reserved/control). Обфускация против DPI у AmneziaWG живёт в **[amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go)**, который форкнут от *upstream* wireguard-go и потому этих sagernet-добавок **не имеет**. + +Значит для sing-box-lx ни один форк по отдельности не подходит: + +| | API под sing-box | обфускация AmneziaWG | +|---|:---:|:---:| +| `sagernet/wireguard-go` | ✅ | ❌ | +| `amnezia-vpn/amneziawg-go` | ❌ | ✅ | +| **этот форк** | ✅ | ✅ | + +Подход **(A): берём sagernet-базу и граффтим обфускацию на неё.** sing-box компилируется с этим форком без изменений, а конфиг без AWG-полей ведёт себя как обычный WireGuard. + +## Как устроен merge + +И `sagernet/wireguard-go`, и `amneziawg-go` происходят от одного upstream `git.zx2c4.com/wireguard-go`, поэтому делят git-историю — а значит возможен настоящий **3-way merge** (а не ручной перенос). + +- **База:** `sagernet/wireguard-go` (тот коммит, что пинит sing-box — сейчас `506b7631853c`). +- **Вливаем:** `amnezia-vpn/amneziawg-go` (тип с AWG2 / I1–I5 + фикс S4-keepalive). +- **Ключевой трюк:** `MessageEncapsulatingTransportSize` выставлен в **`0`** в `device/noise-protocol.go`. sing-box-lx не использует 8-байтный headroom sagernet для `Bind.Send`, и обнуление позволяет обфускации AmneziaWG встать чисто, без конфликтов в send-пути. +- **Изоляция:** обфускация замкнута в `device/` — новые файлы `device/obf*.go`, `device/magic-header.go` + графты в `device/{send,receive,device,uapi}.go`. **`conn/`, `tun/`, `ipc/` остаются чистым sagernet.** +- **Module-path не меняется** (`module github.com/sagernet/wireguard-go`), поэтому потребитель подключает форк через `replace` без правки импортов. + +## Кто потребляет + +[sing-box-lx](https://github.com/Leadaxe/sing-box-lx) подключает это как git submodule + `replace`: + +``` +# sing-box-lx/.gitmodules +[submodule "submodules/wireguard-go"] + url = https://github.com/Leadaxe/wireguard-go-awg2-lx + branch = lx + +# sing-box-lx/go.mod (// lx) +replace github.com/sagernet/wireguard-go => ./submodules/wireguard-go +``` + +Собранный с тегом `with_awg`, он **проверен живым** сервером AmneziaWG 2.0 (handshake + keepalive + трафик наружу) и кросс-компилируется на linux/darwin/windows × amd64/arm64. + +## Сопровождение (ребейз на новый sagernet-тег) + +Когда sing-box бампит `sagernet/wireguard-go`, повторяем merge: + +```sh +git remote add origin https://github.com/sagernet/wireguard-go # база +git remote add amnezia https://github.com/amnezia-vpn/amneziawg-go # источник обфускации +git fetch --all +git checkout -b lx <новый-sagernet-коммит> +git merge amnezia/master # настоящий 3-way merge через общего upstream-предка +``` + +Рецепт разрешения конфликтов: + +1. Новые `device/obf*.go` + `device/magic-header.go` приходят чисто. +2. **Механические** конфликты (import-path amnezia → sagernet, `queueconstants*`, `sticky*`, `tun.go`) → берём **наши** (sagernet). +3. Удаляем amnezia-инфра-дубликаты: `conn/gso_*.go`, `outline/*`, `tun/*_test.go`. +4. `device/device.go` → **union** (sagernet `pauseManager` + obf-поля amnezia). +5. `device/send.go` / `receive.go` → берём обфускацию amnezia, ставим `MessageEncapsulatingTransportSize = 0`, сохраняем 3-арг `bind.Send(…, 0)`. +6. `conn/`, `tun/`, `ipc/`, module-path в `go.mod` → **наши** (sagernet). + +Затем в sing-box-lx: бампим submodule, `make -f Makefile.lx lx-build` и пере-тест против AWG2-сервера. + +## Ссылки + +| | | +|---|---| +| Потребитель | [Leadaxe/sing-box-lx](https://github.com/Leadaxe/sing-box-lx) | +| База | [sagernet/wireguard-go](https://github.com/sagernet/wireguard-go) | +| Источник обфускации | [amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go) · [docs.amnezia.org](https://docs.amnezia.org/documentation/amnezia-wg/) | +| Оригинал | [WireGuard/wireguard-go](https://git.zx2c4.com/wireguard-go/about/) | + +## Лицензия + +MIT, унаследована от WireGuard-Go (см. [`LICENSE`](LICENSE)). Обфускация AmneziaWG — тоже MIT (из amneziawg-go). Это неофициальный форк, не аффилирован с WireGuard, SagerNet или Amnezia. From a2307b51f775dba98f3b467531b3a3a4fc0f6f3f Mon Sep 17 00:00:00 2001 From: Leadaxe <247031499+Leadaxe@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:04:06 +0300 Subject: [PATCH 56/60] docs: drop dangling "(A)" label; explain the merge direction inline --- README.md | 2 +- README.ru.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 42b94d1..f62356a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ So neither fork alone works for sing-box-lx: | `amnezia-vpn/amneziawg-go` | ❌ | ✅ | | **this fork** | ✅ | ✅ | -The approach is **(A): keep the sagernet base, graft the obfuscation onto it.** sing-box compiles unchanged against this fork, and a config without AWG fields behaves exactly like plain WireGuard. +The approach: **keep the sagernet base and graft the obfuscation onto it** — rather than the reverse (adding sagernet's APIs to amneziawg-go, which would route even plain WireGuard through a foreign device). This way sing-box compiles unchanged, the obfuscation is additive and off by default, and a config without AWG fields behaves exactly like plain WireGuard. ## How the merge works diff --git a/README.ru.md b/README.ru.md index 9acd5d5..842ba7c 100644 --- a/README.ru.md +++ b/README.ru.md @@ -21,7 +21,7 @@ WireGuard-endpoint sing-box нуждается в добавках **sagernet/wi | `amnezia-vpn/amneziawg-go` | ❌ | ✅ | | **этот форк** | ✅ | ✅ | -Подход **(A): берём sagernet-базу и граффтим обфускацию на неё.** sing-box компилируется с этим форком без изменений, а конфиг без AWG-полей ведёт себя как обычный WireGuard. +Подход: **берём sagernet-базу и граффтим обфускацию на неё** — а не наоборот (не дотачиваем sagernet-API к amneziawg-go, иначе даже обычный WireGuard шёл бы через чужой device). Так sing-box компилируется без изменений, обфускация аддитивна и выключена по умолчанию, а конфиг без AWG-полей ведёт себя как обычный WireGuard. ## Как устроен merge From 7b15aacbfef2219ba4eea47c8d6245a000a29e36 Mon Sep 17 00:00:00 2001 From: Leadaxe <247031499+Leadaxe@users.noreply.github.com> Date: Tue, 9 Jun 2026 18:05:40 +0300 Subject: [PATCH 57/60] docs: spell out why neither fork alone works (each gives half) before the merge rationale --- README.md | 7 +++++++ README.ru.md | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/README.md b/README.md index f62356a..dd92535 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,13 @@ So neither fork alone works for sing-box-lx: | `amnezia-vpn/amneziawg-go` | ❌ | ✅ | | **this fork** | ✅ | ✅ | +Each existing fork gives exactly **half** of what's needed: + +- **Take `sagernet/wireguard-go`** → sing-box-lx compiles and runs, but the AWG fields (`jc`/`h1`/`i1`…) do nothing → **no obfuscation**; AmneziaWG doesn't actually work. +- **Take `amnezia-vpn/amneziawg-go`** → the obfuscation is there, but sing-box-lx **won't even compile** (the sagernet functions are missing). + +We need **both** ✅ at once, and no ready-made fork has them — so we built one by **merging**: sagernet (for the API) + amnezia (for the obfuscation). That is exactly the **"this fork"** row above. + The approach: **keep the sagernet base and graft the obfuscation onto it** — rather than the reverse (adding sagernet's APIs to amneziawg-go, which would route even plain WireGuard through a foreign device). This way sing-box compiles unchanged, the obfuscation is additive and off by default, and a config without AWG fields behaves exactly like plain WireGuard. ## How the merge works diff --git a/README.ru.md b/README.ru.md index 842ba7c..7482bd4 100644 --- a/README.ru.md +++ b/README.ru.md @@ -21,6 +21,13 @@ WireGuard-endpoint sing-box нуждается в добавках **sagernet/wi | `amnezia-vpn/amneziawg-go` | ❌ | ✅ | | **этот форк** | ✅ | ✅ | +Каждый существующий форк даёт ровно **половину** нужного: + +- **возьмёшь `sagernet/wireguard-go`** → sing-box-lx соберётся и запустится, но AWG-поля (`jc`/`h1`/`i1`…) ничего не сделают → **обфускации нет**; AmneziaWG не работает; +- **возьмёшь `amnezia-vpn/amneziawg-go`** → обфускация есть, но sing-box-lx **не скомпилируется** (нет sagernet-функций). + +Нужны **обе** ✅ сразу, а готового форка с двумя галочками не существует — поэтому мы собрали его **слиянием**: sagernet (за API) + amnezia (за обфускацию). Это ровно строка **«этот форк»** выше. + Подход: **берём sagernet-базу и граффтим обфускацию на неё** — а не наоборот (не дотачиваем sagernet-API к amneziawg-go, иначе даже обычный WireGuard шёл бы через чужой device). Так sing-box компилируется без изменений, обфускация аддитивна и выключена по умолчанию, а конфиг без AWG-полей ведёт себя как обычный WireGuard. ## Как устроен merge From fb8d8d8fcaf39fc5e7e38526b5c95e3d923ad4b8 Mon Sep 17 00:00:00 2001 From: Leadaxe <247031499+Leadaxe@users.noreply.github.com> Date: Sun, 21 Jun 2026 15:49:57 +0300 Subject: [PATCH 58/60] fix(010): never enable UDP_GRO on android (GRO split-brain killed download) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runtime.GOOS=="android" (not "linux"), so StdNetBind enabled UDP_GRO and read rxOffload=true, but the GRO receive dispatcher in bind_std.go is gated on GOOS=="linux" → dead on android. A coalesced GRO super-packet was read as one datagram, corrupting the WG transport stream → download died on no-detour WG-endpoint (detour path uses ClientBind, no offload, so it worked). Gate both the UDP_GRO setsockopt (controlfns_linux.go) and the rxOffload readback (features_linux.go) behind !android. TX/GSO untouched, so non-android linux performance is unchanged. Confirmed on device (CPH2411/Android-15: probe gave rxOffload=true + dispatch=single). See SPECS/010-B-O-WG_ENDPOINT_GRO_SPLIT_BRAIN. --- conn/controlfns_linux.go | 9 +++++++++ conn/features_linux.go | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/conn/controlfns_linux.go b/conn/controlfns_linux.go index 752fbca..7602665 100644 --- a/conn/controlfns_linux.go +++ b/conn/controlfns_linux.go @@ -60,6 +60,15 @@ func init() { // Attempt to enable UDP_GRO func(network, address string, c syscall.RawConn) error { + // lx(010): skip UDP_GRO on android. The GRO receive path in bind_std.go + // is gated on runtime.GOOS=="linux", which is false on android — so a + // coalesced super-packet is never split and corrupts the WG stream + // (download dies). Belt-and-suspenders with the rxOffload guard in + // features_linux.go. TX/GSO untouched. + // See SPECS/010-B-O-WG_ENDPOINT_GRO_SPLIT_BRAIN. + if runtime.GOOS == "android" { + return nil + } c.Control(func(fd uintptr) { _ = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO, 1) }) diff --git a/conn/features_linux.go b/conn/features_linux.go index 513202e..b74c398 100644 --- a/conn/features_linux.go +++ b/conn/features_linux.go @@ -7,6 +7,7 @@ package conn import ( "net" + "runtime" "golang.org/x/sys/unix" ) @@ -29,6 +30,15 @@ func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) { return } txOffload = true + // lx(010): never advertise RX offload on android. runtime.GOOS=="android" + // (not "linux"), so the GRO receive dispatcher in bind_std.go (gated on + // GOOS=="linux") is dead there — a coalesced GRO super-packet would be read + // as one datagram and corrupt the WG transport stream, killing download. + // Confirmed on device (CPH2411/Android-15: rxOffload=true, dispatch=single). + // TX is left untouched. See SPECS/010-B-O-WG_ENDPOINT_GRO_SPLIT_BRAIN. + if runtime.GOOS == "android" { + return + } opt, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO) if errSyscall != nil { return From 6513629626537c5626bdd6c400481a728326b413 Mon Sep 17 00:00:00 2001 From: Leadaxe <247031499+Leadaxe@users.noreply.github.com> Date: Sun, 21 Jun 2026 17:52:53 +0300 Subject: [PATCH 59/60] =?UTF-8?q?docs(010):=20spec=20folder=20O=E2=86=92C?= =?UTF-8?q?=20in=20fix=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conn/controlfns_linux.go | 2 +- conn/features_linux.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conn/controlfns_linux.go b/conn/controlfns_linux.go index 7602665..8c8b652 100644 --- a/conn/controlfns_linux.go +++ b/conn/controlfns_linux.go @@ -65,7 +65,7 @@ func init() { // coalesced super-packet is never split and corrupts the WG stream // (download dies). Belt-and-suspenders with the rxOffload guard in // features_linux.go. TX/GSO untouched. - // See SPECS/010-B-O-WG_ENDPOINT_GRO_SPLIT_BRAIN. + // See SPECS/010-B-C-WG_ENDPOINT_GRO_SPLIT_BRAIN. if runtime.GOOS == "android" { return nil } diff --git a/conn/features_linux.go b/conn/features_linux.go index b74c398..7d12890 100644 --- a/conn/features_linux.go +++ b/conn/features_linux.go @@ -35,7 +35,7 @@ func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) { // GOOS=="linux") is dead there — a coalesced GRO super-packet would be read // as one datagram and corrupt the WG transport stream, killing download. // Confirmed on device (CPH2411/Android-15: rxOffload=true, dispatch=single). - // TX is left untouched. See SPECS/010-B-O-WG_ENDPOINT_GRO_SPLIT_BRAIN. + // TX is left untouched. See SPECS/010-B-C-WG_ENDPOINT_GRO_SPLIT_BRAIN. if runtime.GOOS == "android" { return } From 0c0c10b5d3236796bd3832a6813223d6dc7d0bb1 Mon Sep 17 00:00:00 2001 From: Leadaxe <247031499+Leadaxe@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:27:23 +0300 Subject: [PATCH 60/60] docs(010): use rename-stable SPECS folder ref in fix comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sing-box-lx is dropping type/status from SPEC folder names (NNN-T-S-NAME → NNN-NAME), so the -B-C- tail goes stale. Point at the new stable folder name SPECS/010-WG_ENDPOINT_GRO_SPLIT_BRAIN. Comment-only, no behavior change. --- conn/controlfns_linux.go | 2 +- conn/features_linux.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conn/controlfns_linux.go b/conn/controlfns_linux.go index 8c8b652..ff591f1 100644 --- a/conn/controlfns_linux.go +++ b/conn/controlfns_linux.go @@ -65,7 +65,7 @@ func init() { // coalesced super-packet is never split and corrupts the WG stream // (download dies). Belt-and-suspenders with the rxOffload guard in // features_linux.go. TX/GSO untouched. - // See SPECS/010-B-C-WG_ENDPOINT_GRO_SPLIT_BRAIN. + // See SPECS/010-WG_ENDPOINT_GRO_SPLIT_BRAIN. if runtime.GOOS == "android" { return nil } diff --git a/conn/features_linux.go b/conn/features_linux.go index 7d12890..69dd2f4 100644 --- a/conn/features_linux.go +++ b/conn/features_linux.go @@ -35,7 +35,7 @@ func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) { // GOOS=="linux") is dead there — a coalesced GRO super-packet would be read // as one datagram and corrupt the WG transport stream, killing download. // Confirmed on device (CPH2411/Android-15: rxOffload=true, dispatch=single). - // TX is left untouched. See SPECS/010-B-C-WG_ENDPOINT_GRO_SPLIT_BRAIN. + // TX is left untouched. See SPECS/010-WG_ENDPOINT_GRO_SPLIT_BRAIN. if runtime.GOOS == "android" { return }