docs(010): use rename-stable SPECS folder ref in fix comments

sing-box-lx is dropping type/status from SPEC folder names
(NNN-T-S-NAME → NNN-NAME), so the -B-C- tail goes stale. Point at the
new stable folder name SPECS/010-WG_ENDPOINT_GRO_SPLIT_BRAIN.
Comment-only, no behavior change.

.github/workflows/build-if-tag.yml

@@ -0,0 +1,41 @@
+name: build-if-tag
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+
+env:
+  APP: amneziawg-go
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref_name }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Setup metadata
+        uses: docker/metadata-action@v5
+        id: metadata
+        with:
+          images: amneziavpn/${{ env.APP }}
+          tags: type=semver,pattern={{version}}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          tags: ${{ steps.metadata.outputs.tags }}

Dockerfile

@@ -0,0 +1,18 @@
+FROM golang:1.24.4 as awg
+COPY . /awg
+WORKDIR /awg
+RUN go mod download && \
+    go mod verify && \
+    go build -ldflags '-linkmode external -extldflags "-fno-PIC -static"' -v -o /usr/bin
+
+FROM alpine:3.19
+ARG AWGTOOLS_RELEASE="1.0.20250901"
+
+RUN apk --no-cache add iproute2 iptables bash && \
+    cd /usr/bin/ && \
+    wget https://github.com/amnezia-vpn/amneziawg-tools/releases/download/v${AWGTOOLS_RELEASE}/alpine-3.19-amneziawg-tools.zip && \
+    unzip -j alpine-3.19-amneziawg-tools.zip && \
+    chmod +x /usr/bin/awg /usr/bin/awg-quick && \
+    ln -s /usr/bin/awg /usr/bin/wg && \
+    ln -s /usr/bin/awg-quick /usr/bin/wg-quick
+COPY --from=awg /usr/bin/amneziawg-go /usr/bin/amneziawg-go

LICENSE

@@ -1,3 +1,5 @@
+Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
 the Software without restriction, including without limitation the rights to

Makefile

@@ -9,23 +9,23 @@ MAKEFLAGS += --no-print-directory
 
 generate-version-and-build:
 	@export GIT_CEILING_DIRECTORIES="$(realpath $(CURDIR)/..)" && \
-	tag="$$(git describe --dirty 2>/dev/null)" && \
+	tag="$$(git describe --tags --dirty 2>/dev/null)" && \
 	ver="$$(printf 'package main\n\nconst Version = "%s"\n' "$$tag")" && \
 	[ "$$(cat version.go 2>/dev/null)" != "$$ver" ] && \
 	echo "$$ver" > version.go && \
 	git update-index --assume-unchanged version.go || true
-	@$(MAKE) wireguard-go
+	@$(MAKE) amneziawg-go
 
-wireguard-go: $(wildcard *.go) $(wildcard */*.go)
+amneziawg-go: $(wildcard *.go) $(wildcard */*.go)
 	go build -v -o "$@"
 
-install: wireguard-go
-	@install -v -d "$(DESTDIR)$(BINDIR)" && install -v -m 0755 "$<" "$(DESTDIR)$(BINDIR)/wireguard-go"
+install: amneziawg-go
+	@install -v -d "$(DESTDIR)$(BINDIR)" && install -v -m 0755 "$<" "$(DESTDIR)$(BINDIR)/amneziawg-go"
 
 test:
 	go test ./...
 
 clean:
-	rm -f wireguard-go
+	rm -f amneziawg-go
 
 .PHONY: all clean test install generate-version-and-build

README.md

@@ -0,0 +1,93 @@
+**English** · [Русский](README.ru.md)
+
+# wireguard-go (lx fork) — sagernet + AmneziaWG 2.0
+
+The WireGuard-Go runtime used by **[sing-box-lx](https://github.com/Leadaxe/sing-box-lx)**:
+**[sagernet/wireguard-go](https://github.com/sagernet/wireguard-go)** (the fork sing-box builds on) **+ AmneziaWG 2.0 obfuscation**, merged together.
+
+This is **not** a general-purpose project. It exists for one reason — see below — and lives on the **`lx`** branch.
+
+---
+
+## Why this fork exists
+
+sing-box's WireGuard endpoint needs **sagernet/wireguard-go**'s additions (the `conn.Bind.Send(…, offset)` contract, `device.InputPacket`, reserved/control). AmneziaWG's DPI-evasion obfuscation lives in **[amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go)**, which is a fork of *upstream* wireguard-go and therefore **lacks** those sagernet additions.
+
+So neither fork alone works for sing-box-lx:
+
+| | sing-box-compat API | AmneziaWG obfuscation |
+|---|:---:|:---:|
+| `sagernet/wireguard-go` | ✅ | ❌ |
+| `amnezia-vpn/amneziawg-go` | ❌ | ✅ |
+| **this fork** | ✅ | ✅ |
+
+Each existing fork gives exactly **half** of what's needed:
+
+- **Take `sagernet/wireguard-go`** → sing-box-lx compiles and runs, but the AWG fields (`jc`/`h1`/`i1`…) do nothing → **no obfuscation**; AmneziaWG doesn't actually work.
+- **Take `amnezia-vpn/amneziawg-go`** → the obfuscation is there, but sing-box-lx **won't even compile** (the sagernet functions are missing).
+
+We need **both** ✅ at once, and no ready-made fork has them — so we built one by **merging**: sagernet (for the API) + amnezia (for the obfuscation). That is exactly the **"this fork"** row above.
+
+The approach: **keep the sagernet base and graft the obfuscation onto it** — rather than the reverse (adding sagernet's APIs to amneziawg-go, which would route even plain WireGuard through a foreign device). This way sing-box compiles unchanged, the obfuscation is additive and off by default, and a config without AWG fields behaves exactly like plain WireGuard.
+
+## How the merge works
+
+Both `sagernet/wireguard-go` and `amneziawg-go` descend from the same upstream `git.zx2c4.com/wireguard-go`, so they share git history — which makes a real **3-way merge** possible (not a hand-port).
+
+- **Base:** `sagernet/wireguard-go` (the exact commit sing-box pins — currently `506b7631853c`).
+- **Merged in:** `amnezia-vpn/amneziawg-go` (a tip with AWG2 / I1–I5 + the S4-keepalive fix).
+- **Key trick:** `MessageEncapsulatingTransportSize` is set to **`0`** in `device/noise-protocol.go`. sing-box-lx does not use sagernet's 8-byte `Bind.Send` headroom, and zeroing it makes the AmneziaWG obfuscation compose cleanly with no weave conflicts in the packet send path.
+- **Isolation:** the obfuscation is confined to `device/` — new files `device/obf*.go`, `device/magic-header.go`, plus grafts in `device/{send,receive,device,uapi}.go`. **`conn/`, `tun/`, `ipc/` stay pure sagernet.**
+- **Module path is unchanged** (`module github.com/sagernet/wireguard-go`) so the consumer plugs it in with a `replace` directive and needs **no import edits**.
+
+## Consumed by
+
+[sing-box-lx](https://github.com/Leadaxe/sing-box-lx) wires this in as a git submodule + a `replace`:
+
+```
+# sing-box-lx/.gitmodules
+[submodule "submodules/wireguard-go"]
+    url = https://github.com/Leadaxe/wireguard-go-awg2-lx
+    branch = lx
+
+# sing-box-lx/go.mod   (// lx)
+replace github.com/sagernet/wireguard-go => ./submodules/wireguard-go
+```
+
+Built with the `with_awg` tag, it has been **live-validated** against a real AmneziaWG 2.0 server (handshake + keepalive + outbound traffic) and cross-compiles on linux/darwin/windows × amd64/arm64.
+
+## Maintaining it (rebase onto a new sagernet tag)
+
+When sing-box bumps `sagernet/wireguard-go`, redo the merge:
+
+```sh
+git remote add origin  https://github.com/sagernet/wireguard-go    # base
+git remote add amnezia https://github.com/amnezia-vpn/amneziawg-go # obfuscation source
+git fetch --all
+git checkout -b lx <new-sagernet-commit>
+git merge amnezia/master          # real 3-way merge via the shared upstream ancestor
+```
+
+Conflict resolution recipe:
+
+1. New `device/obf*.go` + `device/magic-header.go` come in clean.
+2. **Mechanical** conflicts (amnezia → sagernet import paths, `queueconstants*`, `sticky*`, `tun.go`) → take **ours** (sagernet).
+3. Remove amnezia-added infra duplicates: `conn/gso_*.go`, `outline/*`, `tun/*_test.go`.
+4. `device/device.go` → **union** (sagernet `pauseManager` + amnezia obf fields).
+5. `device/send.go` / `receive.go` → take amnezia's obfuscation, set `MessageEncapsulatingTransportSize = 0`, and keep the 3-arg `bind.Send(…, 0)` calls.
+6. `conn/`, `tun/`, `ipc/`, `go.mod` module path → **ours** (sagernet).
+
+Then in sing-box-lx: bump the submodule, `make -f Makefile.lx lx-build`, and re-test against an AWG2 server.
+
+## Links
+
+| | |
+|---|---|
+| Consumer | [Leadaxe/sing-box-lx](https://github.com/Leadaxe/sing-box-lx) |
+| Base | [sagernet/wireguard-go](https://github.com/sagernet/wireguard-go) |
+| Obfuscation source | [amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go) · [docs.amnezia.org](https://docs.amnezia.org/documentation/amnezia-wg/) |
+| Original | [WireGuard/wireguard-go](https://git.zx2c4.com/wireguard-go/about/) |
+
+## License
+
+MIT, inherited from WireGuard-Go (see [`LICENSE`](LICENSE)). The AmneziaWG obfuscation is likewise MIT (from amneziawg-go). This is an unofficial fork, not affiliated with WireGuard, SagerNet, or Amnezia.

README.ru.md

@@ -0,0 +1,93 @@
+[English](README.md) · **Русский**
+
+# wireguard-go (lx-форк) — sagernet + AmneziaWG 2.0
+
+Рантайм WireGuard-Go для **[sing-box-lx](https://github.com/Leadaxe/sing-box-lx)**:
+**[sagernet/wireguard-go](https://github.com/sagernet/wireguard-go)** (форк, на котором собирается sing-box) **+ обфускация AmneziaWG 2.0**, слитые вместе.
+
+Это **не** универсальный проект. Он существует ради одной задачи (см. ниже) и живёт на ветке **`lx`**.
+
+---
+
+## Зачем этот форк
+
+WireGuard-endpoint sing-box нуждается в добавках **sagernet/wireguard-go** (контракт `conn.Bind.Send(…, offset)`, `device.InputPacket`, reserved/control). Обфускация против DPI у AmneziaWG живёт в **[amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go)**, который форкнут от *upstream* wireguard-go и потому этих sagernet-добавок **не имеет**.
+
+Значит для sing-box-lx ни один форк по отдельности не подходит:
+
+| | API под sing-box | обфускация AmneziaWG |
+|---|:---:|:---:|
+| `sagernet/wireguard-go` | ✅ | ❌ |
+| `amnezia-vpn/amneziawg-go` | ❌ | ✅ |
+| **этот форк** | ✅ | ✅ |
+
+Каждый существующий форк даёт ровно **половину** нужного:
+
+- **возьмёшь `sagernet/wireguard-go`** → sing-box-lx соберётся и запустится, но AWG-поля (`jc`/`h1`/`i1`…) ничего не сделают → **обфускации нет**; AmneziaWG не работает;
+- **возьмёшь `amnezia-vpn/amneziawg-go`** → обфускация есть, но sing-box-lx **не скомпилируется** (нет sagernet-функций).
+
+Нужны **обе** ✅ сразу, а готового форка с двумя галочками не существует — поэтому мы собрали его **слиянием**: sagernet (за API) + amnezia (за обфускацию). Это ровно строка **«этот форк»** выше.
+
+Подход: **берём sagernet-базу и граффтим обфускацию на неё** — а не наоборот (не дотачиваем sagernet-API к amneziawg-go, иначе даже обычный WireGuard шёл бы через чужой device). Так sing-box компилируется без изменений, обфускация аддитивна и выключена по умолчанию, а конфиг без AWG-полей ведёт себя как обычный WireGuard.
+
+## Как устроен merge
+
+И `sagernet/wireguard-go`, и `amneziawg-go` происходят от одного upstream `git.zx2c4.com/wireguard-go`, поэтому делят git-историю — а значит возможен настоящий **3-way merge** (а не ручной перенос).
+
+- **База:** `sagernet/wireguard-go` (тот коммит, что пинит sing-box — сейчас `506b7631853c`).
+- **Вливаем:** `amnezia-vpn/amneziawg-go` (тип с AWG2 / I1–I5 + фикс S4-keepalive).
+- **Ключевой трюк:** `MessageEncapsulatingTransportSize` выставлен в **`0`** в `device/noise-protocol.go`. sing-box-lx не использует 8-байтный headroom sagernet для `Bind.Send`, и обнуление позволяет обфускации AmneziaWG встать чисто, без конфликтов в send-пути.
+- **Изоляция:** обфускация замкнута в `device/` — новые файлы `device/obf*.go`, `device/magic-header.go` + графты в `device/{send,receive,device,uapi}.go`. **`conn/`, `tun/`, `ipc/` остаются чистым sagernet.**
+- **Module-path не меняется** (`module github.com/sagernet/wireguard-go`), поэтому потребитель подключает форк через `replace` без правки импортов.
+
+## Кто потребляет
+
+[sing-box-lx](https://github.com/Leadaxe/sing-box-lx) подключает это как git submodule + `replace`:
+
+```
+# sing-box-lx/.gitmodules
+[submodule "submodules/wireguard-go"]
+    url = https://github.com/Leadaxe/wireguard-go-awg2-lx
+    branch = lx
+
+# sing-box-lx/go.mod   (// lx)
+replace github.com/sagernet/wireguard-go => ./submodules/wireguard-go
+```
+
+Собранный с тегом `with_awg`, он **проверен живым** сервером AmneziaWG 2.0 (handshake + keepalive + трафик наружу) и кросс-компилируется на linux/darwin/windows × amd64/arm64.
+
+## Сопровождение (ребейз на новый sagernet-тег)
+
+Когда sing-box бампит `sagernet/wireguard-go`, повторяем merge:
+
+```sh
+git remote add origin  https://github.com/sagernet/wireguard-go    # база
+git remote add amnezia https://github.com/amnezia-vpn/amneziawg-go # источник обфускации
+git fetch --all
+git checkout -b lx <новый-sagernet-коммит>
+git merge amnezia/master          # настоящий 3-way merge через общего upstream-предка
+```
+
+Рецепт разрешения конфликтов:
+
+1. Новые `device/obf*.go` + `device/magic-header.go` приходят чисто.
+2. **Механические** конфликты (import-path amnezia → sagernet, `queueconstants*`, `sticky*`, `tun.go`) → берём **наши** (sagernet).
+3. Удаляем amnezia-инфра-дубликаты: `conn/gso_*.go`, `outline/*`, `tun/*_test.go`.
+4. `device/device.go` → **union** (sagernet `pauseManager` + obf-поля amnezia).
+5. `device/send.go` / `receive.go` → берём обфускацию amnezia, ставим `MessageEncapsulatingTransportSize = 0`, сохраняем 3-арг `bind.Send(…, 0)`.
+6. `conn/`, `tun/`, `ipc/`, module-path в `go.mod` → **наши** (sagernet).
+
+Затем в sing-box-lx: бампим submodule, `make -f Makefile.lx lx-build` и пере-тест против AWG2-сервера.
+
+## Ссылки
+
+| | |
+|---|---|
+| Потребитель | [Leadaxe/sing-box-lx](https://github.com/Leadaxe/sing-box-lx) |
+| База | [sagernet/wireguard-go](https://github.com/sagernet/wireguard-go) |
+| Источник обфускации | [amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go) · [docs.amnezia.org](https://docs.amnezia.org/documentation/amnezia-wg/) |
+| Оригинал | [WireGuard/wireguard-go](https://git.zx2c4.com/wireguard-go/about/) |
+
+## Лицензия
+
+MIT, унаследована от WireGuard-Go (см. [`LICENSE`](LICENSE)). Обфускация AmneziaWG — тоже MIT (из amneziawg-go). Это неофициальный форк, не аффилирован с WireGuard, SagerNet или Amnezia.

conn/bind_std.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package conn
@@ -23,13 +23,10 @@ import (
 	"golang.org/x/net/ipv6"
 )
 
-type EgressProvider interface {
-	SetEgressPort(port uint16) bool
-	LookupEgress(destination netip.AddrPort) *net.UDPConn
-	ReceiveEgress(buffer []byte) (int, netip.AddrPort, error)
-}
-
-var _ Bind = (*StdNetBind)(nil)
+var (
+	_ Bind     = (*StdNetBind)(nil)
+	_ Endpoint = (*StdNetEndpoint)(nil)
+)
 
 // StdNetBind implements Bind for all platforms. While Windows has its own Bind
 // (see bind_windows.go), it may fall back to StdNetBind.
@@ -38,7 +35,6 @@ var _ Bind = (*StdNetBind)(nil)
 // proposal in https://github.com/golang/go/issues/45886#issuecomment-1218301564.
 type StdNetBind struct {
 	externalControl     control.Func
-	egressProvider      EgressProvider
 	reservedForEndpoint map[netip.AddrPort][3]uint8
 
 	mu            sync.Mutex // protects all fields except as specified
@@ -46,8 +42,6 @@ type StdNetBind struct {
 	ipv6          *net.UDPConn
 	ipv4PC        *ipv4.PacketConn // will be nil on non-Linux
 	ipv6PC        *ipv6.PacketConn // will be nil on non-Linux
-	ipv4RC        syscall.RawConn  // will be nil on non-Darwin
-	ipv6RC        syscall.RawConn  // will be nil on non-Darwin
 	ipv4TxOffload bool
 	ipv4RxOffload bool
 	ipv6TxOffload bool
@@ -57,8 +51,6 @@ type StdNetBind struct {
 	udpAddrPool sync.Pool
 	msgsPool    sync.Pool
 
-	msgx msgXState
-
 	blackhole4 bool
 	blackhole6 bool
 }
@@ -78,12 +70,10 @@ func NewStdNetBind(externalControl control.Func) Bind {
 
 		msgsPool: sync.Pool{
 			New: func() any {
-				// ipv6.Message and ipv4.Message are interchangeable as they are
-				// both aliases for x/net/internal/socket.Message.
 				msgs := make([]ipv6.Message, IdealBatchSize)
 				for i := range msgs {
 					msgs[i].Buffers = make(net.Buffers, 1)
-					msgs[i].OOB = make([]byte, 0, stickyControlSize+gsoControlSize)
+					msgs[i].OOB = make([]byte, controlSize)
 				}
 				return &msgs
 			},
@@ -126,7 +116,7 @@ func (e *StdNetEndpoint) DstIP() netip.Addr {
 	return e.AddrPort.Addr()
 }
 
-// See control_default,linux, etc for implementations of SrcIP and SrcIfidx.
+// See sticky_default,linux, etc for implementations of SrcIP and SrcIfidx.
 
 func (e *StdNetEndpoint) DstToBytes() []byte {
 	b, _ := e.AddrPort.MarshalBinary()
@@ -176,6 +166,11 @@ func listenNet(externalControl control.Func, network string, port int) (*net.UDP
 	return conn.(*net.UDPConn), uaddr.Port, nil
 }
 
+// errEADDRINUSE is syscall.EADDRINUSE, boxed into an interface once
+// in erraddrinuse.go on almost all platforms. For other platforms,
+// it's at least non-nil.
+var errEADDRINUSE error = errors.New("")
+
 func (s *StdNetBind) Open(uport uint16) ([]ReceiveFunc, uint16, error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -186,7 +181,6 @@ func (s *StdNetBind) Open(uport uint16) ([]ReceiveFunc, uint16, error) {
 	if s.ipv4 != nil || s.ipv6 != nil {
 		return nil, 0, ErrBindAlreadyOpen
 	}
-	s.msgx.reset()
 
 	// Attempt to open ipv4 and ipv6 listeners on the same port.
 	// If uport is 0, we can retry on failure.
@@ -203,7 +197,7 @@ again:
 
 	// Listen on the same port as we're using for ipv4.
 	v6conn, port, err = listenNet(s.externalControl, "udp6", port)
-	if uport == 0 && errors.Is(err, syscall.EADDRINUSE) && tries < 100 {
+	if uport == 0 && errors.Is(err, errEADDRINUSE) && tries < 100 {
 		v4conn.Close()
 		tries++
 		goto again
@@ -215,85 +209,32 @@ again:
 	var fns []ReceiveFunc
 	if v4conn != nil {
 		s.ipv4TxOffload, s.ipv4RxOffload = supportsUDPOffload(v4conn)
-		if runtime.GOOS == "linux" || runtime.GOOS == "android" {
+		if runtime.GOOS == "linux" {
 			v4pc = ipv4.NewPacketConn(v4conn)
 			s.ipv4PC = v4pc
 		}
-		if supportsMsgX {
-			var receiveFn ReceiveFunc
-			receiveFn, err = s.makeReceiveMsgX(v4conn, false)
-			if err != nil {
-				v4conn.Close()
-				return nil, 0, err
-			}
-			s.ipv4RC, err = v4conn.SyscallConn()
-			if err != nil {
-				v4conn.Close()
-				return nil, 0, err
-			}
-			fns = append(fns, receiveFn)
-		} else {
-			fns = append(fns, s.makeReceiveIPv4(v4pc, v4conn, s.ipv4RxOffload))
-		}
+		fns = append(fns, s.makeReceiveIPv4(v4pc, v4conn, s.ipv4RxOffload))
 		s.ipv4 = v4conn
 	}
 	if v6conn != nil {
 		s.ipv6TxOffload, s.ipv6RxOffload = supportsUDPOffload(v6conn)
-		if runtime.GOOS == "linux" || runtime.GOOS == "android" {
+		if runtime.GOOS == "linux" {
 			v6pc = ipv6.NewPacketConn(v6conn)
 			s.ipv6PC = v6pc
 		}
-		if supportsMsgX {
-			var receiveFn ReceiveFunc
-			receiveFn, err = s.makeReceiveMsgX(v6conn, true)
-			if err != nil {
-				v6conn.Close()
-				return nil, 0, err
-			}
-			s.ipv6RC, err = v6conn.SyscallConn()
-			if err != nil {
-				v6conn.Close()
-				return nil, 0, err
-			}
-			fns = append(fns, receiveFn)
-		} else {
-			fns = append(fns, s.makeReceiveIPv6(v6pc, v6conn, s.ipv6RxOffload))
-		}
+		fns = append(fns, s.makeReceiveIPv6(v6pc, v6conn, s.ipv6RxOffload))
 		s.ipv6 = v6conn
 	}
 	if len(fns) == 0 {
 		return nil, 0, syscall.EAFNOSUPPORT
 	}
-	if s.egressProvider != nil {
-		s.egressProvider.SetEgressPort(uint16(port))
-		fns = append(fns, func(bufs [][]byte, sizes []int, endpoints []Endpoint) (int, error) {
-			dataLength, source, err := s.egressProvider.ReceiveEgress(bufs[0])
-			if err != nil {
-				return 0, err
-			}
-			sizes[0] = dataLength
-			if dataLength > 3 && s.hasReserved() { // lx: SPEC 026 — gate reserved-clear on the egress receive path too, so a small-padding AmneziaWG magic in bytes 1-3 survives when no WARP reserved value is set
-				common.ClearArray(bufs[0][1:4])
-			}
-			endpoints[0] = &StdNetEndpoint{AddrPort: source}
-			return 1, nil
-		})
-	}
 
 	return fns, uint16(port), nil
 }
 
-func (s *StdNetBind) SetEgressProvider(provider EgressProvider) {
-	s.egressProvider = provider
-}
-
 func (s *StdNetBind) putMessages(msgs *[]ipv6.Message) {
 	for i := range *msgs {
-		buffers := (*msgs)[i].Buffers
-		for j := range buffers {
-			buffers[j] = nil
-		}
-		(*msgs)[i] = ipv6.Message{Buffers: buffers[:1], OOB: (*msgs)[i].OOB[:0]}
+		(*msgs)[i] = ipv6.Message{Buffers: (*msgs)[i].Buffers, OOB: (*msgs)[i].OOB}
 	}
 	s.msgsPool.Put(msgs)
 }
@@ -328,9 +269,9 @@ func (s *StdNetBind) receiveIP(
 	}
 	defer s.putMessages(msgs)
 	var numMsgs int
-	if runtime.GOOS == "linux" || runtime.GOOS == "android" {
+	if runtime.GOOS == "linux" {
 		if rxOffload {
-			readAt := len(*msgs) - (IdealBatchSize / udpSegmentMaxDatagrams)
+			readAt := len(*msgs) - 2
 			numMsgs, err = br.ReadBatch((*msgs)[readAt:], 0)
 			if err != nil {
 				return 0, err
@@ -359,7 +300,7 @@ func (s *StdNetBind) receiveIP(
 		if sizes[i] == 0 {
 			continue
 		}
-		if msg.N > 3 && s.hasReserved() { // lx: only strip reserved bytes for WARP (see hasReserved)
+		if msg.N > 3 {
 			common.ClearArray(bufs[i][1:4])
 		}
 		ep := &StdNetEndpoint{AddrPort: M.AddrPortFromNet(msg.Addr)} // TODO: remove allocation
@@ -384,12 +325,9 @@ func (s *StdNetBind) makeReceiveIPv6(pc *ipv6.PacketConn, conn *net.UDPConn, rxO
 // TODO: When all Binds handle IdealBatchSize, remove this dynamic function and
 // rename the IdealBatchSize constant to BatchSize.
 func (s *StdNetBind) BatchSize() int {
-	if runtime.GOOS == "linux" || runtime.GOOS == "android" {
+	if runtime.GOOS == "linux" {
 		return IdealBatchSize
 	}
-	if supportsMsgX {
-		return msgXBatchSize
-	}
 	return 1
 }
 
@@ -397,9 +335,6 @@ func (s *StdNetBind) Close() error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	if s.egressProvider != nil {
-		s.egressProvider.SetEgressPort(0)
-	}
 	var err1, err2 error
 	if s.ipv4 != nil {
 		err1 = s.ipv4.Close()
@@ -437,21 +372,13 @@ func (e ErrUDPGSODisabled) Unwrap() error {
 }
 
 func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error {
-	for len(bufs) > IdealBatchSize {
-		err := s.Send(bufs[:IdealBatchSize], endpoint, offset)
-		if err != nil {
-			return err
-		}
-		bufs = bufs[IdealBatchSize:]
-	}
-	standardEndpoint := endpoint.(*StdNetEndpoint)
 	s.mu.Lock()
 	blackhole := s.blackhole4
 	conn := s.ipv4
 	offload := s.ipv4TxOffload
 	br := batchWriter(s.ipv4PC)
 	is6 := false
-	if standardEndpoint.DstIP().Is6() {
+	if endpoint.DstIP().Is6() {
 		blackhole = s.blackhole6
 		conn = s.ipv6
 		br = s.ipv6PC
@@ -472,42 +399,30 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error {
 	ua := s.udpAddrPool.Get().(*net.UDPAddr)
 	defer s.udpAddrPool.Put(ua)
 	if is6 {
-		as16 := standardEndpoint.DstIP().As16()
+		as16 := endpoint.DstIP().As16()
 		copy(ua.IP, as16[:])
 		ua.IP = ua.IP[:16]
 	} else {
-		as4 := standardEndpoint.DstIP().As4()
+		as4 := endpoint.DstIP().As4()
 		copy(ua.IP, as4[:])
 		ua.IP = ua.IP[:4]
 	}
-	ua.Port = int(standardEndpoint.Port())
+	ua.Port = int(endpoint.(*StdNetEndpoint).Port())
 	var (
 		retried bool
 		err     error
 	)
 	for _, buf := range bufs {
 		if len(buf) > offset+3 {
-			reserved, loaded := s.reservedForEndpoint[standardEndpoint.AddrPort]
+			reserved, loaded := s.reservedForEndpoint[endpoint.(*StdNetEndpoint).AddrPort]
 			if loaded {
 				copy(buf[offset+1:offset+4], reserved[:])
 			}
 		}
 	}
-	if s.egressProvider != nil {
-		memberConn := s.egressProvider.LookupEgress(standardEndpoint.AddrPort)
-		if memberConn != nil {
-			for _, buf := range bufs {
-				_, err = memberConn.WriteToUDPAddrPort(buf[offset:], standardEndpoint.AddrPort)
-				if err != nil {
-					return err
-				}
-			}
-			return nil
-		}
-	}
 retry:
 	if offload {
-		n := coalesceMessages(ua, standardEndpoint, bufs, offset, *msgs, setGSOSize)
+		n := coalesceMessages(ua, endpoint.(*StdNetEndpoint), bufs, offset, *msgs, setGSOSize)
 		err = s.send(conn, br, (*msgs)[:n])
 		if err != nil && offload && errShouldDisableUDPGSO(err) {
 			offload = false
@@ -525,7 +440,7 @@ retry:
 		for i := range bufs {
 			(*msgs)[i].Addr = ua
 			(*msgs)[i].Buffers[0] = bufs[i][offset:]
-			setSrcControl(&(*msgs)[i].OOB, standardEndpoint)
+			setSrcControl(&(*msgs)[i].OOB, endpoint.(*StdNetEndpoint))
 		}
 		err = s.send(conn, br, (*msgs)[:len(bufs)])
 	}
@@ -539,27 +454,13 @@ func (s *StdNetBind) SetReservedForEndpoint(destination netip.AddrPort, reserved
 	s.reservedForEndpoint[destination] = reserved
 }
 
-// lx: hasReserved reports whether any Cloudflare "reserved" value is set. The
-// receive path must only zero bytes 1-3 when a reserved value exists (WARP);
-// otherwise an AmneziaWG magic header that lands in bytes 1-3 (small s1/s2/s4
-// padding) would be corrupted and the packet dropped. The send path already
-// gates its stamp on a per-endpoint `loaded` check, so no change is needed there.
-func (s *StdNetBind) hasReserved() bool {
-	for _, reserved := range s.reservedForEndpoint {
-		if reserved != [3]uint8{} {
-			return true
-		}
-	}
-	return false
-}
-
 func (s *StdNetBind) send(conn *net.UDPConn, pc batchWriter, msgs []ipv6.Message) error {
 	var (
 		n     int
 		err   error
 		start int
 	)
-	if runtime.GOOS == "linux" || runtime.GOOS == "android" {
+	if runtime.GOOS == "linux" {
 		for {
 			n, err = pc.WriteBatch(msgs[start:], 0)
 			if err != nil || n == len(msgs[start:]) {
@@ -568,12 +469,6 @@ func (s *StdNetBind) send(conn *net.UDPConn, pc batchWriter, msgs []ipv6.Message
 			start += n
 		}
 	} else {
-		if supportsMsgX {
-			handled, sendErr := s.sendMsgX(conn, msgs)
-			if handled {
-				return sendErr
-			}
-		}
 		for _, msg := range msgs {
 			_, _, err = conn.WriteMsgUDP(msg.Buffers[0], msg.OOB, msg.Addr.(*net.UDPAddr))
 			if err != nil {
@@ -601,7 +496,6 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs
 	var (
 		base     = -1 // index of msg we are currently coalescing into
 		gsoSize  int  // segmentation size of msgs[base]
-		totalLen int  // length of all dgrams coalesced into msgs[base]
 		dgramCnt int  // number of dgrams coalesced into msgs[base]
 		endBatch bool // tracking flag to start a new batch on next iteration of bufs
 	)
@@ -613,14 +507,14 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs
 		buf = buf[offset:]
 		if i > 0 {
 			msgLen := len(buf)
-			if msgLen+totalLen <= maxPayloadLen &&
+			baseLenBefore := len(msgs[base].Buffers[0])
+			freeBaseCap := cap(msgs[base].Buffers[0]) - baseLenBefore
+			if msgLen+baseLenBefore <= maxPayloadLen &&
 				msgLen <= gsoSize &&
+				msgLen <= freeBaseCap &&
 				dgramCnt < udpSegmentMaxDatagrams &&
 				!endBatch {
-				// Coalesce as an additional iovec instead of copying: element
-				// buffers are sized to their packet and have no spare capacity.
-				msgs[base].Buffers = append(msgs[base].Buffers, buf)
-				totalLen += msgLen
+				msgs[base].Buffers[0] = append(msgs[base].Buffers[0], buf...)
 				if i == len(bufs)-1 {
 					setGSO(&msgs[base].OOB, uint16(gsoSize))
 				}
@@ -641,9 +535,8 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs
 		endBatch = false
 		base++
 		gsoSize = len(buf)
-		totalLen = len(buf)
 		setSrcControl(&msgs[base].OOB, ep)
-		msgs[base].Buffers = append(msgs[base].Buffers[:0], buf)
+		msgs[base].Buffers[0] = buf
 		msgs[base].Addr = addr
 		dgramCnt = 1
 	}

conn/bind_windows.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package conn
@@ -461,7 +461,7 @@ func (bind *WinRingBind) receiveIPv4(bufs [][]byte, sizes []int, eps []Endpoint)
 	bind.mu.RLock()
 	defer bind.mu.RUnlock()
 	n, ep, err := bind.v4.Receive(bufs[0], &bind.isOpen)
-	if n > 3 && bind.hasReserved() { // lx: only strip reserved bytes for WARP (see hasReserved)
+	if n > 3 {
 		common.ClearArray(bufs[0][1:4])
 	}
 	sizes[0] = n
@@ -473,7 +473,7 @@ func (bind *WinRingBind) receiveIPv6(bufs [][]byte, sizes []int, eps []Endpoint)
 	bind.mu.RLock()
 	defer bind.mu.RUnlock()
 	n, ep, err := bind.v6.Receive(bufs[0], &bind.isOpen)
-	if n > 3 && bind.hasReserved() { // lx: only strip reserved bytes for WARP (see hasReserved)
+	if n > 3 {
 		common.ClearArray(bufs[0][1:4])
 	}
 	sizes[0] = n
@@ -576,18 +576,6 @@ func (bind *WinRingBind) SetReservedForEndpoint(destination netip.AddrPort, rese
 	bind.reservedForEndpoint[*endpoint.(*WinRingEndpoint)] = reserved
 }
 
-// lx: hasReserved reports whether any Cloudflare "reserved" value is set. See
-// the StdNetBind.hasReserved comment — the unconditional receive clear would
-// corrupt an AmneziaWG magic header sitting in bytes 1-3 (small padding).
-func (bind *WinRingBind) hasReserved() bool {
-	for _, reserved := range bind.reservedForEndpoint {
-		if reserved != [3]uint8{} {
-			return true
-		}
-	}
-	return false
-}
-
 func (s *StdNetBind) BindSocketToInterface4(interfaceIndex uint32, blackhole bool) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()

conn/control_default.go

@@ -1,8 +1,8 @@
-//go:build !linux || android
+//go:build !(linux && !android)
 
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package conn
@@ -22,7 +22,7 @@ func (e *StdNetEndpoint) SrcToString() string {
 }
 
 // TODO: macOS, FreeBSD and other BSDs likely do support the sticky sockets
-// {get,set}srcControl feature set, but use alternatively named flags and need
+// ({get,set}srcControl feature set, but use alternatively named flags and need
 // ports and require testing.
 
 // getSrcFromControl parses the control for PKTINFO and if found updates ep with
@@ -35,8 +35,17 @@ func getSrcFromControl(control []byte, ep *StdNetEndpoint) {
 func setSrcControl(control *[]byte, ep *StdNetEndpoint) {
 }
 
-// stickyControlSize returns the recommended buffer size for pooling sticky
+// getGSOSize parses control for UDP_GRO and if found returns its GSO size data.
+func getGSOSize(control []byte) (int, error) {
+	return 0, nil
+}
+
+// setGSOSize sets a UDP_SEGMENT in control based on gsoSize.
+func setGSOSize(control *[]byte, gsoSize uint16) {
+}
+
+// controlSize returns the recommended buffer size for pooling sticky and UDP
 // offloading control data.
-const stickyControlSize = 0
+const controlSize = 0
 
 const StdNetSupportsStickySockets = false

conn/control_linux.go

@@ -2,12 +2,13 @@
 
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package conn
 
 import (
+	"fmt"
 	"net/netip"
 	"unsafe"
 
@@ -105,8 +106,54 @@ func setSrcControl(control *[]byte, ep *StdNetEndpoint) {
 	*control = append(*control, ep.src...)
 }
 
-// stickyControlSize returns the recommended buffer size for pooling sticky
+const (
+	sizeOfGSOData = 2
+)
+
+// getGSOSize parses control for UDP_GRO and if found returns its GSO size data.
+func getGSOSize(control []byte) (int, error) {
+	var (
+		hdr  unix.Cmsghdr
+		data []byte
+		rem  = control
+		err  error
+	)
+
+	for len(rem) > unix.SizeofCmsghdr {
+		hdr, data, rem, err = unix.ParseOneSocketControlMessage(rem)
+		if err != nil {
+			return 0, fmt.Errorf("error parsing socket control message: %w", err)
+		}
+		if hdr.Level == socketOptionLevelUDP && hdr.Type == socketOptionUDPGRO && len(data) >= sizeOfGSOData {
+			var gso uint16
+			copy(unsafe.Slice((*byte)(unsafe.Pointer(&gso)), sizeOfGSOData), data[:sizeOfGSOData])
+			return int(gso), nil
+		}
+	}
+	return 0, nil
+}
+
+// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. It leaves existing
+// data in control untouched.
+func setGSOSize(control *[]byte, gsoSize uint16) {
+	existingLen := len(*control)
+	avail := cap(*control) - existingLen
+	space := unix.CmsgSpace(sizeOfGSOData)
+	if avail < space {
+		return
+	}
+	*control = (*control)[:cap(*control)]
+	gsoControl := (*control)[existingLen:]
+	hdr := (*unix.Cmsghdr)(unsafe.Pointer(&(gsoControl)[0]))
+	hdr.Level = socketOptionLevelUDP
+	hdr.Type = socketOptionUDPSegment
+	hdr.SetLen(unix.CmsgLen(sizeOfGSOData))
+	copy((gsoControl)[unix.SizeofCmsghdr:], unsafe.Slice((*byte)(unsafe.Pointer(&gsoSize)), sizeOfGSOData))
+	*control = (*control)[:existingLen+space]
+}
+
+// controlSize returns the recommended buffer size for pooling sticky and UDP
 // offloading control data.
-var stickyControlSize = unix.CmsgSpace(unix.SizeofInet6Pktinfo)
+var controlSize = unix.CmsgSpace(unix.SizeofInet6Pktinfo) + unix.CmsgSpace(sizeOfGSOData)
 
 const StdNetSupportsStickySockets = true

conn/controlfns_linux.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package conn
@@ -13,35 +13,6 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-// Taken from go/src/internal/syscall/unix/kernel_version_linux.go
-func kernelVersion() (major, minor int) {
-	var uname unix.Utsname
-	if err := unix.Uname(&uname); err != nil {
-		return
-	}
-
-	var (
-		values    [2]int
-		value, vi int
-	)
-	for _, c := range uname.Release {
-		if '0' <= c && c <= '9' {
-			value = (value * 10) + int(c-'0')
-		} else {
-			// Note that we're assuming N.N.N here.
-			// If we see anything else, we are likely to mis-parse it.
-			values[vi] = value
-			vi++
-			if vi >= len(values) {
-				break
-			}
-			value = 0
-		}
-	}
-
-	return values[0], values[1]
-}
-
 func init() {
 	controlFns = append(controlFns,
 
@@ -89,19 +60,17 @@ func init() {
 
 		// Attempt to enable UDP_GRO
 		func(network, address string, c syscall.RawConn) error {
-			// Kernels below 5.12 are missing 98184612aca0 ("net:
-			// udp: Add support for getsockopt(..., ..., UDP_GRO,
-			// ..., ...);"), which means we can't read this back
-			// later. We could pipe the return value through to
-			// the rest of the code, but UDP_GRO is kind of buggy
-			// anyway, so just gate this here.
-			major, minor := kernelVersion()
-			if major < 5 || (major == 5 && minor < 12) {
+			// lx(010): skip UDP_GRO on android. The GRO receive path in bind_std.go
+			// is gated on runtime.GOOS=="linux", which is false on android — so a
+			// coalesced super-packet is never split and corrupts the WG stream
+			// (download dies). Belt-and-suspenders with the rxOffload guard in
+			// features_linux.go. TX/GSO untouched.
+			// See SPECS/010-WG_ENDPOINT_GRO_SPLIT_BRAIN.
+			if runtime.GOOS == "android" {
 				return nil
 			}
-
 			c.Control(func(fd uintptr) {
-				_ = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_GRO, 1)
+				_ = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO, 1)
 			})
 			return nil
 		},

conn/controlfns_unix.go

@@ -1,4 +1,4 @@
-//go:build !windows && !linux && !wasm
+//go:build !windows && !linux && !wasm && !plan9 && !tamago
 
 /* SPDX-License-Identifier: MIT
  *

conn/erraddrinuse.go

@@ -0,0 +1,14 @@
+//go:build !plan9
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ */
+
+package conn
+
+import "syscall"
+
+func init() {
+	errEADDRINUSE = syscall.EADDRINUSE
+}

conn/errors_default.go

@@ -7,6 +7,6 @@
 
 package conn
 
-func errShouldDisableUDPGSO(_ error) bool {
+func errShouldDisableUDPGSO(err error) bool {
 	return false
 }

conn/errors_linux.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package conn

conn/features_default.go

@@ -3,13 +3,13 @@
 
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package conn
 
 import "net"
 
-func supportsUDPOffload(_ *net.UDPConn) (txOffload, rxOffload bool) {
+func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) {
 	return
 }

conn/features_linux.go

@@ -1,26 +1,49 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package conn
 
 import (
 	"net"
+	"runtime"
 
 	"golang.org/x/sys/unix"
 )
 
+const (
+	// TODO: upstream to x/sys/unix
+	socketOptionLevelUDP   = 17
+	socketOptionUDPSegment = 103
+	socketOptionUDPGRO     = 104
+)
+
 func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) {
 	rc, err := conn.SyscallConn()
 	if err != nil {
 		return
 	}
 	err = rc.Control(func(fd uintptr) {
-		_, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_SEGMENT)
-		txOffload = errSyscall == nil
-		opt, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_GRO)
-		rxOffload = errSyscall == nil && opt == 1
+		_, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPSegment)
+		if errSyscall != nil {
+			return
+		}
+		txOffload = true
+		// lx(010): never advertise RX offload on android. runtime.GOOS=="android"
+		// (not "linux"), so the GRO receive dispatcher in bind_std.go (gated on
+		// GOOS=="linux") is dead there — a coalesced GRO super-packet would be read
+		// as one datagram and corrupt the WG transport stream, killing download.
+		// Confirmed on device (CPH2411/Android-15: rxOffload=true, dispatch=single).
+		// TX is left untouched. See SPECS/010-WG_ENDPOINT_GRO_SPLIT_BRAIN.
+		if runtime.GOOS == "android" {
+			return
+		}
+		opt, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO)
+		if errSyscall != nil {
+			return
+		}
+		rxOffload = opt == 1
 	})
 	if err != nil {
 		return false, false

conn/gso_default.go

@@ -1,21 +0,0 @@
-//go:build !linux
-
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
- */
-
-package conn
-
-// getGSOSize parses control for UDP_GRO and if found returns its GSO size data.
-func getGSOSize(control []byte) (int, error) {
-	return 0, nil
-}
-
-// setGSOSize sets a UDP_SEGMENT in control based on gsoSize.
-func setGSOSize(control *[]byte, gsoSize uint16) {
-}
-
-// gsoControlSize returns the recommended buffer size for pooling sticky and UDP
-// offloading control data.
-const gsoControlSize = 0

conn/gso_linux.go

@@ -1,65 +0,0 @@
-//go:build linux
-
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
- */
-
-package conn
-
-import (
-	"fmt"
-	"unsafe"
-
-	"golang.org/x/sys/unix"
-)
-
-const (
-	sizeOfGSOData = 2
-)
-
-// getGSOSize parses control for UDP_GRO and if found returns its GSO size data.
-func getGSOSize(control []byte) (int, error) {
-	var (
-		hdr  unix.Cmsghdr
-		data []byte
-		rem  = control
-		err  error
-	)
-
-	for len(rem) > unix.SizeofCmsghdr {
-		hdr, data, rem, err = unix.ParseOneSocketControlMessage(rem)
-		if err != nil {
-			return 0, fmt.Errorf("error parsing socket control message: %w", err)
-		}
-		if hdr.Level == unix.SOL_UDP && hdr.Type == unix.UDP_GRO && len(data) >= sizeOfGSOData {
-			var gso uint16
-			copy(unsafe.Slice((*byte)(unsafe.Pointer(&gso)), sizeOfGSOData), data[:sizeOfGSOData])
-			return int(gso), nil
-		}
-	}
-	return 0, nil
-}
-
-// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. It leaves existing
-// data in control untouched.
-func setGSOSize(control *[]byte, gsoSize uint16) {
-	existingLen := len(*control)
-	avail := cap(*control) - existingLen
-	space := unix.CmsgSpace(sizeOfGSOData)
-	if avail < space {
-		return
-	}
-	*control = (*control)[:cap(*control)]
-	gsoControl := (*control)[existingLen:]
-	hdr := (*unix.Cmsghdr)(unsafe.Pointer(&(gsoControl)[0]))
-	hdr.Level = unix.SOL_UDP
-	hdr.Type = unix.UDP_SEGMENT
-	hdr.SetLen(unix.CmsgLen(sizeOfGSOData))
-	copy((gsoControl)[unix.CmsgLen(0):], unsafe.Slice((*byte)(unsafe.Pointer(&gsoSize)), sizeOfGSOData))
-	*control = (*control)[:existingLen+space]
-}
-
-// gsoControlSize returns the recommended buffer size for pooling UDP
-// offloading control data.
-var gsoControlSize = unix.CmsgSpace(sizeOfGSOData)

conn/msgx_darwin.go

@@ -1,325 +0,0 @@
-// On iOS both directions misbehave in the Network Extension (recvmsg_x on
-// unconnected UDP sockets delivers no data, connected sockets stop passing
-// traffic after a rebind), so msgx is macOS only until it can be debugged
-// on a device.
-
-//go:build darwin && !ios
-
-package conn
-
-import (
-	"net"
-	"net/netip"
-	"sync"
-	"sync/atomic"
-	"syscall"
-	"unsafe"
-
-	M "github.com/sagernet/sing/common/metadata"
-	"golang.org/x/net/ipv6"
-	"golang.org/x/sys/unix"
-)
-
-const supportsMsgX = true
-
-const msgXBatchSize = IdealBatchSize
-
-// msghdrX mirrors XNU's struct msghdr_x used by sendmsg_x/recvmsg_x.
-// Per bsd/sys/socket_private.h, sendmsg_x supports neither addresses nor
-// ancillary data (msg_name and msg_control must be zero), so batched sends
-// require a connected socket. recvmsg_x does fill in per-message source
-// addresses (copyout_maddr in uipc_syscalls.c). utun cannot use the send
-// side at all (no ctl_send_list in if_utun.c).
-type msghdrX struct {
-	Msg     unix.Msghdr
-	DataLen uint32
-}
-
-type msgXState struct {
-	singlePeer  atomic.Bool
-	disabled    atomic.Bool // permanent fallback to the generic paths
-	connected4  atomic.Bool
-	connected6  atomic.Bool
-	endpoint    atomic.Pointer[StdNetEndpoint]
-	connectLock sync.Mutex
-}
-
-// reset clears per-socket state; must be called when the bind (re)opens,
-// as the connected state belongs to the previous sockets.
-func (m *msgXState) reset() {
-	m.disabled.Store(false)
-	m.connected4.Store(false)
-	m.connected6.Store(false)
-	m.endpoint.Store(nil)
-}
-
-func (m *msgXState) connectedFlag(isV6 bool) *atomic.Bool {
-	if isV6 {
-		return &m.connected6
-	}
-	return &m.connected4
-}
-
-// SetSinglePeerMode enables connected-socket sendmsg_x batching. Only safe
-// when the bind serves exactly one peer with a fixed endpoint: the kernel
-// will drop datagrams from any other source, so peer roaming stops working.
-func (s *StdNetBind) SetSinglePeerMode() {
-	s.msgx.singlePeer.Store(true)
-}
-
-func sockaddrFromAddrPort(addrPort netip.AddrPort, storage4 *unix.RawSockaddrInet4, storage6 *unix.RawSockaddrInet6) (unsafe.Pointer, uint32) {
-	port := addrPort.Port()<<8 | addrPort.Port()>>8
-	if addrPort.Addr().Unmap().Is4() {
-		*storage4 = unix.RawSockaddrInet4{
-			Len:    unix.SizeofSockaddrInet4,
-			Family: unix.AF_INET,
-			Port:   port,
-			Addr:   addrPort.Addr().Unmap().As4(),
-		}
-		return unsafe.Pointer(storage4), unix.SizeofSockaddrInet4
-	}
-	*storage6 = unix.RawSockaddrInet6{
-		Len:    unix.SizeofSockaddrInet6,
-		Family: unix.AF_INET6,
-		Port:   port,
-		Addr:   addrPort.Addr().As16(),
-	}
-	return unsafe.Pointer(storage6), unix.SizeofSockaddrInet6
-}
-
-// ensureConnected connects the family socket to the single peer on first
-// use, and permanently falls back if a second endpoint shows up.
-func (s *StdNetBind) ensureConnected(rawConn syscall.RawConn, isV6 bool, destination netip.AddrPort) bool {
-	if s.msgx.disabled.Load() || !s.msgx.singlePeer.Load() {
-		return false
-	}
-	connected := s.msgx.connectedFlag(isV6)
-	if connected.Load() {
-		if s.msgx.endpoint.Load().AddrPort == destination {
-			return true
-		}
-		s.msgx.connectLock.Lock()
-		defer s.msgx.connectLock.Unlock()
-		if s.msgx.disabled.Load() {
-			return false
-		}
-		s.msgx.disabled.Store(true)
-		var disconnectErr error
-		controlErr := rawConn.Control(func(fd uintptr) {
-			addr := unix.RawSockaddrAny{}
-			addr.Addr.Family = unix.AF_UNSPEC
-			//nolint:staticcheck
-			_, _, errno := unix.Syscall(unix.SYS_CONNECT, fd, uintptr(unsafe.Pointer(&addr)), unix.SizeofSockaddrAny)
-			if errno != 0 && errno != unix.EAFNOSUPPORT {
-				disconnectErr = errno
-			}
-		})
-		if controlErr == nil && disconnectErr == nil {
-			connected.Store(false)
-		}
-		return false
-	}
-	s.msgx.connectLock.Lock()
-	defer s.msgx.connectLock.Unlock()
-	if s.msgx.disabled.Load() {
-		return false
-	}
-	if connected.Load() {
-		return s.msgx.endpoint.Load().AddrPort == destination
-	}
-	var (
-		storage4   unix.RawSockaddrInet4
-		storage6   unix.RawSockaddrInet6
-		connectErr unix.Errno
-	)
-	name, nameLen := sockaddrFromAddrPort(destination, &storage4, &storage6)
-	controlErr := rawConn.Control(func(fd uintptr) {
-		//nolint:staticcheck
-		_, _, connectErr = unix.Syscall(unix.SYS_CONNECT, fd, uintptr(name), uintptr(nameLen))
-	})
-	if controlErr != nil || connectErr != 0 {
-		s.msgx.disabled.Store(true)
-		return false
-	}
-	s.msgx.endpoint.Store(&StdNetEndpoint{AddrPort: destination})
-	connected.Store(true)
-	return true
-}
-
-type sendMsgXState struct {
-	hdrs []msghdrX
-	iovs []unix.Iovec
-}
-
-var sendMsgXPool = sync.Pool{New: func() any {
-	return &sendMsgXState{
-		hdrs: make([]msghdrX, IdealBatchSize),
-		iovs: make([]unix.Iovec, IdealBatchSize),
-	}
-}}
-
-// sendMsgX sends msgs via sendmsg_x when the socket is connected to their
-// endpoint. handled == false means nothing was sent and the caller must use
-// the generic path; msgs are never partially consumed in that case.
-func (s *StdNetBind) sendMsgX(conn *net.UDPConn, msgs []ipv6.Message) (bool, error) {
-	var (
-		rawConn syscall.RawConn
-		isV6    bool
-	)
-	s.mu.Lock()
-	if conn == s.ipv6 {
-		rawConn = s.ipv6RC
-		isV6 = true
-	} else {
-		rawConn = s.ipv4RC
-	}
-	s.mu.Unlock()
-	if rawConn == nil {
-		return false, nil
-	}
-	destination := M.AddrPortFromNet(msgs[0].Addr)
-	if !s.ensureConnected(rawConn, isV6, destination) {
-		return false, nil
-	}
-	state := sendMsgXPool.Get().(*sendMsgXState)
-	defer sendMsgXPool.Put(state)
-	for i := range msgs {
-		buffer := msgs[i].Buffers[0]
-		state.iovs[i] = unix.Iovec{Base: &buffer[0]}
-		state.iovs[i].SetLen(len(buffer))
-		state.hdrs[i] = msghdrX{}
-		state.hdrs[i].Msg.Iov = &state.iovs[i]
-		state.hdrs[i].Msg.Iovlen = 1
-	}
-	var sent int
-	for sent < len(msgs) {
-		var (
-			n     uintptr
-			errno unix.Errno
-		)
-		writeErr := rawConn.Write(func(fd uintptr) bool {
-			//nolint:staticcheck
-			n, _, errno = unix.RawSyscall6(unix.SYS_SENDMSG_X, fd,
-				uintptr(unsafe.Pointer(&state.hdrs[sent])), uintptr(len(msgs)-sent), unix.MSG_DONTWAIT, 0, 0)
-			return errno != unix.EAGAIN
-		})
-		if writeErr != nil {
-			return true, writeErr
-		}
-		if errno != 0 {
-			if sent == 0 {
-				// The syscall is refusing this socket entirely (sandbox,
-				// disconnected by the system, ...): disable and let the
-				// caller resend everything on the generic path.
-				s.msgx.disabled.Store(true)
-				return false, nil
-			}
-			return true, errno
-		}
-		sent += int(n)
-	}
-	return true, nil
-}
-
-type receiveMsgXState struct {
-	hdrs     []msghdrX
-	iovs     []unix.Iovec
-	names    []unix.RawSockaddrInet6
-	fallback bool
-}
-
-func (s *StdNetBind) receiveSingle(conn *net.UDPConn, bufs [][]byte, sizes []int, eps []Endpoint) (int, error) {
-	n, _, _, addr, err := conn.ReadMsgUDPAddrPort(bufs[0], nil)
-	if err != nil {
-		return 0, err
-	}
-	sizes[0] = n
-	if n > 3 && s.hasReserved() { // lx: only strip reserved bytes for WARP (see hasReserved)
-		bufs[0][1] = 0
-		bufs[0][2] = 0
-		bufs[0][3] = 0
-	}
-	eps[0] = &StdNetEndpoint{AddrPort: addr}
-	return 1, nil
-}
-
-func (s *StdNetBind) makeReceiveMsgX(conn *net.UDPConn, isV6 bool) (ReceiveFunc, error) {
-	rawConn, err := conn.SyscallConn()
-	if err != nil {
-		return nil, err
-	}
-	state := &receiveMsgXState{
-		hdrs:  make([]msghdrX, msgXBatchSize),
-		iovs:  make([]unix.Iovec, msgXBatchSize),
-		names: make([]unix.RawSockaddrInet6, msgXBatchSize),
-	}
-	return func(bufs [][]byte, sizes []int, eps []Endpoint) (int, error) {
-		if state.fallback || s.msgx.disabled.Load() {
-			return s.receiveSingle(conn, bufs, sizes, eps)
-		}
-		connectedEndpoint := s.msgx.endpoint.Load()
-		if !s.msgx.connectedFlag(isV6).Load() {
-			connectedEndpoint = nil
-		}
-		count := len(bufs)
-		if count > msgXBatchSize {
-			count = msgXBatchSize
-		}
-		for i := 0; i < count; i++ {
-			state.iovs[i] = unix.Iovec{Base: &bufs[i][0]}
-			state.iovs[i].SetLen(len(bufs[i]))
-			state.hdrs[i] = msghdrX{}
-			if connectedEndpoint == nil {
-				state.hdrs[i].Msg.Name = (*byte)(unsafe.Pointer(&state.names[i]))
-				state.hdrs[i].Msg.Namelen = unix.SizeofSockaddrInet6
-			}
-			state.hdrs[i].Msg.Iov = &state.iovs[i]
-			state.hdrs[i].Msg.Iovlen = 1
-		}
-		var (
-			n     uintptr
-			errno unix.Errno
-		)
-		readErr := rawConn.Read(func(fd uintptr) bool {
-			//nolint:staticcheck
-			n, _, errno = unix.RawSyscall6(unix.SYS_RECVMSG_X, fd,
-				uintptr(unsafe.Pointer(&state.hdrs[0])), uintptr(count), unix.MSG_DONTWAIT, 0, 0)
-			return errno != unix.EAGAIN
-		})
-		if readErr != nil {
-			return 0, readErr
-		}
-		if errno != 0 {
-			// recvmsg_x is refusing this socket (sandbox, protocol, ...):
-			// serve this and all future calls with a plain single read so
-			// the receive routine keeps running.
-			state.fallback = true
-			return s.receiveSingle(conn, bufs, sizes, eps)
-		}
-		numMsgs := int(n)
-		for i := 0; i < numMsgs; i++ {
-			sizes[i] = int(state.hdrs[i].DataLen)
-			if sizes[i] > 3 && s.hasReserved() { // lx: only strip reserved bytes for WARP (see hasReserved)
-				bufs[i][1] = 0
-				bufs[i][2] = 0
-				bufs[i][3] = 0
-			}
-			if connectedEndpoint != nil {
-				eps[i] = connectedEndpoint
-				continue
-			}
-			var addrPort netip.AddrPort
-			name := &state.names[i]
-			if name.Family == unix.AF_INET6 {
-				port := name.Port<<8 | name.Port>>8
-				addrPort = netip.AddrPortFrom(netip.AddrFrom16(name.Addr).Unmap(), port)
-			} else {
-				name4 := (*unix.RawSockaddrInet4)(unsafe.Pointer(name))
-				port := name4.Port<<8 | name4.Port>>8
-				addrPort = netip.AddrPortFrom(netip.AddrFrom4(name4.Addr), port)
-			}
-			eps[i] = &StdNetEndpoint{AddrPort: addrPort}
-		}
-		return numMsgs, nil
-	}, nil
-}

conn/msgx_default.go

@@ -1,30 +0,0 @@
-//go:build !darwin || ios
-
-package conn
-
-import (
-	"net"
-
-	"golang.org/x/net/ipv6"
-)
-
-const supportsMsgX = false
-
-const msgXBatchSize = 1
-
-type msgXState struct{}
-
-func (m *msgXState) reset() {
-}
-
-// SetSinglePeerMode is a no-op on platforms without sendmsg_x/recvmsg_x.
-func (s *StdNetBind) SetSinglePeerMode() {
-}
-
-func (s *StdNetBind) sendMsgX(conn *net.UDPConn, msgs []ipv6.Message) (bool, error) {
-	return false, nil
-}
-
-func (s *StdNetBind) makeReceiveMsgX(conn *net.UDPConn, isV6 bool) (ReceiveFunc, error) {
-	panic("makeReceiveMsgX is not supported on this platform")
-}

conn/reserved_gate_lx_test.go

@@ -1,56 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * lx: unit coverage for the StdNetBind.hasReserved() gate that guards the
- * receive-side reserved-clear. receiveIP zeroes bytes 1-3 (Cloudflare WARP
- * "reserved") only when a non-zero reserved value is set for some endpoint;
- * otherwise an AmneziaWG magic header landing in bytes 1-3 (small s1/s2/s4
- * padding) would be corrupted and the packet dropped. This test pins the gate
- * itself; the end-to-end handshake proof lives in the device package.
- */
-
-package conn
-
-import (
-	"net/netip"
-	"testing"
-)
-
-func stdNetBindForTest(t *testing.T) *StdNetBind {
-	t.Helper()
-	b, ok := NewStdNetBind(nil).(*StdNetBind)
-	if !ok {
-		t.Fatalf("NewStdNetBind did not return *StdNetBind")
-	}
-	return b
-}
-
-func TestStdNetBindHasReserved(t *testing.T) {
-	b := stdNetBindForTest(t)
-	if b.hasReserved() {
-		t.Fatal("fresh bind must report no reserved value")
-	}
-
-	ep := netip.MustParseAddrPort("127.0.0.1:51820")
-
-	// An all-zero reserved value is indistinguishable from "unset" and must
-	// not arm the clear.
-	b.SetReservedForEndpoint(ep, [3]byte{0, 0, 0})
-	if b.hasReserved() {
-		t.Fatal("all-zero reserved must not count as reserved")
-	}
-
-	// Any non-zero byte (WARP anycast tag) arms the clear.
-	b.SetReservedForEndpoint(ep, [3]byte{0, 0, 1})
-	if !b.hasReserved() {
-		t.Fatal("non-zero reserved (byte 3) must count as reserved")
-	}
-
-	// A second endpoint's non-zero value must also be seen.
-	b2 := stdNetBindForTest(t)
-	ep2 := netip.MustParseAddrPort("192.0.2.1:2408")
-	b2.SetReservedForEndpoint(ep, [3]byte{0, 0, 0})
-	b2.SetReservedForEndpoint(ep2, [3]byte{0xAB, 0, 0})
-	if !b2.hasReserved() {
-		t.Fatal("non-zero reserved on any endpoint must count as reserved")
-	}
-}

device/allowedips.go

@@ -55,25 +55,6 @@ func commonBits(ip1, ip2 []byte) uint8 {
 	}
 }
 
-func commonBits4(ip1 []byte, ip2 [4]byte) uint8 {
-	a := binary.BigEndian.Uint32(ip1)
-	b := binary.BigEndian.Uint32(ip2[:])
-	return uint8(bits.LeadingZeros32(a ^ b))
-}
-
-func commonBits6(ip1 []byte, ip2 [16]byte) uint8 {
-	a := binary.BigEndian.Uint64(ip1)
-	b := binary.BigEndian.Uint64(ip2[:])
-	x := a ^ b
-	if x != 0 {
-		return uint8(bits.LeadingZeros64(x))
-	}
-	a = binary.BigEndian.Uint64(ip1[8:])
-	b = binary.BigEndian.Uint64(ip2[8:])
-	x = a ^ b
-	return 64 + uint8(bits.LeadingZeros64(x))
-}
-
 func (node *trieEntry) addToPeerEntries() {
 	node.perPeerElem = node.peer.trieEntries.PushBack(node)
 }
@@ -207,37 +188,7 @@ func (trie parentIndirection) insert(ip []byte, cidr uint8, peer *Peer) {
 	}
 }
 
-func (node *trieEntry) lookup4(ip [4]byte) *Peer {
-	var found *Peer
-	for node != nil && commonBits4(node.bits, ip) >= node.cidr {
-		if node.peer != nil {
-			found = node.peer
-		}
-		if node.bitAtByte == 4 {
-			break
-		}
-		bit := (ip[node.bitAtByte] >> node.bitAtShift) & 1
-		node = node.child[bit]
-	}
-	return found
-}
-
-func (node *trieEntry) lookup6(ip [16]byte) *Peer {
-	var found *Peer
-	for node != nil && commonBits6(node.bits, ip) >= node.cidr {
-		if node.peer != nil {
-			found = node.peer
-		}
-		if node.bitAtByte == 16 {
-			break
-		}
-		bit := (ip[node.bitAtByte] >> node.bitAtShift) & 1
-		node = node.child[bit]
-	}
-	return found
-}
-
-func (node *trieEntry) lookup(ip net.IP) *Peer {
+func (node *trieEntry) lookup(ip []byte) *Peer {
 	var found *Peer
 	size := uint8(len(ip))
 	for node != nil && commonBits(node.bits, ip) >= node.cidr {
@@ -254,17 +205,14 @@ func (node *trieEntry) lookup(ip net.IP) *Peer {
 }
 
 type AllowedIPs struct {
-	mu   sync.RWMutex
-	ipv4 *trieEntry
-	ipv6 *trieEntry
-
-	peerByIPPacketFunc PeerByIPPacketFunc // if non-nil, called to look up peers by IP
-	device             *Device            // back-reference to parent device; non-nil only if peerByIPPacketFunc is set
+	IPv4  *trieEntry
+	IPv6  *trieEntry
+	mutex sync.RWMutex
 }
 
 func (table *AllowedIPs) EntriesForPeer(peer *Peer, cb func(prefix netip.Prefix) bool) {
-	table.mu.RLock()
-	defer table.mu.RUnlock()
+	table.mutex.RLock()
+	defer table.mutex.RUnlock()
 
 	for elem := peer.trieEntries.Front(); elem != nil; elem = elem.Next() {
 		node := elem.Value.(*trieEntry)
@@ -309,17 +257,17 @@ func (node *trieEntry) remove() {
 }
 
 func (table *AllowedIPs) Remove(prefix netip.Prefix, peer *Peer) {
-	table.mu.Lock()
-	defer table.mu.Unlock()
+	table.mutex.Lock()
+	defer table.mutex.Unlock()
 	var node *trieEntry
 	var exact bool
 
 	if prefix.Addr().Is6() {
 		ip := prefix.Addr().As16()
-		node, exact = table.ipv6.nodePlacement(ip[:], uint8(prefix.Bits()))
+		node, exact = table.IPv6.nodePlacement(ip[:], uint8(prefix.Bits()))
 	} else if prefix.Addr().Is4() {
 		ip := prefix.Addr().As4()
-		node, exact = table.ipv4.nodePlacement(ip[:], uint8(prefix.Bits()))
+		node, exact = table.IPv4.nodePlacement(ip[:], uint8(prefix.Bits()))
 	} else {
 		panic(errors.New("removing unknown address type"))
 	}
@@ -329,25 +277,10 @@ func (table *AllowedIPs) Remove(prefix netip.Prefix, peer *Peer) {
 	node.remove()
 }
 
-// setPeerPrefixes atomically removes all of peer's existing prefixes and adds
-// the provided ones.
-func (table *AllowedIPs) setPeerPrefixes(peer *Peer, prefixes []netip.Prefix) {
-	table.mu.Lock()
-	defer table.mu.Unlock()
-
-	table.removeByPeerLocked(peer)
-	for _, prefix := range prefixes {
-		table.insertLocked(prefix, peer)
-	}
-}
-
 func (table *AllowedIPs) RemoveByPeer(peer *Peer) {
-	table.mu.Lock()
-	defer table.mu.Unlock()
-	table.removeByPeerLocked(peer)
-}
+	table.mutex.Lock()
+	defer table.mutex.Unlock()
 
-func (table *AllowedIPs) removeByPeerLocked(peer *Peer) {
 	var next *list.Element
 	for elem := peer.trieEntries.Front(); elem != nil; elem = next {
 		next = elem.Next()
@@ -356,135 +289,29 @@ func (table *AllowedIPs) removeByPeerLocked(peer *Peer) {
 }
 
 func (table *AllowedIPs) Insert(prefix netip.Prefix, peer *Peer) {
-	table.mu.Lock()
-	defer table.mu.Unlock()
-	table.insertLocked(prefix, peer)
-}
+	table.mutex.Lock()
+	defer table.mutex.Unlock()
 
-func (table *AllowedIPs) insertLocked(prefix netip.Prefix, peer *Peer) {
 	if prefix.Addr().Is6() {
 		ip := prefix.Addr().As16()
-		parentIndirection{&table.ipv6, 2}.insert(ip[:], uint8(prefix.Bits()), peer)
+		parentIndirection{&table.IPv6, 2}.insert(ip[:], uint8(prefix.Bits()), peer)
 	} else if prefix.Addr().Is4() {
 		ip := prefix.Addr().As4()
-		parentIndirection{&table.ipv4, 2}.insert(ip[:], uint8(prefix.Bits()), peer)
+		parentIndirection{&table.IPv4, 2}.insert(ip[:], uint8(prefix.Bits()), peer)
 	} else {
 		panic(errors.New("inserting unknown address type"))
 	}
 }
 
-// LookupFromPacket looks up the peer to which an outbound IP packet should be
-// sent. It lives on [AllowedIPs] for legacy/structural reasons: historically
-// WireGuard's only peer-selection mechanism was the AllowedIPs trie, and the
-// send path already had a reference to the table. When a [PeerByIPPacketFunc]
-// has been registered via [Device.SetPeerByIPPacketFunc], that callback is used
-// instead of the trie and the AllowedIPs table is not consulted at all.
-//
-// When no callback is registered, only dst is used (standard WireGuard
-// AllowedIPs trie lookup). When a callback is registered, all three
-// parameters are forwarded to it; see [PeerByIPPacketFunc] for details.
-func (table *AllowedIPs) LookupFromPacket(src, dst netip.Addr, ipPkt []byte) *Peer {
-	table.mu.RLock()
-	if f := table.peerByIPPacketFunc; f != nil {
-		device := table.device
-		table.mu.RUnlock()
-
-		if pubk, ok := f(src, dst, ipPkt); ok {
-			return device.LookupPeer(pubk)
-		}
-		return nil
-	}
-	defer table.mu.RUnlock()
-
-	switch {
-	case dst.Is6():
-		return table.ipv6.lookup6(dst.As16())
-	case dst.Is4():
-		return table.ipv4.lookup4(dst.As4())
-	default:
-		panic(errors.New("looking up unknown address type"))
-	}
-}
-
-// Deprecated: Lookup is only used by legacy tests. It does not call
-// [PeerByIPPacketFunc]; use [AllowedIPs.LookupFromPacket] for production lookups.
 func (table *AllowedIPs) Lookup(ip []byte) *Peer {
-	table.mu.RLock()
-	defer table.mu.RUnlock()
-	return table.lookupLocked(ip)
-}
-
-// lookupLocked looks up the peer associated with the given IP address.
-// It assumes the caller holds the read lock (or doesn't hold it, but also
-// doesn't concurrently mutate AllowedIP).
-//
-// It returns nil if no peer is associated with the given IP address.
-func (table *AllowedIPs) lookupLocked(ip []byte) *Peer {
+	table.mutex.RLock()
+	defer table.mutex.RUnlock()
 	switch len(ip) {
 	case net.IPv6len:
-		return table.ipv6.lookup(ip)
+		return table.IPv6.lookup(ip)
 	case net.IPv4len:
-		return table.ipv4.lookup(ip)
+		return table.IPv4.lookup(ip)
 	default:
 		panic(errors.New("looking up unknown address type"))
 	}
 }
-
-// AllowedPeerSourceIP reports whether the given source IP address is allowed
-// for the given peer.
-func (peer *Peer) AllowedPeerSourceIP(src netip.Addr) bool {
-	if f := peer.state.testAllowedIP.Load(); f != nil {
-		return (*f)(src)
-	}
-
-	table := &peer.device.allowedips
-	table.mu.RLock()
-	defer table.mu.RUnlock()
-	switch {
-	case src.Is6():
-		return table.ipv6.lookup6(src.As16()) == peer
-	case src.Is4():
-		return table.ipv4.lookup4(src.As4()) == peer
-	}
-	return false
-}
-
-// fakePeer is a zero Peer used only as a placeholder in tries used by mkIPInCIDRsTestFunc.
-var fakePeer Peer
-
-// mkIPInCIDRsTestFunc returns a function that tests whether an IP address is
-// contained in any of the given CIDRs.
-func mkIPInCIDRsTestFunc(cidrs []netip.Prefix) func(netip.Addr) bool {
-	if len(cidrs) == 0 {
-		return func(netip.Addr) bool { return false }
-	}
-	if len(cidrs) == 1 {
-		return func(addr netip.Addr) bool { return cidrs[0].Contains(addr) }
-	}
-	if len(cidrs) <= 4 {
-		// For small numbers of CIDRs, just do a linear search. The trie construction
-		// is more expensive than the linear search, and the test function is faster
-		// than the trie lookup, so this is a net win.
-		return func(addr netip.Addr) bool {
-			for _, c := range cidrs {
-				if c.Contains(addr) {
-					return true
-				}
-			}
-			return false
-		}
-	}
-	// Make a trie for faster lookups. We use a dummy Peer.
-	var a AllowedIPs
-	for _, c := range cidrs {
-		a.Insert(c, &fakePeer)
-	}
-	return func(addr netip.Addr) bool {
-		switch {
-		case addr.Is4():
-			return a.ipv4.lookup4(addr.As4()) == &fakePeer
-		default:
-			return a.ipv6.lookup6(addr.As16()) == &fakePeer
-		}
-	}
-}

device/awg_stdnetbind_reserved_lx_test.go

@@ -1,190 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * lx: e2e regression for the reserved-clear vs AWG magic-header collision,
- * exercised over the StdNetBind path (no detour) with real loopback UDP.
- *
- * Bug model. On receive, StdNetBind.receiveIP unconditionally zeroed bytes
- * 1-3 of every datagram >3 bytes (the Cloudflare WARP "reserved" field).
- * AmneziaWG reads its magic header as LittleEndian.Uint32(packet[padding:]),
- * where padding is s1/s2 (handshake) or s4 (transport). With small padding
- * (0..3) the 4-byte magic overlaps bytes 1-3, so the unconditional clear
- * corrupts it: the value falls outside the ranged h1-h4 window, the packet is
- * classified MessageUnknownType and dropped. WARP was never configured on
- * these binds (no SetReservedForEndpoint), so the clear was pure collateral.
- *
- * The fix gates the clear behind StdNetBind.hasReserved(): bytes 1-3 are only
- * zeroed when a non-zero reserved value is actually set for some endpoint.
- * With no reserved value the magic survives and the handshake completes.
- *
- * This test provokes the worst case: padding = 0 (no s1/s2/s4 at all), so the
- * h1 initiation magic sits in bytes [0..3] and its high bytes (1-3) are the
- * ones the clear would destroy. The h1-h4 ranges are chosen entirely above
- * 0x10000000, so after zeroing bytes 1-3 the surviving value is <= 255 and can
- * never land back inside any range -> guaranteed drop on the buggy tree.
- *
- * GREEN on the fixed tree. To see RED, temporarily restore the unconditional
- * clear in conn/bind_std.go receiveIP:
- *     if msg.N > 3 {
- *         common.ClearArray(bufs[i][1:4])
- *     }
- * and the handshake times out (init magic zeroed in bytes 1-3).
- */
-
-package device
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"encoding/hex"
-	"fmt"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/sagernet/wireguard-go/conn"
-)
-
-// magic header ranges kept entirely above 0x10000000 (268435456). Any value
-// the sender picks therefore has a non-zero byte among positions 1-3; zeroing
-// those bytes collapses the value to <= 0xFF, which is below every range start,
-// so a corrupted magic can never validate. Distinct windows per message type.
-const (
-	lxH1Lo, lxH1Hi = 268500000, 268600000 // init
-	lxH2Lo, lxH2Hi = 300000000, 300100000 // response
-	lxH3Lo, lxH3Hi = 400000000, 400100000 // cookie
-	lxH4Lo, lxH4Hi = 500000000, 500100000 // transport
-)
-
-// lxReadListenPort parses listen_port=<n> out of a device's IpcGet dump.
-func lxReadListenPort(t *testing.T, dev *Device) uint16 {
-	t.Helper()
-	dump, err := dev.IpcGet()
-	if err != nil {
-		t.Fatalf("IpcGet: %v", err)
-	}
-	scanner := bufio.NewScanner(strings.NewReader(dump))
-	for scanner.Scan() {
-		line := scanner.Text()
-		if v, ok := strings.CutPrefix(line, "listen_port="); ok {
-			p, err := strconv.Atoi(v)
-			if err != nil {
-				t.Fatalf("parse listen_port %q: %v", v, err)
-			}
-			return uint16(p)
-		}
-	}
-	t.Fatalf("listen_port not found in dump:\n%s", dump)
-	return 0
-}
-
-// newStdNetPaddedPair builds two Up()'d Devices peered over real loopback UDP
-// (NewStdNetBind), configured with ranged h1-h4 + junk and *no* s1/s2/s4
-// (padding = 0). Endpoints are wired after Up, once the ephemeral ports are
-// known. No reserved value is ever set, so hasReserved() is false.
-func newStdNetPaddedPair(t *testing.T) (devA, devB *Device, tunA, tunB *chanTun) {
-	t.Helper()
-
-	skA, err := newPrivateKey()
-	if err != nil {
-		t.Fatalf("newPrivateKey A: %v", err)
-	}
-	skB, err := newPrivateKey()
-	if err != nil {
-		t.Fatalf("newPrivateKey B: %v", err)
-	}
-	pkA := skA.publicKey()
-	pkB := skB.publicKey()
-
-	tunA = newChanTun()
-	tunB = newChanTun()
-
-	devA = NewDevice(context.Background(), tunA, conn.NewStdNetBind(nil), NewLogger(LogLevelError, "devA: "), 1)
-	devB = NewDevice(context.Background(), tunB, conn.NewStdNetBind(nil), NewLogger(LogLevelError, "devB: "), 1)
-	t.Cleanup(devA.Close)
-	t.Cleanup(devB.Close)
-
-	// obfuscation shared by both ends. Ranged magic headers, junk packets,
-	// and deliberately no s1/s2/s4 so padding stays 0 for every message type.
-	obf := fmt.Sprintf(
-		"jc=3\njmin=8\njmax=16\n"+
-			"h1=%d-%d\nh2=%d-%d\nh3=%d-%d\nh4=%d-%d\n",
-		lxH1Lo, lxH1Hi, lxH2Lo, lxH2Hi, lxH3Lo, lxH3Hi, lxH4Lo, lxH4Hi)
-
-	// Bring both up on an ephemeral port (listen_port=0), no endpoint yet.
-	cfgA := fmt.Sprintf("private_key=%s\nlisten_port=0\n%sreplace_peers=true\npublic_key=%s\nallowed_ip=%s/32\n",
-		hex.EncodeToString(skA[:]), obf, hex.EncodeToString(pkB[:]), testIPB)
-	cfgB := fmt.Sprintf("private_key=%s\nlisten_port=0\n%sreplace_peers=true\npublic_key=%s\nallowed_ip=%s/32\n",
-		hex.EncodeToString(skB[:]), obf, hex.EncodeToString(pkA[:]), testIPA)
-
-	if err := devA.IpcSet(cfgA); err != nil {
-		t.Fatalf("IpcSet A: %v", err)
-	}
-	if err := devB.IpcSet(cfgB); err != nil {
-		t.Fatalf("IpcSet B: %v", err)
-	}
-	if devA.paddings.init != 0 || devA.paddings.response != 0 || devA.paddings.transport != 0 {
-		t.Fatalf("padding must be 0 for this test: init=%d resp=%d transport=%d",
-			devA.paddings.init, devA.paddings.response, devA.paddings.transport)
-	}
-
-	if err := devA.Up(); err != nil {
-		t.Fatalf("Up A: %v", err)
-	}
-	if err := devB.Up(); err != nil {
-		t.Fatalf("Up B: %v", err)
-	}
-
-	portA := lxReadListenPort(t, devA)
-	portB := lxReadListenPort(t, devB)
-	if portA == 0 || portB == 0 {
-		t.Fatalf("ephemeral ports not assigned: A=%d B=%d", portA, portB)
-	}
-
-	// Now that ports are known, point each peer at the other over loopback.
-	if err := devA.IpcSet(fmt.Sprintf("public_key=%s\nupdate_only=true\nendpoint=127.0.0.1:%d\n",
-		hex.EncodeToString(pkB[:]), portB)); err != nil {
-		t.Fatalf("set endpoint A->B: %v", err)
-	}
-	if err := devB.IpcSet(fmt.Sprintf("public_key=%s\nupdate_only=true\nendpoint=127.0.0.1:%d\n",
-		hex.EncodeToString(pkA[:]), portA)); err != nil {
-		t.Fatalf("set endpoint B->A: %v", err)
-	}
-
-	return devA, devB, tunA, tunB
-}
-
-// TestStdNetBindReservedClearVsMagic_ZeroPadding drives a real handshake and a
-// data packet A->B over loopback UDP through StdNetBind, with padding=0 so the
-// h1 magic overlaps the reserved bytes 1-3. It passes only when receive does
-// not blindly clear those bytes (the fix).
-func TestStdNetBindReservedClearVsMagic_ZeroPadding(t *testing.T) {
-	devA, _, tunA, tunB := newStdNetPaddedPair(t)
-	_ = devA
-
-	pkt := buildIPv4Packet(testIPA, testIPB, 8)
-
-	// Re-inject periodically: the first packet triggers the handshake and may
-	// be dropped until keys are established.
-	send := func() { tunA.toDevice <- pkt }
-	send()
-
-	deadline := time.After(15 * time.Second)
-	retry := time.NewTicker(500 * time.Millisecond)
-	defer retry.Stop()
-	for {
-		select {
-		case got := <-tunB.fromDevice:
-			if bytes.Equal(got, pkt) {
-				return // delivered end to end: magic survived, handshake ok
-			}
-			t.Logf("ignoring unexpected packet len=%d", len(got))
-		case <-retry.C:
-			send()
-		case <-deadline:
-			t.Fatal("timed out waiting for packet on peer tun " +
-				"(handshake never completed: reserved-clear likely corrupted the h1 magic)")
-		}
-	}
-}

device/channels.go

@@ -83,21 +83,15 @@ func newAutodrainingInboundQueue(device *Device) *autodrainingInboundQueue {
 	q := &autodrainingInboundQueue{
 		c: make(chan *QueueInboundElementsContainer, QueueInboundSize),
 	}
-	if device.needsInboundQueueFinalizer() {
-		runtime.AddCleanup(q, device.flushInboundQueue, q.c)
-	}
+	runtime.SetFinalizer(q, device.flushInboundQueue)
 	return q
 }
 
-func (device *Device) needsInboundQueueFinalizer() bool {
-	return device.pool.messageBuffers.hasAccounting()
-}
-
-func (device *Device) flushInboundQueue(c <-chan *QueueInboundElementsContainer) {
+func (device *Device) flushInboundQueue(q *autodrainingInboundQueue) {
 	for {
 		select {
-		case elemsContainer := <-c:
-			elemsContainer.filling.Wait()
+		case elemsContainer := <-q.c:
+			elemsContainer.Lock()
 			for _, elem := range elemsContainer.elems {
 				device.PutMessageBuffer(elem.buffer)
 				device.PutInboundElement(elem)
@@ -122,23 +116,17 @@ func newAutodrainingOutboundQueue(device *Device) *autodrainingOutboundQueue {
 	q := &autodrainingOutboundQueue{
 		c: make(chan *QueueOutboundElementsContainer, QueueOutboundSize),
 	}
-	if device.needsOutboundQueueFinalizer() {
-		runtime.AddCleanup(q, device.flushOutboundQueue, q.c)
-	}
+	runtime.SetFinalizer(q, device.flushOutboundQueue)
 	return q
 }
 
-func (device *Device) needsOutboundQueueFinalizer() bool {
-	return device.pool.messageBuffers.hasAccounting()
-}
-
-func (device *Device) flushOutboundQueue(c <-chan *QueueOutboundElementsContainer) {
+func (device *Device) flushOutboundQueue(q *autodrainingOutboundQueue) {
 	for {
 		select {
-		case elemsContainer := <-c:
-			elemsContainer.filling.Wait()
+		case elemsContainer := <-q.c:
+			elemsContainer.Lock()
 			for _, elem := range elemsContainer.elems {
-				device.PutOutboundBuffer(elem.buffer)
+				device.PutMessageBuffer(elem.buffer)
 				device.PutOutboundElement(elem)
 			}
 			device.PutOutboundElementsContainer(elemsContainer)

device/device.go

@@ -7,8 +7,6 @@ package device
 
 import (
 	"context"
-	"errors"
-	"net/netip"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -61,12 +59,8 @@ type Device struct {
 	peers struct {
 		sync.RWMutex // protects keyMap
 		keyMap       map[NoisePublicKey]*Peer
-		lookupFunc   PeerLookupFunc // or nil if unused
 	}
 
-	peerStateFn   atomic.Pointer[PeerSessionStateFunc]    // observes peer session state changes, nil if unset
-	priorityMsgFn atomic.Pointer[PeerPriorityMessageFunc] // returns a priority message to be sent around session establishment, nil if unset
-
 	rate struct {
 		underLoadUntil atomic.Int64
 		limiter        ratelimiter.Ratelimiter
@@ -77,11 +71,11 @@ type Device struct {
 	cookieChecker CookieChecker
 
 	pool struct {
-		inboundElementsContainer  *sync.Pool
-		outboundElementsContainer *sync.Pool
+		inboundElementsContainer  *WaitPool
+		outboundElementsContainer *WaitPool
 		messageBuffers            *WaitPool
-		inboundElements           *sync.Pool
-		outboundElements          *sync.Pool
+		inboundElements           *WaitPool
+		outboundElements          *WaitPool
 	}
 
 	queue struct {
@@ -122,20 +116,6 @@ type Device struct {
 	}
 
 	ipackets [5]*obfChain
-
-	// lx: SPEC 041 — passive self-heal state: reopen the bind once (fresh
-	// ephemeral port when freshPort is set) and immediately re-initiate, to
-	// heal dead per-flow path state (an expired NAT mapping or a poisoned DPI
-	// flow entry) that otherwise pins every retry to the same dead 5-tuple
-	// until a manual reconnect. The whole mechanism — three triggers (giveup /
-	// early / nudge) sharing this state and its debounce — lives in
-	// lx_giveup_rebind.go. Enabled by default; sing-box decides freshPort
-	// from whether the user pinned listen_port.
-	giveUpRebind struct {
-		enabled   atomic.Bool
-		freshPort atomic.Bool
-		last      atomic.Int64 // unix seconds of the last rebind (debounce)
-	}
 }
 
 // deviceState represents the state of a Device.
@@ -227,22 +207,14 @@ func (device *Device) upLocked() error {
 	device.ipcMutex.Lock()
 	defer device.ipcMutex.Unlock()
 
-	// Collect peers under RLock and then release before calling into them,
-	// because SendKeepalive can reach CreateMessageInitiation which acquires
-	// staticIdentity.RLock; holding peers.RLock across that path would
-	// invert the staticIdentity < peers hierarchy (see lock-ordering.md).
 	device.peers.RLock()
-	peers := make([]*Peer, 0, len(device.peers.keyMap))
 	for _, peer := range device.peers.keyMap {
-		peers = append(peers, peer)
-	}
-	device.peers.RUnlock()
-	for _, peer := range peers {
 		peer.Start()
 		if peer.persistentKeepaliveInterval.Load() > 0 {
 			peer.SendKeepalive()
 		}
 	}
+	device.peers.RUnlock()
 	return nil
 }
 
@@ -340,7 +312,6 @@ func (device *Device) SetPrivateKey(sk NoisePrivateKey) error {
 func NewDevice(ctx context.Context, tunDevice tun.Device, bind conn.Bind, logger *Logger, workers int) *Device {
 	device := new(Device)
 	device.pauseManager = service.FromContext[pause.Manager](ctx)
-	device.giveUpRebind.enabled.Store(true) // lx: SPEC 041 — self-heal on by default
 	device.state.state.Store(uint32(deviceStateDown))
 	device.closed = make(chan struct{})
 	device.log = logger
@@ -403,65 +374,12 @@ func (device *Device) BatchSize() int {
 	return size
 }
 
-// LookupPeer looks up a peer by its public key.
-//
-// If the peer does not exist and a [PeerLookupFunc] is set (via
-// [Device.SetPeerLookupFunc]), then that function is used to create the peer
-// before returning it. Peers created via this mechanism exist only until their
-// state machine reaches idle, and then the peers are removed.
-//
-// If the peer does not exist and no [PeerLookupFunc] is set, nil is returned.
-//
-// Use [Device.LookupActivePeer] to only return already-existing peers, without
-// using a [PeerLookupFunc].
 func (device *Device) LookupPeer(pk NoisePublicKey) *Peer {
-	device.peers.RLock()
-	p, ok := device.peers.keyMap[pk]
-	lookupFunc := device.peers.lookupFunc
-	device.peers.RUnlock()
-	if ok || lookupFunc == nil {
-		return p
-	}
-
-	conf, ok := lookupFunc(pk)
-	if !ok || conf == nil {
-		return nil
-	}
-
-	p, err := device.NewPeer(pk)
-	if err != nil {
-		if errors.Is(err, errAddExistingPeer) {
-			device.peers.RLock()
-			defer device.peers.RUnlock()
-			return device.peers.keyMap[pk]
-		}
-		device.log.Errorf("Failed to create peer: %v", err)
-		return nil
-	}
-	p.SetAllowedIPs(conf.AllowedIPs)
-	p.deleteOnIdle = true
-	if conf.Endpoint != nil {
-		p.SetEndpointFromPacket(conf.Endpoint)
-	}
-	p.Start()
-	return p
-}
-
-// LookupActivePeer looks up a peer by its public key.
-//
-// Unlike [Device.LookupPeer], this function does not use a [PeerLookupFunc] to
-// create the peer if it does not already exist.
-//
-// If the peer does not exist or was created lazily via [PeerLookupFunc]
-// and has subsequently idled away, it returns (nil, false).
-func (device *Device) LookupActivePeer(pk NoisePublicKey) (_ *Peer, ok bool) {
 	device.peers.RLock()
 	defer device.peers.RUnlock()
-	p, ok := device.peers.keyMap[pk]
-	return p, ok
-}
 
-var errAddExistingPeer = errors.New("adding existing peer")
+	return device.peers.keyMap[pk]
+}
 
 func (device *Device) RemovePeer(key NoisePublicKey) {
 	device.peers.Lock()
@@ -485,151 +403,6 @@ func (device *Device) RemoveAllPeers() {
 	device.peers.keyMap = make(map[NoisePublicKey]*Peer)
 }
 
-// RemoveMatchingPeers removes all peers for which shouldRemove returns true.
-//
-// It returns the number of peers removed.
-func (device *Device) RemoveMatchingPeers(shouldRemove func(NoisePublicKey) bool) (numRemoved int) {
-	device.peers.Lock()
-	defer device.peers.Unlock()
-
-	for key, peer := range device.peers.keyMap {
-		if shouldRemove(key) {
-			removePeerLocked(device, peer, key)
-			numRemoved++
-		}
-	}
-	return numRemoved
-}
-
-// NewPeerConfig are the configuration parameters for a new peer created via a
-// [PeerLookupFunc] func.
-type NewPeerConfig struct {
-	// AllowedIPs is the initial set of allowed IPs for the new peer.
-	AllowedIPs []netip.Prefix
-
-	// Endpoint, if non-nil, sets the initial endpoint for newly
-	// created peers.
-	Endpoint conn.Endpoint
-}
-
-// PeerLookupFunc is the type of function used to look up peers by public key
-// when receiving packets for unknown peers.
-//
-// If it returns nil, the peer is not known.
-//
-// Otherwise, returning non-nil signals that wireguard-go should create the peer
-// with the provided allowed IPs.
-//
-// See [Device.SetPeerLookupFunc] and [Device.LookupPeer].
-type PeerLookupFunc func(NoisePublicKey) (_ *NewPeerConfig, ok bool)
-
-// PeerByIPPacketFunc is the type of function used to look up a peer to send to
-// for a given src/dst IP pair. The ipPkt parameter is the raw IP packet being
-// routed; callers needing transport-layer ports or other header fields may parse
-// them from ipPkt, but must handle IP fragmentation (ports may be absent on
-// non-first fragments) and protocols that do not use ports (e.g. ICMP).
-//
-// Except for experimental use cases, dst is the only address
-// that should be relied upon when looking up a peer.
-//
-// If it returns ok=false, the peer is not known.
-//
-// See [Device.SetPeerByIPPacketFunc] and [Device.SetPeerLookupFunc].
-type PeerByIPPacketFunc func(src, dst netip.Addr, ipPkt []byte) (_ NoisePublicKey, ok bool)
-
-// PeerSessionState is the current WireGuard session state for a peer.
-type PeerSessionState uint8
-
-const (
-	// PeerSessionNone means there is no handshake in progress and no session key
-	// material retained for this peer.
-	PeerSessionNone PeerSessionState = iota
-
-	// PeerSessionHandshake means a handshake is in progress for this peer, but
-	// there is not currently a usable WireGuard session.
-	PeerSessionHandshake
-
-	// PeerSessionEstablished means the peer has a completed WireGuard session
-	// with usable session key material.
-	PeerSessionEstablished
-
-	// PeerSessionExpired means the peer's session key material is no longer
-	// considered usable, but final key cleanup or lazy peer removal may not have
-	// happened yet.
-	PeerSessionExpired
-)
-
-// PeerSessionStateFunc is called when a peer's WireGuard session state changes.
-//
-// Calls are serialized per peer and delivered in that peer's transition order. The
-// callback must be cheap and must not call back into Device.
-type PeerSessionStateFunc func(peer NoisePublicKey, state PeerSessionState)
-
-// SetPeerLookupFunc sets the function used to look up peers by public key
-// when receiving packets for unknown peers.
-func (device *Device) SetPeerLookupFunc(f PeerLookupFunc) {
-	device.peers.Lock()
-	defer device.peers.Unlock()
-	device.peers.lookupFunc = f
-}
-
-// SetPeerByIPPacketFunc sets the function used to look up peers by IP address
-// when sending packets to unknown peers.
-func (device *Device) SetPeerByIPPacketFunc(f PeerByIPPacketFunc) {
-	device.allowedips.mu.Lock()
-	defer device.allowedips.mu.Unlock()
-	device.allowedips.peerByIPPacketFunc = f
-	device.allowedips.device = device
-}
-
-// SetSessionStateFunc sets the function used to observe peer WireGuard session
-// state changes.
-//
-// It does not replay current state. Callers that need a complete view should set
-// it before peers are started or lazily created, and maintain any snapshots,
-// sequence numbers, and pubsub state outside wireguard-go.
-//
-// The callback must be concurrent-safe and must not call back into Device.
-func (device *Device) SetSessionStateFunc(f PeerSessionStateFunc) {
-	if f == nil {
-		device.peerStateFn.Store(nil)
-		return
-	}
-	device.peerStateFn.Store(&f)
-}
-
-// MaxPriorityMessageContentSize is the maximum size of a message returned by a
-// [PeerPriorityMessageFunc]. It's a power of 2 that leaves significant space
-// when accounting for all WireGuard overhead and encapsulating network protocol
-// headers. Future adjustments to this value should consider all these overheads
-// and any [conn.Bind] implementation limitations.
-const MaxPriorityMessageContentSize = 512
-
-// PeerPriorityMessageFunc is called when a peer's WireGuard session keypair is
-// established (or re-keyed) for forward data transmission.
-//
-// The returned message is transmitted to the peer in priority fashion. Priority
-// means it cannot be evicted from the staged packet queue by non-priority
-// (read from [tun.Device]) packets. It avoids the staged queue altogether.
-//
-// The callback must be cheap and must not call back into [Device]. A zero length
-// message or a message whose length exceeds [MaxPriorityMessageContentSize] will
-// be silently dropped. Message should start with an IPv4 or IPv6 header as it
-// is subject to allowed IPs lookup on the receiver, same as any other transport
-// message.
-type PeerPriorityMessageFunc func(peer NoisePublicKey) (msg []byte)
-
-// SetPriorityMessageOnEstablishmentFunc sets a function to be used for sending
-// a priority message around session establishment. See [PeerPriorityMessageFunc]
-// docs for more details. A nil value clears any previously set value.
-func (device *Device) SetPriorityMessageOnEstablishmentFunc(f PeerPriorityMessageFunc) {
-	if f == nil {
-		device.priorityMsgFn.Store(nil)
-		return
-	}
-	device.priorityMsgFn.Store(&f)
-}
-
 func (device *Device) Close() {
 	device.state.Lock()
 	defer device.state.Unlock()
@@ -671,25 +444,16 @@ func (device *Device) SendKeepalivesToPeersWithCurrentKeypair() {
 		return
 	}
 
-	// Collect the set of peers to keepalive under peers.RLock, then release
-	// before invoking SendKeepalive. SendKeepalive can reach
-	// CreateMessageInitiation which acquires staticIdentity.RLock; holding
-	// peers.RLock across that path would invert the
-	// staticIdentity < peers hierarchy (see lock-ordering.md).
-	var peers []*Peer
 	device.peers.RLock()
 	for _, peer := range device.peers.keyMap {
 		peer.keypairs.RLock()
 		sendKeepalive := peer.keypairs.current != nil && !peer.keypairs.current.created.Add(RejectAfterTime).Before(time.Now())
 		peer.keypairs.RUnlock()
 		if sendKeepalive {
-			peers = append(peers, peer)
+			peer.SendKeepalive()
 		}
 	}
 	device.peers.RUnlock()
-	for _, peer := range peers {
-		peer.SendKeepalive()
-	}
 }
 
 // closeBindLocked closes the device's net.bind.

device/keypair.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package device

device/lx_early_rebind_test.go

@@ -1,58 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * lx: SPEC 041 v2 — behavioural red/green test for the EARLY give-up rebind.
- * Field failure mode (the v1 field leftover, dump 2026-08-01): the v1 give-up
- * rebind heals a dead 5-tuple, but only at ~90s after the first demand — while
- * the user pings within the first 5-35s after device wake and sees every node
- * in ERR. The early trigger fires from the retry branch once >=3 initiations
- * went unanswered AND the session is provably dead (no live keypair, or last
- * handshake older than RejectAfterTime), shrinking the ERR window to ~15-20s.
- *
- * The test reuses the v1 harness (gateBind: first socket generation blackholes
- * every send) and drives the peer to the 3rd retry expiry — the state a real
- * ~15s of unanswered retries ends in. Post-fix the retry branch rebinds and
- * the tunnel comes up; pre-fix (v1 base) the retry keeps dying in the first
- * socket generation and only the ~90s give-up would heal, so the packet never
- * arrives within the test window.
- *
- * This file deliberately uses NO post-fix API, so it compiles and runs RED on
- * the v1 base commit.
- */
-
-package device
-
-import (
-	"testing"
-	"time"
-)
-
-// TestEarlyRebindSelfHeal: dead first socket, cold session (no keypair yet),
-// 3rd retry expiry fires — the tunnel must come up and deliver traffic without
-// waiting out the full 90s give-up cycle.
-func TestEarlyRebindSelfHeal(t *testing.T) {
-	pair := newGiveUpPair(t, true)
-
-	pkt := buildIPv4Packet(testIPA, testIPB, 8)
-	send := func() { pair.tunA.toDevice <- pkt }
-
-	// Traffic demand: stages the packet and sends the first (blackholed)
-	// initiation.
-	send()
-
-	pair.devA.peers.RLock()
-	peer := pair.devA.peers.keyMap[pair.pkB]
-	pair.devA.peers.RUnlock()
-	if peer == nil {
-		t.Fatal("peer not found")
-	}
-
-	// Simulate reaching the 3rd retry expiry (~15s in the field): two retries
-	// already counted, the last initiation older than the retransmit timeout.
-	peer.handshake.mutex.Lock()
-	peer.handshake.lastSentHandshake = time.Now().Add(-2 * RekeyTimeout)
-	peer.handshake.mutex.Unlock()
-	peer.timers.handshakeAttempts.Store(2)
-	expiredRetransmitHandshake(peer)
-
-	awaitPacket(t, pair.tunB, pkt, send)
-}

device/lx_giveup_rebind.go

@@ -1,143 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * lx: SPEC 041 — passive self-heal for a dead per-flow path (an expired NAT
- * mapping or a poisoned DPI flow entry that pins every retry to the same dead
- * 5-tuple until a manual reconnect). One mechanism — reopen the bind (fresh
- * ephemeral port when allowed) and immediately re-initiate — with three
- * triggers sharing one debounce window:
- *
- *   giveup — the handshake retry cycle exhausted (~90s of unanswered
- *            initiations under traffic demand); safety net, covers every path;
- *   early  — >=3 unanswered initiations against a provably dead session
- *            (see sessionProvablyDead): no point waiting out the rest of the
- *            cycle, rebind at ~15s instead of ~90s;
- *   nudge  — the consumer reports "device woke up" via
- *            Device.RebindIfSessionStale (wired through sing-box libbox);
- *            heals without waiting for traffic demand at all.
- *
- * Zero cost while healthy: no timers, no goroutines — triggers 1-2 live in
- * the existing retry cycle, trigger 3 is paid by the caller. The state lives
- * in Device.giveUpRebind (device.go); enabled defaults to true in NewDevice,
- * sing-box decides freshPort from whether the user pinned listen_port.
- */
-
-package device
-
-import "time"
-
-// earlyGiveUpMinAttempts is the number of unanswered initiations (retry timer
-// expiries) after which a provably dead session is rebound early instead of
-// waiting out the full RekeyAttemptTime cycle: ~15s at RekeyTimeout=5s.
-const earlyGiveUpMinAttempts = 3
-
-// SetGiveUpRebind configures the self-heal (see the giveUpRebind field
-// comment in device.go). freshPort must be false when the user pinned an
-// explicit listen_port: the pinned port is preserved, at the cost of the
-// rebind not changing the 5-tuple.
-func (device *Device) SetGiveUpRebind(enabled, freshPort bool) {
-	device.giveUpRebind.enabled.Store(enabled)
-	device.giveUpRebind.freshPort.Store(freshPort)
-}
-
-// handleHandshakeGiveUp is invoked from the give-up branch of
-// expiredRetransmitHandshake: ~90s of initiations went unanswered, so the
-// current socket's 5-tuple is proven dead.
-func (device *Device) handleHandshakeGiveUp(peer *Peer) {
-	device.selfHealRebind("giveup", peer)
-}
-
-// maybeEarlyGiveUpRebind is invoked from the RETRY branch of
-// expiredRetransmitHandshake. Once enough initiations went unanswered AND the
-// session is provably dead there is nothing left to protect — rebind now, at
-// ~15s instead of ~90s. The retry cycle itself continues untouched: this only
-// moves the socket under it. A live session with transient packet loss fails
-// sessionProvablyDead and keeps byte-for-byte upstream behaviour; the shared
-// debounce means this also suppresses the giveup rebind of the same series.
-func (device *Device) maybeEarlyGiveUpRebind(peer *Peer) {
-	if peer.timers.handshakeAttempts.Load() < earlyGiveUpMinAttempts {
-		return
-	}
-	if !device.sessionProvablyDead(peer) {
-		return
-	}
-	device.selfHealRebind("early", peer)
-}
-
-// sessionProvablyDead reports whether the peer's session is beyond saving: no
-// live keypair, or the last successful handshake is older than
-// RejectAfterTime (the keys are invalid after that, so a rebind loses
-// nothing). The stale predicate shared by the early and nudge triggers.
-func (device *Device) sessionProvablyDead(peer *Peer) bool {
-	if peer.keypairs.Current() == nil {
-		return true
-	}
-	return time.Since(time.Unix(0, peer.lastHandshakeNano.Load())) > RejectAfterTime
-}
-
-// RebindIfSessionStale is the wake-nudge entry (trigger 3): the consumer
-// observed a device wake-up and asks for an immediate heal instead of waiting
-// for traffic demand to walk the retry cycle. If any running peer's session
-// is provably dead the bind is reopened once (shared debounce) and every such
-// peer re-initiates immediately; a healthy device is a no-op. Returns whether
-// a rebind was actually scheduled. Never blocks on the rebind itself — the
-// heavy part runs in a goroutine (see selfHealRebind). On a down or closed
-// device it is a no-op, so callers racing idle-suspend or Close are safe.
-func (device *Device) RebindIfSessionStale() bool {
-	if !device.giveUpRebind.enabled.Load() || !device.isUp() {
-		return false
-	}
-	var stale []*Peer
-	device.peers.RLock()
-	for _, peer := range device.peers.keyMap {
-		if peer.isRunning.Load() && device.sessionProvablyDead(peer) {
-			stale = append(stale, peer)
-		}
-	}
-	device.peers.RUnlock()
-	if len(stale) == 0 {
-		return false
-	}
-	return device.selfHealRebind("nudge", stale...)
-}
-
-// selfHealRebind is the shared action behind all three triggers. Runs the
-// heavy part in a goroutine so a timer callback (or a nudge caller) never
-// blocks on BindUpdate's worker drain. Debounced to one rebind per
-// RekeyAttemptTime per device across ALL triggers (CAS on `last` settles
-// concurrent multi-peer races): an early rebind at ~15s suppresses the giveup
-// rebind of the same failed series at ~90s. On a down or closed device
-// BindUpdate does not reopen the socket, so a rebind racing idle-suspend
-// (SPEC 020) or Close degrades to a no-op.
-func (device *Device) selfHealRebind(trigger string, peers ...*Peer) bool {
-	if !device.giveUpRebind.enabled.Load() {
-		return false
-	}
-	if device.isClosed() {
-		return false
-	}
-	now := time.Now().Unix()
-	last := device.giveUpRebind.last.Load()
-	if now-last < int64(RekeyAttemptTime/time.Second) {
-		return false
-	}
-	if !device.giveUpRebind.last.CompareAndSwap(last, now) {
-		return false
-	}
-	fresh := device.giveUpRebind.freshPort.Load()
-	go func() {
-		if fresh {
-			device.net.Lock()
-			device.net.port = 0
-			device.net.Unlock()
-		}
-		if err := device.BindUpdate(); err != nil {
-			device.log.Errorf("%v - Failed self-heal rebind (trigger=%s): %v", peers[0], trigger, err)
-			return
-		}
-		device.log.Verbosef("%v - Rebound socket for self-heal (trigger=%s, fresh port=%v)", peers[0], trigger, fresh)
-		for _, peer := range peers {
-			peer.SendHandshakeInitiation(false)
-		}
-	}()
-	return true
-}

device/lx_giveup_rebind_test.go

@@ -1,83 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * lx: SPEC 041 — unit tests for the give-up rebind mechanics on top of the
- * self-heal harness (lx_giveup_selfheal_test.go): fresh vs pinned port,
- * debounce, and disabled = upstream parity. These use the post-fix API
- * (SetGiveUpRebind) and are NOT expected to compile on the pre-fix base.
- */
-
-package device
-
-import (
-	"testing"
-	"time"
-)
-
-func waitOpens(t *testing.T, bind *gateBind, want int) {
-	t.Helper()
-	deadline := time.Now().Add(5 * time.Second)
-	for bind.openCount() < want {
-		if time.Now().After(deadline) {
-			t.Fatalf("bind reopened %d times, want %d", bind.openCount(), want)
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-}
-
-// Fresh mode (listen_port not pinned): the rebind must ask the OS for a new
-// ephemeral port — Open is called with port 0.
-func TestGiveUpRebindFreshPort(t *testing.T) {
-	pair := newGiveUpPair(t, false)
-	pair.devA.SetGiveUpRebind(true, true)
-
-	triggerGiveUp(t, pair.devA, pair.pkB)
-	waitOpens(t, pair.bindA, 2)
-
-	ports := pair.bindA.portsSnapshot()
-	if ports[1] != 0 {
-		t.Fatalf("rebind requested port %d, want 0 (fresh ephemeral)", ports[1])
-	}
-}
-
-// Pinned mode (explicit listen_port): the rebind must keep the current port.
-// chanBind.Open reports its source id (1) as the actual port, so the device
-// stores net.port=1 after the first Open and must reuse it.
-func TestGiveUpRebindPinnedPortPreserved(t *testing.T) {
-	pair := newGiveUpPair(t, false)
-	pair.devA.SetGiveUpRebind(true, false)
-
-	triggerGiveUp(t, pair.devA, pair.pkB)
-	waitOpens(t, pair.bindA, 2)
-
-	ports := pair.bindA.portsSnapshot()
-	if ports[1] != 1 {
-		t.Fatalf("rebind requested port %d, want 1 (pinned)", ports[1])
-	}
-}
-
-// A second give-up inside the debounce window must not rebind again.
-func TestGiveUpRebindDebounce(t *testing.T) {
-	pair := newGiveUpPair(t, false)
-
-	triggerGiveUp(t, pair.devA, pair.pkB)
-	waitOpens(t, pair.bindA, 2)
-
-	triggerGiveUp(t, pair.devA, pair.pkB)
-	time.Sleep(300 * time.Millisecond)
-	if got := pair.bindA.openCount(); got != 2 {
-		t.Fatalf("debounce failed: bind opened %d times, want 2", got)
-	}
-}
-
-// Disabled: the give-up branch must behave exactly like upstream — flush and
-// stop, no rebind.
-func TestGiveUpRebindDisabled(t *testing.T) {
-	pair := newGiveUpPair(t, false)
-	pair.devA.SetGiveUpRebind(false, false)
-
-	triggerGiveUp(t, pair.devA, pair.pkB)
-	time.Sleep(300 * time.Millisecond)
-	if got := pair.bindA.openCount(); got != 1 {
-		t.Fatalf("disabled mechanism still rebound: %d opens, want 1", got)
-	}
-}

device/lx_giveup_selfheal_test.go

@@ -1,172 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * lx: SPEC 041 — behavioural red/green test for the handshake give-up
- * self-heal. Field failure mode (WARP/AWG after device sleep): the per-flow
- * path state of the socket's 5-tuple dies (expired NAT mapping / poisoned DPI
- * flow entry), every packet sent from the old socket vanishes, and upstream
- * wireguard-go retries into that dead socket forever — only a manual
- * reconnect (new socket, new ephemeral port) heals the peer.
- *
- * The test models the dead 5-tuple with a bind whose FIRST socket generation
- * silently swallows every send; any socket opened after a rebind delivers
- * normally. It then drives the peer into the give-up branch of
- * expiredRetransmitHandshake and expects traffic to flow end to end without
- * any reconnect:
- *
- *   - pre-fix (base): give-up only flushes staged packets, the bind is never
- *     reopened, every retry keeps dying in the first generation -> timeout;
- *   - post-fix: give-up rebinds the socket and re-initiates -> tunnel comes
- *     up and the packet arrives.
- *
- * This file deliberately uses NO post-fix API, so it compiles and runs RED on
- * the pre-fix base commit. Reuses the chanBind/chanTun harness from
- * transport_padding_test.go.
- */
-
-package device
-
-import (
-	"context"
-	"encoding/hex"
-	"fmt"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/sagernet/wireguard-go/conn"
-)
-
-// gateBind wraps chanBind: it records every Open (the port argument the
-// device asked for) and silently swallows sends while the socket generation
-// is at most dropOpens — modelling a dead 5-tuple whose packets vanish on the
-// path without any local error.
-type gateBind struct {
-	*chanBind
-	mu        sync.Mutex
-	openPorts []uint16
-	opens     int
-	dropOpens int // swallow sends while opens <= dropOpens
-}
-
-func (b *gateBind) Open(port uint16) ([]conn.ReceiveFunc, uint16, error) {
-	fns, actual, err := b.chanBind.Open(port)
-	b.mu.Lock()
-	b.opens++
-	b.openPorts = append(b.openPorts, port)
-	b.mu.Unlock()
-	return fns, actual, err
-}
-
-func (b *gateBind) Send(bufs [][]byte, ep conn.Endpoint, offset int) error {
-	b.mu.Lock()
-	drop := b.opens <= b.dropOpens
-	b.mu.Unlock()
-	if drop {
-		return nil // the dead 5-tuple: no local error, the packet just vanishes
-	}
-	return b.chanBind.Send(bufs, ep, offset)
-}
-
-func (b *gateBind) openCount() int {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	return b.opens
-}
-
-func (b *gateBind) portsSnapshot() []uint16 {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	return append([]uint16(nil), b.openPorts...)
-}
-
-type giveUpPair struct {
-	devA, devB *Device
-	tunA, tunB *chanTun
-	bindA      *gateBind
-	pkB        NoisePublicKey
-}
-
-// newGiveUpPair builds two Up()'d peered devices; devA sits on a gateBind
-// whose first socket generation optionally blackholes all sends.
-func newGiveUpPair(t *testing.T, dropFirstOpen bool) *giveUpPair {
-	t.Helper()
-
-	skA, err := newPrivateKey()
-	if err != nil {
-		t.Fatalf("newPrivateKey A: %v", err)
-	}
-	skB, err := newPrivateKey()
-	if err != nil {
-		t.Fatalf("newPrivateKey B: %v", err)
-	}
-	pkA := skA.publicKey()
-	pkB := skB.publicKey()
-
-	rawA, rawB := newChanBindPair()
-	bindA := &gateBind{chanBind: rawA}
-	if dropFirstOpen {
-		bindA.dropOpens = 1
-	}
-	tunA := newChanTun()
-	tunB := newChanTun()
-
-	devA := NewDevice(context.Background(), tunA, bindA, NewLogger(LogLevelError, "devA: "), 1)
-	devB := NewDevice(context.Background(), tunB, rawB, NewLogger(LogLevelError, "devB: "), 1)
-	t.Cleanup(devA.Close)
-	t.Cleanup(devB.Close)
-
-	cfgA := fmt.Sprintf(
-		"private_key=%s\nreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:2\nallowed_ip=%s/32\n",
-		hex.EncodeToString(skA[:]), hex.EncodeToString(pkB[:]), testIPB)
-	cfgB := fmt.Sprintf(
-		"private_key=%s\nreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:1\nallowed_ip=%s/32\n",
-		hex.EncodeToString(skB[:]), hex.EncodeToString(pkA[:]), testIPA)
-
-	if err := devA.IpcSet(cfgA); err != nil {
-		t.Fatalf("IpcSet A: %v", err)
-	}
-	if err := devB.IpcSet(cfgB); err != nil {
-		t.Fatalf("IpcSet B: %v", err)
-	}
-	if err := devA.Up(); err != nil {
-		t.Fatalf("Up A: %v", err)
-	}
-	if err := devB.Up(); err != nil {
-		t.Fatalf("Up B: %v", err)
-	}
-
-	return &giveUpPair{devA: devA, devB: devB, tunA: tunA, tunB: tunB, bindA: bindA, pkB: pkB}
-}
-
-// triggerGiveUp drives dev's peer into the give-up branch of
-// expiredRetransmitHandshake exactly the way 90s of unanswered retries would:
-// attempts past the limit, last initiation older than RekeyTimeout.
-func triggerGiveUp(t *testing.T, dev *Device, pk NoisePublicKey) {
-	t.Helper()
-	dev.peers.RLock()
-	peer := dev.peers.keyMap[pk]
-	dev.peers.RUnlock()
-	if peer == nil {
-		t.Fatal("peer not found")
-	}
-	peer.handshake.mutex.Lock()
-	peer.handshake.lastSentHandshake = time.Now().Add(-2 * RekeyTimeout)
-	peer.handshake.mutex.Unlock()
-	peer.timers.handshakeAttempts.Store(MaxTimerHandshakes + 1)
-	expiredRetransmitHandshake(peer)
-}
-
-// TestHandshakeGiveUpSelfHeal: dead first socket, give-up fires — the tunnel
-// must come up and deliver traffic without any reconnect.
-func TestHandshakeGiveUpSelfHeal(t *testing.T) {
-	pair := newGiveUpPair(t, true)
-
-	pkt := buildIPv4Packet(testIPA, testIPB, 8)
-	send := func() { pair.tunA.toDevice <- pkt }
-
-	// Traffic demand: stages the packet and sends the first (blackholed)
-	// initiation, exactly the state a real give-up cycle ends in.
-	send()
-	triggerGiveUp(t, pair.devA, pair.pkB)
-	awaitPacket(t, pair.tunB, pkt, send)
-}

device/lx_ipcget_awg_test.go

@@ -1,63 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Pins the AWG get path: IpcGet must report every obfuscation parameter it
- * accepted, including i1..i5. The I-slots are emitted by an `i%d=` loop rather
- * than literal per-key sendf calls, which makes them easy to miss when auditing
- * introspection parity against amneziawg-go by grep alone.
- */
-
-package device
-
-import (
-	"context"
-	"encoding/hex"
-	"strings"
-	"testing"
-)
-
-func TestIpcGetReportsAWGParams(t *testing.T) {
-	sk, err := newPrivateKey()
-	if err != nil {
-		t.Fatalf("newPrivateKey: %v", err)
-	}
-
-	bind, _ := newChanBindPair()
-	dev := NewDevice(context.Background(), newChanTun(), bind, NewLogger(LogLevelError, "dev: "), 1)
-	t.Cleanup(dev.Close)
-
-	set := strings.Join([]string{
-		"private_key=" + hex.EncodeToString(sk[:]),
-		"jc=4", "jmin=40", "jmax=70",
-		"s1=15", "s2=20", "s3=25", "s4=30",
-		"h1=1", "h2=2", "h3=3", "h4=100-200",
-		"i1=<b 0xf6a1>", "i3=<r 8>", "i5=<t>",
-		"",
-	}, "\n")
-	if err := dev.IpcSet(set); err != nil {
-		t.Fatalf("IpcSet: %v", err)
-	}
-
-	got, err := dev.IpcGet()
-	if err != nil {
-		t.Fatalf("IpcGet: %v", err)
-	}
-	t.Logf("IpcGet:\n%s", got)
-
-	for _, want := range []string{
-		"jc=4", "jmin=40", "jmax=70",
-		"s1=15", "s2=20", "s3=25", "s4=30",
-		"h1=1", "h2=2", "h3=3", "h4=100-200",
-		"i1=<b 0xf6a1>", "i3=<r 8>", "i5=<t>",
-	} {
-		if !strings.Contains(got, want) {
-			t.Errorf("IpcGet missing %q", want)
-		}
-	}
-
-	// Unset I-slots must stay absent, not surface as empty values.
-	for _, absent := range []string{"i2=", "i4="} {
-		if strings.Contains(got, absent) {
-			t.Errorf("IpcGet reported unset %q", absent)
-		}
-	}
-}

device/lx_stale_rebind_test.go

@@ -1,228 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * lx: SPEC 041 v2 — unit tests for the stale predicate, the wake nudge
- * (RebindIfSessionStale) and the shared debounce across triggers, on top of
- * the v1 harness (lx_giveup_selfheal_test.go). These use the post-fix API and
- * are NOT expected to compile on the pre-fix base.
- */
-
-package device
-
-import (
-	"sync"
-	"testing"
-	"time"
-)
-
-// establishTunnel completes a real handshake over a healthy pair so the peer
-// holds a live keypair and a fresh lastHandshakeNano.
-func establishTunnel(t *testing.T, pair *giveUpPair) {
-	t.Helper()
-	pkt := buildIPv4Packet(testIPA, testIPB, 8)
-	send := func() { pair.tunA.toDevice <- pkt }
-	send()
-	awaitPacket(t, pair.tunB, pkt, send)
-}
-
-func peerOf(t *testing.T, dev *Device, pk NoisePublicKey) *Peer {
-	t.Helper()
-	dev.peers.RLock()
-	peer := dev.peers.keyMap[pk]
-	dev.peers.RUnlock()
-	if peer == nil {
-		t.Fatal("peer not found")
-	}
-	return peer
-}
-
-// A cold peer (no keypair yet, nothing to lose): the nudge must rebind and
-// immediately initiate — the tunnel comes up without any traffic demand.
-func TestNudgeRebindsStaleSession(t *testing.T) {
-	pair := newGiveUpPair(t, true)
-
-	if !pair.devA.RebindIfSessionStale() {
-		t.Fatal("nudge on a cold (keypair-less) session must rebind")
-	}
-	waitOpens(t, pair.bindA, 2)
-
-	// The immediate initiation must bring the tunnel up: traffic sent only
-	// AFTER the nudge flows end to end.
-	pkt := buildIPv4Packet(testIPA, testIPB, 8)
-	send := func() { pair.tunA.toDevice <- pkt }
-	send()
-	awaitPacket(t, pair.tunB, pkt, send)
-}
-
-// A healthy session (live keypair, fresh handshake) must be a strict no-op.
-func TestNudgeHealthySessionNoop(t *testing.T) {
-	pair := newGiveUpPair(t, false)
-	establishTunnel(t, pair)
-
-	opens := pair.bindA.openCount()
-	if pair.devA.RebindIfSessionStale() {
-		t.Fatal("nudge on a healthy session must not rebind")
-	}
-	time.Sleep(100 * time.Millisecond)
-	if got := pair.bindA.openCount(); got != opens {
-		t.Fatalf("healthy nudge reopened the bind: %d opens, want %d", got, opens)
-	}
-}
-
-// A live keypair whose last handshake is older than RejectAfterTime is
-// provably dead (the keys are invalid): the nudge must rebind.
-func TestNudgeExpiredHandshakeIsStale(t *testing.T) {
-	pair := newGiveUpPair(t, false)
-	establishTunnel(t, pair)
-
-	peer := peerOf(t, pair.devA, pair.pkB)
-	peer.lastHandshakeNano.Store(time.Now().Add(-RejectAfterTime - time.Second).UnixNano())
-
-	if !pair.devA.RebindIfSessionStale() {
-		t.Fatal("nudge on an expired session must rebind")
-	}
-	waitOpens(t, pair.bindA, 2)
-}
-
-// A down device (how SPEC 020 idle-suspend leaves it) must be a no-op — the
-// nudge never wakes sleepers.
-func TestNudgeDownDeviceNoop(t *testing.T) {
-	pair := newGiveUpPair(t, true)
-
-	if err := pair.devA.Down(); err != nil {
-		t.Fatalf("Down: %v", err)
-	}
-	if pair.devA.RebindIfSessionStale() {
-		t.Fatal("nudge on a down device must be a no-op")
-	}
-}
-
-// Pinned listen_port survives the nudge rebind.
-func TestNudgePinnedPortPreserved(t *testing.T) {
-	pair := newGiveUpPair(t, true)
-	pair.devA.SetGiveUpRebind(true, false)
-
-	if !pair.devA.RebindIfSessionStale() {
-		t.Fatal("nudge must rebind a cold session")
-	}
-	waitOpens(t, pair.bindA, 2)
-
-	ports := pair.bindA.portsSnapshot()
-	if ports[1] != 1 {
-		t.Fatalf("nudge rebind requested port %d, want 1 (pinned)", ports[1])
-	}
-}
-
-// The debounce window is SHARED across triggers: an early/nudge rebind
-// suppresses the give-up rebind of the same failed series, and a later series
-// (window elapsed) heals again — sliding window, not a latch.
-func TestSharedDebounceAcrossTriggers(t *testing.T) {
-	pair := newGiveUpPair(t, false)
-
-	// First trigger of the series: nudge.
-	if !pair.devA.RebindIfSessionStale() {
-		t.Fatal("first nudge must rebind")
-	}
-	waitOpens(t, pair.bindA, 2)
-
-	// The give-up of the same series lands inside the window: suppressed.
-	triggerGiveUp(t, pair.devA, pair.pkB)
-	time.Sleep(300 * time.Millisecond)
-	if got := pair.bindA.openCount(); got != 2 {
-		t.Fatalf("give-up inside the shared window rebound: %d opens, want 2", got)
-	}
-
-	// Next series: age the window as wall clocks would — heals again.
-	pair.devA.giveUpRebind.last.Store(time.Now().Add(-RekeyAttemptTime - time.Second).Unix())
-	triggerGiveUp(t, pair.devA, pair.pkB)
-	waitOpens(t, pair.bindA, 3)
-}
-
-// The early trigger must NOT fire before enough initiations went unanswered,
-// even against a provably dead session.
-func TestEarlyRebindNeedsMinAttempts(t *testing.T) {
-	pair := newGiveUpPair(t, true)
-	peer := peerOf(t, pair.devA, pair.pkB)
-
-	peer.handshake.mutex.Lock()
-	peer.handshake.lastSentHandshake = time.Now().Add(-2 * RekeyTimeout)
-	peer.handshake.mutex.Unlock()
-	peer.timers.handshakeAttempts.Store(0) // this expiry brings it to 1 (< min)
-	expiredRetransmitHandshake(peer)
-
-	time.Sleep(300 * time.Millisecond)
-	if got := pair.bindA.openCount(); got != 1 {
-		t.Fatalf("early rebind fired below the attempt floor: %d opens, want 1", got)
-	}
-}
-
-// A fresh session (live keypair, recent handshake) must keep the retry branch
-// byte-for-byte upstream even past the attempt floor: no rebind.
-func TestEarlyRebindFreshSessionNoop(t *testing.T) {
-	pair := newGiveUpPair(t, false)
-	establishTunnel(t, pair)
-	peer := peerOf(t, pair.devA, pair.pkB)
-
-	opens := pair.bindA.openCount()
-	peer.handshake.mutex.Lock()
-	peer.handshake.lastSentHandshake = time.Now().Add(-2 * RekeyTimeout)
-	peer.handshake.mutex.Unlock()
-	peer.timers.handshakeAttempts.Store(earlyGiveUpMinAttempts)
-	expiredRetransmitHandshake(peer)
-
-	time.Sleep(300 * time.Millisecond)
-	if got := pair.bindA.openCount(); got != opens {
-		t.Fatalf("early rebind fired on a fresh session: %d opens, want %d", got, opens)
-	}
-}
-
-// Nudge racing Close: no panic, no deadlock, no race-detector report.
-func TestNudgeRacesClose(t *testing.T) {
-	for i := 0; i < 25; i++ {
-		pair := newGiveUpPair(t, false)
-		var wg sync.WaitGroup
-		wg.Add(2)
-		go func() {
-			defer wg.Done()
-			pair.devA.RebindIfSessionStale()
-		}()
-		go func() {
-			defer wg.Done()
-			pair.devA.Close()
-		}()
-		wg.Wait()
-	}
-}
-
-// Nudge racing Down/Up (the SPEC 020 suspend/resume shape): the device must
-// end consistent — up, with a live bind.
-func TestNudgeRacesSuspend(t *testing.T) {
-	for i := 0; i < 25; i++ {
-		pair := newGiveUpPair(t, false)
-		var wg sync.WaitGroup
-		wg.Add(2)
-		go func() {
-			defer wg.Done()
-			pair.devA.RebindIfSessionStale()
-		}()
-		go func() {
-			defer wg.Done()
-			if err := pair.devA.Down(); err != nil {
-				t.Errorf("Down: %v", err)
-			}
-			if err := pair.devA.Up(); err != nil {
-				t.Errorf("Up: %v", err)
-			}
-		}()
-		wg.Wait()
-
-		pair.devA.net.RLock()
-		bindAlive := pair.devA.net.bind != nil
-		pair.devA.net.RUnlock()
-		if !bindAlive || !pair.devA.isUp() {
-			t.Fatalf("iteration %d: device inconsistent after nudge/suspend race (bind=%v up=%v)",
-				i, bindAlive, pair.devA.isUp())
-		}
-		pair.devA.Close()
-		pair.devB.Close()
-	}
-}

device/magic-header.go

@@ -57,9 +57,7 @@ func (h *magicHeader) Validate(val uint32) bool {
 }
 
 func (h *magicHeader) Generate() uint32 {
-	// Widen before arithmetic: end-start+1 in uint32 wraps to 0 for the
-	// full 0..2^32-1 range, which would panic rand.Int (bound <= 0).
-	high := int64(h.end) - int64(h.start) + 1
+	high := int64(h.end - h.start + 1)
 	r, _ := rand.Int(rand.Reader, big.NewInt(high))
 	return h.start + uint32(r.Int64())
 }

device/noise-protocol.go

@@ -351,22 +351,17 @@ func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation, endpoint
 		return nil
 	}
 
-	// Snapshot staticIdentity so we don't hold the RLock across LookupPeer,
-	// which may call NewPeer (reentrant RLock deadlocks against a pending
-	// SetPrivateKey writer; see lock-ordering.md).
 	device.staticIdentity.RLock()
-	publicKey := device.staticIdentity.publicKey
-	privateKey := device.staticIdentity.privateKey
-	device.staticIdentity.RUnlock()
+	defer device.staticIdentity.RUnlock()
 
-	mixHash(&hash, &InitialHash, publicKey[:])
+	mixHash(&hash, &InitialHash, device.staticIdentity.publicKey[:])
 	mixHash(&hash, &hash, msg.Ephemeral[:])
 	mixKey(&chainKey, &InitialChainKey, msg.Ephemeral[:])
 
 	// decrypt static key
 	var peerPK NoisePublicKey
 	var key [chacha20poly1305.KeySize]byte
-	ss, err := privateKey.sharedSecret(msg.Ephemeral)
+	ss, err := device.staticIdentity.privateKey.sharedSecret(msg.Ephemeral)
 	if err != nil {
 		return nil
 	}
@@ -541,14 +536,6 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer {
 		chainKey [blake2s.Size]byte
 	)
 
-	// Snapshot the static private key before acquiring handshake.mutex so
-	// that handshake.mutex is never held while acquiring staticIdentity
-	// (which would invert the staticIdentity < handshake.mutex hierarchy;
-	// see lock-ordering.md).
-	device.staticIdentity.RLock()
-	privateKey := device.staticIdentity.privateKey
-	device.staticIdentity.RUnlock()
-
 	ok := func() bool {
 		// lock handshake state
 
@@ -559,6 +546,11 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer {
 			return false
 		}
 
+		// lock private key for reading
+
+		device.staticIdentity.RLock()
+		defer device.staticIdentity.RUnlock()
+
 		// finish 3-way DH
 
 		mixHash(&hash, &handshake.hash, msg.Ephemeral[:])
@@ -571,7 +563,7 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer {
 		mixKey(&chainKey, &chainKey, ss[:])
 		setZero(ss[:])
 
-		ss, err = privateKey.sharedSecret(msg.Ephemeral)
+		ss, err = device.staticIdentity.privateKey.sharedSecret(msg.Ephemeral)
 		if err != nil {
 			return false
 		}

device/obf.go

@@ -3,26 +3,11 @@ package device
 import (
 	"errors"
 	"fmt"
-	"strconv"
 	"strings"
 )
 
 type obfBuilder func(val string) (obf, error)
 
-// parseObfLen parses and bounds an obfuscator length argument: a negative
-// value would panic slice bounds in obfChain.Obfuscate, a huge one would
-// OOM in the handshake-time make (SendHandshakeInitiation).
-func parseObfLen(val string) (int, error) {
-	length, err := strconv.Atoi(val)
-	if err != nil {
-		return 0, err
-	}
-	if length < 0 || length > MaxMessageSize {
-		return 0, fmt.Errorf("obfuscator length %d out of range [0, %d]", length, MaxMessageSize)
-	}
-	return length, nil
-}
-
 var obfBuilders = map[string]obfBuilder{
 	"b":  newBytesObf,
 	"t":  newTimestampObf,

device/obf_datasize.go

@@ -1,7 +1,9 @@
 package device
 
+import "strconv"
+
 func newDataSizeObf(val string) (obf, error) {
-	length, err := parseObfLen(val)
+	length, err := strconv.Atoi(val)
 	if err != nil {
 		return nil, err
 	}

device/obf_guards_test.go

@@ -1,108 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Guards around AWG obfuscation config values: these tests pin the
- * crash-on-config-value fixes (swapped jmin/jmax, out-of-range obfuscator
- * lengths, full-range magic headers).
- */
-
-package device
-
-import (
-	"context"
-	"encoding/hex"
-	"fmt"
-	"testing"
-)
-
-func TestParseObfLen(t *testing.T) {
-	cases := []struct {
-		val     string
-		want    int
-		wantErr bool
-	}{
-		{"0", 0, false},
-		{"100", 100, false},
-		{fmt.Sprintf("%d", MaxMessageSize), MaxMessageSize, false},
-		{"-1", 0, true}, // would panic slice bounds in Obfuscate
-		{fmt.Sprintf("%d", MaxMessageSize+1), 0, true}, // would OOM the handshake make
-		{"2000000000", 0, true},
-		{"abc", 0, true},
-	}
-	for _, c := range cases {
-		got, err := parseObfLen(c.val)
-		if c.wantErr != (err != nil) {
-			t.Errorf("parseObfLen(%q): err = %v, wantErr = %v", c.val, err, c.wantErr)
-		}
-		if err == nil && got != c.want {
-			t.Errorf("parseObfLen(%q) = %d, want %d", c.val, got, c.want)
-		}
-	}
-}
-
-func TestMagicHeaderGenerateFullRange(t *testing.T) {
-	// end-start+1 computed in uint32 wraps to 0 for the full range and
-	// panics rand.Int; the fix widens to int64 before the arithmetic.
-	h := &magicHeader{start: 0, end: ^uint32(0)}
-	for i := 0; i < 8; i++ {
-		v := h.Generate()
-		if !h.Validate(v) {
-			t.Fatalf("generated value %d outside range", v)
-		}
-	}
-}
-
-// TestJunkSwappedBounds brings up a device pair whose junk config has
-// jmin > jmax (passes per-field UAPI validation); without the swap guard
-// the first handshake panics rand.Int with a non-positive bound.
-func TestJunkSwappedBounds(t *testing.T) {
-	skA, err := newPrivateKey()
-	if err != nil {
-		t.Fatalf("newPrivateKey A: %v", err)
-	}
-	skB, err := newPrivateKey()
-	if err != nil {
-		t.Fatalf("newPrivateKey B: %v", err)
-	}
-	pkA := skA.publicKey()
-	pkB := skB.publicKey()
-
-	bindA, bindB := newChanBindPair()
-	tunA := newChanTun()
-	tunB := newChanTun()
-
-	devA := NewDevice(context.Background(), tunA, bindA, NewLogger(LogLevelError, "devA: "), 1)
-	devB := NewDevice(context.Background(), tunB, bindB, NewLogger(LogLevelError, "devB: "), 1)
-	t.Cleanup(devA.Close)
-	t.Cleanup(devB.Close)
-
-	// jmin deliberately greater than jmax: each field alone is valid.
-	junk := "jc=2\njmin=100\njmax=50\n"
-	cfgA := fmt.Sprintf(
-		"private_key=%s\n%sreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:2\nallowed_ip=%s/32\n",
-		hex.EncodeToString(skA[:]), junk, hex.EncodeToString(pkB[:]), testIPB)
-	cfgB := fmt.Sprintf(
-		"private_key=%s\n%sreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:1\nallowed_ip=%s/32\n",
-		hex.EncodeToString(skB[:]), junk, hex.EncodeToString(pkA[:]), testIPA)
-
-	if err := devA.IpcSet(cfgA); err != nil {
-		t.Fatalf("IpcSet A: %v", err)
-	}
-	if err := devB.IpcSet(cfgB); err != nil {
-		t.Fatalf("IpcSet B: %v", err)
-	}
-
-	if err := devA.Up(); err != nil {
-		t.Fatalf("Up A: %v", err)
-	}
-	if err := devB.Up(); err != nil {
-		t.Fatalf("Up B: %v", err)
-	}
-
-	// Drive a packet end-to-end: the handshake (junk packets included)
-	// must complete without panicking the process.
-	pkt := buildIPv4Packet(testIPA, testIPB, 28)
-	devA.InputPacket(testIPB.AsSlice(), [][]byte{pkt})
-	awaitPacket(t, tunB, pkt, func() {
-		devA.InputPacket(testIPB.AsSlice(), [][]byte{pkt})
-	})
-}

device/obf_rand.go

@@ -2,10 +2,11 @@ package device
 
 import (
 	"crypto/rand"
+	"strconv"
 )
 
 func newRandObf(val string) (obf, error) {
-	length, err := parseObfLen(val)
+	length, err := strconv.Atoi(val)
 	if err != nil {
 		return nil, err
 	}

device/obf_randchars.go

@@ -2,13 +2,14 @@ package device
 
 import (
 	"crypto/rand"
+	"strconv"
 	"unicode"
 )
 
 const chars52 = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
 func newRandCharObf(val string) (obf, error) {
-	length, err := parseObfLen(val)
+	length, err := strconv.Atoi(val)
 	if err != nil {
 		return nil, err
 	}

device/obf_randdigits.go

@@ -2,13 +2,14 @@ package device
 
 import (
 	"crypto/rand"
+	"strconv"
 	"unicode"
 )
 
 const digits10 = "0123456789"
 
 func newRandDigitsObf(val string) (obf, error) {
-	length, err := parseObfLen(val)
+	length, err := strconv.Atoi(val)
 	if err != nil {
 		return nil, err
 	}

device/peer.go

@@ -8,8 +8,6 @@ package device
 import (
 	"container/list"
 	"errors"
-	"net/netip"
-	"slices"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -27,20 +25,6 @@ type Peer struct {
 	rxBytes           atomic.Uint64  // bytes received from peer
 	lastHandshakeNano atomic.Int64   // nano seconds since epoch
 
-	sessionState struct {
-		sync.Mutex
-		current        PeerSessionState
-		sessionExpires time.Time
-	}
-
-	queuedOutboundPackets atomic.Int32 // packets in staged+outbound queues, for input backpressure
-
-	// deleteOnIdle indicates whether the peer should be deleted when idle
-	// because it was auto-created via a Device.PeerLookupFunc.
-	//
-	// This field should only be set once, before the peer is started.
-	deleteOnIdle bool
-
 	endpoint struct {
 		sync.Mutex
 		val            conn.Endpoint
@@ -52,7 +36,6 @@ type Peer struct {
 		retransmitHandshake     *Timer
 		sendKeepalive           *Timer
 		newHandshake            *Timer
-		sessionExpired          *Timer
 		zeroKeyMaterial         *Timer
 		persistentKeepalive     *Timer
 		handshakeAttempts       atomic.Uint32
@@ -61,14 +44,7 @@ type Peer struct {
 	}
 
 	state struct {
-		sync.Mutex // protects against concurrent Start/Stop, and fields below
-
-		allowedIPs []netip.Prefix
-
-		// testAllowedIP, if non-nil, is used to test whether the peer is
-		// allowed to send a packet from the given IP address. It can be read
-		// without locking, but must be set with the state mutex locked.
-		testAllowedIP atomic.Pointer[func(netip.Addr) bool]
+		sync.Mutex // protects against concurrent Start/Stop
 	}
 
 	queue struct {
@@ -111,7 +87,7 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) {
 	// map public key
 	_, ok := device.peers.keyMap[pk]
 	if ok {
-		return nil, errAddExistingPeer
+		return nil, errors.New("adding existing peer")
 	}
 
 	// pre-compute DH
@@ -137,27 +113,6 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) {
 	return peer, nil
 }
 
-// SetAllowedIPs sets the allowed IP prefixes for this peer.
-//
-// If the allowedIPs are unchanged since the last call, this method is a no-op.
-// It's the caller's responsibility to ensure that no two peers have duplicate
-// allowed IPs. If so, the last writer wins.
-func (p *Peer) SetAllowedIPs(allowedIPs []netip.Prefix) {
-	p.state.Lock()
-	defer p.state.Unlock()
-
-	if slices.Equal(p.state.allowedIPs, allowedIPs) {
-		return
-	}
-	p.device.allowedips.setPeerPrefixes(p, allowedIPs)
-
-	allowedIPs = slices.Clone(allowedIPs) // avoid retaining caller's slice
-	p.state.allowedIPs = allowedIPs
-
-	f := mkIPInCIDRsTestFunc(allowedIPs)
-	p.state.testAllowedIP.Store(&f)
-}
-
 // SendBuffers sends buffers to peer. WireGuard packet data in each element of
 // buffers must be preceded by MessageEncapsulatingTransportSize number of
 // bytes.
@@ -238,7 +193,6 @@ func (peer *Peer) Start() {
 	// reset routine state
 	peer.stopping.Wait()
 	peer.stopping.Add(2)
-	peer.queuedOutboundPackets.Store(0)
 
 	peer.handshake.mutex.Lock()
 	peer.handshake.lastSentHandshake = time.Now().Add(-(RekeyTimeout + time.Second))
@@ -248,8 +202,8 @@ func (peer *Peer) Start() {
 
 	peer.timersStart()
 
-	device.flushInboundQueue(peer.queue.inbound.c)
-	device.flushOutboundQueue(peer.queue.outbound.c)
+	device.flushInboundQueue(peer.queue.inbound)
+	device.flushOutboundQueue(peer.queue.outbound)
 
 	// Use the device batch size, not the bind batch size, as the device size is
 	// the size of the batch pools.
@@ -258,21 +212,10 @@ func (peer *Peer) Start() {
 	go peer.RoutineSequentialReceiver(batchSize)
 
 	peer.isRunning.Store(true)
-
-	// A lazily-created peer that never completes a handshake otherwise never
-	// arms its reaping timer. Arm it here, while running under state.Lock, so
-	// it's reclaimed after RejectAfterTime*3 of no session and is guaranteed to
-	// be torn down by a matching Stop. A completed handshake re-Mods it.
-	if peer.deleteOnIdle {
-		peer.timers.zeroKeyMaterial.Mod(RejectAfterTime * 3)
-	}
 }
 
 func (peer *Peer) ZeroAndFlushAll() {
 	device := peer.device
-	if peer.timers.sessionExpired != nil {
-		peer.timers.sessionExpired.Del()
-	}
 
 	// clear key pairs
 
@@ -295,11 +238,6 @@ func (peer *Peer) ZeroAndFlushAll() {
 	handshake.mutex.Unlock()
 
 	peer.FlushStagedPackets()
-
-	peer.sessionState.Lock()
-	peer.sessionState.sessionExpires = time.Time{}
-	peer.noteSessionStateLocked(PeerSessionNone)
-	peer.sessionState.Unlock()
 }
 
 func (peer *Peer) ExpireCurrentKeypairs() {
@@ -319,11 +257,6 @@ func (peer *Peer) ExpireCurrentKeypairs() {
 		next.sendNonce.Store(RejectAfterMessages)
 	}
 	keypairs.Unlock()
-
-	peer.sessionState.Lock()
-	peer.sessionState.sessionExpires = time.Time{}
-	peer.noteSessionStateLocked(PeerSessionExpired)
-	peer.sessionState.Unlock()
 }
 
 func (peer *Peer) Stop() {
@@ -346,51 +279,6 @@ func (peer *Peer) Stop() {
 	peer.ZeroAndFlushAll()
 }
 
-func (peer *Peer) noteSessionState(state PeerSessionState) {
-	peer.sessionState.Lock()
-	defer peer.sessionState.Unlock()
-	peer.noteSessionStateLocked(state)
-}
-
-// noteSessionStateLocked records a session state transition and delivers the
-// callback. The caller must hold peer.sessionState.Mutex during the
-// state determination and transition.
-func (peer *Peer) noteSessionStateLocked(state PeerSessionState) {
-	if peer.sessionState.current == state {
-		return
-	}
-	peer.sessionState.current = state
-	if f := peer.device.peerStateFn.Load(); f != nil {
-		(*f)(peer.handshake.remoteStatic, state)
-	}
-}
-
-func (peer *Peer) noteSessionHandshakeStarted() {
-	peer.sessionState.Lock()
-	defer peer.sessionState.Unlock()
-	if peer.sessionState.current == PeerSessionEstablished {
-		return
-	}
-	peer.noteSessionStateLocked(PeerSessionHandshake)
-}
-
-func (peer *Peer) noteSessionHandshakeStopped() {
-	peer.sessionState.Lock()
-	defer peer.sessionState.Unlock()
-	state := PeerSessionNone
-	if peer.hasKeyMaterial() {
-		state = PeerSessionExpired
-	}
-	peer.noteSessionStateLocked(state)
-}
-
-func (peer *Peer) hasKeyMaterial() bool {
-	keypairs := &peer.keypairs
-	keypairs.RLock()
-	defer keypairs.RUnlock()
-	return keypairs.previous != nil || keypairs.current != nil || keypairs.next.Load() != nil
-}
-
 func (peer *Peer) SetEndpointFromPacket(endpoint conn.Endpoint) {
 	peer.endpoint.Lock()
 	defer peer.endpoint.Unlock()

device/pools.go

@@ -7,8 +7,6 @@ package device
 
 import (
 	"sync"
-
-	"github.com/sagernet/sing/common/buf"
 )
 
 type WaitPool struct {
@@ -25,10 +23,6 @@ func NewWaitPool(max uint32, new func() any) *WaitPool {
 	return p
 }
 
-func (p *WaitPool) hasAccounting() bool {
-	return p != nil && p.max != 0
-}
-
 func (p *WaitPool) Get() any {
 	if p.max != 0 {
 		p.lock.Lock()
@@ -53,27 +47,28 @@ func (p *WaitPool) Put(x any) {
 }
 
 func (device *Device) PopulatePools() {
-	device.pool.inboundElementsContainer = &sync.Pool{New: func() any {
+	device.pool.inboundElementsContainer = NewWaitPool(PreallocatedBuffersPerPool, func() any {
 		s := make([]*QueueInboundElement, 0, device.BatchSize())
 		return &QueueInboundElementsContainer{elems: s}
-	}}
-	device.pool.outboundElementsContainer = &sync.Pool{New: func() any {
+	})
+	device.pool.outboundElementsContainer = NewWaitPool(PreallocatedBuffersPerPool, func() any {
 		s := make([]*QueueOutboundElement, 0, device.BatchSize())
 		return &QueueOutboundElementsContainer{elems: s}
-	}}
+	})
 	device.pool.messageBuffers = NewWaitPool(PreallocatedBuffersPerPool, func() any {
 		return new([MaxMessageSize]byte)
 	})
-	device.pool.inboundElements = &sync.Pool{New: func() any {
+	device.pool.inboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() any {
 		return new(QueueInboundElement)
-	}}
-	device.pool.outboundElements = &sync.Pool{New: func() any {
+	})
+	device.pool.outboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() any {
 		return new(QueueOutboundElement)
-	}}
+	})
 }
 
 func (device *Device) GetInboundElementsContainer() *QueueInboundElementsContainer {
 	c := device.pool.inboundElementsContainer.Get().(*QueueInboundElementsContainer)
+	c.Mutex = sync.Mutex{}
 	return c
 }
 
@@ -87,6 +82,7 @@ func (device *Device) PutInboundElementsContainer(c *QueueInboundElementsContain
 
 func (device *Device) GetOutboundElementsContainer() *QueueOutboundElementsContainer {
 	c := device.pool.outboundElementsContainer.Get().(*QueueOutboundElementsContainer)
+	c.Mutex = sync.Mutex{}
 	return c
 }
 
@@ -106,20 +102,6 @@ func (device *Device) PutMessageBuffer(msg *[MaxMessageSize]byte) {
 	device.pool.messageBuffers.Put(msg)
 }
 
-// Outbound buffers come from the sing allocator instead of the bounded
-// messageBuffers pool: the injection paths (InputPacket/InputPackets) run on
-// the caller's shared read loop, which must never block on pool exhaustion,
-// and their packets are far smaller than MaxMessageSize, so they are allocated
-// by actual size. This also keeps the bounded pool exclusively for the receive
-// path, so outbound backlog can no longer starve it.
-func (device *Device) GetOutboundBuffer(size int) []byte {
-	return buf.Get(size)
-}
-
-func (device *Device) PutOutboundBuffer(buffer []byte) {
-	_ = buf.Put(buffer)
-}
-
 func (device *Device) GetInboundElement() *QueueInboundElement {
 	return device.pool.inboundElements.Get().(*QueueInboundElement)
 }

device/queueconstants_android.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package device
@@ -14,6 +14,6 @@ const (
 	QueueOutboundSize          = 1024
 	QueueInboundSize           = 1024
 	QueueHandshakeSize         = 1024
-	MaxSegmentSize             = (1 << 16) - 1 // largest possible UDP datagram
+	MaxSegmentSize             = 2200
 	PreallocatedBuffersPerPool = 4096
 )

device/queueconstants_default.go

@@ -2,7 +2,7 @@
 
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package device

device/receive.go

@@ -8,9 +8,7 @@ package device
 import (
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"net"
-	"net/netip"
 	"sync"
 	"time"
 
@@ -36,13 +34,8 @@ type QueueInboundElement struct {
 }
 
 type QueueInboundElementsContainer struct {
-	// filling is a one-shot barrier signaling decryption→receive
-	// handoff. RoutineReceiveIncoming calls Add(1) before sending the
-	// container down the decryption and inbound queues; RoutineDecryption
-	// calls Done after decrypting; RoutineSequentialReceiver calls Wait
-	// before reading the decrypted packets.
-	filling sync.WaitGroup
-	elems   []*QueueInboundElement
+	sync.Mutex
+	elems []*QueueInboundElement
 }
 
 // clearPointers clears elem fields that contain pointers.
@@ -103,13 +96,13 @@ func (device *Device) RoutineReceiveIncoming(
 		elemsByPeer = make(map[*Peer]*QueueInboundElementsContainer, maxBatchSize)
 	)
 
-	for i := range bufsArrs {
+	for i := range maxBatchSize {
 		bufsArrs[i] = device.GetMessageBuffer()
 		bufs[i] = bufsArrs[i][:]
 	}
 
 	defer func() {
-		for i := 0; i < maxBatchSize; i++ {
+		for i := range maxBatchSize {
 			if bufsArrs[i] != nil {
 				device.PutMessageBuffer(bufsArrs[i])
 			}
@@ -192,6 +185,7 @@ func (device *Device) RoutineReceiveIncoming(
 				elemsForPeer, ok := elemsByPeer[peer]
 				if !ok {
 					elemsForPeer = device.GetInboundElementsContainer()
+					elemsForPeer.Lock()
 					elemsByPeer[peer] = elemsForPeer
 				}
 				elemsForPeer.elems = append(elemsForPeer.elems, elem)
@@ -235,7 +229,6 @@ func (device *Device) RoutineReceiveIncoming(
 		}
 		for peer, elemsContainer := range elemsByPeer {
 			if peer.isRunning.Load() {
-				elemsContainer.filling.Add(1)
 				peer.queue.inbound.c <- elemsContainer
 				device.queue.decryption.c <- elemsContainer
 			} else {
@@ -277,7 +270,7 @@ func (device *Device) RoutineDecryption(id int) {
 				elem.packet = nil
 			}
 		}
-		elemsContainer.filling.Done()
+		elemsContainer.Unlock()
 	}
 }
 
@@ -439,7 +432,6 @@ func (device *Device) RoutineHandshake(id int) {
 
 			peer.timersSessionDerived()
 			peer.timersHandshakeComplete()
-			peer.SendPriorityMessage()
 			peer.SendKeepalive()
 		}
 	skip:
@@ -461,130 +453,118 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) {
 		if elemsContainer == nil {
 			return
 		}
-		peer.processInboundContainer(elemsContainer, bufs[:0])
-	}
-}
-
-// processInboundContainer waits for the decryption routine to finish
-// filling elemsContainer, then writes the valid packets to the TUN
-// device and returns the container to the pool.
-//
-// scratch is a length-0 slice used to assemble the per-packet buffers
-// passed to tun.device.Write; its backing array is reused across calls.
-func (peer *Peer) processInboundContainer(elemsContainer *QueueInboundElementsContainer, scratch [][]byte) {
-	// Invariants from RoutineSequentialReceiver; all should be unreachable.
-	if len(scratch) != 0 || cap(scratch) == 0 {
-		panic(fmt.Sprintf("processInboundContainer: scratch must be empty with non-zero cap; got len=%d cap=%d",
-			len(scratch), cap(scratch)))
-	}
-	if cap(scratch) < len(elemsContainer.elems) {
-		panic(fmt.Sprintf("processInboundContainer: scratch cap %d < elems %d",
-			cap(scratch), len(elemsContainer.elems)))
-	}
-
-	device := peer.device
-	defer device.PutInboundElementsContainer(elemsContainer)
-
-	// Wait for RoutineDecryption to finish filling the container. After
-	// Wait returns we have happens-before with that goroutine and are the
-	// sole owner of the container until Put hands it back to the pool.
-	elemsContainer.filling.Wait()
-	elems := elemsContainer.elems
-
-	validTailPacket := -1
-	dataPacketReceived := false
-	rxBytesLen := uint64(0)
-	for i, elem := range elems {
-		if elem.packet == nil {
-			// decryption failed
-			continue
-		}
-
-		if !elem.keypair.replayFilter.ValidateCounter(elem.counter, RejectAfterMessages) {
-			continue
-		}
-
-		validTailPacket = i
-		if peer.ReceivedWithKeypair(elem.keypair) {
-			peer.SetEndpointFromPacket(elem.endpoint)
-			peer.timersHandshakeComplete()
-			peer.SendPriorityMessage()
-			peer.SendStagedPackets()
-		}
-		if ep, ok := elem.endpoint.(conn.PeerAwareEndpoint); ok {
-			ep.FromPeer(peer.handshake.remoteStatic)
-		}
-		rxBytesLen += uint64(len(elem.packet) + MinMessageSize)
-
-		if len(elem.packet) == 0 {
-			device.log.Verbosef("%v - Receiving keepalive packet", peer)
-			continue
-		}
-		dataPacketReceived = true
-
-		switch elem.packet[0] >> 4 {
-		case 4:
-			if len(elem.packet) < ipv4.HeaderLen {
-				continue
-			}
-			field := elem.packet[IPv4offsetTotalLength : IPv4offsetTotalLength+2]
-			length := binary.BigEndian.Uint16(field)
-			if int(length) > len(elem.packet) || int(length) < ipv4.HeaderLen {
-				continue
-			}
-			elem.packet = elem.packet[:length]
-			src := elem.packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len]
-			srcAddr, _ := netip.AddrFromSlice(src)
-			if !peer.AllowedPeerSourceIP(srcAddr) {
-				device.log.Verbosef("IPv4 packet with disallowed source address from %v", peer)
+		elemsContainer.Lock()
+		validTailPacket := -1
+		dataPacketReceived := false
+		rxBytesLen := uint64(0)
+		for i, elem := range elemsContainer.elems {
+			if elem.packet == nil {
+				// decryption failed
 				continue
 			}
 
-		case 6:
-			if len(elem.packet) < ipv6.HeaderLen {
-				continue
-			}
-			field := elem.packet[IPv6offsetPayloadLength : IPv6offsetPayloadLength+2]
-			length := binary.BigEndian.Uint16(field)
-			length += ipv6.HeaderLen
-			if int(length) > len(elem.packet) {
-				continue
-			}
-			elem.packet = elem.packet[:length]
-			src := elem.packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len]
-			srcAddr, _ := netip.AddrFromSlice(src)
-			if !peer.AllowedPeerSourceIP(srcAddr) {
-				device.log.Verbosef("IPv6 packet with disallowed source address from %v", peer)
+			if !elem.keypair.replayFilter.ValidateCounter(elem.counter, RejectAfterMessages) {
 				continue
 			}
 
-		default:
-			device.log.Verbosef("Packet with invalid IP version from %v", peer)
-			continue
+			validTailPacket = i
+			if peer.ReceivedWithKeypair(elem.keypair) {
+				peer.SetEndpointFromPacket(elem.endpoint)
+				peer.timersHandshakeComplete()
+				peer.SendStagedPackets()
+			}
+			if ep, ok := elem.endpoint.(conn.PeerAwareEndpoint); ok {
+				ep.FromPeer(peer.handshake.remoteStatic)
+			}
+			rxBytesLen += uint64(len(elem.packet) + MinMessageSize)
+
+			if len(elem.packet) == 0 {
+				device.log.Verbosef("%v - Receiving keepalive packet", peer)
+				continue
+			}
+			dataPacketReceived = true
+
+			switch elem.packet[0] >> 4 {
+			case 4:
+				if len(elem.packet) < ipv4.HeaderLen {
+					continue
+				}
+				field := elem.packet[IPv4offsetTotalLength : IPv4offsetTotalLength+2]
+				length := binary.BigEndian.Uint16(field)
+				if int(length) > len(elem.packet) || int(length) < ipv4.HeaderLen {
+					continue
+				}
+				elem.packet = elem.packet[:length]
+				src := elem.packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len]
+				if device.allowedips.Lookup(src) != peer {
+					device.log.Verbosef("IPv4 packet with disallowed source address from %v", peer)
+					continue
+				}
+
+			case 6:
+				if len(elem.packet) < ipv6.HeaderLen {
+					continue
+				}
+				field := elem.packet[IPv6offsetPayloadLength : IPv6offsetPayloadLength+2]
+				length := binary.BigEndian.Uint16(field)
+				length += ipv6.HeaderLen
+				if int(length) > len(elem.packet) {
+					continue
+				}
+				elem.packet = elem.packet[:length]
+				src := elem.packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len]
+				if device.allowedips.Lookup(src) != peer {
+					device.log.Verbosef("IPv6 packet with disallowed source address from %v", peer)
+					continue
+				}
+
+			default:
+				device.log.Verbosef(
+					"Packet with invalid IP version from %v",
+					peer,
+				)
+				continue
+			}
+
+			bufs = append(
+				bufs,
+				elem.buffer[:MessageTransportOffsetContent+len(elem.packet)],
+			)
 		}
 
-		scratch = append(scratch, elem.buffer[:MessageTransportOffsetContent+len(elem.packet)])
-	}
-
-	peer.rxBytes.Add(rxBytesLen)
-	if validTailPacket >= 0 {
-		peer.SetEndpointFromPacket(elems[validTailPacket].endpoint)
-		peer.keepKeyFreshReceiving()
-		peer.timersAnyAuthenticatedPacketTraversal()
-		peer.timersAnyAuthenticatedPacketReceived()
-	}
-	if dataPacketReceived {
-		peer.timersDataReceived()
-	}
-	if len(scratch) > 0 {
-		_, err := device.tun.device.Write(scratch, MessageTransportOffsetContent)
-		if err != nil && !device.isClosed() {
-			device.log.Errorf("Failed to write packets to TUN device: %v", err)
+		peer.rxBytes.Add(rxBytesLen)
+		if validTailPacket >= 0 {
+			peer.SetEndpointFromPacket(elemsContainer.elems[validTailPacket].endpoint)
+			peer.keepKeyFreshReceiving()
+			peer.timersAnyAuthenticatedPacketTraversal()
+			peer.timersAnyAuthenticatedPacketReceived()
 		}
-	}
-	for _, elem := range elems {
-		device.PutMessageBuffer(elem.buffer)
-		device.PutInboundElement(elem)
+		if dataPacketReceived {
+			peer.timersDataReceived()
+		}
+
+		peer.rxBytes.Add(rxBytesLen)
+		if validTailPacket >= 0 {
+			peer.SetEndpointFromPacket(elemsContainer.elems[validTailPacket].endpoint)
+			peer.keepKeyFreshReceiving()
+			peer.timersAnyAuthenticatedPacketTraversal()
+			peer.timersAnyAuthenticatedPacketReceived()
+		}
+		if dataPacketReceived {
+			peer.timersDataReceived()
+		}
+		if len(bufs) > 0 {
+			_, err := device.tun.device.Write(bufs, MessageTransportOffsetContent)
+			if err != nil && !device.isClosed() {
+				device.log.Errorf("Failed to write packets to TUN device: %v", err)
+			}
+		}
+		for _, elem := range elemsContainer.elems {
+			device.PutMessageBuffer(elem.buffer)
+			device.PutInboundElement(elem)
+		}
+		bufs = bufs[:0]
+		device.PutInboundElementsContainer(elemsContainer)
 	}
 }
 

device/send.go

@@ -10,10 +10,8 @@ import (
 	"crypto/rand"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"math/big"
 	"net"
-	"net/netip"
 	"os"
 	"sync"
 	"time"
@@ -50,7 +48,7 @@ import (
  */
 
 type QueueOutboundElement struct {
-	buffer []byte // sing-allocated buffer holding the packet data
+	buffer *[MaxMessageSize]byte // slice holding the packet data
 	// packet is always a slice of "buffer". The starting offset in buffer
 	// is either:
 	//  a) MessageEncapsulatingTransportSize+MessageTransportHeaderSize (plaintext)
@@ -62,18 +60,13 @@ type QueueOutboundElement struct {
 }
 
 type QueueOutboundElementsContainer struct {
-	// filling is a one-shot barrier signaling encryption→send handoff.
-	// SendStagedPackets calls Add(1) before sending the container down
-	// the encryption and outbound queues; RoutineEncryption calls Done
-	// after encrypting; RoutineSequentialSender calls Wait before
-	// reading the encrypted packets.
-	filling sync.WaitGroup
-	elems   []*QueueOutboundElement
+	sync.Mutex
+	elems []*QueueOutboundElement
 }
 
 func (device *Device) NewOutboundElement() *QueueOutboundElement {
 	elem := device.GetOutboundElement()
-	elem.buffer = device.GetOutboundBuffer(MaxMessageSize)
+	elem.buffer = device.GetMessageBuffer()
 	elem.nonce = 0
 	// keypair and peer were cleared (if necessary) by clearPointers.
 	return elem
@@ -99,10 +92,9 @@ func (peer *Peer) SendKeepalive() {
 		elemsContainer.elems = append(elemsContainer.elems, elem)
 		select {
 		case peer.queue.staged <- elemsContainer:
-			peer.queuedOutboundPackets.Add(1)
 			peer.device.log.Verbosef("%v - Sending keepalive packet", peer)
 		default:
-			peer.device.PutOutboundBuffer(elem.buffer)
+			peer.device.PutMessageBuffer(elem.buffer)
 			peer.device.PutOutboundElement(elem)
 			peer.device.PutOutboundElementsContainer(elemsContainer)
 		}
@@ -110,70 +102,6 @@ func (peer *Peer) SendKeepalive() {
 	peer.SendStagedPackets()
 }
 
-// SendPriorityMessage invokes the [PeerPriorityMessageFunc] callback if one is
-// set, and queues the returned message for encryption and transmission if the
-// current keypair is valid.
-func (peer *Peer) SendPriorityMessage() {
-	f := peer.device.priorityMsgFn.Load()
-	if f == nil {
-		return
-	}
-	keypair := peer.keypairs.Current()
-	if keypair == nil || keypair.sendNonce.Load() >= RejectAfterMessages || time.Since(keypair.created) >= RejectAfterTime {
-		// SendStagedPackets initializes a handshake when the keypair is invalid,
-		// but we explicitly avoid that here. A priority message is only intended
-		// to flow around symmetric session establishment, but it should never
-		// trigger a new session. Reaching this branch due to nonce exhaustion
-		// or keypair expiration is highly unlikely considering where
-		// SendPriorityMessage is called (at current keypair establishment).
-		return
-	}
-
-	// get plaintext message to send
-	msg := (*f)(peer.handshake.remoteStatic)
-	if len(msg) == 0 {
-		return
-	}
-	if len(msg) > MaxPriorityMessageContentSize {
-		peer.device.log.Verbosef("%v - Failed to queue priority message due to size", peer)
-		return
-	}
-
-	// get pooled elements
-	elem := peer.device.NewOutboundElement()
-	elemsContainer := peer.device.GetOutboundElementsContainer()
-	elemsContainer.elems = append(elemsContainer.elems, elem)
-	packetQueued := false
-	defer func() {
-		if !packetQueued {
-			peer.device.PutOutboundBuffer(elem.buffer)
-			peer.device.PutOutboundElement(elem)
-			peer.device.PutOutboundElementsContainer(elemsContainer)
-		}
-	}()
-
-	// initialize outbound element
-	const offset = MessageEncapsulatingTransportSize + MessageTransportHeaderSize
-	n := copy(elem.buffer[offset:], msg)
-	elem.packet = elem.buffer[offset : offset+n]
-	elem.peer = peer
-	elem.nonce = keypair.sendNonce.Add(1) - 1
-	if elem.nonce >= RejectAfterMessages {
-		keypair.sendNonce.Store(RejectAfterMessages)
-		return
-	}
-	elem.keypair = keypair
-
-	// add to parallel and sequential queue
-	if peer.isRunning.Load() {
-		elemsContainer.filling.Add(1)
-		peer.queuedOutboundPackets.Add(1)
-		peer.queue.outbound.c <- elemsContainer
-		peer.device.queue.encryption.c <- elemsContainer
-		packetQueued = true
-	}
-}
-
 func (peer *Peer) SendHandshakeInitiation(isRetry bool) error {
 	if !isRetry {
 		peer.timers.handshakeAttempts.Store(0)
@@ -215,11 +143,6 @@ func (peer *Peer) SendHandshakeInitiation(isRetry bool) error {
 	jc := peer.device.junk.count
 	jmin := peer.device.junk.min
 	jmax := peer.device.junk.max
-	if jmax < jmin {
-		// UAPI validates jmin/jmax only individually; a swapped pair
-		// would panic rand.Int below with a non-positive bound.
-		jmin, jmax = jmax, jmin
-	}
 
 	for i := 0; i < jc; i++ {
 		nBig, _ := rand.Int(rand.Reader, big.NewInt(int64(jmax-jmin+1)))
@@ -375,7 +298,7 @@ func (device *Device) RoutineReadFromTUN() {
 	defer func() {
 		for _, elem := range elems {
 			if elem != nil {
-				device.PutOutboundBuffer(elem.buffer)
+				device.PutMessageBuffer(elem.buffer)
 				device.PutOutboundElement(elem)
 			}
 		}
@@ -399,17 +322,15 @@ func (device *Device) RoutineReadFromTUN() {
 				if len(elem.packet) < ipv4.HeaderLen {
 					continue
 				}
-				src := netip.AddrFrom4([4]byte(elem.packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len]))
-				dst := netip.AddrFrom4([4]byte(elem.packet[IPv4offsetDst : IPv4offsetDst+net.IPv4len]))
-				peer = device.allowedips.LookupFromPacket(src, dst, elem.packet)
+				dst := elem.packet[IPv4offsetDst : IPv4offsetDst+net.IPv4len]
+				peer = device.allowedips.Lookup(dst)
 
 			case 6:
 				if len(elem.packet) < ipv6.HeaderLen {
 					continue
 				}
-				src := netip.AddrFrom16([16]byte(elem.packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len]))
-				dst := netip.AddrFrom16([16]byte(elem.packet[IPv6offsetDst : IPv6offsetDst+net.IPv6len]))
-				peer = device.allowedips.LookupFromPacket(src, dst, elem.packet)
+				dst := elem.packet[IPv6offsetDst : IPv6offsetDst+net.IPv6len]
+				peer = device.allowedips.Lookup(dst)
 
 			default:
 				device.log.Verbosef("Received packet with unknown IP version")
@@ -434,7 +355,7 @@ func (device *Device) RoutineReadFromTUN() {
 				peer.SendStagedPackets()
 			} else {
 				for _, elem := range elemsForPeer.elems {
-					device.PutOutboundBuffer(elem.buffer)
+					device.PutMessageBuffer(elem.buffer)
 					device.PutOutboundElement(elem)
 				}
 				device.PutOutboundElementsContainer(elemsForPeer)
@@ -461,77 +382,12 @@ func (device *Device) RoutineReadFromTUN() {
 	}
 }
 
-// maxQueuedInputPackets bounds the staged+outbound backlog of a peer fed via
-// InputPacket/InputPackets. Injected packets beyond it are dropped before they
-// are copied into pooled message buffers, like a full qdisc: injection has no
-// flow control, and the queues are bounded in containers (up to a full batch
-// each), so without this cap a flood is buffered instead of dropped.
-const maxQueuedInputPackets = 2048
-
-func (device *Device) inputPacketPeer(destination []byte, packetSlices [][]byte) *Peer {
-	var src, dst netip.Addr
-	switch len(destination) {
-	case net.IPv4len:
-		dst = netip.AddrFrom4([4]byte(destination))
-		var srcBytes [net.IPv4len]byte
-		if !gatherPacketBytes(packetSlices, IPv4offsetSrc, srcBytes[:]) {
-			return nil
-		}
-		src = netip.AddrFrom4(srcBytes)
-	case net.IPv6len:
-		dst = netip.AddrFrom16([16]byte(destination))
-		var srcBytes [net.IPv6len]byte
-		if !gatherPacketBytes(packetSlices, IPv6offsetSrc, srcBytes[:]) {
-			return nil
-		}
-		src = netip.AddrFrom16(srcBytes)
-	default:
-		return nil
-	}
-	var ipPkt []byte
-	if len(packetSlices) == 1 {
-		ipPkt = packetSlices[0]
-	}
-	return device.allowedips.LookupFromPacket(src, dst, ipPkt)
-}
-
-func gatherPacketBytes(packetSlices [][]byte, offset int, destination []byte) bool {
-	for _, packetSlice := range packetSlices {
-		if offset >= len(packetSlice) {
-			offset -= len(packetSlice)
-			continue
-		}
-		n := copy(destination, packetSlice[offset:])
-		destination = destination[n:]
-		offset = 0
-		if len(destination) == 0 {
-			return true
-		}
-	}
-	return false
-}
-
 func (device *Device) InputPacket(destination []byte, packetSlices [][]byte) {
-	peer := device.inputPacketPeer(destination, packetSlices)
+	peer := device.allowedips.Lookup(destination)
 	if peer == nil {
 		return
 	}
-	if peer.queuedOutboundPackets.Load() >= maxQueuedInputPackets {
-		return
-	}
-	var totalLength int
-	for _, packetSlice := range packetSlices {
-		totalLength += len(packetSlice)
-	}
-	// paddings.transport (AWG s4) is prepended in-buffer by
-	// RoutineSequentialSender; reserve headroom for the shift.
-	allocLength := MessageEncapsulatingTransportSize + MessageTransportHeaderSize + totalLength + PaddingMultiple + chacha20poly1305.Overhead + device.paddings.transport
-	if allocLength > MaxMessageSize {
-		return
-	}
-	elem := device.GetOutboundElement()
-	elem.buffer = device.GetOutboundBuffer(allocLength)
-	elem.nonce = 0
+	elem := device.NewOutboundElement()
 	packet := elem.buffer[MessageEncapsulatingTransportSize+MessageTransportHeaderSize:]
 	var n int
 	for _, packetSlice := range packetSlices {
@@ -544,77 +400,13 @@ func (device *Device) InputPacket(destination []byte, packetSlices [][]byte) {
 		peer.StagePackets(elemsForPeer)
 		peer.SendStagedPackets()
 	} else {
-		device.PutOutboundBuffer(elem.buffer)
+		device.PutMessageBuffer(elem.buffer)
 		device.PutOutboundElement(elem)
 		device.PutOutboundElementsContainer(elemsForPeer)
 	}
 }
 
-type InputPacketRef struct {
-	Destination  []byte
-	PacketSlices [][]byte
-}
-
-func (device *Device) InputPackets(packets []*InputPacketRef) []*InputPacketRef {
-	var unmatched []*InputPacketRef
-	elemsByPeer := make(map[*Peer][]*QueueOutboundElementsContainer, len(packets))
-	for _, packetRef := range packets {
-		peer := device.inputPacketPeer(packetRef.Destination, packetRef.PacketSlices)
-		if peer == nil {
-			unmatched = append(unmatched, packetRef)
-			continue
-		}
-		if peer.queuedOutboundPackets.Load() >= maxQueuedInputPackets {
-			continue
-		}
-		var totalLength int
-		for _, packetSlice := range packetRef.PacketSlices {
-			totalLength += len(packetSlice)
-		}
-		// paddings.transport (AWG s4) is prepended in-buffer by
-		// RoutineSequentialSender; reserve headroom for the shift.
-		allocLength := MessageEncapsulatingTransportSize + MessageTransportHeaderSize + totalLength + PaddingMultiple + chacha20poly1305.Overhead + device.paddings.transport
-		if allocLength > MaxMessageSize {
-			continue
-		}
-		elem := device.GetOutboundElement()
-		elem.buffer = device.GetOutboundBuffer(allocLength)
-		elem.nonce = 0
-		packet := elem.buffer[MessageEncapsulatingTransportSize+MessageTransportHeaderSize:]
-		var n int
-		for _, packetSlice := range packetRef.PacketSlices {
-			n += copy(packet[n:], packetSlice)
-		}
-		elem.packet = packet[:n]
-		containers := elemsByPeer[peer]
-		if len(containers) == 0 || len(containers[len(containers)-1].elems) >= conn.IdealBatchSize {
-			containers = append(containers, device.GetOutboundElementsContainer())
-			elemsByPeer[peer] = containers
-		}
-		elemsForPeer := containers[len(containers)-1]
-		elemsForPeer.elems = append(elemsForPeer.elems, elem)
-	}
-	for peer, containers := range elemsByPeer {
-		if peer.isRunning.Load() {
-			for _, elemsForPeer := range containers {
-				peer.StagePackets(elemsForPeer)
-			}
-			peer.SendStagedPackets()
-		} else {
-			for _, elemsForPeer := range containers {
-				for _, elem := range elemsForPeer.elems {
-					device.PutOutboundBuffer(elem.buffer)
-					device.PutOutboundElement(elem)
-				}
-				device.PutOutboundElementsContainer(elemsForPeer)
-			}
-		}
-	}
-	return unmatched
-}
-
 func (peer *Peer) StagePackets(elems *QueueOutboundElementsContainer) {
-	peer.queuedOutboundPackets.Add(int32(len(elems.elems)))
 	for {
 		select {
 		case peer.queue.staged <- elems:
@@ -623,9 +415,8 @@ func (peer *Peer) StagePackets(elems *QueueOutboundElementsContainer) {
 		}
 		select {
 		case tooOld := <-peer.queue.staged:
-			peer.queuedOutboundPackets.Add(-int32(len(tooOld.elems)))
 			for _, elem := range tooOld.elems {
-				peer.device.PutOutboundBuffer(elem.buffer)
+				peer.device.PutMessageBuffer(elem.buffer)
 				peer.device.PutOutboundElement(elem)
 			}
 			peer.device.PutOutboundElementsContainer(tooOld)
@@ -668,11 +459,10 @@ top:
 
 				elem.keypair = keypair
 			}
+			elemsContainer.Lock()
 			elemsContainer.elems = elemsContainer.elems[:i]
 
 			if elemsContainerOOO != nil {
-				// Already counted at their original staging; StagePackets will count them again.
-				peer.queuedOutboundPackets.Add(-int32(len(elemsContainerOOO.elems)))
 				peer.StagePackets(elemsContainerOOO) // XXX: Out of order, but we can't front-load go chans
 			}
 
@@ -683,13 +473,11 @@ top:
 
 			// add to parallel and sequential queue
 			if peer.isRunning.Load() {
-				elemsContainer.filling.Add(1)
 				peer.queue.outbound.c <- elemsContainer
 				peer.device.queue.encryption.c <- elemsContainer
 			} else {
-				peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems)))
 				for _, elem := range elemsContainer.elems {
-					peer.device.PutOutboundBuffer(elem.buffer)
+					peer.device.PutMessageBuffer(elem.buffer)
 					peer.device.PutOutboundElement(elem)
 				}
 				peer.device.PutOutboundElementsContainer(elemsContainer)
@@ -708,9 +496,8 @@ func (peer *Peer) FlushStagedPackets() {
 	for {
 		select {
 		case elemsContainer := <-peer.queue.staged:
-			peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems)))
 			for _, elem := range elemsContainer.elems {
-				peer.device.PutOutboundBuffer(elem.buffer)
+				peer.device.PutMessageBuffer(elem.buffer)
 				peer.device.PutOutboundElement(elem)
 			}
 			peer.device.PutOutboundElementsContainer(elemsContainer)
@@ -750,7 +537,7 @@ func (device *Device) RoutineEncryption(id int) {
 	for elemsContainer := range device.queue.encryption.c {
 		for _, elem := range elemsContainer.elems {
 			// populate header fields
-			header := elem.buffer[MessageEncapsulatingTransportSize : MessageEncapsulatingTransportSize+MessageTransportHeaderSize]
+			header := elem.buffer[:MessageTransportHeaderSize]
 
 			fieldType := header[0:4]
 			fieldReceiver := header[4:8]
@@ -776,7 +563,7 @@ func (device *Device) RoutineEncryption(id int) {
 				nil,
 			)
 		}
-		elemsContainer.filling.Done()
+		elemsContainer.Unlock()
 	}
 }
 
@@ -788,100 +575,71 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) {
 	}()
 	device.log.Verbosef("%v - Routine: sequential sender - started", peer)
 
-	bufs := make([][]byte, 0, max(maxBatchSize, conn.IdealBatchSize))
+	bufs := make([][]byte, 0, maxBatchSize)
 
 	for elemsContainer := range peer.queue.outbound.c {
+		bufs = bufs[:0]
 		if elemsContainer == nil {
 			return
 		}
-		peer.processOutboundContainer(elemsContainer, bufs[:0])
-	}
-}
-
-// processOutboundContainer waits for the encryption routine to finish
-// filling elemsContainer, then sends the batch (or drops it, if the peer
-// has been stopped) and returns the container to the pool.
-//
-// scratch is a length-0 slice used to assemble the per-packet buffers
-// passed to SendBuffers; its backing array is reused across calls.
-func (peer *Peer) processOutboundContainer(elemsContainer *QueueOutboundElementsContainer, scratch [][]byte) {
-	// Invariants from RoutineSequentialSender; all should be unreachable.
-	if len(scratch) != 0 || cap(scratch) == 0 {
-		panic(fmt.Sprintf("processOutboundContainer: scratch must be empty with non-zero cap; got len=%d cap=%d",
-			len(scratch), cap(scratch)))
-	}
-	if cap(scratch) < len(elemsContainer.elems) {
-		panic(fmt.Sprintf("processOutboundContainer: scratch cap %d < elems %d",
-			cap(scratch), len(elemsContainer.elems)))
-	}
-
-	device := peer.device
-	defer device.PutOutboundElementsContainer(elemsContainer)
-
-	// Wait for RoutineEncryption to finish filling the container. After
-	// Wait returns we have happens-before with that goroutine and are the
-	// sole owner of the container until Put hands it back to the pool.
-	elemsContainer.filling.Wait()
-
-	if !peer.isRunning.Load() {
-		// peer has been stopped; return re-usable elems to the shared pool.
-		// This is an optimization only. It is possible for the peer to be stopped
-		// immediately after this check, in which case, elem will get processed.
-		// The timers and SendBuffers code are resilient to a few stragglers.
-		// TODO: rework peer shutdown order to ensure
-		// that we never accidentally keep timers alive longer than necessary.
-		peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems)))
+		if !peer.isRunning.Load() {
+			// peer has been stopped; return re-usable elems to the shared pool.
+			// This is an optimization only. It is possible for the peer to be stopped
+			// immediately after this check, in which case, elem will get processed.
+			// The timers and SendBuffers code are resilient to a few stragglers.
+			// TODO: rework peer shutdown order to ensure
+			// that we never accidentally keep timers alive longer than necessary.
+			elemsContainer.Lock()
+			for _, elem := range elemsContainer.elems {
+				device.PutMessageBuffer(elem.buffer)
+				device.PutOutboundElement(elem)
+			}
+			device.PutOutboundElementsContainer(elemsContainer)
+			continue
+		}
+		dataSent := false
+		elemsContainer.Lock()
 		for _, elem := range elemsContainer.elems {
-			device.PutOutboundBuffer(elem.buffer)
+			if len(elem.packet) != MessageKeepaliveSize {
+				dataSent = true
+			}
+			if padding := device.paddings.transport; padding > 0 {
+				// elem.packet is stored at the start of elem.buffer
+				// with zero padding
+				for i := len(elem.packet) - 1; i >= 0; i-- {
+					elem.buffer[i+padding] = elem.buffer[i]
+				}
+				rand.Read(elem.buffer[:padding])
+				elem.packet = elem.buffer[:padding+len(elem.packet)]
+			}
+			bufs = append(bufs, elem.packet)
+		}
+
+		peer.timersAnyAuthenticatedPacketTraversal()
+		peer.timersAnyAuthenticatedPacketSent()
+
+		err := peer.SendBuffers(bufs)
+		if dataSent {
+			peer.timersDataSent()
+		}
+
+		for _, elem := range elemsContainer.elems {
+			device.PutMessageBuffer(elem.buffer)
 			device.PutOutboundElement(elem)
 		}
-		return
-	}
-
-	dataSent := false
-	for _, elem := range elemsContainer.elems {
-		if len(elem.packet[MessageEncapsulatingTransportSize:]) != MessageKeepaliveSize {
-			dataSent = true
-		}
-		// lx:begin awg (SPEC 025 — AmneziaWG transport padding, S4)
-		// Prepend `transport` random bytes ahead of the transport header. The AWG
-		// path zeroes MessageEncapsulatingTransportSize (see noise-protocol.go), so
-		// elem.packet starts at buffer offset 0 and this shift is what creates the
-		// prefix; the buffer is allocated with PaddingMultiple headroom.
-		if padding := device.paddings.transport; padding > 0 {
-			for i := len(elem.packet) - 1; i >= 0; i-- {
-				elem.buffer[i+padding] = elem.buffer[i]
+		device.PutOutboundElementsContainer(elemsContainer)
+		if err != nil {
+			var errGSO conn.ErrUDPGSODisabled
+			if errors.As(err, &errGSO) {
+				device.log.Verbosef(err.Error())
+				err = errGSO.RetryErr
 			}
-			rand.Read(elem.buffer[:padding])
-			elem.packet = elem.buffer[:padding+len(elem.packet)]
 		}
-		// lx:end awg
-		scratch = append(scratch, elem.packet)
-	}
-
-	peer.timersAnyAuthenticatedPacketTraversal()
-	peer.timersAnyAuthenticatedPacketSent()
-
-	err := peer.SendBuffers(scratch)
-	if dataSent {
-		peer.timersDataSent()
-	}
-	peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems)))
-	for _, elem := range elemsContainer.elems {
-		device.PutOutboundBuffer(elem.buffer)
-		device.PutOutboundElement(elem)
-	}
-	if err != nil {
-		var errGSO conn.ErrUDPGSODisabled
-		if errors.As(err, &errGSO) {
-			device.log.Verbosef(err.Error())
-			err = errGSO.RetryErr
+		if err != nil {
+			device.log.Errorf("%v - Failed to send data packets: %v", peer, err)
+			continue
 		}
-	}
-	if err != nil {
-		device.log.Errorf("%v - Failed to send data packets: %v", peer, err)
-		return
-	}
 
-	peer.keepKeyFreshSending()
+		peer.keepKeyFreshSending()
+	}
 }

device/sticky_default.go

@@ -7,6 +7,6 @@ import (
 	"github.com/sagernet/wireguard-go/rwcancel"
 )
 
-func (device *Device) startRouteListener(_ conn.Bind) (*rwcancel.RWCancel, error) {
+func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, error) {
 	return nil, nil
 }

device/sticky_linux.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  *
  * This implements userspace semantics of "sticky sockets", modeled after
  * WireGuard's kernelspace implementation. This is more or less a straight port
@@ -9,7 +9,7 @@
  *
  * Currently there is no way to achieve this within the net package:
  * See e.g. https://github.com/golang/go/issues/17930
- * So this code remains platform dependent.
+ * So this code is remains platform dependent.
  */
 
 package device
@@ -46,7 +46,7 @@ func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, er
 	return netlinkCancel, nil
 }
 
-func (device *Device) routineRouteListener(_ conn.Bind, netlinkSock int, netlinkCancel *rwcancel.RWCancel) {
+func (device *Device) routineRouteListener(bind conn.Bind, netlinkSock int, netlinkCancel *rwcancel.RWCancel) {
 	type peerEndpointPtr struct {
 		peer     *Peer
 		endpoint *conn.Endpoint

device/timers.go

@@ -98,26 +98,10 @@ func expiredRetransmitHandshake(peer *Peer) {
 		if peer.timersActive() && !peer.timers.zeroKeyMaterial.IsPending() {
 			peer.timers.zeroKeyMaterial.Mod(RejectAfterTime * 3)
 		}
-		peer.noteSessionHandshakeStopped()
-
-		/* lx: SPEC 041 — the exhausted cycle just proved the current socket's
-		 * 5-tuple dead (90s of initiations, zero replies). Rebind once and
-		 * re-initiate, so a stale NAT mapping / poisoned DPI flow entry cannot
-		 * pin this peer to a dead socket until a manual reconnect. Runs after
-		 * the session-state notification so a consumer sees "handshake stopped"
-		 * before the socket is recreated. */
-		peer.device.handleHandshakeGiveUp(peer)
 	} else {
 		peer.timers.handshakeAttempts.Add(1)
 		peer.device.log.Verbosef("%s - Handshake did not complete after %d seconds, retrying (try %d)", peer, int(RekeyTimeout.Seconds()), peer.timers.handshakeAttempts.Load()+1)
 
-		/* lx: SPEC 041 v2 — early self-heal: >=3 unanswered initiations against
-		 * a provably dead session (no live keypair, or last handshake older
-		 * than RejectAfterTime) prove the 5-tuple dead without waiting out the
-		 * full cycle. Rebind now (debounced with the give-up trigger below);
-		 * the retry cycle itself continues untouched. */
-		peer.device.maybeEarlyGiveUpRebind(peer)
-
 		/* We clear the endpoint address src address, in case this is the cause of trouble. */
 		peer.markEndpointSrcForClearing()
 
@@ -145,24 +129,6 @@ func expiredNewHandshake(peer *Peer) {
 func expiredZeroKeyMaterial(peer *Peer) {
 	peer.device.log.Verbosef("%s - Removing all keys, since we haven't received a new one in %d seconds", peer, int((RejectAfterTime * 3).Seconds()))
 	peer.ZeroAndFlushAll()
-	if peer.deleteOnIdle {
-		peer.device.log.Verbosef("%s - Removing idle lazy peer", peer)
-		// Remove the peer from the device in a new goroutine as we're currently
-		// holding timer locks which RemovePeer also needs. This is TOCTOU, but
-		// acceptable since the worst case is we remove the peer and the lazy
-		// peerfunc created it again after. We might lose some packets.
-		go peer.device.RemovePeer(peer.handshake.remoteStatic)
-	}
-}
-
-func expiredSession(peer *Peer) {
-	peer.sessionState.Lock()
-	defer peer.sessionState.Unlock()
-	if peer.sessionState.sessionExpires.IsZero() || time.Now().Before(peer.sessionState.sessionExpires) {
-		return
-	}
-	peer.device.log.Verbosef("%s - Session expired after %d seconds", peer, int(RejectAfterTime.Seconds()))
-	peer.noteSessionStateLocked(PeerSessionExpired)
 }
 
 func expiredPersistentKeepalive(peer *Peer) {
@@ -208,7 +174,6 @@ func (peer *Peer) timersHandshakeInitiated() {
 	if peer.timersActive() {
 		peer.timers.retransmitHandshake.Mod(RekeyTimeout + time.Millisecond*time.Duration(fastrandn(RekeyTimeoutJitterMaxMs)))
 	}
-	peer.noteSessionHandshakeStarted()
 }
 
 /* Should be called after a handshake response message is received and processed or when getting key confirmation via the first data message. */
@@ -224,14 +189,7 @@ func (peer *Peer) timersHandshakeComplete() {
 /* Should be called after an ephemeral key is created, which is before sending a handshake response or after receiving a handshake response. */
 func (peer *Peer) timersSessionDerived() {
 	if peer.timersActive() {
-		peer.sessionState.Lock()
-		peer.sessionState.sessionExpires = time.Now().Add(RejectAfterTime)
-		peer.noteSessionStateLocked(PeerSessionEstablished)
-		peer.sessionState.Unlock()
-		peer.timers.sessionExpired.Mod(RejectAfterTime)
 		peer.timers.zeroKeyMaterial.Mod(RejectAfterTime * 3)
-	} else {
-		peer.noteSessionState(PeerSessionEstablished)
 	}
 }
 
@@ -247,7 +205,6 @@ func (peer *Peer) timersInit() {
 	peer.timers.retransmitHandshake = peer.NewTimer(expiredRetransmitHandshake)
 	peer.timers.sendKeepalive = peer.NewTimer(expiredSendKeepalive)
 	peer.timers.newHandshake = peer.NewTimer(expiredNewHandshake)
-	peer.timers.sessionExpired = peer.NewTimer(expiredSession)
 	peer.timers.zeroKeyMaterial = peer.NewTimer(expiredZeroKeyMaterial)
 	peer.timers.persistentKeepalive = peer.NewTimer(expiredPersistentKeepalive)
 }
@@ -262,7 +219,6 @@ func (peer *Peer) timersStop() {
 	peer.timers.retransmitHandshake.DelSync()
 	peer.timers.sendKeepalive.DelSync()
 	peer.timers.newHandshake.DelSync()
-	peer.timers.sessionExpired.DelSync()
 	peer.timers.zeroKeyMaterial.DelSync()
 	peer.timers.persistentKeepalive.DelSync()
 }

device/transport_padding_test.go

@@ -1,361 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Regression test for the AWG transport-padding (uapi "s4") out-of-bounds
- * crash: RoutineSequentialSender shifts elem.packet right by
- * device.paddings.transport bytes inside elem.buffer to prepend a random
- * padding prefix, but the injection paths (InputPacket/InputPackets)
- * allocated elem.buffer tightly, without headroom for that shift:
- *
- *   panic: runtime error: index out of range [123] with length 76
- *   device.(*Peer).RoutineSequentialSender
- *
- * (payload 28 bytes -> allocLength 76, sealed packet 64, s4=60 -> 63+60=123).
- *
- * The tests spin up two real Devices wired together through an in-memory
- * conn.Bind (Go channels) and an in-memory tun.Device, configure s4=60 on
- * both sides, and pass a small IPv4 packet end to end via both outbound
- * paths: Device.InputPacket (the crashing path) and the regular tun read
- * loop (in-place shift path).
- */
-
-package device
-
-import (
-	"bytes"
-	"context"
-	"encoding/binary"
-	"encoding/hex"
-	"fmt"
-	"net"
-	"net/netip"
-	"os"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/sagernet/wireguard-go/conn"
-	"github.com/sagernet/wireguard-go/tun"
-
-	"golang.org/x/net/ipv4"
-)
-
-const testTransportPadding = 60 // uapi s4, matches the on-device crash
-
-// ---------------------------------------------------------------------------
-// In-memory conn.Bind over Go channels (minimal bindtest.ChannelBind clone).
-// ---------------------------------------------------------------------------
-
-type chanEndpoint uint16
-
-func (e chanEndpoint) ClearSrc()           {}
-func (e chanEndpoint) SrcToString() string { return "" }
-func (e chanEndpoint) DstToString() string { return fmt.Sprintf("127.0.0.1:%d", uint16(e)) }
-func (e chanEndpoint) DstToBytes() []byte  { return []byte{byte(e), byte(e >> 8)} }
-func (e chanEndpoint) DstIP() netip.Addr   { return netip.AddrFrom4([4]byte{127, 0, 0, 1}) }
-func (e chanEndpoint) SrcIP() netip.Addr   { return netip.Addr{} }
-
-type chanBind struct {
-	rx, tx chan []byte
-	source chanEndpoint // "port" this bind listens on
-	target chanEndpoint // endpoint of the opposite bind
-
-	mu          sync.Mutex
-	closeSignal chan struct{} // recreated on every Open (BindUpdate closes+reopens)
-}
-
-// newChanBindPair returns two Binds whose Send/Receive are cross-wired.
-func newChanBindPair() (*chanBind, *chanBind) {
-	aToB := make(chan []byte, 1024)
-	bToA := make(chan []byte, 1024)
-	a := &chanBind{rx: bToA, tx: aToB, source: 1, target: 2}
-	b := &chanBind{rx: aToB, tx: bToA, source: 2, target: 1}
-	return a, b
-}
-
-func (b *chanBind) currentCloseSignal() chan struct{} {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	return b.closeSignal
-}
-
-func (b *chanBind) Open(port uint16) ([]conn.ReceiveFunc, uint16, error) {
-	b.mu.Lock()
-	b.closeSignal = make(chan struct{})
-	closeSignal := b.closeSignal
-	b.mu.Unlock()
-	fn := func(packets [][]byte, sizes []int, eps []conn.Endpoint) (int, error) {
-		select {
-		case <-closeSignal:
-			// Must be net.ErrClosed: RoutineReceiveIncoming treats anything
-			// else as a transient error and death-spirals before exiting.
-			return 0, net.ErrClosed
-		case pkt, ok := <-b.rx:
-			if !ok {
-				return 0, net.ErrClosed
-			}
-			sizes[0] = copy(packets[0], pkt)
-			eps[0] = b.target
-			return 1, nil
-		}
-	}
-	return []conn.ReceiveFunc{fn}, uint16(b.source), nil
-}
-
-func (b *chanBind) Close() error {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	if b.closeSignal != nil {
-		select {
-		case <-b.closeSignal:
-		default:
-			close(b.closeSignal)
-		}
-	}
-	return nil
-}
-
-func (b *chanBind) SetMark(mark uint32) error { return nil }
-
-func (b *chanBind) Send(bufs [][]byte, ep conn.Endpoint, offset int) error {
-	closeSignal := b.currentCloseSignal()
-	if closeSignal == nil {
-		return net.ErrClosed
-	}
-	for _, buf := range bufs {
-		pkt := make([]byte, len(buf)-offset)
-		copy(pkt, buf[offset:])
-		select {
-		case <-closeSignal:
-			return net.ErrClosed
-		case b.tx <- pkt:
-		}
-	}
-	return nil
-}
-
-func (b *chanBind) ParseEndpoint(s string) (conn.Endpoint, error) { return b.target, nil }
-
-func (b *chanBind) BatchSize() int { return 1 }
-
-func (b *chanBind) SetReservedForEndpoint(destination netip.AddrPort, reserved [3]byte) {}
-
-// ---------------------------------------------------------------------------
-// In-memory tun.Device over Go channels (minimal tuntest.ChannelTUN clone).
-// ---------------------------------------------------------------------------
-
-type chanTun struct {
-	toDevice   chan []byte // packets the device Reads (outbound plaintext)
-	fromDevice chan []byte // packets the device Writes (inbound plaintext)
-	events     chan tun.Event
-	closed     chan struct{}
-	closeOnce  sync.Once
-}
-
-func newChanTun() *chanTun {
-	return &chanTun{
-		toDevice:   make(chan []byte, 1024),
-		fromDevice: make(chan []byte, 1024),
-		events:     make(chan tun.Event, 4),
-		closed:     make(chan struct{}),
-	}
-}
-
-func (t *chanTun) File() *os.File { return nil }
-
-func (t *chanTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
-	select {
-	case <-t.closed:
-		return 0, os.ErrClosed
-	case pkt, ok := <-t.toDevice:
-		if !ok {
-			return 0, os.ErrClosed
-		}
-		sizes[0] = copy(bufs[0][offset:], pkt)
-		return 1, nil
-	}
-}
-
-func (t *chanTun) Write(bufs [][]byte, offset int) (int, error) {
-	for _, buf := range bufs {
-		pkt := make([]byte, len(buf)-offset)
-		copy(pkt, buf[offset:])
-		select {
-		case <-t.closed:
-			return 0, os.ErrClosed
-		case t.fromDevice <- pkt:
-		}
-	}
-	return len(bufs), nil
-}
-
-func (t *chanTun) MTU() (int, error)        { return DefaultMTU, nil }
-func (t *chanTun) Name() (string, error)    { return "chantun", nil }
-func (t *chanTun) Events() <-chan tun.Event { return t.events }
-func (t *chanTun) BatchSize() int           { return 1 }
-
-func (t *chanTun) Close() error {
-	t.closeOnce.Do(func() {
-		close(t.closed)
-		close(t.events)
-	})
-	return nil
-}
-
-// ---------------------------------------------------------------------------
-// Test scaffolding.
-// ---------------------------------------------------------------------------
-
-var (
-	testIPA = netip.AddrFrom4([4]byte{10, 0, 0, 1})
-	testIPB = netip.AddrFrom4([4]byte{10, 0, 0, 2})
-)
-
-// buildIPv4Packet builds a minimal, routable IPv4/UDP packet whose header
-// fields satisfy the receive-side validation in RoutineSequentialReceiver
-// (version, total-length field, allowed source address).
-func buildIPv4Packet(src, dst netip.Addr, payloadLen int) []byte {
-	total := ipv4.HeaderLen + payloadLen
-	pkt := make([]byte, total)
-	pkt[0] = 0x45 // version 4, IHL 5
-	binary.BigEndian.PutUint16(pkt[IPv4offsetTotalLength:IPv4offsetTotalLength+2], uint16(total))
-	pkt[8] = 64 // TTL
-	pkt[9] = 17 // protocol: UDP
-	copy(pkt[IPv4offsetSrc:], src.AsSlice())
-	copy(pkt[IPv4offsetDst:], dst.AsSlice())
-	for i := ipv4.HeaderLen; i < total; i++ {
-		pkt[i] = byte(i) // deterministic payload
-	}
-	return pkt
-}
-
-type paddedPair struct {
-	devA, devB *Device
-	tunA, tunB *chanTun
-}
-
-// newPaddedDevicePair builds two Up()'d devices peered with each other over
-// the channel bind, both configured with s4 (transport padding) enabled.
-func newPaddedDevicePair(t *testing.T) *paddedPair {
-	t.Helper()
-
-	skA, err := newPrivateKey()
-	if err != nil {
-		t.Fatalf("newPrivateKey A: %v", err)
-	}
-	skB, err := newPrivateKey()
-	if err != nil {
-		t.Fatalf("newPrivateKey B: %v", err)
-	}
-	pkA := skA.publicKey()
-	pkB := skB.publicKey()
-
-	bindA, bindB := newChanBindPair()
-	tunA := newChanTun()
-	tunB := newChanTun()
-
-	devA := NewDevice(context.Background(), tunA, bindA, NewLogger(LogLevelError, "devA: "), 1)
-	devB := NewDevice(context.Background(), tunB, bindB, NewLogger(LogLevelError, "devB: "), 1)
-	t.Cleanup(devA.Close)
-	t.Cleanup(devB.Close)
-
-	cfgA := fmt.Sprintf(
-		"private_key=%s\ns4=%d\nreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:2\nallowed_ip=%s/32\n",
-		hex.EncodeToString(skA[:]), testTransportPadding, hex.EncodeToString(pkB[:]), testIPB)
-	cfgB := fmt.Sprintf(
-		"private_key=%s\ns4=%d\nreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:1\nallowed_ip=%s/32\n",
-		hex.EncodeToString(skB[:]), testTransportPadding, hex.EncodeToString(pkA[:]), testIPA)
-
-	if err := devA.IpcSet(cfgA); err != nil {
-		t.Fatalf("IpcSet A: %v", err)
-	}
-	if err := devB.IpcSet(cfgB); err != nil {
-		t.Fatalf("IpcSet B: %v", err)
-	}
-	if devA.paddings.transport != testTransportPadding {
-		t.Fatalf("s4 not applied: paddings.transport = %d", devA.paddings.transport)
-	}
-
-	if err := devA.Up(); err != nil {
-		t.Fatalf("Up A: %v", err)
-	}
-	if err := devB.Up(); err != nil {
-		t.Fatalf("Up B: %v", err)
-	}
-
-	return &paddedPair{devA: devA, devB: devB, tunA: tunA, tunB: tunB}
-}
-
-// awaitPacket waits for want to arrive on the receiving tun, periodically
-// re-sending via resend (injection has no delivery guarantee before the
-// handshake completes).
-func awaitPacket(t *testing.T, from *chanTun, want []byte, resend func()) {
-	t.Helper()
-	deadline := time.After(20 * time.Second)
-	retry := time.NewTicker(1 * time.Second)
-	defer retry.Stop()
-	for {
-		select {
-		case got := <-from.fromDevice:
-			if bytes.Equal(got, want) {
-				return
-			}
-			t.Logf("ignoring unexpected packet, len=%d", len(got))
-		case <-retry.C:
-			resend()
-		case <-deadline:
-			t.Fatal("timed out waiting for packet on peer tun")
-		}
-	}
-}
-
-// ---------------------------------------------------------------------------
-// Tests.
-// ---------------------------------------------------------------------------
-
-// TestTransportPaddingInputPacket exercises the exact crash path: an injected
-// packet (Device.InputPacket) whose buffer was allocated by payload size.
-// With s4=60 and a 28-byte IPv4 packet the pre-fix buffer was 76 bytes and
-// the padding shift indexed [123] -> index out of range.
-func TestTransportPaddingInputPacket(t *testing.T) {
-	pair := newPaddedDevicePair(t)
-
-	// 20-byte header + 8-byte payload = 28 bytes, the on-device crash size.
-	pkt := buildIPv4Packet(testIPA, testIPB, 8)
-	dst := testIPB.AsSlice()
-
-	send := func() { pair.devA.InputPacket(dst, [][]byte{pkt}) }
-	send()
-	awaitPacket(t, pair.tunB, pkt, send)
-}
-
-// TestTransportPaddingInputPackets covers the batched injection path
-// (Device.InputPackets), which had the same tight allocation.
-func TestTransportPaddingInputPackets(t *testing.T) {
-	pair := newPaddedDevicePair(t)
-
-	pkt := buildIPv4Packet(testIPA, testIPB, 8)
-	refs := []*InputPacketRef{{
-		Destination:  testIPB.AsSlice(),
-		PacketSlices: [][]byte{pkt[:12], pkt[12:]}, // multi-slice on purpose
-	}}
-
-	send := func() {
-		if unmatched := pair.devA.InputPackets(refs); len(unmatched) != 0 {
-			t.Fatalf("InputPackets returned %d unmatched refs", len(unmatched))
-		}
-	}
-	send()
-	awaitPacket(t, pair.tunB, pkt, send)
-}
-
-// TestTransportPaddingTunPath covers the regular outbound path (tun read
-// loop), whose MaxMessageSize buffers take the in-place shift branch.
-func TestTransportPaddingTunPath(t *testing.T) {
-	pair := newPaddedDevicePair(t)
-
-	pkt := buildIPv4Packet(testIPA, testIPB, 8)
-
-	send := func() { pair.tunA.toDevice <- pkt }
-	send()
-	awaitPacket(t, pair.tunB, pkt, send)
-}

device/tun.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package device

go.mod

@@ -1,11 +1,11 @@
 module github.com/sagernet/wireguard-go
 
-go 1.25
+go 1.24
 
 require (
 	github.com/sagernet/sing v0.7.10
 	golang.org/x/crypto v0.13.0
 	golang.org/x/net v0.15.0
-	golang.org/x/sys v0.21.0
+	golang.org/x/sys v0.12.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 )

go.sum

@@ -4,7 +4,7 @@ golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
 golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=

ipc/uapi_fake.go

@@ -0,0 +1,17 @@
+//go:build wasm || plan9 || aix || solaris || illumos
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
+ */
+
+package ipc
+
+// Made up sentinel error codes for {js,wasip1}/wasm, and plan9.
+const (
+	IpcErrorIO        = 1
+	IpcErrorInvalid   = 2
+	IpcErrorPortInUse = 3
+	IpcErrorUnknown   = 4
+	IpcErrorProtocol  = 5
+)

ipc/uapi_linux.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package ipc

ipc/uapi_tamago.go

@@ -1,3 +1,5 @@
+//go:build tamago
+
 /* SPDX-License-Identifier: MIT
  *
  * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
@@ -5,7 +7,7 @@
 
 package ipc
 
-// Made up sentinel error codes for {js,wasip1}/wasm.
+// Made up sentinel error codes for tamago platform.
 const (
 	IpcErrorIO        = 1
 	IpcErrorInvalid   = 2

ipc/uapi_unix.go

@@ -26,7 +26,7 @@ const (
 
 // socketDirectory is variable because it is modified by a linker
 // flag in wireguard-android.
-var socketDirectory = "/var/run/wireguard"
+var socketDirectory = "/var/run/amneziawg"
 
 func sockPath(iface string) string {
 	return fmt.Sprintf("%s/%s.sock", socketDirectory, iface)

ipc/uapi_windows.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package ipc

module_rename.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import argparse
+import fileinput
+
+
+PKG_ORIGINAL = "github.com/tailscale/wireguard-go"
+PKG_NEW = "github.com/sagernet/wireguard-go"
+
+EXTENSIONS = [".go", ".md", ".mod", ".sh"]
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-r", "--reverse", action="store_true")
+args = parser.parse_args()
+
+
+def replace_line(line):
+    if args.reverse:
+        return line.replace(PKG_NEW, PKG_ORIGINAL)
+    return line.replace(PKG_ORIGINAL, PKG_NEW)
+
+
+for dirpath, dirnames, filenames in os.walk("."):
+    # Skip hidden directories like .git
+    dirnames[:] = [d for d in dirnames if not d[0] == "."]
+    filenames = [f for f in filenames if os.path.splitext(f)[1] in EXTENSIONS]
+    for filename in filenames:
+        file_path = os.path.join(dirpath, filename)
+        with fileinput.FileInput(file_path, inplace=True) as file:
+            for line in file:
+                print(replace_line(line), end="")

reformat.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-set -e -o pipefail
-
-GO_FILES=$(find . -name "*.go" | grep -v .git)
-
-gofumpt -l -w $GO_FILES
-gofmt -l -w $GO_FILES
-gci write $GO_FILES

rename-module.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-set -e -o pipefail
-
-OLD_MODULE_NAME="github.com/tailscale/wireguard-go"
-NEW_MODULE_NAME="github.com/sagernet/wireguard-go"
-
-rules=$(cat <<EOF
-id: replace-module
-language: go
-rule:
-  kind: import_spec
-  pattern: \$OLD_IMPORT
-constraints:
-  OLD_IMPORT:
-    has:
-      field: path
-      regex: ^"$OLD_MODULE_NAME
-transform:
-  NEW_IMPORT:
-    replace:
-      source: \$OLD_IMPORT
-      replace: $OLD_MODULE_NAME(?<PATH>.*)
-      by: $NEW_MODULE_NAME\$PATH
-fix: \$NEW_IMPORT
-EOF
-)
-
-sg scan --inline-rules "$rules" -U
-
-sed -i "s|module $OLD_MODULE_NAME|module $NEW_MODULE_NAME|" go.mod
-
-go mod tidy
-
-./reformat.sh
-
-git commit -m "Rename module" -a

rwcancel/rwcancel.go

@@ -1,4 +1,4 @@
-//go:build !windows && !wasm
+//go:build !windows && !wasm && !plan9 && !tamago
 
 /* SPDX-License-Identifier: MIT
  *

rwcancel/rwcancel_stub.go

@@ -1,4 +1,4 @@
-//go:build windows || wasm
+//go:build windows || wasm || plan9 || tamago
 
 // SPDX-License-Identifier: MIT
 

tun/checksum.go

@@ -3,111 +3,710 @@ package tun
 import (
 	"encoding/binary"
 	"math/bits"
+	"strconv"
+
+	"golang.org/x/sys/cpu"
 )
 
-// TODO: Explore SIMD and/or other assembly optimizations.
-func checksumNoFold(b []byte, initial uint64) uint64 {
-	tmp := make([]byte, 8)
-	binary.NativeEndian.PutUint64(tmp, initial)
-	ac := binary.BigEndian.Uint64(tmp)
+// checksumGeneric64 is a reference implementation of checksum using 64 bit
+// arithmetic for use in testing or when an architecture-specific implementation
+// is not available.
+func checksumGeneric64(b []byte, initial uint16) uint16 {
+	var ac uint64
 	var carry uint64
 
+	if cpu.IsBigEndian {
+		ac = uint64(initial)
+	} else {
+		ac = uint64(bits.ReverseBytes16(initial))
+	}
+
 	for len(b) >= 128 {
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[16:24]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[24:32]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[32:40]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[40:48]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[48:56]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[56:64]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[64:72]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[72:80]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[80:88]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[88:96]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[96:104]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[104:112]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[112:120]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[120:128]), carry)
-		ac += carry
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[32:40]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[40:48]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[48:56]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[56:64]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[64:72]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[72:80]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[80:88]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[88:96]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[96:104]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[104:112]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[112:120]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[120:128]), carry)
+		} else {
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[32:40]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[40:48]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[48:56]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[56:64]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[64:72]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[72:80]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[80:88]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[88:96]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[96:104]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[104:112]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[112:120]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[120:128]), carry)
+		}
 		b = b[128:]
 	}
 	if len(b) >= 64 {
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[16:24]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[24:32]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[32:40]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[40:48]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[48:56]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[56:64]), carry)
-		ac += carry
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[32:40]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[40:48]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[48:56]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[56:64]), carry)
+		} else {
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[32:40]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[40:48]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[48:56]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[56:64]), carry)
+		}
 		b = b[64:]
 	}
 	if len(b) >= 32 {
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[16:24]), carry)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[24:32]), carry)
-		ac += carry
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry)
+		} else {
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry)
+		}
 		b = b[32:]
 	}
 	if len(b) >= 16 {
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0)
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry)
-		ac += carry
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry)
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry)
+		} else {
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry)
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry)
+		}
 		b = b[16:]
 	}
 	if len(b) >= 8 {
-		ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0)
-		ac += carry
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b), carry)
+		} else {
+			ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b), carry)
+		}
 		b = b[8:]
 	}
 	if len(b) >= 4 {
-		ac, carry = bits.Add64(ac, uint64(binary.NativeEndian.Uint32(b[:4])), 0)
-		ac += carry
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add64(ac, uint64(binary.BigEndian.Uint32(b)), carry)
+		} else {
+			ac, carry = bits.Add64(ac, uint64(binary.LittleEndian.Uint32(b)), carry)
+		}
 		b = b[4:]
 	}
 	if len(b) >= 2 {
-		ac, carry = bits.Add64(ac, uint64(binary.NativeEndian.Uint16(b[:2])), 0)
-		ac += carry
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add64(ac, uint64(binary.BigEndian.Uint16(b)), carry)
+		} else {
+			ac, carry = bits.Add64(ac, uint64(binary.LittleEndian.Uint16(b)), carry)
+		}
 		b = b[2:]
 	}
-	if len(b) == 1 {
-		tmp := binary.NativeEndian.Uint16([]byte{b[0], 0})
-		ac, carry = bits.Add64(ac, uint64(tmp), 0)
-		ac += carry
+	if len(b) >= 1 {
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add64(ac, uint64(b[0])<<8, carry)
+		} else {
+			ac, carry = bits.Add64(ac, uint64(b[0]), carry)
+		}
 	}
 
-	binary.NativeEndian.PutUint64(tmp, ac)
-	return binary.BigEndian.Uint64(tmp)
+	folded := ipChecksumFold64(ac, carry)
+	if !cpu.IsBigEndian {
+		folded = bits.ReverseBytes16(folded)
+	}
+	return folded
 }
 
-func checksum(b []byte, initial uint64) uint16 {
-	ac := checksumNoFold(b, initial)
-	ac = (ac >> 16) + (ac & 0xffff)
-	ac = (ac >> 16) + (ac & 0xffff)
-	ac = (ac >> 16) + (ac & 0xffff)
-	ac = (ac >> 16) + (ac & 0xffff)
-	return uint16(ac)
+// checksumGeneric32 is a reference implementation of checksum using 32 bit
+// arithmetic for use in testing or when an architecture-specific implementation
+// is not available.
+func checksumGeneric32(b []byte, initial uint16) uint16 {
+	var ac uint32
+	var carry uint32
+
+	if cpu.IsBigEndian {
+		ac = uint32(initial)
+	} else {
+		ac = uint32(bits.ReverseBytes16(initial))
+	}
+
+	for len(b) >= 64 {
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:8]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[16:20]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[20:24]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[24:28]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[28:32]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[32:36]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[36:40]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[40:44]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[44:48]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[48:52]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[52:56]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[56:60]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[60:64]), carry)
+		} else {
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:8]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[16:20]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[20:24]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[24:28]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[28:32]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[32:36]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[36:40]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[40:44]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[44:48]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[48:52]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[52:56]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[56:60]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[60:64]), carry)
+		}
+		b = b[64:]
+	}
+	if len(b) >= 32 {
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[16:20]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[20:24]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[24:28]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[28:32]), carry)
+		} else {
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[16:20]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[20:24]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[24:28]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[28:32]), carry)
+		}
+		b = b[32:]
+	}
+	if len(b) >= 16 {
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry)
+		} else {
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry)
+		}
+		b = b[16:]
+	}
+	if len(b) >= 8 {
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry)
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry)
+		} else {
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry)
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry)
+		}
+		b = b[8:]
+	}
+	if len(b) >= 4 {
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b), carry)
+		} else {
+			ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b), carry)
+		}
+		b = b[4:]
+	}
+	if len(b) >= 2 {
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add32(ac, uint32(binary.BigEndian.Uint16(b)), carry)
+		} else {
+			ac, carry = bits.Add32(ac, uint32(binary.LittleEndian.Uint16(b)), carry)
+		}
+		b = b[2:]
+	}
+	if len(b) >= 1 {
+		if cpu.IsBigEndian {
+			ac, carry = bits.Add32(ac, uint32(b[0])<<8, carry)
+		} else {
+			ac, carry = bits.Add32(ac, uint32(b[0]), carry)
+		}
+	}
+
+	folded := ipChecksumFold32(ac, carry)
+	if !cpu.IsBigEndian {
+		folded = bits.ReverseBytes16(folded)
+	}
+	return folded
 }
 
-// Checksum computes an IP checksum starting with the provided initial value.
-func Checksum(data []byte, initial uint16) uint16 {
-	return checksum(data, uint64(initial))
+// checksumGeneric32Alternate is an alternate reference implementation of
+// checksum using 32 bit arithmetic for use in testing or when an
+// architecture-specific implementation is not available.
+func checksumGeneric32Alternate(b []byte, initial uint16) uint16 {
+	var ac uint32
+
+	if cpu.IsBigEndian {
+		ac = uint32(initial)
+	} else {
+		ac = uint32(bits.ReverseBytes16(initial))
+	}
+
+	for len(b) >= 64 {
+		if cpu.IsBigEndian {
+			ac += uint32(binary.BigEndian.Uint16(b[:2]))
+			ac += uint32(binary.BigEndian.Uint16(b[2:4]))
+			ac += uint32(binary.BigEndian.Uint16(b[4:6]))
+			ac += uint32(binary.BigEndian.Uint16(b[6:8]))
+			ac += uint32(binary.BigEndian.Uint16(b[8:10]))
+			ac += uint32(binary.BigEndian.Uint16(b[10:12]))
+			ac += uint32(binary.BigEndian.Uint16(b[12:14]))
+			ac += uint32(binary.BigEndian.Uint16(b[14:16]))
+			ac += uint32(binary.BigEndian.Uint16(b[16:18]))
+			ac += uint32(binary.BigEndian.Uint16(b[18:20]))
+			ac += uint32(binary.BigEndian.Uint16(b[20:22]))
+			ac += uint32(binary.BigEndian.Uint16(b[22:24]))
+			ac += uint32(binary.BigEndian.Uint16(b[24:26]))
+			ac += uint32(binary.BigEndian.Uint16(b[26:28]))
+			ac += uint32(binary.BigEndian.Uint16(b[28:30]))
+			ac += uint32(binary.BigEndian.Uint16(b[30:32]))
+			ac += uint32(binary.BigEndian.Uint16(b[32:34]))
+			ac += uint32(binary.BigEndian.Uint16(b[34:36]))
+			ac += uint32(binary.BigEndian.Uint16(b[36:38]))
+			ac += uint32(binary.BigEndian.Uint16(b[38:40]))
+			ac += uint32(binary.BigEndian.Uint16(b[40:42]))
+			ac += uint32(binary.BigEndian.Uint16(b[42:44]))
+			ac += uint32(binary.BigEndian.Uint16(b[44:46]))
+			ac += uint32(binary.BigEndian.Uint16(b[46:48]))
+			ac += uint32(binary.BigEndian.Uint16(b[48:50]))
+			ac += uint32(binary.BigEndian.Uint16(b[50:52]))
+			ac += uint32(binary.BigEndian.Uint16(b[52:54]))
+			ac += uint32(binary.BigEndian.Uint16(b[54:56]))
+			ac += uint32(binary.BigEndian.Uint16(b[56:58]))
+			ac += uint32(binary.BigEndian.Uint16(b[58:60]))
+			ac += uint32(binary.BigEndian.Uint16(b[60:62]))
+			ac += uint32(binary.BigEndian.Uint16(b[62:64]))
+		} else {
+			ac += uint32(binary.LittleEndian.Uint16(b[:2]))
+			ac += uint32(binary.LittleEndian.Uint16(b[2:4]))
+			ac += uint32(binary.LittleEndian.Uint16(b[4:6]))
+			ac += uint32(binary.LittleEndian.Uint16(b[6:8]))
+			ac += uint32(binary.LittleEndian.Uint16(b[8:10]))
+			ac += uint32(binary.LittleEndian.Uint16(b[10:12]))
+			ac += uint32(binary.LittleEndian.Uint16(b[12:14]))
+			ac += uint32(binary.LittleEndian.Uint16(b[14:16]))
+			ac += uint32(binary.LittleEndian.Uint16(b[16:18]))
+			ac += uint32(binary.LittleEndian.Uint16(b[18:20]))
+			ac += uint32(binary.LittleEndian.Uint16(b[20:22]))
+			ac += uint32(binary.LittleEndian.Uint16(b[22:24]))
+			ac += uint32(binary.LittleEndian.Uint16(b[24:26]))
+			ac += uint32(binary.LittleEndian.Uint16(b[26:28]))
+			ac += uint32(binary.LittleEndian.Uint16(b[28:30]))
+			ac += uint32(binary.LittleEndian.Uint16(b[30:32]))
+			ac += uint32(binary.LittleEndian.Uint16(b[32:34]))
+			ac += uint32(binary.LittleEndian.Uint16(b[34:36]))
+			ac += uint32(binary.LittleEndian.Uint16(b[36:38]))
+			ac += uint32(binary.LittleEndian.Uint16(b[38:40]))
+			ac += uint32(binary.LittleEndian.Uint16(b[40:42]))
+			ac += uint32(binary.LittleEndian.Uint16(b[42:44]))
+			ac += uint32(binary.LittleEndian.Uint16(b[44:46]))
+			ac += uint32(binary.LittleEndian.Uint16(b[46:48]))
+			ac += uint32(binary.LittleEndian.Uint16(b[48:50]))
+			ac += uint32(binary.LittleEndian.Uint16(b[50:52]))
+			ac += uint32(binary.LittleEndian.Uint16(b[52:54]))
+			ac += uint32(binary.LittleEndian.Uint16(b[54:56]))
+			ac += uint32(binary.LittleEndian.Uint16(b[56:58]))
+			ac += uint32(binary.LittleEndian.Uint16(b[58:60]))
+			ac += uint32(binary.LittleEndian.Uint16(b[60:62]))
+			ac += uint32(binary.LittleEndian.Uint16(b[62:64]))
+		}
+		b = b[64:]
+	}
+	if len(b) >= 32 {
+		if cpu.IsBigEndian {
+			ac += uint32(binary.BigEndian.Uint16(b[:2]))
+			ac += uint32(binary.BigEndian.Uint16(b[2:4]))
+			ac += uint32(binary.BigEndian.Uint16(b[4:6]))
+			ac += uint32(binary.BigEndian.Uint16(b[6:8]))
+			ac += uint32(binary.BigEndian.Uint16(b[8:10]))
+			ac += uint32(binary.BigEndian.Uint16(b[10:12]))
+			ac += uint32(binary.BigEndian.Uint16(b[12:14]))
+			ac += uint32(binary.BigEndian.Uint16(b[14:16]))
+			ac += uint32(binary.BigEndian.Uint16(b[16:18]))
+			ac += uint32(binary.BigEndian.Uint16(b[18:20]))
+			ac += uint32(binary.BigEndian.Uint16(b[20:22]))
+			ac += uint32(binary.BigEndian.Uint16(b[22:24]))
+			ac += uint32(binary.BigEndian.Uint16(b[24:26]))
+			ac += uint32(binary.BigEndian.Uint16(b[26:28]))
+			ac += uint32(binary.BigEndian.Uint16(b[28:30]))
+			ac += uint32(binary.BigEndian.Uint16(b[30:32]))
+		} else {
+			ac += uint32(binary.LittleEndian.Uint16(b[:2]))
+			ac += uint32(binary.LittleEndian.Uint16(b[2:4]))
+			ac += uint32(binary.LittleEndian.Uint16(b[4:6]))
+			ac += uint32(binary.LittleEndian.Uint16(b[6:8]))
+			ac += uint32(binary.LittleEndian.Uint16(b[8:10]))
+			ac += uint32(binary.LittleEndian.Uint16(b[10:12]))
+			ac += uint32(binary.LittleEndian.Uint16(b[12:14]))
+			ac += uint32(binary.LittleEndian.Uint16(b[14:16]))
+			ac += uint32(binary.LittleEndian.Uint16(b[16:18]))
+			ac += uint32(binary.LittleEndian.Uint16(b[18:20]))
+			ac += uint32(binary.LittleEndian.Uint16(b[20:22]))
+			ac += uint32(binary.LittleEndian.Uint16(b[22:24]))
+			ac += uint32(binary.LittleEndian.Uint16(b[24:26]))
+			ac += uint32(binary.LittleEndian.Uint16(b[26:28]))
+			ac += uint32(binary.LittleEndian.Uint16(b[28:30]))
+			ac += uint32(binary.LittleEndian.Uint16(b[30:32]))
+		}
+		b = b[32:]
+	}
+	if len(b) >= 16 {
+		if cpu.IsBigEndian {
+			ac += uint32(binary.BigEndian.Uint16(b[:2]))
+			ac += uint32(binary.BigEndian.Uint16(b[2:4]))
+			ac += uint32(binary.BigEndian.Uint16(b[4:6]))
+			ac += uint32(binary.BigEndian.Uint16(b[6:8]))
+			ac += uint32(binary.BigEndian.Uint16(b[8:10]))
+			ac += uint32(binary.BigEndian.Uint16(b[10:12]))
+			ac += uint32(binary.BigEndian.Uint16(b[12:14]))
+			ac += uint32(binary.BigEndian.Uint16(b[14:16]))
+		} else {
+			ac += uint32(binary.LittleEndian.Uint16(b[:2]))
+			ac += uint32(binary.LittleEndian.Uint16(b[2:4]))
+			ac += uint32(binary.LittleEndian.Uint16(b[4:6]))
+			ac += uint32(binary.LittleEndian.Uint16(b[6:8]))
+			ac += uint32(binary.LittleEndian.Uint16(b[8:10]))
+			ac += uint32(binary.LittleEndian.Uint16(b[10:12]))
+			ac += uint32(binary.LittleEndian.Uint16(b[12:14]))
+			ac += uint32(binary.LittleEndian.Uint16(b[14:16]))
+		}
+		b = b[16:]
+	}
+	if len(b) >= 8 {
+		if cpu.IsBigEndian {
+			ac += uint32(binary.BigEndian.Uint16(b[:2]))
+			ac += uint32(binary.BigEndian.Uint16(b[2:4]))
+			ac += uint32(binary.BigEndian.Uint16(b[4:6]))
+			ac += uint32(binary.BigEndian.Uint16(b[6:8]))
+		} else {
+			ac += uint32(binary.LittleEndian.Uint16(b[:2]))
+			ac += uint32(binary.LittleEndian.Uint16(b[2:4]))
+			ac += uint32(binary.LittleEndian.Uint16(b[4:6]))
+			ac += uint32(binary.LittleEndian.Uint16(b[6:8]))
+		}
+		b = b[8:]
+	}
+	if len(b) >= 4 {
+		if cpu.IsBigEndian {
+			ac += uint32(binary.BigEndian.Uint16(b[:2]))
+			ac += uint32(binary.BigEndian.Uint16(b[2:4]))
+		} else {
+			ac += uint32(binary.LittleEndian.Uint16(b[:2]))
+			ac += uint32(binary.LittleEndian.Uint16(b[2:4]))
+		}
+		b = b[4:]
+	}
+	if len(b) >= 2 {
+		if cpu.IsBigEndian {
+			ac += uint32(binary.BigEndian.Uint16(b))
+		} else {
+			ac += uint32(binary.LittleEndian.Uint16(b))
+		}
+		b = b[2:]
+	}
+	if len(b) >= 1 {
+		if cpu.IsBigEndian {
+			ac += uint32(b[0]) << 8
+		} else {
+			ac += uint32(b[0])
+		}
+	}
+
+	folded := ipChecksumFold32(ac, 0)
+	if !cpu.IsBigEndian {
+		folded = bits.ReverseBytes16(folded)
+	}
+	return folded
 }
 
-func pseudoHeaderChecksumNoFold(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint64 {
-	sum := checksumNoFold(srcAddr, 0)
-	sum = checksumNoFold(dstAddr, sum)
-	sum = checksumNoFold([]byte{0, protocol}, sum)
-	tmp := make([]byte, 2)
-	binary.BigEndian.PutUint16(tmp, totalLen)
-	return checksumNoFold(tmp, sum)
+// checksumGeneric64Alternate is an alternate reference implementation of
+// checksum using 64 bit arithmetic for use in testing or when an
+// architecture-specific implementation is not available.
+func checksumGeneric64Alternate(b []byte, initial uint16) uint16 {
+	var ac uint64
+
+	if cpu.IsBigEndian {
+		ac = uint64(initial)
+	} else {
+		ac = uint64(bits.ReverseBytes16(initial))
+	}
+
+	for len(b) >= 64 {
+		if cpu.IsBigEndian {
+			ac += uint64(binary.BigEndian.Uint32(b[:4]))
+			ac += uint64(binary.BigEndian.Uint32(b[4:8]))
+			ac += uint64(binary.BigEndian.Uint32(b[8:12]))
+			ac += uint64(binary.BigEndian.Uint32(b[12:16]))
+			ac += uint64(binary.BigEndian.Uint32(b[16:20]))
+			ac += uint64(binary.BigEndian.Uint32(b[20:24]))
+			ac += uint64(binary.BigEndian.Uint32(b[24:28]))
+			ac += uint64(binary.BigEndian.Uint32(b[28:32]))
+			ac += uint64(binary.BigEndian.Uint32(b[32:36]))
+			ac += uint64(binary.BigEndian.Uint32(b[36:40]))
+			ac += uint64(binary.BigEndian.Uint32(b[40:44]))
+			ac += uint64(binary.BigEndian.Uint32(b[44:48]))
+			ac += uint64(binary.BigEndian.Uint32(b[48:52]))
+			ac += uint64(binary.BigEndian.Uint32(b[52:56]))
+			ac += uint64(binary.BigEndian.Uint32(b[56:60]))
+			ac += uint64(binary.BigEndian.Uint32(b[60:64]))
+		} else {
+			ac += uint64(binary.LittleEndian.Uint32(b[:4]))
+			ac += uint64(binary.LittleEndian.Uint32(b[4:8]))
+			ac += uint64(binary.LittleEndian.Uint32(b[8:12]))
+			ac += uint64(binary.LittleEndian.Uint32(b[12:16]))
+			ac += uint64(binary.LittleEndian.Uint32(b[16:20]))
+			ac += uint64(binary.LittleEndian.Uint32(b[20:24]))
+			ac += uint64(binary.LittleEndian.Uint32(b[24:28]))
+			ac += uint64(binary.LittleEndian.Uint32(b[28:32]))
+			ac += uint64(binary.LittleEndian.Uint32(b[32:36]))
+			ac += uint64(binary.LittleEndian.Uint32(b[36:40]))
+			ac += uint64(binary.LittleEndian.Uint32(b[40:44]))
+			ac += uint64(binary.LittleEndian.Uint32(b[44:48]))
+			ac += uint64(binary.LittleEndian.Uint32(b[48:52]))
+			ac += uint64(binary.LittleEndian.Uint32(b[52:56]))
+			ac += uint64(binary.LittleEndian.Uint32(b[56:60]))
+			ac += uint64(binary.LittleEndian.Uint32(b[60:64]))
+		}
+		b = b[64:]
+	}
+	if len(b) >= 32 {
+		if cpu.IsBigEndian {
+			ac += uint64(binary.BigEndian.Uint32(b[:4]))
+			ac += uint64(binary.BigEndian.Uint32(b[4:8]))
+			ac += uint64(binary.BigEndian.Uint32(b[8:12]))
+			ac += uint64(binary.BigEndian.Uint32(b[12:16]))
+			ac += uint64(binary.BigEndian.Uint32(b[16:20]))
+			ac += uint64(binary.BigEndian.Uint32(b[20:24]))
+			ac += uint64(binary.BigEndian.Uint32(b[24:28]))
+			ac += uint64(binary.BigEndian.Uint32(b[28:32]))
+		} else {
+			ac += uint64(binary.LittleEndian.Uint32(b[:4]))
+			ac += uint64(binary.LittleEndian.Uint32(b[4:8]))
+			ac += uint64(binary.LittleEndian.Uint32(b[8:12]))
+			ac += uint64(binary.LittleEndian.Uint32(b[12:16]))
+			ac += uint64(binary.LittleEndian.Uint32(b[16:20]))
+			ac += uint64(binary.LittleEndian.Uint32(b[20:24]))
+			ac += uint64(binary.LittleEndian.Uint32(b[24:28]))
+			ac += uint64(binary.LittleEndian.Uint32(b[28:32]))
+		}
+		b = b[32:]
+	}
+	if len(b) >= 16 {
+		if cpu.IsBigEndian {
+			ac += uint64(binary.BigEndian.Uint32(b[:4]))
+			ac += uint64(binary.BigEndian.Uint32(b[4:8]))
+			ac += uint64(binary.BigEndian.Uint32(b[8:12]))
+			ac += uint64(binary.BigEndian.Uint32(b[12:16]))
+		} else {
+			ac += uint64(binary.LittleEndian.Uint32(b[:4]))
+			ac += uint64(binary.LittleEndian.Uint32(b[4:8]))
+			ac += uint64(binary.LittleEndian.Uint32(b[8:12]))
+			ac += uint64(binary.LittleEndian.Uint32(b[12:16]))
+		}
+		b = b[16:]
+	}
+	if len(b) >= 8 {
+		if cpu.IsBigEndian {
+			ac += uint64(binary.BigEndian.Uint32(b[:4]))
+			ac += uint64(binary.BigEndian.Uint32(b[4:8]))
+		} else {
+			ac += uint64(binary.LittleEndian.Uint32(b[:4]))
+			ac += uint64(binary.LittleEndian.Uint32(b[4:8]))
+		}
+		b = b[8:]
+	}
+	if len(b) >= 4 {
+		if cpu.IsBigEndian {
+			ac += uint64(binary.BigEndian.Uint32(b))
+		} else {
+			ac += uint64(binary.LittleEndian.Uint32(b))
+		}
+		b = b[4:]
+	}
+	if len(b) >= 2 {
+		if cpu.IsBigEndian {
+			ac += uint64(binary.BigEndian.Uint16(b))
+		} else {
+			ac += uint64(binary.LittleEndian.Uint16(b))
+		}
+		b = b[2:]
+	}
+	if len(b) >= 1 {
+		if cpu.IsBigEndian {
+			ac += uint64(b[0]) << 8
+		} else {
+			ac += uint64(b[0])
+		}
+	}
+
+	folded := ipChecksumFold64(ac, 0)
+	if !cpu.IsBigEndian {
+		folded = bits.ReverseBytes16(folded)
+	}
+	return folded
+}
+
+func ipChecksumFold64(unfolded uint64, initialCarry uint64) uint16 {
+	sum, carry := bits.Add32(uint32(unfolded>>32), uint32(unfolded&0xffff_ffff), uint32(initialCarry))
+	// if carry != 0, sum <= 0xffff_fffe, otherwise sum <= 0xffff_ffff
+	// therefore (sum >> 16) + (sum & 0xffff) + carry <= 0x1_fffe; so there is
+	// no need to save the carry flag
+	sum = (sum >> 16) + (sum & 0xffff) + carry
+	// sum <= 0x1_fffe therefore this is the last fold needed:
+	//   if (sum >> 16) > 0 then
+	//     (sum >> 16) == 1 && (sum & 0xffff) <= 0xfffe and therefore
+	//     the addition will not overflow
+	// otherwise (sum >> 16) == 0 and sum will be unchanged
+	sum = (sum >> 16) + (sum & 0xffff)
+	return uint16(sum)
+}
+
+func ipChecksumFold32(unfolded uint32, initialCarry uint32) uint16 {
+	sum := (unfolded >> 16) + (unfolded & 0xffff) + initialCarry
+	// sum <= 0x1_ffff:
+	//   0xffff + 0xffff = 0x1_fffe
+	//   initialCarry is 0 or 1, for a combined maximum of 0x1_ffff
+	sum = (sum >> 16) + (sum & 0xffff)
+	// sum <= 0x1_0000 therefore this is the last fold needed:
+	//   if (sum >> 16) > 0 then
+	//     (sum >> 16) == 1 && (sum & 0xffff) == 0 and therefore
+	//     the addition will not overflow
+	// otherwise (sum >> 16) == 0 and sum will be unchanged
+	sum = (sum >> 16) + (sum & 0xffff)
+	return uint16(sum)
+}
+
+func addrPartialChecksum64(addr []byte, initial, carryIn uint64) (sum, carry uint64) {
+	sum, carry = initial, carryIn
+	switch len(addr) {
+	case 4: // IPv4
+		if cpu.IsBigEndian {
+			sum, carry = bits.Add64(sum, uint64(binary.BigEndian.Uint32(addr)), carry)
+		} else {
+			sum, carry = bits.Add64(sum, uint64(binary.LittleEndian.Uint32(addr)), carry)
+		}
+	case 16: // IPv6
+		if cpu.IsBigEndian {
+			sum, carry = bits.Add64(sum, binary.BigEndian.Uint64(addr), carry)
+			sum, carry = bits.Add64(sum, binary.BigEndian.Uint64(addr[8:]), carry)
+		} else {
+			sum, carry = bits.Add64(sum, binary.LittleEndian.Uint64(addr), carry)
+			sum, carry = bits.Add64(sum, binary.LittleEndian.Uint64(addr[8:]), carry)
+		}
+	default:
+		panic("bad addr length")
+	}
+	return sum, carry
+}
+
+func addrPartialChecksum32(addr []byte, initial, carryIn uint32) (sum, carry uint32) {
+	sum, carry = initial, carryIn
+	switch len(addr) {
+	case 4: // IPv4
+		if cpu.IsBigEndian {
+			sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr), carry)
+		} else {
+			sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr), carry)
+		}
+	case 16: // IPv6
+		if cpu.IsBigEndian {
+			sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr), carry)
+			sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[4:8]), carry)
+			sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[8:12]), carry)
+			sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[12:16]), carry)
+		} else {
+			sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr), carry)
+			sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[4:8]), carry)
+			sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[8:12]), carry)
+			sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[12:16]), carry)
+		}
+	default:
+		panic("bad addr length")
+	}
+	return sum, carry
+}
+
+func pseudoHeaderChecksum64(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 {
+	var sum uint64
+	if cpu.IsBigEndian {
+		sum = uint64(totalLen) + uint64(protocol)
+	} else {
+		sum = uint64(bits.ReverseBytes16(totalLen)) + uint64(protocol)<<8
+	}
+	sum, carry := addrPartialChecksum64(srcAddr, sum, 0)
+	sum, carry = addrPartialChecksum64(dstAddr, sum, carry)
+
+	foldedSum := ipChecksumFold64(sum, carry)
+	if !cpu.IsBigEndian {
+		foldedSum = bits.ReverseBytes16(foldedSum)
+	}
+	return foldedSum
+}
+
+func pseudoHeaderChecksum32(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 {
+	var sum uint32
+	if cpu.IsBigEndian {
+		sum = uint32(totalLen) + uint32(protocol)
+	} else {
+		sum = uint32(bits.ReverseBytes16(totalLen)) + uint32(protocol)<<8
+	}
+	sum, carry := addrPartialChecksum32(srcAddr, sum, 0)
+	sum, carry = addrPartialChecksum32(dstAddr, sum, carry)
+
+	foldedSum := ipChecksumFold32(sum, carry)
+	if !cpu.IsBigEndian {
+		foldedSum = bits.ReverseBytes16(foldedSum)
+	}
+	return foldedSum
 }
 
 // PseudoHeaderChecksum computes an IP pseudo-header checksum. srcAddr and
 // dstAddr must be 4 or 16 bytes in length.
 func PseudoHeaderChecksum(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 {
-	return checksum([]byte{}, pseudoHeaderChecksumNoFold(protocol, srcAddr, dstAddr, totalLen))
+	if strconv.IntSize < 64 {
+		return pseudoHeaderChecksum32(protocol, srcAddr, dstAddr, totalLen)
+	}
+	return pseudoHeaderChecksum64(protocol, srcAddr, dstAddr, totalLen)
 }

tun/checksum_amd64.go

@@ -0,0 +1,23 @@
+package tun
+
+import "golang.org/x/sys/cpu"
+
+var checksum = checksumAMD64
+
+// Checksum computes an IP checksum starting with the provided initial value.
+// The length of data should be at least 128 bytes for best performance. Smaller
+// buffers will still compute a correct result.
+func Checksum(data []byte, initial uint16) uint16 {
+	return checksum(data, initial)
+}
+
+func init() {
+	if cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 {
+		checksum = checksumAVX2
+		return
+	}
+	if cpu.X86.HasSSE2 {
+		checksum = checksumSSE2
+		return
+	}
+}

tun/checksum_generated_amd64.go

@@ -0,0 +1,18 @@
+// Code generated by command: go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go. DO NOT EDIT.
+
+package tun
+
+// checksumAVX2 computes an IP checksum using amd64 v3 instructions (AVX2, BMI2)
+//
+//go:noescape
+func checksumAVX2(b []byte, initial uint16) uint16
+
+// checksumSSE2 computes an IP checksum using amd64 baseline instructions (SSE2)
+//
+//go:noescape
+func checksumSSE2(b []byte, initial uint16) uint16
+
+// checksumAMD64 computes an IP checksum using amd64 baseline instructions
+//
+//go:noescape
+func checksumAMD64(b []byte, initial uint16) uint16

tun/checksum_generated_amd64.s

@@ -0,0 +1,851 @@
+// Code generated by command: go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go. DO NOT EDIT.
+
+#include "textflag.h"
+
+DATA xmmLoadMasks<>+0(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff"
+DATA xmmLoadMasks<>+16(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff"
+DATA xmmLoadMasks<>+32(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff"
+DATA xmmLoadMasks<>+48(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff"
+DATA xmmLoadMasks<>+64(SB)/16, $"\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+DATA xmmLoadMasks<>+80(SB)/16, $"\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+DATA xmmLoadMasks<>+96(SB)/16, $"\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+GLOBL xmmLoadMasks<>(SB), RODATA|NOPTR, $112
+
+// func checksumAVX2(b []byte, initial uint16) uint16
+// Requires: AVX, AVX2, BMI2
+TEXT ·checksumAVX2(SB), NOSPLIT|NOFRAME, $0-34
+	MOVWQZX initial+24(FP), AX
+	XCHGB   AH, AL
+	MOVQ    b_base+0(FP), DX
+	MOVQ    b_len+8(FP), BX
+
+	// handle odd length buffers; they are difficult to handle in general
+	TESTQ   $0x00000001, BX
+	JZ      lengthIsEven
+	MOVBQZX -1(DX)(BX*1), CX
+	DECQ    BX
+	ADDQ    CX, AX
+
+lengthIsEven:
+	// handle tiny buffers (<=31 bytes) specially
+	CMPQ BX, $0x1f
+	JGT  bufferIsNotTiny
+	XORQ CX, CX
+	XORQ SI, SI
+	XORQ DI, DI
+
+	// shift twice to start because length is guaranteed to be even
+	// n = n >> 2; CF = originalN & 2
+	SHRQ $0x02, BX
+	JNC  handleTiny4
+
+	// tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:]
+	MOVWQZX (DX), CX
+	ADDQ    $0x02, DX
+
+handleTiny4:
+	// n = n >> 1; CF = originalN & 4
+	SHRQ $0x01, BX
+	JNC  handleTiny8
+
+	// tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:]
+	MOVLQZX (DX), SI
+	ADDQ    $0x04, DX
+
+handleTiny8:
+	// n = n >> 1; CF = originalN & 8
+	SHRQ $0x01, BX
+	JNC  handleTiny16
+
+	// tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:]
+	MOVQ (DX), DI
+	ADDQ $0x08, DX
+
+handleTiny16:
+	// n = n >> 1; CF = originalN & 16
+	// n == 0 now, otherwise we would have branched after comparing with tinyBufferSize
+	SHRQ $0x01, BX
+	JNC  handleTinyFinish
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+
+handleTinyFinish:
+	// CF should be included from the previous add, so we use ADCQ.
+	// If we arrived via the JNC above, then CF=0 due to the branch condition,
+	// so ADCQ will still produce the correct result.
+	ADCQ CX, AX
+	ADCQ SI, AX
+	ADCQ DI, AX
+	JMP  foldAndReturn
+
+bufferIsNotTiny:
+	// skip all SIMD for small buffers
+	CMPQ BX, $0x00000100
+	JGE  startSIMD
+
+	// Accumulate carries in this register. It is never expected to overflow.
+	XORQ SI, SI
+
+	// We will perform an overlapped read for buffers with length not a multiple of 8.
+	// Overlapped in this context means some memory will be read twice, but a shift will
+	// eliminate the duplicated data. This extra read is performed at the end of the buffer to
+	// preserve any alignment that may exist for the start of the buffer.
+	MOVQ BX, CX
+	SHRQ $0x03, BX
+	ANDQ $0x07, CX
+	JZ   handleRemaining8
+	LEAQ (DX)(BX*8), DI
+	MOVQ -8(DI)(CX*1), DI
+
+	// Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8)
+	SHLQ $0x03, CX
+	NEGQ CX
+	ADDQ $0x40, CX
+	SHRQ CL, DI
+	ADDQ DI, AX
+	ADCQ $0x00, SI
+
+handleRemaining8:
+	SHRQ $0x01, BX
+	JNC  handleRemaining16
+	ADDQ (DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x08, DX
+
+handleRemaining16:
+	SHRQ $0x01, BX
+	JNC  handleRemaining32
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x10, DX
+
+handleRemaining32:
+	SHRQ $0x01, BX
+	JNC  handleRemaining64
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ 16(DX), AX
+	ADCQ 24(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x20, DX
+
+handleRemaining64:
+	SHRQ $0x01, BX
+	JNC  handleRemaining128
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ 16(DX), AX
+	ADCQ 24(DX), AX
+	ADCQ 32(DX), AX
+	ADCQ 40(DX), AX
+	ADCQ 48(DX), AX
+	ADCQ 56(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x40, DX
+
+handleRemaining128:
+	SHRQ $0x01, BX
+	JNC  handleRemainingComplete
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ 16(DX), AX
+	ADCQ 24(DX), AX
+	ADCQ 32(DX), AX
+	ADCQ 40(DX), AX
+	ADCQ 48(DX), AX
+	ADCQ 56(DX), AX
+	ADCQ 64(DX), AX
+	ADCQ 72(DX), AX
+	ADCQ 80(DX), AX
+	ADCQ 88(DX), AX
+	ADCQ 96(DX), AX
+	ADCQ 104(DX), AX
+	ADCQ 112(DX), AX
+	ADCQ 120(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x80, DX
+
+handleRemainingComplete:
+	ADDQ SI, AX
+	JMP  foldAndReturn
+
+startSIMD:
+	VPXOR Y0, Y0, Y0
+	VPXOR Y1, Y1, Y1
+	VPXOR Y2, Y2, Y2
+	VPXOR Y3, Y3, Y3
+	MOVQ  BX, CX
+
+	// Update number of bytes remaining after the loop completes
+	ANDQ $0xff, BX
+
+	// Number of 256 byte iterations
+	SHRQ $0x08, CX
+	JZ   smallLoop
+
+bigLoop:
+	VPMOVZXWD (DX), Y4
+	VPADDD    Y4, Y0, Y0
+	VPMOVZXWD 16(DX), Y4
+	VPADDD    Y4, Y1, Y1
+	VPMOVZXWD 32(DX), Y4
+	VPADDD    Y4, Y2, Y2
+	VPMOVZXWD 48(DX), Y4
+	VPADDD    Y4, Y3, Y3
+	VPMOVZXWD 64(DX), Y4
+	VPADDD    Y4, Y0, Y0
+	VPMOVZXWD 80(DX), Y4
+	VPADDD    Y4, Y1, Y1
+	VPMOVZXWD 96(DX), Y4
+	VPADDD    Y4, Y2, Y2
+	VPMOVZXWD 112(DX), Y4
+	VPADDD    Y4, Y3, Y3
+	VPMOVZXWD 128(DX), Y4
+	VPADDD    Y4, Y0, Y0
+	VPMOVZXWD 144(DX), Y4
+	VPADDD    Y4, Y1, Y1
+	VPMOVZXWD 160(DX), Y4
+	VPADDD    Y4, Y2, Y2
+	VPMOVZXWD 176(DX), Y4
+	VPADDD    Y4, Y3, Y3
+	VPMOVZXWD 192(DX), Y4
+	VPADDD    Y4, Y0, Y0
+	VPMOVZXWD 208(DX), Y4
+	VPADDD    Y4, Y1, Y1
+	VPMOVZXWD 224(DX), Y4
+	VPADDD    Y4, Y2, Y2
+	VPMOVZXWD 240(DX), Y4
+	VPADDD    Y4, Y3, Y3
+	ADDQ      $0x00000100, DX
+	DECQ      CX
+	JNZ       bigLoop
+	CMPQ      BX, $0x10
+	JLT       doneSmallLoop
+
+	// now read a single 16 byte unit of data at a time
+smallLoop:
+	VPMOVZXWD (DX), Y4
+	VPADDD    Y4, Y0, Y0
+	ADDQ      $0x10, DX
+	SUBQ      $0x10, BX
+	CMPQ      BX, $0x10
+	JGE       smallLoop
+
+doneSmallLoop:
+	CMPQ BX, $0x00
+	JE   doneSIMD
+
+	// There are between 1 and 15 bytes remaining. Perform an overlapped read.
+	LEAQ      xmmLoadMasks<>+0(SB), CX
+	VMOVDQU   -16(DX)(BX*1), X4
+	VPAND     -16(CX)(BX*8), X4, X4
+	VPMOVZXWD X4, Y4
+	VPADDD    Y4, Y0, Y0
+
+doneSIMD:
+	// Multi-chain loop is done, combine the accumulators
+	VPADDD Y1, Y0, Y0
+	VPADDD Y2, Y0, Y0
+	VPADDD Y3, Y0, Y0
+
+	// extract the YMM into a pair of XMM and sum them
+	VEXTRACTI128 $0x01, Y0, X1
+	VPADDD       X0, X1, X0
+
+	// extract the XMM into GP64
+	VPEXTRQ $0x00, X0, CX
+	VPEXTRQ $0x01, X0, DX
+
+	// no more AVX code, clear upper registers to avoid SSE slowdowns
+	VZEROUPPER
+	ADDQ CX, AX
+	ADCQ DX, AX
+
+foldAndReturn:
+	// add CF and fold
+	RORXQ $0x20, AX, CX
+	ADCL  CX, AX
+	RORXL $0x10, AX, CX
+	ADCW  CX, AX
+	ADCW  $0x00, AX
+	XCHGB AH, AL
+	MOVW  AX, ret+32(FP)
+	RET
+
+// func checksumSSE2(b []byte, initial uint16) uint16
+// Requires: SSE2
+TEXT ·checksumSSE2(SB), NOSPLIT|NOFRAME, $0-34
+	MOVWQZX initial+24(FP), AX
+	XCHGB   AH, AL
+	MOVQ    b_base+0(FP), DX
+	MOVQ    b_len+8(FP), BX
+
+	// handle odd length buffers; they are difficult to handle in general
+	TESTQ   $0x00000001, BX
+	JZ      lengthIsEven
+	MOVBQZX -1(DX)(BX*1), CX
+	DECQ    BX
+	ADDQ    CX, AX
+
+lengthIsEven:
+	// handle tiny buffers (<=31 bytes) specially
+	CMPQ BX, $0x1f
+	JGT  bufferIsNotTiny
+	XORQ CX, CX
+	XORQ SI, SI
+	XORQ DI, DI
+
+	// shift twice to start because length is guaranteed to be even
+	// n = n >> 2; CF = originalN & 2
+	SHRQ $0x02, BX
+	JNC  handleTiny4
+
+	// tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:]
+	MOVWQZX (DX), CX
+	ADDQ    $0x02, DX
+
+handleTiny4:
+	// n = n >> 1; CF = originalN & 4
+	SHRQ $0x01, BX
+	JNC  handleTiny8
+
+	// tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:]
+	MOVLQZX (DX), SI
+	ADDQ    $0x04, DX
+
+handleTiny8:
+	// n = n >> 1; CF = originalN & 8
+	SHRQ $0x01, BX
+	JNC  handleTiny16
+
+	// tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:]
+	MOVQ (DX), DI
+	ADDQ $0x08, DX
+
+handleTiny16:
+	// n = n >> 1; CF = originalN & 16
+	// n == 0 now, otherwise we would have branched after comparing with tinyBufferSize
+	SHRQ $0x01, BX
+	JNC  handleTinyFinish
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+
+handleTinyFinish:
+	// CF should be included from the previous add, so we use ADCQ.
+	// If we arrived via the JNC above, then CF=0 due to the branch condition,
+	// so ADCQ will still produce the correct result.
+	ADCQ CX, AX
+	ADCQ SI, AX
+	ADCQ DI, AX
+	JMP  foldAndReturn
+
+bufferIsNotTiny:
+	// skip all SIMD for small buffers
+	CMPQ BX, $0x00000100
+	JGE  startSIMD
+
+	// Accumulate carries in this register. It is never expected to overflow.
+	XORQ SI, SI
+
+	// We will perform an overlapped read for buffers with length not a multiple of 8.
+	// Overlapped in this context means some memory will be read twice, but a shift will
+	// eliminate the duplicated data. This extra read is performed at the end of the buffer to
+	// preserve any alignment that may exist for the start of the buffer.
+	MOVQ BX, CX
+	SHRQ $0x03, BX
+	ANDQ $0x07, CX
+	JZ   handleRemaining8
+	LEAQ (DX)(BX*8), DI
+	MOVQ -8(DI)(CX*1), DI
+
+	// Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8)
+	SHLQ $0x03, CX
+	NEGQ CX
+	ADDQ $0x40, CX
+	SHRQ CL, DI
+	ADDQ DI, AX
+	ADCQ $0x00, SI
+
+handleRemaining8:
+	SHRQ $0x01, BX
+	JNC  handleRemaining16
+	ADDQ (DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x08, DX
+
+handleRemaining16:
+	SHRQ $0x01, BX
+	JNC  handleRemaining32
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x10, DX
+
+handleRemaining32:
+	SHRQ $0x01, BX
+	JNC  handleRemaining64
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ 16(DX), AX
+	ADCQ 24(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x20, DX
+
+handleRemaining64:
+	SHRQ $0x01, BX
+	JNC  handleRemaining128
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ 16(DX), AX
+	ADCQ 24(DX), AX
+	ADCQ 32(DX), AX
+	ADCQ 40(DX), AX
+	ADCQ 48(DX), AX
+	ADCQ 56(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x40, DX
+
+handleRemaining128:
+	SHRQ $0x01, BX
+	JNC  handleRemainingComplete
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ 16(DX), AX
+	ADCQ 24(DX), AX
+	ADCQ 32(DX), AX
+	ADCQ 40(DX), AX
+	ADCQ 48(DX), AX
+	ADCQ 56(DX), AX
+	ADCQ 64(DX), AX
+	ADCQ 72(DX), AX
+	ADCQ 80(DX), AX
+	ADCQ 88(DX), AX
+	ADCQ 96(DX), AX
+	ADCQ 104(DX), AX
+	ADCQ 112(DX), AX
+	ADCQ 120(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x80, DX
+
+handleRemainingComplete:
+	ADDQ SI, AX
+	JMP  foldAndReturn
+
+startSIMD:
+	PXOR X0, X0
+	PXOR X1, X1
+	PXOR X2, X2
+	PXOR X3, X3
+	PXOR X4, X4
+	MOVQ BX, CX
+
+	// Update number of bytes remaining after the loop completes
+	ANDQ $0xff, BX
+
+	// Number of 256 byte iterations
+	SHRQ $0x08, CX
+	JZ   smallLoop
+
+bigLoop:
+	MOVOU     (DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X0
+	PADDD     X6, X2
+	MOVOU     16(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X1
+	PADDD     X6, X3
+	MOVOU     32(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X2
+	PADDD     X6, X0
+	MOVOU     48(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X3
+	PADDD     X6, X1
+	MOVOU     64(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X0
+	PADDD     X6, X2
+	MOVOU     80(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X1
+	PADDD     X6, X3
+	MOVOU     96(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X2
+	PADDD     X6, X0
+	MOVOU     112(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X3
+	PADDD     X6, X1
+	MOVOU     128(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X0
+	PADDD     X6, X2
+	MOVOU     144(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X1
+	PADDD     X6, X3
+	MOVOU     160(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X2
+	PADDD     X6, X0
+	MOVOU     176(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X3
+	PADDD     X6, X1
+	MOVOU     192(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X0
+	PADDD     X6, X2
+	MOVOU     208(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X1
+	PADDD     X6, X3
+	MOVOU     224(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X2
+	PADDD     X6, X0
+	MOVOU     240(DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X3
+	PADDD     X6, X1
+	ADDQ      $0x00000100, DX
+	DECQ      CX
+	JNZ       bigLoop
+	CMPQ      BX, $0x10
+	JLT       doneSmallLoop
+
+	// now read a single 16 byte unit of data at a time
+smallLoop:
+	MOVOU     (DX), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X0
+	PADDD     X6, X1
+	ADDQ      $0x10, DX
+	SUBQ      $0x10, BX
+	CMPQ      BX, $0x10
+	JGE       smallLoop
+
+doneSmallLoop:
+	CMPQ BX, $0x00
+	JE   doneSIMD
+
+	// There are between 1 and 15 bytes remaining. Perform an overlapped read.
+	LEAQ      xmmLoadMasks<>+0(SB), CX
+	MOVOU     -16(DX)(BX*1), X5
+	PAND      -16(CX)(BX*8), X5
+	MOVOA     X5, X6
+	PUNPCKHWL X4, X5
+	PUNPCKLWL X4, X6
+	PADDD     X5, X0
+	PADDD     X6, X1
+
+doneSIMD:
+	// Multi-chain loop is done, combine the accumulators
+	PADDD X1, X0
+	PADDD X2, X0
+	PADDD X3, X0
+
+	// extract the XMM into GP64
+	MOVQ   X0, CX
+	PSRLDQ $0x08, X0
+	MOVQ   X0, DX
+	ADDQ   CX, AX
+	ADCQ   DX, AX
+
+foldAndReturn:
+	// add CF and fold
+	MOVL    AX, CX
+	ADCQ    $0x00, CX
+	SHRQ    $0x20, AX
+	ADDQ    CX, AX
+	MOVWQZX AX, CX
+	SHRQ    $0x10, AX
+	ADDQ    CX, AX
+	MOVW    AX, CX
+	SHRQ    $0x10, AX
+	ADDW    CX, AX
+	ADCW    $0x00, AX
+	XCHGB   AH, AL
+	MOVW    AX, ret+32(FP)
+	RET
+
+// func checksumAMD64(b []byte, initial uint16) uint16
+TEXT ·checksumAMD64(SB), NOSPLIT|NOFRAME, $0-34
+	MOVWQZX initial+24(FP), AX
+	XCHGB   AH, AL
+	MOVQ    b_base+0(FP), DX
+	MOVQ    b_len+8(FP), BX
+
+	// handle odd length buffers; they are difficult to handle in general
+	TESTQ   $0x00000001, BX
+	JZ      lengthIsEven
+	MOVBQZX -1(DX)(BX*1), CX
+	DECQ    BX
+	ADDQ    CX, AX
+
+lengthIsEven:
+	// handle tiny buffers (<=31 bytes) specially
+	CMPQ BX, $0x1f
+	JGT  bufferIsNotTiny
+	XORQ CX, CX
+	XORQ SI, SI
+	XORQ DI, DI
+
+	// shift twice to start because length is guaranteed to be even
+	// n = n >> 2; CF = originalN & 2
+	SHRQ $0x02, BX
+	JNC  handleTiny4
+
+	// tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:]
+	MOVWQZX (DX), CX
+	ADDQ    $0x02, DX
+
+handleTiny4:
+	// n = n >> 1; CF = originalN & 4
+	SHRQ $0x01, BX
+	JNC  handleTiny8
+
+	// tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:]
+	MOVLQZX (DX), SI
+	ADDQ    $0x04, DX
+
+handleTiny8:
+	// n = n >> 1; CF = originalN & 8
+	SHRQ $0x01, BX
+	JNC  handleTiny16
+
+	// tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:]
+	MOVQ (DX), DI
+	ADDQ $0x08, DX
+
+handleTiny16:
+	// n = n >> 1; CF = originalN & 16
+	// n == 0 now, otherwise we would have branched after comparing with tinyBufferSize
+	SHRQ $0x01, BX
+	JNC  handleTinyFinish
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+
+handleTinyFinish:
+	// CF should be included from the previous add, so we use ADCQ.
+	// If we arrived via the JNC above, then CF=0 due to the branch condition,
+	// so ADCQ will still produce the correct result.
+	ADCQ CX, AX
+	ADCQ SI, AX
+	ADCQ DI, AX
+	JMP  foldAndReturn
+
+bufferIsNotTiny:
+	// Number of 256 byte iterations into loop counter
+	MOVQ BX, CX
+
+	// Update number of bytes remaining after the loop completes
+	ANDQ $0xff, BX
+	SHRQ $0x08, CX
+	JZ   startCleanup
+	CLC
+	XORQ SI, SI
+	XORQ DI, DI
+	XORQ R8, R8
+	XORQ R9, R9
+	XORQ R10, R10
+	XORQ R11, R11
+	XORQ R12, R12
+
+bigLoop:
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ 16(DX), AX
+	ADCQ 24(DX), AX
+	ADCQ $0x00, SI
+	ADDQ 32(DX), DI
+	ADCQ 40(DX), DI
+	ADCQ 48(DX), DI
+	ADCQ 56(DX), DI
+	ADCQ $0x00, R8
+	ADDQ 64(DX), R9
+	ADCQ 72(DX), R9
+	ADCQ 80(DX), R9
+	ADCQ 88(DX), R9
+	ADCQ $0x00, R10
+	ADDQ 96(DX), R11
+	ADCQ 104(DX), R11
+	ADCQ 112(DX), R11
+	ADCQ 120(DX), R11
+	ADCQ $0x00, R12
+	ADDQ 128(DX), AX
+	ADCQ 136(DX), AX
+	ADCQ 144(DX), AX
+	ADCQ 152(DX), AX
+	ADCQ $0x00, SI
+	ADDQ 160(DX), DI
+	ADCQ 168(DX), DI
+	ADCQ 176(DX), DI
+	ADCQ 184(DX), DI
+	ADCQ $0x00, R8
+	ADDQ 192(DX), R9
+	ADCQ 200(DX), R9
+	ADCQ 208(DX), R9
+	ADCQ 216(DX), R9
+	ADCQ $0x00, R10
+	ADDQ 224(DX), R11
+	ADCQ 232(DX), R11
+	ADCQ 240(DX), R11
+	ADCQ 248(DX), R11
+	ADCQ $0x00, R12
+	ADDQ $0x00000100, DX
+	SUBQ $0x01, CX
+	JNZ  bigLoop
+	ADDQ SI, AX
+	ADCQ DI, AX
+	ADCQ R8, AX
+	ADCQ R9, AX
+	ADCQ R10, AX
+	ADCQ R11, AX
+	ADCQ R12, AX
+
+	// accumulate CF (twice, in case the first time overflows)
+	ADCQ $0x00, AX
+	ADCQ $0x00, AX
+
+startCleanup:
+	// Accumulate carries in this register. It is never expected to overflow.
+	XORQ SI, SI
+
+	// We will perform an overlapped read for buffers with length not a multiple of 8.
+	// Overlapped in this context means some memory will be read twice, but a shift will
+	// eliminate the duplicated data. This extra read is performed at the end of the buffer to
+	// preserve any alignment that may exist for the start of the buffer.
+	MOVQ BX, CX
+	SHRQ $0x03, BX
+	ANDQ $0x07, CX
+	JZ   handleRemaining8
+	LEAQ (DX)(BX*8), DI
+	MOVQ -8(DI)(CX*1), DI
+
+	// Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8)
+	SHLQ $0x03, CX
+	NEGQ CX
+	ADDQ $0x40, CX
+	SHRQ CL, DI
+	ADDQ DI, AX
+	ADCQ $0x00, SI
+
+handleRemaining8:
+	SHRQ $0x01, BX
+	JNC  handleRemaining16
+	ADDQ (DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x08, DX
+
+handleRemaining16:
+	SHRQ $0x01, BX
+	JNC  handleRemaining32
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x10, DX
+
+handleRemaining32:
+	SHRQ $0x01, BX
+	JNC  handleRemaining64
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ 16(DX), AX
+	ADCQ 24(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x20, DX
+
+handleRemaining64:
+	SHRQ $0x01, BX
+	JNC  handleRemaining128
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ 16(DX), AX
+	ADCQ 24(DX), AX
+	ADCQ 32(DX), AX
+	ADCQ 40(DX), AX
+	ADCQ 48(DX), AX
+	ADCQ 56(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x40, DX
+
+handleRemaining128:
+	SHRQ $0x01, BX
+	JNC  handleRemainingComplete
+	ADDQ (DX), AX
+	ADCQ 8(DX), AX
+	ADCQ 16(DX), AX
+	ADCQ 24(DX), AX
+	ADCQ 32(DX), AX
+	ADCQ 40(DX), AX
+	ADCQ 48(DX), AX
+	ADCQ 56(DX), AX
+	ADCQ 64(DX), AX
+	ADCQ 72(DX), AX
+	ADCQ 80(DX), AX
+	ADCQ 88(DX), AX
+	ADCQ 96(DX), AX
+	ADCQ 104(DX), AX
+	ADCQ 112(DX), AX
+	ADCQ 120(DX), AX
+	ADCQ $0x00, SI
+	ADDQ $0x80, DX
+
+handleRemainingComplete:
+	ADDQ SI, AX
+
+foldAndReturn:
+	// add CF and fold
+	MOVL    AX, CX
+	ADCQ    $0x00, CX
+	SHRQ    $0x20, AX
+	ADDQ    CX, AX
+	MOVWQZX AX, CX
+	SHRQ    $0x10, AX
+	ADDQ    CX, AX
+	MOVW    AX, CX
+	SHRQ    $0x10, AX
+	ADDW    CX, AX
+	ADCW    $0x00, AX
+	XCHGB   AH, AL
+	MOVW    AX, ret+32(FP)
+	RET

tun/checksum_generic.go

@@ -0,0 +1,15 @@
+// This file contains IP checksum algorithms that are not specific to any
+// architecture and don't use hardware acceleration.
+
+//go:build !amd64
+
+package tun
+
+import "strconv"
+
+func Checksum(data []byte, initial uint16) uint16 {
+	if strconv.IntSize < 64 {
+		return checksumGeneric32(data, initial)
+	}
+	return checksumGeneric64(data, initial)
+}

tun/generate_amd64.go

@@ -0,0 +1,579 @@
+//go:build ignore
+
+//go:generate go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go
+
+package main
+
+import (
+	"fmt"
+	"math"
+	"math/bits"
+
+	. "github.com/mmcloughlin/avo/build"
+	"github.com/mmcloughlin/avo/operand"
+	"github.com/mmcloughlin/avo/reg"
+)
+
+const checksumSignature = "func(b []byte, initial uint16) uint16"
+
+func loadParams() (accum, buf, n reg.GPVirtual) {
+	accum, buf, n = GP64(), GP64(), GP64()
+	Load(Param("initial"), accum)
+	XCHGB(accum.As8H(), accum.As8L())
+	Load(Param("b").Base(), buf)
+	Load(Param("b").Len(), n)
+	return
+}
+
+type simdStrategy int
+
+const (
+	sse2 = iota
+	avx2
+)
+
+const tinyBufferSize = 31 // A buffer is tiny if it has at most 31 bytes.
+
+func generateSIMDChecksum(name, doc string, minSIMDSize, chains int, strategy simdStrategy) {
+	TEXT(name, NOSPLIT|NOFRAME, checksumSignature)
+	Pragma("noescape")
+	Doc(doc)
+
+	accum64, buf, n := loadParams()
+
+	handleOddLength(n, buf, accum64)
+	// no chance of overflow because accum64 was initialized by a uint16 and
+	// handleOddLength adds at most a uint8
+	handleTinyBuffers(n, buf, accum64, operand.LabelRef("foldAndReturn"), operand.LabelRef("bufferIsNotTiny"))
+	Label("bufferIsNotTiny")
+
+	const simdReadSize = 16
+
+	if minSIMDSize > tinyBufferSize {
+		Comment("skip all SIMD for small buffers")
+		if minSIMDSize <= math.MaxUint8 {
+			CMPQ(n, operand.U8(minSIMDSize))
+		} else {
+			CMPQ(n, operand.U32(minSIMDSize))
+		}
+		JGE(operand.LabelRef("startSIMD"))
+
+		handleRemaining(n, buf, accum64, minSIMDSize-1)
+		JMP(operand.LabelRef("foldAndReturn"))
+	}
+
+	Label("startSIMD")
+
+	// chains is the number of accumulators to use. This improves speed via
+	// reduced data dependency. We combine the accumulators once when the big
+	// loop is complete.
+	simdAccumulate := make([]reg.VecVirtual, chains)
+	for i := range simdAccumulate {
+		switch strategy {
+		case sse2:
+			simdAccumulate[i] = XMM()
+			PXOR(simdAccumulate[i], simdAccumulate[i])
+		case avx2:
+			simdAccumulate[i] = YMM()
+			VPXOR(simdAccumulate[i], simdAccumulate[i], simdAccumulate[i])
+		}
+	}
+	var zero reg.VecVirtual
+	if strategy == sse2 {
+		zero = XMM()
+		PXOR(zero, zero)
+	}
+
+	// Number of loads per big loop
+	const unroll = 16
+	// Number of bytes
+	loopSize := uint64(simdReadSize * unroll)
+	if bits.Len64(loopSize) != bits.Len64(loopSize-1)+1 {
+		panic("loopSize is not a power of 2")
+	}
+	loopCount := GP64()
+
+	MOVQ(n, loopCount)
+	Comment("Update number of bytes remaining after the loop completes")
+	ANDQ(operand.Imm(loopSize-1), n)
+	Comment(fmt.Sprintf("Number of %d byte iterations", loopSize))
+	SHRQ(operand.Imm(uint64(bits.Len64(loopSize-1))), loopCount)
+	JZ(operand.LabelRef("smallLoop"))
+	Label("bigLoop")
+	for i := 0; i < unroll; i++ {
+		chain := i % chains
+		switch strategy {
+		case sse2:
+			sse2AccumulateStep(i*simdReadSize, buf, zero, simdAccumulate[chain], simdAccumulate[(chain+chains/2)%chains])
+		case avx2:
+			avx2AccumulateStep(i*simdReadSize, buf, simdAccumulate[chain])
+		}
+	}
+	ADDQ(operand.U32(loopSize), buf)
+	DECQ(loopCount)
+	JNZ(operand.LabelRef("bigLoop"))
+
+	Label("bigCleanup")
+
+	CMPQ(n, operand.Imm(uint64(simdReadSize)))
+	JLT(operand.LabelRef("doneSmallLoop"))
+
+	Commentf("now read a single %d byte unit of data at a time", simdReadSize)
+	Label("smallLoop")
+
+	switch strategy {
+	case sse2:
+		sse2AccumulateStep(0, buf, zero, simdAccumulate[0], simdAccumulate[1])
+	case avx2:
+		avx2AccumulateStep(0, buf, simdAccumulate[0])
+	}
+	ADDQ(operand.Imm(uint64(simdReadSize)), buf)
+	SUBQ(operand.Imm(uint64(simdReadSize)), n)
+	CMPQ(n, operand.Imm(uint64(simdReadSize)))
+	JGE(operand.LabelRef("smallLoop"))
+
+	Label("doneSmallLoop")
+	CMPQ(n, operand.Imm(0))
+	JE(operand.LabelRef("doneSIMD"))
+
+	Commentf("There are between 1 and %d bytes remaining. Perform an overlapped read.", simdReadSize-1)
+
+	maskDataPtr := GP64()
+	LEAQ(operand.NewDataAddr(operand.NewStaticSymbol("xmmLoadMasks"), 0), maskDataPtr)
+	dataAddr := operand.Mem{Index: n, Scale: 1, Base: buf, Disp: -simdReadSize}
+	// scale 8 is only correct here because n is guaranteed to be even and we
+	// do not generate masks for odd lengths
+	maskAddr := operand.Mem{Base: maskDataPtr, Index: n, Scale: 8, Disp: -16}
+	remainder := XMM()
+
+	switch strategy {
+	case sse2:
+		MOVOU(dataAddr, remainder)
+		PAND(maskAddr, remainder)
+		low := XMM()
+		MOVOA(remainder, low)
+		PUNPCKHWL(zero, remainder)
+		PUNPCKLWL(zero, low)
+		PADDD(remainder, simdAccumulate[0])
+		PADDD(low, simdAccumulate[1])
+	case avx2:
+		// Note: this is very similar to the sse2 path but MOVOU has a massive
+		// performance hit if used here, presumably due to switching between SSE
+		// and AVX2 modes.
+		VMOVDQU(dataAddr, remainder)
+		VPAND(maskAddr, remainder, remainder)
+
+		temp := YMM()
+		VPMOVZXWD(remainder, temp)
+		VPADDD(temp, simdAccumulate[0], simdAccumulate[0])
+	}
+
+	Label("doneSIMD")
+
+	Comment("Multi-chain loop is done, combine the accumulators")
+	for i := range simdAccumulate {
+		if i == 0 {
+			continue
+		}
+		switch strategy {
+		case sse2:
+			PADDD(simdAccumulate[i], simdAccumulate[0])
+		case avx2:
+			VPADDD(simdAccumulate[i], simdAccumulate[0], simdAccumulate[0])
+		}
+	}
+
+	if strategy == avx2 {
+		Comment("extract the YMM into a pair of XMM and sum them")
+		tmp := YMM()
+		VEXTRACTI128(operand.Imm(1), simdAccumulate[0], tmp.AsX())
+
+		xAccumulate := XMM()
+		VPADDD(simdAccumulate[0].AsX(), tmp.AsX(), xAccumulate)
+		simdAccumulate = []reg.VecVirtual{xAccumulate}
+	}
+
+	Comment("extract the XMM into GP64")
+	low, high := GP64(), GP64()
+	switch strategy {
+	case sse2:
+		MOVQ(simdAccumulate[0], low)
+		PSRLDQ(operand.Imm(8), simdAccumulate[0])
+		MOVQ(simdAccumulate[0], high)
+	case avx2:
+		VPEXTRQ(operand.Imm(0), simdAccumulate[0], low)
+		VPEXTRQ(operand.Imm(1), simdAccumulate[0], high)
+
+		Comment("no more AVX code, clear upper registers to avoid SSE slowdowns")
+		VZEROUPPER()
+	}
+	ADDQ(low, accum64)
+	ADCQ(high, accum64)
+	Label("foldAndReturn")
+	foldWithCF(accum64, strategy == avx2)
+	XCHGB(accum64.As8H(), accum64.As8L())
+	Store(accum64.As16(), ReturnIndex(0))
+	RET()
+}
+
+// handleOddLength generates instructions to incorporate the last byte into
+// accum64 if the length is odd. CF may be set if accum64 overflows; be sure to
+// handle that if overflow is possible.
+func handleOddLength(n, buf, accum64 reg.GPVirtual) {
+	Comment("handle odd length buffers; they are difficult to handle in general")
+	TESTQ(operand.U32(1), n)
+	JZ(operand.LabelRef("lengthIsEven"))
+
+	tmp := GP64()
+	MOVBQZX(operand.Mem{Base: buf, Index: n, Scale: 1, Disp: -1}, tmp)
+	DECQ(n)
+	ADDQ(tmp, accum64)
+
+	Label("lengthIsEven")
+}
+
+func sse2AccumulateStep(offset int, buf reg.GPVirtual, zero, accumulate1, accumulate2 reg.VecVirtual) {
+	high, low := XMM(), XMM()
+	MOVOU(operand.Mem{Disp: offset, Base: buf}, high)
+	MOVOA(high, low)
+	PUNPCKHWL(zero, high)
+	PUNPCKLWL(zero, low)
+	PADDD(high, accumulate1)
+	PADDD(low, accumulate2)
+}
+
+func avx2AccumulateStep(offset int, buf reg.GPVirtual, accumulate reg.VecVirtual) {
+	tmp := YMM()
+	VPMOVZXWD(operand.Mem{Disp: offset, Base: buf}, tmp)
+	VPADDD(tmp, accumulate, accumulate)
+}
+
+func generateAMD64Checksum(name, doc string) {
+	TEXT(name, NOSPLIT|NOFRAME, checksumSignature)
+	Pragma("noescape")
+	Doc(doc)
+
+	accum64, buf, n := loadParams()
+
+	handleOddLength(n, buf, accum64)
+	// no chance of overflow because accum64 was initialized by a uint16 and
+	// handleOddLength adds at most a uint8
+	handleTinyBuffers(n, buf, accum64, operand.LabelRef("foldAndReturn"), operand.LabelRef("bufferIsNotTiny"))
+	Label("bufferIsNotTiny")
+
+	const (
+		// numChains is the number of accumulators and carry counters to use.
+		// This improves speed via reduced data dependency. We combine the
+		// accumulators and carry counters once when the loop is complete.
+		numChains = 4
+		unroll    = 32         // The number of 64-bit reads to perform per iteration of the loop.
+		loopSize  = 8 * unroll // The number of bytes read per iteration of the loop.
+	)
+	if bits.Len(loopSize) != bits.Len(loopSize-1)+1 {
+		panic("loopSize is not a power of 2")
+	}
+	loopCount := GP64()
+
+	Comment(fmt.Sprintf("Number of %d byte iterations into loop counter", loopSize))
+	MOVQ(n, loopCount)
+	Comment("Update number of bytes remaining after the loop completes")
+	ANDQ(operand.Imm(loopSize-1), n)
+	SHRQ(operand.Imm(uint64(bits.Len(loopSize-1))), loopCount)
+	JZ(operand.LabelRef("startCleanup"))
+	CLC()
+
+	chains := make([]struct {
+		accum   reg.GPVirtual
+		carries reg.GPVirtual
+	}, numChains)
+	for i := range chains {
+		if i == 0 {
+			chains[i].accum = accum64
+		} else {
+			chains[i].accum = GP64()
+			XORQ(chains[i].accum, chains[i].accum)
+		}
+		chains[i].carries = GP64()
+		XORQ(chains[i].carries, chains[i].carries)
+	}
+
+	Label("bigLoop")
+
+	var curChain int
+	for i := 0; i < unroll; i++ {
+		// It is significantly faster to use a ADCX/ADOX pair instead of plain
+		// ADC, which results in two dependency chains, however those require
+		// ADX support, which was added after AVX2. If AVX2 is available, that's
+		// even better than ADCX/ADOX.
+		//
+		// However, multiple dependency chains using multiple accumulators and
+		// occasionally storing CF into temporary counters seems to work almost
+		// as well.
+		addr := operand.Mem{Disp: i * 8, Base: buf}
+
+		if i%4 == 0 {
+			if i > 0 {
+				ADCQ(operand.Imm(0), chains[curChain].carries)
+				curChain = (curChain + 1) % len(chains)
+			}
+			ADDQ(addr, chains[curChain].accum)
+		} else {
+			ADCQ(addr, chains[curChain].accum)
+		}
+	}
+	ADCQ(operand.Imm(0), chains[curChain].carries)
+	ADDQ(operand.U32(loopSize), buf)
+	SUBQ(operand.Imm(1), loopCount)
+	JNZ(operand.LabelRef("bigLoop"))
+	for i := range chains {
+		if i == 0 {
+			ADDQ(chains[i].carries, accum64)
+			continue
+		}
+		ADCQ(chains[i].accum, accum64)
+		ADCQ(chains[i].carries, accum64)
+	}
+
+	accumulateCF(accum64)
+
+	Label("startCleanup")
+	handleRemaining(n, buf, accum64, loopSize-1)
+	Label("foldAndReturn")
+	foldWithCF(accum64, false)
+
+	XCHGB(accum64.As8H(), accum64.As8L())
+	Store(accum64.As16(), ReturnIndex(0))
+	RET()
+}
+
+// handleTinyBuffers computes checksums if the buffer length (the n parameter)
+// is less than 32. After computing the checksum, a jump to returnLabel will
+// be executed. Otherwise, if the buffer length is at least 32, nothing will be
+// modified; a jump to continueLabel will be executed instead.
+//
+// When jumping to returnLabel, CF may be set and must be accommodated e.g.
+// using foldWithCF or accumulateCF.
+//
+// Anecdotally, this appears to be faster than attempting to coordinate an
+// overlapped read (which would also require special handling for buffers
+// smaller than 8).
+func handleTinyBuffers(n, buf, accum reg.GPVirtual, returnLabel, continueLabel operand.LabelRef) {
+	Comment("handle tiny buffers (<=31 bytes) specially")
+	CMPQ(n, operand.Imm(tinyBufferSize))
+	JGT(continueLabel)
+
+	tmp2, tmp4, tmp8 := GP64(), GP64(), GP64()
+	XORQ(tmp2, tmp2)
+	XORQ(tmp4, tmp4)
+	XORQ(tmp8, tmp8)
+
+	Comment("shift twice to start because length is guaranteed to be even",
+		"n = n >> 2; CF = originalN & 2")
+	SHRQ(operand.Imm(2), n)
+	JNC(operand.LabelRef("handleTiny4"))
+	Comment("tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:]")
+	MOVWQZX(operand.Mem{Base: buf}, tmp2)
+	ADDQ(operand.Imm(2), buf)
+
+	Label("handleTiny4")
+	Comment("n = n >> 1; CF = originalN & 4")
+	SHRQ(operand.Imm(1), n)
+	JNC(operand.LabelRef("handleTiny8"))
+	Comment("tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:]")
+	MOVLQZX(operand.Mem{Base: buf}, tmp4)
+	ADDQ(operand.Imm(4), buf)
+
+	Label("handleTiny8")
+	Comment("n = n >> 1; CF = originalN & 8")
+	SHRQ(operand.Imm(1), n)
+	JNC(operand.LabelRef("handleTiny16"))
+	Comment("tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:]")
+	MOVQ(operand.Mem{Base: buf}, tmp8)
+	ADDQ(operand.Imm(8), buf)
+
+	Label("handleTiny16")
+	Comment("n = n >> 1; CF = originalN & 16",
+		"n == 0 now, otherwise we would have branched after comparing with tinyBufferSize")
+	SHRQ(operand.Imm(1), n)
+	JNC(operand.LabelRef("handleTinyFinish"))
+	ADDQ(operand.Mem{Base: buf}, accum)
+	ADCQ(operand.Mem{Base: buf, Disp: 8}, accum)
+
+	Label("handleTinyFinish")
+	Comment("CF should be included from the previous add, so we use ADCQ.",
+		"If we arrived via the JNC above, then CF=0 due to the branch condition,",
+		"so ADCQ will still produce the correct result.")
+	ADCQ(tmp2, accum)
+	ADCQ(tmp4, accum)
+	ADCQ(tmp8, accum)
+
+	JMP(returnLabel)
+}
+
+// handleRemaining generates a series of conditional unrolled additions,
+// starting with 8 bytes long and doubling each time until the length reaches
+// max. This is the reverse order of what may be intuitive, but makes the branch
+// conditions convenient to compute: perform one right shift each time and test
+// against CF.
+//
+// When done, CF may be set and must be accommodated e.g., using foldWithCF or
+// accumulateCF.
+//
+// If n is not a multiple of 8, an extra 64 bit read at the end of the buffer
+// will be performed, overlapping with data that will be read later. The
+// duplicate data will be shifted off.
+//
+// The original buffer length must have been at least 8 bytes long, even if
+// n < 8, otherwise this will access memory before the start of the buffer,
+// which may be unsafe.
+func handleRemaining(n, buf, accum64 reg.GPVirtual, max int) {
+	Comment("Accumulate carries in this register. It is never expected to overflow.")
+	carries := GP64()
+	XORQ(carries, carries)
+
+	Comment("We will perform an overlapped read for buffers with length not a multiple of 8.",
+		"Overlapped in this context means some memory will be read twice, but a shift will",
+		"eliminate the duplicated data. This extra read is performed at the end of the buffer to",
+		"preserve any alignment that may exist for the start of the buffer.")
+	leftover := reg.RCX
+	MOVQ(n, leftover)
+	SHRQ(operand.Imm(3), n)          // n is now the number of 64 bit reads remaining
+	ANDQ(operand.Imm(0x7), leftover) // leftover is now the number of bytes to read from the end
+	JZ(operand.LabelRef("handleRemaining8"))
+	endBuf := GP64()
+	// endBuf is the position near the end of the buffer that is just past the
+	// last multiple of 8: (buf + len(buf)) & ^0x7
+	LEAQ(operand.Mem{Base: buf, Index: n, Scale: 8}, endBuf)
+
+	overlapRead := GP64()
+	// equivalent to overlapRead = binary.LittleEndian.Uint64(buf[len(buf)-8:len(buf)])
+	MOVQ(operand.Mem{Base: endBuf, Index: leftover, Scale: 1, Disp: -8}, overlapRead)
+
+	Comment("Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8)")
+	SHLQ(operand.Imm(3), leftover)  // leftover = leftover * 8
+	NEGQ(leftover)                  // leftover = -leftover; this completes the (-leftoverBytes*8) part of the expression
+	ADDQ(operand.Imm(64), leftover) // now we have (64 - leftoverBytes*8)
+	SHRQ(reg.CL, overlapRead)       // shift right by (64 - leftoverBytes*8); CL is the low 8 bits of leftover (set to RCX above) and variable shift only accepts CL
+
+	ADDQ(overlapRead, accum64)
+	ADCQ(operand.Imm(0), carries)
+
+	for curBytes := 8; curBytes <= max; curBytes *= 2 {
+		Label(fmt.Sprintf("handleRemaining%d", curBytes))
+		SHRQ(operand.Imm(1), n)
+		if curBytes*2 <= max {
+			JNC(operand.LabelRef(fmt.Sprintf("handleRemaining%d", curBytes*2)))
+		} else {
+			JNC(operand.LabelRef("handleRemainingComplete"))
+		}
+
+		numLoads := curBytes / 8
+		for i := 0; i < numLoads; i++ {
+			addr := operand.Mem{Base: buf, Disp: i * 8}
+			// It is possible to add the multiple dependency chains trick here
+			// that generateAMD64Checksum uses but anecdotally it does not
+			// appear to outweigh the cost.
+			if i == 0 {
+				ADDQ(addr, accum64)
+				continue
+			}
+			ADCQ(addr, accum64)
+		}
+		ADCQ(operand.Imm(0), carries)
+
+		if curBytes > math.MaxUint8 {
+			ADDQ(operand.U32(uint64(curBytes)), buf)
+		} else {
+			ADDQ(operand.U8(uint64(curBytes)), buf)
+		}
+		if curBytes*2 >= max {
+			continue
+		}
+		JMP(operand.LabelRef(fmt.Sprintf("handleRemaining%d", curBytes*2)))
+	}
+	Label("handleRemainingComplete")
+	ADDQ(carries, accum64)
+}
+
+func accumulateCF(accum64 reg.GPVirtual) {
+	Comment("accumulate CF (twice, in case the first time overflows)")
+	// accum64 += CF
+	ADCQ(operand.Imm(0), accum64)
+	// accum64 += CF again if the previous add overflowed. The previous add was
+	// 0 or 1. If it overflowed, then accum64 == 0, so adding another 1 can
+	// never overflow.
+	ADCQ(operand.Imm(0), accum64)
+}
+
+// foldWithCF generates instructions to fold accum (a GP64) into a 16-bit value
+// according to ones-complement arithmetic. BMI2 instructions will be used if
+// allowBMI2 is true (requires fewer instructions).
+func foldWithCF(accum reg.GPVirtual, allowBMI2 bool) {
+	Comment("add CF and fold")
+
+	// CF|accum max value starts as 0x1_ffff_ffff_ffff_ffff
+
+	tmp := GP64()
+	if allowBMI2 {
+		// effectively, tmp = accum >> 32 (technically, this is a rotate)
+		RORXQ(operand.Imm(32), accum, tmp)
+		// accum as uint32 = uint32(accum) + uint32(tmp64) + CF; max value 0xffff_ffff + CF set
+		ADCL(tmp.As32(), accum.As32())
+		// effectively, tmp64 as uint32 = uint32(accum) >> 16 (also a rotate)
+		RORXL(operand.Imm(16), accum.As32(), tmp.As32())
+		// accum as uint16 = uint16(accum) + uint16(tmp) + CF; max value 0xffff + CF unset or 0xfffe + CF set
+		ADCW(tmp.As16(), accum.As16())
+	} else {
+		// tmp = uint32(accum); max value 0xffff_ffff
+		// MOVL clears the upper 32 bits of a GP64 so this is equivalent to the
+		// non-existent MOVLQZX.
+		MOVL(accum.As32(), tmp.As32())
+		// tmp += CF; max value 0x1_0000_0000, CF unset
+		ADCQ(operand.Imm(0), tmp)
+		// accum = accum >> 32; max value 0xffff_ffff
+		SHRQ(operand.Imm(32), accum)
+		// accum = accum + tmp; max value 0x1_ffff_ffff + CF unset
+		ADDQ(tmp, accum)
+		// tmp = uint16(accum); max value 0xffff
+		MOVWQZX(accum.As16(), tmp)
+		// accum = accum >> 16; max value 0x1_ffff
+		SHRQ(operand.Imm(16), accum)
+		// accum = accum + tmp; max value 0x2_fffe + CF unset
+		ADDQ(tmp, accum)
+		// tmp as uint16 = uint16(accum); max value 0xffff
+		MOVW(accum.As16(), tmp.As16())
+		// accum = accum >> 16; max value 0x2
+		SHRQ(operand.Imm(16), accum)
+		// accum as uint16 = uint16(accum) + uint16(tmp); max value 0xffff + CF unset or 0x2 + CF set
+		ADDW(tmp.As16(), accum.As16())
+	}
+	// accum as uint16 += CF; will not overflow: either CF was 0 or accum <= 0xfffe
+	ADCW(operand.Imm(0), accum.As16())
+}
+
+func generateLoadMasks() {
+	var offset int
+	// xmmLoadMasks is a table of masks that can be used with PAND to zero all but the last N bytes in an XMM, N=2,4,6,8,10,12,14
+	GLOBL("xmmLoadMasks", RODATA|NOPTR)
+
+	for n := 2; n < 16; n += 2 {
+		var pattern [16]byte
+		for i := 0; i < len(pattern); i++ {
+			if i < len(pattern)-n {
+				pattern[i] = 0
+				continue
+			}
+			pattern[i] = 0xff
+		}
+		DATA(offset, operand.String(pattern[:]))
+		offset += len(pattern)
+	}
+}
+
+func main() {
+	generateLoadMasks()
+	generateSIMDChecksum("checksumAVX2", "checksumAVX2 computes an IP checksum using amd64 v3 instructions (AVX2, BMI2)", 256, 4, avx2)
+	generateSIMDChecksum("checksumSSE2", "checksumSSE2 computes an IP checksum using amd64 baseline instructions (SSE2)", 256, 4, sse2)
+	generateAMD64Checksum("checksumAMD64", "checksumAMD64 computes an IP checksum using amd64 baseline instructions")
+	Generate()
+}

tun/offload.go

@@ -55,25 +55,32 @@ type GSOOptions struct {
 }
 
 const (
-	gsoIPv4SrcAddrOffset = 12
-	gsoIPv6SrcAddrOffset = 8
-	gsoTCPFlagsOffset    = 13
-	gsoIPProtoTCP        = 6
-	gsoIPProtoUDP        = 17
+	ipv4SrcAddrOffset = 12
+	ipv6SrcAddrOffset = 8
+)
+
+const tcpFlagsOffset = 13
+
+const (
+	tcpFlagFIN uint8 = 0x01
+	tcpFlagPSH uint8 = 0x08
+	tcpFlagACK uint8 = 0x10
 )
 
 const (
-	gsoTCPFlagFIN uint8 = 0x01
-	gsoTCPFlagPSH uint8 = 0x08
+	// defined here in order to avoid importation of any platform-specific pkgs
+	ipProtoTCP = 6
+	ipProtoUDP = 17
 )
 
-// GSOSplit splits packets from in into outBufs[<index>][outOffset:], writing
+// GSOSplit splits packets from 'in' into outBufs[<index>][outOffset:], writing
 // the size of each element into sizes. It returns the number of buffers
-// populated, and/or an error. Callers may pass an in slice that overlaps with
-// the first element of outBufs, i.e. &in[0] may be equal to
+// populated, and/or an error. Callers may pass an 'in' slice that overlaps with
+// the first element of outBuffers, i.e. &in[0] may be equal to
 // &outBufs[0][outOffset]. GSONone is a valid options.GSOType regardless of the
 // value of options.NeedsCsum. Length of each outBufs element must be greater
-// than or equal to the length of in, otherwise output may be silently truncated.
+// than or equal to the length of 'in', otherwise output may be silently
+// truncated.
 func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outOffset int) (int, error) {
 	cSumAt := int(options.CsumStart) + int(options.CsumOffset)
 	if cSumAt+1 >= len(in) {
@@ -84,12 +91,15 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 		return 0, fmt.Errorf("length of packet (%d) < GSO HdrLen (%d)", len(in), options.HdrLen)
 	}
 
+	// Handle the conditions where we are copying a single element to outBuffs.
 	payloadLen := len(in) - int(options.HdrLen)
 	if options.GSOType == GSONone || payloadLen < int(options.GSOSize) {
 		if len(in) > len(outBufs[0][outOffset:]) {
 			return 0, fmt.Errorf("length of packet (%d) exceeds output element length (%d)", len(in), len(outBufs[0][outOffset:]))
 		}
 		if options.NeedsCsum {
+			// The initial value at the checksum offset should be summed with
+			// the checksum we compute. This is typically the pseudo-header sum.
 			initial := binary.BigEndian.Uint16(in[cSumAt:])
 			in[cSumAt], in[cSumAt+1] = 0, 0
 			binary.BigEndian.PutUint16(in[cSumAt:], ^Checksum(in[options.CsumStart:], initial))
@@ -123,24 +133,24 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 	}
 
 	iphLen := int(options.CsumStart)
-	srcAddrOffset := gsoIPv6SrcAddrOffset
+	srcAddrOffset := ipv6SrcAddrOffset
 	addrLen := 16
 	if ipVersion == 4 {
-		srcAddrOffset = gsoIPv4SrcAddrOffset
+		srcAddrOffset = ipv4SrcAddrOffset
 		addrLen = 4
 	}
 	transportCsumAt := int(options.CsumStart + options.CsumOffset)
 	var firstTCPSeqNum uint32
 	var protocol uint8
 	if options.GSOType == GSOTCPv4 || options.GSOType == GSOTCPv6 {
-		protocol = gsoIPProtoTCP
+		protocol = ipProtoTCP
 		if len(in) < int(options.CsumStart)+20 {
 			return 0, fmt.Errorf("length of packet (%d) < GSO CsumStart (%d) + minimum TCP header size (%d)",
 				len(in), options.CsumStart, 20)
 		}
 		firstTCPSeqNum = binary.BigEndian.Uint32(in[options.CsumStart+4:])
 	} else {
-		protocol = gsoIPProtoUDP
+		protocol = ipProtoUDP
 	}
 	nextSegmentDataAt := int(options.HdrLen)
 	i := 0
@@ -159,35 +169,45 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 
 		copy(out, in[:iphLen])
 		if ipVersion == 4 {
+			// For IPv4 we are responsible for incrementing the ID field,
+			// updating the total len field, and recalculating the header
+			// checksum.
 			if i > 0 {
 				id := binary.BigEndian.Uint16(out[4:])
 				id += uint16(i)
 				binary.BigEndian.PutUint16(out[4:], id)
 			}
-			out[10], out[11] = 0, 0
+			out[10], out[11] = 0, 0 // clear ipv4 header checksum
 			binary.BigEndian.PutUint16(out[2:], uint16(totalLen))
 			ipv4CSum := ^Checksum(out[:iphLen], 0)
 			binary.BigEndian.PutUint16(out[10:], ipv4CSum)
 		} else {
+			// For IPv6 we are responsible for updating the payload length field.
 			binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen))
 		}
 
+		// copy transport header
 		copy(out[options.CsumStart:options.HdrLen], in[options.CsumStart:options.HdrLen])
 
-		if protocol == gsoIPProtoTCP {
+		if protocol == ipProtoTCP {
+			// set TCP seq and adjust TCP flags
 			tcpSeq := firstTCPSeqNum + uint32(options.GSOSize*uint16(i))
 			binary.BigEndian.PutUint32(out[options.CsumStart+4:], tcpSeq)
 			if nextSegmentEnd != len(in) {
-				clearFlags := gsoTCPFlagFIN | gsoTCPFlagPSH
-				out[options.CsumStart+gsoTCPFlagsOffset] &^= clearFlags
+				// FIN and PSH should only be set on last segment
+				clearFlags := tcpFlagFIN | tcpFlagPSH
+				out[options.CsumStart+tcpFlagsOffset] &^= clearFlags
 			}
 		} else {
+			// set UDP header len
 			binary.BigEndian.PutUint16(out[options.CsumStart+4:], uint16(segmentDataLen)+(options.HdrLen-options.CsumStart))
 		}
 
+		// payload
 		copy(out[options.HdrLen:], in[nextSegmentDataAt:nextSegmentEnd])
 
-		out[transportCsumAt], out[transportCsumAt+1] = 0, 0
+		// transport checksum
+		out[transportCsumAt], out[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum
 		transportHeaderLen := int(options.HdrLen - options.CsumStart)
 		lenForPseudo := uint16(transportHeaderLen + segmentDataLen)
 		transportCSum := PseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo)

tun/offload_linux.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package tun
@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
 	"unsafe"
 
@@ -16,14 +17,6 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-const tcpFlagsOffset = 13
-
-const (
-	tcpFlagFIN uint8 = 0x01
-	tcpFlagPSH uint8 = 0x08
-	tcpFlagACK uint8 = 0x10
-)
-
 // virtioNetHdr is defined in the kernel in include/uapi/linux/virtio_net.h. The
 // kernel symbol is virtio_net_hdr.
 type virtioNetHdr struct {
@@ -35,6 +28,30 @@ type virtioNetHdr struct {
 	csumOffset uint16
 }
 
+func (v *virtioNetHdr) toGSOOptions() (GSOOptions, error) {
+	var gsoType GSOType
+	switch v.gsoType {
+	case unix.VIRTIO_NET_HDR_GSO_NONE:
+		gsoType = GSONone
+	case unix.VIRTIO_NET_HDR_GSO_TCPV4:
+		gsoType = GSOTCPv4
+	case unix.VIRTIO_NET_HDR_GSO_TCPV6:
+		gsoType = GSOTCPv6
+	case unix.VIRTIO_NET_HDR_GSO_UDP_L4:
+		gsoType = GSOUDPL4
+	default:
+		return GSOOptions{}, fmt.Errorf("unsupported virtio gsoType: %d", v.gsoType)
+	}
+	return GSOOptions{
+		GSOType:    gsoType,
+		HdrLen:     v.hdrLen,
+		CsumStart:  v.csumStart,
+		CsumOffset: v.csumOffset,
+		GSOSize:    v.gsoSize,
+		NeedsCsum:  v.flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0,
+	}, nil
+}
+
 func (v *virtioNetHdr) decode(b []byte) error {
 	if len(b) < virtioNetHdrLen {
 		return io.ErrShortBuffer
@@ -393,8 +410,8 @@ func checksumValid(pkt []byte, iphLen, proto uint8, isV6 bool) bool {
 		addrSize = 16
 	}
 	lenForPseudo := uint16(len(pkt) - int(iphLen))
-	cSum := pseudoHeaderChecksumNoFold(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo)
-	return ^checksum(pkt[iphLen:], cSum) == 0
+	cSum := PseudoHeaderChecksum(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo)
+	return ^Checksum(pkt[iphLen:], cSum) == 0
 }
 
 // coalesceResult represents the result of attempting to coalesce two TCP
@@ -510,9 +527,7 @@ const (
 )
 
 const (
-	ipv4SrcAddrOffset = 12
-	ipv6SrcAddrOffset = 8
-	maxUint16         = 1<<16 - 1
+	maxUint16 = 1<<16 - 1
 )
 
 type groResult int
@@ -644,7 +659,7 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e
 					hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4
 					pkt[10], pkt[11] = 0, 0
 					binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length
-					iphCSum := ^checksum(pkt[:item.iphLen], 0)            // compute IPv4 header checksum
+					iphCSum := ^Checksum(pkt[:item.iphLen], 0)            // compute IPv4 header checksum
 					binary.BigEndian.PutUint16(pkt[10:], iphCSum)         // set IPv4 header checksum field
 				}
 				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
@@ -664,8 +679,8 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e
 				srcAddrAt := offset + addrOffset
 				srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen]
 				dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2]
-				psum := pseudoHeaderChecksumNoFold(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen)))
-				binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum))
+				psum := PseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen)))
+				binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum))
 			} else {
 				hdr := virtioNetHdr{}
 				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
@@ -701,7 +716,7 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e
 				} else {
 					pkt[10], pkt[11] = 0, 0
 					binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length
-					iphCSum := ^checksum(pkt[:item.iphLen], 0)            // compute IPv4 header checksum
+					iphCSum := ^Checksum(pkt[:item.iphLen], 0)            // compute IPv4 header checksum
 					binary.BigEndian.PutUint16(pkt[10:], iphCSum)         // set IPv4 header checksum field
 				}
 				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
@@ -724,8 +739,8 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e
 				srcAddrAt := offset + addrOffset
 				srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen]
 				dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2]
-				psum := pseudoHeaderChecksumNoFold(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen)))
-				binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum))
+				psum := PseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen)))
+				binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum))
 			} else {
 				hdr := virtioNetHdr{}
 				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
@@ -894,100 +909,3 @@ func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGR
 	errUDP := applyUDPCoalesceAccounting(bufs, offset, udpTable)
 	return errors.Join(errTCP, errUDP)
 }
-
-// gsoSplit splits packets from in into outBuffs, writing the size of each
-// element into sizes. It returns the number of buffers populated, and/or an
-// error.
-func gsoSplit(in []byte, hdr virtioNetHdr, outBuffs [][]byte, sizes []int, outOffset int, isV6 bool) (int, error) {
-	iphLen := int(hdr.csumStart)
-	srcAddrOffset := ipv6SrcAddrOffset
-	addrLen := 16
-	if !isV6 {
-		in[10], in[11] = 0, 0 // clear ipv4 header checksum
-		srcAddrOffset = ipv4SrcAddrOffset
-		addrLen = 4
-	}
-	transportCsumAt := int(hdr.csumStart + hdr.csumOffset)
-	in[transportCsumAt], in[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum
-	var firstTCPSeqNum uint32
-	var protocol uint8
-	if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV4 || hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV6 {
-		protocol = unix.IPPROTO_TCP
-		firstTCPSeqNum = binary.BigEndian.Uint32(in[hdr.csumStart+4:])
-	} else {
-		protocol = unix.IPPROTO_UDP
-	}
-	nextSegmentDataAt := int(hdr.hdrLen)
-	i := 0
-	for ; nextSegmentDataAt < len(in); i++ {
-		if i == len(outBuffs) {
-			return i - 1, ErrTooManySegments
-		}
-		nextSegmentEnd := nextSegmentDataAt + int(hdr.gsoSize)
-		if nextSegmentEnd > len(in) {
-			nextSegmentEnd = len(in)
-		}
-		segmentDataLen := nextSegmentEnd - nextSegmentDataAt
-		totalLen := int(hdr.hdrLen) + segmentDataLen
-		sizes[i] = totalLen
-		out := outBuffs[i][outOffset:]
-
-		copy(out, in[:iphLen])
-		if !isV6 {
-			// For IPv4 we are responsible for incrementing the ID field,
-			// updating the total len field, and recalculating the header
-			// checksum.
-			if i > 0 {
-				id := binary.BigEndian.Uint16(out[4:])
-				id += uint16(i)
-				binary.BigEndian.PutUint16(out[4:], id)
-			}
-			binary.BigEndian.PutUint16(out[2:], uint16(totalLen))
-			ipv4CSum := ^checksum(out[:iphLen], 0)
-			binary.BigEndian.PutUint16(out[10:], ipv4CSum)
-		} else {
-			// For IPv6 we are responsible for updating the payload length field.
-			binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen))
-		}
-
-		// copy transport header
-		copy(out[hdr.csumStart:hdr.hdrLen], in[hdr.csumStart:hdr.hdrLen])
-
-		if protocol == unix.IPPROTO_TCP {
-			// set TCP seq and adjust TCP flags
-			tcpSeq := firstTCPSeqNum + uint32(hdr.gsoSize*uint16(i))
-			binary.BigEndian.PutUint32(out[hdr.csumStart+4:], tcpSeq)
-			if nextSegmentEnd != len(in) {
-				// FIN and PSH should only be set on last segment
-				clearFlags := tcpFlagFIN | tcpFlagPSH
-				out[hdr.csumStart+tcpFlagsOffset] &^= clearFlags
-			}
-		} else {
-			// set UDP header len
-			binary.BigEndian.PutUint16(out[hdr.csumStart+4:], uint16(segmentDataLen)+(hdr.hdrLen-hdr.csumStart))
-		}
-
-		// payload
-		copy(out[hdr.hdrLen:], in[nextSegmentDataAt:nextSegmentEnd])
-
-		// transport checksum
-		transportHeaderLen := int(hdr.hdrLen - hdr.csumStart)
-		lenForPseudo := uint16(transportHeaderLen + segmentDataLen)
-		transportCSumNoFold := pseudoHeaderChecksumNoFold(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo)
-		transportCSum := ^checksum(out[hdr.csumStart:totalLen], transportCSumNoFold)
-		binary.BigEndian.PutUint16(out[hdr.csumStart+hdr.csumOffset:], transportCSum)
-
-		nextSegmentDataAt += int(hdr.gsoSize)
-	}
-	return i, nil
-}
-
-func gsoNoneChecksum(in []byte, cSumStart, cSumOffset uint16) error {
-	cSumAt := cSumStart + cSumOffset
-	// The initial value at the checksum offset should be summed with the
-	// checksum we compute. This is typically the pseudo-header checksum.
-	initial := binary.BigEndian.Uint16(in[cSumAt:])
-	in[cSumAt], in[cSumAt+1] = 0, 0
-	binary.BigEndian.PutUint16(in[cSumAt:], ^checksum(in[cSumStart:], uint64(initial)))
-	return nil
-}

tun/tun.go

@@ -56,12 +56,21 @@ type Device interface {
 // versions may have offload bugs. Where these bugs negatively impact throughput
 // or break connectivity entirely we can use these methods to disable the
 // related offload.
+//
+// Linux has the following known, GRO bugs.
+//
+// torvalds/linux@e269d79c7d35aa3808b1f3c1737d63dab504ddc8 broke virtio_net
+// TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then
+// resolved later in
+// torvalds/linux@89add40066f9ed9abe5f7f886fe5789ff7e0c50e. The offending
+// commit was pulled into various LTS releases.
+//
+// UDP GRO writes end up blackholing/dropping packets destined for a
+// vxlan/geneve interface on kernel versions prior to 6.8.5.
 type GRODevice interface {
 	Device
-
 	// DisableUDPGRO disables UDP GRO if it is enabled.
 	DisableUDPGRO()
-
 	// DisableTCPGRO disables TCP GRO if it is enabled.
 	DisableTCPGRO()
 }

tun/tun_linux.go

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: MIT
  *
- * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved.
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
  */
 
 package tun
@@ -48,7 +48,7 @@ type NativeTun struct {
 	readOpMu sync.Mutex                    // readOpMu guards readBuff
 	readBuff [virtioNetHdrLen + 65535]byte // if vnetHdr every read() is prefixed by virtioNetHdr
 
-	writeOpMu   sync.Mutex // writeOpMu guards toWrite, tcpGROTable, udpGROTable, gro
+	writeOpMu   sync.Mutex // writeOpMu guards the following fields
 	toWrite     []int
 	tcpGROTable *tcpGROTable
 	udpGROTable *udpGROTable
@@ -269,21 +269,15 @@ func (tun *NativeTun) setMTU(n int) error {
 
 	defer unix.Close(fd)
 
-	// do ioctl call
-	var ifr [ifReqSize]byte
-	copy(ifr[:], name)
-	*(*uint32)(unsafe.Pointer(&ifr[unix.IFNAMSIZ])) = uint32(n)
-	_, _, errno := unix.Syscall(
-		unix.SYS_IOCTL,
-		uintptr(fd),
-		uintptr(unix.SIOCSIFMTU),
-		uintptr(unsafe.Pointer(&ifr[0])),
-	)
-
-	if errno != 0 {
-		return fmt.Errorf("failed to set MTU of TUN device: %w", errno)
+	req, err := unix.NewIfreq(name)
+	if err != nil {
+		return fmt.Errorf("unix.NewIfreq(%q): %w", name, err)
+	}
+	req.SetUint32(uint32(n))
+	err = unix.IoctlIfreq(fd, unix.SIOCSIFMTU, req)
+	if err != nil {
+		return fmt.Errorf("failed to set MTU of TUN device %q: %w", name, err)
 	}
-
 	return nil
 }
 
@@ -402,73 +396,32 @@ func handleVirtioRead(in []byte, bufs [][]byte, sizes []int, offset int) (int, e
 		return 0, err
 	}
 	in = in[virtioNetHdrLen:]
-	if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_NONE {
-		if hdr.flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0 {
-			// This means CHECKSUM_PARTIAL in skb context. We are responsible
-			// for computing the checksum starting at hdr.csumStart and placing
-			// at hdr.csumOffset.
-			err = gsoNoneChecksum(in, hdr.csumStart, hdr.csumOffset)
-			if err != nil {
-				return 0, err
-			}
-		}
-		if len(in) > len(bufs[0][offset:]) {
-			return 0, fmt.Errorf("read len %d overflows bufs element len %d", len(in), len(bufs[0][offset:]))
-		}
-		n := copy(bufs[0][offset:], in)
-		sizes[0] = n
-		return 1, nil
-	}
-	if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 {
-		return 0, fmt.Errorf("unsupported virtio GSO type: %d", hdr.gsoType)
+
+	options, err := hdr.toGSOOptions()
+	if err != nil {
+		return 0, err
 	}
 
-	ipVersion := in[0] >> 4
-	switch ipVersion {
-	case 4:
-		if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 {
-			return 0, fmt.Errorf("ip header version: %d, GSO type: %d", ipVersion, hdr.gsoType)
-		}
-	case 6:
-		if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 {
-			return 0, fmt.Errorf("ip header version: %d, GSO type: %d", ipVersion, hdr.gsoType)
-		}
-	default:
-		return 0, fmt.Errorf("invalid ip header version: %d", ipVersion)
-	}
-
-	// Don't trust hdr.hdrLen from the kernel as it can be equal to the length
+	// Don't trust HdrLen from the kernel as it can be equal to the length
 	// of the entire first packet when the kernel is handling it as part of a
 	// FORWARD path. Instead, parse the transport header length and add it onto
-	// csumStart, which is synonymous for IP header length.
-	if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_UDP_L4 {
-		hdr.hdrLen = hdr.csumStart + 8
-	} else {
-		if len(in) <= int(hdr.csumStart+12) {
+	// CsumStart, which is synonymous for IP header length.
+	if options.GSOType == GSOUDPL4 {
+		options.HdrLen = options.CsumStart + 8
+	} else if options.GSOType != GSONone {
+		if len(in) <= int(options.CsumStart+12) {
 			return 0, errors.New("packet is too short")
 		}
 
-		tcpHLen := uint16(in[hdr.csumStart+12] >> 4 * 4)
+		tcpHLen := uint16(in[options.CsumStart+12] >> 4 * 4)
 		if tcpHLen < 20 || tcpHLen > 60 {
 			// A TCP header must be between 20 and 60 bytes in length.
 			return 0, fmt.Errorf("tcp header len is invalid: %d", tcpHLen)
 		}
-		hdr.hdrLen = hdr.csumStart + tcpHLen
+		options.HdrLen = options.CsumStart + tcpHLen
 	}
 
-	if len(in) < int(hdr.hdrLen) {
-		return 0, fmt.Errorf("length of packet (%d) < virtioNetHdr.hdrLen (%d)", len(in), hdr.hdrLen)
-	}
-
-	if hdr.hdrLen < hdr.csumStart {
-		return 0, fmt.Errorf("virtioNetHdr.hdrLen (%d) < virtioNetHdr.csumStart (%d)", hdr.hdrLen, hdr.csumStart)
-	}
-	cSumAt := int(hdr.csumStart + hdr.csumOffset)
-	if cSumAt+1 >= len(in) {
-		return 0, fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(in))
-	}
-
-	return gsoSplit(in, hdr, bufs, sizes, offset, ipVersion == 6)
+	return GSOSplit(in, options, bufs, sizes, offset)
 }
 
 func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
@@ -525,14 +478,16 @@ func (tun *NativeTun) BatchSize() int {
 	return tun.batchSize
 }
 
-// DisableUDPGRO disables UDP GRO if it is enabled.
+// DisableUDPGRO disables UDP GRO if it is enabled. See the GRODevice interface
+// for cases where it should be called.
 func (tun *NativeTun) DisableUDPGRO() {
 	tun.writeOpMu.Lock()
 	tun.gro.disableUDPGRO()
 	tun.writeOpMu.Unlock()
 }
 
-// DisableTCPGRO disables TCP GRO if it is enabled.
+// DisableTCPGRO disables TCP GRO if it is enabled. See the GRODevice interface
+// for cases where it should be called.
 func (tun *NativeTun) DisableTCPGRO() {
 	tun.writeOpMu.Lock()
 	tun.gro.disableTCPGRO()

tun/tun_plan9.go

@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
+ */
+
+package tun
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type NativeTun struct {
+	name      string // "/net/ipifc/2"
+	ctlFile   *os.File
+	dataFile  *os.File
+	events    chan Event
+	errors    chan error
+	closeOnce sync.Once
+}
+
+func CreateTUN(_ string, mtu int) (Device, error) {
+	ctl, err := os.OpenFile("/net/ipifc/clone", os.O_RDWR, 0)
+	if err != nil {
+		return nil, err
+	}
+	nbuf := make([]byte, 5)
+	n, err := ctl.Read(nbuf)
+	if err != nil {
+		ctl.Close()
+		return nil, fmt.Errorf("error reading from clone file: %w", err)
+	}
+	ifn, err := strconv.Atoi(strings.TrimSpace(string(nbuf[:n])))
+	if err != nil {
+		ctl.Close()
+		return nil, fmt.Errorf("error converting clone result %q to int: %w", nbuf[:n], err)
+	}
+
+	if _, err := fmt.Fprintf(ctl, "bind pkt\n"); err != nil {
+		ctl.Close()
+		return nil, fmt.Errorf("error binding to pkt: %w", err)
+	}
+	if mtu > 0 {
+		if _, err := fmt.Fprintf(ctl, "mtu %d\n", mtu); err != nil {
+			ctl.Close()
+			return nil, fmt.Errorf("error setting MTU: %w", err)
+		}
+	}
+
+	dataFile, err := os.OpenFile(fmt.Sprintf("/net/ipifc/%d/data", ifn), os.O_RDWR, 0)
+	if err != nil {
+		ctl.Close()
+		return nil, err
+	}
+
+	tun := &NativeTun{
+		ctlFile:  ctl,
+		dataFile: dataFile,
+		name:     fmt.Sprintf("/net/ipifc/%d", ifn),
+		events:   make(chan Event, 10),
+		errors:   make(chan error, 5),
+	}
+	tun.events <- EventUp
+
+	return tun, nil
+}
+
+func (tun *NativeTun) Name() (string, error) {
+	return tun.name, nil
+}
+
+func (tun *NativeTun) File() *os.File {
+	return tun.ctlFile
+}
+
+func (tun *NativeTun) Events() <-chan Event {
+	return tun.events
+}
+
+func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+	select {
+	case err := <-tun.errors:
+		return 0, err
+	default:
+		n, err := tun.dataFile.Read(bufs[0][offset:])
+		if n == 1 && bufs[0][offset] == 0 {
+			// EOF
+			err = io.EOF
+			n = 0
+		}
+		sizes[0] = n
+		return 1, err
+	}
+}
+
+func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) {
+	for i, buf := range bufs {
+		if _, err := tun.dataFile.Write(buf[offset:]); err != nil {
+			return i, err
+		}
+	}
+	return len(bufs), nil
+}
+
+func (tun *NativeTun) Close() error {
+	var err1, err2 error
+	tun.closeOnce.Do(func() {
+		_, err1 := fmt.Fprintf(tun.ctlFile, "unbind\n")
+		if err := tun.ctlFile.Close(); err != nil && err1 == nil {
+			err1 = err
+		}
+		err2 = tun.dataFile.Close()
+	})
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+func (tun *NativeTun) MTU() (int, error) {
+	var buf [100]byte
+	f, err := os.Open(tun.name + "/status")
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	n, err := f.Read(buf[:])
+	_, res, ok := strings.Cut(string(buf[:n]), " maxtu ")
+	if ok {
+		if mtus, _, ok := strings.Cut(res, " "); ok {
+			mtu, err := strconv.Atoi(mtus)
+			if err != nil {
+				return 0, fmt.Errorf("error converting mtu %q to int: %w", mtus, err)
+			}
+			return mtu, nil
+		}
+	}
+	return 0, fmt.Errorf("no 'maxtu' field found in %s/status", tun.name)
+}
+
+func (tun *NativeTun) BatchSize() int {
+	return 1
+}