diff --git a/.github/workflows/build-if-tag.yml b/.github/workflows/build-if-tag.yml deleted file mode 100644 index 4fa0198..0000000 --- a/.github/workflows/build-if-tag.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: build-if-tag - -on: - push: - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - -env: - APP: amneziawg-go - -jobs: - build: - runs-on: ubuntu-latest - name: build - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.ref_name }} - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Setup metadata - uses: docker/metadata-action@v5 - id: metadata - with: - images: amneziavpn/${{ env.APP }} - tags: type=semver,pattern={{version}} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build - uses: docker/build-push-action@v5 - with: - push: true - tags: ${{ steps.metadata.outputs.tags }} diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 98a7e9e..0000000 --- a/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM golang:1.24.4 as awg -COPY . /awg -WORKDIR /awg -RUN go mod download && \ - go mod verify && \ - go build -ldflags '-linkmode external -extldflags "-fno-PIC -static"' -v -o /usr/bin - -FROM alpine:3.19 -ARG AWGTOOLS_RELEASE="1.0.20250901" - -RUN apk --no-cache add iproute2 iptables bash && \ - cd /usr/bin/ && \ - wget https://github.com/amnezia-vpn/amneziawg-tools/releases/download/v${AWGTOOLS_RELEASE}/alpine-3.19-amneziawg-tools.zip && \ - unzip -j alpine-3.19-amneziawg-tools.zip && \ - chmod +x /usr/bin/awg /usr/bin/awg-quick && \ - ln -s /usr/bin/awg /usr/bin/wg && \ - ln -s /usr/bin/awg-quick /usr/bin/wg-quick -COPY --from=awg /usr/bin/amneziawg-go /usr/bin/amneziawg-go diff --git a/LICENSE b/LICENSE index ab45fb3..f85e365 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,3 @@ -Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. - Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to diff --git a/Makefile b/Makefile index 7a88647..3f6e407 100644 --- a/Makefile +++ b/Makefile @@ -9,23 +9,23 @@ MAKEFLAGS += --no-print-directory generate-version-and-build: @export GIT_CEILING_DIRECTORIES="$(realpath $(CURDIR)/..)" && \ - tag="$$(git describe --tags --dirty 2>/dev/null)" && \ + tag="$$(git describe --dirty 2>/dev/null)" && \ ver="$$(printf 'package main\n\nconst Version = "%s"\n' "$$tag")" && \ [ "$$(cat version.go 2>/dev/null)" != "$$ver" ] && \ echo "$$ver" > version.go && \ git update-index --assume-unchanged version.go || true - @$(MAKE) amneziawg-go + @$(MAKE) wireguard-go -amneziawg-go: $(wildcard *.go) $(wildcard */*.go) +wireguard-go: $(wildcard *.go) $(wildcard */*.go) go build -v -o "$@" -install: amneziawg-go - @install -v -d "$(DESTDIR)$(BINDIR)" && install -v -m 0755 "$<" "$(DESTDIR)$(BINDIR)/amneziawg-go" +install: wireguard-go + @install -v -d "$(DESTDIR)$(BINDIR)" && install -v -m 0755 "$<" "$(DESTDIR)$(BINDIR)/wireguard-go" test: go test ./... clean: - rm -f amneziawg-go + rm -f wireguard-go .PHONY: all clean test install generate-version-and-build diff --git a/README.md b/README.md deleted file mode 100644 index dd92535..0000000 --- a/README.md +++ /dev/null @@ -1,93 +0,0 @@ -**English** · [Русский](README.ru.md) - -# wireguard-go (lx fork) — sagernet + AmneziaWG 2.0 - -The WireGuard-Go runtime used by **[sing-box-lx](https://github.com/Leadaxe/sing-box-lx)**: -**[sagernet/wireguard-go](https://github.com/sagernet/wireguard-go)** (the fork sing-box builds on) **+ AmneziaWG 2.0 obfuscation**, merged together. - -This is **not** a general-purpose project. It exists for one reason — see below — and lives on the **`lx`** branch. - ---- - -## Why this fork exists - -sing-box's WireGuard endpoint needs **sagernet/wireguard-go**'s additions (the `conn.Bind.Send(…, offset)` contract, `device.InputPacket`, reserved/control). AmneziaWG's DPI-evasion obfuscation lives in **[amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go)**, which is a fork of *upstream* wireguard-go and therefore **lacks** those sagernet additions. - -So neither fork alone works for sing-box-lx: - -| | sing-box-compat API | AmneziaWG obfuscation | -|---|:---:|:---:| -| `sagernet/wireguard-go` | ✅ | ❌ | -| `amnezia-vpn/amneziawg-go` | ❌ | ✅ | -| **this fork** | ✅ | ✅ | - -Each existing fork gives exactly **half** of what's needed: - -- **Take `sagernet/wireguard-go`** → sing-box-lx compiles and runs, but the AWG fields (`jc`/`h1`/`i1`…) do nothing → **no obfuscation**; AmneziaWG doesn't actually work. -- **Take `amnezia-vpn/amneziawg-go`** → the obfuscation is there, but sing-box-lx **won't even compile** (the sagernet functions are missing). - -We need **both** ✅ at once, and no ready-made fork has them — so we built one by **merging**: sagernet (for the API) + amnezia (for the obfuscation). That is exactly the **"this fork"** row above. - -The approach: **keep the sagernet base and graft the obfuscation onto it** — rather than the reverse (adding sagernet's APIs to amneziawg-go, which would route even plain WireGuard through a foreign device). This way sing-box compiles unchanged, the obfuscation is additive and off by default, and a config without AWG fields behaves exactly like plain WireGuard. - -## How the merge works - -Both `sagernet/wireguard-go` and `amneziawg-go` descend from the same upstream `git.zx2c4.com/wireguard-go`, so they share git history — which makes a real **3-way merge** possible (not a hand-port). - -- **Base:** `sagernet/wireguard-go` (the exact commit sing-box pins — currently `506b7631853c`). -- **Merged in:** `amnezia-vpn/amneziawg-go` (a tip with AWG2 / I1–I5 + the S4-keepalive fix). -- **Key trick:** `MessageEncapsulatingTransportSize` is set to **`0`** in `device/noise-protocol.go`. sing-box-lx does not use sagernet's 8-byte `Bind.Send` headroom, and zeroing it makes the AmneziaWG obfuscation compose cleanly with no weave conflicts in the packet send path. -- **Isolation:** the obfuscation is confined to `device/` — new files `device/obf*.go`, `device/magic-header.go`, plus grafts in `device/{send,receive,device,uapi}.go`. **`conn/`, `tun/`, `ipc/` stay pure sagernet.** -- **Module path is unchanged** (`module github.com/sagernet/wireguard-go`) so the consumer plugs it in with a `replace` directive and needs **no import edits**. - -## Consumed by - -[sing-box-lx](https://github.com/Leadaxe/sing-box-lx) wires this in as a git submodule + a `replace`: - -``` -# sing-box-lx/.gitmodules -[submodule "submodules/wireguard-go"] - url = https://github.com/Leadaxe/wireguard-go-awg2-lx - branch = lx - -# sing-box-lx/go.mod (// lx) -replace github.com/sagernet/wireguard-go => ./submodules/wireguard-go -``` - -Built with the `with_awg` tag, it has been **live-validated** against a real AmneziaWG 2.0 server (handshake + keepalive + outbound traffic) and cross-compiles on linux/darwin/windows × amd64/arm64. - -## Maintaining it (rebase onto a new sagernet tag) - -When sing-box bumps `sagernet/wireguard-go`, redo the merge: - -```sh -git remote add origin https://github.com/sagernet/wireguard-go # base -git remote add amnezia https://github.com/amnezia-vpn/amneziawg-go # obfuscation source -git fetch --all -git checkout -b lx -git merge amnezia/master # real 3-way merge via the shared upstream ancestor -``` - -Conflict resolution recipe: - -1. New `device/obf*.go` + `device/magic-header.go` come in clean. -2. **Mechanical** conflicts (amnezia → sagernet import paths, `queueconstants*`, `sticky*`, `tun.go`) → take **ours** (sagernet). -3. Remove amnezia-added infra duplicates: `conn/gso_*.go`, `outline/*`, `tun/*_test.go`. -4. `device/device.go` → **union** (sagernet `pauseManager` + amnezia obf fields). -5. `device/send.go` / `receive.go` → take amnezia's obfuscation, set `MessageEncapsulatingTransportSize = 0`, and keep the 3-arg `bind.Send(…, 0)` calls. -6. `conn/`, `tun/`, `ipc/`, `go.mod` module path → **ours** (sagernet). - -Then in sing-box-lx: bump the submodule, `make -f Makefile.lx lx-build`, and re-test against an AWG2 server. - -## Links - -| | | -|---|---| -| Consumer | [Leadaxe/sing-box-lx](https://github.com/Leadaxe/sing-box-lx) | -| Base | [sagernet/wireguard-go](https://github.com/sagernet/wireguard-go) | -| Obfuscation source | [amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go) · [docs.amnezia.org](https://docs.amnezia.org/documentation/amnezia-wg/) | -| Original | [WireGuard/wireguard-go](https://git.zx2c4.com/wireguard-go/about/) | - -## License - -MIT, inherited from WireGuard-Go (see [`LICENSE`](LICENSE)). The AmneziaWG obfuscation is likewise MIT (from amneziawg-go). This is an unofficial fork, not affiliated with WireGuard, SagerNet, or Amnezia. diff --git a/README.ru.md b/README.ru.md deleted file mode 100644 index 7482bd4..0000000 --- a/README.ru.md +++ /dev/null @@ -1,93 +0,0 @@ -[English](README.md) · **Русский** - -# wireguard-go (lx-форк) — sagernet + AmneziaWG 2.0 - -Рантайм WireGuard-Go для **[sing-box-lx](https://github.com/Leadaxe/sing-box-lx)**: -**[sagernet/wireguard-go](https://github.com/sagernet/wireguard-go)** (форк, на котором собирается sing-box) **+ обфускация AmneziaWG 2.0**, слитые вместе. - -Это **не** универсальный проект. Он существует ради одной задачи (см. ниже) и живёт на ветке **`lx`**. - ---- - -## Зачем этот форк - -WireGuard-endpoint sing-box нуждается в добавках **sagernet/wireguard-go** (контракт `conn.Bind.Send(…, offset)`, `device.InputPacket`, reserved/control). Обфускация против DPI у AmneziaWG живёт в **[amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go)**, который форкнут от *upstream* wireguard-go и потому этих sagernet-добавок **не имеет**. - -Значит для sing-box-lx ни один форк по отдельности не подходит: - -| | API под sing-box | обфускация AmneziaWG | -|---|:---:|:---:| -| `sagernet/wireguard-go` | ✅ | ❌ | -| `amnezia-vpn/amneziawg-go` | ❌ | ✅ | -| **этот форк** | ✅ | ✅ | - -Каждый существующий форк даёт ровно **половину** нужного: - -- **возьмёшь `sagernet/wireguard-go`** → sing-box-lx соберётся и запустится, но AWG-поля (`jc`/`h1`/`i1`…) ничего не сделают → **обфускации нет**; AmneziaWG не работает; -- **возьмёшь `amnezia-vpn/amneziawg-go`** → обфускация есть, но sing-box-lx **не скомпилируется** (нет sagernet-функций). - -Нужны **обе** ✅ сразу, а готового форка с двумя галочками не существует — поэтому мы собрали его **слиянием**: sagernet (за API) + amnezia (за обфускацию). Это ровно строка **«этот форк»** выше. - -Подход: **берём sagernet-базу и граффтим обфускацию на неё** — а не наоборот (не дотачиваем sagernet-API к amneziawg-go, иначе даже обычный WireGuard шёл бы через чужой device). Так sing-box компилируется без изменений, обфускация аддитивна и выключена по умолчанию, а конфиг без AWG-полей ведёт себя как обычный WireGuard. - -## Как устроен merge - -И `sagernet/wireguard-go`, и `amneziawg-go` происходят от одного upstream `git.zx2c4.com/wireguard-go`, поэтому делят git-историю — а значит возможен настоящий **3-way merge** (а не ручной перенос). - -- **База:** `sagernet/wireguard-go` (тот коммит, что пинит sing-box — сейчас `506b7631853c`). -- **Вливаем:** `amnezia-vpn/amneziawg-go` (тип с AWG2 / I1–I5 + фикс S4-keepalive). -- **Ключевой трюк:** `MessageEncapsulatingTransportSize` выставлен в **`0`** в `device/noise-protocol.go`. sing-box-lx не использует 8-байтный headroom sagernet для `Bind.Send`, и обнуление позволяет обфускации AmneziaWG встать чисто, без конфликтов в send-пути. -- **Изоляция:** обфускация замкнута в `device/` — новые файлы `device/obf*.go`, `device/magic-header.go` + графты в `device/{send,receive,device,uapi}.go`. **`conn/`, `tun/`, `ipc/` остаются чистым sagernet.** -- **Module-path не меняется** (`module github.com/sagernet/wireguard-go`), поэтому потребитель подключает форк через `replace` без правки импортов. - -## Кто потребляет - -[sing-box-lx](https://github.com/Leadaxe/sing-box-lx) подключает это как git submodule + `replace`: - -``` -# sing-box-lx/.gitmodules -[submodule "submodules/wireguard-go"] - url = https://github.com/Leadaxe/wireguard-go-awg2-lx - branch = lx - -# sing-box-lx/go.mod (// lx) -replace github.com/sagernet/wireguard-go => ./submodules/wireguard-go -``` - -Собранный с тегом `with_awg`, он **проверен живым** сервером AmneziaWG 2.0 (handshake + keepalive + трафик наружу) и кросс-компилируется на linux/darwin/windows × amd64/arm64. - -## Сопровождение (ребейз на новый sagernet-тег) - -Когда sing-box бампит `sagernet/wireguard-go`, повторяем merge: - -```sh -git remote add origin https://github.com/sagernet/wireguard-go # база -git remote add amnezia https://github.com/amnezia-vpn/amneziawg-go # источник обфускации -git fetch --all -git checkout -b lx <новый-sagernet-коммит> -git merge amnezia/master # настоящий 3-way merge через общего upstream-предка -``` - -Рецепт разрешения конфликтов: - -1. Новые `device/obf*.go` + `device/magic-header.go` приходят чисто. -2. **Механические** конфликты (import-path amnezia → sagernet, `queueconstants*`, `sticky*`, `tun.go`) → берём **наши** (sagernet). -3. Удаляем amnezia-инфра-дубликаты: `conn/gso_*.go`, `outline/*`, `tun/*_test.go`. -4. `device/device.go` → **union** (sagernet `pauseManager` + obf-поля amnezia). -5. `device/send.go` / `receive.go` → берём обфускацию amnezia, ставим `MessageEncapsulatingTransportSize = 0`, сохраняем 3-арг `bind.Send(…, 0)`. -6. `conn/`, `tun/`, `ipc/`, module-path в `go.mod` → **наши** (sagernet). - -Затем в sing-box-lx: бампим submodule, `make -f Makefile.lx lx-build` и пере-тест против AWG2-сервера. - -## Ссылки - -| | | -|---|---| -| Потребитель | [Leadaxe/sing-box-lx](https://github.com/Leadaxe/sing-box-lx) | -| База | [sagernet/wireguard-go](https://github.com/sagernet/wireguard-go) | -| Источник обфускации | [amnezia-vpn/amneziawg-go](https://github.com/amnezia-vpn/amneziawg-go) · [docs.amnezia.org](https://docs.amnezia.org/documentation/amnezia-wg/) | -| Оригинал | [WireGuard/wireguard-go](https://git.zx2c4.com/wireguard-go/about/) | - -## Лицензия - -MIT, унаследована от WireGuard-Go (см. [`LICENSE`](LICENSE)). Обфускация AmneziaWG — тоже MIT (из amneziawg-go). Это неофициальный форк, не аффилирован с WireGuard, SagerNet или Amnezia. diff --git a/conn/bind_std.go b/conn/bind_std.go index 6bf5978..eb27e10 100644 --- a/conn/bind_std.go +++ b/conn/bind_std.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package conn @@ -23,10 +23,13 @@ import ( "golang.org/x/net/ipv6" ) -var ( - _ Bind = (*StdNetBind)(nil) - _ Endpoint = (*StdNetEndpoint)(nil) -) +type EgressProvider interface { + SetEgressPort(port uint16) bool + LookupEgress(destination netip.AddrPort) *net.UDPConn + ReceiveEgress(buffer []byte) (int, netip.AddrPort, error) +} + +var _ Bind = (*StdNetBind)(nil) // StdNetBind implements Bind for all platforms. While Windows has its own Bind // (see bind_windows.go), it may fall back to StdNetBind. @@ -35,6 +38,7 @@ var ( // proposal in https://github.com/golang/go/issues/45886#issuecomment-1218301564. type StdNetBind struct { externalControl control.Func + egressProvider EgressProvider reservedForEndpoint map[netip.AddrPort][3]uint8 mu sync.Mutex // protects all fields except as specified @@ -42,6 +46,8 @@ type StdNetBind struct { ipv6 *net.UDPConn ipv4PC *ipv4.PacketConn // will be nil on non-Linux ipv6PC *ipv6.PacketConn // will be nil on non-Linux + ipv4RC syscall.RawConn // will be nil on non-Darwin + ipv6RC syscall.RawConn // will be nil on non-Darwin ipv4TxOffload bool ipv4RxOffload bool ipv6TxOffload bool @@ -51,6 +57,8 @@ type StdNetBind struct { udpAddrPool sync.Pool msgsPool sync.Pool + msgx msgXState + blackhole4 bool blackhole6 bool } @@ -70,10 +78,12 @@ func NewStdNetBind(externalControl control.Func) Bind { msgsPool: sync.Pool{ New: func() any { + // ipv6.Message and ipv4.Message are interchangeable as they are + // both aliases for x/net/internal/socket.Message. msgs := make([]ipv6.Message, IdealBatchSize) for i := range msgs { msgs[i].Buffers = make(net.Buffers, 1) - msgs[i].OOB = make([]byte, controlSize) + msgs[i].OOB = make([]byte, 0, stickyControlSize+gsoControlSize) } return &msgs }, @@ -116,7 +126,7 @@ func (e *StdNetEndpoint) DstIP() netip.Addr { return e.AddrPort.Addr() } -// See sticky_default,linux, etc for implementations of SrcIP and SrcIfidx. +// See control_default,linux, etc for implementations of SrcIP and SrcIfidx. func (e *StdNetEndpoint) DstToBytes() []byte { b, _ := e.AddrPort.MarshalBinary() @@ -166,11 +176,6 @@ func listenNet(externalControl control.Func, network string, port int) (*net.UDP return conn.(*net.UDPConn), uaddr.Port, nil } -// errEADDRINUSE is syscall.EADDRINUSE, boxed into an interface once -// in erraddrinuse.go on almost all platforms. For other platforms, -// it's at least non-nil. -var errEADDRINUSE error = errors.New("") - func (s *StdNetBind) Open(uport uint16) ([]ReceiveFunc, uint16, error) { s.mu.Lock() defer s.mu.Unlock() @@ -181,6 +186,7 @@ func (s *StdNetBind) Open(uport uint16) ([]ReceiveFunc, uint16, error) { if s.ipv4 != nil || s.ipv6 != nil { return nil, 0, ErrBindAlreadyOpen } + s.msgx.reset() // Attempt to open ipv4 and ipv6 listeners on the same port. // If uport is 0, we can retry on failure. @@ -197,7 +203,7 @@ again: // Listen on the same port as we're using for ipv4. v6conn, port, err = listenNet(s.externalControl, "udp6", port) - if uport == 0 && errors.Is(err, errEADDRINUSE) && tries < 100 { + if uport == 0 && errors.Is(err, syscall.EADDRINUSE) && tries < 100 { v4conn.Close() tries++ goto again @@ -209,32 +215,85 @@ again: var fns []ReceiveFunc if v4conn != nil { s.ipv4TxOffload, s.ipv4RxOffload = supportsUDPOffload(v4conn) - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "android" { v4pc = ipv4.NewPacketConn(v4conn) s.ipv4PC = v4pc } - fns = append(fns, s.makeReceiveIPv4(v4pc, v4conn, s.ipv4RxOffload)) + if supportsMsgX { + var receiveFn ReceiveFunc + receiveFn, err = s.makeReceiveMsgX(v4conn, false) + if err != nil { + v4conn.Close() + return nil, 0, err + } + s.ipv4RC, err = v4conn.SyscallConn() + if err != nil { + v4conn.Close() + return nil, 0, err + } + fns = append(fns, receiveFn) + } else { + fns = append(fns, s.makeReceiveIPv4(v4pc, v4conn, s.ipv4RxOffload)) + } s.ipv4 = v4conn } if v6conn != nil { s.ipv6TxOffload, s.ipv6RxOffload = supportsUDPOffload(v6conn) - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "android" { v6pc = ipv6.NewPacketConn(v6conn) s.ipv6PC = v6pc } - fns = append(fns, s.makeReceiveIPv6(v6pc, v6conn, s.ipv6RxOffload)) + if supportsMsgX { + var receiveFn ReceiveFunc + receiveFn, err = s.makeReceiveMsgX(v6conn, true) + if err != nil { + v6conn.Close() + return nil, 0, err + } + s.ipv6RC, err = v6conn.SyscallConn() + if err != nil { + v6conn.Close() + return nil, 0, err + } + fns = append(fns, receiveFn) + } else { + fns = append(fns, s.makeReceiveIPv6(v6pc, v6conn, s.ipv6RxOffload)) + } s.ipv6 = v6conn } if len(fns) == 0 { return nil, 0, syscall.EAFNOSUPPORT } + if s.egressProvider != nil { + s.egressProvider.SetEgressPort(uint16(port)) + fns = append(fns, func(bufs [][]byte, sizes []int, endpoints []Endpoint) (int, error) { + dataLength, source, err := s.egressProvider.ReceiveEgress(bufs[0]) + if err != nil { + return 0, err + } + sizes[0] = dataLength + if dataLength > 3 && s.hasReserved() { // lx: SPEC 026 — gate reserved-clear on the egress receive path too, so a small-padding AmneziaWG magic in bytes 1-3 survives when no WARP reserved value is set + common.ClearArray(bufs[0][1:4]) + } + endpoints[0] = &StdNetEndpoint{AddrPort: source} + return 1, nil + }) + } return fns, uint16(port), nil } +func (s *StdNetBind) SetEgressProvider(provider EgressProvider) { + s.egressProvider = provider +} + func (s *StdNetBind) putMessages(msgs *[]ipv6.Message) { for i := range *msgs { - (*msgs)[i] = ipv6.Message{Buffers: (*msgs)[i].Buffers, OOB: (*msgs)[i].OOB} + buffers := (*msgs)[i].Buffers + for j := range buffers { + buffers[j] = nil + } + (*msgs)[i] = ipv6.Message{Buffers: buffers[:1], OOB: (*msgs)[i].OOB[:0]} } s.msgsPool.Put(msgs) } @@ -269,9 +328,9 @@ func (s *StdNetBind) receiveIP( } defer s.putMessages(msgs) var numMsgs int - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "android" { if rxOffload { - readAt := len(*msgs) - 2 + readAt := len(*msgs) - (IdealBatchSize / udpSegmentMaxDatagrams) numMsgs, err = br.ReadBatch((*msgs)[readAt:], 0) if err != nil { return 0, err @@ -300,7 +359,7 @@ func (s *StdNetBind) receiveIP( if sizes[i] == 0 { continue } - if msg.N > 3 { + if msg.N > 3 && s.hasReserved() { // lx: only strip reserved bytes for WARP (see hasReserved) common.ClearArray(bufs[i][1:4]) } ep := &StdNetEndpoint{AddrPort: M.AddrPortFromNet(msg.Addr)} // TODO: remove allocation @@ -325,9 +384,12 @@ func (s *StdNetBind) makeReceiveIPv6(pc *ipv6.PacketConn, conn *net.UDPConn, rxO // TODO: When all Binds handle IdealBatchSize, remove this dynamic function and // rename the IdealBatchSize constant to BatchSize. func (s *StdNetBind) BatchSize() int { - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "android" { return IdealBatchSize } + if supportsMsgX { + return msgXBatchSize + } return 1 } @@ -335,6 +397,9 @@ func (s *StdNetBind) Close() error { s.mu.Lock() defer s.mu.Unlock() + if s.egressProvider != nil { + s.egressProvider.SetEgressPort(0) + } var err1, err2 error if s.ipv4 != nil { err1 = s.ipv4.Close() @@ -372,13 +437,21 @@ func (e ErrUDPGSODisabled) Unwrap() error { } func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error { + for len(bufs) > IdealBatchSize { + err := s.Send(bufs[:IdealBatchSize], endpoint, offset) + if err != nil { + return err + } + bufs = bufs[IdealBatchSize:] + } + standardEndpoint := endpoint.(*StdNetEndpoint) s.mu.Lock() blackhole := s.blackhole4 conn := s.ipv4 offload := s.ipv4TxOffload br := batchWriter(s.ipv4PC) is6 := false - if endpoint.DstIP().Is6() { + if standardEndpoint.DstIP().Is6() { blackhole = s.blackhole6 conn = s.ipv6 br = s.ipv6PC @@ -399,30 +472,42 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error { ua := s.udpAddrPool.Get().(*net.UDPAddr) defer s.udpAddrPool.Put(ua) if is6 { - as16 := endpoint.DstIP().As16() + as16 := standardEndpoint.DstIP().As16() copy(ua.IP, as16[:]) ua.IP = ua.IP[:16] } else { - as4 := endpoint.DstIP().As4() + as4 := standardEndpoint.DstIP().As4() copy(ua.IP, as4[:]) ua.IP = ua.IP[:4] } - ua.Port = int(endpoint.(*StdNetEndpoint).Port()) + ua.Port = int(standardEndpoint.Port()) var ( retried bool err error ) for _, buf := range bufs { if len(buf) > offset+3 { - reserved, loaded := s.reservedForEndpoint[endpoint.(*StdNetEndpoint).AddrPort] + reserved, loaded := s.reservedForEndpoint[standardEndpoint.AddrPort] if loaded { copy(buf[offset+1:offset+4], reserved[:]) } } } + if s.egressProvider != nil { + memberConn := s.egressProvider.LookupEgress(standardEndpoint.AddrPort) + if memberConn != nil { + for _, buf := range bufs { + _, err = memberConn.WriteToUDPAddrPort(buf[offset:], standardEndpoint.AddrPort) + if err != nil { + return err + } + } + return nil + } + } retry: if offload { - n := coalesceMessages(ua, endpoint.(*StdNetEndpoint), bufs, offset, *msgs, setGSOSize) + n := coalesceMessages(ua, standardEndpoint, bufs, offset, *msgs, setGSOSize) err = s.send(conn, br, (*msgs)[:n]) if err != nil && offload && errShouldDisableUDPGSO(err) { offload = false @@ -440,7 +525,7 @@ retry: for i := range bufs { (*msgs)[i].Addr = ua (*msgs)[i].Buffers[0] = bufs[i][offset:] - setSrcControl(&(*msgs)[i].OOB, endpoint.(*StdNetEndpoint)) + setSrcControl(&(*msgs)[i].OOB, standardEndpoint) } err = s.send(conn, br, (*msgs)[:len(bufs)]) } @@ -454,13 +539,27 @@ func (s *StdNetBind) SetReservedForEndpoint(destination netip.AddrPort, reserved s.reservedForEndpoint[destination] = reserved } +// lx: hasReserved reports whether any Cloudflare "reserved" value is set. The +// receive path must only zero bytes 1-3 when a reserved value exists (WARP); +// otherwise an AmneziaWG magic header that lands in bytes 1-3 (small s1/s2/s4 +// padding) would be corrupted and the packet dropped. The send path already +// gates its stamp on a per-endpoint `loaded` check, so no change is needed there. +func (s *StdNetBind) hasReserved() bool { + for _, reserved := range s.reservedForEndpoint { + if reserved != [3]uint8{} { + return true + } + } + return false +} + func (s *StdNetBind) send(conn *net.UDPConn, pc batchWriter, msgs []ipv6.Message) error { var ( n int err error start int ) - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "android" { for { n, err = pc.WriteBatch(msgs[start:], 0) if err != nil || n == len(msgs[start:]) { @@ -469,6 +568,12 @@ func (s *StdNetBind) send(conn *net.UDPConn, pc batchWriter, msgs []ipv6.Message start += n } } else { + if supportsMsgX { + handled, sendErr := s.sendMsgX(conn, msgs) + if handled { + return sendErr + } + } for _, msg := range msgs { _, _, err = conn.WriteMsgUDP(msg.Buffers[0], msg.OOB, msg.Addr.(*net.UDPAddr)) if err != nil { @@ -496,6 +601,7 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs var ( base = -1 // index of msg we are currently coalescing into gsoSize int // segmentation size of msgs[base] + totalLen int // length of all dgrams coalesced into msgs[base] dgramCnt int // number of dgrams coalesced into msgs[base] endBatch bool // tracking flag to start a new batch on next iteration of bufs ) @@ -507,14 +613,14 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs buf = buf[offset:] if i > 0 { msgLen := len(buf) - baseLenBefore := len(msgs[base].Buffers[0]) - freeBaseCap := cap(msgs[base].Buffers[0]) - baseLenBefore - if msgLen+baseLenBefore <= maxPayloadLen && + if msgLen+totalLen <= maxPayloadLen && msgLen <= gsoSize && - msgLen <= freeBaseCap && dgramCnt < udpSegmentMaxDatagrams && !endBatch { - msgs[base].Buffers[0] = append(msgs[base].Buffers[0], buf...) + // Coalesce as an additional iovec instead of copying: element + // buffers are sized to their packet and have no spare capacity. + msgs[base].Buffers = append(msgs[base].Buffers, buf) + totalLen += msgLen if i == len(bufs)-1 { setGSO(&msgs[base].OOB, uint16(gsoSize)) } @@ -535,8 +641,9 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs endBatch = false base++ gsoSize = len(buf) + totalLen = len(buf) setSrcControl(&msgs[base].OOB, ep) - msgs[base].Buffers[0] = buf + msgs[base].Buffers = append(msgs[base].Buffers[:0], buf) msgs[base].Addr = addr dgramCnt = 1 } diff --git a/conn/bind_windows.go b/conn/bind_windows.go index 51c0974..c31bb35 100644 --- a/conn/bind_windows.go +++ b/conn/bind_windows.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package conn @@ -461,7 +461,7 @@ func (bind *WinRingBind) receiveIPv4(bufs [][]byte, sizes []int, eps []Endpoint) bind.mu.RLock() defer bind.mu.RUnlock() n, ep, err := bind.v4.Receive(bufs[0], &bind.isOpen) - if n > 3 { + if n > 3 && bind.hasReserved() { // lx: only strip reserved bytes for WARP (see hasReserved) common.ClearArray(bufs[0][1:4]) } sizes[0] = n @@ -473,7 +473,7 @@ func (bind *WinRingBind) receiveIPv6(bufs [][]byte, sizes []int, eps []Endpoint) bind.mu.RLock() defer bind.mu.RUnlock() n, ep, err := bind.v6.Receive(bufs[0], &bind.isOpen) - if n > 3 { + if n > 3 && bind.hasReserved() { // lx: only strip reserved bytes for WARP (see hasReserved) common.ClearArray(bufs[0][1:4]) } sizes[0] = n @@ -576,6 +576,18 @@ func (bind *WinRingBind) SetReservedForEndpoint(destination netip.AddrPort, rese bind.reservedForEndpoint[*endpoint.(*WinRingEndpoint)] = reserved } +// lx: hasReserved reports whether any Cloudflare "reserved" value is set. See +// the StdNetBind.hasReserved comment — the unconditional receive clear would +// corrupt an AmneziaWG magic header sitting in bytes 1-3 (small padding). +func (bind *WinRingBind) hasReserved() bool { + for _, reserved := range bind.reservedForEndpoint { + if reserved != [3]uint8{} { + return true + } + } + return false +} + func (s *StdNetBind) BindSocketToInterface4(interfaceIndex uint32, blackhole bool) error { s.mu.Lock() defer s.mu.Unlock() diff --git a/conn/controlfns_linux.go b/conn/controlfns_linux.go index ff591f1..f0deefa 100644 --- a/conn/controlfns_linux.go +++ b/conn/controlfns_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package conn @@ -13,6 +13,35 @@ import ( "golang.org/x/sys/unix" ) +// Taken from go/src/internal/syscall/unix/kernel_version_linux.go +func kernelVersion() (major, minor int) { + var uname unix.Utsname + if err := unix.Uname(&uname); err != nil { + return + } + + var ( + values [2]int + value, vi int + ) + for _, c := range uname.Release { + if '0' <= c && c <= '9' { + value = (value * 10) + int(c-'0') + } else { + // Note that we're assuming N.N.N here. + // If we see anything else, we are likely to mis-parse it. + values[vi] = value + vi++ + if vi >= len(values) { + break + } + value = 0 + } + } + + return values[0], values[1] +} + func init() { controlFns = append(controlFns, @@ -60,17 +89,19 @@ func init() { // Attempt to enable UDP_GRO func(network, address string, c syscall.RawConn) error { - // lx(010): skip UDP_GRO on android. The GRO receive path in bind_std.go - // is gated on runtime.GOOS=="linux", which is false on android — so a - // coalesced super-packet is never split and corrupts the WG stream - // (download dies). Belt-and-suspenders with the rxOffload guard in - // features_linux.go. TX/GSO untouched. - // See SPECS/010-WG_ENDPOINT_GRO_SPLIT_BRAIN. - if runtime.GOOS == "android" { + // Kernels below 5.12 are missing 98184612aca0 ("net: + // udp: Add support for getsockopt(..., ..., UDP_GRO, + // ..., ...);"), which means we can't read this back + // later. We could pipe the return value through to + // the rest of the code, but UDP_GRO is kind of buggy + // anyway, so just gate this here. + major, minor := kernelVersion() + if major < 5 || (major == 5 && minor < 12) { return nil } + c.Control(func(fd uintptr) { - _ = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO, 1) + _ = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_GRO, 1) }) return nil }, diff --git a/conn/controlfns_unix.go b/conn/controlfns_unix.go index 2bfccab..b2e7570 100644 --- a/conn/controlfns_unix.go +++ b/conn/controlfns_unix.go @@ -1,4 +1,4 @@ -//go:build !windows && !linux && !wasm && !plan9 && !tamago +//go:build !windows && !linux && !wasm /* SPDX-License-Identifier: MIT * diff --git a/conn/erraddrinuse.go b/conn/erraddrinuse.go deleted file mode 100644 index a6563a4..0000000 --- a/conn/erraddrinuse.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build !plan9 - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. - */ - -package conn - -import "syscall" - -func init() { - errEADDRINUSE = syscall.EADDRINUSE -} diff --git a/conn/errors_default.go b/conn/errors_default.go index d967518..3c9b223 100644 --- a/conn/errors_default.go +++ b/conn/errors_default.go @@ -7,6 +7,6 @@ package conn -func errShouldDisableUDPGSO(err error) bool { +func errShouldDisableUDPGSO(_ error) bool { return false } diff --git a/conn/errors_linux.go b/conn/errors_linux.go index 8e61000..037d820 100644 --- a/conn/errors_linux.go +++ b/conn/errors_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package conn diff --git a/conn/features_default.go b/conn/features_default.go index d53ff5f..9fc5088 100644 --- a/conn/features_default.go +++ b/conn/features_default.go @@ -3,13 +3,13 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package conn import "net" -func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) { +func supportsUDPOffload(_ *net.UDPConn) (txOffload, rxOffload bool) { return } diff --git a/conn/features_linux.go b/conn/features_linux.go index 69dd2f4..6386023 100644 --- a/conn/features_linux.go +++ b/conn/features_linux.go @@ -1,49 +1,26 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package conn import ( "net" - "runtime" "golang.org/x/sys/unix" ) -const ( - // TODO: upstream to x/sys/unix - socketOptionLevelUDP = 17 - socketOptionUDPSegment = 103 - socketOptionUDPGRO = 104 -) - func supportsUDPOffload(conn *net.UDPConn) (txOffload, rxOffload bool) { rc, err := conn.SyscallConn() if err != nil { return } err = rc.Control(func(fd uintptr) { - _, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPSegment) - if errSyscall != nil { - return - } - txOffload = true - // lx(010): never advertise RX offload on android. runtime.GOOS=="android" - // (not "linux"), so the GRO receive dispatcher in bind_std.go (gated on - // GOOS=="linux") is dead there — a coalesced GRO super-packet would be read - // as one datagram and corrupt the WG transport stream, killing download. - // Confirmed on device (CPH2411/Android-15: rxOffload=true, dispatch=single). - // TX is left untouched. See SPECS/010-WG_ENDPOINT_GRO_SPLIT_BRAIN. - if runtime.GOOS == "android" { - return - } - opt, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, socketOptionUDPGRO) - if errSyscall != nil { - return - } - rxOffload = opt == 1 + _, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_SEGMENT) + txOffload = errSyscall == nil + opt, errSyscall := unix.GetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_GRO) + rxOffload = errSyscall == nil && opt == 1 }) if err != nil { return false, false diff --git a/conn/gso_default.go b/conn/gso_default.go new file mode 100644 index 0000000..a9a3e80 --- /dev/null +++ b/conn/gso_default.go @@ -0,0 +1,21 @@ +//go:build !linux + +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + */ + +package conn + +// getGSOSize parses control for UDP_GRO and if found returns its GSO size data. +func getGSOSize(control []byte) (int, error) { + return 0, nil +} + +// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. +func setGSOSize(control *[]byte, gsoSize uint16) { +} + +// gsoControlSize returns the recommended buffer size for pooling sticky and UDP +// offloading control data. +const gsoControlSize = 0 diff --git a/conn/gso_linux.go b/conn/gso_linux.go new file mode 100644 index 0000000..4ee31fa --- /dev/null +++ b/conn/gso_linux.go @@ -0,0 +1,65 @@ +//go:build linux + +/* SPDX-License-Identifier: MIT + * + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. + */ + +package conn + +import ( + "fmt" + "unsafe" + + "golang.org/x/sys/unix" +) + +const ( + sizeOfGSOData = 2 +) + +// getGSOSize parses control for UDP_GRO and if found returns its GSO size data. +func getGSOSize(control []byte) (int, error) { + var ( + hdr unix.Cmsghdr + data []byte + rem = control + err error + ) + + for len(rem) > unix.SizeofCmsghdr { + hdr, data, rem, err = unix.ParseOneSocketControlMessage(rem) + if err != nil { + return 0, fmt.Errorf("error parsing socket control message: %w", err) + } + if hdr.Level == unix.SOL_UDP && hdr.Type == unix.UDP_GRO && len(data) >= sizeOfGSOData { + var gso uint16 + copy(unsafe.Slice((*byte)(unsafe.Pointer(&gso)), sizeOfGSOData), data[:sizeOfGSOData]) + return int(gso), nil + } + } + return 0, nil +} + +// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. It leaves existing +// data in control untouched. +func setGSOSize(control *[]byte, gsoSize uint16) { + existingLen := len(*control) + avail := cap(*control) - existingLen + space := unix.CmsgSpace(sizeOfGSOData) + if avail < space { + return + } + *control = (*control)[:cap(*control)] + gsoControl := (*control)[existingLen:] + hdr := (*unix.Cmsghdr)(unsafe.Pointer(&(gsoControl)[0])) + hdr.Level = unix.SOL_UDP + hdr.Type = unix.UDP_SEGMENT + hdr.SetLen(unix.CmsgLen(sizeOfGSOData)) + copy((gsoControl)[unix.CmsgLen(0):], unsafe.Slice((*byte)(unsafe.Pointer(&gsoSize)), sizeOfGSOData)) + *control = (*control)[:existingLen+space] +} + +// gsoControlSize returns the recommended buffer size for pooling UDP +// offloading control data. +var gsoControlSize = unix.CmsgSpace(sizeOfGSOData) diff --git a/conn/msgx_darwin.go b/conn/msgx_darwin.go new file mode 100644 index 0000000..da9bb07 --- /dev/null +++ b/conn/msgx_darwin.go @@ -0,0 +1,325 @@ +// On iOS both directions misbehave in the Network Extension (recvmsg_x on +// unconnected UDP sockets delivers no data, connected sockets stop passing +// traffic after a rebind), so msgx is macOS only until it can be debugged +// on a device. + +//go:build darwin && !ios + +package conn + +import ( + "net" + "net/netip" + "sync" + "sync/atomic" + "syscall" + "unsafe" + + M "github.com/sagernet/sing/common/metadata" + "golang.org/x/net/ipv6" + "golang.org/x/sys/unix" +) + +const supportsMsgX = true + +const msgXBatchSize = IdealBatchSize + +// msghdrX mirrors XNU's struct msghdr_x used by sendmsg_x/recvmsg_x. +// Per bsd/sys/socket_private.h, sendmsg_x supports neither addresses nor +// ancillary data (msg_name and msg_control must be zero), so batched sends +// require a connected socket. recvmsg_x does fill in per-message source +// addresses (copyout_maddr in uipc_syscalls.c). utun cannot use the send +// side at all (no ctl_send_list in if_utun.c). +type msghdrX struct { + Msg unix.Msghdr + DataLen uint32 +} + +type msgXState struct { + singlePeer atomic.Bool + disabled atomic.Bool // permanent fallback to the generic paths + connected4 atomic.Bool + connected6 atomic.Bool + endpoint atomic.Pointer[StdNetEndpoint] + connectLock sync.Mutex +} + +// reset clears per-socket state; must be called when the bind (re)opens, +// as the connected state belongs to the previous sockets. +func (m *msgXState) reset() { + m.disabled.Store(false) + m.connected4.Store(false) + m.connected6.Store(false) + m.endpoint.Store(nil) +} + +func (m *msgXState) connectedFlag(isV6 bool) *atomic.Bool { + if isV6 { + return &m.connected6 + } + return &m.connected4 +} + +// SetSinglePeerMode enables connected-socket sendmsg_x batching. Only safe +// when the bind serves exactly one peer with a fixed endpoint: the kernel +// will drop datagrams from any other source, so peer roaming stops working. +func (s *StdNetBind) SetSinglePeerMode() { + s.msgx.singlePeer.Store(true) +} + +func sockaddrFromAddrPort(addrPort netip.AddrPort, storage4 *unix.RawSockaddrInet4, storage6 *unix.RawSockaddrInet6) (unsafe.Pointer, uint32) { + port := addrPort.Port()<<8 | addrPort.Port()>>8 + if addrPort.Addr().Unmap().Is4() { + *storage4 = unix.RawSockaddrInet4{ + Len: unix.SizeofSockaddrInet4, + Family: unix.AF_INET, + Port: port, + Addr: addrPort.Addr().Unmap().As4(), + } + return unsafe.Pointer(storage4), unix.SizeofSockaddrInet4 + } + *storage6 = unix.RawSockaddrInet6{ + Len: unix.SizeofSockaddrInet6, + Family: unix.AF_INET6, + Port: port, + Addr: addrPort.Addr().As16(), + } + return unsafe.Pointer(storage6), unix.SizeofSockaddrInet6 +} + +// ensureConnected connects the family socket to the single peer on first +// use, and permanently falls back if a second endpoint shows up. +func (s *StdNetBind) ensureConnected(rawConn syscall.RawConn, isV6 bool, destination netip.AddrPort) bool { + if s.msgx.disabled.Load() || !s.msgx.singlePeer.Load() { + return false + } + connected := s.msgx.connectedFlag(isV6) + if connected.Load() { + if s.msgx.endpoint.Load().AddrPort == destination { + return true + } + s.msgx.connectLock.Lock() + defer s.msgx.connectLock.Unlock() + if s.msgx.disabled.Load() { + return false + } + s.msgx.disabled.Store(true) + var disconnectErr error + controlErr := rawConn.Control(func(fd uintptr) { + addr := unix.RawSockaddrAny{} + addr.Addr.Family = unix.AF_UNSPEC + //nolint:staticcheck + _, _, errno := unix.Syscall(unix.SYS_CONNECT, fd, uintptr(unsafe.Pointer(&addr)), unix.SizeofSockaddrAny) + if errno != 0 && errno != unix.EAFNOSUPPORT { + disconnectErr = errno + } + }) + if controlErr == nil && disconnectErr == nil { + connected.Store(false) + } + return false + } + s.msgx.connectLock.Lock() + defer s.msgx.connectLock.Unlock() + if s.msgx.disabled.Load() { + return false + } + if connected.Load() { + return s.msgx.endpoint.Load().AddrPort == destination + } + var ( + storage4 unix.RawSockaddrInet4 + storage6 unix.RawSockaddrInet6 + connectErr unix.Errno + ) + name, nameLen := sockaddrFromAddrPort(destination, &storage4, &storage6) + controlErr := rawConn.Control(func(fd uintptr) { + //nolint:staticcheck + _, _, connectErr = unix.Syscall(unix.SYS_CONNECT, fd, uintptr(name), uintptr(nameLen)) + }) + if controlErr != nil || connectErr != 0 { + s.msgx.disabled.Store(true) + return false + } + s.msgx.endpoint.Store(&StdNetEndpoint{AddrPort: destination}) + connected.Store(true) + return true +} + +type sendMsgXState struct { + hdrs []msghdrX + iovs []unix.Iovec +} + +var sendMsgXPool = sync.Pool{New: func() any { + return &sendMsgXState{ + hdrs: make([]msghdrX, IdealBatchSize), + iovs: make([]unix.Iovec, IdealBatchSize), + } +}} + +// sendMsgX sends msgs via sendmsg_x when the socket is connected to their +// endpoint. handled == false means nothing was sent and the caller must use +// the generic path; msgs are never partially consumed in that case. +func (s *StdNetBind) sendMsgX(conn *net.UDPConn, msgs []ipv6.Message) (bool, error) { + var ( + rawConn syscall.RawConn + isV6 bool + ) + s.mu.Lock() + if conn == s.ipv6 { + rawConn = s.ipv6RC + isV6 = true + } else { + rawConn = s.ipv4RC + } + s.mu.Unlock() + if rawConn == nil { + return false, nil + } + destination := M.AddrPortFromNet(msgs[0].Addr) + if !s.ensureConnected(rawConn, isV6, destination) { + return false, nil + } + state := sendMsgXPool.Get().(*sendMsgXState) + defer sendMsgXPool.Put(state) + for i := range msgs { + buffer := msgs[i].Buffers[0] + state.iovs[i] = unix.Iovec{Base: &buffer[0]} + state.iovs[i].SetLen(len(buffer)) + state.hdrs[i] = msghdrX{} + state.hdrs[i].Msg.Iov = &state.iovs[i] + state.hdrs[i].Msg.Iovlen = 1 + } + var sent int + for sent < len(msgs) { + var ( + n uintptr + errno unix.Errno + ) + writeErr := rawConn.Write(func(fd uintptr) bool { + //nolint:staticcheck + n, _, errno = unix.RawSyscall6(unix.SYS_SENDMSG_X, fd, + uintptr(unsafe.Pointer(&state.hdrs[sent])), uintptr(len(msgs)-sent), unix.MSG_DONTWAIT, 0, 0) + return errno != unix.EAGAIN + }) + if writeErr != nil { + return true, writeErr + } + if errno != 0 { + if sent == 0 { + // The syscall is refusing this socket entirely (sandbox, + // disconnected by the system, ...): disable and let the + // caller resend everything on the generic path. + s.msgx.disabled.Store(true) + return false, nil + } + return true, errno + } + sent += int(n) + } + return true, nil +} + +type receiveMsgXState struct { + hdrs []msghdrX + iovs []unix.Iovec + names []unix.RawSockaddrInet6 + fallback bool +} + +func (s *StdNetBind) receiveSingle(conn *net.UDPConn, bufs [][]byte, sizes []int, eps []Endpoint) (int, error) { + n, _, _, addr, err := conn.ReadMsgUDPAddrPort(bufs[0], nil) + if err != nil { + return 0, err + } + sizes[0] = n + if n > 3 && s.hasReserved() { // lx: only strip reserved bytes for WARP (see hasReserved) + bufs[0][1] = 0 + bufs[0][2] = 0 + bufs[0][3] = 0 + } + eps[0] = &StdNetEndpoint{AddrPort: addr} + return 1, nil +} + +func (s *StdNetBind) makeReceiveMsgX(conn *net.UDPConn, isV6 bool) (ReceiveFunc, error) { + rawConn, err := conn.SyscallConn() + if err != nil { + return nil, err + } + state := &receiveMsgXState{ + hdrs: make([]msghdrX, msgXBatchSize), + iovs: make([]unix.Iovec, msgXBatchSize), + names: make([]unix.RawSockaddrInet6, msgXBatchSize), + } + return func(bufs [][]byte, sizes []int, eps []Endpoint) (int, error) { + if state.fallback || s.msgx.disabled.Load() { + return s.receiveSingle(conn, bufs, sizes, eps) + } + connectedEndpoint := s.msgx.endpoint.Load() + if !s.msgx.connectedFlag(isV6).Load() { + connectedEndpoint = nil + } + count := len(bufs) + if count > msgXBatchSize { + count = msgXBatchSize + } + for i := 0; i < count; i++ { + state.iovs[i] = unix.Iovec{Base: &bufs[i][0]} + state.iovs[i].SetLen(len(bufs[i])) + state.hdrs[i] = msghdrX{} + if connectedEndpoint == nil { + state.hdrs[i].Msg.Name = (*byte)(unsafe.Pointer(&state.names[i])) + state.hdrs[i].Msg.Namelen = unix.SizeofSockaddrInet6 + } + state.hdrs[i].Msg.Iov = &state.iovs[i] + state.hdrs[i].Msg.Iovlen = 1 + } + var ( + n uintptr + errno unix.Errno + ) + readErr := rawConn.Read(func(fd uintptr) bool { + //nolint:staticcheck + n, _, errno = unix.RawSyscall6(unix.SYS_RECVMSG_X, fd, + uintptr(unsafe.Pointer(&state.hdrs[0])), uintptr(count), unix.MSG_DONTWAIT, 0, 0) + return errno != unix.EAGAIN + }) + if readErr != nil { + return 0, readErr + } + if errno != 0 { + // recvmsg_x is refusing this socket (sandbox, protocol, ...): + // serve this and all future calls with a plain single read so + // the receive routine keeps running. + state.fallback = true + return s.receiveSingle(conn, bufs, sizes, eps) + } + numMsgs := int(n) + for i := 0; i < numMsgs; i++ { + sizes[i] = int(state.hdrs[i].DataLen) + if sizes[i] > 3 && s.hasReserved() { // lx: only strip reserved bytes for WARP (see hasReserved) + bufs[i][1] = 0 + bufs[i][2] = 0 + bufs[i][3] = 0 + } + if connectedEndpoint != nil { + eps[i] = connectedEndpoint + continue + } + var addrPort netip.AddrPort + name := &state.names[i] + if name.Family == unix.AF_INET6 { + port := name.Port<<8 | name.Port>>8 + addrPort = netip.AddrPortFrom(netip.AddrFrom16(name.Addr).Unmap(), port) + } else { + name4 := (*unix.RawSockaddrInet4)(unsafe.Pointer(name)) + port := name4.Port<<8 | name4.Port>>8 + addrPort = netip.AddrPortFrom(netip.AddrFrom4(name4.Addr), port) + } + eps[i] = &StdNetEndpoint{AddrPort: addrPort} + } + return numMsgs, nil + }, nil +} diff --git a/conn/msgx_default.go b/conn/msgx_default.go new file mode 100644 index 0000000..6fffdd5 --- /dev/null +++ b/conn/msgx_default.go @@ -0,0 +1,30 @@ +//go:build !darwin || ios + +package conn + +import ( + "net" + + "golang.org/x/net/ipv6" +) + +const supportsMsgX = false + +const msgXBatchSize = 1 + +type msgXState struct{} + +func (m *msgXState) reset() { +} + +// SetSinglePeerMode is a no-op on platforms without sendmsg_x/recvmsg_x. +func (s *StdNetBind) SetSinglePeerMode() { +} + +func (s *StdNetBind) sendMsgX(conn *net.UDPConn, msgs []ipv6.Message) (bool, error) { + return false, nil +} + +func (s *StdNetBind) makeReceiveMsgX(conn *net.UDPConn, isV6 bool) (ReceiveFunc, error) { + panic("makeReceiveMsgX is not supported on this platform") +} diff --git a/conn/reserved_gate_lx_test.go b/conn/reserved_gate_lx_test.go new file mode 100644 index 0000000..3502a79 --- /dev/null +++ b/conn/reserved_gate_lx_test.go @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: MIT + * + * lx: unit coverage for the StdNetBind.hasReserved() gate that guards the + * receive-side reserved-clear. receiveIP zeroes bytes 1-3 (Cloudflare WARP + * "reserved") only when a non-zero reserved value is set for some endpoint; + * otherwise an AmneziaWG magic header landing in bytes 1-3 (small s1/s2/s4 + * padding) would be corrupted and the packet dropped. This test pins the gate + * itself; the end-to-end handshake proof lives in the device package. + */ + +package conn + +import ( + "net/netip" + "testing" +) + +func stdNetBindForTest(t *testing.T) *StdNetBind { + t.Helper() + b, ok := NewStdNetBind(nil).(*StdNetBind) + if !ok { + t.Fatalf("NewStdNetBind did not return *StdNetBind") + } + return b +} + +func TestStdNetBindHasReserved(t *testing.T) { + b := stdNetBindForTest(t) + if b.hasReserved() { + t.Fatal("fresh bind must report no reserved value") + } + + ep := netip.MustParseAddrPort("127.0.0.1:51820") + + // An all-zero reserved value is indistinguishable from "unset" and must + // not arm the clear. + b.SetReservedForEndpoint(ep, [3]byte{0, 0, 0}) + if b.hasReserved() { + t.Fatal("all-zero reserved must not count as reserved") + } + + // Any non-zero byte (WARP anycast tag) arms the clear. + b.SetReservedForEndpoint(ep, [3]byte{0, 0, 1}) + if !b.hasReserved() { + t.Fatal("non-zero reserved (byte 3) must count as reserved") + } + + // A second endpoint's non-zero value must also be seen. + b2 := stdNetBindForTest(t) + ep2 := netip.MustParseAddrPort("192.0.2.1:2408") + b2.SetReservedForEndpoint(ep, [3]byte{0, 0, 0}) + b2.SetReservedForEndpoint(ep2, [3]byte{0xAB, 0, 0}) + if !b2.hasReserved() { + t.Fatal("non-zero reserved on any endpoint must count as reserved") + } +} diff --git a/conn/control_default.go b/conn/sticky_default.go similarity index 59% rename from conn/control_default.go rename to conn/sticky_default.go index a8bc06a..15b65af 100644 --- a/conn/control_default.go +++ b/conn/sticky_default.go @@ -1,8 +1,8 @@ -//go:build !(linux && !android) +//go:build !linux || android /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package conn @@ -22,7 +22,7 @@ func (e *StdNetEndpoint) SrcToString() string { } // TODO: macOS, FreeBSD and other BSDs likely do support the sticky sockets -// ({get,set}srcControl feature set, but use alternatively named flags and need +// {get,set}srcControl feature set, but use alternatively named flags and need // ports and require testing. // getSrcFromControl parses the control for PKTINFO and if found updates ep with @@ -35,17 +35,8 @@ func getSrcFromControl(control []byte, ep *StdNetEndpoint) { func setSrcControl(control *[]byte, ep *StdNetEndpoint) { } -// getGSOSize parses control for UDP_GRO and if found returns its GSO size data. -func getGSOSize(control []byte) (int, error) { - return 0, nil -} - -// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. -func setGSOSize(control *[]byte, gsoSize uint16) { -} - -// controlSize returns the recommended buffer size for pooling sticky and UDP +// stickyControlSize returns the recommended buffer size for pooling sticky // offloading control data. -const controlSize = 0 +const stickyControlSize = 0 const StdNetSupportsStickySockets = false diff --git a/conn/control_linux.go b/conn/sticky_linux.go similarity index 64% rename from conn/control_linux.go rename to conn/sticky_linux.go index f32f26a..adfedc1 100644 --- a/conn/control_linux.go +++ b/conn/sticky_linux.go @@ -2,13 +2,12 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package conn import ( - "fmt" "net/netip" "unsafe" @@ -106,54 +105,8 @@ func setSrcControl(control *[]byte, ep *StdNetEndpoint) { *control = append(*control, ep.src...) } -const ( - sizeOfGSOData = 2 -) - -// getGSOSize parses control for UDP_GRO and if found returns its GSO size data. -func getGSOSize(control []byte) (int, error) { - var ( - hdr unix.Cmsghdr - data []byte - rem = control - err error - ) - - for len(rem) > unix.SizeofCmsghdr { - hdr, data, rem, err = unix.ParseOneSocketControlMessage(rem) - if err != nil { - return 0, fmt.Errorf("error parsing socket control message: %w", err) - } - if hdr.Level == socketOptionLevelUDP && hdr.Type == socketOptionUDPGRO && len(data) >= sizeOfGSOData { - var gso uint16 - copy(unsafe.Slice((*byte)(unsafe.Pointer(&gso)), sizeOfGSOData), data[:sizeOfGSOData]) - return int(gso), nil - } - } - return 0, nil -} - -// setGSOSize sets a UDP_SEGMENT in control based on gsoSize. It leaves existing -// data in control untouched. -func setGSOSize(control *[]byte, gsoSize uint16) { - existingLen := len(*control) - avail := cap(*control) - existingLen - space := unix.CmsgSpace(sizeOfGSOData) - if avail < space { - return - } - *control = (*control)[:cap(*control)] - gsoControl := (*control)[existingLen:] - hdr := (*unix.Cmsghdr)(unsafe.Pointer(&(gsoControl)[0])) - hdr.Level = socketOptionLevelUDP - hdr.Type = socketOptionUDPSegment - hdr.SetLen(unix.CmsgLen(sizeOfGSOData)) - copy((gsoControl)[unix.SizeofCmsghdr:], unsafe.Slice((*byte)(unsafe.Pointer(&gsoSize)), sizeOfGSOData)) - *control = (*control)[:existingLen+space] -} - -// controlSize returns the recommended buffer size for pooling sticky and UDP +// stickyControlSize returns the recommended buffer size for pooling sticky // offloading control data. -var controlSize = unix.CmsgSpace(unix.SizeofInet6Pktinfo) + unix.CmsgSpace(sizeOfGSOData) +var stickyControlSize = unix.CmsgSpace(unix.SizeofInet6Pktinfo) const StdNetSupportsStickySockets = true diff --git a/device/allowedips.go b/device/allowedips.go index d15373c..2271af1 100644 --- a/device/allowedips.go +++ b/device/allowedips.go @@ -55,6 +55,25 @@ func commonBits(ip1, ip2 []byte) uint8 { } } +func commonBits4(ip1 []byte, ip2 [4]byte) uint8 { + a := binary.BigEndian.Uint32(ip1) + b := binary.BigEndian.Uint32(ip2[:]) + return uint8(bits.LeadingZeros32(a ^ b)) +} + +func commonBits6(ip1 []byte, ip2 [16]byte) uint8 { + a := binary.BigEndian.Uint64(ip1) + b := binary.BigEndian.Uint64(ip2[:]) + x := a ^ b + if x != 0 { + return uint8(bits.LeadingZeros64(x)) + } + a = binary.BigEndian.Uint64(ip1[8:]) + b = binary.BigEndian.Uint64(ip2[8:]) + x = a ^ b + return 64 + uint8(bits.LeadingZeros64(x)) +} + func (node *trieEntry) addToPeerEntries() { node.perPeerElem = node.peer.trieEntries.PushBack(node) } @@ -188,7 +207,37 @@ func (trie parentIndirection) insert(ip []byte, cidr uint8, peer *Peer) { } } -func (node *trieEntry) lookup(ip []byte) *Peer { +func (node *trieEntry) lookup4(ip [4]byte) *Peer { + var found *Peer + for node != nil && commonBits4(node.bits, ip) >= node.cidr { + if node.peer != nil { + found = node.peer + } + if node.bitAtByte == 4 { + break + } + bit := (ip[node.bitAtByte] >> node.bitAtShift) & 1 + node = node.child[bit] + } + return found +} + +func (node *trieEntry) lookup6(ip [16]byte) *Peer { + var found *Peer + for node != nil && commonBits6(node.bits, ip) >= node.cidr { + if node.peer != nil { + found = node.peer + } + if node.bitAtByte == 16 { + break + } + bit := (ip[node.bitAtByte] >> node.bitAtShift) & 1 + node = node.child[bit] + } + return found +} + +func (node *trieEntry) lookup(ip net.IP) *Peer { var found *Peer size := uint8(len(ip)) for node != nil && commonBits(node.bits, ip) >= node.cidr { @@ -205,14 +254,17 @@ func (node *trieEntry) lookup(ip []byte) *Peer { } type AllowedIPs struct { - IPv4 *trieEntry - IPv6 *trieEntry - mutex sync.RWMutex + mu sync.RWMutex + ipv4 *trieEntry + ipv6 *trieEntry + + peerByIPPacketFunc PeerByIPPacketFunc // if non-nil, called to look up peers by IP + device *Device // back-reference to parent device; non-nil only if peerByIPPacketFunc is set } func (table *AllowedIPs) EntriesForPeer(peer *Peer, cb func(prefix netip.Prefix) bool) { - table.mutex.RLock() - defer table.mutex.RUnlock() + table.mu.RLock() + defer table.mu.RUnlock() for elem := peer.trieEntries.Front(); elem != nil; elem = elem.Next() { node := elem.Value.(*trieEntry) @@ -257,17 +309,17 @@ func (node *trieEntry) remove() { } func (table *AllowedIPs) Remove(prefix netip.Prefix, peer *Peer) { - table.mutex.Lock() - defer table.mutex.Unlock() + table.mu.Lock() + defer table.mu.Unlock() var node *trieEntry var exact bool if prefix.Addr().Is6() { ip := prefix.Addr().As16() - node, exact = table.IPv6.nodePlacement(ip[:], uint8(prefix.Bits())) + node, exact = table.ipv6.nodePlacement(ip[:], uint8(prefix.Bits())) } else if prefix.Addr().Is4() { ip := prefix.Addr().As4() - node, exact = table.IPv4.nodePlacement(ip[:], uint8(prefix.Bits())) + node, exact = table.ipv4.nodePlacement(ip[:], uint8(prefix.Bits())) } else { panic(errors.New("removing unknown address type")) } @@ -277,10 +329,25 @@ func (table *AllowedIPs) Remove(prefix netip.Prefix, peer *Peer) { node.remove() } -func (table *AllowedIPs) RemoveByPeer(peer *Peer) { - table.mutex.Lock() - defer table.mutex.Unlock() +// setPeerPrefixes atomically removes all of peer's existing prefixes and adds +// the provided ones. +func (table *AllowedIPs) setPeerPrefixes(peer *Peer, prefixes []netip.Prefix) { + table.mu.Lock() + defer table.mu.Unlock() + table.removeByPeerLocked(peer) + for _, prefix := range prefixes { + table.insertLocked(prefix, peer) + } +} + +func (table *AllowedIPs) RemoveByPeer(peer *Peer) { + table.mu.Lock() + defer table.mu.Unlock() + table.removeByPeerLocked(peer) +} + +func (table *AllowedIPs) removeByPeerLocked(peer *Peer) { var next *list.Element for elem := peer.trieEntries.Front(); elem != nil; elem = next { next = elem.Next() @@ -289,29 +356,135 @@ func (table *AllowedIPs) RemoveByPeer(peer *Peer) { } func (table *AllowedIPs) Insert(prefix netip.Prefix, peer *Peer) { - table.mutex.Lock() - defer table.mutex.Unlock() + table.mu.Lock() + defer table.mu.Unlock() + table.insertLocked(prefix, peer) +} +func (table *AllowedIPs) insertLocked(prefix netip.Prefix, peer *Peer) { if prefix.Addr().Is6() { ip := prefix.Addr().As16() - parentIndirection{&table.IPv6, 2}.insert(ip[:], uint8(prefix.Bits()), peer) + parentIndirection{&table.ipv6, 2}.insert(ip[:], uint8(prefix.Bits()), peer) } else if prefix.Addr().Is4() { ip := prefix.Addr().As4() - parentIndirection{&table.IPv4, 2}.insert(ip[:], uint8(prefix.Bits()), peer) + parentIndirection{&table.ipv4, 2}.insert(ip[:], uint8(prefix.Bits()), peer) } else { panic(errors.New("inserting unknown address type")) } } -func (table *AllowedIPs) Lookup(ip []byte) *Peer { - table.mutex.RLock() - defer table.mutex.RUnlock() - switch len(ip) { - case net.IPv6len: - return table.IPv6.lookup(ip) - case net.IPv4len: - return table.IPv4.lookup(ip) +// LookupFromPacket looks up the peer to which an outbound IP packet should be +// sent. It lives on [AllowedIPs] for legacy/structural reasons: historically +// WireGuard's only peer-selection mechanism was the AllowedIPs trie, and the +// send path already had a reference to the table. When a [PeerByIPPacketFunc] +// has been registered via [Device.SetPeerByIPPacketFunc], that callback is used +// instead of the trie and the AllowedIPs table is not consulted at all. +// +// When no callback is registered, only dst is used (standard WireGuard +// AllowedIPs trie lookup). When a callback is registered, all three +// parameters are forwarded to it; see [PeerByIPPacketFunc] for details. +func (table *AllowedIPs) LookupFromPacket(src, dst netip.Addr, ipPkt []byte) *Peer { + table.mu.RLock() + if f := table.peerByIPPacketFunc; f != nil { + device := table.device + table.mu.RUnlock() + + if pubk, ok := f(src, dst, ipPkt); ok { + return device.LookupPeer(pubk) + } + return nil + } + defer table.mu.RUnlock() + + switch { + case dst.Is6(): + return table.ipv6.lookup6(dst.As16()) + case dst.Is4(): + return table.ipv4.lookup4(dst.As4()) default: panic(errors.New("looking up unknown address type")) } } + +// Deprecated: Lookup is only used by legacy tests. It does not call +// [PeerByIPPacketFunc]; use [AllowedIPs.LookupFromPacket] for production lookups. +func (table *AllowedIPs) Lookup(ip []byte) *Peer { + table.mu.RLock() + defer table.mu.RUnlock() + return table.lookupLocked(ip) +} + +// lookupLocked looks up the peer associated with the given IP address. +// It assumes the caller holds the read lock (or doesn't hold it, but also +// doesn't concurrently mutate AllowedIP). +// +// It returns nil if no peer is associated with the given IP address. +func (table *AllowedIPs) lookupLocked(ip []byte) *Peer { + switch len(ip) { + case net.IPv6len: + return table.ipv6.lookup(ip) + case net.IPv4len: + return table.ipv4.lookup(ip) + default: + panic(errors.New("looking up unknown address type")) + } +} + +// AllowedPeerSourceIP reports whether the given source IP address is allowed +// for the given peer. +func (peer *Peer) AllowedPeerSourceIP(src netip.Addr) bool { + if f := peer.state.testAllowedIP.Load(); f != nil { + return (*f)(src) + } + + table := &peer.device.allowedips + table.mu.RLock() + defer table.mu.RUnlock() + switch { + case src.Is6(): + return table.ipv6.lookup6(src.As16()) == peer + case src.Is4(): + return table.ipv4.lookup4(src.As4()) == peer + } + return false +} + +// fakePeer is a zero Peer used only as a placeholder in tries used by mkIPInCIDRsTestFunc. +var fakePeer Peer + +// mkIPInCIDRsTestFunc returns a function that tests whether an IP address is +// contained in any of the given CIDRs. +func mkIPInCIDRsTestFunc(cidrs []netip.Prefix) func(netip.Addr) bool { + if len(cidrs) == 0 { + return func(netip.Addr) bool { return false } + } + if len(cidrs) == 1 { + return func(addr netip.Addr) bool { return cidrs[0].Contains(addr) } + } + if len(cidrs) <= 4 { + // For small numbers of CIDRs, just do a linear search. The trie construction + // is more expensive than the linear search, and the test function is faster + // than the trie lookup, so this is a net win. + return func(addr netip.Addr) bool { + for _, c := range cidrs { + if c.Contains(addr) { + return true + } + } + return false + } + } + // Make a trie for faster lookups. We use a dummy Peer. + var a AllowedIPs + for _, c := range cidrs { + a.Insert(c, &fakePeer) + } + return func(addr netip.Addr) bool { + switch { + case addr.Is4(): + return a.ipv4.lookup4(addr.As4()) == &fakePeer + default: + return a.ipv6.lookup6(addr.As16()) == &fakePeer + } + } +} diff --git a/device/awg_stdnetbind_reserved_lx_test.go b/device/awg_stdnetbind_reserved_lx_test.go new file mode 100644 index 0000000..0d431e1 --- /dev/null +++ b/device/awg_stdnetbind_reserved_lx_test.go @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: MIT + * + * lx: e2e regression for the reserved-clear vs AWG magic-header collision, + * exercised over the StdNetBind path (no detour) with real loopback UDP. + * + * Bug model. On receive, StdNetBind.receiveIP unconditionally zeroed bytes + * 1-3 of every datagram >3 bytes (the Cloudflare WARP "reserved" field). + * AmneziaWG reads its magic header as LittleEndian.Uint32(packet[padding:]), + * where padding is s1/s2 (handshake) or s4 (transport). With small padding + * (0..3) the 4-byte magic overlaps bytes 1-3, so the unconditional clear + * corrupts it: the value falls outside the ranged h1-h4 window, the packet is + * classified MessageUnknownType and dropped. WARP was never configured on + * these binds (no SetReservedForEndpoint), so the clear was pure collateral. + * + * The fix gates the clear behind StdNetBind.hasReserved(): bytes 1-3 are only + * zeroed when a non-zero reserved value is actually set for some endpoint. + * With no reserved value the magic survives and the handshake completes. + * + * This test provokes the worst case: padding = 0 (no s1/s2/s4 at all), so the + * h1 initiation magic sits in bytes [0..3] and its high bytes (1-3) are the + * ones the clear would destroy. The h1-h4 ranges are chosen entirely above + * 0x10000000, so after zeroing bytes 1-3 the surviving value is <= 255 and can + * never land back inside any range -> guaranteed drop on the buggy tree. + * + * GREEN on the fixed tree. To see RED, temporarily restore the unconditional + * clear in conn/bind_std.go receiveIP: + * if msg.N > 3 { + * common.ClearArray(bufs[i][1:4]) + * } + * and the handshake times out (init magic zeroed in bytes 1-3). + */ + +package device + +import ( + "bufio" + "bytes" + "context" + "encoding/hex" + "fmt" + "strconv" + "strings" + "testing" + "time" + + "github.com/sagernet/wireguard-go/conn" +) + +// magic header ranges kept entirely above 0x10000000 (268435456). Any value +// the sender picks therefore has a non-zero byte among positions 1-3; zeroing +// those bytes collapses the value to <= 0xFF, which is below every range start, +// so a corrupted magic can never validate. Distinct windows per message type. +const ( + lxH1Lo, lxH1Hi = 268500000, 268600000 // init + lxH2Lo, lxH2Hi = 300000000, 300100000 // response + lxH3Lo, lxH3Hi = 400000000, 400100000 // cookie + lxH4Lo, lxH4Hi = 500000000, 500100000 // transport +) + +// lxReadListenPort parses listen_port= out of a device's IpcGet dump. +func lxReadListenPort(t *testing.T, dev *Device) uint16 { + t.Helper() + dump, err := dev.IpcGet() + if err != nil { + t.Fatalf("IpcGet: %v", err) + } + scanner := bufio.NewScanner(strings.NewReader(dump)) + for scanner.Scan() { + line := scanner.Text() + if v, ok := strings.CutPrefix(line, "listen_port="); ok { + p, err := strconv.Atoi(v) + if err != nil { + t.Fatalf("parse listen_port %q: %v", v, err) + } + return uint16(p) + } + } + t.Fatalf("listen_port not found in dump:\n%s", dump) + return 0 +} + +// newStdNetPaddedPair builds two Up()'d Devices peered over real loopback UDP +// (NewStdNetBind), configured with ranged h1-h4 + junk and *no* s1/s2/s4 +// (padding = 0). Endpoints are wired after Up, once the ephemeral ports are +// known. No reserved value is ever set, so hasReserved() is false. +func newStdNetPaddedPair(t *testing.T) (devA, devB *Device, tunA, tunB *chanTun) { + t.Helper() + + skA, err := newPrivateKey() + if err != nil { + t.Fatalf("newPrivateKey A: %v", err) + } + skB, err := newPrivateKey() + if err != nil { + t.Fatalf("newPrivateKey B: %v", err) + } + pkA := skA.publicKey() + pkB := skB.publicKey() + + tunA = newChanTun() + tunB = newChanTun() + + devA = NewDevice(context.Background(), tunA, conn.NewStdNetBind(nil), NewLogger(LogLevelError, "devA: "), 1) + devB = NewDevice(context.Background(), tunB, conn.NewStdNetBind(nil), NewLogger(LogLevelError, "devB: "), 1) + t.Cleanup(devA.Close) + t.Cleanup(devB.Close) + + // obfuscation shared by both ends. Ranged magic headers, junk packets, + // and deliberately no s1/s2/s4 so padding stays 0 for every message type. + obf := fmt.Sprintf( + "jc=3\njmin=8\njmax=16\n"+ + "h1=%d-%d\nh2=%d-%d\nh3=%d-%d\nh4=%d-%d\n", + lxH1Lo, lxH1Hi, lxH2Lo, lxH2Hi, lxH3Lo, lxH3Hi, lxH4Lo, lxH4Hi) + + // Bring both up on an ephemeral port (listen_port=0), no endpoint yet. + cfgA := fmt.Sprintf("private_key=%s\nlisten_port=0\n%sreplace_peers=true\npublic_key=%s\nallowed_ip=%s/32\n", + hex.EncodeToString(skA[:]), obf, hex.EncodeToString(pkB[:]), testIPB) + cfgB := fmt.Sprintf("private_key=%s\nlisten_port=0\n%sreplace_peers=true\npublic_key=%s\nallowed_ip=%s/32\n", + hex.EncodeToString(skB[:]), obf, hex.EncodeToString(pkA[:]), testIPA) + + if err := devA.IpcSet(cfgA); err != nil { + t.Fatalf("IpcSet A: %v", err) + } + if err := devB.IpcSet(cfgB); err != nil { + t.Fatalf("IpcSet B: %v", err) + } + if devA.paddings.init != 0 || devA.paddings.response != 0 || devA.paddings.transport != 0 { + t.Fatalf("padding must be 0 for this test: init=%d resp=%d transport=%d", + devA.paddings.init, devA.paddings.response, devA.paddings.transport) + } + + if err := devA.Up(); err != nil { + t.Fatalf("Up A: %v", err) + } + if err := devB.Up(); err != nil { + t.Fatalf("Up B: %v", err) + } + + portA := lxReadListenPort(t, devA) + portB := lxReadListenPort(t, devB) + if portA == 0 || portB == 0 { + t.Fatalf("ephemeral ports not assigned: A=%d B=%d", portA, portB) + } + + // Now that ports are known, point each peer at the other over loopback. + if err := devA.IpcSet(fmt.Sprintf("public_key=%s\nupdate_only=true\nendpoint=127.0.0.1:%d\n", + hex.EncodeToString(pkB[:]), portB)); err != nil { + t.Fatalf("set endpoint A->B: %v", err) + } + if err := devB.IpcSet(fmt.Sprintf("public_key=%s\nupdate_only=true\nendpoint=127.0.0.1:%d\n", + hex.EncodeToString(pkA[:]), portA)); err != nil { + t.Fatalf("set endpoint B->A: %v", err) + } + + return devA, devB, tunA, tunB +} + +// TestStdNetBindReservedClearVsMagic_ZeroPadding drives a real handshake and a +// data packet A->B over loopback UDP through StdNetBind, with padding=0 so the +// h1 magic overlaps the reserved bytes 1-3. It passes only when receive does +// not blindly clear those bytes (the fix). +func TestStdNetBindReservedClearVsMagic_ZeroPadding(t *testing.T) { + devA, _, tunA, tunB := newStdNetPaddedPair(t) + _ = devA + + pkt := buildIPv4Packet(testIPA, testIPB, 8) + + // Re-inject periodically: the first packet triggers the handshake and may + // be dropped until keys are established. + send := func() { tunA.toDevice <- pkt } + send() + + deadline := time.After(15 * time.Second) + retry := time.NewTicker(500 * time.Millisecond) + defer retry.Stop() + for { + select { + case got := <-tunB.fromDevice: + if bytes.Equal(got, pkt) { + return // delivered end to end: magic survived, handshake ok + } + t.Logf("ignoring unexpected packet len=%d", len(got)) + case <-retry.C: + send() + case <-deadline: + t.Fatal("timed out waiting for packet on peer tun " + + "(handshake never completed: reserved-clear likely corrupted the h1 magic)") + } + } +} diff --git a/device/channels.go b/device/channels.go index be15d1c..9af6e3d 100644 --- a/device/channels.go +++ b/device/channels.go @@ -83,15 +83,21 @@ func newAutodrainingInboundQueue(device *Device) *autodrainingInboundQueue { q := &autodrainingInboundQueue{ c: make(chan *QueueInboundElementsContainer, QueueInboundSize), } - runtime.SetFinalizer(q, device.flushInboundQueue) + if device.needsInboundQueueFinalizer() { + runtime.AddCleanup(q, device.flushInboundQueue, q.c) + } return q } -func (device *Device) flushInboundQueue(q *autodrainingInboundQueue) { +func (device *Device) needsInboundQueueFinalizer() bool { + return device.pool.messageBuffers.hasAccounting() +} + +func (device *Device) flushInboundQueue(c <-chan *QueueInboundElementsContainer) { for { select { - case elemsContainer := <-q.c: - elemsContainer.Lock() + case elemsContainer := <-c: + elemsContainer.filling.Wait() for _, elem := range elemsContainer.elems { device.PutMessageBuffer(elem.buffer) device.PutInboundElement(elem) @@ -116,17 +122,23 @@ func newAutodrainingOutboundQueue(device *Device) *autodrainingOutboundQueue { q := &autodrainingOutboundQueue{ c: make(chan *QueueOutboundElementsContainer, QueueOutboundSize), } - runtime.SetFinalizer(q, device.flushOutboundQueue) + if device.needsOutboundQueueFinalizer() { + runtime.AddCleanup(q, device.flushOutboundQueue, q.c) + } return q } -func (device *Device) flushOutboundQueue(q *autodrainingOutboundQueue) { +func (device *Device) needsOutboundQueueFinalizer() bool { + return device.pool.messageBuffers.hasAccounting() +} + +func (device *Device) flushOutboundQueue(c <-chan *QueueOutboundElementsContainer) { for { select { - case elemsContainer := <-q.c: - elemsContainer.Lock() + case elemsContainer := <-c: + elemsContainer.filling.Wait() for _, elem := range elemsContainer.elems { - device.PutMessageBuffer(elem.buffer) + device.PutOutboundBuffer(elem.buffer) device.PutOutboundElement(elem) } device.PutOutboundElementsContainer(elemsContainer) diff --git a/device/device.go b/device/device.go index 140a5df..fe11b7a 100644 --- a/device/device.go +++ b/device/device.go @@ -7,6 +7,8 @@ package device import ( "context" + "errors" + "net/netip" "runtime" "sync" "sync/atomic" @@ -59,8 +61,12 @@ type Device struct { peers struct { sync.RWMutex // protects keyMap keyMap map[NoisePublicKey]*Peer + lookupFunc PeerLookupFunc // or nil if unused } + peerStateFn atomic.Pointer[PeerSessionStateFunc] // observes peer session state changes, nil if unset + priorityMsgFn atomic.Pointer[PeerPriorityMessageFunc] // returns a priority message to be sent around session establishment, nil if unset + rate struct { underLoadUntil atomic.Int64 limiter ratelimiter.Ratelimiter @@ -71,11 +77,11 @@ type Device struct { cookieChecker CookieChecker pool struct { - inboundElementsContainer *WaitPool - outboundElementsContainer *WaitPool + inboundElementsContainer *sync.Pool + outboundElementsContainer *sync.Pool messageBuffers *WaitPool - inboundElements *WaitPool - outboundElements *WaitPool + inboundElements *sync.Pool + outboundElements *sync.Pool } queue struct { @@ -116,6 +122,20 @@ type Device struct { } ipackets [5]*obfChain + + // lx: SPEC 041 — passive self-heal state: reopen the bind once (fresh + // ephemeral port when freshPort is set) and immediately re-initiate, to + // heal dead per-flow path state (an expired NAT mapping or a poisoned DPI + // flow entry) that otherwise pins every retry to the same dead 5-tuple + // until a manual reconnect. The whole mechanism — three triggers (giveup / + // early / nudge) sharing this state and its debounce — lives in + // lx_giveup_rebind.go. Enabled by default; sing-box decides freshPort + // from whether the user pinned listen_port. + giveUpRebind struct { + enabled atomic.Bool + freshPort atomic.Bool + last atomic.Int64 // unix seconds of the last rebind (debounce) + } } // deviceState represents the state of a Device. @@ -207,14 +227,22 @@ func (device *Device) upLocked() error { device.ipcMutex.Lock() defer device.ipcMutex.Unlock() + // Collect peers under RLock and then release before calling into them, + // because SendKeepalive can reach CreateMessageInitiation which acquires + // staticIdentity.RLock; holding peers.RLock across that path would + // invert the staticIdentity < peers hierarchy (see lock-ordering.md). device.peers.RLock() + peers := make([]*Peer, 0, len(device.peers.keyMap)) for _, peer := range device.peers.keyMap { + peers = append(peers, peer) + } + device.peers.RUnlock() + for _, peer := range peers { peer.Start() if peer.persistentKeepaliveInterval.Load() > 0 { peer.SendKeepalive() } } - device.peers.RUnlock() return nil } @@ -312,6 +340,7 @@ func (device *Device) SetPrivateKey(sk NoisePrivateKey) error { func NewDevice(ctx context.Context, tunDevice tun.Device, bind conn.Bind, logger *Logger, workers int) *Device { device := new(Device) device.pauseManager = service.FromContext[pause.Manager](ctx) + device.giveUpRebind.enabled.Store(true) // lx: SPEC 041 — self-heal on by default device.state.state.Store(uint32(deviceStateDown)) device.closed = make(chan struct{}) device.log = logger @@ -374,13 +403,66 @@ func (device *Device) BatchSize() int { return size } +// LookupPeer looks up a peer by its public key. +// +// If the peer does not exist and a [PeerLookupFunc] is set (via +// [Device.SetPeerLookupFunc]), then that function is used to create the peer +// before returning it. Peers created via this mechanism exist only until their +// state machine reaches idle, and then the peers are removed. +// +// If the peer does not exist and no [PeerLookupFunc] is set, nil is returned. +// +// Use [Device.LookupActivePeer] to only return already-existing peers, without +// using a [PeerLookupFunc]. func (device *Device) LookupPeer(pk NoisePublicKey) *Peer { device.peers.RLock() - defer device.peers.RUnlock() + p, ok := device.peers.keyMap[pk] + lookupFunc := device.peers.lookupFunc + device.peers.RUnlock() + if ok || lookupFunc == nil { + return p + } - return device.peers.keyMap[pk] + conf, ok := lookupFunc(pk) + if !ok || conf == nil { + return nil + } + + p, err := device.NewPeer(pk) + if err != nil { + if errors.Is(err, errAddExistingPeer) { + device.peers.RLock() + defer device.peers.RUnlock() + return device.peers.keyMap[pk] + } + device.log.Errorf("Failed to create peer: %v", err) + return nil + } + p.SetAllowedIPs(conf.AllowedIPs) + p.deleteOnIdle = true + if conf.Endpoint != nil { + p.SetEndpointFromPacket(conf.Endpoint) + } + p.Start() + return p } +// LookupActivePeer looks up a peer by its public key. +// +// Unlike [Device.LookupPeer], this function does not use a [PeerLookupFunc] to +// create the peer if it does not already exist. +// +// If the peer does not exist or was created lazily via [PeerLookupFunc] +// and has subsequently idled away, it returns (nil, false). +func (device *Device) LookupActivePeer(pk NoisePublicKey) (_ *Peer, ok bool) { + device.peers.RLock() + defer device.peers.RUnlock() + p, ok := device.peers.keyMap[pk] + return p, ok +} + +var errAddExistingPeer = errors.New("adding existing peer") + func (device *Device) RemovePeer(key NoisePublicKey) { device.peers.Lock() defer device.peers.Unlock() @@ -403,6 +485,151 @@ func (device *Device) RemoveAllPeers() { device.peers.keyMap = make(map[NoisePublicKey]*Peer) } +// RemoveMatchingPeers removes all peers for which shouldRemove returns true. +// +// It returns the number of peers removed. +func (device *Device) RemoveMatchingPeers(shouldRemove func(NoisePublicKey) bool) (numRemoved int) { + device.peers.Lock() + defer device.peers.Unlock() + + for key, peer := range device.peers.keyMap { + if shouldRemove(key) { + removePeerLocked(device, peer, key) + numRemoved++ + } + } + return numRemoved +} + +// NewPeerConfig are the configuration parameters for a new peer created via a +// [PeerLookupFunc] func. +type NewPeerConfig struct { + // AllowedIPs is the initial set of allowed IPs for the new peer. + AllowedIPs []netip.Prefix + + // Endpoint, if non-nil, sets the initial endpoint for newly + // created peers. + Endpoint conn.Endpoint +} + +// PeerLookupFunc is the type of function used to look up peers by public key +// when receiving packets for unknown peers. +// +// If it returns nil, the peer is not known. +// +// Otherwise, returning non-nil signals that wireguard-go should create the peer +// with the provided allowed IPs. +// +// See [Device.SetPeerLookupFunc] and [Device.LookupPeer]. +type PeerLookupFunc func(NoisePublicKey) (_ *NewPeerConfig, ok bool) + +// PeerByIPPacketFunc is the type of function used to look up a peer to send to +// for a given src/dst IP pair. The ipPkt parameter is the raw IP packet being +// routed; callers needing transport-layer ports or other header fields may parse +// them from ipPkt, but must handle IP fragmentation (ports may be absent on +// non-first fragments) and protocols that do not use ports (e.g. ICMP). +// +// Except for experimental use cases, dst is the only address +// that should be relied upon when looking up a peer. +// +// If it returns ok=false, the peer is not known. +// +// See [Device.SetPeerByIPPacketFunc] and [Device.SetPeerLookupFunc]. +type PeerByIPPacketFunc func(src, dst netip.Addr, ipPkt []byte) (_ NoisePublicKey, ok bool) + +// PeerSessionState is the current WireGuard session state for a peer. +type PeerSessionState uint8 + +const ( + // PeerSessionNone means there is no handshake in progress and no session key + // material retained for this peer. + PeerSessionNone PeerSessionState = iota + + // PeerSessionHandshake means a handshake is in progress for this peer, but + // there is not currently a usable WireGuard session. + PeerSessionHandshake + + // PeerSessionEstablished means the peer has a completed WireGuard session + // with usable session key material. + PeerSessionEstablished + + // PeerSessionExpired means the peer's session key material is no longer + // considered usable, but final key cleanup or lazy peer removal may not have + // happened yet. + PeerSessionExpired +) + +// PeerSessionStateFunc is called when a peer's WireGuard session state changes. +// +// Calls are serialized per peer and delivered in that peer's transition order. The +// callback must be cheap and must not call back into Device. +type PeerSessionStateFunc func(peer NoisePublicKey, state PeerSessionState) + +// SetPeerLookupFunc sets the function used to look up peers by public key +// when receiving packets for unknown peers. +func (device *Device) SetPeerLookupFunc(f PeerLookupFunc) { + device.peers.Lock() + defer device.peers.Unlock() + device.peers.lookupFunc = f +} + +// SetPeerByIPPacketFunc sets the function used to look up peers by IP address +// when sending packets to unknown peers. +func (device *Device) SetPeerByIPPacketFunc(f PeerByIPPacketFunc) { + device.allowedips.mu.Lock() + defer device.allowedips.mu.Unlock() + device.allowedips.peerByIPPacketFunc = f + device.allowedips.device = device +} + +// SetSessionStateFunc sets the function used to observe peer WireGuard session +// state changes. +// +// It does not replay current state. Callers that need a complete view should set +// it before peers are started or lazily created, and maintain any snapshots, +// sequence numbers, and pubsub state outside wireguard-go. +// +// The callback must be concurrent-safe and must not call back into Device. +func (device *Device) SetSessionStateFunc(f PeerSessionStateFunc) { + if f == nil { + device.peerStateFn.Store(nil) + return + } + device.peerStateFn.Store(&f) +} + +// MaxPriorityMessageContentSize is the maximum size of a message returned by a +// [PeerPriorityMessageFunc]. It's a power of 2 that leaves significant space +// when accounting for all WireGuard overhead and encapsulating network protocol +// headers. Future adjustments to this value should consider all these overheads +// and any [conn.Bind] implementation limitations. +const MaxPriorityMessageContentSize = 512 + +// PeerPriorityMessageFunc is called when a peer's WireGuard session keypair is +// established (or re-keyed) for forward data transmission. +// +// The returned message is transmitted to the peer in priority fashion. Priority +// means it cannot be evicted from the staged packet queue by non-priority +// (read from [tun.Device]) packets. It avoids the staged queue altogether. +// +// The callback must be cheap and must not call back into [Device]. A zero length +// message or a message whose length exceeds [MaxPriorityMessageContentSize] will +// be silently dropped. Message should start with an IPv4 or IPv6 header as it +// is subject to allowed IPs lookup on the receiver, same as any other transport +// message. +type PeerPriorityMessageFunc func(peer NoisePublicKey) (msg []byte) + +// SetPriorityMessageOnEstablishmentFunc sets a function to be used for sending +// a priority message around session establishment. See [PeerPriorityMessageFunc] +// docs for more details. A nil value clears any previously set value. +func (device *Device) SetPriorityMessageOnEstablishmentFunc(f PeerPriorityMessageFunc) { + if f == nil { + device.priorityMsgFn.Store(nil) + return + } + device.priorityMsgFn.Store(&f) +} + func (device *Device) Close() { device.state.Lock() defer device.state.Unlock() @@ -444,16 +671,25 @@ func (device *Device) SendKeepalivesToPeersWithCurrentKeypair() { return } + // Collect the set of peers to keepalive under peers.RLock, then release + // before invoking SendKeepalive. SendKeepalive can reach + // CreateMessageInitiation which acquires staticIdentity.RLock; holding + // peers.RLock across that path would invert the + // staticIdentity < peers hierarchy (see lock-ordering.md). + var peers []*Peer device.peers.RLock() for _, peer := range device.peers.keyMap { peer.keypairs.RLock() sendKeepalive := peer.keypairs.current != nil && !peer.keypairs.current.created.Add(RejectAfterTime).Before(time.Now()) peer.keypairs.RUnlock() if sendKeepalive { - peer.SendKeepalive() + peers = append(peers, peer) } } device.peers.RUnlock() + for _, peer := range peers { + peer.SendKeepalive() + } } // closeBindLocked closes the device's net.bind. diff --git a/device/keypair.go b/device/keypair.go index fb3d392..0704748 100644 --- a/device/keypair.go +++ b/device/keypair.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package device diff --git a/device/lx_early_rebind_test.go b/device/lx_early_rebind_test.go new file mode 100644 index 0000000..30910c1 --- /dev/null +++ b/device/lx_early_rebind_test.go @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT + * + * lx: SPEC 041 v2 — behavioural red/green test for the EARLY give-up rebind. + * Field failure mode (the v1 field leftover, dump 2026-08-01): the v1 give-up + * rebind heals a dead 5-tuple, but only at ~90s after the first demand — while + * the user pings within the first 5-35s after device wake and sees every node + * in ERR. The early trigger fires from the retry branch once >=3 initiations + * went unanswered AND the session is provably dead (no live keypair, or last + * handshake older than RejectAfterTime), shrinking the ERR window to ~15-20s. + * + * The test reuses the v1 harness (gateBind: first socket generation blackholes + * every send) and drives the peer to the 3rd retry expiry — the state a real + * ~15s of unanswered retries ends in. Post-fix the retry branch rebinds and + * the tunnel comes up; pre-fix (v1 base) the retry keeps dying in the first + * socket generation and only the ~90s give-up would heal, so the packet never + * arrives within the test window. + * + * This file deliberately uses NO post-fix API, so it compiles and runs RED on + * the v1 base commit. + */ + +package device + +import ( + "testing" + "time" +) + +// TestEarlyRebindSelfHeal: dead first socket, cold session (no keypair yet), +// 3rd retry expiry fires — the tunnel must come up and deliver traffic without +// waiting out the full 90s give-up cycle. +func TestEarlyRebindSelfHeal(t *testing.T) { + pair := newGiveUpPair(t, true) + + pkt := buildIPv4Packet(testIPA, testIPB, 8) + send := func() { pair.tunA.toDevice <- pkt } + + // Traffic demand: stages the packet and sends the first (blackholed) + // initiation. + send() + + pair.devA.peers.RLock() + peer := pair.devA.peers.keyMap[pair.pkB] + pair.devA.peers.RUnlock() + if peer == nil { + t.Fatal("peer not found") + } + + // Simulate reaching the 3rd retry expiry (~15s in the field): two retries + // already counted, the last initiation older than the retransmit timeout. + peer.handshake.mutex.Lock() + peer.handshake.lastSentHandshake = time.Now().Add(-2 * RekeyTimeout) + peer.handshake.mutex.Unlock() + peer.timers.handshakeAttempts.Store(2) + expiredRetransmitHandshake(peer) + + awaitPacket(t, pair.tunB, pkt, send) +} diff --git a/device/lx_giveup_rebind.go b/device/lx_giveup_rebind.go new file mode 100644 index 0000000..60d6684 --- /dev/null +++ b/device/lx_giveup_rebind.go @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: MIT + * + * lx: SPEC 041 — passive self-heal for a dead per-flow path (an expired NAT + * mapping or a poisoned DPI flow entry that pins every retry to the same dead + * 5-tuple until a manual reconnect). One mechanism — reopen the bind (fresh + * ephemeral port when allowed) and immediately re-initiate — with three + * triggers sharing one debounce window: + * + * giveup — the handshake retry cycle exhausted (~90s of unanswered + * initiations under traffic demand); safety net, covers every path; + * early — >=3 unanswered initiations against a provably dead session + * (see sessionProvablyDead): no point waiting out the rest of the + * cycle, rebind at ~15s instead of ~90s; + * nudge — the consumer reports "device woke up" via + * Device.RebindIfSessionStale (wired through sing-box libbox); + * heals without waiting for traffic demand at all. + * + * Zero cost while healthy: no timers, no goroutines — triggers 1-2 live in + * the existing retry cycle, trigger 3 is paid by the caller. The state lives + * in Device.giveUpRebind (device.go); enabled defaults to true in NewDevice, + * sing-box decides freshPort from whether the user pinned listen_port. + */ + +package device + +import "time" + +// earlyGiveUpMinAttempts is the number of unanswered initiations (retry timer +// expiries) after which a provably dead session is rebound early instead of +// waiting out the full RekeyAttemptTime cycle: ~15s at RekeyTimeout=5s. +const earlyGiveUpMinAttempts = 3 + +// SetGiveUpRebind configures the self-heal (see the giveUpRebind field +// comment in device.go). freshPort must be false when the user pinned an +// explicit listen_port: the pinned port is preserved, at the cost of the +// rebind not changing the 5-tuple. +func (device *Device) SetGiveUpRebind(enabled, freshPort bool) { + device.giveUpRebind.enabled.Store(enabled) + device.giveUpRebind.freshPort.Store(freshPort) +} + +// handleHandshakeGiveUp is invoked from the give-up branch of +// expiredRetransmitHandshake: ~90s of initiations went unanswered, so the +// current socket's 5-tuple is proven dead. +func (device *Device) handleHandshakeGiveUp(peer *Peer) { + device.selfHealRebind("giveup", peer) +} + +// maybeEarlyGiveUpRebind is invoked from the RETRY branch of +// expiredRetransmitHandshake. Once enough initiations went unanswered AND the +// session is provably dead there is nothing left to protect — rebind now, at +// ~15s instead of ~90s. The retry cycle itself continues untouched: this only +// moves the socket under it. A live session with transient packet loss fails +// sessionProvablyDead and keeps byte-for-byte upstream behaviour; the shared +// debounce means this also suppresses the giveup rebind of the same series. +func (device *Device) maybeEarlyGiveUpRebind(peer *Peer) { + if peer.timers.handshakeAttempts.Load() < earlyGiveUpMinAttempts { + return + } + if !device.sessionProvablyDead(peer) { + return + } + device.selfHealRebind("early", peer) +} + +// sessionProvablyDead reports whether the peer's session is beyond saving: no +// live keypair, or the last successful handshake is older than +// RejectAfterTime (the keys are invalid after that, so a rebind loses +// nothing). The stale predicate shared by the early and nudge triggers. +func (device *Device) sessionProvablyDead(peer *Peer) bool { + if peer.keypairs.Current() == nil { + return true + } + return time.Since(time.Unix(0, peer.lastHandshakeNano.Load())) > RejectAfterTime +} + +// RebindIfSessionStale is the wake-nudge entry (trigger 3): the consumer +// observed a device wake-up and asks for an immediate heal instead of waiting +// for traffic demand to walk the retry cycle. If any running peer's session +// is provably dead the bind is reopened once (shared debounce) and every such +// peer re-initiates immediately; a healthy device is a no-op. Returns whether +// a rebind was actually scheduled. Never blocks on the rebind itself — the +// heavy part runs in a goroutine (see selfHealRebind). On a down or closed +// device it is a no-op, so callers racing idle-suspend or Close are safe. +func (device *Device) RebindIfSessionStale() bool { + if !device.giveUpRebind.enabled.Load() || !device.isUp() { + return false + } + var stale []*Peer + device.peers.RLock() + for _, peer := range device.peers.keyMap { + if peer.isRunning.Load() && device.sessionProvablyDead(peer) { + stale = append(stale, peer) + } + } + device.peers.RUnlock() + if len(stale) == 0 { + return false + } + return device.selfHealRebind("nudge", stale...) +} + +// selfHealRebind is the shared action behind all three triggers. Runs the +// heavy part in a goroutine so a timer callback (or a nudge caller) never +// blocks on BindUpdate's worker drain. Debounced to one rebind per +// RekeyAttemptTime per device across ALL triggers (CAS on `last` settles +// concurrent multi-peer races): an early rebind at ~15s suppresses the giveup +// rebind of the same failed series at ~90s. On a down or closed device +// BindUpdate does not reopen the socket, so a rebind racing idle-suspend +// (SPEC 020) or Close degrades to a no-op. +func (device *Device) selfHealRebind(trigger string, peers ...*Peer) bool { + if !device.giveUpRebind.enabled.Load() { + return false + } + if device.isClosed() { + return false + } + now := time.Now().Unix() + last := device.giveUpRebind.last.Load() + if now-last < int64(RekeyAttemptTime/time.Second) { + return false + } + if !device.giveUpRebind.last.CompareAndSwap(last, now) { + return false + } + fresh := device.giveUpRebind.freshPort.Load() + go func() { + if fresh { + device.net.Lock() + device.net.port = 0 + device.net.Unlock() + } + if err := device.BindUpdate(); err != nil { + device.log.Errorf("%v - Failed self-heal rebind (trigger=%s): %v", peers[0], trigger, err) + return + } + device.log.Verbosef("%v - Rebound socket for self-heal (trigger=%s, fresh port=%v)", peers[0], trigger, fresh) + for _, peer := range peers { + peer.SendHandshakeInitiation(false) + } + }() + return true +} diff --git a/device/lx_giveup_rebind_test.go b/device/lx_giveup_rebind_test.go new file mode 100644 index 0000000..bbd4706 --- /dev/null +++ b/device/lx_giveup_rebind_test.go @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: MIT + * + * lx: SPEC 041 — unit tests for the give-up rebind mechanics on top of the + * self-heal harness (lx_giveup_selfheal_test.go): fresh vs pinned port, + * debounce, and disabled = upstream parity. These use the post-fix API + * (SetGiveUpRebind) and are NOT expected to compile on the pre-fix base. + */ + +package device + +import ( + "testing" + "time" +) + +func waitOpens(t *testing.T, bind *gateBind, want int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for bind.openCount() < want { + if time.Now().After(deadline) { + t.Fatalf("bind reopened %d times, want %d", bind.openCount(), want) + } + time.Sleep(10 * time.Millisecond) + } +} + +// Fresh mode (listen_port not pinned): the rebind must ask the OS for a new +// ephemeral port — Open is called with port 0. +func TestGiveUpRebindFreshPort(t *testing.T) { + pair := newGiveUpPair(t, false) + pair.devA.SetGiveUpRebind(true, true) + + triggerGiveUp(t, pair.devA, pair.pkB) + waitOpens(t, pair.bindA, 2) + + ports := pair.bindA.portsSnapshot() + if ports[1] != 0 { + t.Fatalf("rebind requested port %d, want 0 (fresh ephemeral)", ports[1]) + } +} + +// Pinned mode (explicit listen_port): the rebind must keep the current port. +// chanBind.Open reports its source id (1) as the actual port, so the device +// stores net.port=1 after the first Open and must reuse it. +func TestGiveUpRebindPinnedPortPreserved(t *testing.T) { + pair := newGiveUpPair(t, false) + pair.devA.SetGiveUpRebind(true, false) + + triggerGiveUp(t, pair.devA, pair.pkB) + waitOpens(t, pair.bindA, 2) + + ports := pair.bindA.portsSnapshot() + if ports[1] != 1 { + t.Fatalf("rebind requested port %d, want 1 (pinned)", ports[1]) + } +} + +// A second give-up inside the debounce window must not rebind again. +func TestGiveUpRebindDebounce(t *testing.T) { + pair := newGiveUpPair(t, false) + + triggerGiveUp(t, pair.devA, pair.pkB) + waitOpens(t, pair.bindA, 2) + + triggerGiveUp(t, pair.devA, pair.pkB) + time.Sleep(300 * time.Millisecond) + if got := pair.bindA.openCount(); got != 2 { + t.Fatalf("debounce failed: bind opened %d times, want 2", got) + } +} + +// Disabled: the give-up branch must behave exactly like upstream — flush and +// stop, no rebind. +func TestGiveUpRebindDisabled(t *testing.T) { + pair := newGiveUpPair(t, false) + pair.devA.SetGiveUpRebind(false, false) + + triggerGiveUp(t, pair.devA, pair.pkB) + time.Sleep(300 * time.Millisecond) + if got := pair.bindA.openCount(); got != 1 { + t.Fatalf("disabled mechanism still rebound: %d opens, want 1", got) + } +} diff --git a/device/lx_giveup_selfheal_test.go b/device/lx_giveup_selfheal_test.go new file mode 100644 index 0000000..2f59471 --- /dev/null +++ b/device/lx_giveup_selfheal_test.go @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: MIT + * + * lx: SPEC 041 — behavioural red/green test for the handshake give-up + * self-heal. Field failure mode (WARP/AWG after device sleep): the per-flow + * path state of the socket's 5-tuple dies (expired NAT mapping / poisoned DPI + * flow entry), every packet sent from the old socket vanishes, and upstream + * wireguard-go retries into that dead socket forever — only a manual + * reconnect (new socket, new ephemeral port) heals the peer. + * + * The test models the dead 5-tuple with a bind whose FIRST socket generation + * silently swallows every send; any socket opened after a rebind delivers + * normally. It then drives the peer into the give-up branch of + * expiredRetransmitHandshake and expects traffic to flow end to end without + * any reconnect: + * + * - pre-fix (base): give-up only flushes staged packets, the bind is never + * reopened, every retry keeps dying in the first generation -> timeout; + * - post-fix: give-up rebinds the socket and re-initiates -> tunnel comes + * up and the packet arrives. + * + * This file deliberately uses NO post-fix API, so it compiles and runs RED on + * the pre-fix base commit. Reuses the chanBind/chanTun harness from + * transport_padding_test.go. + */ + +package device + +import ( + "context" + "encoding/hex" + "fmt" + "sync" + "testing" + "time" + + "github.com/sagernet/wireguard-go/conn" +) + +// gateBind wraps chanBind: it records every Open (the port argument the +// device asked for) and silently swallows sends while the socket generation +// is at most dropOpens — modelling a dead 5-tuple whose packets vanish on the +// path without any local error. +type gateBind struct { + *chanBind + mu sync.Mutex + openPorts []uint16 + opens int + dropOpens int // swallow sends while opens <= dropOpens +} + +func (b *gateBind) Open(port uint16) ([]conn.ReceiveFunc, uint16, error) { + fns, actual, err := b.chanBind.Open(port) + b.mu.Lock() + b.opens++ + b.openPorts = append(b.openPorts, port) + b.mu.Unlock() + return fns, actual, err +} + +func (b *gateBind) Send(bufs [][]byte, ep conn.Endpoint, offset int) error { + b.mu.Lock() + drop := b.opens <= b.dropOpens + b.mu.Unlock() + if drop { + return nil // the dead 5-tuple: no local error, the packet just vanishes + } + return b.chanBind.Send(bufs, ep, offset) +} + +func (b *gateBind) openCount() int { + b.mu.Lock() + defer b.mu.Unlock() + return b.opens +} + +func (b *gateBind) portsSnapshot() []uint16 { + b.mu.Lock() + defer b.mu.Unlock() + return append([]uint16(nil), b.openPorts...) +} + +type giveUpPair struct { + devA, devB *Device + tunA, tunB *chanTun + bindA *gateBind + pkB NoisePublicKey +} + +// newGiveUpPair builds two Up()'d peered devices; devA sits on a gateBind +// whose first socket generation optionally blackholes all sends. +func newGiveUpPair(t *testing.T, dropFirstOpen bool) *giveUpPair { + t.Helper() + + skA, err := newPrivateKey() + if err != nil { + t.Fatalf("newPrivateKey A: %v", err) + } + skB, err := newPrivateKey() + if err != nil { + t.Fatalf("newPrivateKey B: %v", err) + } + pkA := skA.publicKey() + pkB := skB.publicKey() + + rawA, rawB := newChanBindPair() + bindA := &gateBind{chanBind: rawA} + if dropFirstOpen { + bindA.dropOpens = 1 + } + tunA := newChanTun() + tunB := newChanTun() + + devA := NewDevice(context.Background(), tunA, bindA, NewLogger(LogLevelError, "devA: "), 1) + devB := NewDevice(context.Background(), tunB, rawB, NewLogger(LogLevelError, "devB: "), 1) + t.Cleanup(devA.Close) + t.Cleanup(devB.Close) + + cfgA := fmt.Sprintf( + "private_key=%s\nreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:2\nallowed_ip=%s/32\n", + hex.EncodeToString(skA[:]), hex.EncodeToString(pkB[:]), testIPB) + cfgB := fmt.Sprintf( + "private_key=%s\nreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:1\nallowed_ip=%s/32\n", + hex.EncodeToString(skB[:]), hex.EncodeToString(pkA[:]), testIPA) + + if err := devA.IpcSet(cfgA); err != nil { + t.Fatalf("IpcSet A: %v", err) + } + if err := devB.IpcSet(cfgB); err != nil { + t.Fatalf("IpcSet B: %v", err) + } + if err := devA.Up(); err != nil { + t.Fatalf("Up A: %v", err) + } + if err := devB.Up(); err != nil { + t.Fatalf("Up B: %v", err) + } + + return &giveUpPair{devA: devA, devB: devB, tunA: tunA, tunB: tunB, bindA: bindA, pkB: pkB} +} + +// triggerGiveUp drives dev's peer into the give-up branch of +// expiredRetransmitHandshake exactly the way 90s of unanswered retries would: +// attempts past the limit, last initiation older than RekeyTimeout. +func triggerGiveUp(t *testing.T, dev *Device, pk NoisePublicKey) { + t.Helper() + dev.peers.RLock() + peer := dev.peers.keyMap[pk] + dev.peers.RUnlock() + if peer == nil { + t.Fatal("peer not found") + } + peer.handshake.mutex.Lock() + peer.handshake.lastSentHandshake = time.Now().Add(-2 * RekeyTimeout) + peer.handshake.mutex.Unlock() + peer.timers.handshakeAttempts.Store(MaxTimerHandshakes + 1) + expiredRetransmitHandshake(peer) +} + +// TestHandshakeGiveUpSelfHeal: dead first socket, give-up fires — the tunnel +// must come up and deliver traffic without any reconnect. +func TestHandshakeGiveUpSelfHeal(t *testing.T) { + pair := newGiveUpPair(t, true) + + pkt := buildIPv4Packet(testIPA, testIPB, 8) + send := func() { pair.tunA.toDevice <- pkt } + + // Traffic demand: stages the packet and sends the first (blackholed) + // initiation, exactly the state a real give-up cycle ends in. + send() + triggerGiveUp(t, pair.devA, pair.pkB) + awaitPacket(t, pair.tunB, pkt, send) +} diff --git a/device/lx_ipcget_awg_test.go b/device/lx_ipcget_awg_test.go new file mode 100644 index 0000000..a5954e6 --- /dev/null +++ b/device/lx_ipcget_awg_test.go @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT + * + * Pins the AWG get path: IpcGet must report every obfuscation parameter it + * accepted, including i1..i5. The I-slots are emitted by an `i%d=` loop rather + * than literal per-key sendf calls, which makes them easy to miss when auditing + * introspection parity against amneziawg-go by grep alone. + */ + +package device + +import ( + "context" + "encoding/hex" + "strings" + "testing" +) + +func TestIpcGetReportsAWGParams(t *testing.T) { + sk, err := newPrivateKey() + if err != nil { + t.Fatalf("newPrivateKey: %v", err) + } + + bind, _ := newChanBindPair() + dev := NewDevice(context.Background(), newChanTun(), bind, NewLogger(LogLevelError, "dev: "), 1) + t.Cleanup(dev.Close) + + set := strings.Join([]string{ + "private_key=" + hex.EncodeToString(sk[:]), + "jc=4", "jmin=40", "jmax=70", + "s1=15", "s2=20", "s3=25", "s4=30", + "h1=1", "h2=2", "h3=3", "h4=100-200", + "i1=", "i3=", "i5=", + "", + }, "\n") + if err := dev.IpcSet(set); err != nil { + t.Fatalf("IpcSet: %v", err) + } + + got, err := dev.IpcGet() + if err != nil { + t.Fatalf("IpcGet: %v", err) + } + t.Logf("IpcGet:\n%s", got) + + for _, want := range []string{ + "jc=4", "jmin=40", "jmax=70", + "s1=15", "s2=20", "s3=25", "s4=30", + "h1=1", "h2=2", "h3=3", "h4=100-200", + "i1=", "i3=", "i5=", + } { + if !strings.Contains(got, want) { + t.Errorf("IpcGet missing %q", want) + } + } + + // Unset I-slots must stay absent, not surface as empty values. + for _, absent := range []string{"i2=", "i4="} { + if strings.Contains(got, absent) { + t.Errorf("IpcGet reported unset %q", absent) + } + } +} diff --git a/device/lx_stale_rebind_test.go b/device/lx_stale_rebind_test.go new file mode 100644 index 0000000..91b8a31 --- /dev/null +++ b/device/lx_stale_rebind_test.go @@ -0,0 +1,228 @@ +/* SPDX-License-Identifier: MIT + * + * lx: SPEC 041 v2 — unit tests for the stale predicate, the wake nudge + * (RebindIfSessionStale) and the shared debounce across triggers, on top of + * the v1 harness (lx_giveup_selfheal_test.go). These use the post-fix API and + * are NOT expected to compile on the pre-fix base. + */ + +package device + +import ( + "sync" + "testing" + "time" +) + +// establishTunnel completes a real handshake over a healthy pair so the peer +// holds a live keypair and a fresh lastHandshakeNano. +func establishTunnel(t *testing.T, pair *giveUpPair) { + t.Helper() + pkt := buildIPv4Packet(testIPA, testIPB, 8) + send := func() { pair.tunA.toDevice <- pkt } + send() + awaitPacket(t, pair.tunB, pkt, send) +} + +func peerOf(t *testing.T, dev *Device, pk NoisePublicKey) *Peer { + t.Helper() + dev.peers.RLock() + peer := dev.peers.keyMap[pk] + dev.peers.RUnlock() + if peer == nil { + t.Fatal("peer not found") + } + return peer +} + +// A cold peer (no keypair yet, nothing to lose): the nudge must rebind and +// immediately initiate — the tunnel comes up without any traffic demand. +func TestNudgeRebindsStaleSession(t *testing.T) { + pair := newGiveUpPair(t, true) + + if !pair.devA.RebindIfSessionStale() { + t.Fatal("nudge on a cold (keypair-less) session must rebind") + } + waitOpens(t, pair.bindA, 2) + + // The immediate initiation must bring the tunnel up: traffic sent only + // AFTER the nudge flows end to end. + pkt := buildIPv4Packet(testIPA, testIPB, 8) + send := func() { pair.tunA.toDevice <- pkt } + send() + awaitPacket(t, pair.tunB, pkt, send) +} + +// A healthy session (live keypair, fresh handshake) must be a strict no-op. +func TestNudgeHealthySessionNoop(t *testing.T) { + pair := newGiveUpPair(t, false) + establishTunnel(t, pair) + + opens := pair.bindA.openCount() + if pair.devA.RebindIfSessionStale() { + t.Fatal("nudge on a healthy session must not rebind") + } + time.Sleep(100 * time.Millisecond) + if got := pair.bindA.openCount(); got != opens { + t.Fatalf("healthy nudge reopened the bind: %d opens, want %d", got, opens) + } +} + +// A live keypair whose last handshake is older than RejectAfterTime is +// provably dead (the keys are invalid): the nudge must rebind. +func TestNudgeExpiredHandshakeIsStale(t *testing.T) { + pair := newGiveUpPair(t, false) + establishTunnel(t, pair) + + peer := peerOf(t, pair.devA, pair.pkB) + peer.lastHandshakeNano.Store(time.Now().Add(-RejectAfterTime - time.Second).UnixNano()) + + if !pair.devA.RebindIfSessionStale() { + t.Fatal("nudge on an expired session must rebind") + } + waitOpens(t, pair.bindA, 2) +} + +// A down device (how SPEC 020 idle-suspend leaves it) must be a no-op — the +// nudge never wakes sleepers. +func TestNudgeDownDeviceNoop(t *testing.T) { + pair := newGiveUpPair(t, true) + + if err := pair.devA.Down(); err != nil { + t.Fatalf("Down: %v", err) + } + if pair.devA.RebindIfSessionStale() { + t.Fatal("nudge on a down device must be a no-op") + } +} + +// Pinned listen_port survives the nudge rebind. +func TestNudgePinnedPortPreserved(t *testing.T) { + pair := newGiveUpPair(t, true) + pair.devA.SetGiveUpRebind(true, false) + + if !pair.devA.RebindIfSessionStale() { + t.Fatal("nudge must rebind a cold session") + } + waitOpens(t, pair.bindA, 2) + + ports := pair.bindA.portsSnapshot() + if ports[1] != 1 { + t.Fatalf("nudge rebind requested port %d, want 1 (pinned)", ports[1]) + } +} + +// The debounce window is SHARED across triggers: an early/nudge rebind +// suppresses the give-up rebind of the same failed series, and a later series +// (window elapsed) heals again — sliding window, not a latch. +func TestSharedDebounceAcrossTriggers(t *testing.T) { + pair := newGiveUpPair(t, false) + + // First trigger of the series: nudge. + if !pair.devA.RebindIfSessionStale() { + t.Fatal("first nudge must rebind") + } + waitOpens(t, pair.bindA, 2) + + // The give-up of the same series lands inside the window: suppressed. + triggerGiveUp(t, pair.devA, pair.pkB) + time.Sleep(300 * time.Millisecond) + if got := pair.bindA.openCount(); got != 2 { + t.Fatalf("give-up inside the shared window rebound: %d opens, want 2", got) + } + + // Next series: age the window as wall clocks would — heals again. + pair.devA.giveUpRebind.last.Store(time.Now().Add(-RekeyAttemptTime - time.Second).Unix()) + triggerGiveUp(t, pair.devA, pair.pkB) + waitOpens(t, pair.bindA, 3) +} + +// The early trigger must NOT fire before enough initiations went unanswered, +// even against a provably dead session. +func TestEarlyRebindNeedsMinAttempts(t *testing.T) { + pair := newGiveUpPair(t, true) + peer := peerOf(t, pair.devA, pair.pkB) + + peer.handshake.mutex.Lock() + peer.handshake.lastSentHandshake = time.Now().Add(-2 * RekeyTimeout) + peer.handshake.mutex.Unlock() + peer.timers.handshakeAttempts.Store(0) // this expiry brings it to 1 (< min) + expiredRetransmitHandshake(peer) + + time.Sleep(300 * time.Millisecond) + if got := pair.bindA.openCount(); got != 1 { + t.Fatalf("early rebind fired below the attempt floor: %d opens, want 1", got) + } +} + +// A fresh session (live keypair, recent handshake) must keep the retry branch +// byte-for-byte upstream even past the attempt floor: no rebind. +func TestEarlyRebindFreshSessionNoop(t *testing.T) { + pair := newGiveUpPair(t, false) + establishTunnel(t, pair) + peer := peerOf(t, pair.devA, pair.pkB) + + opens := pair.bindA.openCount() + peer.handshake.mutex.Lock() + peer.handshake.lastSentHandshake = time.Now().Add(-2 * RekeyTimeout) + peer.handshake.mutex.Unlock() + peer.timers.handshakeAttempts.Store(earlyGiveUpMinAttempts) + expiredRetransmitHandshake(peer) + + time.Sleep(300 * time.Millisecond) + if got := pair.bindA.openCount(); got != opens { + t.Fatalf("early rebind fired on a fresh session: %d opens, want %d", got, opens) + } +} + +// Nudge racing Close: no panic, no deadlock, no race-detector report. +func TestNudgeRacesClose(t *testing.T) { + for i := 0; i < 25; i++ { + pair := newGiveUpPair(t, false) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + pair.devA.RebindIfSessionStale() + }() + go func() { + defer wg.Done() + pair.devA.Close() + }() + wg.Wait() + } +} + +// Nudge racing Down/Up (the SPEC 020 suspend/resume shape): the device must +// end consistent — up, with a live bind. +func TestNudgeRacesSuspend(t *testing.T) { + for i := 0; i < 25; i++ { + pair := newGiveUpPair(t, false) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + pair.devA.RebindIfSessionStale() + }() + go func() { + defer wg.Done() + if err := pair.devA.Down(); err != nil { + t.Errorf("Down: %v", err) + } + if err := pair.devA.Up(); err != nil { + t.Errorf("Up: %v", err) + } + }() + wg.Wait() + + pair.devA.net.RLock() + bindAlive := pair.devA.net.bind != nil + pair.devA.net.RUnlock() + if !bindAlive || !pair.devA.isUp() { + t.Fatalf("iteration %d: device inconsistent after nudge/suspend race (bind=%v up=%v)", + i, bindAlive, pair.devA.isUp()) + } + pair.devA.Close() + pair.devB.Close() + } +} diff --git a/device/magic-header.go b/device/magic-header.go index 78e59d6..6ea0ce5 100644 --- a/device/magic-header.go +++ b/device/magic-header.go @@ -57,7 +57,9 @@ func (h *magicHeader) Validate(val uint32) bool { } func (h *magicHeader) Generate() uint32 { - high := int64(h.end - h.start + 1) + // Widen before arithmetic: end-start+1 in uint32 wraps to 0 for the + // full 0..2^32-1 range, which would panic rand.Int (bound <= 0). + high := int64(h.end) - int64(h.start) + 1 r, _ := rand.Int(rand.Reader, big.NewInt(high)) return h.start + uint32(r.Int64()) } diff --git a/device/noise-protocol.go b/device/noise-protocol.go index 745edd1..75fa025 100644 --- a/device/noise-protocol.go +++ b/device/noise-protocol.go @@ -351,17 +351,22 @@ func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation, endpoint return nil } + // Snapshot staticIdentity so we don't hold the RLock across LookupPeer, + // which may call NewPeer (reentrant RLock deadlocks against a pending + // SetPrivateKey writer; see lock-ordering.md). device.staticIdentity.RLock() - defer device.staticIdentity.RUnlock() + publicKey := device.staticIdentity.publicKey + privateKey := device.staticIdentity.privateKey + device.staticIdentity.RUnlock() - mixHash(&hash, &InitialHash, device.staticIdentity.publicKey[:]) + mixHash(&hash, &InitialHash, publicKey[:]) mixHash(&hash, &hash, msg.Ephemeral[:]) mixKey(&chainKey, &InitialChainKey, msg.Ephemeral[:]) // decrypt static key var peerPK NoisePublicKey var key [chacha20poly1305.KeySize]byte - ss, err := device.staticIdentity.privateKey.sharedSecret(msg.Ephemeral) + ss, err := privateKey.sharedSecret(msg.Ephemeral) if err != nil { return nil } @@ -536,6 +541,14 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer { chainKey [blake2s.Size]byte ) + // Snapshot the static private key before acquiring handshake.mutex so + // that handshake.mutex is never held while acquiring staticIdentity + // (which would invert the staticIdentity < handshake.mutex hierarchy; + // see lock-ordering.md). + device.staticIdentity.RLock() + privateKey := device.staticIdentity.privateKey + device.staticIdentity.RUnlock() + ok := func() bool { // lock handshake state @@ -546,11 +559,6 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer { return false } - // lock private key for reading - - device.staticIdentity.RLock() - defer device.staticIdentity.RUnlock() - // finish 3-way DH mixHash(&hash, &handshake.hash, msg.Ephemeral[:]) @@ -563,7 +571,7 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer { mixKey(&chainKey, &chainKey, ss[:]) setZero(ss[:]) - ss, err = device.staticIdentity.privateKey.sharedSecret(msg.Ephemeral) + ss, err = privateKey.sharedSecret(msg.Ephemeral) if err != nil { return false } diff --git a/device/obf.go b/device/obf.go index 53c55ff..269007c 100644 --- a/device/obf.go +++ b/device/obf.go @@ -3,11 +3,26 @@ package device import ( "errors" "fmt" + "strconv" "strings" ) type obfBuilder func(val string) (obf, error) +// parseObfLen parses and bounds an obfuscator length argument: a negative +// value would panic slice bounds in obfChain.Obfuscate, a huge one would +// OOM in the handshake-time make (SendHandshakeInitiation). +func parseObfLen(val string) (int, error) { + length, err := strconv.Atoi(val) + if err != nil { + return 0, err + } + if length < 0 || length > MaxMessageSize { + return 0, fmt.Errorf("obfuscator length %d out of range [0, %d]", length, MaxMessageSize) + } + return length, nil +} + var obfBuilders = map[string]obfBuilder{ "b": newBytesObf, "t": newTimestampObf, diff --git a/device/obf_datasize.go b/device/obf_datasize.go index 7267e2a..8ad1a71 100644 --- a/device/obf_datasize.go +++ b/device/obf_datasize.go @@ -1,9 +1,7 @@ package device -import "strconv" - func newDataSizeObf(val string) (obf, error) { - length, err := strconv.Atoi(val) + length, err := parseObfLen(val) if err != nil { return nil, err } diff --git a/device/obf_guards_test.go b/device/obf_guards_test.go new file mode 100644 index 0000000..5080196 --- /dev/null +++ b/device/obf_guards_test.go @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: MIT + * + * Guards around AWG obfuscation config values: these tests pin the + * crash-on-config-value fixes (swapped jmin/jmax, out-of-range obfuscator + * lengths, full-range magic headers). + */ + +package device + +import ( + "context" + "encoding/hex" + "fmt" + "testing" +) + +func TestParseObfLen(t *testing.T) { + cases := []struct { + val string + want int + wantErr bool + }{ + {"0", 0, false}, + {"100", 100, false}, + {fmt.Sprintf("%d", MaxMessageSize), MaxMessageSize, false}, + {"-1", 0, true}, // would panic slice bounds in Obfuscate + {fmt.Sprintf("%d", MaxMessageSize+1), 0, true}, // would OOM the handshake make + {"2000000000", 0, true}, + {"abc", 0, true}, + } + for _, c := range cases { + got, err := parseObfLen(c.val) + if c.wantErr != (err != nil) { + t.Errorf("parseObfLen(%q): err = %v, wantErr = %v", c.val, err, c.wantErr) + } + if err == nil && got != c.want { + t.Errorf("parseObfLen(%q) = %d, want %d", c.val, got, c.want) + } + } +} + +func TestMagicHeaderGenerateFullRange(t *testing.T) { + // end-start+1 computed in uint32 wraps to 0 for the full range and + // panics rand.Int; the fix widens to int64 before the arithmetic. + h := &magicHeader{start: 0, end: ^uint32(0)} + for i := 0; i < 8; i++ { + v := h.Generate() + if !h.Validate(v) { + t.Fatalf("generated value %d outside range", v) + } + } +} + +// TestJunkSwappedBounds brings up a device pair whose junk config has +// jmin > jmax (passes per-field UAPI validation); without the swap guard +// the first handshake panics rand.Int with a non-positive bound. +func TestJunkSwappedBounds(t *testing.T) { + skA, err := newPrivateKey() + if err != nil { + t.Fatalf("newPrivateKey A: %v", err) + } + skB, err := newPrivateKey() + if err != nil { + t.Fatalf("newPrivateKey B: %v", err) + } + pkA := skA.publicKey() + pkB := skB.publicKey() + + bindA, bindB := newChanBindPair() + tunA := newChanTun() + tunB := newChanTun() + + devA := NewDevice(context.Background(), tunA, bindA, NewLogger(LogLevelError, "devA: "), 1) + devB := NewDevice(context.Background(), tunB, bindB, NewLogger(LogLevelError, "devB: "), 1) + t.Cleanup(devA.Close) + t.Cleanup(devB.Close) + + // jmin deliberately greater than jmax: each field alone is valid. + junk := "jc=2\njmin=100\njmax=50\n" + cfgA := fmt.Sprintf( + "private_key=%s\n%sreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:2\nallowed_ip=%s/32\n", + hex.EncodeToString(skA[:]), junk, hex.EncodeToString(pkB[:]), testIPB) + cfgB := fmt.Sprintf( + "private_key=%s\n%sreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:1\nallowed_ip=%s/32\n", + hex.EncodeToString(skB[:]), junk, hex.EncodeToString(pkA[:]), testIPA) + + if err := devA.IpcSet(cfgA); err != nil { + t.Fatalf("IpcSet A: %v", err) + } + if err := devB.IpcSet(cfgB); err != nil { + t.Fatalf("IpcSet B: %v", err) + } + + if err := devA.Up(); err != nil { + t.Fatalf("Up A: %v", err) + } + if err := devB.Up(); err != nil { + t.Fatalf("Up B: %v", err) + } + + // Drive a packet end-to-end: the handshake (junk packets included) + // must complete without panicking the process. + pkt := buildIPv4Packet(testIPA, testIPB, 28) + devA.InputPacket(testIPB.AsSlice(), [][]byte{pkt}) + awaitPacket(t, tunB, pkt, func() { + devA.InputPacket(testIPB.AsSlice(), [][]byte{pkt}) + }) +} diff --git a/device/obf_rand.go b/device/obf_rand.go index edf461e..1560460 100644 --- a/device/obf_rand.go +++ b/device/obf_rand.go @@ -2,11 +2,10 @@ package device import ( "crypto/rand" - "strconv" ) func newRandObf(val string) (obf, error) { - length, err := strconv.Atoi(val) + length, err := parseObfLen(val) if err != nil { return nil, err } diff --git a/device/obf_randchars.go b/device/obf_randchars.go index 1d9968c..470ca6f 100644 --- a/device/obf_randchars.go +++ b/device/obf_randchars.go @@ -2,14 +2,13 @@ package device import ( "crypto/rand" - "strconv" "unicode" ) const chars52 = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" func newRandCharObf(val string) (obf, error) { - length, err := strconv.Atoi(val) + length, err := parseObfLen(val) if err != nil { return nil, err } diff --git a/device/obf_randdigits.go b/device/obf_randdigits.go index 4794bb1..d3585a0 100644 --- a/device/obf_randdigits.go +++ b/device/obf_randdigits.go @@ -2,14 +2,13 @@ package device import ( "crypto/rand" - "strconv" "unicode" ) const digits10 = "0123456789" func newRandDigitsObf(val string) (obf, error) { - length, err := strconv.Atoi(val) + length, err := parseObfLen(val) if err != nil { return nil, err } diff --git a/device/peer.go b/device/peer.go index bca121d..9726f90 100644 --- a/device/peer.go +++ b/device/peer.go @@ -8,6 +8,8 @@ package device import ( "container/list" "errors" + "net/netip" + "slices" "sync" "sync/atomic" "time" @@ -25,6 +27,20 @@ type Peer struct { rxBytes atomic.Uint64 // bytes received from peer lastHandshakeNano atomic.Int64 // nano seconds since epoch + sessionState struct { + sync.Mutex + current PeerSessionState + sessionExpires time.Time + } + + queuedOutboundPackets atomic.Int32 // packets in staged+outbound queues, for input backpressure + + // deleteOnIdle indicates whether the peer should be deleted when idle + // because it was auto-created via a Device.PeerLookupFunc. + // + // This field should only be set once, before the peer is started. + deleteOnIdle bool + endpoint struct { sync.Mutex val conn.Endpoint @@ -36,6 +52,7 @@ type Peer struct { retransmitHandshake *Timer sendKeepalive *Timer newHandshake *Timer + sessionExpired *Timer zeroKeyMaterial *Timer persistentKeepalive *Timer handshakeAttempts atomic.Uint32 @@ -44,7 +61,14 @@ type Peer struct { } state struct { - sync.Mutex // protects against concurrent Start/Stop + sync.Mutex // protects against concurrent Start/Stop, and fields below + + allowedIPs []netip.Prefix + + // testAllowedIP, if non-nil, is used to test whether the peer is + // allowed to send a packet from the given IP address. It can be read + // without locking, but must be set with the state mutex locked. + testAllowedIP atomic.Pointer[func(netip.Addr) bool] } queue struct { @@ -87,7 +111,7 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) { // map public key _, ok := device.peers.keyMap[pk] if ok { - return nil, errors.New("adding existing peer") + return nil, errAddExistingPeer } // pre-compute DH @@ -113,6 +137,27 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) { return peer, nil } +// SetAllowedIPs sets the allowed IP prefixes for this peer. +// +// If the allowedIPs are unchanged since the last call, this method is a no-op. +// It's the caller's responsibility to ensure that no two peers have duplicate +// allowed IPs. If so, the last writer wins. +func (p *Peer) SetAllowedIPs(allowedIPs []netip.Prefix) { + p.state.Lock() + defer p.state.Unlock() + + if slices.Equal(p.state.allowedIPs, allowedIPs) { + return + } + p.device.allowedips.setPeerPrefixes(p, allowedIPs) + + allowedIPs = slices.Clone(allowedIPs) // avoid retaining caller's slice + p.state.allowedIPs = allowedIPs + + f := mkIPInCIDRsTestFunc(allowedIPs) + p.state.testAllowedIP.Store(&f) +} + // SendBuffers sends buffers to peer. WireGuard packet data in each element of // buffers must be preceded by MessageEncapsulatingTransportSize number of // bytes. @@ -193,6 +238,7 @@ func (peer *Peer) Start() { // reset routine state peer.stopping.Wait() peer.stopping.Add(2) + peer.queuedOutboundPackets.Store(0) peer.handshake.mutex.Lock() peer.handshake.lastSentHandshake = time.Now().Add(-(RekeyTimeout + time.Second)) @@ -202,8 +248,8 @@ func (peer *Peer) Start() { peer.timersStart() - device.flushInboundQueue(peer.queue.inbound) - device.flushOutboundQueue(peer.queue.outbound) + device.flushInboundQueue(peer.queue.inbound.c) + device.flushOutboundQueue(peer.queue.outbound.c) // Use the device batch size, not the bind batch size, as the device size is // the size of the batch pools. @@ -212,10 +258,21 @@ func (peer *Peer) Start() { go peer.RoutineSequentialReceiver(batchSize) peer.isRunning.Store(true) + + // A lazily-created peer that never completes a handshake otherwise never + // arms its reaping timer. Arm it here, while running under state.Lock, so + // it's reclaimed after RejectAfterTime*3 of no session and is guaranteed to + // be torn down by a matching Stop. A completed handshake re-Mods it. + if peer.deleteOnIdle { + peer.timers.zeroKeyMaterial.Mod(RejectAfterTime * 3) + } } func (peer *Peer) ZeroAndFlushAll() { device := peer.device + if peer.timers.sessionExpired != nil { + peer.timers.sessionExpired.Del() + } // clear key pairs @@ -238,6 +295,11 @@ func (peer *Peer) ZeroAndFlushAll() { handshake.mutex.Unlock() peer.FlushStagedPackets() + + peer.sessionState.Lock() + peer.sessionState.sessionExpires = time.Time{} + peer.noteSessionStateLocked(PeerSessionNone) + peer.sessionState.Unlock() } func (peer *Peer) ExpireCurrentKeypairs() { @@ -257,6 +319,11 @@ func (peer *Peer) ExpireCurrentKeypairs() { next.sendNonce.Store(RejectAfterMessages) } keypairs.Unlock() + + peer.sessionState.Lock() + peer.sessionState.sessionExpires = time.Time{} + peer.noteSessionStateLocked(PeerSessionExpired) + peer.sessionState.Unlock() } func (peer *Peer) Stop() { @@ -279,6 +346,51 @@ func (peer *Peer) Stop() { peer.ZeroAndFlushAll() } +func (peer *Peer) noteSessionState(state PeerSessionState) { + peer.sessionState.Lock() + defer peer.sessionState.Unlock() + peer.noteSessionStateLocked(state) +} + +// noteSessionStateLocked records a session state transition and delivers the +// callback. The caller must hold peer.sessionState.Mutex during the +// state determination and transition. +func (peer *Peer) noteSessionStateLocked(state PeerSessionState) { + if peer.sessionState.current == state { + return + } + peer.sessionState.current = state + if f := peer.device.peerStateFn.Load(); f != nil { + (*f)(peer.handshake.remoteStatic, state) + } +} + +func (peer *Peer) noteSessionHandshakeStarted() { + peer.sessionState.Lock() + defer peer.sessionState.Unlock() + if peer.sessionState.current == PeerSessionEstablished { + return + } + peer.noteSessionStateLocked(PeerSessionHandshake) +} + +func (peer *Peer) noteSessionHandshakeStopped() { + peer.sessionState.Lock() + defer peer.sessionState.Unlock() + state := PeerSessionNone + if peer.hasKeyMaterial() { + state = PeerSessionExpired + } + peer.noteSessionStateLocked(state) +} + +func (peer *Peer) hasKeyMaterial() bool { + keypairs := &peer.keypairs + keypairs.RLock() + defer keypairs.RUnlock() + return keypairs.previous != nil || keypairs.current != nil || keypairs.next.Load() != nil +} + func (peer *Peer) SetEndpointFromPacket(endpoint conn.Endpoint) { peer.endpoint.Lock() defer peer.endpoint.Unlock() diff --git a/device/pools.go b/device/pools.go index 2c18f41..6a52472 100644 --- a/device/pools.go +++ b/device/pools.go @@ -7,6 +7,8 @@ package device import ( "sync" + + "github.com/sagernet/sing/common/buf" ) type WaitPool struct { @@ -23,6 +25,10 @@ func NewWaitPool(max uint32, new func() any) *WaitPool { return p } +func (p *WaitPool) hasAccounting() bool { + return p != nil && p.max != 0 +} + func (p *WaitPool) Get() any { if p.max != 0 { p.lock.Lock() @@ -47,28 +53,27 @@ func (p *WaitPool) Put(x any) { } func (device *Device) PopulatePools() { - device.pool.inboundElementsContainer = NewWaitPool(PreallocatedBuffersPerPool, func() any { + device.pool.inboundElementsContainer = &sync.Pool{New: func() any { s := make([]*QueueInboundElement, 0, device.BatchSize()) return &QueueInboundElementsContainer{elems: s} - }) - device.pool.outboundElementsContainer = NewWaitPool(PreallocatedBuffersPerPool, func() any { + }} + device.pool.outboundElementsContainer = &sync.Pool{New: func() any { s := make([]*QueueOutboundElement, 0, device.BatchSize()) return &QueueOutboundElementsContainer{elems: s} - }) + }} device.pool.messageBuffers = NewWaitPool(PreallocatedBuffersPerPool, func() any { return new([MaxMessageSize]byte) }) - device.pool.inboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() any { + device.pool.inboundElements = &sync.Pool{New: func() any { return new(QueueInboundElement) - }) - device.pool.outboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() any { + }} + device.pool.outboundElements = &sync.Pool{New: func() any { return new(QueueOutboundElement) - }) + }} } func (device *Device) GetInboundElementsContainer() *QueueInboundElementsContainer { c := device.pool.inboundElementsContainer.Get().(*QueueInboundElementsContainer) - c.Mutex = sync.Mutex{} return c } @@ -82,7 +87,6 @@ func (device *Device) PutInboundElementsContainer(c *QueueInboundElementsContain func (device *Device) GetOutboundElementsContainer() *QueueOutboundElementsContainer { c := device.pool.outboundElementsContainer.Get().(*QueueOutboundElementsContainer) - c.Mutex = sync.Mutex{} return c } @@ -102,6 +106,20 @@ func (device *Device) PutMessageBuffer(msg *[MaxMessageSize]byte) { device.pool.messageBuffers.Put(msg) } +// Outbound buffers come from the sing allocator instead of the bounded +// messageBuffers pool: the injection paths (InputPacket/InputPackets) run on +// the caller's shared read loop, which must never block on pool exhaustion, +// and their packets are far smaller than MaxMessageSize, so they are allocated +// by actual size. This also keeps the bounded pool exclusively for the receive +// path, so outbound backlog can no longer starve it. +func (device *Device) GetOutboundBuffer(size int) []byte { + return buf.Get(size) +} + +func (device *Device) PutOutboundBuffer(buffer []byte) { + _ = buf.Put(buffer) +} + func (device *Device) GetInboundElement() *QueueInboundElement { return device.pool.inboundElements.Get().(*QueueInboundElement) } diff --git a/device/queueconstants_android.go b/device/queueconstants_android.go index d0718d2..a3bee69 100644 --- a/device/queueconstants_android.go +++ b/device/queueconstants_android.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package device @@ -14,6 +14,6 @@ const ( QueueOutboundSize = 1024 QueueInboundSize = 1024 QueueHandshakeSize = 1024 - MaxSegmentSize = 2200 + MaxSegmentSize = (1 << 16) - 1 // largest possible UDP datagram PreallocatedBuffersPerPool = 4096 ) diff --git a/device/queueconstants_default.go b/device/queueconstants_default.go index 79b74b0..1d09285 100644 --- a/device/queueconstants_default.go +++ b/device/queueconstants_default.go @@ -2,7 +2,7 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package device diff --git a/device/receive.go b/device/receive.go index d9a0f69..8064f61 100644 --- a/device/receive.go +++ b/device/receive.go @@ -8,7 +8,9 @@ package device import ( "encoding/binary" "errors" + "fmt" "net" + "net/netip" "sync" "time" @@ -34,8 +36,13 @@ type QueueInboundElement struct { } type QueueInboundElementsContainer struct { - sync.Mutex - elems []*QueueInboundElement + // filling is a one-shot barrier signaling decryption→receive + // handoff. RoutineReceiveIncoming calls Add(1) before sending the + // container down the decryption and inbound queues; RoutineDecryption + // calls Done after decrypting; RoutineSequentialReceiver calls Wait + // before reading the decrypted packets. + filling sync.WaitGroup + elems []*QueueInboundElement } // clearPointers clears elem fields that contain pointers. @@ -96,13 +103,13 @@ func (device *Device) RoutineReceiveIncoming( elemsByPeer = make(map[*Peer]*QueueInboundElementsContainer, maxBatchSize) ) - for i := range maxBatchSize { + for i := range bufsArrs { bufsArrs[i] = device.GetMessageBuffer() bufs[i] = bufsArrs[i][:] } defer func() { - for i := range maxBatchSize { + for i := 0; i < maxBatchSize; i++ { if bufsArrs[i] != nil { device.PutMessageBuffer(bufsArrs[i]) } @@ -185,7 +192,6 @@ func (device *Device) RoutineReceiveIncoming( elemsForPeer, ok := elemsByPeer[peer] if !ok { elemsForPeer = device.GetInboundElementsContainer() - elemsForPeer.Lock() elemsByPeer[peer] = elemsForPeer } elemsForPeer.elems = append(elemsForPeer.elems, elem) @@ -229,6 +235,7 @@ func (device *Device) RoutineReceiveIncoming( } for peer, elemsContainer := range elemsByPeer { if peer.isRunning.Load() { + elemsContainer.filling.Add(1) peer.queue.inbound.c <- elemsContainer device.queue.decryption.c <- elemsContainer } else { @@ -270,7 +277,7 @@ func (device *Device) RoutineDecryption(id int) { elem.packet = nil } } - elemsContainer.Unlock() + elemsContainer.filling.Done() } } @@ -432,6 +439,7 @@ func (device *Device) RoutineHandshake(id int) { peer.timersSessionDerived() peer.timersHandshakeComplete() + peer.SendPriorityMessage() peer.SendKeepalive() } skip: @@ -453,118 +461,130 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) { if elemsContainer == nil { return } - elemsContainer.Lock() - validTailPacket := -1 - dataPacketReceived := false - rxBytesLen := uint64(0) - for i, elem := range elemsContainer.elems { - if elem.packet == nil { - // decryption failed + peer.processInboundContainer(elemsContainer, bufs[:0]) + } +} + +// processInboundContainer waits for the decryption routine to finish +// filling elemsContainer, then writes the valid packets to the TUN +// device and returns the container to the pool. +// +// scratch is a length-0 slice used to assemble the per-packet buffers +// passed to tun.device.Write; its backing array is reused across calls. +func (peer *Peer) processInboundContainer(elemsContainer *QueueInboundElementsContainer, scratch [][]byte) { + // Invariants from RoutineSequentialReceiver; all should be unreachable. + if len(scratch) != 0 || cap(scratch) == 0 { + panic(fmt.Sprintf("processInboundContainer: scratch must be empty with non-zero cap; got len=%d cap=%d", + len(scratch), cap(scratch))) + } + if cap(scratch) < len(elemsContainer.elems) { + panic(fmt.Sprintf("processInboundContainer: scratch cap %d < elems %d", + cap(scratch), len(elemsContainer.elems))) + } + + device := peer.device + defer device.PutInboundElementsContainer(elemsContainer) + + // Wait for RoutineDecryption to finish filling the container. After + // Wait returns we have happens-before with that goroutine and are the + // sole owner of the container until Put hands it back to the pool. + elemsContainer.filling.Wait() + elems := elemsContainer.elems + + validTailPacket := -1 + dataPacketReceived := false + rxBytesLen := uint64(0) + for i, elem := range elems { + if elem.packet == nil { + // decryption failed + continue + } + + if !elem.keypair.replayFilter.ValidateCounter(elem.counter, RejectAfterMessages) { + continue + } + + validTailPacket = i + if peer.ReceivedWithKeypair(elem.keypair) { + peer.SetEndpointFromPacket(elem.endpoint) + peer.timersHandshakeComplete() + peer.SendPriorityMessage() + peer.SendStagedPackets() + } + if ep, ok := elem.endpoint.(conn.PeerAwareEndpoint); ok { + ep.FromPeer(peer.handshake.remoteStatic) + } + rxBytesLen += uint64(len(elem.packet) + MinMessageSize) + + if len(elem.packet) == 0 { + device.log.Verbosef("%v - Receiving keepalive packet", peer) + continue + } + dataPacketReceived = true + + switch elem.packet[0] >> 4 { + case 4: + if len(elem.packet) < ipv4.HeaderLen { + continue + } + field := elem.packet[IPv4offsetTotalLength : IPv4offsetTotalLength+2] + length := binary.BigEndian.Uint16(field) + if int(length) > len(elem.packet) || int(length) < ipv4.HeaderLen { + continue + } + elem.packet = elem.packet[:length] + src := elem.packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len] + srcAddr, _ := netip.AddrFromSlice(src) + if !peer.AllowedPeerSourceIP(srcAddr) { + device.log.Verbosef("IPv4 packet with disallowed source address from %v", peer) continue } - if !elem.keypair.replayFilter.ValidateCounter(elem.counter, RejectAfterMessages) { + case 6: + if len(elem.packet) < ipv6.HeaderLen { + continue + } + field := elem.packet[IPv6offsetPayloadLength : IPv6offsetPayloadLength+2] + length := binary.BigEndian.Uint16(field) + length += ipv6.HeaderLen + if int(length) > len(elem.packet) { + continue + } + elem.packet = elem.packet[:length] + src := elem.packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len] + srcAddr, _ := netip.AddrFromSlice(src) + if !peer.AllowedPeerSourceIP(srcAddr) { + device.log.Verbosef("IPv6 packet with disallowed source address from %v", peer) continue } - validTailPacket = i - if peer.ReceivedWithKeypair(elem.keypair) { - peer.SetEndpointFromPacket(elem.endpoint) - peer.timersHandshakeComplete() - peer.SendStagedPackets() - } - if ep, ok := elem.endpoint.(conn.PeerAwareEndpoint); ok { - ep.FromPeer(peer.handshake.remoteStatic) - } - rxBytesLen += uint64(len(elem.packet) + MinMessageSize) - - if len(elem.packet) == 0 { - device.log.Verbosef("%v - Receiving keepalive packet", peer) - continue - } - dataPacketReceived = true - - switch elem.packet[0] >> 4 { - case 4: - if len(elem.packet) < ipv4.HeaderLen { - continue - } - field := elem.packet[IPv4offsetTotalLength : IPv4offsetTotalLength+2] - length := binary.BigEndian.Uint16(field) - if int(length) > len(elem.packet) || int(length) < ipv4.HeaderLen { - continue - } - elem.packet = elem.packet[:length] - src := elem.packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len] - if device.allowedips.Lookup(src) != peer { - device.log.Verbosef("IPv4 packet with disallowed source address from %v", peer) - continue - } - - case 6: - if len(elem.packet) < ipv6.HeaderLen { - continue - } - field := elem.packet[IPv6offsetPayloadLength : IPv6offsetPayloadLength+2] - length := binary.BigEndian.Uint16(field) - length += ipv6.HeaderLen - if int(length) > len(elem.packet) { - continue - } - elem.packet = elem.packet[:length] - src := elem.packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len] - if device.allowedips.Lookup(src) != peer { - device.log.Verbosef("IPv6 packet with disallowed source address from %v", peer) - continue - } - - default: - device.log.Verbosef( - "Packet with invalid IP version from %v", - peer, - ) - continue - } - - bufs = append( - bufs, - elem.buffer[:MessageTransportOffsetContent+len(elem.packet)], - ) + default: + device.log.Verbosef("Packet with invalid IP version from %v", peer) + continue } - peer.rxBytes.Add(rxBytesLen) - if validTailPacket >= 0 { - peer.SetEndpointFromPacket(elemsContainer.elems[validTailPacket].endpoint) - peer.keepKeyFreshReceiving() - peer.timersAnyAuthenticatedPacketTraversal() - peer.timersAnyAuthenticatedPacketReceived() - } - if dataPacketReceived { - peer.timersDataReceived() - } + scratch = append(scratch, elem.buffer[:MessageTransportOffsetContent+len(elem.packet)]) + } - peer.rxBytes.Add(rxBytesLen) - if validTailPacket >= 0 { - peer.SetEndpointFromPacket(elemsContainer.elems[validTailPacket].endpoint) - peer.keepKeyFreshReceiving() - peer.timersAnyAuthenticatedPacketTraversal() - peer.timersAnyAuthenticatedPacketReceived() + peer.rxBytes.Add(rxBytesLen) + if validTailPacket >= 0 { + peer.SetEndpointFromPacket(elems[validTailPacket].endpoint) + peer.keepKeyFreshReceiving() + peer.timersAnyAuthenticatedPacketTraversal() + peer.timersAnyAuthenticatedPacketReceived() + } + if dataPacketReceived { + peer.timersDataReceived() + } + if len(scratch) > 0 { + _, err := device.tun.device.Write(scratch, MessageTransportOffsetContent) + if err != nil && !device.isClosed() { + device.log.Errorf("Failed to write packets to TUN device: %v", err) } - if dataPacketReceived { - peer.timersDataReceived() - } - if len(bufs) > 0 { - _, err := device.tun.device.Write(bufs, MessageTransportOffsetContent) - if err != nil && !device.isClosed() { - device.log.Errorf("Failed to write packets to TUN device: %v", err) - } - } - for _, elem := range elemsContainer.elems { - device.PutMessageBuffer(elem.buffer) - device.PutInboundElement(elem) - } - bufs = bufs[:0] - device.PutInboundElementsContainer(elemsContainer) + } + for _, elem := range elems { + device.PutMessageBuffer(elem.buffer) + device.PutInboundElement(elem) } } diff --git a/device/send.go b/device/send.go index 7653f62..982d61f 100644 --- a/device/send.go +++ b/device/send.go @@ -10,8 +10,10 @@ import ( "crypto/rand" "encoding/binary" "errors" + "fmt" "math/big" "net" + "net/netip" "os" "sync" "time" @@ -48,7 +50,7 @@ import ( */ type QueueOutboundElement struct { - buffer *[MaxMessageSize]byte // slice holding the packet data + buffer []byte // sing-allocated buffer holding the packet data // packet is always a slice of "buffer". The starting offset in buffer // is either: // a) MessageEncapsulatingTransportSize+MessageTransportHeaderSize (plaintext) @@ -60,13 +62,18 @@ type QueueOutboundElement struct { } type QueueOutboundElementsContainer struct { - sync.Mutex - elems []*QueueOutboundElement + // filling is a one-shot barrier signaling encryption→send handoff. + // SendStagedPackets calls Add(1) before sending the container down + // the encryption and outbound queues; RoutineEncryption calls Done + // after encrypting; RoutineSequentialSender calls Wait before + // reading the encrypted packets. + filling sync.WaitGroup + elems []*QueueOutboundElement } func (device *Device) NewOutboundElement() *QueueOutboundElement { elem := device.GetOutboundElement() - elem.buffer = device.GetMessageBuffer() + elem.buffer = device.GetOutboundBuffer(MaxMessageSize) elem.nonce = 0 // keypair and peer were cleared (if necessary) by clearPointers. return elem @@ -92,9 +99,10 @@ func (peer *Peer) SendKeepalive() { elemsContainer.elems = append(elemsContainer.elems, elem) select { case peer.queue.staged <- elemsContainer: + peer.queuedOutboundPackets.Add(1) peer.device.log.Verbosef("%v - Sending keepalive packet", peer) default: - peer.device.PutMessageBuffer(elem.buffer) + peer.device.PutOutboundBuffer(elem.buffer) peer.device.PutOutboundElement(elem) peer.device.PutOutboundElementsContainer(elemsContainer) } @@ -102,6 +110,70 @@ func (peer *Peer) SendKeepalive() { peer.SendStagedPackets() } +// SendPriorityMessage invokes the [PeerPriorityMessageFunc] callback if one is +// set, and queues the returned message for encryption and transmission if the +// current keypair is valid. +func (peer *Peer) SendPriorityMessage() { + f := peer.device.priorityMsgFn.Load() + if f == nil { + return + } + keypair := peer.keypairs.Current() + if keypair == nil || keypair.sendNonce.Load() >= RejectAfterMessages || time.Since(keypair.created) >= RejectAfterTime { + // SendStagedPackets initializes a handshake when the keypair is invalid, + // but we explicitly avoid that here. A priority message is only intended + // to flow around symmetric session establishment, but it should never + // trigger a new session. Reaching this branch due to nonce exhaustion + // or keypair expiration is highly unlikely considering where + // SendPriorityMessage is called (at current keypair establishment). + return + } + + // get plaintext message to send + msg := (*f)(peer.handshake.remoteStatic) + if len(msg) == 0 { + return + } + if len(msg) > MaxPriorityMessageContentSize { + peer.device.log.Verbosef("%v - Failed to queue priority message due to size", peer) + return + } + + // get pooled elements + elem := peer.device.NewOutboundElement() + elemsContainer := peer.device.GetOutboundElementsContainer() + elemsContainer.elems = append(elemsContainer.elems, elem) + packetQueued := false + defer func() { + if !packetQueued { + peer.device.PutOutboundBuffer(elem.buffer) + peer.device.PutOutboundElement(elem) + peer.device.PutOutboundElementsContainer(elemsContainer) + } + }() + + // initialize outbound element + const offset = MessageEncapsulatingTransportSize + MessageTransportHeaderSize + n := copy(elem.buffer[offset:], msg) + elem.packet = elem.buffer[offset : offset+n] + elem.peer = peer + elem.nonce = keypair.sendNonce.Add(1) - 1 + if elem.nonce >= RejectAfterMessages { + keypair.sendNonce.Store(RejectAfterMessages) + return + } + elem.keypair = keypair + + // add to parallel and sequential queue + if peer.isRunning.Load() { + elemsContainer.filling.Add(1) + peer.queuedOutboundPackets.Add(1) + peer.queue.outbound.c <- elemsContainer + peer.device.queue.encryption.c <- elemsContainer + packetQueued = true + } +} + func (peer *Peer) SendHandshakeInitiation(isRetry bool) error { if !isRetry { peer.timers.handshakeAttempts.Store(0) @@ -143,6 +215,11 @@ func (peer *Peer) SendHandshakeInitiation(isRetry bool) error { jc := peer.device.junk.count jmin := peer.device.junk.min jmax := peer.device.junk.max + if jmax < jmin { + // UAPI validates jmin/jmax only individually; a swapped pair + // would panic rand.Int below with a non-positive bound. + jmin, jmax = jmax, jmin + } for i := 0; i < jc; i++ { nBig, _ := rand.Int(rand.Reader, big.NewInt(int64(jmax-jmin+1))) @@ -298,7 +375,7 @@ func (device *Device) RoutineReadFromTUN() { defer func() { for _, elem := range elems { if elem != nil { - device.PutMessageBuffer(elem.buffer) + device.PutOutboundBuffer(elem.buffer) device.PutOutboundElement(elem) } } @@ -322,15 +399,17 @@ func (device *Device) RoutineReadFromTUN() { if len(elem.packet) < ipv4.HeaderLen { continue } - dst := elem.packet[IPv4offsetDst : IPv4offsetDst+net.IPv4len] - peer = device.allowedips.Lookup(dst) + src := netip.AddrFrom4([4]byte(elem.packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len])) + dst := netip.AddrFrom4([4]byte(elem.packet[IPv4offsetDst : IPv4offsetDst+net.IPv4len])) + peer = device.allowedips.LookupFromPacket(src, dst, elem.packet) case 6: if len(elem.packet) < ipv6.HeaderLen { continue } - dst := elem.packet[IPv6offsetDst : IPv6offsetDst+net.IPv6len] - peer = device.allowedips.Lookup(dst) + src := netip.AddrFrom16([16]byte(elem.packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len])) + dst := netip.AddrFrom16([16]byte(elem.packet[IPv6offsetDst : IPv6offsetDst+net.IPv6len])) + peer = device.allowedips.LookupFromPacket(src, dst, elem.packet) default: device.log.Verbosef("Received packet with unknown IP version") @@ -355,7 +434,7 @@ func (device *Device) RoutineReadFromTUN() { peer.SendStagedPackets() } else { for _, elem := range elemsForPeer.elems { - device.PutMessageBuffer(elem.buffer) + device.PutOutboundBuffer(elem.buffer) device.PutOutboundElement(elem) } device.PutOutboundElementsContainer(elemsForPeer) @@ -382,12 +461,77 @@ func (device *Device) RoutineReadFromTUN() { } } +// maxQueuedInputPackets bounds the staged+outbound backlog of a peer fed via +// InputPacket/InputPackets. Injected packets beyond it are dropped before they +// are copied into pooled message buffers, like a full qdisc: injection has no +// flow control, and the queues are bounded in containers (up to a full batch +// each), so without this cap a flood is buffered instead of dropped. +const maxQueuedInputPackets = 2048 + +func (device *Device) inputPacketPeer(destination []byte, packetSlices [][]byte) *Peer { + var src, dst netip.Addr + switch len(destination) { + case net.IPv4len: + dst = netip.AddrFrom4([4]byte(destination)) + var srcBytes [net.IPv4len]byte + if !gatherPacketBytes(packetSlices, IPv4offsetSrc, srcBytes[:]) { + return nil + } + src = netip.AddrFrom4(srcBytes) + case net.IPv6len: + dst = netip.AddrFrom16([16]byte(destination)) + var srcBytes [net.IPv6len]byte + if !gatherPacketBytes(packetSlices, IPv6offsetSrc, srcBytes[:]) { + return nil + } + src = netip.AddrFrom16(srcBytes) + default: + return nil + } + var ipPkt []byte + if len(packetSlices) == 1 { + ipPkt = packetSlices[0] + } + return device.allowedips.LookupFromPacket(src, dst, ipPkt) +} + +func gatherPacketBytes(packetSlices [][]byte, offset int, destination []byte) bool { + for _, packetSlice := range packetSlices { + if offset >= len(packetSlice) { + offset -= len(packetSlice) + continue + } + n := copy(destination, packetSlice[offset:]) + destination = destination[n:] + offset = 0 + if len(destination) == 0 { + return true + } + } + return false +} + func (device *Device) InputPacket(destination []byte, packetSlices [][]byte) { - peer := device.allowedips.Lookup(destination) + peer := device.inputPacketPeer(destination, packetSlices) if peer == nil { return } - elem := device.NewOutboundElement() + if peer.queuedOutboundPackets.Load() >= maxQueuedInputPackets { + return + } + var totalLength int + for _, packetSlice := range packetSlices { + totalLength += len(packetSlice) + } + // paddings.transport (AWG s4) is prepended in-buffer by + // RoutineSequentialSender; reserve headroom for the shift. + allocLength := MessageEncapsulatingTransportSize + MessageTransportHeaderSize + totalLength + PaddingMultiple + chacha20poly1305.Overhead + device.paddings.transport + if allocLength > MaxMessageSize { + return + } + elem := device.GetOutboundElement() + elem.buffer = device.GetOutboundBuffer(allocLength) + elem.nonce = 0 packet := elem.buffer[MessageEncapsulatingTransportSize+MessageTransportHeaderSize:] var n int for _, packetSlice := range packetSlices { @@ -400,13 +544,77 @@ func (device *Device) InputPacket(destination []byte, packetSlices [][]byte) { peer.StagePackets(elemsForPeer) peer.SendStagedPackets() } else { - device.PutMessageBuffer(elem.buffer) + device.PutOutboundBuffer(elem.buffer) device.PutOutboundElement(elem) device.PutOutboundElementsContainer(elemsForPeer) } } +type InputPacketRef struct { + Destination []byte + PacketSlices [][]byte +} + +func (device *Device) InputPackets(packets []*InputPacketRef) []*InputPacketRef { + var unmatched []*InputPacketRef + elemsByPeer := make(map[*Peer][]*QueueOutboundElementsContainer, len(packets)) + for _, packetRef := range packets { + peer := device.inputPacketPeer(packetRef.Destination, packetRef.PacketSlices) + if peer == nil { + unmatched = append(unmatched, packetRef) + continue + } + if peer.queuedOutboundPackets.Load() >= maxQueuedInputPackets { + continue + } + var totalLength int + for _, packetSlice := range packetRef.PacketSlices { + totalLength += len(packetSlice) + } + // paddings.transport (AWG s4) is prepended in-buffer by + // RoutineSequentialSender; reserve headroom for the shift. + allocLength := MessageEncapsulatingTransportSize + MessageTransportHeaderSize + totalLength + PaddingMultiple + chacha20poly1305.Overhead + device.paddings.transport + if allocLength > MaxMessageSize { + continue + } + elem := device.GetOutboundElement() + elem.buffer = device.GetOutboundBuffer(allocLength) + elem.nonce = 0 + packet := elem.buffer[MessageEncapsulatingTransportSize+MessageTransportHeaderSize:] + var n int + for _, packetSlice := range packetRef.PacketSlices { + n += copy(packet[n:], packetSlice) + } + elem.packet = packet[:n] + containers := elemsByPeer[peer] + if len(containers) == 0 || len(containers[len(containers)-1].elems) >= conn.IdealBatchSize { + containers = append(containers, device.GetOutboundElementsContainer()) + elemsByPeer[peer] = containers + } + elemsForPeer := containers[len(containers)-1] + elemsForPeer.elems = append(elemsForPeer.elems, elem) + } + for peer, containers := range elemsByPeer { + if peer.isRunning.Load() { + for _, elemsForPeer := range containers { + peer.StagePackets(elemsForPeer) + } + peer.SendStagedPackets() + } else { + for _, elemsForPeer := range containers { + for _, elem := range elemsForPeer.elems { + device.PutOutboundBuffer(elem.buffer) + device.PutOutboundElement(elem) + } + device.PutOutboundElementsContainer(elemsForPeer) + } + } + } + return unmatched +} + func (peer *Peer) StagePackets(elems *QueueOutboundElementsContainer) { + peer.queuedOutboundPackets.Add(int32(len(elems.elems))) for { select { case peer.queue.staged <- elems: @@ -415,8 +623,9 @@ func (peer *Peer) StagePackets(elems *QueueOutboundElementsContainer) { } select { case tooOld := <-peer.queue.staged: + peer.queuedOutboundPackets.Add(-int32(len(tooOld.elems))) for _, elem := range tooOld.elems { - peer.device.PutMessageBuffer(elem.buffer) + peer.device.PutOutboundBuffer(elem.buffer) peer.device.PutOutboundElement(elem) } peer.device.PutOutboundElementsContainer(tooOld) @@ -459,10 +668,11 @@ top: elem.keypair = keypair } - elemsContainer.Lock() elemsContainer.elems = elemsContainer.elems[:i] if elemsContainerOOO != nil { + // Already counted at their original staging; StagePackets will count them again. + peer.queuedOutboundPackets.Add(-int32(len(elemsContainerOOO.elems))) peer.StagePackets(elemsContainerOOO) // XXX: Out of order, but we can't front-load go chans } @@ -473,11 +683,13 @@ top: // add to parallel and sequential queue if peer.isRunning.Load() { + elemsContainer.filling.Add(1) peer.queue.outbound.c <- elemsContainer peer.device.queue.encryption.c <- elemsContainer } else { + peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems))) for _, elem := range elemsContainer.elems { - peer.device.PutMessageBuffer(elem.buffer) + peer.device.PutOutboundBuffer(elem.buffer) peer.device.PutOutboundElement(elem) } peer.device.PutOutboundElementsContainer(elemsContainer) @@ -496,8 +708,9 @@ func (peer *Peer) FlushStagedPackets() { for { select { case elemsContainer := <-peer.queue.staged: + peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems))) for _, elem := range elemsContainer.elems { - peer.device.PutMessageBuffer(elem.buffer) + peer.device.PutOutboundBuffer(elem.buffer) peer.device.PutOutboundElement(elem) } peer.device.PutOutboundElementsContainer(elemsContainer) @@ -537,7 +750,7 @@ func (device *Device) RoutineEncryption(id int) { for elemsContainer := range device.queue.encryption.c { for _, elem := range elemsContainer.elems { // populate header fields - header := elem.buffer[:MessageTransportHeaderSize] + header := elem.buffer[MessageEncapsulatingTransportSize : MessageEncapsulatingTransportSize+MessageTransportHeaderSize] fieldType := header[0:4] fieldReceiver := header[4:8] @@ -563,7 +776,7 @@ func (device *Device) RoutineEncryption(id int) { nil, ) } - elemsContainer.Unlock() + elemsContainer.filling.Done() } } @@ -575,71 +788,100 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) { }() device.log.Verbosef("%v - Routine: sequential sender - started", peer) - bufs := make([][]byte, 0, maxBatchSize) + bufs := make([][]byte, 0, max(maxBatchSize, conn.IdealBatchSize)) for elemsContainer := range peer.queue.outbound.c { - bufs = bufs[:0] if elemsContainer == nil { return } - if !peer.isRunning.Load() { - // peer has been stopped; return re-usable elems to the shared pool. - // This is an optimization only. It is possible for the peer to be stopped - // immediately after this check, in which case, elem will get processed. - // The timers and SendBuffers code are resilient to a few stragglers. - // TODO: rework peer shutdown order to ensure - // that we never accidentally keep timers alive longer than necessary. - elemsContainer.Lock() - for _, elem := range elemsContainer.elems { - device.PutMessageBuffer(elem.buffer) - device.PutOutboundElement(elem) - } - device.PutOutboundElementsContainer(elemsContainer) - continue - } - dataSent := false - elemsContainer.Lock() - for _, elem := range elemsContainer.elems { - if len(elem.packet) != MessageKeepaliveSize { - dataSent = true - } - if padding := device.paddings.transport; padding > 0 { - // elem.packet is stored at the start of elem.buffer - // with zero padding - for i := len(elem.packet) - 1; i >= 0; i-- { - elem.buffer[i+padding] = elem.buffer[i] - } - rand.Read(elem.buffer[:padding]) - elem.packet = elem.buffer[:padding+len(elem.packet)] - } - bufs = append(bufs, elem.packet) - } - - peer.timersAnyAuthenticatedPacketTraversal() - peer.timersAnyAuthenticatedPacketSent() - - err := peer.SendBuffers(bufs) - if dataSent { - peer.timersDataSent() - } - - for _, elem := range elemsContainer.elems { - device.PutMessageBuffer(elem.buffer) - device.PutOutboundElement(elem) - } - device.PutOutboundElementsContainer(elemsContainer) - if err != nil { - var errGSO conn.ErrUDPGSODisabled - if errors.As(err, &errGSO) { - device.log.Verbosef(err.Error()) - err = errGSO.RetryErr - } - } - if err != nil { - device.log.Errorf("%v - Failed to send data packets: %v", peer, err) - continue - } - - peer.keepKeyFreshSending() + peer.processOutboundContainer(elemsContainer, bufs[:0]) } } + +// processOutboundContainer waits for the encryption routine to finish +// filling elemsContainer, then sends the batch (or drops it, if the peer +// has been stopped) and returns the container to the pool. +// +// scratch is a length-0 slice used to assemble the per-packet buffers +// passed to SendBuffers; its backing array is reused across calls. +func (peer *Peer) processOutboundContainer(elemsContainer *QueueOutboundElementsContainer, scratch [][]byte) { + // Invariants from RoutineSequentialSender; all should be unreachable. + if len(scratch) != 0 || cap(scratch) == 0 { + panic(fmt.Sprintf("processOutboundContainer: scratch must be empty with non-zero cap; got len=%d cap=%d", + len(scratch), cap(scratch))) + } + if cap(scratch) < len(elemsContainer.elems) { + panic(fmt.Sprintf("processOutboundContainer: scratch cap %d < elems %d", + cap(scratch), len(elemsContainer.elems))) + } + + device := peer.device + defer device.PutOutboundElementsContainer(elemsContainer) + + // Wait for RoutineEncryption to finish filling the container. After + // Wait returns we have happens-before with that goroutine and are the + // sole owner of the container until Put hands it back to the pool. + elemsContainer.filling.Wait() + + if !peer.isRunning.Load() { + // peer has been stopped; return re-usable elems to the shared pool. + // This is an optimization only. It is possible for the peer to be stopped + // immediately after this check, in which case, elem will get processed. + // The timers and SendBuffers code are resilient to a few stragglers. + // TODO: rework peer shutdown order to ensure + // that we never accidentally keep timers alive longer than necessary. + peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems))) + for _, elem := range elemsContainer.elems { + device.PutOutboundBuffer(elem.buffer) + device.PutOutboundElement(elem) + } + return + } + + dataSent := false + for _, elem := range elemsContainer.elems { + if len(elem.packet[MessageEncapsulatingTransportSize:]) != MessageKeepaliveSize { + dataSent = true + } + // lx:begin awg (SPEC 025 — AmneziaWG transport padding, S4) + // Prepend `transport` random bytes ahead of the transport header. The AWG + // path zeroes MessageEncapsulatingTransportSize (see noise-protocol.go), so + // elem.packet starts at buffer offset 0 and this shift is what creates the + // prefix; the buffer is allocated with PaddingMultiple headroom. + if padding := device.paddings.transport; padding > 0 { + for i := len(elem.packet) - 1; i >= 0; i-- { + elem.buffer[i+padding] = elem.buffer[i] + } + rand.Read(elem.buffer[:padding]) + elem.packet = elem.buffer[:padding+len(elem.packet)] + } + // lx:end awg + scratch = append(scratch, elem.packet) + } + + peer.timersAnyAuthenticatedPacketTraversal() + peer.timersAnyAuthenticatedPacketSent() + + err := peer.SendBuffers(scratch) + if dataSent { + peer.timersDataSent() + } + peer.queuedOutboundPackets.Add(-int32(len(elemsContainer.elems))) + for _, elem := range elemsContainer.elems { + device.PutOutboundBuffer(elem.buffer) + device.PutOutboundElement(elem) + } + if err != nil { + var errGSO conn.ErrUDPGSODisabled + if errors.As(err, &errGSO) { + device.log.Verbosef(err.Error()) + err = errGSO.RetryErr + } + } + if err != nil { + device.log.Errorf("%v - Failed to send data packets: %v", peer, err) + return + } + + peer.keepKeyFreshSending() +} diff --git a/device/sticky_default.go b/device/sticky_default.go index 0d02174..cac7add 100644 --- a/device/sticky_default.go +++ b/device/sticky_default.go @@ -7,6 +7,6 @@ import ( "github.com/sagernet/wireguard-go/rwcancel" ) -func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, error) { +func (device *Device) startRouteListener(_ conn.Bind) (*rwcancel.RWCancel, error) { return nil, nil } diff --git a/device/sticky_linux.go b/device/sticky_linux.go index b5b0e51..9fcfeeb 100644 --- a/device/sticky_linux.go +++ b/device/sticky_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. * * This implements userspace semantics of "sticky sockets", modeled after * WireGuard's kernelspace implementation. This is more or less a straight port @@ -9,7 +9,7 @@ * * Currently there is no way to achieve this within the net package: * See e.g. https://github.com/golang/go/issues/17930 - * So this code is remains platform dependent. + * So this code remains platform dependent. */ package device @@ -46,7 +46,7 @@ func (device *Device) startRouteListener(bind conn.Bind) (*rwcancel.RWCancel, er return netlinkCancel, nil } -func (device *Device) routineRouteListener(bind conn.Bind, netlinkSock int, netlinkCancel *rwcancel.RWCancel) { +func (device *Device) routineRouteListener(_ conn.Bind, netlinkSock int, netlinkCancel *rwcancel.RWCancel) { type peerEndpointPtr struct { peer *Peer endpoint *conn.Endpoint diff --git a/device/timers.go b/device/timers.go index 80fb7d9..9ec3d18 100644 --- a/device/timers.go +++ b/device/timers.go @@ -98,10 +98,26 @@ func expiredRetransmitHandshake(peer *Peer) { if peer.timersActive() && !peer.timers.zeroKeyMaterial.IsPending() { peer.timers.zeroKeyMaterial.Mod(RejectAfterTime * 3) } + peer.noteSessionHandshakeStopped() + + /* lx: SPEC 041 — the exhausted cycle just proved the current socket's + * 5-tuple dead (90s of initiations, zero replies). Rebind once and + * re-initiate, so a stale NAT mapping / poisoned DPI flow entry cannot + * pin this peer to a dead socket until a manual reconnect. Runs after + * the session-state notification so a consumer sees "handshake stopped" + * before the socket is recreated. */ + peer.device.handleHandshakeGiveUp(peer) } else { peer.timers.handshakeAttempts.Add(1) peer.device.log.Verbosef("%s - Handshake did not complete after %d seconds, retrying (try %d)", peer, int(RekeyTimeout.Seconds()), peer.timers.handshakeAttempts.Load()+1) + /* lx: SPEC 041 v2 — early self-heal: >=3 unanswered initiations against + * a provably dead session (no live keypair, or last handshake older + * than RejectAfterTime) prove the 5-tuple dead without waiting out the + * full cycle. Rebind now (debounced with the give-up trigger below); + * the retry cycle itself continues untouched. */ + peer.device.maybeEarlyGiveUpRebind(peer) + /* We clear the endpoint address src address, in case this is the cause of trouble. */ peer.markEndpointSrcForClearing() @@ -129,6 +145,24 @@ func expiredNewHandshake(peer *Peer) { func expiredZeroKeyMaterial(peer *Peer) { peer.device.log.Verbosef("%s - Removing all keys, since we haven't received a new one in %d seconds", peer, int((RejectAfterTime * 3).Seconds())) peer.ZeroAndFlushAll() + if peer.deleteOnIdle { + peer.device.log.Verbosef("%s - Removing idle lazy peer", peer) + // Remove the peer from the device in a new goroutine as we're currently + // holding timer locks which RemovePeer also needs. This is TOCTOU, but + // acceptable since the worst case is we remove the peer and the lazy + // peerfunc created it again after. We might lose some packets. + go peer.device.RemovePeer(peer.handshake.remoteStatic) + } +} + +func expiredSession(peer *Peer) { + peer.sessionState.Lock() + defer peer.sessionState.Unlock() + if peer.sessionState.sessionExpires.IsZero() || time.Now().Before(peer.sessionState.sessionExpires) { + return + } + peer.device.log.Verbosef("%s - Session expired after %d seconds", peer, int(RejectAfterTime.Seconds())) + peer.noteSessionStateLocked(PeerSessionExpired) } func expiredPersistentKeepalive(peer *Peer) { @@ -174,6 +208,7 @@ func (peer *Peer) timersHandshakeInitiated() { if peer.timersActive() { peer.timers.retransmitHandshake.Mod(RekeyTimeout + time.Millisecond*time.Duration(fastrandn(RekeyTimeoutJitterMaxMs))) } + peer.noteSessionHandshakeStarted() } /* Should be called after a handshake response message is received and processed or when getting key confirmation via the first data message. */ @@ -189,7 +224,14 @@ func (peer *Peer) timersHandshakeComplete() { /* Should be called after an ephemeral key is created, which is before sending a handshake response or after receiving a handshake response. */ func (peer *Peer) timersSessionDerived() { if peer.timersActive() { + peer.sessionState.Lock() + peer.sessionState.sessionExpires = time.Now().Add(RejectAfterTime) + peer.noteSessionStateLocked(PeerSessionEstablished) + peer.sessionState.Unlock() + peer.timers.sessionExpired.Mod(RejectAfterTime) peer.timers.zeroKeyMaterial.Mod(RejectAfterTime * 3) + } else { + peer.noteSessionState(PeerSessionEstablished) } } @@ -205,6 +247,7 @@ func (peer *Peer) timersInit() { peer.timers.retransmitHandshake = peer.NewTimer(expiredRetransmitHandshake) peer.timers.sendKeepalive = peer.NewTimer(expiredSendKeepalive) peer.timers.newHandshake = peer.NewTimer(expiredNewHandshake) + peer.timers.sessionExpired = peer.NewTimer(expiredSession) peer.timers.zeroKeyMaterial = peer.NewTimer(expiredZeroKeyMaterial) peer.timers.persistentKeepalive = peer.NewTimer(expiredPersistentKeepalive) } @@ -219,6 +262,7 @@ func (peer *Peer) timersStop() { peer.timers.retransmitHandshake.DelSync() peer.timers.sendKeepalive.DelSync() peer.timers.newHandshake.DelSync() + peer.timers.sessionExpired.DelSync() peer.timers.zeroKeyMaterial.DelSync() peer.timers.persistentKeepalive.DelSync() } diff --git a/device/transport_padding_test.go b/device/transport_padding_test.go new file mode 100644 index 0000000..c235846 --- /dev/null +++ b/device/transport_padding_test.go @@ -0,0 +1,361 @@ +/* SPDX-License-Identifier: MIT + * + * Regression test for the AWG transport-padding (uapi "s4") out-of-bounds + * crash: RoutineSequentialSender shifts elem.packet right by + * device.paddings.transport bytes inside elem.buffer to prepend a random + * padding prefix, but the injection paths (InputPacket/InputPackets) + * allocated elem.buffer tightly, without headroom for that shift: + * + * panic: runtime error: index out of range [123] with length 76 + * device.(*Peer).RoutineSequentialSender + * + * (payload 28 bytes -> allocLength 76, sealed packet 64, s4=60 -> 63+60=123). + * + * The tests spin up two real Devices wired together through an in-memory + * conn.Bind (Go channels) and an in-memory tun.Device, configure s4=60 on + * both sides, and pass a small IPv4 packet end to end via both outbound + * paths: Device.InputPacket (the crashing path) and the regular tun read + * loop (in-place shift path). + */ + +package device + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/hex" + "fmt" + "net" + "net/netip" + "os" + "sync" + "testing" + "time" + + "github.com/sagernet/wireguard-go/conn" + "github.com/sagernet/wireguard-go/tun" + + "golang.org/x/net/ipv4" +) + +const testTransportPadding = 60 // uapi s4, matches the on-device crash + +// --------------------------------------------------------------------------- +// In-memory conn.Bind over Go channels (minimal bindtest.ChannelBind clone). +// --------------------------------------------------------------------------- + +type chanEndpoint uint16 + +func (e chanEndpoint) ClearSrc() {} +func (e chanEndpoint) SrcToString() string { return "" } +func (e chanEndpoint) DstToString() string { return fmt.Sprintf("127.0.0.1:%d", uint16(e)) } +func (e chanEndpoint) DstToBytes() []byte { return []byte{byte(e), byte(e >> 8)} } +func (e chanEndpoint) DstIP() netip.Addr { return netip.AddrFrom4([4]byte{127, 0, 0, 1}) } +func (e chanEndpoint) SrcIP() netip.Addr { return netip.Addr{} } + +type chanBind struct { + rx, tx chan []byte + source chanEndpoint // "port" this bind listens on + target chanEndpoint // endpoint of the opposite bind + + mu sync.Mutex + closeSignal chan struct{} // recreated on every Open (BindUpdate closes+reopens) +} + +// newChanBindPair returns two Binds whose Send/Receive are cross-wired. +func newChanBindPair() (*chanBind, *chanBind) { + aToB := make(chan []byte, 1024) + bToA := make(chan []byte, 1024) + a := &chanBind{rx: bToA, tx: aToB, source: 1, target: 2} + b := &chanBind{rx: aToB, tx: bToA, source: 2, target: 1} + return a, b +} + +func (b *chanBind) currentCloseSignal() chan struct{} { + b.mu.Lock() + defer b.mu.Unlock() + return b.closeSignal +} + +func (b *chanBind) Open(port uint16) ([]conn.ReceiveFunc, uint16, error) { + b.mu.Lock() + b.closeSignal = make(chan struct{}) + closeSignal := b.closeSignal + b.mu.Unlock() + fn := func(packets [][]byte, sizes []int, eps []conn.Endpoint) (int, error) { + select { + case <-closeSignal: + // Must be net.ErrClosed: RoutineReceiveIncoming treats anything + // else as a transient error and death-spirals before exiting. + return 0, net.ErrClosed + case pkt, ok := <-b.rx: + if !ok { + return 0, net.ErrClosed + } + sizes[0] = copy(packets[0], pkt) + eps[0] = b.target + return 1, nil + } + } + return []conn.ReceiveFunc{fn}, uint16(b.source), nil +} + +func (b *chanBind) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closeSignal != nil { + select { + case <-b.closeSignal: + default: + close(b.closeSignal) + } + } + return nil +} + +func (b *chanBind) SetMark(mark uint32) error { return nil } + +func (b *chanBind) Send(bufs [][]byte, ep conn.Endpoint, offset int) error { + closeSignal := b.currentCloseSignal() + if closeSignal == nil { + return net.ErrClosed + } + for _, buf := range bufs { + pkt := make([]byte, len(buf)-offset) + copy(pkt, buf[offset:]) + select { + case <-closeSignal: + return net.ErrClosed + case b.tx <- pkt: + } + } + return nil +} + +func (b *chanBind) ParseEndpoint(s string) (conn.Endpoint, error) { return b.target, nil } + +func (b *chanBind) BatchSize() int { return 1 } + +func (b *chanBind) SetReservedForEndpoint(destination netip.AddrPort, reserved [3]byte) {} + +// --------------------------------------------------------------------------- +// In-memory tun.Device over Go channels (minimal tuntest.ChannelTUN clone). +// --------------------------------------------------------------------------- + +type chanTun struct { + toDevice chan []byte // packets the device Reads (outbound plaintext) + fromDevice chan []byte // packets the device Writes (inbound plaintext) + events chan tun.Event + closed chan struct{} + closeOnce sync.Once +} + +func newChanTun() *chanTun { + return &chanTun{ + toDevice: make(chan []byte, 1024), + fromDevice: make(chan []byte, 1024), + events: make(chan tun.Event, 4), + closed: make(chan struct{}), + } +} + +func (t *chanTun) File() *os.File { return nil } + +func (t *chanTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) { + select { + case <-t.closed: + return 0, os.ErrClosed + case pkt, ok := <-t.toDevice: + if !ok { + return 0, os.ErrClosed + } + sizes[0] = copy(bufs[0][offset:], pkt) + return 1, nil + } +} + +func (t *chanTun) Write(bufs [][]byte, offset int) (int, error) { + for _, buf := range bufs { + pkt := make([]byte, len(buf)-offset) + copy(pkt, buf[offset:]) + select { + case <-t.closed: + return 0, os.ErrClosed + case t.fromDevice <- pkt: + } + } + return len(bufs), nil +} + +func (t *chanTun) MTU() (int, error) { return DefaultMTU, nil } +func (t *chanTun) Name() (string, error) { return "chantun", nil } +func (t *chanTun) Events() <-chan tun.Event { return t.events } +func (t *chanTun) BatchSize() int { return 1 } + +func (t *chanTun) Close() error { + t.closeOnce.Do(func() { + close(t.closed) + close(t.events) + }) + return nil +} + +// --------------------------------------------------------------------------- +// Test scaffolding. +// --------------------------------------------------------------------------- + +var ( + testIPA = netip.AddrFrom4([4]byte{10, 0, 0, 1}) + testIPB = netip.AddrFrom4([4]byte{10, 0, 0, 2}) +) + +// buildIPv4Packet builds a minimal, routable IPv4/UDP packet whose header +// fields satisfy the receive-side validation in RoutineSequentialReceiver +// (version, total-length field, allowed source address). +func buildIPv4Packet(src, dst netip.Addr, payloadLen int) []byte { + total := ipv4.HeaderLen + payloadLen + pkt := make([]byte, total) + pkt[0] = 0x45 // version 4, IHL 5 + binary.BigEndian.PutUint16(pkt[IPv4offsetTotalLength:IPv4offsetTotalLength+2], uint16(total)) + pkt[8] = 64 // TTL + pkt[9] = 17 // protocol: UDP + copy(pkt[IPv4offsetSrc:], src.AsSlice()) + copy(pkt[IPv4offsetDst:], dst.AsSlice()) + for i := ipv4.HeaderLen; i < total; i++ { + pkt[i] = byte(i) // deterministic payload + } + return pkt +} + +type paddedPair struct { + devA, devB *Device + tunA, tunB *chanTun +} + +// newPaddedDevicePair builds two Up()'d devices peered with each other over +// the channel bind, both configured with s4 (transport padding) enabled. +func newPaddedDevicePair(t *testing.T) *paddedPair { + t.Helper() + + skA, err := newPrivateKey() + if err != nil { + t.Fatalf("newPrivateKey A: %v", err) + } + skB, err := newPrivateKey() + if err != nil { + t.Fatalf("newPrivateKey B: %v", err) + } + pkA := skA.publicKey() + pkB := skB.publicKey() + + bindA, bindB := newChanBindPair() + tunA := newChanTun() + tunB := newChanTun() + + devA := NewDevice(context.Background(), tunA, bindA, NewLogger(LogLevelError, "devA: "), 1) + devB := NewDevice(context.Background(), tunB, bindB, NewLogger(LogLevelError, "devB: "), 1) + t.Cleanup(devA.Close) + t.Cleanup(devB.Close) + + cfgA := fmt.Sprintf( + "private_key=%s\ns4=%d\nreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:2\nallowed_ip=%s/32\n", + hex.EncodeToString(skA[:]), testTransportPadding, hex.EncodeToString(pkB[:]), testIPB) + cfgB := fmt.Sprintf( + "private_key=%s\ns4=%d\nreplace_peers=true\npublic_key=%s\nendpoint=127.0.0.1:1\nallowed_ip=%s/32\n", + hex.EncodeToString(skB[:]), testTransportPadding, hex.EncodeToString(pkA[:]), testIPA) + + if err := devA.IpcSet(cfgA); err != nil { + t.Fatalf("IpcSet A: %v", err) + } + if err := devB.IpcSet(cfgB); err != nil { + t.Fatalf("IpcSet B: %v", err) + } + if devA.paddings.transport != testTransportPadding { + t.Fatalf("s4 not applied: paddings.transport = %d", devA.paddings.transport) + } + + if err := devA.Up(); err != nil { + t.Fatalf("Up A: %v", err) + } + if err := devB.Up(); err != nil { + t.Fatalf("Up B: %v", err) + } + + return &paddedPair{devA: devA, devB: devB, tunA: tunA, tunB: tunB} +} + +// awaitPacket waits for want to arrive on the receiving tun, periodically +// re-sending via resend (injection has no delivery guarantee before the +// handshake completes). +func awaitPacket(t *testing.T, from *chanTun, want []byte, resend func()) { + t.Helper() + deadline := time.After(20 * time.Second) + retry := time.NewTicker(1 * time.Second) + defer retry.Stop() + for { + select { + case got := <-from.fromDevice: + if bytes.Equal(got, want) { + return + } + t.Logf("ignoring unexpected packet, len=%d", len(got)) + case <-retry.C: + resend() + case <-deadline: + t.Fatal("timed out waiting for packet on peer tun") + } + } +} + +// --------------------------------------------------------------------------- +// Tests. +// --------------------------------------------------------------------------- + +// TestTransportPaddingInputPacket exercises the exact crash path: an injected +// packet (Device.InputPacket) whose buffer was allocated by payload size. +// With s4=60 and a 28-byte IPv4 packet the pre-fix buffer was 76 bytes and +// the padding shift indexed [123] -> index out of range. +func TestTransportPaddingInputPacket(t *testing.T) { + pair := newPaddedDevicePair(t) + + // 20-byte header + 8-byte payload = 28 bytes, the on-device crash size. + pkt := buildIPv4Packet(testIPA, testIPB, 8) + dst := testIPB.AsSlice() + + send := func() { pair.devA.InputPacket(dst, [][]byte{pkt}) } + send() + awaitPacket(t, pair.tunB, pkt, send) +} + +// TestTransportPaddingInputPackets covers the batched injection path +// (Device.InputPackets), which had the same tight allocation. +func TestTransportPaddingInputPackets(t *testing.T) { + pair := newPaddedDevicePair(t) + + pkt := buildIPv4Packet(testIPA, testIPB, 8) + refs := []*InputPacketRef{{ + Destination: testIPB.AsSlice(), + PacketSlices: [][]byte{pkt[:12], pkt[12:]}, // multi-slice on purpose + }} + + send := func() { + if unmatched := pair.devA.InputPackets(refs); len(unmatched) != 0 { + t.Fatalf("InputPackets returned %d unmatched refs", len(unmatched)) + } + } + send() + awaitPacket(t, pair.tunB, pkt, send) +} + +// TestTransportPaddingTunPath covers the regular outbound path (tun read +// loop), whose MaxMessageSize buffers take the in-place shift branch. +func TestTransportPaddingTunPath(t *testing.T) { + pair := newPaddedDevicePair(t) + + pkt := buildIPv4Packet(testIPA, testIPB, 8) + + send := func() { pair.tunA.toDevice <- pkt } + send() + awaitPacket(t, pair.tunB, pkt, send) +} diff --git a/device/tun.go b/device/tun.go index 891575d..01a92ed 100644 --- a/device/tun.go +++ b/device/tun.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package device diff --git a/go.mod b/go.mod index 8679506..d445678 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ module github.com/sagernet/wireguard-go -go 1.24 +go 1.25 require ( github.com/sagernet/sing v0.7.10 golang.org/x/crypto v0.13.0 golang.org/x/net v0.15.0 - golang.org/x/sys v0.12.0 + golang.org/x/sys v0.21.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 ) diff --git a/go.sum b/go.sum index c405e0a..9ce9725 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,7 @@ golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= diff --git a/ipc/uapi_fake.go b/ipc/uapi_fake.go deleted file mode 100644 index a2e0f85..0000000 --- a/ipc/uapi_fake.go +++ /dev/null @@ -1,17 +0,0 @@ -//go:build wasm || plan9 || aix || solaris || illumos - -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package ipc - -// Made up sentinel error codes for {js,wasip1}/wasm, and plan9. -const ( - IpcErrorIO = 1 - IpcErrorInvalid = 2 - IpcErrorPortInUse = 3 - IpcErrorUnknown = 4 - IpcErrorProtocol = 5 -) diff --git a/ipc/uapi_linux.go b/ipc/uapi_linux.go index c14d5d0..be59e58 100644 --- a/ipc/uapi_linux.go +++ b/ipc/uapi_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package ipc diff --git a/ipc/uapi_unix.go b/ipc/uapi_unix.go index 79604ee..dcce167 100644 --- a/ipc/uapi_unix.go +++ b/ipc/uapi_unix.go @@ -26,7 +26,7 @@ const ( // socketDirectory is variable because it is modified by a linker // flag in wireguard-android. -var socketDirectory = "/var/run/amneziawg" +var socketDirectory = "/var/run/wireguard" func sockPath(iface string) string { return fmt.Sprintf("%s/%s.sock", socketDirectory, iface) diff --git a/ipc/uapi_tamago.go b/ipc/uapi_wasm.go similarity index 76% rename from ipc/uapi_tamago.go rename to ipc/uapi_wasm.go index 3584a4d..50ac091 100644 --- a/ipc/uapi_tamago.go +++ b/ipc/uapi_wasm.go @@ -1,5 +1,3 @@ -//go:build tamago - /* SPDX-License-Identifier: MIT * * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. @@ -7,7 +5,7 @@ package ipc -// Made up sentinel error codes for tamago platform. +// Made up sentinel error codes for {js,wasip1}/wasm. const ( IpcErrorIO = 1 IpcErrorInvalid = 2 diff --git a/ipc/uapi_windows.go b/ipc/uapi_windows.go index 5d236b3..a146f1a 100644 --- a/ipc/uapi_windows.go +++ b/ipc/uapi_windows.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package ipc diff --git a/module_rename.py b/module_rename.py deleted file mode 100644 index bf03152..0000000 --- a/module_rename.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import os -import argparse -import fileinput - - -PKG_ORIGINAL = "github.com/tailscale/wireguard-go" -PKG_NEW = "github.com/sagernet/wireguard-go" - -EXTENSIONS = [".go", ".md", ".mod", ".sh"] - -parser = argparse.ArgumentParser() -parser.add_argument("-r", "--reverse", action="store_true") -args = parser.parse_args() - - -def replace_line(line): - if args.reverse: - return line.replace(PKG_NEW, PKG_ORIGINAL) - return line.replace(PKG_ORIGINAL, PKG_NEW) - - -for dirpath, dirnames, filenames in os.walk("."): - # Skip hidden directories like .git - dirnames[:] = [d for d in dirnames if not d[0] == "."] - filenames = [f for f in filenames if os.path.splitext(f)[1] in EXTENSIONS] - for filename in filenames: - file_path = os.path.join(dirpath, filename) - with fileinput.FileInput(file_path, inplace=True) as file: - for line in file: - print(replace_line(line), end="") diff --git a/reformat.sh b/reformat.sh new file mode 100755 index 0000000..8d3ac05 --- /dev/null +++ b/reformat.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +GO_FILES=$(find . -name "*.go" | grep -v .git) + +gofumpt -l -w $GO_FILES +gofmt -l -w $GO_FILES +gci write $GO_FILES diff --git a/rename-module.sh b/rename-module.sh new file mode 100755 index 0000000..a6bd016 --- /dev/null +++ b/rename-module.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -e -o pipefail + +OLD_MODULE_NAME="github.com/tailscale/wireguard-go" +NEW_MODULE_NAME="github.com/sagernet/wireguard-go" + +rules=$(cat <.*) + by: $NEW_MODULE_NAME\$PATH +fix: \$NEW_IMPORT +EOF +) + +sg scan --inline-rules "$rules" -U + +sed -i "s|module $OLD_MODULE_NAME|module $NEW_MODULE_NAME|" go.mod + +go mod tidy + +./reformat.sh + +git commit -m "Rename module" -a diff --git a/rwcancel/rwcancel.go b/rwcancel/rwcancel.go index a4668f0..4372453 100644 --- a/rwcancel/rwcancel.go +++ b/rwcancel/rwcancel.go @@ -1,4 +1,4 @@ -//go:build !windows && !wasm && !plan9 && !tamago +//go:build !windows && !wasm /* SPDX-License-Identifier: MIT * diff --git a/rwcancel/rwcancel_stub.go b/rwcancel/rwcancel_stub.go index 60ae9af..2a98b2b 100644 --- a/rwcancel/rwcancel_stub.go +++ b/rwcancel/rwcancel_stub.go @@ -1,4 +1,4 @@ -//go:build windows || wasm || plan9 || tamago +//go:build windows || wasm // SPDX-License-Identifier: MIT diff --git a/tun/checksum.go b/tun/checksum.go index 6634050..ac16569 100644 --- a/tun/checksum.go +++ b/tun/checksum.go @@ -3,710 +3,111 @@ package tun import ( "encoding/binary" "math/bits" - "strconv" - - "golang.org/x/sys/cpu" ) -// checksumGeneric64 is a reference implementation of checksum using 64 bit -// arithmetic for use in testing or when an architecture-specific implementation -// is not available. -func checksumGeneric64(b []byte, initial uint16) uint16 { - var ac uint64 +// TODO: Explore SIMD and/or other assembly optimizations. +func checksumNoFold(b []byte, initial uint64) uint64 { + tmp := make([]byte, 8) + binary.NativeEndian.PutUint64(tmp, initial) + ac := binary.BigEndian.Uint64(tmp) var carry uint64 - if cpu.IsBigEndian { - ac = uint64(initial) - } else { - ac = uint64(bits.ReverseBytes16(initial)) - } - for len(b) >= 128 { - if cpu.IsBigEndian { - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[32:40]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[40:48]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[48:56]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[56:64]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[64:72]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[72:80]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[80:88]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[88:96]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[96:104]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[104:112]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[112:120]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[120:128]), carry) - } else { - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[32:40]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[40:48]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[48:56]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[56:64]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[64:72]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[72:80]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[80:88]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[88:96]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[96:104]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[104:112]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[112:120]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[120:128]), carry) - } + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[24:32]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[32:40]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[40:48]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[48:56]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[56:64]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[64:72]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[72:80]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[80:88]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[88:96]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[96:104]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[104:112]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[112:120]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[120:128]), carry) + ac += carry b = b[128:] } if len(b) >= 64 { - if cpu.IsBigEndian { - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[32:40]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[40:48]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[48:56]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[56:64]), carry) - } else { - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[32:40]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[40:48]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[48:56]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[56:64]), carry) - } + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[24:32]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[32:40]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[40:48]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[48:56]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[56:64]), carry) + ac += carry b = b[64:] } if len(b) >= 32 { - if cpu.IsBigEndian { - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[16:24]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[24:32]), carry) - } else { - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[16:24]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[24:32]), carry) - } + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[16:24]), carry) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[24:32]), carry) + ac += carry b = b[32:] } if len(b) >= 16 { - if cpu.IsBigEndian { - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[:8]), carry) - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b[8:16]), carry) - } else { - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[:8]), carry) - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b[8:16]), carry) - } + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0) + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[8:16]), carry) + ac += carry b = b[16:] } if len(b) >= 8 { - if cpu.IsBigEndian { - ac, carry = bits.Add64(ac, binary.BigEndian.Uint64(b), carry) - } else { - ac, carry = bits.Add64(ac, binary.LittleEndian.Uint64(b), carry) - } + ac, carry = bits.Add64(ac, binary.NativeEndian.Uint64(b[:8]), 0) + ac += carry b = b[8:] } if len(b) >= 4 { - if cpu.IsBigEndian { - ac, carry = bits.Add64(ac, uint64(binary.BigEndian.Uint32(b)), carry) - } else { - ac, carry = bits.Add64(ac, uint64(binary.LittleEndian.Uint32(b)), carry) - } + ac, carry = bits.Add64(ac, uint64(binary.NativeEndian.Uint32(b[:4])), 0) + ac += carry b = b[4:] } if len(b) >= 2 { - if cpu.IsBigEndian { - ac, carry = bits.Add64(ac, uint64(binary.BigEndian.Uint16(b)), carry) - } else { - ac, carry = bits.Add64(ac, uint64(binary.LittleEndian.Uint16(b)), carry) - } + ac, carry = bits.Add64(ac, uint64(binary.NativeEndian.Uint16(b[:2])), 0) + ac += carry b = b[2:] } - if len(b) >= 1 { - if cpu.IsBigEndian { - ac, carry = bits.Add64(ac, uint64(b[0])<<8, carry) - } else { - ac, carry = bits.Add64(ac, uint64(b[0]), carry) - } + if len(b) == 1 { + tmp := binary.NativeEndian.Uint16([]byte{b[0], 0}) + ac, carry = bits.Add64(ac, uint64(tmp), 0) + ac += carry } - folded := ipChecksumFold64(ac, carry) - if !cpu.IsBigEndian { - folded = bits.ReverseBytes16(folded) - } - return folded + binary.NativeEndian.PutUint64(tmp, ac) + return binary.BigEndian.Uint64(tmp) } -// checksumGeneric32 is a reference implementation of checksum using 32 bit -// arithmetic for use in testing or when an architecture-specific implementation -// is not available. -func checksumGeneric32(b []byte, initial uint16) uint16 { - var ac uint32 - var carry uint32 - - if cpu.IsBigEndian { - ac = uint32(initial) - } else { - ac = uint32(bits.ReverseBytes16(initial)) - } - - for len(b) >= 64 { - if cpu.IsBigEndian { - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:8]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[16:20]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[20:24]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[24:28]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[28:32]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[32:36]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[36:40]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[40:44]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[44:48]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[48:52]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[52:56]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[56:60]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[60:64]), carry) - } else { - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:8]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[16:20]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[20:24]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[24:28]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[28:32]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[32:36]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[36:40]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[40:44]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[44:48]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[48:52]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[52:56]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[56:60]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[60:64]), carry) - } - b = b[64:] - } - if len(b) >= 32 { - if cpu.IsBigEndian { - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[16:20]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[20:24]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[24:28]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[28:32]), carry) - } else { - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[16:20]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[20:24]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[24:28]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[28:32]), carry) - } - b = b[32:] - } - if len(b) >= 16 { - if cpu.IsBigEndian { - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[8:12]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[12:16]), carry) - } else { - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[8:12]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[12:16]), carry) - } - b = b[16:] - } - if len(b) >= 8 { - if cpu.IsBigEndian { - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[:4]), carry) - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b[4:8]), carry) - } else { - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[:4]), carry) - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b[4:8]), carry) - } - b = b[8:] - } - if len(b) >= 4 { - if cpu.IsBigEndian { - ac, carry = bits.Add32(ac, binary.BigEndian.Uint32(b), carry) - } else { - ac, carry = bits.Add32(ac, binary.LittleEndian.Uint32(b), carry) - } - b = b[4:] - } - if len(b) >= 2 { - if cpu.IsBigEndian { - ac, carry = bits.Add32(ac, uint32(binary.BigEndian.Uint16(b)), carry) - } else { - ac, carry = bits.Add32(ac, uint32(binary.LittleEndian.Uint16(b)), carry) - } - b = b[2:] - } - if len(b) >= 1 { - if cpu.IsBigEndian { - ac, carry = bits.Add32(ac, uint32(b[0])<<8, carry) - } else { - ac, carry = bits.Add32(ac, uint32(b[0]), carry) - } - } - - folded := ipChecksumFold32(ac, carry) - if !cpu.IsBigEndian { - folded = bits.ReverseBytes16(folded) - } - return folded +func checksum(b []byte, initial uint64) uint16 { + ac := checksumNoFold(b, initial) + ac = (ac >> 16) + (ac & 0xffff) + ac = (ac >> 16) + (ac & 0xffff) + ac = (ac >> 16) + (ac & 0xffff) + ac = (ac >> 16) + (ac & 0xffff) + return uint16(ac) } -// checksumGeneric32Alternate is an alternate reference implementation of -// checksum using 32 bit arithmetic for use in testing or when an -// architecture-specific implementation is not available. -func checksumGeneric32Alternate(b []byte, initial uint16) uint16 { - var ac uint32 - - if cpu.IsBigEndian { - ac = uint32(initial) - } else { - ac = uint32(bits.ReverseBytes16(initial)) - } - - for len(b) >= 64 { - if cpu.IsBigEndian { - ac += uint32(binary.BigEndian.Uint16(b[:2])) - ac += uint32(binary.BigEndian.Uint16(b[2:4])) - ac += uint32(binary.BigEndian.Uint16(b[4:6])) - ac += uint32(binary.BigEndian.Uint16(b[6:8])) - ac += uint32(binary.BigEndian.Uint16(b[8:10])) - ac += uint32(binary.BigEndian.Uint16(b[10:12])) - ac += uint32(binary.BigEndian.Uint16(b[12:14])) - ac += uint32(binary.BigEndian.Uint16(b[14:16])) - ac += uint32(binary.BigEndian.Uint16(b[16:18])) - ac += uint32(binary.BigEndian.Uint16(b[18:20])) - ac += uint32(binary.BigEndian.Uint16(b[20:22])) - ac += uint32(binary.BigEndian.Uint16(b[22:24])) - ac += uint32(binary.BigEndian.Uint16(b[24:26])) - ac += uint32(binary.BigEndian.Uint16(b[26:28])) - ac += uint32(binary.BigEndian.Uint16(b[28:30])) - ac += uint32(binary.BigEndian.Uint16(b[30:32])) - ac += uint32(binary.BigEndian.Uint16(b[32:34])) - ac += uint32(binary.BigEndian.Uint16(b[34:36])) - ac += uint32(binary.BigEndian.Uint16(b[36:38])) - ac += uint32(binary.BigEndian.Uint16(b[38:40])) - ac += uint32(binary.BigEndian.Uint16(b[40:42])) - ac += uint32(binary.BigEndian.Uint16(b[42:44])) - ac += uint32(binary.BigEndian.Uint16(b[44:46])) - ac += uint32(binary.BigEndian.Uint16(b[46:48])) - ac += uint32(binary.BigEndian.Uint16(b[48:50])) - ac += uint32(binary.BigEndian.Uint16(b[50:52])) - ac += uint32(binary.BigEndian.Uint16(b[52:54])) - ac += uint32(binary.BigEndian.Uint16(b[54:56])) - ac += uint32(binary.BigEndian.Uint16(b[56:58])) - ac += uint32(binary.BigEndian.Uint16(b[58:60])) - ac += uint32(binary.BigEndian.Uint16(b[60:62])) - ac += uint32(binary.BigEndian.Uint16(b[62:64])) - } else { - ac += uint32(binary.LittleEndian.Uint16(b[:2])) - ac += uint32(binary.LittleEndian.Uint16(b[2:4])) - ac += uint32(binary.LittleEndian.Uint16(b[4:6])) - ac += uint32(binary.LittleEndian.Uint16(b[6:8])) - ac += uint32(binary.LittleEndian.Uint16(b[8:10])) - ac += uint32(binary.LittleEndian.Uint16(b[10:12])) - ac += uint32(binary.LittleEndian.Uint16(b[12:14])) - ac += uint32(binary.LittleEndian.Uint16(b[14:16])) - ac += uint32(binary.LittleEndian.Uint16(b[16:18])) - ac += uint32(binary.LittleEndian.Uint16(b[18:20])) - ac += uint32(binary.LittleEndian.Uint16(b[20:22])) - ac += uint32(binary.LittleEndian.Uint16(b[22:24])) - ac += uint32(binary.LittleEndian.Uint16(b[24:26])) - ac += uint32(binary.LittleEndian.Uint16(b[26:28])) - ac += uint32(binary.LittleEndian.Uint16(b[28:30])) - ac += uint32(binary.LittleEndian.Uint16(b[30:32])) - ac += uint32(binary.LittleEndian.Uint16(b[32:34])) - ac += uint32(binary.LittleEndian.Uint16(b[34:36])) - ac += uint32(binary.LittleEndian.Uint16(b[36:38])) - ac += uint32(binary.LittleEndian.Uint16(b[38:40])) - ac += uint32(binary.LittleEndian.Uint16(b[40:42])) - ac += uint32(binary.LittleEndian.Uint16(b[42:44])) - ac += uint32(binary.LittleEndian.Uint16(b[44:46])) - ac += uint32(binary.LittleEndian.Uint16(b[46:48])) - ac += uint32(binary.LittleEndian.Uint16(b[48:50])) - ac += uint32(binary.LittleEndian.Uint16(b[50:52])) - ac += uint32(binary.LittleEndian.Uint16(b[52:54])) - ac += uint32(binary.LittleEndian.Uint16(b[54:56])) - ac += uint32(binary.LittleEndian.Uint16(b[56:58])) - ac += uint32(binary.LittleEndian.Uint16(b[58:60])) - ac += uint32(binary.LittleEndian.Uint16(b[60:62])) - ac += uint32(binary.LittleEndian.Uint16(b[62:64])) - } - b = b[64:] - } - if len(b) >= 32 { - if cpu.IsBigEndian { - ac += uint32(binary.BigEndian.Uint16(b[:2])) - ac += uint32(binary.BigEndian.Uint16(b[2:4])) - ac += uint32(binary.BigEndian.Uint16(b[4:6])) - ac += uint32(binary.BigEndian.Uint16(b[6:8])) - ac += uint32(binary.BigEndian.Uint16(b[8:10])) - ac += uint32(binary.BigEndian.Uint16(b[10:12])) - ac += uint32(binary.BigEndian.Uint16(b[12:14])) - ac += uint32(binary.BigEndian.Uint16(b[14:16])) - ac += uint32(binary.BigEndian.Uint16(b[16:18])) - ac += uint32(binary.BigEndian.Uint16(b[18:20])) - ac += uint32(binary.BigEndian.Uint16(b[20:22])) - ac += uint32(binary.BigEndian.Uint16(b[22:24])) - ac += uint32(binary.BigEndian.Uint16(b[24:26])) - ac += uint32(binary.BigEndian.Uint16(b[26:28])) - ac += uint32(binary.BigEndian.Uint16(b[28:30])) - ac += uint32(binary.BigEndian.Uint16(b[30:32])) - } else { - ac += uint32(binary.LittleEndian.Uint16(b[:2])) - ac += uint32(binary.LittleEndian.Uint16(b[2:4])) - ac += uint32(binary.LittleEndian.Uint16(b[4:6])) - ac += uint32(binary.LittleEndian.Uint16(b[6:8])) - ac += uint32(binary.LittleEndian.Uint16(b[8:10])) - ac += uint32(binary.LittleEndian.Uint16(b[10:12])) - ac += uint32(binary.LittleEndian.Uint16(b[12:14])) - ac += uint32(binary.LittleEndian.Uint16(b[14:16])) - ac += uint32(binary.LittleEndian.Uint16(b[16:18])) - ac += uint32(binary.LittleEndian.Uint16(b[18:20])) - ac += uint32(binary.LittleEndian.Uint16(b[20:22])) - ac += uint32(binary.LittleEndian.Uint16(b[22:24])) - ac += uint32(binary.LittleEndian.Uint16(b[24:26])) - ac += uint32(binary.LittleEndian.Uint16(b[26:28])) - ac += uint32(binary.LittleEndian.Uint16(b[28:30])) - ac += uint32(binary.LittleEndian.Uint16(b[30:32])) - } - b = b[32:] - } - if len(b) >= 16 { - if cpu.IsBigEndian { - ac += uint32(binary.BigEndian.Uint16(b[:2])) - ac += uint32(binary.BigEndian.Uint16(b[2:4])) - ac += uint32(binary.BigEndian.Uint16(b[4:6])) - ac += uint32(binary.BigEndian.Uint16(b[6:8])) - ac += uint32(binary.BigEndian.Uint16(b[8:10])) - ac += uint32(binary.BigEndian.Uint16(b[10:12])) - ac += uint32(binary.BigEndian.Uint16(b[12:14])) - ac += uint32(binary.BigEndian.Uint16(b[14:16])) - } else { - ac += uint32(binary.LittleEndian.Uint16(b[:2])) - ac += uint32(binary.LittleEndian.Uint16(b[2:4])) - ac += uint32(binary.LittleEndian.Uint16(b[4:6])) - ac += uint32(binary.LittleEndian.Uint16(b[6:8])) - ac += uint32(binary.LittleEndian.Uint16(b[8:10])) - ac += uint32(binary.LittleEndian.Uint16(b[10:12])) - ac += uint32(binary.LittleEndian.Uint16(b[12:14])) - ac += uint32(binary.LittleEndian.Uint16(b[14:16])) - } - b = b[16:] - } - if len(b) >= 8 { - if cpu.IsBigEndian { - ac += uint32(binary.BigEndian.Uint16(b[:2])) - ac += uint32(binary.BigEndian.Uint16(b[2:4])) - ac += uint32(binary.BigEndian.Uint16(b[4:6])) - ac += uint32(binary.BigEndian.Uint16(b[6:8])) - } else { - ac += uint32(binary.LittleEndian.Uint16(b[:2])) - ac += uint32(binary.LittleEndian.Uint16(b[2:4])) - ac += uint32(binary.LittleEndian.Uint16(b[4:6])) - ac += uint32(binary.LittleEndian.Uint16(b[6:8])) - } - b = b[8:] - } - if len(b) >= 4 { - if cpu.IsBigEndian { - ac += uint32(binary.BigEndian.Uint16(b[:2])) - ac += uint32(binary.BigEndian.Uint16(b[2:4])) - } else { - ac += uint32(binary.LittleEndian.Uint16(b[:2])) - ac += uint32(binary.LittleEndian.Uint16(b[2:4])) - } - b = b[4:] - } - if len(b) >= 2 { - if cpu.IsBigEndian { - ac += uint32(binary.BigEndian.Uint16(b)) - } else { - ac += uint32(binary.LittleEndian.Uint16(b)) - } - b = b[2:] - } - if len(b) >= 1 { - if cpu.IsBigEndian { - ac += uint32(b[0]) << 8 - } else { - ac += uint32(b[0]) - } - } - - folded := ipChecksumFold32(ac, 0) - if !cpu.IsBigEndian { - folded = bits.ReverseBytes16(folded) - } - return folded +// Checksum computes an IP checksum starting with the provided initial value. +func Checksum(data []byte, initial uint16) uint16 { + return checksum(data, uint64(initial)) } -// checksumGeneric64Alternate is an alternate reference implementation of -// checksum using 64 bit arithmetic for use in testing or when an -// architecture-specific implementation is not available. -func checksumGeneric64Alternate(b []byte, initial uint16) uint16 { - var ac uint64 - - if cpu.IsBigEndian { - ac = uint64(initial) - } else { - ac = uint64(bits.ReverseBytes16(initial)) - } - - for len(b) >= 64 { - if cpu.IsBigEndian { - ac += uint64(binary.BigEndian.Uint32(b[:4])) - ac += uint64(binary.BigEndian.Uint32(b[4:8])) - ac += uint64(binary.BigEndian.Uint32(b[8:12])) - ac += uint64(binary.BigEndian.Uint32(b[12:16])) - ac += uint64(binary.BigEndian.Uint32(b[16:20])) - ac += uint64(binary.BigEndian.Uint32(b[20:24])) - ac += uint64(binary.BigEndian.Uint32(b[24:28])) - ac += uint64(binary.BigEndian.Uint32(b[28:32])) - ac += uint64(binary.BigEndian.Uint32(b[32:36])) - ac += uint64(binary.BigEndian.Uint32(b[36:40])) - ac += uint64(binary.BigEndian.Uint32(b[40:44])) - ac += uint64(binary.BigEndian.Uint32(b[44:48])) - ac += uint64(binary.BigEndian.Uint32(b[48:52])) - ac += uint64(binary.BigEndian.Uint32(b[52:56])) - ac += uint64(binary.BigEndian.Uint32(b[56:60])) - ac += uint64(binary.BigEndian.Uint32(b[60:64])) - } else { - ac += uint64(binary.LittleEndian.Uint32(b[:4])) - ac += uint64(binary.LittleEndian.Uint32(b[4:8])) - ac += uint64(binary.LittleEndian.Uint32(b[8:12])) - ac += uint64(binary.LittleEndian.Uint32(b[12:16])) - ac += uint64(binary.LittleEndian.Uint32(b[16:20])) - ac += uint64(binary.LittleEndian.Uint32(b[20:24])) - ac += uint64(binary.LittleEndian.Uint32(b[24:28])) - ac += uint64(binary.LittleEndian.Uint32(b[28:32])) - ac += uint64(binary.LittleEndian.Uint32(b[32:36])) - ac += uint64(binary.LittleEndian.Uint32(b[36:40])) - ac += uint64(binary.LittleEndian.Uint32(b[40:44])) - ac += uint64(binary.LittleEndian.Uint32(b[44:48])) - ac += uint64(binary.LittleEndian.Uint32(b[48:52])) - ac += uint64(binary.LittleEndian.Uint32(b[52:56])) - ac += uint64(binary.LittleEndian.Uint32(b[56:60])) - ac += uint64(binary.LittleEndian.Uint32(b[60:64])) - } - b = b[64:] - } - if len(b) >= 32 { - if cpu.IsBigEndian { - ac += uint64(binary.BigEndian.Uint32(b[:4])) - ac += uint64(binary.BigEndian.Uint32(b[4:8])) - ac += uint64(binary.BigEndian.Uint32(b[8:12])) - ac += uint64(binary.BigEndian.Uint32(b[12:16])) - ac += uint64(binary.BigEndian.Uint32(b[16:20])) - ac += uint64(binary.BigEndian.Uint32(b[20:24])) - ac += uint64(binary.BigEndian.Uint32(b[24:28])) - ac += uint64(binary.BigEndian.Uint32(b[28:32])) - } else { - ac += uint64(binary.LittleEndian.Uint32(b[:4])) - ac += uint64(binary.LittleEndian.Uint32(b[4:8])) - ac += uint64(binary.LittleEndian.Uint32(b[8:12])) - ac += uint64(binary.LittleEndian.Uint32(b[12:16])) - ac += uint64(binary.LittleEndian.Uint32(b[16:20])) - ac += uint64(binary.LittleEndian.Uint32(b[20:24])) - ac += uint64(binary.LittleEndian.Uint32(b[24:28])) - ac += uint64(binary.LittleEndian.Uint32(b[28:32])) - } - b = b[32:] - } - if len(b) >= 16 { - if cpu.IsBigEndian { - ac += uint64(binary.BigEndian.Uint32(b[:4])) - ac += uint64(binary.BigEndian.Uint32(b[4:8])) - ac += uint64(binary.BigEndian.Uint32(b[8:12])) - ac += uint64(binary.BigEndian.Uint32(b[12:16])) - } else { - ac += uint64(binary.LittleEndian.Uint32(b[:4])) - ac += uint64(binary.LittleEndian.Uint32(b[4:8])) - ac += uint64(binary.LittleEndian.Uint32(b[8:12])) - ac += uint64(binary.LittleEndian.Uint32(b[12:16])) - } - b = b[16:] - } - if len(b) >= 8 { - if cpu.IsBigEndian { - ac += uint64(binary.BigEndian.Uint32(b[:4])) - ac += uint64(binary.BigEndian.Uint32(b[4:8])) - } else { - ac += uint64(binary.LittleEndian.Uint32(b[:4])) - ac += uint64(binary.LittleEndian.Uint32(b[4:8])) - } - b = b[8:] - } - if len(b) >= 4 { - if cpu.IsBigEndian { - ac += uint64(binary.BigEndian.Uint32(b)) - } else { - ac += uint64(binary.LittleEndian.Uint32(b)) - } - b = b[4:] - } - if len(b) >= 2 { - if cpu.IsBigEndian { - ac += uint64(binary.BigEndian.Uint16(b)) - } else { - ac += uint64(binary.LittleEndian.Uint16(b)) - } - b = b[2:] - } - if len(b) >= 1 { - if cpu.IsBigEndian { - ac += uint64(b[0]) << 8 - } else { - ac += uint64(b[0]) - } - } - - folded := ipChecksumFold64(ac, 0) - if !cpu.IsBigEndian { - folded = bits.ReverseBytes16(folded) - } - return folded -} - -func ipChecksumFold64(unfolded uint64, initialCarry uint64) uint16 { - sum, carry := bits.Add32(uint32(unfolded>>32), uint32(unfolded&0xffff_ffff), uint32(initialCarry)) - // if carry != 0, sum <= 0xffff_fffe, otherwise sum <= 0xffff_ffff - // therefore (sum >> 16) + (sum & 0xffff) + carry <= 0x1_fffe; so there is - // no need to save the carry flag - sum = (sum >> 16) + (sum & 0xffff) + carry - // sum <= 0x1_fffe therefore this is the last fold needed: - // if (sum >> 16) > 0 then - // (sum >> 16) == 1 && (sum & 0xffff) <= 0xfffe and therefore - // the addition will not overflow - // otherwise (sum >> 16) == 0 and sum will be unchanged - sum = (sum >> 16) + (sum & 0xffff) - return uint16(sum) -} - -func ipChecksumFold32(unfolded uint32, initialCarry uint32) uint16 { - sum := (unfolded >> 16) + (unfolded & 0xffff) + initialCarry - // sum <= 0x1_ffff: - // 0xffff + 0xffff = 0x1_fffe - // initialCarry is 0 or 1, for a combined maximum of 0x1_ffff - sum = (sum >> 16) + (sum & 0xffff) - // sum <= 0x1_0000 therefore this is the last fold needed: - // if (sum >> 16) > 0 then - // (sum >> 16) == 1 && (sum & 0xffff) == 0 and therefore - // the addition will not overflow - // otherwise (sum >> 16) == 0 and sum will be unchanged - sum = (sum >> 16) + (sum & 0xffff) - return uint16(sum) -} - -func addrPartialChecksum64(addr []byte, initial, carryIn uint64) (sum, carry uint64) { - sum, carry = initial, carryIn - switch len(addr) { - case 4: // IPv4 - if cpu.IsBigEndian { - sum, carry = bits.Add64(sum, uint64(binary.BigEndian.Uint32(addr)), carry) - } else { - sum, carry = bits.Add64(sum, uint64(binary.LittleEndian.Uint32(addr)), carry) - } - case 16: // IPv6 - if cpu.IsBigEndian { - sum, carry = bits.Add64(sum, binary.BigEndian.Uint64(addr), carry) - sum, carry = bits.Add64(sum, binary.BigEndian.Uint64(addr[8:]), carry) - } else { - sum, carry = bits.Add64(sum, binary.LittleEndian.Uint64(addr), carry) - sum, carry = bits.Add64(sum, binary.LittleEndian.Uint64(addr[8:]), carry) - } - default: - panic("bad addr length") - } - return sum, carry -} - -func addrPartialChecksum32(addr []byte, initial, carryIn uint32) (sum, carry uint32) { - sum, carry = initial, carryIn - switch len(addr) { - case 4: // IPv4 - if cpu.IsBigEndian { - sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr), carry) - } else { - sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr), carry) - } - case 16: // IPv6 - if cpu.IsBigEndian { - sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr), carry) - sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[4:8]), carry) - sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[8:12]), carry) - sum, carry = bits.Add32(sum, binary.BigEndian.Uint32(addr[12:16]), carry) - } else { - sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr), carry) - sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[4:8]), carry) - sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[8:12]), carry) - sum, carry = bits.Add32(sum, binary.LittleEndian.Uint32(addr[12:16]), carry) - } - default: - panic("bad addr length") - } - return sum, carry -} - -func pseudoHeaderChecksum64(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { - var sum uint64 - if cpu.IsBigEndian { - sum = uint64(totalLen) + uint64(protocol) - } else { - sum = uint64(bits.ReverseBytes16(totalLen)) + uint64(protocol)<<8 - } - sum, carry := addrPartialChecksum64(srcAddr, sum, 0) - sum, carry = addrPartialChecksum64(dstAddr, sum, carry) - - foldedSum := ipChecksumFold64(sum, carry) - if !cpu.IsBigEndian { - foldedSum = bits.ReverseBytes16(foldedSum) - } - return foldedSum -} - -func pseudoHeaderChecksum32(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { - var sum uint32 - if cpu.IsBigEndian { - sum = uint32(totalLen) + uint32(protocol) - } else { - sum = uint32(bits.ReverseBytes16(totalLen)) + uint32(protocol)<<8 - } - sum, carry := addrPartialChecksum32(srcAddr, sum, 0) - sum, carry = addrPartialChecksum32(dstAddr, sum, carry) - - foldedSum := ipChecksumFold32(sum, carry) - if !cpu.IsBigEndian { - foldedSum = bits.ReverseBytes16(foldedSum) - } - return foldedSum +func pseudoHeaderChecksumNoFold(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint64 { + sum := checksumNoFold(srcAddr, 0) + sum = checksumNoFold(dstAddr, sum) + sum = checksumNoFold([]byte{0, protocol}, sum) + tmp := make([]byte, 2) + binary.BigEndian.PutUint16(tmp, totalLen) + return checksumNoFold(tmp, sum) } // PseudoHeaderChecksum computes an IP pseudo-header checksum. srcAddr and // dstAddr must be 4 or 16 bytes in length. func PseudoHeaderChecksum(protocol uint8, srcAddr, dstAddr []byte, totalLen uint16) uint16 { - if strconv.IntSize < 64 { - return pseudoHeaderChecksum32(protocol, srcAddr, dstAddr, totalLen) - } - return pseudoHeaderChecksum64(protocol, srcAddr, dstAddr, totalLen) + return checksum([]byte{}, pseudoHeaderChecksumNoFold(protocol, srcAddr, dstAddr, totalLen)) } diff --git a/tun/checksum_amd64.go b/tun/checksum_amd64.go deleted file mode 100644 index 4fb684e..0000000 --- a/tun/checksum_amd64.go +++ /dev/null @@ -1,23 +0,0 @@ -package tun - -import "golang.org/x/sys/cpu" - -var checksum = checksumAMD64 - -// Checksum computes an IP checksum starting with the provided initial value. -// The length of data should be at least 128 bytes for best performance. Smaller -// buffers will still compute a correct result. -func Checksum(data []byte, initial uint16) uint16 { - return checksum(data, initial) -} - -func init() { - if cpu.X86.HasAVX && cpu.X86.HasAVX2 && cpu.X86.HasBMI2 { - checksum = checksumAVX2 - return - } - if cpu.X86.HasSSE2 { - checksum = checksumSSE2 - return - } -} diff --git a/tun/checksum_generated_amd64.go b/tun/checksum_generated_amd64.go deleted file mode 100644 index b4a2941..0000000 --- a/tun/checksum_generated_amd64.go +++ /dev/null @@ -1,18 +0,0 @@ -// Code generated by command: go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go. DO NOT EDIT. - -package tun - -// checksumAVX2 computes an IP checksum using amd64 v3 instructions (AVX2, BMI2) -// -//go:noescape -func checksumAVX2(b []byte, initial uint16) uint16 - -// checksumSSE2 computes an IP checksum using amd64 baseline instructions (SSE2) -// -//go:noescape -func checksumSSE2(b []byte, initial uint16) uint16 - -// checksumAMD64 computes an IP checksum using amd64 baseline instructions -// -//go:noescape -func checksumAMD64(b []byte, initial uint16) uint16 diff --git a/tun/checksum_generated_amd64.s b/tun/checksum_generated_amd64.s deleted file mode 100644 index 5f2e4c5..0000000 --- a/tun/checksum_generated_amd64.s +++ /dev/null @@ -1,851 +0,0 @@ -// Code generated by command: go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go. DO NOT EDIT. - -#include "textflag.h" - -DATA xmmLoadMasks<>+0(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff" -DATA xmmLoadMasks<>+16(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff" -DATA xmmLoadMasks<>+32(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff" -DATA xmmLoadMasks<>+48(SB)/16, $"\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff" -DATA xmmLoadMasks<>+64(SB)/16, $"\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" -DATA xmmLoadMasks<>+80(SB)/16, $"\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" -DATA xmmLoadMasks<>+96(SB)/16, $"\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" -GLOBL xmmLoadMasks<>(SB), RODATA|NOPTR, $112 - -// func checksumAVX2(b []byte, initial uint16) uint16 -// Requires: AVX, AVX2, BMI2 -TEXT ·checksumAVX2(SB), NOSPLIT|NOFRAME, $0-34 - MOVWQZX initial+24(FP), AX - XCHGB AH, AL - MOVQ b_base+0(FP), DX - MOVQ b_len+8(FP), BX - - // handle odd length buffers; they are difficult to handle in general - TESTQ $0x00000001, BX - JZ lengthIsEven - MOVBQZX -1(DX)(BX*1), CX - DECQ BX - ADDQ CX, AX - -lengthIsEven: - // handle tiny buffers (<=31 bytes) specially - CMPQ BX, $0x1f - JGT bufferIsNotTiny - XORQ CX, CX - XORQ SI, SI - XORQ DI, DI - - // shift twice to start because length is guaranteed to be even - // n = n >> 2; CF = originalN & 2 - SHRQ $0x02, BX - JNC handleTiny4 - - // tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:] - MOVWQZX (DX), CX - ADDQ $0x02, DX - -handleTiny4: - // n = n >> 1; CF = originalN & 4 - SHRQ $0x01, BX - JNC handleTiny8 - - // tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:] - MOVLQZX (DX), SI - ADDQ $0x04, DX - -handleTiny8: - // n = n >> 1; CF = originalN & 8 - SHRQ $0x01, BX - JNC handleTiny16 - - // tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:] - MOVQ (DX), DI - ADDQ $0x08, DX - -handleTiny16: - // n = n >> 1; CF = originalN & 16 - // n == 0 now, otherwise we would have branched after comparing with tinyBufferSize - SHRQ $0x01, BX - JNC handleTinyFinish - ADDQ (DX), AX - ADCQ 8(DX), AX - -handleTinyFinish: - // CF should be included from the previous add, so we use ADCQ. - // If we arrived via the JNC above, then CF=0 due to the branch condition, - // so ADCQ will still produce the correct result. - ADCQ CX, AX - ADCQ SI, AX - ADCQ DI, AX - JMP foldAndReturn - -bufferIsNotTiny: - // skip all SIMD for small buffers - CMPQ BX, $0x00000100 - JGE startSIMD - - // Accumulate carries in this register. It is never expected to overflow. - XORQ SI, SI - - // We will perform an overlapped read for buffers with length not a multiple of 8. - // Overlapped in this context means some memory will be read twice, but a shift will - // eliminate the duplicated data. This extra read is performed at the end of the buffer to - // preserve any alignment that may exist for the start of the buffer. - MOVQ BX, CX - SHRQ $0x03, BX - ANDQ $0x07, CX - JZ handleRemaining8 - LEAQ (DX)(BX*8), DI - MOVQ -8(DI)(CX*1), DI - - // Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8) - SHLQ $0x03, CX - NEGQ CX - ADDQ $0x40, CX - SHRQ CL, DI - ADDQ DI, AX - ADCQ $0x00, SI - -handleRemaining8: - SHRQ $0x01, BX - JNC handleRemaining16 - ADDQ (DX), AX - ADCQ $0x00, SI - ADDQ $0x08, DX - -handleRemaining16: - SHRQ $0x01, BX - JNC handleRemaining32 - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ $0x00, SI - ADDQ $0x10, DX - -handleRemaining32: - SHRQ $0x01, BX - JNC handleRemaining64 - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ 16(DX), AX - ADCQ 24(DX), AX - ADCQ $0x00, SI - ADDQ $0x20, DX - -handleRemaining64: - SHRQ $0x01, BX - JNC handleRemaining128 - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ 16(DX), AX - ADCQ 24(DX), AX - ADCQ 32(DX), AX - ADCQ 40(DX), AX - ADCQ 48(DX), AX - ADCQ 56(DX), AX - ADCQ $0x00, SI - ADDQ $0x40, DX - -handleRemaining128: - SHRQ $0x01, BX - JNC handleRemainingComplete - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ 16(DX), AX - ADCQ 24(DX), AX - ADCQ 32(DX), AX - ADCQ 40(DX), AX - ADCQ 48(DX), AX - ADCQ 56(DX), AX - ADCQ 64(DX), AX - ADCQ 72(DX), AX - ADCQ 80(DX), AX - ADCQ 88(DX), AX - ADCQ 96(DX), AX - ADCQ 104(DX), AX - ADCQ 112(DX), AX - ADCQ 120(DX), AX - ADCQ $0x00, SI - ADDQ $0x80, DX - -handleRemainingComplete: - ADDQ SI, AX - JMP foldAndReturn - -startSIMD: - VPXOR Y0, Y0, Y0 - VPXOR Y1, Y1, Y1 - VPXOR Y2, Y2, Y2 - VPXOR Y3, Y3, Y3 - MOVQ BX, CX - - // Update number of bytes remaining after the loop completes - ANDQ $0xff, BX - - // Number of 256 byte iterations - SHRQ $0x08, CX - JZ smallLoop - -bigLoop: - VPMOVZXWD (DX), Y4 - VPADDD Y4, Y0, Y0 - VPMOVZXWD 16(DX), Y4 - VPADDD Y4, Y1, Y1 - VPMOVZXWD 32(DX), Y4 - VPADDD Y4, Y2, Y2 - VPMOVZXWD 48(DX), Y4 - VPADDD Y4, Y3, Y3 - VPMOVZXWD 64(DX), Y4 - VPADDD Y4, Y0, Y0 - VPMOVZXWD 80(DX), Y4 - VPADDD Y4, Y1, Y1 - VPMOVZXWD 96(DX), Y4 - VPADDD Y4, Y2, Y2 - VPMOVZXWD 112(DX), Y4 - VPADDD Y4, Y3, Y3 - VPMOVZXWD 128(DX), Y4 - VPADDD Y4, Y0, Y0 - VPMOVZXWD 144(DX), Y4 - VPADDD Y4, Y1, Y1 - VPMOVZXWD 160(DX), Y4 - VPADDD Y4, Y2, Y2 - VPMOVZXWD 176(DX), Y4 - VPADDD Y4, Y3, Y3 - VPMOVZXWD 192(DX), Y4 - VPADDD Y4, Y0, Y0 - VPMOVZXWD 208(DX), Y4 - VPADDD Y4, Y1, Y1 - VPMOVZXWD 224(DX), Y4 - VPADDD Y4, Y2, Y2 - VPMOVZXWD 240(DX), Y4 - VPADDD Y4, Y3, Y3 - ADDQ $0x00000100, DX - DECQ CX - JNZ bigLoop - CMPQ BX, $0x10 - JLT doneSmallLoop - - // now read a single 16 byte unit of data at a time -smallLoop: - VPMOVZXWD (DX), Y4 - VPADDD Y4, Y0, Y0 - ADDQ $0x10, DX - SUBQ $0x10, BX - CMPQ BX, $0x10 - JGE smallLoop - -doneSmallLoop: - CMPQ BX, $0x00 - JE doneSIMD - - // There are between 1 and 15 bytes remaining. Perform an overlapped read. - LEAQ xmmLoadMasks<>+0(SB), CX - VMOVDQU -16(DX)(BX*1), X4 - VPAND -16(CX)(BX*8), X4, X4 - VPMOVZXWD X4, Y4 - VPADDD Y4, Y0, Y0 - -doneSIMD: - // Multi-chain loop is done, combine the accumulators - VPADDD Y1, Y0, Y0 - VPADDD Y2, Y0, Y0 - VPADDD Y3, Y0, Y0 - - // extract the YMM into a pair of XMM and sum them - VEXTRACTI128 $0x01, Y0, X1 - VPADDD X0, X1, X0 - - // extract the XMM into GP64 - VPEXTRQ $0x00, X0, CX - VPEXTRQ $0x01, X0, DX - - // no more AVX code, clear upper registers to avoid SSE slowdowns - VZEROUPPER - ADDQ CX, AX - ADCQ DX, AX - -foldAndReturn: - // add CF and fold - RORXQ $0x20, AX, CX - ADCL CX, AX - RORXL $0x10, AX, CX - ADCW CX, AX - ADCW $0x00, AX - XCHGB AH, AL - MOVW AX, ret+32(FP) - RET - -// func checksumSSE2(b []byte, initial uint16) uint16 -// Requires: SSE2 -TEXT ·checksumSSE2(SB), NOSPLIT|NOFRAME, $0-34 - MOVWQZX initial+24(FP), AX - XCHGB AH, AL - MOVQ b_base+0(FP), DX - MOVQ b_len+8(FP), BX - - // handle odd length buffers; they are difficult to handle in general - TESTQ $0x00000001, BX - JZ lengthIsEven - MOVBQZX -1(DX)(BX*1), CX - DECQ BX - ADDQ CX, AX - -lengthIsEven: - // handle tiny buffers (<=31 bytes) specially - CMPQ BX, $0x1f - JGT bufferIsNotTiny - XORQ CX, CX - XORQ SI, SI - XORQ DI, DI - - // shift twice to start because length is guaranteed to be even - // n = n >> 2; CF = originalN & 2 - SHRQ $0x02, BX - JNC handleTiny4 - - // tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:] - MOVWQZX (DX), CX - ADDQ $0x02, DX - -handleTiny4: - // n = n >> 1; CF = originalN & 4 - SHRQ $0x01, BX - JNC handleTiny8 - - // tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:] - MOVLQZX (DX), SI - ADDQ $0x04, DX - -handleTiny8: - // n = n >> 1; CF = originalN & 8 - SHRQ $0x01, BX - JNC handleTiny16 - - // tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:] - MOVQ (DX), DI - ADDQ $0x08, DX - -handleTiny16: - // n = n >> 1; CF = originalN & 16 - // n == 0 now, otherwise we would have branched after comparing with tinyBufferSize - SHRQ $0x01, BX - JNC handleTinyFinish - ADDQ (DX), AX - ADCQ 8(DX), AX - -handleTinyFinish: - // CF should be included from the previous add, so we use ADCQ. - // If we arrived via the JNC above, then CF=0 due to the branch condition, - // so ADCQ will still produce the correct result. - ADCQ CX, AX - ADCQ SI, AX - ADCQ DI, AX - JMP foldAndReturn - -bufferIsNotTiny: - // skip all SIMD for small buffers - CMPQ BX, $0x00000100 - JGE startSIMD - - // Accumulate carries in this register. It is never expected to overflow. - XORQ SI, SI - - // We will perform an overlapped read for buffers with length not a multiple of 8. - // Overlapped in this context means some memory will be read twice, but a shift will - // eliminate the duplicated data. This extra read is performed at the end of the buffer to - // preserve any alignment that may exist for the start of the buffer. - MOVQ BX, CX - SHRQ $0x03, BX - ANDQ $0x07, CX - JZ handleRemaining8 - LEAQ (DX)(BX*8), DI - MOVQ -8(DI)(CX*1), DI - - // Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8) - SHLQ $0x03, CX - NEGQ CX - ADDQ $0x40, CX - SHRQ CL, DI - ADDQ DI, AX - ADCQ $0x00, SI - -handleRemaining8: - SHRQ $0x01, BX - JNC handleRemaining16 - ADDQ (DX), AX - ADCQ $0x00, SI - ADDQ $0x08, DX - -handleRemaining16: - SHRQ $0x01, BX - JNC handleRemaining32 - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ $0x00, SI - ADDQ $0x10, DX - -handleRemaining32: - SHRQ $0x01, BX - JNC handleRemaining64 - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ 16(DX), AX - ADCQ 24(DX), AX - ADCQ $0x00, SI - ADDQ $0x20, DX - -handleRemaining64: - SHRQ $0x01, BX - JNC handleRemaining128 - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ 16(DX), AX - ADCQ 24(DX), AX - ADCQ 32(DX), AX - ADCQ 40(DX), AX - ADCQ 48(DX), AX - ADCQ 56(DX), AX - ADCQ $0x00, SI - ADDQ $0x40, DX - -handleRemaining128: - SHRQ $0x01, BX - JNC handleRemainingComplete - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ 16(DX), AX - ADCQ 24(DX), AX - ADCQ 32(DX), AX - ADCQ 40(DX), AX - ADCQ 48(DX), AX - ADCQ 56(DX), AX - ADCQ 64(DX), AX - ADCQ 72(DX), AX - ADCQ 80(DX), AX - ADCQ 88(DX), AX - ADCQ 96(DX), AX - ADCQ 104(DX), AX - ADCQ 112(DX), AX - ADCQ 120(DX), AX - ADCQ $0x00, SI - ADDQ $0x80, DX - -handleRemainingComplete: - ADDQ SI, AX - JMP foldAndReturn - -startSIMD: - PXOR X0, X0 - PXOR X1, X1 - PXOR X2, X2 - PXOR X3, X3 - PXOR X4, X4 - MOVQ BX, CX - - // Update number of bytes remaining after the loop completes - ANDQ $0xff, BX - - // Number of 256 byte iterations - SHRQ $0x08, CX - JZ smallLoop - -bigLoop: - MOVOU (DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X0 - PADDD X6, X2 - MOVOU 16(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X1 - PADDD X6, X3 - MOVOU 32(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X2 - PADDD X6, X0 - MOVOU 48(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X3 - PADDD X6, X1 - MOVOU 64(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X0 - PADDD X6, X2 - MOVOU 80(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X1 - PADDD X6, X3 - MOVOU 96(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X2 - PADDD X6, X0 - MOVOU 112(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X3 - PADDD X6, X1 - MOVOU 128(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X0 - PADDD X6, X2 - MOVOU 144(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X1 - PADDD X6, X3 - MOVOU 160(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X2 - PADDD X6, X0 - MOVOU 176(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X3 - PADDD X6, X1 - MOVOU 192(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X0 - PADDD X6, X2 - MOVOU 208(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X1 - PADDD X6, X3 - MOVOU 224(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X2 - PADDD X6, X0 - MOVOU 240(DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X3 - PADDD X6, X1 - ADDQ $0x00000100, DX - DECQ CX - JNZ bigLoop - CMPQ BX, $0x10 - JLT doneSmallLoop - - // now read a single 16 byte unit of data at a time -smallLoop: - MOVOU (DX), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X0 - PADDD X6, X1 - ADDQ $0x10, DX - SUBQ $0x10, BX - CMPQ BX, $0x10 - JGE smallLoop - -doneSmallLoop: - CMPQ BX, $0x00 - JE doneSIMD - - // There are between 1 and 15 bytes remaining. Perform an overlapped read. - LEAQ xmmLoadMasks<>+0(SB), CX - MOVOU -16(DX)(BX*1), X5 - PAND -16(CX)(BX*8), X5 - MOVOA X5, X6 - PUNPCKHWL X4, X5 - PUNPCKLWL X4, X6 - PADDD X5, X0 - PADDD X6, X1 - -doneSIMD: - // Multi-chain loop is done, combine the accumulators - PADDD X1, X0 - PADDD X2, X0 - PADDD X3, X0 - - // extract the XMM into GP64 - MOVQ X0, CX - PSRLDQ $0x08, X0 - MOVQ X0, DX - ADDQ CX, AX - ADCQ DX, AX - -foldAndReturn: - // add CF and fold - MOVL AX, CX - ADCQ $0x00, CX - SHRQ $0x20, AX - ADDQ CX, AX - MOVWQZX AX, CX - SHRQ $0x10, AX - ADDQ CX, AX - MOVW AX, CX - SHRQ $0x10, AX - ADDW CX, AX - ADCW $0x00, AX - XCHGB AH, AL - MOVW AX, ret+32(FP) - RET - -// func checksumAMD64(b []byte, initial uint16) uint16 -TEXT ·checksumAMD64(SB), NOSPLIT|NOFRAME, $0-34 - MOVWQZX initial+24(FP), AX - XCHGB AH, AL - MOVQ b_base+0(FP), DX - MOVQ b_len+8(FP), BX - - // handle odd length buffers; they are difficult to handle in general - TESTQ $0x00000001, BX - JZ lengthIsEven - MOVBQZX -1(DX)(BX*1), CX - DECQ BX - ADDQ CX, AX - -lengthIsEven: - // handle tiny buffers (<=31 bytes) specially - CMPQ BX, $0x1f - JGT bufferIsNotTiny - XORQ CX, CX - XORQ SI, SI - XORQ DI, DI - - // shift twice to start because length is guaranteed to be even - // n = n >> 2; CF = originalN & 2 - SHRQ $0x02, BX - JNC handleTiny4 - - // tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:] - MOVWQZX (DX), CX - ADDQ $0x02, DX - -handleTiny4: - // n = n >> 1; CF = originalN & 4 - SHRQ $0x01, BX - JNC handleTiny8 - - // tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:] - MOVLQZX (DX), SI - ADDQ $0x04, DX - -handleTiny8: - // n = n >> 1; CF = originalN & 8 - SHRQ $0x01, BX - JNC handleTiny16 - - // tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:] - MOVQ (DX), DI - ADDQ $0x08, DX - -handleTiny16: - // n = n >> 1; CF = originalN & 16 - // n == 0 now, otherwise we would have branched after comparing with tinyBufferSize - SHRQ $0x01, BX - JNC handleTinyFinish - ADDQ (DX), AX - ADCQ 8(DX), AX - -handleTinyFinish: - // CF should be included from the previous add, so we use ADCQ. - // If we arrived via the JNC above, then CF=0 due to the branch condition, - // so ADCQ will still produce the correct result. - ADCQ CX, AX - ADCQ SI, AX - ADCQ DI, AX - JMP foldAndReturn - -bufferIsNotTiny: - // Number of 256 byte iterations into loop counter - MOVQ BX, CX - - // Update number of bytes remaining after the loop completes - ANDQ $0xff, BX - SHRQ $0x08, CX - JZ startCleanup - CLC - XORQ SI, SI - XORQ DI, DI - XORQ R8, R8 - XORQ R9, R9 - XORQ R10, R10 - XORQ R11, R11 - XORQ R12, R12 - -bigLoop: - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ 16(DX), AX - ADCQ 24(DX), AX - ADCQ $0x00, SI - ADDQ 32(DX), DI - ADCQ 40(DX), DI - ADCQ 48(DX), DI - ADCQ 56(DX), DI - ADCQ $0x00, R8 - ADDQ 64(DX), R9 - ADCQ 72(DX), R9 - ADCQ 80(DX), R9 - ADCQ 88(DX), R9 - ADCQ $0x00, R10 - ADDQ 96(DX), R11 - ADCQ 104(DX), R11 - ADCQ 112(DX), R11 - ADCQ 120(DX), R11 - ADCQ $0x00, R12 - ADDQ 128(DX), AX - ADCQ 136(DX), AX - ADCQ 144(DX), AX - ADCQ 152(DX), AX - ADCQ $0x00, SI - ADDQ 160(DX), DI - ADCQ 168(DX), DI - ADCQ 176(DX), DI - ADCQ 184(DX), DI - ADCQ $0x00, R8 - ADDQ 192(DX), R9 - ADCQ 200(DX), R9 - ADCQ 208(DX), R9 - ADCQ 216(DX), R9 - ADCQ $0x00, R10 - ADDQ 224(DX), R11 - ADCQ 232(DX), R11 - ADCQ 240(DX), R11 - ADCQ 248(DX), R11 - ADCQ $0x00, R12 - ADDQ $0x00000100, DX - SUBQ $0x01, CX - JNZ bigLoop - ADDQ SI, AX - ADCQ DI, AX - ADCQ R8, AX - ADCQ R9, AX - ADCQ R10, AX - ADCQ R11, AX - ADCQ R12, AX - - // accumulate CF (twice, in case the first time overflows) - ADCQ $0x00, AX - ADCQ $0x00, AX - -startCleanup: - // Accumulate carries in this register. It is never expected to overflow. - XORQ SI, SI - - // We will perform an overlapped read for buffers with length not a multiple of 8. - // Overlapped in this context means some memory will be read twice, but a shift will - // eliminate the duplicated data. This extra read is performed at the end of the buffer to - // preserve any alignment that may exist for the start of the buffer. - MOVQ BX, CX - SHRQ $0x03, BX - ANDQ $0x07, CX - JZ handleRemaining8 - LEAQ (DX)(BX*8), DI - MOVQ -8(DI)(CX*1), DI - - // Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8) - SHLQ $0x03, CX - NEGQ CX - ADDQ $0x40, CX - SHRQ CL, DI - ADDQ DI, AX - ADCQ $0x00, SI - -handleRemaining8: - SHRQ $0x01, BX - JNC handleRemaining16 - ADDQ (DX), AX - ADCQ $0x00, SI - ADDQ $0x08, DX - -handleRemaining16: - SHRQ $0x01, BX - JNC handleRemaining32 - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ $0x00, SI - ADDQ $0x10, DX - -handleRemaining32: - SHRQ $0x01, BX - JNC handleRemaining64 - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ 16(DX), AX - ADCQ 24(DX), AX - ADCQ $0x00, SI - ADDQ $0x20, DX - -handleRemaining64: - SHRQ $0x01, BX - JNC handleRemaining128 - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ 16(DX), AX - ADCQ 24(DX), AX - ADCQ 32(DX), AX - ADCQ 40(DX), AX - ADCQ 48(DX), AX - ADCQ 56(DX), AX - ADCQ $0x00, SI - ADDQ $0x40, DX - -handleRemaining128: - SHRQ $0x01, BX - JNC handleRemainingComplete - ADDQ (DX), AX - ADCQ 8(DX), AX - ADCQ 16(DX), AX - ADCQ 24(DX), AX - ADCQ 32(DX), AX - ADCQ 40(DX), AX - ADCQ 48(DX), AX - ADCQ 56(DX), AX - ADCQ 64(DX), AX - ADCQ 72(DX), AX - ADCQ 80(DX), AX - ADCQ 88(DX), AX - ADCQ 96(DX), AX - ADCQ 104(DX), AX - ADCQ 112(DX), AX - ADCQ 120(DX), AX - ADCQ $0x00, SI - ADDQ $0x80, DX - -handleRemainingComplete: - ADDQ SI, AX - -foldAndReturn: - // add CF and fold - MOVL AX, CX - ADCQ $0x00, CX - SHRQ $0x20, AX - ADDQ CX, AX - MOVWQZX AX, CX - SHRQ $0x10, AX - ADDQ CX, AX - MOVW AX, CX - SHRQ $0x10, AX - ADDW CX, AX - ADCW $0x00, AX - XCHGB AH, AL - MOVW AX, ret+32(FP) - RET diff --git a/tun/checksum_generic.go b/tun/checksum_generic.go deleted file mode 100644 index 2ef201a..0000000 --- a/tun/checksum_generic.go +++ /dev/null @@ -1,15 +0,0 @@ -// This file contains IP checksum algorithms that are not specific to any -// architecture and don't use hardware acceleration. - -//go:build !amd64 - -package tun - -import "strconv" - -func Checksum(data []byte, initial uint16) uint16 { - if strconv.IntSize < 64 { - return checksumGeneric32(data, initial) - } - return checksumGeneric64(data, initial) -} diff --git a/tun/generate_amd64.go b/tun/generate_amd64.go deleted file mode 100644 index 543e211..0000000 --- a/tun/generate_amd64.go +++ /dev/null @@ -1,579 +0,0 @@ -//go:build ignore - -//go:generate go run generate_amd64.go -out checksum_generated_amd64.s -stubs checksum_generated_amd64.go - -package main - -import ( - "fmt" - "math" - "math/bits" - - . "github.com/mmcloughlin/avo/build" - "github.com/mmcloughlin/avo/operand" - "github.com/mmcloughlin/avo/reg" -) - -const checksumSignature = "func(b []byte, initial uint16) uint16" - -func loadParams() (accum, buf, n reg.GPVirtual) { - accum, buf, n = GP64(), GP64(), GP64() - Load(Param("initial"), accum) - XCHGB(accum.As8H(), accum.As8L()) - Load(Param("b").Base(), buf) - Load(Param("b").Len(), n) - return -} - -type simdStrategy int - -const ( - sse2 = iota - avx2 -) - -const tinyBufferSize = 31 // A buffer is tiny if it has at most 31 bytes. - -func generateSIMDChecksum(name, doc string, minSIMDSize, chains int, strategy simdStrategy) { - TEXT(name, NOSPLIT|NOFRAME, checksumSignature) - Pragma("noescape") - Doc(doc) - - accum64, buf, n := loadParams() - - handleOddLength(n, buf, accum64) - // no chance of overflow because accum64 was initialized by a uint16 and - // handleOddLength adds at most a uint8 - handleTinyBuffers(n, buf, accum64, operand.LabelRef("foldAndReturn"), operand.LabelRef("bufferIsNotTiny")) - Label("bufferIsNotTiny") - - const simdReadSize = 16 - - if minSIMDSize > tinyBufferSize { - Comment("skip all SIMD for small buffers") - if minSIMDSize <= math.MaxUint8 { - CMPQ(n, operand.U8(minSIMDSize)) - } else { - CMPQ(n, operand.U32(minSIMDSize)) - } - JGE(operand.LabelRef("startSIMD")) - - handleRemaining(n, buf, accum64, minSIMDSize-1) - JMP(operand.LabelRef("foldAndReturn")) - } - - Label("startSIMD") - - // chains is the number of accumulators to use. This improves speed via - // reduced data dependency. We combine the accumulators once when the big - // loop is complete. - simdAccumulate := make([]reg.VecVirtual, chains) - for i := range simdAccumulate { - switch strategy { - case sse2: - simdAccumulate[i] = XMM() - PXOR(simdAccumulate[i], simdAccumulate[i]) - case avx2: - simdAccumulate[i] = YMM() - VPXOR(simdAccumulate[i], simdAccumulate[i], simdAccumulate[i]) - } - } - var zero reg.VecVirtual - if strategy == sse2 { - zero = XMM() - PXOR(zero, zero) - } - - // Number of loads per big loop - const unroll = 16 - // Number of bytes - loopSize := uint64(simdReadSize * unroll) - if bits.Len64(loopSize) != bits.Len64(loopSize-1)+1 { - panic("loopSize is not a power of 2") - } - loopCount := GP64() - - MOVQ(n, loopCount) - Comment("Update number of bytes remaining after the loop completes") - ANDQ(operand.Imm(loopSize-1), n) - Comment(fmt.Sprintf("Number of %d byte iterations", loopSize)) - SHRQ(operand.Imm(uint64(bits.Len64(loopSize-1))), loopCount) - JZ(operand.LabelRef("smallLoop")) - Label("bigLoop") - for i := 0; i < unroll; i++ { - chain := i % chains - switch strategy { - case sse2: - sse2AccumulateStep(i*simdReadSize, buf, zero, simdAccumulate[chain], simdAccumulate[(chain+chains/2)%chains]) - case avx2: - avx2AccumulateStep(i*simdReadSize, buf, simdAccumulate[chain]) - } - } - ADDQ(operand.U32(loopSize), buf) - DECQ(loopCount) - JNZ(operand.LabelRef("bigLoop")) - - Label("bigCleanup") - - CMPQ(n, operand.Imm(uint64(simdReadSize))) - JLT(operand.LabelRef("doneSmallLoop")) - - Commentf("now read a single %d byte unit of data at a time", simdReadSize) - Label("smallLoop") - - switch strategy { - case sse2: - sse2AccumulateStep(0, buf, zero, simdAccumulate[0], simdAccumulate[1]) - case avx2: - avx2AccumulateStep(0, buf, simdAccumulate[0]) - } - ADDQ(operand.Imm(uint64(simdReadSize)), buf) - SUBQ(operand.Imm(uint64(simdReadSize)), n) - CMPQ(n, operand.Imm(uint64(simdReadSize))) - JGE(operand.LabelRef("smallLoop")) - - Label("doneSmallLoop") - CMPQ(n, operand.Imm(0)) - JE(operand.LabelRef("doneSIMD")) - - Commentf("There are between 1 and %d bytes remaining. Perform an overlapped read.", simdReadSize-1) - - maskDataPtr := GP64() - LEAQ(operand.NewDataAddr(operand.NewStaticSymbol("xmmLoadMasks"), 0), maskDataPtr) - dataAddr := operand.Mem{Index: n, Scale: 1, Base: buf, Disp: -simdReadSize} - // scale 8 is only correct here because n is guaranteed to be even and we - // do not generate masks for odd lengths - maskAddr := operand.Mem{Base: maskDataPtr, Index: n, Scale: 8, Disp: -16} - remainder := XMM() - - switch strategy { - case sse2: - MOVOU(dataAddr, remainder) - PAND(maskAddr, remainder) - low := XMM() - MOVOA(remainder, low) - PUNPCKHWL(zero, remainder) - PUNPCKLWL(zero, low) - PADDD(remainder, simdAccumulate[0]) - PADDD(low, simdAccumulate[1]) - case avx2: - // Note: this is very similar to the sse2 path but MOVOU has a massive - // performance hit if used here, presumably due to switching between SSE - // and AVX2 modes. - VMOVDQU(dataAddr, remainder) - VPAND(maskAddr, remainder, remainder) - - temp := YMM() - VPMOVZXWD(remainder, temp) - VPADDD(temp, simdAccumulate[0], simdAccumulate[0]) - } - - Label("doneSIMD") - - Comment("Multi-chain loop is done, combine the accumulators") - for i := range simdAccumulate { - if i == 0 { - continue - } - switch strategy { - case sse2: - PADDD(simdAccumulate[i], simdAccumulate[0]) - case avx2: - VPADDD(simdAccumulate[i], simdAccumulate[0], simdAccumulate[0]) - } - } - - if strategy == avx2 { - Comment("extract the YMM into a pair of XMM and sum them") - tmp := YMM() - VEXTRACTI128(operand.Imm(1), simdAccumulate[0], tmp.AsX()) - - xAccumulate := XMM() - VPADDD(simdAccumulate[0].AsX(), tmp.AsX(), xAccumulate) - simdAccumulate = []reg.VecVirtual{xAccumulate} - } - - Comment("extract the XMM into GP64") - low, high := GP64(), GP64() - switch strategy { - case sse2: - MOVQ(simdAccumulate[0], low) - PSRLDQ(operand.Imm(8), simdAccumulate[0]) - MOVQ(simdAccumulate[0], high) - case avx2: - VPEXTRQ(operand.Imm(0), simdAccumulate[0], low) - VPEXTRQ(operand.Imm(1), simdAccumulate[0], high) - - Comment("no more AVX code, clear upper registers to avoid SSE slowdowns") - VZEROUPPER() - } - ADDQ(low, accum64) - ADCQ(high, accum64) - Label("foldAndReturn") - foldWithCF(accum64, strategy == avx2) - XCHGB(accum64.As8H(), accum64.As8L()) - Store(accum64.As16(), ReturnIndex(0)) - RET() -} - -// handleOddLength generates instructions to incorporate the last byte into -// accum64 if the length is odd. CF may be set if accum64 overflows; be sure to -// handle that if overflow is possible. -func handleOddLength(n, buf, accum64 reg.GPVirtual) { - Comment("handle odd length buffers; they are difficult to handle in general") - TESTQ(operand.U32(1), n) - JZ(operand.LabelRef("lengthIsEven")) - - tmp := GP64() - MOVBQZX(operand.Mem{Base: buf, Index: n, Scale: 1, Disp: -1}, tmp) - DECQ(n) - ADDQ(tmp, accum64) - - Label("lengthIsEven") -} - -func sse2AccumulateStep(offset int, buf reg.GPVirtual, zero, accumulate1, accumulate2 reg.VecVirtual) { - high, low := XMM(), XMM() - MOVOU(operand.Mem{Disp: offset, Base: buf}, high) - MOVOA(high, low) - PUNPCKHWL(zero, high) - PUNPCKLWL(zero, low) - PADDD(high, accumulate1) - PADDD(low, accumulate2) -} - -func avx2AccumulateStep(offset int, buf reg.GPVirtual, accumulate reg.VecVirtual) { - tmp := YMM() - VPMOVZXWD(operand.Mem{Disp: offset, Base: buf}, tmp) - VPADDD(tmp, accumulate, accumulate) -} - -func generateAMD64Checksum(name, doc string) { - TEXT(name, NOSPLIT|NOFRAME, checksumSignature) - Pragma("noescape") - Doc(doc) - - accum64, buf, n := loadParams() - - handleOddLength(n, buf, accum64) - // no chance of overflow because accum64 was initialized by a uint16 and - // handleOddLength adds at most a uint8 - handleTinyBuffers(n, buf, accum64, operand.LabelRef("foldAndReturn"), operand.LabelRef("bufferIsNotTiny")) - Label("bufferIsNotTiny") - - const ( - // numChains is the number of accumulators and carry counters to use. - // This improves speed via reduced data dependency. We combine the - // accumulators and carry counters once when the loop is complete. - numChains = 4 - unroll = 32 // The number of 64-bit reads to perform per iteration of the loop. - loopSize = 8 * unroll // The number of bytes read per iteration of the loop. - ) - if bits.Len(loopSize) != bits.Len(loopSize-1)+1 { - panic("loopSize is not a power of 2") - } - loopCount := GP64() - - Comment(fmt.Sprintf("Number of %d byte iterations into loop counter", loopSize)) - MOVQ(n, loopCount) - Comment("Update number of bytes remaining after the loop completes") - ANDQ(operand.Imm(loopSize-1), n) - SHRQ(operand.Imm(uint64(bits.Len(loopSize-1))), loopCount) - JZ(operand.LabelRef("startCleanup")) - CLC() - - chains := make([]struct { - accum reg.GPVirtual - carries reg.GPVirtual - }, numChains) - for i := range chains { - if i == 0 { - chains[i].accum = accum64 - } else { - chains[i].accum = GP64() - XORQ(chains[i].accum, chains[i].accum) - } - chains[i].carries = GP64() - XORQ(chains[i].carries, chains[i].carries) - } - - Label("bigLoop") - - var curChain int - for i := 0; i < unroll; i++ { - // It is significantly faster to use a ADCX/ADOX pair instead of plain - // ADC, which results in two dependency chains, however those require - // ADX support, which was added after AVX2. If AVX2 is available, that's - // even better than ADCX/ADOX. - // - // However, multiple dependency chains using multiple accumulators and - // occasionally storing CF into temporary counters seems to work almost - // as well. - addr := operand.Mem{Disp: i * 8, Base: buf} - - if i%4 == 0 { - if i > 0 { - ADCQ(operand.Imm(0), chains[curChain].carries) - curChain = (curChain + 1) % len(chains) - } - ADDQ(addr, chains[curChain].accum) - } else { - ADCQ(addr, chains[curChain].accum) - } - } - ADCQ(operand.Imm(0), chains[curChain].carries) - ADDQ(operand.U32(loopSize), buf) - SUBQ(operand.Imm(1), loopCount) - JNZ(operand.LabelRef("bigLoop")) - for i := range chains { - if i == 0 { - ADDQ(chains[i].carries, accum64) - continue - } - ADCQ(chains[i].accum, accum64) - ADCQ(chains[i].carries, accum64) - } - - accumulateCF(accum64) - - Label("startCleanup") - handleRemaining(n, buf, accum64, loopSize-1) - Label("foldAndReturn") - foldWithCF(accum64, false) - - XCHGB(accum64.As8H(), accum64.As8L()) - Store(accum64.As16(), ReturnIndex(0)) - RET() -} - -// handleTinyBuffers computes checksums if the buffer length (the n parameter) -// is less than 32. After computing the checksum, a jump to returnLabel will -// be executed. Otherwise, if the buffer length is at least 32, nothing will be -// modified; a jump to continueLabel will be executed instead. -// -// When jumping to returnLabel, CF may be set and must be accommodated e.g. -// using foldWithCF or accumulateCF. -// -// Anecdotally, this appears to be faster than attempting to coordinate an -// overlapped read (which would also require special handling for buffers -// smaller than 8). -func handleTinyBuffers(n, buf, accum reg.GPVirtual, returnLabel, continueLabel operand.LabelRef) { - Comment("handle tiny buffers (<=31 bytes) specially") - CMPQ(n, operand.Imm(tinyBufferSize)) - JGT(continueLabel) - - tmp2, tmp4, tmp8 := GP64(), GP64(), GP64() - XORQ(tmp2, tmp2) - XORQ(tmp4, tmp4) - XORQ(tmp8, tmp8) - - Comment("shift twice to start because length is guaranteed to be even", - "n = n >> 2; CF = originalN & 2") - SHRQ(operand.Imm(2), n) - JNC(operand.LabelRef("handleTiny4")) - Comment("tmp2 = binary.LittleEndian.Uint16(buf[:2]); buf = buf[2:]") - MOVWQZX(operand.Mem{Base: buf}, tmp2) - ADDQ(operand.Imm(2), buf) - - Label("handleTiny4") - Comment("n = n >> 1; CF = originalN & 4") - SHRQ(operand.Imm(1), n) - JNC(operand.LabelRef("handleTiny8")) - Comment("tmp4 = binary.LittleEndian.Uint32(buf[:4]); buf = buf[4:]") - MOVLQZX(operand.Mem{Base: buf}, tmp4) - ADDQ(operand.Imm(4), buf) - - Label("handleTiny8") - Comment("n = n >> 1; CF = originalN & 8") - SHRQ(operand.Imm(1), n) - JNC(operand.LabelRef("handleTiny16")) - Comment("tmp8 = binary.LittleEndian.Uint64(buf[:8]); buf = buf[8:]") - MOVQ(operand.Mem{Base: buf}, tmp8) - ADDQ(operand.Imm(8), buf) - - Label("handleTiny16") - Comment("n = n >> 1; CF = originalN & 16", - "n == 0 now, otherwise we would have branched after comparing with tinyBufferSize") - SHRQ(operand.Imm(1), n) - JNC(operand.LabelRef("handleTinyFinish")) - ADDQ(operand.Mem{Base: buf}, accum) - ADCQ(operand.Mem{Base: buf, Disp: 8}, accum) - - Label("handleTinyFinish") - Comment("CF should be included from the previous add, so we use ADCQ.", - "If we arrived via the JNC above, then CF=0 due to the branch condition,", - "so ADCQ will still produce the correct result.") - ADCQ(tmp2, accum) - ADCQ(tmp4, accum) - ADCQ(tmp8, accum) - - JMP(returnLabel) -} - -// handleRemaining generates a series of conditional unrolled additions, -// starting with 8 bytes long and doubling each time until the length reaches -// max. This is the reverse order of what may be intuitive, but makes the branch -// conditions convenient to compute: perform one right shift each time and test -// against CF. -// -// When done, CF may be set and must be accommodated e.g., using foldWithCF or -// accumulateCF. -// -// If n is not a multiple of 8, an extra 64 bit read at the end of the buffer -// will be performed, overlapping with data that will be read later. The -// duplicate data will be shifted off. -// -// The original buffer length must have been at least 8 bytes long, even if -// n < 8, otherwise this will access memory before the start of the buffer, -// which may be unsafe. -func handleRemaining(n, buf, accum64 reg.GPVirtual, max int) { - Comment("Accumulate carries in this register. It is never expected to overflow.") - carries := GP64() - XORQ(carries, carries) - - Comment("We will perform an overlapped read for buffers with length not a multiple of 8.", - "Overlapped in this context means some memory will be read twice, but a shift will", - "eliminate the duplicated data. This extra read is performed at the end of the buffer to", - "preserve any alignment that may exist for the start of the buffer.") - leftover := reg.RCX - MOVQ(n, leftover) - SHRQ(operand.Imm(3), n) // n is now the number of 64 bit reads remaining - ANDQ(operand.Imm(0x7), leftover) // leftover is now the number of bytes to read from the end - JZ(operand.LabelRef("handleRemaining8")) - endBuf := GP64() - // endBuf is the position near the end of the buffer that is just past the - // last multiple of 8: (buf + len(buf)) & ^0x7 - LEAQ(operand.Mem{Base: buf, Index: n, Scale: 8}, endBuf) - - overlapRead := GP64() - // equivalent to overlapRead = binary.LittleEndian.Uint64(buf[len(buf)-8:len(buf)]) - MOVQ(operand.Mem{Base: endBuf, Index: leftover, Scale: 1, Disp: -8}, overlapRead) - - Comment("Shift out the duplicated data: overlapRead = overlapRead >> (64 - leftoverBytes*8)") - SHLQ(operand.Imm(3), leftover) // leftover = leftover * 8 - NEGQ(leftover) // leftover = -leftover; this completes the (-leftoverBytes*8) part of the expression - ADDQ(operand.Imm(64), leftover) // now we have (64 - leftoverBytes*8) - SHRQ(reg.CL, overlapRead) // shift right by (64 - leftoverBytes*8); CL is the low 8 bits of leftover (set to RCX above) and variable shift only accepts CL - - ADDQ(overlapRead, accum64) - ADCQ(operand.Imm(0), carries) - - for curBytes := 8; curBytes <= max; curBytes *= 2 { - Label(fmt.Sprintf("handleRemaining%d", curBytes)) - SHRQ(operand.Imm(1), n) - if curBytes*2 <= max { - JNC(operand.LabelRef(fmt.Sprintf("handleRemaining%d", curBytes*2))) - } else { - JNC(operand.LabelRef("handleRemainingComplete")) - } - - numLoads := curBytes / 8 - for i := 0; i < numLoads; i++ { - addr := operand.Mem{Base: buf, Disp: i * 8} - // It is possible to add the multiple dependency chains trick here - // that generateAMD64Checksum uses but anecdotally it does not - // appear to outweigh the cost. - if i == 0 { - ADDQ(addr, accum64) - continue - } - ADCQ(addr, accum64) - } - ADCQ(operand.Imm(0), carries) - - if curBytes > math.MaxUint8 { - ADDQ(operand.U32(uint64(curBytes)), buf) - } else { - ADDQ(operand.U8(uint64(curBytes)), buf) - } - if curBytes*2 >= max { - continue - } - JMP(operand.LabelRef(fmt.Sprintf("handleRemaining%d", curBytes*2))) - } - Label("handleRemainingComplete") - ADDQ(carries, accum64) -} - -func accumulateCF(accum64 reg.GPVirtual) { - Comment("accumulate CF (twice, in case the first time overflows)") - // accum64 += CF - ADCQ(operand.Imm(0), accum64) - // accum64 += CF again if the previous add overflowed. The previous add was - // 0 or 1. If it overflowed, then accum64 == 0, so adding another 1 can - // never overflow. - ADCQ(operand.Imm(0), accum64) -} - -// foldWithCF generates instructions to fold accum (a GP64) into a 16-bit value -// according to ones-complement arithmetic. BMI2 instructions will be used if -// allowBMI2 is true (requires fewer instructions). -func foldWithCF(accum reg.GPVirtual, allowBMI2 bool) { - Comment("add CF and fold") - - // CF|accum max value starts as 0x1_ffff_ffff_ffff_ffff - - tmp := GP64() - if allowBMI2 { - // effectively, tmp = accum >> 32 (technically, this is a rotate) - RORXQ(operand.Imm(32), accum, tmp) - // accum as uint32 = uint32(accum) + uint32(tmp64) + CF; max value 0xffff_ffff + CF set - ADCL(tmp.As32(), accum.As32()) - // effectively, tmp64 as uint32 = uint32(accum) >> 16 (also a rotate) - RORXL(operand.Imm(16), accum.As32(), tmp.As32()) - // accum as uint16 = uint16(accum) + uint16(tmp) + CF; max value 0xffff + CF unset or 0xfffe + CF set - ADCW(tmp.As16(), accum.As16()) - } else { - // tmp = uint32(accum); max value 0xffff_ffff - // MOVL clears the upper 32 bits of a GP64 so this is equivalent to the - // non-existent MOVLQZX. - MOVL(accum.As32(), tmp.As32()) - // tmp += CF; max value 0x1_0000_0000, CF unset - ADCQ(operand.Imm(0), tmp) - // accum = accum >> 32; max value 0xffff_ffff - SHRQ(operand.Imm(32), accum) - // accum = accum + tmp; max value 0x1_ffff_ffff + CF unset - ADDQ(tmp, accum) - // tmp = uint16(accum); max value 0xffff - MOVWQZX(accum.As16(), tmp) - // accum = accum >> 16; max value 0x1_ffff - SHRQ(operand.Imm(16), accum) - // accum = accum + tmp; max value 0x2_fffe + CF unset - ADDQ(tmp, accum) - // tmp as uint16 = uint16(accum); max value 0xffff - MOVW(accum.As16(), tmp.As16()) - // accum = accum >> 16; max value 0x2 - SHRQ(operand.Imm(16), accum) - // accum as uint16 = uint16(accum) + uint16(tmp); max value 0xffff + CF unset or 0x2 + CF set - ADDW(tmp.As16(), accum.As16()) - } - // accum as uint16 += CF; will not overflow: either CF was 0 or accum <= 0xfffe - ADCW(operand.Imm(0), accum.As16()) -} - -func generateLoadMasks() { - var offset int - // xmmLoadMasks is a table of masks that can be used with PAND to zero all but the last N bytes in an XMM, N=2,4,6,8,10,12,14 - GLOBL("xmmLoadMasks", RODATA|NOPTR) - - for n := 2; n < 16; n += 2 { - var pattern [16]byte - for i := 0; i < len(pattern); i++ { - if i < len(pattern)-n { - pattern[i] = 0 - continue - } - pattern[i] = 0xff - } - DATA(offset, operand.String(pattern[:])) - offset += len(pattern) - } -} - -func main() { - generateLoadMasks() - generateSIMDChecksum("checksumAVX2", "checksumAVX2 computes an IP checksum using amd64 v3 instructions (AVX2, BMI2)", 256, 4, avx2) - generateSIMDChecksum("checksumSSE2", "checksumSSE2 computes an IP checksum using amd64 baseline instructions (SSE2)", 256, 4, sse2) - generateAMD64Checksum("checksumAMD64", "checksumAMD64 computes an IP checksum using amd64 baseline instructions") - Generate() -} diff --git a/tun/offload.go b/tun/offload.go index 6db437c..4e84db4 100644 --- a/tun/offload.go +++ b/tun/offload.go @@ -55,32 +55,25 @@ type GSOOptions struct { } const ( - ipv4SrcAddrOffset = 12 - ipv6SrcAddrOffset = 8 -) - -const tcpFlagsOffset = 13 - -const ( - tcpFlagFIN uint8 = 0x01 - tcpFlagPSH uint8 = 0x08 - tcpFlagACK uint8 = 0x10 + gsoIPv4SrcAddrOffset = 12 + gsoIPv6SrcAddrOffset = 8 + gsoTCPFlagsOffset = 13 + gsoIPProtoTCP = 6 + gsoIPProtoUDP = 17 ) const ( - // defined here in order to avoid importation of any platform-specific pkgs - ipProtoTCP = 6 - ipProtoUDP = 17 + gsoTCPFlagFIN uint8 = 0x01 + gsoTCPFlagPSH uint8 = 0x08 ) -// GSOSplit splits packets from 'in' into outBufs[][outOffset:], writing +// GSOSplit splits packets from in into outBufs[][outOffset:], writing // the size of each element into sizes. It returns the number of buffers -// populated, and/or an error. Callers may pass an 'in' slice that overlaps with -// the first element of outBuffers, i.e. &in[0] may be equal to +// populated, and/or an error. Callers may pass an in slice that overlaps with +// the first element of outBufs, i.e. &in[0] may be equal to // &outBufs[0][outOffset]. GSONone is a valid options.GSOType regardless of the // value of options.NeedsCsum. Length of each outBufs element must be greater -// than or equal to the length of 'in', otherwise output may be silently -// truncated. +// than or equal to the length of in, otherwise output may be silently truncated. func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outOffset int) (int, error) { cSumAt := int(options.CsumStart) + int(options.CsumOffset) if cSumAt+1 >= len(in) { @@ -91,15 +84,12 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO return 0, fmt.Errorf("length of packet (%d) < GSO HdrLen (%d)", len(in), options.HdrLen) } - // Handle the conditions where we are copying a single element to outBuffs. payloadLen := len(in) - int(options.HdrLen) if options.GSOType == GSONone || payloadLen < int(options.GSOSize) { if len(in) > len(outBufs[0][outOffset:]) { return 0, fmt.Errorf("length of packet (%d) exceeds output element length (%d)", len(in), len(outBufs[0][outOffset:])) } if options.NeedsCsum { - // The initial value at the checksum offset should be summed with - // the checksum we compute. This is typically the pseudo-header sum. initial := binary.BigEndian.Uint16(in[cSumAt:]) in[cSumAt], in[cSumAt+1] = 0, 0 binary.BigEndian.PutUint16(in[cSumAt:], ^Checksum(in[options.CsumStart:], initial)) @@ -133,24 +123,24 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO } iphLen := int(options.CsumStart) - srcAddrOffset := ipv6SrcAddrOffset + srcAddrOffset := gsoIPv6SrcAddrOffset addrLen := 16 if ipVersion == 4 { - srcAddrOffset = ipv4SrcAddrOffset + srcAddrOffset = gsoIPv4SrcAddrOffset addrLen = 4 } transportCsumAt := int(options.CsumStart + options.CsumOffset) var firstTCPSeqNum uint32 var protocol uint8 if options.GSOType == GSOTCPv4 || options.GSOType == GSOTCPv6 { - protocol = ipProtoTCP + protocol = gsoIPProtoTCP if len(in) < int(options.CsumStart)+20 { return 0, fmt.Errorf("length of packet (%d) < GSO CsumStart (%d) + minimum TCP header size (%d)", len(in), options.CsumStart, 20) } firstTCPSeqNum = binary.BigEndian.Uint32(in[options.CsumStart+4:]) } else { - protocol = ipProtoUDP + protocol = gsoIPProtoUDP } nextSegmentDataAt := int(options.HdrLen) i := 0 @@ -169,45 +159,35 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO copy(out, in[:iphLen]) if ipVersion == 4 { - // For IPv4 we are responsible for incrementing the ID field, - // updating the total len field, and recalculating the header - // checksum. if i > 0 { id := binary.BigEndian.Uint16(out[4:]) id += uint16(i) binary.BigEndian.PutUint16(out[4:], id) } - out[10], out[11] = 0, 0 // clear ipv4 header checksum + out[10], out[11] = 0, 0 binary.BigEndian.PutUint16(out[2:], uint16(totalLen)) ipv4CSum := ^Checksum(out[:iphLen], 0) binary.BigEndian.PutUint16(out[10:], ipv4CSum) } else { - // For IPv6 we are responsible for updating the payload length field. binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen)) } - // copy transport header copy(out[options.CsumStart:options.HdrLen], in[options.CsumStart:options.HdrLen]) - if protocol == ipProtoTCP { - // set TCP seq and adjust TCP flags + if protocol == gsoIPProtoTCP { tcpSeq := firstTCPSeqNum + uint32(options.GSOSize*uint16(i)) binary.BigEndian.PutUint32(out[options.CsumStart+4:], tcpSeq) if nextSegmentEnd != len(in) { - // FIN and PSH should only be set on last segment - clearFlags := tcpFlagFIN | tcpFlagPSH - out[options.CsumStart+tcpFlagsOffset] &^= clearFlags + clearFlags := gsoTCPFlagFIN | gsoTCPFlagPSH + out[options.CsumStart+gsoTCPFlagsOffset] &^= clearFlags } } else { - // set UDP header len binary.BigEndian.PutUint16(out[options.CsumStart+4:], uint16(segmentDataLen)+(options.HdrLen-options.CsumStart)) } - // payload copy(out[options.HdrLen:], in[nextSegmentDataAt:nextSegmentEnd]) - // transport checksum - out[transportCsumAt], out[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum + out[transportCsumAt], out[transportCsumAt+1] = 0, 0 transportHeaderLen := int(options.HdrLen - options.CsumStart) lenForPseudo := uint16(transportHeaderLen + segmentDataLen) transportCSum := PseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) diff --git a/tun/offload_linux.go b/tun/offload_linux.go index 6ba8547..6825bbc 100644 --- a/tun/offload_linux.go +++ b/tun/offload_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package tun @@ -9,7 +9,6 @@ import ( "bytes" "encoding/binary" "errors" - "fmt" "io" "unsafe" @@ -17,6 +16,14 @@ import ( "golang.org/x/sys/unix" ) +const tcpFlagsOffset = 13 + +const ( + tcpFlagFIN uint8 = 0x01 + tcpFlagPSH uint8 = 0x08 + tcpFlagACK uint8 = 0x10 +) + // virtioNetHdr is defined in the kernel in include/uapi/linux/virtio_net.h. The // kernel symbol is virtio_net_hdr. type virtioNetHdr struct { @@ -28,30 +35,6 @@ type virtioNetHdr struct { csumOffset uint16 } -func (v *virtioNetHdr) toGSOOptions() (GSOOptions, error) { - var gsoType GSOType - switch v.gsoType { - case unix.VIRTIO_NET_HDR_GSO_NONE: - gsoType = GSONone - case unix.VIRTIO_NET_HDR_GSO_TCPV4: - gsoType = GSOTCPv4 - case unix.VIRTIO_NET_HDR_GSO_TCPV6: - gsoType = GSOTCPv6 - case unix.VIRTIO_NET_HDR_GSO_UDP_L4: - gsoType = GSOUDPL4 - default: - return GSOOptions{}, fmt.Errorf("unsupported virtio gsoType: %d", v.gsoType) - } - return GSOOptions{ - GSOType: gsoType, - HdrLen: v.hdrLen, - CsumStart: v.csumStart, - CsumOffset: v.csumOffset, - GSOSize: v.gsoSize, - NeedsCsum: v.flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0, - }, nil -} - func (v *virtioNetHdr) decode(b []byte) error { if len(b) < virtioNetHdrLen { return io.ErrShortBuffer @@ -410,8 +393,8 @@ func checksumValid(pkt []byte, iphLen, proto uint8, isV6 bool) bool { addrSize = 16 } lenForPseudo := uint16(len(pkt) - int(iphLen)) - cSum := PseudoHeaderChecksum(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo) - return ^Checksum(pkt[iphLen:], cSum) == 0 + cSum := pseudoHeaderChecksumNoFold(proto, pkt[srcAddrAt:srcAddrAt+addrSize], pkt[srcAddrAt+addrSize:srcAddrAt+addrSize*2], lenForPseudo) + return ^checksum(pkt[iphLen:], cSum) == 0 } // coalesceResult represents the result of attempting to coalesce two TCP @@ -527,7 +510,9 @@ const ( ) const ( - maxUint16 = 1<<16 - 1 + ipv4SrcAddrOffset = 12 + ipv6SrcAddrOffset = 8 + maxUint16 = 1<<16 - 1 ) type groResult int @@ -659,7 +644,7 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4 pkt[10], pkt[11] = 0, 0 binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length - iphCSum := ^Checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum + iphCSum := ^checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field } err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -679,8 +664,8 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e srcAddrAt := offset + addrOffset srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] - psum := PseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) - binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum)) + psum := pseudoHeaderChecksumNoFold(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) + binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) } else { hdr := virtioNetHdr{} err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -716,7 +701,7 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e } else { pkt[10], pkt[11] = 0, 0 binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length - iphCSum := ^Checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum + iphCSum := ^checksum(pkt[:item.iphLen], 0) // compute IPv4 header checksum binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field } err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -739,8 +724,8 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e srcAddrAt := offset + addrOffset srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen] dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2] - psum := PseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) - binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum)) + psum := pseudoHeaderChecksumNoFold(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen))) + binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], checksum([]byte{}, psum)) } else { hdr := virtioNetHdr{} err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:]) @@ -909,3 +894,100 @@ func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGR errUDP := applyUDPCoalesceAccounting(bufs, offset, udpTable) return errors.Join(errTCP, errUDP) } + +// gsoSplit splits packets from in into outBuffs, writing the size of each +// element into sizes. It returns the number of buffers populated, and/or an +// error. +func gsoSplit(in []byte, hdr virtioNetHdr, outBuffs [][]byte, sizes []int, outOffset int, isV6 bool) (int, error) { + iphLen := int(hdr.csumStart) + srcAddrOffset := ipv6SrcAddrOffset + addrLen := 16 + if !isV6 { + in[10], in[11] = 0, 0 // clear ipv4 header checksum + srcAddrOffset = ipv4SrcAddrOffset + addrLen = 4 + } + transportCsumAt := int(hdr.csumStart + hdr.csumOffset) + in[transportCsumAt], in[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum + var firstTCPSeqNum uint32 + var protocol uint8 + if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV4 || hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV6 { + protocol = unix.IPPROTO_TCP + firstTCPSeqNum = binary.BigEndian.Uint32(in[hdr.csumStart+4:]) + } else { + protocol = unix.IPPROTO_UDP + } + nextSegmentDataAt := int(hdr.hdrLen) + i := 0 + for ; nextSegmentDataAt < len(in); i++ { + if i == len(outBuffs) { + return i - 1, ErrTooManySegments + } + nextSegmentEnd := nextSegmentDataAt + int(hdr.gsoSize) + if nextSegmentEnd > len(in) { + nextSegmentEnd = len(in) + } + segmentDataLen := nextSegmentEnd - nextSegmentDataAt + totalLen := int(hdr.hdrLen) + segmentDataLen + sizes[i] = totalLen + out := outBuffs[i][outOffset:] + + copy(out, in[:iphLen]) + if !isV6 { + // For IPv4 we are responsible for incrementing the ID field, + // updating the total len field, and recalculating the header + // checksum. + if i > 0 { + id := binary.BigEndian.Uint16(out[4:]) + id += uint16(i) + binary.BigEndian.PutUint16(out[4:], id) + } + binary.BigEndian.PutUint16(out[2:], uint16(totalLen)) + ipv4CSum := ^checksum(out[:iphLen], 0) + binary.BigEndian.PutUint16(out[10:], ipv4CSum) + } else { + // For IPv6 we are responsible for updating the payload length field. + binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen)) + } + + // copy transport header + copy(out[hdr.csumStart:hdr.hdrLen], in[hdr.csumStart:hdr.hdrLen]) + + if protocol == unix.IPPROTO_TCP { + // set TCP seq and adjust TCP flags + tcpSeq := firstTCPSeqNum + uint32(hdr.gsoSize*uint16(i)) + binary.BigEndian.PutUint32(out[hdr.csumStart+4:], tcpSeq) + if nextSegmentEnd != len(in) { + // FIN and PSH should only be set on last segment + clearFlags := tcpFlagFIN | tcpFlagPSH + out[hdr.csumStart+tcpFlagsOffset] &^= clearFlags + } + } else { + // set UDP header len + binary.BigEndian.PutUint16(out[hdr.csumStart+4:], uint16(segmentDataLen)+(hdr.hdrLen-hdr.csumStart)) + } + + // payload + copy(out[hdr.hdrLen:], in[nextSegmentDataAt:nextSegmentEnd]) + + // transport checksum + transportHeaderLen := int(hdr.hdrLen - hdr.csumStart) + lenForPseudo := uint16(transportHeaderLen + segmentDataLen) + transportCSumNoFold := pseudoHeaderChecksumNoFold(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo) + transportCSum := ^checksum(out[hdr.csumStart:totalLen], transportCSumNoFold) + binary.BigEndian.PutUint16(out[hdr.csumStart+hdr.csumOffset:], transportCSum) + + nextSegmentDataAt += int(hdr.gsoSize) + } + return i, nil +} + +func gsoNoneChecksum(in []byte, cSumStart, cSumOffset uint16) error { + cSumAt := cSumStart + cSumOffset + // The initial value at the checksum offset should be summed with the + // checksum we compute. This is typically the pseudo-header checksum. + initial := binary.BigEndian.Uint16(in[cSumAt:]) + in[cSumAt], in[cSumAt+1] = 0, 0 + binary.BigEndian.PutUint16(in[cSumAt:], ^checksum(in[cSumStart:], uint64(initial))) + return nil +} diff --git a/tun/tun.go b/tun/tun.go index 733b24c..6fb5c56 100644 --- a/tun/tun.go +++ b/tun/tun.go @@ -56,21 +56,12 @@ type Device interface { // versions may have offload bugs. Where these bugs negatively impact throughput // or break connectivity entirely we can use these methods to disable the // related offload. -// -// Linux has the following known, GRO bugs. -// -// torvalds/linux@e269d79c7d35aa3808b1f3c1737d63dab504ddc8 broke virtio_net -// TCP & UDP GRO causing GRO writes to return EINVAL. The bug was then -// resolved later in -// torvalds/linux@89add40066f9ed9abe5f7f886fe5789ff7e0c50e. The offending -// commit was pulled into various LTS releases. -// -// UDP GRO writes end up blackholing/dropping packets destined for a -// vxlan/geneve interface on kernel versions prior to 6.8.5. type GRODevice interface { Device + // DisableUDPGRO disables UDP GRO if it is enabled. DisableUDPGRO() + // DisableTCPGRO disables TCP GRO if it is enabled. DisableTCPGRO() } diff --git a/tun/tun_linux.go b/tun/tun_linux.go index 8f4aed1..4b7866d 100644 --- a/tun/tun_linux.go +++ b/tun/tun_linux.go @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: MIT * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. + * Copyright (C) 2017-2025 WireGuard LLC. All Rights Reserved. */ package tun @@ -48,7 +48,7 @@ type NativeTun struct { readOpMu sync.Mutex // readOpMu guards readBuff readBuff [virtioNetHdrLen + 65535]byte // if vnetHdr every read() is prefixed by virtioNetHdr - writeOpMu sync.Mutex // writeOpMu guards the following fields + writeOpMu sync.Mutex // writeOpMu guards toWrite, tcpGROTable, udpGROTable, gro toWrite []int tcpGROTable *tcpGROTable udpGROTable *udpGROTable @@ -269,15 +269,21 @@ func (tun *NativeTun) setMTU(n int) error { defer unix.Close(fd) - req, err := unix.NewIfreq(name) - if err != nil { - return fmt.Errorf("unix.NewIfreq(%q): %w", name, err) - } - req.SetUint32(uint32(n)) - err = unix.IoctlIfreq(fd, unix.SIOCSIFMTU, req) - if err != nil { - return fmt.Errorf("failed to set MTU of TUN device %q: %w", name, err) + // do ioctl call + var ifr [ifReqSize]byte + copy(ifr[:], name) + *(*uint32)(unsafe.Pointer(&ifr[unix.IFNAMSIZ])) = uint32(n) + _, _, errno := unix.Syscall( + unix.SYS_IOCTL, + uintptr(fd), + uintptr(unix.SIOCSIFMTU), + uintptr(unsafe.Pointer(&ifr[0])), + ) + + if errno != 0 { + return fmt.Errorf("failed to set MTU of TUN device: %w", errno) } + return nil } @@ -396,32 +402,73 @@ func handleVirtioRead(in []byte, bufs [][]byte, sizes []int, offset int) (int, e return 0, err } in = in[virtioNetHdrLen:] - - options, err := hdr.toGSOOptions() - if err != nil { - return 0, err + if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_NONE { + if hdr.flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0 { + // This means CHECKSUM_PARTIAL in skb context. We are responsible + // for computing the checksum starting at hdr.csumStart and placing + // at hdr.csumOffset. + err = gsoNoneChecksum(in, hdr.csumStart, hdr.csumOffset) + if err != nil { + return 0, err + } + } + if len(in) > len(bufs[0][offset:]) { + return 0, fmt.Errorf("read len %d overflows bufs element len %d", len(in), len(bufs[0][offset:])) + } + n := copy(bufs[0][offset:], in) + sizes[0] = n + return 1, nil + } + if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { + return 0, fmt.Errorf("unsupported virtio GSO type: %d", hdr.gsoType) } - // Don't trust HdrLen from the kernel as it can be equal to the length + ipVersion := in[0] >> 4 + switch ipVersion { + case 4: + if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV4 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { + return 0, fmt.Errorf("ip header version: %d, GSO type: %d", ipVersion, hdr.gsoType) + } + case 6: + if hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_TCPV6 && hdr.gsoType != unix.VIRTIO_NET_HDR_GSO_UDP_L4 { + return 0, fmt.Errorf("ip header version: %d, GSO type: %d", ipVersion, hdr.gsoType) + } + default: + return 0, fmt.Errorf("invalid ip header version: %d", ipVersion) + } + + // Don't trust hdr.hdrLen from the kernel as it can be equal to the length // of the entire first packet when the kernel is handling it as part of a // FORWARD path. Instead, parse the transport header length and add it onto - // CsumStart, which is synonymous for IP header length. - if options.GSOType == GSOUDPL4 { - options.HdrLen = options.CsumStart + 8 - } else if options.GSOType != GSONone { - if len(in) <= int(options.CsumStart+12) { + // csumStart, which is synonymous for IP header length. + if hdr.gsoType == unix.VIRTIO_NET_HDR_GSO_UDP_L4 { + hdr.hdrLen = hdr.csumStart + 8 + } else { + if len(in) <= int(hdr.csumStart+12) { return 0, errors.New("packet is too short") } - tcpHLen := uint16(in[options.CsumStart+12] >> 4 * 4) + tcpHLen := uint16(in[hdr.csumStart+12] >> 4 * 4) if tcpHLen < 20 || tcpHLen > 60 { // A TCP header must be between 20 and 60 bytes in length. return 0, fmt.Errorf("tcp header len is invalid: %d", tcpHLen) } - options.HdrLen = options.CsumStart + tcpHLen + hdr.hdrLen = hdr.csumStart + tcpHLen } - return GSOSplit(in, options, bufs, sizes, offset) + if len(in) < int(hdr.hdrLen) { + return 0, fmt.Errorf("length of packet (%d) < virtioNetHdr.hdrLen (%d)", len(in), hdr.hdrLen) + } + + if hdr.hdrLen < hdr.csumStart { + return 0, fmt.Errorf("virtioNetHdr.hdrLen (%d) < virtioNetHdr.csumStart (%d)", hdr.hdrLen, hdr.csumStart) + } + cSumAt := int(hdr.csumStart + hdr.csumOffset) + if cSumAt+1 >= len(in) { + return 0, fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(in)) + } + + return gsoSplit(in, hdr, bufs, sizes, offset, ipVersion == 6) } func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) { @@ -478,16 +525,14 @@ func (tun *NativeTun) BatchSize() int { return tun.batchSize } -// DisableUDPGRO disables UDP GRO if it is enabled. See the GRODevice interface -// for cases where it should be called. +// DisableUDPGRO disables UDP GRO if it is enabled. func (tun *NativeTun) DisableUDPGRO() { tun.writeOpMu.Lock() tun.gro.disableUDPGRO() tun.writeOpMu.Unlock() } -// DisableTCPGRO disables TCP GRO if it is enabled. See the GRODevice interface -// for cases where it should be called. +// DisableTCPGRO disables TCP GRO if it is enabled. func (tun *NativeTun) DisableTCPGRO() { tun.writeOpMu.Lock() tun.gro.disableTCPGRO() diff --git a/tun/tun_plan9.go b/tun/tun_plan9.go deleted file mode 100644 index 7b66ead..0000000 --- a/tun/tun_plan9.go +++ /dev/null @@ -1,147 +0,0 @@ -/* SPDX-License-Identifier: MIT - * - * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved. - */ - -package tun - -import ( - "fmt" - "io" - "os" - "strconv" - "strings" - "sync" -) - -type NativeTun struct { - name string // "/net/ipifc/2" - ctlFile *os.File - dataFile *os.File - events chan Event - errors chan error - closeOnce sync.Once -} - -func CreateTUN(_ string, mtu int) (Device, error) { - ctl, err := os.OpenFile("/net/ipifc/clone", os.O_RDWR, 0) - if err != nil { - return nil, err - } - nbuf := make([]byte, 5) - n, err := ctl.Read(nbuf) - if err != nil { - ctl.Close() - return nil, fmt.Errorf("error reading from clone file: %w", err) - } - ifn, err := strconv.Atoi(strings.TrimSpace(string(nbuf[:n]))) - if err != nil { - ctl.Close() - return nil, fmt.Errorf("error converting clone result %q to int: %w", nbuf[:n], err) - } - - if _, err := fmt.Fprintf(ctl, "bind pkt\n"); err != nil { - ctl.Close() - return nil, fmt.Errorf("error binding to pkt: %w", err) - } - if mtu > 0 { - if _, err := fmt.Fprintf(ctl, "mtu %d\n", mtu); err != nil { - ctl.Close() - return nil, fmt.Errorf("error setting MTU: %w", err) - } - } - - dataFile, err := os.OpenFile(fmt.Sprintf("/net/ipifc/%d/data", ifn), os.O_RDWR, 0) - if err != nil { - ctl.Close() - return nil, err - } - - tun := &NativeTun{ - ctlFile: ctl, - dataFile: dataFile, - name: fmt.Sprintf("/net/ipifc/%d", ifn), - events: make(chan Event, 10), - errors: make(chan error, 5), - } - tun.events <- EventUp - - return tun, nil -} - -func (tun *NativeTun) Name() (string, error) { - return tun.name, nil -} - -func (tun *NativeTun) File() *os.File { - return tun.ctlFile -} - -func (tun *NativeTun) Events() <-chan Event { - return tun.events -} - -func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) { - select { - case err := <-tun.errors: - return 0, err - default: - n, err := tun.dataFile.Read(bufs[0][offset:]) - if n == 1 && bufs[0][offset] == 0 { - // EOF - err = io.EOF - n = 0 - } - sizes[0] = n - return 1, err - } -} - -func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) { - for i, buf := range bufs { - if _, err := tun.dataFile.Write(buf[offset:]); err != nil { - return i, err - } - } - return len(bufs), nil -} - -func (tun *NativeTun) Close() error { - var err1, err2 error - tun.closeOnce.Do(func() { - _, err1 := fmt.Fprintf(tun.ctlFile, "unbind\n") - if err := tun.ctlFile.Close(); err != nil && err1 == nil { - err1 = err - } - err2 = tun.dataFile.Close() - }) - if err1 != nil { - return err1 - } - return err2 -} - -func (tun *NativeTun) MTU() (int, error) { - var buf [100]byte - f, err := os.Open(tun.name + "/status") - if err != nil { - return 0, err - } - defer f.Close() - n, err := f.Read(buf[:]) - _, res, ok := strings.Cut(string(buf[:n]), " maxtu ") - if ok { - if mtus, _, ok := strings.Cut(res, " "); ok { - mtu, err := strconv.Atoi(mtus) - if err != nil { - return 0, fmt.Errorf("error converting mtu %q to int: %w", mtus, err) - } - return mtu, nil - } - } - return 0, fmt.Errorf("no 'maxtu' field found in %s/status", tun.name) -} - -func (tun *NativeTun) BatchSize() int { - return 1 -}